+ . /ovnkube-lib/ovnkube-lib.sh ++ set -x ++ K8S_NODE=ip-10-0-1-71.ec2.internal ++ [[ -n ip-10-0-1-71.ec2.internal ]] ++ [[ -f /env/ip-10-0-1-71.ec2.internal ]] ++ northd_pidfile=/var/run/ovn/ovn-northd.pid ++ controller_pidfile=/var/run/ovn/ovn-controller.pid ++ controller_logfile=/var/log/ovn/acl-audit-log.log ++ vswitch_dbsock=/var/run/openvswitch/db.sock ++ nbdb_pidfile=/var/run/ovn/ovnnb_db.pid ++ nbdb_sock=/var/run/ovn/ovnnb_db.sock ++ nbdb_ctl=/var/run/ovn/ovnnb_db.ctl ++ sbdb_pidfile=/var/run/ovn/ovnsb_db.pid ++ sbdb_sock=/var/run/ovn/ovnsb_db.sock ++ sbdb_ctl=/var/run/ovn/ovnsb_db.ctl + start-ovnkube-node 4 29103 29105 + local log_level=4 + local metrics_port=29103 + local ovn_metrics_port=29105 + ovn_advertised_udn_isolation_mode_flag= + openflow_probe_flag= + [[ 3 -ne 3 ]] + ovs-vsctl br-exists br-ex + add_garp_drop_flow br-ex + local bridge=br-ex + local cookie=0x0305 + local priority=499 ++ ovs-vsctl list-ports br-ex + for port_name in $(ovs-vsctl list-ports "$bridge") + [[ ens5 == *to-br-int ]] + cni-bin-copy + . /host/etc/os-release ++ NAME='Red Hat Enterprise Linux CoreOS' ++ VERSION='9.6.20260401-0 (Plow)' ++ ID=rhel ++ ID_LIKE=fedora ++ VERSION_ID=9.6 ++ PLATFORM_ID=platform:el9 ++ PRETTY_NAME='Red Hat Enterprise Linux CoreOS 9.6.20260401-0 (Plow)' ++ ANSI_COLOR='0;31' ++ LOGO=fedora-logo-icon ++ CPE_NAME=cpe:/o:redhat:enterprise_linux:9::baseos ++ HOME_URL=https://www.redhat.com/ ++ DOCUMENTATION_URL=https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9 ++ BUG_REPORT_URL=https://issues.redhat.com/ ++ REDHAT_BUGZILLA_PRODUCT='Red Hat Enterprise Linux 9' ++ REDHAT_BUGZILLA_PRODUCT_VERSION=9.6 ++ REDHAT_SUPPORT_PRODUCT='Red Hat Enterprise Linux' ++ REDHAT_SUPPORT_PRODUCT_VERSION=9.6 ++ OSTREE_VERSION=9.6.20260401-0 ++ VARIANT=CoreOS ++ VARIANT_ID=coreos ++ OPENSHIFT_VERSION=4.21 + rhelmajor= + case "${ID}" in ++ echo 9.6 ++ cut -f 1 -d . + rhelmajor=9 + sourcedir=/usr/libexec/cni/ + case "${rhelmajor}" in + sourcedir=/usr/libexec/cni/rhel9 + cp -f /usr/libexec/cni/rhel9/ovn-k8s-cni-overlay /cni-bin-dir/ ++ date '+%m%d %H:%M:%S.%N' + echo 'I0505 08:20:47.625206644 - disable conntrack on geneve port' I0505 08:20:47.625206644 - disable conntrack on geneve port + iptables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + iptables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK ++ date '+%m%d %H:%M:%S.%N' I0505 08:20:47.668550832 - starting ovnkube-node + echo 'I0505 08:20:47.668550832 - starting ovnkube-node' + egress_features_enable_flag='--enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true' + init_ovnkube_controller='--init-ovnkube-controller ip-10-0-1-71.ec2.internal' + multi_external_gateway_enable_flag=--enable-multi-external-gateway=true + gateway_interface=br-ex + enable_multicast_flag=--enable-multicast + OVN_NODE_MODE=full + '[' full == dpu-host ']' + '[' shared == shared ']' + gateway_mode_flags='--gateway-mode shared --gateway-interface br-ex' + export_network_flows_flags= + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + gw_interface_flag= + '[' -d /sys/class/net/br-ex1 ']' + node_mgmt_port_netdev_flags= + [[ -n '' ]] + [[ -n '' ]] + multi_network_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag=--enable-network-segmentation + route_advertisements_enable_flag= + [[ false == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag= + [[ true == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag=--enable-preconfigured-udn-addresses + network_observability_enabled_flag= + [[ false == \t\r\u\e ]] + multi_network_policy_enabled_flag= + [[ false == \t\r\u\e ]] + admin_network_policy_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + admin_network_policy_enabled_flag=--enable-admin-network-policy + dns_name_resolver_enabled_flag= + [[ false == \t\r\u\e ]] + ip_forwarding_flag= + '[' '' == Global ']' + ip_forwarding_flag=--disable-forwarding + sysctl -w net.ipv4.ip_forward=0 net.ipv4.ip_forward = 0 + sysctl -w net.ipv6.conf.all.forwarding=0 net.ipv6.conf.all.forwarding = 0 + [[ '' != '' ]] + [[ '' != '' ]] + NETWORK_NODE_IDENTITY_ENABLE= + [[ true == \t\r\u\e ]] + NETWORK_NODE_IDENTITY_ENABLE=' --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h ' + ovn_v4_join_subnet_opt= + [[ '' != '' ]] + ovn_v6_join_subnet_opt= + [[ '' != '' ]] + ovn_v4_masquerade_subnet_opt= + [[ 169.254.0.0/17 != '' ]] + ovn_v4_masquerade_subnet_opt='--gateway-v4-masquerade-subnet 169.254.0.0/17' + ovn_v6_masquerade_subnet_opt= + [[ fd69::/112 != '' ]] + ovn_v6_masquerade_subnet_opt='--gateway-v6-masquerade-subnet fd69::/112' + ovn_v4_transit_switch_subnet_opt= + [[ '' != '' ]] + ovn_v6_transit_switch_subnet_opt= + [[ '' != '' ]] + exec /usr/bin/ovnkube --init-ovnkube-controller ip-10-0-1-71.ec2.internal --init-node ip-10-0-1-71.ec2.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4 --inactivity-probe=180000 --gateway-mode shared --gateway-interface br-ex --metrics-bind-address 127.0.0.1:29103 --ovn-metrics-bind-address 127.0.0.1:29105 --metrics-enable-pprof --metrics-enable-config-duration --export-ovs-metrics --disable-snat-multiple-gws --enable-multi-network --enable-network-segmentation --enable-preconfigured-udn-addresses --enable-admin-network-policy --enable-multicast --zone ip-10-0-1-71.ec2.internal --enable-interconnect --acl-logging-rate-limit 20 --disable-forwarding --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h --gateway-v4-masquerade-subnet 169.254.0.0/17 --gateway-v6-masquerade-subnet fd69::/112 --enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true --enable-multi-external-gateway=true I0505 08:20:47.728226 3788 config.go:2726] Parsed config file /run/ovnkube-config/ovnkube.conf I0505 08:20:47.728307 3788 config.go:2727] Parsed config: {Default:{MTU:8901 RoutableMTU:0 ConntrackZone:64000 HostMasqConntrackZone:0 OVNMasqConntrackZone:0 HostNodePortConntrackZone:0 ReassemblyConntrackZone:0 EncapType:geneve EncapIP: EffectiveEncapIP: EncapPort:6081 InactivityProbe:100000 OpenFlowProbe:0 OfctrlWaitBeforeClear:0 MonitorAll:true OVSDBTxnTimeout:1m40s LFlowCacheEnable:true LFlowCacheLimit:0 LFlowCacheLimitKb:1048576 RawClusterSubnets:10.128.0.0/14/23 ClusterSubnets:[] EnableUDPAggregation:true Zone:global RawUDNAllowedDefaultServices:default/kubernetes,openshift-dns/dns-default UDNAllowedDefaultServices:[] Transport:geneve} Logging:{File: CNIFile: LibovsdbFile: Level:4 LogFileMaxSize:100 LogFileMaxBackups:5 LogFileMaxAge:5 ACLLoggingRateLimit:20} Monitoring:{RawNetFlowTargets: RawSFlowTargets: RawIPFIXTargets: NetFlowTargets:[] SFlowTargets:[] IPFIXTargets:[]} IPFIX:{Sampling:400 CacheActiveTimeout:60 CacheMaxFlows:0} CNI:{ConfDir:/etc/cni/net.d Plugin:ovn-k8s-cni-overlay} OVNKubernetesFeature:{EnableAdminNetworkPolicy:false EnableEgressIP:false EgressIPReachabiltyTotalTimeout:1 EnableEgressFirewall:false EnableEgressQoS:false EnableEgressService:false EgressIPNodeHealthCheckPort:9107 EnableMultiNetwork:false EnableNetworkSegmentation:true EnableNetworkConnect:false EnablePreconfiguredUDNAddresses:true EnableRouteAdvertisements:false EnableEVPN:false EnableMultiNetworkPolicy:false EnableStatelessNetPol:false EnableInterconnect:false EnableMultiExternalGateway:false EnablePersistentIPs:false EnableDNSNameResolver:false EnableServiceTemplateSupport:false EnableObservability:false EnableNetworkQoS:false AdvertisedUDNIsolationMode:strict EnableDynamicUDNAllocation:false UDNDeletionGracePeriod:2m0s} Kubernetes:{BootstrapKubeconfig: CertDir: CertDuration:10m0s Kubeconfig: CACert: CACertData: CAData:[] APIServer:https://api.kx-99c9d17dd1.hypershift.local:443 Token: TokenFile: CompatServiceCIDR: RawServiceCIDRs:172.30.0.0/16 ServiceCIDRs:[] OVNConfigNamespace:openshift-ovn-kubernetes OVNEmptyLbEvents:false RawNoHostSubnetNodes: NoHostSubnetNodes: HostNetworkNamespace:openshift-host-network DisableRequestedChassis:false PlatformType:AWS HealthzBindAddress:0.0.0.0:10256 CompatMetricsBindAddress: CompatOVNMetricsBindAddress: CompatMetricsEnablePprof:false DNSServiceNamespace:openshift-dns DNSServiceName:dns-default} Metrics:{BindAddress: OVNMetricsBindAddress: ExportOVSMetrics:false EnablePprof:false NodeServerPrivKey: NodeServerCert: EnableConfigDuration:false EnableScaleMetrics:false} OvnNorth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnnb_db.db exec:} OvnSouth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnsb_db.db exec:} Gateway:{Mode:shared Interface: GatewayAcceleratedInterface: EgressGWInterface: NextHop: VLANID:0 NodeportEnable:true DisableSNATMultipleGWs:false V4JoinSubnet:100.64.0.0/16 V6JoinSubnet:fd98::/64 V4MasqueradeSubnet:169.254.169.0/29 V6MasqueradeSubnet:fd69::/125 MasqueradeIPs:{V4OVNMasqueradeIP:169.254.169.1 V6OVNMasqueradeIP:fd69::1 V4HostMasqueradeIP:169.254.169.2 V6HostMasqueradeIP:fd69::2 V4HostETPLocalMasqueradeIP:169.254.169.3 V6HostETPLocalMasqueradeIP:fd69::3 V4DummyNextHopMasqueradeIP:169.254.169.4 V6DummyNextHopMasqueradeIP:fd69::4 V4OVNServiceHairpinMasqueradeIP:169.254.169.5 V6OVNServiceHairpinMasqueradeIP:fd69::5} DisablePacketMTUCheck:false RouterSubnet: SingleNode:false DisableForwarding:false AllowNoUplink:false EphemeralPortRange:} MasterHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} ClusterMgrHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} HybridOverlay:{Enabled:false RawClusterSubnets: ClusterSubnets:[] VXLANPort:4789} OvnKubeNode:{Mode:full MgmtPortNetdev: MgmtPortDPResourceName:} ClusterManager:{V4TransitSubnet:100.88.0.0/16 V6TransitSubnet:fd97::/64} OvsPaths:{RunDir:/var/run/openvswitch/} NoOverlay:{OutboundSNAT: Routing:} ManagedBGP:{ASNumber:64512 Topology:}} I0505 08:20:47.730061 3788 kube.go:431] Waiting for certificate I0505 08:20:47.730111 3788 certificate_manager.go:422] "Certificate rotation is enabled" logger="kubernetes.io/kube-apiserver-client" I0505 08:20:47.730147 3788 certificate_manager.go:566] "Rotating certificates" logger="kubernetes.io/kube-apiserver-client" I0505 08:20:47.730872 3788 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0505 08:20:47.757275 3788 reflector.go:358] "Starting reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0505 08:20:47.757302 3788 reflector.go:404] "Listing and watching" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0505 08:20:47.760000 3788 reflector.go:436] "Caches populated" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0505 08:20:47.762389 3788 csr.go:274] "Certificate signing request is approved, waiting to be issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-qmq4q" I0505 08:20:47.770734 3788 csr.go:270] "Certificate signing request is issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-qmq4q" I0505 08:20:47.770810 3788 reflector.go:364] "Stopping reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0505 08:20:48.730384 3788 kube.go:438] Certificate found I0505 08:20:48.731059 3788 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0505 08:20:48.732473 3788 metrics.go:532] Starting metrics server at address "127.0.0.1:29103" I0505 08:20:48.799325 3788 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0505 08:20:48.812063 3788 certificate_manager.go:715] "Certificate rotation deadline determined" logger="kubernetes.io/kube-apiserver-client" expiration="2026-05-06 08:15:47 +0000 UTC" deadline="2026-05-06 01:56:22.648998353 +0000 UTC" I0505 08:20:48.812172 3788 certificate_manager.go:431] "Waiting for next certificate rotation" logger="kubernetes.io/kube-apiserver-client" sleep="17h35m33.836839519s" I0505 08:20:48.822978 3788 metrics.go:512] Create OVN Metrics Server on address: 127.0.0.1:29105 I0505 08:20:48.823049 3788 server.go:101] MetricServer registers OVS metrics I0505 08:20:48.823314 3788 server.go:105] MetricServer registers OVN DB metrics I0505 08:20:48.826281 3788 node_controller_manager.go:338] Starting the node network controller manager, Mode: full I0505 08:20:48.826352 3788 factory.go:561] Starting watch factory I0505 08:20:48.826481 3788 reflector.go:358] "Starting reflector" type="*v1.Service" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0505 08:20:48.826561 3788 reflector.go:404] "Listing and watching" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0505 08:20:48.826491 3788 reflector.go:358] "Starting reflector" type="*v1.EndpointSlice" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0505 08:20:48.826689 3788 reflector.go:404] "Listing and watching" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0505 08:20:48.826490 3788 reflector.go:358] "Starting reflector" type="*v1.Node" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0505 08:20:48.826798 3788 reflector.go:404] "Listing and watching" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0505 08:20:48.826540 3788 reflector.go:358] "Starting reflector" type="*v1.NetworkPolicy" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0505 08:20:48.827009 3788 reflector.go:358] "Starting reflector" type="*v1.Namespace" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0505 08:20:48.827045 3788 reflector.go:404] "Listing and watching" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0505 08:20:48.827374 3788 reflector.go:404] "Listing and watching" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0505 08:20:48.827399 3788 reflector.go:358] "Starting reflector" type="*v1.Pod" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0505 08:20:48.827416 3788 reflector.go:404] "Listing and watching" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0505 08:20:48.828195 3788 ovn_db.go:330] /var/run/openvswitch/ovnnb_db.sock getting info failed: stat /var/run/openvswitch/ovnnb_db.sock: no such file or directory I0505 08:20:48.828265 3788 ovn_db.go:327] ovnnb_db.sock found at /var/run/ovn/ I0505 08:20:48.829164 3788 controller_manager.go:370] Starting the ovnkube controller I0505 08:20:48.829176 3788 controller_manager.go:375] Waiting up to 5m0s for NBDB zone to match: ip-10-0-1-71.ec2.internal I0505 08:20:48.829232 3788 controller_manager.go:395] NBDB zone sync took: 48.078µs I0505 08:20:48.829243 3788 factory.go:561] Starting watch factory I0505 08:20:48.842997 3788 ovn_db.go:388] Found OVN NB DB: &{0x1e9e6e0 /etc/ovn/ovnnb_db.db OVN_Northbound 0} I0505 08:20:48.843026 3788 ovn_db.go:395] Found OVN SB DB: &{0x1e9e9a0 /etc/ovn/ovnsb_db.db OVN_Southbound 0} I0505 08:20:48.846884 3788 ovn_db.go:410] Found db is standalone, don't register db_cluster metrics I0505 08:20:48.846924 3788 server.go:109] MetricServer registers OVN Controller metrics I0505 08:20:48.847154 3788 reflector.go:436] "Caches populated" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0505 08:20:48.847192 3788 reflector.go:436] "Caches populated" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0505 08:20:48.847361 3788 reflector.go:436] "Caches populated" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0505 08:20:48.847999 3788 reflector.go:436] "Caches populated" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0505 08:20:48.849668 3788 reflector.go:436] "Caches populated" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0505 08:20:48.853322 3788 server.go:113] MetricServer registers OVN Northd metrics I0505 08:20:48.855195 3788 reflector.go:436] "Caches populated" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0505 08:20:48.858608 3788 metrics.go:519] OVN Metrics Server starts to run ... I0505 08:20:48.932477 3788 factory.go:1977] *v1.Namespace informer cache synced successfully I0505 08:20:48.932502 3788 factory.go:1977] *v1.Node informer cache synced successfully I0505 08:20:48.932507 3788 factory.go:1977] *v1.Service informer cache synced successfully I0505 08:20:48.932512 3788 factory.go:1977] *v1.EndpointSlice informer cache synced successfully I0505 08:20:48.932517 3788 factory.go:1977] *v1.Pod informer cache synced successfully I0505 08:20:48.932522 3788 factory.go:1977] *v1.NetworkPolicy informer cache synced successfully I0505 08:20:48.932612 3788 reflector.go:358] "Starting reflector" type="*v1alpha1.BaselineAdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0505 08:20:48.932626 3788 reflector.go:404] "Listing and watching" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0505 08:20:48.932643 3788 reflector.go:358] "Starting reflector" type="*v1alpha1.AdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0505 08:20:48.932666 3788 reflector.go:404] "Listing and watching" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0505 08:20:48.934630 3788 factory.go:1977] *v1.Pod informer cache synced successfully I0505 08:20:48.934647 3788 factory.go:1977] *v1.NetworkPolicy informer cache synced successfully I0505 08:20:48.934652 3788 factory.go:1977] *v1.Namespace informer cache synced successfully I0505 08:20:48.934655 3788 factory.go:1977] *v1.Node informer cache synced successfully I0505 08:20:48.934658 3788 factory.go:1977] *v1.Service informer cache synced successfully I0505 08:20:48.934661 3788 factory.go:1977] *v1.EndpointSlice informer cache synced successfully I0505 08:20:48.935560 3788 reflector.go:436] "Caches populated" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0505 08:20:48.935580 3788 reflector.go:436] "Caches populated" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0505 08:20:48.937642 3788 factory.go:1977] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0505 08:20:48.937654 3788 factory.go:1977] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0505 08:20:48.937718 3788 reflector.go:358] "Starting reflector" type="*v1.EgressIP" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0505 08:20:48.937730 3788 reflector.go:404] "Listing and watching" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0505 08:20:48.939805 3788 reflector.go:436] "Caches populated" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0505 08:20:48.939808 3788 factory.go:1977] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0505 08:20:48.939874 3788 factory.go:1977] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0505 08:20:48.942844 3788 factory.go:1977] *v1.EgressIP informer cache synced successfully I0505 08:20:48.942907 3788 reflector.go:358] "Starting reflector" type="*v1.EgressFirewall" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0505 08:20:48.942920 3788 reflector.go:404] "Listing and watching" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0505 08:20:48.945129 3788 factory.go:1977] *v1.EgressIP informer cache synced successfully I0505 08:20:48.945383 3788 reflector.go:436] "Caches populated" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0505 08:20:48.948577 3788 factory.go:1977] *v1.EgressFirewall informer cache synced successfully I0505 08:20:48.948660 3788 reflector.go:358] "Starting reflector" type="*v1.EgressQoS" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0505 08:20:48.948673 3788 reflector.go:404] "Listing and watching" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0505 08:20:48.950926 3788 factory.go:1977] *v1.EgressFirewall informer cache synced successfully I0505 08:20:48.951625 3788 reflector.go:436] "Caches populated" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0505 08:20:48.954580 3788 factory.go:1977] *v1.EgressQoS informer cache synced successfully I0505 08:20:48.954644 3788 reflector.go:358] "Starting reflector" type="*v1.EgressService" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0505 08:20:48.954655 3788 reflector.go:404] "Listing and watching" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0505 08:20:48.955977 3788 factory.go:1977] *v1.EgressQoS informer cache synced successfully I0505 08:20:48.957879 3788 reflector.go:436] "Caches populated" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0505 08:20:48.959865 3788 factory.go:1977] *v1.EgressService informer cache synced successfully I0505 08:20:48.959949 3788 reflector.go:358] "Starting reflector" type="*v1.AdminPolicyBasedExternalRoute" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0505 08:20:48.959970 3788 reflector.go:404] "Listing and watching" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0505 08:20:48.961142 3788 factory.go:1977] *v1.EgressService informer cache synced successfully I0505 08:20:48.962604 3788 reflector.go:436] "Caches populated" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0505 08:20:48.964903 3788 factory.go:1977] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0505 08:20:48.965090 3788 reflector.go:358] "Starting reflector" type="*v1.NetworkAttachmentDefinition" resyncPeriod="0s" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0505 08:20:48.965106 3788 reflector.go:404] "Listing and watching" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0505 08:20:48.967019 3788 factory.go:1977] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0505 08:20:48.968243 3788 reflector.go:436] "Caches populated" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0505 08:20:48.970138 3788 factory.go:1977] *v1.NetworkAttachmentDefinition informer cache synced successfully I0505 08:20:48.970204 3788 reflector.go:358] "Starting reflector" type="*v1.UserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0505 08:20:48.970217 3788 reflector.go:404] "Listing and watching" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0505 08:20:48.970261 3788 reflector.go:358] "Starting reflector" type="*v1.ClusterUserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0505 08:20:48.970276 3788 reflector.go:404] "Listing and watching" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0505 08:20:48.972058 3788 factory.go:1977] *v1.NetworkAttachmentDefinition informer cache synced successfully I0505 08:20:48.972898 3788 reflector.go:436] "Caches populated" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0505 08:20:48.972940 3788 reflector.go:436] "Caches populated" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0505 08:20:48.975204 3788 factory.go:1977] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0505 08:20:48.975216 3788 factory.go:1977] *v1.UserDefinedNetwork informer cache synced successfully I0505 08:20:48.975223 3788 factory.go:680] Watch Factory start up complete, took: 148.876021ms I0505 08:20:48.975290 3788 default_node_network_controller.go:171] Enable node proxy healthz server on 0.0.0.0:10256 I0505 08:20:48.977581 3788 factory.go:1977] *v1.UserDefinedNetwork informer cache synced successfully I0505 08:20:48.977592 3788 factory.go:1977] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0505 08:20:48.977600 3788 factory.go:680] Watch Factory start up complete, took: 148.356428ms I0505 08:20:48.977607 3788 controller_manager.go:406] Waiting up to 5m0s for a node to have "ip-10-0-1-71.ec2.internal" zone I0505 08:20:49.020224 3788 default_node_network_controller.go:710] Initializing the default node network controller I0505 08:20:49.101647 3788 udn_isolation.go:101] Starting UDN host isolation manager I0505 08:20:49.107319 3788 udn_isolation.go:112] Found kubelet cgroup path: system.slice/kubelet.service I0505 08:20:49.173268 3788 controller.go:133] Adding controller udn-host-isolation-manager event handlers I0505 08:20:49.173393 3788 shared_informer.go:349] "Waiting for caches to sync" controller="udn-host-isolation-manager" I0505 08:20:49.173406 3788 shared_informer.go:356] "Caches are synced" controller="udn-host-isolation-manager" I0505 08:20:49.207146 3788 controller.go:157] Starting controller udn-host-isolation-manager with 1 workers I0505 08:20:49.207215 3788 default_node_network_controller.go:809] Node ip-10-0-1-71.ec2.internal ready for ovn initialization with subnet 10.129.0.0/23 I0505 08:20:49.213064 3788 kube.go:131] Setting annotations map[k8s.ovn.org/node-encap-ips:["10.0.1.71"] k8s.ovn.org/zone-name:ip-10-0-1-71.ec2.internal] on node ip-10-0-1-71.ec2.internal I0505 08:20:49.228371 3788 controller_manager.go:429] Waiting for node in zone sync took: 250.749972ms I0505 08:20:49.233315 3788 controller_manager.go:611] Switching to transit router for layer2 networks I0505 08:20:49.233330 3788 kube.go:131] Setting annotations map[k8s.ovn.org/layer2-topology-version:2.0] on node ip-10-0-1-71.ec2.internal I0505 08:20:49.233601 3788 gateway_init.go:234] Initializing Gateway Functionality for Gateway PreStart I0505 08:20:49.240864 3788 helper_linux.go:93] Provided gateway interface "br-ex", found as index: 5 I0505 08:20:49.240978 3788 helper_linux.go:118] Found default gateway interface br-ex 10.0.1.1 I0505 08:20:49.241122 3788 gateway_init.go:263] Preparing Gateway I0505 08:20:49.241135 3788 gateway_shared_intf.go:1687] Creating new gateway I0505 08:20:49.252566 3788 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0505 08:20:49.259495 3788 controller_manager.go:290] SCTP support detected in OVN I0505 08:20:49.260218 3788 services_controller.go:69] Creating services controller for network=default I0505 08:20:49.260303 3788 udn_enabled_svc.go:80] Waiting for service informer to sync I0505 08:20:49.260358 3788 controller.go:133] Adding controller [zone-nad-controller NAD controller] event handlers I0505 08:20:49.260393 3788 shared_informer.go:349] "Waiting for caches to sync" controller="[zone-nad-controller NAD controller]" I0505 08:20:49.260406 3788 shared_informer.go:356] "Caches are synced" controller="[zone-nad-controller NAD controller]" I0505 08:20:49.260416 3788 nad_controller.go:541] [zone-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0505 08:20:49.260597 3788 controller.go:157] Starting controller [zone-nad-controller NAD controller] with 1 workers I0505 08:20:49.260646 3788 network_controller.go:373] [zone-nad-controller network controller]: syncing all networks I0505 08:20:49.260705 3788 nad_controller.go:580] [zone-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 67.491µs I0505 08:20:49.260728 3788 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 74.254µs I0505 08:20:49.260734 3788 network_controller.go:384] [zone-nad-controller network controller]: finished syncing all networks. Time taken: 87.987µs I0505 08:20:49.260740 3788 controller.go:157] Starting controller [zone-nad-controller network controller] with 1 workers I0505 08:20:49.260750 3788 nad_controller.go:379] [zone-nad-controller NAD controller]: started I0505 08:20:49.260761 3788 default_network_controller.go:325] Starting the default network controller I0505 08:20:49.260810 3788 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 14.178µs I0505 08:20:49.260827 3788 udn_enabled_svc.go:95] Performing full resync I0505 08:20:49.261138 3788 udn_enabled_svc.go:99] Waiting for handler to sync I0505 08:20:49.261151 3788 udn_enabled_svc.go:104] Starting worker I0505 08:20:49.272035 3788 default_network_controller.go:395] Cleaning External Gateway ECMP routes I0505 08:20:49.272110 3788 repair.go:33] Syncing exgw routes took 59.74µs I0505 08:20:49.272146 3788 default_network_controller.go:414] Starting all the Watchers... I0505 08:20:49.272348 3788 namespace.go:98] [openshift-must-gather-operator] adding namespace I0505 08:20:49.272354 3788 namespace.go:98] [openshift-backplane-srep-ro] adding namespace I0505 08:20:49.272362 3788 namespace.go:98] [openshift-route-controller-manager] adding namespace I0505 08:20:49.272348 3788 namespace.go:98] [openshift-backplane-tam] adding namespace I0505 08:20:49.272374 3788 namespace.go:98] [openshift-apiserver] adding namespace I0505 08:20:49.272379 3788 namespace.go:98] [openshift-console] adding namespace I0505 08:20:49.272385 3788 namespace.go:98] [openshift-kube-scheduler] adding namespace I0505 08:20:49.272386 3788 namespace.go:98] [openshift-logging] adding namespace I0505 08:20:49.272399 3788 namespace.go:98] [openshift-backplane-cee] adding namespace I0505 08:20:49.272402 3788 namespace.go:98] [openshift-backplane-srep] adding namespace I0505 08:20:49.272403 3788 namespace.go:98] [openshift] adding namespace I0505 08:20:49.272406 3788 namespace.go:98] [openshift-kube-controller-manager-operator] adding namespace I0505 08:20:49.272403 3788 namespace.go:98] [openshift-operators-redhat] adding namespace I0505 08:20:49.272447 3788 namespace.go:98] [openshift-kube-apiserver-operator] adding namespace I0505 08:20:49.272348 3788 namespace.go:98] [openshift-kube-scheduler-operator] adding namespace I0505 08:20:49.273062 3788 namespace.go:102] [openshift-must-gather-operator] adding namespace took 698.446µs I0505 08:20:49.273078 3788 namespace.go:98] [openshift-kube-storage-version-migrator-operator] adding namespace I0505 08:20:49.273653 3788 namespace.go:102] [openshift-kube-storage-version-migrator-operator] adding namespace took 568.318µs I0505 08:20:49.273672 3788 namespace.go:98] [openshift-network-diagnostics] adding namespace W0505 08:20:49.273723 3788 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-wg7jm: pod openshift-network-diagnostics/network-check-target-wg7jm: no pod IPs found W0505 08:20:49.273736 3788 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-zqqrr: pod openshift-network-diagnostics/network-check-target-zqqrr: no pod IPs found I0505 08:20:49.274364 3788 namespace.go:102] [openshift-network-diagnostics] adding namespace took 686.139µs I0505 08:20:49.274379 3788 namespace.go:98] [open-cluster-management-2q3ldim9a28b4bs5p55oas8jpfchqces] adding namespace I0505 08:20:49.275038 3788 namespace.go:102] [openshift-backplane-srep-ro] adding namespace took 2.669592ms I0505 08:20:49.275053 3788 namespace.go:98] [openshift-backplane] adding namespace I0505 08:20:49.275645 3788 namespace.go:102] [openshift-route-controller-manager] adding namespace took 3.272798ms I0505 08:20:49.275664 3788 namespace.go:98] [openshift-node] adding namespace I0505 08:20:49.276220 3788 namespace.go:102] [openshift-backplane-tam] adding namespace took 3.843685ms I0505 08:20:49.276239 3788 namespace.go:98] [openshift-network-console] adding namespace I0505 08:20:49.276964 3788 namespace.go:102] [openshift-apiserver] adding namespace took 4.577553ms I0505 08:20:49.276981 3788 namespace.go:98] [dedicated-admin] adding namespace I0505 08:20:49.277566 3788 namespace.go:102] [openshift-console] adding namespace took 5.178007ms I0505 08:20:49.277582 3788 namespace.go:98] [openshift-cluster-machine-approver] adding namespace I0505 08:20:49.278168 3788 namespace.go:102] [openshift-kube-scheduler] adding namespace took 5.77433ms I0505 08:20:49.278187 3788 namespace.go:98] [open-cluster-management-agent-addon] adding namespace I0505 08:20:49.278853 3788 namespace.go:102] [openshift-logging] adding namespace took 6.454939ms I0505 08:20:49.278868 3788 namespace.go:98] [openshift-image-registry] adding namespace I0505 08:20:49.279417 3788 namespace.go:102] [openshift-backplane-cee] adding namespace took 7.011931ms I0505 08:20:49.279434 3788 namespace.go:98] [openshift-operator-lifecycle-manager] adding namespace I0505 08:20:49.280004 3788 namespace.go:102] [openshift-backplane-srep] adding namespace took 7.595401ms I0505 08:20:49.280020 3788 namespace.go:98] [openshift-cluster-version] adding namespace I0505 08:20:49.280528 3788 namespace.go:102] [openshift] adding namespace took 8.114007ms I0505 08:20:49.280543 3788 namespace.go:98] [openshift-config-managed] adding namespace I0505 08:20:49.281217 3788 namespace.go:102] [openshift-kube-controller-manager-operator] adding namespace took 8.800577ms I0505 08:20:49.281232 3788 namespace.go:98] [openshift-monitoring] adding namespace I0505 08:20:49.281858 3788 namespace.go:102] [openshift-operators-redhat] adding namespace took 9.437466ms I0505 08:20:49.281875 3788 namespace.go:98] [openshift-backplane-mobb] adding namespace I0505 08:20:49.282424 3788 namespace.go:102] [openshift-kube-apiserver-operator] adding namespace took 9.950988ms I0505 08:20:49.282436 3788 namespace.go:98] [openshift-operators] adding namespace I0505 08:20:49.282979 3788 namespace.go:102] [openshift-kube-scheduler-operator] adding namespace took 10.499057ms I0505 08:20:49.283001 3788 namespace.go:98] [openshift-config] adding namespace I0505 08:20:49.283583 3788 namespace.go:102] [open-cluster-management-2q3ldim9a28b4bs5p55oas8jpfchqces] adding namespace took 9.196115ms I0505 08:20:49.283604 3788 namespace.go:98] [openshift-network-operator] adding namespace I0505 08:20:49.284172 3788 namespace.go:102] [openshift-backplane] adding namespace took 9.112432ms I0505 08:20:49.284188 3788 namespace.go:98] [openshift-console-user-settings] adding namespace I0505 08:20:49.284736 3788 namespace.go:102] [openshift-node] adding namespace took 9.065029ms I0505 08:20:49.284751 3788 namespace.go:98] [openshift-network-node-identity] adding namespace I0505 08:20:49.285271 3788 namespace.go:102] [openshift-network-console] adding namespace took 9.027796ms I0505 08:20:49.285281 3788 namespace.go:98] [openshift-cloud-network-config-controller] adding namespace I0505 08:20:49.285880 3788 namespace.go:102] [dedicated-admin] adding namespace took 8.891469ms I0505 08:20:49.285897 3788 namespace.go:98] [openshift-infra] adding namespace I0505 08:20:49.286362 3788 namespace.go:102] [openshift-cluster-machine-approver] adding namespace took 8.773871ms I0505 08:20:49.286379 3788 namespace.go:98] [openshift-service-ca-operator] adding namespace I0505 08:20:49.286842 3788 namespace.go:102] [open-cluster-management-agent-addon] adding namespace took 8.648766ms I0505 08:20:49.286857 3788 namespace.go:98] [openshift-kube-controller-manager] adding namespace I0505 08:20:49.287564 3788 namespace.go:102] [openshift-image-registry] adding namespace took 8.689992ms I0505 08:20:49.287578 3788 namespace.go:98] [openshift-machine-api] adding namespace I0505 08:20:49.288091 3788 namespace.go:102] [openshift-operator-lifecycle-manager] adding namespace took 8.64886ms I0505 08:20:49.288105 3788 namespace.go:98] [openshift-authentication-operator] adding namespace I0505 08:20:49.288583 3788 namespace.go:102] [openshift-cluster-version] adding namespace took 8.556064ms I0505 08:20:49.288603 3788 namespace.go:98] [openshift-ingress-operator] adding namespace I0505 08:20:49.289227 3788 namespace.go:102] [openshift-config-managed] adding namespace took 8.677719ms I0505 08:20:49.289239 3788 namespace.go:98] [openshift-backplane-cse] adding namespace I0505 08:20:49.289825 3788 namespace.go:102] [openshift-monitoring] adding namespace took 8.587667ms I0505 08:20:49.289840 3788 namespace.go:98] [openshift-deployment-validation-operator] adding namespace I0505 08:20:49.290347 3788 namespace.go:102] [openshift-backplane-mobb] adding namespace took 8.466099ms I0505 08:20:49.290362 3788 namespace.go:98] [openshift-marketplace] adding namespace I0505 08:20:49.290937 3788 namespace.go:102] [openshift-operators] adding namespace took 8.494853ms I0505 08:20:49.290952 3788 namespace.go:98] [openshift-host-network] adding namespace I0505 08:20:49.291487 3788 namespace.go:102] [openshift-config] adding namespace took 8.478916ms I0505 08:20:49.291502 3788 namespace.go:98] [openshift-backplane-csm] adding namespace I0505 08:20:49.292163 3788 namespace.go:102] [openshift-network-operator] adding namespace took 8.552994ms I0505 08:20:49.292826 3788 namespace.go:102] [openshift-console-user-settings] adding namespace took 8.63307ms I0505 08:20:49.292850 3788 namespace.go:98] [openshift-multus] adding namespace I0505 08:20:49.293453 3788 namespace.go:102] [openshift-network-node-identity] adding namespace took 8.694929ms I0505 08:20:49.293484 3788 namespace.go:98] [openshift-backplane-mcs-tier-two] adding namespace I0505 08:20:49.294491 3788 namespace.go:102] [openshift-cloud-network-config-controller] adding namespace took 9.205789ms I0505 08:20:49.294500 3788 namespace.go:98] [openshift-controller-manager] adding namespace I0505 08:20:49.295098 3788 namespace.go:102] [openshift-infra] adding namespace took 9.194873ms I0505 08:20:49.295112 3788 namespace.go:98] [openshift-cloud-controller-manager] adding namespace I0505 08:20:49.295709 3788 namespace.go:102] [openshift-service-ca-operator] adding namespace took 9.323987ms I0505 08:20:49.295722 3788 namespace.go:98] [openshift-customer-monitoring] adding namespace I0505 08:20:49.296217 3788 namespace.go:102] [openshift-kube-controller-manager] adding namespace took 9.353968ms I0505 08:20:49.296237 3788 namespace.go:98] [kube-public] adding namespace I0505 08:20:49.296779 3788 namespace.go:102] [openshift-machine-api] adding namespace took 9.196615ms I0505 08:20:49.296793 3788 namespace.go:98] [kube-node-lease] adding namespace I0505 08:20:49.297347 3788 namespace.go:102] [openshift-authentication-operator] adding namespace took 9.236646ms I0505 08:20:49.297358 3788 namespace.go:98] [openshift-user-workload-monitoring] adding namespace I0505 08:20:49.297930 3788 namespace.go:102] [openshift-ingress-operator] adding namespace took 9.317715ms I0505 08:20:49.297955 3788 namespace.go:98] [kube-system] adding namespace I0505 08:20:49.298492 3788 namespace.go:102] [openshift-backplane-cse] adding namespace took 9.244185ms I0505 08:20:49.298514 3788 namespace.go:98] [openshift-backplane-managed-scripts] adding namespace I0505 08:20:49.299142 3788 namespace.go:102] [openshift-deployment-validation-operator] adding namespace took 9.295798ms I0505 08:20:49.299159 3788 namespace.go:98] [openshift-ingress-canary] adding namespace I0505 08:20:49.299598 3788 namespace.go:102] [openshift-marketplace] adding namespace took 9.230823ms I0505 08:20:49.299611 3788 namespace.go:98] [openshift-console-operator] adding namespace I0505 08:20:49.300320 3788 namespace.go:102] [openshift-host-network] adding namespace took 9.362491ms I0505 08:20:49.300336 3788 namespace.go:98] [openshift-ovn-kubernetes] adding namespace W0505 08:20:49.300847 3788 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-multus/network-metrics-daemon-hd7q7: pod openshift-multus/network-metrics-daemon-hd7q7: no pod IPs found I0505 08:20:49.300873 3788 namespace.go:102] [openshift-backplane-csm] adding namespace took 9.365254ms I0505 08:20:49.300886 3788 namespace.go:98] [openshift-backplane-lpsre] adding namespace W0505 08:20:49.300889 3788 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-multus/network-metrics-daemon-b2tcb: pod openshift-multus/network-metrics-daemon-b2tcb: no pod IPs found I0505 08:20:49.301617 3788 namespace.go:102] [openshift-multus] adding namespace took 8.760635ms I0505 08:20:49.301631 3788 namespace.go:98] [openshift-controller-manager-operator] adding namespace I0505 08:20:49.302153 3788 namespace.go:102] [openshift-backplane-mcs-tier-two] adding namespace took 8.663724ms I0505 08:20:49.302167 3788 namespace.go:98] [openshift-ingress] adding namespace I0505 08:20:49.302899 3788 namespace.go:102] [openshift-controller-manager] adding namespace took 8.392103ms I0505 08:20:49.302914 3788 namespace.go:98] [openshift-authentication] adding namespace I0505 08:20:49.303426 3788 namespace.go:102] [openshift-cloud-controller-manager] adding namespace took 8.306223ms I0505 08:20:49.303444 3788 namespace.go:98] [openshift-dns] adding namespace I0505 08:20:49.304051 3788 namespace.go:102] [openshift-customer-monitoring] adding namespace took 8.321905ms I0505 08:20:49.304072 3788 namespace.go:98] [openshift-etcd] adding namespace I0505 08:20:49.304555 3788 namespace.go:102] [kube-public] adding namespace took 8.307381ms I0505 08:20:49.304572 3788 namespace.go:98] [openshift-kube-apiserver] adding namespace I0505 08:20:49.305131 3788 namespace.go:102] [kube-node-lease] adding namespace took 8.332873ms I0505 08:20:49.305149 3788 namespace.go:98] [openshift-config-operator] adding namespace I0505 08:20:49.305736 3788 namespace.go:102] [openshift-user-workload-monitoring] adding namespace took 8.370992ms I0505 08:20:49.305755 3788 namespace.go:98] [openshift-cluster-storage-operator] adding namespace I0505 08:20:49.306293 3788 namespace.go:102] [kube-system] adding namespace took 8.331512ms I0505 08:20:49.306308 3788 namespace.go:98] [openshift-cluster-samples-operator] adding namespace I0505 08:20:49.306866 3788 namespace.go:102] [openshift-backplane-managed-scripts] adding namespace took 8.346233ms I0505 08:20:49.306880 3788 namespace.go:98] [openshift-insights] adding namespace I0505 08:20:49.307543 3788 namespace.go:102] [openshift-ingress-canary] adding namespace took 8.3783ms I0505 08:20:49.307555 3788 namespace.go:98] [openshift-package-operator] adding namespace I0505 08:20:49.308092 3788 namespace.go:102] [openshift-console-operator] adding namespace took 8.475174ms I0505 08:20:49.308107 3788 namespace.go:98] [openshift-cluster-node-tuning-operator] adding namespace I0505 08:20:49.308745 3788 namespace.go:102] [openshift-ovn-kubernetes] adding namespace took 8.404559ms I0505 08:20:49.308758 3788 namespace.go:98] [openshift-cluster-csi-drivers] adding namespace I0505 08:20:49.309273 3788 namespace.go:102] [openshift-backplane-lpsre] adding namespace took 8.380494ms I0505 08:20:49.309287 3788 namespace.go:98] [default] adding namespace I0505 08:20:49.309901 3788 namespace.go:102] [openshift-controller-manager-operator] adding namespace took 8.264663ms I0505 08:20:49.310612 3788 namespace.go:102] [openshift-ingress] adding namespace took 8.437317ms I0505 08:20:49.310632 3788 namespace.go:98] [openshift-cloud-credential-operator] adding namespace I0505 08:20:49.311100 3788 namespace.go:102] [openshift-authentication] adding namespace took 8.180672ms I0505 08:20:49.311822 3788 namespace.go:102] [openshift-dns] adding namespace took 8.370869ms I0505 08:20:49.311838 3788 namespace.go:98] [openshift-machine-config-operator] adding namespace I0505 08:20:49.312394 3788 namespace.go:102] [openshift-etcd] adding namespace took 8.315772ms I0505 08:20:49.313029 3788 namespace.go:102] [openshift-kube-apiserver] adding namespace took 8.451556ms I0505 08:20:49.313558 3788 namespace.go:102] [openshift-config-operator] adding namespace took 8.39971ms I0505 08:20:49.313577 3788 namespace.go:98] [openshift-apiserver-operator] adding namespace I0505 08:20:49.314151 3788 namespace.go:102] [openshift-cluster-storage-operator] adding namespace took 8.389981ms I0505 08:20:49.314170 3788 namespace.go:98] [openshift-dns-operator] adding namespace I0505 08:20:49.314713 3788 namespace.go:102] [openshift-cluster-samples-operator] adding namespace took 8.399298ms I0505 08:20:49.315220 3788 namespace.go:102] [openshift-insights] adding namespace took 8.335976ms I0505 08:20:49.315750 3788 namespace.go:102] [openshift-package-operator] adding namespace took 8.189932ms I0505 08:20:49.316358 3788 namespace.go:102] [openshift-cluster-node-tuning-operator] adding namespace took 8.247706ms I0505 08:20:49.316963 3788 namespace.go:102] [openshift-cluster-csi-drivers] adding namespace took 8.197628ms I0505 08:20:49.317474 3788 namespace.go:102] [default] adding namespace took 8.165014ms I0505 08:20:49.318335 3788 namespace.go:102] [openshift-cloud-credential-operator] adding namespace took 7.696181ms I0505 08:20:49.318988 3788 namespace.go:102] [openshift-machine-config-operator] adding namespace took 7.145245ms I0505 08:20:49.319660 3788 namespace.go:102] [openshift-apiserver-operator] adding namespace took 6.073247ms I0505 08:20:49.320328 3788 namespace.go:102] [openshift-dns-operator] adding namespace took 6.150162ms W0505 08:20:49.320444 3788 master.go:458] Unable to parse local node ip-10-0-1-71.ec2.internal chassis-id annotation. Chassis may be removed during sync W0505 08:20:49.320480 3788 master.go:471] Unable to parse remote node ip-10-0-2-96.ec2.internal chassis-id annotation. Chassis may be removed during sync I0505 08:20:49.320489 3788 master.go:481] Removing stale chassis with ID/Name: 73100b4c-91b6-4220-a10d-36c72e0b3758, hostname: ip-10-0-1-71.ec2.internal I0505 08:20:49.321005 3788 master.go:548] Adding or Updating local node "ip-10-0-1-71.ec2.internal" for network "default" I0505 08:20:49.321148 3788 obj_retry.go:578] Failed to create *v1.Node ip-10-0-2-96.ec2.internal, error: adding or updating remote node chassis ip-10-0-2-96.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-2-96.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-2-96.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-96.ec2.internal I0505 08:20:49.322634 3788 zone_ic_handler.go:195] Time taken to create transit switch: 635.894µs I0505 08:20:49.322654 3788 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-0-32.ec2.internal for the network default I0505 08:20:49.323963 3788 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-0-32.ec2.internal" on network "default" took: 1.981981ms I0505 08:20:49.325497 3788 zone_ic_handler.go:203] Creating interconnect resources for local zone node ip-10-0-1-71.ec2.internal for the network default I0505 08:20:49.325714 3788 zone_ic_handler.go:195] Time taken to create transit switch: 201.109µs I0505 08:20:49.327201 3788 default_network_controller.go:823] Node add failed for ip-10-0-1-71.ec2.internal, will try again later: [k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-71.ec2.internal, error getting gateway config for node ip-10-0-1-71.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-71.ec2.internal", failed to update chassis to local for local node ip-10-0-1-71.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-1-71.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-71.ec2.internal] E0505 08:20:49.327252 3788 obj_retry.go:575] Failed to create *v1.Node ip-10-0-1-71.ec2.internal, error: [k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-71.ec2.internal, error getting gateway config for node ip-10-0-1-71.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-71.ec2.internal", failed to update chassis to local for local node ip-10-0-1-71.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-1-71.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-71.ec2.internal] I0505 08:20:49.327316 3788 services_controller.go:192] Starting controller ovn-lb-controller for network=default I0505 08:20:49.327345 3788 services_controller.go:200] Waiting for node tracker handler to sync for network=default I0505 08:20:49.327365 3788 shared_informer.go:349] "Waiting for caches to sync" controller="node-tracker-controller" I0505 08:20:49.327389 3788 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-32.ec2.internal in network "default" I0505 08:20:49.327395 3788 event.go:377] Event(v1.ObjectReference{Kind:"Node", Namespace:"", Name:"ip-10-0-1-71.ec2.internal", UID:"0026441f-432f-47e6-93f1-91cf929c8f8a", APIVersion:"v1", ResourceVersion:"8979", FieldPath:""}): type: 'Warning' reason: 'ErrorAddingResource' [k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-71.ec2.internal, error getting gateway config for node ip-10-0-1-71.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-71.ec2.internal", failed to update chassis to local for local node ip-10-0-1-71.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-1-71.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-71.ec2.internal] I0505 08:20:49.327530 3788 node_tracker.go:169] Node ip-10-0-0-32.ec2.internal switch + router changed, syncing services in network "default" I0505 08:20:49.327549 3788 services_controller.go:574] Full service sync requested for network=default I0505 08:20:49.327562 3788 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-71.ec2.internal in network "default" I0505 08:20:49.327574 3788 node_tracker.go:235] Node ip-10-0-1-71.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-71.ec2.internal" I0505 08:20:49.327582 3788 node_tracker.go:169] Node ip-10-0-1-71.ec2.internal switch + router changed, syncing services in network "default" I0505 08:20:49.327587 3788 services_controller.go:574] Full service sync requested for network=default I0505 08:20:49.327593 3788 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-96.ec2.internal in network "default" I0505 08:20:49.327601 3788 node_tracker.go:235] Node ip-10-0-2-96.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-96.ec2.internal" I0505 08:20:49.327607 3788 node_tracker.go:169] Node ip-10-0-2-96.ec2.internal switch + router changed, syncing services in network "default" I0505 08:20:49.327611 3788 services_controller.go:574] Full service sync requested for network=default I0505 08:20:49.353285 3788 iptables.go:108] Creating table: filter chain: FORWARD I0505 08:20:49.359648 3788 iptables.go:108] Creating table: filter chain: OUTPUT I0505 08:20:49.366128 3788 gateway_shared_intf.go:1816] Gateway Creation Complete I0505 08:20:49.366147 3788 kube.go:131] Setting annotations map[k8s.ovn.org/gateway-mtu-support: k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-1-71.ec2.internal","mac-address":"0a:ff:c4:6f:76:79","ip-addresses":["10.0.1.71/24"],"ip-address":"10.0.1.71/24","next-hops":["10.0.1.1"],"next-hop":"10.0.1.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:73100b4c-91b6-4220-a10d-36c72e0b3758 k8s.ovn.org/node-encap-ips:["10.0.1.71"] k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.1.71/24"} k8s.ovn.org/zone-name:ip-10-0-1-71.ec2.internal] on node ip-10-0-1-71.ec2.internal I0505 08:20:49.382257 3788 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-71.ec2.internal in network "default" I0505 08:20:49.382277 3788 master.go:548] Adding or Updating local node "ip-10-0-1-71.ec2.internal" for network "default" I0505 08:20:49.382317 3788 node_tracker.go:169] Node ip-10-0-1-71.ec2.internal switch + router changed, syncing services in network "default" I0505 08:20:49.382336 3788 services_controller.go:574] Full service sync requested for network=default I0505 08:20:49.387288 3788 default_node_network_controller.go:519] Node connection status = connected W0505 08:20:49.388141 3788 gateway.go:207] Unable to fetch podIPs for pod openshift-multus/network-metrics-daemon-b2tcb: pod openshift-multus/network-metrics-daemon-b2tcb: no pod IPs found W0505 08:20:49.388163 3788 gateway.go:207] Unable to fetch podIPs for pod openshift-network-diagnostics/network-check-target-wg7jm: pod openshift-network-diagnostics/network-check-target-wg7jm: no pod IPs found I0505 08:20:49.389314 3788 zone_ic_handler.go:203] Creating interconnect resources for local zone node ip-10-0-1-71.ec2.internal for the network default I0505 08:20:49.389524 3788 zone_ic_handler.go:195] Time taken to create transit switch: 193.759µs I0505 08:20:49.428404 3788 shared_informer.go:356] "Caches are synced" controller="node-tracker-controller" I0505 08:20:49.428430 3788 services_controller.go:208] Setting up event handlers for services for network=default I0505 08:20:49.428520 3788 services_controller.go:218] Setting up event handlers for endpoint slices for network=default I0505 08:20:49.428570 3788 services_controller.go:233] Waiting for service and endpoint handlers to sync for network=default I0505 08:20:49.428587 3788 shared_informer.go:349] "Waiting for caches to sync" controller="ovn-lb-controller" I0505 08:20:49.529054 3788 shared_informer.go:356] "Caches are synced" controller="ovn-lb-controller" I0505 08:20:49.529079 3788 repair.go:60] Starting repairing loop for services I0505 08:20:49.529163 3788 repair.go:131] Deleted 0 stale service LBs I0505 08:20:49.529207 3788 repair.go:137] Deleted 0 stale Chassis Template Vars I0505 08:20:49.529226 3788 repair.go:62] Finished repairing loop for services: 148.962µs I0505 08:20:49.529249 3788 services_controller.go:363] Controller cache of 0 load balancers initialized for 0 services for network=default I0505 08:20:49.529258 3788 services_controller.go:254] Starting workers for network=default I0505 08:20:49.529584 3788 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-wg7jm] creating logical port openshift-network-diagnostics_network-check-target-wg7jm for pod on switch ip-10-0-1-71.ec2.internal I0505 08:20:49.529725 3788 obj_retry.go:578] Failed to create *v1.Pod openshift-network-diagnostics/network-check-target-zqqrr, error: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-zqqrr: suppressed error logged: pod openshift-network-diagnostics/network-check-target-zqqrr: no pod IPs found I0505 08:20:49.529587 3788 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-b2tcb] creating logical port openshift-multus_network-metrics-daemon-b2tcb for pod on switch ip-10-0-1-71.ec2.internal I0505 08:20:49.529764 3788 kube.go:256] Updating pod openshift-network-diagnostics/network-check-target-wg7jm I0505 08:20:49.529835 3788 kube.go:256] Updating pod openshift-multus/network-metrics-daemon-b2tcb I0505 08:20:49.529863 3788 obj_retry.go:578] Failed to create *v1.Pod openshift-multus/network-metrics-daemon-hd7q7, error: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-hd7q7: suppressed error logged: pod openshift-multus/network-metrics-daemon-hd7q7: no pod IPs found W0505 08:20:49.535356 3788 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0505 08:20:49.545269 3788 pod.go:62] [openshift-network-diagnostics/network-check-target-wg7jm] pod update took 15.543169ms I0505 08:20:49.545290 3788 base_network_controller_pods.go:951] [default/openshift-network-diagnostics/network-check-target-wg7jm] addLogicalPort annotation time took 15.638003ms I0505 08:20:49.545846 3788 pod.go:62] [openshift-multus/network-metrics-daemon-b2tcb] pod update took 16.026805ms I0505 08:20:49.545866 3788 base_network_controller_pods.go:951] [default/openshift-multus/network-metrics-daemon-b2tcb] addLogicalPort annotation time took 16.052785ms I0505 08:20:49.546409 3788 pods.go:271] [openshift-network-diagnostics/network-check-target-wg7jm] addLogicalPort took 16.842592ms, libovsdb time 841.985µs I0505 08:20:49.546727 3788 pods.go:271] [openshift-multus/network-metrics-daemon-b2tcb] addLogicalPort took 17.151659ms, libovsdb time 627.871µs I0505 08:20:49.547189 3788 admin_network_policy_controller.go:227] Starting controller default-network-controller I0505 08:20:49.547209 3788 shared_informer.go:349] "Waiting for caches to sync" controller="default-network-controller" I0505 08:20:49.547223 3788 shared_informer.go:356] "Caches are synced" controller="default-network-controller" I0505 08:20:49.547229 3788 admin_network_policy_controller.go:237] Repairing Admin Network Policies I0505 08:20:49.547400 3788 repair.go:29] Repairing admin network policies took 165.868µs I0505 08:20:49.547510 3788 repair.go:92] Repairing baseline admin network policies took 102.703µs I0505 08:20:49.547870 3788 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles for network default I0505 08:20:49.547898 3788 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles added to peer address sets [] I0505 08:20:49.550396 3788 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles resources completed, update namespace loglevel I0505 08:20:49.550425 3788 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-egress for network default I0505 08:20:49.550438 3788 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-egress added to peer address sets [] I0505 08:20:49.552396 3788 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-egress resources completed, update namespace loglevel I0505 08:20:49.552422 3788 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-egress-to-api-server for network default I0505 08:20:49.552433 3788 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-egress-to-api-server added to peer address sets [] I0505 08:20:49.554050 3788 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-egress-to-api-server resources completed, update namespace loglevel I0505 08:20:49.554073 3788 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-all-egress for network default I0505 08:20:49.554081 3788 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-all-egress added to peer address sets [] I0505 08:20:49.555653 3788 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-all-egress resources completed, update namespace loglevel I0505 08:20:49.555674 3788 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-olm for network default I0505 08:20:49.555681 3788 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-olm added to peer address sets [] I0505 08:20:49.556254 3788 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-olm resources completed, update namespace loglevel I0505 08:20:49.556273 3788 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/unpack-bundles for network default I0505 08:20:49.556284 3788 base_network_controller_policy.go:1054] Policy openshift-marketplace/unpack-bundles added to peer address sets [] I0505 08:20:49.558246 3788 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/unpack-bundles resources completed, update namespace loglevel I0505 08:20:49.558265 3788 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-webhook for network default I0505 08:20:49.558277 3788 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-webhook added to peer address sets [] I0505 08:20:49.558815 3788 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-webhook resources completed, update namespace loglevel I0505 08:20:49.558835 3788 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server for network default I0505 08:20:49.558843 3788 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server added to peer address sets [] I0505 08:20:49.559413 3788 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server resources completed, update namespace loglevel I0505 08:20:49.559431 3788 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/catalog-operator for network default I0505 08:20:49.559445 3788 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/catalog-operator added to peer address sets [] I0505 08:20:49.561853 3788 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/catalog-operator resources completed, update namespace loglevel I0505 08:20:49.561874 3788 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/default-deny for network default I0505 08:20:49.561882 3788 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/default-deny added to peer address sets [] I0505 08:20:49.562207 3788 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/default-deny resources completed, update namespace loglevel I0505 08:20:49.562226 3788 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/default-deny for network default I0505 08:20:49.562234 3788 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/default-deny added to peer address sets [] I0505 08:20:49.562640 3788 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/default-deny resources completed, update namespace loglevel I0505 08:20:49.562666 3788 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics for network default I0505 08:20:49.562679 3788 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics added to peer address sets [] I0505 08:20:49.563260 3788 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics resources completed, update namespace loglevel I0505 08:20:49.563289 3788 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-insights for network default I0505 08:20:49.563299 3788 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-insights added to peer address sets [] I0505 08:20:49.564575 3788 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-insights resources completed, update namespace loglevel I0505 08:20:49.564593 3788 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/package-server-manager for network default I0505 08:20:49.564603 3788 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/package-server-manager added to peer address sets [] I0505 08:20:49.565968 3788 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/package-server-manager resources completed, update namespace loglevel I0505 08:20:49.565991 3788 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/olm-operator for network default I0505 08:20:49.566007 3788 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/olm-operator added to peer address sets [] I0505 08:20:49.567575 3788 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/olm-operator resources completed, update namespace loglevel I0505 08:20:49.567594 3788 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/packageserver for network default I0505 08:20:49.567604 3788 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/packageserver added to peer address sets [] I0505 08:20:49.568811 3788 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/packageserver resources completed, update namespace loglevel I0505 08:20:49.568833 3788 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range for network default I0505 08:20:49.568843 3788 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range added to peer address sets [] I0505 08:20:49.569374 3788 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range resources completed, update namespace loglevel I0505 08:20:49.569391 3788 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range for network default I0505 08:20:49.569399 3788 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range added to peer address sets [] I0505 08:20:49.569914 3788 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range resources completed, update namespace loglevel I0505 08:20:49.569931 3788 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-version/default-deny for network default I0505 08:20:49.569938 3788 base_network_controller_policy.go:1054] Policy openshift-cluster-version/default-deny added to peer address sets [] I0505 08:20:49.571265 3788 base_network_controller_policy.go:1232] Create network policy openshift-cluster-version/default-deny resources completed, update namespace loglevel I0505 08:20:49.571286 3788 base_network_controller_policy.go:1179] Adding network policy openshift-operators/default-allow-all for network default I0505 08:20:49.571296 3788 base_network_controller_policy.go:1054] Policy openshift-operators/default-allow-all added to peer address sets [] I0505 08:20:49.573000 3788 base_network_controller_policy.go:1232] Create network policy openshift-operators/default-allow-all resources completed, update namespace loglevel I0505 08:20:49.573020 3788 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-metrics for network default I0505 08:20:49.573029 3788 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-metrics added to peer address sets [] I0505 08:20:49.573515 3788 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-metrics resources completed, update namespace loglevel I0505 08:20:49.573534 3788 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-pprof for network default I0505 08:20:49.573553 3788 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-pprof added to peer address sets [] I0505 08:20:49.574030 3788 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-pprof resources completed, update namespace loglevel I0505 08:20:49.574049 3788 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-all-egress for network default I0505 08:20:49.574057 3788 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-all-egress added to peer address sets [] I0505 08:20:49.574606 3788 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-all-egress resources completed, update namespace loglevel I0505 08:20:49.574624 3788 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-to-dns for network default I0505 08:20:49.575404 3788 pod_selector_address_set.go:213] Created shared address set for pod selector LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},} I0505 08:20:49.575421 3788 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0505 08:20:49.576406 3788 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-to-dns resources completed, update namespace loglevel I0505 08:20:49.576425 3788 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/default-deny-all for network default I0505 08:20:49.576432 3788 base_network_controller_policy.go:1054] Policy openshift-marketplace/default-deny-all added to peer address sets [] I0505 08:20:49.576770 3788 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/default-deny-all resources completed, update namespace loglevel I0505 08:20:49.576795 3788 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/marketplace-operator for network default I0505 08:20:49.576809 3788 base_network_controller_policy.go:1054] Policy openshift-marketplace/marketplace-operator added to peer address sets [] I0505 08:20:49.577868 3788 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/marketplace-operator resources completed, update namespace loglevel I0505 08:20:49.577886 3788 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/default-deny-all-traffic for network default I0505 08:20:49.577893 3788 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/default-deny-all-traffic added to peer address sets [] I0505 08:20:49.578222 3788 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/default-deny-all-traffic resources completed, update namespace loglevel I0505 08:20:49.578243 3788 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-egress-to-api-server for network default I0505 08:20:49.578256 3788 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-egress-to-api-server added to peer address sets [] I0505 08:20:49.578785 3788 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-egress-to-api-server resources completed, update namespace loglevel I0505 08:20:49.578804 3788 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-to-dns for network default I0505 08:20:49.578829 3788 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0505 08:20:49.579595 3788 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-to-dns resources completed, update namespace loglevel I0505 08:20:49.586914 3788 egressfirewall.go:374] Starting EgressFirewall controller I0505 08:20:49.586958 3788 controller.go:133] Adding controller egress-firewall-controller event handlers I0505 08:20:49.586958 3788 controller.go:133] Adding controller egress-firewall-controller-node event handlers I0505 08:20:49.587007 3788 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller-node" I0505 08:20:49.587015 3788 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller" I0505 08:20:49.587026 3788 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller" I0505 08:20:49.587029 3788 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller-node" I0505 08:20:49.587346 3788 controller.go:157] Starting controller egress-firewall-controller with 1 workers I0505 08:20:49.587360 3788 controller.go:157] Starting controller egress-firewall-controller-node with 1 workers I0505 08:20:49.587374 3788 controller.go:157] Starting controller egress-firewall-controller-NAD with 1 workers I0505 08:20:49.587387 3788 egressqos.go:193] Setting up event handlers for EgressQoS I0505 08:20:49.587453 3788 egressfirewall.go:723] Syncing node "ip-10-0-0-32.ec2.internal" for egress firewall I0505 08:20:49.587482 3788 egressfirewall.go:723] Syncing node "ip-10-0-1-71.ec2.internal" for egress firewall I0505 08:20:49.587487 3788 egressfirewall.go:723] Syncing node "ip-10-0-2-96.ec2.internal" for egress firewall I0505 08:20:49.587529 3788 egressqos.go:245] Starting EgressQoS Controller I0505 08:20:49.587540 3788 shared_informer.go:349] "Waiting for caches to sync" controller="egressqosnodes" I0505 08:20:49.587549 3788 shared_informer.go:356] "Caches are synced" controller="egressqosnodes" I0505 08:20:49.587558 3788 shared_informer.go:349] "Waiting for caches to sync" controller="egressqospods" I0505 08:20:49.587566 3788 shared_informer.go:356] "Caches are synced" controller="egressqospods" I0505 08:20:49.587574 3788 shared_informer.go:349] "Waiting for caches to sync" controller="egressqos" I0505 08:20:49.587581 3788 shared_informer.go:356] "Caches are synced" controller="egressqos" I0505 08:20:49.587587 3788 egressqos.go:259] Repairing EgressQoSes I0505 08:20:49.587593 3788 egressqos.go:399] Starting repairing loop for egressqos I0505 08:20:49.587686 3788 egressqos.go:401] Finished repairing loop for egressqos: 94.552µs I0505 08:20:49.587707 3788 egressservice_zone.go:132] Setting up event handlers for Egress Services I0505 08:20:49.587742 3788 egressqos.go:1007] Processing sync for EgressQoS node ip-10-0-1-71.ec2.internal I0505 08:20:49.587794 3788 egressqos.go:1010] Finished syncing EgressQoS node ip-10-0-1-71.ec2.internal : 55.019µs I0505 08:20:49.587928 3788 egressservice_zone.go:210] Starting Egress Services Controller I0505 08:20:49.587941 3788 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0505 08:20:49.587946 3788 shared_informer.go:356] "Caches are synced" controller="egressservices" I0505 08:20:49.587952 3788 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0505 08:20:49.587957 3788 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0505 08:20:49.587962 3788 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0505 08:20:49.587967 3788 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0505 08:20:49.587972 3788 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_nodes" I0505 08:20:49.587976 3788 shared_informer.go:356] "Caches are synced" controller="egressservices_nodes" I0505 08:20:49.587980 3788 egressservice_zone.go:228] Repairing Egress Services I0505 08:20:49.588273 3788 master_controller.go:88] Starting Admin Policy Based Route Controller I0505 08:20:49.588284 3788 external_controller.go:278] Starting Admin Policy Based Route Controller I0505 08:20:49.588294 3788 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-71.ec2.internal I0505 08:20:49.588304 3788 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-71.ec2.internal: 12.475µs I0505 08:20:49.588313 3788 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-96.ec2.internal I0505 08:20:49.588318 3788 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-96.ec2.internal: 5.936µs I0505 08:20:49.588325 3788 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-32.ec2.internal I0505 08:20:49.588336 3788 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-32.ec2.internal: 12.2µs I0505 08:20:49.588398 3788 default_network_controller.go:585] Completing all the Watchers took 316.239695ms I0505 08:20:49.588412 3788 default_network_controller.go:589] Starting unidling controllers I0505 08:20:49.588425 3788 unidle.go:46] Registering OVN SB ControllerEvent handler I0505 08:20:49.588433 3788 unidle.go:63] Populating Initial ContollerEvent events I0505 08:20:49.588447 3788 unidle.go:79] Setting up event handlers for services I0505 08:20:49.588537 3788 ovnkube.go:545] Waiting for OVN northbound database changes to be processed by ovn-controller I0505 08:20:49.887696 3788 default_node_network_controller.go:519] Node connection status = connected I0505 08:20:50.386874 3788 default_node_network_controller.go:519] Node connection status = connected I0505 08:20:50.888102 3788 default_node_network_controller.go:519] Node connection status = connected I0505 08:20:51.386970 3788 default_node_network_controller.go:519] Node connection status = connected I0505 08:20:51.614043 3788 ovnkube.go:550] Finished waiting for OVN northbound database changes to be processed by ovn-controller I0505 08:20:51.887329 3788 default_node_network_controller.go:519] Node connection status = connected I0505 08:20:51.905975 3788 bridgeconfig.go:537] Gateway is ready I0505 08:20:51.905998 3788 gateway_shared_intf.go:1733] Creating Gateway Openflow Manager I0505 08:20:51.919119 3788 node_ip_handler_linux.go:247] Node primary address changed to 10.0.1.71. Updating OVN encap IP. I0505 08:20:51.925286 3788 node_ip_handler_linux.go:542] Will not update encap IP 10.0.1.71 - it is already configured I0505 08:20:51.925304 3788 node_ip_handler_linux.go:485] Node address changed to map[10.0.1.71/24:{} 172.20.0.1/32:{}]. Updating annotations. I0505 08:20:51.925528 3788 kube.go:131] Setting annotations map[k8s.ovn.org/host-cidrs:["10.0.1.71/24","172.20.0.1/32"] k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-1-71.ec2.internal","mac-address":"0a:ff:c4:6f:76:79","ip-addresses":["10.0.1.71/24"],"ip-address":"10.0.1.71/24","next-hops":["10.0.1.1"],"next-hop":"10.0.1.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:73100b4c-91b6-4220-a10d-36c72e0b3758 k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.1.71/24"}] on node ip-10-0-1-71.ec2.internal I0505 08:20:51.942983 3788 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-71.ec2.internal I0505 08:20:51.943007 3788 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-71.ec2.internal: 27.196µs I0505 08:20:51.943008 3788 egressfirewall.go:723] Syncing node "ip-10-0-1-71.ec2.internal" for egress firewall I0505 08:20:51.943073 3788 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-71.ec2.internal in network "default" I0505 08:20:51.943104 3788 master.go:548] Adding or Updating local node "ip-10-0-1-71.ec2.internal" for network "default" I0505 08:20:51.943139 3788 node_tracker.go:169] Node ip-10-0-1-71.ec2.internal switch + router changed, syncing services in network "default" I0505 08:20:51.943158 3788 services_controller.go:574] Full service sync requested for network=default I0505 08:20:51.943261 3788 gateway_shared_intf.go:2023] Setting OVN Masquerade route with source: 10.0.1.71 I0505 08:20:51.943375 3788 kube.go:131] Setting annotations map[k8s.ovn.org/node-masquerade-subnet:{"ipv4":"169.254.0.0/17","ipv6":"fd69::/112"}] on node ip-10-0-1-71.ec2.internal W0505 08:20:51.943397 3788 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0505 08:20:51.947926 3788 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-1-71.ec2.internal. Updating no re-route policies I0505 08:20:51.947996 3788 obj_retry.go:334] Retry object setup: *v1.Pod openshift-cluster-node-tuning-operator/tuned-xbntd I0505 08:20:51.948014 3788 obj_retry.go:334] Retry object setup: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-1-71.ec2.internal I0505 08:20:51.948019 3788 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-wg7jm I0505 08:20:51.948025 3788 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/multus-7gffq I0505 08:20:51.948028 3788 obj_retry.go:398] Adding new object: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-1-71.ec2.internal I0505 08:20:51.948032 3788 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-wg7jm I0505 08:20:51.947998 3788 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-zqqrr I0505 08:20:51.948033 3788 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-1-71.ec2.internal I0505 08:20:51.948045 3788 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-operator/iptables-alerter-s64td I0505 08:20:51.948049 3788 obj_retry.go:398] Adding new object: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-1-71.ec2.internal I0505 08:20:51.948048 3788 obj_retry.go:334] Retry object setup: *v1.Pod openshift-image-registry/node-ca-vrr54 I0505 08:20:51.948058 3788 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-b2tcb I0505 08:20:51.948059 3788 obj_retry.go:334] Retry object setup: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-xdt46 I0505 08:20:51.948061 3788 obj_retry.go:398] Adding new object: *v1.Pod openshift-image-registry/node-ca-vrr54 I0505 08:20:51.948064 3788 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-b2tcb I0505 08:20:51.948066 3788 obj_retry.go:398] Adding new object: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-xdt46 I0505 08:20:51.948067 3788 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/multus-additional-cni-plugins-z4knk I0505 08:20:51.948070 3788 obj_retry.go:334] Retry object setup: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-5pw7z I0505 08:20:51.948076 3788 obj_retry.go:418] Retry successful for *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-xdt46 after 0 failed attempt(s) I0505 08:20:51.948078 3788 obj_retry.go:418] Retry successful for *v1.Pod openshift-image-registry/node-ca-vrr54 after 0 failed attempt(s) I0505 08:20:51.948080 3788 obj_retry.go:398] Adding new object: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-5pw7z I0505 08:20:51.948079 3788 obj_retry.go:334] Retry object setup: *v1.Pod openshift-dns/node-resolver-fzg45 I0505 08:20:51.948084 3788 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-hd7q7 I0505 08:20:51.948092 3788 obj_retry.go:398] Adding new object: *v1.Pod openshift-dns/node-resolver-fzg45 I0505 08:20:51.948094 3788 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-hd7q7 I0505 08:20:51.948097 3788 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-b2tcb] creating logical port openshift-multus_network-metrics-daemon-b2tcb for pod on switch ip-10-0-1-71.ec2.internal I0505 08:20:51.948102 3788 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/konnectivity-agent-cwcx2 I0505 08:20:51.948108 3788 obj_retry.go:418] Retry successful for *v1.Pod openshift-dns/node-resolver-fzg45 after 0 failed attempt(s) I0505 08:20:51.948113 3788 obj_retry.go:398] Adding new object: *v1.Pod kube-system/konnectivity-agent-cwcx2 I0505 08:20:51.948062 3788 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-wg7jm] creating logical port openshift-network-diagnostics_network-check-target-wg7jm for pod on switch ip-10-0-1-71.ec2.internal I0505 08:20:51.948134 3788 obj_retry.go:418] Retry successful for *v1.Pod kube-system/konnectivity-agent-cwcx2 after 0 failed attempt(s) I0505 08:20:51.948038 3788 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-zqqrr I0505 08:20:51.948033 3788 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/multus-7gffq I0505 08:20:51.948172 3788 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/multus-7gffq after 0 failed attempt(s) I0505 08:20:51.948017 3788 obj_retry.go:398] Adding new object: *v1.Pod openshift-cluster-node-tuning-operator/tuned-xbntd I0505 08:20:51.948186 3788 obj_retry.go:409] Retry add failed for *v1.Pod openshift-network-diagnostics/network-check-target-zqqrr, will try again later: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-zqqrr: suppressed error logged: pod openshift-network-diagnostics/network-check-target-zqqrr: no pod IPs found I0505 08:20:51.948072 3788 obj_retry.go:418] Retry successful for *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-1-71.ec2.internal after 0 failed attempt(s) I0505 08:20:51.948191 3788 obj_retry.go:418] Retry successful for *v1.Pod openshift-cluster-node-tuning-operator/tuned-xbntd after 0 failed attempt(s) I0505 08:20:51.948134 3788 obj_retry.go:409] Retry add failed for *v1.Pod openshift-multus/network-metrics-daemon-hd7q7, will try again later: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-hd7q7: suppressed error logged: pod openshift-multus/network-metrics-daemon-hd7q7: no pod IPs found I0505 08:20:51.948050 3788 obj_retry.go:418] Retry successful for *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-1-71.ec2.internal after 0 failed attempt(s) I0505 08:20:51.948081 3788 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/multus-additional-cni-plugins-z4knk I0505 08:20:51.948221 3788 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/multus-additional-cni-plugins-z4knk after 0 failed attempt(s) I0505 08:20:51.948095 3788 obj_retry.go:418] Retry successful for *v1.Pod openshift-ovn-kubernetes/ovnkube-node-5pw7z after 0 failed attempt(s) I0505 08:20:51.948051 3788 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-operator/iptables-alerter-s64td I0505 08:20:51.948256 3788 obj_retry.go:418] Retry successful for *v1.Pod openshift-network-operator/iptables-alerter-s64td after 0 failed attempt(s) I0505 08:20:51.948823 3788 pods.go:271] [openshift-network-diagnostics/network-check-target-wg7jm] addLogicalPort took 768.778µs, libovsdb time 283.186µs I0505 08:20:51.948836 3788 obj_retry.go:418] Retry successful for *v1.Pod openshift-network-diagnostics/network-check-target-wg7jm after 0 failed attempt(s) I0505 08:20:51.948900 3788 pods.go:271] [openshift-multus/network-metrics-daemon-b2tcb] addLogicalPort took 812.979µs, libovsdb time 183.952µs I0505 08:20:51.948907 3788 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/network-metrics-daemon-b2tcb after 0 failed attempt(s) I0505 08:20:51.961227 3788 gateway_shared_intf.go:1799] Creating Gateway Node Port Watcher I0505 08:20:51.973738 3788 iptables.go:108] Creating table: mangle chain: OUTPUT I0505 08:20:51.977111 3788 iptables.go:108] Creating table: nat chain: OUTPUT I0505 08:20:52.031674 3788 iptables.go:108] Creating table: nat chain: PREROUTING I0505 08:20:52.090995 3788 iptables.go:108] Creating table: filter chain: FORWARD I0505 08:20:52.110714 3788 gateway_shared_intf.go:2124] Ensuring IP Neighbor entry for: 169.254.0.1 W0505 08:20:52.110834 3788 gateway_shared_intf.go:2130] Failed to remove IP neighbor entry for ip 169.254.0.1, on iface br-ex: failed to delete neighbour entry 169.254.0.1 : no such file or directory I0505 08:20:52.110905 3788 gateway_shared_intf.go:2124] Ensuring IP Neighbor entry for: 169.254.0.4 W0505 08:20:52.110990 3788 gateway_shared_intf.go:2130] Failed to remove IP neighbor entry for ip 169.254.0.4, on iface br-ex: failed to delete neighbour entry 169.254.0.4 : no such file or directory I0505 08:20:52.111050 3788 gateway_init.go:325] Gateway and management port readiness took 2.728988907s I0505 08:20:52.111078 3788 controller.go:133] Adding controller [node-nad-controller NAD controller] event handlers I0505 08:20:52.111116 3788 shared_informer.go:349] "Waiting for caches to sync" controller="[node-nad-controller NAD controller]" I0505 08:20:52.111129 3788 shared_informer.go:356] "Caches are synced" controller="[node-nad-controller NAD controller]" I0505 08:20:52.111138 3788 nad_controller.go:541] [node-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0505 08:20:52.111202 3788 controller.go:157] Starting controller [node-nad-controller NAD controller] with 1 workers I0505 08:20:52.111288 3788 nad_controller.go:580] [node-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 56.219µs I0505 08:20:52.111539 3788 network_controller.go:373] [node-nad-controller network controller]: syncing all networks I0505 08:20:52.111565 3788 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 15.796µs I0505 08:20:52.111573 3788 network_controller.go:384] [node-nad-controller network controller]: finished syncing all networks. Time taken: 36.118µs I0505 08:20:52.111580 3788 controller.go:157] Starting controller [node-nad-controller network controller] with 1 workers I0505 08:20:52.111590 3788 nad_controller.go:379] [node-nad-controller NAD controller]: started I0505 08:20:52.111596 3788 default_node_network_controller.go:899] Starting the default node network controller I0505 08:20:52.111601 3788 gateway_init.go:335] Initializing Gateway Functionality for gateway Start I0505 08:20:52.111618 3788 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 13.073µs I0505 08:20:52.111788 3788 default_node_network_controller.go:1460] MTU (9001) of network interface br-ex is big enough to deal with Geneve header overhead (sum 8959). I0505 08:20:52.111797 3788 default_node_network_controller.go:935] Waiting for gateway and management port readiness... I0505 08:20:52.111820 3788 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000a27780 0xc0003418f0 0xc000341960 0xc0003419d0 0xc000341a40 0xc000341b20 0xc000341b90 0xc000a66d20 0xc000341c00 0xc000341c70 0xc000a66d70 0xc00657e4d0 0xc000341ce0 map[0x2ff9080:0xc004014ba0 0x2ff95c0:0xc004014d20 0x2ffa2e0:0xc00397a0c0 0x2ffa580:0xc004014b40 0x2ffad60:0xc004014c00 0x2ffb000:0xc004014c60 0x2ffb2a0:0xc004014cc0 0x2ffba80:0xc002814d80 0x2ffbd20:0xc002814de0 0x3026980:0xc000c1dda0 0x3026d20:0xc000c1de00 0x3027800:0xc000a57da0 0x3029160:0xc000f80d80 0x30298a0:0xc00397a060 0x302a720:0xc000c1dd40] 0xc00016c8c0 0} I0505 08:20:52.111890 3788 gateway.go:151] Starting gateway service sync I0505 08:20:52.112343 3788 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ITP I0505 08:20:52.114839 3788 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-NODEPORT I0505 08:20:52.138862 3788 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-EXTERNALIP I0505 08:20:52.140229 3788 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ETP I0505 08:20:52.141435 3788 gateway_iptables.go:474] Recreating iptables rules for table: mangle, chain: OVN-KUBE-ITP I0505 08:20:52.177557 3788 gateway.go:168] Gateway service sync done. Time taken: 65.652073ms I0505 08:20:52.178008 3788 healthcheck.go:145] Opening healthcheck "openshift-ingress/router-default" on port 31745 I0505 08:20:52.178630 3788 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000a27780 0xc0003418f0 0xc000341960 0xc0003419d0 0xc000341a40 0xc000341b20 0xc000341b90 0xc000a66d20 0xc000341c00 0xc000341c70 0xc000a66d70 0xc00657e4d0 0xc000341ce0 map[0x2ff9080:0xc004014ba0 0x2ff95c0:0xc004014d20 0x2ffa2e0:0xc00397a0c0 0x2ffa580:0xc004014b40 0x2ffad60:0xc004014c00 0x2ffb000:0xc004014c60 0x2ffb2a0:0xc004014cc0 0x2ffba80:0xc002814d80 0x2ffbd20:0xc002814de0 0x3026980:0xc000c1dda0 0x3026d20:0xc000c1de00 0x3027800:0xc000a57da0 0x3029160:0xc000f80d80 0x30298a0:0xc00397a060 0x302a720:0xc000c1dd40] 0xc00016c8c0 0} I0505 08:20:52.179114 3788 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000a27780 0xc0003418f0 0xc000341960 0xc0003419d0 0xc000341a40 0xc000341b20 0xc000341b90 0xc000a66d20 0xc000341c00 0xc000341c70 0xc000a66d70 0xc00657e4d0 0xc000341ce0 map[0x2ff9080:0xc004014ba0 0x2ff95c0:0xc004014d20 0x2ffa2e0:0xc00397a0c0 0x2ffa580:0xc004014b40 0x2ffad60:0xc004014c00 0x2ffb000:0xc004014c60 0x2ffb2a0:0xc004014cc0 0x2ffba80:0xc002814d80 0x2ffbd20:0xc002814de0 0x3026980:0xc000c1dda0 0x3026d20:0xc000c1de00 0x3027800:0xc000a57da0 0x3029160:0xc000f80d80 0x30298a0:0xc00397a060 0x302a720:0xc000c1dd40] 0xc00016c8c0 0} I0505 08:20:52.179158 3788 gateway.go:532] Reconciling gateway with updates I0505 08:20:52.184931 3788 gateway.go:339] Spawning Conntrack Rule Check Thread I0505 08:20:52.185019 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0505 08:20:52.185034 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0505 08:20:52.185040 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0505 08:20:52.185053 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0505 08:20:52.185058 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0505 08:20:52.185066 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0505 08:20:52.185076 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0505 08:20:52.185080 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0505 08:20:52.185078 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0505 08:20:52.185086 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0505 08:20:52.185035 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0505 08:20:52.185092 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/kubernetes I0505 08:20:52.185096 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0505 08:20:52.185098 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0505 08:20:52.185106 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/kubernetes I0505 08:20:52.185106 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0505 08:20:52.185116 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0505 08:20:52.185122 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0505 08:20:52.185124 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0505 08:20:52.185126 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0505 08:20:52.185132 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0505 08:20:52.185141 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0505 08:20:52.185148 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0505 08:20:52.185156 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0505 08:20:52.185157 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0505 08:20:52.185162 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0505 08:20:52.185163 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0505 08:20:52.185163 3788 port_claim.go:71] Svc openshift-ingress/router-default: port 32373 is already open, no action needed I0505 08:20:52.185165 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0505 08:20:52.185172 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0505 08:20:52.185174 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0505 08:20:52.185171 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0505 08:20:52.185123 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0505 08:20:52.185183 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0505 08:20:52.185160 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0505 08:20:52.185190 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0505 08:20:52.185198 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0505 08:20:52.185038 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0505 08:20:52.185194 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0505 08:20:52.185217 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0505 08:20:52.185218 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0505 08:20:52.185226 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0505 08:20:52.185144 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0505 08:20:52.185163 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0505 08:20:52.185243 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0505 08:20:52.185247 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0505 08:20:52.185250 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0505 08:20:52.185253 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0505 08:20:52.185242 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0505 08:20:52.185193 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0505 08:20:52.185278 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0505 08:20:52.185285 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0505 08:20:52.185289 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0505 08:20:52.185054 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0505 08:20:52.185303 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0505 08:20:52.185326 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0505 08:20:52.185279 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0505 08:20:52.185338 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0505 08:20:52.185342 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0505 08:20:52.185346 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0505 08:20:52.185348 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0505 08:20:52.185291 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0505 08:20:52.185352 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0505 08:20:52.185362 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0505 08:20:52.185360 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0505 08:20:52.185371 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0505 08:20:52.185375 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0505 08:20:52.185377 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0505 08:20:52.185379 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0505 08:20:52.185380 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0505 08:20:52.185386 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0505 08:20:52.185176 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0505 08:20:52.185397 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0505 08:20:52.185178 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0505 08:20:52.185172 3788 port_claim.go:71] Svc openshift-ingress/router-default: port 32723 is already open, no action needed I0505 08:20:52.185165 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/packageserver I0505 08:20:52.185424 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0505 08:20:52.185182 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0505 08:20:52.185432 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/packageserver I0505 08:20:52.185437 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0505 08:20:52.185439 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0505 08:20:52.185440 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0505 08:20:52.185450 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0505 08:20:52.185194 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0505 08:20:52.185190 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0505 08:20:52.185497 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0505 08:20:52.185499 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0505 08:20:52.185508 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0505 08:20:52.185200 3788 default_node_network_controller.go:944] Gateway and management port readiness took 73.394688ms I0505 08:20:52.185206 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0505 08:20:52.185266 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0505 08:20:52.185266 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift I0505 08:20:52.185586 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift I0505 08:20:52.185024 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0505 08:20:52.185129 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0505 08:20:52.185597 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0505 08:20:52.185192 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0505 08:20:52.185617 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0505 08:20:52.185603 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0505 08:20:52.185346 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0505 08:20:52.185678 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0505 08:20:52.185179 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0505 08:20:52.185703 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0505 08:20:52.186544 3788 node_ip_handler_linux.go:160] Node IP manager is running W0505 08:20:52.226538 3788 port_linux.go:245] Missing or unable to find route entry for subnet 10.128.0.0/14 via gateway 10.129.0.1 on link ovn-k8s-mp0 with MTU: 0 W0505 08:20:52.226727 3788 port_linux.go:245] Missing or unable to find route entry for subnet 169.254.0.3/32 via gateway 10.129.0.1 on link ovn-k8s-mp0 with MTU: 0 W0505 08:20:52.226891 3788 port_linux.go:261] Missing arp entry for MAC/IP binding (0a:58:0a:81:00:01/10.129.0.1) on link ovn-k8s-mp0 I0505 08:20:52.233351 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0505 08:20:52.233381 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0505 08:20:52.233388 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0505 08:20:52.233397 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0505 08:20:52.233405 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0505 08:20:52.233421 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0505 08:20:52.233442 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0505 08:20:52.233451 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0505 08:20:52.233470 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0505 08:20:52.233484 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0505 08:20:52.233496 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0505 08:20:52.233508 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0505 08:20:52.233514 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0505 08:20:52.233523 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0505 08:20:52.233530 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0505 08:20:52.233538 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0505 08:20:52.233546 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0505 08:20:52.233555 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0505 08:20:52.233562 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0505 08:20:52.233567 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0505 08:20:52.233571 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0505 08:20:52.274597 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0505 08:20:52.274619 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0505 08:20:52.274635 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0505 08:20:52.274642 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0505 08:20:52.274651 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0505 08:20:52.274659 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0505 08:20:52.274666 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0505 08:20:52.274671 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0505 08:20:52.274677 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0505 08:20:52.274682 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0505 08:20:52.274687 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0505 08:20:52.925412 3788 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-96.ec2.internal I0505 08:20:52.925439 3788 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-96.ec2.internal: 36.568µs I0505 08:20:52.925422 3788 egressfirewall.go:723] Syncing node "ip-10-0-2-96.ec2.internal" for egress firewall I0505 08:20:52.925451 3788 default_network_controller.go:1025] Node "ip-10-0-2-96.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false W0505 08:20:53.713964 3788 egressip_healthcheck.go:76] Health checking using insecure connection I0505 08:20:53.714013 3788 egressip_healthcheck.go:109] Starting Egress IP Health Server on 10.129.0.2:9107 I0505 08:20:53.714047 3788 default_node_network_controller.go:1380] Starting node controller node sync I0505 08:20:53.725855 3788 default_node_network_controller.go:1422] Node controller node sync done. Time taken: 11.790085ms I0505 08:20:53.725957 3788 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-32.ec2.internal", IP: 10.0.0.32 to PMTUD blocking rules I0505 08:20:53.725970 3788 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-32.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0505 08:20:53.731850 3788 healthcheck_node.go:124] "Starting node proxy healthz server" address="0.0.0.0:10256" I0505 08:20:53.732152 3788 egressservice_node.go:93] Setting up event handlers for Egress Services I0505 08:20:53.732348 3788 egressservice_node.go:183] Starting Egress Services Controller I0505 08:20:53.732365 3788 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0505 08:20:53.732376 3788 shared_informer.go:356] "Caches are synced" controller="egressservices" I0505 08:20:53.732381 3788 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0505 08:20:53.732386 3788 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0505 08:20:53.732394 3788 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0505 08:20:53.732398 3788 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0505 08:20:53.732403 3788 egressservice_node.go:197] Repairing Egress Services I0505 08:20:53.742821 3788 node_controller.go:43] Starting Admin Policy Based Route Node Controller I0505 08:20:53.742835 3788 external_controller.go:278] Starting Admin Policy Based Route Controller I0505 08:20:53.743374 3788 default_node_network_controller.go:1077] Egress IP for secondary host network is disabled I0505 08:20:53.743388 3788 link_network_manager.go:119] Link manager is running I0505 08:20:53.743396 3788 default_node_network_controller.go:1098] Default node network controller initialized and ready. I0505 08:20:53.743620 3788 ovspinning_linux.go:50] OVS CPU affinity pinning disabled I0505 08:20:53.743723 3788 vrf_manager.go:128] VRF manager is running I0505 08:20:53.743780 3788 node_controller_manager.go:437] Removing flows to drop GARP I0505 08:20:53.743789 3788 gateway.go:532] Reconciling gateway with updates I0505 08:20:53.768452 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0505 08:20:53.768483 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0505 08:20:53.768484 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0505 08:20:53.768498 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0505 08:20:53.768498 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0505 08:20:53.768444 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift I0505 08:20:53.768504 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0505 08:20:53.768514 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0505 08:20:53.768521 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0505 08:20:53.768524 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0505 08:20:53.768523 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0505 08:20:53.768533 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0505 08:20:53.768535 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0505 08:20:53.768539 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0505 08:20:53.768547 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0505 08:20:53.768548 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0505 08:20:53.768558 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0505 08:20:53.768560 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0505 08:20:53.768451 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0505 08:20:53.768567 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0505 08:20:53.768572 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0505 08:20:53.768535 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0505 08:20:53.768580 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0505 08:20:53.768582 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0505 08:20:53.768583 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0505 08:20:53.768586 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0505 08:20:53.768589 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0505 08:20:53.768591 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0505 08:20:53.768503 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0505 08:20:53.768596 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0505 08:20:53.768598 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0505 08:20:53.768603 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0505 08:20:53.768606 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0505 08:20:53.768608 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0505 08:20:53.768607 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/packageserver I0505 08:20:53.768617 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0505 08:20:53.768621 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0505 08:20:53.768623 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0505 08:20:53.768626 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0505 08:20:53.768629 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/packageserver I0505 08:20:53.768635 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0505 08:20:53.768633 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0505 08:20:53.768640 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/kubernetes I0505 08:20:53.768643 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0505 08:20:53.768644 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0505 08:20:53.768640 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0505 08:20:53.768655 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0505 08:20:53.768660 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0505 08:20:53.768663 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0505 08:20:53.768665 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0505 08:20:53.768671 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0505 08:20:53.768671 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0505 08:20:53.768616 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0505 08:20:53.768679 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0505 08:20:53.768687 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0505 08:20:53.768688 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0505 08:20:53.768691 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0505 08:20:53.768645 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0505 08:20:53.768514 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift I0505 08:20:53.768701 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0505 08:20:53.768704 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0505 08:20:53.768620 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0505 08:20:53.768713 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0505 08:20:53.768721 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0505 08:20:53.768731 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0505 08:20:53.768740 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0505 08:20:53.768583 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0505 08:20:53.768661 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0505 08:20:53.768693 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0505 08:20:53.768774 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0505 08:20:53.768780 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0505 08:20:53.768785 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0505 08:20:53.768788 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0505 08:20:53.768809 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0505 08:20:53.768701 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0505 08:20:53.768608 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0505 08:20:53.768572 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0505 08:20:53.768624 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0505 08:20:53.768846 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0505 08:20:53.768573 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0505 08:20:53.768572 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0505 08:20:53.768585 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0505 08:20:53.768901 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0505 08:20:53.768613 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0505 08:20:53.768914 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0505 08:20:53.768916 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0505 08:20:53.768593 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0505 08:20:53.768632 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0505 08:20:53.768629 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0505 08:20:53.768953 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0505 08:20:53.768648 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/kubernetes I0505 08:20:53.768642 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0505 08:20:53.768564 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0505 08:20:53.768989 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0505 08:20:53.768995 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0505 08:20:53.768660 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0505 08:20:53.769002 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0505 08:20:53.768664 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0505 08:20:53.769025 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0505 08:20:53.768676 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0505 08:20:53.769051 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0505 08:20:53.769055 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0505 08:20:53.768705 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0505 08:20:53.768711 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0505 08:20:53.768766 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0505 08:20:53.768490 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0505 08:20:53.769083 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0505 08:20:53.768858 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0505 08:20:53.768572 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0505 08:20:53.769110 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0505 08:20:53.768620 3788 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0505 08:20:53.769133 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0505 08:20:53.768926 3788 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0505 08:20:53.769162 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0505 08:20:53.769016 3788 port_claim.go:71] Svc openshift-ingress/router-default: port 32373 is already open, no action needed I0505 08:20:53.769174 3788 port_claim.go:71] Svc openshift-ingress/router-default: port 32723 is already open, no action needed I0505 08:20:53.768669 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0505 08:20:53.809653 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0505 08:20:53.809678 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0505 08:20:53.809686 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0505 08:20:53.809691 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0505 08:20:53.809703 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0505 08:20:53.809710 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0505 08:20:53.809715 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0505 08:20:53.809722 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0505 08:20:53.809727 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0505 08:20:53.850076 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0505 08:20:53.850104 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0505 08:20:53.850115 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0505 08:20:53.850124 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0505 08:20:53.850131 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0505 08:20:53.850137 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0505 08:20:53.850143 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0505 08:20:53.890298 3788 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0505 08:20:54.884385 3788 default_network_controller.go:1025] Node "ip-10-0-2-96.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0505 08:20:55.381790 3788 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-71.ec2.internal I0505 08:20:55.381822 3788 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-71.ec2.internal: 35.406µs I0505 08:20:55.406615 3788 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-5f557bd69b-lnjm9] creating logical port openshift-image-registry_image-registry-5f557bd69b-lnjm9 for pod on switch ip-10-0-1-71.ec2.internal I0505 08:20:55.406739 3788 kube.go:256] Updating pod openshift-image-registry/image-registry-5f557bd69b-lnjm9 I0505 08:20:55.425801 3788 pod.go:62] [openshift-image-registry/image-registry-5f557bd69b-lnjm9] pod update took 19.090547ms I0505 08:20:55.425832 3788 base_network_controller_pods.go:951] [default/openshift-image-registry/image-registry-5f557bd69b-lnjm9] addLogicalPort annotation time took 19.126028ms I0505 08:20:55.426983 3788 pods.go:271] [openshift-image-registry/image-registry-5f557bd69b-lnjm9] addLogicalPort took 20.384529ms, libovsdb time 847.242µs I0505 08:20:55.441691 3788 base_network_controller_pods.go:487] [default/openshift-dns/dns-default-lcv6b] creating logical port openshift-dns_dns-default-lcv6b for pod on switch ip-10-0-1-71.ec2.internal I0505 08:20:55.441842 3788 kube.go:256] Updating pod openshift-dns/dns-default-lcv6b I0505 08:20:55.441923 3788 base_network_controller_pods.go:487] [default/openshift-ingress-canary/ingress-canary-95cf2] creating logical port openshift-ingress-canary_ingress-canary-95cf2 for pod on switch ip-10-0-1-71.ec2.internal I0505 08:20:55.442094 3788 kube.go:256] Updating pod openshift-ingress-canary/ingress-canary-95cf2 I0505 08:20:55.465401 3788 pod.go:62] [openshift-ingress-canary/ingress-canary-95cf2] pod update took 23.335248ms I0505 08:20:55.465433 3788 base_network_controller_pods.go:951] [default/openshift-ingress-canary/ingress-canary-95cf2] addLogicalPort annotation time took 23.372685ms I0505 08:20:55.465718 3788 pod.go:62] [openshift-dns/dns-default-lcv6b] pod update took 23.914375ms I0505 08:20:55.465737 3788 base_network_controller_pods.go:951] [default/openshift-dns/dns-default-lcv6b] addLogicalPort annotation time took 23.938232ms I0505 08:20:55.467086 3788 pods.go:271] [openshift-ingress-canary/ingress-canary-95cf2] addLogicalPort took 25.170439ms, libovsdb time 1.112738ms I0505 08:20:55.467523 3788 pods.go:271] [openshift-dns/dns-default-lcv6b] addLogicalPort took 25.85456ms, libovsdb time 1.424274ms I0505 08:20:55.863966 3788 cni.go:452] [openshift-ingress-canary/ingress-canary-95cf2 535dd7b8dcd58d8f8dec3d097ca5349cfb3d7154214bd9b13cd2218e61a3b692 network default NAD default NAD key ] ADD starting CNI request [openshift-ingress-canary/ingress-canary-95cf2 535dd7b8dcd58d8f8dec3d097ca5349cfb3d7154214bd9b13cd2218e61a3b692 network default NAD default NAD key ] I0505 08:20:55.896020 3788 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress-canary, podName: ingress-canary-95cf2, hostIfaceName: 535dd7b8dcd58d8, network: default, NAD default, SandboxID: "535dd7b8dcd58d8f8dec3d097ca5349cfb3d7154214bd9b13cd2218e61a3b692", PCI device ID: , UID: "32ea188b-f5d7-4889-9260-ce2258abb573", MAC: 0a:58:0a:81:00:07, IPs: [10.129.0.7/23] I0505 08:20:55.947225 3788 cni.go:473] [openshift-ingress-canary/ingress-canary-95cf2 535dd7b8dcd58d8f8dec3d097ca5349cfb3d7154214bd9b13cd2218e61a3b692 network default NAD default NAD key default] ADD finished CNI request [openshift-ingress-canary/ingress-canary-95cf2 535dd7b8dcd58d8f8dec3d097ca5349cfb3d7154214bd9b13cd2218e61a3b692 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"76:1d:9a:79:81:49\",\"name\":\"535dd7b8dcd58d8\"},{\"mac\":\"0a:58:0a:81:00:07\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/981ae71e-b572-43db-8da5-b5ca0f9c6ff5\"}],\"ips\":[{\"address\":\"10.129.0.7/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0505 08:20:59.411842 3788 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-96.ec2.internal in network "default" I0505 08:20:59.411855 3788 default_network_controller.go:1025] Node "ip-10-0-2-96.ec2.internal" in remote zone "ip-10-0-2-96.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0505 08:20:59.411870 3788 node_tracker.go:235] Node ip-10-0-2-96.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-96.ec2.internal" I0505 08:20:59.411893 3788 node_tracker.go:169] Node ip-10-0-2-96.ec2.internal switch + router changed, syncing services in network "default" I0505 08:20:59.411908 3788 services_controller.go:574] Full service sync requested for network=default I0505 08:20:59.496001 3788 default_network_controller.go:1025] Node "ip-10-0-2-96.ec2.internal" in remote zone "ip-10-0-2-96.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0505 08:20:59.584113 3788 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-96.ec2.internal in network "default" I0505 08:20:59.584179 3788 node_tracker.go:169] Node ip-10-0-2-96.ec2.internal switch + router changed, syncing services in network "default" I0505 08:20:59.584198 3788 services_controller.go:574] Full service sync requested for network=default I0505 08:20:59.584114 3788 default_network_controller.go:1025] Node "ip-10-0-2-96.ec2.internal" in remote zone "ip-10-0-2-96.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0505 08:20:59.585180 3788 zone_ic_handler.go:195] Time taken to create transit switch: 280.855µs I0505 08:20:59.585198 3788 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-2-96.ec2.internal for the network default I0505 08:20:59.586145 3788 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-2-96.ec2.internal" on network "default" took: 1.255925ms I0505 08:20:59.708255 3788 egressfirewall.go:723] Syncing node "ip-10-0-2-96.ec2.internal" for egress firewall I0505 08:20:59.708310 3788 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-96.ec2.internal in network "default" I0505 08:20:59.708320 3788 default_node_network_controller.go:1293] Adding remote node "ip-10-0-2-96.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0505 08:20:59.708334 3788 default_node_network_controller.go:1293] Adding remote node "ip-10-0-2-96.ec2.internal", IP: 10.0.2.96 to PMTUD blocking rules I0505 08:20:59.708339 3788 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-96.ec2.internal I0505 08:20:59.708357 3788 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-96.ec2.internal: 22.286µs I0505 08:20:59.708362 3788 node_tracker.go:169] Node ip-10-0-2-96.ec2.internal switch + router changed, syncing services in network "default" I0505 08:20:59.708375 3788 services_controller.go:574] Full service sync requested for network=default I0505 08:20:59.715088 3788 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-2-96.ec2.internal. Updating no re-route policies I0505 08:20:59.746776 3788 default_node_network_controller.go:1293] Adding remote node "ip-10-0-2-96.ec2.internal", IP: 10.0.2.96 to PMTUD blocking rules I0505 08:20:59.746796 3788 default_node_network_controller.go:1293] Adding remote node "ip-10-0-2-96.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0505 08:21:03.282297 3788 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-96.ec2.internal I0505 08:21:03.282325 3788 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-96.ec2.internal: 36.72µs W0505 08:21:13.196085 3788 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0505 08:21:14.106527 3788 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0505 08:21:20.338961 3788 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dsvs4t of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 08:21:20.339618 3788 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dsvs4t of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 08:21:20.339650 3788 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dsvs4t of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0505 08:21:20.339660 3788 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... I0505 08:21:20.339716 3788 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dsvs4t of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0505 08:21:20.339724 3788 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles to delete 1 local pods... I0505 08:21:21.365494 3788 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dsvs4t of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:21:21.365517 3788 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dsvs4t of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:21:21.365535 3788 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dsvs4t of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:21:21.365547 3788 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dsvs4t of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:21:24.430451 3788 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0505 08:21:24.439516 3788 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0505 08:21:28.117639 3788 cni.go:452] [openshift-network-diagnostics/network-check-target-wg7jm 47565e63262824fb7dfa6c8a132770628914598f8f3bdbdef24e12169e06d65a network default NAD default NAD key ] ADD starting CNI request [openshift-network-diagnostics/network-check-target-wg7jm 47565e63262824fb7dfa6c8a132770628914598f8f3bdbdef24e12169e06d65a network default NAD default NAD key ] I0505 08:21:28.127397 3788 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-target-wg7jm, hostIfaceName: 47565e63262824f, network: default, NAD default, SandboxID: "47565e63262824fb7dfa6c8a132770628914598f8f3bdbdef24e12169e06d65a", PCI device ID: , UID: "90f7caa0-099f-440c-abaa-5664fc407a5e", MAC: 0a:58:0a:81:00:03, IPs: [10.129.0.3/23] I0505 08:21:28.176051 3788 cni.go:473] [openshift-network-diagnostics/network-check-target-wg7jm 47565e63262824fb7dfa6c8a132770628914598f8f3bdbdef24e12169e06d65a network default NAD default NAD key default] ADD finished CNI request [openshift-network-diagnostics/network-check-target-wg7jm 47565e63262824fb7dfa6c8a132770628914598f8f3bdbdef24e12169e06d65a network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"c2:f1:1f:b5:7e:20\",\"name\":\"47565e63262824f\"},{\"mac\":\"0a:58:0a:81:00:03\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/23974260-46c0-4ff1-9013-bc537d9d6209\"}],\"ips\":[{\"address\":\"10.129.0.3/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0505 08:21:32.197628 3788 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0505 08:21:44.204298 3788 util.go:959] Failed to build node endpoints for node ip-10-0-1-71.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0505 08:21:44.204318 3788 util.go:959] Failed to build node endpoints for node ip-10-0-1-71.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0505 08:21:45.113450 3788 util.go:959] Failed to build node endpoints for node ip-10-0-1-71.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0505 08:21:45.113495 3788 util.go:959] Failed to build node endpoints for node ip-10-0-1-71.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0505 08:22:03.201918 3788 util.go:959] Failed to build node endpoints for node ip-10-0-1-71.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0505 08:22:42.792688 3788 base_network_controller_pods.go:487] [default/openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-p7j6r] creating logical port openshift-kube-storage-version-migrator-operator_kube-storage-version-migrator-operator-74478b59c6-p7j6r for pod on switch ip-10-0-1-71.ec2.internal I0505 08:22:42.792812 3788 kube.go:256] Updating pod openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-p7j6r I0505 08:22:42.800871 3788 base_network_controller_pods.go:487] [default/openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-lm9wl] creating logical port openshift-cluster-storage-operator_volume-data-source-validator-6944588dcc-lm9wl for pod on switch ip-10-0-1-71.ec2.internal I0505 08:22:42.801015 3788 kube.go:256] Updating pod openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-lm9wl I0505 08:22:42.801079 3788 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-5b9c989d7f-xqzkn] creating logical port openshift-ingress_router-default-5b9c989d7f-xqzkn for pod on switch ip-10-0-1-71.ec2.internal I0505 08:22:42.801158 3788 kube.go:256] Updating pod openshift-ingress/router-default-5b9c989d7f-xqzkn I0505 08:22:42.850976 3788 pod.go:62] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-p7j6r] pod update took 58.185378ms I0505 08:22:42.851000 3788 base_network_controller_pods.go:951] [default/openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-p7j6r] addLogicalPort annotation time took 58.216264ms I0505 08:22:42.852155 3788 pods.go:271] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-p7j6r] addLogicalPort took 59.482913ms, libovsdb time 868.28µs I0505 08:22:42.852214 3788 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-96cd775b9-xpgcv] creating logical port openshift-ingress_router-default-96cd775b9-xpgcv for pod on switch ip-10-0-1-71.ec2.internal I0505 08:22:42.852294 3788 kube.go:256] Updating pod openshift-ingress/router-default-96cd775b9-xpgcv I0505 08:22:42.855965 3788 pod.go:62] [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-lm9wl] pod update took 54.975937ms I0505 08:22:42.855985 3788 base_network_controller_pods.go:951] [default/openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-lm9wl] addLogicalPort annotation time took 55.001033ms I0505 08:22:42.857098 3788 pods.go:271] [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-lm9wl] addLogicalPort took 56.246079ms, libovsdb time 880.807µs I0505 08:22:42.864737 3788 pod.go:62] [openshift-ingress/router-default-5b9c989d7f-xqzkn] pod update took 63.602008ms I0505 08:22:42.864764 3788 base_network_controller_pods.go:951] [default/openshift-ingress/router-default-5b9c989d7f-xqzkn] addLogicalPort annotation time took 63.627463ms I0505 08:22:42.865764 3788 pods.go:271] [openshift-ingress/router-default-5b9c989d7f-xqzkn] addLogicalPort took 64.699921ms, libovsdb time 754.567µs I0505 08:22:42.878443 3788 pod.go:62] [openshift-ingress/router-default-96cd775b9-xpgcv] pod update took 26.181504ms I0505 08:22:42.878489 3788 base_network_controller_pods.go:951] [default/openshift-ingress/router-default-96cd775b9-xpgcv] addLogicalPort annotation time took 26.233096ms I0505 08:22:42.879669 3788 pods.go:271] [openshift-ingress/router-default-96cd775b9-xpgcv] addLogicalPort took 27.458626ms, libovsdb time 792.608µs I0505 08:22:42.928933 3788 base_network_controller_pods.go:487] [default/openshift-network-console/networking-console-plugin-7f449dfbfd-t8kmd] creating logical port openshift-network-console_networking-console-plugin-7f449dfbfd-t8kmd for pod on switch ip-10-0-1-71.ec2.internal I0505 08:22:42.929037 3788 kube.go:256] Updating pod openshift-network-console/networking-console-plugin-7f449dfbfd-t8kmd I0505 08:22:42.955247 3788 pod.go:62] [openshift-network-console/networking-console-plugin-7f449dfbfd-t8kmd] pod update took 26.237408ms I0505 08:22:42.955282 3788 base_network_controller_pods.go:951] [default/openshift-network-console/networking-console-plugin-7f449dfbfd-t8kmd] addLogicalPort annotation time took 26.277446ms I0505 08:22:42.956630 3788 pods.go:271] [openshift-network-console/networking-console-plugin-7f449dfbfd-t8kmd] addLogicalPort took 27.711164ms, libovsdb time 959.543µs I0505 08:22:43.155819 3788 cni.go:452] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-p7j6r 45410c0d24eb1d66bf682c2d427c82f04dc9b495e49b6317e86ec9623b01272a network default NAD default NAD key ] ADD starting CNI request [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-p7j6r 45410c0d24eb1d66bf682c2d427c82f04dc9b495e49b6317e86ec9623b01272a network default NAD default NAD key ] I0505 08:22:43.155886 3788 cni.go:452] [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-lm9wl 7d40bd0d36b1afe94bd8ce77994ce96107273d6a8e70a1d36015a4a00d6646f5 network default NAD default NAD key ] ADD starting CNI request [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-lm9wl 7d40bd0d36b1afe94bd8ce77994ce96107273d6a8e70a1d36015a4a00d6646f5 network default NAD default NAD key ] I0505 08:22:43.169266 3788 helper_linux.go:556] ConfigureOVS: namespace: openshift-kube-storage-version-migrator-operator, podName: kube-storage-version-migrator-operator-74478b59c6-p7j6r, hostIfaceName: 45410c0d24eb1d6, network: default, NAD default, SandboxID: "45410c0d24eb1d66bf682c2d427c82f04dc9b495e49b6317e86ec9623b01272a", PCI device ID: , UID: "d00177d8-5073-48d8-8bb2-742b5089881e", MAC: 0a:58:0a:81:00:08, IPs: [10.129.0.8/23] I0505 08:22:43.170264 3788 helper_linux.go:556] ConfigureOVS: namespace: openshift-cluster-storage-operator, podName: volume-data-source-validator-6944588dcc-lm9wl, hostIfaceName: 7d40bd0d36b1afe, network: default, NAD default, SandboxID: "7d40bd0d36b1afe94bd8ce77994ce96107273d6a8e70a1d36015a4a00d6646f5", PCI device ID: , UID: "0491f948-3306-4267-a388-95245665b534", MAC: 0a:58:0a:81:00:09, IPs: [10.129.0.9/23] I0505 08:22:43.219566 3788 cni.go:473] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-p7j6r 45410c0d24eb1d66bf682c2d427c82f04dc9b495e49b6317e86ec9623b01272a network default NAD default NAD key default] ADD finished CNI request [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-p7j6r 45410c0d24eb1d66bf682c2d427c82f04dc9b495e49b6317e86ec9623b01272a network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"66:ef:59:44:f3:19\",\"name\":\"45410c0d24eb1d6\"},{\"mac\":\"0a:58:0a:81:00:08\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/88d2637b-4fd9-445f-b9eb-e1c74516eada\"}],\"ips\":[{\"address\":\"10.129.0.8/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0505 08:22:43.234192 3788 cni.go:473] [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-lm9wl 7d40bd0d36b1afe94bd8ce77994ce96107273d6a8e70a1d36015a4a00d6646f5 network default NAD default NAD key default] ADD finished CNI request [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-lm9wl 7d40bd0d36b1afe94bd8ce77994ce96107273d6a8e70a1d36015a4a00d6646f5 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"e6:1b:94:46:ee:ec\",\"name\":\"7d40bd0d36b1afe\"},{\"mac\":\"0a:58:0a:81:00:09\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4b3503ae-9129-422c-a87c-64d8e02e4887\"}],\"ips\":[{\"address\":\"10.129.0.9/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0505 08:22:47.384693 3788 namespace.go:98] [openshift-kube-storage-version-migrator] adding namespace I0505 08:22:47.385774 3788 namespace.go:102] [openshift-kube-storage-version-migrator] adding namespace took 1.054021ms I0505 08:22:47.399276 3788 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace W0505 08:22:47.487525 3788 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0505 08:22:48.518140 3788 namespace.go:98] [openshift-service-ca] adding namespace I0505 08:22:48.519052 3788 namespace.go:102] [openshift-service-ca] adding namespace took 884.715µs I0505 08:22:48.541102 3788 namespace.go:142] [openshift-service-ca] updating namespace I0505 08:22:50.188212 3788 base_network_controller_pods.go:487] [default/openshift-service-ca/service-ca-d9cb48497-v5k8c] creating logical port openshift-service-ca_service-ca-d9cb48497-v5k8c for pod on switch ip-10-0-1-71.ec2.internal I0505 08:22:50.188313 3788 kube.go:256] Updating pod openshift-service-ca/service-ca-d9cb48497-v5k8c I0505 08:22:50.204493 3788 pod.go:62] [openshift-service-ca/service-ca-d9cb48497-v5k8c] pod update took 16.203127ms I0505 08:22:50.204516 3788 base_network_controller_pods.go:951] [default/openshift-service-ca/service-ca-d9cb48497-v5k8c] addLogicalPort annotation time took 16.233007ms I0505 08:22:50.205708 3788 pods.go:271] [openshift-service-ca/service-ca-d9cb48497-v5k8c] addLogicalPort took 17.517995ms, libovsdb time 847.029µs I0505 08:22:50.549406 3788 cni.go:452] [openshift-service-ca/service-ca-d9cb48497-v5k8c 3fd886b541b9f07c82ebe5e4dfb3b5996e1692b1d224b39a8f80cea8f23194b6 network default NAD default NAD key ] ADD starting CNI request [openshift-service-ca/service-ca-d9cb48497-v5k8c 3fd886b541b9f07c82ebe5e4dfb3b5996e1692b1d224b39a8f80cea8f23194b6 network default NAD default NAD key ] I0505 08:22:50.559195 3788 helper_linux.go:556] ConfigureOVS: namespace: openshift-service-ca, podName: service-ca-d9cb48497-v5k8c, hostIfaceName: 3fd886b541b9f07, network: default, NAD default, SandboxID: "3fd886b541b9f07c82ebe5e4dfb3b5996e1692b1d224b39a8f80cea8f23194b6", PCI device ID: , UID: "36595ffc-cdec-4927-83db-de5296c958c8", MAC: 0a:58:0a:81:00:0d, IPs: [10.129.0.13/23] I0505 08:22:50.611002 3788 cni.go:473] [openshift-service-ca/service-ca-d9cb48497-v5k8c 3fd886b541b9f07c82ebe5e4dfb3b5996e1692b1d224b39a8f80cea8f23194b6 network default NAD default NAD key default] ADD finished CNI request [openshift-service-ca/service-ca-d9cb48497-v5k8c 3fd886b541b9f07c82ebe5e4dfb3b5996e1692b1d224b39a8f80cea8f23194b6 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ca:75:1a:bf:04:cf\",\"name\":\"3fd886b541b9f07\"},{\"mac\":\"0a:58:0a:81:00:0d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a3c716fc-4191-4ee6-aac5-77ca7fca58ef\"}],\"ips\":[{\"address\":\"10.129.0.13/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0505 08:22:57.410400 3788 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0505 08:23:03.631034 3788 cni.go:452] [openshift-dns/dns-default-lcv6b 318eca45f958d94503ad8dac77e1156d6caaaeb892df93b7b8fb02049ea614ce network default NAD default NAD key ] ADD starting CNI request [openshift-dns/dns-default-lcv6b 318eca45f958d94503ad8dac77e1156d6caaaeb892df93b7b8fb02049ea614ce network default NAD default NAD key ] I0505 08:23:03.640351 3788 helper_linux.go:556] ConfigureOVS: namespace: openshift-dns, podName: dns-default-lcv6b, hostIfaceName: 318eca45f958d94, network: default, NAD default, SandboxID: "318eca45f958d94503ad8dac77e1156d6caaaeb892df93b7b8fb02049ea614ce", PCI device ID: , UID: "c59ef289-d239-4f95-a675-6d11c46fd143", MAC: 0a:58:0a:81:00:06, IPs: [10.129.0.6/23] I0505 08:23:03.690175 3788 cni.go:473] [openshift-dns/dns-default-lcv6b 318eca45f958d94503ad8dac77e1156d6caaaeb892df93b7b8fb02049ea614ce network default NAD default NAD key default] ADD finished CNI request [openshift-dns/dns-default-lcv6b 318eca45f958d94503ad8dac77e1156d6caaaeb892df93b7b8fb02049ea614ce network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"fe:9b:19:7f:00:14\",\"name\":\"318eca45f958d94\"},{\"mac\":\"0a:58:0a:81:00:06\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/67ae9b70-b344-4e2d-b43c-91c4c890cdc6\"}],\"ips\":[{\"address\":\"10.129.0.6/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0505 08:23:09.000020 3788 cni.go:452] [openshift-image-registry/image-registry-5f557bd69b-lnjm9 42d925b93dfe912813d63d09f846370893ab897c7ce8686a21df1a45620d9368 network default NAD default NAD key ] ADD starting CNI request [openshift-image-registry/image-registry-5f557bd69b-lnjm9 42d925b93dfe912813d63d09f846370893ab897c7ce8686a21df1a45620d9368 network default NAD default NAD key ] I0505 08:23:09.009073 3788 helper_linux.go:556] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-5f557bd69b-lnjm9, hostIfaceName: 42d925b93dfe912, network: default, NAD default, SandboxID: "42d925b93dfe912813d63d09f846370893ab897c7ce8686a21df1a45620d9368", PCI device ID: , UID: "539d7f8c-72bb-4829-91df-a864a406f10d", MAC: 0a:58:0a:81:00:05, IPs: [10.129.0.5/23] I0505 08:23:09.058778 3788 cni.go:473] [openshift-image-registry/image-registry-5f557bd69b-lnjm9 42d925b93dfe912813d63d09f846370893ab897c7ce8686a21df1a45620d9368 network default NAD default NAD key default] ADD finished CNI request [openshift-image-registry/image-registry-5f557bd69b-lnjm9 42d925b93dfe912813d63d09f846370893ab897c7ce8686a21df1a45620d9368 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ea:7e:6d:9f:4f:90\",\"name\":\"42d925b93dfe912\"},{\"mac\":\"0a:58:0a:81:00:05\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/182e9cae-9cdc-4898-b047-d96ef52d27a7\"}],\"ips\":[{\"address\":\"10.129.0.5/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0505 08:23:09.956557 3788 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0505 08:23:09.956582 3788 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0505 08:23:09.956588 3788 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0505 08:23:09.973443 3788 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0505 08:23:10.340531 3788 base_network_controller_pods.go:487] [default/openshift-insights/insights-runtime-extractor-pvpmf] creating logical port openshift-insights_insights-runtime-extractor-pvpmf for pod on switch ip-10-0-1-71.ec2.internal I0505 08:23:10.340628 3788 kube.go:256] Updating pod openshift-insights/insights-runtime-extractor-pvpmf I0505 08:23:10.359281 3788 pod.go:62] [openshift-insights/insights-runtime-extractor-pvpmf] pod update took 18.68229ms I0505 08:23:10.359303 3788 base_network_controller_pods.go:951] [default/openshift-insights/insights-runtime-extractor-pvpmf] addLogicalPort annotation time took 18.71026ms I0505 08:23:10.360491 3788 pods.go:271] [openshift-insights/insights-runtime-extractor-pvpmf] addLogicalPort took 19.97409ms, libovsdb time 890.755µs I0505 08:23:10.721791 3788 cni.go:452] [openshift-insights/insights-runtime-extractor-pvpmf cacdd34d9090f3653c65e4d395e80d7476556e69f65d5fce5e6af57d546a32e5 network default NAD default NAD key ] ADD starting CNI request [openshift-insights/insights-runtime-extractor-pvpmf cacdd34d9090f3653c65e4d395e80d7476556e69f65d5fce5e6af57d546a32e5 network default NAD default NAD key ] I0505 08:23:10.731900 3788 helper_linux.go:556] ConfigureOVS: namespace: openshift-insights, podName: insights-runtime-extractor-pvpmf, hostIfaceName: cacdd34d9090f36, network: default, NAD default, SandboxID: "cacdd34d9090f3653c65e4d395e80d7476556e69f65d5fce5e6af57d546a32e5", PCI device ID: , UID: "51f94598-4a72-498e-a9d0-8c0bef044669", MAC: 0a:58:0a:81:00:0e, IPs: [10.129.0.14/23] I0505 08:23:10.781700 3788 cni.go:473] [openshift-insights/insights-runtime-extractor-pvpmf cacdd34d9090f3653c65e4d395e80d7476556e69f65d5fce5e6af57d546a32e5 network default NAD default NAD key default] ADD finished CNI request [openshift-insights/insights-runtime-extractor-pvpmf cacdd34d9090f3653c65e4d395e80d7476556e69f65d5fce5e6af57d546a32e5 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"d2:3d:76:ef:3c:46\",\"name\":\"cacdd34d9090f36\"},{\"mac\":\"0a:58:0a:81:00:0e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/fc3e17a4-f78b-4e2e-bc3d-f1f300b317e2\"}],\"ips\":[{\"address\":\"10.129.0.14/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0505 08:23:11.407431 3788 obj_retry.go:498] Detected object openshift-image-registry/image-registry-5f557bd69b-fmt48 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 08:23:11.408120 3788 obj_retry.go:498] Detected object openshift-image-registry/image-registry-5f557bd69b-fmt48 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 08:23:11.429563 3788 obj_retry.go:492] Detected object openshift-image-registry/image-registry-5f557bd69b-fmt48 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:23:11.429589 3788 obj_retry.go:492] Detected object openshift-image-registry/image-registry-5f557bd69b-fmt48 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:23:11.439492 3788 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-5f557bd69b-fmt48 I0505 08:23:11.439520 3788 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-5f557bd69b-fmt48 I0505 08:23:11.581383 3788 obj_retry.go:498] Detected object openshift-image-registry/image-registry-749985c8d7-k8l7z of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 08:23:11.582020 3788 obj_retry.go:498] Detected object openshift-image-registry/image-registry-749985c8d7-k8l7z of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 08:23:11.605664 3788 obj_retry.go:492] Detected object openshift-image-registry/image-registry-749985c8d7-k8l7z of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:23:11.605691 3788 obj_retry.go:492] Detected object openshift-image-registry/image-registry-749985c8d7-k8l7z of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:23:11.614622 3788 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-749985c8d7-k8l7z I0505 08:23:11.614642 3788 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-749985c8d7-k8l7z W0505 08:23:12.406537 3788 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0505 08:23:12.562582 3788 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0505 08:23:13.412595 3788 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0505 08:23:13.412615 3788 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0505 08:23:13.412624 3788 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default I0505 08:23:14.966565 3788 cni.go:452] [openshift-ingress/router-default-5b9c989d7f-xqzkn 125bde7d074590fd54abbb11a0b968510bf2bdc5e0a5bdc8070be8413811642f network default NAD default NAD key ] ADD starting CNI request [openshift-ingress/router-default-5b9c989d7f-xqzkn 125bde7d074590fd54abbb11a0b968510bf2bdc5e0a5bdc8070be8413811642f network default NAD default NAD key ] I0505 08:23:14.966784 3788 cni.go:452] [openshift-ingress/router-default-96cd775b9-xpgcv ece6957f6265cd837e9f6e77a0149eba0c2ff97dffe18aee82562564b3b8a06e network default NAD default NAD key ] ADD starting CNI request [openshift-ingress/router-default-96cd775b9-xpgcv ece6957f6265cd837e9f6e77a0149eba0c2ff97dffe18aee82562564b3b8a06e network default NAD default NAD key ] I0505 08:23:14.976752 3788 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress, podName: router-default-96cd775b9-xpgcv, hostIfaceName: ece6957f6265cd8, network: default, NAD default, SandboxID: "ece6957f6265cd837e9f6e77a0149eba0c2ff97dffe18aee82562564b3b8a06e", PCI device ID: , UID: "e35e8650-602d-49a2-b690-bd12f1be7fc4", MAC: 0a:58:0a:81:00:0b, IPs: [10.129.0.11/23] I0505 08:23:14.976964 3788 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress, podName: router-default-5b9c989d7f-xqzkn, hostIfaceName: 125bde7d074590f, network: default, NAD default, SandboxID: "125bde7d074590fd54abbb11a0b968510bf2bdc5e0a5bdc8070be8413811642f", PCI device ID: , UID: "e82273b3-52fc-4d1d-9fc6-374616aeaf81", MAC: 0a:58:0a:81:00:0a, IPs: [10.129.0.10/23] I0505 08:23:15.032688 3788 cni.go:473] [openshift-ingress/router-default-96cd775b9-xpgcv ece6957f6265cd837e9f6e77a0149eba0c2ff97dffe18aee82562564b3b8a06e network default NAD default NAD key default] ADD finished CNI request [openshift-ingress/router-default-96cd775b9-xpgcv ece6957f6265cd837e9f6e77a0149eba0c2ff97dffe18aee82562564b3b8a06e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"76:15:f8:99:ee:bb\",\"name\":\"ece6957f6265cd8\"},{\"mac\":\"0a:58:0a:81:00:0b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1792e1c5-de9e-4911-8d01-40edecb54eb2\"}],\"ips\":[{\"address\":\"10.129.0.11/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0505 08:23:15.045823 3788 cni.go:473] [openshift-ingress/router-default-5b9c989d7f-xqzkn 125bde7d074590fd54abbb11a0b968510bf2bdc5e0a5bdc8070be8413811642f network default NAD default NAD key default] ADD finished CNI request [openshift-ingress/router-default-5b9c989d7f-xqzkn 125bde7d074590fd54abbb11a0b968510bf2bdc5e0a5bdc8070be8413811642f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"9e:dc:0d:88:c3:77\",\"name\":\"125bde7d074590f\"},{\"mac\":\"0a:58:0a:81:00:0a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/62cf38de-d764-4cd9-bb40-836d546d4b78\"}],\"ips\":[{\"address\":\"10.129.0.10/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0505 08:23:15.089880 3788 cni.go:452] [openshift-network-console/networking-console-plugin-7f449dfbfd-t8kmd 4ead4ab400662adf3f286ec95a12c73ce6225b6d129d2041555eeb2740459165 network default NAD default NAD key ] ADD starting CNI request [openshift-network-console/networking-console-plugin-7f449dfbfd-t8kmd 4ead4ab400662adf3f286ec95a12c73ce6225b6d129d2041555eeb2740459165 network default NAD default NAD key ] I0505 08:23:15.100533 3788 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-console, podName: networking-console-plugin-7f449dfbfd-t8kmd, hostIfaceName: 4ead4ab400662ad, network: default, NAD default, SandboxID: "4ead4ab400662adf3f286ec95a12c73ce6225b6d129d2041555eeb2740459165", PCI device ID: , UID: "4bd0edaa-f762-4ea3-9a7f-8b16c59c8292", MAC: 0a:58:0a:81:00:0c, IPs: [10.129.0.12/23] I0505 08:23:15.152941 3788 cni.go:473] [openshift-network-console/networking-console-plugin-7f449dfbfd-t8kmd 4ead4ab400662adf3f286ec95a12c73ce6225b6d129d2041555eeb2740459165 network default NAD default NAD key default] ADD finished CNI request [openshift-network-console/networking-console-plugin-7f449dfbfd-t8kmd 4ead4ab400662adf3f286ec95a12c73ce6225b6d129d2041555eeb2740459165 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"da:3a:d4:ff:ce:b5\",\"name\":\"4ead4ab400662ad\"},{\"mac\":\"0a:58:0a:81:00:0c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/73c670f8-b296-42bf-98f6-972e370d700a\"}],\"ips\":[{\"address\":\"10.129.0.12/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0505 08:23:15.465615 3788 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0505 08:23:15.465639 3788 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0505 08:23:15.465658 3788 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0505 08:23:15.465674 3788 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0505 08:23:15.465680 3788 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0505 08:23:15.496290 3788 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0505 08:23:15.496317 3788 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0505 08:23:15.496300 3788 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0505 08:23:15.496405 3788 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0505 08:23:15.496414 3788 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0505 08:23:15.576636 3788 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0505 08:23:15.576751 3788 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0505 08:23:15.576717 3788 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0505 08:23:15.576768 3788 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0505 08:23:15.576778 3788 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0505 08:23:16.477009 3788 util.go:959] Failed to build node endpoints for node ip-10-0-1-71.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-internal-default W0505 08:23:16.477030 3788 util.go:959] Failed to build node endpoints for node ip-10-0-1-71.ec2.internal port 1936: empty IP address endpoints for service openshift-ingress/router-internal-default W0505 08:23:16.477038 3788 util.go:959] Failed to build node endpoints for node ip-10-0-1-71.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-internal-default W0505 08:23:16.480001 3788 util.go:959] Failed to build node endpoints for node ip-10-0-1-71.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-default W0505 08:23:16.480019 3788 util.go:959] Failed to build node endpoints for node ip-10-0-1-71.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-default I0505 08:23:16.621500 3788 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0505 08:23:16.726482 3788 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0505 08:23:17.440193 3788 util.go:959] Failed to build node endpoints for node ip-10-0-1-71.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0505 08:23:17.440222 3788 util.go:959] Failed to build node endpoints for node ip-10-0-1-71.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0505 08:23:17.440231 3788 util.go:959] Failed to build node endpoints for node ip-10-0-1-71.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default I0505 08:23:18.061707 3788 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-lp676] creating logical port openshift-monitoring_prometheus-operator-admission-webhook-59b79769fb-lp676 for pod on switch ip-10-0-1-71.ec2.internal I0505 08:23:18.061804 3788 kube.go:256] Updating pod openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-lp676 I0505 08:23:18.086001 3788 pod.go:62] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-lp676] pod update took 24.225294ms I0505 08:23:18.086030 3788 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-lp676] addLogicalPort annotation time took 24.260542ms I0505 08:23:18.087170 3788 pods.go:271] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-lp676] addLogicalPort took 25.479234ms, libovsdb time 831.624µs I0505 08:23:19.021204 3788 cni.go:452] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-lp676 6ab74843fe41afc3c8c01a40e51677fbd954b7c148d4c22b8911e9d2616e4cf4 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-lp676 6ab74843fe41afc3c8c01a40e51677fbd954b7c148d4c22b8911e9d2616e4cf4 network default NAD default NAD key ] I0505 08:23:19.031155 3788 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-operator-admission-webhook-59b79769fb-lp676, hostIfaceName: 6ab74843fe41afc, network: default, NAD default, SandboxID: "6ab74843fe41afc3c8c01a40e51677fbd954b7c148d4c22b8911e9d2616e4cf4", PCI device ID: , UID: "96e66c61-6613-44e0-b0ee-b0082c28e0b9", MAC: 0a:58:0a:81:00:0f, IPs: [10.129.0.15/23] I0505 08:23:19.081240 3788 cni.go:473] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-lp676 6ab74843fe41afc3c8c01a40e51677fbd954b7c148d4c22b8911e9d2616e4cf4 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-lp676 6ab74843fe41afc3c8c01a40e51677fbd954b7c148d4c22b8911e9d2616e4cf4 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"da:ca:31:4a:1f:d2\",\"name\":\"6ab74843fe41afc\"},{\"mac\":\"0a:58:0a:81:00:0f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2955190f-56c1-4c57-b8a3-3befb79540ea\"}],\"ips\":[{\"address\":\"10.129.0.15/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0505 08:23:19.968853 3788 base_network_controller_pods.go:487] [default/openshift-console/console-7585666dd5-zttv2] creating logical port openshift-console_console-7585666dd5-zttv2 for pod on switch ip-10-0-1-71.ec2.internal I0505 08:23:19.968941 3788 kube.go:256] Updating pod openshift-console/console-7585666dd5-zttv2 I0505 08:23:19.991020 3788 pod.go:62] [openshift-console/console-7585666dd5-zttv2] pod update took 22.099779ms I0505 08:23:19.991077 3788 base_network_controller_pods.go:951] [default/openshift-console/console-7585666dd5-zttv2] addLogicalPort annotation time took 22.160652ms I0505 08:23:19.992429 3788 pods.go:271] [openshift-console/console-7585666dd5-zttv2] addLogicalPort took 23.592888ms, libovsdb time 1.010077ms I0505 08:23:20.350591 3788 cni.go:452] [openshift-console/console-7585666dd5-zttv2 b4f5b0f53d5608dc0c1cfa974e35cee0ccfda680a64ab318353afaa0ea806229 network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-7585666dd5-zttv2 b4f5b0f53d5608dc0c1cfa974e35cee0ccfda680a64ab318353afaa0ea806229 network default NAD default NAD key ] I0505 08:23:20.360406 3788 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-7585666dd5-zttv2, hostIfaceName: b4f5b0f53d5608d, network: default, NAD default, SandboxID: "b4f5b0f53d5608dc0c1cfa974e35cee0ccfda680a64ab318353afaa0ea806229", PCI device ID: , UID: "4ea5ac33-4028-4b59-b17b-98563c011edb", MAC: 0a:58:0a:81:00:10, IPs: [10.129.0.16/23] I0505 08:23:20.413637 3788 cni.go:473] [openshift-console/console-7585666dd5-zttv2 b4f5b0f53d5608dc0c1cfa974e35cee0ccfda680a64ab318353afaa0ea806229 network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-7585666dd5-zttv2 b4f5b0f53d5608dc0c1cfa974e35cee0ccfda680a64ab318353afaa0ea806229 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"5e:55:e9:e5:d5:04\",\"name\":\"b4f5b0f53d5608d\"},{\"mac\":\"0a:58:0a:81:00:10\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/37a3c69e-64b8-45db-9684-182a3be4ab03\"}],\"ips\":[{\"address\":\"10.129.0.16/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0505 08:23:20.482715 3788 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0505 08:23:20.510644 3788 util.go:959] Failed to build node endpoints for node ip-10-0-1-71.ec2.internal port 8443: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook I0505 08:23:22.125955 3788 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-operator-76b589d668-6k55r] creating logical port openshift-monitoring_prometheus-operator-76b589d668-6k55r for pod on switch ip-10-0-1-71.ec2.internal I0505 08:23:22.126066 3788 kube.go:256] Updating pod openshift-monitoring/prometheus-operator-76b589d668-6k55r I0505 08:23:22.143489 3788 pod.go:62] [openshift-monitoring/prometheus-operator-76b589d668-6k55r] pod update took 17.445686ms I0505 08:23:22.143524 3788 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-operator-76b589d668-6k55r] addLogicalPort annotation time took 17.486863ms I0505 08:23:22.145100 3788 pods.go:271] [openshift-monitoring/prometheus-operator-76b589d668-6k55r] addLogicalPort took 19.165898ms, libovsdb time 1.125986ms I0505 08:23:22.485064 3788 cni.go:452] [openshift-monitoring/prometheus-operator-76b589d668-6k55r ba7ae0707b19437a07eee01fe35c78ff8c5ad19325709d40cd3dd4d7c422569e network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-operator-76b589d668-6k55r ba7ae0707b19437a07eee01fe35c78ff8c5ad19325709d40cd3dd4d7c422569e network default NAD default NAD key ] I0505 08:23:22.494259 3788 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-operator-76b589d668-6k55r, hostIfaceName: ba7ae0707b19437, network: default, NAD default, SandboxID: "ba7ae0707b19437a07eee01fe35c78ff8c5ad19325709d40cd3dd4d7c422569e", PCI device ID: , UID: "89a63044-9ad9-4552-8ea6-67b162a20d82", MAC: 0a:58:0a:81:00:11, IPs: [10.129.0.17/23] I0505 08:23:22.547384 3788 cni.go:473] [openshift-monitoring/prometheus-operator-76b589d668-6k55r ba7ae0707b19437a07eee01fe35c78ff8c5ad19325709d40cd3dd4d7c422569e network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-operator-76b589d668-6k55r ba7ae0707b19437a07eee01fe35c78ff8c5ad19325709d40cd3dd4d7c422569e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"c6:15:5c:2e:c1:e1\",\"name\":\"ba7ae0707b19437\"},{\"mac\":\"0a:58:0a:81:00:11\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b3dd62c1-24be-457f-b840-1fb88e17b323\"}],\"ips\":[{\"address\":\"10.129.0.17/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0505 08:23:24.468541 3788 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0505 08:23:24.515848 3788 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0505 08:23:25.607206 3788 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console I0505 08:23:29.811560 3788 base_network_controller_pods.go:487] [default/openshift-monitoring/alertmanager-main-0] creating logical port openshift-monitoring_alertmanager-main-0 for pod on switch ip-10-0-1-71.ec2.internal I0505 08:23:29.811698 3788 kube.go:256] Updating pod openshift-monitoring/alertmanager-main-0 I0505 08:23:29.854053 3788 pod.go:62] [openshift-monitoring/alertmanager-main-0] pod update took 42.409704ms I0505 08:23:29.854079 3788 base_network_controller_pods.go:951] [default/openshift-monitoring/alertmanager-main-0] addLogicalPort annotation time took 42.44384ms I0505 08:23:29.855261 3788 pods.go:271] [openshift-monitoring/alertmanager-main-0] addLogicalPort took 43.716026ms, libovsdb time 867.116µs I0505 08:23:30.255054 3788 cni.go:452] [openshift-monitoring/alertmanager-main-0 93c30d006a5d4191a00b64a78dc9fa338b1f50b004184b75216c9cea69d6dbb9 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/alertmanager-main-0 93c30d006a5d4191a00b64a78dc9fa338b1f50b004184b75216c9cea69d6dbb9 network default NAD default NAD key ] I0505 08:23:30.264601 3788 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: alertmanager-main-0, hostIfaceName: 93c30d006a5d419, network: default, NAD default, SandboxID: "93c30d006a5d4191a00b64a78dc9fa338b1f50b004184b75216c9cea69d6dbb9", PCI device ID: , UID: "eacc6d18-7824-4c76-a775-918337501ad4", MAC: 0a:58:0a:81:00:12, IPs: [10.129.0.18/23] I0505 08:23:30.316753 3788 cni.go:473] [openshift-monitoring/alertmanager-main-0 93c30d006a5d4191a00b64a78dc9fa338b1f50b004184b75216c9cea69d6dbb9 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/alertmanager-main-0 93c30d006a5d4191a00b64a78dc9fa338b1f50b004184b75216c9cea69d6dbb9 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"56:a5:b9:ce:0b:b2\",\"name\":\"93c30d006a5d419\"},{\"mac\":\"0a:58:0a:81:00:12\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4dce8a13-177e-4fd0-ac71-e361980a5105\"}],\"ips\":[{\"address\":\"10.129.0.18/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0505 08:23:30.462083 3788 util.go:959] Failed to build node endpoints for node ip-10-0-1-71.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0505 08:23:30.568480 3788 util.go:959] Failed to build node endpoints for node ip-10-0-1-71.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0505 08:23:32.493124 3788 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads W0505 08:23:32.560891 3788 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0505 08:23:32.560908 3788 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0505 08:23:32.560913 3788 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0505 08:23:33.249250 3788 base_network_controller_pods.go:487] [default/openshift-monitoring/metrics-server-77488cdcbc-vwhs4] creating logical port openshift-monitoring_metrics-server-77488cdcbc-vwhs4 for pod on switch ip-10-0-1-71.ec2.internal I0505 08:23:33.249400 3788 kube.go:256] Updating pod openshift-monitoring/metrics-server-77488cdcbc-vwhs4 I0505 08:23:33.273274 3788 pod.go:62] [openshift-monitoring/metrics-server-77488cdcbc-vwhs4] pod update took 23.900393ms I0505 08:23:33.273302 3788 base_network_controller_pods.go:951] [default/openshift-monitoring/metrics-server-77488cdcbc-vwhs4] addLogicalPort annotation time took 23.935631ms I0505 08:23:33.274548 3788 pods.go:271] [openshift-monitoring/metrics-server-77488cdcbc-vwhs4] addLogicalPort took 25.320007ms, libovsdb time 855.94µs I0505 08:23:33.402181 3788 base_network_controller_pods.go:487] [default/openshift-monitoring/monitoring-plugin-5cbb48bf6b-v5bvp] creating logical port openshift-monitoring_monitoring-plugin-5cbb48bf6b-v5bvp for pod on switch ip-10-0-1-71.ec2.internal I0505 08:23:33.402278 3788 kube.go:256] Updating pod openshift-monitoring/monitoring-plugin-5cbb48bf6b-v5bvp I0505 08:23:33.419644 3788 pod.go:62] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-v5bvp] pod update took 17.390272ms I0505 08:23:33.419699 3788 base_network_controller_pods.go:951] [default/openshift-monitoring/monitoring-plugin-5cbb48bf6b-v5bvp] addLogicalPort annotation time took 17.423581ms I0505 08:23:33.420949 3788 pods.go:271] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-v5bvp] addLogicalPort took 18.782816ms, libovsdb time 894.822µs I0505 08:23:33.618435 3788 cni.go:452] [openshift-monitoring/metrics-server-77488cdcbc-vwhs4 206fdb31523514cfbc44fa8b6b355a1f922d41a5550b65a99dc30a9e7a983ea4 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/metrics-server-77488cdcbc-vwhs4 206fdb31523514cfbc44fa8b6b355a1f922d41a5550b65a99dc30a9e7a983ea4 network default NAD default NAD key ] I0505 08:23:33.629628 3788 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: metrics-server-77488cdcbc-vwhs4, hostIfaceName: 206fdb31523514c, network: default, NAD default, SandboxID: "206fdb31523514cfbc44fa8b6b355a1f922d41a5550b65a99dc30a9e7a983ea4", PCI device ID: , UID: "e06b2ea4-9d07-4206-8ed3-b5a401016a07", MAC: 0a:58:0a:81:00:13, IPs: [10.129.0.19/23] I0505 08:23:33.685093 3788 cni.go:473] [openshift-monitoring/metrics-server-77488cdcbc-vwhs4 206fdb31523514cfbc44fa8b6b355a1f922d41a5550b65a99dc30a9e7a983ea4 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/metrics-server-77488cdcbc-vwhs4 206fdb31523514cfbc44fa8b6b355a1f922d41a5550b65a99dc30a9e7a983ea4 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"96:18:74:49:aa:44\",\"name\":\"206fdb31523514c\"},{\"mac\":\"0a:58:0a:81:00:13\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f2f15603-27aa-43ed-ba43-fd1f84e733fc\"}],\"ips\":[{\"address\":\"10.129.0.19/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0505 08:23:33.689254 3788 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0505 08:23:33.689271 3788 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0505 08:23:33.689276 3788 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0505 08:23:34.363790 3788 cni.go:452] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-v5bvp 42dfc2cc8d1f82246032fbfb003cc96a4e6f2ec5f6e9de1fb993b394811bcd46 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/monitoring-plugin-5cbb48bf6b-v5bvp 42dfc2cc8d1f82246032fbfb003cc96a4e6f2ec5f6e9de1fb993b394811bcd46 network default NAD default NAD key ] I0505 08:23:34.374868 3788 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: monitoring-plugin-5cbb48bf6b-v5bvp, hostIfaceName: 42dfc2cc8d1f822, network: default, NAD default, SandboxID: "42dfc2cc8d1f82246032fbfb003cc96a4e6f2ec5f6e9de1fb993b394811bcd46", PCI device ID: , UID: "d5d3ecb3-11fa-4069-abec-63a003ce4a0c", MAC: 0a:58:0a:81:00:14, IPs: [10.129.0.20/23] I0505 08:23:34.430926 3788 cni.go:473] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-v5bvp 42dfc2cc8d1f82246032fbfb003cc96a4e6f2ec5f6e9de1fb993b394811bcd46 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/monitoring-plugin-5cbb48bf6b-v5bvp 42dfc2cc8d1f82246032fbfb003cc96a4e6f2ec5f6e9de1fb993b394811bcd46 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"12:e9:67:49:7a:5f\",\"name\":\"42dfc2cc8d1f822\"},{\"mac\":\"0a:58:0a:81:00:14\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f1321d60-e270-4a0c-9e0d-ae83c5c0c218\"}],\"ips\":[{\"address\":\"10.129.0.20/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0505 08:23:35.073596 3788 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-k8s-0] creating logical port openshift-monitoring_prometheus-k8s-0 for pod on switch ip-10-0-1-71.ec2.internal I0505 08:23:35.073719 3788 kube.go:256] Updating pod openshift-monitoring/prometheus-k8s-0 I0505 08:23:35.106315 3788 pod.go:62] [openshift-monitoring/prometheus-k8s-0] pod update took 32.640127ms I0505 08:23:35.106346 3788 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-k8s-0] addLogicalPort annotation time took 32.675427ms I0505 08:23:35.107614 3788 pods.go:271] [openshift-monitoring/prometheus-k8s-0] addLogicalPort took 34.042948ms, libovsdb time 898.335µs I0505 08:23:35.668755 3788 cni.go:452] [openshift-monitoring/prometheus-k8s-0 433e36fba82e383f97cbf394a2d445604ba4f49fa879aa05e59c2e7b4ee7c00a network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-k8s-0 433e36fba82e383f97cbf394a2d445604ba4f49fa879aa05e59c2e7b4ee7c00a network default NAD default NAD key ] I0505 08:23:35.679678 3788 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-k8s-0, hostIfaceName: 433e36fba82e383, network: default, NAD default, SandboxID: "433e36fba82e383f97cbf394a2d445604ba4f49fa879aa05e59c2e7b4ee7c00a", PCI device ID: , UID: "7916c118-5c1e-4f37-b0fe-63370a53bae7", MAC: 0a:58:0a:81:00:15, IPs: [10.129.0.21/23] I0505 08:23:35.734260 3788 cni.go:473] [openshift-monitoring/prometheus-k8s-0 433e36fba82e383f97cbf394a2d445604ba4f49fa879aa05e59c2e7b4ee7c00a network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-k8s-0 433e36fba82e383f97cbf394a2d445604ba4f49fa879aa05e59c2e7b4ee7c00a network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"26:72:42:22:66:af\",\"name\":\"433e36fba82e383\"},{\"mac\":\"0a:58:0a:81:00:15\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7d9935b9-909f-4fde-9d7e-f13e882b6a42\"}],\"ips\":[{\"address\":\"10.129.0.21/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0505 08:23:36.338792 3788 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-console/console-578895dbb6-2pjs8 IP address from the namespace address-set, err: pod openshift-console/console-578895dbb6-2pjs8: no pod IPs found W0505 08:23:36.539003 3788 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0505 08:23:36.539025 3788 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0505 08:23:36.539031 3788 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0505 08:23:36.539038 3788 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0505 08:23:36.571846 3788 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0505 08:23:36.577123 3788 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/monitoring-plugin W0505 08:23:36.601089 3788 util.go:959] Failed to build node endpoints for node ip-10-0-1-71.ec2.internal port 9443: empty IP address endpoints for service openshift-monitoring/monitoring-plugin W0505 08:23:36.647047 3788 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0505 08:23:36.647069 3788 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0505 08:23:36.673175 3788 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0505 08:23:36.692247 3788 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0505 08:23:36.692264 3788 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0505 08:23:37.762417 3788 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0505 08:23:37.762436 3788 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0505 08:23:37.762441 3788 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0505 08:23:37.762445 3788 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier I0505 08:23:46.150406 3788 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0505 08:23:49.737363 3788 obj_retry.go:498] Detected object openshift-console/console-7585666dd5-8slff of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 08:23:49.737949 3788 obj_retry.go:498] Detected object openshift-console/console-7585666dd5-8slff of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 08:23:50.591931 3788 obj_retry.go:492] Detected object openshift-console/console-7585666dd5-8slff of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:23:50.591960 3788 obj_retry.go:492] Detected object openshift-console/console-7585666dd5-8slff of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:23:50.596736 3788 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-7585666dd5-8slff I0505 08:23:50.596764 3788 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-7585666dd5-8slff W0505 08:23:57.542178 3788 util.go:959] Failed to build node endpoints for node ip-10-0-1-71.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server W0505 08:23:57.542208 3788 util.go:959] Failed to build node endpoints for node ip-10-0-1-71.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server W0505 08:23:57.586101 3788 util.go:959] Failed to build node endpoints for node ip-10-0-1-71.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server W0505 08:23:57.618186 3788 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-insights/metrics I0505 08:23:57.885068 3788 cni.go:452] [openshift-console/console-7585666dd5-zttv2 b4f5b0f53d5608dc0c1cfa974e35cee0ccfda680a64ab318353afaa0ea806229 network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-7585666dd5-zttv2 b4f5b0f53d5608dc0c1cfa974e35cee0ccfda680a64ab318353afaa0ea806229 network default NAD default NAD key ] I0505 08:23:57.969263 3788 cni.go:473] [openshift-console/console-7585666dd5-zttv2 b4f5b0f53d5608dc0c1cfa974e35cee0ccfda680a64ab318353afaa0ea806229 network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-7585666dd5-zttv2 b4f5b0f53d5608dc0c1cfa974e35cee0ccfda680a64ab318353afaa0ea806229 network default NAD default NAD key default], result "{}", err I0505 08:23:58.039722 3788 obj_retry.go:498] Detected object openshift-console/console-7585666dd5-zttv2 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 08:23:58.039762 3788 pods.go:176] Deleting pod: openshift-console/console-7585666dd5-zttv2 I0505 08:23:58.041159 3788 pods.go:236] Attempting to release IPs for pod: openshift-console/console-7585666dd5-zttv2, ips: 10.129.0.16 I0505 08:23:58.041196 3788 obj_retry.go:498] Detected object openshift-console/console-7585666dd5-zttv2 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 08:23:58.047966 3788 base_network_controller_pods.go:487] [default/openshift-console/console-794776bbc8-vcmbr] creating logical port openshift-console_console-794776bbc8-vcmbr for pod on switch ip-10-0-1-71.ec2.internal I0505 08:23:58.048166 3788 kube.go:256] Updating pod openshift-console/console-794776bbc8-vcmbr I0505 08:23:58.062167 3788 pod.go:62] [openshift-console/console-794776bbc8-vcmbr] pod update took 14.065163ms I0505 08:23:58.062196 3788 base_network_controller_pods.go:951] [default/openshift-console/console-794776bbc8-vcmbr] addLogicalPort annotation time took 14.100149ms I0505 08:23:58.063410 3788 pods.go:271] [openshift-console/console-794776bbc8-vcmbr] addLogicalPort took 15.468598ms, libovsdb time 860.714µs I0505 08:23:58.478026 3788 cni.go:452] [openshift-console/console-794776bbc8-vcmbr 32c0e4f9907b4422f338b2b2ccefdde80b3f6bcc25c4e00eb0e7407e02bd5d1d network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-794776bbc8-vcmbr 32c0e4f9907b4422f338b2b2ccefdde80b3f6bcc25c4e00eb0e7407e02bd5d1d network default NAD default NAD key ] I0505 08:23:58.488387 3788 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-794776bbc8-vcmbr, hostIfaceName: 32c0e4f9907b442, network: default, NAD default, SandboxID: "32c0e4f9907b4422f338b2b2ccefdde80b3f6bcc25c4e00eb0e7407e02bd5d1d", PCI device ID: , UID: "1dab04ca-2e00-4451-892b-2c0b2d75c309", MAC: 0a:58:0a:81:00:16, IPs: [10.129.0.22/23] I0505 08:23:58.545744 3788 cni.go:473] [openshift-console/console-794776bbc8-vcmbr 32c0e4f9907b4422f338b2b2ccefdde80b3f6bcc25c4e00eb0e7407e02bd5d1d network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-794776bbc8-vcmbr 32c0e4f9907b4422f338b2b2ccefdde80b3f6bcc25c4e00eb0e7407e02bd5d1d network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"e2:c8:4f:88:37:43\",\"name\":\"32c0e4f9907b442\"},{\"mac\":\"0a:58:0a:81:00:16\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2ebd6771-f9e8-41b1-97dc-2ad2e1a08cea\"}],\"ips\":[{\"address\":\"10.129.0.22/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0505 08:23:58.680677 3788 cni.go:452] [openshift-image-registry/image-registry-5f557bd69b-lnjm9 42d925b93dfe912813d63d09f846370893ab897c7ce8686a21df1a45620d9368 network default NAD default NAD key ] DEL starting CNI request [openshift-image-registry/image-registry-5f557bd69b-lnjm9 42d925b93dfe912813d63d09f846370893ab897c7ce8686a21df1a45620d9368 network default NAD default NAD key ] I0505 08:23:58.685129 3788 obj_retry.go:492] Detected object openshift-console/console-7585666dd5-zttv2 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:23:58.685202 3788 obj_retry.go:492] Detected object openshift-console/console-7585666dd5-zttv2 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:23:58.689190 3788 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-7585666dd5-zttv2 I0505 08:23:58.689226 3788 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-7585666dd5-zttv2 I0505 08:23:58.778281 3788 cni.go:473] [openshift-image-registry/image-registry-5f557bd69b-lnjm9 42d925b93dfe912813d63d09f846370893ab897c7ce8686a21df1a45620d9368 network default NAD default NAD key default] DEL finished CNI request [openshift-image-registry/image-registry-5f557bd69b-lnjm9 42d925b93dfe912813d63d09f846370893ab897c7ce8686a21df1a45620d9368 network default NAD default NAD key default], result "{}", err I0505 08:23:58.838989 3788 obj_retry.go:498] Detected object openshift-image-registry/image-registry-5f557bd69b-lnjm9 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 08:23:58.839029 3788 pods.go:176] Deleting pod: openshift-image-registry/image-registry-5f557bd69b-lnjm9 I0505 08:23:58.840319 3788 pods.go:236] Attempting to release IPs for pod: openshift-image-registry/image-registry-5f557bd69b-lnjm9, ips: 10.129.0.5 I0505 08:23:58.840349 3788 obj_retry.go:498] Detected object openshift-image-registry/image-registry-5f557bd69b-lnjm9 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 08:23:59.698054 3788 obj_retry.go:492] Detected object openshift-image-registry/image-registry-5f557bd69b-lnjm9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:23:59.698075 3788 obj_retry.go:492] Detected object openshift-image-registry/image-registry-5f557bd69b-lnjm9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:23:59.706605 3788 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-5f557bd69b-lnjm9 I0505 08:23:59.706637 3788 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-5f557bd69b-lnjm9 W0505 08:24:00.220348 3788 util.go:959] Failed to build node endpoints for node ip-10-0-1-71.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0505 08:24:00.220375 3788 util.go:959] Failed to build node endpoints for node ip-10-0-1-71.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0505 08:24:00.220382 3788 util.go:959] Failed to build node endpoints for node ip-10-0-1-71.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0505 08:24:00.633142 3788 util.go:959] Failed to build node endpoints for node ip-10-0-1-71.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0505 08:24:00.633169 3788 util.go:959] Failed to build node endpoints for node ip-10-0-1-71.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0505 08:24:00.633192 3788 util.go:959] Failed to build node endpoints for node ip-10-0-1-71.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0505 08:24:00.633233 3788 util.go:959] Failed to build node endpoints for node ip-10-0-1-71.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0505 08:24:00.673831 3788 util.go:959] Failed to build node endpoints for node ip-10-0-1-71.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0505 08:24:00.673854 3788 util.go:959] Failed to build node endpoints for node ip-10-0-1-71.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0505 08:24:08.724700 3788 util.go:959] Failed to build node endpoints for node ip-10-0-1-71.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0505 08:24:17.759285 3788 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0505 08:24:17.759311 3788 util.go:959] Failed to build node endpoints for node ip-10-0-1-71.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0505 08:24:17.759359 3788 util.go:959] Failed to build node endpoints for node ip-10-0-1-71.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0505 08:24:17.891020 3788 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-service-ca-operator/metrics W0505 08:24:18.765479 3788 util.go:959] Failed to build node endpoints for node ip-10-0-1-71.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics I0505 08:24:25.951720 3788 obj_retry.go:498] Detected object openshift-console/console-578895dbb6-gg97r of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 08:24:25.952288 3788 obj_retry.go:498] Detected object openshift-console/console-578895dbb6-gg97r of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 08:24:26.912832 3788 obj_retry.go:492] Detected object openshift-console/console-578895dbb6-gg97r of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:24:26.912855 3788 obj_retry.go:492] Detected object openshift-console/console-578895dbb6-gg97r of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:24:26.917895 3788 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-578895dbb6-gg97r I0505 08:24:26.917922 3788 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-578895dbb6-gg97r I0505 08:24:32.215817 3788 cni.go:452] [openshift-ingress/router-default-96cd775b9-xpgcv ece6957f6265cd837e9f6e77a0149eba0c2ff97dffe18aee82562564b3b8a06e network default NAD default NAD key ] DEL starting CNI request [openshift-ingress/router-default-96cd775b9-xpgcv ece6957f6265cd837e9f6e77a0149eba0c2ff97dffe18aee82562564b3b8a06e network default NAD default NAD key ] I0505 08:24:32.295940 3788 cni.go:473] [openshift-ingress/router-default-96cd775b9-xpgcv ece6957f6265cd837e9f6e77a0149eba0c2ff97dffe18aee82562564b3b8a06e network default NAD default NAD key default] DEL finished CNI request [openshift-ingress/router-default-96cd775b9-xpgcv ece6957f6265cd837e9f6e77a0149eba0c2ff97dffe18aee82562564b3b8a06e network default NAD default NAD key default], result "{}", err I0505 08:24:32.347615 3788 obj_retry.go:498] Detected object openshift-ingress/router-default-96cd775b9-xpgcv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 08:24:32.347653 3788 pods.go:176] Deleting pod: openshift-ingress/router-default-96cd775b9-xpgcv I0505 08:24:32.349023 3788 pods.go:236] Attempting to release IPs for pod: openshift-ingress/router-default-96cd775b9-xpgcv, ips: 10.129.0.11 I0505 08:24:32.349053 3788 obj_retry.go:498] Detected object openshift-ingress/router-default-96cd775b9-xpgcv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 08:24:32.815763 3788 obj_retry.go:492] Detected object openshift-ingress/router-default-96cd775b9-xpgcv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:24:32.815789 3788 obj_retry.go:492] Detected object openshift-ingress/router-default-96cd775b9-xpgcv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:24:32.821613 3788 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/router-default-96cd775b9-xpgcv I0505 08:24:32.821643 3788 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/router-default-96cd775b9-xpgcv I0505 08:24:34.102834 3788 cni.go:452] [openshift-multus/network-metrics-daemon-b2tcb 244bb9d9a9653ae35b3bfee17ebc16f181c7c68822a0f20defd9ba439f805924 network default NAD default NAD key ] ADD starting CNI request [openshift-multus/network-metrics-daemon-b2tcb 244bb9d9a9653ae35b3bfee17ebc16f181c7c68822a0f20defd9ba439f805924 network default NAD default NAD key ] I0505 08:24:34.112695 3788 helper_linux.go:556] ConfigureOVS: namespace: openshift-multus, podName: network-metrics-daemon-b2tcb, hostIfaceName: 244bb9d9a9653ae, network: default, NAD default, SandboxID: "244bb9d9a9653ae35b3bfee17ebc16f181c7c68822a0f20defd9ba439f805924", PCI device ID: , UID: "67ba7abd-f980-4892-ab5c-406e6a371954", MAC: 0a:58:0a:81:00:04, IPs: [10.129.0.4/23] I0505 08:24:34.165289 3788 cni.go:473] [openshift-multus/network-metrics-daemon-b2tcb 244bb9d9a9653ae35b3bfee17ebc16f181c7c68822a0f20defd9ba439f805924 network default NAD default NAD key default] ADD finished CNI request [openshift-multus/network-metrics-daemon-b2tcb 244bb9d9a9653ae35b3bfee17ebc16f181c7c68822a0f20defd9ba439f805924 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"56:9f:d1:7d:09:9c\",\"name\":\"244bb9d9a9653ae\"},{\"mac\":\"0a:58:0a:81:00:04\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5492405c-c0ce-440b-9ffe-e7d490cc9c05\"}],\"ips\":[{\"address\":\"10.129.0.4/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0505 08:24:35.890513 3788 util.go:959] Failed to build node endpoints for node ip-10-0-1-71.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0505 08:24:35.890539 3788 util.go:959] Failed to build node endpoints for node ip-10-0-1-71.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0505 08:24:49.948907 3788 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 08:24:49.949539 3788 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 08:24:50.311000 3788 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:24:50.311028 3788 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:24:50.319437 3788 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-1 I0505 08:24:50.319500 3788 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-1 I0505 08:24:53.328273 3788 base_network_controller_pods.go:487] [default/openshift-monitoring/telemeter-client-68747f5f56-vt2jw] creating logical port openshift-monitoring_telemeter-client-68747f5f56-vt2jw for pod on switch ip-10-0-1-71.ec2.internal I0505 08:24:53.328375 3788 kube.go:256] Updating pod openshift-monitoring/telemeter-client-68747f5f56-vt2jw I0505 08:24:53.359677 3788 pod.go:62] [openshift-monitoring/telemeter-client-68747f5f56-vt2jw] pod update took 31.327629ms I0505 08:24:53.359703 3788 base_network_controller_pods.go:951] [default/openshift-monitoring/telemeter-client-68747f5f56-vt2jw] addLogicalPort annotation time took 31.360621ms I0505 08:24:53.361806 3788 pods.go:271] [openshift-monitoring/telemeter-client-68747f5f56-vt2jw] addLogicalPort took 33.551575ms, libovsdb time 1.715869ms I0505 08:24:53.695876 3788 cni.go:452] [openshift-monitoring/telemeter-client-68747f5f56-vt2jw 8210d34d2bab39226fc80c758f608bcd698b3ff83e5a0c8e55d2ceb46e3e9dd0 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/telemeter-client-68747f5f56-vt2jw 8210d34d2bab39226fc80c758f608bcd698b3ff83e5a0c8e55d2ceb46e3e9dd0 network default NAD default NAD key ] I0505 08:24:53.708442 3788 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: telemeter-client-68747f5f56-vt2jw, hostIfaceName: 8210d34d2bab392, network: default, NAD default, SandboxID: "8210d34d2bab39226fc80c758f608bcd698b3ff83e5a0c8e55d2ceb46e3e9dd0", PCI device ID: , UID: "6edb891a-ec21-49b7-b986-9e7c3c0a19d2", MAC: 0a:58:0a:81:00:17, IPs: [10.129.0.23/23] I0505 08:24:53.771695 3788 cni.go:473] [openshift-monitoring/telemeter-client-68747f5f56-vt2jw 8210d34d2bab39226fc80c758f608bcd698b3ff83e5a0c8e55d2ceb46e3e9dd0 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/telemeter-client-68747f5f56-vt2jw 8210d34d2bab39226fc80c758f608bcd698b3ff83e5a0c8e55d2ceb46e3e9dd0 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"b2:fb:94:62:86:d1\",\"name\":\"8210d34d2bab392\"},{\"mac\":\"0a:58:0a:81:00:17\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/81ae74f6-3d25-4907-82c9-374f912d0169\"}],\"ips\":[{\"address\":\"10.129.0.23/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0505 08:24:54.960283 3788 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 08:24:54.960845 3788 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 08:24:55.346995 3788 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:24:55.347022 3788 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:24:55.352110 3788 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-1 I0505 08:24:55.352141 3788 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-1 I0505 08:25:21.938762 3788 cni.go:452] [openshift-monitoring/alertmanager-main-0 93c30d006a5d4191a00b64a78dc9fa338b1f50b004184b75216c9cea69d6dbb9 network default NAD default NAD key ] DEL starting CNI request [openshift-monitoring/alertmanager-main-0 93c30d006a5d4191a00b64a78dc9fa338b1f50b004184b75216c9cea69d6dbb9 network default NAD default NAD key ] I0505 08:25:22.022312 3788 cni.go:473] [openshift-monitoring/alertmanager-main-0 93c30d006a5d4191a00b64a78dc9fa338b1f50b004184b75216c9cea69d6dbb9 network default NAD default NAD key default] DEL finished CNI request [openshift-monitoring/alertmanager-main-0 93c30d006a5d4191a00b64a78dc9fa338b1f50b004184b75216c9cea69d6dbb9 network default NAD default NAD key default], result "{}", err I0505 08:25:22.088510 3788 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 08:25:22.088663 3788 pods.go:176] Deleting pod: openshift-monitoring/alertmanager-main-0 I0505 08:25:22.090106 3788 pods.go:236] Attempting to release IPs for pod: openshift-monitoring/alertmanager-main-0, ips: 10.129.0.18 I0505 08:25:22.090134 3788 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 08:25:22.439253 3788 obj_retry.go:498] Detected object openshift-console/console-794776bbc8-qcpf4 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 08:25:22.439824 3788 obj_retry.go:498] Detected object openshift-console/console-794776bbc8-qcpf4 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 08:25:22.931532 3788 obj_retry.go:492] Detected object openshift-console/console-794776bbc8-qcpf4 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:25:22.931556 3788 obj_retry.go:492] Detected object openshift-console/console-794776bbc8-qcpf4 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:25:22.937133 3788 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-794776bbc8-qcpf4 I0505 08:25:22.937154 3788 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-794776bbc8-qcpf4 I0505 08:25:23.036578 3788 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:25:23.036604 3788 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:25:23.043660 3788 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-0 I0505 08:25:23.043693 3788 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-0 I0505 08:25:23.086237 3788 base_network_controller_pods.go:487] [default/openshift-monitoring/alertmanager-main-0] creating logical port openshift-monitoring_alertmanager-main-0 for pod on switch ip-10-0-1-71.ec2.internal I0505 08:25:23.086375 3788 kube.go:256] Updating pod openshift-monitoring/alertmanager-main-0 I0505 08:25:23.111129 3788 pod.go:62] [openshift-monitoring/alertmanager-main-0] pod update took 24.793361ms I0505 08:25:23.111154 3788 base_network_controller_pods.go:951] [default/openshift-monitoring/alertmanager-main-0] addLogicalPort annotation time took 24.823743ms I0505 08:25:23.112370 3788 pods.go:271] [openshift-monitoring/alertmanager-main-0] addLogicalPort took 26.150157ms, libovsdb time 878.507µs I0505 08:25:23.447774 3788 cni.go:452] [openshift-monitoring/alertmanager-main-0 b8f9290f784c86eace2391e1e753a415ad1fea0d121bff7e5679d9a613dc0ade network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/alertmanager-main-0 b8f9290f784c86eace2391e1e753a415ad1fea0d121bff7e5679d9a613dc0ade network default NAD default NAD key ] I0505 08:25:23.457133 3788 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: alertmanager-main-0, hostIfaceName: b8f9290f784c86e, network: default, NAD default, SandboxID: "b8f9290f784c86eace2391e1e753a415ad1fea0d121bff7e5679d9a613dc0ade", PCI device ID: , UID: "95479e32-2f32-42f0-8ad5-92349465dfec", MAC: 0a:58:0a:81:00:18, IPs: [10.129.0.24/23] I0505 08:25:23.507046 3788 cni.go:473] [openshift-monitoring/alertmanager-main-0 b8f9290f784c86eace2391e1e753a415ad1fea0d121bff7e5679d9a613dc0ade network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/alertmanager-main-0 b8f9290f784c86eace2391e1e753a415ad1fea0d121bff7e5679d9a613dc0ade network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"06:0d:db:d6:80:2d\",\"name\":\"b8f9290f784c86e\"},{\"mac\":\"0a:58:0a:81:00:18\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/06c6fa40-f9d6-4337-a42c-da15ed77da06\"}],\"ips\":[{\"address\":\"10.129.0.24/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0505 08:25:33.321240 3788 cni.go:452] [openshift-console/console-794776bbc8-vcmbr 32c0e4f9907b4422f338b2b2ccefdde80b3f6bcc25c4e00eb0e7407e02bd5d1d network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-794776bbc8-vcmbr 32c0e4f9907b4422f338b2b2ccefdde80b3f6bcc25c4e00eb0e7407e02bd5d1d network default NAD default NAD key ] I0505 08:25:33.401199 3788 cni.go:473] [openshift-console/console-794776bbc8-vcmbr 32c0e4f9907b4422f338b2b2ccefdde80b3f6bcc25c4e00eb0e7407e02bd5d1d network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-794776bbc8-vcmbr 32c0e4f9907b4422f338b2b2ccefdde80b3f6bcc25c4e00eb0e7407e02bd5d1d network default NAD default NAD key default], result "{}", err I0505 08:25:33.451682 3788 obj_retry.go:498] Detected object openshift-console/console-794776bbc8-vcmbr of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 08:25:33.451717 3788 pods.go:176] Deleting pod: openshift-console/console-794776bbc8-vcmbr I0505 08:25:33.453072 3788 pods.go:236] Attempting to release IPs for pod: openshift-console/console-794776bbc8-vcmbr, ips: 10.129.0.22 I0505 08:25:33.453096 3788 obj_retry.go:498] Detected object openshift-console/console-794776bbc8-vcmbr of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 08:25:34.082207 3788 obj_retry.go:492] Detected object openshift-console/console-794776bbc8-vcmbr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:25:34.082231 3788 obj_retry.go:492] Detected object openshift-console/console-794776bbc8-vcmbr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:25:34.088847 3788 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-794776bbc8-vcmbr I0505 08:25:34.088870 3788 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-794776bbc8-vcmbr W0505 08:25:53.500085 3788 util.go:959] Failed to build node endpoints for node ip-10-0-1-71.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0505 08:25:53.500108 3788 util.go:959] Failed to build node endpoints for node ip-10-0-1-71.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0505 08:25:53.500115 3788 util.go:959] Failed to build node endpoints for node ip-10-0-1-71.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0505 08:25:56.942586 3788 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=6 I0505 08:25:57.504109 3788 cni.go:452] [openshift-monitoring/prometheus-k8s-0 433e36fba82e383f97cbf394a2d445604ba4f49fa879aa05e59c2e7b4ee7c00a network default NAD default NAD key ] DEL starting CNI request [openshift-monitoring/prometheus-k8s-0 433e36fba82e383f97cbf394a2d445604ba4f49fa879aa05e59c2e7b4ee7c00a network default NAD default NAD key ] I0505 08:25:57.583599 3788 cni.go:473] [openshift-monitoring/prometheus-k8s-0 433e36fba82e383f97cbf394a2d445604ba4f49fa879aa05e59c2e7b4ee7c00a network default NAD default NAD key default] DEL finished CNI request [openshift-monitoring/prometheus-k8s-0 433e36fba82e383f97cbf394a2d445604ba4f49fa879aa05e59c2e7b4ee7c00a network default NAD default NAD key default], result "{}", err I0505 08:25:57.675932 3788 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 08:25:57.675967 3788 pods.go:176] Deleting pod: openshift-monitoring/prometheus-k8s-0 I0505 08:25:57.677396 3788 pods.go:236] Attempting to release IPs for pod: openshift-monitoring/prometheus-k8s-0, ips: 10.129.0.21 I0505 08:25:57.677425 3788 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 08:25:58.203513 3788 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:25:58.203538 3788 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:25:58.214720 3788 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-0 I0505 08:25:58.214750 3788 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-0 I0505 08:25:58.266631 3788 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-k8s-0] creating logical port openshift-monitoring_prometheus-k8s-0 for pod on switch ip-10-0-1-71.ec2.internal I0505 08:25:58.266742 3788 kube.go:256] Updating pod openshift-monitoring/prometheus-k8s-0 I0505 08:25:58.296682 3788 pod.go:62] [openshift-monitoring/prometheus-k8s-0] pod update took 29.984193ms I0505 08:25:58.296706 3788 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-k8s-0] addLogicalPort annotation time took 30.012513ms I0505 08:25:58.297912 3788 pods.go:271] [openshift-monitoring/prometheus-k8s-0] addLogicalPort took 31.298244ms, libovsdb time 882.841µs I0505 08:25:58.631301 3788 cni.go:452] [openshift-monitoring/prometheus-k8s-0 4a40142db414c29571e26468eb4ca1e90a66ee7be49e28652c8119ba76226ab9 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-k8s-0 4a40142db414c29571e26468eb4ca1e90a66ee7be49e28652c8119ba76226ab9 network default NAD default NAD key ] I0505 08:25:58.641473 3788 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-k8s-0, hostIfaceName: 4a40142db414c29, network: default, NAD default, SandboxID: "4a40142db414c29571e26468eb4ca1e90a66ee7be49e28652c8119ba76226ab9", PCI device ID: , UID: "726f41fc-9181-4fd8-8221-fca6f4db5a0a", MAC: 0a:58:0a:81:00:19, IPs: [10.129.0.25/23] I0505 08:25:58.691648 3788 cni.go:473] [openshift-monitoring/prometheus-k8s-0 4a40142db414c29571e26468eb4ca1e90a66ee7be49e28652c8119ba76226ab9 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-k8s-0 4a40142db414c29571e26468eb4ca1e90a66ee7be49e28652c8119ba76226ab9 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"66:7f:ee:77:fb:63\",\"name\":\"4a40142db414c29\"},{\"mac\":\"0a:58:0a:81:00:19\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/fd4fb3dd-332a-4311-addd-68a0ca504db5\"}],\"ips\":[{\"address\":\"10.129.0.25/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0505 08:26:02.938550 3788 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=7 I0505 08:26:08.851859 3788 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=69 I0505 08:26:14.971513 3788 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=7 I0505 08:26:28.568622 3788 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29632820-5v8c9 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 08:26:28.569168 3788 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29632820-5v8c9 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0505 08:26:29.187001 3788 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0505 08:26:29.610929 3788 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29632820-5v8c9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:26:29.610954 3788 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29632820-5v8c9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:26:40.235698 3788 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... W0505 08:26:40.235789 3788 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-deployment-validation-operator/deployment-validation-operator-65d68cbd6c-fm54j IP address from the namespace address-set, err: pod openshift-deployment-validation-operator/deployment-validation-operator-65d68cbd6c-fm54j: no pod IPs found I0505 08:26:40.309305 3788 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0505 08:26:51.966078 3788 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=6 I0505 08:26:52.975778 3788 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=6 I0505 08:26:56.960665 3788 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=7 W0505 08:26:59.472541 3788 util.go:959] Failed to build node endpoints for node ip-10-0-1-71.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0505 08:26:59.472571 3788 util.go:959] Failed to build node endpoints for node ip-10-0-1-71.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0505 08:27:06.010766 3788 base_network_controller_pods.go:487] [default/openshift-console/console-7cffd994fd-6hbg2] creating logical port openshift-console_console-7cffd994fd-6hbg2 for pod on switch ip-10-0-1-71.ec2.internal I0505 08:27:06.010914 3788 kube.go:256] Updating pod openshift-console/console-7cffd994fd-6hbg2 I0505 08:27:06.032524 3788 pod.go:62] [openshift-console/console-7cffd994fd-6hbg2] pod update took 21.64192ms I0505 08:27:06.032546 3788 base_network_controller_pods.go:951] [default/openshift-console/console-7cffd994fd-6hbg2] addLogicalPort annotation time took 21.670917ms I0505 08:27:06.033755 3788 pods.go:271] [openshift-console/console-7cffd994fd-6hbg2] addLogicalPort took 23.004734ms, libovsdb time 900.201µs I0505 08:27:06.371308 3788 cni.go:452] [openshift-console/console-7cffd994fd-6hbg2 45e63288061ec231dd0b8108e83053a97ebf43160cab226676fcf1515086d5a8 network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-7cffd994fd-6hbg2 45e63288061ec231dd0b8108e83053a97ebf43160cab226676fcf1515086d5a8 network default NAD default NAD key ] I0505 08:27:06.380478 3788 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-7cffd994fd-6hbg2, hostIfaceName: 45e63288061ec23, network: default, NAD default, SandboxID: "45e63288061ec231dd0b8108e83053a97ebf43160cab226676fcf1515086d5a8", PCI device ID: , UID: "00a64453-4743-4416-9bc2-f0cec487ae87", MAC: 0a:58:0a:81:00:1a, IPs: [10.129.0.26/23] I0505 08:27:06.430222 3788 cni.go:473] [openshift-console/console-7cffd994fd-6hbg2 45e63288061ec231dd0b8108e83053a97ebf43160cab226676fcf1515086d5a8 network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-7cffd994fd-6hbg2 45e63288061ec231dd0b8108e83053a97ebf43160cab226676fcf1515086d5a8 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"02:0a:87:9a:9e:49\",\"name\":\"45e63288061ec23\"},{\"mac\":\"0a:58:0a:81:00:1a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f6870d9d-5d0e-4af3-a7db-10610c41c1ce\"}],\"ips\":[{\"address\":\"10.129.0.26/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0505 08:27:13.850229 3788 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=67 I0505 08:27:15.948408 3788 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=7 W0505 08:27:16.502210 3788 util.go:959] Failed to build node endpoints for node ip-10-0-1-71.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0505 08:27:31.314793 3788 obj_retry.go:498] Detected object openshift-console/console-bcc697d8c-jppmh of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 08:27:31.315355 3788 obj_retry.go:498] Detected object openshift-console/console-bcc697d8c-jppmh of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 08:27:31.689722 3788 obj_retry.go:492] Detected object openshift-console/console-bcc697d8c-jppmh of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:27:31.689746 3788 obj_retry.go:492] Detected object openshift-console/console-bcc697d8c-jppmh of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:27:31.693992 3788 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-bcc697d8c-jppmh I0505 08:27:31.694019 3788 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-bcc697d8c-jppmh I0505 08:27:41.866199 3788 obj_retry.go:498] Detected object openshift-console/console-bcc697d8c-bhqdz of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 08:27:41.866773 3788 obj_retry.go:498] Detected object openshift-console/console-bcc697d8c-bhqdz of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 08:27:42.464773 3788 obj_retry.go:492] Detected object openshift-console/console-bcc697d8c-bhqdz of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:27:42.464804 3788 obj_retry.go:492] Detected object openshift-console/console-bcc697d8c-bhqdz of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:27:42.469987 3788 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-bcc697d8c-bhqdz I0505 08:27:42.470014 3788 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-bcc697d8c-bhqdz I0505 08:28:00.976300 3788 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=8 I0505 08:28:40.852377 3788 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=15 I0505 08:29:29.857611 3788 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=594 I0505 08:29:54.850057 3788 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=148 I0505 08:30:00.851102 3788 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=11 I0505 08:30:01.938584 3788 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=10 I0505 08:30:04.453784 3788 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29632830-rshpg of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 08:30:04.454377 3788 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29632830-rshpg of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 08:30:05.476099 3788 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29632830-rshpg of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:30:05.476126 3788 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29632830-rshpg of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:30:33.957692 3788 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=10 I0505 08:31:04.232379 3788 namespace.go:98] [tsf] adding namespace I0505 08:31:04.233301 3788 namespace.go:102] [tsf] adding namespace took 897.841µs I0505 08:31:04.251085 3788 namespace.go:142] [tsf] updating namespace I0505 08:31:04.267169 3788 namespace.go:142] [tsf] updating namespace I0505 08:31:08.170166 3788 namespace.go:98] [tsf-tpa] adding namespace I0505 08:31:08.171195 3788 namespace.go:102] [tsf-tpa] adding namespace took 1.005087ms I0505 08:31:08.173781 3788 namespace.go:98] [konflux-ui] adding namespace I0505 08:31:08.174477 3788 namespace.go:102] [konflux-ui] adding namespace took 662.021µs I0505 08:31:08.177369 3788 namespace.go:98] [tsf-tas] adding namespace I0505 08:31:08.178019 3788 namespace.go:102] [tsf-tas] adding namespace took 636.821µs I0505 08:31:08.190245 3788 namespace.go:142] [tsf-tpa] updating namespace I0505 08:31:08.198250 3788 namespace.go:98] [cert-manager-operator] adding namespace I0505 08:31:08.198335 3788 namespace.go:98] [rhbk-operator] adding namespace I0505 08:31:08.198516 3788 namespace.go:98] [rhtpa-operator] adding namespace I0505 08:31:08.198641 3788 namespace.go:98] [konflux-operator] adding namespace I0505 08:31:08.199009 3788 namespace.go:102] [cert-manager-operator] adding namespace took 741.502µs I0505 08:31:08.199634 3788 namespace.go:102] [rhbk-operator] adding namespace took 1.284257ms I0505 08:31:08.199659 3788 namespace.go:98] [tsf-keycloak] adding namespace I0505 08:31:08.200264 3788 namespace.go:102] [tsf-keycloak] adding namespace took 600.416µs I0505 08:31:08.200889 3788 namespace.go:102] [rhtpa-operator] adding namespace took 2.359733ms I0505 08:31:08.201358 3788 namespace.go:102] [konflux-operator] adding namespace took 2.698473ms I0505 08:31:08.228348 3788 namespace.go:142] [tsf-tpa] updating namespace I0505 08:31:08.234254 3788 namespace.go:142] [konflux-ui] updating namespace I0505 08:31:08.234386 3788 namespace.go:142] [tsf-tas] updating namespace I0505 08:31:08.234437 3788 namespace.go:142] [cert-manager-operator] updating namespace I0505 08:31:08.237193 3788 namespace.go:142] [rhbk-operator] updating namespace I0505 08:31:08.258234 3788 namespace.go:142] [rhtpa-operator] updating namespace I0505 08:31:08.258345 3788 namespace.go:142] [konflux-ui] updating namespace I0505 08:31:08.261542 3788 namespace.go:142] [tsf-tas] updating namespace I0505 08:31:08.261566 3788 namespace.go:142] [konflux-operator] updating namespace I0505 08:31:08.261664 3788 namespace.go:142] [tsf-keycloak] updating namespace I0505 08:31:08.271077 3788 namespace.go:142] [tsf-tpa] updating namespace I0505 08:31:08.280586 3788 namespace.go:142] [tsf-tas] updating namespace I0505 08:31:08.289567 3788 namespace.go:142] [cert-manager-operator] updating namespace I0505 08:31:08.310175 3788 namespace.go:142] [cert-manager-operator] updating namespace I0505 08:31:08.336801 3788 namespace.go:142] [rhbk-operator] updating namespace I0505 08:31:08.353328 3788 namespace.go:142] [rhbk-operator] updating namespace I0505 08:31:08.373223 3788 namespace.go:142] [tsf-keycloak] updating namespace I0505 08:31:08.383253 3788 namespace.go:142] [tsf-keycloak] updating namespace I0505 08:31:08.393069 3788 namespace.go:142] [konflux-ui] updating namespace I0505 08:31:08.424314 3788 namespace.go:142] [rhtpa-operator] updating namespace I0505 08:31:08.430163 3788 namespace.go:142] [rhtpa-operator] updating namespace I0505 08:31:08.463224 3788 namespace.go:142] [konflux-operator] updating namespace I0505 08:31:08.485395 3788 namespace.go:142] [konflux-operator] updating namespace I0505 08:31:09.336358 3788 namespace.go:142] [tsf-tpa] updating namespace I0505 08:31:09.361060 3788 namespace.go:142] [tsf-keycloak] updating namespace I0505 08:31:09.523929 3788 base_network_controller_pods.go:487] [default/tsf/test-tsf-subscriptions] creating logical port tsf_test-tsf-subscriptions for pod on switch ip-10-0-1-71.ec2.internal I0505 08:31:09.524037 3788 kube.go:256] Updating pod tsf/test-tsf-subscriptions I0505 08:31:09.549247 3788 pod.go:62] [tsf/test-tsf-subscriptions] pod update took 25.247193ms I0505 08:31:09.549268 3788 base_network_controller_pods.go:951] [default/tsf/test-tsf-subscriptions] addLogicalPort annotation time took 25.27503ms I0505 08:31:09.550651 3788 pods.go:271] [tsf/test-tsf-subscriptions] addLogicalPort took 26.740826ms, libovsdb time 1.049181ms I0505 08:31:09.891975 3788 cni.go:452] [tsf/test-tsf-subscriptions d26e3da24e8631d92c8cf455ecf364ea56dc0e892c65150a1d380cb44d9d5e82 network default NAD default NAD key ] ADD starting CNI request [tsf/test-tsf-subscriptions d26e3da24e8631d92c8cf455ecf364ea56dc0e892c65150a1d380cb44d9d5e82 network default NAD default NAD key ] I0505 08:31:09.901186 3788 helper_linux.go:556] ConfigureOVS: namespace: tsf, podName: test-tsf-subscriptions, hostIfaceName: d26e3da24e8631d, network: default, NAD default, SandboxID: "d26e3da24e8631d92c8cf455ecf364ea56dc0e892c65150a1d380cb44d9d5e82", PCI device ID: , UID: "1cc61b29-8ff7-49ab-a106-28731b5caf3a", MAC: 0a:58:0a:81:00:1b, IPs: [10.129.0.27/23] I0505 08:31:09.951132 3788 cni.go:473] [tsf/test-tsf-subscriptions d26e3da24e8631d92c8cf455ecf364ea56dc0e892c65150a1d380cb44d9d5e82 network default NAD default NAD key default] ADD finished CNI request [tsf/test-tsf-subscriptions d26e3da24e8631d92c8cf455ecf364ea56dc0e892c65150a1d380cb44d9d5e82 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"32:c8:20:ce:5e:a0\",\"name\":\"d26e3da24e8631d\"},{\"mac\":\"0a:58:0a:81:00:1b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/709886a8-b4bb-4c18-a715-03a96ee8f6d3\"}],\"ips\":[{\"address\":\"10.129.0.27/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0505 08:31:16.749103 3788 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dk4424 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 08:31:16.749748 3788 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dk4424 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 08:31:16.749822 3788 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dk4424 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0505 08:31:16.749836 3788 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0505 08:31:16.749852 3788 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dk4424 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0505 08:31:16.749859 3788 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0505 08:31:17.783997 3788 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dk4424 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:31:17.784038 3788 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dk4424 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:31:17.784056 3788 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dk4424 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:31:17.784070 3788 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dk4424 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:31:18.769368 3788 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51njqvp of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 08:31:18.769971 3788 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51njqvp of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 08:31:18.769999 3788 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51njqvp of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0505 08:31:18.770006 3788 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0505 08:31:18.770017 3788 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51njqvp of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0505 08:31:18.770021 3788 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0505 08:31:19.795953 3788 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51njqvp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:31:19.795982 3788 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51njqvp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:31:19.796002 3788 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51njqvp of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:31:19.796012 3788 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51njqvp of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:31:19.862600 3788 obj_retry.go:498] Detected object openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6espg6l of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 08:31:19.863117 3788 obj_retry.go:498] Detected object openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6espg6l of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 08:31:19.863156 3788 obj_retry.go:498] Detected object openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6espg6l of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0505 08:31:19.863169 3788 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0505 08:31:19.863180 3788 obj_retry.go:498] Detected object openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6espg6l of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0505 08:31:19.863189 3788 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0505 08:31:20.889090 3788 obj_retry.go:492] Detected object openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6espg6l of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:31:20.889116 3788 obj_retry.go:492] Detected object openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6espg6l of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:31:20.889130 3788 obj_retry.go:492] Detected object openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6espg6l of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:31:20.889141 3788 obj_retry.go:492] Detected object openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6espg6l of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:31:21.810211 3788 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf24fk9l of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 08:31:21.810756 3788 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf24fk9l of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 08:31:21.810801 3788 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf24fk9l of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0505 08:31:21.810811 3788 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0505 08:31:21.810821 3788 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf24fk9l of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0505 08:31:21.810825 3788 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0505 08:31:22.867719 3788 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136dn78n of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 08:31:22.867741 3788 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf24fk9l of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:31:22.867757 3788 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf24fk9l of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:31:22.867775 3788 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf24fk9l of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:31:22.867787 3788 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf24fk9l of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:31:22.868257 3788 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136dn78n of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 08:31:22.868283 3788 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136dn78n of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0505 08:31:22.868295 3788 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0505 08:31:22.868308 3788 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136dn78n of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0505 08:31:22.868315 3788 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0505 08:31:22.915533 3788 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7vxmzt of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 08:31:22.916069 3788 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7vxmzt of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 08:31:22.916096 3788 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7vxmzt of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0505 08:31:22.916106 3788 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0505 08:31:22.916116 3788 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7vxmzt of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0505 08:31:22.916120 3788 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0505 08:31:23.893853 3788 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136dn78n of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:31:23.893880 3788 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136dn78n of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:31:23.893898 3788 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136dn78n of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:31:23.893913 3788 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136dn78n of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:31:23.944154 3788 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7vxmzt of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:31:23.944180 3788 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7vxmzt of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:31:23.944198 3788 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7vxmzt of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:31:23.944206 3788 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7vxmzt of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:31:29.678157 3788 namespace.go:98] [cert-manager] adding namespace I0505 08:31:29.679070 3788 namespace.go:102] [cert-manager] adding namespace took 886.948µs I0505 08:31:29.723216 3788 namespace.go:142] [cert-manager] updating namespace I0505 08:31:29.752399 3788 namespace.go:142] [cert-manager] updating namespace W0505 08:31:35.372792 3788 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service cert-manager/cert-manager-webhook W0505 08:31:35.372811 3788 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service cert-manager/cert-manager-webhook I0505 08:31:43.065666 3788 namespace.go:142] [openshift-operators] updating namespace I0505 08:31:43.077030 3788 namespace.go:142] [openshift-operators] updating namespace W0505 08:31:44.875289 3788 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0505 08:31:45.829270 3788 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0505 08:31:45.839119 3788 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0505 08:31:45.855287 3788 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0505 08:31:45.863407 3788 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0505 08:31:50.429853 3788 namespace.go:142] [openshift-operators] updating namespace W0505 08:31:57.531576 3788 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0505 08:31:58.945511 3788 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=6 I0505 08:31:59.495124 3788 namespace.go:98] [trusted-artifact-signer] adding namespace I0505 08:31:59.495933 3788 namespace.go:102] [trusted-artifact-signer] adding namespace took 786.046µs I0505 08:31:59.543102 3788 namespace.go:142] [trusted-artifact-signer] updating namespace I0505 08:31:59.592220 3788 namespace.go:142] [trusted-artifact-signer] updating namespace W0505 08:31:59.908398 3788 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0505 08:32:06.262293 3788 namespace.go:98] [openshift-pipelines] adding namespace I0505 08:32:06.263148 3788 namespace.go:102] [openshift-pipelines] adding namespace took 833.093µs I0505 08:32:06.314147 3788 namespace.go:142] [openshift-pipelines] updating namespace I0505 08:32:06.977753 3788 namespace.go:142] [cert-manager] updating namespace I0505 08:32:07.177269 3788 namespace.go:142] [cert-manager-operator] updating namespace I0505 08:32:07.566518 3788 namespace.go:142] [konflux-operator] updating namespace I0505 08:32:07.768819 3788 namespace.go:142] [konflux-ui] updating namespace I0505 08:32:07.969813 3788 namespace.go:142] [open-cluster-management-2q3ldim9a28b4bs5p55oas8jpfchqces] updating namespace I0505 08:32:08.369879 3788 namespace.go:142] [rhbk-operator] updating namespace I0505 08:32:08.568344 3788 namespace.go:142] [rhtpa-operator] updating namespace I0505 08:32:08.775136 3788 namespace.go:142] [trusted-artifact-signer] updating namespace I0505 08:32:08.973874 3788 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=7 I0505 08:32:08.988216 3788 namespace.go:142] [tsf] updating namespace I0505 08:32:09.195755 3788 namespace.go:142] [tsf-keycloak] updating namespace I0505 08:32:09.396430 3788 namespace.go:142] [tsf-tas] updating namespace I0505 08:32:09.611965 3788 namespace.go:142] [tsf-tpa] updating namespace I0505 08:32:09.744758 3788 namespace.go:142] [cert-manager] updating namespace I0505 08:32:09.755004 3788 namespace.go:142] [cert-manager-operator] updating namespace I0505 08:32:09.762500 3788 namespace.go:142] [default] updating namespace I0505 08:32:09.772888 3788 namespace.go:142] [konflux-operator] updating namespace I0505 08:32:09.784428 3788 namespace.go:142] [konflux-ui] updating namespace I0505 08:32:09.793275 3788 namespace.go:142] [open-cluster-management-2q3ldim9a28b4bs5p55oas8jpfchqces] updating namespace I0505 08:32:09.803730 3788 namespace.go:142] [openshift] updating namespace I0505 08:32:09.816718 3788 namespace.go:142] [rhbk-operator] updating namespace I0505 08:32:09.825127 3788 namespace.go:142] [rhtpa-operator] updating namespace I0505 08:32:09.835814 3788 namespace.go:142] [trusted-artifact-signer] updating namespace I0505 08:32:09.843918 3788 namespace.go:142] [tsf] updating namespace I0505 08:32:09.855638 3788 namespace.go:142] [tsf-keycloak] updating namespace I0505 08:32:09.874804 3788 namespace.go:142] [tsf-tas] updating namespace I0505 08:32:09.897635 3788 namespace.go:142] [tsf-tpa] updating namespace I0505 08:32:09.993251 3788 namespace.go:142] [cert-manager] updating namespace I0505 08:32:10.095010 3788 namespace.go:142] [cert-manager-operator] updating namespace I0505 08:32:10.195946 3788 namespace.go:142] [default] updating namespace I0505 08:32:10.308919 3788 namespace.go:142] [konflux-operator] updating namespace I0505 08:32:10.402773 3788 namespace.go:142] [konflux-ui] updating namespace I0505 08:32:10.498271 3788 namespace.go:142] [open-cluster-management-2q3ldim9a28b4bs5p55oas8jpfchqces] updating namespace I0505 08:32:10.599396 3788 namespace.go:142] [openshift] updating namespace I0505 08:32:10.716315 3788 namespace.go:142] [rhbk-operator] updating namespace I0505 08:32:10.799450 3788 namespace.go:142] [rhtpa-operator] updating namespace I0505 08:32:10.905295 3788 namespace.go:142] [trusted-artifact-signer] updating namespace I0505 08:32:10.998862 3788 namespace.go:142] [tsf] updating namespace I0505 08:32:11.100758 3788 namespace.go:142] [tsf-keycloak] updating namespace I0505 08:32:11.200450 3788 namespace.go:142] [tsf-tas] updating namespace I0505 08:32:11.307289 3788 namespace.go:142] [tsf-tpa] updating namespace I0505 08:32:12.059618 3788 namespace.go:142] [openshift-pipelines] updating namespace I0505 08:32:12.695043 3788 cni.go:452] [tsf/test-tsf-subscriptions d26e3da24e8631d92c8cf455ecf364ea56dc0e892c65150a1d380cb44d9d5e82 network default NAD default NAD key ] DEL starting CNI request [tsf/test-tsf-subscriptions d26e3da24e8631d92c8cf455ecf364ea56dc0e892c65150a1d380cb44d9d5e82 network default NAD default NAD key ] I0505 08:32:12.781190 3788 cni.go:473] [tsf/test-tsf-subscriptions d26e3da24e8631d92c8cf455ecf364ea56dc0e892c65150a1d380cb44d9d5e82 network default NAD default NAD key default] DEL finished CNI request [tsf/test-tsf-subscriptions d26e3da24e8631d92c8cf455ecf364ea56dc0e892c65150a1d380cb44d9d5e82 network default NAD default NAD key default], result "{}", err I0505 08:32:12.949163 3788 obj_retry.go:498] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 08:32:12.949199 3788 pods.go:176] Deleting pod: tsf/test-tsf-subscriptions I0505 08:32:12.950434 3788 pods.go:236] Attempting to release IPs for pod: tsf/test-tsf-subscriptions, ips: 10.129.0.27 I0505 08:32:12.950480 3788 obj_retry.go:498] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 08:32:12.980551 3788 obj_retry.go:492] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:32:12.980578 3788 obj_retry.go:492] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:32:12.988683 3788 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-subscriptions I0505 08:32:12.988707 3788 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-subscriptions I0505 08:32:15.638480 3788 base_network_controller_pods.go:487] [default/tsf/test-tsf-infrastructure] creating logical port tsf_test-tsf-infrastructure for pod on switch ip-10-0-1-71.ec2.internal I0505 08:32:15.638612 3788 kube.go:256] Updating pod tsf/test-tsf-infrastructure I0505 08:32:15.653005 3788 pod.go:62] [tsf/test-tsf-infrastructure] pod update took 14.418723ms I0505 08:32:15.653034 3788 base_network_controller_pods.go:951] [default/tsf/test-tsf-infrastructure] addLogicalPort annotation time took 14.449776ms I0505 08:32:15.654222 3788 pods.go:271] [tsf/test-tsf-infrastructure] addLogicalPort took 15.782458ms, libovsdb time 843.374µs I0505 08:32:16.012203 3788 cni.go:452] [tsf/test-tsf-infrastructure 0588f29fb3cfa24ca4a79e6955a242b48e13cb627a0aadea9964417cfdb303bc network default NAD default NAD key ] ADD starting CNI request [tsf/test-tsf-infrastructure 0588f29fb3cfa24ca4a79e6955a242b48e13cb627a0aadea9964417cfdb303bc network default NAD default NAD key ] I0505 08:32:16.022245 3788 helper_linux.go:556] ConfigureOVS: namespace: tsf, podName: test-tsf-infrastructure, hostIfaceName: 0588f29fb3cfa24, network: default, NAD default, SandboxID: "0588f29fb3cfa24ca4a79e6955a242b48e13cb627a0aadea9964417cfdb303bc", PCI device ID: , UID: "9e03f7ff-cf87-4e68-8af4-070c8629abe6", MAC: 0a:58:0a:81:00:1c, IPs: [10.129.0.28/23] I0505 08:32:16.073380 3788 cni.go:473] [tsf/test-tsf-infrastructure 0588f29fb3cfa24ca4a79e6955a242b48e13cb627a0aadea9964417cfdb303bc network default NAD default NAD key default] ADD finished CNI request [tsf/test-tsf-infrastructure 0588f29fb3cfa24ca4a79e6955a242b48e13cb627a0aadea9964417cfdb303bc network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"da:51:4b:20:7b:1d\",\"name\":\"0588f29fb3cfa24\"},{\"mac\":\"0a:58:0a:81:00:1c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d526cb60-6e93-4f14-b161-becb67f5b23a\"}],\"ips\":[{\"address\":\"10.129.0.28/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0505 08:32:18.437626 3788 base_network_controller_pods.go:487] [default/tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-5dqgw] creating logical port tsf-keycloak_keycloak-pgsql-bee-744f6fc4d9-5dqgw for pod on switch ip-10-0-1-71.ec2.internal I0505 08:32:18.437751 3788 kube.go:256] Updating pod tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-5dqgw I0505 08:32:18.456716 3788 pod.go:62] [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-5dqgw] pod update took 18.990832ms I0505 08:32:18.456740 3788 base_network_controller_pods.go:951] [default/tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-5dqgw] addLogicalPort annotation time took 19.020094ms I0505 08:32:18.458038 3788 pods.go:271] [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-5dqgw] addLogicalPort took 20.431164ms, libovsdb time 963.078µs I0505 08:32:19.884733 3788 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-hng62] creating logical port openshift-pipelines_tekton-pipelines-controller-86bd8d9bfc-hng62 for pod on switch ip-10-0-1-71.ec2.internal I0505 08:32:19.884827 3788 kube.go:256] Updating pod openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-hng62 I0505 08:32:19.901998 3788 pod.go:62] [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-hng62] pod update took 17.19756ms I0505 08:32:19.902020 3788 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-hng62] addLogicalPort annotation time took 17.226856ms I0505 08:32:19.903438 3788 pods.go:271] [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-hng62] addLogicalPort took 18.720605ms, libovsdb time 1.062476ms I0505 08:32:19.969272 3788 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-events-controller-57d7f78b8d-cpfs4] creating logical port openshift-pipelines_tekton-events-controller-57d7f78b8d-cpfs4 for pod on switch ip-10-0-1-71.ec2.internal I0505 08:32:19.969387 3788 kube.go:256] Updating pod openshift-pipelines/tekton-events-controller-57d7f78b8d-cpfs4 I0505 08:32:19.984125 3788 pod.go:62] [openshift-pipelines/tekton-events-controller-57d7f78b8d-cpfs4] pod update took 14.760989ms I0505 08:32:19.984149 3788 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-events-controller-57d7f78b8d-cpfs4] addLogicalPort annotation time took 14.790994ms I0505 08:32:19.985419 3788 pods.go:271] [openshift-pipelines/tekton-events-controller-57d7f78b8d-cpfs4] addLogicalPort took 16.15969ms, libovsdb time 905.385µs I0505 08:32:20.244497 3788 cni.go:452] [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-hng62 a1f604c4baf260529f6c682302c80d958798343bbcb3c6a07b3f390ff0cced98 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-hng62 a1f604c4baf260529f6c682302c80d958798343bbcb3c6a07b3f390ff0cced98 network default NAD default NAD key ] I0505 08:32:20.254734 3788 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-controller-86bd8d9bfc-hng62, hostIfaceName: a1f604c4baf2605, network: default, NAD default, SandboxID: "a1f604c4baf260529f6c682302c80d958798343bbcb3c6a07b3f390ff0cced98", PCI device ID: , UID: "f7ed43bd-8184-494d-8557-5f8b3cd70e7a", MAC: 0a:58:0a:81:00:1e, IPs: [10.129.0.30/23] I0505 08:32:20.307516 3788 cni.go:473] [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-hng62 a1f604c4baf260529f6c682302c80d958798343bbcb3c6a07b3f390ff0cced98 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-hng62 a1f604c4baf260529f6c682302c80d958798343bbcb3c6a07b3f390ff0cced98 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"72:a7:18:89:8e:67\",\"name\":\"a1f604c4baf2605\"},{\"mac\":\"0a:58:0a:81:00:1e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b0345553-22f7-4837-856e-595af06d608c\"}],\"ips\":[{\"address\":\"10.129.0.30/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0505 08:32:20.347060 3788 cni.go:452] [openshift-pipelines/tekton-events-controller-57d7f78b8d-cpfs4 5265dd18c8d62014144cff862f7d520aabb4435e34b60ce23c2627f86c4c851b network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-events-controller-57d7f78b8d-cpfs4 5265dd18c8d62014144cff862f7d520aabb4435e34b60ce23c2627f86c4c851b network default NAD default NAD key ] I0505 08:32:20.357315 3788 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-events-controller-57d7f78b8d-cpfs4, hostIfaceName: 5265dd18c8d6201, network: default, NAD default, SandboxID: "5265dd18c8d62014144cff862f7d520aabb4435e34b60ce23c2627f86c4c851b", PCI device ID: , UID: "009d3c81-cb00-4712-ae83-be9d074548f8", MAC: 0a:58:0a:81:00:1f, IPs: [10.129.0.31/23] I0505 08:32:20.408276 3788 cni.go:473] [openshift-pipelines/tekton-events-controller-57d7f78b8d-cpfs4 5265dd18c8d62014144cff862f7d520aabb4435e34b60ce23c2627f86c4c851b network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-events-controller-57d7f78b8d-cpfs4 5265dd18c8d62014144cff862f7d520aabb4435e34b60ce23c2627f86c4c851b network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"f6:94:af:3d:6c:01\",\"name\":\"5265dd18c8d6201\"},{\"mac\":\"0a:58:0a:81:00:1f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/da39930a-6114-4de1-b632-f3824b6665d1\"}],\"ips\":[{\"address\":\"10.129.0.31/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0505 08:32:23.569189 3788 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service trusted-artifact-signer/cli-server W0505 08:32:24.587338 3788 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0505 08:32:24.587354 3788 util.go:870] Failed to build global endpoints for port TCP/https-webhook: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0505 08:32:24.587360 3788 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0505 08:32:24.587365 3788 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0505 08:32:24.772475 3788 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0505 08:32:24.772496 3788 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0505 08:32:24.772501 3788 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0505 08:32:24.797435 3788 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0505 08:32:24.797475 3788 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0505 08:32:24.797482 3788 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-events-controller I0505 08:32:29.028120 3788 cni.go:452] [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-5dqgw 6d4a0b1b9909b00043b1117f35215ef2478d4ec1238b59e7a8833543ff421f2d network default NAD default NAD key ] ADD starting CNI request [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-5dqgw 6d4a0b1b9909b00043b1117f35215ef2478d4ec1238b59e7a8833543ff421f2d network default NAD default NAD key ] I0505 08:32:29.037610 3788 helper_linux.go:556] ConfigureOVS: namespace: tsf-keycloak, podName: keycloak-pgsql-bee-744f6fc4d9-5dqgw, hostIfaceName: 6d4a0b1b9909b00, network: default, NAD default, SandboxID: "6d4a0b1b9909b00043b1117f35215ef2478d4ec1238b59e7a8833543ff421f2d", PCI device ID: , UID: "df824397-947b-49d1-9a01-fc350d7aacfd", MAC: 0a:58:0a:81:00:1d, IPs: [10.129.0.29/23] I0505 08:32:29.089384 3788 cni.go:473] [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-5dqgw 6d4a0b1b9909b00043b1117f35215ef2478d4ec1238b59e7a8833543ff421f2d network default NAD default NAD key default] ADD finished CNI request [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-5dqgw 6d4a0b1b9909b00043b1117f35215ef2478d4ec1238b59e7a8833543ff421f2d network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"8e:0b:fd:10:92:3a\",\"name\":\"6d4a0b1b9909b00\"},{\"mac\":\"0a:58:0a:81:00:1d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c7674cad-0eee-49e3-b1e5-98eadcc64896\"}],\"ips\":[{\"address\":\"10.129.0.29/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0505 08:32:34.630835 3788 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-tpa/tpa-pgsql W0505 08:32:35.764715 3788 util.go:959] Failed to build node endpoints for node ip-10-0-1-71.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0505 08:32:35.764735 3788 util.go:959] Failed to build node endpoints for node ip-10-0-1-71.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0505 08:32:35.764741 3788 util.go:959] Failed to build node endpoints for node ip-10-0-1-71.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0505 08:32:35.791371 3788 util.go:959] Failed to build node endpoints for node ip-10-0-1-71.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0505 08:32:35.791390 3788 util.go:959] Failed to build node endpoints for node ip-10-0-1-71.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0505 08:32:35.791395 3788 util.go:959] Failed to build node endpoints for node ip-10-0-1-71.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0505 08:32:35.840852 3788 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-keycloak/keycloak-pgsql I0505 08:32:43.099921 3788 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-triggers-webhook-6b5b45569f-7q27d] creating logical port openshift-pipelines_tekton-triggers-webhook-6b5b45569f-7q27d for pod on switch ip-10-0-1-71.ec2.internal I0505 08:32:43.100009 3788 kube.go:256] Updating pod openshift-pipelines/tekton-triggers-webhook-6b5b45569f-7q27d I0505 08:32:43.124557 3788 pod.go:62] [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-7q27d] pod update took 24.564247ms I0505 08:32:43.124587 3788 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-triggers-webhook-6b5b45569f-7q27d] addLogicalPort annotation time took 24.601758ms I0505 08:32:43.125777 3788 pods.go:271] [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-7q27d] addLogicalPort took 25.876411ms, libovsdb time 844.44µs I0505 08:32:43.524600 3788 cni.go:452] [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-7q27d 8d8ed94b9b4c369b5844b50f3d5fa7e0e7eccf6d0e208e8b93bc825d9fb4420f network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-7q27d 8d8ed94b9b4c369b5844b50f3d5fa7e0e7eccf6d0e208e8b93bc825d9fb4420f network default NAD default NAD key ] I0505 08:32:43.533880 3788 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-triggers-webhook-6b5b45569f-7q27d, hostIfaceName: 8d8ed94b9b4c369, network: default, NAD default, SandboxID: "8d8ed94b9b4c369b5844b50f3d5fa7e0e7eccf6d0e208e8b93bc825d9fb4420f", PCI device ID: , UID: "1b69970f-7a17-4e53-b205-a4877f2e70fc", MAC: 0a:58:0a:81:00:20, IPs: [10.129.0.32/23] I0505 08:32:43.587398 3788 cni.go:473] [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-7q27d 8d8ed94b9b4c369b5844b50f3d5fa7e0e7eccf6d0e208e8b93bc825d9fb4420f network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-7q27d 8d8ed94b9b4c369b5844b50f3d5fa7e0e7eccf6d0e208e8b93bc825d9fb4420f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"96:d1:ca:01:dd:16\",\"name\":\"8d8ed94b9b4c369\"},{\"mac\":\"0a:58:0a:81:00:20\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/90fb54ca-57f0-4ce1-99eb-f7bdce32d802\"}],\"ips\":[{\"address\":\"10.129.0.32/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0505 08:32:47.146254 3788 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors I0505 08:32:49.854931 3788 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=39 W0505 08:32:56.814705 3788 util.go:959] Failed to build node endpoints for node ip-10-0-1-71.ec2.internal port 5432: empty IP address endpoints for service tsf-keycloak/keycloak-pgsql I0505 08:32:58.928729 3788 cni.go:452] [tsf/test-tsf-infrastructure 0588f29fb3cfa24ca4a79e6955a242b48e13cb627a0aadea9964417cfdb303bc network default NAD default NAD key ] DEL starting CNI request [tsf/test-tsf-infrastructure 0588f29fb3cfa24ca4a79e6955a242b48e13cb627a0aadea9964417cfdb303bc network default NAD default NAD key ] I0505 08:32:59.018075 3788 cni.go:473] [tsf/test-tsf-infrastructure 0588f29fb3cfa24ca4a79e6955a242b48e13cb627a0aadea9964417cfdb303bc network default NAD default NAD key default] DEL finished CNI request [tsf/test-tsf-infrastructure 0588f29fb3cfa24ca4a79e6955a242b48e13cb627a0aadea9964417cfdb303bc network default NAD default NAD key default], result "{}", err I0505 08:32:59.070421 3788 obj_retry.go:498] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 08:32:59.070484 3788 pods.go:176] Deleting pod: tsf/test-tsf-infrastructure I0505 08:32:59.071825 3788 pods.go:236] Attempting to release IPs for pod: tsf/test-tsf-infrastructure, ips: 10.129.0.28 I0505 08:32:59.071852 3788 obj_retry.go:498] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 08:32:59.089253 3788 obj_retry.go:492] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:32:59.089276 3788 obj_retry.go:492] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:32:59.095238 3788 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-infrastructure I0505 08:32:59.095265 3788 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-infrastructure I0505 08:33:00.103889 3788 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-chains-controller-55fb698565-tlxvs] creating logical port openshift-pipelines_tekton-chains-controller-55fb698565-tlxvs for pod on switch ip-10-0-1-71.ec2.internal I0505 08:33:00.103992 3788 kube.go:256] Updating pod openshift-pipelines/tekton-chains-controller-55fb698565-tlxvs I0505 08:33:00.120813 3788 pod.go:62] [openshift-pipelines/tekton-chains-controller-55fb698565-tlxvs] pod update took 16.848382ms I0505 08:33:00.120838 3788 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-chains-controller-55fb698565-tlxvs] addLogicalPort annotation time took 16.878292ms I0505 08:33:00.122137 3788 pods.go:271] [openshift-pipelines/tekton-chains-controller-55fb698565-tlxvs] addLogicalPort took 18.264651ms, libovsdb time 919.197µs I0505 08:33:00.237572 3788 base_network_controller_pods.go:487] [default/tsf/test-tsf-iam] creating logical port tsf_test-tsf-iam for pod on switch ip-10-0-1-71.ec2.internal I0505 08:33:00.237673 3788 kube.go:256] Updating pod tsf/test-tsf-iam I0505 08:33:00.251506 3788 pod.go:62] [tsf/test-tsf-iam] pod update took 13.85638ms I0505 08:33:00.251534 3788 base_network_controller_pods.go:951] [default/tsf/test-tsf-iam] addLogicalPort annotation time took 13.887964ms I0505 08:33:00.252801 3788 pods.go:271] [tsf/test-tsf-iam] addLogicalPort took 15.242055ms, libovsdb time 922.365µs I0505 08:33:00.596364 3788 cni.go:452] [tsf/test-tsf-iam 85fb5d0c807d81358b334dc8c452edcb5c0608f04e06affa49da5fb1a3195529 network default NAD default NAD key ] ADD starting CNI request [tsf/test-tsf-iam 85fb5d0c807d81358b334dc8c452edcb5c0608f04e06affa49da5fb1a3195529 network default NAD default NAD key ] I0505 08:33:00.610774 3788 helper_linux.go:556] ConfigureOVS: namespace: tsf, podName: test-tsf-iam, hostIfaceName: 85fb5d0c807d813, network: default, NAD default, SandboxID: "85fb5d0c807d81358b334dc8c452edcb5c0608f04e06affa49da5fb1a3195529", PCI device ID: , UID: "df8ba5ec-fc83-44ff-b215-998751acbbf4", MAC: 0a:58:0a:81:00:22, IPs: [10.129.0.34/23] I0505 08:33:00.671660 3788 cni.go:473] [tsf/test-tsf-iam 85fb5d0c807d81358b334dc8c452edcb5c0608f04e06affa49da5fb1a3195529 network default NAD default NAD key default] ADD finished CNI request [tsf/test-tsf-iam 85fb5d0c807d81358b334dc8c452edcb5c0608f04e06affa49da5fb1a3195529 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"52:fc:45:e0:b4:1f\",\"name\":\"85fb5d0c807d813\"},{\"mac\":\"0a:58:0a:81:00:22\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ca45dcdd-a71a-41bf-81bb-ecd101de3d36\"}],\"ips\":[{\"address\":\"10.129.0.34/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0505 08:33:00.764849 3788 cni.go:452] [openshift-pipelines/tekton-chains-controller-55fb698565-tlxvs a1bd4aabe9586d04b81452d635d7a952f1a2b168e394257de271289974cd6a63 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-chains-controller-55fb698565-tlxvs a1bd4aabe9586d04b81452d635d7a952f1a2b168e394257de271289974cd6a63 network default NAD default NAD key ] I0505 08:33:00.774676 3788 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-chains-controller-55fb698565-tlxvs, hostIfaceName: a1bd4aabe9586d0, network: default, NAD default, SandboxID: "a1bd4aabe9586d04b81452d635d7a952f1a2b168e394257de271289974cd6a63", PCI device ID: , UID: "8fb0c2cb-36fb-4375-a02d-9afabcce4ae4", MAC: 0a:58:0a:81:00:21, IPs: [10.129.0.33/23] I0505 08:33:00.825698 3788 cni.go:473] [openshift-pipelines/tekton-chains-controller-55fb698565-tlxvs a1bd4aabe9586d04b81452d635d7a952f1a2b168e394257de271289974cd6a63 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-chains-controller-55fb698565-tlxvs a1bd4aabe9586d04b81452d635d7a952f1a2b168e394257de271289974cd6a63 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"aa:25:28:de:3e:9b\",\"name\":\"a1bd4aabe9586d0\"},{\"mac\":\"0a:58:0a:81:00:21\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9d87741d-1933-4857-b9f4-f44b440090b8\"}],\"ips\":[{\"address\":\"10.129.0.33/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0505 08:33:01.069008 3788 base_network_controller_policy.go:1179] Adding network policy tsf-keycloak/keycloak-network-policy for network default I0505 08:33:01.069736 3788 pod_selector_address_set.go:213] Created shared address set for pod selector tsf-keycloak_LS{ML:{app: keycloak,app.kubernetes.io/instance: keycloak,app.kubernetes.io/managed-by: keycloak-operator,},} I0505 08:33:01.069767 3788 base_network_controller_policy.go:1054] Policy tsf-keycloak/keycloak-network-policy added to peer address sets [tsf-keycloak_LS{ML:{app: keycloak,app.kubernetes.io/instance: keycloak,app.kubernetes.io/managed-by: keycloak-operator,},}] I0505 08:33:01.072074 3788 base_network_controller_policy.go:1232] Create network policy tsf-keycloak/keycloak-network-policy resources completed, update namespace loglevel I0505 08:33:04.653203 3788 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0505 08:33:07.733698 3788 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-postgres-0] creating logical port openshift-pipelines_tekton-results-postgres-0 for pod on switch ip-10-0-1-71.ec2.internal I0505 08:33:07.733799 3788 kube.go:256] Updating pod openshift-pipelines/tekton-results-postgres-0 I0505 08:33:07.746125 3788 pod.go:62] [openshift-pipelines/tekton-results-postgres-0] pod update took 12.349799ms I0505 08:33:07.746147 3788 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-results-postgres-0] addLogicalPort annotation time took 12.377487ms I0505 08:33:07.747512 3788 pods.go:271] [openshift-pipelines/tekton-results-postgres-0] addLogicalPort took 13.834483ms, libovsdb time 937.249µs W0505 08:33:08.790762 3788 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0505 08:33:08.790786 3788 util.go:870] Failed to build global endpoints for port TCP/management: empty IP address endpoints for service tsf-keycloak/keycloak-service W0505 08:33:08.793652 3788 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0505 08:33:08.793662 3788 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak I0505 08:33:10.495178 3788 cni.go:452] [openshift-pipelines/tekton-results-postgres-0 072b279d43cdde221d04d29a52b7a082ce158f00564afa027107c2a761cd0ca9 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-results-postgres-0 072b279d43cdde221d04d29a52b7a082ce158f00564afa027107c2a761cd0ca9 network default NAD default NAD key ] I0505 08:33:10.504339 3788 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-postgres-0, hostIfaceName: 072b279d43cdde2, network: default, NAD default, SandboxID: "072b279d43cdde221d04d29a52b7a082ce158f00564afa027107c2a761cd0ca9", PCI device ID: , UID: "c2f06ccc-d93e-495c-8dd2-2a715abfe3e0", MAC: 0a:58:0a:81:00:23, IPs: [10.129.0.35/23] I0505 08:33:10.554984 3788 cni.go:473] [openshift-pipelines/tekton-results-postgres-0 072b279d43cdde221d04d29a52b7a082ce158f00564afa027107c2a761cd0ca9 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-results-postgres-0 072b279d43cdde221d04d29a52b7a082ce158f00564afa027107c2a761cd0ca9 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"12:d8:39:bb:b9:bb\",\"name\":\"072b279d43cdde2\"},{\"mac\":\"0a:58:0a:81:00:23\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e20cbe1f-ca4b-4a3a-919c-02105bb0abcf\"}],\"ips\":[{\"address\":\"10.129.0.35/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0505 08:33:17.072421 3788 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0505 08:33:18.505067 3788 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-retention-policy-agent-585694b54-m8tp7] creating logical port openshift-pipelines_tekton-results-retention-policy-agent-585694b54-m8tp7 for pod on switch ip-10-0-1-71.ec2.internal I0505 08:33:18.505205 3788 kube.go:256] Updating pod openshift-pipelines/tekton-results-retention-policy-agent-585694b54-m8tp7 I0505 08:33:18.522947 3788 pod.go:62] [openshift-pipelines/tekton-results-retention-policy-agent-585694b54-m8tp7] pod update took 17.807179ms I0505 08:33:18.522972 3788 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-results-retention-policy-agent-585694b54-m8tp7] addLogicalPort annotation time took 17.837549ms I0505 08:33:18.524494 3788 pods.go:271] [openshift-pipelines/tekton-results-retention-policy-agent-585694b54-m8tp7] addLogicalPort took 19.439659ms, libovsdb time 1.052996ms I0505 08:33:18.608768 3788 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-watcher-7777c5dbd5-ppw25] creating logical port openshift-pipelines_tekton-results-watcher-7777c5dbd5-ppw25 for pod on switch ip-10-0-1-71.ec2.internal I0505 08:33:18.608857 3788 kube.go:256] Updating pod openshift-pipelines/tekton-results-watcher-7777c5dbd5-ppw25 I0505 08:33:18.625782 3788 pod.go:62] [openshift-pipelines/tekton-results-watcher-7777c5dbd5-ppw25] pod update took 16.94538ms I0505 08:33:18.625810 3788 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-results-watcher-7777c5dbd5-ppw25] addLogicalPort annotation time took 16.978693ms I0505 08:33:18.627041 3788 pods.go:271] [openshift-pipelines/tekton-results-watcher-7777c5dbd5-ppw25] addLogicalPort took 18.294206ms, libovsdb time 849.301µs I0505 08:33:18.969113 3788 cni.go:452] [openshift-pipelines/tekton-results-watcher-7777c5dbd5-ppw25 69948e74001cf7c28c56b7c0d3915eea3091a216e0a884a9d5dcb27f36dd3992 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-results-watcher-7777c5dbd5-ppw25 69948e74001cf7c28c56b7c0d3915eea3091a216e0a884a9d5dcb27f36dd3992 network default NAD default NAD key ] I0505 08:33:18.978255 3788 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-watcher-7777c5dbd5-ppw25, hostIfaceName: 69948e74001cf7c, network: default, NAD default, SandboxID: "69948e74001cf7c28c56b7c0d3915eea3091a216e0a884a9d5dcb27f36dd3992", PCI device ID: , UID: "4bb23a7f-5442-4167-b8f5-896b313fbd65", MAC: 0a:58:0a:81:00:25, IPs: [10.129.0.37/23] I0505 08:33:19.031984 3788 cni.go:473] [openshift-pipelines/tekton-results-watcher-7777c5dbd5-ppw25 69948e74001cf7c28c56b7c0d3915eea3091a216e0a884a9d5dcb27f36dd3992 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-results-watcher-7777c5dbd5-ppw25 69948e74001cf7c28c56b7c0d3915eea3091a216e0a884a9d5dcb27f36dd3992 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"12:f5:8b:79:50:40\",\"name\":\"69948e74001cf7c\"},{\"mac\":\"0a:58:0a:81:00:25\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b3c04fef-c1d4-4198-86a0-e9d66536e7af\"}],\"ips\":[{\"address\":\"10.129.0.37/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0505 08:33:19.472399 3788 cni.go:452] [openshift-pipelines/tekton-results-retention-policy-agent-585694b54-m8tp7 30f2c9d7e28d98f6865fd6e33d709442509a8642162bd8ac8895d95341e6910c network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-results-retention-policy-agent-585694b54-m8tp7 30f2c9d7e28d98f6865fd6e33d709442509a8642162bd8ac8895d95341e6910c network default NAD default NAD key ] I0505 08:33:19.481968 3788 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-retention-policy-agent-585694b54-m8tp7, hostIfaceName: 30f2c9d7e28d98f, network: default, NAD default, SandboxID: "30f2c9d7e28d98f6865fd6e33d709442509a8642162bd8ac8895d95341e6910c", PCI device ID: , UID: "43c66860-b4d5-493d-8a5b-762959d9bded", MAC: 0a:58:0a:81:00:24, IPs: [10.129.0.36/23] I0505 08:33:19.532108 3788 cni.go:473] [openshift-pipelines/tekton-results-retention-policy-agent-585694b54-m8tp7 30f2c9d7e28d98f6865fd6e33d709442509a8642162bd8ac8895d95341e6910c network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-results-retention-policy-agent-585694b54-m8tp7 30f2c9d7e28d98f6865fd6e33d709442509a8642162bd8ac8895d95341e6910c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"e6:57:66:80:b4:c2\",\"name\":\"30f2c9d7e28d98f\"},{\"mac\":\"0a:58:0a:81:00:24\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2bdc57b3-62e8-41f4-b473-9e2501625c54\"}],\"ips\":[{\"address\":\"10.129.0.36/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0505 08:33:25.356386 3788 util.go:870] Failed to build global endpoints for port TCP/prometheus: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0505 08:33:25.356412 3788 util.go:870] Failed to build global endpoints for port TCP/profiling: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0505 08:33:25.356419 3788 util.go:870] Failed to build global endpoints for port TCP/server: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service I0505 08:33:51.208660 3788 cni.go:452] [tsf/test-tsf-iam 85fb5d0c807d81358b334dc8c452edcb5c0608f04e06affa49da5fb1a3195529 network default NAD default NAD key ] DEL starting CNI request [tsf/test-tsf-iam 85fb5d0c807d81358b334dc8c452edcb5c0608f04e06affa49da5fb1a3195529 network default NAD default NAD key ] I0505 08:33:51.290507 3788 cni.go:473] [tsf/test-tsf-iam 85fb5d0c807d81358b334dc8c452edcb5c0608f04e06affa49da5fb1a3195529 network default NAD default NAD key default] DEL finished CNI request [tsf/test-tsf-iam 85fb5d0c807d81358b334dc8c452edcb5c0608f04e06affa49da5fb1a3195529 network default NAD default NAD key default], result "{}", err I0505 08:33:51.356426 3788 obj_retry.go:498] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 08:33:51.356486 3788 pods.go:176] Deleting pod: tsf/test-tsf-iam I0505 08:33:51.357979 3788 pods.go:236] Attempting to release IPs for pod: tsf/test-tsf-iam, ips: 10.129.0.34 I0505 08:33:51.358009 3788 obj_retry.go:498] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 08:33:51.396903 3788 obj_retry.go:492] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:33:51.396923 3788 obj_retry.go:492] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:33:51.406236 3788 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0505 08:33:51.406261 3788 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam I0505 08:33:52.175798 3788 base_network_controller_pods.go:487] [default/tsf-tas/test-tsf-tas] creating logical port tsf-tas_test-tsf-tas for pod on switch ip-10-0-1-71.ec2.internal I0505 08:33:52.175948 3788 kube.go:256] Updating pod tsf-tas/test-tsf-tas I0505 08:33:52.196760 3788 pod.go:62] [tsf-tas/test-tsf-tas] pod update took 20.839227ms I0505 08:33:52.196783 3788 base_network_controller_pods.go:951] [default/tsf-tas/test-tsf-tas] addLogicalPort annotation time took 20.869043ms I0505 08:33:52.198077 3788 pods.go:271] [tsf-tas/test-tsf-tas] addLogicalPort took 22.299673ms, libovsdb time 925.936µs I0505 08:33:52.541333 3788 cni.go:452] [tsf-tas/test-tsf-tas 99ab4941d4c1b941ca23ce1d95fb25b0267700d99bb033b123bd676b0d72966f network default NAD default NAD key ] ADD starting CNI request [tsf-tas/test-tsf-tas 99ab4941d4c1b941ca23ce1d95fb25b0267700d99bb033b123bd676b0d72966f network default NAD default NAD key ] I0505 08:33:52.550823 3788 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: test-tsf-tas, hostIfaceName: 99ab4941d4c1b94, network: default, NAD default, SandboxID: "99ab4941d4c1b941ca23ce1d95fb25b0267700d99bb033b123bd676b0d72966f", PCI device ID: , UID: "279c2379-5155-46bb-b4c4-3deb89d99d10", MAC: 0a:58:0a:81:00:26, IPs: [10.129.0.38/23] I0505 08:33:52.605553 3788 cni.go:473] [tsf-tas/test-tsf-tas 99ab4941d4c1b941ca23ce1d95fb25b0267700d99bb033b123bd676b0d72966f network default NAD default NAD key default] ADD finished CNI request [tsf-tas/test-tsf-tas 99ab4941d4c1b941ca23ce1d95fb25b0267700d99bb033b123bd676b0d72966f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"8e:80:ca:86:98:a3\",\"name\":\"99ab4941d4c1b94\"},{\"mac\":\"0a:58:0a:81:00:26\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b7ff1207-36ed-4cd6-aec5-449174e6c0a9\"}],\"ips\":[{\"address\":\"10.129.0.38/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0505 08:33:53.100729 3788 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-825xg] creating logical port openshift-pipelines_pipelines-as-code-watcher-7947cf58bc-825xg for pod on switch ip-10-0-1-71.ec2.internal I0505 08:33:53.100827 3788 kube.go:256] Updating pod openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-825xg I0505 08:33:53.116244 3788 pod.go:62] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-825xg] pod update took 15.437021ms I0505 08:33:53.116267 3788 base_network_controller_pods.go:951] [default/openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-825xg] addLogicalPort annotation time took 15.464972ms I0505 08:33:53.117568 3788 pods.go:271] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-825xg] addLogicalPort took 16.858031ms, libovsdb time 896.726µs I0505 08:33:53.199884 3788 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-as-code-webhook-68965d5857-x6x6j] creating logical port openshift-pipelines_pipelines-as-code-webhook-68965d5857-x6x6j for pod on switch ip-10-0-1-71.ec2.internal I0505 08:33:53.199977 3788 kube.go:256] Updating pod openshift-pipelines/pipelines-as-code-webhook-68965d5857-x6x6j I0505 08:33:53.217952 3788 pod.go:62] [openshift-pipelines/pipelines-as-code-webhook-68965d5857-x6x6j] pod update took 17.997848ms I0505 08:33:53.217975 3788 base_network_controller_pods.go:951] [default/openshift-pipelines/pipelines-as-code-webhook-68965d5857-x6x6j] addLogicalPort annotation time took 18.024646ms I0505 08:33:53.219118 3788 pods.go:271] [openshift-pipelines/pipelines-as-code-webhook-68965d5857-x6x6j] addLogicalPort took 19.255336ms, libovsdb time 779.943µs I0505 08:33:53.565516 3788 cni.go:452] [openshift-pipelines/pipelines-as-code-webhook-68965d5857-x6x6j 6f71e2ba78284329afdc2ce035ab1a9866daf93d9ef2c39d7959affaff725eb9 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/pipelines-as-code-webhook-68965d5857-x6x6j 6f71e2ba78284329afdc2ce035ab1a9866daf93d9ef2c39d7959affaff725eb9 network default NAD default NAD key ] I0505 08:33:53.575761 3788 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-as-code-webhook-68965d5857-x6x6j, hostIfaceName: 6f71e2ba7828432, network: default, NAD default, SandboxID: "6f71e2ba78284329afdc2ce035ab1a9866daf93d9ef2c39d7959affaff725eb9", PCI device ID: , UID: "3e8b747d-cca2-4b03-9cc2-f7c060b18ea9", MAC: 0a:58:0a:81:00:28, IPs: [10.129.0.40/23] I0505 08:33:53.628963 3788 cni.go:473] [openshift-pipelines/pipelines-as-code-webhook-68965d5857-x6x6j 6f71e2ba78284329afdc2ce035ab1a9866daf93d9ef2c39d7959affaff725eb9 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/pipelines-as-code-webhook-68965d5857-x6x6j 6f71e2ba78284329afdc2ce035ab1a9866daf93d9ef2c39d7959affaff725eb9 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"fa:99:f6:c6:59:ea\",\"name\":\"6f71e2ba7828432\"},{\"mac\":\"0a:58:0a:81:00:28\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/50643955-8b43-4e6a-ba49-8da018233128\"}],\"ips\":[{\"address\":\"10.129.0.40/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0505 08:33:53.763133 3788 cni.go:452] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-825xg bc03c784a09495cff63f3c48df30450ff9ad4ab0547031eb64f77aa0d9e6e80a network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-825xg bc03c784a09495cff63f3c48df30450ff9ad4ab0547031eb64f77aa0d9e6e80a network default NAD default NAD key ] I0505 08:33:53.773536 3788 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-as-code-watcher-7947cf58bc-825xg, hostIfaceName: bc03c784a09495c, network: default, NAD default, SandboxID: "bc03c784a09495cff63f3c48df30450ff9ad4ab0547031eb64f77aa0d9e6e80a", PCI device ID: , UID: "fbf13e50-44c2-479e-a059-669c4ebdde0d", MAC: 0a:58:0a:81:00:27, IPs: [10.129.0.39/23] I0505 08:33:53.825343 3788 cni.go:473] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-825xg bc03c784a09495cff63f3c48df30450ff9ad4ab0547031eb64f77aa0d9e6e80a network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-825xg bc03c784a09495cff63f3c48df30450ff9ad4ab0547031eb64f77aa0d9e6e80a network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"86:65:2c:3c:e7:ee\",\"name\":\"bc03c784a09495c\"},{\"mac\":\"0a:58:0a:81:00:27\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c010a329-2d39-4763-acaa-8fd728aa8dd9\"}],\"ips\":[{\"address\":\"10.129.0.39/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0505 08:33:54.207082 3788 base_network_controller_pods.go:487] [default/tsf-tas/fulcio-server-77c4cd777d-tvkkq] creating logical port tsf-tas_fulcio-server-77c4cd777d-tvkkq for pod on switch ip-10-0-1-71.ec2.internal I0505 08:33:54.207196 3788 kube.go:256] Updating pod tsf-tas/fulcio-server-77c4cd777d-tvkkq I0505 08:33:54.238339 3788 pod.go:62] [tsf-tas/fulcio-server-77c4cd777d-tvkkq] pod update took 31.163379ms I0505 08:33:54.238362 3788 base_network_controller_pods.go:951] [default/tsf-tas/fulcio-server-77c4cd777d-tvkkq] addLogicalPort annotation time took 31.191824ms I0505 08:33:54.239582 3788 pods.go:271] [tsf-tas/fulcio-server-77c4cd777d-tvkkq] addLogicalPort took 32.513329ms, libovsdb time 785.472µs I0505 08:33:54.581493 3788 cni.go:452] [tsf-tas/fulcio-server-77c4cd777d-tvkkq 901163499bc64dd2fa49b178022e8a8ed281604d99f64ec4708c43a62c9d0b60 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/fulcio-server-77c4cd777d-tvkkq 901163499bc64dd2fa49b178022e8a8ed281604d99f64ec4708c43a62c9d0b60 network default NAD default NAD key ] I0505 08:33:54.594303 3788 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: fulcio-server-77c4cd777d-tvkkq, hostIfaceName: 901163499bc64dd, network: default, NAD default, SandboxID: "901163499bc64dd2fa49b178022e8a8ed281604d99f64ec4708c43a62c9d0b60", PCI device ID: , UID: "0dcc672f-618d-4dc1-a684-2e536362d590", MAC: 0a:58:0a:81:00:29, IPs: [10.129.0.41/23] I0505 08:33:54.654447 3788 cni.go:473] [tsf-tas/fulcio-server-77c4cd777d-tvkkq 901163499bc64dd2fa49b178022e8a8ed281604d99f64ec4708c43a62c9d0b60 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/fulcio-server-77c4cd777d-tvkkq 901163499bc64dd2fa49b178022e8a8ed281604d99f64ec4708c43a62c9d0b60 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"52:cd:9e:c6:2e:a6\",\"name\":\"901163499bc64dd\"},{\"mac\":\"0a:58:0a:81:00:29\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9189a23f-cf11-4d7d-9136-fb7b3bd50b7f\"}],\"ips\":[{\"address\":\"10.129.0.41/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0505 08:33:54.736941 3788 base_network_controller_pods.go:487] [default/tsf-tas/trillian-logserver-7c9cc44b6d-5lt2c] creating logical port tsf-tas_trillian-logserver-7c9cc44b6d-5lt2c for pod on switch ip-10-0-1-71.ec2.internal I0505 08:33:54.737037 3788 kube.go:256] Updating pod tsf-tas/trillian-logserver-7c9cc44b6d-5lt2c I0505 08:33:54.756263 3788 pod.go:62] [tsf-tas/trillian-logserver-7c9cc44b6d-5lt2c] pod update took 19.246926ms I0505 08:33:54.756295 3788 base_network_controller_pods.go:951] [default/tsf-tas/trillian-logserver-7c9cc44b6d-5lt2c] addLogicalPort annotation time took 19.285299ms I0505 08:33:54.757757 3788 pods.go:271] [tsf-tas/trillian-logserver-7c9cc44b6d-5lt2c] addLogicalPort took 20.83398ms, libovsdb time 967.753µs I0505 08:33:55.113226 3788 cni.go:452] [tsf-tas/trillian-logserver-7c9cc44b6d-5lt2c 8f3b6b4b8b73c8f49256ee73d26ae3cf53bbbd1540a8fbc2db5fcefe433c5c18 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/trillian-logserver-7c9cc44b6d-5lt2c 8f3b6b4b8b73c8f49256ee73d26ae3cf53bbbd1540a8fbc2db5fcefe433c5c18 network default NAD default NAD key ] I0505 08:33:55.125693 3788 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: trillian-logserver-7c9cc44b6d-5lt2c, hostIfaceName: 8f3b6b4b8b73c8f, network: default, NAD default, SandboxID: "8f3b6b4b8b73c8f49256ee73d26ae3cf53bbbd1540a8fbc2db5fcefe433c5c18", PCI device ID: , UID: "1e2100b4-1f42-4890-b3ab-a6d0545d2039", MAC: 0a:58:0a:81:00:2a, IPs: [10.129.0.42/23] I0505 08:33:55.183835 3788 cni.go:473] [tsf-tas/trillian-logserver-7c9cc44b6d-5lt2c 8f3b6b4b8b73c8f49256ee73d26ae3cf53bbbd1540a8fbc2db5fcefe433c5c18 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/trillian-logserver-7c9cc44b6d-5lt2c 8f3b6b4b8b73c8f49256ee73d26ae3cf53bbbd1540a8fbc2db5fcefe433c5c18 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"be:2d:5a:09:57:e0\",\"name\":\"8f3b6b4b8b73c8f\"},{\"mac\":\"0a:58:0a:81:00:2a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/542a4107-85a7-4a42-823f-2ff6766b5775\"}],\"ips\":[{\"address\":\"10.129.0.42/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0505 08:33:56.291574 3788 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher W0505 08:33:56.570866 3788 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0505 08:33:56.570890 3788 util.go:870] Failed to build global endpoints for port TCP/http-listener: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller I0505 08:33:57.523068 3788 base_network_controller_pods.go:487] [default/tsf-tas/trillian-db-6dc8d65d94-69vrt] creating logical port tsf-tas_trillian-db-6dc8d65d94-69vrt for pod on switch ip-10-0-1-71.ec2.internal I0505 08:33:57.523167 3788 kube.go:256] Updating pod tsf-tas/trillian-db-6dc8d65d94-69vrt I0505 08:33:57.539945 3788 pod.go:62] [tsf-tas/trillian-db-6dc8d65d94-69vrt] pod update took 16.800454ms I0505 08:33:57.539975 3788 base_network_controller_pods.go:951] [default/tsf-tas/trillian-db-6dc8d65d94-69vrt] addLogicalPort annotation time took 16.840045ms I0505 08:33:57.541804 3788 pods.go:271] [tsf-tas/trillian-db-6dc8d65d94-69vrt] addLogicalPort took 18.753482ms, libovsdb time 1.207202ms I0505 08:34:00.203762 3788 obj_retry.go:498] Detected object tsf-keycloak/tsf-iam-l8ghp of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 08:34:00.204415 3788 obj_retry.go:498] Detected object tsf-keycloak/tsf-iam-l8ghp of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 08:34:01.087135 3788 obj_retry.go:492] Detected object tsf-keycloak/tsf-iam-l8ghp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:34:01.087164 3788 obj_retry.go:492] Detected object tsf-keycloak/tsf-iam-l8ghp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:34:01.799484 3788 cni.go:452] [tsf-tas/trillian-db-6dc8d65d94-69vrt 39f7964dbccb23846b3e454eaed657fd4ad57d4cb0e4704ac54c93ec2fed86bb network default NAD default NAD key ] ADD starting CNI request [tsf-tas/trillian-db-6dc8d65d94-69vrt 39f7964dbccb23846b3e454eaed657fd4ad57d4cb0e4704ac54c93ec2fed86bb network default NAD default NAD key ] I0505 08:34:01.808504 3788 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: trillian-db-6dc8d65d94-69vrt, hostIfaceName: 39f7964dbccb238, network: default, NAD default, SandboxID: "39f7964dbccb23846b3e454eaed657fd4ad57d4cb0e4704ac54c93ec2fed86bb", PCI device ID: , UID: "5301ea17-fa74-4695-bd90-3c5cf818d704", MAC: 0a:58:0a:81:00:2b, IPs: [10.129.0.43/23] I0505 08:34:01.863310 3788 cni.go:473] [tsf-tas/trillian-db-6dc8d65d94-69vrt 39f7964dbccb23846b3e454eaed657fd4ad57d4cb0e4704ac54c93ec2fed86bb network default NAD default NAD key default] ADD finished CNI request [tsf-tas/trillian-db-6dc8d65d94-69vrt 39f7964dbccb23846b3e454eaed657fd4ad57d4cb0e4704ac54c93ec2fed86bb network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"de:28:ff:70:50:d6\",\"name\":\"39f7964dbccb238\"},{\"mac\":\"0a:58:0a:81:00:2b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/706ff8aa-4dab-442f-9879-cf0ca4c8c6da\"}],\"ips\":[{\"address\":\"10.129.0.43/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0505 08:34:02.277815 3788 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logserver W0505 08:34:03.285102 3788 util.go:870] Failed to build global endpoints for port TCP/trillian-mysql: empty IP address endpoints for service tsf-tas/trillian-mysql W0505 08:34:03.581562 3788 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logsigner I0505 08:34:06.978306 3788 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=8 W0505 08:34:07.252249 3788 util.go:959] Failed to build node endpoints for node ip-10-0-1-71.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher I0505 08:34:22.951157 3788 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=9 I0505 08:34:59.964118 3788 namespace.go:142] [openshift-pipelines] updating namespace I0505 08:35:21.969566 3788 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=10 I0505 08:35:29.963731 3788 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=10 I0505 08:35:55.941251 3788 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=11 I0505 08:36:13.853755 3788 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=59 I0505 08:36:31.860708 3788 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=412 I0505 08:36:36.941806 3788 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=7 I0505 08:37:30.948806 3788 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=6 I0505 08:37:43.979394 3788 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=11 I0505 08:38:19.855622 3788 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=101 I0505 08:38:25.976859 3788 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=8 I0505 08:39:24.852737 3788 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=98 I0505 08:39:27.853688 3788 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=12 I0505 08:40:03.740934 3788 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29632840-mr97v of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 08:40:03.741554 3788 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29632840-mr97v of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 08:40:04.615716 3788 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29632840-mr97v of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:40:04.615740 3788 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29632840-mr97v of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:40:16.857648 3788 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=30 I0505 08:40:24.960130 3788 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=10 I0505 08:40:43.981502 3788 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=8 I0505 08:42:16.944678 3788 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=8 I0505 08:42:27.967298 3788 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=8 I0505 08:42:36.863725 3788 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=41 I0505 08:43:00.955034 3788 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=9 I0505 08:43:27.858313 3788 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=6 I0505 08:43:33.982164 3788 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=7 I0505 08:43:43.972146 3788 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=9 I0505 08:43:53.891126 3788 obj_retry.go:498] Detected object tsf-tas/segment-backup-installation-2xl7c-9v7j8 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 08:43:53.891744 3788 obj_retry.go:498] Detected object tsf-tas/segment-backup-installation-2xl7c-9v7j8 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 08:43:53.902735 3788 obj_retry.go:492] Detected object tsf-tas/segment-backup-installation-2xl7c-9v7j8 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:43:53.902758 3788 obj_retry.go:492] Detected object tsf-tas/segment-backup-installation-2xl7c-9v7j8 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:43:53.910812 3788 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tas/segment-backup-installation-2xl7c-9v7j8 I0505 08:43:53.910841 3788 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tas/segment-backup-installation-2xl7c-9v7j8 I0505 08:43:54.922305 3788 obj_retry.go:498] Detected object tsf-tas/ctlog-createtree-job-52zpb-xfwp8 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 08:43:54.922886 3788 obj_retry.go:498] Detected object tsf-tas/ctlog-createtree-job-52zpb-xfwp8 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 08:43:54.934690 3788 obj_retry.go:492] Detected object tsf-tas/ctlog-createtree-job-52zpb-xfwp8 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:43:54.934710 3788 obj_retry.go:492] Detected object tsf-tas/ctlog-createtree-job-52zpb-xfwp8 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:43:54.943631 3788 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tas/ctlog-createtree-job-52zpb-xfwp8 I0505 08:43:54.943655 3788 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tas/ctlog-createtree-job-52zpb-xfwp8 I0505 08:43:54.981410 3788 obj_retry.go:498] Detected object tsf-tas/rekor-createtree-job-9d6kt-phmmf of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 08:43:54.982015 3788 obj_retry.go:498] Detected object tsf-tas/rekor-createtree-job-9d6kt-phmmf of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 08:43:54.999174 3788 obj_retry.go:492] Detected object tsf-tas/rekor-createtree-job-9d6kt-phmmf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:43:54.999198 3788 obj_retry.go:492] Detected object tsf-tas/rekor-createtree-job-9d6kt-phmmf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:43:55.002288 3788 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tas/rekor-createtree-job-9d6kt-phmmf I0505 08:43:55.002311 3788 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tas/rekor-createtree-job-9d6kt-phmmf I0505 08:43:59.856153 3788 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=39 I0505 08:44:42.855895 3788 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=6 I0505 08:44:53.857166 3788 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=6 I0505 08:44:58.979559 3788 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=8 I0505 08:45:04.951955 3788 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=8 I0505 08:46:22.944637 3788 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=10 I0505 08:47:35.947130 3788 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=5 I0505 08:48:18.534489 3788 cni.go:452] [tsf-tas/test-tsf-tas 99ab4941d4c1b941ca23ce1d95fb25b0267700d99bb033b123bd676b0d72966f network default NAD default NAD key ] DEL starting CNI request [tsf-tas/test-tsf-tas 99ab4941d4c1b941ca23ce1d95fb25b0267700d99bb033b123bd676b0d72966f network default NAD default NAD key ] I0505 08:48:18.624580 3788 cni.go:473] [tsf-tas/test-tsf-tas 99ab4941d4c1b941ca23ce1d95fb25b0267700d99bb033b123bd676b0d72966f network default NAD default NAD key default] DEL finished CNI request [tsf-tas/test-tsf-tas 99ab4941d4c1b941ca23ce1d95fb25b0267700d99bb033b123bd676b0d72966f network default NAD default NAD key default], result "{}", err I0505 08:48:18.679575 3788 obj_retry.go:498] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 08:48:18.679642 3788 pods.go:176] Deleting pod: tsf-tas/test-tsf-tas I0505 08:48:18.681009 3788 pods.go:236] Attempting to release IPs for pod: tsf-tas/test-tsf-tas, ips: 10.129.0.38 I0505 08:48:18.681035 3788 obj_retry.go:498] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 08:48:35.860532 3788 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=9 I0505 08:49:18.786759 3788 obj_retry.go:492] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:49:18.786787 3788 obj_retry.go:492] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:49:18.791214 3788 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tas/test-tsf-tas I0505 08:49:18.791247 3788 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tas/test-tsf-tas I0505 08:49:18.922197 3788 base_network_controller_pods.go:487] [default/tsf-tas/test-tsf-tas] creating logical port tsf-tas_test-tsf-tas for pod on switch ip-10-0-1-71.ec2.internal I0505 08:49:18.922319 3788 kube.go:256] Updating pod tsf-tas/test-tsf-tas I0505 08:49:18.965061 3788 kube.go:256] Updating pod tsf-tas/test-tsf-tas I0505 08:49:18.980319 3788 pod.go:62] [tsf-tas/test-tsf-tas] pod update took 58.026142ms I0505 08:49:18.980346 3788 base_network_controller_pods.go:951] [default/tsf-tas/test-tsf-tas] addLogicalPort annotation time took 58.056033ms I0505 08:49:18.981911 3788 pods.go:271] [tsf-tas/test-tsf-tas] addLogicalPort took 59.733146ms, libovsdb time 1.06957ms I0505 08:49:19.281018 3788 cni.go:452] [tsf-tas/test-tsf-tas 426286ab62a0287449a3c761d6649aae0bb4993f624aac46bb0433dfae37e945 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/test-tsf-tas 426286ab62a0287449a3c761d6649aae0bb4993f624aac46bb0433dfae37e945 network default NAD default NAD key ] I0505 08:49:19.290118 3788 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: test-tsf-tas, hostIfaceName: 426286ab62a0287, network: default, NAD default, SandboxID: "426286ab62a0287449a3c761d6649aae0bb4993f624aac46bb0433dfae37e945", PCI device ID: , UID: "5b03d680-99f2-452f-9a40-9f26e212a900", MAC: 0a:58:0a:81:00:2c, IPs: [10.129.0.44/23] I0505 08:49:19.344422 3788 cni.go:473] [tsf-tas/test-tsf-tas 426286ab62a0287449a3c761d6649aae0bb4993f624aac46bb0433dfae37e945 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/test-tsf-tas 426286ab62a0287449a3c761d6649aae0bb4993f624aac46bb0433dfae37e945 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"f6:5d:50:ab:0e:b2\",\"name\":\"426286ab62a0287\"},{\"mac\":\"0a:58:0a:81:00:2c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2cff5714-bf0c-4408-b1c5-fa44ec1eb083\"}],\"ips\":[{\"address\":\"10.129.0.44/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0505 08:49:19.859315 3788 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=27 I0505 08:49:59.984551 3788 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=10 I0505 08:50:04.083854 3788 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29632850-mhq2d of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 08:50:04.084452 3788 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29632850-mhq2d of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 08:50:04.965637 3788 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29632850-mhq2d of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:50:04.965682 3788 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29632850-mhq2d of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:50:05.001751 3788 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29632820-5v8c9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:50:05.001775 3788 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29632820-5v8c9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:50:05.007421 3788 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-backplane/osd-delete-backplane-serviceaccounts-29632820-5v8c9 I0505 08:50:05.007448 3788 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-backplane/osd-delete-backplane-serviceaccounts-29632820-5v8c9 I0505 08:50:15.963521 3788 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=11 I0505 08:50:31.955487 3788 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=6 I0505 08:50:46.975276 3788 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=9 I0505 08:51:05.867115 3788 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=8 I0505 08:51:27.970029 3788 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=9 I0505 08:51:43.958354 3788 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=10 I0505 08:51:48.985626 3788 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=10 I0505 08:51:59.858401 3788 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=8 I0505 08:52:24.866343 3788 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=61 I0505 08:53:03.950212 3788 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=7 W0505 08:53:21.008825 3788 util.go:959] Failed to build node endpoints for node ip-10-0-1-71.ec2.internal port 3306: empty IP address endpoints for service tsf-tas/trillian-mysql W0505 08:53:40.078966 3788 util.go:959] Failed to build node endpoints for node ip-10-0-1-71.ec2.internal port 8091: empty IP address endpoints for service tsf-tas/trillian-logserver I0505 08:53:56.860433 3788 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=11 W0505 08:54:04.260436 3788 duration.go:461] Config duration recorder: measurement expired for pod/tsf-keycloak/tsf-iam-l8ghp I0505 08:54:11.267402 3788 cni.go:452] [tsf-tas/test-tsf-tas 426286ab62a0287449a3c761d6649aae0bb4993f624aac46bb0433dfae37e945 network default NAD default NAD key ] DEL starting CNI request [tsf-tas/test-tsf-tas 426286ab62a0287449a3c761d6649aae0bb4993f624aac46bb0433dfae37e945 network default NAD default NAD key ] I0505 08:54:11.354485 3788 cni.go:473] [tsf-tas/test-tsf-tas 426286ab62a0287449a3c761d6649aae0bb4993f624aac46bb0433dfae37e945 network default NAD default NAD key default] DEL finished CNI request [tsf-tas/test-tsf-tas 426286ab62a0287449a3c761d6649aae0bb4993f624aac46bb0433dfae37e945 network default NAD default NAD key default], result "{}", err I0505 08:54:11.418035 3788 obj_retry.go:498] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 08:54:11.418068 3788 pods.go:176] Deleting pod: tsf-tas/test-tsf-tas I0505 08:54:11.419316 3788 pods.go:236] Attempting to release IPs for pod: tsf-tas/test-tsf-tas, ips: 10.129.0.44 I0505 08:54:11.419340 3788 obj_retry.go:498] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 08:54:11.456684 3788 obj_retry.go:492] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:54:11.456736 3788 obj_retry.go:492] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:54:11.465334 3788 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tas/test-tsf-tas I0505 08:54:11.465362 3788 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tas/test-tsf-tas I0505 08:54:14.983055 3788 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=11 I0505 08:54:19.528532 3788 obj_retry.go:498] Detected object tsf/patch-tekton-config-lc6nw of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 08:54:19.529067 3788 obj_retry.go:498] Detected object tsf/patch-tekton-config-lc6nw of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 08:54:20.547283 3788 obj_retry.go:492] Detected object tsf/patch-tekton-config-lc6nw of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:54:20.547328 3788 obj_retry.go:492] Detected object tsf/patch-tekton-config-lc6nw of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:54:26.554338 3788 obj_retry.go:498] Detected object tsf/tsf-tekton-configuration-j5wfh of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 08:54:26.554904 3788 obj_retry.go:498] Detected object tsf/tsf-tekton-configuration-j5wfh of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 08:54:27.443144 3788 obj_retry.go:492] Detected object tsf/tsf-tekton-configuration-j5wfh of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:54:27.443178 3788 obj_retry.go:492] Detected object tsf/tsf-tekton-configuration-j5wfh of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:54:27.475176 3788 obj_retry.go:492] Detected object tsf/patch-tekton-config-lc6nw of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:54:27.475199 3788 obj_retry.go:492] Detected object tsf/patch-tekton-config-lc6nw of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:54:27.479102 3788 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/patch-tekton-config-lc6nw I0505 08:54:27.479122 3788 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/patch-tekton-config-lc6nw I0505 08:54:28.098511 3788 base_network_controller_pods.go:487] [default/tsf-tpa/pre-install-tsf-tpa] creating logical port tsf-tpa_pre-install-tsf-tpa for pod on switch ip-10-0-1-71.ec2.internal I0505 08:54:28.098598 3788 kube.go:256] Updating pod tsf-tpa/pre-install-tsf-tpa I0505 08:54:28.112069 3788 pod.go:62] [tsf-tpa/pre-install-tsf-tpa] pod update took 13.491532ms I0505 08:54:28.112093 3788 base_network_controller_pods.go:951] [default/tsf-tpa/pre-install-tsf-tpa] addLogicalPort annotation time took 13.523174ms I0505 08:54:28.113558 3788 pods.go:271] [tsf-tpa/pre-install-tsf-tpa] addLogicalPort took 15.062395ms, libovsdb time 922.685µs I0505 08:54:28.458911 3788 cni.go:452] [tsf-tpa/pre-install-tsf-tpa 6160f71c1453fcbc1a024d38df83091478d616479d20748f879e90dcce4f04ac network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/pre-install-tsf-tpa 6160f71c1453fcbc1a024d38df83091478d616479d20748f879e90dcce4f04ac network default NAD default NAD key ] I0505 08:54:28.468582 3788 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: pre-install-tsf-tpa, hostIfaceName: 6160f71c1453fcb, network: default, NAD default, SandboxID: "6160f71c1453fcbc1a024d38df83091478d616479d20748f879e90dcce4f04ac", PCI device ID: , UID: "e8066ab6-bc41-4e07-8b11-62160995c6f9", MAC: 0a:58:0a:81:00:2d, IPs: [10.129.0.45/23] I0505 08:54:28.522728 3788 cni.go:473] [tsf-tpa/pre-install-tsf-tpa 6160f71c1453fcbc1a024d38df83091478d616479d20748f879e90dcce4f04ac network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/pre-install-tsf-tpa 6160f71c1453fcbc1a024d38df83091478d616479d20748f879e90dcce4f04ac network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"7a:62:4d:84:04:27\",\"name\":\"6160f71c1453fcb\"},{\"mac\":\"0a:58:0a:81:00:2d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ed507d0b-c855-4a24-b79f-89a172816c15\"}],\"ips\":[{\"address\":\"10.129.0.45/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0505 08:54:32.382026 3788 cni.go:452] [tsf-tpa/pre-install-tsf-tpa 6160f71c1453fcbc1a024d38df83091478d616479d20748f879e90dcce4f04ac network default NAD default NAD key ] DEL starting CNI request [tsf-tpa/pre-install-tsf-tpa 6160f71c1453fcbc1a024d38df83091478d616479d20748f879e90dcce4f04ac network default NAD default NAD key ] I0505 08:54:32.471273 3788 cni.go:473] [tsf-tpa/pre-install-tsf-tpa 6160f71c1453fcbc1a024d38df83091478d616479d20748f879e90dcce4f04ac network default NAD default NAD key default] DEL finished CNI request [tsf-tpa/pre-install-tsf-tpa 6160f71c1453fcbc1a024d38df83091478d616479d20748f879e90dcce4f04ac network default NAD default NAD key default], result "{}", err I0505 08:54:32.526628 3788 obj_retry.go:498] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 08:54:32.526689 3788 pods.go:176] Deleting pod: tsf-tpa/pre-install-tsf-tpa I0505 08:54:32.527929 3788 pods.go:236] Attempting to release IPs for pod: tsf-tpa/pre-install-tsf-tpa, ips: 10.129.0.45 I0505 08:54:32.527963 3788 obj_retry.go:498] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 08:54:32.547386 3788 obj_retry.go:492] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:54:32.547408 3788 obj_retry.go:492] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:54:32.553155 3788 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/pre-install-tsf-tpa I0505 08:54:32.553188 3788 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/pre-install-tsf-tpa I0505 08:54:32.748796 3788 base_network_controller_pods.go:487] [default/tsf-tpa/test-tsf-tpa] creating logical port tsf-tpa_test-tsf-tpa for pod on switch ip-10-0-1-71.ec2.internal I0505 08:54:32.748910 3788 kube.go:256] Updating pod tsf-tpa/test-tsf-tpa I0505 08:54:32.762721 3788 pod.go:62] [tsf-tpa/test-tsf-tpa] pod update took 13.837399ms I0505 08:54:32.762819 3788 base_network_controller_pods.go:951] [default/tsf-tpa/test-tsf-tpa] addLogicalPort annotation time took 13.940481ms I0505 08:54:32.764068 3788 pods.go:271] [tsf-tpa/test-tsf-tpa] addLogicalPort took 15.289602ms, libovsdb time 847.788µs I0505 08:54:33.108959 3788 cni.go:452] [tsf-tpa/test-tsf-tpa 446efc775da0f671ff2e53d01ab5086c0fd3ad04677020ce1176a9ad59fbb912 network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/test-tsf-tpa 446efc775da0f671ff2e53d01ab5086c0fd3ad04677020ce1176a9ad59fbb912 network default NAD default NAD key ] I0505 08:54:33.118499 3788 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: test-tsf-tpa, hostIfaceName: 446efc775da0f67, network: default, NAD default, SandboxID: "446efc775da0f671ff2e53d01ab5086c0fd3ad04677020ce1176a9ad59fbb912", PCI device ID: , UID: "25b3b46c-b412-4ec0-85e1-f4a09cc26399", MAC: 0a:58:0a:81:00:2e, IPs: [10.129.0.46/23] I0505 08:54:33.171281 3788 cni.go:473] [tsf-tpa/test-tsf-tpa 446efc775da0f671ff2e53d01ab5086c0fd3ad04677020ce1176a9ad59fbb912 network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/test-tsf-tpa 446efc775da0f671ff2e53d01ab5086c0fd3ad04677020ce1176a9ad59fbb912 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"06:fb:f9:ba:2d:db\",\"name\":\"446efc775da0f67\"},{\"mac\":\"0a:58:0a:81:00:2e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2fb7a3e8-636e-4161-bb2b-76d3615b9187\"}],\"ips\":[{\"address\":\"10.129.0.46/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0505 08:54:38.640187 3788 obj_retry.go:498] Detected object tsf-tpa/create-db-pqdqg of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 08:54:38.640744 3788 obj_retry.go:498] Detected object tsf-tpa/create-db-pqdqg of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 08:54:39.664950 3788 obj_retry.go:492] Detected object tsf-tpa/create-db-pqdqg of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:54:39.664996 3788 obj_retry.go:492] Detected object tsf-tpa/create-db-pqdqg of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:54:42.663475 3788 obj_retry.go:498] Detected object tsf-tpa/migrate-db-q9lx7 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 08:54:42.664007 3788 obj_retry.go:498] Detected object tsf-tpa/migrate-db-q9lx7 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 08:54:42.803840 3788 obj_retry.go:492] Detected object tsf-tpa/migrate-db-q9lx7 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:54:42.803864 3788 obj_retry.go:492] Detected object tsf-tpa/migrate-db-q9lx7 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:54:48.361016 3788 obj_retry.go:498] Detected object tsf-tpa/create-importers-sxgn8 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 08:54:48.361547 3788 obj_retry.go:498] Detected object tsf-tpa/create-importers-sxgn8 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 08:54:49.385781 3788 obj_retry.go:492] Detected object tsf-tpa/create-importers-sxgn8 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:54:49.385802 3788 obj_retry.go:492] Detected object tsf-tpa/create-importers-sxgn8 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0505 08:54:50.577587 3788 util.go:870] Failed to build global endpoints for port TCP/endpoint: empty IP address endpoints for service tsf-tpa/server I0505 08:55:18.619769 3788 cni.go:452] [tsf-tpa/test-tsf-tpa 446efc775da0f671ff2e53d01ab5086c0fd3ad04677020ce1176a9ad59fbb912 network default NAD default NAD key ] DEL starting CNI request [tsf-tpa/test-tsf-tpa 446efc775da0f671ff2e53d01ab5086c0fd3ad04677020ce1176a9ad59fbb912 network default NAD default NAD key ] I0505 08:55:18.702646 3788 cni.go:473] [tsf-tpa/test-tsf-tpa 446efc775da0f671ff2e53d01ab5086c0fd3ad04677020ce1176a9ad59fbb912 network default NAD default NAD key default] DEL finished CNI request [tsf-tpa/test-tsf-tpa 446efc775da0f671ff2e53d01ab5086c0fd3ad04677020ce1176a9ad59fbb912 network default NAD default NAD key default], result "{}", err I0505 08:55:18.758451 3788 obj_retry.go:498] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 08:55:18.758514 3788 pods.go:176] Deleting pod: tsf-tpa/test-tsf-tpa I0505 08:55:18.759799 3788 pods.go:236] Attempting to release IPs for pod: tsf-tpa/test-tsf-tpa, ips: 10.129.0.46 I0505 08:55:18.759829 3788 obj_retry.go:498] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 08:55:18.777195 3788 obj_retry.go:492] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:55:18.777220 3788 obj_retry.go:492] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:55:18.791821 3788 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/test-tsf-tpa I0505 08:55:18.791849 3788 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/test-tsf-tpa I0505 08:55:19.490724 3788 namespace.go:98] [integration-service] adding namespace I0505 08:55:19.491545 3788 namespace.go:102] [integration-service] adding namespace took 798.968µs I0505 08:55:19.498281 3788 namespace.go:98] [build-service] adding namespace I0505 08:55:19.498366 3788 namespace.go:98] [image-controller] adding namespace I0505 08:55:19.498983 3788 namespace.go:102] [build-service] adding namespace took 684.554µs I0505 08:55:19.499527 3788 namespace.go:102] [image-controller] adding namespace took 1.14674ms I0505 08:55:19.506138 3788 namespace.go:98] [default-tenant] adding namespace I0505 08:55:19.506753 3788 namespace.go:102] [default-tenant] adding namespace took 602.71µs I0505 08:55:19.555144 3788 namespace.go:142] [integration-service] updating namespace I0505 08:55:19.595243 3788 namespace.go:142] [integration-service] updating namespace I0505 08:55:19.625997 3788 namespace.go:142] [build-service] updating namespace I0505 08:55:19.649803 3788 namespace.go:142] [build-service] updating namespace I0505 08:55:19.674672 3788 namespace.go:142] [image-controller] updating namespace I0505 08:55:19.759212 3788 namespace.go:142] [default-tenant] updating namespace I0505 08:55:19.783323 3788 namespace.go:142] [image-controller] updating namespace I0505 08:55:19.810636 3788 base_network_controller_pods.go:487] [default/konflux-ui/test-tsf-konflux] creating logical port konflux-ui_test-tsf-konflux for pod on switch ip-10-0-1-71.ec2.internal I0505 08:55:19.810782 3788 kube.go:256] Updating pod konflux-ui/test-tsf-konflux I0505 08:55:19.811678 3788 namespace.go:142] [default-tenant] updating namespace I0505 08:55:19.861182 3788 pod.go:62] [konflux-ui/test-tsf-konflux] pod update took 50.433797ms I0505 08:55:19.861207 3788 base_network_controller_pods.go:951] [default/konflux-ui/test-tsf-konflux] addLogicalPort annotation time took 50.466111ms I0505 08:55:19.861860 3788 namespace.go:142] [build-service] updating namespace I0505 08:55:19.862707 3788 pods.go:271] [konflux-ui/test-tsf-konflux] addLogicalPort took 52.084584ms, libovsdb time 1.069297ms I0505 08:55:19.915153 3788 namespace.go:142] [integration-service] updating namespace I0505 08:55:19.926156 3788 namespace.go:98] [release-service] adding namespace I0505 08:55:19.927073 3788 namespace.go:102] [release-service] adding namespace took 893.51µs I0505 08:55:20.010547 3788 namespace.go:142] [release-service] updating namespace I0505 08:55:20.038311 3788 namespace.go:142] [konflux-ui] updating namespace I0505 08:55:20.078169 3788 namespace.go:142] [release-service] updating namespace I0505 08:55:20.172167 3788 cni.go:452] [konflux-ui/test-tsf-konflux 285ec5987a74eb8f94249dddb0fb805ab4d1680f55a31c4d8309252fe41ccbf4 network default NAD default NAD key ] ADD starting CNI request [konflux-ui/test-tsf-konflux 285ec5987a74eb8f94249dddb0fb805ab4d1680f55a31c4d8309252fe41ccbf4 network default NAD default NAD key ] I0505 08:55:20.181644 3788 helper_linux.go:556] ConfigureOVS: namespace: konflux-ui, podName: test-tsf-konflux, hostIfaceName: 285ec5987a74eb8, network: default, NAD default, SandboxID: "285ec5987a74eb8f94249dddb0fb805ab4d1680f55a31c4d8309252fe41ccbf4", PCI device ID: , UID: "d9f181af-ddc1-44d5-9bf2-1bf319d6d241", MAC: 0a:58:0a:81:00:2f, IPs: [10.129.0.47/23] I0505 08:55:20.234319 3788 cni.go:473] [konflux-ui/test-tsf-konflux 285ec5987a74eb8f94249dddb0fb805ab4d1680f55a31c4d8309252fe41ccbf4 network default NAD default NAD key default] ADD finished CNI request [konflux-ui/test-tsf-konflux 285ec5987a74eb8f94249dddb0fb805ab4d1680f55a31c4d8309252fe41ccbf4 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"fa:94:89:79:51:0d\",\"name\":\"285ec5987a74eb8\"},{\"mac\":\"0a:58:0a:81:00:2f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d2e8aae7-f912-46c4-9e04-b4bc6566b861\"}],\"ips\":[{\"address\":\"10.129.0.47/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0505 08:55:20.254241 3788 namespace.go:98] [konflux-info] adding namespace I0505 08:55:20.255151 3788 namespace.go:102] [konflux-info] adding namespace took 888.156µs I0505 08:55:20.319188 3788 namespace.go:98] [namespace-lister] adding namespace I0505 08:55:20.320137 3788 namespace.go:102] [namespace-lister] adding namespace took 926.159µs I0505 08:55:20.336721 3788 namespace.go:142] [konflux-info] updating namespace I0505 08:55:20.443284 3788 namespace.go:98] [enterprise-contract-service] adding namespace I0505 08:55:20.444165 3788 namespace.go:102] [enterprise-contract-service] adding namespace took 859.663µs I0505 08:55:20.452654 3788 namespace.go:142] [konflux-info] updating namespace I0505 08:55:20.583037 3788 namespace.go:142] [image-controller] updating namespace I0505 08:55:20.583174 3788 namespace.go:142] [namespace-lister] updating namespace I0505 08:55:20.785192 3788 namespace.go:142] [namespace-lister] updating namespace I0505 08:55:20.896362 3788 namespace.go:142] [enterprise-contract-service] updating namespace I0505 08:55:20.939200 3788 namespace.go:142] [default-tenant] updating namespace I0505 08:55:21.009480 3788 namespace.go:142] [enterprise-contract-service] updating namespace I0505 08:55:21.304211 3788 namespace.go:98] [segment-bridge] adding namespace I0505 08:55:21.305198 3788 namespace.go:102] [segment-bridge] adding namespace took 961.567µs I0505 08:55:21.472570 3788 namespace.go:98] [konflux-cli] adding namespace I0505 08:55:21.473328 3788 namespace.go:102] [konflux-cli] adding namespace took 739.557µs I0505 08:55:21.602365 3788 namespace.go:142] [segment-bridge] updating namespace I0505 08:55:21.834881 3788 namespace.go:142] [segment-bridge] updating namespace I0505 08:55:22.450103 3788 namespace.go:142] [konflux-cli] updating namespace I0505 08:55:22.607153 3788 namespace.go:142] [konflux-cli] updating namespace I0505 08:55:22.621188 3788 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-from-konfluxui for network default I0505 08:55:22.621806 3788 pod_selector_address_set.go:213] Created shared address set for pod selector namespace-lister_LS{ML:{app: proxy,},} I0505 08:55:22.621821 3788 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-from-konfluxui added to peer address sets [namespace-lister_LS{ML:{app: proxy,},}] I0505 08:55:22.624530 3788 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-from-konfluxui resources completed, update namespace loglevel I0505 08:55:22.687551 3788 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-to-apiserver for network default I0505 08:55:22.687574 3788 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-to-apiserver added to peer address sets [] I0505 08:55:22.688378 3788 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-to-apiserver resources completed, update namespace loglevel I0505 08:55:23.151557 3788 namespace.go:142] [build-service] updating namespace I0505 08:55:23.199641 3788 base_network_controller_pods.go:487] [default/konflux-ui/dex-66599f95df-24c7c] creating logical port konflux-ui_dex-66599f95df-24c7c for pod on switch ip-10-0-1-71.ec2.internal I0505 08:55:23.199747 3788 kube.go:256] Updating pod konflux-ui/dex-66599f95df-24c7c I0505 08:55:23.246516 3788 pod.go:62] [konflux-ui/dex-66599f95df-24c7c] pod update took 46.795171ms I0505 08:55:23.246548 3788 base_network_controller_pods.go:951] [default/konflux-ui/dex-66599f95df-24c7c] addLogicalPort annotation time took 46.832089ms I0505 08:55:23.247887 3788 pods.go:271] [konflux-ui/dex-66599f95df-24c7c] addLogicalPort took 48.262798ms, libovsdb time 836.31µs I0505 08:55:23.682551 3788 namespace.go:142] [default-tenant] updating namespace I0505 08:55:23.960972 3788 base_network_controller_pods.go:487] [default/integration-service/integration-service-controller-manager-7b666bf6d8-l7nr8] creating logical port integration-service_integration-service-controller-manager-7b666bf6d8-l7nr8 for pod on switch ip-10-0-1-71.ec2.internal I0505 08:55:23.961055 3788 base_network_controller_pods.go:487] [default/build-service/build-service-controller-manager-7cbf488fcf-4mrjf] creating logical port build-service_build-service-controller-manager-7cbf488fcf-4mrjf for pod on switch ip-10-0-1-71.ec2.internal I0505 08:55:23.961077 3788 kube.go:256] Updating pod integration-service/integration-service-controller-manager-7b666bf6d8-l7nr8 I0505 08:55:23.961113 3788 kube.go:256] Updating pod build-service/build-service-controller-manager-7cbf488fcf-4mrjf I0505 08:55:23.966193 3788 namespace.go:142] [enterprise-contract-service] updating namespace I0505 08:55:24.000787 3788 pod.go:62] [integration-service/integration-service-controller-manager-7b666bf6d8-l7nr8] pod update took 39.741571ms I0505 08:55:24.000821 3788 base_network_controller_pods.go:951] [default/integration-service/integration-service-controller-manager-7b666bf6d8-l7nr8] addLogicalPort annotation time took 39.778854ms I0505 08:55:24.002242 3788 pods.go:271] [integration-service/integration-service-controller-manager-7b666bf6d8-l7nr8] addLogicalPort took 41.289312ms, libovsdb time 892.383µs I0505 08:55:24.008839 3788 pod.go:62] [build-service/build-service-controller-manager-7cbf488fcf-4mrjf] pod update took 47.735191ms I0505 08:55:24.008865 3788 base_network_controller_pods.go:951] [default/build-service/build-service-controller-manager-7cbf488fcf-4mrjf] addLogicalPort annotation time took 47.768983ms I0505 08:55:24.010444 3788 pods.go:271] [build-service/build-service-controller-manager-7cbf488fcf-4mrjf] addLogicalPort took 49.394429ms, libovsdb time 990.259µs I0505 08:55:24.253100 3788 namespace.go:142] [image-controller] updating namespace I0505 08:55:24.338975 3788 cni.go:452] [build-service/build-service-controller-manager-7cbf488fcf-4mrjf 50eedd6cec85c689730c968097f6d783f2bcf94a005f865d1c80f4e27b4e7f0c network default NAD default NAD key ] ADD starting CNI request [build-service/build-service-controller-manager-7cbf488fcf-4mrjf 50eedd6cec85c689730c968097f6d783f2bcf94a005f865d1c80f4e27b4e7f0c network default NAD default NAD key ] I0505 08:55:24.348662 3788 helper_linux.go:556] ConfigureOVS: namespace: build-service, podName: build-service-controller-manager-7cbf488fcf-4mrjf, hostIfaceName: 50eedd6cec85c68, network: default, NAD default, SandboxID: "50eedd6cec85c689730c968097f6d783f2bcf94a005f865d1c80f4e27b4e7f0c", PCI device ID: , UID: "2a74d381-803b-46a4-a50e-53feefb4467d", MAC: 0a:58:0a:81:00:32, IPs: [10.129.0.50/23] I0505 08:55:24.404798 3788 cni.go:473] [build-service/build-service-controller-manager-7cbf488fcf-4mrjf 50eedd6cec85c689730c968097f6d783f2bcf94a005f865d1c80f4e27b4e7f0c network default NAD default NAD key default] ADD finished CNI request [build-service/build-service-controller-manager-7cbf488fcf-4mrjf 50eedd6cec85c689730c968097f6d783f2bcf94a005f865d1c80f4e27b4e7f0c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"72:9e:ba:ab:af:b7\",\"name\":\"50eedd6cec85c68\"},{\"mac\":\"0a:58:0a:81:00:32\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0c219dbc-7691-43cd-9bc4-a2b41e990015\"}],\"ips\":[{\"address\":\"10.129.0.50/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0505 08:55:24.566180 3788 namespace.go:142] [integration-service] updating namespace I0505 08:55:24.781599 3788 namespace.go:142] [konflux-cli] updating namespace I0505 08:55:24.909130 3788 base_network_controller_pods.go:487] [default/konflux-ui/proxy-567b5b6747-g5hq2] creating logical port konflux-ui_proxy-567b5b6747-g5hq2 for pod on switch ip-10-0-1-71.ec2.internal I0505 08:55:24.909268 3788 kube.go:256] Updating pod konflux-ui/proxy-567b5b6747-g5hq2 I0505 08:55:24.955398 3788 pod.go:62] [konflux-ui/proxy-567b5b6747-g5hq2] pod update took 46.161286ms I0505 08:55:24.955421 3788 base_network_controller_pods.go:951] [default/konflux-ui/proxy-567b5b6747-g5hq2] addLogicalPort annotation time took 46.191981ms I0505 08:55:24.956895 3788 pods.go:271] [konflux-ui/proxy-567b5b6747-g5hq2] addLogicalPort took 47.778136ms, libovsdb time 1.004019ms I0505 08:55:25.079177 3788 namespace.go:142] [konflux-info] updating namespace I0505 08:55:25.328298 3788 namespace.go:142] [namespace-lister] updating namespace I0505 08:55:25.591493 3788 namespace.go:142] [release-service] updating namespace I0505 08:55:25.782974 3788 namespace.go:142] [segment-bridge] updating namespace I0505 08:55:25.893862 3788 namespace.go:142] [build-service] updating namespace I0505 08:55:25.909595 3788 namespace.go:142] [default-tenant] updating namespace I0505 08:55:25.933594 3788 namespace.go:142] [enterprise-contract-service] updating namespace I0505 08:55:25.949326 3788 namespace.go:142] [image-controller] updating namespace I0505 08:55:25.959911 3788 namespace.go:142] [integration-service] updating namespace I0505 08:55:25.973716 3788 namespace.go:142] [konflux-cli] updating namespace I0505 08:55:25.985929 3788 namespace.go:142] [konflux-info] updating namespace I0505 08:55:26.003163 3788 namespace.go:142] [namespace-lister] updating namespace I0505 08:55:26.021085 3788 namespace.go:142] [release-service] updating namespace I0505 08:55:26.047257 3788 namespace.go:142] [segment-bridge] updating namespace I0505 08:55:26.107221 3788 namespace.go:142] [build-service] updating namespace I0505 08:55:26.189239 3788 namespace.go:142] [default-tenant] updating namespace I0505 08:55:26.257215 3788 namespace.go:142] [enterprise-contract-service] updating namespace I0505 08:55:26.335527 3788 namespace.go:142] [image-controller] updating namespace I0505 08:55:26.438002 3788 namespace.go:142] [integration-service] updating namespace I0505 08:55:26.549099 3788 namespace.go:142] [konflux-cli] updating namespace I0505 08:55:26.697214 3788 namespace.go:142] [konflux-info] updating namespace I0505 08:55:26.796152 3788 namespace.go:142] [namespace-lister] updating namespace I0505 08:55:26.895059 3788 namespace.go:142] [release-service] updating namespace I0505 08:55:27.004168 3788 namespace.go:142] [segment-bridge] updating namespace I0505 08:55:27.864060 3788 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=21 W0505 08:55:29.049174 3788 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service image-controller/image-controller-controller-manager-metrics-service W0505 08:55:29.078835 3788 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service namespace-lister/namespace-lister W0505 08:55:29.708170 3788 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service I0505 08:55:32.134390 3788 cni.go:452] [integration-service/integration-service-controller-manager-7b666bf6d8-l7nr8 7c79a1d8d4a4ee3723bbde0994bc3229aaefc4e88c82d3ef72e470849c66b295 network default NAD default NAD key ] ADD starting CNI request [integration-service/integration-service-controller-manager-7b666bf6d8-l7nr8 7c79a1d8d4a4ee3723bbde0994bc3229aaefc4e88c82d3ef72e470849c66b295 network default NAD default NAD key ] I0505 08:55:32.146361 3788 helper_linux.go:556] ConfigureOVS: namespace: integration-service, podName: integration-service-controller-manager-7b666bf6d8-l7nr8, hostIfaceName: 7c79a1d8d4a4ee3, network: default, NAD default, SandboxID: "7c79a1d8d4a4ee3723bbde0994bc3229aaefc4e88c82d3ef72e470849c66b295", PCI device ID: , UID: "ffdff3c9-e918-4ab8-8458-b9a0a11676aa", MAC: 0a:58:0a:81:00:31, IPs: [10.129.0.49/23] I0505 08:55:32.205125 3788 cni.go:473] [integration-service/integration-service-controller-manager-7b666bf6d8-l7nr8 7c79a1d8d4a4ee3723bbde0994bc3229aaefc4e88c82d3ef72e470849c66b295 network default NAD default NAD key default] ADD finished CNI request [integration-service/integration-service-controller-manager-7b666bf6d8-l7nr8 7c79a1d8d4a4ee3723bbde0994bc3229aaefc4e88c82d3ef72e470849c66b295 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ea:06:43:60:4e:db\",\"name\":\"7c79a1d8d4a4ee3\"},{\"mac\":\"0a:58:0a:81:00:31\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/50a3335f-846a-496f-8ae6-e2801c1475c4\"}],\"ips\":[{\"address\":\"10.129.0.49/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0505 08:55:32.775274 3788 cni.go:452] [konflux-ui/proxy-567b5b6747-g5hq2 71002ed0b96a6b27ff5d077ba3c0e658500e92ff4e67496a07a85c8724f8b170 network default NAD default NAD key ] ADD starting CNI request [konflux-ui/proxy-567b5b6747-g5hq2 71002ed0b96a6b27ff5d077ba3c0e658500e92ff4e67496a07a85c8724f8b170 network default NAD default NAD key ] I0505 08:55:32.786117 3788 helper_linux.go:556] ConfigureOVS: namespace: konflux-ui, podName: proxy-567b5b6747-g5hq2, hostIfaceName: 71002ed0b96a6b2, network: default, NAD default, SandboxID: "71002ed0b96a6b27ff5d077ba3c0e658500e92ff4e67496a07a85c8724f8b170", PCI device ID: , UID: "90bd0226-6ec4-43ab-97f6-07df63a5ce09", MAC: 0a:58:0a:81:00:33, IPs: [10.129.0.51/23] I0505 08:55:32.852757 3788 cni.go:473] [konflux-ui/proxy-567b5b6747-g5hq2 71002ed0b96a6b27ff5d077ba3c0e658500e92ff4e67496a07a85c8724f8b170 network default NAD default NAD key default] ADD finished CNI request [konflux-ui/proxy-567b5b6747-g5hq2 71002ed0b96a6b27ff5d077ba3c0e658500e92ff4e67496a07a85c8724f8b170 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"5a:09:59:8f:55:e5\",\"name\":\"71002ed0b96a6b2\"},{\"mac\":\"0a:58:0a:81:00:33\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/aed9b2db-6138-48ee-b393-0bd6633b39d3\"}],\"ips\":[{\"address\":\"10.129.0.51/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0505 08:55:38.736970 3788 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0505 08:55:38.737639 3788 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0505 08:55:38.766745 3788 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0505 08:55:38.766763 3788 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0505 08:55:39.111649 3788 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0505 08:55:39.111672 3788 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0505 08:55:39.133501 3788 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0505 08:55:39.133632 3788 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0505 08:55:39.151239 3788 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service W0505 08:55:39.152100 3788 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service release-service/release-service-webhook-service I0505 08:55:39.166384 3788 cni.go:452] [konflux-ui/dex-66599f95df-24c7c f359f6970b56d21908deb2f505bed30bc0bcdb6d2af63c4fd6417509dc50135c network default NAD default NAD key ] ADD starting CNI request [konflux-ui/dex-66599f95df-24c7c f359f6970b56d21908deb2f505bed30bc0bcdb6d2af63c4fd6417509dc50135c network default NAD default NAD key ] I0505 08:55:39.175606 3788 helper_linux.go:556] ConfigureOVS: namespace: konflux-ui, podName: dex-66599f95df-24c7c, hostIfaceName: f359f6970b56d21, network: default, NAD default, SandboxID: "f359f6970b56d21908deb2f505bed30bc0bcdb6d2af63c4fd6417509dc50135c", PCI device ID: , UID: "ce79c2de-aea0-4323-944e-4907a9049597", MAC: 0a:58:0a:81:00:30, IPs: [10.129.0.48/23] I0505 08:55:39.231890 3788 cni.go:473] [konflux-ui/dex-66599f95df-24c7c f359f6970b56d21908deb2f505bed30bc0bcdb6d2af63c4fd6417509dc50135c network default NAD default NAD key default] ADD finished CNI request [konflux-ui/dex-66599f95df-24c7c f359f6970b56d21908deb2f505bed30bc0bcdb6d2af63c4fd6417509dc50135c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ce:92:fc:ce:e8:8d\",\"name\":\"f359f6970b56d21\"},{\"mac\":\"0a:58:0a:81:00:30\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7d20586d-643b-46f4-950e-277e71577db2\"}],\"ips\":[{\"address\":\"10.129.0.48/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0505 08:55:40.698588 3788 util.go:959] Failed to build node endpoints for node ip-10-0-1-71.ec2.internal port 8443: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service W0505 08:55:41.818436 3788 util.go:870] Failed to build global endpoints for port TCP/dex: empty IP address endpoints for service konflux-ui/dex I0505 08:55:42.966423 3788 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=7 W0505 08:55:49.754238 3788 util.go:959] Failed to build node endpoints for node ip-10-0-1-71.ec2.internal port 9443: empty IP address endpoints for service integration-service/integration-service-webhook-service W0505 08:55:49.756746 3788 util.go:959] Failed to build node endpoints for node ip-10-0-1-71.ec2.internal port 8080: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service I0505 08:55:49.947768 3788 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=10 I0505 08:55:50.311222 3788 cni.go:452] [integration-service/integration-service-controller-manager-7b666bf6d8-l7nr8 7c79a1d8d4a4ee3723bbde0994bc3229aaefc4e88c82d3ef72e470849c66b295 network default NAD default NAD key ] DEL starting CNI request [integration-service/integration-service-controller-manager-7b666bf6d8-l7nr8 7c79a1d8d4a4ee3723bbde0994bc3229aaefc4e88c82d3ef72e470849c66b295 network default NAD default NAD key ] I0505 08:55:50.406849 3788 cni.go:473] [integration-service/integration-service-controller-manager-7b666bf6d8-l7nr8 7c79a1d8d4a4ee3723bbde0994bc3229aaefc4e88c82d3ef72e470849c66b295 network default NAD default NAD key default] DEL finished CNI request [integration-service/integration-service-controller-manager-7b666bf6d8-l7nr8 7c79a1d8d4a4ee3723bbde0994bc3229aaefc4e88c82d3ef72e470849c66b295 network default NAD default NAD key default], result "{}", err I0505 08:55:50.465486 3788 obj_retry.go:498] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-l7nr8 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 08:55:50.465529 3788 pods.go:176] Deleting pod: integration-service/integration-service-controller-manager-7b666bf6d8-l7nr8 I0505 08:55:50.466884 3788 pods.go:236] Attempting to release IPs for pod: integration-service/integration-service-controller-manager-7b666bf6d8-l7nr8, ips: 10.129.0.49 I0505 08:55:50.466913 3788 obj_retry.go:498] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-l7nr8 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 08:55:50.827171 3788 obj_retry.go:492] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-l7nr8 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:55:50.827194 3788 obj_retry.go:492] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-l7nr8 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 08:55:50.832370 3788 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod integration-service/integration-service-controller-manager-7b666bf6d8-l7nr8 I0505 08:55:50.832393 3788 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod integration-service/integration-service-controller-manager-7b666bf6d8-l7nr8 W0505 08:55:54.784703 3788 util.go:959] Failed to build node endpoints for node ip-10-0-1-71.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/dex