+ . /ovnkube-lib/ovnkube-lib.sh ++ set -x ++ K8S_NODE=ip-10-0-2-63.ec2.internal ++ [[ -n ip-10-0-2-63.ec2.internal ]] ++ [[ -f /env/ip-10-0-2-63.ec2.internal ]] ++ northd_pidfile=/var/run/ovn/ovn-northd.pid ++ controller_pidfile=/var/run/ovn/ovn-controller.pid ++ controller_logfile=/var/log/ovn/acl-audit-log.log ++ vswitch_dbsock=/var/run/openvswitch/db.sock ++ nbdb_pidfile=/var/run/ovn/ovnnb_db.pid ++ nbdb_sock=/var/run/ovn/ovnnb_db.sock ++ nbdb_ctl=/var/run/ovn/ovnnb_db.ctl ++ sbdb_pidfile=/var/run/ovn/ovnsb_db.pid ++ sbdb_sock=/var/run/ovn/ovnsb_db.sock ++ sbdb_ctl=/var/run/ovn/ovnsb_db.ctl + start-ovnkube-node 4 29103 29105 + local log_level=4 + local metrics_port=29103 + local ovn_metrics_port=29105 + ovn_advertised_udn_isolation_mode_flag= + openflow_probe_flag= + [[ 3 -ne 3 ]] + ovs-vsctl br-exists br-ex + add_garp_drop_flow br-ex + local bridge=br-ex + local cookie=0x0305 + local priority=499 ++ ovs-vsctl list-ports br-ex + for port_name in $(ovs-vsctl list-ports "$bridge") + [[ ens5 == *to-br-int ]] + cni-bin-copy + . /host/etc/os-release ++ NAME='Red Hat Enterprise Linux CoreOS' ++ VERSION='9.6.20260401-0 (Plow)' ++ ID=rhel ++ ID_LIKE=fedora ++ VERSION_ID=9.6 ++ PLATFORM_ID=platform:el9 ++ PRETTY_NAME='Red Hat Enterprise Linux CoreOS 9.6.20260401-0 (Plow)' ++ ANSI_COLOR='0;31' ++ LOGO=fedora-logo-icon ++ CPE_NAME=cpe:/o:redhat:enterprise_linux:9::baseos ++ HOME_URL=https://www.redhat.com/ ++ DOCUMENTATION_URL=https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9 ++ BUG_REPORT_URL=https://issues.redhat.com/ ++ REDHAT_BUGZILLA_PRODUCT='Red Hat Enterprise Linux 9' ++ REDHAT_BUGZILLA_PRODUCT_VERSION=9.6 ++ REDHAT_SUPPORT_PRODUCT='Red Hat Enterprise Linux' ++ REDHAT_SUPPORT_PRODUCT_VERSION=9.6 ++ OSTREE_VERSION=9.6.20260401-0 ++ VARIANT=CoreOS ++ VARIANT_ID=coreos ++ OPENSHIFT_VERSION=4.21 + rhelmajor= + case "${ID}" in ++ echo 9.6 ++ cut -f 1 -d . + rhelmajor=9 + sourcedir=/usr/libexec/cni/ + case "${rhelmajor}" in + sourcedir=/usr/libexec/cni/rhel9 + cp -f /usr/libexec/cni/rhel9/ovn-k8s-cni-overlay /cni-bin-dir/ ++ date '+%m%d %H:%M:%S.%N' + echo 'I0504 12:23:37.586672272 - disable conntrack on geneve port' I0504 12:23:37.586672272 - disable conntrack on geneve port + iptables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + iptables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK ++ date '+%m%d %H:%M:%S.%N' I0504 12:23:37.627344810 - starting ovnkube-node + echo 'I0504 12:23:37.627344810 - starting ovnkube-node' + egress_features_enable_flag='--enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true' + init_ovnkube_controller='--init-ovnkube-controller ip-10-0-2-63.ec2.internal' + multi_external_gateway_enable_flag=--enable-multi-external-gateway=true + gateway_interface=br-ex + enable_multicast_flag=--enable-multicast + OVN_NODE_MODE=full + '[' full == dpu-host ']' + '[' shared == shared ']' + gateway_mode_flags='--gateway-mode shared --gateway-interface br-ex' + export_network_flows_flags= + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + gw_interface_flag= + '[' -d /sys/class/net/br-ex1 ']' + node_mgmt_port_netdev_flags= + [[ -n '' ]] + [[ -n '' ]] + multi_network_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag=--enable-network-segmentation + route_advertisements_enable_flag= + [[ false == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag= + [[ true == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag=--enable-preconfigured-udn-addresses + network_observability_enabled_flag= + [[ false == \t\r\u\e ]] + multi_network_policy_enabled_flag= + [[ false == \t\r\u\e ]] + admin_network_policy_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + admin_network_policy_enabled_flag=--enable-admin-network-policy + dns_name_resolver_enabled_flag= + [[ false == \t\r\u\e ]] + ip_forwarding_flag= + '[' '' == Global ']' + ip_forwarding_flag=--disable-forwarding + sysctl -w net.ipv4.ip_forward=0 net.ipv4.ip_forward = 0 + sysctl -w net.ipv6.conf.all.forwarding=0 net.ipv6.conf.all.forwarding = 0 + [[ '' != '' ]] + [[ '' != '' ]] + NETWORK_NODE_IDENTITY_ENABLE= + [[ true == \t\r\u\e ]] + NETWORK_NODE_IDENTITY_ENABLE=' --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h ' + ovn_v4_join_subnet_opt= + [[ '' != '' ]] + ovn_v6_join_subnet_opt= + [[ '' != '' ]] + ovn_v4_masquerade_subnet_opt= + [[ 169.254.0.0/17 != '' ]] + ovn_v4_masquerade_subnet_opt='--gateway-v4-masquerade-subnet 169.254.0.0/17' + ovn_v6_masquerade_subnet_opt= + [[ fd69::/112 != '' ]] + ovn_v6_masquerade_subnet_opt='--gateway-v6-masquerade-subnet fd69::/112' + ovn_v4_transit_switch_subnet_opt= + [[ '' != '' ]] + ovn_v6_transit_switch_subnet_opt= + [[ '' != '' ]] + exec /usr/bin/ovnkube --init-ovnkube-controller ip-10-0-2-63.ec2.internal --init-node ip-10-0-2-63.ec2.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4 --inactivity-probe=180000 --gateway-mode shared --gateway-interface br-ex --metrics-bind-address 127.0.0.1:29103 --ovn-metrics-bind-address 127.0.0.1:29105 --metrics-enable-pprof --metrics-enable-config-duration --export-ovs-metrics --disable-snat-multiple-gws --enable-multi-network --enable-network-segmentation --enable-preconfigured-udn-addresses --enable-admin-network-policy --enable-multicast --zone ip-10-0-2-63.ec2.internal --enable-interconnect --acl-logging-rate-limit 20 --disable-forwarding --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h --gateway-v4-masquerade-subnet 169.254.0.0/17 --gateway-v6-masquerade-subnet fd69::/112 --enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true --enable-multi-external-gateway=true I0504 12:23:37.676103 3659 config.go:2726] Parsed config file /run/ovnkube-config/ovnkube.conf I0504 12:23:37.676156 3659 config.go:2727] Parsed config: {Default:{MTU:8901 RoutableMTU:0 ConntrackZone:64000 HostMasqConntrackZone:0 OVNMasqConntrackZone:0 HostNodePortConntrackZone:0 ReassemblyConntrackZone:0 EncapType:geneve EncapIP: EffectiveEncapIP: EncapPort:6081 InactivityProbe:100000 OpenFlowProbe:0 OfctrlWaitBeforeClear:0 MonitorAll:true OVSDBTxnTimeout:1m40s LFlowCacheEnable:true LFlowCacheLimit:0 LFlowCacheLimitKb:1048576 RawClusterSubnets:10.128.0.0/14/23 ClusterSubnets:[] EnableUDPAggregation:true Zone:global RawUDNAllowedDefaultServices:default/kubernetes,openshift-dns/dns-default UDNAllowedDefaultServices:[] Transport:geneve} Logging:{File: CNIFile: LibovsdbFile: Level:4 LogFileMaxSize:100 LogFileMaxBackups:5 LogFileMaxAge:5 ACLLoggingRateLimit:20} Monitoring:{RawNetFlowTargets: RawSFlowTargets: RawIPFIXTargets: NetFlowTargets:[] SFlowTargets:[] IPFIXTargets:[]} IPFIX:{Sampling:400 CacheActiveTimeout:60 CacheMaxFlows:0} CNI:{ConfDir:/etc/cni/net.d Plugin:ovn-k8s-cni-overlay} OVNKubernetesFeature:{EnableAdminNetworkPolicy:false EnableEgressIP:false EgressIPReachabiltyTotalTimeout:1 EnableEgressFirewall:false EnableEgressQoS:false EnableEgressService:false EgressIPNodeHealthCheckPort:9107 EnableMultiNetwork:false EnableNetworkSegmentation:true EnableNetworkConnect:false EnablePreconfiguredUDNAddresses:true EnableRouteAdvertisements:false EnableEVPN:false EnableMultiNetworkPolicy:false EnableStatelessNetPol:false EnableInterconnect:false EnableMultiExternalGateway:false EnablePersistentIPs:false EnableDNSNameResolver:false EnableServiceTemplateSupport:false EnableObservability:false EnableNetworkQoS:false AdvertisedUDNIsolationMode:strict EnableDynamicUDNAllocation:false UDNDeletionGracePeriod:2m0s} Kubernetes:{BootstrapKubeconfig: CertDir: CertDuration:10m0s Kubeconfig: CACert: CACertData: CAData:[] APIServer:https://api.kx-99ffb1110b.hypershift.local:443 Token: TokenFile: CompatServiceCIDR: RawServiceCIDRs:172.30.0.0/16 ServiceCIDRs:[] OVNConfigNamespace:openshift-ovn-kubernetes OVNEmptyLbEvents:false RawNoHostSubnetNodes: NoHostSubnetNodes: HostNetworkNamespace:openshift-host-network DisableRequestedChassis:false PlatformType:AWS HealthzBindAddress:0.0.0.0:10256 CompatMetricsBindAddress: CompatOVNMetricsBindAddress: CompatMetricsEnablePprof:false DNSServiceNamespace:openshift-dns DNSServiceName:dns-default} Metrics:{BindAddress: OVNMetricsBindAddress: ExportOVSMetrics:false EnablePprof:false NodeServerPrivKey: NodeServerCert: EnableConfigDuration:false EnableScaleMetrics:false} OvnNorth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnnb_db.db exec:} OvnSouth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnsb_db.db exec:} Gateway:{Mode:shared Interface: GatewayAcceleratedInterface: EgressGWInterface: NextHop: VLANID:0 NodeportEnable:true DisableSNATMultipleGWs:false V4JoinSubnet:100.64.0.0/16 V6JoinSubnet:fd98::/64 V4MasqueradeSubnet:169.254.169.0/29 V6MasqueradeSubnet:fd69::/125 MasqueradeIPs:{V4OVNMasqueradeIP:169.254.169.1 V6OVNMasqueradeIP:fd69::1 V4HostMasqueradeIP:169.254.169.2 V6HostMasqueradeIP:fd69::2 V4HostETPLocalMasqueradeIP:169.254.169.3 V6HostETPLocalMasqueradeIP:fd69::3 V4DummyNextHopMasqueradeIP:169.254.169.4 V6DummyNextHopMasqueradeIP:fd69::4 V4OVNServiceHairpinMasqueradeIP:169.254.169.5 V6OVNServiceHairpinMasqueradeIP:fd69::5} DisablePacketMTUCheck:false RouterSubnet: SingleNode:false DisableForwarding:false AllowNoUplink:false EphemeralPortRange:} MasterHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} ClusterMgrHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} HybridOverlay:{Enabled:false RawClusterSubnets: ClusterSubnets:[] VXLANPort:4789} OvnKubeNode:{Mode:full MgmtPortNetdev: MgmtPortDPResourceName:} ClusterManager:{V4TransitSubnet:100.88.0.0/16 V6TransitSubnet:fd97::/64} OvsPaths:{RunDir:/var/run/openvswitch/} NoOverlay:{OutboundSNAT: Routing:} ManagedBGP:{ASNumber:64512 Topology:}} I0504 12:23:37.677995 3659 kube.go:431] Waiting for certificate I0504 12:23:37.678015 3659 certificate_manager.go:422] "Certificate rotation is enabled" logger="kubernetes.io/kube-apiserver-client" I0504 12:23:37.678055 3659 certificate_manager.go:566] "Rotating certificates" logger="kubernetes.io/kube-apiserver-client" I0504 12:23:37.678793 3659 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0504 12:23:37.711419 3659 reflector.go:358] "Starting reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0504 12:23:37.711598 3659 reflector.go:404] "Listing and watching" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0504 12:23:37.714890 3659 reflector.go:436] "Caches populated" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0504 12:23:37.714987 3659 csr.go:274] "Certificate signing request is approved, waiting to be issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-8x2lk" I0504 12:23:37.723287 3659 csr.go:270] "Certificate signing request is issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-8x2lk" I0504 12:23:37.723347 3659 reflector.go:364] "Stopping reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0504 12:23:38.678565 3659 kube.go:438] Certificate found I0504 12:23:38.679247 3659 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0504 12:23:38.680282 3659 metrics.go:532] Starting metrics server at address "127.0.0.1:29103" I0504 12:23:38.743557 3659 certificate_manager.go:715] "Certificate rotation deadline determined" logger="kubernetes.io/kube-apiserver-client" expiration="2026-05-05 12:18:37 +0000 UTC" deadline="2026-05-05 07:01:14.112267495 +0000 UTC" I0504 12:23:38.743594 3659 certificate_manager.go:431] "Waiting for next certificate rotation" logger="kubernetes.io/kube-apiserver-client" sleep="18h37m35.368684906s" I0504 12:23:38.749625 3659 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0504 12:23:38.762569 3659 metrics.go:512] Create OVN Metrics Server on address: 127.0.0.1:29105 I0504 12:23:38.762744 3659 server.go:101] MetricServer registers OVS metrics I0504 12:23:38.763769 3659 server.go:105] MetricServer registers OVN DB metrics I0504 12:23:38.764195 3659 node_controller_manager.go:338] Starting the node network controller manager, Mode: full I0504 12:23:38.764380 3659 factory.go:561] Starting watch factory I0504 12:23:38.764513 3659 reflector.go:358] "Starting reflector" type="*v1.Namespace" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0504 12:23:38.764558 3659 reflector.go:404] "Listing and watching" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0504 12:23:38.764615 3659 reflector.go:358] "Starting reflector" type="*v1.Service" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0504 12:23:38.764778 3659 reflector.go:404] "Listing and watching" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0504 12:23:38.765216 3659 reflector.go:358] "Starting reflector" type="*v1.Pod" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0504 12:23:38.765464 3659 reflector.go:404] "Listing and watching" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0504 12:23:38.765621 3659 reflector.go:358] "Starting reflector" type="*v1.NetworkPolicy" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0504 12:23:38.765688 3659 reflector.go:404] "Listing and watching" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0504 12:23:38.764573 3659 reflector.go:358] "Starting reflector" type="*v1.Node" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0504 12:23:38.766282 3659 reflector.go:404] "Listing and watching" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0504 12:23:38.764518 3659 reflector.go:358] "Starting reflector" type="*v1.EndpointSlice" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0504 12:23:38.766910 3659 reflector.go:404] "Listing and watching" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0504 12:23:38.769281 3659 ovn_db.go:330] /var/run/openvswitch/ovnnb_db.sock getting info failed: stat /var/run/openvswitch/ovnnb_db.sock: no such file or directory I0504 12:23:38.769343 3659 ovn_db.go:327] ovnnb_db.sock found at /var/run/ovn/ I0504 12:23:38.772345 3659 controller_manager.go:370] Starting the ovnkube controller I0504 12:23:38.772363 3659 controller_manager.go:375] Waiting up to 5m0s for NBDB zone to match: ip-10-0-2-63.ec2.internal I0504 12:23:38.772479 3659 controller_manager.go:395] NBDB zone sync took: 82.04µs I0504 12:23:38.772501 3659 factory.go:561] Starting watch factory I0504 12:23:38.783938 3659 ovn_db.go:388] Found OVN NB DB: &{0x1e9e6e0 /etc/ovn/ovnnb_db.db OVN_Northbound 0} I0504 12:23:38.783957 3659 ovn_db.go:395] Found OVN SB DB: &{0x1e9e9a0 /etc/ovn/ovnsb_db.db OVN_Southbound 0} I0504 12:23:38.787009 3659 reflector.go:436] "Caches populated" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0504 12:23:38.787401 3659 reflector.go:436] "Caches populated" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0504 12:23:38.787766 3659 reflector.go:436] "Caches populated" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0504 12:23:38.787822 3659 reflector.go:436] "Caches populated" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0504 12:23:38.788172 3659 reflector.go:436] "Caches populated" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0504 12:23:38.788943 3659 ovn_db.go:410] Found db is standalone, don't register db_cluster metrics I0504 12:23:38.789009 3659 server.go:109] MetricServer registers OVN Controller metrics I0504 12:23:38.794609 3659 server.go:113] MetricServer registers OVN Northd metrics I0504 12:23:38.798880 3659 reflector.go:436] "Caches populated" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0504 12:23:38.799048 3659 metrics.go:519] OVN Metrics Server starts to run ... I0504 12:23:38.870392 3659 factory.go:1977] *v1.Node informer cache synced successfully I0504 12:23:38.870408 3659 factory.go:1977] *v1.Service informer cache synced successfully I0504 12:23:38.870412 3659 factory.go:1977] *v1.EndpointSlice informer cache synced successfully I0504 12:23:38.870415 3659 factory.go:1977] *v1.Pod informer cache synced successfully I0504 12:23:38.870418 3659 factory.go:1977] *v1.NetworkPolicy informer cache synced successfully I0504 12:23:38.870421 3659 factory.go:1977] *v1.Namespace informer cache synced successfully I0504 12:23:38.870494 3659 reflector.go:358] "Starting reflector" type="*v1alpha1.BaselineAdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0504 12:23:38.870514 3659 reflector.go:404] "Listing and watching" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0504 12:23:38.870498 3659 reflector.go:358] "Starting reflector" type="*v1alpha1.AdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0504 12:23:38.870554 3659 reflector.go:404] "Listing and watching" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0504 12:23:38.873815 3659 reflector.go:436] "Caches populated" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0504 12:23:38.873820 3659 reflector.go:436] "Caches populated" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0504 12:23:38.875691 3659 factory.go:1977] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0504 12:23:38.875702 3659 factory.go:1977] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0504 12:23:38.875791 3659 reflector.go:358] "Starting reflector" type="*v1.EgressIP" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0504 12:23:38.875813 3659 reflector.go:404] "Listing and watching" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0504 12:23:38.877856 3659 factory.go:1977] *v1.Namespace informer cache synced successfully I0504 12:23:38.877869 3659 factory.go:1977] *v1.Node informer cache synced successfully I0504 12:23:38.877873 3659 factory.go:1977] *v1.Service informer cache synced successfully I0504 12:23:38.877876 3659 factory.go:1977] *v1.EndpointSlice informer cache synced successfully I0504 12:23:38.877879 3659 factory.go:1977] *v1.Pod informer cache synced successfully I0504 12:23:38.877881 3659 factory.go:1977] *v1.NetworkPolicy informer cache synced successfully I0504 12:23:38.878242 3659 reflector.go:436] "Caches populated" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0504 12:23:38.881248 3659 factory.go:1977] *v1.EgressIP informer cache synced successfully I0504 12:23:38.881311 3659 reflector.go:358] "Starting reflector" type="*v1.EgressFirewall" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0504 12:23:38.881329 3659 reflector.go:404] "Listing and watching" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0504 12:23:38.883870 3659 factory.go:1977] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0504 12:23:38.883885 3659 factory.go:1977] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0504 12:23:38.887545 3659 reflector.go:436] "Caches populated" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0504 12:23:38.889525 3659 factory.go:1977] *v1.EgressIP informer cache synced successfully I0504 12:23:38.894888 3659 factory.go:1977] *v1.EgressFirewall informer cache synced successfully I0504 12:23:38.895454 3659 reflector.go:358] "Starting reflector" type="*v1.EgressQoS" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0504 12:23:38.895469 3659 reflector.go:404] "Listing and watching" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0504 12:23:38.898432 3659 reflector.go:436] "Caches populated" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0504 12:23:38.900418 3659 factory.go:1977] *v1.EgressQoS informer cache synced successfully I0504 12:23:38.900472 3659 reflector.go:358] "Starting reflector" type="*v1.EgressService" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0504 12:23:38.900484 3659 reflector.go:404] "Listing and watching" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0504 12:23:38.904010 3659 reflector.go:436] "Caches populated" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0504 12:23:38.906019 3659 factory.go:1977] *v1.EgressService informer cache synced successfully I0504 12:23:38.906090 3659 reflector.go:358] "Starting reflector" type="*v1.AdminPolicyBasedExternalRoute" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0504 12:23:38.906108 3659 reflector.go:404] "Listing and watching" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0504 12:23:38.908708 3659 reflector.go:436] "Caches populated" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0504 12:23:38.911084 3659 factory.go:1977] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0504 12:23:38.911262 3659 reflector.go:358] "Starting reflector" type="*v1.NetworkAttachmentDefinition" resyncPeriod="0s" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0504 12:23:38.911278 3659 reflector.go:404] "Listing and watching" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0504 12:23:38.914093 3659 reflector.go:436] "Caches populated" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0504 12:23:38.916520 3659 factory.go:1977] *v1.NetworkAttachmentDefinition informer cache synced successfully I0504 12:23:38.916578 3659 reflector.go:358] "Starting reflector" type="*v1.ClusterUserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0504 12:23:38.916589 3659 reflector.go:404] "Listing and watching" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0504 12:23:38.916586 3659 reflector.go:358] "Starting reflector" type="*v1.UserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0504 12:23:38.916638 3659 reflector.go:404] "Listing and watching" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0504 12:23:38.920147 3659 reflector.go:436] "Caches populated" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0504 12:23:38.920263 3659 reflector.go:436] "Caches populated" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0504 12:23:38.922213 3659 factory.go:1977] *v1.UserDefinedNetwork informer cache synced successfully I0504 12:23:38.922225 3659 factory.go:1977] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0504 12:23:38.922234 3659 factory.go:680] Watch Factory start up complete, took: 149.732838ms I0504 12:23:38.922244 3659 controller_manager.go:406] Waiting up to 5m0s for a node to have "ip-10-0-2-63.ec2.internal" zone I0504 12:23:38.986893 3659 factory.go:1977] *v1.EgressFirewall informer cache synced successfully I0504 12:23:38.992006 3659 factory.go:1977] *v1.EgressQoS informer cache synced successfully I0504 12:23:38.997154 3659 factory.go:1977] *v1.EgressService informer cache synced successfully I0504 12:23:39.002813 3659 factory.go:1977] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0504 12:23:39.008488 3659 factory.go:1977] *v1.NetworkAttachmentDefinition informer cache synced successfully I0504 12:23:39.013632 3659 factory.go:1977] *v1.UserDefinedNetwork informer cache synced successfully I0504 12:23:39.013640 3659 factory.go:1977] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0504 12:23:39.013647 3659 factory.go:680] Watch Factory start up complete, took: 249.294985ms I0504 12:23:39.013707 3659 default_node_network_controller.go:171] Enable node proxy healthz server on 0.0.0.0:10256 I0504 12:23:39.057469 3659 default_node_network_controller.go:710] Initializing the default node network controller I0504 12:23:39.135705 3659 udn_isolation.go:101] Starting UDN host isolation manager I0504 12:23:39.142238 3659 udn_isolation.go:112] Found kubelet cgroup path: system.slice/kubelet.service I0504 12:23:39.200024 3659 controller.go:133] Adding controller udn-host-isolation-manager event handlers I0504 12:23:39.200172 3659 shared_informer.go:349] "Waiting for caches to sync" controller="udn-host-isolation-manager" I0504 12:23:39.200189 3659 shared_informer.go:356] "Caches are synced" controller="udn-host-isolation-manager" I0504 12:23:39.232732 3659 controller.go:157] Starting controller udn-host-isolation-manager with 1 workers I0504 12:23:39.232789 3659 default_node_network_controller.go:809] Node ip-10-0-2-63.ec2.internal ready for ovn initialization with subnet 10.130.0.0/23 I0504 12:23:39.238325 3659 kube.go:131] Setting annotations map[k8s.ovn.org/node-encap-ips:["10.0.2.63"] k8s.ovn.org/zone-name:ip-10-0-2-63.ec2.internal] on node ip-10-0-2-63.ec2.internal I0504 12:23:39.259825 3659 gateway_init.go:234] Initializing Gateway Functionality for Gateway PreStart I0504 12:23:39.266108 3659 helper_linux.go:93] Provided gateway interface "br-ex", found as index: 5 I0504 12:23:39.266308 3659 helper_linux.go:118] Found default gateway interface br-ex 10.0.2.1 I0504 12:23:39.266480 3659 gateway_init.go:263] Preparing Gateway I0504 12:23:39.266494 3659 gateway_shared_intf.go:1687] Creating new gateway I0504 12:23:39.347684 3659 iptables.go:108] Creating table: filter chain: FORWARD I0504 12:23:39.353905 3659 iptables.go:108] Creating table: filter chain: OUTPUT I0504 12:23:39.359776 3659 gateway_shared_intf.go:1816] Gateway Creation Complete I0504 12:23:39.359790 3659 kube.go:131] Setting annotations map[k8s.ovn.org/gateway-mtu-support: k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-2-63.ec2.internal","mac-address":"0e:85:df:20:9b:f3","ip-addresses":["10.0.2.63/24"],"ip-address":"10.0.2.63/24","next-hops":["10.0.2.1"],"next-hop":"10.0.2.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:3476f505-374e-4d61-b849-02650d2a6378 k8s.ovn.org/node-encap-ips:["10.0.2.63"] k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.2.63/24"} k8s.ovn.org/zone-name:ip-10-0-2-63.ec2.internal] on node ip-10-0-2-63.ec2.internal I0504 12:23:39.379931 3659 default_node_network_controller.go:519] Node connection status = connected I0504 12:23:39.422280 3659 controller_manager.go:429] Waiting for node in zone sync took: 500.021406ms I0504 12:23:39.428510 3659 controller_manager.go:611] Switching to transit router for layer2 networks I0504 12:23:39.428524 3659 kube.go:131] Setting annotations map[k8s.ovn.org/layer2-topology-version:2.0] on node ip-10-0-2-63.ec2.internal I0504 12:23:39.447411 3659 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0504 12:23:39.453834 3659 controller_manager.go:290] SCTP support detected in OVN I0504 12:23:39.454447 3659 services_controller.go:69] Creating services controller for network=default I0504 12:23:39.454553 3659 udn_enabled_svc.go:80] Waiting for service informer to sync I0504 12:23:39.454701 3659 controller.go:133] Adding controller [zone-nad-controller NAD controller] event handlers I0504 12:23:39.454744 3659 shared_informer.go:349] "Waiting for caches to sync" controller="[zone-nad-controller NAD controller]" I0504 12:23:39.454759 3659 shared_informer.go:356] "Caches are synced" controller="[zone-nad-controller NAD controller]" I0504 12:23:39.454771 3659 nad_controller.go:541] [zone-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0504 12:23:39.455000 3659 controller.go:157] Starting controller [zone-nad-controller NAD controller] with 1 workers I0504 12:23:39.455020 3659 udn_enabled_svc.go:95] Performing full resync I0504 12:23:39.455104 3659 network_controller.go:373] [zone-nad-controller network controller]: syncing all networks I0504 12:23:39.455118 3659 nad_controller.go:580] [zone-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 53.461µs I0504 12:23:39.455152 3659 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 35.254µs I0504 12:23:39.455163 3659 network_controller.go:384] [zone-nad-controller network controller]: finished syncing all networks. Time taken: 57.402µs I0504 12:23:39.455171 3659 controller.go:157] Starting controller [zone-nad-controller network controller] with 1 workers I0504 12:23:39.455185 3659 nad_controller.go:379] [zone-nad-controller NAD controller]: started I0504 12:23:39.455200 3659 default_network_controller.go:325] Starting the default network controller I0504 12:23:39.455223 3659 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 8.151µs I0504 12:23:39.455360 3659 udn_enabled_svc.go:99] Waiting for handler to sync I0504 12:23:39.455372 3659 udn_enabled_svc.go:104] Starting worker I0504 12:23:39.464376 3659 default_network_controller.go:395] Cleaning External Gateway ECMP routes I0504 12:23:39.464431 3659 repair.go:33] Syncing exgw routes took 43.403µs I0504 12:23:39.464451 3659 default_network_controller.go:414] Starting all the Watchers... I0504 12:23:39.464581 3659 namespace.go:98] [openshift-package-operator] adding namespace I0504 12:23:39.464589 3659 namespace.go:98] [openshift-infra] adding namespace I0504 12:23:39.464609 3659 namespace.go:98] [openshift-backplane-srep-ro] adding namespace I0504 12:23:39.464608 3659 namespace.go:98] [openshift-user-workload-monitoring] adding namespace I0504 12:23:39.464618 3659 namespace.go:98] [openshift-console-operator] adding namespace I0504 12:23:39.464619 3659 namespace.go:98] [openshift-operators] adding namespace I0504 12:23:39.464634 3659 namespace.go:98] [openshift-controller-manager] adding namespace I0504 12:23:39.464642 3659 namespace.go:98] [openshift-console-user-settings] adding namespace I0504 12:23:39.464645 3659 namespace.go:98] [openshift] adding namespace I0504 12:23:39.464646 3659 namespace.go:98] [openshift-marketplace] adding namespace I0504 12:23:39.464652 3659 namespace.go:98] [openshift-backplane-mobb] adding namespace I0504 12:23:39.464657 3659 namespace.go:98] [openshift-kube-scheduler] adding namespace I0504 12:23:39.464655 3659 namespace.go:98] [openshift-backplane-srep] adding namespace I0504 12:23:39.464723 3659 namespace.go:98] [openshift-ingress] adding namespace I0504 12:23:39.464741 3659 namespace.go:98] [openshift-ingress-canary] adding namespace I0504 12:23:39.465220 3659 namespace.go:102] [openshift-package-operator] adding namespace took 626.155µs I0504 12:23:39.465235 3659 namespace.go:98] [open-cluster-management-2q33rmlhe485oeqr03tcagaphvlpoaiv] adding namespace I0504 12:23:39.465797 3659 namespace.go:102] [openshift-infra] adding namespace took 1.194869ms I0504 12:23:39.465813 3659 namespace.go:98] [openshift-deployment-validation-operator] adding namespace I0504 12:23:39.466405 3659 namespace.go:102] [openshift-deployment-validation-operator] adding namespace took 586.572µs I0504 12:23:39.466419 3659 namespace.go:98] [openshift-dns-operator] adding namespace I0504 12:23:39.466838 3659 namespace.go:102] [openshift-backplane-srep-ro] adding namespace took 2.21521ms I0504 12:23:39.466863 3659 namespace.go:98] [openshift-apiserver] adding namespace I0504 12:23:39.467266 3659 namespace.go:102] [openshift-user-workload-monitoring] adding namespace took 2.641056ms I0504 12:23:39.467278 3659 namespace.go:98] [openshift-backplane-csm] adding namespace I0504 12:23:39.467727 3659 namespace.go:102] [openshift-console-operator] adding namespace took 3.092363ms I0504 12:23:39.467744 3659 namespace.go:98] [openshift-ingress-operator] adding namespace I0504 12:23:39.468283 3659 namespace.go:102] [openshift-operators] adding namespace took 3.645572ms I0504 12:23:39.468294 3659 namespace.go:98] [openshift-cluster-samples-operator] adding namespace I0504 12:23:39.468780 3659 namespace.go:102] [openshift-controller-manager] adding namespace took 4.13609ms I0504 12:23:39.468796 3659 namespace.go:98] [default] adding namespace I0504 12:23:39.469314 3659 namespace.go:102] [openshift-console-user-settings] adding namespace took 4.663872ms I0504 12:23:39.469329 3659 namespace.go:98] [openshift-insights] adding namespace I0504 12:23:39.469810 3659 namespace.go:102] [openshift] adding namespace took 5.154458ms I0504 12:23:39.469828 3659 namespace.go:98] [openshift-config] adding namespace I0504 12:23:39.470322 3659 namespace.go:102] [openshift-backplane-mobb] adding namespace took 5.660084ms I0504 12:23:39.470335 3659 namespace.go:98] [openshift-etcd] adding namespace I0504 12:23:39.470799 3659 namespace.go:102] [openshift-marketplace] adding namespace took 6.140045ms I0504 12:23:39.470830 3659 namespace.go:98] [openshift-controller-manager-operator] adding namespace I0504 12:23:39.471279 3659 namespace.go:102] [openshift-kube-scheduler] adding namespace took 6.61589ms I0504 12:23:39.471291 3659 namespace.go:98] [openshift-backplane-mcs-tier-two] adding namespace I0504 12:23:39.471769 3659 namespace.go:102] [openshift-backplane-srep] adding namespace took 7.085621ms I0504 12:23:39.471784 3659 namespace.go:98] [openshift-apiserver-operator] adding namespace I0504 12:23:39.472349 3659 namespace.go:102] [openshift-ingress] adding namespace took 7.590236ms I0504 12:23:39.472361 3659 namespace.go:98] [kube-public] adding namespace I0504 12:23:39.472862 3659 namespace.go:102] [openshift-ingress-canary] adding namespace took 8.112011ms I0504 12:23:39.472871 3659 namespace.go:98] [openshift-ovn-kubernetes] adding namespace I0504 12:23:39.473308 3659 namespace.go:102] [open-cluster-management-2q33rmlhe485oeqr03tcagaphvlpoaiv] adding namespace took 8.06711ms I0504 12:23:39.473323 3659 namespace.go:98] [openshift-backplane-managed-scripts] adding namespace I0504 12:23:39.474059 3659 namespace.go:102] [openshift-dns-operator] adding namespace took 7.634197ms I0504 12:23:39.474073 3659 namespace.go:98] [openshift-route-controller-manager] adding namespace I0504 12:23:39.474535 3659 namespace.go:102] [openshift-apiserver] adding namespace took 7.665733ms I0504 12:23:39.474550 3659 namespace.go:98] [openshift-cloud-credential-operator] adding namespace I0504 12:23:39.474940 3659 namespace.go:102] [openshift-backplane-csm] adding namespace took 7.655652ms I0504 12:23:39.474952 3659 namespace.go:98] [openshift-backplane] adding namespace I0504 12:23:39.475323 3659 namespace.go:102] [openshift-ingress-operator] adding namespace took 7.574093ms I0504 12:23:39.475334 3659 namespace.go:98] [openshift-logging] adding namespace I0504 12:23:39.475861 3659 namespace.go:102] [openshift-cluster-samples-operator] adding namespace took 7.561796ms I0504 12:23:39.475875 3659 namespace.go:98] [openshift-kube-controller-manager] adding namespace I0504 12:23:39.476327 3659 namespace.go:102] [default] adding namespace took 7.522206ms I0504 12:23:39.476339 3659 namespace.go:98] [openshift-kube-scheduler-operator] adding namespace I0504 12:23:39.476754 3659 namespace.go:102] [openshift-insights] adding namespace took 7.419588ms I0504 12:23:39.476769 3659 namespace.go:98] [openshift-cloud-network-config-controller] adding namespace I0504 12:23:39.477233 3659 namespace.go:102] [openshift-config] adding namespace took 7.399381ms I0504 12:23:39.477252 3659 namespace.go:98] [openshift-config-managed] adding namespace I0504 12:23:39.477598 3659 namespace.go:102] [openshift-etcd] adding namespace took 7.256214ms I0504 12:23:39.477617 3659 namespace.go:98] [openshift-machine-config-operator] adding namespace I0504 12:23:39.478099 3659 namespace.go:102] [openshift-controller-manager-operator] adding namespace took 7.260139ms I0504 12:23:39.478116 3659 namespace.go:98] [openshift-host-network] adding namespace I0504 12:23:39.478612 3659 namespace.go:102] [openshift-backplane-mcs-tier-two] adding namespace took 7.315808ms I0504 12:23:39.478625 3659 namespace.go:98] [openshift-backplane-lpsre] adding namespace I0504 12:23:39.479127 3659 namespace.go:102] [openshift-apiserver-operator] adding namespace took 7.3368ms I0504 12:23:39.479143 3659 namespace.go:98] [openshift-cluster-version] adding namespace I0504 12:23:39.479624 3659 namespace.go:102] [kube-public] adding namespace took 7.255011ms I0504 12:23:39.479642 3659 namespace.go:98] [openshift-network-console] adding namespace I0504 12:23:39.480070 3659 namespace.go:102] [openshift-ovn-kubernetes] adding namespace took 7.19375ms I0504 12:23:39.480084 3659 namespace.go:98] [openshift-console] adding namespace I0504 12:23:39.480506 3659 namespace.go:102] [openshift-backplane-managed-scripts] adding namespace took 7.177172ms I0504 12:23:39.480519 3659 namespace.go:98] [openshift-image-registry] adding namespace I0504 12:23:39.480903 3659 namespace.go:102] [openshift-route-controller-manager] adding namespace took 6.825662ms I0504 12:23:39.480915 3659 namespace.go:98] [openshift-cluster-machine-approver] adding namespace I0504 12:23:39.481397 3659 namespace.go:102] [openshift-cloud-credential-operator] adding namespace took 6.84058ms I0504 12:23:39.481426 3659 namespace.go:98] [openshift-network-operator] adding namespace I0504 12:23:39.481785 3659 namespace.go:102] [openshift-backplane] adding namespace took 6.827763ms I0504 12:23:39.481799 3659 namespace.go:98] [openshift-must-gather-operator] adding namespace I0504 12:23:39.482174 3659 namespace.go:102] [openshift-logging] adding namespace took 6.835012ms I0504 12:23:39.482185 3659 namespace.go:98] [openshift-kube-apiserver] adding namespace I0504 12:23:39.482537 3659 namespace.go:102] [openshift-kube-controller-manager] adding namespace took 6.657041ms I0504 12:23:39.482547 3659 namespace.go:98] [openshift-backplane-tam] adding namespace I0504 12:23:39.482966 3659 namespace.go:102] [openshift-kube-scheduler-operator] adding namespace took 6.621571ms I0504 12:23:39.482979 3659 namespace.go:98] [openshift-monitoring] adding namespace I0504 12:23:39.483345 3659 namespace.go:102] [openshift-cloud-network-config-controller] adding namespace took 6.572182ms I0504 12:23:39.483356 3659 namespace.go:98] [openshift-authentication] adding namespace I0504 12:23:39.483745 3659 namespace.go:102] [openshift-config-managed] adding namespace took 6.487527ms I0504 12:23:39.483763 3659 namespace.go:98] [dedicated-admin] adding namespace I0504 12:23:39.484152 3659 namespace.go:102] [openshift-machine-config-operator] adding namespace took 6.530573ms I0504 12:23:39.484165 3659 namespace.go:98] [openshift-cluster-storage-operator] adding namespace I0504 12:23:39.484589 3659 namespace.go:102] [openshift-host-network] adding namespace took 6.468537ms I0504 12:23:39.484602 3659 namespace.go:98] [openshift-node] adding namespace I0504 12:23:39.484941 3659 namespace.go:102] [openshift-backplane-lpsre] adding namespace took 6.312121ms I0504 12:23:39.484951 3659 namespace.go:98] [kube-node-lease] adding namespace I0504 12:23:39.485451 3659 namespace.go:102] [openshift-cluster-version] adding namespace took 6.304104ms I0504 12:23:39.485461 3659 namespace.go:98] [openshift-cluster-csi-drivers] adding namespace I0504 12:23:39.485860 3659 namespace.go:102] [openshift-network-console] adding namespace took 6.212787ms I0504 12:23:39.485875 3659 namespace.go:98] [openshift-backplane-cse] adding namespace I0504 12:23:39.486248 3659 namespace.go:102] [openshift-console] adding namespace took 6.158437ms I0504 12:23:39.486265 3659 namespace.go:98] [openshift-kube-apiserver-operator] adding namespace I0504 12:23:39.486660 3659 namespace.go:102] [openshift-image-registry] adding namespace took 6.136541ms I0504 12:23:39.486671 3659 namespace.go:98] [openshift-multus] adding namespace I0504 12:23:39.487072 3659 namespace.go:102] [openshift-cluster-machine-approver] adding namespace took 6.152965ms I0504 12:23:39.487423 3659 namespace.go:102] [openshift-network-operator] adding namespace took 5.990287ms I0504 12:23:39.487433 3659 namespace.go:98] [openshift-service-ca] adding namespace I0504 12:23:39.487883 3659 namespace.go:102] [openshift-must-gather-operator] adding namespace took 6.078764ms I0504 12:23:39.487896 3659 namespace.go:98] [openshift-dns] adding namespace I0504 12:23:39.488246 3659 namespace.go:102] [openshift-kube-apiserver] adding namespace took 6.057264ms I0504 12:23:39.488256 3659 namespace.go:98] [openshift-service-ca-operator] adding namespace I0504 12:23:39.488852 3659 namespace.go:102] [openshift-backplane-tam] adding namespace took 6.299315ms I0504 12:23:39.488869 3659 namespace.go:98] [openshift-config-operator] adding namespace I0504 12:23:39.489399 3659 namespace.go:102] [openshift-monitoring] adding namespace took 6.41406ms I0504 12:23:39.489413 3659 namespace.go:98] [openshift-network-node-identity] adding namespace I0504 12:23:39.489765 3659 namespace.go:102] [openshift-authentication] adding namespace took 6.40371ms I0504 12:23:39.489774 3659 namespace.go:98] [openshift-operators-redhat] adding namespace I0504 12:23:39.490319 3659 namespace.go:102] [dedicated-admin] adding namespace took 6.549827ms I0504 12:23:39.490333 3659 namespace.go:98] [openshift-authentication-operator] adding namespace I0504 12:23:39.490765 3659 namespace.go:102] [openshift-cluster-storage-operator] adding namespace took 6.5937ms I0504 12:23:39.490790 3659 namespace.go:98] [openshift-cloud-controller-manager] adding namespace I0504 12:23:39.491240 3659 namespace.go:102] [openshift-node] adding namespace took 6.632911ms I0504 12:23:39.491254 3659 namespace.go:98] [openshift-kube-storage-version-migrator-operator] adding namespace I0504 12:23:39.491670 3659 namespace.go:102] [kube-node-lease] adding namespace took 6.714387ms I0504 12:23:39.491682 3659 namespace.go:98] [kube-system] adding namespace I0504 12:23:39.492133 3659 namespace.go:102] [openshift-cluster-csi-drivers] adding namespace took 6.666905ms I0504 12:23:39.492147 3659 namespace.go:98] [openshift-operator-lifecycle-manager] adding namespace I0504 12:23:39.492556 3659 namespace.go:102] [openshift-backplane-cse] adding namespace took 6.673391ms I0504 12:23:39.492576 3659 namespace.go:98] [openshift-network-diagnostics] adding namespace W0504 12:23:39.492980 3659 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-multus/network-metrics-daemon-528nq: pod openshift-multus/network-metrics-daemon-528nq: no pod IPs found I0504 12:23:39.493043 3659 namespace.go:102] [openshift-kube-apiserver-operator] adding namespace took 6.758478ms I0504 12:23:39.493058 3659 namespace.go:98] [open-cluster-management-agent-addon] adding namespace I0504 12:23:39.493484 3659 namespace.go:102] [openshift-multus] adding namespace took 6.806505ms I0504 12:23:39.493499 3659 namespace.go:98] [openshift-cluster-node-tuning-operator] adding namespace I0504 12:23:39.493956 3659 namespace.go:102] [openshift-service-ca] adding namespace took 6.520022ms I0504 12:23:39.494490 3659 namespace.go:102] [openshift-dns] adding namespace took 6.588039ms I0504 12:23:39.495057 3659 namespace.go:102] [openshift-service-ca-operator] adding namespace took 6.796171ms I0504 12:23:39.495073 3659 namespace.go:98] [openshift-backplane-cee] adding namespace I0504 12:23:39.495545 3659 namespace.go:102] [openshift-config-operator] adding namespace took 6.669303ms I0504 12:23:39.495964 3659 namespace.go:102] [openshift-network-node-identity] adding namespace took 6.545997ms I0504 12:23:39.496388 3659 namespace.go:102] [openshift-operators-redhat] adding namespace took 6.607615ms I0504 12:23:39.496406 3659 namespace.go:98] [openshift-machine-api] adding namespace I0504 12:23:39.496841 3659 namespace.go:102] [openshift-authentication-operator] adding namespace took 6.503597ms I0504 12:23:39.497286 3659 namespace.go:102] [openshift-cloud-controller-manager] adding namespace took 6.488512ms I0504 12:23:39.497706 3659 namespace.go:102] [openshift-kube-storage-version-migrator-operator] adding namespace took 6.448598ms I0504 12:23:39.498544 3659 namespace.go:102] [kube-system] adding namespace took 6.855936ms I0504 12:23:39.498558 3659 namespace.go:98] [openshift-kube-controller-manager-operator] adding namespace W0504 12:23:39.498936 3659 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-jl9gk: pod openshift-network-diagnostics/network-check-target-jl9gk: no pod IPs found I0504 12:23:39.498976 3659 namespace.go:102] [openshift-operator-lifecycle-manager] adding namespace took 6.822547ms I0504 12:23:39.499546 3659 namespace.go:102] [openshift-network-diagnostics] adding namespace took 6.964409ms I0504 12:23:39.499988 3659 namespace.go:102] [open-cluster-management-agent-addon] adding namespace took 6.923103ms I0504 12:23:39.500001 3659 namespace.go:98] [openshift-customer-monitoring] adding namespace I0504 12:23:39.500456 3659 namespace.go:102] [openshift-cluster-node-tuning-operator] adding namespace took 6.951458ms I0504 12:23:39.500842 3659 namespace.go:102] [openshift-backplane-cee] adding namespace took 5.764953ms I0504 12:23:39.501224 3659 namespace.go:102] [openshift-machine-api] adding namespace took 4.812499ms I0504 12:23:39.501725 3659 namespace.go:102] [openshift-kube-controller-manager-operator] adding namespace took 3.16308ms I0504 12:23:39.502085 3659 namespace.go:102] [openshift-customer-monitoring] adding namespace took 2.078963ms I0504 12:23:39.502291 3659 master.go:548] Adding or Updating local node "ip-10-0-2-63.ec2.internal" for network "default" I0504 12:23:39.503279 3659 zone_ic_handler.go:195] Time taken to create transit switch: 325.714µs I0504 12:23:39.503292 3659 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-0-182.ec2.internal for the network default E0504 12:23:39.503361 3659 obj_retry.go:575] Failed to create *v1.Node ip-10-0-1-18.ec2.internal, error: adding or updating remote node IC resources ip-10-0-1-18.ec2.internal failed, err - ensuring transit switch for remote zone node ip-10-0-1-18.ec2.internal for the network default failed : err - failed to create/update transit switch transit_switch: error in transact with ops [{Op:wait Table:Logical_Switch Row:map[] Rows:[map[name:transit_switch]] Columns:[name] Mutations:[] Timeout:0xc009b9f1b8 Where:[where column name == transit_switch] Until:!= Durable: Comment: Lock: UUID: UUIDName:} {Op:insert Table:Logical_Switch Row:map[name:transit_switch other_config:{GoMap:map[interconn-ts:transit_switch mcast_flood_unregistered:true mcast_querier:false mcast_snoop:true requested-tnl-key:16711683]}] Rows:[] Columns:[] Mutations:[] Timeout: Where:[] Until: Durable: Comment: Lock: UUID: UUIDName:u1849753249}] results [{Count:0 Error:timed out Details:"where" clause test failed UUID:{GoUUID:} Rows:[]} {Count:0 Error: Details: UUID:{GoUUID:} Rows:[]}] and errors [timed out: "where" clause test failed]: 1 ovsdb operations failed I0504 12:23:39.503453 3659 event.go:377] Event(v1.ObjectReference{Kind:"Node", Namespace:"", Name:"ip-10-0-1-18.ec2.internal", UID:"61096e6b-95ce-4d36-b7a6-854fc719f20b", APIVersion:"v1", ResourceVersion:"9145", FieldPath:""}): type: 'Warning' reason: 'ErrorAddingResource' adding or updating remote node IC resources ip-10-0-1-18.ec2.internal failed, err - ensuring transit switch for remote zone node ip-10-0-1-18.ec2.internal for the network default failed : err - failed to create/update transit switch transit_switch: error in transact with ops [{Op:wait Table:Logical_Switch Row:map[] Rows:[map[name:transit_switch]] Columns:[name] Mutations:[] Timeout:0xc009b9f1b8 Where:[where column name == transit_switch] Until:!= Durable: Comment: Lock: UUID: UUIDName:} {Op:insert Table:Logical_Switch Row:map[name:transit_switch other_config:{GoMap:map[interconn-ts:transit_switch mcast_flood_unregistered:true mcast_querier:false mcast_snoop:true requested-tnl-key:16711683]}] Rows:[] Columns:[] Mutations:[] Timeout: Where:[] Until: Durable: Comment: Lock: UUID: UUIDName:u1849753249}] results [{Count:0 Error:timed out Details:"where" clause test failed UUID:{GoUUID:} Rows:[]} {Count:0 Error: Details: UUID:{GoUUID:} Rows:[]}] and errors [timed out: "where" clause test failed]: 1 ovsdb operations failed I0504 12:23:39.504412 3659 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-0-182.ec2.internal" on network "default" took: 1.471219ms W0504 12:23:39.510786 3659 gateway.go:207] Unable to fetch podIPs for pod openshift-multus/network-metrics-daemon-528nq: pod openshift-multus/network-metrics-daemon-528nq: no pod IPs found W0504 12:23:39.510809 3659 gateway.go:207] Unable to fetch podIPs for pod openshift-network-diagnostics/network-check-target-jl9gk: pod openshift-network-diagnostics/network-check-target-jl9gk: no pod IPs found I0504 12:23:39.511712 3659 zone_ic_handler.go:203] Creating interconnect resources for local zone node ip-10-0-2-63.ec2.internal for the network default I0504 12:23:39.511926 3659 zone_ic_handler.go:195] Time taken to create transit switch: 202.785µs I0504 12:23:39.513170 3659 services_controller.go:192] Starting controller ovn-lb-controller for network=default I0504 12:23:39.513199 3659 services_controller.go:200] Waiting for node tracker handler to sync for network=default I0504 12:23:39.513212 3659 shared_informer.go:349] "Waiting for caches to sync" controller="node-tracker-controller" I0504 12:23:39.513237 3659 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-182.ec2.internal in network "default" I0504 12:23:39.513282 3659 node_tracker.go:169] Node ip-10-0-0-182.ec2.internal switch + router changed, syncing services in network "default" I0504 12:23:39.513297 3659 services_controller.go:574] Full service sync requested for network=default I0504 12:23:39.513305 3659 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-18.ec2.internal in network "default" I0504 12:23:39.513325 3659 node_tracker.go:169] Node ip-10-0-1-18.ec2.internal switch + router changed, syncing services in network "default" I0504 12:23:39.513332 3659 services_controller.go:574] Full service sync requested for network=default I0504 12:23:39.513337 3659 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-63.ec2.internal in network "default" I0504 12:23:39.513355 3659 node_tracker.go:169] Node ip-10-0-2-63.ec2.internal switch + router changed, syncing services in network "default" I0504 12:23:39.513361 3659 services_controller.go:574] Full service sync requested for network=default I0504 12:23:39.613860 3659 shared_informer.go:356] "Caches are synced" controller="node-tracker-controller" I0504 12:23:39.613878 3659 services_controller.go:208] Setting up event handlers for services for network=default I0504 12:23:39.614022 3659 services_controller.go:218] Setting up event handlers for endpoint slices for network=default I0504 12:23:39.614096 3659 services_controller.go:233] Waiting for service and endpoint handlers to sync for network=default I0504 12:23:39.614111 3659 shared_informer.go:349] "Waiting for caches to sync" controller="ovn-lb-controller" I0504 12:23:39.714332 3659 shared_informer.go:356] "Caches are synced" controller="ovn-lb-controller" I0504 12:23:39.714347 3659 repair.go:60] Starting repairing loop for services I0504 12:23:39.714419 3659 repair.go:131] Deleted 0 stale service LBs I0504 12:23:39.714436 3659 repair.go:137] Deleted 0 stale Chassis Template Vars I0504 12:23:39.714451 3659 repair.go:62] Finished repairing loop for services: 105.873µs I0504 12:23:39.714475 3659 services_controller.go:363] Controller cache of 0 load balancers initialized for 0 services for network=default I0504 12:23:39.714486 3659 services_controller.go:254] Starting workers for network=default W0504 12:23:39.714671 3659 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0504 12:23:39.714852 3659 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-jl9gk] creating logical port openshift-network-diagnostics_network-check-target-jl9gk for pod on switch ip-10-0-2-63.ec2.internal I0504 12:23:39.714950 3659 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-528nq] creating logical port openshift-multus_network-metrics-daemon-528nq for pod on switch ip-10-0-2-63.ec2.internal I0504 12:23:39.715068 3659 kube.go:256] Updating pod openshift-multus/network-metrics-daemon-528nq I0504 12:23:39.715098 3659 kube.go:256] Updating pod openshift-network-diagnostics/network-check-target-jl9gk W0504 12:23:39.715999 3659 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics W0504 12:23:39.716080 3659 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0504 12:23:39.735666 3659 pod.go:62] [openshift-multus/network-metrics-daemon-528nq] pod update took 20.664248ms I0504 12:23:39.735686 3659 base_network_controller_pods.go:951] [default/openshift-multus/network-metrics-daemon-528nq] addLogicalPort annotation time took 20.686144ms I0504 12:23:39.736569 3659 pods.go:271] [openshift-multus/network-metrics-daemon-528nq] addLogicalPort took 21.626073ms, libovsdb time 640.73µs I0504 12:23:39.736844 3659 pod.go:62] [openshift-network-diagnostics/network-check-target-jl9gk] pod update took 21.844586ms I0504 12:23:39.736868 3659 base_network_controller_pods.go:951] [default/openshift-network-diagnostics/network-check-target-jl9gk] addLogicalPort annotation time took 21.884861ms I0504 12:23:39.737787 3659 pods.go:271] [openshift-network-diagnostics/network-check-target-jl9gk] addLogicalPort took 22.949165ms, libovsdb time 606.515µs I0504 12:23:39.738363 3659 admin_network_policy_controller.go:227] Starting controller default-network-controller I0504 12:23:39.738381 3659 shared_informer.go:349] "Waiting for caches to sync" controller="default-network-controller" I0504 12:23:39.738392 3659 shared_informer.go:356] "Caches are synced" controller="default-network-controller" I0504 12:23:39.738396 3659 admin_network_policy_controller.go:237] Repairing Admin Network Policies I0504 12:23:39.738572 3659 repair.go:29] Repairing admin network policies took 170.729µs I0504 12:23:39.738668 3659 repair.go:92] Repairing baseline admin network policies took 89.086µs I0504 12:23:39.739064 3659 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-egress-to-api-server for network default I0504 12:23:39.739099 3659 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-egress-to-api-server added to peer address sets [] I0504 12:23:39.740680 3659 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-egress-to-api-server resources completed, update namespace loglevel I0504 12:23:39.740699 3659 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics for network default I0504 12:23:39.740708 3659 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics added to peer address sets [] I0504 12:23:39.742321 3659 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics resources completed, update namespace loglevel I0504 12:23:39.742338 3659 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/default-deny for network default I0504 12:23:39.742344 3659 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/default-deny added to peer address sets [] I0504 12:23:39.742688 3659 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/default-deny resources completed, update namespace loglevel I0504 12:23:39.742706 3659 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/package-server-manager for network default I0504 12:23:39.742720 3659 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/package-server-manager added to peer address sets [] I0504 12:23:39.745011 3659 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/package-server-manager resources completed, update namespace loglevel I0504 12:23:39.745049 3659 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server for network default I0504 12:23:39.745059 3659 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server added to peer address sets [] I0504 12:23:39.746680 3659 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server resources completed, update namespace loglevel I0504 12:23:39.746704 3659 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/default-deny-all-traffic for network default I0504 12:23:39.746716 3659 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/default-deny-all-traffic added to peer address sets [] I0504 12:23:39.747008 3659 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/default-deny-all-traffic resources completed, update namespace loglevel I0504 12:23:39.747024 3659 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/default-deny for network default I0504 12:23:39.747044 3659 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/default-deny added to peer address sets [] I0504 12:23:39.748400 3659 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/default-deny resources completed, update namespace loglevel I0504 12:23:39.748419 3659 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles for network default I0504 12:23:39.748430 3659 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles added to peer address sets [] I0504 12:23:39.749339 3659 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles resources completed, update namespace loglevel I0504 12:23:39.749363 3659 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/unpack-bundles for network default I0504 12:23:39.749375 3659 base_network_controller_policy.go:1054] Policy openshift-marketplace/unpack-bundles added to peer address sets [] I0504 12:23:39.751383 3659 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/unpack-bundles resources completed, update namespace loglevel I0504 12:23:39.751407 3659 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-pprof for network default I0504 12:23:39.751417 3659 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-pprof added to peer address sets [] I0504 12:23:39.751951 3659 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-pprof resources completed, update namespace loglevel I0504 12:23:39.751967 3659 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-webhook for network default I0504 12:23:39.751974 3659 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-webhook added to peer address sets [] I0504 12:23:39.752483 3659 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-webhook resources completed, update namespace loglevel I0504 12:23:39.752502 3659 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-to-dns for network default I0504 12:23:39.753302 3659 pod_selector_address_set.go:213] Created shared address set for pod selector LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},} I0504 12:23:39.753316 3659 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0504 12:23:39.754121 3659 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-to-dns resources completed, update namespace loglevel I0504 12:23:39.754139 3659 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-all-egress for network default I0504 12:23:39.754147 3659 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-all-egress added to peer address sets [] I0504 12:23:39.754624 3659 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-all-egress resources completed, update namespace loglevel I0504 12:23:39.754641 3659 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/default-deny-all for network default I0504 12:23:39.754648 3659 base_network_controller_policy.go:1054] Policy openshift-marketplace/default-deny-all added to peer address sets [] I0504 12:23:39.754918 3659 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/default-deny-all resources completed, update namespace loglevel I0504 12:23:39.754934 3659 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/marketplace-operator for network default I0504 12:23:39.754947 3659 base_network_controller_policy.go:1054] Policy openshift-marketplace/marketplace-operator added to peer address sets [] I0504 12:23:39.755986 3659 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/marketplace-operator resources completed, update namespace loglevel I0504 12:23:39.756016 3659 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-metrics for network default I0504 12:23:39.756039 3659 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-metrics added to peer address sets [] I0504 12:23:39.756590 3659 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-metrics resources completed, update namespace loglevel I0504 12:23:39.756608 3659 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-to-dns for network default I0504 12:23:39.756628 3659 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0504 12:23:39.757383 3659 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-to-dns resources completed, update namespace loglevel I0504 12:23:39.757400 3659 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-insights for network default I0504 12:23:39.757408 3659 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-insights added to peer address sets [] I0504 12:23:39.758474 3659 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-insights resources completed, update namespace loglevel I0504 12:23:39.758489 3659 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/catalog-operator for network default I0504 12:23:39.758500 3659 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/catalog-operator added to peer address sets [] I0504 12:23:39.759763 3659 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/catalog-operator resources completed, update namespace loglevel I0504 12:23:39.759788 3659 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/olm-operator for network default I0504 12:23:39.759808 3659 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/olm-operator added to peer address sets [] I0504 12:23:39.760883 3659 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/olm-operator resources completed, update namespace loglevel I0504 12:23:39.760899 3659 base_network_controller_policy.go:1179] Adding network policy openshift-operators/default-allow-all for network default I0504 12:23:39.760913 3659 base_network_controller_policy.go:1054] Policy openshift-operators/default-allow-all added to peer address sets [] I0504 12:23:39.762586 3659 base_network_controller_policy.go:1232] Create network policy openshift-operators/default-allow-all resources completed, update namespace loglevel I0504 12:23:39.762602 3659 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-all-egress for network default I0504 12:23:39.762608 3659 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-all-egress added to peer address sets [] I0504 12:23:39.763146 3659 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-all-egress resources completed, update namespace loglevel I0504 12:23:39.763164 3659 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-olm for network default I0504 12:23:39.763171 3659 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-olm added to peer address sets [] I0504 12:23:39.763694 3659 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-olm resources completed, update namespace loglevel I0504 12:23:39.763708 3659 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-egress-to-api-server for network default I0504 12:23:39.763715 3659 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-egress-to-api-server added to peer address sets [] I0504 12:23:39.764220 3659 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-egress-to-api-server resources completed, update namespace loglevel I0504 12:23:39.764233 3659 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-version/default-deny for network default I0504 12:23:39.764239 3659 base_network_controller_policy.go:1054] Policy openshift-cluster-version/default-deny added to peer address sets [] I0504 12:23:39.765621 3659 base_network_controller_policy.go:1232] Create network policy openshift-cluster-version/default-deny resources completed, update namespace loglevel I0504 12:23:39.765642 3659 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/packageserver for network default I0504 12:23:39.765652 3659 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/packageserver added to peer address sets [] I0504 12:23:39.767185 3659 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/packageserver resources completed, update namespace loglevel I0504 12:23:39.767202 3659 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-egress for network default I0504 12:23:39.767210 3659 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-egress added to peer address sets [] I0504 12:23:39.767848 3659 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-egress resources completed, update namespace loglevel I0504 12:23:39.767867 3659 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range for network default I0504 12:23:39.767878 3659 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range added to peer address sets [] I0504 12:23:39.768334 3659 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range resources completed, update namespace loglevel I0504 12:23:39.768351 3659 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range for network default I0504 12:23:39.768361 3659 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range added to peer address sets [] I0504 12:23:39.768832 3659 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range resources completed, update namespace loglevel I0504 12:23:39.774368 3659 egressfirewall.go:374] Starting EgressFirewall controller I0504 12:23:39.774396 3659 controller.go:133] Adding controller egress-firewall-controller-node event handlers I0504 12:23:39.774399 3659 controller.go:133] Adding controller egress-firewall-controller event handlers I0504 12:23:39.774421 3659 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller" I0504 12:23:39.774426 3659 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller-node" I0504 12:23:39.774433 3659 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller" I0504 12:23:39.774436 3659 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller-node" I0504 12:23:39.774696 3659 controller.go:157] Starting controller egress-firewall-controller with 1 workers I0504 12:23:39.774709 3659 controller.go:157] Starting controller egress-firewall-controller-node with 1 workers I0504 12:23:39.774714 3659 controller.go:157] Starting controller egress-firewall-controller-NAD with 1 workers I0504 12:23:39.774724 3659 egressqos.go:193] Setting up event handlers for EgressQoS I0504 12:23:39.774740 3659 egressfirewall.go:723] Syncing node "ip-10-0-2-63.ec2.internal" for egress firewall I0504 12:23:39.774756 3659 egressfirewall.go:723] Syncing node "ip-10-0-0-182.ec2.internal" for egress firewall I0504 12:23:39.774762 3659 egressfirewall.go:723] Syncing node "ip-10-0-1-18.ec2.internal" for egress firewall I0504 12:23:39.774832 3659 egressqos.go:245] Starting EgressQoS Controller I0504 12:23:39.774844 3659 shared_informer.go:349] "Waiting for caches to sync" controller="egressqosnodes" I0504 12:23:39.774851 3659 shared_informer.go:356] "Caches are synced" controller="egressqosnodes" I0504 12:23:39.774857 3659 shared_informer.go:349] "Waiting for caches to sync" controller="egressqospods" I0504 12:23:39.774861 3659 shared_informer.go:356] "Caches are synced" controller="egressqospods" I0504 12:23:39.774866 3659 shared_informer.go:349] "Waiting for caches to sync" controller="egressqos" I0504 12:23:39.774870 3659 shared_informer.go:356] "Caches are synced" controller="egressqos" I0504 12:23:39.774874 3659 egressqos.go:259] Repairing EgressQoSes I0504 12:23:39.774878 3659 egressqos.go:399] Starting repairing loop for egressqos I0504 12:23:39.774949 3659 egressqos.go:401] Finished repairing loop for egressqos: 70.997µs I0504 12:23:39.774962 3659 egressservice_zone.go:132] Setting up event handlers for Egress Services I0504 12:23:39.775001 3659 egressqos.go:1007] Processing sync for EgressQoS node ip-10-0-2-63.ec2.internal I0504 12:23:39.775093 3659 egressqos.go:1010] Finished syncing EgressQoS node ip-10-0-2-63.ec2.internal : 93.371µs I0504 12:23:39.775158 3659 egressservice_zone.go:210] Starting Egress Services Controller I0504 12:23:39.775172 3659 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0504 12:23:39.775181 3659 shared_informer.go:356] "Caches are synced" controller="egressservices" I0504 12:23:39.775188 3659 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0504 12:23:39.775193 3659 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0504 12:23:39.775198 3659 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0504 12:23:39.775202 3659 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0504 12:23:39.775207 3659 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_nodes" I0504 12:23:39.775212 3659 shared_informer.go:356] "Caches are synced" controller="egressservices_nodes" I0504 12:23:39.775215 3659 egressservice_zone.go:228] Repairing Egress Services I0504 12:23:39.775469 3659 master_controller.go:88] Starting Admin Policy Based Route Controller I0504 12:23:39.775477 3659 external_controller.go:278] Starting Admin Policy Based Route Controller I0504 12:23:39.775546 3659 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-63.ec2.internal I0504 12:23:39.775557 3659 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-63.ec2.internal: 14.074µs I0504 12:23:39.775563 3659 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-182.ec2.internal I0504 12:23:39.775573 3659 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-182.ec2.internal: 10.134µs I0504 12:23:39.775578 3659 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-18.ec2.internal I0504 12:23:39.775583 3659 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-18.ec2.internal: 5.616µs I0504 12:23:39.775738 3659 default_network_controller.go:585] Completing all the Watchers took 311.27545ms I0504 12:23:39.775758 3659 default_network_controller.go:589] Starting unidling controllers I0504 12:23:39.775773 3659 unidle.go:46] Registering OVN SB ControllerEvent handler I0504 12:23:39.775784 3659 unidle.go:63] Populating Initial ContollerEvent events I0504 12:23:39.775823 3659 unidle.go:79] Setting up event handlers for services I0504 12:23:39.775879 3659 ovnkube.go:545] Waiting for OVN northbound database changes to be processed by ovn-controller I0504 12:23:39.880929 3659 default_node_network_controller.go:519] Node connection status = connected I0504 12:23:40.164265 3659 namespace.go:98] [openshift-kube-storage-version-migrator] adding namespace I0504 12:23:40.165016 3659 namespace.go:102] [openshift-kube-storage-version-migrator] adding namespace took 731.572µs I0504 12:23:40.191006 3659 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0504 12:23:40.381455 3659 default_node_network_controller.go:519] Node connection status = connected I0504 12:23:40.836460 3659 ovnkube.go:550] Finished waiting for OVN northbound database changes to be processed by ovn-controller I0504 12:23:40.880790 3659 default_node_network_controller.go:519] Node connection status = connected I0504 12:23:40.897164 3659 bridgeconfig.go:537] Gateway is ready I0504 12:23:40.897179 3659 gateway_shared_intf.go:1733] Creating Gateway Openflow Manager I0504 12:23:40.908004 3659 node_ip_handler_linux.go:247] Node primary address changed to 10.0.2.63. Updating OVN encap IP. I0504 12:23:40.913492 3659 node_ip_handler_linux.go:542] Will not update encap IP 10.0.2.63 - it is already configured I0504 12:23:40.913505 3659 node_ip_handler_linux.go:485] Node address changed to map[10.0.2.63/24:{} 172.20.0.1/32:{}]. Updating annotations. I0504 12:23:40.913721 3659 kube.go:131] Setting annotations map[k8s.ovn.org/host-cidrs:["10.0.2.63/24","172.20.0.1/32"] k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-2-63.ec2.internal","mac-address":"0e:85:df:20:9b:f3","ip-addresses":["10.0.2.63/24"],"ip-address":"10.0.2.63/24","next-hops":["10.0.2.1"],"next-hop":"10.0.2.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:3476f505-374e-4d61-b849-02650d2a6378 k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.2.63/24"}] on node ip-10-0-2-63.ec2.internal I0504 12:23:40.930340 3659 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-63.ec2.internal I0504 12:23:40.930361 3659 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-63.ec2.internal: 24.965µs I0504 12:23:40.930431 3659 master.go:548] Adding or Updating local node "ip-10-0-2-63.ec2.internal" for network "default" I0504 12:23:40.930439 3659 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-63.ec2.internal in network "default" I0504 12:23:40.930460 3659 egressfirewall.go:723] Syncing node "ip-10-0-2-63.ec2.internal" for egress firewall I0504 12:23:40.930483 3659 node_tracker.go:169] Node ip-10-0-2-63.ec2.internal switch + router changed, syncing services in network "default" I0504 12:23:40.930500 3659 services_controller.go:574] Full service sync requested for network=default W0504 12:23:40.931147 3659 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog W0504 12:23:40.931207 3659 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics W0504 12:23:40.931215 3659 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0504 12:23:40.931412 3659 gateway_shared_intf.go:2023] Setting OVN Masquerade route with source: 10.0.2.63 I0504 12:23:40.931555 3659 kube.go:131] Setting annotations map[k8s.ovn.org/node-masquerade-subnet:{"ipv4":"169.254.0.0/17","ipv6":"fd69::/112"}] on node ip-10-0-2-63.ec2.internal I0504 12:23:40.934916 3659 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-2-63.ec2.internal. Updating no re-route policies I0504 12:23:40.934986 3659 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/multus-8fpwx I0504 12:23:40.935001 3659 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/multus-8fpwx I0504 12:23:40.935042 3659 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/multus-8fpwx after 0 failed attempt(s) I0504 12:23:40.935058 3659 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-jl9gk I0504 12:23:40.935069 3659 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-jl9gk I0504 12:23:40.935075 3659 obj_retry.go:334] Retry object setup: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-vkm5x I0504 12:23:40.935080 3659 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-63.ec2.internal I0504 12:23:40.935088 3659 obj_retry.go:398] Adding new object: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-vkm5x I0504 12:23:40.935089 3659 obj_retry.go:334] Retry object setup: *v1.Pod openshift-dns/node-resolver-xm8zj I0504 12:23:40.935089 3659 obj_retry.go:334] Retry object setup: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-63.ec2.internal I0504 12:23:40.935103 3659 obj_retry.go:398] Adding new object: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-63.ec2.internal I0504 12:23:40.935061 3659 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-operator/iptables-alerter-cw55f I0504 12:23:40.935113 3659 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-jl9gk] creating logical port openshift-network-diagnostics_network-check-target-jl9gk for pod on switch ip-10-0-2-63.ec2.internal I0504 12:23:40.935114 3659 obj_retry.go:334] Retry object setup: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-846z4 I0504 12:23:40.935117 3659 obj_retry.go:334] Retry object setup: *v1.Pod openshift-image-registry/node-ca-fn4c8 I0504 12:23:40.935115 3659 obj_retry.go:334] Retry object setup: *v1.Pod openshift-cluster-node-tuning-operator/tuned-gkztf I0504 12:23:40.935125 3659 obj_retry.go:418] Retry successful for *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-63.ec2.internal after 0 failed attempt(s) I0504 12:23:40.935124 3659 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-operator/iptables-alerter-cw55f I0504 12:23:40.935091 3659 obj_retry.go:398] Adding new object: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-63.ec2.internal I0504 12:23:40.935130 3659 obj_retry.go:398] Adding new object: *v1.Pod openshift-dns/node-resolver-xm8zj I0504 12:23:40.935132 3659 obj_retry.go:398] Adding new object: *v1.Pod openshift-cluster-node-tuning-operator/tuned-gkztf I0504 12:23:40.935146 3659 obj_retry.go:418] Retry successful for *v1.Pod openshift-network-operator/iptables-alerter-cw55f after 0 failed attempt(s) I0504 12:23:40.935146 3659 obj_retry.go:418] Retry successful for *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-63.ec2.internal after 0 failed attempt(s) I0504 12:23:40.935149 3659 obj_retry.go:418] Retry successful for *v1.Pod openshift-dns/node-resolver-xm8zj after 0 failed attempt(s) I0504 12:23:40.935103 3659 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/konnectivity-agent-t6f2b I0504 12:23:40.935148 3659 obj_retry.go:418] Retry successful for *v1.Pod openshift-cluster-node-tuning-operator/tuned-gkztf after 0 failed attempt(s) I0504 12:23:40.935158 3659 obj_retry.go:398] Adding new object: *v1.Pod kube-system/konnectivity-agent-t6f2b I0504 12:23:40.935131 3659 obj_retry.go:398] Adding new object: *v1.Pod openshift-image-registry/node-ca-fn4c8 I0504 12:23:40.935169 3659 obj_retry.go:418] Retry successful for *v1.Pod kube-system/konnectivity-agent-t6f2b after 0 failed attempt(s) I0504 12:23:40.935172 3659 obj_retry.go:418] Retry successful for *v1.Pod openshift-image-registry/node-ca-fn4c8 after 0 failed attempt(s) I0504 12:23:40.935103 3659 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/multus-additional-cni-plugins-dnq4p I0504 12:23:40.935125 3659 obj_retry.go:398] Adding new object: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-846z4 I0504 12:23:40.935225 3659 obj_retry.go:418] Retry successful for *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-846z4 after 0 failed attempt(s) I0504 12:23:40.935071 3659 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-528nq I0504 12:23:40.935242 3659 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-528nq I0504 12:23:40.935285 3659 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-528nq] creating logical port openshift-multus_network-metrics-daemon-528nq for pod on switch ip-10-0-2-63.ec2.internal I0504 12:23:40.935107 3659 obj_retry.go:418] Retry successful for *v1.Pod openshift-ovn-kubernetes/ovnkube-node-vkm5x after 0 failed attempt(s) I0504 12:23:40.935193 3659 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/multus-additional-cni-plugins-dnq4p I0504 12:23:40.935313 3659 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/multus-additional-cni-plugins-dnq4p after 0 failed attempt(s) I0504 12:23:40.936046 3659 pods.go:271] [openshift-network-diagnostics/network-check-target-jl9gk] addLogicalPort took 945.09µs, libovsdb time 291.556µs I0504 12:23:40.936057 3659 obj_retry.go:418] Retry successful for *v1.Pod openshift-network-diagnostics/network-check-target-jl9gk after 0 failed attempt(s) I0504 12:23:40.936164 3659 pods.go:271] [openshift-multus/network-metrics-daemon-528nq] addLogicalPort took 907.152µs, libovsdb time 268.094µs I0504 12:23:40.936173 3659 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/network-metrics-daemon-528nq after 0 failed attempt(s) I0504 12:23:40.947453 3659 gateway_shared_intf.go:1799] Creating Gateway Node Port Watcher I0504 12:23:40.958468 3659 iptables.go:108] Creating table: mangle chain: OUTPUT I0504 12:23:40.961702 3659 iptables.go:108] Creating table: nat chain: OUTPUT I0504 12:23:41.005624 3659 iptables.go:108] Creating table: nat chain: PREROUTING I0504 12:23:41.071864 3659 iptables.go:108] Creating table: filter chain: FORWARD I0504 12:23:41.076507 3659 default_network_controller.go:1025] Node "ip-10-0-1-18.ec2.internal" in remote zone "ip-10-0-1-18.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0504 12:23:41.077266 3659 zone_ic_handler.go:195] Time taken to create transit switch: 271.638µs I0504 12:23:41.077279 3659 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-1-18.ec2.internal for the network default I0504 12:23:41.078328 3659 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-1-18.ec2.internal" on network "default" took: 1.355667ms I0504 12:23:41.088649 3659 gateway_shared_intf.go:2124] Ensuring IP Neighbor entry for: 169.254.0.1 W0504 12:23:41.088740 3659 gateway_shared_intf.go:2130] Failed to remove IP neighbor entry for ip 169.254.0.1, on iface br-ex: failed to delete neighbour entry 169.254.0.1 : no such file or directory I0504 12:23:41.088780 3659 gateway_shared_intf.go:2124] Ensuring IP Neighbor entry for: 169.254.0.4 W0504 12:23:41.088846 3659 gateway_shared_intf.go:2130] Failed to remove IP neighbor entry for ip 169.254.0.4, on iface br-ex: failed to delete neighbour entry 169.254.0.4 : no such file or directory I0504 12:23:41.088892 3659 gateway_init.go:325] Gateway and management port readiness took 1.7123591s I0504 12:23:41.088918 3659 controller.go:133] Adding controller [node-nad-controller NAD controller] event handlers I0504 12:23:41.088962 3659 shared_informer.go:349] "Waiting for caches to sync" controller="[node-nad-controller NAD controller]" I0504 12:23:41.088973 3659 shared_informer.go:356] "Caches are synced" controller="[node-nad-controller NAD controller]" I0504 12:23:41.088982 3659 nad_controller.go:541] [node-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0504 12:23:41.089056 3659 controller.go:157] Starting controller [node-nad-controller NAD controller] with 1 workers I0504 12:23:41.089144 3659 nad_controller.go:580] [node-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 54.744µs I0504 12:23:41.089357 3659 network_controller.go:373] [node-nad-controller network controller]: syncing all networks I0504 12:23:41.089381 3659 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 14.824µs I0504 12:23:41.089389 3659 network_controller.go:384] [node-nad-controller network controller]: finished syncing all networks. Time taken: 32.079µs I0504 12:23:41.089394 3659 controller.go:157] Starting controller [node-nad-controller network controller] with 1 workers I0504 12:23:41.089400 3659 nad_controller.go:379] [node-nad-controller NAD controller]: started I0504 12:23:41.089404 3659 default_node_network_controller.go:899] Starting the default node network controller I0504 12:23:41.089409 3659 gateway_init.go:335] Initializing Gateway Functionality for gateway Start I0504 12:23:41.089510 3659 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 25.345µs I0504 12:23:41.089541 3659 default_node_network_controller.go:1460] MTU (9001) of network interface br-ex is big enough to deal with Geneve header overhead (sum 8959). I0504 12:23:41.089549 3659 default_node_network_controller.go:935] Waiting for gateway and management port readiness... I0504 12:23:41.089574 3659 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000b15510 0xc000342460 0xc0003424d0 0xc0003425b0 0xc000342620 0xc000342690 0xc000342700 0xc000ca0870 0xc000342770 0xc0003427e0 0xc000ca08c0 0xc0063ff340 0xc000342850 map[0x2ff9080:0xc002593560 0x2ff95c0:0xc0025936e0 0x2ffa2e0:0xc00238e720 0x2ffa580:0xc002593500 0x2ffad60:0xc0025935c0 0x2ffb000:0xc002593620 0x2ffb2a0:0xc002593680 0x2ffba80:0xc002593740 0x2ffbd20:0xc0025937a0 0x3026980:0xc000e0daa0 0x3026d20:0xc000e0db00 0x3027800:0xc000b75b00 0x3029160:0xc001bea000 0x30298a0:0xc00238e6c0 0x302a720:0xc001bea060] 0xc00016d1f0 0} I0504 12:23:41.089631 3659 gateway.go:151] Starting gateway service sync I0504 12:23:41.090022 3659 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ITP I0504 12:23:41.092001 3659 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-NODEPORT I0504 12:23:41.114356 3659 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-EXTERNALIP I0504 12:23:41.115464 3659 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ETP I0504 12:23:41.116498 3659 gateway_iptables.go:474] Recreating iptables rules for table: mangle, chain: OVN-KUBE-ITP I0504 12:23:41.149732 3659 gateway.go:168] Gateway service sync done. Time taken: 60.089546ms I0504 12:23:41.150268 3659 healthcheck.go:145] Opening healthcheck "openshift-ingress/router-default" on port 32347 I0504 12:23:41.150485 3659 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000b15510 0xc000342460 0xc0003424d0 0xc0003425b0 0xc000342620 0xc000342690 0xc000342700 0xc000ca0870 0xc000342770 0xc0003427e0 0xc000ca08c0 0xc0063ff340 0xc000342850 map[0x2ff9080:0xc002593560 0x2ff95c0:0xc0025936e0 0x2ffa2e0:0xc00238e720 0x2ffa580:0xc002593500 0x2ffad60:0xc0025935c0 0x2ffb000:0xc002593620 0x2ffb2a0:0xc002593680 0x2ffba80:0xc002593740 0x2ffbd20:0xc0025937a0 0x3026980:0xc000e0daa0 0x3026d20:0xc000e0db00 0x3027800:0xc000b75b00 0x3029160:0xc001bea000 0x30298a0:0xc00238e6c0 0x302a720:0xc001bea060] 0xc00016d1f0 0} I0504 12:23:41.150931 3659 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000b15510 0xc000342460 0xc0003424d0 0xc0003425b0 0xc000342620 0xc000342690 0xc000342700 0xc000ca0870 0xc000342770 0xc0003427e0 0xc000ca08c0 0xc0063ff340 0xc000342850 map[0x2ff9080:0xc002593560 0x2ff95c0:0xc0025936e0 0x2ffa2e0:0xc00238e720 0x2ffa580:0xc002593500 0x2ffad60:0xc0025935c0 0x2ffb000:0xc002593620 0x2ffb2a0:0xc002593680 0x2ffba80:0xc002593740 0x2ffbd20:0xc0025937a0 0x3026980:0xc000e0daa0 0x3026d20:0xc000e0db00 0x3027800:0xc000b75b00 0x3029160:0xc001bea000 0x30298a0:0xc00238e6c0 0x302a720:0xc001bea060] 0xc00016d1f0 0} I0504 12:23:41.150966 3659 gateway.go:532] Reconciling gateway with updates I0504 12:23:41.156751 3659 gateway.go:339] Spawning Conntrack Rule Check Thread I0504 12:23:41.156774 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0504 12:23:41.156779 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0504 12:23:41.156790 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0504 12:23:41.156793 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0504 12:23:41.156793 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0504 12:23:41.156800 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0504 12:23:41.156801 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0504 12:23:41.156811 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0504 12:23:41.156813 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0504 12:23:41.156810 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0504 12:23:41.156823 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0504 12:23:41.156828 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0504 12:23:41.156827 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0504 12:23:41.156835 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0504 12:23:41.156842 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0504 12:23:41.156848 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0504 12:23:41.156850 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0504 12:23:41.156849 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0504 12:23:41.156864 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0504 12:23:41.156865 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0504 12:23:41.156869 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0504 12:23:41.156873 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0504 12:23:41.156866 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0504 12:23:41.156879 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0504 12:23:41.156882 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0504 12:23:41.156883 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0504 12:23:41.156885 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0504 12:23:41.156886 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0504 12:23:41.156895 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0504 12:23:41.156895 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0504 12:23:41.156897 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift I0504 12:23:41.156900 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0504 12:23:41.156904 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0504 12:23:41.156899 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0504 12:23:41.156909 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0504 12:23:41.156912 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0504 12:23:41.156914 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0504 12:23:41.156913 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/kubernetes I0504 12:23:41.156918 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0504 12:23:41.156917 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0504 12:23:41.156925 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/kubernetes I0504 12:23:41.156925 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0504 12:23:41.156933 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0504 12:23:41.156937 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0504 12:23:41.156934 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0504 12:23:41.156942 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0504 12:23:41.156944 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0504 12:23:41.156952 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0504 12:23:41.156956 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0504 12:23:41.156961 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0504 12:23:41.156960 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0504 12:23:41.156965 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0504 12:23:41.156967 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0504 12:23:41.156977 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0504 12:23:41.156970 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0504 12:23:41.156984 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0504 12:23:41.156988 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0504 12:23:41.156883 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0504 12:23:41.156999 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0504 12:23:41.156888 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0504 12:23:41.156976 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0504 12:23:41.157054 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/packageserver I0504 12:23:41.157069 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/packageserver I0504 12:23:41.157067 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0504 12:23:41.156867 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0504 12:23:41.156938 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0504 12:23:41.156900 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0504 12:23:41.156896 3659 default_node_network_controller.go:944] Gateway and management port readiness took 67.339888ms I0504 12:23:41.157113 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0504 12:23:41.156985 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0504 12:23:41.157181 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0504 12:23:41.156952 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0504 12:23:41.157191 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0504 12:23:41.157195 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0504 12:23:41.157202 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0504 12:23:41.157203 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0504 12:23:41.157216 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0504 12:23:41.156945 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0504 12:23:41.156914 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0504 12:23:41.157259 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0504 12:23:41.156958 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0504 12:23:41.156880 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0504 12:23:41.157353 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0504 12:23:41.156890 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0504 12:23:41.157432 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0504 12:23:41.156901 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0504 12:23:41.156791 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0504 12:23:41.157519 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0504 12:23:41.156903 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0504 12:23:41.157596 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0504 12:23:41.156913 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift I0504 12:23:41.157620 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0504 12:23:41.156923 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0504 12:23:41.156867 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0504 12:23:41.157695 3659 node_ip_handler_linux.go:160] Node IP manager is running I0504 12:23:41.157702 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0504 12:23:41.157722 3659 port_claim.go:71] Svc openshift-ingress/router-default: port 30776 is already open, no action needed I0504 12:23:41.156925 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0504 12:23:41.157736 3659 port_claim.go:71] Svc openshift-ingress/router-default: port 30843 is already open, no action needed I0504 12:23:41.157744 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0504 12:23:41.157761 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0504 12:23:41.156949 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0504 12:23:41.156969 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0504 12:23:41.157828 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0504 12:23:41.156966 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0504 12:23:41.156983 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0504 12:23:41.157846 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0504 12:23:41.156985 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0504 12:23:41.157885 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0504 12:23:41.157893 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0504 12:23:41.157269 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0504 12:23:41.156852 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller W0504 12:23:41.193491 3659 port_linux.go:245] Missing or unable to find route entry for subnet 10.128.0.0/14 via gateway 10.130.0.1 on link ovn-k8s-mp0 with MTU: 0 W0504 12:23:41.193619 3659 port_linux.go:245] Missing or unable to find route entry for subnet 169.254.0.3/32 via gateway 10.130.0.1 on link ovn-k8s-mp0 with MTU: 0 W0504 12:23:41.193772 3659 port_linux.go:261] Missing arp entry for MAC/IP binding (0a:58:0a:82:00:01/10.130.0.1) on link ovn-k8s-mp0 I0504 12:23:41.202275 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0504 12:23:41.202298 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0504 12:23:41.202307 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0504 12:23:41.202314 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0504 12:23:41.202320 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0504 12:23:41.202325 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0504 12:23:41.202331 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0504 12:23:41.202336 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0504 12:23:41.202341 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0504 12:23:41.202346 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0504 12:23:41.202351 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0504 12:23:41.202356 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0504 12:23:41.202361 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0504 12:23:41.202366 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0504 12:23:41.202373 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0504 12:23:41.202378 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0504 12:23:41.202384 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0504 12:23:41.202410 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0504 12:23:41.202420 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0504 12:23:41.202425 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0504 12:23:41.226849 3659 egressfirewall.go:723] Syncing node "ip-10-0-1-18.ec2.internal" for egress firewall I0504 12:23:41.226876 3659 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-1-18.ec2.internal. Updating no re-route policies I0504 12:23:41.226879 3659 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-18.ec2.internal in network "default" I0504 12:23:41.226908 3659 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-18.ec2.internal I0504 12:23:41.226921 3659 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-18.ec2.internal: 14.78µs I0504 12:23:41.226929 3659 node_tracker.go:169] Node ip-10-0-1-18.ec2.internal switch + router changed, syncing services in network "default" I0504 12:23:41.226941 3659 services_controller.go:574] Full service sync requested for network=default W0504 12:23:41.227242 3659 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog W0504 12:23:41.227298 3659 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics W0504 12:23:41.227312 3659 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0504 12:23:41.241203 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0504 12:23:41.241265 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0504 12:23:41.241287 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0504 12:23:41.241316 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0504 12:23:42.402506 3659 default_node_network_controller.go:1380] Starting node controller node sync W0504 12:23:42.402557 3659 egressip_healthcheck.go:76] Health checking using insecure connection I0504 12:23:42.402624 3659 egressip_healthcheck.go:109] Starting Egress IP Health Server on 10.130.0.2:9107 I0504 12:23:42.413163 3659 default_node_network_controller.go:1422] Node controller node sync done. Time taken: 10.639691ms I0504 12:23:42.413295 3659 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-182.ec2.internal", IP: 10.0.0.182 to PMTUD blocking rules I0504 12:23:42.413312 3659 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-182.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0504 12:23:42.413300 3659 default_node_network_controller.go:1293] Adding remote node "ip-10-0-1-18.ec2.internal", IP: 10.0.1.18 to PMTUD blocking rules I0504 12:23:42.413361 3659 default_node_network_controller.go:1293] Adding remote node "ip-10-0-1-18.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0504 12:23:42.424276 3659 healthcheck_node.go:124] "Starting node proxy healthz server" address="0.0.0.0:10256" I0504 12:23:42.424595 3659 egressservice_node.go:93] Setting up event handlers for Egress Services I0504 12:23:42.424794 3659 egressservice_node.go:183] Starting Egress Services Controller I0504 12:23:42.424811 3659 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0504 12:23:42.424823 3659 shared_informer.go:356] "Caches are synced" controller="egressservices" I0504 12:23:42.424832 3659 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0504 12:23:42.424839 3659 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0504 12:23:42.424865 3659 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0504 12:23:42.424875 3659 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0504 12:23:42.424882 3659 egressservice_node.go:197] Repairing Egress Services I0504 12:23:42.434568 3659 node_controller.go:43] Starting Admin Policy Based Route Node Controller I0504 12:23:42.434578 3659 external_controller.go:278] Starting Admin Policy Based Route Controller I0504 12:23:42.434873 3659 default_node_network_controller.go:1077] Egress IP for secondary host network is disabled I0504 12:23:42.434887 3659 link_network_manager.go:119] Link manager is running I0504 12:23:42.434897 3659 default_node_network_controller.go:1098] Default node network controller initialized and ready. I0504 12:23:42.435130 3659 ovspinning_linux.go:50] OVS CPU affinity pinning disabled I0504 12:23:42.435385 3659 vrf_manager.go:128] VRF manager is running I0504 12:23:42.435513 3659 node_controller_manager.go:437] Removing flows to drop GARP I0504 12:23:42.435526 3659 gateway.go:532] Reconciling gateway with updates I0504 12:23:42.458152 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0504 12:23:42.458158 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0504 12:23:42.458174 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0504 12:23:42.458176 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0504 12:23:42.458174 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/packageserver I0504 12:23:42.458182 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0504 12:23:42.458195 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0504 12:23:42.458195 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/packageserver I0504 12:23:42.458232 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/kubernetes I0504 12:23:42.458244 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0504 12:23:42.458246 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0504 12:23:42.458255 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/kubernetes I0504 12:23:42.458262 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0504 12:23:42.458267 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift I0504 12:23:42.458272 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0504 12:23:42.458278 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0504 12:23:42.458283 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift I0504 12:23:42.458282 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0504 12:23:42.458288 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0504 12:23:42.458293 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0504 12:23:42.458294 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0504 12:23:42.458294 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0504 12:23:42.458300 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0504 12:23:42.458308 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0504 12:23:42.458310 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0504 12:23:42.458312 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0504 12:23:42.458316 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0504 12:23:42.458319 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0504 12:23:42.458323 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0504 12:23:42.458256 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0504 12:23:42.458330 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0504 12:23:42.458338 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0504 12:23:42.458340 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0504 12:23:42.458347 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0504 12:23:42.458353 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0504 12:23:42.458355 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0504 12:23:42.458357 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0504 12:23:42.458360 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0504 12:23:42.458367 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0504 12:23:42.458372 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0504 12:23:42.458249 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0504 12:23:42.458393 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0504 12:23:42.458265 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0504 12:23:42.458405 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0504 12:23:42.458255 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0504 12:23:42.458238 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0504 12:23:42.458418 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0504 12:23:42.458240 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0504 12:23:42.458422 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0504 12:23:42.458428 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0504 12:23:42.458429 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0504 12:23:42.458430 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0504 12:23:42.458438 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0504 12:23:42.458228 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0504 12:23:42.458444 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0504 12:23:42.458448 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0504 12:23:42.458454 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0504 12:23:42.458287 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0504 12:23:42.458462 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0504 12:23:42.458463 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0504 12:23:42.458461 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0504 12:23:42.458279 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0504 12:23:42.458479 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0504 12:23:42.458296 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0504 12:23:42.458402 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0504 12:23:42.458300 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0504 12:23:42.458283 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0504 12:23:42.458524 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0504 12:23:42.458531 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0504 12:23:42.458304 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0504 12:23:42.458324 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0504 12:23:42.458542 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0504 12:23:42.458433 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0504 12:23:42.458269 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0504 12:23:42.458582 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0504 12:23:42.458441 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0504 12:23:42.458599 3659 port_claim.go:71] Svc openshift-ingress/router-default: port 30776 is already open, no action needed I0504 12:23:42.458602 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0504 12:23:42.458607 3659 port_claim.go:71] Svc openshift-ingress/router-default: port 30843 is already open, no action needed I0504 12:23:42.458281 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0504 12:23:42.458645 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0504 12:23:42.458654 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0504 12:23:42.458668 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0504 12:23:42.458646 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0504 12:23:42.458682 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0504 12:23:42.458656 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0504 12:23:42.458499 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0504 12:23:42.458761 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0504 12:23:42.458768 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0504 12:23:42.458777 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0504 12:23:42.458789 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0504 12:23:42.458533 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0504 12:23:42.458407 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0504 12:23:42.458443 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0504 12:23:42.458843 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0504 12:23:42.458263 3659 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0504 12:23:42.458882 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0504 12:23:42.458289 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0504 12:23:42.458464 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0504 12:23:42.458301 3659 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0504 12:23:42.483139 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0504 12:23:42.483148 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0504 12:23:42.483163 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0504 12:23:42.483169 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0504 12:23:42.483173 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0504 12:23:42.483179 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0504 12:23:42.483184 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0504 12:23:42.483189 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0504 12:23:42.483194 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0504 12:23:42.483209 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0504 12:23:42.483227 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0504 12:23:42.483233 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0504 12:23:42.483238 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0504 12:23:42.483246 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0504 12:23:42.483261 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0504 12:23:42.483270 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0504 12:23:42.483275 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0504 12:23:42.483280 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0504 12:23:42.523728 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0504 12:23:42.523768 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0504 12:23:42.523776 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0504 12:23:42.523782 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0504 12:23:42.523786 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0504 12:23:42.523791 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0504 12:23:42.575244 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0504 12:23:42.575265 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0504 12:23:42.575293 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0504 12:23:42.575303 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0504 12:23:42.575310 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0504 12:23:42.575315 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0504 12:23:42.575321 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0504 12:23:42.575326 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0504 12:23:42.575332 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0504 12:23:42.575337 3659 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0504 12:23:44.793792 3659 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-18.ec2.internal I0504 12:23:44.793831 3659 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-18.ec2.internal: 33.251µs I0504 12:23:45.000275 3659 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-63.ec2.internal I0504 12:23:45.000308 3659 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-63.ec2.internal: 41.923µs W0504 12:23:47.255059 3659 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0504 12:23:49.074997 3659 cni.go:452] [openshift-network-diagnostics/network-check-target-jl9gk 5552aedb2130d349448a849e4f2713907b45839a0d693a4cf48cec38f61b1b77 network default NAD default NAD key ] ADD starting CNI request [openshift-network-diagnostics/network-check-target-jl9gk 5552aedb2130d349448a849e4f2713907b45839a0d693a4cf48cec38f61b1b77 network default NAD default NAD key ] I0504 12:23:49.106783 3659 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-target-jl9gk, hostIfaceName: 5552aedb2130d34, network: default, NAD default, SandboxID: "5552aedb2130d349448a849e4f2713907b45839a0d693a4cf48cec38f61b1b77", PCI device ID: , UID: "5022eaea-8c91-4c3e-9ab1-2177c1acfb91", MAC: 0a:58:0a:82:00:03, IPs: [10.130.0.3/23] I0504 12:23:49.174127 3659 cni.go:473] [openshift-network-diagnostics/network-check-target-jl9gk 5552aedb2130d349448a849e4f2713907b45839a0d693a4cf48cec38f61b1b77 network default NAD default NAD key default] ADD finished CNI request [openshift-network-diagnostics/network-check-target-jl9gk 5552aedb2130d349448a849e4f2713907b45839a0d693a4cf48cec38f61b1b77 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ca:bb:27:67:6d:ab\",\"name\":\"5552aedb2130d34\"},{\"mac\":\"0a:58:0a:82:00:03\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c5b1fb0b-1481-422d-bba6-eb3e3a3e9d14\"}],\"ips\":[{\"address\":\"10.130.0.3/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0504 12:24:02.675650 3659 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0504 12:24:02.711118 3659 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0504 12:24:02.730327 3659 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-85b84bf8b4-rwvjm] creating logical port openshift-image-registry_image-registry-85b84bf8b4-rwvjm for pod on switch ip-10-0-2-63.ec2.internal I0504 12:24:02.730425 3659 kube.go:256] Updating pod openshift-image-registry/image-registry-85b84bf8b4-rwvjm I0504 12:24:02.752408 3659 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-c9dd8b8-7vqj6] creating logical port openshift-ingress_router-default-c9dd8b8-7vqj6 for pod on switch ip-10-0-2-63.ec2.internal I0504 12:24:02.752518 3659 kube.go:256] Updating pod openshift-ingress/router-default-c9dd8b8-7vqj6 I0504 12:24:02.757382 3659 pod.go:62] [openshift-image-registry/image-registry-85b84bf8b4-rwvjm] pod update took 26.983867ms I0504 12:24:02.757400 3659 base_network_controller_pods.go:951] [default/openshift-image-registry/image-registry-85b84bf8b4-rwvjm] addLogicalPort annotation time took 27.004363ms I0504 12:24:02.758294 3659 pods.go:271] [openshift-image-registry/image-registry-85b84bf8b4-rwvjm] addLogicalPort took 27.976166ms, libovsdb time 681.825µs I0504 12:24:02.803719 3659 pod.go:62] [openshift-ingress/router-default-c9dd8b8-7vqj6] pod update took 51.258627ms I0504 12:24:02.803736 3659 base_network_controller_pods.go:951] [default/openshift-ingress/router-default-c9dd8b8-7vqj6] addLogicalPort annotation time took 51.278864ms I0504 12:24:02.804661 3659 pods.go:271] [openshift-ingress/router-default-c9dd8b8-7vqj6] addLogicalPort took 52.265617ms, libovsdb time 700.559µs W0504 12:24:02.854823 3659 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-image-registry/image-registry-f75ff85c9-wbqnf IP address from the namespace address-set, err: pod openshift-image-registry/image-registry-f75ff85c9-wbqnf: no pod IPs found I0504 12:24:02.998349 3659 base_network_controller_pods.go:487] [default/openshift-dns/dns-default-vwbr6] creating logical port openshift-dns_dns-default-vwbr6 for pod on switch ip-10-0-2-63.ec2.internal I0504 12:24:02.998430 3659 base_network_controller_pods.go:487] [default/openshift-console/downloads-5c54f4944f-8xjk7] creating logical port openshift-console_downloads-5c54f4944f-8xjk7 for pod on switch ip-10-0-2-63.ec2.internal I0504 12:24:02.998518 3659 kube.go:256] Updating pod openshift-console/downloads-5c54f4944f-8xjk7 I0504 12:24:02.998432 3659 kube.go:256] Updating pod openshift-dns/dns-default-vwbr6 I0504 12:24:03.070234 3659 pod.go:62] [openshift-console/downloads-5c54f4944f-8xjk7] pod update took 71.731391ms I0504 12:24:03.070260 3659 base_network_controller_pods.go:951] [default/openshift-console/downloads-5c54f4944f-8xjk7] addLogicalPort annotation time took 71.758063ms I0504 12:24:03.070234 3659 pod.go:62] [openshift-dns/dns-default-vwbr6] pod update took 71.832621ms I0504 12:24:03.070345 3659 base_network_controller_pods.go:951] [default/openshift-dns/dns-default-vwbr6] addLogicalPort annotation time took 71.945777ms I0504 12:24:03.071274 3659 pods.go:271] [openshift-console/downloads-5c54f4944f-8xjk7] addLogicalPort took 72.852312ms, libovsdb time 736.618µs I0504 12:24:03.071550 3659 pods.go:271] [openshift-dns/dns-default-vwbr6] addLogicalPort took 73.215223ms, libovsdb time 989.682µs I0504 12:24:03.087048 3659 base_network_controller_pods.go:487] [default/openshift-ingress-canary/ingress-canary-wgjjc] creating logical port openshift-ingress-canary_ingress-canary-wgjjc for pod on switch ip-10-0-2-63.ec2.internal I0504 12:24:03.087210 3659 kube.go:256] Updating pod openshift-ingress-canary/ingress-canary-wgjjc I0504 12:24:03.117478 3659 cni.go:452] [openshift-ingress/router-default-c9dd8b8-7vqj6 789e5e64cd98110f659b7ab9017ac2ed0aca48967f21c43490b0f0b234313a51 network default NAD default NAD key ] ADD starting CNI request [openshift-ingress/router-default-c9dd8b8-7vqj6 789e5e64cd98110f659b7ab9017ac2ed0aca48967f21c43490b0f0b234313a51 network default NAD default NAD key ] I0504 12:24:03.119359 3659 pod.go:62] [openshift-ingress-canary/ingress-canary-wgjjc] pod update took 32.174708ms I0504 12:24:03.119381 3659 base_network_controller_pods.go:951] [default/openshift-ingress-canary/ingress-canary-wgjjc] addLogicalPort annotation time took 32.202446ms I0504 12:24:03.120660 3659 pods.go:271] [openshift-ingress-canary/ingress-canary-wgjjc] addLogicalPort took 33.643701ms, libovsdb time 1.007783ms I0504 12:24:03.126567 3659 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress, podName: router-default-c9dd8b8-7vqj6, hostIfaceName: 789e5e64cd98110, network: default, NAD default, SandboxID: "789e5e64cd98110f659b7ab9017ac2ed0aca48967f21c43490b0f0b234313a51", PCI device ID: , UID: "b513b99c-ce5c-465a-86c3-68b061796932", MAC: 0a:58:0a:82:00:06, IPs: [10.130.0.6/23] I0504 12:24:03.133477 3659 base_network_controller_pods.go:487] [default/openshift-insights/insights-runtime-extractor-jflpt] creating logical port openshift-insights_insights-runtime-extractor-jflpt for pod on switch ip-10-0-2-63.ec2.internal I0504 12:24:03.133591 3659 kube.go:256] Updating pod openshift-insights/insights-runtime-extractor-jflpt I0504 12:24:03.156979 3659 pod.go:62] [openshift-insights/insights-runtime-extractor-jflpt] pod update took 23.427525ms I0504 12:24:03.156999 3659 base_network_controller_pods.go:951] [default/openshift-insights/insights-runtime-extractor-jflpt] addLogicalPort annotation time took 23.452818ms I0504 12:24:03.157975 3659 pods.go:271] [openshift-insights/insights-runtime-extractor-jflpt] addLogicalPort took 24.51416ms, libovsdb time 677.621µs I0504 12:24:03.172214 3659 cni.go:473] [openshift-ingress/router-default-c9dd8b8-7vqj6 789e5e64cd98110f659b7ab9017ac2ed0aca48967f21c43490b0f0b234313a51 network default NAD default NAD key default] ADD finished CNI request [openshift-ingress/router-default-c9dd8b8-7vqj6 789e5e64cd98110f659b7ab9017ac2ed0aca48967f21c43490b0f0b234313a51 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"4e:51:c7:f8:d7:91\",\"name\":\"789e5e64cd98110\"},{\"mac\":\"0a:58:0a:82:00:06\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/00d696aa-95bb-4df0-92a4-7c182e078581\"}],\"ips\":[{\"address\":\"10.130.0.6/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0504 12:24:03.297275 3659 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0504 12:24:03.297295 3659 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0504 12:24:03.297300 3659 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default I0504 12:24:03.507384 3659 cni.go:452] [openshift-insights/insights-runtime-extractor-jflpt cf7934df991313b73fb4dd7d3224183cb8975cf152cd506b5f91234cb9c8386f network default NAD default NAD key ] ADD starting CNI request [openshift-insights/insights-runtime-extractor-jflpt cf7934df991313b73fb4dd7d3224183cb8975cf152cd506b5f91234cb9c8386f network default NAD default NAD key ] I0504 12:24:03.516987 3659 helper_linux.go:556] ConfigureOVS: namespace: openshift-insights, podName: insights-runtime-extractor-jflpt, hostIfaceName: cf7934df991313b, network: default, NAD default, SandboxID: "cf7934df991313b73fb4dd7d3224183cb8975cf152cd506b5f91234cb9c8386f", PCI device ID: , UID: "22ab2f81-9ffe-4035-96a4-fac7b6a25a4a", MAC: 0a:58:0a:82:00:0a, IPs: [10.130.0.10/23] I0504 12:24:03.560747 3659 cni.go:473] [openshift-insights/insights-runtime-extractor-jflpt cf7934df991313b73fb4dd7d3224183cb8975cf152cd506b5f91234cb9c8386f network default NAD default NAD key default] ADD finished CNI request [openshift-insights/insights-runtime-extractor-jflpt cf7934df991313b73fb4dd7d3224183cb8975cf152cd506b5f91234cb9c8386f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"76:8d:a1:26:e8:3c\",\"name\":\"cf7934df991313b\"},{\"mac\":\"0a:58:0a:82:00:0a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a76835b5-1d6d-41ec-ab59-5c0a832ee04e\"}],\"ips\":[{\"address\":\"10.130.0.10/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0504 12:24:03.678573 3659 cni.go:452] [openshift-dns/dns-default-vwbr6 13706b2ea5e5a6b622b1f6b716af661ea03e883918c66fce4fadbd8533183df8 network default NAD default NAD key ] ADD starting CNI request [openshift-dns/dns-default-vwbr6 13706b2ea5e5a6b622b1f6b716af661ea03e883918c66fce4fadbd8533183df8 network default NAD default NAD key ] I0504 12:24:03.682812 3659 cni.go:452] [openshift-console/downloads-5c54f4944f-8xjk7 329866f699c38de74c0af51e970e17f6265ad7ef89e42df31c28ac0eb74db89e network default NAD default NAD key ] ADD starting CNI request [openshift-console/downloads-5c54f4944f-8xjk7 329866f699c38de74c0af51e970e17f6265ad7ef89e42df31c28ac0eb74db89e network default NAD default NAD key ] I0504 12:24:03.694414 3659 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: downloads-5c54f4944f-8xjk7, hostIfaceName: 329866f699c38de, network: default, NAD default, SandboxID: "329866f699c38de74c0af51e970e17f6265ad7ef89e42df31c28ac0eb74db89e", PCI device ID: , UID: "9875d0c8-b67d-4fa2-9b03-2e58ba43d9c4", MAC: 0a:58:0a:82:00:08, IPs: [10.130.0.8/23] I0504 12:24:03.694555 3659 helper_linux.go:556] ConfigureOVS: namespace: openshift-dns, podName: dns-default-vwbr6, hostIfaceName: 13706b2ea5e5a6b, network: default, NAD default, SandboxID: "13706b2ea5e5a6b622b1f6b716af661ea03e883918c66fce4fadbd8533183df8", PCI device ID: , UID: "239b9885-f6ea-4252-b191-eeb872e30d1c", MAC: 0a:58:0a:82:00:07, IPs: [10.130.0.7/23] I0504 12:24:03.739098 3659 cni.go:473] [openshift-console/downloads-5c54f4944f-8xjk7 329866f699c38de74c0af51e970e17f6265ad7ef89e42df31c28ac0eb74db89e network default NAD default NAD key default] ADD finished CNI request [openshift-console/downloads-5c54f4944f-8xjk7 329866f699c38de74c0af51e970e17f6265ad7ef89e42df31c28ac0eb74db89e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"3e:79:36:7c:9a:4a\",\"name\":\"329866f699c38de\"},{\"mac\":\"0a:58:0a:82:00:08\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0ca22481-717c-4f70-9970-599d525e5af2\"}],\"ips\":[{\"address\":\"10.130.0.8/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0504 12:24:03.753708 3659 cni.go:473] [openshift-dns/dns-default-vwbr6 13706b2ea5e5a6b622b1f6b716af661ea03e883918c66fce4fadbd8533183df8 network default NAD default NAD key default] ADD finished CNI request [openshift-dns/dns-default-vwbr6 13706b2ea5e5a6b622b1f6b716af661ea03e883918c66fce4fadbd8533183df8 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"06:fa:7f:70:03:cd\",\"name\":\"13706b2ea5e5a6b\"},{\"mac\":\"0a:58:0a:82:00:07\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/32fe2349-d14f-47ba-96ec-621129aa498f\"}],\"ips\":[{\"address\":\"10.130.0.7/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0504 12:24:03.756125 3659 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-w5jk9] creating logical port openshift-monitoring_prometheus-operator-admission-webhook-59b79769fb-w5jk9 for pod on switch ip-10-0-2-63.ec2.internal I0504 12:24:03.756221 3659 kube.go:256] Updating pod openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-w5jk9 I0504 12:24:03.818829 3659 pod.go:62] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-w5jk9] pod update took 62.636843ms I0504 12:24:03.818849 3659 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-w5jk9] addLogicalPort annotation time took 62.66243ms I0504 12:24:03.819860 3659 pods.go:271] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-w5jk9] addLogicalPort took 63.750367ms, libovsdb time 735.099µs I0504 12:24:04.123964 3659 cni.go:452] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-w5jk9 a526de079dc72441546cbbaa9bc16e94f2542c5fcaed14d2100f31e04cb91e54 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-w5jk9 a526de079dc72441546cbbaa9bc16e94f2542c5fcaed14d2100f31e04cb91e54 network default NAD default NAD key ] I0504 12:24:04.134926 3659 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-operator-admission-webhook-59b79769fb-w5jk9, hostIfaceName: a526de079dc7244, network: default, NAD default, SandboxID: "a526de079dc72441546cbbaa9bc16e94f2542c5fcaed14d2100f31e04cb91e54", PCI device ID: , UID: "6b306b9b-20b0-4724-9084-e0c92d65f58f", MAC: 0a:58:0a:82:00:0b, IPs: [10.130.0.11/23] I0504 12:24:04.178645 3659 cni.go:473] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-w5jk9 a526de079dc72441546cbbaa9bc16e94f2542c5fcaed14d2100f31e04cb91e54 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-w5jk9 a526de079dc72441546cbbaa9bc16e94f2542c5fcaed14d2100f31e04cb91e54 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"0e:ce:46:3b:cf:d1\",\"name\":\"a526de079dc7244\"},{\"mac\":\"0a:58:0a:82:00:0b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a29ced1c-8790-4a5b-9fd3-3e2ce948b72f\"}],\"ips\":[{\"address\":\"10.130.0.11/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0504 12:24:04.351366 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-default W0504 12:24:04.351391 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-default I0504 12:24:04.392721 3659 obj_retry.go:498] Detected object openshift-image-registry/image-registry-85b84bf8b4-rwvjm of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0504 12:24:04.392795 3659 pods.go:176] Deleting pod: openshift-image-registry/image-registry-85b84bf8b4-rwvjm I0504 12:24:04.394263 3659 pods.go:236] Attempting to release IPs for pod: openshift-image-registry/image-registry-85b84bf8b4-rwvjm, ips: 10.130.0.5 I0504 12:24:04.394292 3659 obj_retry.go:498] Detected object openshift-image-registry/image-registry-85b84bf8b4-rwvjm of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0504 12:24:04.410347 3659 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-59d855649b-d7k5t] creating logical port openshift-image-registry_image-registry-59d855649b-d7k5t for pod on switch ip-10-0-2-63.ec2.internal I0504 12:24:04.410460 3659 kube.go:256] Updating pod openshift-image-registry/image-registry-59d855649b-d7k5t I0504 12:24:04.418225 3659 obj_retry.go:492] Detected object openshift-image-registry/image-registry-85b84bf8b4-rwvjm of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:24:04.418258 3659 obj_retry.go:492] Detected object openshift-image-registry/image-registry-85b84bf8b4-rwvjm of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:24:04.438627 3659 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-85b84bf8b4-rwvjm I0504 12:24:04.438710 3659 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-85b84bf8b4-rwvjm I0504 12:24:04.452222 3659 pod.go:62] [openshift-image-registry/image-registry-59d855649b-d7k5t] pod update took 41.788067ms I0504 12:24:04.452309 3659 base_network_controller_pods.go:951] [default/openshift-image-registry/image-registry-59d855649b-d7k5t] addLogicalPort annotation time took 41.877259ms I0504 12:24:04.453742 3659 pods.go:271] [openshift-image-registry/image-registry-59d855649b-d7k5t] addLogicalPort took 43.407661ms, libovsdb time 1.020896ms I0504 12:24:04.593666 3659 cni.go:452] [openshift-ingress-canary/ingress-canary-wgjjc f93e87f6f1276ebb2c8cf413bc2acb098a2534772e4cbcbbfe7a5c139f229540 network default NAD default NAD key ] ADD starting CNI request [openshift-ingress-canary/ingress-canary-wgjjc f93e87f6f1276ebb2c8cf413bc2acb098a2534772e4cbcbbfe7a5c139f229540 network default NAD default NAD key ] I0504 12:24:04.607231 3659 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress-canary, podName: ingress-canary-wgjjc, hostIfaceName: f93e87f6f1276eb, network: default, NAD default, SandboxID: "f93e87f6f1276ebb2c8cf413bc2acb098a2534772e4cbcbbfe7a5c139f229540", PCI device ID: , UID: "ee3a5b51-1e3a-4717-982b-8081f8f56a7d", MAC: 0a:58:0a:82:00:09, IPs: [10.130.0.9/23] I0504 12:24:04.657917 3659 cni.go:473] [openshift-ingress-canary/ingress-canary-wgjjc f93e87f6f1276ebb2c8cf413bc2acb098a2534772e4cbcbbfe7a5c139f229540 network default NAD default NAD key default] ADD finished CNI request [openshift-ingress-canary/ingress-canary-wgjjc f93e87f6f1276ebb2c8cf413bc2acb098a2534772e4cbcbbfe7a5c139f229540 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"c6:08:1d:13:d3:98\",\"name\":\"f93e87f6f1276eb\"},{\"mac\":\"0a:58:0a:82:00:09\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3fa0bd5b-8375-4a1e-8899-4dca86916aad\"}],\"ips\":[{\"address\":\"10.130.0.9/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0504 12:24:05.156884 3659 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0504 12:24:05.323506 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-internal-default W0504 12:24:05.323531 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 1936: empty IP address endpoints for service openshift-ingress/router-internal-default W0504 12:24:05.323539 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-internal-default W0504 12:24:05.323741 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-default W0504 12:24:05.323761 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-default I0504 12:24:05.452954 3659 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0504 12:24:05.486692 3659 cni.go:452] [openshift-image-registry/image-registry-59d855649b-d7k5t 1a1cf19c8565929862c78097336233de5254c56ea14a6bd440f19918b69ffa47 network default NAD default NAD key ] ADD starting CNI request [openshift-image-registry/image-registry-59d855649b-d7k5t 1a1cf19c8565929862c78097336233de5254c56ea14a6bd440f19918b69ffa47 network default NAD default NAD key ] I0504 12:24:05.496730 3659 helper_linux.go:556] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-59d855649b-d7k5t, hostIfaceName: 1a1cf19c8565929, network: default, NAD default, SandboxID: "1a1cf19c8565929862c78097336233de5254c56ea14a6bd440f19918b69ffa47", PCI device ID: , UID: "58d8461e-3db8-4ea2-afc7-5b82a3c52b3f", MAC: 0a:58:0a:82:00:0c, IPs: [10.130.0.12/23] I0504 12:24:05.543148 3659 cni.go:473] [openshift-image-registry/image-registry-59d855649b-d7k5t 1a1cf19c8565929862c78097336233de5254c56ea14a6bd440f19918b69ffa47 network default NAD default NAD key default] ADD finished CNI request [openshift-image-registry/image-registry-59d855649b-d7k5t 1a1cf19c8565929862c78097336233de5254c56ea14a6bd440f19918b69ffa47 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"d6:35:91:55:52:3b\",\"name\":\"1a1cf19c8565929\"},{\"mac\":\"0a:58:0a:82:00:0c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5186e97d-8704-4899-a232-4729b290026d\"}],\"ips\":[{\"address\":\"10.130.0.12/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0504 12:24:06.325376 3659 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0504 12:24:07.178545 3659 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0504 12:24:07.178568 3659 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0504 12:24:07.178578 3659 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0504 12:24:07.325753 3659 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0504 12:24:07.325771 3659 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0504 12:24:07.325777 3659 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0504 12:24:08.355343 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 8443: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook I0504 12:24:08.812362 3659 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-operator-76b589d668-pwgwm] creating logical port openshift-monitoring_prometheus-operator-76b589d668-pwgwm for pod on switch ip-10-0-2-63.ec2.internal I0504 12:24:08.812494 3659 kube.go:256] Updating pod openshift-monitoring/prometheus-operator-76b589d668-pwgwm I0504 12:24:08.832448 3659 pod.go:62] [openshift-monitoring/prometheus-operator-76b589d668-pwgwm] pod update took 19.983897ms I0504 12:24:08.832470 3659 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-operator-76b589d668-pwgwm] addLogicalPort annotation time took 20.011776ms I0504 12:24:08.833965 3659 pods.go:271] [openshift-monitoring/prometheus-operator-76b589d668-pwgwm] addLogicalPort took 21.624359ms, libovsdb time 1.188943ms I0504 12:24:09.199936 3659 cni.go:452] [openshift-monitoring/prometheus-operator-76b589d668-pwgwm 6dbdfa9180607746c0b7d1e106e27b6832a3564be5ac72423b6aaffa4131ffef network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-operator-76b589d668-pwgwm 6dbdfa9180607746c0b7d1e106e27b6832a3564be5ac72423b6aaffa4131ffef network default NAD default NAD key ] I0504 12:24:09.215221 3659 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-operator-76b589d668-pwgwm, hostIfaceName: 6dbdfa918060774, network: default, NAD default, SandboxID: "6dbdfa9180607746c0b7d1e106e27b6832a3564be5ac72423b6aaffa4131ffef", PCI device ID: , UID: "5b66e8d7-6b41-49ca-91fe-f0280e2ad5ad", MAC: 0a:58:0a:82:00:0d, IPs: [10.130.0.13/23] I0504 12:24:09.263374 3659 cni.go:473] [openshift-monitoring/prometheus-operator-76b589d668-pwgwm 6dbdfa9180607746c0b7d1e106e27b6832a3564be5ac72423b6aaffa4131ffef network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-operator-76b589d668-pwgwm 6dbdfa9180607746c0b7d1e106e27b6832a3564be5ac72423b6aaffa4131ffef network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"c6:71:90:fc:a2:db\",\"name\":\"6dbdfa918060774\"},{\"mac\":\"0a:58:0a:82:00:0d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4e7ec689-9103-4a4e-9906-f93f778d5856\"}],\"ips\":[{\"address\":\"10.130.0.13/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0504 12:24:09.884140 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0504 12:24:09.884172 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0504 12:24:10.317909 3659 base_network_controller_pods.go:487] [default/openshift-console/console-7d559c6f65-h7dn6] creating logical port openshift-console_console-7d559c6f65-h7dn6 for pod on switch ip-10-0-2-63.ec2.internal I0504 12:24:10.318074 3659 kube.go:256] Updating pod openshift-console/console-7d559c6f65-h7dn6 I0504 12:24:10.337211 3659 pod.go:62] [openshift-console/console-7d559c6f65-h7dn6] pod update took 19.185173ms I0504 12:24:10.337230 3659 base_network_controller_pods.go:951] [default/openshift-console/console-7d559c6f65-h7dn6] addLogicalPort annotation time took 19.209133ms I0504 12:24:10.338413 3659 pods.go:271] [openshift-console/console-7d559c6f65-h7dn6] addLogicalPort took 20.525402ms, libovsdb time 909.53µs I0504 12:24:10.988473 3659 cni.go:452] [openshift-console/console-7d559c6f65-h7dn6 c6587cd9f34743e73617e8cad54b3b85de1ac92c352f6fb5f5c5d9c6e77605ff network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-7d559c6f65-h7dn6 c6587cd9f34743e73617e8cad54b3b85de1ac92c352f6fb5f5c5d9c6e77605ff network default NAD default NAD key ] I0504 12:24:10.999453 3659 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-7d559c6f65-h7dn6, hostIfaceName: c6587cd9f34743e, network: default, NAD default, SandboxID: "c6587cd9f34743e73617e8cad54b3b85de1ac92c352f6fb5f5c5d9c6e77605ff", PCI device ID: , UID: "1ded6e4a-1ced-46dd-87a2-3f730e94f6a3", MAC: 0a:58:0a:82:00:0e, IPs: [10.130.0.14/23] I0504 12:24:11.049924 3659 cni.go:473] [openshift-console/console-7d559c6f65-h7dn6 c6587cd9f34743e73617e8cad54b3b85de1ac92c352f6fb5f5c5d9c6e77605ff network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-7d559c6f65-h7dn6 c6587cd9f34743e73617e8cad54b3b85de1ac92c352f6fb5f5c5d9c6e77605ff network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"da:90:9d:29:8d:1b\",\"name\":\"c6587cd9f34743e\"},{\"mac\":\"0a:58:0a:82:00:0e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d3ae3c27-0946-4de8-a6df-b94c34ffd1d6\"}],\"ips\":[{\"address\":\"10.130.0.14/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0504 12:24:13.002333 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0504 12:24:13.002429 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0504 12:24:13.002446 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0504 12:24:13.002466 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0504 12:24:13.002474 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0504 12:24:13.002480 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0504 12:24:14.384714 3659 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console I0504 12:24:14.616568 3659 base_network_controller_pods.go:487] [default/openshift-monitoring/alertmanager-main-0] creating logical port openshift-monitoring_alertmanager-main-0 for pod on switch ip-10-0-2-63.ec2.internal I0504 12:24:14.616700 3659 kube.go:256] Updating pod openshift-monitoring/alertmanager-main-0 I0504 12:24:14.638524 3659 pod.go:62] [openshift-monitoring/alertmanager-main-0] pod update took 21.865804ms I0504 12:24:14.638607 3659 base_network_controller_pods.go:951] [default/openshift-monitoring/alertmanager-main-0] addLogicalPort annotation time took 21.954123ms I0504 12:24:14.639929 3659 pods.go:271] [openshift-monitoring/alertmanager-main-0] addLogicalPort took 23.377801ms, libovsdb time 1.024139ms I0504 12:24:14.978755 3659 cni.go:452] [openshift-monitoring/alertmanager-main-0 480c4178cbf9b10dabbe04963aa03b173bfebc1a8a6e4a0f14041dd1b2e4a8d0 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/alertmanager-main-0 480c4178cbf9b10dabbe04963aa03b173bfebc1a8a6e4a0f14041dd1b2e4a8d0 network default NAD default NAD key ] I0504 12:24:14.988856 3659 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: alertmanager-main-0, hostIfaceName: 480c4178cbf9b10, network: default, NAD default, SandboxID: "480c4178cbf9b10dabbe04963aa03b173bfebc1a8a6e4a0f14041dd1b2e4a8d0", PCI device ID: , UID: "d84d3ac7-d5e6-49eb-b2bf-2d7b0e12e4c8", MAC: 0a:58:0a:82:00:0f, IPs: [10.130.0.15/23] I0504 12:24:15.037809 3659 cni.go:473] [openshift-monitoring/alertmanager-main-0 480c4178cbf9b10dabbe04963aa03b173bfebc1a8a6e4a0f14041dd1b2e4a8d0 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/alertmanager-main-0 480c4178cbf9b10dabbe04963aa03b173bfebc1a8a6e4a0f14041dd1b2e4a8d0 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"12:3b:3c:e4:ee:1e\",\"name\":\"480c4178cbf9b10\"},{\"mac\":\"0a:58:0a:82:00:0f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5ce22b69-0e9e-4ab0-8fc8-2c7f1665e14f\"}],\"ips\":[{\"address\":\"10.130.0.15/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0504 12:24:15.206680 3659 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console I0504 12:24:15.351968 3659 base_network_controller_pods.go:487] [default/openshift-monitoring/thanos-querier-79bc8f874b-2j7vr] creating logical port openshift-monitoring_thanos-querier-79bc8f874b-2j7vr for pod on switch ip-10-0-2-63.ec2.internal I0504 12:24:15.352102 3659 kube.go:256] Updating pod openshift-monitoring/thanos-querier-79bc8f874b-2j7vr I0504 12:24:15.372276 3659 pod.go:62] [openshift-monitoring/thanos-querier-79bc8f874b-2j7vr] pod update took 20.205101ms I0504 12:24:15.372298 3659 base_network_controller_pods.go:951] [default/openshift-monitoring/thanos-querier-79bc8f874b-2j7vr] addLogicalPort annotation time took 20.232358ms I0504 12:24:15.373549 3659 pods.go:271] [openshift-monitoring/thanos-querier-79bc8f874b-2j7vr] addLogicalPort took 21.599264ms, libovsdb time 915.363µs I0504 12:24:15.726202 3659 cni.go:452] [openshift-monitoring/thanos-querier-79bc8f874b-2j7vr 2df7c61025c791d07ca2e76123379e0f45abf139cd1e709f57821cfc10eb5890 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/thanos-querier-79bc8f874b-2j7vr 2df7c61025c791d07ca2e76123379e0f45abf139cd1e709f57821cfc10eb5890 network default NAD default NAD key ] I0504 12:24:15.735749 3659 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: thanos-querier-79bc8f874b-2j7vr, hostIfaceName: 2df7c61025c791d, network: default, NAD default, SandboxID: "2df7c61025c791d07ca2e76123379e0f45abf139cd1e709f57821cfc10eb5890", PCI device ID: , UID: "12ae2b39-3623-45bb-a258-61b8431b7986", MAC: 0a:58:0a:82:00:10, IPs: [10.130.0.16/23] I0504 12:24:15.782117 3659 cni.go:473] [openshift-monitoring/thanos-querier-79bc8f874b-2j7vr 2df7c61025c791d07ca2e76123379e0f45abf139cd1e709f57821cfc10eb5890 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/thanos-querier-79bc8f874b-2j7vr 2df7c61025c791d07ca2e76123379e0f45abf139cd1e709f57821cfc10eb5890 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"52:98:fd:eb:7e:88\",\"name\":\"2df7c61025c791d\"},{\"mac\":\"0a:58:0a:82:00:10\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/92eadf8c-110f-4635-b220-3de3fd955a93\"}],\"ips\":[{\"address\":\"10.130.0.16/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0504 12:24:17.176356 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0504 12:24:17.176376 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0504 12:24:17.176382 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0504 12:24:17.176396 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0504 12:24:17.176402 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0504 12:24:17.176406 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0504 12:24:17.340989 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0504 12:24:17.341008 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0504 12:24:17.341014 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default I0504 12:24:17.987062 3659 base_network_controller_pods.go:487] [default/openshift-monitoring/metrics-server-65f66884d8-9fddk] creating logical port openshift-monitoring_metrics-server-65f66884d8-9fddk for pod on switch ip-10-0-2-63.ec2.internal I0504 12:24:17.987157 3659 kube.go:256] Updating pod openshift-monitoring/metrics-server-65f66884d8-9fddk I0504 12:24:18.008647 3659 pod.go:62] [openshift-monitoring/metrics-server-65f66884d8-9fddk] pod update took 21.512875ms I0504 12:24:18.008671 3659 base_network_controller_pods.go:951] [default/openshift-monitoring/metrics-server-65f66884d8-9fddk] addLogicalPort annotation time took 21.542721ms I0504 12:24:18.009846 3659 pods.go:271] [openshift-monitoring/metrics-server-65f66884d8-9fddk] addLogicalPort took 22.812889ms, libovsdb time 865.478µs I0504 12:24:18.080393 3659 base_network_controller_pods.go:487] [default/openshift-monitoring/monitoring-plugin-5cbb48bf6b-4gkgb] creating logical port openshift-monitoring_monitoring-plugin-5cbb48bf6b-4gkgb for pod on switch ip-10-0-2-63.ec2.internal I0504 12:24:18.080498 3659 kube.go:256] Updating pod openshift-monitoring/monitoring-plugin-5cbb48bf6b-4gkgb I0504 12:24:18.097234 3659 pod.go:62] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-4gkgb] pod update took 16.768755ms I0504 12:24:18.097260 3659 base_network_controller_pods.go:951] [default/openshift-monitoring/monitoring-plugin-5cbb48bf6b-4gkgb] addLogicalPort annotation time took 16.802101ms I0504 12:24:18.098523 3659 pods.go:271] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-4gkgb] addLogicalPort took 18.146753ms, libovsdb time 955.843µs W0504 12:24:18.907385 3659 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console I0504 12:24:19.748967 3659 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-k8s-1] creating logical port openshift-monitoring_prometheus-k8s-1 for pod on switch ip-10-0-2-63.ec2.internal I0504 12:24:19.749089 3659 kube.go:256] Updating pod openshift-monitoring/prometheus-k8s-1 I0504 12:24:19.790489 3659 pod.go:62] [openshift-monitoring/prometheus-k8s-1] pod update took 41.434083ms I0504 12:24:19.790516 3659 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-k8s-1] addLogicalPort annotation time took 41.48243ms I0504 12:24:19.791762 3659 pods.go:271] [openshift-monitoring/prometheus-k8s-1] addLogicalPort took 42.813681ms, libovsdb time 893.779µs W0504 12:24:20.983183 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0504 12:24:20.983215 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0504 12:24:21.405583 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0504 12:24:22.949955 3659 cni.go:452] [openshift-monitoring/metrics-server-65f66884d8-9fddk a2af46a0ad8322d2f4cb6ea63e7a034ed09e475f7ae156cd50b6d4cf63785ebc network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/metrics-server-65f66884d8-9fddk a2af46a0ad8322d2f4cb6ea63e7a034ed09e475f7ae156cd50b6d4cf63785ebc network default NAD default NAD key ] I0504 12:24:22.950206 3659 cni.go:452] [openshift-monitoring/prometheus-k8s-1 8993c3e3e90b79db9ef08982661d4ed5936cad9b912d3c4c55d70c20b1d0a531 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-k8s-1 8993c3e3e90b79db9ef08982661d4ed5936cad9b912d3c4c55d70c20b1d0a531 network default NAD default NAD key ] I0504 12:24:22.952681 3659 cni.go:452] [openshift-multus/network-metrics-daemon-528nq 3b53a9b457b6ad64e7c6ca78687a1c083df28086834992dd3e0f6d368b81a574 network default NAD default NAD key ] ADD starting CNI request [openshift-multus/network-metrics-daemon-528nq 3b53a9b457b6ad64e7c6ca78687a1c083df28086834992dd3e0f6d368b81a574 network default NAD default NAD key ] I0504 12:24:22.954852 3659 cni.go:452] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-4gkgb ce63bbffde4ba248e65e28790cb1e6f70e43193948d8069cd7e14812a09f0843 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/monitoring-plugin-5cbb48bf6b-4gkgb ce63bbffde4ba248e65e28790cb1e6f70e43193948d8069cd7e14812a09f0843 network default NAD default NAD key ] I0504 12:24:22.965815 3659 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: metrics-server-65f66884d8-9fddk, hostIfaceName: a2af46a0ad8322d, network: default, NAD default, SandboxID: "a2af46a0ad8322d2f4cb6ea63e7a034ed09e475f7ae156cd50b6d4cf63785ebc", PCI device ID: , UID: "1cc695a9-64e3-4d9b-a4ba-5bbd5c713424", MAC: 0a:58:0a:82:00:11, IPs: [10.130.0.17/23] I0504 12:24:22.966338 3659 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-k8s-1, hostIfaceName: 8993c3e3e90b79d, network: default, NAD default, SandboxID: "8993c3e3e90b79db9ef08982661d4ed5936cad9b912d3c4c55d70c20b1d0a531", PCI device ID: , UID: "011ea044-2579-4571-8416-d968d195ec5d", MAC: 0a:58:0a:82:00:13, IPs: [10.130.0.19/23] I0504 12:24:22.970013 3659 helper_linux.go:556] ConfigureOVS: namespace: openshift-multus, podName: network-metrics-daemon-528nq, hostIfaceName: 3b53a9b457b6ad6, network: default, NAD default, SandboxID: "3b53a9b457b6ad64e7c6ca78687a1c083df28086834992dd3e0f6d368b81a574", PCI device ID: , UID: "b90b299e-4d95-48f1-a520-34e1ce70bd01", MAC: 0a:58:0a:82:00:04, IPs: [10.130.0.4/23] I0504 12:24:22.970067 3659 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: monitoring-plugin-5cbb48bf6b-4gkgb, hostIfaceName: ce63bbffde4ba24, network: default, NAD default, SandboxID: "ce63bbffde4ba248e65e28790cb1e6f70e43193948d8069cd7e14812a09f0843", PCI device ID: , UID: "f794230b-0754-46b0-ac77-d02de5868d7e", MAC: 0a:58:0a:82:00:12, IPs: [10.130.0.18/23] I0504 12:24:23.026309 3659 cni.go:473] [openshift-monitoring/prometheus-k8s-1 8993c3e3e90b79db9ef08982661d4ed5936cad9b912d3c4c55d70c20b1d0a531 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-k8s-1 8993c3e3e90b79db9ef08982661d4ed5936cad9b912d3c4c55d70c20b1d0a531 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"7e:84:92:7a:47:17\",\"name\":\"8993c3e3e90b79d\"},{\"mac\":\"0a:58:0a:82:00:13\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/68ffff2c-ed78-4b10-894d-413dd38c95ce\"}],\"ips\":[{\"address\":\"10.130.0.19/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0504 12:24:23.039401 3659 cni.go:473] [openshift-monitoring/metrics-server-65f66884d8-9fddk a2af46a0ad8322d2f4cb6ea63e7a034ed09e475f7ae156cd50b6d4cf63785ebc network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/metrics-server-65f66884d8-9fddk a2af46a0ad8322d2f4cb6ea63e7a034ed09e475f7ae156cd50b6d4cf63785ebc network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"d6:40:a7:4d:6a:65\",\"name\":\"a2af46a0ad8322d\"},{\"mac\":\"0a:58:0a:82:00:11\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a7ffd3b3-15aa-4040-824a-bd2f1e859cad\"}],\"ips\":[{\"address\":\"10.130.0.17/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0504 12:24:23.263986 3659 cni.go:473] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-4gkgb ce63bbffde4ba248e65e28790cb1e6f70e43193948d8069cd7e14812a09f0843 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/monitoring-plugin-5cbb48bf6b-4gkgb ce63bbffde4ba248e65e28790cb1e6f70e43193948d8069cd7e14812a09f0843 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ce:7d:97:05:4f:2f\",\"name\":\"ce63bbffde4ba24\"},{\"mac\":\"0a:58:0a:82:00:12\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a4edb5ce-d0a2-4039-b848-e9c9b9dd343b\"}],\"ips\":[{\"address\":\"10.130.0.18/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0504 12:24:23.264138 3659 cni.go:473] [openshift-multus/network-metrics-daemon-528nq 3b53a9b457b6ad64e7c6ca78687a1c083df28086834992dd3e0f6d368b81a574 network default NAD default NAD key default] ADD finished CNI request [openshift-multus/network-metrics-daemon-528nq 3b53a9b457b6ad64e7c6ca78687a1c083df28086834992dd3e0f6d368b81a574 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"0a:7e:e8:c6:f8:dd\",\"name\":\"3b53a9b457b6ad6\"},{\"mac\":\"0a:58:0a:82:00:04\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/683e24c4-3689-47cd-8893-dc69ee9d48c3\"}],\"ips\":[{\"address\":\"10.130.0.4/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0504 12:24:23.409230 3659 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads W0504 12:24:23.451214 3659 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0504 12:24:23.451240 3659 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0504 12:24:23.477342 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 8080: empty IP address endpoints for service openshift-console/downloads W0504 12:24:23.519815 3659 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0504 12:24:23.519841 3659 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0504 12:24:23.519850 3659 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0504 12:24:24.120958 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0504 12:24:24.120991 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0504 12:24:24.275411 3659 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0504 12:24:24.275429 3659 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0504 12:24:24.275436 3659 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0504 12:24:24.282067 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0504 12:24:24.346345 3659 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0504 12:24:24.346370 3659 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0504 12:24:26.174910 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0504 12:24:26.174948 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0504 12:24:26.219005 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0504 12:24:26.219045 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0504 12:24:27.286821 3659 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/monitoring-plugin W0504 12:24:27.307548 3659 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0504 12:24:27.342411 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0504 12:24:27.472584 3659 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0504 12:24:29.360179 3659 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0504 12:24:29.360204 3659 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0504 12:24:29.360210 3659 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0504 12:24:29.360214 3659 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0504 12:24:29.513750 3659 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0504 12:24:29.513771 3659 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0504 12:24:29.513777 3659 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0504 12:24:29.513782 3659 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0504 12:24:31.360927 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 9093: empty IP address endpoints for service openshift-monitoring/thanos-querier W0504 12:24:31.360955 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/thanos-querier W0504 12:24:31.360961 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 9094: empty IP address endpoints for service openshift-monitoring/thanos-querier W0504 12:24:31.360966 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/thanos-querier W0504 12:24:31.360977 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/thanos-querier W0504 12:24:31.360982 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 9094: empty IP address endpoints for service openshift-monitoring/thanos-querier W0504 12:24:31.360986 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/thanos-querier W0504 12:24:31.360990 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 9093: empty IP address endpoints for service openshift-monitoring/thanos-querier W0504 12:24:36.499222 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/thanos-querier W0504 12:24:36.499320 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 9094: empty IP address endpoints for service openshift-monitoring/thanos-querier W0504 12:24:36.499352 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/thanos-querier W0504 12:24:36.499380 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 9093: empty IP address endpoints for service openshift-monitoring/thanos-querier I0504 12:24:44.223639 3659 obj_retry.go:498] Detected object openshift-console/console-7d559c6f65-nfcdc of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0504 12:24:44.224285 3659 obj_retry.go:498] Detected object openshift-console/console-7d559c6f65-nfcdc of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0504 12:24:44.387145 3659 obj_retry.go:492] Detected object openshift-console/console-7d559c6f65-nfcdc of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:24:44.387166 3659 obj_retry.go:492] Detected object openshift-console/console-7d559c6f65-nfcdc of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:24:44.393555 3659 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-7d559c6f65-nfcdc I0504 12:24:44.393581 3659 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-7d559c6f65-nfcdc W0504 12:24:48.317998 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server W0504 12:24:48.318022 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server W0504 12:24:48.462133 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server W0504 12:24:49.185758 3659 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-service-ca-operator/metrics I0504 12:24:51.492071 3659 obj_retry.go:498] Detected object openshift-image-registry/image-registry-85b84bf8b4-vll5v of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0504 12:24:51.492660 3659 obj_retry.go:498] Detected object openshift-image-registry/image-registry-85b84bf8b4-vll5v of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0504 12:24:52.192231 3659 obj_retry.go:492] Detected object openshift-image-registry/image-registry-85b84bf8b4-vll5v of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:24:52.192262 3659 obj_retry.go:492] Detected object openshift-image-registry/image-registry-85b84bf8b4-vll5v of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:24:52.195623 3659 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-85b84bf8b4-vll5v I0504 12:24:52.195656 3659 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-85b84bf8b4-vll5v W0504 12:24:55.027548 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0504 12:24:55.027572 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0504 12:24:55.027579 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0504 12:24:55.421871 3659 cni.go:452] [openshift-console/console-7d559c6f65-h7dn6 c6587cd9f34743e73617e8cad54b3b85de1ac92c352f6fb5f5c5d9c6e77605ff network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-7d559c6f65-h7dn6 c6587cd9f34743e73617e8cad54b3b85de1ac92c352f6fb5f5c5d9c6e77605ff network default NAD default NAD key ] I0504 12:24:55.489831 3659 cni.go:473] [openshift-console/console-7d559c6f65-h7dn6 c6587cd9f34743e73617e8cad54b3b85de1ac92c352f6fb5f5c5d9c6e77605ff network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-7d559c6f65-h7dn6 c6587cd9f34743e73617e8cad54b3b85de1ac92c352f6fb5f5c5d9c6e77605ff network default NAD default NAD key default], result "{}", err I0504 12:24:55.560615 3659 obj_retry.go:498] Detected object openshift-console/console-7d559c6f65-h7dn6 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0504 12:24:55.560655 3659 pods.go:176] Deleting pod: openshift-console/console-7d559c6f65-h7dn6 I0504 12:24:55.561931 3659 pods.go:236] Attempting to release IPs for pod: openshift-console/console-7d559c6f65-h7dn6, ips: 10.130.0.14 I0504 12:24:55.561959 3659 obj_retry.go:498] Detected object openshift-console/console-7d559c6f65-h7dn6 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0504 12:24:55.867439 3659 obj_retry.go:492] Detected object openshift-console/console-7d559c6f65-h7dn6 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:24:55.867465 3659 obj_retry.go:492] Detected object openshift-console/console-7d559c6f65-h7dn6 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:24:55.873754 3659 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-7d559c6f65-h7dn6 I0504 12:24:55.873780 3659 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-7d559c6f65-h7dn6 W0504 12:25:04.245342 3659 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0504 12:25:04.274082 3659 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-insights/metrics I0504 12:25:16.368973 3659 obj_retry.go:498] Detected object openshift-ingress/router-default-7c6689f965-sncd8 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0504 12:25:16.369647 3659 obj_retry.go:498] Detected object openshift-ingress/router-default-7c6689f965-sncd8 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0504 12:25:17.297020 3659 obj_retry.go:492] Detected object openshift-ingress/router-default-7c6689f965-sncd8 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:25:17.297055 3659 obj_retry.go:492] Detected object openshift-ingress/router-default-7c6689f965-sncd8 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:25:17.303632 3659 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/router-default-7c6689f965-sncd8 I0504 12:25:17.303659 3659 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/router-default-7c6689f965-sncd8 W0504 12:25:20.578310 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0504 12:25:20.578338 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0504 12:25:20.578355 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0504 12:25:20.578364 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0504 12:25:20.727116 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0504 12:25:20.727141 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0504 12:25:35.201489 3659 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0504 12:25:35.201992 3659 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0504 12:25:35.606058 3659 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:25:35.606081 3659 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:25:35.613090 3659 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-1 I0504 12:25:35.613115 3659 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-1 I0504 12:25:39.434694 3659 cni.go:452] [openshift-monitoring/prometheus-k8s-1 8993c3e3e90b79db9ef08982661d4ed5936cad9b912d3c4c55d70c20b1d0a531 network default NAD default NAD key ] DEL starting CNI request [openshift-monitoring/prometheus-k8s-1 8993c3e3e90b79db9ef08982661d4ed5936cad9b912d3c4c55d70c20b1d0a531 network default NAD default NAD key ] I0504 12:25:39.501764 3659 cni.go:473] [openshift-monitoring/prometheus-k8s-1 8993c3e3e90b79db9ef08982661d4ed5936cad9b912d3c4c55d70c20b1d0a531 network default NAD default NAD key default] DEL finished CNI request [openshift-monitoring/prometheus-k8s-1 8993c3e3e90b79db9ef08982661d4ed5936cad9b912d3c4c55d70c20b1d0a531 network default NAD default NAD key default], result "{}", err I0504 12:25:39.563161 3659 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0504 12:25:39.563190 3659 pods.go:176] Deleting pod: openshift-monitoring/prometheus-k8s-1 I0504 12:25:39.564491 3659 pods.go:236] Attempting to release IPs for pod: openshift-monitoring/prometheus-k8s-1, ips: 10.130.0.19 I0504 12:25:39.564525 3659 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0504 12:25:39.752711 3659 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:25:39.752735 3659 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:25:39.760638 3659 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-1 I0504 12:25:39.760666 3659 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-1 I0504 12:25:39.806890 3659 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-k8s-1] creating logical port openshift-monitoring_prometheus-k8s-1 for pod on switch ip-10-0-2-63.ec2.internal I0504 12:25:39.806996 3659 kube.go:256] Updating pod openshift-monitoring/prometheus-k8s-1 I0504 12:25:39.831184 3659 pod.go:62] [openshift-monitoring/prometheus-k8s-1] pod update took 24.229492ms I0504 12:25:39.831204 3659 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-k8s-1] addLogicalPort annotation time took 24.253831ms I0504 12:25:39.832356 3659 pods.go:271] [openshift-monitoring/prometheus-k8s-1] addLogicalPort took 25.480133ms, libovsdb time 848.435µs I0504 12:25:40.165544 3659 cni.go:452] [openshift-monitoring/prometheus-k8s-1 1af6c0d3cbc4e485e9f6c1034e55f0deba83c7cff9e1792ced36b2115b6b72e6 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-k8s-1 1af6c0d3cbc4e485e9f6c1034e55f0deba83c7cff9e1792ced36b2115b6b72e6 network default NAD default NAD key ] I0504 12:25:40.182246 3659 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-k8s-1, hostIfaceName: 1af6c0d3cbc4e48, network: default, NAD default, SandboxID: "1af6c0d3cbc4e485e9f6c1034e55f0deba83c7cff9e1792ced36b2115b6b72e6", PCI device ID: , UID: "bd6eaafd-c017-4735-b5b8-b820dc96d543", MAC: 0a:58:0a:82:00:14, IPs: [10.130.0.20/23] I0504 12:25:40.232186 3659 cni.go:473] [openshift-monitoring/prometheus-k8s-1 1af6c0d3cbc4e485e9f6c1034e55f0deba83c7cff9e1792ced36b2115b6b72e6 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-k8s-1 1af6c0d3cbc4e485e9f6c1034e55f0deba83c7cff9e1792ced36b2115b6b72e6 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"12:03:09:8a:9c:5b\",\"name\":\"1af6c0d3cbc4e48\"},{\"mac\":\"0a:58:0a:82:00:14\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/404b72f9-5e33-4e0d-b045-a8913773be90\"}],\"ips\":[{\"address\":\"10.130.0.20/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0504 12:25:42.154710 3659 base_network_controller_pods.go:487] [default/openshift-console/console-86f5dfb94f-vj5rk] creating logical port openshift-console_console-86f5dfb94f-vj5rk for pod on switch ip-10-0-2-63.ec2.internal I0504 12:25:42.154867 3659 kube.go:256] Updating pod openshift-console/console-86f5dfb94f-vj5rk I0504 12:25:42.182501 3659 pod.go:62] [openshift-console/console-86f5dfb94f-vj5rk] pod update took 27.667107ms I0504 12:25:42.182526 3659 base_network_controller_pods.go:951] [default/openshift-console/console-86f5dfb94f-vj5rk] addLogicalPort annotation time took 27.696335ms I0504 12:25:42.183752 3659 pods.go:271] [openshift-console/console-86f5dfb94f-vj5rk] addLogicalPort took 29.061677ms, libovsdb time 929.31µs I0504 12:25:42.520886 3659 cni.go:452] [openshift-console/console-86f5dfb94f-vj5rk b57b34278cc4af63698c11d47eccad6ace5d06f347b62420bfac492fd091f9bd network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-86f5dfb94f-vj5rk b57b34278cc4af63698c11d47eccad6ace5d06f347b62420bfac492fd091f9bd network default NAD default NAD key ] I0504 12:25:42.530098 3659 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-86f5dfb94f-vj5rk, hostIfaceName: b57b34278cc4af6, network: default, NAD default, SandboxID: "b57b34278cc4af63698c11d47eccad6ace5d06f347b62420bfac492fd091f9bd", PCI device ID: , UID: "889e15f8-7c4e-4ab4-aedc-750a5b66af9f", MAC: 0a:58:0a:82:00:15, IPs: [10.130.0.21/23] I0504 12:25:42.785682 3659 cni.go:473] [openshift-console/console-86f5dfb94f-vj5rk b57b34278cc4af63698c11d47eccad6ace5d06f347b62420bfac492fd091f9bd network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-86f5dfb94f-vj5rk b57b34278cc4af63698c11d47eccad6ace5d06f347b62420bfac492fd091f9bd network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"22:55:4c:d2:02:d4\",\"name\":\"b57b34278cc4af6\"},{\"mac\":\"0a:58:0a:82:00:15\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b3b5ff88-5441-4b3d-9d52-fc3b21b1e69e\"}],\"ips\":[{\"address\":\"10.130.0.21/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0504 12:25:52.835066 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0504 12:26:07.229741 3659 cni.go:452] [openshift-monitoring/alertmanager-main-0 480c4178cbf9b10dabbe04963aa03b173bfebc1a8a6e4a0f14041dd1b2e4a8d0 network default NAD default NAD key ] DEL starting CNI request [openshift-monitoring/alertmanager-main-0 480c4178cbf9b10dabbe04963aa03b173bfebc1a8a6e4a0f14041dd1b2e4a8d0 network default NAD default NAD key ] I0504 12:26:07.299284 3659 cni.go:473] [openshift-monitoring/alertmanager-main-0 480c4178cbf9b10dabbe04963aa03b173bfebc1a8a6e4a0f14041dd1b2e4a8d0 network default NAD default NAD key default] DEL finished CNI request [openshift-monitoring/alertmanager-main-0 480c4178cbf9b10dabbe04963aa03b173bfebc1a8a6e4a0f14041dd1b2e4a8d0 network default NAD default NAD key default], result "{}", err I0504 12:26:07.356933 3659 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0504 12:26:07.356977 3659 pods.go:176] Deleting pod: openshift-monitoring/alertmanager-main-0 I0504 12:26:07.358218 3659 pods.go:236] Attempting to release IPs for pod: openshift-monitoring/alertmanager-main-0, ips: 10.130.0.15 I0504 12:26:07.358245 3659 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0504 12:26:07.477766 3659 obj_retry.go:498] Detected object openshift-console/console-75cfb74f4-flfh4 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0504 12:26:07.478310 3659 obj_retry.go:498] Detected object openshift-console/console-75cfb74f4-flfh4 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0504 12:26:07.744238 3659 obj_retry.go:492] Detected object openshift-console/console-75cfb74f4-flfh4 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:26:07.744265 3659 obj_retry.go:492] Detected object openshift-console/console-75cfb74f4-flfh4 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:26:07.747417 3659 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-75cfb74f4-flfh4 I0504 12:26:07.747446 3659 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-75cfb74f4-flfh4 I0504 12:26:07.891172 3659 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:26:07.891198 3659 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:26:07.900317 3659 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-0 I0504 12:26:07.900350 3659 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-0 I0504 12:26:07.941895 3659 base_network_controller_pods.go:487] [default/openshift-monitoring/alertmanager-main-0] creating logical port openshift-monitoring_alertmanager-main-0 for pod on switch ip-10-0-2-63.ec2.internal I0504 12:26:07.942026 3659 kube.go:256] Updating pod openshift-monitoring/alertmanager-main-0 I0504 12:26:07.968866 3659 pod.go:62] [openshift-monitoring/alertmanager-main-0] pod update took 26.878035ms I0504 12:26:07.968888 3659 base_network_controller_pods.go:951] [default/openshift-monitoring/alertmanager-main-0] addLogicalPort annotation time took 26.903577ms I0504 12:26:07.970115 3659 pods.go:271] [openshift-monitoring/alertmanager-main-0] addLogicalPort took 28.237511ms, libovsdb time 891.657µs I0504 12:26:08.304910 3659 cni.go:452] [openshift-monitoring/alertmanager-main-0 27aa49b57be38315b95b6d1cce23b38b5f136cb50ff15580cc8186099f99148e network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/alertmanager-main-0 27aa49b57be38315b95b6d1cce23b38b5f136cb50ff15580cc8186099f99148e network default NAD default NAD key ] I0504 12:26:08.313871 3659 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: alertmanager-main-0, hostIfaceName: 27aa49b57be3831, network: default, NAD default, SandboxID: "27aa49b57be38315b95b6d1cce23b38b5f136cb50ff15580cc8186099f99148e", PCI device ID: , UID: "f7bfe815-3215-4404-afd1-bdbbbd1ac8ea", MAC: 0a:58:0a:82:00:16, IPs: [10.130.0.22/23] I0504 12:26:08.359998 3659 cni.go:473] [openshift-monitoring/alertmanager-main-0 27aa49b57be38315b95b6d1cce23b38b5f136cb50ff15580cc8186099f99148e network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/alertmanager-main-0 27aa49b57be38315b95b6d1cce23b38b5f136cb50ff15580cc8186099f99148e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"32:df:8c:41:82:eb\",\"name\":\"27aa49b57be3831\"},{\"mac\":\"0a:58:0a:82:00:16\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5e1b6a1a-6bd5-45ad-a878-31e6e30e159c\"}],\"ips\":[{\"address\":\"10.130.0.22/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0504 12:26:17.662357 3659 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d5gcfb of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0504 12:26:17.662947 3659 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d5gcfb of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0504 12:26:17.662977 3659 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d5gcfb of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0504 12:26:17.662985 3659 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... I0504 12:26:17.662998 3659 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d5gcfb of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0504 12:26:17.663002 3659 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles to delete 1 local pods... I0504 12:26:18.155243 3659 obj_retry.go:498] Detected object openshift-console/console-75cfb74f4-4fz5p of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0504 12:26:18.155902 3659 obj_retry.go:498] Detected object openshift-console/console-75cfb74f4-4fz5p of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0504 12:26:18.538735 3659 obj_retry.go:492] Detected object openshift-console/console-75cfb74f4-4fz5p of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:26:18.538757 3659 obj_retry.go:492] Detected object openshift-console/console-75cfb74f4-4fz5p of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:26:18.545368 3659 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-75cfb74f4-4fz5p I0504 12:26:18.545398 3659 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-75cfb74f4-4fz5p I0504 12:26:18.688976 3659 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d5gcfb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:26:18.688997 3659 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d5gcfb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:26:18.689011 3659 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d5gcfb of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:26:18.689022 3659 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d5gcfb of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:26:22.710694 3659 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29631620-9lblp of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0504 12:26:22.711249 3659 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29631620-9lblp of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0504 12:26:22.913175 3659 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0504 12:26:22.924558 3659 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0504 12:26:23.585518 3659 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29631620-9lblp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:26:23.585544 3659 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29631620-9lblp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0504 12:26:28.359545 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0504 12:26:28.359572 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0504 12:26:28.359580 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0504 12:26:41.085615 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0504 12:26:41.085638 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0504 12:26:42.383832 3659 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0504 12:26:42.384484 3659 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0504 12:26:42.898001 3659 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:26:42.898055 3659 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:26:42.906625 3659 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-0 I0504 12:26:42.906657 3659 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-0 I0504 12:28:25.591413 3659 cni.go:452] [openshift-console/console-86f5dfb94f-vj5rk b57b34278cc4af63698c11d47eccad6ace5d06f347b62420bfac492fd091f9bd network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-86f5dfb94f-vj5rk b57b34278cc4af63698c11d47eccad6ace5d06f347b62420bfac492fd091f9bd network default NAD default NAD key ] I0504 12:28:25.668468 3659 cni.go:473] [openshift-console/console-86f5dfb94f-vj5rk b57b34278cc4af63698c11d47eccad6ace5d06f347b62420bfac492fd091f9bd network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-86f5dfb94f-vj5rk b57b34278cc4af63698c11d47eccad6ace5d06f347b62420bfac492fd091f9bd network default NAD default NAD key default], result "{}", err I0504 12:28:25.717409 3659 obj_retry.go:498] Detected object openshift-console/console-86f5dfb94f-vj5rk of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0504 12:28:25.717443 3659 pods.go:176] Deleting pod: openshift-console/console-86f5dfb94f-vj5rk I0504 12:28:25.718633 3659 pods.go:236] Attempting to release IPs for pod: openshift-console/console-86f5dfb94f-vj5rk, ips: 10.130.0.21 I0504 12:28:25.718660 3659 obj_retry.go:498] Detected object openshift-console/console-86f5dfb94f-vj5rk of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0504 12:28:25.729452 3659 base_network_controller_pods.go:487] [default/openshift-console/console-775db98b4d-vc48w] creating logical port openshift-console_console-775db98b4d-vc48w for pod on switch ip-10-0-2-63.ec2.internal I0504 12:28:25.729609 3659 kube.go:256] Updating pod openshift-console/console-775db98b4d-vc48w I0504 12:28:25.745515 3659 pod.go:62] [openshift-console/console-775db98b4d-vc48w] pod update took 15.942644ms I0504 12:28:25.745541 3659 base_network_controller_pods.go:951] [default/openshift-console/console-775db98b4d-vc48w] addLogicalPort annotation time took 15.972529ms I0504 12:28:25.746684 3659 pods.go:271] [openshift-console/console-775db98b4d-vc48w] addLogicalPort took 17.246448ms, libovsdb time 847.765µs I0504 12:28:26.085745 3659 cni.go:452] [openshift-console/console-775db98b4d-vc48w a698b8a89ac218e5219fc8df16ecff81ef8a4095460f3d37e6b0a4f66d0e5f7c network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-775db98b4d-vc48w a698b8a89ac218e5219fc8df16ecff81ef8a4095460f3d37e6b0a4f66d0e5f7c network default NAD default NAD key ] I0504 12:28:26.095798 3659 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-775db98b4d-vc48w, hostIfaceName: a698b8a89ac218e, network: default, NAD default, SandboxID: "a698b8a89ac218e5219fc8df16ecff81ef8a4095460f3d37e6b0a4f66d0e5f7c", PCI device ID: , UID: "555412c9-0c4c-4007-a7ed-c5ad10b29136", MAC: 0a:58:0a:82:00:17, IPs: [10.130.0.23/23] I0504 12:28:26.142067 3659 cni.go:473] [openshift-console/console-775db98b4d-vc48w a698b8a89ac218e5219fc8df16ecff81ef8a4095460f3d37e6b0a4f66d0e5f7c network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-775db98b4d-vc48w a698b8a89ac218e5219fc8df16ecff81ef8a4095460f3d37e6b0a4f66d0e5f7c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"32:da:05:49:0c:1d\",\"name\":\"a698b8a89ac218e\"},{\"mac\":\"0a:58:0a:82:00:17\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d1278e82-5541-436c-9cc9-af816f21df24\"}],\"ips\":[{\"address\":\"10.130.0.23/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0504 12:28:26.460820 3659 obj_retry.go:492] Detected object openshift-console/console-86f5dfb94f-vj5rk of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:28:26.460847 3659 obj_retry.go:492] Detected object openshift-console/console-86f5dfb94f-vj5rk of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:28:26.469937 3659 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-86f5dfb94f-vj5rk I0504 12:28:26.469960 3659 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-86f5dfb94f-vj5rk I0504 12:28:36.313533 3659 obj_retry.go:498] Detected object openshift-console/console-86f5dfb94f-mrsfl of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0504 12:28:36.314090 3659 obj_retry.go:498] Detected object openshift-console/console-86f5dfb94f-mrsfl of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0504 12:28:36.505170 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0504 12:28:36.646180 3659 obj_retry.go:492] Detected object openshift-console/console-86f5dfb94f-mrsfl of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:28:36.646207 3659 obj_retry.go:492] Detected object openshift-console/console-86f5dfb94f-mrsfl of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:28:36.652911 3659 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-86f5dfb94f-mrsfl I0504 12:28:36.652936 3659 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-86f5dfb94f-mrsfl I0504 12:28:45.390326 3659 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0504 12:29:05.793376 3659 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=7 I0504 12:29:27.911359 3659 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=6 I0504 12:29:34.901319 3659 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=7 I0504 12:29:52.791809 3659 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=70 I0504 12:30:04.543589 3659 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29631630-jbhln of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0504 12:30:04.544254 3659 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29631630-jbhln of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0504 12:30:05.434879 3659 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29631630-jbhln of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:30:05.434903 3659 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29631630-jbhln of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:30:07.880830 3659 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=7 I0504 12:30:36.922765 3659 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=8 I0504 12:30:48.794737 3659 namespace.go:98] [tsf] adding namespace I0504 12:30:48.795785 3659 namespace.go:102] [tsf] adding namespace took 1.017169ms I0504 12:30:48.815277 3659 namespace.go:142] [tsf] updating namespace I0504 12:30:48.826675 3659 namespace.go:142] [tsf] updating namespace I0504 12:30:53.821817 3659 namespace.go:98] [tsf-tas] adding namespace I0504 12:30:53.821980 3659 namespace.go:98] [tsf-keycloak] adding namespace I0504 12:30:53.822220 3659 namespace.go:98] [konflux-operator] adding namespace I0504 12:30:53.822333 3659 namespace.go:98] [konflux-ui] adding namespace I0504 12:30:53.822510 3659 namespace.go:98] [rhbk-operator] adding namespace I0504 12:30:53.822659 3659 namespace.go:98] [tsf-tpa] adding namespace I0504 12:30:53.822942 3659 namespace.go:102] [tsf-tas] adding namespace took 1.093197ms I0504 12:30:53.822982 3659 namespace.go:98] [rhtpa-operator] adding namespace I0504 12:30:53.823692 3659 namespace.go:102] [rhtpa-operator] adding namespace took 696.665µs I0504 12:30:53.824369 3659 namespace.go:102] [tsf-keycloak] adding namespace took 2.367704ms I0504 12:30:53.824802 3659 namespace.go:102] [konflux-operator] adding namespace took 2.560103ms I0504 12:30:53.825417 3659 namespace.go:102] [konflux-ui] adding namespace took 3.067832ms I0504 12:30:53.825453 3659 namespace.go:98] [cert-manager-operator] adding namespace I0504 12:30:53.825924 3659 namespace.go:102] [rhbk-operator] adding namespace took 3.393006ms I0504 12:30:53.826468 3659 namespace.go:102] [tsf-tpa] adding namespace took 3.793238ms I0504 12:30:53.826875 3659 namespace.go:102] [cert-manager-operator] adding namespace took 1.413958ms I0504 12:30:53.842125 3659 namespace.go:142] [tsf-tas] updating namespace I0504 12:30:53.867428 3659 namespace.go:142] [tsf-tas] updating namespace I0504 12:30:53.879333 3659 namespace.go:142] [tsf-keycloak] updating namespace I0504 12:30:53.905368 3659 namespace.go:142] [tsf-keycloak] updating namespace I0504 12:30:53.909657 3659 namespace.go:142] [rhtpa-operator] updating namespace I0504 12:30:53.909852 3659 namespace.go:142] [konflux-operator] updating namespace I0504 12:30:53.912308 3659 namespace.go:142] [konflux-ui] updating namespace I0504 12:30:53.917263 3659 namespace.go:142] [tsf-tpa] updating namespace I0504 12:30:53.921778 3659 namespace.go:142] [rhbk-operator] updating namespace I0504 12:30:53.922717 3659 namespace.go:142] [cert-manager-operator] updating namespace I0504 12:30:53.973671 3659 namespace.go:142] [tsf-keycloak] updating namespace I0504 12:30:53.973814 3659 namespace.go:142] [tsf-tas] updating namespace I0504 12:30:53.979675 3659 namespace.go:142] [konflux-operator] updating namespace I0504 12:30:53.994607 3659 namespace.go:142] [konflux-operator] updating namespace I0504 12:30:54.002107 3659 namespace.go:142] [konflux-ui] updating namespace I0504 12:30:54.021206 3659 namespace.go:142] [rhbk-operator] updating namespace I0504 12:30:54.033607 3659 namespace.go:142] [konflux-ui] updating namespace I0504 12:30:54.056649 3659 namespace.go:142] [tsf-tpa] updating namespace I0504 12:30:54.062004 3659 namespace.go:142] [rhbk-operator] updating namespace I0504 12:30:54.079357 3659 namespace.go:142] [tsf-tpa] updating namespace I0504 12:30:54.082579 3659 namespace.go:142] [cert-manager-operator] updating namespace I0504 12:30:54.097089 3659 namespace.go:142] [cert-manager-operator] updating namespace I0504 12:30:54.115087 3659 namespace.go:142] [rhtpa-operator] updating namespace I0504 12:30:54.135212 3659 namespace.go:142] [rhtpa-operator] updating namespace I0504 12:30:55.064367 3659 namespace.go:142] [tsf-keycloak] updating namespace I0504 12:30:55.064425 3659 namespace.go:142] [tsf-tpa] updating namespace I0504 12:31:01.804616 3659 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2blgpl of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0504 12:31:01.805159 3659 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2blgpl of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0504 12:31:01.805185 3659 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2blgpl of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0504 12:31:01.805194 3659 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0504 12:31:01.805206 3659 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2blgpl of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0504 12:31:01.805211 3659 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0504 12:31:02.796566 3659 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51gc6dr of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0504 12:31:02.797112 3659 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51gc6dr of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0504 12:31:02.797135 3659 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51gc6dr of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0504 12:31:02.797143 3659 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0504 12:31:02.797156 3659 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51gc6dr of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0504 12:31:02.797160 3659 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0504 12:31:02.829484 3659 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2blgpl of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:31:02.829506 3659 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2blgpl of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:31:02.829519 3659 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2blgpl of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:31:02.829538 3659 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2blgpl of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:31:03.827535 3659 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51gc6dr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:31:03.827559 3659 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51gc6dr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:31:03.827574 3659 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51gc6dr of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:31:03.827588 3659 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51gc6dr of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:31:05.837107 3659 obj_retry.go:498] Detected object openshift-marketplace/5ec6e5efb8778306fa5cc8dea8c4cbe66f61728c35992f4c74484f827bmtkkn of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0504 12:31:05.837714 3659 obj_retry.go:498] Detected object openshift-marketplace/5ec6e5efb8778306fa5cc8dea8c4cbe66f61728c35992f4c74484f827bmtkkn of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0504 12:31:05.837737 3659 obj_retry.go:498] Detected object openshift-marketplace/5ec6e5efb8778306fa5cc8dea8c4cbe66f61728c35992f4c74484f827bmtkkn of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0504 12:31:05.837746 3659 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0504 12:31:05.837759 3659 obj_retry.go:498] Detected object openshift-marketplace/5ec6e5efb8778306fa5cc8dea8c4cbe66f61728c35992f4c74484f827bmtkkn of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0504 12:31:05.837763 3659 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0504 12:31:06.943111 3659 obj_retry.go:492] Detected object openshift-marketplace/5ec6e5efb8778306fa5cc8dea8c4cbe66f61728c35992f4c74484f827bmtkkn of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:31:06.943147 3659 obj_retry.go:492] Detected object openshift-marketplace/5ec6e5efb8778306fa5cc8dea8c4cbe66f61728c35992f4c74484f827bmtkkn of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:31:06.943165 3659 obj_retry.go:492] Detected object openshift-marketplace/5ec6e5efb8778306fa5cc8dea8c4cbe66f61728c35992f4c74484f827bmtkkn of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:31:06.943184 3659 obj_retry.go:492] Detected object openshift-marketplace/5ec6e5efb8778306fa5cc8dea8c4cbe66f61728c35992f4c74484f827bmtkkn of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:31:07.857636 3659 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d645bf of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0504 12:31:07.858139 3659 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d645bf of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0504 12:31:07.858167 3659 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d645bf of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0504 12:31:07.858175 3659 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0504 12:31:07.858192 3659 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d645bf of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0504 12:31:07.858199 3659 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0504 12:31:07.878509 3659 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7kjq7t of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0504 12:31:07.878882 3659 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7kjq7t of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0504 12:31:07.878903 3659 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7kjq7t of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0504 12:31:07.878912 3659 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0504 12:31:07.878923 3659 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7kjq7t of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0504 12:31:07.878927 3659 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0504 12:31:07.924022 3659 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=9 I0504 12:31:07.994864 3659 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7kjq7t of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:31:07.994880 3659 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7kjq7t of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:31:07.994891 3659 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7kjq7t of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:31:07.994902 3659 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7kjq7t of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:31:08.015545 3659 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d645bf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:31:08.015560 3659 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d645bf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:31:08.015572 3659 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d645bf of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:31:08.015584 3659 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d645bf of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:31:08.866791 3659 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136gbpjx of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0504 12:31:08.867457 3659 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136gbpjx of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0504 12:31:08.867487 3659 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136gbpjx of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0504 12:31:08.867501 3659 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0504 12:31:08.867516 3659 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136gbpjx of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0504 12:31:08.867524 3659 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0504 12:31:09.892669 3659 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136gbpjx of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:31:09.892691 3659 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136gbpjx of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:31:09.892707 3659 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136gbpjx of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:31:09.892721 3659 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136gbpjx of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:31:13.793192 3659 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=53 W0504 12:31:17.001312 3659 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0504 12:31:24.598145 3659 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-z9hhm] creating logical port openshift-deployment-validation-operator_deployment-validation-operator-5d7684fb5d-z9hhm for pod on switch ip-10-0-2-63.ec2.internal I0504 12:31:24.598266 3659 kube.go:256] Updating pod openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-z9hhm I0504 12:31:24.615329 3659 pod.go:62] [openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-z9hhm] pod update took 17.090599ms I0504 12:31:24.615356 3659 base_network_controller_pods.go:951] [default/openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-z9hhm] addLogicalPort annotation time took 17.123573ms I0504 12:31:24.616706 3659 pods.go:271] [openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-z9hhm] addLogicalPort took 18.578305ms, libovsdb time 949.541µs I0504 12:31:24.789835 3659 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=63 I0504 12:31:24.955016 3659 cni.go:452] [openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-z9hhm 4b2f601748514f93e3b1674a91584a8b87ee77825527e37349a2fddf86bdf884 network default NAD default NAD key ] ADD starting CNI request [openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-z9hhm 4b2f601748514f93e3b1674a91584a8b87ee77825527e37349a2fddf86bdf884 network default NAD default NAD key ] I0504 12:31:24.964367 3659 helper_linux.go:556] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: deployment-validation-operator-5d7684fb5d-z9hhm, hostIfaceName: 4b2f601748514f9, network: default, NAD default, SandboxID: "4b2f601748514f93e3b1674a91584a8b87ee77825527e37349a2fddf86bdf884", PCI device ID: , UID: "a6c8645c-747c-4d5b-b74b-82d8462a5c77", MAC: 0a:58:0a:82:00:18, IPs: [10.130.0.24/23] I0504 12:31:25.015494 3659 cni.go:473] [openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-z9hhm 4b2f601748514f93e3b1674a91584a8b87ee77825527e37349a2fddf86bdf884 network default NAD default NAD key default] ADD finished CNI request [openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-z9hhm 4b2f601748514f93e3b1674a91584a8b87ee77825527e37349a2fddf86bdf884 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"4e:8e:15:32:f7:80\",\"name\":\"4b2f601748514f9\"},{\"mac\":\"0a:58:0a:82:00:18\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/fecafc4c-259e-42dd-b20e-86a6b773801d\"}],\"ips\":[{\"address\":\"10.130.0.24/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0504 12:31:28.190205 3659 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0504 12:31:36.877925 3659 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=9 I0504 12:31:37.917562 3659 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=8 W0504 12:31:39.207044 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 8383: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0504 12:31:39.277221 3659 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... W0504 12:31:39.277279 3659 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-deployment-validation-operator/deployment-validation-operator-65d68cbd6c-jcq2w IP address from the namespace address-set, err: pod openshift-deployment-validation-operator/deployment-validation-operator-65d68cbd6c-jcq2w: no pod IPs found I0504 12:31:40.746609 3659 namespace.go:142] [openshift-operators] updating namespace I0504 12:31:40.758322 3659 namespace.go:142] [openshift-operators] updating namespace I0504 12:31:43.562252 3659 namespace.go:98] [cert-manager] adding namespace I0504 12:31:43.563148 3659 namespace.go:102] [cert-manager] adding namespace took 871.955µs I0504 12:31:43.573194 3659 namespace.go:142] [cert-manager] updating namespace I0504 12:31:43.573229 3659 namespace.go:142] [cert-manager] updating namespace I0504 12:31:47.612794 3659 base_network_controller_pods.go:487] [default/cert-manager/cert-manager-cainjector-5d99449ffd-55m5s] creating logical port cert-manager_cert-manager-cainjector-5d99449ffd-55m5s for pod on switch ip-10-0-2-63.ec2.internal I0504 12:31:47.612925 3659 kube.go:256] Updating pod cert-manager/cert-manager-cainjector-5d99449ffd-55m5s I0504 12:31:47.631180 3659 pod.go:62] [cert-manager/cert-manager-cainjector-5d99449ffd-55m5s] pod update took 18.281423ms I0504 12:31:47.631203 3659 base_network_controller_pods.go:951] [default/cert-manager/cert-manager-cainjector-5d99449ffd-55m5s] addLogicalPort annotation time took 18.307489ms I0504 12:31:47.632346 3659 pods.go:271] [cert-manager/cert-manager-cainjector-5d99449ffd-55m5s] addLogicalPort took 19.566046ms, libovsdb time 823.164µs I0504 12:31:47.987803 3659 cni.go:452] [cert-manager/cert-manager-cainjector-5d99449ffd-55m5s 3b3cab2b9eadbcb1c8b85805f39a4be4581cfcadc6a788328af7c719d249f454 network default NAD default NAD key ] ADD starting CNI request [cert-manager/cert-manager-cainjector-5d99449ffd-55m5s 3b3cab2b9eadbcb1c8b85805f39a4be4581cfcadc6a788328af7c719d249f454 network default NAD default NAD key ] I0504 12:31:47.996582 3659 helper_linux.go:556] ConfigureOVS: namespace: cert-manager, podName: cert-manager-cainjector-5d99449ffd-55m5s, hostIfaceName: 3b3cab2b9eadbcb, network: default, NAD default, SandboxID: "3b3cab2b9eadbcb1c8b85805f39a4be4581cfcadc6a788328af7c719d249f454", PCI device ID: , UID: "99c27e97-4525-4bad-807e-7a73f526785c", MAC: 0a:58:0a:82:00:19, IPs: [10.130.0.25/23] I0504 12:31:48.025689 3659 base_network_controller_pods.go:487] [default/konflux-operator/konflux-operator-controller-manager-6f57cf498-k9tlr] creating logical port konflux-operator_konflux-operator-controller-manager-6f57cf498-k9tlr for pod on switch ip-10-0-2-63.ec2.internal I0504 12:31:48.025851 3659 kube.go:256] Updating pod konflux-operator/konflux-operator-controller-manager-6f57cf498-k9tlr I0504 12:31:48.045325 3659 cni.go:473] [cert-manager/cert-manager-cainjector-5d99449ffd-55m5s 3b3cab2b9eadbcb1c8b85805f39a4be4581cfcadc6a788328af7c719d249f454 network default NAD default NAD key default] ADD finished CNI request [cert-manager/cert-manager-cainjector-5d99449ffd-55m5s 3b3cab2b9eadbcb1c8b85805f39a4be4581cfcadc6a788328af7c719d249f454 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"32:4b:be:06:bf:c9\",\"name\":\"3b3cab2b9eadbcb\"},{\"mac\":\"0a:58:0a:82:00:19\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/10c37e38-abc7-4bf3-8242-4c6ba73b905a\"}],\"ips\":[{\"address\":\"10.130.0.25/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0504 12:31:48.050065 3659 pod.go:62] [konflux-operator/konflux-operator-controller-manager-6f57cf498-k9tlr] pod update took 24.238013ms I0504 12:31:48.050089 3659 base_network_controller_pods.go:951] [default/konflux-operator/konflux-operator-controller-manager-6f57cf498-k9tlr] addLogicalPort annotation time took 24.265115ms I0504 12:31:48.051194 3659 pods.go:271] [konflux-operator/konflux-operator-controller-manager-6f57cf498-k9tlr] addLogicalPort took 25.51951ms, libovsdb time 796.445µs I0504 12:31:48.385990 3659 cni.go:452] [konflux-operator/konflux-operator-controller-manager-6f57cf498-k9tlr bbc5fcc22504c6e15fa0ee473ed1f2f13bfccfa04fbbd46f149e2770240d2546 network default NAD default NAD key ] ADD starting CNI request [konflux-operator/konflux-operator-controller-manager-6f57cf498-k9tlr bbc5fcc22504c6e15fa0ee473ed1f2f13bfccfa04fbbd46f149e2770240d2546 network default NAD default NAD key ] I0504 12:31:48.395521 3659 helper_linux.go:556] ConfigureOVS: namespace: konflux-operator, podName: konflux-operator-controller-manager-6f57cf498-k9tlr, hostIfaceName: bbc5fcc22504c6e, network: default, NAD default, SandboxID: "bbc5fcc22504c6e15fa0ee473ed1f2f13bfccfa04fbbd46f149e2770240d2546", PCI device ID: , UID: "ef2e547d-f435-4835-b58c-df140b45d92e", MAC: 0a:58:0a:82:00:1a, IPs: [10.130.0.26/23] I0504 12:31:48.443277 3659 cni.go:473] [konflux-operator/konflux-operator-controller-manager-6f57cf498-k9tlr bbc5fcc22504c6e15fa0ee473ed1f2f13bfccfa04fbbd46f149e2770240d2546 network default NAD default NAD key default] ADD finished CNI request [konflux-operator/konflux-operator-controller-manager-6f57cf498-k9tlr bbc5fcc22504c6e15fa0ee473ed1f2f13bfccfa04fbbd46f149e2770240d2546 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"fa:1f:77:7b:a6:ff\",\"name\":\"bbc5fcc22504c6e\"},{\"mac\":\"0a:58:0a:82:00:1a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8f9d21a1-cab2-412c-9ac6-359139da7246\"}],\"ips\":[{\"address\":\"10.130.0.26/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0504 12:31:48.665475 3659 namespace.go:142] [openshift-operators] updating namespace W0504 12:31:49.162226 3659 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service cert-manager/cert-manager-webhook W0504 12:31:49.162251 3659 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service cert-manager/cert-manager-webhook I0504 12:31:49.434422 3659 base_network_controller_pods.go:487] [default/openshift-operators/tekton-operator-webhook-5864bd5898-78qn5] creating logical port openshift-operators_tekton-operator-webhook-5864bd5898-78qn5 for pod on switch ip-10-0-2-63.ec2.internal I0504 12:31:49.434613 3659 kube.go:256] Updating pod openshift-operators/tekton-operator-webhook-5864bd5898-78qn5 I0504 12:31:49.464954 3659 pod.go:62] [openshift-operators/tekton-operator-webhook-5864bd5898-78qn5] pod update took 30.38094ms I0504 12:31:49.464980 3659 base_network_controller_pods.go:951] [default/openshift-operators/tekton-operator-webhook-5864bd5898-78qn5] addLogicalPort annotation time took 30.409347ms I0504 12:31:49.466359 3659 pods.go:271] [openshift-operators/tekton-operator-webhook-5864bd5898-78qn5] addLogicalPort took 31.956458ms, libovsdb time 1.010967ms I0504 12:31:51.670883 3659 cni.go:452] [openshift-operators/tekton-operator-webhook-5864bd5898-78qn5 42b68804bbfb3e5f3f8fdf9c4d7c3c4f5c8631e067c27f900f0d550eb65e7368 network default NAD default NAD key ] ADD starting CNI request [openshift-operators/tekton-operator-webhook-5864bd5898-78qn5 42b68804bbfb3e5f3f8fdf9c4d7c3c4f5c8631e067c27f900f0d550eb65e7368 network default NAD default NAD key ] I0504 12:31:51.680611 3659 helper_linux.go:556] ConfigureOVS: namespace: openshift-operators, podName: tekton-operator-webhook-5864bd5898-78qn5, hostIfaceName: 42b68804bbfb3e5, network: default, NAD default, SandboxID: "42b68804bbfb3e5f3f8fdf9c4d7c3c4f5c8631e067c27f900f0d550eb65e7368", PCI device ID: , UID: "8e014f9e-d11f-4a7f-a0cc-73201379c042", MAC: 0a:58:0a:82:00:1b, IPs: [10.130.0.27/23] I0504 12:31:51.731762 3659 cni.go:473] [openshift-operators/tekton-operator-webhook-5864bd5898-78qn5 42b68804bbfb3e5f3f8fdf9c4d7c3c4f5c8631e067c27f900f0d550eb65e7368 network default NAD default NAD key default] ADD finished CNI request [openshift-operators/tekton-operator-webhook-5864bd5898-78qn5 42b68804bbfb3e5f3f8fdf9c4d7c3c4f5c8631e067c27f900f0d550eb65e7368 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"e6:26:09:9d:79:69\",\"name\":\"42b68804bbfb3e5\"},{\"mac\":\"0a:58:0a:82:00:1b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/caa05388-2dc7-459a-9014-d62652122268\"}],\"ips\":[{\"address\":\"10.130.0.27/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0504 12:31:52.609555 3659 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0504 12:31:55.932020 3659 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=9 W0504 12:31:56.620231 3659 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0504 12:31:56.646406 3659 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0504 12:31:56.828294 3659 namespace.go:98] [trusted-artifact-signer] adding namespace I0504 12:31:56.829174 3659 namespace.go:102] [trusted-artifact-signer] adding namespace took 826.366µs I0504 12:31:56.836020 3659 namespace.go:142] [trusted-artifact-signer] updating namespace I0504 12:31:56.870260 3659 namespace.go:142] [trusted-artifact-signer] updating namespace I0504 12:31:56.990131 3659 base_network_controller_pods.go:487] [default/trusted-artifact-signer/cli-server-7d56d9fd65-6nxq9] creating logical port trusted-artifact-signer_cli-server-7d56d9fd65-6nxq9 for pod on switch ip-10-0-2-63.ec2.internal I0504 12:31:56.990220 3659 kube.go:256] Updating pod trusted-artifact-signer/cli-server-7d56d9fd65-6nxq9 I0504 12:31:57.016165 3659 kube.go:256] Updating pod trusted-artifact-signer/cli-server-7d56d9fd65-6nxq9 I0504 12:31:57.062441 3659 pod.go:62] [trusted-artifact-signer/cli-server-7d56d9fd65-6nxq9] pod update took 72.245381ms I0504 12:31:57.062463 3659 base_network_controller_pods.go:951] [default/trusted-artifact-signer/cli-server-7d56d9fd65-6nxq9] addLogicalPort annotation time took 72.27195ms I0504 12:31:57.063577 3659 pods.go:271] [trusted-artifact-signer/cli-server-7d56d9fd65-6nxq9] addLogicalPort took 73.460376ms, libovsdb time 753.955µs W0504 12:31:57.074058 3659 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0504 12:31:57.089130 3659 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0504 12:31:57.193168 3659 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0504 12:31:57.324865 3659 cni.go:452] [trusted-artifact-signer/cli-server-7d56d9fd65-6nxq9 48d637c64cd9401ffce6ca268dc2e24b9096600cff10d9255397b91967bc38e9 network default NAD default NAD key ] ADD starting CNI request [trusted-artifact-signer/cli-server-7d56d9fd65-6nxq9 48d637c64cd9401ffce6ca268dc2e24b9096600cff10d9255397b91967bc38e9 network default NAD default NAD key ] I0504 12:31:57.334553 3659 helper_linux.go:556] ConfigureOVS: namespace: trusted-artifact-signer, podName: cli-server-7d56d9fd65-6nxq9, hostIfaceName: 48d637c64cd9401, network: default, NAD default, SandboxID: "48d637c64cd9401ffce6ca268dc2e24b9096600cff10d9255397b91967bc38e9", PCI device ID: , UID: "9ca4b7b4-55df-4d28-b2bb-394b4d0218ac", MAC: 0a:58:0a:82:00:1c, IPs: [10.130.0.28/23] I0504 12:31:57.383781 3659 cni.go:473] [trusted-artifact-signer/cli-server-7d56d9fd65-6nxq9 48d637c64cd9401ffce6ca268dc2e24b9096600cff10d9255397b91967bc38e9 network default NAD default NAD key default] ADD finished CNI request [trusted-artifact-signer/cli-server-7d56d9fd65-6nxq9 48d637c64cd9401ffce6ca268dc2e24b9096600cff10d9255397b91967bc38e9 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ea:01:45:1c:45:de\",\"name\":\"48d637c64cd9401\"},{\"mac\":\"0a:58:0a:82:00:1c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/89450788-ce6c-4710-b256-0712515a1ad5\"}],\"ips\":[{\"address\":\"10.130.0.28/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0504 12:32:03.308329 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 8443: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0504 12:32:03.891319 3659 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=9 I0504 12:32:04.230150 3659 namespace.go:98] [openshift-pipelines] adding namespace I0504 12:32:04.230927 3659 namespace.go:102] [openshift-pipelines] adding namespace took 753.228µs I0504 12:32:04.339231 3659 namespace.go:142] [openshift-pipelines] updating namespace I0504 12:32:05.214539 3659 obj_retry.go:498] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0504 12:32:05.215095 3659 obj_retry.go:498] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0504 12:32:05.308671 3659 obj_retry.go:492] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:32:05.308692 3659 obj_retry.go:492] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:32:05.345979 3659 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-subscriptions I0504 12:32:05.346005 3659 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-subscriptions I0504 12:32:05.366828 3659 namespace.go:142] [cert-manager] updating namespace I0504 12:32:05.562199 3659 namespace.go:142] [cert-manager-operator] updating namespace I0504 12:32:05.957096 3659 namespace.go:142] [konflux-operator] updating namespace I0504 12:32:06.152201 3659 namespace.go:142] [konflux-ui] updating namespace I0504 12:32:06.385916 3659 namespace.go:142] [open-cluster-management-2q33rmlhe485oeqr03tcagaphvlpoaiv] updating namespace I0504 12:32:06.807924 3659 namespace.go:142] [rhbk-operator] updating namespace I0504 12:32:07.012833 3659 namespace.go:142] [rhtpa-operator] updating namespace I0504 12:32:07.218993 3659 namespace.go:142] [trusted-artifact-signer] updating namespace I0504 12:32:07.422011 3659 namespace.go:142] [tsf] updating namespace I0504 12:32:07.613178 3659 namespace.go:142] [tsf-keycloak] updating namespace I0504 12:32:07.814959 3659 namespace.go:142] [tsf-tas] updating namespace I0504 12:32:08.014500 3659 namespace.go:142] [tsf-tpa] updating namespace I0504 12:32:08.164027 3659 namespace.go:142] [cert-manager] updating namespace I0504 12:32:08.176128 3659 namespace.go:142] [cert-manager-operator] updating namespace I0504 12:32:08.193961 3659 namespace.go:142] [default] updating namespace I0504 12:32:08.213183 3659 namespace.go:142] [konflux-operator] updating namespace I0504 12:32:08.230812 3659 namespace.go:142] [konflux-ui] updating namespace I0504 12:32:08.246168 3659 namespace.go:142] [open-cluster-management-2q33rmlhe485oeqr03tcagaphvlpoaiv] updating namespace I0504 12:32:08.259063 3659 namespace.go:142] [openshift] updating namespace I0504 12:32:08.269685 3659 namespace.go:142] [rhbk-operator] updating namespace I0504 12:32:08.281486 3659 namespace.go:142] [rhtpa-operator] updating namespace I0504 12:32:08.289862 3659 namespace.go:142] [trusted-artifact-signer] updating namespace I0504 12:32:08.299240 3659 namespace.go:142] [tsf] updating namespace I0504 12:32:08.311171 3659 namespace.go:142] [tsf-keycloak] updating namespace I0504 12:32:08.321385 3659 namespace.go:142] [tsf-tas] updating namespace I0504 12:32:08.332754 3659 namespace.go:142] [tsf-tpa] updating namespace I0504 12:32:08.391618 3659 namespace.go:142] [cert-manager] updating namespace I0504 12:32:08.492876 3659 namespace.go:142] [cert-manager-operator] updating namespace I0504 12:32:08.624595 3659 namespace.go:142] [default] updating namespace I0504 12:32:08.748114 3659 namespace.go:142] [konflux-operator] updating namespace I0504 12:32:08.875512 3659 namespace.go:142] [konflux-ui] updating namespace I0504 12:32:09.234848 3659 namespace.go:142] [open-cluster-management-2q33rmlhe485oeqr03tcagaphvlpoaiv] updating namespace I0504 12:32:09.631426 3659 namespace.go:142] [openshift] updating namespace I0504 12:32:09.796198 3659 namespace.go:142] [rhbk-operator] updating namespace I0504 12:32:09.983448 3659 namespace.go:142] [rhtpa-operator] updating namespace I0504 12:32:10.084721 3659 namespace.go:142] [trusted-artifact-signer] updating namespace I0504 12:32:10.194780 3659 namespace.go:142] [tsf] updating namespace I0504 12:32:10.308952 3659 namespace.go:142] [tsf-keycloak] updating namespace I0504 12:32:10.409065 3659 namespace.go:142] [tsf-tas] updating namespace I0504 12:32:10.479712 3659 namespace.go:142] [tsf-tpa] updating namespace I0504 12:32:11.268496 3659 namespace.go:142] [openshift-pipelines] updating namespace W0504 12:32:17.108070 3659 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-tpa/tpa-pgsql W0504 12:32:17.137427 3659 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-keycloak/keycloak-pgsql I0504 12:32:21.297407 3659 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-spsj9] creating logical port openshift-pipelines_tekton-pipelines-controller-86bd8d9bfc-spsj9 for pod on switch ip-10-0-2-63.ec2.internal I0504 12:32:21.297524 3659 kube.go:256] Updating pod openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-spsj9 I0504 12:32:21.324984 3659 pod.go:62] [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-spsj9] pod update took 27.501715ms I0504 12:32:21.325012 3659 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-spsj9] addLogicalPort annotation time took 27.534997ms I0504 12:32:21.326394 3659 pods.go:271] [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-spsj9] addLogicalPort took 29.013555ms, libovsdb time 966.165µs I0504 12:32:21.614424 3659 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-dxzjg] creating logical port openshift-pipelines_tekton-pipelines-webhook-7b6d6f4c94-dxzjg for pod on switch ip-10-0-2-63.ec2.internal I0504 12:32:21.614540 3659 kube.go:256] Updating pod openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-dxzjg I0504 12:32:21.629925 3659 pod.go:62] [openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-dxzjg] pod update took 15.412712ms I0504 12:32:21.629952 3659 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-dxzjg] addLogicalPort annotation time took 15.453014ms I0504 12:32:21.631322 3659 pods.go:271] [openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-dxzjg] addLogicalPort took 16.917232ms, libovsdb time 924.431µs I0504 12:32:21.657493 3659 cni.go:452] [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-spsj9 163106076e51d9649a78fce9d867279114bdc8ef16866bf9ebda67a435a3f6d7 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-spsj9 163106076e51d9649a78fce9d867279114bdc8ef16866bf9ebda67a435a3f6d7 network default NAD default NAD key ] I0504 12:32:21.667514 3659 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-controller-86bd8d9bfc-spsj9, hostIfaceName: 163106076e51d96, network: default, NAD default, SandboxID: "163106076e51d9649a78fce9d867279114bdc8ef16866bf9ebda67a435a3f6d7", PCI device ID: , UID: "ecb38f26-6608-4371-9453-d23e226df134", MAC: 0a:58:0a:82:00:1d, IPs: [10.130.0.29/23] I0504 12:32:21.721148 3659 cni.go:473] [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-spsj9 163106076e51d9649a78fce9d867279114bdc8ef16866bf9ebda67a435a3f6d7 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-spsj9 163106076e51d9649a78fce9d867279114bdc8ef16866bf9ebda67a435a3f6d7 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"92:46:f1:d2:c4:e5\",\"name\":\"163106076e51d96\"},{\"mac\":\"0a:58:0a:82:00:1d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a5c7ff75-049a-428f-b2cd-3f5cfb199f85\"}],\"ips\":[{\"address\":\"10.130.0.29/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0504 12:32:21.987338 3659 cni.go:452] [openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-dxzjg 59d92518efa3f2deb2841df935edc9e293708c2450341ecdc3993bd08155cbab network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-dxzjg 59d92518efa3f2deb2841df935edc9e293708c2450341ecdc3993bd08155cbab network default NAD default NAD key ] I0504 12:32:21.998557 3659 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-webhook-7b6d6f4c94-dxzjg, hostIfaceName: 59d92518efa3f2d, network: default, NAD default, SandboxID: "59d92518efa3f2deb2841df935edc9e293708c2450341ecdc3993bd08155cbab", PCI device ID: , UID: "26841a35-74bd-4f93-ad0a-3fb62cd4eb45", MAC: 0a:58:0a:82:00:1e, IPs: [10.130.0.30/23] I0504 12:32:22.049892 3659 cni.go:473] [openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-dxzjg 59d92518efa3f2deb2841df935edc9e293708c2450341ecdc3993bd08155cbab network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-dxzjg 59d92518efa3f2deb2841df935edc9e293708c2450341ecdc3993bd08155cbab network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"c2:88:ac:f9:e1:76\",\"name\":\"59d92518efa3f2d\"},{\"mac\":\"0a:58:0a:82:00:1e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/db4be642-9d3a-4655-8622-c18c0e65b6af\"}],\"ips\":[{\"address\":\"10.130.0.30/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0504 12:32:25.453596 3659 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0504 12:32:25.453624 3659 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0504 12:32:25.453632 3659 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0504 12:32:26.337454 3659 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0504 12:32:26.337485 3659 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0504 12:32:26.337493 3659 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0504 12:32:28.519483 3659 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0504 12:32:28.519513 3659 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0504 12:32:28.519521 3659 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0504 12:32:28.519527 3659 util.go:870] Failed to build global endpoints for port TCP/https-webhook: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook I0504 12:32:30.794097 3659 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=183 W0504 12:32:36.458026 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0504 12:32:36.458072 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0504 12:32:36.458080 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0504 12:32:39.475524 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0504 12:32:39.475547 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 8443: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0504 12:32:39.475555 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0504 12:32:39.475560 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook I0504 12:32:41.397851 3659 obj_retry.go:498] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0504 12:32:41.398376 3659 obj_retry.go:498] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0504 12:32:41.420897 3659 obj_retry.go:492] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:32:41.420917 3659 obj_retry.go:492] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:32:41.430001 3659 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-infrastructure I0504 12:32:41.430048 3659 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-infrastructure I0504 12:32:43.746109 3659 base_network_controller_policy.go:1179] Adding network policy tsf-keycloak/keycloak-network-policy for network default I0504 12:32:43.746763 3659 pod_selector_address_set.go:213] Created shared address set for pod selector tsf-keycloak_LS{ML:{app: keycloak,app.kubernetes.io/instance: keycloak,app.kubernetes.io/managed-by: keycloak-operator,},} I0504 12:32:43.746785 3659 base_network_controller_policy.go:1054] Policy tsf-keycloak/keycloak-network-policy added to peer address sets [tsf-keycloak_LS{ML:{app: keycloak,app.kubernetes.io/instance: keycloak,app.kubernetes.io/managed-by: keycloak-operator,},}] I0504 12:32:43.748676 3659 base_network_controller_policy.go:1232] Create network policy tsf-keycloak/keycloak-network-policy resources completed, update namespace loglevel I0504 12:32:45.877251 3659 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=11 I0504 12:32:46.421330 3659 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-triggers-webhook-6b5b45569f-kjlfp] creating logical port openshift-pipelines_tekton-triggers-webhook-6b5b45569f-kjlfp for pod on switch ip-10-0-2-63.ec2.internal I0504 12:32:46.421412 3659 kube.go:256] Updating pod openshift-pipelines/tekton-triggers-webhook-6b5b45569f-kjlfp I0504 12:32:46.445866 3659 pod.go:62] [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-kjlfp] pod update took 24.475489ms I0504 12:32:46.445886 3659 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-triggers-webhook-6b5b45569f-kjlfp] addLogicalPort annotation time took 24.501313ms I0504 12:32:46.447023 3659 pods.go:271] [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-kjlfp] addLogicalPort took 25.711533ms, libovsdb time 802.281µs I0504 12:32:46.521152 3659 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-4phsc] creating logical port openshift-pipelines_tekton-triggers-core-interceptors-76f587ddc9-4phsc for pod on switch ip-10-0-2-63.ec2.internal I0504 12:32:46.521290 3659 kube.go:256] Updating pod openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-4phsc I0504 12:32:46.551380 3659 pod.go:62] [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-4phsc] pod update took 30.120993ms I0504 12:32:46.551400 3659 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-4phsc] addLogicalPort annotation time took 30.147857ms I0504 12:32:46.552554 3659 pods.go:271] [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-4phsc] addLogicalPort took 31.420631ms, libovsdb time 816.096µs I0504 12:32:46.783243 3659 cni.go:452] [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-kjlfp caa707eb8abe491d7a6b1d72f82062365da75e7899b30a100bc80db05b221bc1 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-kjlfp caa707eb8abe491d7a6b1d72f82062365da75e7899b30a100bc80db05b221bc1 network default NAD default NAD key ] I0504 12:32:46.792382 3659 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-triggers-webhook-6b5b45569f-kjlfp, hostIfaceName: caa707eb8abe491, network: default, NAD default, SandboxID: "caa707eb8abe491d7a6b1d72f82062365da75e7899b30a100bc80db05b221bc1", PCI device ID: , UID: "806b7076-a5c5-4390-b1d5-159eee5a1327", MAC: 0a:58:0a:82:00:1f, IPs: [10.130.0.31/23] I0504 12:32:46.842487 3659 cni.go:473] [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-kjlfp caa707eb8abe491d7a6b1d72f82062365da75e7899b30a100bc80db05b221bc1 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-kjlfp caa707eb8abe491d7a6b1d72f82062365da75e7899b30a100bc80db05b221bc1 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"3a:3e:b0:62:18:ee\",\"name\":\"caa707eb8abe491\"},{\"mac\":\"0a:58:0a:82:00:1f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8037656c-695e-43f8-8b58-b435aa22c07f\"}],\"ips\":[{\"address\":\"10.130.0.31/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0504 12:32:46.875634 3659 cni.go:452] [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-4phsc 776324222be0f1c7154ecd7aca53ffecc07481fd302d85dd5e5908305dbc72aa network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-4phsc 776324222be0f1c7154ecd7aca53ffecc07481fd302d85dd5e5908305dbc72aa network default NAD default NAD key ] I0504 12:32:46.885899 3659 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-triggers-core-interceptors-76f587ddc9-4phsc, hostIfaceName: 776324222be0f1c, network: default, NAD default, SandboxID: "776324222be0f1c7154ecd7aca53ffecc07481fd302d85dd5e5908305dbc72aa", PCI device ID: , UID: "50a1d056-9057-471c-a2e6-47c32016487e", MAC: 0a:58:0a:82:00:20, IPs: [10.130.0.32/23] I0504 12:32:46.934541 3659 cni.go:473] [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-4phsc 776324222be0f1c7154ecd7aca53ffecc07481fd302d85dd5e5908305dbc72aa network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-4phsc 776324222be0f1c7154ecd7aca53ffecc07481fd302d85dd5e5908305dbc72aa network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"da:88:05:90:1b:43\",\"name\":\"776324222be0f1c\"},{\"mac\":\"0a:58:0a:82:00:20\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/24bb7f79-4c7b-41a0-b225-716c02944874\"}],\"ips\":[{\"address\":\"10.130.0.32/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0504 12:32:50.571627 3659 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors W0504 12:33:01.591761 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 8443: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors I0504 12:33:03.801459 3659 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=757 I0504 12:33:08.284647 3659 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0504 12:33:11.383217 3659 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-postgres-0] creating logical port openshift-pipelines_tekton-results-postgres-0 for pod on switch ip-10-0-2-63.ec2.internal I0504 12:33:11.383326 3659 kube.go:256] Updating pod openshift-pipelines/tekton-results-postgres-0 I0504 12:33:11.396265 3659 pod.go:62] [openshift-pipelines/tekton-results-postgres-0] pod update took 12.969402ms I0504 12:33:11.396286 3659 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-results-postgres-0] addLogicalPort annotation time took 12.996361ms I0504 12:33:11.397490 3659 pods.go:271] [openshift-pipelines/tekton-results-postgres-0] addLogicalPort took 14.29662ms, libovsdb time 827.998µs I0504 12:33:19.251686 3659 cni.go:452] [openshift-pipelines/tekton-results-postgres-0 59bae2998a109f499b797f7c77e00b0338d39276d755b78c54328553fa55e766 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-results-postgres-0 59bae2998a109f499b797f7c77e00b0338d39276d755b78c54328553fa55e766 network default NAD default NAD key ] I0504 12:33:19.260939 3659 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-postgres-0, hostIfaceName: 59bae2998a109f4, network: default, NAD default, SandboxID: "59bae2998a109f499b797f7c77e00b0338d39276d755b78c54328553fa55e766", PCI device ID: , UID: "9d5b6b41-6035-4062-9f15-a6a85a72c939", MAC: 0a:58:0a:82:00:21, IPs: [10.130.0.33/23] I0504 12:33:19.310022 3659 cni.go:473] [openshift-pipelines/tekton-results-postgres-0 59bae2998a109f499b797f7c77e00b0338d39276d755b78c54328553fa55e766 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-results-postgres-0 59bae2998a109f499b797f7c77e00b0338d39276d755b78c54328553fa55e766 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ca:85:a6:35:78:57\",\"name\":\"59bae2998a109f4\"},{\"mac\":\"0a:58:0a:82:00:21\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/395e3035-9dc2-44dc-96f8-9e043a74372b\"}],\"ips\":[{\"address\":\"10.130.0.33/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0504 12:33:25.803986 3659 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0504 12:33:27.136718 3659 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-api-64c5c8474c-ntpkv] creating logical port openshift-pipelines_tekton-results-api-64c5c8474c-ntpkv for pod on switch ip-10-0-2-63.ec2.internal I0504 12:33:27.136812 3659 kube.go:256] Updating pod openshift-pipelines/tekton-results-api-64c5c8474c-ntpkv I0504 12:33:27.153107 3659 pod.go:62] [openshift-pipelines/tekton-results-api-64c5c8474c-ntpkv] pod update took 16.32032ms I0504 12:33:27.153131 3659 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-results-api-64c5c8474c-ntpkv] addLogicalPort annotation time took 16.34903ms I0504 12:33:27.154381 3659 pods.go:271] [openshift-pipelines/tekton-results-api-64c5c8474c-ntpkv] addLogicalPort took 17.680111ms, libovsdb time 876.731µs I0504 12:33:28.096487 3659 cni.go:452] [openshift-pipelines/tekton-results-api-64c5c8474c-ntpkv 6cc92ac29918bab268524ebbe70b7bb100a830df96880873811308e207d36bfc network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-results-api-64c5c8474c-ntpkv 6cc92ac29918bab268524ebbe70b7bb100a830df96880873811308e207d36bfc network default NAD default NAD key ] I0504 12:33:28.105589 3659 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-api-64c5c8474c-ntpkv, hostIfaceName: 6cc92ac29918bab, network: default, NAD default, SandboxID: "6cc92ac29918bab268524ebbe70b7bb100a830df96880873811308e207d36bfc", PCI device ID: , UID: "28139701-8801-41e3-af1f-d686547a76f9", MAC: 0a:58:0a:82:00:22, IPs: [10.130.0.34/23] I0504 12:33:28.156137 3659 cni.go:473] [openshift-pipelines/tekton-results-api-64c5c8474c-ntpkv 6cc92ac29918bab268524ebbe70b7bb100a830df96880873811308e207d36bfc network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-results-api-64c5c8474c-ntpkv 6cc92ac29918bab268524ebbe70b7bb100a830df96880873811308e207d36bfc network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"22:84:d0:72:6c:09\",\"name\":\"6cc92ac29918bab\"},{\"mac\":\"0a:58:0a:82:00:22\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c97e0c37-14a3-488b-a5e0-0f7c9f97a43f\"}],\"ips\":[{\"address\":\"10.130.0.34/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0504 12:33:32.783115 3659 util.go:870] Failed to build global endpoints for port TCP/prometheus: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0504 12:33:32.783140 3659 util.go:870] Failed to build global endpoints for port TCP/profiling: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0504 12:33:32.783146 3659 util.go:870] Failed to build global endpoints for port TCP/server: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0504 12:33:38.820843 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0504 12:33:38.820869 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 6060: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0504 12:33:38.820875 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service I0504 12:33:58.524070 3659 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-fzfm4] creating logical port openshift-pipelines_pipelines-as-code-watcher-7947cf58bc-fzfm4 for pod on switch ip-10-0-2-63.ec2.internal I0504 12:33:58.524157 3659 kube.go:256] Updating pod openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-fzfm4 I0504 12:33:58.540134 3659 pod.go:62] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-fzfm4] pod update took 15.99831ms I0504 12:33:58.540153 3659 base_network_controller_pods.go:951] [default/openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-fzfm4] addLogicalPort annotation time took 16.023628ms I0504 12:33:58.541378 3659 pods.go:271] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-fzfm4] addLogicalPort took 17.322919ms, libovsdb time 802.105µs I0504 12:33:58.904873 3659 cni.go:452] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-fzfm4 17e02fa3e0054e5e10c13dc935cd4a9595aff477baa0f9838bd8078215540514 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-fzfm4 17e02fa3e0054e5e10c13dc935cd4a9595aff477baa0f9838bd8078215540514 network default NAD default NAD key ] I0504 12:33:58.914208 3659 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-as-code-watcher-7947cf58bc-fzfm4, hostIfaceName: 17e02fa3e0054e5, network: default, NAD default, SandboxID: "17e02fa3e0054e5e10c13dc935cd4a9595aff477baa0f9838bd8078215540514", PCI device ID: , UID: "3b675ed1-a667-4eee-811f-9719693984e3", MAC: 0a:58:0a:82:00:23, IPs: [10.130.0.35/23] I0504 12:33:58.964614 3659 cni.go:473] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-fzfm4 17e02fa3e0054e5e10c13dc935cd4a9595aff477baa0f9838bd8078215540514 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-fzfm4 17e02fa3e0054e5e10c13dc935cd4a9595aff477baa0f9838bd8078215540514 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ea:5c:79:35:e5:d0\",\"name\":\"17e02fa3e0054e5\"},{\"mac\":\"0a:58:0a:82:00:23\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/318d99c2-26d4-4862-b0d3-4cc51f6f7f9f\"}],\"ips\":[{\"address\":\"10.130.0.35/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0504 12:34:03.652965 3659 util.go:870] Failed to build global endpoints for port TCP/http-listener: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0504 12:34:03.652986 3659 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0504 12:34:06.945854 3659 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher W0504 12:34:13.717900 3659 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0504 12:34:13.717919 3659 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0504 12:34:13.718931 3659 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0504 12:34:13.718950 3659 util.go:870] Failed to build global endpoints for port TCP/management: empty IP address endpoints for service tsf-keycloak/keycloak-service W0504 12:34:17.978997 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher I0504 12:34:30.765913 3659 namespace.go:142] [openshift-pipelines] updating namespace I0504 12:34:42.006286 3659 obj_retry.go:498] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0504 12:34:42.006984 3659 obj_retry.go:498] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0504 12:35:03.129384 3659 obj_retry.go:498] Detected object tsf-keycloak/tsf-iam-88vn2 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0504 12:35:03.129934 3659 obj_retry.go:498] Detected object tsf-keycloak/tsf-iam-88vn2 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0504 12:35:04.032564 3659 obj_retry.go:492] Detected object tsf-keycloak/tsf-iam-88vn2 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:35:04.032590 3659 obj_retry.go:492] Detected object tsf-keycloak/tsf-iam-88vn2 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:35:42.157891 3659 obj_retry.go:492] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:35:42.157912 3659 obj_retry.go:492] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:35:42.163817 3659 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0504 12:35:42.163838 3659 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam I0504 12:35:45.883999 3659 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=5 I0504 12:35:50.381370 3659 obj_retry.go:498] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0504 12:35:50.381961 3659 obj_retry.go:498] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0504 12:35:50.408337 3659 obj_retry.go:492] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:35:50.408360 3659 obj_retry.go:492] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:35:50.414610 3659 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0504 12:35:50.414635 3659 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam I0504 12:35:52.376349 3659 base_network_controller_pods.go:487] [default/tsf-tas/segment-backup-installation-zsw85-8w25p] creating logical port tsf-tas_segment-backup-installation-zsw85-8w25p for pod on switch ip-10-0-2-63.ec2.internal I0504 12:35:52.376464 3659 kube.go:256] Updating pod tsf-tas/segment-backup-installation-zsw85-8w25p I0504 12:35:52.392700 3659 pod.go:62] [tsf-tas/segment-backup-installation-zsw85-8w25p] pod update took 16.268911ms I0504 12:35:52.392723 3659 base_network_controller_pods.go:951] [default/tsf-tas/segment-backup-installation-zsw85-8w25p] addLogicalPort annotation time took 16.295643ms I0504 12:35:52.394042 3659 pods.go:271] [tsf-tas/segment-backup-installation-zsw85-8w25p] addLogicalPort took 17.697437ms, libovsdb time 884.98µs I0504 12:35:52.736646 3659 cni.go:452] [tsf-tas/segment-backup-installation-zsw85-8w25p 05279ec80ef2587d71d1994278f94db16a42c24b1ea717b9692a3f418576143b network default NAD default NAD key ] ADD starting CNI request [tsf-tas/segment-backup-installation-zsw85-8w25p 05279ec80ef2587d71d1994278f94db16a42c24b1ea717b9692a3f418576143b network default NAD default NAD key ] I0504 12:35:52.745743 3659 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: segment-backup-installation-zsw85-8w25p, hostIfaceName: 05279ec80ef2587, network: default, NAD default, SandboxID: "05279ec80ef2587d71d1994278f94db16a42c24b1ea717b9692a3f418576143b", PCI device ID: , UID: "ebb4f649-88b5-4d6e-a5b9-3216ea2b2703", MAC: 0a:58:0a:82:00:24, IPs: [10.130.0.36/23] I0504 12:35:52.796452 3659 cni.go:473] [tsf-tas/segment-backup-installation-zsw85-8w25p 05279ec80ef2587d71d1994278f94db16a42c24b1ea717b9692a3f418576143b network default NAD default NAD key default] ADD finished CNI request [tsf-tas/segment-backup-installation-zsw85-8w25p 05279ec80ef2587d71d1994278f94db16a42c24b1ea717b9692a3f418576143b network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"62:64:3f:d9:cd:37\",\"name\":\"05279ec80ef2587\"},{\"mac\":\"0a:58:0a:82:00:24\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/dd0ff893-c2bd-4f46-ae80-3d9630aec83c\"}],\"ips\":[{\"address\":\"10.130.0.36/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0504 12:35:52.989140 3659 base_network_controller_pods.go:487] [default/tsf-tas/fulcio-server-c4f59b775-6hrxk] creating logical port tsf-tas_fulcio-server-c4f59b775-6hrxk for pod on switch ip-10-0-2-63.ec2.internal I0504 12:35:52.989260 3659 kube.go:256] Updating pod tsf-tas/fulcio-server-c4f59b775-6hrxk I0504 12:35:53.017212 3659 pod.go:62] [tsf-tas/fulcio-server-c4f59b775-6hrxk] pod update took 27.975945ms I0504 12:35:53.017232 3659 base_network_controller_pods.go:951] [default/tsf-tas/fulcio-server-c4f59b775-6hrxk] addLogicalPort annotation time took 28.000008ms I0504 12:35:53.018459 3659 pods.go:271] [tsf-tas/fulcio-server-c4f59b775-6hrxk] addLogicalPort took 29.334717ms, libovsdb time 832.583µs I0504 12:35:53.342083 3659 cni.go:452] [tsf-tas/fulcio-server-c4f59b775-6hrxk e1c2fe0a3caa5625f4216e4340d65ed077f91467db90273985bcbaf3d945e269 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/fulcio-server-c4f59b775-6hrxk e1c2fe0a3caa5625f4216e4340d65ed077f91467db90273985bcbaf3d945e269 network default NAD default NAD key ] I0504 12:35:53.352170 3659 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: fulcio-server-c4f59b775-6hrxk, hostIfaceName: e1c2fe0a3caa562, network: default, NAD default, SandboxID: "e1c2fe0a3caa5625f4216e4340d65ed077f91467db90273985bcbaf3d945e269", PCI device ID: , UID: "b4c86f10-53c8-4443-87e1-1cbf078cef68", MAC: 0a:58:0a:82:00:25, IPs: [10.130.0.37/23] I0504 12:35:53.403370 3659 cni.go:473] [tsf-tas/fulcio-server-c4f59b775-6hrxk e1c2fe0a3caa5625f4216e4340d65ed077f91467db90273985bcbaf3d945e269 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/fulcio-server-c4f59b775-6hrxk e1c2fe0a3caa5625f4216e4340d65ed077f91467db90273985bcbaf3d945e269 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"0e:2d:c0:60:72:6e\",\"name\":\"e1c2fe0a3caa562\"},{\"mac\":\"0a:58:0a:82:00:25\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3ab5e7c1-b437-44f6-9209-262f6c0df7cf\"}],\"ips\":[{\"address\":\"10.130.0.37/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0504 12:35:53.733279 3659 base_network_controller_pods.go:487] [default/tsf-tas/trillian-logsigner-6d7df4d6f5-c55tc] creating logical port tsf-tas_trillian-logsigner-6d7df4d6f5-c55tc for pod on switch ip-10-0-2-63.ec2.internal I0504 12:35:53.733428 3659 kube.go:256] Updating pod tsf-tas/trillian-logsigner-6d7df4d6f5-c55tc I0504 12:35:53.778867 3659 pod.go:62] [tsf-tas/trillian-logsigner-6d7df4d6f5-c55tc] pod update took 45.475287ms I0504 12:35:53.778889 3659 base_network_controller_pods.go:951] [default/tsf-tas/trillian-logsigner-6d7df4d6f5-c55tc] addLogicalPort annotation time took 45.500741ms I0504 12:35:53.780022 3659 pods.go:271] [tsf-tas/trillian-logsigner-6d7df4d6f5-c55tc] addLogicalPort took 46.764733ms, libovsdb time 755.334µs I0504 12:35:54.698652 3659 cni.go:452] [tsf-tas/trillian-logsigner-6d7df4d6f5-c55tc 4b610b12b4c668f527423f388df4ecdaa2889352247fbec0b65a8a9589c88a6d network default NAD default NAD key ] ADD starting CNI request [tsf-tas/trillian-logsigner-6d7df4d6f5-c55tc 4b610b12b4c668f527423f388df4ecdaa2889352247fbec0b65a8a9589c88a6d network default NAD default NAD key ] I0504 12:35:54.708480 3659 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: trillian-logsigner-6d7df4d6f5-c55tc, hostIfaceName: 4b610b12b4c668f, network: default, NAD default, SandboxID: "4b610b12b4c668f527423f388df4ecdaa2889352247fbec0b65a8a9589c88a6d", PCI device ID: , UID: "6e672309-1a33-4573-9c7f-9ff6d7b312ef", MAC: 0a:58:0a:82:00:26, IPs: [10.130.0.38/23] I0504 12:35:54.760478 3659 cni.go:473] [tsf-tas/trillian-logsigner-6d7df4d6f5-c55tc 4b610b12b4c668f527423f388df4ecdaa2889352247fbec0b65a8a9589c88a6d network default NAD default NAD key default] ADD finished CNI request [tsf-tas/trillian-logsigner-6d7df4d6f5-c55tc 4b610b12b4c668f527423f388df4ecdaa2889352247fbec0b65a8a9589c88a6d network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"6a:06:64:bf:63:a3\",\"name\":\"4b610b12b4c668f\"},{\"mac\":\"0a:58:0a:82:00:26\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/134b3448-90ce-45b5-a312-98fefac8cffa\"}],\"ips\":[{\"address\":\"10.130.0.38/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0504 12:35:59.483108 3659 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/fulcio-server W0504 12:35:59.483131 3659 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/fulcio-server W0504 12:35:59.506228 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 5554: empty IP address endpoints for service tsf-tas/fulcio-server W0504 12:35:59.506248 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 5555: empty IP address endpoints for service tsf-tas/fulcio-server W0504 12:36:10.457607 3659 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logserver W0504 12:36:11.568607 3659 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logsigner I0504 12:36:14.570218 3659 cni.go:452] [tsf-tas/segment-backup-installation-zsw85-8w25p 05279ec80ef2587d71d1994278f94db16a42c24b1ea717b9692a3f418576143b network default NAD default NAD key ] DEL starting CNI request [tsf-tas/segment-backup-installation-zsw85-8w25p 05279ec80ef2587d71d1994278f94db16a42c24b1ea717b9692a3f418576143b network default NAD default NAD key ] I0504 12:36:14.651083 3659 cni.go:473] [tsf-tas/segment-backup-installation-zsw85-8w25p 05279ec80ef2587d71d1994278f94db16a42c24b1ea717b9692a3f418576143b network default NAD default NAD key default] DEL finished CNI request [tsf-tas/segment-backup-installation-zsw85-8w25p 05279ec80ef2587d71d1994278f94db16a42c24b1ea717b9692a3f418576143b network default NAD default NAD key default], result "{}", err I0504 12:36:14.700785 3659 obj_retry.go:498] Detected object tsf-tas/segment-backup-installation-zsw85-8w25p of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0504 12:36:14.700821 3659 pods.go:176] Deleting pod: tsf-tas/segment-backup-installation-zsw85-8w25p I0504 12:36:14.702446 3659 pods.go:236] Attempting to release IPs for pod: tsf-tas/segment-backup-installation-zsw85-8w25p, ips: 10.130.0.36 I0504 12:36:14.702475 3659 obj_retry.go:498] Detected object tsf-tas/segment-backup-installation-zsw85-8w25p of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0504 12:36:15.723487 3659 obj_retry.go:492] Detected object tsf-tas/segment-backup-installation-zsw85-8w25p of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:36:15.723512 3659 obj_retry.go:492] Detected object tsf-tas/segment-backup-installation-zsw85-8w25p of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0504 12:36:18.401244 3659 util.go:870] Failed to build global endpoints for port TCP/trillian-mysql: empty IP address endpoints for service tsf-tas/trillian-mysql W0504 12:36:44.681388 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 8091: empty IP address endpoints for service tsf-tas/trillian-logsigner I0504 12:36:46.696400 3659 obj_retry.go:498] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0504 12:36:46.696982 3659 obj_retry.go:498] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0504 12:36:46.737696 3659 obj_retry.go:492] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:36:46.737718 3659 obj_retry.go:492] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:36:46.744679 3659 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tas/test-tsf-tas I0504 12:36:46.744703 3659 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tas/test-tsf-tas I0504 12:36:55.776519 3659 obj_retry.go:498] Detected object tsf/patch-tekton-config-cpwwt of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0504 12:36:55.777063 3659 obj_retry.go:498] Detected object tsf/patch-tekton-config-cpwwt of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0504 12:36:55.795533 3659 obj_retry.go:498] Detected object tsf-tas/rekor-createtree-job-gcjhf-v5pwm of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0504 12:36:55.795911 3659 obj_retry.go:498] Detected object tsf-tas/rekor-createtree-job-gcjhf-v5pwm of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0504 12:36:56.648353 3659 obj_retry.go:492] Detected object tsf-tas/rekor-createtree-job-gcjhf-v5pwm of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:36:56.648375 3659 obj_retry.go:492] Detected object tsf-tas/rekor-createtree-job-gcjhf-v5pwm of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:36:56.801344 3659 obj_retry.go:492] Detected object tsf/patch-tekton-config-cpwwt of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:36:56.801365 3659 obj_retry.go:492] Detected object tsf/patch-tekton-config-cpwwt of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:37:00.921969 3659 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=6 I0504 12:37:02.734589 3659 base_network_controller_pods.go:487] [default/tsf-tas/rekor-server-c69d9c978-lmtnn] creating logical port tsf-tas_rekor-server-c69d9c978-lmtnn for pod on switch ip-10-0-2-63.ec2.internal I0504 12:37:02.734682 3659 kube.go:256] Updating pod tsf-tas/rekor-server-c69d9c978-lmtnn I0504 12:37:02.747691 3659 pod.go:62] [tsf-tas/rekor-server-c69d9c978-lmtnn] pod update took 13.034374ms I0504 12:37:02.747712 3659 base_network_controller_pods.go:951] [default/tsf-tas/rekor-server-c69d9c978-lmtnn] addLogicalPort annotation time took 13.061776ms I0504 12:37:02.749144 3659 pods.go:271] [tsf-tas/rekor-server-c69d9c978-lmtnn] addLogicalPort took 14.57101ms, libovsdb time 980.351µs I0504 12:37:03.477811 3659 obj_retry.go:498] Detected object tsf/tsf-tekton-configuration-gvg72 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0504 12:37:03.478714 3659 obj_retry.go:498] Detected object tsf/tsf-tekton-configuration-gvg72 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0504 12:37:04.501196 3659 obj_retry.go:492] Detected object tsf/tsf-tekton-configuration-gvg72 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:37:04.501219 3659 obj_retry.go:492] Detected object tsf/tsf-tekton-configuration-gvg72 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:37:04.539814 3659 obj_retry.go:492] Detected object tsf/patch-tekton-config-cpwwt of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:37:04.539838 3659 obj_retry.go:492] Detected object tsf/patch-tekton-config-cpwwt of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:37:04.543108 3659 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/patch-tekton-config-cpwwt I0504 12:37:04.543145 3659 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/patch-tekton-config-cpwwt W0504 12:37:04.674673 3659 util.go:870] Failed to build global endpoints for port TCP/resp: empty IP address endpoints for service tsf-tas/rekor-redis W0504 12:37:04.695001 3659 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-search-ui I0504 12:37:07.042248 3659 cni.go:452] [tsf-tas/rekor-server-c69d9c978-lmtnn 1e28b995177a82e7d76456e4d1b425e054d9eba7e6ee804450e5da1f0f6071a9 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/rekor-server-c69d9c978-lmtnn 1e28b995177a82e7d76456e4d1b425e054d9eba7e6ee804450e5da1f0f6071a9 network default NAD default NAD key ] I0504 12:37:07.051233 3659 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: rekor-server-c69d9c978-lmtnn, hostIfaceName: 1e28b995177a82e, network: default, NAD default, SandboxID: "1e28b995177a82e7d76456e4d1b425e054d9eba7e6ee804450e5da1f0f6071a9", PCI device ID: , UID: "33fe5a36-53b8-4fb5-9614-d970f03de7f2", MAC: 0a:58:0a:82:00:27, IPs: [10.130.0.39/23] I0504 12:37:07.101350 3659 cni.go:473] [tsf-tas/rekor-server-c69d9c978-lmtnn 1e28b995177a82e7d76456e4d1b425e054d9eba7e6ee804450e5da1f0f6071a9 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/rekor-server-c69d9c978-lmtnn 1e28b995177a82e7d76456e4d1b425e054d9eba7e6ee804450e5da1f0f6071a9 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"aa:ee:93:00:0c:9b\",\"name\":\"1e28b995177a82e\"},{\"mac\":\"0a:58:0a:82:00:27\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b0009004-e3d6-49b7-8243-6e29b78e79e3\"}],\"ips\":[{\"address\":\"10.130.0.39/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0504 12:37:08.840779 3659 obj_retry.go:498] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0504 12:37:08.841341 3659 obj_retry.go:498] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0504 12:37:08.862092 3659 obj_retry.go:492] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:37:08.862115 3659 obj_retry.go:492] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:37:08.869367 3659 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/pre-install-tsf-tpa I0504 12:37:08.869392 3659 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/pre-install-tsf-tpa W0504 12:37:09.847528 3659 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-server W0504 12:37:20.857377 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 3000: empty IP address endpoints for service tsf-tas/rekor-server I0504 12:37:23.974512 3659 obj_retry.go:498] Detected object tsf-tas/ctlog-createtree-job-m7sdz-mp4t4 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0504 12:37:23.975098 3659 obj_retry.go:498] Detected object tsf-tas/ctlog-createtree-job-m7sdz-mp4t4 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0504 12:37:24.039329 3659 obj_retry.go:498] Detected object tsf-tpa/create-db-sxrrz of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0504 12:37:24.039785 3659 obj_retry.go:498] Detected object tsf-tpa/create-db-sxrrz of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0504 12:37:24.796241 3659 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=11 I0504 12:37:24.854249 3659 obj_retry.go:492] Detected object tsf-tas/ctlog-createtree-job-m7sdz-mp4t4 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:37:24.854269 3659 obj_retry.go:492] Detected object tsf-tas/ctlog-createtree-job-m7sdz-mp4t4 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:37:25.062816 3659 obj_retry.go:492] Detected object tsf-tpa/create-db-sxrrz of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:37:25.062842 3659 obj_retry.go:492] Detected object tsf-tpa/create-db-sxrrz of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:37:27.967900 3659 obj_retry.go:498] Detected object tsf-tpa/migrate-db-jjqtc of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0504 12:37:27.968539 3659 obj_retry.go:498] Detected object tsf-tpa/migrate-db-jjqtc of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0504 12:37:28.989958 3659 obj_retry.go:492] Detected object tsf-tpa/migrate-db-jjqtc of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:37:28.989978 3659 obj_retry.go:492] Detected object tsf-tpa/migrate-db-jjqtc of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0504 12:37:30.883828 3659 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/ctlog I0504 12:37:30.972581 3659 obj_retry.go:498] Detected object tsf-tpa/create-importers-fkqzn of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0504 12:37:30.973181 3659 obj_retry.go:498] Detected object tsf-tpa/create-importers-fkqzn of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0504 12:37:31.996406 3659 obj_retry.go:492] Detected object tsf-tpa/create-importers-fkqzn of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:37:31.996429 3659 obj_retry.go:492] Detected object tsf-tpa/create-importers-fkqzn of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:37:41.039392 3659 obj_retry.go:498] Detected object tsf-tas/tuf-repository-init-gszps-csgm7 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0504 12:37:41.039968 3659 obj_retry.go:498] Detected object tsf-tas/tuf-repository-init-gszps-csgm7 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0504 12:37:41.922826 3659 obj_retry.go:492] Detected object tsf-tas/tuf-repository-init-gszps-csgm7 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:37:41.922846 3659 obj_retry.go:492] Detected object tsf-tas/tuf-repository-init-gszps-csgm7 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:37:46.797800 3659 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=60 W0504 12:37:46.971662 3659 util.go:870] Failed to build global endpoints for port TCP/endpoint: empty IP address endpoints for service tsf-tpa/server I0504 12:37:50.095340 3659 obj_retry.go:498] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0504 12:37:50.095919 3659 obj_retry.go:498] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0504 12:37:50.118635 3659 obj_retry.go:492] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:37:50.118653 3659 obj_retry.go:492] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:37:50.123499 3659 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/test-tsf-tpa I0504 12:37:50.123521 3659 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/test-tsf-tpa I0504 12:37:50.916261 3659 namespace.go:98] [image-controller] adding namespace I0504 12:37:50.917148 3659 namespace.go:102] [image-controller] adding namespace took 862.249µs I0504 12:37:50.917511 3659 namespace.go:98] [build-service] adding namespace I0504 12:37:50.918275 3659 namespace.go:102] [build-service] adding namespace took 745.9µs I0504 12:37:50.919699 3659 namespace.go:98] [integration-service] adding namespace I0504 12:37:50.920657 3659 namespace.go:102] [integration-service] adding namespace took 934.756µs I0504 12:37:50.927510 3659 namespace.go:98] [default-tenant] adding namespace I0504 12:37:50.928301 3659 namespace.go:102] [default-tenant] adding namespace took 774.256µs I0504 12:37:51.018345 3659 namespace.go:142] [image-controller] updating namespace I0504 12:37:51.068641 3659 namespace.go:142] [image-controller] updating namespace I0504 12:37:51.096987 3659 namespace.go:142] [build-service] updating namespace I0504 12:37:51.126236 3659 namespace.go:142] [build-service] updating namespace I0504 12:37:51.169311 3659 namespace.go:142] [integration-service] updating namespace I0504 12:37:51.202099 3659 namespace.go:142] [build-service] updating namespace I0504 12:37:51.225006 3659 namespace.go:142] [integration-service] updating namespace I0504 12:37:51.233538 3659 namespace.go:142] [default-tenant] updating namespace I0504 12:37:51.261552 3659 namespace.go:142] [default-tenant] updating namespace I0504 12:37:51.273433 3659 namespace.go:142] [integration-service] updating namespace I0504 12:37:51.375571 3659 namespace.go:98] [release-service] adding namespace I0504 12:37:51.376423 3659 namespace.go:102] [release-service] adding namespace took 838.25µs I0504 12:37:51.387548 3659 namespace.go:142] [release-service] updating namespace I0504 12:37:51.410285 3659 namespace.go:142] [konflux-ui] updating namespace I0504 12:37:51.410395 3659 namespace.go:142] [release-service] updating namespace I0504 12:37:51.737985 3659 namespace.go:98] [konflux-info] adding namespace I0504 12:37:51.738832 3659 namespace.go:102] [konflux-info] adding namespace took 829.023µs I0504 12:37:51.796929 3659 namespace.go:98] [namespace-lister] adding namespace I0504 12:37:51.797075 3659 namespace.go:142] [konflux-info] updating namespace I0504 12:37:51.797616 3659 namespace.go:102] [namespace-lister] adding namespace took 671.359µs I0504 12:37:51.860310 3659 namespace.go:142] [konflux-info] updating namespace I0504 12:37:51.860364 3659 namespace.go:98] [enterprise-contract-service] adding namespace I0504 12:37:51.861076 3659 namespace.go:102] [enterprise-contract-service] adding namespace took 701.678µs I0504 12:37:51.876946 3659 namespace.go:142] [namespace-lister] updating namespace I0504 12:37:51.962212 3659 namespace.go:142] [image-controller] updating namespace I0504 12:37:51.990703 3659 namespace.go:142] [namespace-lister] updating namespace I0504 12:37:52.117188 3659 namespace.go:142] [enterprise-contract-service] updating namespace I0504 12:37:52.234768 3659 namespace.go:142] [default-tenant] updating namespace I0504 12:37:52.315076 3659 namespace.go:98] [segment-bridge] adding namespace I0504 12:37:52.315952 3659 namespace.go:102] [segment-bridge] adding namespace took 860.348µs I0504 12:37:52.325160 3659 namespace.go:142] [enterprise-contract-service] updating namespace I0504 12:37:52.540335 3659 namespace.go:98] [konflux-cli] adding namespace I0504 12:37:52.541298 3659 namespace.go:102] [konflux-cli] adding namespace took 941.762µs I0504 12:37:52.637318 3659 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-from-konfluxui for network default I0504 12:37:52.637818 3659 pod_selector_address_set.go:213] Created shared address set for pod selector namespace-lister_LS{ML:{app: proxy,},} I0504 12:37:52.637837 3659 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-from-konfluxui added to peer address sets [namespace-lister_LS{ML:{app: proxy,},}] I0504 12:37:52.640306 3659 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-from-konfluxui resources completed, update namespace loglevel I0504 12:37:52.839991 3659 namespace.go:142] [segment-bridge] updating namespace I0504 12:37:53.033070 3659 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-to-apiserver for network default I0504 12:37:53.033092 3659 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-to-apiserver added to peer address sets [] I0504 12:37:53.033900 3659 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-to-apiserver resources completed, update namespace loglevel I0504 12:37:53.072443 3659 namespace.go:142] [segment-bridge] updating namespace I0504 12:37:53.155824 3659 namespace.go:142] [build-service] updating namespace I0504 12:37:53.215958 3659 namespace.go:142] [konflux-cli] updating namespace I0504 12:37:53.261094 3659 namespace.go:142] [konflux-cli] updating namespace I0504 12:37:53.591111 3659 namespace.go:142] [default-tenant] updating namespace I0504 12:37:53.783250 3659 namespace.go:142] [image-controller] updating namespace I0504 12:37:54.100354 3659 namespace.go:142] [integration-service] updating namespace I0504 12:37:54.295192 3659 namespace.go:142] [release-service] updating namespace I0504 12:37:54.438079 3659 namespace.go:142] [build-service] updating namespace I0504 12:37:54.458058 3659 namespace.go:142] [default-tenant] updating namespace I0504 12:37:54.475496 3659 namespace.go:142] [image-controller] updating namespace I0504 12:37:54.518275 3659 namespace.go:142] [integration-service] updating namespace I0504 12:37:54.525140 3659 namespace.go:142] [release-service] updating namespace I0504 12:37:54.567054 3659 base_network_controller_pods.go:487] [default/konflux-ui/dex-b576bbdfc-k88dv] creating logical port konflux-ui_dex-b576bbdfc-k88dv for pod on switch ip-10-0-2-63.ec2.internal I0504 12:37:54.567147 3659 kube.go:256] Updating pod konflux-ui/dex-b576bbdfc-k88dv I0504 12:37:54.593183 3659 namespace.go:142] [build-service] updating namespace I0504 12:37:54.651181 3659 pod.go:62] [konflux-ui/dex-b576bbdfc-k88dv] pod update took 84.06059ms I0504 12:37:54.651202 3659 base_network_controller_pods.go:951] [default/konflux-ui/dex-b576bbdfc-k88dv] addLogicalPort annotation time took 84.088085ms I0504 12:37:54.652541 3659 pods.go:271] [konflux-ui/dex-b576bbdfc-k88dv] addLogicalPort took 85.526404ms, libovsdb time 840.89µs I0504 12:37:54.700725 3659 namespace.go:142] [default-tenant] updating namespace I0504 12:37:54.714565 3659 base_network_controller_pods.go:487] [default/konflux-ui/proxy-c8cfcf87d-sgt59] creating logical port konflux-ui_proxy-c8cfcf87d-sgt59 for pod on switch ip-10-0-2-63.ec2.internal I0504 12:37:54.714689 3659 kube.go:256] Updating pod konflux-ui/proxy-c8cfcf87d-sgt59 I0504 12:37:54.811822 3659 namespace.go:142] [image-controller] updating namespace I0504 12:37:54.837059 3659 pod.go:62] [konflux-ui/proxy-c8cfcf87d-sgt59] pod update took 122.406618ms I0504 12:37:54.837084 3659 base_network_controller_pods.go:951] [default/konflux-ui/proxy-c8cfcf87d-sgt59] addLogicalPort annotation time took 122.434697ms I0504 12:37:54.838385 3659 pods.go:271] [konflux-ui/proxy-c8cfcf87d-sgt59] addLogicalPort took 123.837432ms, libovsdb time 869.963µs I0504 12:37:54.880051 3659 namespace.go:142] [integration-service] updating namespace I0504 12:37:54.978938 3659 namespace.go:142] [release-service] updating namespace I0504 12:37:56.281718 3659 namespace.go:142] [enterprise-contract-service] updating namespace I0504 12:37:56.471300 3659 namespace.go:142] [konflux-cli] updating namespace I0504 12:37:56.675711 3659 namespace.go:142] [konflux-info] updating namespace I0504 12:37:56.876108 3659 namespace.go:142] [namespace-lister] updating namespace I0504 12:37:57.083653 3659 namespace.go:142] [segment-bridge] updating namespace I0504 12:37:57.239349 3659 namespace.go:142] [enterprise-contract-service] updating namespace I0504 12:37:57.251774 3659 namespace.go:142] [konflux-cli] updating namespace I0504 12:37:57.287425 3659 namespace.go:142] [konflux-info] updating namespace I0504 12:37:57.300325 3659 namespace.go:142] [namespace-lister] updating namespace I0504 12:37:57.325048 3659 namespace.go:142] [segment-bridge] updating namespace I0504 12:37:57.416009 3659 namespace.go:142] [enterprise-contract-service] updating namespace I0504 12:37:57.490231 3659 namespace.go:142] [konflux-cli] updating namespace I0504 12:37:57.576190 3659 namespace.go:142] [konflux-info] updating namespace I0504 12:37:57.650266 3659 namespace.go:142] [namespace-lister] updating namespace I0504 12:37:57.724233 3659 namespace.go:142] [segment-bridge] updating namespace I0504 12:37:58.851409 3659 base_network_controller_pods.go:487] [default/integration-service/integration-service-controller-manager-b59fbfbf8-27dv9] creating logical port integration-service_integration-service-controller-manager-b59fbfbf8-27dv9 for pod on switch ip-10-0-2-63.ec2.internal I0504 12:37:58.851505 3659 kube.go:256] Updating pod integration-service/integration-service-controller-manager-b59fbfbf8-27dv9 I0504 12:37:58.873512 3659 pod.go:62] [integration-service/integration-service-controller-manager-b59fbfbf8-27dv9] pod update took 22.034048ms I0504 12:37:58.873535 3659 base_network_controller_pods.go:951] [default/integration-service/integration-service-controller-manager-b59fbfbf8-27dv9] addLogicalPort annotation time took 22.060565ms I0504 12:37:58.874726 3659 pods.go:271] [integration-service/integration-service-controller-manager-b59fbfbf8-27dv9] addLogicalPort took 23.338057ms, libovsdb time 770.318µs I0504 12:37:59.427874 3659 cni.go:452] [konflux-ui/dex-b576bbdfc-k88dv 059491064fc3f610c70b5bd5ee6a552eb8bbc6a21c587a97bf5ab50d1dbf44d4 network default NAD default NAD key ] ADD starting CNI request [konflux-ui/dex-b576bbdfc-k88dv 059491064fc3f610c70b5bd5ee6a552eb8bbc6a21c587a97bf5ab50d1dbf44d4 network default NAD default NAD key ] I0504 12:37:59.439208 3659 helper_linux.go:556] ConfigureOVS: namespace: konflux-ui, podName: dex-b576bbdfc-k88dv, hostIfaceName: 059491064fc3f61, network: default, NAD default, SandboxID: "059491064fc3f610c70b5bd5ee6a552eb8bbc6a21c587a97bf5ab50d1dbf44d4", PCI device ID: , UID: "3d43cbdf-0ae3-4cbb-814d-0833705958c4", MAC: 0a:58:0a:82:00:28, IPs: [10.130.0.40/23] I0504 12:37:59.489763 3659 cni.go:473] [konflux-ui/dex-b576bbdfc-k88dv 059491064fc3f610c70b5bd5ee6a552eb8bbc6a21c587a97bf5ab50d1dbf44d4 network default NAD default NAD key default] ADD finished CNI request [konflux-ui/dex-b576bbdfc-k88dv 059491064fc3f610c70b5bd5ee6a552eb8bbc6a21c587a97bf5ab50d1dbf44d4 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"5e:52:78:93:25:74\",\"name\":\"059491064fc3f61\"},{\"mac\":\"0a:58:0a:82:00:28\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/17e46e09-6db9-47c0-9101-1478501f7786\"}],\"ips\":[{\"address\":\"10.130.0.40/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0504 12:37:59.842722 3659 cni.go:452] [integration-service/integration-service-controller-manager-b59fbfbf8-27dv9 d4b2e8b4f59b3481f04ba21be5120db5a6490cc867a1b81a15bcba58a797530e network default NAD default NAD key ] ADD starting CNI request [integration-service/integration-service-controller-manager-b59fbfbf8-27dv9 d4b2e8b4f59b3481f04ba21be5120db5a6490cc867a1b81a15bcba58a797530e network default NAD default NAD key ] I0504 12:37:59.853137 3659 helper_linux.go:556] ConfigureOVS: namespace: integration-service, podName: integration-service-controller-manager-b59fbfbf8-27dv9, hostIfaceName: d4b2e8b4f59b348, network: default, NAD default, SandboxID: "d4b2e8b4f59b3481f04ba21be5120db5a6490cc867a1b81a15bcba58a797530e", PCI device ID: , UID: "a8060cda-69eb-4a75-a701-269c02e76d0a", MAC: 0a:58:0a:82:00:2a, IPs: [10.130.0.42/23] I0504 12:37:59.903411 3659 cni.go:473] [integration-service/integration-service-controller-manager-b59fbfbf8-27dv9 d4b2e8b4f59b3481f04ba21be5120db5a6490cc867a1b81a15bcba58a797530e network default NAD default NAD key default] ADD finished CNI request [integration-service/integration-service-controller-manager-b59fbfbf8-27dv9 d4b2e8b4f59b3481f04ba21be5120db5a6490cc867a1b81a15bcba58a797530e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"32:fa:2a:77:5f:51\",\"name\":\"d4b2e8b4f59b348\"},{\"mac\":\"0a:58:0a:82:00:2a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0a126a67-8d19-4ee7-9f1a-37dc6f16096a\"}],\"ips\":[{\"address\":\"10.130.0.42/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0504 12:38:01.057383 3659 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service namespace-lister/namespace-lister W0504 12:38:01.084701 3659 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/tuf W0504 12:38:01.139730 3659 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service image-controller/image-controller-controller-manager-metrics-service W0504 12:38:02.120360 3659 util.go:870] Failed to build global endpoints for port TCP/dex: empty IP address endpoints for service konflux-ui/dex I0504 12:38:02.873121 3659 cni.go:452] [konflux-ui/proxy-c8cfcf87d-sgt59 e76e7a360810fa2cc352e79e6f52d15eab676f882b1274dad3e5adf7ba1f4496 network default NAD default NAD key ] ADD starting CNI request [konflux-ui/proxy-c8cfcf87d-sgt59 e76e7a360810fa2cc352e79e6f52d15eab676f882b1274dad3e5adf7ba1f4496 network default NAD default NAD key ] I0504 12:38:02.883665 3659 helper_linux.go:556] ConfigureOVS: namespace: konflux-ui, podName: proxy-c8cfcf87d-sgt59, hostIfaceName: e76e7a360810fa2, network: default, NAD default, SandboxID: "e76e7a360810fa2cc352e79e6f52d15eab676f882b1274dad3e5adf7ba1f4496", PCI device ID: , UID: "b190928f-88ff-42bf-9bba-efcb8dd5910d", MAC: 0a:58:0a:82:00:29, IPs: [10.130.0.41/23] I0504 12:38:02.933746 3659 cni.go:473] [konflux-ui/proxy-c8cfcf87d-sgt59 e76e7a360810fa2cc352e79e6f52d15eab676f882b1274dad3e5adf7ba1f4496 network default NAD default NAD key default] ADD finished CNI request [konflux-ui/proxy-c8cfcf87d-sgt59 e76e7a360810fa2cc352e79e6f52d15eab676f882b1274dad3e5adf7ba1f4496 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"3e:ec:23:0f:95:07\",\"name\":\"e76e7a360810fa2\"},{\"mac\":\"0a:58:0a:82:00:29\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/04d8d2be-bf9b-4910-8f88-93f610d15e71\"}],\"ips\":[{\"address\":\"10.130.0.41/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0504 12:38:03.128195 3659 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0504 12:38:03.129508 3659 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0504 12:38:04.100997 3659 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0504 12:38:04.101143 3659 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0504 12:38:04.128887 3659 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service W0504 12:38:08.130257 3659 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0504 12:38:08.130283 3659 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0504 12:38:08.167967 3659 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0504 12:38:08.167985 3659 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy I0504 12:38:08.793049 3659 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=73 W0504 12:38:10.181161 3659 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service W0504 12:38:10.185591 3659 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service release-service/release-service-webhook-service I0504 12:38:11.880902 3659 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=8 W0504 12:38:14.149863 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 8080: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0504 12:38:14.151747 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 9443: empty IP address endpoints for service integration-service/integration-service-webhook-service I0504 12:38:15.081464 3659 obj_retry.go:498] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-rjvc2 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0504 12:38:15.082100 3659 obj_retry.go:498] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-rjvc2 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0504 12:38:15.169310 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/dex I0504 12:38:15.199327 3659 obj_retry.go:492] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-rjvc2 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:38:15.199348 3659 obj_retry.go:492] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-rjvc2 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:38:15.207669 3659 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod integration-service/integration-service-controller-manager-7b666bf6d8-rjvc2 I0504 12:38:15.207706 3659 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod integration-service/integration-service-controller-manager-7b666bf6d8-rjvc2 I0504 12:38:31.794491 3659 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=88 I0504 12:38:33.914648 3659 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=10 W0504 12:38:47.250789 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 8080: empty IP address endpoints for service konflux-ui/proxy W0504 12:38:47.250807 3659 util.go:959] Failed to build node endpoints for node ip-10-0-2-63.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/proxy I0504 12:39:15.904629 3659 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=10 I0504 12:39:18.197421 3659 cni.go:452] [konflux-ui/proxy-c8cfcf87d-sgt59 e76e7a360810fa2cc352e79e6f52d15eab676f882b1274dad3e5adf7ba1f4496 network default NAD default NAD key ] DEL starting CNI request [konflux-ui/proxy-c8cfcf87d-sgt59 e76e7a360810fa2cc352e79e6f52d15eab676f882b1274dad3e5adf7ba1f4496 network default NAD default NAD key ] I0504 12:39:18.289318 3659 cni.go:473] [konflux-ui/proxy-c8cfcf87d-sgt59 e76e7a360810fa2cc352e79e6f52d15eab676f882b1274dad3e5adf7ba1f4496 network default NAD default NAD key default] DEL finished CNI request [konflux-ui/proxy-c8cfcf87d-sgt59 e76e7a360810fa2cc352e79e6f52d15eab676f882b1274dad3e5adf7ba1f4496 network default NAD default NAD key default], result "{}", err I0504 12:39:18.353629 3659 obj_retry.go:498] Detected object konflux-ui/proxy-c8cfcf87d-sgt59 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0504 12:39:18.353657 3659 pods.go:176] Deleting pod: konflux-ui/proxy-c8cfcf87d-sgt59 I0504 12:39:18.355100 3659 pods.go:236] Attempting to release IPs for pod: konflux-ui/proxy-c8cfcf87d-sgt59, ips: 10.130.0.41 I0504 12:39:18.355131 3659 obj_retry.go:498] Detected object konflux-ui/proxy-c8cfcf87d-sgt59 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0504 12:39:18.820920 3659 obj_retry.go:492] Detected object konflux-ui/proxy-c8cfcf87d-sgt59 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:39:18.820946 3659 obj_retry.go:492] Detected object konflux-ui/proxy-c8cfcf87d-sgt59 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:39:18.829005 3659 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/proxy-c8cfcf87d-sgt59 I0504 12:39:18.829044 3659 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/proxy-c8cfcf87d-sgt59 I0504 12:39:22.738551 3659 obj_retry.go:498] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0504 12:39:22.739201 3659 obj_retry.go:498] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0504 12:39:22.778200 3659 obj_retry.go:492] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:39:22.778220 3659 obj_retry.go:492] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:39:22.786549 3659 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/test-tsf-konflux I0504 12:39:22.786573 3659 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/test-tsf-konflux I0504 12:39:29.926335 3659 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=9 I0504 12:39:45.904181 3659 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=9 I0504 12:39:53.800665 3659 namespace.go:98] [managed-tenant-9g2ln] adding namespace I0504 12:39:53.801628 3659 namespace.go:102] [managed-tenant-9g2ln] adding namespace took 935.865µs I0504 12:39:53.840195 3659 namespace.go:142] [managed-tenant-9g2ln] updating namespace I0504 12:39:53.860601 3659 namespace.go:142] [managed-tenant-9g2ln] updating namespace I0504 12:39:55.020554 3659 namespace.go:142] [managed-tenant-9g2ln] updating namespace I0504 12:39:55.150999 3659 namespace.go:142] [managed-tenant-9g2ln] updating namespace I0504 12:39:55.267661 3659 namespace.go:142] [managed-tenant-9g2ln] updating namespace I0504 12:40:03.990900 3659 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29631640-7xrzw of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0504 12:40:03.991530 3659 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29631640-7xrzw of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0504 12:40:04.865996 3659 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29631640-7xrzw of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:40:04.866023 3659 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29631640-7xrzw of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:40:19.804447 3659 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=391 I0504 12:40:20.926763 3659 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=11 I0504 12:40:21.963099 3659 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=9 I0504 12:40:40.880025 3659 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=9 I0504 12:42:10.797669 3659 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=118 I0504 12:42:30.886859 3659 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=7 I0504 12:44:20.800061 3659 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=9 I0504 12:44:49.907724 3659 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=7 I0504 12:45:31.797768 3659 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=7 I0504 12:45:48.917760 3659 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=8 I0504 12:46:01.907058 3659 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=6 I0504 12:46:10.929133 3659 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=10 I0504 12:46:22.801334 3659 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=81 I0504 12:47:08.883885 3659 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=9 I0504 12:47:08.966416 3659 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=7 I0504 12:47:14.930163 3659 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=7 I0504 12:47:25.050733 3659 obj_retry.go:492] Detected object tsf-tpa/create-db-sxrrz of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:47:25.050779 3659 obj_retry.go:492] Detected object tsf-tpa/create-db-sxrrz of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:47:25.055513 3659 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-db-sxrrz I0504 12:47:25.055532 3659 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-db-sxrrz I0504 12:47:28.032495 3659 obj_retry.go:492] Detected object tsf-tpa/migrate-db-jjqtc of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:47:28.032539 3659 obj_retry.go:492] Detected object tsf-tpa/migrate-db-jjqtc of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:47:28.035318 3659 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/migrate-db-jjqtc I0504 12:47:28.035343 3659 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/migrate-db-jjqtc I0504 12:47:31.033661 3659 obj_retry.go:492] Detected object tsf-tpa/create-importers-fkqzn of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:47:31.033711 3659 obj_retry.go:492] Detected object tsf-tpa/create-importers-fkqzn of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:47:31.039435 3659 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-importers-fkqzn I0504 12:47:31.039461 3659 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-importers-fkqzn I0504 12:47:35.796527 3659 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=52 I0504 12:48:07.930221 3659 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=8 I0504 12:49:05.883438 3659 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=9 I0504 12:49:30.807071 3659 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=16 I0504 12:49:55.801523 3659 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=8 I0504 12:50:04.475255 3659 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29631650-zkzm8 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0504 12:50:04.475857 3659 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29631650-zkzm8 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0504 12:50:05.328776 3659 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29631650-zkzm8 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:50:05.328818 3659 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29631650-zkzm8 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:50:05.363210 3659 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29631620-9lblp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:50:05.363235 3659 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29631620-9lblp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0504 12:50:05.366790 3659 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-backplane/osd-delete-backplane-serviceaccounts-29631620-9lblp I0504 12:50:05.366812 3659 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-backplane/osd-delete-backplane-serviceaccounts-29631620-9lblp I0504 12:50:33.801288 3659 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=5 W0504 12:51:04.454921 3659 duration.go:461] Config duration recorder: measurement expired for pod/openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2blgpl