+ . /ovnkube-lib/ovnkube-lib.sh ++ set -x ++ K8S_NODE=ip-10-0-1-214.ec2.internal ++ [[ -n ip-10-0-1-214.ec2.internal ]] ++ [[ -f /env/ip-10-0-1-214.ec2.internal ]] ++ northd_pidfile=/var/run/ovn/ovn-northd.pid ++ controller_pidfile=/var/run/ovn/ovn-controller.pid ++ controller_logfile=/var/log/ovn/acl-audit-log.log ++ vswitch_dbsock=/var/run/openvswitch/db.sock ++ nbdb_pidfile=/var/run/ovn/ovnnb_db.pid ++ nbdb_sock=/var/run/ovn/ovnnb_db.sock ++ nbdb_ctl=/var/run/ovn/ovnnb_db.ctl ++ sbdb_pidfile=/var/run/ovn/ovnsb_db.pid ++ sbdb_sock=/var/run/ovn/ovnsb_db.sock ++ sbdb_ctl=/var/run/ovn/ovnsb_db.ctl + start-ovnkube-node 4 29103 29105 + local log_level=4 + local metrics_port=29103 + local ovn_metrics_port=29105 + ovn_advertised_udn_isolation_mode_flag= + openflow_probe_flag= + [[ 3 -ne 3 ]] + ovs-vsctl br-exists br-ex + add_garp_drop_flow br-ex + local bridge=br-ex + local cookie=0x0305 + local priority=499 ++ ovs-vsctl list-ports br-ex + for port_name in $(ovs-vsctl list-ports "$bridge") + [[ ens5 == *to-br-int ]] + cni-bin-copy + . /host/etc/os-release ++ NAME='Red Hat Enterprise Linux CoreOS' ++ VERSION='9.6.20260401-0 (Plow)' ++ ID=rhel ++ ID_LIKE=fedora ++ VERSION_ID=9.6 ++ PLATFORM_ID=platform:el9 ++ PRETTY_NAME='Red Hat Enterprise Linux CoreOS 9.6.20260401-0 (Plow)' ++ ANSI_COLOR='0;31' ++ LOGO=fedora-logo-icon ++ CPE_NAME=cpe:/o:redhat:enterprise_linux:9::baseos ++ HOME_URL=https://www.redhat.com/ ++ DOCUMENTATION_URL=https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9 ++ BUG_REPORT_URL=https://issues.redhat.com/ ++ REDHAT_BUGZILLA_PRODUCT='Red Hat Enterprise Linux 9' ++ REDHAT_BUGZILLA_PRODUCT_VERSION=9.6 ++ REDHAT_SUPPORT_PRODUCT='Red Hat Enterprise Linux' ++ REDHAT_SUPPORT_PRODUCT_VERSION=9.6 ++ OSTREE_VERSION=9.6.20260401-0 ++ VARIANT=CoreOS ++ VARIANT_ID=coreos ++ OPENSHIFT_VERSION=4.21 + rhelmajor= + case "${ID}" in ++ echo 9.6 ++ cut -f 1 -d . + rhelmajor=9 + sourcedir=/usr/libexec/cni/ + case "${rhelmajor}" in + sourcedir=/usr/libexec/cni/rhel9 + cp -f /usr/libexec/cni/rhel9/ovn-k8s-cni-overlay /cni-bin-dir/ ++ date '+%m%d %H:%M:%S.%N' + echo 'I0428 11:03:26.881380156 - disable conntrack on geneve port' I0428 11:03:26.881380156 - disable conntrack on geneve port + iptables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + iptables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK ++ date '+%m%d %H:%M:%S.%N' + echo 'I0428 11:03:26.900044402 - starting ovnkube-node' I0428 11:03:26.900044402 - starting ovnkube-node + egress_features_enable_flag='--enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true' + init_ovnkube_controller='--init-ovnkube-controller ip-10-0-1-214.ec2.internal' + multi_external_gateway_enable_flag=--enable-multi-external-gateway=true + gateway_interface=br-ex + enable_multicast_flag=--enable-multicast + OVN_NODE_MODE=full + '[' full == dpu-host ']' + '[' shared == shared ']' + gateway_mode_flags='--gateway-mode shared --gateway-interface br-ex' + export_network_flows_flags= + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + gw_interface_flag= + '[' -d /sys/class/net/br-ex1 ']' + node_mgmt_port_netdev_flags= + [[ -n '' ]] + [[ -n '' ]] + multi_network_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag=--enable-network-segmentation + route_advertisements_enable_flag= + [[ false == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag= + [[ true == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag=--enable-preconfigured-udn-addresses + network_observability_enabled_flag= + [[ false == \t\r\u\e ]] + multi_network_policy_enabled_flag= + [[ false == \t\r\u\e ]] + admin_network_policy_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + admin_network_policy_enabled_flag=--enable-admin-network-policy + dns_name_resolver_enabled_flag= + [[ false == \t\r\u\e ]] + ip_forwarding_flag= + '[' '' == Global ']' + ip_forwarding_flag=--disable-forwarding + sysctl -w net.ipv4.ip_forward=0 net.ipv4.ip_forward = 0 + sysctl -w net.ipv6.conf.all.forwarding=0 net.ipv6.conf.all.forwarding = 0 + [[ '' != '' ]] + [[ '' != '' ]] + NETWORK_NODE_IDENTITY_ENABLE= + [[ true == \t\r\u\e ]] + NETWORK_NODE_IDENTITY_ENABLE=' --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h ' + ovn_v4_join_subnet_opt= + [[ '' != '' ]] + ovn_v6_join_subnet_opt= + [[ '' != '' ]] + ovn_v4_masquerade_subnet_opt= + [[ 169.254.0.0/17 != '' ]] + ovn_v4_masquerade_subnet_opt='--gateway-v4-masquerade-subnet 169.254.0.0/17' + ovn_v6_masquerade_subnet_opt= + [[ fd69::/112 != '' ]] + ovn_v6_masquerade_subnet_opt='--gateway-v6-masquerade-subnet fd69::/112' + ovn_v4_transit_switch_subnet_opt= + [[ '' != '' ]] + ovn_v6_transit_switch_subnet_opt= + [[ '' != '' ]] + exec /usr/bin/ovnkube --init-ovnkube-controller ip-10-0-1-214.ec2.internal --init-node ip-10-0-1-214.ec2.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4 --inactivity-probe=180000 --gateway-mode shared --gateway-interface br-ex --metrics-bind-address 127.0.0.1:29103 --ovn-metrics-bind-address 127.0.0.1:29105 --metrics-enable-pprof --metrics-enable-config-duration --export-ovs-metrics --disable-snat-multiple-gws --enable-multi-network --enable-network-segmentation --enable-preconfigured-udn-addresses --enable-admin-network-policy --enable-multicast --zone ip-10-0-1-214.ec2.internal --enable-interconnect --acl-logging-rate-limit 20 --disable-forwarding --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h --gateway-v4-masquerade-subnet 169.254.0.0/17 --gateway-v6-masquerade-subnet fd69::/112 --enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true --enable-multi-external-gateway=true I0428 11:03:26.939619 3656 config.go:2726] Parsed config file /run/ovnkube-config/ovnkube.conf I0428 11:03:26.939700 3656 config.go:2727] Parsed config: {Default:{MTU:8901 RoutableMTU:0 ConntrackZone:64000 HostMasqConntrackZone:0 OVNMasqConntrackZone:0 HostNodePortConntrackZone:0 ReassemblyConntrackZone:0 EncapType:geneve EncapIP: EffectiveEncapIP: EncapPort:6081 InactivityProbe:100000 OpenFlowProbe:0 OfctrlWaitBeforeClear:0 MonitorAll:true OVSDBTxnTimeout:1m40s LFlowCacheEnable:true LFlowCacheLimit:0 LFlowCacheLimitKb:1048576 RawClusterSubnets:10.128.0.0/14/23 ClusterSubnets:[] EnableUDPAggregation:true Zone:global RawUDNAllowedDefaultServices:default/kubernetes,openshift-dns/dns-default UDNAllowedDefaultServices:[] Transport:geneve} Logging:{File: CNIFile: LibovsdbFile: Level:4 LogFileMaxSize:100 LogFileMaxBackups:5 LogFileMaxAge:5 ACLLoggingRateLimit:20} Monitoring:{RawNetFlowTargets: RawSFlowTargets: RawIPFIXTargets: NetFlowTargets:[] SFlowTargets:[] IPFIXTargets:[]} IPFIX:{Sampling:400 CacheActiveTimeout:60 CacheMaxFlows:0} CNI:{ConfDir:/etc/cni/net.d Plugin:ovn-k8s-cni-overlay} OVNKubernetesFeature:{EnableAdminNetworkPolicy:false EnableEgressIP:false EgressIPReachabiltyTotalTimeout:1 EnableEgressFirewall:false EnableEgressQoS:false EnableEgressService:false EgressIPNodeHealthCheckPort:9107 EnableMultiNetwork:false EnableNetworkSegmentation:true EnableNetworkConnect:false EnablePreconfiguredUDNAddresses:true EnableRouteAdvertisements:false EnableEVPN:false EnableMultiNetworkPolicy:false EnableStatelessNetPol:false EnableInterconnect:false EnableMultiExternalGateway:false EnablePersistentIPs:false EnableDNSNameResolver:false EnableServiceTemplateSupport:false EnableObservability:false EnableNetworkQoS:false AdvertisedUDNIsolationMode:strict EnableDynamicUDNAllocation:false UDNDeletionGracePeriod:2m0s} Kubernetes:{BootstrapKubeconfig: CertDir: CertDuration:10m0s Kubeconfig: CACert: CACertData: CAData:[] APIServer:https://api.kx-0b0c8db5ba.hypershift.local:443 Token: TokenFile: CompatServiceCIDR: RawServiceCIDRs:172.30.0.0/16 ServiceCIDRs:[] OVNConfigNamespace:openshift-ovn-kubernetes OVNEmptyLbEvents:false RawNoHostSubnetNodes: NoHostSubnetNodes: HostNetworkNamespace:openshift-host-network DisableRequestedChassis:false PlatformType:AWS HealthzBindAddress:0.0.0.0:10256 CompatMetricsBindAddress: CompatOVNMetricsBindAddress: CompatMetricsEnablePprof:false DNSServiceNamespace:openshift-dns DNSServiceName:dns-default} Metrics:{BindAddress: OVNMetricsBindAddress: ExportOVSMetrics:false EnablePprof:false NodeServerPrivKey: NodeServerCert: EnableConfigDuration:false EnableScaleMetrics:false} OvnNorth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnnb_db.db exec:} OvnSouth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnsb_db.db exec:} Gateway:{Mode:shared Interface: GatewayAcceleratedInterface: EgressGWInterface: NextHop: VLANID:0 NodeportEnable:true DisableSNATMultipleGWs:false V4JoinSubnet:100.64.0.0/16 V6JoinSubnet:fd98::/64 V4MasqueradeSubnet:169.254.169.0/29 V6MasqueradeSubnet:fd69::/125 MasqueradeIPs:{V4OVNMasqueradeIP:169.254.169.1 V6OVNMasqueradeIP:fd69::1 V4HostMasqueradeIP:169.254.169.2 V6HostMasqueradeIP:fd69::2 V4HostETPLocalMasqueradeIP:169.254.169.3 V6HostETPLocalMasqueradeIP:fd69::3 V4DummyNextHopMasqueradeIP:169.254.169.4 V6DummyNextHopMasqueradeIP:fd69::4 V4OVNServiceHairpinMasqueradeIP:169.254.169.5 V6OVNServiceHairpinMasqueradeIP:fd69::5} DisablePacketMTUCheck:false RouterSubnet: SingleNode:false DisableForwarding:false AllowNoUplink:false EphemeralPortRange:} MasterHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} ClusterMgrHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} HybridOverlay:{Enabled:false RawClusterSubnets: ClusterSubnets:[] VXLANPort:4789} OvnKubeNode:{Mode:full MgmtPortNetdev: MgmtPortDPResourceName:} ClusterManager:{V4TransitSubnet:100.88.0.0/16 V6TransitSubnet:fd97::/64} OvsPaths:{RunDir:/var/run/openvswitch/} NoOverlay:{OutboundSNAT: Routing:} ManagedBGP:{ASNumber:64512 Topology:}} I0428 11:03:26.941433 3656 kube.go:431] Waiting for certificate I0428 11:03:26.941469 3656 certificate_manager.go:422] "Certificate rotation is enabled" logger="kubernetes.io/kube-apiserver-client" I0428 11:03:26.941496 3656 certificate_manager.go:566] "Rotating certificates" logger="kubernetes.io/kube-apiserver-client" I0428 11:03:26.942153 3656 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0428 11:03:26.972781 3656 reflector.go:358] "Starting reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0428 11:03:26.972803 3656 reflector.go:404] "Listing and watching" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0428 11:03:26.975735 3656 reflector.go:436] "Caches populated" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0428 11:03:26.975819 3656 csr.go:274] "Certificate signing request is approved, waiting to be issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-m585h" I0428 11:03:26.982799 3656 csr.go:270] "Certificate signing request is issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-m585h" I0428 11:03:26.982856 3656 reflector.go:364] "Stopping reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0428 11:03:27.942042 3656 kube.go:438] Certificate found I0428 11:03:27.942829 3656 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0428 11:03:27.944160 3656 metrics.go:532] Starting metrics server at address "127.0.0.1:29103" I0428 11:03:27.995009 3656 certificate_manager.go:715] "Certificate rotation deadline determined" logger="kubernetes.io/kube-apiserver-client" expiration="2026-04-29 10:58:26 +0000 UTC" deadline="2026-04-29 05:34:45.039202069 +0000 UTC" I0428 11:03:27.995093 3656 certificate_manager.go:431] "Waiting for next certificate rotation" logger="kubernetes.io/kube-apiserver-client" sleep="18h31m17.044124293s" I0428 11:03:28.016036 3656 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0428 11:03:28.026116 3656 metrics.go:512] Create OVN Metrics Server on address: 127.0.0.1:29105 I0428 11:03:28.026174 3656 server.go:101] MetricServer registers OVS metrics I0428 11:03:28.026445 3656 server.go:105] MetricServer registers OVN DB metrics I0428 11:03:28.029722 3656 node_controller_manager.go:338] Starting the node network controller manager, Mode: full I0428 11:03:28.029790 3656 factory.go:561] Starting watch factory I0428 11:03:28.029873 3656 reflector.go:358] "Starting reflector" type="*v1.EndpointSlice" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0428 11:03:28.029892 3656 reflector.go:404] "Listing and watching" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0428 11:03:28.029899 3656 reflector.go:358] "Starting reflector" type="*v1.NetworkPolicy" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0428 11:03:28.029887 3656 reflector.go:358] "Starting reflector" type="*v1.Node" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0428 11:03:28.029934 3656 reflector.go:404] "Listing and watching" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0428 11:03:28.029946 3656 reflector.go:404] "Listing and watching" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0428 11:03:28.029875 3656 reflector.go:358] "Starting reflector" type="*v1.Service" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0428 11:03:28.029878 3656 reflector.go:358] "Starting reflector" type="*v1.Namespace" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0428 11:03:28.030243 3656 reflector.go:404] "Listing and watching" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0428 11:03:28.030582 3656 reflector.go:404] "Listing and watching" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0428 11:03:28.031744 3656 reflector.go:358] "Starting reflector" type="*v1.Pod" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0428 11:03:28.031769 3656 reflector.go:404] "Listing and watching" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0428 11:03:28.031333 3656 ovn_db.go:330] /var/run/openvswitch/ovnnb_db.sock getting info failed: stat /var/run/openvswitch/ovnnb_db.sock: no such file or directory I0428 11:03:28.032525 3656 ovn_db.go:327] ovnnb_db.sock found at /var/run/ovn/ I0428 11:03:28.034803 3656 controller_manager.go:370] Starting the ovnkube controller I0428 11:03:28.034818 3656 controller_manager.go:375] Waiting up to 5m0s for NBDB zone to match: ip-10-0-1-214.ec2.internal I0428 11:03:28.034912 3656 controller_manager.go:395] NBDB zone sync took: 69.18µs I0428 11:03:28.034947 3656 factory.go:561] Starting watch factory I0428 11:03:28.047657 3656 reflector.go:436] "Caches populated" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0428 11:03:28.048151 3656 reflector.go:436] "Caches populated" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0428 11:03:28.048213 3656 reflector.go:436] "Caches populated" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0428 11:03:28.049076 3656 reflector.go:436] "Caches populated" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0428 11:03:28.050003 3656 reflector.go:436] "Caches populated" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0428 11:03:28.057266 3656 ovn_db.go:388] Found OVN NB DB: &{0x1e9e6e0 /etc/ovn/ovnnb_db.db OVN_Northbound 0} I0428 11:03:28.057300 3656 ovn_db.go:395] Found OVN SB DB: &{0x1e9e9a0 /etc/ovn/ovnsb_db.db OVN_Southbound 0} I0428 11:03:28.058113 3656 reflector.go:436] "Caches populated" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0428 11:03:28.061961 3656 ovn_db.go:410] Found db is standalone, don't register db_cluster metrics I0428 11:03:28.061995 3656 server.go:109] MetricServer registers OVN Controller metrics I0428 11:03:28.066525 3656 server.go:113] MetricServer registers OVN Northd metrics I0428 11:03:28.070475 3656 metrics.go:519] OVN Metrics Server starts to run ... I0428 11:03:28.135311 3656 factory.go:1977] *v1.NetworkPolicy informer cache synced successfully I0428 11:03:28.135347 3656 factory.go:1977] *v1.Namespace informer cache synced successfully I0428 11:03:28.135350 3656 factory.go:1977] *v1.Node informer cache synced successfully I0428 11:03:28.135354 3656 factory.go:1977] *v1.Service informer cache synced successfully I0428 11:03:28.135357 3656 factory.go:1977] *v1.EndpointSlice informer cache synced successfully I0428 11:03:28.135364 3656 factory.go:1977] *v1.Pod informer cache synced successfully I0428 11:03:28.135489 3656 reflector.go:358] "Starting reflector" type="*v1alpha1.AdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0428 11:03:28.135515 3656 reflector.go:404] "Listing and watching" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0428 11:03:28.135490 3656 reflector.go:358] "Starting reflector" type="*v1alpha1.BaselineAdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0428 11:03:28.135552 3656 reflector.go:404] "Listing and watching" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0428 11:03:28.138378 3656 reflector.go:436] "Caches populated" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0428 11:03:28.138389 3656 reflector.go:436] "Caches populated" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0428 11:03:28.140492 3656 factory.go:1977] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0428 11:03:28.140505 3656 factory.go:1977] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0428 11:03:28.140598 3656 reflector.go:358] "Starting reflector" type="*v1.EgressIP" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0428 11:03:28.140619 3656 reflector.go:404] "Listing and watching" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0428 11:03:28.140798 3656 factory.go:1977] *v1.Namespace informer cache synced successfully I0428 11:03:28.140814 3656 factory.go:1977] *v1.Node informer cache synced successfully I0428 11:03:28.140818 3656 factory.go:1977] *v1.Service informer cache synced successfully I0428 11:03:28.140821 3656 factory.go:1977] *v1.EndpointSlice informer cache synced successfully I0428 11:03:28.140825 3656 factory.go:1977] *v1.Pod informer cache synced successfully I0428 11:03:28.140828 3656 factory.go:1977] *v1.NetworkPolicy informer cache synced successfully I0428 11:03:28.142766 3656 reflector.go:436] "Caches populated" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0428 11:03:28.146040 3656 factory.go:1977] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0428 11:03:28.146050 3656 factory.go:1977] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0428 11:03:28.146114 3656 factory.go:1977] *v1.EgressIP informer cache synced successfully I0428 11:03:28.146180 3656 reflector.go:358] "Starting reflector" type="*v1.EgressFirewall" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0428 11:03:28.146199 3656 reflector.go:404] "Listing and watching" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0428 11:03:28.148499 3656 reflector.go:436] "Caches populated" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0428 11:03:28.151283 3656 factory.go:1977] *v1.EgressIP informer cache synced successfully I0428 11:03:28.151289 3656 factory.go:1977] *v1.EgressFirewall informer cache synced successfully I0428 11:03:28.151348 3656 reflector.go:358] "Starting reflector" type="*v1.EgressQoS" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0428 11:03:28.151359 3656 reflector.go:404] "Listing and watching" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0428 11:03:28.153301 3656 reflector.go:436] "Caches populated" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0428 11:03:28.157044 3656 factory.go:1977] *v1.EgressQoS informer cache synced successfully I0428 11:03:28.157059 3656 factory.go:1977] *v1.EgressFirewall informer cache synced successfully I0428 11:03:28.157117 3656 reflector.go:358] "Starting reflector" type="*v1.EgressService" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0428 11:03:28.157130 3656 reflector.go:404] "Listing and watching" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0428 11:03:28.159128 3656 reflector.go:436] "Caches populated" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0428 11:03:28.162734 3656 factory.go:1977] *v1.EgressService informer cache synced successfully I0428 11:03:28.162811 3656 reflector.go:358] "Starting reflector" type="*v1.AdminPolicyBasedExternalRoute" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0428 11:03:28.162827 3656 reflector.go:404] "Listing and watching" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0428 11:03:28.162840 3656 factory.go:1977] *v1.EgressQoS informer cache synced successfully I0428 11:03:28.165029 3656 reflector.go:436] "Caches populated" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0428 11:03:28.168713 3656 factory.go:1977] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0428 11:03:28.168724 3656 factory.go:1977] *v1.EgressService informer cache synced successfully I0428 11:03:28.168890 3656 reflector.go:358] "Starting reflector" type="*v1.NetworkAttachmentDefinition" resyncPeriod="0s" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0428 11:03:28.168907 3656 reflector.go:404] "Listing and watching" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0428 11:03:28.171013 3656 reflector.go:436] "Caches populated" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0428 11:03:28.174719 3656 factory.go:1977] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0428 11:03:28.174727 3656 factory.go:1977] *v1.NetworkAttachmentDefinition informer cache synced successfully I0428 11:03:28.174818 3656 reflector.go:358] "Starting reflector" type="*v1.UserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0428 11:03:28.174841 3656 reflector.go:358] "Starting reflector" type="*v1.ClusterUserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0428 11:03:28.174856 3656 reflector.go:404] "Listing and watching" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0428 11:03:28.174839 3656 reflector.go:404] "Listing and watching" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0428 11:03:28.177293 3656 reflector.go:436] "Caches populated" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0428 11:03:28.177322 3656 reflector.go:436] "Caches populated" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0428 11:03:28.180273 3656 factory.go:1977] *v1.UserDefinedNetwork informer cache synced successfully I0428 11:03:28.180282 3656 factory.go:1977] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0428 11:03:28.180284 3656 factory.go:1977] *v1.NetworkAttachmentDefinition informer cache synced successfully I0428 11:03:28.180291 3656 factory.go:680] Watch Factory start up complete, took: 150.502449ms I0428 11:03:28.180365 3656 default_node_network_controller.go:171] Enable node proxy healthz server on 0.0.0.0:10256 I0428 11:03:28.185790 3656 factory.go:1977] *v1.UserDefinedNetwork informer cache synced successfully I0428 11:03:28.185801 3656 factory.go:1977] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0428 11:03:28.185809 3656 factory.go:680] Watch Factory start up complete, took: 150.861282ms I0428 11:03:28.185816 3656 controller_manager.go:406] Waiting up to 5m0s for a node to have "ip-10-0-1-214.ec2.internal" zone I0428 11:03:28.226960 3656 default_node_network_controller.go:710] Initializing the default node network controller I0428 11:03:28.308985 3656 udn_isolation.go:101] Starting UDN host isolation manager I0428 11:03:28.315410 3656 udn_isolation.go:112] Found kubelet cgroup path: system.slice/kubelet.service I0428 11:03:28.382083 3656 controller.go:133] Adding controller udn-host-isolation-manager event handlers I0428 11:03:28.382188 3656 shared_informer.go:349] "Waiting for caches to sync" controller="udn-host-isolation-manager" I0428 11:03:28.382207 3656 shared_informer.go:356] "Caches are synced" controller="udn-host-isolation-manager" I0428 11:03:28.417616 3656 controller.go:157] Starting controller udn-host-isolation-manager with 1 workers I0428 11:03:28.417695 3656 default_node_network_controller.go:809] Node ip-10-0-1-214.ec2.internal ready for ovn initialization with subnet 10.130.0.0/23 I0428 11:03:28.423515 3656 kube.go:131] Setting annotations map[k8s.ovn.org/node-encap-ips:["10.0.1.214"] k8s.ovn.org/zone-name:ip-10-0-1-214.ec2.internal] on node ip-10-0-1-214.ec2.internal I0428 11:03:28.443689 3656 gateway_init.go:234] Initializing Gateway Functionality for Gateway PreStart I0428 11:03:28.450123 3656 helper_linux.go:93] Provided gateway interface "br-ex", found as index: 5 I0428 11:03:28.450303 3656 helper_linux.go:118] Found default gateway interface br-ex 10.0.1.1 I0428 11:03:28.450453 3656 gateway_init.go:263] Preparing Gateway I0428 11:03:28.450465 3656 gateway_shared_intf.go:1687] Creating new gateway I0428 11:03:28.584717 3656 iptables.go:108] Creating table: filter chain: FORWARD I0428 11:03:28.591110 3656 iptables.go:108] Creating table: filter chain: OUTPUT I0428 11:03:28.597896 3656 gateway_shared_intf.go:1816] Gateway Creation Complete I0428 11:03:28.597912 3656 kube.go:131] Setting annotations map[k8s.ovn.org/gateway-mtu-support: k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-1-214.ec2.internal","mac-address":"0a:ff:f1:76:ae:cf","ip-addresses":["10.0.1.214/24"],"ip-address":"10.0.1.214/24","next-hops":["10.0.1.1"],"next-hop":"10.0.1.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:ce5d33ea-36a8-4c3e-8972-2035a81837fa k8s.ovn.org/node-encap-ips:["10.0.1.214"] k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.1.214/24"} k8s.ovn.org/zone-name:ip-10-0-1-214.ec2.internal] on node ip-10-0-1-214.ec2.internal I0428 11:03:28.616398 3656 default_node_network_controller.go:519] Node connection status = connected I0428 11:03:28.686496 3656 controller_manager.go:429] Waiting for node in zone sync took: 500.664024ms I0428 11:03:28.691515 3656 controller_manager.go:611] Switching to transit router for layer2 networks I0428 11:03:28.691533 3656 kube.go:131] Setting annotations map[k8s.ovn.org/layer2-topology-version:2.0] on node ip-10-0-1-214.ec2.internal I0428 11:03:28.706967 3656 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0428 11:03:28.714055 3656 controller_manager.go:290] SCTP support detected in OVN I0428 11:03:28.714769 3656 services_controller.go:69] Creating services controller for network=default I0428 11:03:28.714880 3656 udn_enabled_svc.go:80] Waiting for service informer to sync I0428 11:03:28.715070 3656 controller.go:133] Adding controller [zone-nad-controller NAD controller] event handlers I0428 11:03:28.715135 3656 shared_informer.go:349] "Waiting for caches to sync" controller="[zone-nad-controller NAD controller]" I0428 11:03:28.715150 3656 shared_informer.go:356] "Caches are synced" controller="[zone-nad-controller NAD controller]" I0428 11:03:28.715159 3656 nad_controller.go:541] [zone-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0428 11:03:28.715344 3656 controller.go:157] Starting controller [zone-nad-controller NAD controller] with 1 workers I0428 11:03:28.715385 3656 udn_enabled_svc.go:95] Performing full resync I0428 11:03:28.715403 3656 network_controller.go:373] [zone-nad-controller network controller]: syncing all networks I0428 11:03:28.715457 3656 nad_controller.go:580] [zone-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 57.943µs I0428 11:03:28.715484 3656 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 73.604µs I0428 11:03:28.715495 3656 network_controller.go:384] [zone-nad-controller network controller]: finished syncing all networks. Time taken: 91.298µs I0428 11:03:28.715504 3656 controller.go:157] Starting controller [zone-nad-controller network controller] with 1 workers I0428 11:03:28.715521 3656 nad_controller.go:379] [zone-nad-controller NAD controller]: started I0428 11:03:28.715537 3656 default_network_controller.go:325] Starting the default network controller I0428 11:03:28.715544 3656 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 11.526µs I0428 11:03:28.715729 3656 udn_enabled_svc.go:99] Waiting for handler to sync I0428 11:03:28.715739 3656 udn_enabled_svc.go:104] Starting worker I0428 11:03:28.726003 3656 default_network_controller.go:395] Cleaning External Gateway ECMP routes I0428 11:03:28.726058 3656 repair.go:33] Syncing exgw routes took 42.725µs I0428 11:03:28.726082 3656 default_network_controller.go:414] Starting all the Watchers... I0428 11:03:28.726214 3656 namespace.go:98] [openshift-cluster-version] adding namespace I0428 11:03:28.726227 3656 namespace.go:98] [openshift-operators-redhat] adding namespace I0428 11:03:28.726231 3656 namespace.go:98] [openshift-kube-storage-version-migrator-operator] adding namespace I0428 11:03:28.726231 3656 namespace.go:98] [openshift-kube-controller-manager-operator] adding namespace I0428 11:03:28.726241 3656 namespace.go:98] [openshift-ovn-kubernetes] adding namespace I0428 11:03:28.726216 3656 namespace.go:98] [openshift-kube-apiserver-operator] adding namespace I0428 11:03:28.726250 3656 namespace.go:98] [openshift-network-operator] adding namespace I0428 11:03:28.726244 3656 namespace.go:98] [openshift-image-registry] adding namespace I0428 11:03:28.726258 3656 namespace.go:98] [openshift-operator-lifecycle-manager] adding namespace I0428 11:03:28.726257 3656 namespace.go:98] [openshift-customer-monitoring] adding namespace I0428 11:03:28.726266 3656 namespace.go:98] [openshift-authentication-operator] adding namespace I0428 11:03:28.726266 3656 namespace.go:98] [openshift-controller-manager-operator] adding namespace I0428 11:03:28.726272 3656 namespace.go:98] [openshift-operators] adding namespace I0428 11:03:28.726277 3656 namespace.go:98] [openshift-backplane-mobb] adding namespace I0428 11:03:28.726216 3656 namespace.go:98] [openshift-machine-api] adding namespace I0428 11:03:28.726860 3656 namespace.go:102] [openshift-cluster-version] adding namespace took 629.787µs I0428 11:03:28.726881 3656 namespace.go:98] [openshift-backplane-tam] adding namespace I0428 11:03:28.727398 3656 namespace.go:102] [openshift-backplane-tam] adding namespace took 509.991µs I0428 11:03:28.727413 3656 namespace.go:98] [openshift-backplane-cse] adding namespace I0428 11:03:28.727941 3656 namespace.go:102] [openshift-backplane-cse] adding namespace took 522.881µs I0428 11:03:28.727955 3656 namespace.go:98] [openshift-backplane-srep-ro] adding namespace I0428 11:03:28.728523 3656 namespace.go:102] [openshift-operators-redhat] adding namespace took 2.28655ms I0428 11:03:28.728537 3656 namespace.go:98] [openshift-network-diagnostics] adding namespace I0428 11:03:28.729100 3656 namespace.go:102] [openshift-kube-storage-version-migrator-operator] adding namespace took 2.852333ms I0428 11:03:28.729119 3656 namespace.go:98] [openshift-backplane-srep] adding namespace I0428 11:03:28.730073 3656 namespace.go:102] [openshift-kube-controller-manager-operator] adding namespace took 3.825756ms I0428 11:03:28.730088 3656 namespace.go:98] [default] adding namespace I0428 11:03:28.730573 3656 namespace.go:102] [openshift-ovn-kubernetes] adding namespace took 4.322042ms I0428 11:03:28.730588 3656 namespace.go:98] [openshift-config-operator] adding namespace I0428 11:03:28.731120 3656 namespace.go:102] [openshift-kube-apiserver-operator] adding namespace took 4.866739ms I0428 11:03:28.731134 3656 namespace.go:98] [openshift-etcd] adding namespace I0428 11:03:28.731571 3656 namespace.go:102] [openshift-network-operator] adding namespace took 5.316731ms I0428 11:03:28.731583 3656 namespace.go:98] [openshift-service-ca-operator] adding namespace I0428 11:03:28.732093 3656 namespace.go:102] [openshift-image-registry] adding namespace took 5.836149ms I0428 11:03:28.732106 3656 namespace.go:98] [openshift-dns-operator] adding namespace I0428 11:03:28.732653 3656 namespace.go:102] [openshift-operator-lifecycle-manager] adding namespace took 6.387658ms I0428 11:03:28.732666 3656 namespace.go:98] [openshift-cluster-storage-operator] adding namespace I0428 11:03:28.733237 3656 namespace.go:102] [openshift-customer-monitoring] adding namespace took 6.968417ms I0428 11:03:28.733253 3656 namespace.go:98] [openshift-ingress-operator] adding namespace I0428 11:03:28.733755 3656 namespace.go:102] [openshift-authentication-operator] adding namespace took 7.484444ms I0428 11:03:28.733765 3656 namespace.go:98] [openshift-machine-config-operator] adding namespace I0428 11:03:28.734258 3656 namespace.go:102] [openshift-controller-manager-operator] adding namespace took 7.983881ms I0428 11:03:28.734270 3656 namespace.go:98] [openshift-cloud-controller-manager] adding namespace I0428 11:03:28.734867 3656 namespace.go:102] [openshift-operators] adding namespace took 8.590387ms I0428 11:03:28.734876 3656 namespace.go:98] [openshift-multus] adding namespace I0428 11:03:28.735398 3656 namespace.go:102] [openshift-backplane-mobb] adding namespace took 9.112395ms I0428 11:03:28.735415 3656 namespace.go:98] [openshift-ingress-canary] adding namespace I0428 11:03:28.735785 3656 namespace.go:102] [openshift-machine-api] adding namespace took 9.447577ms I0428 11:03:28.735799 3656 namespace.go:98] [openshift-monitoring] adding namespace I0428 11:03:28.736265 3656 namespace.go:102] [openshift-backplane-srep-ro] adding namespace took 8.303571ms W0428 11:03:28.736265 3656 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-mxd6k: pod openshift-network-diagnostics/network-check-target-mxd6k: no pod IPs found I0428 11:03:28.736286 3656 namespace.go:98] [openshift-console-user-settings] adding namespace I0428 11:03:28.736842 3656 namespace.go:102] [openshift-network-diagnostics] adding namespace took 8.29952ms I0428 11:03:28.736856 3656 namespace.go:98] [openshift-cluster-node-tuning-operator] adding namespace I0428 11:03:28.737312 3656 namespace.go:102] [openshift-backplane-srep] adding namespace took 8.184378ms I0428 11:03:28.737330 3656 namespace.go:98] [openshift-config] adding namespace I0428 11:03:28.737695 3656 namespace.go:102] [default] adding namespace took 7.601328ms I0428 11:03:28.737710 3656 namespace.go:98] [openshift-ingress] adding namespace I0428 11:03:28.738134 3656 namespace.go:102] [openshift-config-operator] adding namespace took 7.538971ms I0428 11:03:28.738148 3656 namespace.go:98] [openshift-cluster-samples-operator] adding namespace I0428 11:03:28.738638 3656 namespace.go:102] [openshift-etcd] adding namespace took 7.496626ms I0428 11:03:28.738657 3656 namespace.go:98] [kube-node-lease] adding namespace I0428 11:03:28.739084 3656 namespace.go:102] [openshift-service-ca-operator] adding namespace took 7.496458ms I0428 11:03:28.739096 3656 namespace.go:98] [openshift] adding namespace I0428 11:03:28.739652 3656 namespace.go:102] [openshift-dns-operator] adding namespace took 7.53864ms I0428 11:03:28.739669 3656 namespace.go:98] [open-cluster-management-2pv452ledagn57tf47lfif0ho4ffo09v] adding namespace I0428 11:03:28.740125 3656 namespace.go:102] [openshift-cluster-storage-operator] adding namespace took 7.450812ms I0428 11:03:28.740143 3656 namespace.go:98] [openshift-kube-controller-manager] adding namespace I0428 11:03:28.740648 3656 namespace.go:102] [openshift-ingress-operator] adding namespace took 7.390278ms I0428 11:03:28.740660 3656 namespace.go:98] [openshift-backplane-mcs-tier-two] adding namespace I0428 11:03:28.741114 3656 namespace.go:102] [openshift-machine-config-operator] adding namespace took 7.342865ms I0428 11:03:28.741130 3656 namespace.go:98] [openshift-console-operator] adding namespace W0428 11:03:28.741600 3656 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-multus/network-metrics-daemon-6hd9d: pod openshift-multus/network-metrics-daemon-6hd9d: no pod IPs found I0428 11:03:28.741602 3656 namespace.go:102] [openshift-cloud-controller-manager] adding namespace took 7.325021ms I0428 11:03:28.741617 3656 namespace.go:98] [openshift-dns] adding namespace I0428 11:03:28.742288 3656 namespace.go:102] [openshift-multus] adding namespace took 7.406157ms I0428 11:03:28.742305 3656 namespace.go:98] [openshift-backplane] adding namespace I0428 11:03:28.742831 3656 namespace.go:102] [openshift-ingress-canary] adding namespace took 7.408095ms I0428 11:03:28.742852 3656 namespace.go:98] [openshift-authentication] adding namespace I0428 11:03:28.743364 3656 namespace.go:102] [openshift-monitoring] adding namespace took 7.559926ms I0428 11:03:28.743381 3656 namespace.go:98] [openshift-host-network] adding namespace I0428 11:03:28.743964 3656 namespace.go:102] [openshift-console-user-settings] adding namespace took 7.671031ms I0428 11:03:28.744588 3656 namespace.go:102] [openshift-cluster-node-tuning-operator] adding namespace took 7.726002ms I0428 11:03:28.744602 3656 namespace.go:98] [openshift-cluster-csi-drivers] adding namespace I0428 11:03:28.745161 3656 namespace.go:102] [openshift-config] adding namespace took 7.825248ms I0428 11:03:28.745174 3656 namespace.go:98] [openshift-backplane-lpsre] adding namespace I0428 11:03:28.745805 3656 namespace.go:102] [openshift-ingress] adding namespace took 8.087658ms I0428 11:03:28.745824 3656 namespace.go:98] [openshift-cloud-credential-operator] adding namespace I0428 11:03:28.746295 3656 namespace.go:102] [openshift-cluster-samples-operator] adding namespace took 8.141089ms I0428 11:03:28.746312 3656 namespace.go:98] [openshift-network-console] adding namespace I0428 11:03:28.746843 3656 namespace.go:102] [kube-node-lease] adding namespace took 8.180298ms I0428 11:03:28.746856 3656 namespace.go:98] [openshift-logging] adding namespace I0428 11:03:28.747390 3656 namespace.go:102] [openshift] adding namespace took 8.288547ms I0428 11:03:28.747405 3656 namespace.go:98] [openshift-marketplace] adding namespace I0428 11:03:28.747962 3656 namespace.go:102] [open-cluster-management-2pv452ledagn57tf47lfif0ho4ffo09v] adding namespace took 8.286326ms I0428 11:03:28.747980 3656 namespace.go:98] [openshift-user-workload-monitoring] adding namespace I0428 11:03:28.748461 3656 namespace.go:102] [openshift-kube-controller-manager] adding namespace took 8.312459ms I0428 11:03:28.748482 3656 namespace.go:98] [openshift-route-controller-manager] adding namespace I0428 11:03:28.748976 3656 namespace.go:102] [openshift-backplane-mcs-tier-two] adding namespace took 8.309434ms I0428 11:03:28.748993 3656 namespace.go:98] [openshift-node] adding namespace I0428 11:03:28.749512 3656 namespace.go:102] [openshift-console-operator] adding namespace took 8.377306ms I0428 11:03:28.749524 3656 namespace.go:98] [openshift-apiserver] adding namespace I0428 11:03:28.750050 3656 namespace.go:102] [openshift-dns] adding namespace took 8.428152ms I0428 11:03:28.750063 3656 namespace.go:98] [openshift-network-node-identity] adding namespace I0428 11:03:28.750586 3656 namespace.go:102] [openshift-backplane] adding namespace took 8.273051ms I0428 11:03:28.750605 3656 namespace.go:98] [openshift-cloud-network-config-controller] adding namespace I0428 11:03:28.751088 3656 namespace.go:102] [openshift-authentication] adding namespace took 8.23011ms I0428 11:03:28.751103 3656 namespace.go:98] [open-cluster-management-agent-addon] adding namespace I0428 11:03:28.752112 3656 namespace.go:102] [openshift-host-network] adding namespace took 8.724728ms I0428 11:03:28.752130 3656 namespace.go:98] [openshift-package-operator] adding namespace I0428 11:03:28.752662 3656 namespace.go:102] [openshift-cluster-csi-drivers] adding namespace took 8.053515ms I0428 11:03:28.752676 3656 namespace.go:98] [openshift-deployment-validation-operator] adding namespace I0428 11:03:28.753226 3656 namespace.go:102] [openshift-backplane-lpsre] adding namespace took 8.046091ms I0428 11:03:28.753243 3656 namespace.go:98] [openshift-backplane-cee] adding namespace I0428 11:03:28.753717 3656 namespace.go:102] [openshift-cloud-credential-operator] adding namespace took 7.887199ms I0428 11:03:28.753735 3656 namespace.go:98] [openshift-kube-scheduler-operator] adding namespace I0428 11:03:28.754277 3656 namespace.go:102] [openshift-network-console] adding namespace took 7.958298ms I0428 11:03:28.754294 3656 namespace.go:98] [openshift-kube-apiserver] adding namespace I0428 11:03:28.754860 3656 namespace.go:102] [openshift-logging] adding namespace took 7.995959ms I0428 11:03:28.754880 3656 namespace.go:98] [openshift-kube-scheduler] adding namespace I0428 11:03:28.755391 3656 namespace.go:102] [openshift-marketplace] adding namespace took 7.979337ms I0428 11:03:28.755407 3656 namespace.go:98] [openshift-insights] adding namespace I0428 11:03:28.755868 3656 namespace.go:102] [openshift-user-workload-monitoring] adding namespace took 7.882211ms I0428 11:03:28.755884 3656 namespace.go:98] [openshift-cluster-machine-approver] adding namespace I0428 11:03:28.756407 3656 namespace.go:102] [openshift-route-controller-manager] adding namespace took 7.917765ms I0428 11:03:28.756424 3656 namespace.go:98] [openshift-console] adding namespace I0428 11:03:28.756957 3656 namespace.go:102] [openshift-node] adding namespace took 7.958558ms I0428 11:03:28.756968 3656 namespace.go:98] [kube-public] adding namespace I0428 11:03:28.757535 3656 namespace.go:102] [openshift-apiserver] adding namespace took 8.003249ms I0428 11:03:28.757554 3656 namespace.go:98] [dedicated-admin] adding namespace I0428 11:03:28.758043 3656 namespace.go:102] [openshift-network-node-identity] adding namespace took 7.974966ms I0428 11:03:28.758053 3656 namespace.go:98] [openshift-backplane-csm] adding namespace I0428 11:03:28.758525 3656 namespace.go:102] [openshift-cloud-network-config-controller] adding namespace took 7.914386ms I0428 11:03:28.758539 3656 namespace.go:98] [openshift-config-managed] adding namespace I0428 11:03:28.759046 3656 namespace.go:102] [open-cluster-management-agent-addon] adding namespace took 7.936885ms I0428 11:03:28.759063 3656 namespace.go:98] [openshift-infra] adding namespace I0428 11:03:28.759473 3656 namespace.go:102] [openshift-package-operator] adding namespace took 7.336705ms I0428 11:03:28.759489 3656 namespace.go:98] [kube-system] adding namespace I0428 11:03:28.759934 3656 namespace.go:102] [openshift-deployment-validation-operator] adding namespace took 7.254142ms I0428 11:03:28.760456 3656 namespace.go:102] [openshift-backplane-cee] adding namespace took 7.205194ms I0428 11:03:28.760480 3656 namespace.go:98] [openshift-backplane-managed-scripts] adding namespace I0428 11:03:28.760884 3656 namespace.go:102] [openshift-kube-scheduler-operator] adding namespace took 7.143325ms I0428 11:03:28.760897 3656 namespace.go:98] [openshift-apiserver-operator] adding namespace I0428 11:03:28.761297 3656 namespace.go:102] [openshift-kube-apiserver] adding namespace took 6.999235ms I0428 11:03:28.761853 3656 namespace.go:102] [openshift-kube-scheduler] adding namespace took 6.967557ms I0428 11:03:28.761868 3656 namespace.go:98] [openshift-controller-manager] adding namespace I0428 11:03:28.762301 3656 namespace.go:102] [openshift-insights] adding namespace took 6.889159ms I0428 11:03:28.762315 3656 namespace.go:98] [openshift-must-gather-operator] adding namespace I0428 11:03:28.762742 3656 namespace.go:102] [openshift-cluster-machine-approver] adding namespace took 6.852944ms I0428 11:03:28.763221 3656 namespace.go:102] [openshift-console] adding namespace took 6.791348ms I0428 11:03:28.763714 3656 namespace.go:102] [kube-public] adding namespace took 6.741473ms I0428 11:03:28.764206 3656 namespace.go:102] [dedicated-admin] adding namespace took 6.646351ms I0428 11:03:28.764634 3656 namespace.go:102] [openshift-backplane-csm] adding namespace took 6.576234ms I0428 11:03:28.765139 3656 namespace.go:102] [openshift-config-managed] adding namespace took 6.593789ms I0428 11:03:28.765681 3656 namespace.go:102] [openshift-infra] adding namespace took 6.612659ms I0428 11:03:28.766234 3656 namespace.go:102] [kube-system] adding namespace took 6.738119ms I0428 11:03:28.766740 3656 namespace.go:102] [openshift-backplane-managed-scripts] adding namespace took 6.252116ms I0428 11:03:28.767349 3656 namespace.go:102] [openshift-apiserver-operator] adding namespace took 6.445326ms I0428 11:03:28.767890 3656 namespace.go:102] [openshift-controller-manager] adding namespace took 6.015233ms I0428 11:03:28.768394 3656 namespace.go:102] [openshift-must-gather-operator] adding namespace took 6.074286ms I0428 11:03:28.768579 3656 master.go:548] Adding or Updating local node "ip-10-0-1-214.ec2.internal" for network "default" I0428 11:03:28.769657 3656 zone_ic_handler.go:195] Time taken to create transit switch: 306.168µs I0428 11:03:28.769671 3656 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-0-142.ec2.internal for the network default E0428 11:03:28.769748 3656 obj_retry.go:575] Failed to create *v1.Node ip-10-0-2-127.ec2.internal, error: adding or updating remote node IC resources ip-10-0-2-127.ec2.internal failed, err - ensuring transit switch for remote zone node ip-10-0-2-127.ec2.internal for the network default failed : err - failed to create/update transit switch transit_switch: error in transact with ops [{Op:wait Table:Logical_Switch Row:map[] Rows:[map[name:transit_switch]] Columns:[name] Mutations:[] Timeout:0xc0097710f0 Where:[where column name == transit_switch] Until:!= Durable: Comment: Lock: UUID: UUIDName:} {Op:insert Table:Logical_Switch Row:map[name:transit_switch other_config:{GoMap:map[interconn-ts:transit_switch mcast_flood_unregistered:true mcast_querier:false mcast_snoop:true requested-tnl-key:16711683]}] Rows:[] Columns:[] Mutations:[] Timeout: Where:[] Until: Durable: Comment: Lock: UUID: UUIDName:u2610654604}] results [{Count:0 Error:timed out Details:"where" clause test failed UUID:{GoUUID:} Rows:[]} {Count:0 Error: Details: UUID:{GoUUID:} Rows:[]}] and errors [timed out: "where" clause test failed]: 1 ovsdb operations failed I0428 11:03:28.769815 3656 event.go:377] Event(v1.ObjectReference{Kind:"Node", Namespace:"", Name:"ip-10-0-2-127.ec2.internal", UID:"b3823e9d-17ee-4dc3-88cd-3899ffe4a35d", APIVersion:"v1", ResourceVersion:"8964", FieldPath:""}): type: 'Warning' reason: 'ErrorAddingResource' adding or updating remote node IC resources ip-10-0-2-127.ec2.internal failed, err - ensuring transit switch for remote zone node ip-10-0-2-127.ec2.internal for the network default failed : err - failed to create/update transit switch transit_switch: error in transact with ops [{Op:wait Table:Logical_Switch Row:map[] Rows:[map[name:transit_switch]] Columns:[name] Mutations:[] Timeout:0xc0097710f0 Where:[where column name == transit_switch] Until:!= Durable: Comment: Lock: UUID: UUIDName:} {Op:insert Table:Logical_Switch Row:map[name:transit_switch other_config:{GoMap:map[interconn-ts:transit_switch mcast_flood_unregistered:true mcast_querier:false mcast_snoop:true requested-tnl-key:16711683]}] Rows:[] Columns:[] Mutations:[] Timeout: Where:[] Until: Durable: Comment: Lock: UUID: UUIDName:u2610654604}] results [{Count:0 Error:timed out Details:"where" clause test failed UUID:{GoUUID:} Rows:[]} {Count:0 Error: Details: UUID:{GoUUID:} Rows:[]}] and errors [timed out: "where" clause test failed]: 1 ovsdb operations failed I0428 11:03:28.771020 3656 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-0-142.ec2.internal" on network "default" took: 1.676976ms W0428 11:03:28.777624 3656 gateway.go:207] Unable to fetch podIPs for pod openshift-network-diagnostics/network-check-target-mxd6k: pod openshift-network-diagnostics/network-check-target-mxd6k: no pod IPs found W0428 11:03:28.777649 3656 gateway.go:207] Unable to fetch podIPs for pod openshift-multus/network-metrics-daemon-6hd9d: pod openshift-multus/network-metrics-daemon-6hd9d: no pod IPs found I0428 11:03:28.778670 3656 zone_ic_handler.go:203] Creating interconnect resources for local zone node ip-10-0-1-214.ec2.internal for the network default I0428 11:03:28.778867 3656 zone_ic_handler.go:195] Time taken to create transit switch: 184.639µs I0428 11:03:28.779694 3656 services_controller.go:192] Starting controller ovn-lb-controller for network=default I0428 11:03:28.779731 3656 services_controller.go:200] Waiting for node tracker handler to sync for network=default I0428 11:03:28.779752 3656 shared_informer.go:349] "Waiting for caches to sync" controller="node-tracker-controller" I0428 11:03:28.779733 3656 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-142.ec2.internal in network "default" I0428 11:03:28.779827 3656 node_tracker.go:169] Node ip-10-0-0-142.ec2.internal switch + router changed, syncing services in network "default" I0428 11:03:28.779848 3656 services_controller.go:574] Full service sync requested for network=default I0428 11:03:28.779860 3656 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-214.ec2.internal in network "default" I0428 11:03:28.779886 3656 node_tracker.go:169] Node ip-10-0-1-214.ec2.internal switch + router changed, syncing services in network "default" I0428 11:03:28.779894 3656 services_controller.go:574] Full service sync requested for network=default I0428 11:03:28.779900 3656 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-127.ec2.internal in network "default" I0428 11:03:28.779941 3656 node_tracker.go:169] Node ip-10-0-2-127.ec2.internal switch + router changed, syncing services in network "default" I0428 11:03:28.779950 3656 services_controller.go:574] Full service sync requested for network=default I0428 11:03:28.880856 3656 shared_informer.go:356] "Caches are synced" controller="node-tracker-controller" I0428 11:03:28.880877 3656 services_controller.go:208] Setting up event handlers for services for network=default I0428 11:03:28.881003 3656 services_controller.go:218] Setting up event handlers for endpoint slices for network=default I0428 11:03:28.881081 3656 services_controller.go:233] Waiting for service and endpoint handlers to sync for network=default I0428 11:03:28.881105 3656 shared_informer.go:349] "Waiting for caches to sync" controller="ovn-lb-controller" I0428 11:03:28.981292 3656 shared_informer.go:356] "Caches are synced" controller="ovn-lb-controller" I0428 11:03:28.981311 3656 repair.go:60] Starting repairing loop for services I0428 11:03:28.981396 3656 repair.go:131] Deleted 0 stale service LBs I0428 11:03:28.981415 3656 repair.go:137] Deleted 0 stale Chassis Template Vars I0428 11:03:28.981433 3656 repair.go:62] Finished repairing loop for services: 123.543µs I0428 11:03:28.981464 3656 services_controller.go:363] Controller cache of 0 load balancers initialized for 0 services for network=default I0428 11:03:28.981475 3656 services_controller.go:254] Starting workers for network=default I0428 11:03:28.982349 3656 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-6hd9d] creating logical port openshift-multus_network-metrics-daemon-6hd9d for pod on switch ip-10-0-1-214.ec2.internal I0428 11:03:28.982498 3656 kube.go:256] Updating pod openshift-multus/network-metrics-daemon-6hd9d I0428 11:03:28.982566 3656 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-mxd6k] creating logical port openshift-network-diagnostics_network-check-target-mxd6k for pod on switch ip-10-0-1-214.ec2.internal I0428 11:03:28.982659 3656 kube.go:256] Updating pod openshift-network-diagnostics/network-check-target-mxd6k I0428 11:03:28.997257 3656 pod.go:62] [openshift-network-diagnostics/network-check-target-mxd6k] pod update took 14.624597ms I0428 11:03:28.997277 3656 base_network_controller_pods.go:951] [default/openshift-network-diagnostics/network-check-target-mxd6k] addLogicalPort annotation time took 14.647718ms I0428 11:03:28.998270 3656 pod.go:62] [openshift-multus/network-metrics-daemon-6hd9d] pod update took 15.831651ms I0428 11:03:28.998291 3656 base_network_controller_pods.go:951] [default/openshift-multus/network-metrics-daemon-6hd9d] addLogicalPort annotation time took 15.85607ms I0428 11:03:28.998372 3656 pods.go:271] [openshift-network-diagnostics/network-check-target-mxd6k] addLogicalPort took 15.816587ms, libovsdb time 876.894µs I0428 11:03:28.999302 3656 pods.go:271] [openshift-multus/network-metrics-daemon-6hd9d] addLogicalPort took 16.977483ms, libovsdb time 695.859µs I0428 11:03:29.000029 3656 admin_network_policy_controller.go:227] Starting controller default-network-controller I0428 11:03:29.000057 3656 shared_informer.go:349] "Waiting for caches to sync" controller="default-network-controller" I0428 11:03:29.000074 3656 shared_informer.go:356] "Caches are synced" controller="default-network-controller" I0428 11:03:29.000081 3656 admin_network_policy_controller.go:237] Repairing Admin Network Policies I0428 11:03:29.000276 3656 repair.go:29] Repairing admin network policies took 188.912µs I0428 11:03:29.000390 3656 repair.go:92] Repairing baseline admin network policies took 104.432µs I0428 11:03:29.000844 3656 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/package-server-manager for network default I0428 11:03:29.000864 3656 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/package-server-manager added to peer address sets [] I0428 11:03:29.003660 3656 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/package-server-manager resources completed, update namespace loglevel I0428 11:03:29.003689 3656 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-egress for network default I0428 11:03:29.003698 3656 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-egress added to peer address sets [] I0428 11:03:29.005563 3656 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-egress resources completed, update namespace loglevel I0428 11:03:29.005589 3656 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-metrics for network default I0428 11:03:29.005600 3656 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-metrics added to peer address sets [] I0428 11:03:29.006200 3656 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-metrics resources completed, update namespace loglevel I0428 11:03:29.006220 3656 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-pprof for network default I0428 11:03:29.006228 3656 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-pprof added to peer address sets [] I0428 11:03:29.006859 3656 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-pprof resources completed, update namespace loglevel I0428 11:03:29.006877 3656 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-to-dns for network default I0428 11:03:29.007427 3656 pod_selector_address_set.go:213] Created shared address set for pod selector LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},} I0428 11:03:29.007444 3656 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0428 11:03:29.009897 3656 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-to-dns resources completed, update namespace loglevel I0428 11:03:29.009929 3656 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-insights for network default I0428 11:03:29.009949 3656 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-insights added to peer address sets [] I0428 11:03:29.012128 3656 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-insights resources completed, update namespace loglevel I0428 11:03:29.012149 3656 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-egress-to-api-server for network default I0428 11:03:29.012158 3656 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-egress-to-api-server added to peer address sets [] I0428 11:03:29.012704 3656 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-egress-to-api-server resources completed, update namespace loglevel I0428 11:03:29.012726 3656 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/default-deny for network default I0428 11:03:29.012733 3656 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/default-deny added to peer address sets [] I0428 11:03:29.014355 3656 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/default-deny resources completed, update namespace loglevel I0428 11:03:29.014380 3656 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-version/default-deny for network default I0428 11:03:29.014396 3656 base_network_controller_policy.go:1054] Policy openshift-cluster-version/default-deny added to peer address sets [] I0428 11:03:29.015767 3656 base_network_controller_policy.go:1232] Create network policy openshift-cluster-version/default-deny resources completed, update namespace loglevel I0428 11:03:29.015789 3656 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server for network default I0428 11:03:29.015800 3656 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server added to peer address sets [] I0428 11:03:29.016393 3656 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server resources completed, update namespace loglevel I0428 11:03:29.016415 3656 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles for network default I0428 11:03:29.016424 3656 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles added to peer address sets [] I0428 11:03:29.017349 3656 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles resources completed, update namespace loglevel I0428 11:03:29.017369 3656 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/unpack-bundles for network default I0428 11:03:29.017378 3656 base_network_controller_policy.go:1054] Policy openshift-marketplace/unpack-bundles added to peer address sets [] I0428 11:03:29.019432 3656 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/unpack-bundles resources completed, update namespace loglevel I0428 11:03:29.019451 3656 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/catalog-operator for network default I0428 11:03:29.019461 3656 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/catalog-operator added to peer address sets [] I0428 11:03:29.020869 3656 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/catalog-operator resources completed, update namespace loglevel I0428 11:03:29.020892 3656 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/default-deny-all for network default I0428 11:03:29.020902 3656 base_network_controller_policy.go:1054] Policy openshift-marketplace/default-deny-all added to peer address sets [] I0428 11:03:29.021272 3656 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/default-deny-all resources completed, update namespace loglevel I0428 11:03:29.021296 3656 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-all-egress for network default I0428 11:03:29.021309 3656 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-all-egress added to peer address sets [] I0428 11:03:29.021986 3656 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-all-egress resources completed, update namespace loglevel I0428 11:03:29.022006 3656 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/default-deny-all-traffic for network default I0428 11:03:29.022013 3656 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/default-deny-all-traffic added to peer address sets [] I0428 11:03:29.022391 3656 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/default-deny-all-traffic resources completed, update namespace loglevel I0428 11:03:29.022413 3656 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/marketplace-operator for network default I0428 11:03:29.022423 3656 base_network_controller_policy.go:1054] Policy openshift-marketplace/marketplace-operator added to peer address sets [] I0428 11:03:29.023600 3656 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/marketplace-operator resources completed, update namespace loglevel I0428 11:03:29.023621 3656 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-webhook for network default I0428 11:03:29.023633 3656 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-webhook added to peer address sets [] I0428 11:03:29.024221 3656 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-webhook resources completed, update namespace loglevel I0428 11:03:29.024240 3656 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/default-deny for network default I0428 11:03:29.024249 3656 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/default-deny added to peer address sets [] I0428 11:03:29.024572 3656 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/default-deny resources completed, update namespace loglevel I0428 11:03:29.024590 3656 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics for network default I0428 11:03:29.024598 3656 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics added to peer address sets [] I0428 11:03:29.025084 3656 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics resources completed, update namespace loglevel I0428 11:03:29.025106 3656 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range for network default I0428 11:03:29.025117 3656 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range added to peer address sets [] I0428 11:03:29.025630 3656 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range resources completed, update namespace loglevel I0428 11:03:29.025648 3656 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-egress-to-api-server for network default I0428 11:03:29.025656 3656 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-egress-to-api-server added to peer address sets [] I0428 11:03:29.026214 3656 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-egress-to-api-server resources completed, update namespace loglevel I0428 11:03:29.026231 3656 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-to-dns for network default I0428 11:03:29.026260 3656 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0428 11:03:29.027073 3656 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-to-dns resources completed, update namespace loglevel I0428 11:03:29.027092 3656 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-olm for network default I0428 11:03:29.027100 3656 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-olm added to peer address sets [] I0428 11:03:29.027676 3656 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-olm resources completed, update namespace loglevel I0428 11:03:29.027697 3656 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/packageserver for network default I0428 11:03:29.027707 3656 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/packageserver added to peer address sets [] I0428 11:03:29.029031 3656 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/packageserver resources completed, update namespace loglevel I0428 11:03:29.029054 3656 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/olm-operator for network default I0428 11:03:29.029067 3656 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/olm-operator added to peer address sets [] I0428 11:03:29.030132 3656 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/olm-operator resources completed, update namespace loglevel I0428 11:03:29.030150 3656 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-all-egress for network default I0428 11:03:29.030161 3656 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-all-egress added to peer address sets [] I0428 11:03:29.030675 3656 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-all-egress resources completed, update namespace loglevel I0428 11:03:29.030699 3656 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range for network default I0428 11:03:29.030711 3656 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range added to peer address sets [] I0428 11:03:29.031237 3656 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range resources completed, update namespace loglevel I0428 11:03:29.031275 3656 base_network_controller_policy.go:1179] Adding network policy openshift-operators/default-allow-all for network default I0428 11:03:29.031296 3656 base_network_controller_policy.go:1054] Policy openshift-operators/default-allow-all added to peer address sets [] I0428 11:03:29.033240 3656 base_network_controller_policy.go:1232] Create network policy openshift-operators/default-allow-all resources completed, update namespace loglevel I0428 11:03:29.040283 3656 egressfirewall.go:374] Starting EgressFirewall controller I0428 11:03:29.040322 3656 controller.go:133] Adding controller egress-firewall-controller event handlers I0428 11:03:29.040342 3656 controller.go:133] Adding controller egress-firewall-controller-node event handlers I0428 11:03:29.040354 3656 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller" I0428 11:03:29.040374 3656 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller" I0428 11:03:29.040381 3656 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller-node" I0428 11:03:29.040393 3656 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller-node" I0428 11:03:29.040673 3656 controller.go:157] Starting controller egress-firewall-controller with 1 workers I0428 11:03:29.040689 3656 controller.go:157] Starting controller egress-firewall-controller-node with 1 workers I0428 11:03:29.040697 3656 controller.go:157] Starting controller egress-firewall-controller-NAD with 1 workers I0428 11:03:29.040706 3656 egressqos.go:193] Setting up event handlers for EgressQoS I0428 11:03:29.040796 3656 egressfirewall.go:723] Syncing node "ip-10-0-0-142.ec2.internal" for egress firewall I0428 11:03:29.040819 3656 egressfirewall.go:723] Syncing node "ip-10-0-1-214.ec2.internal" for egress firewall I0428 11:03:29.040826 3656 egressqos.go:245] Starting EgressQoS Controller I0428 11:03:29.040842 3656 shared_informer.go:349] "Waiting for caches to sync" controller="egressqosnodes" I0428 11:03:29.040850 3656 shared_informer.go:356] "Caches are synced" controller="egressqosnodes" I0428 11:03:29.040851 3656 egressfirewall.go:723] Syncing node "ip-10-0-2-127.ec2.internal" for egress firewall I0428 11:03:29.040856 3656 shared_informer.go:349] "Waiting for caches to sync" controller="egressqospods" I0428 11:03:29.040861 3656 shared_informer.go:356] "Caches are synced" controller="egressqospods" I0428 11:03:29.040865 3656 shared_informer.go:349] "Waiting for caches to sync" controller="egressqos" I0428 11:03:29.040870 3656 shared_informer.go:356] "Caches are synced" controller="egressqos" I0428 11:03:29.040874 3656 egressqos.go:259] Repairing EgressQoSes I0428 11:03:29.040878 3656 egressqos.go:399] Starting repairing loop for egressqos I0428 11:03:29.040975 3656 egressqos.go:401] Finished repairing loop for egressqos: 96.998µs I0428 11:03:29.040996 3656 egressservice_zone.go:132] Setting up event handlers for Egress Services I0428 11:03:29.041046 3656 egressqos.go:1007] Processing sync for EgressQoS node ip-10-0-1-214.ec2.internal I0428 11:03:29.041089 3656 egressqos.go:1010] Finished syncing EgressQoS node ip-10-0-1-214.ec2.internal : 47.082µs I0428 11:03:29.041186 3656 egressservice_zone.go:210] Starting Egress Services Controller I0428 11:03:29.041198 3656 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0428 11:03:29.041207 3656 shared_informer.go:356] "Caches are synced" controller="egressservices" I0428 11:03:29.041214 3656 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0428 11:03:29.041222 3656 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0428 11:03:29.041230 3656 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0428 11:03:29.041237 3656 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0428 11:03:29.041243 3656 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_nodes" I0428 11:03:29.041248 3656 shared_informer.go:356] "Caches are synced" controller="egressservices_nodes" I0428 11:03:29.041252 3656 egressservice_zone.go:228] Repairing Egress Services I0428 11:03:29.041555 3656 master_controller.go:88] Starting Admin Policy Based Route Controller I0428 11:03:29.041565 3656 external_controller.go:278] Starting Admin Policy Based Route Controller I0428 11:03:29.041627 3656 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-142.ec2.internal I0428 11:03:29.041641 3656 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-142.ec2.internal: 15.52µs I0428 11:03:29.041651 3656 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-214.ec2.internal I0428 11:03:29.041656 3656 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-214.ec2.internal: 6.325µs I0428 11:03:29.041663 3656 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-127.ec2.internal I0428 11:03:29.041668 3656 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-127.ec2.internal: 5.582µs I0428 11:03:29.041681 3656 default_network_controller.go:585] Completing all the Watchers took 315.58903ms I0428 11:03:29.041694 3656 default_network_controller.go:589] Starting unidling controllers I0428 11:03:29.041708 3656 unidle.go:46] Registering OVN SB ControllerEvent handler I0428 11:03:29.041720 3656 unidle.go:63] Populating Initial ContollerEvent events I0428 11:03:29.041732 3656 unidle.go:79] Setting up event handlers for services I0428 11:03:29.041905 3656 ovnkube.go:545] Waiting for OVN northbound database changes to be processed by ovn-controller I0428 11:03:29.116846 3656 default_node_network_controller.go:519] Node connection status = connected I0428 11:03:29.616780 3656 default_node_network_controller.go:519] Node connection status = connected I0428 11:03:30.117048 3656 default_node_network_controller.go:519] Node connection status = connected I0428 11:03:30.617409 3656 default_node_network_controller.go:519] Node connection status = connected I0428 11:03:30.892328 3656 ovnkube.go:550] Finished waiting for OVN northbound database changes to be processed by ovn-controller I0428 11:03:31.116701 3656 default_node_network_controller.go:519] Node connection status = connected I0428 11:03:31.136199 3656 bridgeconfig.go:537] Gateway is ready I0428 11:03:31.136226 3656 gateway_shared_intf.go:1733] Creating Gateway Openflow Manager I0428 11:03:31.148869 3656 node_ip_handler_linux.go:247] Node primary address changed to 10.0.1.214. Updating OVN encap IP. I0428 11:03:31.154469 3656 node_ip_handler_linux.go:542] Will not update encap IP 10.0.1.214 - it is already configured I0428 11:03:31.154485 3656 node_ip_handler_linux.go:485] Node address changed to map[10.0.1.214/24:{} 172.20.0.1/32:{}]. Updating annotations. I0428 11:03:31.154708 3656 kube.go:131] Setting annotations map[k8s.ovn.org/host-cidrs:["10.0.1.214/24","172.20.0.1/32"] k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-1-214.ec2.internal","mac-address":"0a:ff:f1:76:ae:cf","ip-addresses":["10.0.1.214/24"],"ip-address":"10.0.1.214/24","next-hops":["10.0.1.1"],"next-hop":"10.0.1.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:ce5d33ea-36a8-4c3e-8972-2035a81837fa k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.1.214/24"}] on node ip-10-0-1-214.ec2.internal I0428 11:03:31.168779 3656 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-214.ec2.internal in network "default" I0428 11:03:31.168838 3656 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-214.ec2.internal I0428 11:03:31.168859 3656 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-214.ec2.internal: 71.709µs I0428 11:03:31.168871 3656 master.go:548] Adding or Updating local node "ip-10-0-1-214.ec2.internal" for network "default" I0428 11:03:31.168893 3656 node_tracker.go:169] Node ip-10-0-1-214.ec2.internal switch + router changed, syncing services in network "default" I0428 11:03:31.168901 3656 egressfirewall.go:723] Syncing node "ip-10-0-1-214.ec2.internal" for egress firewall I0428 11:03:31.168928 3656 services_controller.go:574] Full service sync requested for network=default I0428 11:03:31.169176 3656 gateway_shared_intf.go:2023] Setting OVN Masquerade route with source: 10.0.1.214 I0428 11:03:31.169298 3656 kube.go:131] Setting annotations map[k8s.ovn.org/node-masquerade-subnet:{"ipv4":"169.254.0.0/17","ipv6":"fd69::/112"}] on node ip-10-0-1-214.ec2.internal I0428 11:03:31.173453 3656 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-1-214.ec2.internal. Updating no re-route policies I0428 11:03:31.173537 3656 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/multus-additional-cni-plugins-lbjwg I0428 11:03:31.173554 3656 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/multus-additional-cni-plugins-lbjwg I0428 11:03:31.173552 3656 obj_retry.go:334] Retry object setup: *v1.Pod openshift-image-registry/node-ca-wfzwq I0428 11:03:31.173557 3656 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-6hd9d I0428 11:03:31.173570 3656 obj_retry.go:398] Adding new object: *v1.Pod openshift-image-registry/node-ca-wfzwq I0428 11:03:31.173574 3656 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-6hd9d I0428 11:03:31.173578 3656 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/multus-additional-cni-plugins-lbjwg after 0 failed attempt(s) I0428 11:03:31.173581 3656 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/konnectivity-agent-cxltc I0428 11:03:31.173539 3656 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-1-214.ec2.internal I0428 11:03:31.173596 3656 obj_retry.go:398] Adding new object: *v1.Pod kube-system/konnectivity-agent-cxltc I0428 11:03:31.173596 3656 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-operator/iptables-alerter-shk8c I0428 11:03:31.173607 3656 obj_retry.go:334] Retry object setup: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-bdlg2 I0428 11:03:31.173622 3656 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-6hd9d] creating logical port openshift-multus_network-metrics-daemon-6hd9d for pod on switch ip-10-0-1-214.ec2.internal I0428 11:03:31.173624 3656 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-mxd6k I0428 11:03:31.173630 3656 obj_retry.go:334] Retry object setup: *v1.Pod openshift-dns/node-resolver-5vjsr I0428 11:03:31.173633 3656 obj_retry.go:398] Adding new object: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-bdlg2 I0428 11:03:31.173639 3656 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-mxd6k I0428 11:03:31.173599 3656 obj_retry.go:398] Adding new object: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-1-214.ec2.internal I0428 11:03:31.173643 3656 obj_retry.go:398] Adding new object: *v1.Pod openshift-dns/node-resolver-5vjsr I0428 11:03:31.173647 3656 obj_retry.go:418] Retry successful for *v1.Pod kube-system/konnectivity-agent-cxltc after 0 failed attempt(s) I0428 11:03:31.173662 3656 obj_retry.go:418] Retry successful for *v1.Pod openshift-dns/node-resolver-5vjsr after 0 failed attempt(s) I0428 11:03:31.173615 3656 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-operator/iptables-alerter-shk8c I0428 11:03:31.173675 3656 obj_retry.go:418] Retry successful for *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-1-214.ec2.internal after 0 failed attempt(s) I0428 11:03:31.173676 3656 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-mxd6k] creating logical port openshift-network-diagnostics_network-check-target-mxd6k for pod on switch ip-10-0-1-214.ec2.internal I0428 11:03:31.173689 3656 obj_retry.go:418] Retry successful for *v1.Pod openshift-network-operator/iptables-alerter-shk8c after 0 failed attempt(s) I0428 11:03:31.173631 3656 obj_retry.go:334] Retry object setup: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-1-214.ec2.internal I0428 11:03:31.173765 3656 obj_retry.go:398] Adding new object: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-1-214.ec2.internal I0428 11:03:31.173664 3656 obj_retry.go:418] Retry successful for *v1.Pod openshift-ovn-kubernetes/ovnkube-node-bdlg2 after 0 failed attempt(s) I0428 11:03:31.173855 3656 obj_retry.go:418] Retry successful for *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-1-214.ec2.internal after 0 failed attempt(s) I0428 11:03:31.173599 3656 obj_retry.go:418] Retry successful for *v1.Pod openshift-image-registry/node-ca-wfzwq after 0 failed attempt(s) I0428 11:03:31.173582 3656 obj_retry.go:334] Retry object setup: *v1.Pod openshift-cluster-node-tuning-operator/tuned-bjsvn I0428 11:03:31.173877 3656 obj_retry.go:398] Adding new object: *v1.Pod openshift-cluster-node-tuning-operator/tuned-bjsvn I0428 11:03:31.173893 3656 obj_retry.go:418] Retry successful for *v1.Pod openshift-cluster-node-tuning-operator/tuned-bjsvn after 0 failed attempt(s) I0428 11:03:31.173612 3656 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/multus-hnps9 I0428 11:03:31.173908 3656 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/multus-hnps9 I0428 11:03:31.173940 3656 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/multus-hnps9 after 0 failed attempt(s) I0428 11:03:31.173616 3656 obj_retry.go:334] Retry object setup: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-ktx6b I0428 11:03:31.173953 3656 obj_retry.go:398] Adding new object: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-ktx6b I0428 11:03:31.173968 3656 obj_retry.go:418] Retry successful for *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-ktx6b after 0 failed attempt(s) I0428 11:03:31.174510 3656 pods.go:271] [openshift-multus/network-metrics-daemon-6hd9d] addLogicalPort took 912.832µs, libovsdb time 245.433µs I0428 11:03:31.174523 3656 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/network-metrics-daemon-6hd9d after 0 failed attempt(s) I0428 11:03:31.174647 3656 pods.go:271] [openshift-network-diagnostics/network-check-target-mxd6k] addLogicalPort took 979.987µs, libovsdb time 260.755µs I0428 11:03:31.174660 3656 obj_retry.go:418] Retry successful for *v1.Pod openshift-network-diagnostics/network-check-target-mxd6k after 0 failed attempt(s) I0428 11:03:31.185004 3656 gateway_shared_intf.go:1799] Creating Gateway Node Port Watcher I0428 11:03:31.196678 3656 iptables.go:108] Creating table: mangle chain: OUTPUT I0428 11:03:31.199704 3656 iptables.go:108] Creating table: nat chain: OUTPUT I0428 11:03:31.237614 3656 iptables.go:108] Creating table: nat chain: PREROUTING I0428 11:03:31.283467 3656 iptables.go:108] Creating table: filter chain: FORWARD I0428 11:03:31.299741 3656 gateway_shared_intf.go:2124] Ensuring IP Neighbor entry for: 169.254.0.1 W0428 11:03:31.300000 3656 gateway_shared_intf.go:2130] Failed to remove IP neighbor entry for ip 169.254.0.1, on iface br-ex: failed to delete neighbour entry 169.254.0.1 : no such file or directory I0428 11:03:31.300115 3656 gateway_shared_intf.go:2124] Ensuring IP Neighbor entry for: 169.254.0.4 W0428 11:03:31.300206 3656 gateway_shared_intf.go:2130] Failed to remove IP neighbor entry for ip 169.254.0.4, on iface br-ex: failed to delete neighbour entry 169.254.0.4 : no such file or directory I0428 11:03:31.300254 3656 gateway_init.go:325] Gateway and management port readiness took 2.688166642s I0428 11:03:31.300287 3656 controller.go:133] Adding controller [node-nad-controller NAD controller] event handlers I0428 11:03:31.300330 3656 shared_informer.go:349] "Waiting for caches to sync" controller="[node-nad-controller NAD controller]" I0428 11:03:31.300346 3656 shared_informer.go:356] "Caches are synced" controller="[node-nad-controller NAD controller]" I0428 11:03:31.300357 3656 nad_controller.go:541] [node-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0428 11:03:31.300424 3656 controller.go:157] Starting controller [node-nad-controller NAD controller] with 1 workers I0428 11:03:31.300518 3656 nad_controller.go:580] [node-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 65.864µs I0428 11:03:31.300821 3656 network_controller.go:373] [node-nad-controller network controller]: syncing all networks I0428 11:03:31.300846 3656 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 15.889µs I0428 11:03:31.300854 3656 network_controller.go:384] [node-nad-controller network controller]: finished syncing all networks. Time taken: 34.395µs I0428 11:03:31.300861 3656 controller.go:157] Starting controller [node-nad-controller network controller] with 1 workers I0428 11:03:31.300873 3656 nad_controller.go:379] [node-nad-controller NAD controller]: started I0428 11:03:31.300881 3656 default_node_network_controller.go:899] Starting the default node network controller I0428 11:03:31.300888 3656 gateway_init.go:335] Initializing Gateway Functionality for gateway Start I0428 11:03:31.300964 3656 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 16.898µs I0428 11:03:31.301093 3656 default_node_network_controller.go:1460] MTU (9001) of network interface br-ex is big enough to deal with Geneve header overhead (sum 8959). I0428 11:03:31.301102 3656 default_node_network_controller.go:935] Waiting for gateway and management port readiness... I0428 11:03:31.301128 3656 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc0004a97f0 0xc0002ec380 0xc0002ec3f0 0xc0002ec4d0 0xc0002ec540 0xc0002ec5b0 0xc0002ec620 0xc0006c5ef0 0xc0002ec690 0xc0002ec700 0xc000a7e190 0xc0064f4a10 0xc0002ec770 map[0x2ff9080:0xc00260aba0 0x2ff95c0:0xc0057807e0 0x2ffa2e0:0xc00278ef00 0x2ffa580:0xc00260ab40 0x2ffad60:0xc00260ac00 0x2ffb000:0xc00260ac60 0x2ffb2a0:0xc005780780 0x2ffba80:0xc005780840 0x2ffbd20:0xc0057808a0 0x3026980:0xc000f36540 0x3026d20:0xc000f365a0 0x3027800:0xc000bc4d20 0x3029160:0xc001bee000 0x30298a0:0xc000f36840 0x302a720:0xc001bee060] 0xc0000d2770 0} I0428 11:03:31.301192 3656 gateway.go:151] Starting gateway service sync I0428 11:03:31.301626 3656 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ITP I0428 11:03:31.303707 3656 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-NODEPORT I0428 11:03:31.331500 3656 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-EXTERNALIP I0428 11:03:31.332739 3656 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ETP I0428 11:03:31.333879 3656 gateway_iptables.go:474] Recreating iptables rules for table: mangle, chain: OVN-KUBE-ITP I0428 11:03:31.367881 3656 gateway.go:168] Gateway service sync done. Time taken: 66.676971ms I0428 11:03:31.368394 3656 healthcheck.go:145] Opening healthcheck "openshift-ingress/router-default" on port 30408 I0428 11:03:31.368667 3656 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc0004a97f0 0xc0002ec380 0xc0002ec3f0 0xc0002ec4d0 0xc0002ec540 0xc0002ec5b0 0xc0002ec620 0xc0006c5ef0 0xc0002ec690 0xc0002ec700 0xc000a7e190 0xc0064f4a10 0xc0002ec770 map[0x2ff9080:0xc00260aba0 0x2ff95c0:0xc0057807e0 0x2ffa2e0:0xc00278ef00 0x2ffa580:0xc00260ab40 0x2ffad60:0xc00260ac00 0x2ffb000:0xc00260ac60 0x2ffb2a0:0xc005780780 0x2ffba80:0xc005780840 0x2ffbd20:0xc0057808a0 0x3026980:0xc000f36540 0x3026d20:0xc000f365a0 0x3027800:0xc000bc4d20 0x3029160:0xc001bee000 0x30298a0:0xc000f36840 0x302a720:0xc001bee060] 0xc0000d2770 0} I0428 11:03:31.369103 3656 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc0004a97f0 0xc0002ec380 0xc0002ec3f0 0xc0002ec4d0 0xc0002ec540 0xc0002ec5b0 0xc0002ec620 0xc0006c5ef0 0xc0002ec690 0xc0002ec700 0xc000a7e190 0xc0064f4a10 0xc0002ec770 map[0x2ff9080:0xc00260aba0 0x2ff95c0:0xc0057807e0 0x2ffa2e0:0xc00278ef00 0x2ffa580:0xc00260ab40 0x2ffad60:0xc00260ac00 0x2ffb000:0xc00260ac60 0x2ffb2a0:0xc005780780 0x2ffba80:0xc005780840 0x2ffbd20:0xc0057808a0 0x3026980:0xc000f36540 0x3026d20:0xc000f365a0 0x3027800:0xc000bc4d20 0x3029160:0xc001bee000 0x30298a0:0xc000f36840 0x302a720:0xc001bee060] 0xc0000d2770 0} I0428 11:03:31.369137 3656 gateway.go:532] Reconciling gateway with updates I0428 11:03:31.374305 3656 gateway.go:339] Spawning Conntrack Rule Check Thread I0428 11:03:31.374338 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0428 11:03:31.374357 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0428 11:03:31.374368 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0428 11:03:31.374380 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0428 11:03:31.374385 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0428 11:03:31.374363 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0428 11:03:31.374444 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0428 11:03:31.374449 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0428 11:03:31.374460 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0428 11:03:31.374459 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0428 11:03:31.374469 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0428 11:03:31.374470 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0428 11:03:31.374474 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0428 11:03:31.374339 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift I0428 11:03:31.374487 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0428 11:03:31.374479 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0428 11:03:31.374498 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift I0428 11:03:31.374506 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0428 11:03:31.374507 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0428 11:03:31.374512 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0428 11:03:31.374517 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0428 11:03:31.374496 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0428 11:03:31.374523 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0428 11:03:31.374521 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0428 11:03:31.374357 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0428 11:03:31.374539 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0428 11:03:31.374543 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0428 11:03:31.374502 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0428 11:03:31.374549 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0428 11:03:31.374552 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0428 11:03:31.374549 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0428 11:03:31.374556 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0428 11:03:31.374450 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0428 11:03:31.374560 3656 default_node_network_controller.go:944] Gateway and management port readiness took 73.448093ms I0428 11:03:31.374542 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0428 11:03:31.374570 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0428 11:03:31.374508 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0428 11:03:31.374527 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0428 11:03:31.374522 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0428 11:03:31.374581 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0428 11:03:31.374591 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0428 11:03:31.374592 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0428 11:03:31.374594 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0428 11:03:31.374601 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0428 11:03:31.374545 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0428 11:03:31.374607 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/kubernetes I0428 11:03:31.374615 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0428 11:03:31.374623 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0428 11:03:31.374625 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/packageserver I0428 11:03:31.374630 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/kubernetes I0428 11:03:31.374635 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0428 11:03:31.374639 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/packageserver I0428 11:03:31.374645 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0428 11:03:31.374645 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0428 11:03:31.374653 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0428 11:03:31.374659 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0428 11:03:31.374665 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0428 11:03:31.374669 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0428 11:03:31.374447 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0428 11:03:31.374629 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0428 11:03:31.374713 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0428 11:03:31.374722 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0428 11:03:31.374729 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0428 11:03:31.374731 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0428 11:03:31.374737 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0428 11:03:31.374571 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0428 11:03:31.374564 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0428 11:03:31.374553 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0428 11:03:31.374602 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0428 11:03:31.374542 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0428 11:03:31.374544 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0428 11:03:31.374848 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0428 11:03:31.374863 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0428 11:03:31.374878 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0428 11:03:31.374880 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0428 11:03:31.374885 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0428 11:03:31.374889 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0428 11:03:31.374892 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0428 11:03:31.374896 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0428 11:03:31.374905 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0428 11:03:31.374906 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0428 11:03:31.374554 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0428 11:03:31.374935 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0428 11:03:31.374943 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0428 11:03:31.374896 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0428 11:03:31.374887 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0428 11:03:31.375018 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0428 11:03:31.374865 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0428 11:03:31.375098 3656 port_claim.go:71] Svc openshift-ingress/router-default: port 32323 is already open, no action needed I0428 11:03:31.375110 3656 port_claim.go:71] Svc openshift-ingress/router-default: port 30139 is already open, no action needed I0428 11:03:31.375119 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0428 11:03:31.375138 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0428 11:03:31.374560 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0428 11:03:31.374584 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0428 11:03:31.374608 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0428 11:03:31.374610 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0428 11:03:31.374613 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0428 11:03:31.374613 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0428 11:03:31.375340 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0428 11:03:31.374624 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0428 11:03:31.375372 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0428 11:03:31.374850 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0428 11:03:31.374554 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0428 11:03:31.375439 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0428 11:03:31.374897 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0428 11:03:31.375492 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0428 11:03:31.374913 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0428 11:03:31.376298 3656 node_ip_handler_linux.go:160] Node IP manager is running W0428 11:03:31.413771 3656 port_linux.go:245] Missing or unable to find route entry for subnet 10.128.0.0/14 via gateway 10.130.0.1 on link ovn-k8s-mp0 with MTU: 0 W0428 11:03:31.413904 3656 port_linux.go:245] Missing or unable to find route entry for subnet 169.254.0.3/32 via gateway 10.130.0.1 on link ovn-k8s-mp0 with MTU: 0 W0428 11:03:31.414036 3656 port_linux.go:261] Missing arp entry for MAC/IP binding (0a:58:0a:82:00:01/10.130.0.1) on link ovn-k8s-mp0 I0428 11:03:31.421430 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0428 11:03:31.421451 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0428 11:03:31.421457 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0428 11:03:31.421464 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0428 11:03:31.421476 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0428 11:03:31.421483 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0428 11:03:31.421488 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0428 11:03:31.421493 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0428 11:03:31.421499 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0428 11:03:31.421504 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0428 11:03:31.421510 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0428 11:03:31.421516 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0428 11:03:31.461043 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0428 11:03:31.461072 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0428 11:03:31.461084 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0428 11:03:31.461093 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0428 11:03:31.461103 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0428 11:03:31.461113 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0428 11:03:31.500727 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0428 11:03:31.500742 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0428 11:03:31.500731 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0428 11:03:31.500757 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0428 11:03:31.500763 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0428 11:03:31.500770 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0428 11:03:31.500775 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0428 11:03:31.500781 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0428 11:03:31.500787 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0428 11:03:31.500800 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0428 11:03:32.257492 3656 default_network_controller.go:1025] Node "ip-10-0-2-127.ec2.internal" in remote zone "ip-10-0-2-127.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0428 11:03:32.257506 3656 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-127.ec2.internal I0428 11:03:32.257532 3656 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-127.ec2.internal: 31.175µs I0428 11:03:32.258296 3656 zone_ic_handler.go:195] Time taken to create transit switch: 271.227µs I0428 11:03:32.258311 3656 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-2-127.ec2.internal for the network default I0428 11:03:32.259268 3656 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-2-127.ec2.internal" on network "default" took: 1.257486ms I0428 11:03:32.728339 3656 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-142.ec2.internal I0428 11:03:32.728378 3656 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-142.ec2.internal: 46.222µs I0428 11:03:32.891318 3656 default_node_network_controller.go:1380] Starting node controller node sync W0428 11:03:32.891387 3656 egressip_healthcheck.go:76] Health checking using insecure connection I0428 11:03:32.891454 3656 egressip_healthcheck.go:109] Starting Egress IP Health Server on 10.130.0.2:9107 I0428 11:03:32.903238 3656 default_node_network_controller.go:1422] Node controller node sync done. Time taken: 11.901083ms I0428 11:03:32.903338 3656 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-142.ec2.internal", IP: 10.0.0.142 to PMTUD blocking rules I0428 11:03:32.903355 3656 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-142.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0428 11:03:32.903338 3656 default_node_network_controller.go:1293] Adding remote node "ip-10-0-2-127.ec2.internal", IP: 10.0.2.127 to PMTUD blocking rules I0428 11:03:32.903369 3656 default_node_network_controller.go:1293] Adding remote node "ip-10-0-2-127.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0428 11:03:32.915000 3656 healthcheck_node.go:124] "Starting node proxy healthz server" address="0.0.0.0:10256" I0428 11:03:32.915262 3656 egressservice_node.go:93] Setting up event handlers for Egress Services I0428 11:03:32.915367 3656 egressservice_node.go:183] Starting Egress Services Controller I0428 11:03:32.915389 3656 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0428 11:03:32.915405 3656 shared_informer.go:356] "Caches are synced" controller="egressservices" I0428 11:03:32.915414 3656 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0428 11:03:32.915422 3656 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0428 11:03:32.915430 3656 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0428 11:03:32.915437 3656 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0428 11:03:32.915444 3656 egressservice_node.go:197] Repairing Egress Services I0428 11:03:32.925588 3656 node_controller.go:43] Starting Admin Policy Based Route Node Controller I0428 11:03:32.925602 3656 external_controller.go:278] Starting Admin Policy Based Route Controller I0428 11:03:32.925734 3656 default_node_network_controller.go:1077] Egress IP for secondary host network is disabled I0428 11:03:32.925749 3656 link_network_manager.go:119] Link manager is running I0428 11:03:32.925761 3656 default_node_network_controller.go:1098] Default node network controller initialized and ready. I0428 11:03:32.925947 3656 ovspinning_linux.go:50] OVS CPU affinity pinning disabled I0428 11:03:32.926150 3656 vrf_manager.go:128] VRF manager is running I0428 11:03:32.926214 3656 node_controller_manager.go:437] Removing flows to drop GARP I0428 11:03:32.926228 3656 gateway.go:532] Reconciling gateway with updates I0428 11:03:32.985819 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/packageserver I0428 11:03:32.985833 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/kubernetes I0428 11:03:32.985841 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/packageserver I0428 11:03:32.985845 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0428 11:03:32.985846 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0428 11:03:32.985836 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0428 11:03:32.985853 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0428 11:03:32.985858 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0428 11:03:32.985867 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0428 11:03:32.985870 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0428 11:03:32.985871 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0428 11:03:32.985872 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0428 11:03:32.985877 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0428 11:03:32.985879 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0428 11:03:32.985877 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0428 11:03:32.985892 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0428 11:03:32.985893 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0428 11:03:32.985868 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0428 11:03:32.985893 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0428 11:03:32.985902 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0428 11:03:32.985902 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0428 11:03:32.985906 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0428 11:03:32.985907 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0428 11:03:32.985936 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0428 11:03:32.985895 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0428 11:03:32.985942 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0428 11:03:32.985945 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0428 11:03:32.985949 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0428 11:03:32.985951 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0428 11:03:32.985951 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0428 11:03:32.985959 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0428 11:03:32.985962 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0428 11:03:32.985965 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0428 11:03:32.985969 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0428 11:03:32.985966 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0428 11:03:32.985973 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0428 11:03:32.985975 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0428 11:03:32.985976 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0428 11:03:32.985980 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0428 11:03:32.985983 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0428 11:03:32.985982 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0428 11:03:32.985985 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0428 11:03:32.985990 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0428 11:03:32.985998 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0428 11:03:32.985998 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0428 11:03:32.986001 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0428 11:03:32.986008 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0428 11:03:32.985939 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0428 11:03:32.986024 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0428 11:03:32.985858 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0428 11:03:32.985961 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0428 11:03:32.985982 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0428 11:03:32.986057 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0428 11:03:32.985879 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0428 11:03:32.985945 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0428 11:03:32.985912 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0428 11:03:32.986082 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0428 11:03:32.985967 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0428 11:03:32.986101 3656 port_claim.go:71] Svc openshift-ingress/router-default: port 32323 is already open, no action needed I0428 11:03:32.986114 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0428 11:03:32.986127 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0428 11:03:32.985874 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0428 11:03:32.985905 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0428 11:03:32.985849 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/kubernetes I0428 11:03:32.986117 3656 port_claim.go:71] Svc openshift-ingress/router-default: port 30139 is already open, no action needed I0428 11:03:32.985853 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0428 11:03:32.985825 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift I0428 11:03:32.986233 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift I0428 11:03:32.986254 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0428 11:03:32.985859 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0428 11:03:32.985859 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0428 11:03:32.986284 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0428 11:03:32.985838 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0428 11:03:32.986306 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0428 11:03:32.985882 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0428 11:03:32.985883 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0428 11:03:32.985886 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0428 11:03:32.985893 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0428 11:03:32.986408 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0428 11:03:32.985899 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0428 11:03:32.985934 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0428 11:03:32.985948 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0428 11:03:32.985952 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0428 11:03:32.985947 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0428 11:03:32.986545 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0428 11:03:32.985954 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0428 11:03:32.986576 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0428 11:03:32.985955 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0428 11:03:32.986582 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0428 11:03:32.985826 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0428 11:03:32.986589 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0428 11:03:32.985967 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0428 11:03:32.986614 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0428 11:03:32.985969 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0428 11:03:32.985969 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0428 11:03:32.986659 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0428 11:03:32.985967 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0428 11:03:32.986682 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0428 11:03:32.985977 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0428 11:03:32.986706 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0428 11:03:32.985987 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0428 11:03:32.986049 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0428 11:03:32.986745 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0428 11:03:33.025703 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0428 11:03:33.025724 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0428 11:03:33.025730 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0428 11:03:33.025751 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0428 11:03:33.025758 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0428 11:03:33.025764 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0428 11:03:33.025771 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0428 11:03:33.025780 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0428 11:03:33.025785 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0428 11:03:33.025790 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0428 11:03:33.025796 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0428 11:03:33.025802 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0428 11:03:33.064678 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0428 11:03:33.084260 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0428 11:03:33.084291 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0428 11:03:33.084306 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0428 11:03:33.084312 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0428 11:03:33.084317 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0428 11:03:33.084322 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0428 11:03:33.084328 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0428 11:03:33.084334 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0428 11:03:33.084339 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0428 11:03:33.084345 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0428 11:03:33.084351 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0428 11:03:33.084359 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0428 11:03:33.084367 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0428 11:03:33.084374 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0428 11:03:33.084380 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0428 11:03:33.084385 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0428 11:03:33.084391 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0428 11:03:33.084396 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0428 11:03:34.616353 3656 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-214.ec2.internal I0428 11:03:34.616390 3656 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-214.ec2.internal: 44.807µs I0428 11:03:34.643843 3656 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-6f4c55d5d-d8w4p] creating logical port openshift-image-registry_image-registry-6f4c55d5d-d8w4p for pod on switch ip-10-0-1-214.ec2.internal I0428 11:03:34.643966 3656 kube.go:256] Updating pod openshift-image-registry/image-registry-6f4c55d5d-d8w4p I0428 11:03:34.672084 3656 base_network_controller_pods.go:487] [default/openshift-ingress-canary/ingress-canary-ldtzw] creating logical port openshift-ingress-canary_ingress-canary-ldtzw for pod on switch ip-10-0-1-214.ec2.internal I0428 11:03:34.672214 3656 kube.go:256] Updating pod openshift-ingress-canary/ingress-canary-ldtzw I0428 11:03:34.672228 3656 base_network_controller_pods.go:487] [default/openshift-dns/dns-default-2gfr5] creating logical port openshift-dns_dns-default-2gfr5 for pod on switch ip-10-0-1-214.ec2.internal I0428 11:03:34.672255 3656 pod.go:62] [openshift-image-registry/image-registry-6f4c55d5d-d8w4p] pod update took 28.343285ms I0428 11:03:34.672273 3656 base_network_controller_pods.go:951] [default/openshift-image-registry/image-registry-6f4c55d5d-d8w4p] addLogicalPort annotation time took 28.367101ms I0428 11:03:34.672306 3656 kube.go:256] Updating pod openshift-dns/dns-default-2gfr5 I0428 11:03:34.673644 3656 pods.go:271] [openshift-image-registry/image-registry-6f4c55d5d-d8w4p] addLogicalPort took 30.241716ms, libovsdb time 1.033285ms I0428 11:03:34.693415 3656 pod.go:62] [openshift-ingress-canary/ingress-canary-ldtzw] pod update took 21.2405ms I0428 11:03:34.693447 3656 base_network_controller_pods.go:951] [default/openshift-ingress-canary/ingress-canary-ldtzw] addLogicalPort annotation time took 21.275119ms I0428 11:03:34.694618 3656 pods.go:271] [openshift-ingress-canary/ingress-canary-ldtzw] addLogicalPort took 22.556353ms, libovsdb time 887.568µs I0428 11:03:34.702116 3656 pod.go:62] [openshift-dns/dns-default-2gfr5] pod update took 29.832594ms I0428 11:03:34.702144 3656 base_network_controller_pods.go:951] [default/openshift-dns/dns-default-2gfr5] addLogicalPort annotation time took 29.862762ms I0428 11:03:34.703181 3656 pods.go:271] [openshift-dns/dns-default-2gfr5] addLogicalPort took 30.996668ms, libovsdb time 754.948µs I0428 11:03:35.082347 3656 cni.go:452] [openshift-ingress-canary/ingress-canary-ldtzw 214484ab0a33dfbe08e3d565365cd56763d2087b80e924e24b95ca568946d841 network default NAD default NAD key ] ADD starting CNI request [openshift-ingress-canary/ingress-canary-ldtzw 214484ab0a33dfbe08e3d565365cd56763d2087b80e924e24b95ca568946d841 network default NAD default NAD key ] I0428 11:03:35.127824 3656 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress-canary, podName: ingress-canary-ldtzw, hostIfaceName: 214484ab0a33dfb, network: default, NAD default, SandboxID: "214484ab0a33dfbe08e3d565365cd56763d2087b80e924e24b95ca568946d841", PCI device ID: , UID: "bad28cfd-9c2b-46fe-b95a-405f6da6dd1f", MAC: 0a:58:0a:82:00:06, IPs: [10.130.0.6/23] I0428 11:03:35.194649 3656 cni.go:473] [openshift-ingress-canary/ingress-canary-ldtzw 214484ab0a33dfbe08e3d565365cd56763d2087b80e924e24b95ca568946d841 network default NAD default NAD key default] ADD finished CNI request [openshift-ingress-canary/ingress-canary-ldtzw 214484ab0a33dfbe08e3d565365cd56763d2087b80e924e24b95ca568946d841 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"fa:65:e8:1e:de:3e\",\"name\":\"214484ab0a33dfb\"},{\"mac\":\"0a:58:0a:82:00:06\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1b9a7985-1709-40c6-83a8-0094d6605ad6\"}],\"ips\":[{\"address\":\"10.130.0.6/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0428 11:03:38.832233 3656 cni.go:452] [openshift-network-diagnostics/network-check-target-mxd6k 11961a20d97e3a61b3ed0aeeb745e74dcc59519e91656d9b47e7ea7e72c9d58e network default NAD default NAD key ] ADD starting CNI request [openshift-network-diagnostics/network-check-target-mxd6k 11961a20d97e3a61b3ed0aeeb745e74dcc59519e91656d9b47e7ea7e72c9d58e network default NAD default NAD key ] I0428 11:03:38.843211 3656 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-target-mxd6k, hostIfaceName: 11961a20d97e3a6, network: default, NAD default, SandboxID: "11961a20d97e3a61b3ed0aeeb745e74dcc59519e91656d9b47e7ea7e72c9d58e", PCI device ID: , UID: "aba627cc-9f1e-4e2a-90e5-fbf9dc16a097", MAC: 0a:58:0a:82:00:04, IPs: [10.130.0.4/23] I0428 11:03:38.898594 3656 cni.go:473] [openshift-network-diagnostics/network-check-target-mxd6k 11961a20d97e3a61b3ed0aeeb745e74dcc59519e91656d9b47e7ea7e72c9d58e network default NAD default NAD key default] ADD finished CNI request [openshift-network-diagnostics/network-check-target-mxd6k 11961a20d97e3a61b3ed0aeeb745e74dcc59519e91656d9b47e7ea7e72c9d58e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"9a:b6:80:d0:86:c0\",\"name\":\"11961a20d97e3a6\"},{\"mac\":\"0a:58:0a:82:00:04\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/bb30abc8-1acf-48a3-9be9-46fcbf573659\"}],\"ips\":[{\"address\":\"10.130.0.4/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0428 11:03:39.193777 3656 namespace.go:98] [openshift-kube-storage-version-migrator] adding namespace I0428 11:03:39.194798 3656 namespace.go:102] [openshift-kube-storage-version-migrator] adding namespace took 996.83µs I0428 11:03:39.222550 3656 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace W0428 11:03:42.173065 3656 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics W0428 11:03:42.296541 3656 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0428 11:03:42.992763 3656 namespace.go:98] [openshift-service-ca] adding namespace I0428 11:03:42.993649 3656 namespace.go:102] [openshift-service-ca] adding namespace took 862.412µs I0428 11:03:43.018081 3656 namespace.go:142] [openshift-service-ca] updating namespace W0428 11:03:43.037500 3656 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0428 11:03:44.663178 3656 base_network_controller_pods.go:487] [default/openshift-service-ca/service-ca-d9cb48497-xwvvg] creating logical port openshift-service-ca_service-ca-d9cb48497-xwvvg for pod on switch ip-10-0-1-214.ec2.internal I0428 11:03:44.663300 3656 kube.go:256] Updating pod openshift-service-ca/service-ca-d9cb48497-xwvvg I0428 11:03:44.678352 3656 pod.go:62] [openshift-service-ca/service-ca-d9cb48497-xwvvg] pod update took 15.073944ms I0428 11:03:44.678382 3656 base_network_controller_pods.go:951] [default/openshift-service-ca/service-ca-d9cb48497-xwvvg] addLogicalPort annotation time took 15.109495ms I0428 11:03:44.679515 3656 pods.go:271] [openshift-service-ca/service-ca-d9cb48497-xwvvg] addLogicalPort took 16.358221ms, libovsdb time 858.923µs I0428 11:03:45.086856 3656 cni.go:452] [openshift-service-ca/service-ca-d9cb48497-xwvvg 6a686e1c01c71214504b809484a1cff4acbd02c4357785acd4f583f9f82d6f32 network default NAD default NAD key ] ADD starting CNI request [openshift-service-ca/service-ca-d9cb48497-xwvvg 6a686e1c01c71214504b809484a1cff4acbd02c4357785acd4f583f9f82d6f32 network default NAD default NAD key ] I0428 11:03:45.096023 3656 helper_linux.go:556] ConfigureOVS: namespace: openshift-service-ca, podName: service-ca-d9cb48497-xwvvg, hostIfaceName: 6a686e1c01c7121, network: default, NAD default, SandboxID: "6a686e1c01c71214504b809484a1cff4acbd02c4357785acd4f583f9f82d6f32", PCI device ID: , UID: "9ece899f-4d87-444b-83c5-670a7569ab9f", MAC: 0a:58:0a:82:00:08, IPs: [10.130.0.8/23] I0428 11:03:45.142405 3656 cni.go:473] [openshift-service-ca/service-ca-d9cb48497-xwvvg 6a686e1c01c71214504b809484a1cff4acbd02c4357785acd4f583f9f82d6f32 network default NAD default NAD key default] ADD finished CNI request [openshift-service-ca/service-ca-d9cb48497-xwvvg 6a686e1c01c71214504b809484a1cff4acbd02c4357785acd4f583f9f82d6f32 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"42:fb:04:f2:78:0d\",\"name\":\"6a686e1c01c7121\"},{\"mac\":\"0a:58:0a:82:00:08\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5b8c088f-bf5f-4f0e-bb1f-958076717b98\"}],\"ips\":[{\"address\":\"10.130.0.8/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0428 11:03:52.575274 3656 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics W0428 11:04:06.551528 3656 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0428 11:04:06.569020 3656 base_network_controller_pods.go:487] [default/openshift-insights/insights-runtime-extractor-fnhw5] creating logical port openshift-insights_insights-runtime-extractor-fnhw5 for pod on switch ip-10-0-1-214.ec2.internal I0428 11:04:06.569132 3656 kube.go:256] Updating pod openshift-insights/insights-runtime-extractor-fnhw5 I0428 11:04:06.603034 3656 pod.go:62] [openshift-insights/insights-runtime-extractor-fnhw5] pod update took 33.936549ms I0428 11:04:06.603065 3656 base_network_controller_pods.go:951] [default/openshift-insights/insights-runtime-extractor-fnhw5] addLogicalPort annotation time took 33.974561ms W0428 11:04:06.603128 3656 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0428 11:04:06.604188 3656 pods.go:271] [openshift-insights/insights-runtime-extractor-fnhw5] addLogicalPort took 35.18539ms, libovsdb time 790.252µs I0428 11:04:06.689867 3656 base_network_controller_pods.go:487] [default/openshift-console/downloads-5c54f4944f-grblx] creating logical port openshift-console_downloads-5c54f4944f-grblx for pod on switch ip-10-0-1-214.ec2.internal I0428 11:04:06.689979 3656 kube.go:256] Updating pod openshift-console/downloads-5c54f4944f-grblx I0428 11:04:06.722115 3656 cni.go:452] [openshift-image-registry/image-registry-6f4c55d5d-d8w4p f69cb794a675452ec688e61e9237784ac6fb6a5fe88011ee071caca2257faac0 network default NAD default NAD key ] ADD starting CNI request [openshift-image-registry/image-registry-6f4c55d5d-d8w4p f69cb794a675452ec688e61e9237784ac6fb6a5fe88011ee071caca2257faac0 network default NAD default NAD key ] I0428 11:04:06.728979 3656 pod.go:62] [openshift-console/downloads-5c54f4944f-grblx] pod update took 39.023367ms I0428 11:04:06.729001 3656 base_network_controller_pods.go:951] [default/openshift-console/downloads-5c54f4944f-grblx] addLogicalPort annotation time took 39.050549ms W0428 11:04:06.729468 3656 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0428 11:04:06.730111 3656 pods.go:271] [openshift-console/downloads-5c54f4944f-grblx] addLogicalPort took 40.259442ms, libovsdb time 836.66µs I0428 11:04:06.731524 3656 helper_linux.go:556] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-6f4c55d5d-d8w4p, hostIfaceName: f69cb794a675452, network: default, NAD default, SandboxID: "f69cb794a675452ec688e61e9237784ac6fb6a5fe88011ee071caca2257faac0", PCI device ID: , UID: "463e4956-3355-487c-a599-13ed348a10fa", MAC: 0a:58:0a:82:00:05, IPs: [10.130.0.5/23] I0428 11:04:06.779357 3656 cni.go:473] [openshift-image-registry/image-registry-6f4c55d5d-d8w4p f69cb794a675452ec688e61e9237784ac6fb6a5fe88011ee071caca2257faac0 network default NAD default NAD key default] ADD finished CNI request [openshift-image-registry/image-registry-6f4c55d5d-d8w4p f69cb794a675452ec688e61e9237784ac6fb6a5fe88011ee071caca2257faac0 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"96:55:42:0c:0e:0a\",\"name\":\"f69cb794a675452\"},{\"mac\":\"0a:58:0a:82:00:05\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/836cc1df-4bdb-4e1e-930a-344fee8e62fe\"}],\"ips\":[{\"address\":\"10.130.0.5/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0428 11:04:06.800712 3656 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0428 11:04:06.811281 3656 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0428 11:04:06.865150 3656 cni.go:452] [openshift-dns/dns-default-2gfr5 ee4ba45c17b8564c40a4391df9fd5a306abc53508fc53963cc42aff66f697686 network default NAD default NAD key ] ADD starting CNI request [openshift-dns/dns-default-2gfr5 ee4ba45c17b8564c40a4391df9fd5a306abc53508fc53963cc42aff66f697686 network default NAD default NAD key ] I0428 11:04:06.875473 3656 helper_linux.go:556] ConfigureOVS: namespace: openshift-dns, podName: dns-default-2gfr5, hostIfaceName: ee4ba45c17b8564, network: default, NAD default, SandboxID: "ee4ba45c17b8564c40a4391df9fd5a306abc53508fc53963cc42aff66f697686", PCI device ID: , UID: "8ac44998-14fe-472a-bd7d-673c77cf4ea3", MAC: 0a:58:0a:82:00:07, IPs: [10.130.0.7/23] I0428 11:04:06.924168 3656 cni.go:473] [openshift-dns/dns-default-2gfr5 ee4ba45c17b8564c40a4391df9fd5a306abc53508fc53963cc42aff66f697686 network default NAD default NAD key default] ADD finished CNI request [openshift-dns/dns-default-2gfr5 ee4ba45c17b8564c40a4391df9fd5a306abc53508fc53963cc42aff66f697686 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"16:00:50:02:66:5a\",\"name\":\"ee4ba45c17b8564\"},{\"mac\":\"0a:58:0a:82:00:07\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/79975849-e5e5-496a-a86f-b7b86d0b6223\"}],\"ips\":[{\"address\":\"10.130.0.7/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0428 11:04:06.928521 3656 cni.go:452] [openshift-insights/insights-runtime-extractor-fnhw5 5af5aa02c7eeb49cb9263eed8b077e85234b213dceabe1f008e4df1a1b525ddd network default NAD default NAD key ] ADD starting CNI request [openshift-insights/insights-runtime-extractor-fnhw5 5af5aa02c7eeb49cb9263eed8b077e85234b213dceabe1f008e4df1a1b525ddd network default NAD default NAD key ] I0428 11:04:06.939283 3656 helper_linux.go:556] ConfigureOVS: namespace: openshift-insights, podName: insights-runtime-extractor-fnhw5, hostIfaceName: 5af5aa02c7eeb49, network: default, NAD default, SandboxID: "5af5aa02c7eeb49cb9263eed8b077e85234b213dceabe1f008e4df1a1b525ddd", PCI device ID: , UID: "c45f1f91-e1c1-4cd3-bcce-fbab30347796", MAC: 0a:58:0a:82:00:09, IPs: [10.130.0.9/23] I0428 11:04:06.986381 3656 cni.go:473] [openshift-insights/insights-runtime-extractor-fnhw5 5af5aa02c7eeb49cb9263eed8b077e85234b213dceabe1f008e4df1a1b525ddd network default NAD default NAD key default] ADD finished CNI request [openshift-insights/insights-runtime-extractor-fnhw5 5af5aa02c7eeb49cb9263eed8b077e85234b213dceabe1f008e4df1a1b525ddd network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"da:c6:1a:d7:8d:74\",\"name\":\"5af5aa02c7eeb49\"},{\"mac\":\"0a:58:0a:82:00:09\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/21806c62-e087-4db9-810f-d24b84212b6a\"}],\"ips\":[{\"address\":\"10.130.0.9/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0428 11:04:07.046196 3656 cni.go:452] [openshift-console/downloads-5c54f4944f-grblx d1709e40b75b6716cbc2b413b470880a5790535783b5988c82bf8f5f7c02b321 network default NAD default NAD key ] ADD starting CNI request [openshift-console/downloads-5c54f4944f-grblx d1709e40b75b6716cbc2b413b470880a5790535783b5988c82bf8f5f7c02b321 network default NAD default NAD key ] I0428 11:04:07.057385 3656 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: downloads-5c54f4944f-grblx, hostIfaceName: d1709e40b75b671, network: default, NAD default, SandboxID: "d1709e40b75b6716cbc2b413b470880a5790535783b5988c82bf8f5f7c02b321", PCI device ID: , UID: "cf57e7ca-5be9-4585-a59c-1c0b314e6207", MAC: 0a:58:0a:82:00:0a, IPs: [10.130.0.10/23] I0428 11:04:07.107682 3656 cni.go:473] [openshift-console/downloads-5c54f4944f-grblx d1709e40b75b6716cbc2b413b470880a5790535783b5988c82bf8f5f7c02b321 network default NAD default NAD key default] ADD finished CNI request [openshift-console/downloads-5c54f4944f-grblx d1709e40b75b6716cbc2b413b470880a5790535783b5988c82bf8f5f7c02b321 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"5e:57:9e:4e:d3:e6\",\"name\":\"d1709e40b75b671\"},{\"mac\":\"0a:58:0a:82:00:0a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/363a5780-20fc-4dd0-bd6f-4db8191cf09e\"}],\"ips\":[{\"address\":\"10.130.0.10/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0428 11:04:07.196255 3656 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0428 11:04:07.659203 3656 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0428 11:04:07.659221 3656 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0428 11:04:07.659226 3656 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0428 11:04:08.336523 3656 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0428 11:04:08.336542 3656 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0428 11:04:08.336547 3656 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default I0428 11:04:08.864907 3656 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-mfwfw] creating logical port openshift-monitoring_prometheus-operator-admission-webhook-59b79769fb-mfwfw for pod on switch ip-10-0-1-214.ec2.internal I0428 11:04:08.865008 3656 kube.go:256] Updating pod openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-mfwfw I0428 11:04:08.881506 3656 pod.go:62] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-mfwfw] pod update took 16.517849ms I0428 11:04:08.881532 3656 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-mfwfw] addLogicalPort annotation time took 16.550712ms I0428 11:04:08.882697 3656 pods.go:271] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-mfwfw] addLogicalPort took 17.802747ms, libovsdb time 879.586µs W0428 11:04:09.103043 3656 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0428 11:04:09.103067 3656 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0428 11:04:09.103075 3656 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default I0428 11:04:09.228636 3656 cni.go:452] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-mfwfw ddb1fb24eb18eeee8c71768e3f7e3c0f38006ae5a918295364dfeb62a7a13ba8 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-mfwfw ddb1fb24eb18eeee8c71768e3f7e3c0f38006ae5a918295364dfeb62a7a13ba8 network default NAD default NAD key ] I0428 11:04:09.239193 3656 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-operator-admission-webhook-59b79769fb-mfwfw, hostIfaceName: ddb1fb24eb18eee, network: default, NAD default, SandboxID: "ddb1fb24eb18eeee8c71768e3f7e3c0f38006ae5a918295364dfeb62a7a13ba8", PCI device ID: , UID: "58ce9311-1c22-4003-9efc-17db65fb0540", MAC: 0a:58:0a:82:00:0b, IPs: [10.130.0.11/23] I0428 11:04:09.286627 3656 cni.go:473] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-mfwfw ddb1fb24eb18eeee8c71768e3f7e3c0f38006ae5a918295364dfeb62a7a13ba8 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-mfwfw ddb1fb24eb18eeee8c71768e3f7e3c0f38006ae5a918295364dfeb62a7a13ba8 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"46:06:50:a5:cc:8c\",\"name\":\"ddb1fb24eb18eee\"},{\"mac\":\"0a:58:0a:82:00:0b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d77a887a-29f7-4b09-a685-b13c76f42bb7\"}],\"ips\":[{\"address\":\"10.130.0.11/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0428 11:04:10.636256 3656 cni.go:452] [openshift-multus/network-metrics-daemon-6hd9d 09e9f664fd7a5291063aebc5c740d2b34705d40eb7ea9b86d93886a7cc960e61 network default NAD default NAD key ] ADD starting CNI request [openshift-multus/network-metrics-daemon-6hd9d 09e9f664fd7a5291063aebc5c740d2b34705d40eb7ea9b86d93886a7cc960e61 network default NAD default NAD key ] I0428 11:04:10.645870 3656 helper_linux.go:556] ConfigureOVS: namespace: openshift-multus, podName: network-metrics-daemon-6hd9d, hostIfaceName: 09e9f664fd7a529, network: default, NAD default, SandboxID: "09e9f664fd7a5291063aebc5c740d2b34705d40eb7ea9b86d93886a7cc960e61", PCI device ID: , UID: "87bbe9e7-fe43-4e4b-9702-a2dd853bc7ca", MAC: 0a:58:0a:82:00:03, IPs: [10.130.0.3/23] I0428 11:04:10.703969 3656 cni.go:473] [openshift-multus/network-metrics-daemon-6hd9d 09e9f664fd7a5291063aebc5c740d2b34705d40eb7ea9b86d93886a7cc960e61 network default NAD default NAD key default] ADD finished CNI request [openshift-multus/network-metrics-daemon-6hd9d 09e9f664fd7a5291063aebc5c740d2b34705d40eb7ea9b86d93886a7cc960e61 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"36:f2:e4:62:6e:48\",\"name\":\"09e9f664fd7a529\"},{\"mac\":\"0a:58:0a:82:00:03\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c6bd4444-11d8-4b33-8f91-c40dd3a201bc\"}],\"ips\":[{\"address\":\"10.130.0.3/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0428 11:04:11.140002 3656 base_network_controller_pods.go:487] [default/openshift-console/console-5bcfcd7b98-mcfnm] creating logical port openshift-console_console-5bcfcd7b98-mcfnm for pod on switch ip-10-0-1-214.ec2.internal I0428 11:04:11.140126 3656 kube.go:256] Updating pod openshift-console/console-5bcfcd7b98-mcfnm I0428 11:04:11.158126 3656 pod.go:62] [openshift-console/console-5bcfcd7b98-mcfnm] pod update took 18.032134ms I0428 11:04:11.158148 3656 base_network_controller_pods.go:951] [default/openshift-console/console-5bcfcd7b98-mcfnm] addLogicalPort annotation time took 18.061333ms I0428 11:04:11.159260 3656 pods.go:271] [openshift-console/console-5bcfcd7b98-mcfnm] addLogicalPort took 19.285827ms, libovsdb time 845.122µs I0428 11:04:11.539542 3656 cni.go:452] [openshift-console/console-5bcfcd7b98-mcfnm c170b29704ffeca7a9fb1d52d396eba64302d5061464a8cc59a4f38966ebce34 network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-5bcfcd7b98-mcfnm c170b29704ffeca7a9fb1d52d396eba64302d5061464a8cc59a4f38966ebce34 network default NAD default NAD key ] I0428 11:04:11.549963 3656 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-5bcfcd7b98-mcfnm, hostIfaceName: c170b29704ffeca, network: default, NAD default, SandboxID: "c170b29704ffeca7a9fb1d52d396eba64302d5061464a8cc59a4f38966ebce34", PCI device ID: , UID: "56295179-e42e-4d02-be49-8ab01054ce05", MAC: 0a:58:0a:82:00:0c, IPs: [10.130.0.12/23] I0428 11:04:11.604683 3656 cni.go:473] [openshift-console/console-5bcfcd7b98-mcfnm c170b29704ffeca7a9fb1d52d396eba64302d5061464a8cc59a4f38966ebce34 network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-5bcfcd7b98-mcfnm c170b29704ffeca7a9fb1d52d396eba64302d5061464a8cc59a4f38966ebce34 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"b2:fc:d3:88:e4:f6\",\"name\":\"c170b29704ffeca\"},{\"mac\":\"0a:58:0a:82:00:0c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/aa16b3d3-6174-4a81-a03f-7c1028b5b131\"}],\"ips\":[{\"address\":\"10.130.0.12/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0428 11:04:12.119655 3656 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0428 11:04:12.149753 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-214.ec2.internal port 8443: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0428 11:04:14.022550 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-214.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0428 11:04:15.138586 3656 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0428 11:04:15.688576 3656 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0428 11:04:17.662638 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-214.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0428 11:04:17.662658 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-214.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0428 11:04:17.662663 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-214.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0428 11:04:17.662682 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-214.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0428 11:04:17.662687 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-214.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0428 11:04:17.662691 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-214.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0428 11:04:18.117582 3656 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0428 11:04:18.297350 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-214.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0428 11:04:18.297379 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-214.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0428 11:04:18.297388 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-214.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0428 11:04:18.297405 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-214.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0428 11:04:18.297412 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-214.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0428 11:04:18.297419 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-214.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0428 11:04:19.109753 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-214.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0428 11:04:19.109775 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-214.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0428 11:04:19.109783 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-214.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0428 11:04:21.332183 3656 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0428 11:04:21.718680 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-214.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0428 11:04:21.718702 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-214.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0428 11:04:27.269756 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-214.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0428 11:04:27.269781 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-214.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0428 11:04:27.666741 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-214.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0428 11:04:27.666764 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-214.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0428 11:04:28.733259 3656 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads W0428 11:04:29.221687 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-214.ec2.internal port 8080: empty IP address endpoints for service openshift-console/downloads W0428 11:04:29.397641 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-214.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0428 11:04:29.397667 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-214.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0428 11:04:29.492494 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-214.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0428 11:04:29.492527 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-214.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0428 11:04:32.226068 3656 cni.go:452] [openshift-image-registry/image-registry-6f4c55d5d-d8w4p f69cb794a675452ec688e61e9237784ac6fb6a5fe88011ee071caca2257faac0 network default NAD default NAD key ] DEL starting CNI request [openshift-image-registry/image-registry-6f4c55d5d-d8w4p f69cb794a675452ec688e61e9237784ac6fb6a5fe88011ee071caca2257faac0 network default NAD default NAD key ] I0428 11:04:32.301875 3656 cni.go:473] [openshift-image-registry/image-registry-6f4c55d5d-d8w4p f69cb794a675452ec688e61e9237784ac6fb6a5fe88011ee071caca2257faac0 network default NAD default NAD key default] DEL finished CNI request [openshift-image-registry/image-registry-6f4c55d5d-d8w4p f69cb794a675452ec688e61e9237784ac6fb6a5fe88011ee071caca2257faac0 network default NAD default NAD key default], result "{}", err I0428 11:04:32.359544 3656 obj_retry.go:498] Detected object openshift-image-registry/image-registry-6f4c55d5d-d8w4p of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:04:32.359583 3656 pods.go:176] Deleting pod: openshift-image-registry/image-registry-6f4c55d5d-d8w4p I0428 11:04:32.360936 3656 pods.go:236] Attempting to release IPs for pod: openshift-image-registry/image-registry-6f4c55d5d-d8w4p, ips: 10.130.0.5 I0428 11:04:32.360968 3656 obj_retry.go:498] Detected object openshift-image-registry/image-registry-6f4c55d5d-d8w4p of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:04:32.371651 3656 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-6bb7584977-wtpdc] creating logical port openshift-image-registry_image-registry-6bb7584977-wtpdc for pod on switch ip-10-0-1-214.ec2.internal I0428 11:04:32.371780 3656 kube.go:256] Updating pod openshift-image-registry/image-registry-6bb7584977-wtpdc I0428 11:04:32.389739 3656 pod.go:62] [openshift-image-registry/image-registry-6bb7584977-wtpdc] pod update took 17.986893ms I0428 11:04:32.389766 3656 base_network_controller_pods.go:951] [default/openshift-image-registry/image-registry-6bb7584977-wtpdc] addLogicalPort annotation time took 18.020626ms I0428 11:04:32.390992 3656 pods.go:271] [openshift-image-registry/image-registry-6bb7584977-wtpdc] addLogicalPort took 19.358356ms, libovsdb time 905.901µs I0428 11:04:32.516199 3656 obj_retry.go:498] Detected object openshift-image-registry/image-registry-6c8946d869-rc9f4 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:04:32.516783 3656 obj_retry.go:498] Detected object openshift-image-registry/image-registry-6c8946d869-rc9f4 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:04:32.733666 3656 cni.go:452] [openshift-image-registry/image-registry-6bb7584977-wtpdc 856b16f244d0bb0b0a2ec870e790086ad9576cc95c19e60c0dbbc9d2405fab89 network default NAD default NAD key ] ADD starting CNI request [openshift-image-registry/image-registry-6bb7584977-wtpdc 856b16f244d0bb0b0a2ec870e790086ad9576cc95c19e60c0dbbc9d2405fab89 network default NAD default NAD key ] I0428 11:04:32.744399 3656 helper_linux.go:556] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-6bb7584977-wtpdc, hostIfaceName: 856b16f244d0bb0, network: default, NAD default, SandboxID: "856b16f244d0bb0b0a2ec870e790086ad9576cc95c19e60c0dbbc9d2405fab89", PCI device ID: , UID: "39eeff00-0947-43c1-aaaa-5a6d699426fb", MAC: 0a:58:0a:82:00:0d, IPs: [10.130.0.13/23] I0428 11:04:32.794524 3656 cni.go:473] [openshift-image-registry/image-registry-6bb7584977-wtpdc 856b16f244d0bb0b0a2ec870e790086ad9576cc95c19e60c0dbbc9d2405fab89 network default NAD default NAD key default] ADD finished CNI request [openshift-image-registry/image-registry-6bb7584977-wtpdc 856b16f244d0bb0b0a2ec870e790086ad9576cc95c19e60c0dbbc9d2405fab89 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"96:08:ac:b2:e8:99\",\"name\":\"856b16f244d0bb0\"},{\"mac\":\"0a:58:0a:82:00:0d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/21ed611d-22ab-4e46-bdf3-fc6384918d73\"}],\"ips\":[{\"address\":\"10.130.0.13/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0428 11:04:33.215062 3656 obj_retry.go:492] Detected object openshift-image-registry/image-registry-6f4c55d5d-d8w4p of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:04:33.215092 3656 obj_retry.go:492] Detected object openshift-image-registry/image-registry-6f4c55d5d-d8w4p of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:04:33.220723 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-6f4c55d5d-d8w4p I0428 11:04:33.220753 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-6f4c55d5d-d8w4p I0428 11:04:33.382856 3656 obj_retry.go:492] Detected object openshift-image-registry/image-registry-6c8946d869-rc9f4 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:04:33.382888 3656 obj_retry.go:492] Detected object openshift-image-registry/image-registry-6c8946d869-rc9f4 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:04:33.385630 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-6c8946d869-rc9f4 I0428 11:04:33.385652 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-6c8946d869-rc9f4 I0428 11:04:43.292715 3656 cni.go:452] [openshift-console/console-5bcfcd7b98-mcfnm c170b29704ffeca7a9fb1d52d396eba64302d5061464a8cc59a4f38966ebce34 network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-5bcfcd7b98-mcfnm c170b29704ffeca7a9fb1d52d396eba64302d5061464a8cc59a4f38966ebce34 network default NAD default NAD key ] I0428 11:04:43.364669 3656 cni.go:473] [openshift-console/console-5bcfcd7b98-mcfnm c170b29704ffeca7a9fb1d52d396eba64302d5061464a8cc59a4f38966ebce34 network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-5bcfcd7b98-mcfnm c170b29704ffeca7a9fb1d52d396eba64302d5061464a8cc59a4f38966ebce34 network default NAD default NAD key default], result "{}", err I0428 11:04:43.427666 3656 obj_retry.go:498] Detected object openshift-console/console-5bcfcd7b98-mcfnm of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:04:43.427703 3656 pods.go:176] Deleting pod: openshift-console/console-5bcfcd7b98-mcfnm I0428 11:04:43.428860 3656 pods.go:236] Attempting to release IPs for pod: openshift-console/console-5bcfcd7b98-mcfnm, ips: 10.130.0.12 I0428 11:04:43.428888 3656 obj_retry.go:498] Detected object openshift-console/console-5bcfcd7b98-mcfnm of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:04:43.437081 3656 base_network_controller_pods.go:487] [default/openshift-console/console-c669869f6-5ngzn] creating logical port openshift-console_console-c669869f6-5ngzn for pod on switch ip-10-0-1-214.ec2.internal I0428 11:04:43.437227 3656 kube.go:256] Updating pod openshift-console/console-c669869f6-5ngzn I0428 11:04:43.456066 3656 pod.go:62] [openshift-console/console-c669869f6-5ngzn] pod update took 18.871258ms I0428 11:04:43.456089 3656 base_network_controller_pods.go:951] [default/openshift-console/console-c669869f6-5ngzn] addLogicalPort annotation time took 18.900536ms I0428 11:04:43.457222 3656 pods.go:271] [openshift-console/console-c669869f6-5ngzn] addLogicalPort took 20.158017ms, libovsdb time 836.653µs I0428 11:04:43.841266 3656 cni.go:452] [openshift-console/console-c669869f6-5ngzn 993eecde3dcd6618d84d0cf373407b15a09ea6f22cb8320b9e2d1231bbb2c09c network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-c669869f6-5ngzn 993eecde3dcd6618d84d0cf373407b15a09ea6f22cb8320b9e2d1231bbb2c09c network default NAD default NAD key ] I0428 11:04:43.850415 3656 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-c669869f6-5ngzn, hostIfaceName: 993eecde3dcd661, network: default, NAD default, SandboxID: "993eecde3dcd6618d84d0cf373407b15a09ea6f22cb8320b9e2d1231bbb2c09c", PCI device ID: , UID: "6f454dd3-4272-4100-97cd-e434dda31cac", MAC: 0a:58:0a:82:00:0e, IPs: [10.130.0.14/23] I0428 11:04:43.897999 3656 cni.go:473] [openshift-console/console-c669869f6-5ngzn 993eecde3dcd6618d84d0cf373407b15a09ea6f22cb8320b9e2d1231bbb2c09c network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-c669869f6-5ngzn 993eecde3dcd6618d84d0cf373407b15a09ea6f22cb8320b9e2d1231bbb2c09c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"82:fe:c6:a0:52:ca\",\"name\":\"993eecde3dcd661\"},{\"mac\":\"0a:58:0a:82:00:0e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/80506789-ba09-4bce-8aef-17f8b707de25\"}],\"ips\":[{\"address\":\"10.130.0.14/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0428 11:04:44.251968 3656 obj_retry.go:492] Detected object openshift-console/console-5bcfcd7b98-mcfnm of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:04:44.252000 3656 obj_retry.go:492] Detected object openshift-console/console-5bcfcd7b98-mcfnm of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:04:44.254771 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-5bcfcd7b98-mcfnm I0428 11:04:44.254790 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-5bcfcd7b98-mcfnm W0428 11:04:47.421367 3656 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-insights/metrics I0428 11:04:52.513965 3656 base_network_controller_pods.go:487] [default/openshift-monitoring/openshift-state-metrics-b8587444b-kbswr] creating logical port openshift-monitoring_openshift-state-metrics-b8587444b-kbswr for pod on switch ip-10-0-1-214.ec2.internal I0428 11:04:52.514094 3656 kube.go:256] Updating pod openshift-monitoring/openshift-state-metrics-b8587444b-kbswr I0428 11:04:52.533196 3656 pod.go:62] [openshift-monitoring/openshift-state-metrics-b8587444b-kbswr] pod update took 19.145315ms I0428 11:04:52.533219 3656 base_network_controller_pods.go:951] [default/openshift-monitoring/openshift-state-metrics-b8587444b-kbswr] addLogicalPort annotation time took 19.175294ms I0428 11:04:52.534368 3656 pods.go:271] [openshift-monitoring/openshift-state-metrics-b8587444b-kbswr] addLogicalPort took 20.453252ms, libovsdb time 874.778µs I0428 11:04:52.569512 3656 base_network_controller_pods.go:487] [default/openshift-monitoring/kube-state-metrics-6dfffbb599-4j2v5] creating logical port openshift-monitoring_kube-state-metrics-6dfffbb599-4j2v5 for pod on switch ip-10-0-1-214.ec2.internal I0428 11:04:52.569605 3656 kube.go:256] Updating pod openshift-monitoring/kube-state-metrics-6dfffbb599-4j2v5 I0428 11:04:52.595778 3656 pod.go:62] [openshift-monitoring/kube-state-metrics-6dfffbb599-4j2v5] pod update took 26.196345ms I0428 11:04:52.595801 3656 base_network_controller_pods.go:951] [default/openshift-monitoring/kube-state-metrics-6dfffbb599-4j2v5] addLogicalPort annotation time took 26.226354ms I0428 11:04:52.596996 3656 pods.go:271] [openshift-monitoring/kube-state-metrics-6dfffbb599-4j2v5] addLogicalPort took 27.502356ms, libovsdb time 887.747µs I0428 11:04:52.939173 3656 cni.go:452] [openshift-monitoring/openshift-state-metrics-b8587444b-kbswr cd9e6416b121e46078b8af1bb970a23d01daa9f1c6d82a338eee6d31e8042038 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/openshift-state-metrics-b8587444b-kbswr cd9e6416b121e46078b8af1bb970a23d01daa9f1c6d82a338eee6d31e8042038 network default NAD default NAD key ] I0428 11:04:52.949017 3656 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: openshift-state-metrics-b8587444b-kbswr, hostIfaceName: cd9e6416b121e46, network: default, NAD default, SandboxID: "cd9e6416b121e46078b8af1bb970a23d01daa9f1c6d82a338eee6d31e8042038", PCI device ID: , UID: "b01248c8-94fd-422b-a788-85f1792532a2", MAC: 0a:58:0a:82:00:0f, IPs: [10.130.0.15/23] I0428 11:04:52.980012 3656 cni.go:452] [openshift-monitoring/kube-state-metrics-6dfffbb599-4j2v5 9d25735d7929f9b306cef12e4ee51a79c9cb6d58b64cbee549b2e8886e739124 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/kube-state-metrics-6dfffbb599-4j2v5 9d25735d7929f9b306cef12e4ee51a79c9cb6d58b64cbee549b2e8886e739124 network default NAD default NAD key ] I0428 11:04:52.991820 3656 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: kube-state-metrics-6dfffbb599-4j2v5, hostIfaceName: 9d25735d7929f9b, network: default, NAD default, SandboxID: "9d25735d7929f9b306cef12e4ee51a79c9cb6d58b64cbee549b2e8886e739124", PCI device ID: , UID: "0167166f-68a4-4434-973b-aa4747e6c507", MAC: 0a:58:0a:82:00:10, IPs: [10.130.0.16/23] I0428 11:04:53.001695 3656 cni.go:473] [openshift-monitoring/openshift-state-metrics-b8587444b-kbswr cd9e6416b121e46078b8af1bb970a23d01daa9f1c6d82a338eee6d31e8042038 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/openshift-state-metrics-b8587444b-kbswr cd9e6416b121e46078b8af1bb970a23d01daa9f1c6d82a338eee6d31e8042038 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"aa:fb:43:f4:2d:82\",\"name\":\"cd9e6416b121e46\"},{\"mac\":\"0a:58:0a:82:00:0f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c5c67d76-fe14-49a8-8651-8db6eaa42e4c\"}],\"ips\":[{\"address\":\"10.130.0.15/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0428 11:04:53.038859 3656 cni.go:473] [openshift-monitoring/kube-state-metrics-6dfffbb599-4j2v5 9d25735d7929f9b306cef12e4ee51a79c9cb6d58b64cbee549b2e8886e739124 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/kube-state-metrics-6dfffbb599-4j2v5 9d25735d7929f9b306cef12e4ee51a79c9cb6d58b64cbee549b2e8886e739124 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"2e:0e:5a:f6:ae:fc\",\"name\":\"9d25735d7929f9b\"},{\"mac\":\"0a:58:0a:82:00:10\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ecf0420d-5909-4820-92ca-2d28f2407791\"}],\"ips\":[{\"address\":\"10.130.0.16/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0428 11:04:53.577175 3656 base_network_controller_pods.go:487] [default/openshift-monitoring/alertmanager-main-0] creating logical port openshift-monitoring_alertmanager-main-0 for pod on switch ip-10-0-1-214.ec2.internal I0428 11:04:53.577283 3656 kube.go:256] Updating pod openshift-monitoring/alertmanager-main-0 I0428 11:04:53.601547 3656 pod.go:62] [openshift-monitoring/alertmanager-main-0] pod update took 24.302271ms I0428 11:04:53.601571 3656 base_network_controller_pods.go:951] [default/openshift-monitoring/alertmanager-main-0] addLogicalPort annotation time took 24.331747ms I0428 11:04:53.602662 3656 pods.go:271] [openshift-monitoring/alertmanager-main-0] addLogicalPort took 25.507325ms, libovsdb time 815.391µs W0428 11:04:54.225948 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-214.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0428 11:04:54.306290 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-214.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0428 11:04:55.024078 3656 obj_retry.go:498] Detected object openshift-console/console-5bcfcd7b98-zkmg8 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:04:55.024624 3656 obj_retry.go:498] Detected object openshift-console/console-5bcfcd7b98-zkmg8 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:04:55.370108 3656 cni.go:452] [openshift-monitoring/alertmanager-main-0 eed3f6079926dcb91c6bb8c23aa9b22cee0444b61abe47a64626646b2579f4a1 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/alertmanager-main-0 eed3f6079926dcb91c6bb8c23aa9b22cee0444b61abe47a64626646b2579f4a1 network default NAD default NAD key ] I0428 11:04:55.380638 3656 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: alertmanager-main-0, hostIfaceName: eed3f6079926dcb, network: default, NAD default, SandboxID: "eed3f6079926dcb91c6bb8c23aa9b22cee0444b61abe47a64626646b2579f4a1", PCI device ID: , UID: "d0a8ef0b-e599-422d-9ab0-b374b4403344", MAC: 0a:58:0a:82:00:11, IPs: [10.130.0.17/23] I0428 11:04:55.431948 3656 cni.go:473] [openshift-monitoring/alertmanager-main-0 eed3f6079926dcb91c6bb8c23aa9b22cee0444b61abe47a64626646b2579f4a1 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/alertmanager-main-0 eed3f6079926dcb91c6bb8c23aa9b22cee0444b61abe47a64626646b2579f4a1 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"9a:08:53:53:82:cb\",\"name\":\"eed3f6079926dcb\"},{\"mac\":\"0a:58:0a:82:00:11\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/23182572-8766-4d1a-a0ee-e3ca02320e48\"}],\"ips\":[{\"address\":\"10.130.0.17/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0428 11:04:55.585515 3656 base_network_controller_pods.go:487] [default/openshift-monitoring/thanos-querier-5b6555c46c-xb5r4] creating logical port openshift-monitoring_thanos-querier-5b6555c46c-xb5r4 for pod on switch ip-10-0-1-214.ec2.internal I0428 11:04:55.585619 3656 kube.go:256] Updating pod openshift-monitoring/thanos-querier-5b6555c46c-xb5r4 I0428 11:04:55.607501 3656 pod.go:62] [openshift-monitoring/thanos-querier-5b6555c46c-xb5r4] pod update took 21.92306ms I0428 11:04:55.607529 3656 base_network_controller_pods.go:951] [default/openshift-monitoring/thanos-querier-5b6555c46c-xb5r4] addLogicalPort annotation time took 21.951686ms I0428 11:04:55.608701 3656 pods.go:271] [openshift-monitoring/thanos-querier-5b6555c46c-xb5r4] addLogicalPort took 23.203347ms, libovsdb time 818.631µs I0428 11:04:55.848062 3656 obj_retry.go:492] Detected object openshift-console/console-5bcfcd7b98-zkmg8 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:04:55.848088 3656 obj_retry.go:492] Detected object openshift-console/console-5bcfcd7b98-zkmg8 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:04:55.854598 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-5bcfcd7b98-zkmg8 I0428 11:04:55.854623 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-5bcfcd7b98-zkmg8 I0428 11:04:56.035904 3656 cni.go:452] [openshift-monitoring/thanos-querier-5b6555c46c-xb5r4 3a46c89a055c465ec2dc9d13d082d220b0901a738f57543b012bd8474f80f4e6 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/thanos-querier-5b6555c46c-xb5r4 3a46c89a055c465ec2dc9d13d082d220b0901a738f57543b012bd8474f80f4e6 network default NAD default NAD key ] I0428 11:04:56.045714 3656 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: thanos-querier-5b6555c46c-xb5r4, hostIfaceName: 3a46c89a055c465, network: default, NAD default, SandboxID: "3a46c89a055c465ec2dc9d13d082d220b0901a738f57543b012bd8474f80f4e6", PCI device ID: , UID: "6ead087a-5dbd-48e9-a146-b317f101dedc", MAC: 0a:58:0a:82:00:12, IPs: [10.130.0.18/23] I0428 11:04:56.097577 3656 cni.go:473] [openshift-monitoring/thanos-querier-5b6555c46c-xb5r4 3a46c89a055c465ec2dc9d13d082d220b0901a738f57543b012bd8474f80f4e6 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/thanos-querier-5b6555c46c-xb5r4 3a46c89a055c465ec2dc9d13d082d220b0901a738f57543b012bd8474f80f4e6 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"b6:08:27:ff:42:76\",\"name\":\"3a46c89a055c465\"},{\"mac\":\"0a:58:0a:82:00:12\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/91bcd4e2-0bc0-4cf1-a293-c4505bca98ea\"}],\"ips\":[{\"address\":\"10.130.0.18/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0428 11:04:57.093895 3656 base_network_controller_pods.go:487] [default/openshift-monitoring/metrics-server-56b749db89-6r65g] creating logical port openshift-monitoring_metrics-server-56b749db89-6r65g for pod on switch ip-10-0-1-214.ec2.internal I0428 11:04:57.094044 3656 kube.go:256] Updating pod openshift-monitoring/metrics-server-56b749db89-6r65g I0428 11:04:57.112336 3656 pod.go:62] [openshift-monitoring/metrics-server-56b749db89-6r65g] pod update took 18.319829ms I0428 11:04:57.112362 3656 base_network_controller_pods.go:951] [default/openshift-monitoring/metrics-server-56b749db89-6r65g] addLogicalPort annotation time took 18.348108ms I0428 11:04:57.113643 3656 pods.go:271] [openshift-monitoring/metrics-server-56b749db89-6r65g] addLogicalPort took 19.768493ms, libovsdb time 945.088µs I0428 11:04:57.264181 3656 base_network_controller_pods.go:487] [default/openshift-monitoring/monitoring-plugin-5cbb48bf6b-82gfr] creating logical port openshift-monitoring_monitoring-plugin-5cbb48bf6b-82gfr for pod on switch ip-10-0-1-214.ec2.internal I0428 11:04:57.264280 3656 kube.go:256] Updating pod openshift-monitoring/monitoring-plugin-5cbb48bf6b-82gfr I0428 11:04:57.284176 3656 pod.go:62] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-82gfr] pod update took 19.917355ms I0428 11:04:57.284208 3656 base_network_controller_pods.go:951] [default/openshift-monitoring/monitoring-plugin-5cbb48bf6b-82gfr] addLogicalPort annotation time took 19.955051ms I0428 11:04:57.285462 3656 pods.go:271] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-82gfr] addLogicalPort took 21.298933ms, libovsdb time 841.618µs W0428 11:04:57.484579 3656 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0428 11:04:57.484604 3656 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0428 11:04:57.484611 3656 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0428 11:04:57.560974 3656 cni.go:452] [openshift-monitoring/metrics-server-56b749db89-6r65g 3c4b64d561e082bcc92a67691ddfe71017933aba4e438379f7dfbcbdf7ca7d59 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/metrics-server-56b749db89-6r65g 3c4b64d561e082bcc92a67691ddfe71017933aba4e438379f7dfbcbdf7ca7d59 network default NAD default NAD key ] I0428 11:04:57.571600 3656 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: metrics-server-56b749db89-6r65g, hostIfaceName: 3c4b64d561e082b, network: default, NAD default, SandboxID: "3c4b64d561e082bcc92a67691ddfe71017933aba4e438379f7dfbcbdf7ca7d59", PCI device ID: , UID: "15926229-84c3-4f29-8f79-189d316d7e05", MAC: 0a:58:0a:82:00:13, IPs: [10.130.0.19/23] I0428 11:04:57.623465 3656 cni.go:473] [openshift-monitoring/metrics-server-56b749db89-6r65g 3c4b64d561e082bcc92a67691ddfe71017933aba4e438379f7dfbcbdf7ca7d59 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/metrics-server-56b749db89-6r65g 3c4b64d561e082bcc92a67691ddfe71017933aba4e438379f7dfbcbdf7ca7d59 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"16:17:7e:a0:d9:e9\",\"name\":\"3c4b64d561e082b\"},{\"mac\":\"0a:58:0a:82:00:13\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/45150d85-ebb7-48e6-bfd5-c56cfb769952\"}],\"ips\":[{\"address\":\"10.130.0.19/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0428 11:04:57.664440 3656 cni.go:452] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-82gfr c0e89372a33b76dcfea1561c2fb149af87489414d343d235d45fe9c397cf126c network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/monitoring-plugin-5cbb48bf6b-82gfr c0e89372a33b76dcfea1561c2fb149af87489414d343d235d45fe9c397cf126c network default NAD default NAD key ] I0428 11:04:57.674092 3656 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: monitoring-plugin-5cbb48bf6b-82gfr, hostIfaceName: c0e89372a33b76d, network: default, NAD default, SandboxID: "c0e89372a33b76dcfea1561c2fb149af87489414d343d235d45fe9c397cf126c", PCI device ID: , UID: "64bf1906-c0be-4f64-bac8-5c5a3f01cfb1", MAC: 0a:58:0a:82:00:14, IPs: [10.130.0.20/23] I0428 11:04:57.722784 3656 cni.go:473] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-82gfr c0e89372a33b76dcfea1561c2fb149af87489414d343d235d45fe9c397cf126c network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/monitoring-plugin-5cbb48bf6b-82gfr c0e89372a33b76dcfea1561c2fb149af87489414d343d235d45fe9c397cf126c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"2a:4f:35:59:a8:e5\",\"name\":\"c0e89372a33b76d\"},{\"mac\":\"0a:58:0a:82:00:14\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f2eca091-2c3a-4808-bd97-8b932b3914c9\"}],\"ips\":[{\"address\":\"10.130.0.20/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0428 11:04:57.861850 3656 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0428 11:04:57.861868 3656 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0428 11:04:57.861874 3656 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0428 11:04:58.470937 3656 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics I0428 11:04:59.077697 3656 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-k8s-1] creating logical port openshift-monitoring_prometheus-k8s-1 for pod on switch ip-10-0-1-214.ec2.internal I0428 11:04:59.077827 3656 kube.go:256] Updating pod openshift-monitoring/prometheus-k8s-1 I0428 11:04:59.117763 3656 pod.go:62] [openshift-monitoring/prometheus-k8s-1] pod update took 39.994024ms I0428 11:04:59.117792 3656 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-k8s-1] addLogicalPort annotation time took 40.026286ms I0428 11:04:59.119146 3656 pods.go:271] [openshift-monitoring/prometheus-k8s-1] addLogicalPort took 41.467786ms, libovsdb time 942.34µs I0428 11:05:00.989484 3656 cni.go:452] [openshift-monitoring/prometheus-k8s-1 1817ad7b9f5ab213bd8690573eab0d76618f3c8577ffc32a684751ba8ef82382 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-k8s-1 1817ad7b9f5ab213bd8690573eab0d76618f3c8577ffc32a684751ba8ef82382 network default NAD default NAD key ] I0428 11:05:01.014542 3656 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-k8s-1, hostIfaceName: 1817ad7b9f5ab21, network: default, NAD default, SandboxID: "1817ad7b9f5ab213bd8690573eab0d76618f3c8577ffc32a684751ba8ef82382", PCI device ID: , UID: "dd3c2210-2fe3-4123-b33c-c1b8e2fc7ec5", MAC: 0a:58:0a:82:00:15, IPs: [10.130.0.21/23] I0428 11:05:01.066369 3656 cni.go:473] [openshift-monitoring/prometheus-k8s-1 1817ad7b9f5ab213bd8690573eab0d76618f3c8577ffc32a684751ba8ef82382 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-k8s-1 1817ad7b9f5ab213bd8690573eab0d76618f3c8577ffc32a684751ba8ef82382 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"82:05:86:34:12:2b\",\"name\":\"1817ad7b9f5ab21\"},{\"mac\":\"0a:58:0a:82:00:15\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2042b3f6-8d39-40ec-a524-d2bd15f0ad0e\"}],\"ips\":[{\"address\":\"10.130.0.21/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0428 11:05:01.332344 3656 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0428 11:05:01.382523 3656 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0428 11:05:01.382542 3656 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0428 11:05:01.869975 3656 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0428 11:05:01.913275 3656 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0428 11:05:01.913290 3656 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0428 11:05:03.943541 3656 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0428 11:05:03.943632 3656 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0428 11:05:03.943668 3656 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0428 11:05:03.943692 3656 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0428 11:05:04.373996 3656 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0428 11:05:04.374023 3656 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0428 11:05:04.374031 3656 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0428 11:05:04.374036 3656 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0428 11:05:06.396576 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-214.ec2.internal port 9093: empty IP address endpoints for service openshift-monitoring/thanos-querier W0428 11:05:06.396687 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-214.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/thanos-querier W0428 11:05:06.396724 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-214.ec2.internal port 9094: empty IP address endpoints for service openshift-monitoring/thanos-querier W0428 11:05:06.396756 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-214.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/thanos-querier I0428 11:05:12.676000 3656 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29622900-d5prm of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:05:12.676560 3656 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29622900-d5prm of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:05:13.547136 3656 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29622900-d5prm of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:05:13.547161 3656 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29622900-d5prm of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0428 11:05:17.546324 3656 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-service-ca-operator/metrics I0428 11:05:19.591333 3656 obj_retry.go:498] Detected object openshift-image-registry/image-registry-6c8946d869-j79sv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:05:19.591942 3656 obj_retry.go:498] Detected object openshift-image-registry/image-registry-6c8946d869-j79sv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:05:19.968164 3656 obj_retry.go:492] Detected object openshift-image-registry/image-registry-6c8946d869-j79sv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:05:19.968200 3656 obj_retry.go:492] Detected object openshift-image-registry/image-registry-6c8946d869-j79sv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:05:19.978041 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-6c8946d869-j79sv I0428 11:05:19.978074 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-6c8946d869-j79sv I0428 11:05:22.345374 3656 obj_retry.go:498] Detected object openshift-ingress/router-default-74bcd9648c-v9ffn of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:05:22.345909 3656 obj_retry.go:498] Detected object openshift-ingress/router-default-74bcd9648c-v9ffn of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0428 11:05:22.368465 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-214.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server I0428 11:05:22.449672 3656 cni.go:452] [openshift-console/console-c669869f6-5ngzn 993eecde3dcd6618d84d0cf373407b15a09ea6f22cb8320b9e2d1231bbb2c09c network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-c669869f6-5ngzn 993eecde3dcd6618d84d0cf373407b15a09ea6f22cb8320b9e2d1231bbb2c09c network default NAD default NAD key ] I0428 11:05:22.521442 3656 cni.go:473] [openshift-console/console-c669869f6-5ngzn 993eecde3dcd6618d84d0cf373407b15a09ea6f22cb8320b9e2d1231bbb2c09c network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-c669869f6-5ngzn 993eecde3dcd6618d84d0cf373407b15a09ea6f22cb8320b9e2d1231bbb2c09c network default NAD default NAD key default], result "{}", err I0428 11:05:22.565243 3656 obj_retry.go:492] Detected object openshift-ingress/router-default-74bcd9648c-v9ffn of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:05:22.565263 3656 obj_retry.go:492] Detected object openshift-ingress/router-default-74bcd9648c-v9ffn of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:05:22.574666 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/router-default-74bcd9648c-v9ffn I0428 11:05:22.574695 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/router-default-74bcd9648c-v9ffn I0428 11:05:22.585248 3656 obj_retry.go:498] Detected object openshift-console/console-c669869f6-5ngzn of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:05:22.585307 3656 pods.go:176] Deleting pod: openshift-console/console-c669869f6-5ngzn I0428 11:05:22.586557 3656 pods.go:236] Attempting to release IPs for pod: openshift-console/console-c669869f6-5ngzn, ips: 10.130.0.14 I0428 11:05:22.586588 3656 obj_retry.go:498] Detected object openshift-console/console-c669869f6-5ngzn of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:05:22.594385 3656 base_network_controller_pods.go:487] [default/openshift-console/console-64774ffbc8-pnb2m] creating logical port openshift-console_console-64774ffbc8-pnb2m for pod on switch ip-10-0-1-214.ec2.internal I0428 11:05:22.594466 3656 kube.go:256] Updating pod openshift-console/console-64774ffbc8-pnb2m I0428 11:05:22.608566 3656 pod.go:62] [openshift-console/console-64774ffbc8-pnb2m] pod update took 14.12395ms I0428 11:05:22.608594 3656 base_network_controller_pods.go:951] [default/openshift-console/console-64774ffbc8-pnb2m] addLogicalPort annotation time took 14.15431ms I0428 11:05:22.609983 3656 pods.go:271] [openshift-console/console-64774ffbc8-pnb2m] addLogicalPort took 15.609704ms, libovsdb time 1.067688ms I0428 11:05:22.959580 3656 cni.go:452] [openshift-console/console-64774ffbc8-pnb2m 68d61705b90e78eab5eb93c3a105b264b3bcd5c5d0891f35ae65b0fd325fbc8f network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-64774ffbc8-pnb2m 68d61705b90e78eab5eb93c3a105b264b3bcd5c5d0891f35ae65b0fd325fbc8f network default NAD default NAD key ] I0428 11:05:22.969097 3656 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-64774ffbc8-pnb2m, hostIfaceName: 68d61705b90e78e, network: default, NAD default, SandboxID: "68d61705b90e78eab5eb93c3a105b264b3bcd5c5d0891f35ae65b0fd325fbc8f", PCI device ID: , UID: "0e42684e-b97e-4654-bccb-c1e9cf67c60c", MAC: 0a:58:0a:82:00:16, IPs: [10.130.0.22/23] I0428 11:05:23.017485 3656 cni.go:473] [openshift-console/console-64774ffbc8-pnb2m 68d61705b90e78eab5eb93c3a105b264b3bcd5c5d0891f35ae65b0fd325fbc8f network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-64774ffbc8-pnb2m 68d61705b90e78eab5eb93c3a105b264b3bcd5c5d0891f35ae65b0fd325fbc8f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"9a:0f:01:f7:60:61\",\"name\":\"68d61705b90e78e\"},{\"mac\":\"0a:58:0a:82:00:16\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/85d6a4ff-3b45-453c-968a-b5f666e26f0e\"}],\"ips\":[{\"address\":\"10.130.0.22/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0428 11:05:23.446193 3656 obj_retry.go:492] Detected object openshift-console/console-c669869f6-5ngzn of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:05:23.446217 3656 obj_retry.go:492] Detected object openshift-console/console-c669869f6-5ngzn of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:05:23.454286 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-c669869f6-5ngzn I0428 11:05:23.454313 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-c669869f6-5ngzn W0428 11:05:24.609582 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-214.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0428 11:05:24.609610 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-214.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0428 11:05:24.609617 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-214.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0428 11:05:33.280155 3656 obj_retry.go:498] Detected object openshift-console/console-c669869f6-mlxxc of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:05:33.280709 3656 obj_retry.go:498] Detected object openshift-console/console-c669869f6-mlxxc of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0428 11:05:33.499364 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-214.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0428 11:05:33.612299 3656 obj_retry.go:492] Detected object openshift-console/console-c669869f6-mlxxc of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:05:33.612324 3656 obj_retry.go:492] Detected object openshift-console/console-c669869f6-mlxxc of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:05:33.622434 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-c669869f6-mlxxc I0428 11:05:33.622461 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-c669869f6-mlxxc W0428 11:05:59.644666 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-214.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0428 11:05:59.644691 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-214.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0428 11:06:14.562217 3656 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:06:14.562791 3656 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:06:15.197156 3656 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:06:15.197186 3656 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:06:15.205371 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-1 I0428 11:06:15.205392 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-1 I0428 11:06:17.366988 3656 base_network_controller_pods.go:487] [default/openshift-monitoring/telemeter-client-648759467f-74xjv] creating logical port openshift-monitoring_telemeter-client-648759467f-74xjv for pod on switch ip-10-0-1-214.ec2.internal I0428 11:06:17.367091 3656 kube.go:256] Updating pod openshift-monitoring/telemeter-client-648759467f-74xjv I0428 11:06:17.388541 3656 pod.go:62] [openshift-monitoring/telemeter-client-648759467f-74xjv] pod update took 21.475473ms I0428 11:06:17.388564 3656 base_network_controller_pods.go:951] [default/openshift-monitoring/telemeter-client-648759467f-74xjv] addLogicalPort annotation time took 21.50453ms I0428 11:06:17.389873 3656 pods.go:271] [openshift-monitoring/telemeter-client-648759467f-74xjv] addLogicalPort took 22.904659ms, libovsdb time 983.286µs I0428 11:06:17.727985 3656 cni.go:452] [openshift-monitoring/telemeter-client-648759467f-74xjv 44085284cd82befc53511d3c1849acf07d6385951233da41c62748f591bb0f86 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/telemeter-client-648759467f-74xjv 44085284cd82befc53511d3c1849acf07d6385951233da41c62748f591bb0f86 network default NAD default NAD key ] I0428 11:06:17.737220 3656 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: telemeter-client-648759467f-74xjv, hostIfaceName: 44085284cd82bef, network: default, NAD default, SandboxID: "44085284cd82befc53511d3c1849acf07d6385951233da41c62748f591bb0f86", PCI device ID: , UID: "429003cb-545c-4a38-8ae0-373672ecc6c9", MAC: 0a:58:0a:82:00:17, IPs: [10.130.0.23/23] I0428 11:06:17.788853 3656 cni.go:473] [openshift-monitoring/telemeter-client-648759467f-74xjv 44085284cd82befc53511d3c1849acf07d6385951233da41c62748f591bb0f86 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/telemeter-client-648759467f-74xjv 44085284cd82befc53511d3c1849acf07d6385951233da41c62748f591bb0f86 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"1e:7c:88:83:36:f1\",\"name\":\"44085284cd82bef\"},{\"mac\":\"0a:58:0a:82:00:17\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c4f731b4-debe-4c1b-b8a0-636ed1b4ac5d\"}],\"ips\":[{\"address\":\"10.130.0.23/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0428 11:06:18.820504 3656 cni.go:452] [openshift-monitoring/prometheus-k8s-1 1817ad7b9f5ab213bd8690573eab0d76618f3c8577ffc32a684751ba8ef82382 network default NAD default NAD key ] DEL starting CNI request [openshift-monitoring/prometheus-k8s-1 1817ad7b9f5ab213bd8690573eab0d76618f3c8577ffc32a684751ba8ef82382 network default NAD default NAD key ] I0428 11:06:18.891012 3656 cni.go:473] [openshift-monitoring/prometheus-k8s-1 1817ad7b9f5ab213bd8690573eab0d76618f3c8577ffc32a684751ba8ef82382 network default NAD default NAD key default] DEL finished CNI request [openshift-monitoring/prometheus-k8s-1 1817ad7b9f5ab213bd8690573eab0d76618f3c8577ffc32a684751ba8ef82382 network default NAD default NAD key default], result "{}", err I0428 11:06:18.910550 3656 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d2p75x of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:06:18.911098 3656 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d2p75x of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:06:18.911128 3656 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d2p75x of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 11:06:18.911137 3656 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... I0428 11:06:18.911152 3656 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d2p75x of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 11:06:18.911159 3656 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles to delete 1 local pods... I0428 11:06:18.952977 3656 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:06:18.953016 3656 pods.go:176] Deleting pod: openshift-monitoring/prometheus-k8s-1 I0428 11:06:18.954607 3656 pods.go:236] Attempting to release IPs for pod: openshift-monitoring/prometheus-k8s-1, ips: 10.130.0.21 I0428 11:06:18.954670 3656 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:06:19.686448 3656 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:06:19.686480 3656 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:06:19.694021 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-1 I0428 11:06:19.694070 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-1 I0428 11:06:19.735052 3656 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-k8s-1] creating logical port openshift-monitoring_prometheus-k8s-1 for pod on switch ip-10-0-1-214.ec2.internal I0428 11:06:19.735164 3656 kube.go:256] Updating pod openshift-monitoring/prometheus-k8s-1 I0428 11:06:19.754127 3656 pod.go:62] [openshift-monitoring/prometheus-k8s-1] pod update took 18.9947ms I0428 11:06:19.754153 3656 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-k8s-1] addLogicalPort annotation time took 19.031959ms I0428 11:06:19.755505 3656 pods.go:271] [openshift-monitoring/prometheus-k8s-1] addLogicalPort took 20.468805ms, libovsdb time 1.011073ms I0428 11:06:19.936345 3656 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d2p75x of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:06:19.936367 3656 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d2p75x of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:06:19.936380 3656 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d2p75x of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:06:19.936392 3656 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d2p75x of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:06:20.099634 3656 cni.go:452] [openshift-monitoring/prometheus-k8s-1 0e1e73d3a81c4ea0b82d57419ce201fe8e8a53a4fae27a62cec8e9c6ed544ea8 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-k8s-1 0e1e73d3a81c4ea0b82d57419ce201fe8e8a53a4fae27a62cec8e9c6ed544ea8 network default NAD default NAD key ] I0428 11:06:20.109777 3656 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-k8s-1, hostIfaceName: 0e1e73d3a81c4ea, network: default, NAD default, SandboxID: "0e1e73d3a81c4ea0b82d57419ce201fe8e8a53a4fae27a62cec8e9c6ed544ea8", PCI device ID: , UID: "9c9e894e-8131-4a2e-a231-48f0898c423d", MAC: 0a:58:0a:82:00:18, IPs: [10.130.0.24/23] I0428 11:06:20.162641 3656 cni.go:473] [openshift-monitoring/prometheus-k8s-1 0e1e73d3a81c4ea0b82d57419ce201fe8e8a53a4fae27a62cec8e9c6ed544ea8 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-k8s-1 0e1e73d3a81c4ea0b82d57419ce201fe8e8a53a4fae27a62cec8e9c6ed544ea8 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"f2:7d:d8:9a:16:73\",\"name\":\"0e1e73d3a81c4ea\"},{\"mac\":\"0a:58:0a:82:00:18\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e47eeb5a-56df-4fcb-8a40-b2fe5c1facac\"}],\"ips\":[{\"address\":\"10.130.0.24/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0428 11:06:20.983779 3656 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0428 11:06:20.997475 3656 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0428 11:06:46.821163 3656 cni.go:452] [openshift-monitoring/alertmanager-main-0 eed3f6079926dcb91c6bb8c23aa9b22cee0444b61abe47a64626646b2579f4a1 network default NAD default NAD key ] DEL starting CNI request [openshift-monitoring/alertmanager-main-0 eed3f6079926dcb91c6bb8c23aa9b22cee0444b61abe47a64626646b2579f4a1 network default NAD default NAD key ] I0428 11:06:46.914656 3656 cni.go:473] [openshift-monitoring/alertmanager-main-0 eed3f6079926dcb91c6bb8c23aa9b22cee0444b61abe47a64626646b2579f4a1 network default NAD default NAD key default] DEL finished CNI request [openshift-monitoring/alertmanager-main-0 eed3f6079926dcb91c6bb8c23aa9b22cee0444b61abe47a64626646b2579f4a1 network default NAD default NAD key default], result "{}", err I0428 11:06:46.974305 3656 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:06:46.974348 3656 pods.go:176] Deleting pod: openshift-monitoring/alertmanager-main-0 I0428 11:06:46.975605 3656 pods.go:236] Attempting to release IPs for pod: openshift-monitoring/alertmanager-main-0, ips: 10.130.0.17 I0428 11:06:46.975641 3656 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:06:47.119272 3656 cni.go:452] [openshift-console/console-64774ffbc8-pnb2m 68d61705b90e78eab5eb93c3a105b264b3bcd5c5d0891f35ae65b0fd325fbc8f network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-64774ffbc8-pnb2m 68d61705b90e78eab5eb93c3a105b264b3bcd5c5d0891f35ae65b0fd325fbc8f network default NAD default NAD key ] I0428 11:06:47.188553 3656 cni.go:473] [openshift-console/console-64774ffbc8-pnb2m 68d61705b90e78eab5eb93c3a105b264b3bcd5c5d0891f35ae65b0fd325fbc8f network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-64774ffbc8-pnb2m 68d61705b90e78eab5eb93c3a105b264b3bcd5c5d0891f35ae65b0fd325fbc8f network default NAD default NAD key default], result "{}", err I0428 11:06:47.233396 3656 obj_retry.go:498] Detected object openshift-console/console-64774ffbc8-pnb2m of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:06:47.233445 3656 pods.go:176] Deleting pod: openshift-console/console-64774ffbc8-pnb2m I0428 11:06:47.235143 3656 pods.go:236] Attempting to release IPs for pod: openshift-console/console-64774ffbc8-pnb2m, ips: 10.130.0.22 I0428 11:06:47.235243 3656 obj_retry.go:498] Detected object openshift-console/console-64774ffbc8-pnb2m of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:06:47.243571 3656 base_network_controller_pods.go:487] [default/openshift-console/console-845c47766f-qggjh] creating logical port openshift-console_console-845c47766f-qggjh for pod on switch ip-10-0-1-214.ec2.internal I0428 11:06:47.243682 3656 kube.go:256] Updating pod openshift-console/console-845c47766f-qggjh I0428 11:06:47.257256 3656 pod.go:62] [openshift-console/console-845c47766f-qggjh] pod update took 13.602098ms I0428 11:06:47.257283 3656 base_network_controller_pods.go:951] [default/openshift-console/console-845c47766f-qggjh] addLogicalPort annotation time took 13.633555ms I0428 11:06:47.258470 3656 pods.go:271] [openshift-console/console-845c47766f-qggjh] addLogicalPort took 14.913648ms, libovsdb time 873.713µs I0428 11:06:47.602134 3656 cni.go:452] [openshift-console/console-845c47766f-qggjh 77c656e15dd372a7a40bbd6359c330d7fa0986c26a543bb8d5d88f7cb639c9fc network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-845c47766f-qggjh 77c656e15dd372a7a40bbd6359c330d7fa0986c26a543bb8d5d88f7cb639c9fc network default NAD default NAD key ] I0428 11:06:47.611325 3656 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-845c47766f-qggjh, hostIfaceName: 77c656e15dd372a, network: default, NAD default, SandboxID: "77c656e15dd372a7a40bbd6359c330d7fa0986c26a543bb8d5d88f7cb639c9fc", PCI device ID: , UID: "266111fc-1641-40ec-bdd2-add5c2e71ec5", MAC: 0a:58:0a:82:00:19, IPs: [10.130.0.25/23] I0428 11:06:47.663176 3656 cni.go:473] [openshift-console/console-845c47766f-qggjh 77c656e15dd372a7a40bbd6359c330d7fa0986c26a543bb8d5d88f7cb639c9fc network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-845c47766f-qggjh 77c656e15dd372a7a40bbd6359c330d7fa0986c26a543bb8d5d88f7cb639c9fc network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"42:e1:a7:0c:0b:1b\",\"name\":\"77c656e15dd372a\"},{\"mac\":\"0a:58:0a:82:00:19\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0c7205cb-86f9-4270-9b16-050259a7bd34\"}],\"ips\":[{\"address\":\"10.130.0.25/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0428 11:06:47.841615 3656 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:06:47.841644 3656 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:06:47.848619 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-0 I0428 11:06:47.848652 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-0 I0428 11:06:47.864571 3656 obj_retry.go:492] Detected object openshift-console/console-64774ffbc8-pnb2m of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:06:47.864597 3656 obj_retry.go:492] Detected object openshift-console/console-64774ffbc8-pnb2m of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:06:47.871965 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-64774ffbc8-pnb2m I0428 11:06:47.872046 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-64774ffbc8-pnb2m I0428 11:06:47.891824 3656 base_network_controller_pods.go:487] [default/openshift-monitoring/alertmanager-main-0] creating logical port openshift-monitoring_alertmanager-main-0 for pod on switch ip-10-0-1-214.ec2.internal I0428 11:06:47.892042 3656 kube.go:256] Updating pod openshift-monitoring/alertmanager-main-0 I0428 11:06:47.912773 3656 pod.go:62] [openshift-monitoring/alertmanager-main-0] pod update took 20.769404ms I0428 11:06:47.912797 3656 base_network_controller_pods.go:951] [default/openshift-monitoring/alertmanager-main-0] addLogicalPort annotation time took 20.799131ms I0428 11:06:47.914104 3656 pods.go:271] [openshift-monitoring/alertmanager-main-0] addLogicalPort took 22.303626ms, libovsdb time 945.208µs I0428 11:06:48.254835 3656 cni.go:452] [openshift-monitoring/alertmanager-main-0 4447ca4e855f33275a888c5c8ab9522f630bf801b4f3615aa4229fb5d62989cf network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/alertmanager-main-0 4447ca4e855f33275a888c5c8ab9522f630bf801b4f3615aa4229fb5d62989cf network default NAD default NAD key ] I0428 11:06:48.264473 3656 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: alertmanager-main-0, hostIfaceName: 4447ca4e855f332, network: default, NAD default, SandboxID: "4447ca4e855f33275a888c5c8ab9522f630bf801b4f3615aa4229fb5d62989cf", PCI device ID: , UID: "843dcf13-8395-4a32-b653-263275d0a481", MAC: 0a:58:0a:82:00:1a, IPs: [10.130.0.26/23] I0428 11:06:48.316858 3656 cni.go:473] [openshift-monitoring/alertmanager-main-0 4447ca4e855f33275a888c5c8ab9522f630bf801b4f3615aa4229fb5d62989cf network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/alertmanager-main-0 4447ca4e855f33275a888c5c8ab9522f630bf801b4f3615aa4229fb5d62989cf network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"9e:ee:ca:85:33:73\",\"name\":\"4447ca4e855f332\"},{\"mac\":\"0a:58:0a:82:00:1a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/806a7e09-8c32-47ee-b2f6-510871867500\"}],\"ips\":[{\"address\":\"10.130.0.26/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0428 11:06:57.888689 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-214.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0428 11:06:58.167687 3656 obj_retry.go:498] Detected object openshift-console/console-64774ffbc8-ztldw of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:06:58.168265 3656 obj_retry.go:498] Detected object openshift-console/console-64774ffbc8-ztldw of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:06:58.378636 3656 obj_retry.go:492] Detected object openshift-console/console-64774ffbc8-ztldw of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:06:58.378660 3656 obj_retry.go:492] Detected object openshift-console/console-64774ffbc8-ztldw of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:06:58.386236 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-64774ffbc8-ztldw I0428 11:06:58.386271 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-64774ffbc8-ztldw W0428 11:07:08.308602 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-214.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0428 11:07:08.308622 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-214.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0428 11:07:08.308628 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-214.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0428 11:07:11.542180 3656 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace W0428 11:07:21.015364 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-214.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0428 11:07:21.015389 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-214.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0428 11:07:22.324632 3656 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:07:22.325304 3656 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:07:22.478214 3656 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:07:22.478246 3656 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:07:22.486189 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-0 I0428 11:07:22.486211 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-0 I0428 11:08:28.052783 3656 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=43 I0428 11:08:36.050386 3656 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=140 I0428 11:09:19.382131 3656 cni.go:452] [openshift-console/console-845c47766f-qggjh 77c656e15dd372a7a40bbd6359c330d7fa0986c26a543bb8d5d88f7cb639c9fc network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-845c47766f-qggjh 77c656e15dd372a7a40bbd6359c330d7fa0986c26a543bb8d5d88f7cb639c9fc network default NAD default NAD key ] I0428 11:09:19.463384 3656 cni.go:473] [openshift-console/console-845c47766f-qggjh 77c656e15dd372a7a40bbd6359c330d7fa0986c26a543bb8d5d88f7cb639c9fc network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-845c47766f-qggjh 77c656e15dd372a7a40bbd6359c330d7fa0986c26a543bb8d5d88f7cb639c9fc network default NAD default NAD key default], result "{}", err I0428 11:09:19.515103 3656 obj_retry.go:498] Detected object openshift-console/console-845c47766f-qggjh of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:09:19.515151 3656 pods.go:176] Deleting pod: openshift-console/console-845c47766f-qggjh I0428 11:09:19.516555 3656 pods.go:236] Attempting to release IPs for pod: openshift-console/console-845c47766f-qggjh, ips: 10.130.0.25 I0428 11:09:19.516587 3656 obj_retry.go:498] Detected object openshift-console/console-845c47766f-qggjh of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:09:19.524440 3656 base_network_controller_pods.go:487] [default/openshift-console/console-55c9744697-lgk2c] creating logical port openshift-console_console-55c9744697-lgk2c for pod on switch ip-10-0-1-214.ec2.internal I0428 11:09:19.524566 3656 kube.go:256] Updating pod openshift-console/console-55c9744697-lgk2c I0428 11:09:19.540876 3656 pod.go:62] [openshift-console/console-55c9744697-lgk2c] pod update took 16.336054ms I0428 11:09:19.540897 3656 base_network_controller_pods.go:951] [default/openshift-console/console-55c9744697-lgk2c] addLogicalPort annotation time took 16.362239ms I0428 11:09:19.542232 3656 pods.go:271] [openshift-console/console-55c9744697-lgk2c] addLogicalPort took 17.804705ms, libovsdb time 1.008917ms I0428 11:09:19.887973 3656 cni.go:452] [openshift-console/console-55c9744697-lgk2c 11e4c664c92dc748274931729bc515fb76541aabaf7cd8702ff2f739bcf01db7 network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-55c9744697-lgk2c 11e4c664c92dc748274931729bc515fb76541aabaf7cd8702ff2f739bcf01db7 network default NAD default NAD key ] I0428 11:09:19.898350 3656 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-55c9744697-lgk2c, hostIfaceName: 11e4c664c92dc74, network: default, NAD default, SandboxID: "11e4c664c92dc748274931729bc515fb76541aabaf7cd8702ff2f739bcf01db7", PCI device ID: , UID: "1758da41-dbb9-4445-86e0-712ff802b24f", MAC: 0a:58:0a:82:00:1b, IPs: [10.130.0.27/23] I0428 11:09:19.948217 3656 cni.go:473] [openshift-console/console-55c9744697-lgk2c 11e4c664c92dc748274931729bc515fb76541aabaf7cd8702ff2f739bcf01db7 network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-55c9744697-lgk2c 11e4c664c92dc748274931729bc515fb76541aabaf7cd8702ff2f739bcf01db7 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"b2:e7:77:1c:1c:84\",\"name\":\"11e4c664c92dc74\"},{\"mac\":\"0a:58:0a:82:00:1b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0c060206-d642-49b4-8f14-428c63652674\"}],\"ips\":[{\"address\":\"10.130.0.27/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0428 11:09:20.480933 3656 obj_retry.go:492] Detected object openshift-console/console-845c47766f-qggjh of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:09:20.480983 3656 obj_retry.go:492] Detected object openshift-console/console-845c47766f-qggjh of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:09:20.489238 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-845c47766f-qggjh I0428 11:09:20.489271 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-845c47766f-qggjh I0428 11:09:30.213023 3656 obj_retry.go:498] Detected object openshift-console/console-845c47766f-tz7h4 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:09:30.213635 3656 obj_retry.go:498] Detected object openshift-console/console-845c47766f-tz7h4 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0428 11:09:30.513718 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-214.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0428 11:09:30.524051 3656 obj_retry.go:492] Detected object openshift-console/console-845c47766f-tz7h4 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:09:30.524071 3656 obj_retry.go:492] Detected object openshift-console/console-845c47766f-tz7h4 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:09:30.530349 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-845c47766f-tz7h4 I0428 11:09:30.530376 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-845c47766f-tz7h4 I0428 11:09:32.179643 3656 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=8 I0428 11:09:46.050999 3656 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=71 I0428 11:10:03.787772 3656 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29622910-7w5sx of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:10:03.788355 3656 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29622910-7w5sx of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:10:04.675987 3656 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29622910-7w5sx of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:10:04.676013 3656 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29622910-7w5sx of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:10:25.060419 3656 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=616 I0428 11:10:34.141266 3656 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=8 I0428 11:10:36.161088 3656 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=8 I0428 11:10:38.140638 3656 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=8 I0428 11:10:51.145675 3656 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=9 I0428 11:11:00.150913 3656 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=8 W0428 11:11:25.975160 3656 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0428 11:11:28.847776 3656 namespace.go:98] [tsf] adding namespace I0428 11:11:28.848706 3656 namespace.go:102] [tsf] adding namespace took 900.581µs I0428 11:11:28.862776 3656 namespace.go:142] [tsf] updating namespace I0428 11:11:28.878075 3656 namespace.go:142] [tsf] updating namespace I0428 11:11:32.753591 3656 namespace.go:98] [konflux-ui] adding namespace I0428 11:11:32.753720 3656 namespace.go:98] [cert-manager-operator] adding namespace I0428 11:11:32.753756 3656 namespace.go:98] [tsf-tas] adding namespace I0428 11:11:32.754022 3656 namespace.go:98] [rhbk-operator] adding namespace I0428 11:11:32.754017 3656 namespace.go:98] [tsf-tpa] adding namespace I0428 11:11:32.754158 3656 namespace.go:98] [konflux-operator] adding namespace I0428 11:11:32.754514 3656 namespace.go:102] [konflux-ui] adding namespace took 902.916µs I0428 11:11:32.754546 3656 namespace.go:98] [tsf-keycloak] adding namespace I0428 11:11:32.755040 3656 namespace.go:102] [tsf-keycloak] adding namespace took 486.083µs I0428 11:11:32.755552 3656 namespace.go:102] [cert-manager-operator] adding namespace took 1.812564ms I0428 11:11:32.756033 3656 namespace.go:102] [tsf-tas] adding namespace took 2.259573ms I0428 11:11:32.756515 3656 namespace.go:102] [rhbk-operator] adding namespace took 2.478753ms I0428 11:11:32.757059 3656 namespace.go:102] [tsf-tpa] adding namespace took 3.01893ms I0428 11:11:32.757551 3656 namespace.go:102] [konflux-operator] adding namespace took 3.378765ms I0428 11:11:32.765652 3656 namespace.go:142] [konflux-ui] updating namespace I0428 11:11:32.765728 3656 namespace.go:98] [rhtpa-operator] adding namespace I0428 11:11:32.766292 3656 namespace.go:102] [rhtpa-operator] adding namespace took 551.094µs I0428 11:11:32.788523 3656 namespace.go:142] [konflux-ui] updating namespace I0428 11:11:32.788568 3656 namespace.go:142] [cert-manager-operator] updating namespace I0428 11:11:32.796352 3656 namespace.go:142] [tsf-tpa] updating namespace I0428 11:11:32.806621 3656 namespace.go:142] [tsf-keycloak] updating namespace I0428 11:11:32.806639 3656 namespace.go:142] [tsf-tas] updating namespace I0428 11:11:32.806793 3656 namespace.go:142] [cert-manager-operator] updating namespace I0428 11:11:32.825931 3656 namespace.go:142] [rhbk-operator] updating namespace I0428 11:11:32.826009 3656 namespace.go:142] [konflux-operator] updating namespace I0428 11:11:32.826099 3656 namespace.go:142] [rhtpa-operator] updating namespace I0428 11:11:32.839514 3656 namespace.go:142] [cert-manager-operator] updating namespace I0428 11:11:32.849767 3656 namespace.go:142] [konflux-ui] updating namespace I0428 11:11:32.860595 3656 namespace.go:142] [tsf-tpa] updating namespace I0428 11:11:32.872660 3656 namespace.go:142] [rhbk-operator] updating namespace I0428 11:11:32.872710 3656 namespace.go:142] [tsf-tpa] updating namespace I0428 11:11:32.882987 3656 namespace.go:142] [konflux-operator] updating namespace I0428 11:11:32.886616 3656 namespace.go:142] [rhbk-operator] updating namespace I0428 11:11:32.904560 3656 namespace.go:142] [konflux-operator] updating namespace I0428 11:11:32.904696 3656 namespace.go:142] [rhtpa-operator] updating namespace I0428 11:11:32.929693 3656 namespace.go:142] [rhtpa-operator] updating namespace I0428 11:11:32.964479 3656 namespace.go:142] [tsf-tas] updating namespace I0428 11:11:32.972985 3656 namespace.go:142] [tsf-tas] updating namespace I0428 11:11:33.158331 3656 namespace.go:142] [tsf-keycloak] updating namespace I0428 11:11:33.167446 3656 namespace.go:142] [tsf-keycloak] updating namespace I0428 11:11:33.880910 3656 namespace.go:142] [tsf-tpa] updating namespace I0428 11:11:33.885535 3656 namespace.go:142] [tsf-keycloak] updating namespace I0428 11:11:37.030216 3656 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... W0428 11:11:37.030277 3656 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-deployment-validation-operator/deployment-validation-operator-65d68cbd6c-6qndl IP address from the namespace address-set, err: pod openshift-deployment-validation-operator/deployment-validation-operator-65d68cbd6c-6qndl: no pod IPs found I0428 11:11:40.191115 3656 obj_retry.go:498] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa0158pvc4q of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:11:40.191693 3656 obj_retry.go:498] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa0158pvc4q of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:11:40.191722 3656 obj_retry.go:498] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa0158pvc4q of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 11:11:40.191732 3656 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0428 11:11:40.191751 3656 obj_retry.go:498] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa0158pvc4q of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 11:11:40.191759 3656 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0428 11:11:41.167551 3656 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=10 I0428 11:11:41.181319 3656 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19djxp4q of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:11:41.181876 3656 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19djxp4q of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:11:41.181904 3656 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19djxp4q of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 11:11:41.181912 3656 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0428 11:11:41.181937 3656 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19djxp4q of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 11:11:41.181942 3656 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0428 11:11:41.215985 3656 obj_retry.go:492] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa0158pvc4q of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:11:41.216013 3656 obj_retry.go:492] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa0158pvc4q of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:11:41.216025 3656 obj_retry.go:492] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa0158pvc4q of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:11:41.216043 3656 obj_retry.go:492] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa0158pvc4q of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:11:42.209238 3656 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19djxp4q of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:11:42.209265 3656 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19djxp4q of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:11:42.209276 3656 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19djxp4q of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:11:42.209293 3656 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19djxp4q of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:11:44.225361 3656 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf26tmnq of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:11:44.225941 3656 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf26tmnq of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:11:44.225973 3656 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf26tmnq of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 11:11:44.225981 3656 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0428 11:11:44.225992 3656 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf26tmnq of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 11:11:44.225998 3656 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0428 11:11:44.246511 3656 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51gwtzk of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:11:44.246940 3656 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51gwtzk of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:11:44.246958 3656 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51gwtzk of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 11:11:44.246963 3656 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0428 11:11:44.246970 3656 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51gwtzk of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 11:11:44.246976 3656 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0428 11:11:45.276206 3656 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf26tmnq of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:11:45.276233 3656 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf26tmnq of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:11:45.276247 3656 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf26tmnq of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:11:45.276259 3656 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf26tmnq of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:11:45.334622 3656 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51gwtzk of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:11:45.334655 3656 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51gwtzk of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:11:45.334669 3656 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51gwtzk of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:11:45.334683 3656 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51gwtzk of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:11:46.233557 3656 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136s9zfw of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:11:46.234117 3656 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136s9zfw of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:11:46.234138 3656 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136s9zfw of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 11:11:46.234146 3656 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0428 11:11:46.234156 3656 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136s9zfw of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 11:11:46.234160 3656 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0428 11:11:47.319828 3656 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dt77d of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:11:47.320350 3656 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dt77d of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:11:47.320375 3656 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dt77d of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 11:11:47.320383 3656 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0428 11:11:47.320399 3656 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dt77d of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 11:11:47.320403 3656 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0428 11:11:47.331778 3656 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136s9zfw of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:11:47.331798 3656 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136s9zfw of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:11:47.331808 3656 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136s9zfw of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:11:47.331817 3656 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136s9zfw of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:11:48.358759 3656 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dt77d of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:11:48.358781 3656 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dt77d of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:11:48.358799 3656 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dt77d of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:11:48.358811 3656 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dt77d of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:11:53.957739 3656 namespace.go:98] [cert-manager] adding namespace I0428 11:11:53.958557 3656 namespace.go:102] [cert-manager] adding namespace took 790.931µs I0428 11:11:54.036506 3656 namespace.go:142] [cert-manager] updating namespace I0428 11:11:54.087539 3656 namespace.go:142] [cert-manager] updating namespace W0428 11:11:58.347394 3656 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service cert-manager/cert-manager-webhook W0428 11:11:58.347420 3656 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service cert-manager/cert-manager-webhook I0428 11:11:58.444322 3656 base_network_controller_pods.go:487] [default/cert-manager/cert-manager-cainjector-5d99449ffd-2lv85] creating logical port cert-manager_cert-manager-cainjector-5d99449ffd-2lv85 for pod on switch ip-10-0-1-214.ec2.internal I0428 11:11:58.444416 3656 kube.go:256] Updating pod cert-manager/cert-manager-cainjector-5d99449ffd-2lv85 I0428 11:11:58.518893 3656 kube.go:256] Updating pod cert-manager/cert-manager-cainjector-5d99449ffd-2lv85 I0428 11:11:58.545001 3656 pod.go:62] [cert-manager/cert-manager-cainjector-5d99449ffd-2lv85] pod update took 100.606936ms I0428 11:11:58.545039 3656 base_network_controller_pods.go:951] [default/cert-manager/cert-manager-cainjector-5d99449ffd-2lv85] addLogicalPort annotation time took 100.649121ms I0428 11:11:58.546364 3656 pods.go:271] [cert-manager/cert-manager-cainjector-5d99449ffd-2lv85] addLogicalPort took 102.062336ms, libovsdb time 901.613µs I0428 11:11:59.109039 3656 cni.go:452] [cert-manager/cert-manager-cainjector-5d99449ffd-2lv85 8b519f8662b9b2e346e3f8d87093e69f020f4e36c5eed5261cc8307267440357 network default NAD default NAD key ] ADD starting CNI request [cert-manager/cert-manager-cainjector-5d99449ffd-2lv85 8b519f8662b9b2e346e3f8d87093e69f020f4e36c5eed5261cc8307267440357 network default NAD default NAD key ] I0428 11:11:59.118059 3656 helper_linux.go:556] ConfigureOVS: namespace: cert-manager, podName: cert-manager-cainjector-5d99449ffd-2lv85, hostIfaceName: 8b519f8662b9b2e, network: default, NAD default, SandboxID: "8b519f8662b9b2e346e3f8d87093e69f020f4e36c5eed5261cc8307267440357", PCI device ID: , UID: "3f13e83c-b1b9-483b-84a9-2e8244ffc3a7", MAC: 0a:58:0a:82:00:1c, IPs: [10.130.0.28/23] I0428 11:11:59.167961 3656 cni.go:473] [cert-manager/cert-manager-cainjector-5d99449ffd-2lv85 8b519f8662b9b2e346e3f8d87093e69f020f4e36c5eed5261cc8307267440357 network default NAD default NAD key default] ADD finished CNI request [cert-manager/cert-manager-cainjector-5d99449ffd-2lv85 8b519f8662b9b2e346e3f8d87093e69f020f4e36c5eed5261cc8307267440357 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"f2:b8:54:1c:b5:49\",\"name\":\"8b519f8662b9b2e\"},{\"mac\":\"0a:58:0a:82:00:1c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9db25a93-13b4-4b79-9c2f-d2077b403e33\"}],\"ips\":[{\"address\":\"10.130.0.28/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0428 11:12:01.301423 3656 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0428 11:12:01.444687 3656 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0428 11:12:01.851453 3656 namespace.go:142] [openshift-operators] updating namespace I0428 11:12:01.869270 3656 namespace.go:142] [openshift-operators] updating namespace I0428 11:12:04.896542 3656 base_network_controller_pods.go:487] [default/cert-manager/cert-manager-59c77684cc-4wg7k] creating logical port cert-manager_cert-manager-59c77684cc-4wg7k for pod on switch ip-10-0-1-214.ec2.internal I0428 11:12:04.896640 3656 kube.go:256] Updating pod cert-manager/cert-manager-59c77684cc-4wg7k I0428 11:12:04.931675 3656 kube.go:256] Updating pod cert-manager/cert-manager-59c77684cc-4wg7k I0428 11:12:04.950836 3656 pod.go:62] [cert-manager/cert-manager-59c77684cc-4wg7k] pod update took 54.219085ms I0428 11:12:04.950863 3656 base_network_controller_pods.go:951] [default/cert-manager/cert-manager-59c77684cc-4wg7k] addLogicalPort annotation time took 54.250733ms I0428 11:12:04.952182 3656 pods.go:271] [cert-manager/cert-manager-59c77684cc-4wg7k] addLogicalPort took 55.658431ms, libovsdb time 970.232µs I0428 11:12:05.237433 3656 cni.go:452] [cert-manager/cert-manager-59c77684cc-4wg7k b99aa6f23be171c4cea310a5330977cdb50939a78211c5bb330697af5b97b5c7 network default NAD default NAD key ] ADD starting CNI request [cert-manager/cert-manager-59c77684cc-4wg7k b99aa6f23be171c4cea310a5330977cdb50939a78211c5bb330697af5b97b5c7 network default NAD default NAD key ] I0428 11:12:05.246751 3656 helper_linux.go:556] ConfigureOVS: namespace: cert-manager, podName: cert-manager-59c77684cc-4wg7k, hostIfaceName: b99aa6f23be171c, network: default, NAD default, SandboxID: "b99aa6f23be171c4cea310a5330977cdb50939a78211c5bb330697af5b97b5c7", PCI device ID: , UID: "35334c98-b26c-46ea-9df7-bb2bb7924f26", MAC: 0a:58:0a:82:00:1d, IPs: [10.130.0.29/23] I0428 11:12:05.297543 3656 cni.go:473] [cert-manager/cert-manager-59c77684cc-4wg7k b99aa6f23be171c4cea310a5330977cdb50939a78211c5bb330697af5b97b5c7 network default NAD default NAD key default] ADD finished CNI request [cert-manager/cert-manager-59c77684cc-4wg7k b99aa6f23be171c4cea310a5330977cdb50939a78211c5bb330697af5b97b5c7 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"3e:33:a4:77:8e:9a\",\"name\":\"b99aa6f23be171c\"},{\"mac\":\"0a:58:0a:82:00:1d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2745d228-dc77-4ede-85d6-ed0141902360\"}],\"ips\":[{\"address\":\"10.130.0.29/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0428 11:12:08.171933 3656 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=9 W0428 11:12:09.400563 3656 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0428 11:12:09.410747 3656 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0428 11:12:09.438163 3656 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0428 11:12:09.450414 3656 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0428 11:12:10.015168 3656 namespace.go:142] [openshift-operators] updating namespace I0428 11:12:10.549865 3656 base_network_controller_pods.go:487] [default/openshift-operators/openshift-pipelines-operator-6bdf6c8794-9m9w6] creating logical port openshift-operators_openshift-pipelines-operator-6bdf6c8794-9m9w6 for pod on switch ip-10-0-1-214.ec2.internal I0428 11:12:10.550049 3656 kube.go:256] Updating pod openshift-operators/openshift-pipelines-operator-6bdf6c8794-9m9w6 I0428 11:12:10.603803 3656 base_network_controller_pods.go:487] [default/openshift-operators/tekton-operator-webhook-5864bd5898-b5mrb] creating logical port openshift-operators_tekton-operator-webhook-5864bd5898-b5mrb for pod on switch ip-10-0-1-214.ec2.internal I0428 11:12:10.603942 3656 kube.go:256] Updating pod openshift-operators/tekton-operator-webhook-5864bd5898-b5mrb I0428 11:12:10.642584 3656 kube.go:256] Updating pod openshift-operators/openshift-pipelines-operator-6bdf6c8794-9m9w6 I0428 11:12:10.747433 3656 kube.go:256] Updating pod openshift-operators/tekton-operator-webhook-5864bd5898-b5mrb I0428 11:12:10.835618 3656 pod.go:62] [openshift-operators/openshift-pipelines-operator-6bdf6c8794-9m9w6] pod update took 285.594284ms I0428 11:12:10.835643 3656 base_network_controller_pods.go:951] [default/openshift-operators/openshift-pipelines-operator-6bdf6c8794-9m9w6] addLogicalPort annotation time took 285.625445ms I0428 11:12:10.836873 3656 pods.go:271] [openshift-operators/openshift-pipelines-operator-6bdf6c8794-9m9w6] addLogicalPort took 287.029054ms, libovsdb time 859.167µs I0428 11:12:10.904723 3656 cni.go:452] [openshift-operators/openshift-pipelines-operator-6bdf6c8794-9m9w6 187f403fa7b8dccb985096dc08d29e4d51655fe9fb5f9695b62ab2e4d0c30a0b network default NAD default NAD key ] ADD starting CNI request [openshift-operators/openshift-pipelines-operator-6bdf6c8794-9m9w6 187f403fa7b8dccb985096dc08d29e4d51655fe9fb5f9695b62ab2e4d0c30a0b network default NAD default NAD key ] I0428 11:12:10.914435 3656 helper_linux.go:556] ConfigureOVS: namespace: openshift-operators, podName: openshift-pipelines-operator-6bdf6c8794-9m9w6, hostIfaceName: 187f403fa7b8dcc, network: default, NAD default, SandboxID: "187f403fa7b8dccb985096dc08d29e4d51655fe9fb5f9695b62ab2e4d0c30a0b", PCI device ID: , UID: "5860f455-d55a-484e-ac24-01e57856df88", MAC: 0a:58:0a:82:00:1e, IPs: [10.130.0.30/23] I0428 11:12:10.950268 3656 pod.go:62] [openshift-operators/tekton-operator-webhook-5864bd5898-b5mrb] pod update took 346.367013ms I0428 11:12:10.950290 3656 base_network_controller_pods.go:951] [default/openshift-operators/tekton-operator-webhook-5864bd5898-b5mrb] addLogicalPort annotation time took 346.392918ms I0428 11:12:10.951444 3656 pods.go:271] [openshift-operators/tekton-operator-webhook-5864bd5898-b5mrb] addLogicalPort took 347.656486ms, libovsdb time 842.131µs I0428 11:12:10.960423 3656 cni.go:452] [openshift-operators/tekton-operator-webhook-5864bd5898-b5mrb fd4074640ce5b847d45b6a91f14e739fc73b605dff409c03e959a3dad34f888c network default NAD default NAD key ] ADD starting CNI request [openshift-operators/tekton-operator-webhook-5864bd5898-b5mrb fd4074640ce5b847d45b6a91f14e739fc73b605dff409c03e959a3dad34f888c network default NAD default NAD key ] I0428 11:12:10.969325 3656 cni.go:473] [openshift-operators/openshift-pipelines-operator-6bdf6c8794-9m9w6 187f403fa7b8dccb985096dc08d29e4d51655fe9fb5f9695b62ab2e4d0c30a0b network default NAD default NAD key default] ADD finished CNI request [openshift-operators/openshift-pipelines-operator-6bdf6c8794-9m9w6 187f403fa7b8dccb985096dc08d29e4d51655fe9fb5f9695b62ab2e4d0c30a0b network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"82:2e:16:49:1c:00\",\"name\":\"187f403fa7b8dcc\"},{\"mac\":\"0a:58:0a:82:00:1e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/09621fa5-6f66-4f9c-8f3d-0dd33670e118\"}],\"ips\":[{\"address\":\"10.130.0.30/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0428 11:12:10.971147 3656 helper_linux.go:556] ConfigureOVS: namespace: openshift-operators, podName: tekton-operator-webhook-5864bd5898-b5mrb, hostIfaceName: fd4074640ce5b84, network: default, NAD default, SandboxID: "fd4074640ce5b847d45b6a91f14e739fc73b605dff409c03e959a3dad34f888c", PCI device ID: , UID: "4c9b56af-eb4c-476d-a3dc-e79038bed204", MAC: 0a:58:0a:82:00:1f, IPs: [10.130.0.31/23] I0428 11:12:11.022889 3656 cni.go:473] [openshift-operators/tekton-operator-webhook-5864bd5898-b5mrb fd4074640ce5b847d45b6a91f14e739fc73b605dff409c03e959a3dad34f888c network default NAD default NAD key default] ADD finished CNI request [openshift-operators/tekton-operator-webhook-5864bd5898-b5mrb fd4074640ce5b847d45b6a91f14e739fc73b605dff409c03e959a3dad34f888c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"72:62:42:30:43:f8\",\"name\":\"fd4074640ce5b84\"},{\"mac\":\"0a:58:0a:82:00:1f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/51087365-dc61-4429-b589-04795d5e482a\"}],\"ips\":[{\"address\":\"10.130.0.31/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0428 11:12:19.071343 3656 namespace.go:98] [trusted-artifact-signer] adding namespace I0428 11:12:19.072268 3656 namespace.go:102] [trusted-artifact-signer] adding namespace took 901.732µs I0428 11:12:19.158318 3656 namespace.go:142] [trusted-artifact-signer] updating namespace I0428 11:12:19.203609 3656 namespace.go:142] [trusted-artifact-signer] updating namespace W0428 11:12:19.314959 3656 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0428 11:12:25.705869 3656 namespace.go:98] [openshift-pipelines] adding namespace I0428 11:12:25.706912 3656 namespace.go:102] [openshift-pipelines] adding namespace took 1.012333ms I0428 11:12:25.736650 3656 namespace.go:142] [openshift-pipelines] updating namespace I0428 11:12:26.409207 3656 namespace.go:142] [cert-manager] updating namespace I0428 11:12:26.592211 3656 namespace.go:142] [cert-manager-operator] updating namespace I0428 11:12:26.882254 3656 obj_retry.go:498] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:12:26.882814 3656 obj_retry.go:498] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:12:26.908083 3656 obj_retry.go:492] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:12:26.908108 3656 obj_retry.go:492] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:12:26.924328 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-subscriptions I0428 11:12:26.924356 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-subscriptions I0428 11:12:27.004558 3656 namespace.go:142] [konflux-operator] updating namespace I0428 11:12:27.206540 3656 namespace.go:142] [konflux-ui] updating namespace I0428 11:12:27.422489 3656 namespace.go:142] [open-cluster-management-2pv452ledagn57tf47lfif0ho4ffo09v] updating namespace I0428 11:12:27.828678 3656 namespace.go:142] [rhbk-operator] updating namespace I0428 11:12:28.038439 3656 namespace.go:142] [rhtpa-operator] updating namespace I0428 11:12:28.231967 3656 namespace.go:142] [trusted-artifact-signer] updating namespace I0428 11:12:28.434762 3656 namespace.go:142] [tsf] updating namespace I0428 11:12:28.640145 3656 namespace.go:142] [tsf-keycloak] updating namespace I0428 11:12:28.833240 3656 namespace.go:142] [tsf-tas] updating namespace I0428 11:12:29.037430 3656 namespace.go:142] [tsf-tpa] updating namespace I0428 11:12:29.163518 3656 namespace.go:142] [cert-manager] updating namespace I0428 11:12:29.173827 3656 namespace.go:142] [cert-manager-operator] updating namespace I0428 11:12:29.181051 3656 namespace.go:142] [default] updating namespace I0428 11:12:29.188615 3656 namespace.go:142] [konflux-operator] updating namespace I0428 11:12:29.197093 3656 namespace.go:142] [konflux-ui] updating namespace I0428 11:12:29.205803 3656 namespace.go:142] [open-cluster-management-2pv452ledagn57tf47lfif0ho4ffo09v] updating namespace I0428 11:12:29.212886 3656 namespace.go:142] [openshift] updating namespace I0428 11:12:29.221907 3656 namespace.go:142] [rhbk-operator] updating namespace I0428 11:12:29.230537 3656 namespace.go:142] [rhtpa-operator] updating namespace I0428 11:12:29.238140 3656 namespace.go:142] [trusted-artifact-signer] updating namespace I0428 11:12:29.252978 3656 namespace.go:142] [tsf] updating namespace I0428 11:12:29.276701 3656 namespace.go:142] [tsf-keycloak] updating namespace I0428 11:12:29.293015 3656 namespace.go:142] [tsf-tas] updating namespace I0428 11:12:29.314176 3656 namespace.go:142] [tsf-tpa] updating namespace I0428 11:12:29.414469 3656 namespace.go:142] [cert-manager] updating namespace I0428 11:12:29.522113 3656 namespace.go:142] [cert-manager-operator] updating namespace I0428 11:12:29.615234 3656 namespace.go:142] [default] updating namespace I0428 11:12:29.715112 3656 namespace.go:142] [konflux-operator] updating namespace I0428 11:12:29.821951 3656 namespace.go:142] [konflux-ui] updating namespace I0428 11:12:29.920452 3656 namespace.go:142] [open-cluster-management-2pv452ledagn57tf47lfif0ho4ffo09v] updating namespace I0428 11:12:30.028442 3656 namespace.go:142] [openshift] updating namespace I0428 11:12:30.116196 3656 namespace.go:142] [rhbk-operator] updating namespace I0428 11:12:30.214852 3656 namespace.go:142] [rhtpa-operator] updating namespace I0428 11:12:30.314822 3656 namespace.go:142] [trusted-artifact-signer] updating namespace I0428 11:12:30.412477 3656 namespace.go:142] [tsf] updating namespace I0428 11:12:30.519455 3656 namespace.go:142] [tsf-keycloak] updating namespace I0428 11:12:30.617215 3656 namespace.go:142] [tsf-tas] updating namespace I0428 11:12:30.732445 3656 namespace.go:142] [tsf-tpa] updating namespace I0428 11:12:31.173036 3656 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=10 I0428 11:12:31.355778 3656 namespace.go:142] [openshift-pipelines] updating namespace I0428 11:12:39.107731 3656 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-7s598] creating logical port openshift-pipelines_tekton-pipelines-controller-86bd8d9bfc-7s598 for pod on switch ip-10-0-1-214.ec2.internal I0428 11:12:39.107847 3656 kube.go:256] Updating pod openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-7s598 I0428 11:12:39.123295 3656 pod.go:62] [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-7s598] pod update took 15.482857ms I0428 11:12:39.123320 3656 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-7s598] addLogicalPort annotation time took 15.514062ms I0428 11:12:39.124761 3656 pods.go:271] [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-7s598] addLogicalPort took 17.05575ms, libovsdb time 1.070139ms I0428 11:12:39.475112 3656 cni.go:452] [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-7s598 04b72e62e7bd08035b4f7fd2ac58cfa1517184314d74231a06784f35f2c3434b network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-7s598 04b72e62e7bd08035b4f7fd2ac58cfa1517184314d74231a06784f35f2c3434b network default NAD default NAD key ] I0428 11:12:39.485209 3656 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-controller-86bd8d9bfc-7s598, hostIfaceName: 04b72e62e7bd080, network: default, NAD default, SandboxID: "04b72e62e7bd08035b4f7fd2ac58cfa1517184314d74231a06784f35f2c3434b", PCI device ID: , UID: "761bc4fd-3027-4668-b375-6c2644ab4af5", MAC: 0a:58:0a:82:00:20, IPs: [10.130.0.32/23] I0428 11:12:39.537075 3656 cni.go:473] [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-7s598 04b72e62e7bd08035b4f7fd2ac58cfa1517184314d74231a06784f35f2c3434b network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-7s598 04b72e62e7bd08035b4f7fd2ac58cfa1517184314d74231a06784f35f2c3434b network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"b6:9a:29:f0:58:22\",\"name\":\"04b72e62e7bd080\"},{\"mac\":\"0a:58:0a:82:00:20\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4ae73449-b9de-406e-818d-36eecd30db94\"}],\"ips\":[{\"address\":\"10.130.0.32/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0428 11:12:40.179356 3656 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=10 W0428 11:12:40.799115 3656 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-keycloak/keycloak-pgsql W0428 11:12:40.821041 3656 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-tpa/tpa-pgsql W0428 11:12:43.317312 3656 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0428 11:12:43.317330 3656 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0428 11:12:43.317335 3656 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller I0428 11:12:45.050519 3656 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=11 W0428 11:12:45.456566 3656 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0428 11:12:45.456586 3656 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0428 11:12:45.456590 3656 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0428 11:12:45.456595 3656 util.go:870] Failed to build global endpoints for port TCP/https-webhook: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0428 11:12:45.493509 3656 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0428 11:12:45.493527 3656 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0428 11:12:45.493531 3656 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-events-controller I0428 11:12:53.052349 3656 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=107 W0428 11:12:54.327606 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-214.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0428 11:12:54.327628 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-214.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0428 11:12:54.327633 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-214.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller I0428 11:13:03.577329 3656 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-triggers-controller-785c845664-4mdmm] creating logical port openshift-pipelines_tekton-triggers-controller-785c845664-4mdmm for pod on switch ip-10-0-1-214.ec2.internal I0428 11:13:03.577441 3656 kube.go:256] Updating pod openshift-pipelines/tekton-triggers-controller-785c845664-4mdmm I0428 11:13:03.595213 3656 pod.go:62] [openshift-pipelines/tekton-triggers-controller-785c845664-4mdmm] pod update took 17.805082ms I0428 11:13:03.595238 3656 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-triggers-controller-785c845664-4mdmm] addLogicalPort annotation time took 17.838151ms I0428 11:13:03.596587 3656 pods.go:271] [openshift-pipelines/tekton-triggers-controller-785c845664-4mdmm] addLogicalPort took 19.274219ms, libovsdb time 986.31µs I0428 11:13:03.669521 3656 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-triggers-webhook-6b5b45569f-l9sxv] creating logical port openshift-pipelines_tekton-triggers-webhook-6b5b45569f-l9sxv for pod on switch ip-10-0-1-214.ec2.internal I0428 11:13:03.669610 3656 kube.go:256] Updating pod openshift-pipelines/tekton-triggers-webhook-6b5b45569f-l9sxv I0428 11:13:03.686908 3656 pod.go:62] [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-l9sxv] pod update took 17.318671ms I0428 11:13:03.686946 3656 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-triggers-webhook-6b5b45569f-l9sxv] addLogicalPort annotation time took 17.361216ms I0428 11:13:03.688197 3656 pods.go:271] [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-l9sxv] addLogicalPort took 18.69492ms, libovsdb time 857.585µs I0428 11:13:03.936361 3656 cni.go:452] [openshift-pipelines/tekton-triggers-controller-785c845664-4mdmm 845e46c95e4ebfab057d4228bb04e6e311b153b12b20913bd0bf5ae75087b8a9 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-triggers-controller-785c845664-4mdmm 845e46c95e4ebfab057d4228bb04e6e311b153b12b20913bd0bf5ae75087b8a9 network default NAD default NAD key ] I0428 11:13:03.945546 3656 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-triggers-controller-785c845664-4mdmm, hostIfaceName: 845e46c95e4ebfa, network: default, NAD default, SandboxID: "845e46c95e4ebfab057d4228bb04e6e311b153b12b20913bd0bf5ae75087b8a9", PCI device ID: , UID: "388779b8-152c-47b4-8a48-d8c0d92a9707", MAC: 0a:58:0a:82:00:21, IPs: [10.130.0.33/23] I0428 11:13:03.999898 3656 cni.go:473] [openshift-pipelines/tekton-triggers-controller-785c845664-4mdmm 845e46c95e4ebfab057d4228bb04e6e311b153b12b20913bd0bf5ae75087b8a9 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-triggers-controller-785c845664-4mdmm 845e46c95e4ebfab057d4228bb04e6e311b153b12b20913bd0bf5ae75087b8a9 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"d2:b6:b1:82:bc:10\",\"name\":\"845e46c95e4ebfa\"},{\"mac\":\"0a:58:0a:82:00:21\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1dfb94fa-4773-40ce-a47c-df4f556ffedb\"}],\"ips\":[{\"address\":\"10.130.0.33/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0428 11:13:04.032183 3656 cni.go:452] [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-l9sxv fd577fc0a66f8ddd944f0443eeae8cf0c6906b02add0d79c8acc5ff5209c2138 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-l9sxv fd577fc0a66f8ddd944f0443eeae8cf0c6906b02add0d79c8acc5ff5209c2138 network default NAD default NAD key ] I0428 11:13:04.041983 3656 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-triggers-webhook-6b5b45569f-l9sxv, hostIfaceName: fd577fc0a66f8dd, network: default, NAD default, SandboxID: "fd577fc0a66f8ddd944f0443eeae8cf0c6906b02add0d79c8acc5ff5209c2138", PCI device ID: , UID: "b555719f-fabb-418d-802a-168a56539e15", MAC: 0a:58:0a:82:00:22, IPs: [10.130.0.34/23] I0428 11:13:04.094288 3656 cni.go:473] [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-l9sxv fd577fc0a66f8ddd944f0443eeae8cf0c6906b02add0d79c8acc5ff5209c2138 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-l9sxv fd577fc0a66f8ddd944f0443eeae8cf0c6906b02add0d79c8acc5ff5209c2138 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"3a:ce:1c:73:b3:f5\",\"name\":\"fd577fc0a66f8dd\"},{\"mac\":\"0a:58:0a:82:00:22\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/34f003c2-5e41-4b07-96c5-8dc00b7d34c8\"}],\"ips\":[{\"address\":\"10.130.0.34/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0428 11:13:06.944971 3656 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors I0428 11:13:08.079142 3656 obj_retry.go:498] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:13:08.079699 3656 obj_retry.go:498] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:13:08.095676 3656 obj_retry.go:492] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:13:08.095697 3656 obj_retry.go:492] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:13:08.106640 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-infrastructure I0428 11:13:08.106666 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-infrastructure I0428 11:13:10.083847 3656 base_network_controller_policy.go:1179] Adding network policy tsf-keycloak/keycloak-network-policy for network default I0428 11:13:10.084648 3656 pod_selector_address_set.go:213] Created shared address set for pod selector tsf-keycloak_LS{ML:{app: keycloak,app.kubernetes.io/instance: keycloak,app.kubernetes.io/managed-by: keycloak-operator,},} I0428 11:13:10.084672 3656 base_network_controller_policy.go:1054] Policy tsf-keycloak/keycloak-network-policy added to peer address sets [tsf-keycloak_LS{ML:{app: keycloak,app.kubernetes.io/instance: keycloak,app.kubernetes.io/managed-by: keycloak-operator,},}] I0428 11:13:10.086816 3656 base_network_controller_policy.go:1232] Create network policy tsf-keycloak/keycloak-network-policy resources completed, update namespace loglevel W0428 11:13:20.646143 3656 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0428 11:13:20.646162 3656 util.go:870] Failed to build global endpoints for port TCP/management: empty IP address endpoints for service tsf-keycloak/keycloak-service W0428 11:13:20.649171 3656 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0428 11:13:20.649188 3656 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak I0428 11:13:23.741034 3656 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0428 11:13:26.814742 3656 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-postgres-0] creating logical port openshift-pipelines_tekton-results-postgres-0 for pod on switch ip-10-0-1-214.ec2.internal I0428 11:13:26.814856 3656 kube.go:256] Updating pod openshift-pipelines/tekton-results-postgres-0 I0428 11:13:26.827114 3656 pod.go:62] [openshift-pipelines/tekton-results-postgres-0] pod update took 12.286903ms I0428 11:13:26.827142 3656 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-results-postgres-0] addLogicalPort annotation time took 12.319598ms I0428 11:13:26.828527 3656 pods.go:271] [openshift-pipelines/tekton-results-postgres-0] addLogicalPort took 13.79514ms, libovsdb time 960.439µs I0428 11:13:31.074793 3656 cni.go:452] [openshift-pipelines/tekton-results-postgres-0 70d0d294be2d805436de1d5e6564b931fde18418b728ab0d026defec0dca3f64 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-results-postgres-0 70d0d294be2d805436de1d5e6564b931fde18418b728ab0d026defec0dca3f64 network default NAD default NAD key ] I0428 11:13:31.084569 3656 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-postgres-0, hostIfaceName: 70d0d294be2d805, network: default, NAD default, SandboxID: "70d0d294be2d805436de1d5e6564b931fde18418b728ab0d026defec0dca3f64", PCI device ID: , UID: "5ed3fe6b-cf60-46de-95ad-139a54e8c316", MAC: 0a:58:0a:82:00:23, IPs: [10.130.0.35/23] I0428 11:13:31.137782 3656 cni.go:473] [openshift-pipelines/tekton-results-postgres-0 70d0d294be2d805436de1d5e6564b931fde18418b728ab0d026defec0dca3f64 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-results-postgres-0 70d0d294be2d805436de1d5e6564b931fde18418b728ab0d026defec0dca3f64 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"12:4b:21:8d:37:06\",\"name\":\"70d0d294be2d805\"},{\"mac\":\"0a:58:0a:82:00:23\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/cc2c4fed-b023-4667-89a8-6010f56a6093\"}],\"ips\":[{\"address\":\"10.130.0.35/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0428 11:13:36.624940 3656 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0428 11:13:38.160715 3656 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-watcher-7777c5dbd5-x2tnb] creating logical port openshift-pipelines_tekton-results-watcher-7777c5dbd5-x2tnb for pod on switch ip-10-0-1-214.ec2.internal I0428 11:13:38.160820 3656 kube.go:256] Updating pod openshift-pipelines/tekton-results-watcher-7777c5dbd5-x2tnb I0428 11:13:38.177507 3656 pod.go:62] [openshift-pipelines/tekton-results-watcher-7777c5dbd5-x2tnb] pod update took 16.715023ms I0428 11:13:38.177530 3656 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-results-watcher-7777c5dbd5-x2tnb] addLogicalPort annotation time took 16.745362ms I0428 11:13:38.178785 3656 pods.go:271] [openshift-pipelines/tekton-results-watcher-7777c5dbd5-x2tnb] addLogicalPort took 18.090388ms, libovsdb time 885.404µs I0428 11:13:38.524752 3656 cni.go:452] [openshift-pipelines/tekton-results-watcher-7777c5dbd5-x2tnb 000d854ada24b86dc14e98346aea1d419fc9bf2e4cea7511db87a914a5dfd68b network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-results-watcher-7777c5dbd5-x2tnb 000d854ada24b86dc14e98346aea1d419fc9bf2e4cea7511db87a914a5dfd68b network default NAD default NAD key ] I0428 11:13:38.534288 3656 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-watcher-7777c5dbd5-x2tnb, hostIfaceName: 000d854ada24b86, network: default, NAD default, SandboxID: "000d854ada24b86dc14e98346aea1d419fc9bf2e4cea7511db87a914a5dfd68b", PCI device ID: , UID: "d8b7202a-73ae-4f3e-ae06-ac7eb13a5875", MAC: 0a:58:0a:82:00:24, IPs: [10.130.0.36/23] I0428 11:13:38.587860 3656 cni.go:473] [openshift-pipelines/tekton-results-watcher-7777c5dbd5-x2tnb 000d854ada24b86dc14e98346aea1d419fc9bf2e4cea7511db87a914a5dfd68b network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-results-watcher-7777c5dbd5-x2tnb 000d854ada24b86dc14e98346aea1d419fc9bf2e4cea7511db87a914a5dfd68b network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"76:80:aa:ea:a0:65\",\"name\":\"000d854ada24b86\"},{\"mac\":\"0a:58:0a:82:00:24\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a497a477-3bdf-4599-8c4d-2a91ad5d4bb9\"}],\"ips\":[{\"address\":\"10.130.0.36/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0428 11:13:43.122821 3656 util.go:870] Failed to build global endpoints for port TCP/server: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0428 11:13:43.122838 3656 util.go:870] Failed to build global endpoints for port TCP/prometheus: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0428 11:13:43.122843 3656 util.go:870] Failed to build global endpoints for port TCP/profiling: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service I0428 11:13:55.168450 3656 base_network_controller_pods.go:487] [default/openshift-pipelines/tkn-cli-serve-784b566968-5vqmh] creating logical port openshift-pipelines_tkn-cli-serve-784b566968-5vqmh for pod on switch ip-10-0-1-214.ec2.internal I0428 11:13:55.168547 3656 kube.go:256] Updating pod openshift-pipelines/tkn-cli-serve-784b566968-5vqmh I0428 11:13:55.182029 3656 pod.go:62] [openshift-pipelines/tkn-cli-serve-784b566968-5vqmh] pod update took 13.501852ms I0428 11:13:55.182054 3656 base_network_controller_pods.go:951] [default/openshift-pipelines/tkn-cli-serve-784b566968-5vqmh] addLogicalPort annotation time took 13.532784ms I0428 11:13:55.183347 3656 pods.go:271] [openshift-pipelines/tkn-cli-serve-784b566968-5vqmh] addLogicalPort took 14.913057ms, libovsdb time 903.157µs I0428 11:13:55.526779 3656 cni.go:452] [openshift-pipelines/tkn-cli-serve-784b566968-5vqmh 874deadff23d6c3142f910f2c6ede08f56b9e1127ad7e176dce0b69ec3acc9cc network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tkn-cli-serve-784b566968-5vqmh 874deadff23d6c3142f910f2c6ede08f56b9e1127ad7e176dce0b69ec3acc9cc network default NAD default NAD key ] I0428 11:13:55.535967 3656 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tkn-cli-serve-784b566968-5vqmh, hostIfaceName: 874deadff23d6c3, network: default, NAD default, SandboxID: "874deadff23d6c3142f910f2c6ede08f56b9e1127ad7e176dce0b69ec3acc9cc", PCI device ID: , UID: "adba9e71-8b6e-4a09-814c-c1a09c823a02", MAC: 0a:58:0a:82:00:25, IPs: [10.130.0.37/23] I0428 11:13:55.589854 3656 cni.go:473] [openshift-pipelines/tkn-cli-serve-784b566968-5vqmh 874deadff23d6c3142f910f2c6ede08f56b9e1127ad7e176dce0b69ec3acc9cc network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tkn-cli-serve-784b566968-5vqmh 874deadff23d6c3142f910f2c6ede08f56b9e1127ad7e176dce0b69ec3acc9cc network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"b2:dc:c3:3d:63:ee\",\"name\":\"874deadff23d6c3\"},{\"mac\":\"0a:58:0a:82:00:25\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5aff9371-4a6f-4b6b-bbd5-ffca8fad6606\"}],\"ips\":[{\"address\":\"10.130.0.37/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0428 11:14:00.332111 3656 obj_retry.go:498] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:14:00.332734 3656 obj_retry.go:498] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:14:00.368654 3656 obj_retry.go:492] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:14:00.368676 3656 obj_retry.go:492] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:14:00.375282 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0428 11:14:00.375304 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam I0428 11:14:02.766691 3656 base_network_controller_pods.go:487] [default/tsf-tas/fulcio-server-7dcbf985f8-r9tb4] creating logical port tsf-tas_fulcio-server-7dcbf985f8-r9tb4 for pod on switch ip-10-0-1-214.ec2.internal I0428 11:14:02.766786 3656 kube.go:256] Updating pod tsf-tas/fulcio-server-7dcbf985f8-r9tb4 I0428 11:14:02.786555 3656 pod.go:62] [tsf-tas/fulcio-server-7dcbf985f8-r9tb4] pod update took 19.795004ms I0428 11:14:02.786578 3656 base_network_controller_pods.go:951] [default/tsf-tas/fulcio-server-7dcbf985f8-r9tb4] addLogicalPort annotation time took 19.823616ms I0428 11:14:02.788189 3656 pods.go:271] [tsf-tas/fulcio-server-7dcbf985f8-r9tb4] addLogicalPort took 21.519675ms, libovsdb time 1.119641ms I0428 11:14:03.135118 3656 cni.go:452] [tsf-tas/fulcio-server-7dcbf985f8-r9tb4 7f0d1d69637ed0a080698ea0829dbe77cc4ee63b8bc5a2f14596020cc1eb5644 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/fulcio-server-7dcbf985f8-r9tb4 7f0d1d69637ed0a080698ea0829dbe77cc4ee63b8bc5a2f14596020cc1eb5644 network default NAD default NAD key ] I0428 11:14:03.149959 3656 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: fulcio-server-7dcbf985f8-r9tb4, hostIfaceName: 7f0d1d69637ed0a, network: default, NAD default, SandboxID: "7f0d1d69637ed0a080698ea0829dbe77cc4ee63b8bc5a2f14596020cc1eb5644", PCI device ID: , UID: "778bbdbd-33dd-4209-bf87-41eaa9b9a2a4", MAC: 0a:58:0a:82:00:26, IPs: [10.130.0.38/23] I0428 11:14:03.204776 3656 base_network_controller_pods.go:487] [default/tsf-tas/rekor-createtree-job-2xpgm-lwqnz] creating logical port tsf-tas_rekor-createtree-job-2xpgm-lwqnz for pod on switch ip-10-0-1-214.ec2.internal I0428 11:14:03.204954 3656 kube.go:256] Updating pod tsf-tas/rekor-createtree-job-2xpgm-lwqnz I0428 11:14:03.208348 3656 cni.go:473] [tsf-tas/fulcio-server-7dcbf985f8-r9tb4 7f0d1d69637ed0a080698ea0829dbe77cc4ee63b8bc5a2f14596020cc1eb5644 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/fulcio-server-7dcbf985f8-r9tb4 7f0d1d69637ed0a080698ea0829dbe77cc4ee63b8bc5a2f14596020cc1eb5644 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"52:25:d1:07:7f:68\",\"name\":\"7f0d1d69637ed0a\"},{\"mac\":\"0a:58:0a:82:00:26\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/97e6cb8e-2651-4a65-afc4-a829853c1df1\"}],\"ips\":[{\"address\":\"10.130.0.38/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0428 11:14:03.237582 3656 base_network_controller_pods.go:487] [default/tsf-tas/trillian-logsigner-dc68764b8-djk6x] creating logical port tsf-tas_trillian-logsigner-dc68764b8-djk6x for pod on switch ip-10-0-1-214.ec2.internal I0428 11:14:03.237701 3656 kube.go:256] Updating pod tsf-tas/trillian-logsigner-dc68764b8-djk6x I0428 11:14:03.252023 3656 kube.go:256] Updating pod tsf-tas/rekor-createtree-job-2xpgm-lwqnz I0428 11:14:03.265612 3656 pod.go:62] [tsf-tas/trillian-logsigner-dc68764b8-djk6x] pod update took 27.932225ms I0428 11:14:03.265696 3656 base_network_controller_pods.go:951] [default/tsf-tas/trillian-logsigner-dc68764b8-djk6x] addLogicalPort annotation time took 28.02036ms I0428 11:14:03.267341 3656 pods.go:271] [tsf-tas/trillian-logsigner-dc68764b8-djk6x] addLogicalPort took 29.776522ms, libovsdb time 1.103471ms I0428 11:14:03.271521 3656 pod.go:62] [tsf-tas/rekor-createtree-job-2xpgm-lwqnz] pod update took 66.620798ms I0428 11:14:03.271549 3656 base_network_controller_pods.go:951] [default/tsf-tas/rekor-createtree-job-2xpgm-lwqnz] addLogicalPort annotation time took 66.655638ms I0428 11:14:03.273222 3656 pods.go:271] [tsf-tas/rekor-createtree-job-2xpgm-lwqnz] addLogicalPort took 68.462119ms, libovsdb time 1.176523ms I0428 11:14:03.564115 3656 cni.go:452] [tsf-tas/rekor-createtree-job-2xpgm-lwqnz 8d36030b7fd8f1564871f210a344711b1576398bca74726f86f321b8f9e062b5 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/rekor-createtree-job-2xpgm-lwqnz 8d36030b7fd8f1564871f210a344711b1576398bca74726f86f321b8f9e062b5 network default NAD default NAD key ] I0428 11:14:03.574310 3656 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: rekor-createtree-job-2xpgm-lwqnz, hostIfaceName: 8d36030b7fd8f15, network: default, NAD default, SandboxID: "8d36030b7fd8f1564871f210a344711b1576398bca74726f86f321b8f9e062b5", PCI device ID: , UID: "908aa25e-a7c3-402a-9ba4-4ff9016f6c31", MAC: 0a:58:0a:82:00:27, IPs: [10.130.0.39/23] I0428 11:14:03.599541 3656 cni.go:452] [tsf-tas/trillian-logsigner-dc68764b8-djk6x 6f187e90ed85018b935523822775ed33ea075407ccdf14fdfd02aaa9ffd62148 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/trillian-logsigner-dc68764b8-djk6x 6f187e90ed85018b935523822775ed33ea075407ccdf14fdfd02aaa9ffd62148 network default NAD default NAD key ] I0428 11:14:03.608717 3656 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: trillian-logsigner-dc68764b8-djk6x, hostIfaceName: 6f187e90ed85018, network: default, NAD default, SandboxID: "6f187e90ed85018b935523822775ed33ea075407ccdf14fdfd02aaa9ffd62148", PCI device ID: , UID: "367b9311-182d-49d0-be82-84fae9b9edb2", MAC: 0a:58:0a:82:00:28, IPs: [10.130.0.40/23] I0428 11:14:03.629700 3656 cni.go:473] [tsf-tas/rekor-createtree-job-2xpgm-lwqnz 8d36030b7fd8f1564871f210a344711b1576398bca74726f86f321b8f9e062b5 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/rekor-createtree-job-2xpgm-lwqnz 8d36030b7fd8f1564871f210a344711b1576398bca74726f86f321b8f9e062b5 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"92:96:e8:ab:35:b1\",\"name\":\"8d36030b7fd8f15\"},{\"mac\":\"0a:58:0a:82:00:27\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3d6117a4-b89e-4d8c-92c2-a107309e0289\"}],\"ips\":[{\"address\":\"10.130.0.39/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0428 11:14:03.663276 3656 cni.go:473] [tsf-tas/trillian-logsigner-dc68764b8-djk6x 6f187e90ed85018b935523822775ed33ea075407ccdf14fdfd02aaa9ffd62148 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/trillian-logsigner-dc68764b8-djk6x 6f187e90ed85018b935523822775ed33ea075407ccdf14fdfd02aaa9ffd62148 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"f2:45:d9:dd:0f:11\",\"name\":\"6f187e90ed85018\"},{\"mac\":\"0a:58:0a:82:00:28\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3ba90559-5162-4c43-8ef3-8eeb10ac02cb\"}],\"ips\":[{\"address\":\"10.130.0.40/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0428 11:14:06.752676 3656 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/fulcio-server W0428 11:14:06.752697 3656 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/fulcio-server W0428 11:14:06.778022 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-214.ec2.internal port 5554: empty IP address endpoints for service tsf-tas/fulcio-server W0428 11:14:06.778045 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-214.ec2.internal port 5555: empty IP address endpoints for service tsf-tas/fulcio-server I0428 11:14:08.360762 3656 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-wdb6q] creating logical port openshift-pipelines_pipelines-as-code-controller-79b6fbbc5c-wdb6q for pod on switch ip-10-0-1-214.ec2.internal I0428 11:14:08.360870 3656 kube.go:256] Updating pod openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-wdb6q I0428 11:14:08.378311 3656 pod.go:62] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-wdb6q] pod update took 17.470137ms I0428 11:14:08.378340 3656 base_network_controller_pods.go:951] [default/openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-wdb6q] addLogicalPort annotation time took 17.502463ms I0428 11:14:08.379792 3656 pods.go:271] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-wdb6q] addLogicalPort took 19.052914ms, libovsdb time 943.692µs I0428 11:14:08.462314 3656 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-fxncz] creating logical port openshift-pipelines_pipelines-as-code-watcher-7947cf58bc-fxncz for pod on switch ip-10-0-1-214.ec2.internal I0428 11:14:08.462476 3656 kube.go:256] Updating pod openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-fxncz I0428 11:14:08.477809 3656 pod.go:62] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-fxncz] pod update took 15.360139ms I0428 11:14:08.477838 3656 base_network_controller_pods.go:951] [default/openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-fxncz] addLogicalPort annotation time took 15.392494ms I0428 11:14:08.479503 3656 pods.go:271] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-fxncz] addLogicalPort took 17.214918ms, libovsdb time 1.092306ms I0428 11:14:09.091092 3656 obj_retry.go:498] Detected object tsf-keycloak/tsf-iam-6blsw of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:14:09.091864 3656 obj_retry.go:498] Detected object tsf-keycloak/tsf-iam-6blsw of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:14:09.923005 3656 cni.go:452] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-wdb6q eef557f657a9edf075fb61818e46b465499de7f63f9174208786ce02d2491e7f network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-wdb6q eef557f657a9edf075fb61818e46b465499de7f63f9174208786ce02d2491e7f network default NAD default NAD key ] I0428 11:14:09.923220 3656 cni.go:452] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-fxncz 89d60bd5e4b442b2f4bd5c211fec4fa19aeb9a0ebc5bbc767cc156787c83089b network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-fxncz 89d60bd5e4b442b2f4bd5c211fec4fa19aeb9a0ebc5bbc767cc156787c83089b network default NAD default NAD key ] I0428 11:14:09.933601 3656 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-as-code-watcher-7947cf58bc-fxncz, hostIfaceName: 89d60bd5e4b442b, network: default, NAD default, SandboxID: "89d60bd5e4b442b2f4bd5c211fec4fa19aeb9a0ebc5bbc767cc156787c83089b", PCI device ID: , UID: "ee549f33-af0d-42e8-a6a6-e70372e43894", MAC: 0a:58:0a:82:00:2a, IPs: [10.130.0.42/23] I0428 11:14:09.933851 3656 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-as-code-controller-79b6fbbc5c-wdb6q, hostIfaceName: eef557f657a9edf, network: default, NAD default, SandboxID: "eef557f657a9edf075fb61818e46b465499de7f63f9174208786ce02d2491e7f", PCI device ID: , UID: "e268ca1b-fd92-4667-98fb-d29038980763", MAC: 0a:58:0a:82:00:29, IPs: [10.130.0.41/23] I0428 11:14:09.976458 3656 obj_retry.go:492] Detected object tsf-keycloak/tsf-iam-6blsw of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:14:09.976548 3656 obj_retry.go:492] Detected object tsf-keycloak/tsf-iam-6blsw of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:14:09.990830 3656 cni.go:473] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-wdb6q eef557f657a9edf075fb61818e46b465499de7f63f9174208786ce02d2491e7f network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-wdb6q eef557f657a9edf075fb61818e46b465499de7f63f9174208786ce02d2491e7f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"02:7b:4d:52:65:31\",\"name\":\"eef557f657a9edf\"},{\"mac\":\"0a:58:0a:82:00:29\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/73bd9b78-2e4b-4898-80a4-8824b6ed3503\"}],\"ips\":[{\"address\":\"10.130.0.41/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0428 11:14:10.011742 3656 cni.go:473] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-fxncz 89d60bd5e4b442b2f4bd5c211fec4fa19aeb9a0ebc5bbc767cc156787c83089b network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-fxncz 89d60bd5e4b442b2f4bd5c211fec4fa19aeb9a0ebc5bbc767cc156787c83089b network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"b6:63:a6:84:02:90\",\"name\":\"89d60bd5e4b442b\"},{\"mac\":\"0a:58:0a:82:00:2a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3ae69261-dfa8-40bd-a38e-ced13fec4f7a\"}],\"ips\":[{\"address\":\"10.130.0.42/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0428 11:14:10.783339 3656 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logsigner W0428 11:14:13.800379 3656 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher W0428 11:14:17.302145 3656 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logserver W0428 11:14:20.353269 3656 util.go:870] Failed to build global endpoints for port TCP/trillian-mysql: empty IP address endpoints for service tsf-tas/trillian-mysql I0428 11:14:20.467461 3656 obj_retry.go:498] Detected object tsf-tas/segment-backup-installation-7g2kg-96d2z of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:14:20.468029 3656 obj_retry.go:498] Detected object tsf-tas/segment-backup-installation-7g2kg-96d2z of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:14:21.489696 3656 obj_retry.go:492] Detected object tsf-tas/segment-backup-installation-7g2kg-96d2z of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:14:21.489728 3656 obj_retry.go:492] Detected object tsf-tas/segment-backup-installation-7g2kg-96d2z of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0428 11:14:24.817645 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-214.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher I0428 11:14:36.055102 3656 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=51 W0428 11:14:49.947834 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-214.ec2.internal port 8091: empty IP address endpoints for service tsf-tas/trillian-logsigner I0428 11:14:51.652114 3656 obj_retry.go:498] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:14:51.652723 3656 obj_retry.go:498] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:14:51.670624 3656 obj_retry.go:492] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:14:51.670649 3656 obj_retry.go:492] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:14:51.677939 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tas/test-tsf-tas I0428 11:14:51.677966 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tas/test-tsf-tas I0428 11:14:53.053063 3656 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=106 I0428 11:14:53.756248 3656 obj_retry.go:498] Detected object tsf-tas/ctlog-createtree-job-8f966-6r72g of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:14:53.756807 3656 obj_retry.go:498] Detected object tsf-tas/ctlog-createtree-job-8f966-6r72g of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:14:54.520332 3656 obj_retry.go:492] Detected object tsf-tas/ctlog-createtree-job-8f966-6r72g of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:14:54.520365 3656 obj_retry.go:492] Detected object tsf-tas/ctlog-createtree-job-8f966-6r72g of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:14:56.035331 3656 cni.go:452] [tsf-tas/rekor-createtree-job-2xpgm-lwqnz 8d36030b7fd8f1564871f210a344711b1576398bca74726f86f321b8f9e062b5 network default NAD default NAD key ] DEL starting CNI request [tsf-tas/rekor-createtree-job-2xpgm-lwqnz 8d36030b7fd8f1564871f210a344711b1576398bca74726f86f321b8f9e062b5 network default NAD default NAD key ] I0428 11:14:56.129712 3656 cni.go:473] [tsf-tas/rekor-createtree-job-2xpgm-lwqnz 8d36030b7fd8f1564871f210a344711b1576398bca74726f86f321b8f9e062b5 network default NAD default NAD key default] DEL finished CNI request [tsf-tas/rekor-createtree-job-2xpgm-lwqnz 8d36030b7fd8f1564871f210a344711b1576398bca74726f86f321b8f9e062b5 network default NAD default NAD key default], result "{}", err I0428 11:14:56.180749 3656 obj_retry.go:498] Detected object tsf-tas/rekor-createtree-job-2xpgm-lwqnz of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:14:56.180783 3656 pods.go:176] Deleting pod: tsf-tas/rekor-createtree-job-2xpgm-lwqnz I0428 11:14:56.182135 3656 pods.go:236] Attempting to release IPs for pod: tsf-tas/rekor-createtree-job-2xpgm-lwqnz, ips: 10.130.0.39 I0428 11:14:56.182166 3656 obj_retry.go:498] Detected object tsf-tas/rekor-createtree-job-2xpgm-lwqnz of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:14:57.046894 3656 obj_retry.go:492] Detected object tsf-tas/rekor-createtree-job-2xpgm-lwqnz of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:14:57.046936 3656 obj_retry.go:492] Detected object tsf-tas/rekor-createtree-job-2xpgm-lwqnz of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:14:57.053953 3656 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=58 I0428 11:14:59.254130 3656 base_network_controller_pods.go:487] [default/tsf-tas/rekor-redis-c64588d97-xfxjt] creating logical port tsf-tas_rekor-redis-c64588d97-xfxjt for pod on switch ip-10-0-1-214.ec2.internal I0428 11:14:59.254230 3656 kube.go:256] Updating pod tsf-tas/rekor-redis-c64588d97-xfxjt I0428 11:14:59.271260 3656 pod.go:62] [tsf-tas/rekor-redis-c64588d97-xfxjt] pod update took 17.054039ms I0428 11:14:59.271285 3656 base_network_controller_pods.go:951] [default/tsf-tas/rekor-redis-c64588d97-xfxjt] addLogicalPort annotation time took 17.083943ms I0428 11:14:59.272793 3656 pods.go:271] [tsf-tas/rekor-redis-c64588d97-xfxjt] addLogicalPort took 18.688054ms, libovsdb time 971.367µs I0428 11:15:00.214282 3656 cni.go:452] [tsf-tas/rekor-redis-c64588d97-xfxjt 737a06627a3403179c6d71e5ab3023f1eaea8bf6a1fd72d6644fba9b5177f715 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/rekor-redis-c64588d97-xfxjt 737a06627a3403179c6d71e5ab3023f1eaea8bf6a1fd72d6644fba9b5177f715 network default NAD default NAD key ] I0428 11:15:00.223481 3656 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: rekor-redis-c64588d97-xfxjt, hostIfaceName: 737a06627a34031, network: default, NAD default, SandboxID: "737a06627a3403179c6d71e5ab3023f1eaea8bf6a1fd72d6644fba9b5177f715", PCI device ID: , UID: "d8b5544f-9df4-4414-94c4-60196f79b284", MAC: 0a:58:0a:82:00:2b, IPs: [10.130.0.43/23] I0428 11:15:00.278605 3656 cni.go:473] [tsf-tas/rekor-redis-c64588d97-xfxjt 737a06627a3403179c6d71e5ab3023f1eaea8bf6a1fd72d6644fba9b5177f715 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/rekor-redis-c64588d97-xfxjt 737a06627a3403179c6d71e5ab3023f1eaea8bf6a1fd72d6644fba9b5177f715 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"2a:6d:cd:6b:1d:87\",\"name\":\"737a06627a34031\"},{\"mac\":\"0a:58:0a:82:00:2b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0be94024-7058-4131-a19f-42a56b1fd7c4\"}],\"ips\":[{\"address\":\"10.130.0.43/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0428 11:15:00.369326 3656 obj_retry.go:498] Detected object tsf/patch-tekton-config-mmrll of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:15:00.369897 3656 obj_retry.go:498] Detected object tsf/patch-tekton-config-mmrll of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0428 11:15:00.544723 3656 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/ctlog I0428 11:15:01.272908 3656 obj_retry.go:492] Detected object tsf/patch-tekton-config-mmrll of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:15:01.272975 3656 obj_retry.go:492] Detected object tsf/patch-tekton-config-mmrll of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0428 11:15:03.269685 3656 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-search-ui W0428 11:15:05.081382 3656 util.go:870] Failed to build global endpoints for port TCP/resp: empty IP address endpoints for service tsf-tas/rekor-redis I0428 11:15:07.717511 3656 obj_retry.go:498] Detected object tsf/tsf-tekton-configuration-pcg6b of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:15:07.718110 3656 obj_retry.go:498] Detected object tsf/tsf-tekton-configuration-pcg6b of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:15:08.740273 3656 obj_retry.go:492] Detected object tsf/tsf-tekton-configuration-pcg6b of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:15:08.740300 3656 obj_retry.go:492] Detected object tsf/tsf-tekton-configuration-pcg6b of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:15:08.775694 3656 obj_retry.go:492] Detected object tsf/patch-tekton-config-mmrll of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:15:08.775726 3656 obj_retry.go:492] Detected object tsf/patch-tekton-config-mmrll of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:15:08.780720 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/patch-tekton-config-mmrll I0428 11:15:08.780749 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/patch-tekton-config-mmrll I0428 11:15:12.748939 3656 obj_retry.go:498] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:15:12.749548 3656 obj_retry.go:498] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:15:12.767670 3656 obj_retry.go:492] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:15:12.767693 3656 obj_retry.go:492] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:15:12.771827 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/pre-install-tsf-tpa I0428 11:15:12.771846 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/pre-install-tsf-tpa W0428 11:15:15.634775 3656 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-server W0428 11:15:17.126179 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-214.ec2.internal port 6379: empty IP address endpoints for service tsf-tas/rekor-redis I0428 11:15:18.498086 3656 obj_retry.go:498] Detected object tsf-tpa/create-db-zbjfx of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:15:18.498621 3656 obj_retry.go:498] Detected object tsf-tpa/create-db-zbjfx of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:15:19.521897 3656 obj_retry.go:492] Detected object tsf-tpa/create-db-zbjfx of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:15:19.521950 3656 obj_retry.go:492] Detected object tsf-tpa/create-db-zbjfx of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:15:22.523892 3656 obj_retry.go:498] Detected object tsf-tpa/migrate-db-m6x9s of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:15:22.524467 3656 obj_retry.go:498] Detected object tsf-tpa/migrate-db-m6x9s of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:15:22.661220 3656 obj_retry.go:492] Detected object tsf-tpa/migrate-db-m6x9s of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:15:22.661248 3656 obj_retry.go:492] Detected object tsf-tpa/migrate-db-m6x9s of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:15:25.544334 3656 obj_retry.go:498] Detected object tsf-tpa/create-importers-pg867 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:15:25.544909 3656 obj_retry.go:498] Detected object tsf-tpa/create-importers-pg867 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:15:26.566035 3656 obj_retry.go:492] Detected object tsf-tpa/create-importers-pg867 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:15:26.566059 3656 obj_retry.go:492] Detected object tsf-tpa/create-importers-pg867 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0428 11:15:37.752789 3656 util.go:870] Failed to build global endpoints for port TCP/endpoint: empty IP address endpoints for service tsf-tpa/server I0428 11:15:48.672745 3656 obj_retry.go:498] Detected object tsf-tas/tuf-repository-init-wh98r-lkd4h of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:15:48.673407 3656 obj_retry.go:498] Detected object tsf-tas/tuf-repository-init-wh98r-lkd4h of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:15:49.574847 3656 obj_retry.go:492] Detected object tsf-tas/tuf-repository-init-wh98r-lkd4h of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:15:49.574875 3656 obj_retry.go:492] Detected object tsf-tas/tuf-repository-init-wh98r-lkd4h of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0428 11:16:04.634834 3656 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/tuf I0428 11:16:06.045459 3656 obj_retry.go:498] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:16:06.046050 3656 obj_retry.go:498] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:16:06.083447 3656 obj_retry.go:492] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:16:06.083475 3656 obj_retry.go:492] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:16:06.089978 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/test-tsf-tpa I0428 11:16:06.089996 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/test-tsf-tpa I0428 11:16:06.791372 3656 namespace.go:98] [integration-service] adding namespace I0428 11:16:06.792379 3656 namespace.go:102] [integration-service] adding namespace took 982.745µs I0428 11:16:06.804363 3656 namespace.go:98] [default-tenant] adding namespace I0428 11:16:06.804415 3656 namespace.go:98] [image-controller] adding namespace I0428 11:16:06.804551 3656 namespace.go:98] [build-service] adding namespace I0428 11:16:06.805183 3656 namespace.go:102] [default-tenant] adding namespace took 801.531µs I0428 11:16:06.805700 3656 namespace.go:102] [image-controller] adding namespace took 1.271578ms I0428 11:16:06.806253 3656 namespace.go:102] [build-service] adding namespace took 1.680334ms I0428 11:16:06.861551 3656 namespace.go:142] [integration-service] updating namespace I0428 11:16:06.889309 3656 namespace.go:142] [integration-service] updating namespace I0428 11:16:06.932493 3656 namespace.go:142] [default-tenant] updating namespace I0428 11:16:06.932631 3656 namespace.go:142] [default-tenant] updating namespace I0428 11:16:06.932720 3656 namespace.go:142] [image-controller] updating namespace I0428 11:16:06.952855 3656 namespace.go:142] [image-controller] updating namespace I0428 11:16:06.956755 3656 namespace.go:142] [build-service] updating namespace I0428 11:16:06.980557 3656 namespace.go:142] [build-service] updating namespace I0428 11:16:07.087529 3656 namespace.go:142] [build-service] updating namespace I0428 11:16:07.120666 3656 namespace.go:142] [integration-service] updating namespace I0428 11:16:07.171359 3656 namespace.go:98] [release-service] adding namespace I0428 11:16:07.172284 3656 namespace.go:102] [release-service] adding namespace took 903.336µs I0428 11:16:07.207313 3656 namespace.go:142] [release-service] updating namespace I0428 11:16:07.219631 3656 namespace.go:142] [release-service] updating namespace I0428 11:16:07.279507 3656 namespace.go:142] [konflux-ui] updating namespace I0428 11:16:07.309562 3656 base_network_controller_pods.go:487] [default/konflux-ui/test-tsf-konflux] creating logical port konflux-ui_test-tsf-konflux for pod on switch ip-10-0-1-214.ec2.internal I0428 11:16:07.309680 3656 kube.go:256] Updating pod konflux-ui/test-tsf-konflux I0428 11:16:07.382262 3656 pod.go:62] [konflux-ui/test-tsf-konflux] pod update took 72.610168ms I0428 11:16:07.382284 3656 base_network_controller_pods.go:951] [default/konflux-ui/test-tsf-konflux] addLogicalPort annotation time took 72.639291ms I0428 11:16:07.383509 3656 pods.go:271] [konflux-ui/test-tsf-konflux] addLogicalPort took 73.967002ms, libovsdb time 846.583µs I0428 11:16:07.392830 3656 namespace.go:98] [konflux-info] adding namespace I0428 11:16:07.393900 3656 namespace.go:102] [konflux-info] adding namespace took 1.046238ms I0428 11:16:07.415502 3656 namespace.go:142] [konflux-info] updating namespace I0428 11:16:07.470410 3656 namespace.go:98] [namespace-lister] adding namespace I0428 11:16:07.471268 3656 namespace.go:102] [namespace-lister] adding namespace took 839.015µs I0428 11:16:07.497569 3656 namespace.go:142] [konflux-info] updating namespace I0428 11:16:07.497724 3656 namespace.go:142] [namespace-lister] updating namespace I0428 11:16:07.612359 3656 namespace.go:98] [enterprise-contract-service] adding namespace I0428 11:16:07.613260 3656 namespace.go:102] [enterprise-contract-service] adding namespace took 876.778µs I0428 11:16:07.677612 3656 namespace.go:142] [enterprise-contract-service] updating namespace I0428 11:16:07.677675 3656 namespace.go:142] [namespace-lister] updating namespace I0428 11:16:07.716686 3656 namespace.go:142] [image-controller] updating namespace I0428 11:16:07.738676 3656 namespace.go:142] [enterprise-contract-service] updating namespace I0428 11:16:07.960901 3656 cni.go:452] [konflux-ui/test-tsf-konflux f23ee6ef32021cf5dcbbca2c4a9986d198e85fe96fb878c08930215fe10fa842 network default NAD default NAD key ] ADD starting CNI request [konflux-ui/test-tsf-konflux f23ee6ef32021cf5dcbbca2c4a9986d198e85fe96fb878c08930215fe10fa842 network default NAD default NAD key ] I0428 11:16:07.970059 3656 helper_linux.go:556] ConfigureOVS: namespace: konflux-ui, podName: test-tsf-konflux, hostIfaceName: f23ee6ef32021cf, network: default, NAD default, SandboxID: "f23ee6ef32021cf5dcbbca2c4a9986d198e85fe96fb878c08930215fe10fa842", PCI device ID: , UID: "9eb5c787-5558-4fc8-a76d-b058f4f05704", MAC: 0a:58:0a:82:00:2c, IPs: [10.130.0.44/23] I0428 11:16:08.023740 3656 cni.go:473] [konflux-ui/test-tsf-konflux f23ee6ef32021cf5dcbbca2c4a9986d198e85fe96fb878c08930215fe10fa842 network default NAD default NAD key default] ADD finished CNI request [konflux-ui/test-tsf-konflux f23ee6ef32021cf5dcbbca2c4a9986d198e85fe96fb878c08930215fe10fa842 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"b2:6b:ce:09:2e:c4\",\"name\":\"f23ee6ef32021cf\"},{\"mac\":\"0a:58:0a:82:00:2c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e833481f-516d-4417-abaf-69817fbe7410\"}],\"ips\":[{\"address\":\"10.130.0.44/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0428 11:16:08.397146 3656 namespace.go:142] [default-tenant] updating namespace I0428 11:16:08.708516 3656 namespace.go:98] [segment-bridge] adding namespace I0428 11:16:08.709520 3656 namespace.go:102] [segment-bridge] adding namespace took 981.816µs I0428 11:16:08.709568 3656 namespace.go:142] [segment-bridge] updating namespace I0428 11:16:08.893766 3656 namespace.go:142] [segment-bridge] updating namespace I0428 11:16:09.315071 3656 namespace.go:98] [konflux-cli] adding namespace I0428 11:16:09.315877 3656 namespace.go:102] [konflux-cli] adding namespace took 780.861µs I0428 11:16:09.315928 3656 namespace.go:142] [konflux-cli] updating namespace I0428 11:16:09.371063 3656 namespace.go:142] [konflux-cli] updating namespace I0428 11:16:09.679813 3656 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-from-konfluxui for network default I0428 11:16:09.680486 3656 pod_selector_address_set.go:213] Created shared address set for pod selector namespace-lister_LS{ML:{app: proxy,},} I0428 11:16:09.680506 3656 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-from-konfluxui added to peer address sets [namespace-lister_LS{ML:{app: proxy,},}] I0428 11:16:09.683398 3656 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-from-konfluxui resources completed, update namespace loglevel I0428 11:16:09.783616 3656 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-to-apiserver for network default I0428 11:16:09.783649 3656 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-to-apiserver added to peer address sets [] I0428 11:16:09.784688 3656 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-to-apiserver resources completed, update namespace loglevel I0428 11:16:11.216585 3656 namespace.go:142] [build-service] updating namespace I0428 11:16:11.962474 3656 namespace.go:142] [default-tenant] updating namespace I0428 11:16:12.692543 3656 namespace.go:142] [enterprise-contract-service] updating namespace W0428 11:16:12.767609 3656 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service namespace-lister/namespace-lister I0428 11:16:13.100670 3656 namespace.go:142] [image-controller] updating namespace I0428 11:16:13.460069 3656 namespace.go:142] [integration-service] updating namespace I0428 11:16:13.528410 3656 base_network_controller_pods.go:487] [default/konflux-ui/proxy-c8d7646c7-prm4p] creating logical port konflux-ui_proxy-c8d7646c7-prm4p for pod on switch ip-10-0-1-214.ec2.internal I0428 11:16:13.528516 3656 kube.go:256] Updating pod konflux-ui/proxy-c8d7646c7-prm4p I0428 11:16:13.592682 3656 pod.go:62] [konflux-ui/proxy-c8d7646c7-prm4p] pod update took 64.2038ms I0428 11:16:13.592708 3656 base_network_controller_pods.go:951] [default/konflux-ui/proxy-c8d7646c7-prm4p] addLogicalPort annotation time took 64.236679ms I0428 11:16:13.594139 3656 pods.go:271] [konflux-ui/proxy-c8d7646c7-prm4p] addLogicalPort took 65.744071ms, libovsdb time 959.843µs I0428 11:16:13.771430 3656 namespace.go:142] [konflux-cli] updating namespace I0428 11:16:13.937760 3656 namespace.go:142] [konflux-info] updating namespace I0428 11:16:14.135961 3656 namespace.go:142] [namespace-lister] updating namespace I0428 11:16:14.330460 3656 namespace.go:142] [release-service] updating namespace I0428 11:16:14.533976 3656 namespace.go:142] [segment-bridge] updating namespace I0428 11:16:14.664262 3656 namespace.go:142] [build-service] updating namespace I0428 11:16:14.681780 3656 namespace.go:142] [default-tenant] updating namespace I0428 11:16:14.706708 3656 namespace.go:142] [enterprise-contract-service] updating namespace I0428 11:16:14.724001 3656 namespace.go:142] [image-controller] updating namespace I0428 11:16:14.738678 3656 namespace.go:142] [integration-service] updating namespace I0428 11:16:14.758439 3656 namespace.go:142] [konflux-cli] updating namespace I0428 11:16:14.773830 3656 namespace.go:142] [konflux-info] updating namespace I0428 11:16:14.791786 3656 namespace.go:142] [namespace-lister] updating namespace I0428 11:16:14.809367 3656 namespace.go:142] [release-service] updating namespace I0428 11:16:14.839904 3656 namespace.go:142] [segment-bridge] updating namespace I0428 11:16:14.909351 3656 namespace.go:142] [build-service] updating namespace I0428 11:16:14.978142 3656 namespace.go:142] [default-tenant] updating namespace I0428 11:16:15.045630 3656 namespace.go:142] [enterprise-contract-service] updating namespace I0428 11:16:15.156715 3656 namespace.go:142] [image-controller] updating namespace I0428 11:16:15.358586 3656 namespace.go:142] [integration-service] updating namespace I0428 11:16:15.542457 3656 namespace.go:142] [konflux-cli] updating namespace I0428 11:16:15.662828 3656 namespace.go:142] [konflux-info] updating namespace W0428 11:16:15.768262 3656 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service W0428 11:16:15.779076 3656 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service image-controller/image-controller-controller-manager-metrics-service I0428 11:16:15.828466 3656 namespace.go:142] [namespace-lister] updating namespace I0428 11:16:15.906794 3656 namespace.go:142] [release-service] updating namespace I0428 11:16:15.986977 3656 namespace.go:142] [segment-bridge] updating namespace I0428 11:16:21.387240 3656 cni.go:452] [konflux-ui/proxy-c8d7646c7-prm4p 3177bab6504e84064dbcd68c21a9955a2e2dd1697ecec3337ec6dad9ceb90679 network default NAD default NAD key ] ADD starting CNI request [konflux-ui/proxy-c8d7646c7-prm4p 3177bab6504e84064dbcd68c21a9955a2e2dd1697ecec3337ec6dad9ceb90679 network default NAD default NAD key ] I0428 11:16:21.397673 3656 helper_linux.go:556] ConfigureOVS: namespace: konflux-ui, podName: proxy-c8d7646c7-prm4p, hostIfaceName: 3177bab6504e840, network: default, NAD default, SandboxID: "3177bab6504e84064dbcd68c21a9955a2e2dd1697ecec3337ec6dad9ceb90679", PCI device ID: , UID: "d4ce8531-2788-4e25-8c7e-c103eb280e02", MAC: 0a:58:0a:82:00:2d, IPs: [10.130.0.45/23] I0428 11:16:21.455988 3656 cni.go:473] [konflux-ui/proxy-c8d7646c7-prm4p 3177bab6504e84064dbcd68c21a9955a2e2dd1697ecec3337ec6dad9ceb90679 network default NAD default NAD key default] ADD finished CNI request [konflux-ui/proxy-c8d7646c7-prm4p 3177bab6504e84064dbcd68c21a9955a2e2dd1697ecec3337ec6dad9ceb90679 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"f6:9a:18:7c:39:ef\",\"name\":\"3177bab6504e840\"},{\"mac\":\"0a:58:0a:82:00:2d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9cc39f14-03d1-4a60-b571-b337a2ec964e\"}],\"ips\":[{\"address\":\"10.130.0.45/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0428 11:16:21.758598 3656 util.go:870] Failed to build global endpoints for port TCP/dex: empty IP address endpoints for service konflux-ui/dex W0428 11:16:21.778813 3656 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0428 11:16:21.782137 3656 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0428 11:16:21.803683 3656 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0428 11:16:21.805897 3656 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0428 11:16:24.795142 3656 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0428 11:16:24.795166 3656 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0428 11:16:26.530711 3656 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0428 11:16:26.530736 3656 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0428 11:16:26.838329 3656 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service release-service/release-service-webhook-service W0428 11:16:26.839391 3656 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service I0428 11:16:33.107628 3656 obj_retry.go:498] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-ghwrv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:16:33.108283 3656 obj_retry.go:498] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-ghwrv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:16:33.855011 3656 obj_retry.go:492] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-ghwrv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:16:33.855040 3656 obj_retry.go:492] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-ghwrv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:16:33.857586 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod integration-service/integration-service-controller-manager-7b666bf6d8-ghwrv I0428 11:16:33.857612 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod integration-service/integration-service-controller-manager-7b666bf6d8-ghwrv W0428 11:17:01.862936 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-214.ec2.internal port 8080: empty IP address endpoints for service konflux-ui/proxy W0428 11:17:01.862958 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-214.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/proxy W0428 11:17:01.862972 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-214.ec2.internal port 8080: empty IP address endpoints for service konflux-ui/proxy W0428 11:17:01.862979 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-214.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/proxy W0428 11:17:05.598725 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-214.ec2.internal port 8080: empty IP address endpoints for service konflux-ui/proxy W0428 11:17:05.598748 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-214.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/proxy I0428 11:17:19.025392 3656 namespace.go:142] [openshift-pipelines] updating namespace I0428 11:17:27.143996 3656 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=8 I0428 11:17:31.182355 3656 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=9 I0428 11:17:35.450172 3656 obj_retry.go:498] Detected object konflux-ui/proxy-ffb94955b-cwmvh of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:17:35.450776 3656 obj_retry.go:498] Detected object konflux-ui/proxy-ffb94955b-cwmvh of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:17:36.242036 3656 obj_retry.go:492] Detected object konflux-ui/proxy-ffb94955b-cwmvh of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:17:36.242063 3656 obj_retry.go:492] Detected object konflux-ui/proxy-ffb94955b-cwmvh of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:17:36.249981 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/proxy-ffb94955b-cwmvh I0428 11:17:36.250007 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/proxy-ffb94955b-cwmvh I0428 11:17:37.933631 3656 cni.go:452] [konflux-ui/test-tsf-konflux f23ee6ef32021cf5dcbbca2c4a9986d198e85fe96fb878c08930215fe10fa842 network default NAD default NAD key ] DEL starting CNI request [konflux-ui/test-tsf-konflux f23ee6ef32021cf5dcbbca2c4a9986d198e85fe96fb878c08930215fe10fa842 network default NAD default NAD key ] I0428 11:17:38.021042 3656 cni.go:473] [konflux-ui/test-tsf-konflux f23ee6ef32021cf5dcbbca2c4a9986d198e85fe96fb878c08930215fe10fa842 network default NAD default NAD key default] DEL finished CNI request [konflux-ui/test-tsf-konflux f23ee6ef32021cf5dcbbca2c4a9986d198e85fe96fb878c08930215fe10fa842 network default NAD default NAD key default], result "{}", err I0428 11:17:38.072054 3656 obj_retry.go:498] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:17:38.072086 3656 pods.go:176] Deleting pod: konflux-ui/test-tsf-konflux I0428 11:17:38.073487 3656 pods.go:236] Attempting to release IPs for pod: konflux-ui/test-tsf-konflux, ips: 10.130.0.44 I0428 11:17:38.073518 3656 obj_retry.go:498] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:17:38.091377 3656 obj_retry.go:492] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:17:38.091409 3656 obj_retry.go:492] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:17:38.096824 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/test-tsf-konflux I0428 11:17:38.096854 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/test-tsf-konflux I0428 11:18:04.482563 3656 namespace.go:98] [default-managed-tenant-0xzgp] adding namespace I0428 11:18:04.483584 3656 namespace.go:102] [default-managed-tenant-0xzgp] adding namespace took 996.511µs I0428 11:18:04.528038 3656 namespace.go:142] [default-managed-tenant-0xzgp] updating namespace I0428 11:18:04.541615 3656 namespace.go:142] [default-managed-tenant-0xzgp] updating namespace I0428 11:18:05.734428 3656 namespace.go:142] [default-managed-tenant-0xzgp] updating namespace I0428 11:18:05.866947 3656 namespace.go:142] [default-managed-tenant-0xzgp] updating namespace I0428 11:18:05.961259 3656 namespace.go:142] [default-managed-tenant-0xzgp] updating namespace I0428 11:18:15.142981 3656 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=8 I0428 11:18:48.053212 3656 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=10 I0428 11:18:49.055052 3656 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=78 I0428 11:19:14.169985 3656 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=9 I0428 11:19:21.153048 3656 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=8 I0428 11:19:30.175224 3656 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=8 I0428 11:19:36.163677 3656 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=9 I0428 11:19:42.182736 3656 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=8 I0428 11:19:45.062467 3656 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=624 I0428 11:19:55.148101 3656 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=10 I0428 11:20:04.262438 3656 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29622920-r4h2f of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 11:20:04.263085 3656 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29622920-r4h2f of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 11:20:05.150136 3656 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29622920-r4h2f of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 11:20:05.150196 3656 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29622920-r4h2f of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed