+ . /ovnkube-lib/ovnkube-lib.sh ++ set -x ++ K8S_NODE=ip-10-0-1-106.ec2.internal ++ [[ -n ip-10-0-1-106.ec2.internal ]] ++ [[ -f /env/ip-10-0-1-106.ec2.internal ]] ++ northd_pidfile=/var/run/ovn/ovn-northd.pid ++ controller_pidfile=/var/run/ovn/ovn-controller.pid ++ controller_logfile=/var/log/ovn/acl-audit-log.log ++ vswitch_dbsock=/var/run/openvswitch/db.sock ++ nbdb_pidfile=/var/run/ovn/ovnnb_db.pid ++ nbdb_sock=/var/run/ovn/ovnnb_db.sock ++ nbdb_ctl=/var/run/ovn/ovnnb_db.ctl ++ sbdb_pidfile=/var/run/ovn/ovnsb_db.pid ++ sbdb_sock=/var/run/ovn/ovnsb_db.sock ++ sbdb_ctl=/var/run/ovn/ovnsb_db.ctl + start-ovnkube-node 4 29103 29105 + local log_level=4 + local metrics_port=29103 + local ovn_metrics_port=29105 + ovn_advertised_udn_isolation_mode_flag= + openflow_probe_flag= + [[ 3 -ne 3 ]] + ovs-vsctl br-exists br-ex + add_garp_drop_flow br-ex + local bridge=br-ex + local cookie=0x0305 + local priority=499 ++ ovs-vsctl list-ports br-ex + for port_name in $(ovs-vsctl list-ports "$bridge") + [[ ens5 == *to-br-int ]] + cni-bin-copy + . /host/etc/os-release ++ NAME='Red Hat Enterprise Linux CoreOS' ++ VERSION='9.6.20260401-0 (Plow)' ++ ID=rhel ++ ID_LIKE=fedora ++ VERSION_ID=9.6 ++ PLATFORM_ID=platform:el9 ++ PRETTY_NAME='Red Hat Enterprise Linux CoreOS 9.6.20260401-0 (Plow)' ++ ANSI_COLOR='0;31' ++ LOGO=fedora-logo-icon ++ CPE_NAME=cpe:/o:redhat:enterprise_linux:9::baseos ++ HOME_URL=https://www.redhat.com/ ++ DOCUMENTATION_URL=https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9 ++ BUG_REPORT_URL=https://issues.redhat.com/ ++ REDHAT_BUGZILLA_PRODUCT='Red Hat Enterprise Linux 9' ++ REDHAT_BUGZILLA_PRODUCT_VERSION=9.6 ++ REDHAT_SUPPORT_PRODUCT='Red Hat Enterprise Linux' ++ REDHAT_SUPPORT_PRODUCT_VERSION=9.6 ++ OSTREE_VERSION=9.6.20260401-0 ++ VARIANT=CoreOS ++ VARIANT_ID=coreos ++ OPENSHIFT_VERSION=4.21 + rhelmajor= + case "${ID}" in ++ echo 9.6 ++ cut -f 1 -d . + rhelmajor=9 + sourcedir=/usr/libexec/cni/ + case "${rhelmajor}" in + sourcedir=/usr/libexec/cni/rhel9 + cp -f /usr/libexec/cni/rhel9/ovn-k8s-cni-overlay /cni-bin-dir/ ++ date '+%m%d %H:%M:%S.%N' + echo 'I0428 15:37:44.198343608 - disable conntrack on geneve port' I0428 15:37:44.198343608 - disable conntrack on geneve port + iptables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + iptables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK ++ date '+%m%d %H:%M:%S.%N' I0428 15:37:44.225946507 - starting ovnkube-node + echo 'I0428 15:37:44.225946507 - starting ovnkube-node' + egress_features_enable_flag='--enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true' + init_ovnkube_controller='--init-ovnkube-controller ip-10-0-1-106.ec2.internal' + multi_external_gateway_enable_flag=--enable-multi-external-gateway=true + gateway_interface=br-ex + enable_multicast_flag=--enable-multicast + OVN_NODE_MODE=full + '[' full == dpu-host ']' + '[' shared == shared ']' + gateway_mode_flags='--gateway-mode shared --gateway-interface br-ex' + export_network_flows_flags= + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + gw_interface_flag= + '[' -d /sys/class/net/br-ex1 ']' + node_mgmt_port_netdev_flags= + [[ -n '' ]] + [[ -n '' ]] + multi_network_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag=--enable-network-segmentation + route_advertisements_enable_flag= + [[ false == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag= + [[ true == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag=--enable-preconfigured-udn-addresses + network_observability_enabled_flag= + [[ false == \t\r\u\e ]] + multi_network_policy_enabled_flag= + [[ false == \t\r\u\e ]] + admin_network_policy_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + admin_network_policy_enabled_flag=--enable-admin-network-policy + dns_name_resolver_enabled_flag= + [[ false == \t\r\u\e ]] + ip_forwarding_flag= + '[' '' == Global ']' + ip_forwarding_flag=--disable-forwarding + sysctl -w net.ipv4.ip_forward=0 net.ipv4.ip_forward = 0 + sysctl -w net.ipv6.conf.all.forwarding=0 net.ipv6.conf.all.forwarding = 0 + [[ '' != '' ]] + [[ '' != '' ]] + NETWORK_NODE_IDENTITY_ENABLE= + [[ true == \t\r\u\e ]] + NETWORK_NODE_IDENTITY_ENABLE=' --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h ' + ovn_v4_join_subnet_opt= + [[ '' != '' ]] + ovn_v6_join_subnet_opt= + [[ '' != '' ]] + ovn_v4_masquerade_subnet_opt= + [[ 169.254.0.0/17 != '' ]] + ovn_v4_masquerade_subnet_opt='--gateway-v4-masquerade-subnet 169.254.0.0/17' + ovn_v6_masquerade_subnet_opt= + [[ fd69::/112 != '' ]] + ovn_v6_masquerade_subnet_opt='--gateway-v6-masquerade-subnet fd69::/112' + ovn_v4_transit_switch_subnet_opt= + [[ '' != '' ]] + ovn_v6_transit_switch_subnet_opt= + [[ '' != '' ]] + exec /usr/bin/ovnkube --init-ovnkube-controller ip-10-0-1-106.ec2.internal --init-node ip-10-0-1-106.ec2.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4 --inactivity-probe=180000 --gateway-mode shared --gateway-interface br-ex --metrics-bind-address 127.0.0.1:29103 --ovn-metrics-bind-address 127.0.0.1:29105 --metrics-enable-pprof --metrics-enable-config-duration --export-ovs-metrics --disable-snat-multiple-gws --enable-multi-network --enable-network-segmentation --enable-preconfigured-udn-addresses --enable-admin-network-policy --enable-multicast --zone ip-10-0-1-106.ec2.internal --enable-interconnect --acl-logging-rate-limit 20 --disable-forwarding --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h --gateway-v4-masquerade-subnet 169.254.0.0/17 --gateway-v6-masquerade-subnet fd69::/112 --enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true --enable-multi-external-gateway=true I0428 15:37:44.271180 3647 config.go:2726] Parsed config file /run/ovnkube-config/ovnkube.conf I0428 15:37:44.271259 3647 config.go:2727] Parsed config: {Default:{MTU:8901 RoutableMTU:0 ConntrackZone:64000 HostMasqConntrackZone:0 OVNMasqConntrackZone:0 HostNodePortConntrackZone:0 ReassemblyConntrackZone:0 EncapType:geneve EncapIP: EffectiveEncapIP: EncapPort:6081 InactivityProbe:100000 OpenFlowProbe:0 OfctrlWaitBeforeClear:0 MonitorAll:true OVSDBTxnTimeout:1m40s LFlowCacheEnable:true LFlowCacheLimit:0 LFlowCacheLimitKb:1048576 RawClusterSubnets:10.128.0.0/14/23 ClusterSubnets:[] EnableUDPAggregation:true Zone:global RawUDNAllowedDefaultServices:default/kubernetes,openshift-dns/dns-default UDNAllowedDefaultServices:[] Transport:geneve} Logging:{File: CNIFile: LibovsdbFile: Level:4 LogFileMaxSize:100 LogFileMaxBackups:5 LogFileMaxAge:5 ACLLoggingRateLimit:20} Monitoring:{RawNetFlowTargets: RawSFlowTargets: RawIPFIXTargets: NetFlowTargets:[] SFlowTargets:[] IPFIXTargets:[]} IPFIX:{Sampling:400 CacheActiveTimeout:60 CacheMaxFlows:0} CNI:{ConfDir:/etc/cni/net.d Plugin:ovn-k8s-cni-overlay} OVNKubernetesFeature:{EnableAdminNetworkPolicy:false EnableEgressIP:false EgressIPReachabiltyTotalTimeout:1 EnableEgressFirewall:false EnableEgressQoS:false EnableEgressService:false EgressIPNodeHealthCheckPort:9107 EnableMultiNetwork:false EnableNetworkSegmentation:true EnableNetworkConnect:false EnablePreconfiguredUDNAddresses:true EnableRouteAdvertisements:false EnableEVPN:false EnableMultiNetworkPolicy:false EnableStatelessNetPol:false EnableInterconnect:false EnableMultiExternalGateway:false EnablePersistentIPs:false EnableDNSNameResolver:false EnableServiceTemplateSupport:false EnableObservability:false EnableNetworkQoS:false AdvertisedUDNIsolationMode:strict EnableDynamicUDNAllocation:false UDNDeletionGracePeriod:2m0s} Kubernetes:{BootstrapKubeconfig: CertDir: CertDuration:10m0s Kubeconfig: CACert: CACertData: CAData:[] APIServer:https://api.kx-6952a9a844.hypershift.local:443 Token: TokenFile: CompatServiceCIDR: RawServiceCIDRs:172.30.0.0/16 ServiceCIDRs:[] OVNConfigNamespace:openshift-ovn-kubernetes OVNEmptyLbEvents:false RawNoHostSubnetNodes: NoHostSubnetNodes: HostNetworkNamespace:openshift-host-network DisableRequestedChassis:false PlatformType:AWS HealthzBindAddress:0.0.0.0:10256 CompatMetricsBindAddress: CompatOVNMetricsBindAddress: CompatMetricsEnablePprof:false DNSServiceNamespace:openshift-dns DNSServiceName:dns-default} Metrics:{BindAddress: OVNMetricsBindAddress: ExportOVSMetrics:false EnablePprof:false NodeServerPrivKey: NodeServerCert: EnableConfigDuration:false EnableScaleMetrics:false} OvnNorth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnnb_db.db exec:} OvnSouth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnsb_db.db exec:} Gateway:{Mode:shared Interface: GatewayAcceleratedInterface: EgressGWInterface: NextHop: VLANID:0 NodeportEnable:true DisableSNATMultipleGWs:false V4JoinSubnet:100.64.0.0/16 V6JoinSubnet:fd98::/64 V4MasqueradeSubnet:169.254.169.0/29 V6MasqueradeSubnet:fd69::/125 MasqueradeIPs:{V4OVNMasqueradeIP:169.254.169.1 V6OVNMasqueradeIP:fd69::1 V4HostMasqueradeIP:169.254.169.2 V6HostMasqueradeIP:fd69::2 V4HostETPLocalMasqueradeIP:169.254.169.3 V6HostETPLocalMasqueradeIP:fd69::3 V4DummyNextHopMasqueradeIP:169.254.169.4 V6DummyNextHopMasqueradeIP:fd69::4 V4OVNServiceHairpinMasqueradeIP:169.254.169.5 V6OVNServiceHairpinMasqueradeIP:fd69::5} DisablePacketMTUCheck:false RouterSubnet: SingleNode:false DisableForwarding:false AllowNoUplink:false EphemeralPortRange:} MasterHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} ClusterMgrHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} HybridOverlay:{Enabled:false RawClusterSubnets: ClusterSubnets:[] VXLANPort:4789} OvnKubeNode:{Mode:full MgmtPortNetdev: MgmtPortDPResourceName:} ClusterManager:{V4TransitSubnet:100.88.0.0/16 V6TransitSubnet:fd97::/64} OvsPaths:{RunDir:/var/run/openvswitch/} NoOverlay:{OutboundSNAT: Routing:} ManagedBGP:{ASNumber:64512 Topology:}} I0428 15:37:44.273011 3647 kube.go:431] Waiting for certificate I0428 15:37:44.273036 3647 certificate_manager.go:422] "Certificate rotation is enabled" logger="kubernetes.io/kube-apiserver-client" I0428 15:37:44.273074 3647 certificate_manager.go:566] "Rotating certificates" logger="kubernetes.io/kube-apiserver-client" I0428 15:37:44.273737 3647 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0428 15:37:44.300021 3647 reflector.go:358] "Starting reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0428 15:37:44.300035 3647 reflector.go:404] "Listing and watching" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0428 15:37:44.302096 3647 reflector.go:436] "Caches populated" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0428 15:37:44.302196 3647 csr.go:274] "Certificate signing request is approved, waiting to be issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-fkg2r" I0428 15:37:44.309740 3647 csr.go:270] "Certificate signing request is issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-fkg2r" I0428 15:37:44.309796 3647 reflector.go:364] "Stopping reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0428 15:37:45.273152 3647 kube.go:438] Certificate found I0428 15:37:45.273744 3647 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0428 15:37:45.274677 3647 metrics.go:532] Starting metrics server at address "127.0.0.1:29103" I0428 15:37:45.314316 3647 certificate_manager.go:715] "Certificate rotation deadline determined" logger="kubernetes.io/kube-apiserver-client" expiration="2026-04-29 15:32:44 +0000 UTC" deadline="2026-04-29 11:31:54.988641063 +0000 UTC" I0428 15:37:45.314527 3647 certificate_manager.go:431] "Waiting for next certificate rotation" logger="kubernetes.io/kube-apiserver-client" sleep="19h54m9.674168799s" I0428 15:37:45.343722 3647 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0428 15:37:45.362990 3647 node_controller_manager.go:338] Starting the node network controller manager, Mode: full I0428 15:37:45.363001 3647 metrics.go:512] Create OVN Metrics Server on address: 127.0.0.1:29105 I0428 15:37:45.363062 3647 server.go:101] MetricServer registers OVS metrics I0428 15:37:45.363076 3647 factory.go:561] Starting watch factory I0428 15:37:45.363156 3647 reflector.go:358] "Starting reflector" type="*v1.Namespace" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:37:45.363176 3647 reflector.go:404] "Listing and watching" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:37:45.363173 3647 reflector.go:358] "Starting reflector" type="*v1.Service" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:37:45.363222 3647 reflector.go:404] "Listing and watching" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:37:45.363159 3647 reflector.go:358] "Starting reflector" type="*v1.Node" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:37:45.363252 3647 reflector.go:404] "Listing and watching" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:37:45.363160 3647 reflector.go:358] "Starting reflector" type="*v1.Pod" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:37:45.363370 3647 reflector.go:358] "Starting reflector" type="*v1.NetworkPolicy" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:37:45.363397 3647 reflector.go:404] "Listing and watching" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:37:45.363384 3647 reflector.go:404] "Listing and watching" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:37:45.363526 3647 server.go:105] MetricServer registers OVN DB metrics I0428 15:37:45.363788 3647 reflector.go:358] "Starting reflector" type="*v1.EndpointSlice" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:37:45.363893 3647 reflector.go:404] "Listing and watching" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:37:45.365604 3647 controller_manager.go:370] Starting the ovnkube controller I0428 15:37:45.365624 3647 controller_manager.go:375] Waiting up to 5m0s for NBDB zone to match: ip-10-0-1-106.ec2.internal I0428 15:37:45.365684 3647 controller_manager.go:395] NBDB zone sync took: 49.162µs I0428 15:37:45.365714 3647 factory.go:561] Starting watch factory I0428 15:37:45.368249 3647 ovn_db.go:330] /var/run/openvswitch/ovnnb_db.sock getting info failed: stat /var/run/openvswitch/ovnnb_db.sock: no such file or directory I0428 15:37:45.368268 3647 ovn_db.go:327] ovnnb_db.sock found at /var/run/ovn/ I0428 15:37:45.382124 3647 ovn_db.go:388] Found OVN NB DB: &{0x1e9e6e0 /etc/ovn/ovnnb_db.db OVN_Northbound 0} I0428 15:37:45.382156 3647 ovn_db.go:395] Found OVN SB DB: &{0x1e9e9a0 /etc/ovn/ovnsb_db.db OVN_Southbound 0} I0428 15:37:45.386289 3647 ovn_db.go:410] Found db is standalone, don't register db_cluster metrics I0428 15:37:45.386369 3647 server.go:109] MetricServer registers OVN Controller metrics I0428 15:37:45.390643 3647 server.go:113] MetricServer registers OVN Northd metrics I0428 15:37:45.394748 3647 metrics.go:519] OVN Metrics Server starts to run ... I0428 15:37:45.396652 3647 reflector.go:436] "Caches populated" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:37:45.399550 3647 reflector.go:436] "Caches populated" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:37:45.399614 3647 reflector.go:436] "Caches populated" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:37:45.399805 3647 reflector.go:436] "Caches populated" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:37:45.404343 3647 reflector.go:436] "Caches populated" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:37:45.427469 3647 reflector.go:436] "Caches populated" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:37:45.469285 3647 factory.go:1977] *v1.Namespace informer cache synced successfully I0428 15:37:45.469303 3647 factory.go:1977] *v1.Node informer cache synced successfully I0428 15:37:45.469307 3647 factory.go:1977] *v1.Service informer cache synced successfully I0428 15:37:45.469310 3647 factory.go:1977] *v1.EndpointSlice informer cache synced successfully I0428 15:37:45.469313 3647 factory.go:1977] *v1.Pod informer cache synced successfully I0428 15:37:45.469316 3647 factory.go:1977] *v1.NetworkPolicy informer cache synced successfully I0428 15:37:45.469392 3647 reflector.go:358] "Starting reflector" type="*v1alpha1.BaselineAdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0428 15:37:45.469411 3647 reflector.go:404] "Listing and watching" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0428 15:37:45.469421 3647 reflector.go:358] "Starting reflector" type="*v1alpha1.AdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0428 15:37:45.469439 3647 reflector.go:404] "Listing and watching" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0428 15:37:45.471682 3647 factory.go:1977] *v1.Namespace informer cache synced successfully I0428 15:37:45.471694 3647 factory.go:1977] *v1.Node informer cache synced successfully I0428 15:37:45.471711 3647 factory.go:1977] *v1.Service informer cache synced successfully I0428 15:37:45.471714 3647 factory.go:1977] *v1.EndpointSlice informer cache synced successfully I0428 15:37:45.471717 3647 factory.go:1977] *v1.Pod informer cache synced successfully I0428 15:37:45.471720 3647 factory.go:1977] *v1.NetworkPolicy informer cache synced successfully I0428 15:37:45.475224 3647 reflector.go:436] "Caches populated" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0428 15:37:45.475239 3647 reflector.go:436] "Caches populated" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0428 15:37:45.477167 3647 factory.go:1977] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0428 15:37:45.477178 3647 factory.go:1977] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0428 15:37:45.477230 3647 reflector.go:358] "Starting reflector" type="*v1.EgressIP" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0428 15:37:45.477244 3647 reflector.go:404] "Listing and watching" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0428 15:37:45.484197 3647 reflector.go:436] "Caches populated" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0428 15:37:45.576043 3647 factory.go:1977] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0428 15:37:45.576062 3647 factory.go:1977] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0428 15:37:45.581226 3647 factory.go:1977] *v1.EgressIP informer cache synced successfully I0428 15:37:45.581310 3647 reflector.go:358] "Starting reflector" type="*v1.EgressFirewall" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0428 15:37:45.581330 3647 reflector.go:404] "Listing and watching" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0428 15:37:45.583541 3647 factory.go:1977] *v1.EgressIP informer cache synced successfully I0428 15:37:45.585202 3647 reflector.go:436] "Caches populated" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0428 15:37:45.587077 3647 factory.go:1977] *v1.EgressFirewall informer cache synced successfully I0428 15:37:45.587153 3647 reflector.go:358] "Starting reflector" type="*v1.EgressQoS" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0428 15:37:45.587171 3647 reflector.go:404] "Listing and watching" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0428 15:37:45.589524 3647 factory.go:1977] *v1.EgressFirewall informer cache synced successfully I0428 15:37:45.592144 3647 reflector.go:436] "Caches populated" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0428 15:37:45.592154 3647 factory.go:1977] *v1.EgressQoS informer cache synced successfully I0428 15:37:45.592255 3647 reflector.go:358] "Starting reflector" type="*v1.EgressService" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0428 15:37:45.592269 3647 reflector.go:404] "Listing and watching" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0428 15:37:45.595174 3647 factory.go:1977] *v1.EgressQoS informer cache synced successfully I0428 15:37:45.596008 3647 reflector.go:436] "Caches populated" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0428 15:37:45.598061 3647 factory.go:1977] *v1.EgressService informer cache synced successfully I0428 15:37:45.598120 3647 reflector.go:358] "Starting reflector" type="*v1.AdminPolicyBasedExternalRoute" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0428 15:37:45.598132 3647 reflector.go:404] "Listing and watching" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0428 15:37:45.600980 3647 factory.go:1977] *v1.EgressService informer cache synced successfully I0428 15:37:45.601161 3647 reflector.go:436] "Caches populated" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0428 15:37:45.603535 3647 factory.go:1977] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0428 15:37:45.603763 3647 reflector.go:358] "Starting reflector" type="*v1.NetworkAttachmentDefinition" resyncPeriod="0s" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0428 15:37:45.603783 3647 reflector.go:404] "Listing and watching" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0428 15:37:45.606295 3647 factory.go:1977] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0428 15:37:45.606513 3647 reflector.go:436] "Caches populated" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0428 15:37:45.609347 3647 factory.go:1977] *v1.NetworkAttachmentDefinition informer cache synced successfully I0428 15:37:45.609404 3647 reflector.go:358] "Starting reflector" type="*v1.ClusterUserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0428 15:37:45.609417 3647 reflector.go:404] "Listing and watching" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0428 15:37:45.609494 3647 reflector.go:358] "Starting reflector" type="*v1.UserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0428 15:37:45.609515 3647 reflector.go:404] "Listing and watching" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0428 15:37:45.611733 3647 factory.go:1977] *v1.NetworkAttachmentDefinition informer cache synced successfully I0428 15:37:45.612532 3647 reflector.go:436] "Caches populated" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0428 15:37:45.612577 3647 reflector.go:436] "Caches populated" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0428 15:37:45.614464 3647 factory.go:1977] *v1.UserDefinedNetwork informer cache synced successfully I0428 15:37:45.614477 3647 factory.go:1977] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0428 15:37:45.614486 3647 factory.go:680] Watch Factory start up complete, took: 251.415329ms I0428 15:37:45.614570 3647 default_node_network_controller.go:171] Enable node proxy healthz server on 0.0.0.0:10256 I0428 15:37:45.617227 3647 factory.go:1977] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0428 15:37:45.617242 3647 factory.go:1977] *v1.UserDefinedNetwork informer cache synced successfully I0428 15:37:45.617250 3647 factory.go:680] Watch Factory start up complete, took: 251.552495ms I0428 15:37:45.617258 3647 controller_manager.go:406] Waiting up to 5m0s for a node to have "ip-10-0-1-106.ec2.internal" zone I0428 15:37:45.662161 3647 default_node_network_controller.go:710] Initializing the default node network controller I0428 15:37:45.736049 3647 udn_isolation.go:101] Starting UDN host isolation manager I0428 15:37:45.742234 3647 udn_isolation.go:112] Found kubelet cgroup path: system.slice/kubelet.service I0428 15:37:45.798749 3647 controller.go:133] Adding controller udn-host-isolation-manager event handlers I0428 15:37:45.799081 3647 shared_informer.go:349] "Waiting for caches to sync" controller="udn-host-isolation-manager" I0428 15:37:45.799107 3647 shared_informer.go:356] "Caches are synced" controller="udn-host-isolation-manager" I0428 15:37:45.833942 3647 controller.go:157] Starting controller udn-host-isolation-manager with 1 workers I0428 15:37:45.834007 3647 default_node_network_controller.go:809] Node ip-10-0-1-106.ec2.internal ready for ovn initialization with subnet 10.130.0.0/23 I0428 15:37:45.839778 3647 kube.go:131] Setting annotations map[k8s.ovn.org/node-encap-ips:["10.0.1.106"] k8s.ovn.org/zone-name:ip-10-0-1-106.ec2.internal] on node ip-10-0-1-106.ec2.internal I0428 15:37:45.866472 3647 gateway_init.go:234] Initializing Gateway Functionality for Gateway PreStart I0428 15:37:45.867638 3647 controller_manager.go:429] Waiting for node in zone sync took: 250.368492ms I0428 15:37:45.873162 3647 helper_linux.go:93] Provided gateway interface "br-ex", found as index: 5 I0428 15:37:45.873304 3647 helper_linux.go:118] Found default gateway interface br-ex 10.0.1.1 I0428 15:37:45.873423 3647 gateway_init.go:263] Preparing Gateway I0428 15:37:45.873434 3647 gateway_shared_intf.go:1687] Creating new gateway I0428 15:37:45.876987 3647 controller_manager.go:611] Switching to transit router for layer2 networks I0428 15:37:45.877000 3647 kube.go:131] Setting annotations map[k8s.ovn.org/layer2-topology-version:2.0] on node ip-10-0-1-106.ec2.internal I0428 15:37:45.894392 3647 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0428 15:37:45.900735 3647 controller_manager.go:290] SCTP support detected in OVN I0428 15:37:45.901422 3647 services_controller.go:69] Creating services controller for network=default I0428 15:37:45.901554 3647 udn_enabled_svc.go:80] Waiting for service informer to sync I0428 15:37:45.901721 3647 controller.go:133] Adding controller [zone-nad-controller NAD controller] event handlers I0428 15:37:45.901880 3647 shared_informer.go:349] "Waiting for caches to sync" controller="[zone-nad-controller NAD controller]" I0428 15:37:45.901894 3647 shared_informer.go:356] "Caches are synced" controller="[zone-nad-controller NAD controller]" I0428 15:37:45.901903 3647 nad_controller.go:541] [zone-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0428 15:37:45.902160 3647 controller.go:157] Starting controller [zone-nad-controller NAD controller] with 1 workers I0428 15:37:45.902229 3647 udn_enabled_svc.go:95] Performing full resync I0428 15:37:45.902266 3647 nad_controller.go:580] [zone-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 62.928µs I0428 15:37:45.902319 3647 network_controller.go:373] [zone-nad-controller network controller]: syncing all networks I0428 15:37:45.902376 3647 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 39.347µs I0428 15:37:45.902387 3647 network_controller.go:384] [zone-nad-controller network controller]: finished syncing all networks. Time taken: 71.666µs I0428 15:37:45.902393 3647 controller.go:157] Starting controller [zone-nad-controller network controller] with 1 workers I0428 15:37:45.902404 3647 nad_controller.go:379] [zone-nad-controller NAD controller]: started I0428 15:37:45.902416 3647 default_network_controller.go:325] Starting the default network controller I0428 15:37:45.902476 3647 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 14.305µs I0428 15:37:45.902617 3647 udn_enabled_svc.go:99] Waiting for handler to sync I0428 15:37:45.902633 3647 udn_enabled_svc.go:104] Starting worker I0428 15:37:45.912881 3647 default_network_controller.go:395] Cleaning External Gateway ECMP routes I0428 15:37:45.912948 3647 repair.go:33] Syncing exgw routes took 52.353µs I0428 15:37:45.912972 3647 default_network_controller.go:414] Starting all the Watchers... I0428 15:37:45.913134 3647 namespace.go:98] [openshift-route-controller-manager] adding namespace I0428 15:37:45.913136 3647 namespace.go:98] [openshift-kube-storage-version-migrator] adding namespace I0428 15:37:45.913142 3647 namespace.go:98] [openshift-console-operator] adding namespace I0428 15:37:45.913150 3647 namespace.go:98] [openshift-logging] adding namespace I0428 15:37:45.913156 3647 namespace.go:98] [openshift-backplane-mcs-tier-two] adding namespace I0428 15:37:45.913166 3647 namespace.go:98] [openshift-authentication] adding namespace I0428 15:37:45.913170 3647 namespace.go:98] [openshift-marketplace] adding namespace I0428 15:37:45.913175 3647 namespace.go:98] [openshift-dns] adding namespace I0428 15:37:45.913182 3647 namespace.go:98] [openshift-multus] adding namespace I0428 15:37:45.913184 3647 namespace.go:98] [open-cluster-management-agent-addon] adding namespace I0428 15:37:45.913190 3647 namespace.go:98] [openshift-cloud-controller-manager] adding namespace I0428 15:37:45.913199 3647 namespace.go:98] [openshift-cloud-network-config-controller] adding namespace I0428 15:37:45.913250 3647 namespace.go:98] [openshift-console] adding namespace I0428 15:37:45.913261 3647 namespace.go:98] [openshift-cluster-node-tuning-operator] adding namespace I0428 15:37:45.913311 3647 namespace.go:98] [openshift-customer-monitoring] adding namespace I0428 15:37:45.913959 3647 namespace.go:102] [openshift-route-controller-manager] adding namespace took 812.375µs I0428 15:37:45.913975 3647 namespace.go:98] [openshift-kube-scheduler-operator] adding namespace I0428 15:37:45.914490 3647 namespace.go:102] [openshift-kube-scheduler-operator] adding namespace took 509.318µs I0428 15:37:45.914508 3647 namespace.go:98] [openshift-console-user-settings] adding namespace I0428 15:37:45.915303 3647 namespace.go:102] [openshift-kube-storage-version-migrator] adding namespace took 2.151631ms I0428 15:37:45.915326 3647 namespace.go:98] [openshift-operators-redhat] adding namespace I0428 15:37:45.915961 3647 namespace.go:102] [openshift-operators-redhat] adding namespace took 626.688µs I0428 15:37:45.915981 3647 namespace.go:98] [openshift-backplane-managed-scripts] adding namespace I0428 15:37:45.916482 3647 namespace.go:102] [openshift-console-operator] adding namespace took 3.326057ms I0428 15:37:45.916502 3647 namespace.go:98] [openshift-kube-apiserver-operator] adding namespace I0428 15:37:45.917079 3647 namespace.go:102] [openshift-backplane-mcs-tier-two] adding namespace took 3.91304ms I0428 15:37:45.917094 3647 namespace.go:98] [openshift-cluster-csi-drivers] adding namespace I0428 15:37:45.917600 3647 namespace.go:102] [openshift-logging] adding namespace took 4.43834ms I0428 15:37:45.917617 3647 namespace.go:98] [openshift-monitoring] adding namespace W0428 15:37:45.918296 3647 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-multus/network-metrics-daemon-c85fc: pod openshift-multus/network-metrics-daemon-c85fc: no pod IPs found I0428 15:37:45.918351 3647 namespace.go:102] [openshift-authentication] adding namespace took 5.17853ms W0428 15:37:45.918357 3647 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-multus/network-metrics-daemon-xdscj: pod openshift-multus/network-metrics-daemon-xdscj: no pod IPs found I0428 15:37:45.918368 3647 namespace.go:98] [openshift-must-gather-operator] adding namespace I0428 15:37:45.918983 3647 namespace.go:102] [openshift-multus] adding namespace took 5.794868ms I0428 15:37:45.918996 3647 namespace.go:98] [openshift-network-console] adding namespace I0428 15:37:45.919592 3647 namespace.go:102] [openshift-marketplace] adding namespace took 6.411891ms I0428 15:37:45.919607 3647 namespace.go:98] [openshift-backplane-cse] adding namespace I0428 15:37:45.920237 3647 namespace.go:102] [openshift-dns] adding namespace took 7.050297ms I0428 15:37:45.920252 3647 namespace.go:98] [openshift] adding namespace I0428 15:37:45.920969 3647 namespace.go:102] [open-cluster-management-agent-addon] adding namespace took 7.775348ms I0428 15:37:45.920982 3647 namespace.go:98] [openshift-cloud-credential-operator] adding namespace I0428 15:37:45.921611 3647 namespace.go:102] [openshift-cloud-controller-manager] adding namespace took 8.41318ms I0428 15:37:45.921623 3647 namespace.go:98] [open-cluster-management-2pv84nia58gje8norbmmjqtj26oprnd8] adding namespace I0428 15:37:45.922214 3647 namespace.go:102] [openshift-cloud-network-config-controller] adding namespace took 9.004357ms I0428 15:37:45.922237 3647 namespace.go:98] [openshift-network-diagnostics] adding namespace I0428 15:37:45.922796 3647 namespace.go:102] [openshift-console] adding namespace took 9.531864ms I0428 15:37:45.922818 3647 namespace.go:98] [openshift-kube-controller-manager-operator] adding namespace I0428 15:37:45.923267 3647 namespace.go:102] [openshift-cluster-node-tuning-operator] adding namespace took 9.993798ms I0428 15:37:45.923288 3647 namespace.go:98] [openshift-ingress] adding namespace I0428 15:37:45.923828 3647 namespace.go:102] [openshift-customer-monitoring] adding namespace took 10.507714ms I0428 15:37:45.923847 3647 namespace.go:98] [openshift-operators] adding namespace I0428 15:37:45.924268 3647 namespace.go:102] [openshift-console-user-settings] adding namespace took 9.751934ms I0428 15:37:45.924288 3647 namespace.go:98] [openshift-kube-scheduler] adding namespace I0428 15:37:45.924680 3647 namespace.go:102] [openshift-backplane-managed-scripts] adding namespace took 8.69331ms I0428 15:37:45.924693 3647 namespace.go:98] [openshift-controller-manager-operator] adding namespace I0428 15:37:45.925152 3647 namespace.go:102] [openshift-kube-apiserver-operator] adding namespace took 8.643731ms I0428 15:37:45.925170 3647 namespace.go:98] [openshift-config-managed] adding namespace I0428 15:37:45.925619 3647 namespace.go:102] [openshift-cluster-csi-drivers] adding namespace took 8.518265ms I0428 15:37:45.925640 3647 namespace.go:98] [openshift-user-workload-monitoring] adding namespace I0428 15:37:45.926266 3647 namespace.go:102] [openshift-monitoring] adding namespace took 8.640289ms I0428 15:37:45.926289 3647 namespace.go:98] [openshift-apiserver-operator] adding namespace I0428 15:37:45.926798 3647 namespace.go:102] [openshift-must-gather-operator] adding namespace took 8.423215ms I0428 15:37:45.926812 3647 namespace.go:98] [openshift-service-ca] adding namespace I0428 15:37:45.927427 3647 namespace.go:102] [openshift-network-console] adding namespace took 8.424361ms I0428 15:37:45.927445 3647 namespace.go:98] [openshift-ingress-operator] adding namespace I0428 15:37:45.928103 3647 namespace.go:102] [openshift-backplane-cse] adding namespace took 8.488096ms I0428 15:37:45.928125 3647 namespace.go:98] [openshift-cluster-samples-operator] adding namespace I0428 15:37:45.928690 3647 namespace.go:102] [openshift] adding namespace took 8.431112ms I0428 15:37:45.928726 3647 namespace.go:98] [openshift-backplane-cee] adding namespace I0428 15:37:45.929333 3647 namespace.go:102] [openshift-cloud-credential-operator] adding namespace took 8.345304ms I0428 15:37:45.929351 3647 namespace.go:98] [openshift-kube-apiserver] adding namespace I0428 15:37:45.929991 3647 namespace.go:102] [open-cluster-management-2pv84nia58gje8norbmmjqtj26oprnd8] adding namespace took 8.361938ms W0428 15:37:45.929999 3647 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-gzrrf: pod openshift-network-diagnostics/network-check-target-gzrrf: no pod IPs found I0428 15:37:45.930012 3647 namespace.go:98] [openshift-machine-api] adding namespace W0428 15:37:45.930015 3647 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-w6lhl: pod openshift-network-diagnostics/network-check-target-w6lhl: no pod IPs found I0428 15:37:45.930682 3647 namespace.go:102] [openshift-network-diagnostics] adding namespace took 8.436306ms I0428 15:37:45.930716 3647 namespace.go:98] [default] adding namespace I0428 15:37:45.931398 3647 namespace.go:102] [openshift-kube-controller-manager-operator] adding namespace took 8.568297ms I0428 15:37:45.931419 3647 namespace.go:98] [openshift-package-operator] adding namespace I0428 15:37:45.932006 3647 namespace.go:102] [openshift-ingress] adding namespace took 8.71055ms I0428 15:37:45.932023 3647 namespace.go:98] [kube-public] adding namespace I0428 15:37:45.932497 3647 namespace.go:102] [openshift-operators] adding namespace took 8.642955ms I0428 15:37:45.932512 3647 namespace.go:98] [openshift-machine-config-operator] adding namespace I0428 15:37:45.932999 3647 namespace.go:102] [openshift-kube-scheduler] adding namespace took 8.705414ms I0428 15:37:45.933011 3647 namespace.go:98] [openshift-apiserver] adding namespace I0428 15:37:45.933481 3647 namespace.go:102] [openshift-controller-manager-operator] adding namespace took 8.783879ms I0428 15:37:45.933493 3647 namespace.go:98] [openshift-host-network] adding namespace I0428 15:37:45.934008 3647 namespace.go:102] [openshift-config-managed] adding namespace took 8.831011ms I0428 15:37:45.934021 3647 namespace.go:98] [openshift-deployment-validation-operator] adding namespace I0428 15:37:45.934469 3647 namespace.go:102] [openshift-user-workload-monitoring] adding namespace took 8.820993ms I0428 15:37:45.934488 3647 namespace.go:98] [openshift-backplane-srep-ro] adding namespace I0428 15:37:45.934949 3647 namespace.go:102] [openshift-apiserver-operator] adding namespace took 8.640954ms I0428 15:37:45.934963 3647 namespace.go:98] [openshift-ingress-canary] adding namespace I0428 15:37:45.935324 3647 namespace.go:102] [openshift-service-ca] adding namespace took 8.506919ms I0428 15:37:45.935343 3647 namespace.go:98] [openshift-backplane-csm] adding namespace I0428 15:37:45.935935 3647 namespace.go:102] [openshift-ingress-operator] adding namespace took 8.481783ms I0428 15:37:45.935956 3647 namespace.go:98] [openshift-cluster-storage-operator] adding namespace I0428 15:37:45.936817 3647 namespace.go:102] [openshift-cluster-samples-operator] adding namespace took 8.686497ms I0428 15:37:45.936831 3647 namespace.go:98] [openshift-backplane-srep] adding namespace I0428 15:37:45.937282 3647 namespace.go:102] [openshift-backplane-cee] adding namespace took 8.547828ms I0428 15:37:45.937296 3647 namespace.go:98] [openshift-controller-manager] adding namespace I0428 15:37:45.938446 3647 namespace.go:102] [openshift-kube-apiserver] adding namespace took 9.084762ms I0428 15:37:45.938466 3647 namespace.go:98] [openshift-authentication-operator] adding namespace I0428 15:37:45.939009 3647 namespace.go:102] [openshift-machine-api] adding namespace took 8.990791ms I0428 15:37:45.939022 3647 namespace.go:98] [openshift-infra] adding namespace I0428 15:37:45.939499 3647 namespace.go:102] [default] adding namespace took 8.775977ms I0428 15:37:45.939516 3647 namespace.go:98] [openshift-kube-storage-version-migrator-operator] adding namespace I0428 15:37:45.939995 3647 namespace.go:102] [openshift-package-operator] adding namespace took 8.566464ms I0428 15:37:45.940018 3647 namespace.go:98] [openshift-etcd] adding namespace I0428 15:37:45.940443 3647 namespace.go:102] [kube-public] adding namespace took 8.413112ms I0428 15:37:45.940466 3647 namespace.go:98] [kube-node-lease] adding namespace I0428 15:37:45.941036 3647 namespace.go:102] [openshift-machine-config-operator] adding namespace took 8.517399ms I0428 15:37:45.941055 3647 namespace.go:98] [openshift-config-operator] adding namespace I0428 15:37:45.941650 3647 namespace.go:102] [openshift-apiserver] adding namespace took 8.632838ms I0428 15:37:45.942300 3647 namespace.go:102] [openshift-host-network] adding namespace took 8.800887ms I0428 15:37:45.942313 3647 namespace.go:98] [openshift-cluster-version] adding namespace I0428 15:37:45.942813 3647 namespace.go:102] [openshift-deployment-validation-operator] adding namespace took 8.786801ms I0428 15:37:45.942825 3647 namespace.go:98] [dedicated-admin] adding namespace I0428 15:37:45.943329 3647 namespace.go:102] [openshift-backplane-srep-ro] adding namespace took 8.835701ms I0428 15:37:45.943343 3647 namespace.go:98] [openshift-network-operator] adding namespace I0428 15:37:45.943938 3647 namespace.go:102] [openshift-ingress-canary] adding namespace took 8.966943ms I0428 15:37:45.943951 3647 namespace.go:98] [openshift-service-ca-operator] adding namespace I0428 15:37:45.944469 3647 namespace.go:102] [openshift-backplane-csm] adding namespace took 9.118417ms I0428 15:37:45.944487 3647 namespace.go:98] [openshift-ovn-kubernetes] adding namespace I0428 15:37:45.944967 3647 namespace.go:102] [openshift-cluster-storage-operator] adding namespace took 9.005338ms I0428 15:37:45.944981 3647 namespace.go:98] [openshift-config] adding namespace I0428 15:37:45.945437 3647 namespace.go:102] [openshift-backplane-srep] adding namespace took 8.598873ms I0428 15:37:45.945450 3647 namespace.go:98] [openshift-kube-controller-manager] adding namespace I0428 15:37:45.945901 3647 namespace.go:102] [openshift-controller-manager] adding namespace took 8.600224ms I0428 15:37:45.945920 3647 namespace.go:98] [openshift-backplane-mobb] adding namespace I0428 15:37:45.946333 3647 namespace.go:102] [openshift-authentication-operator] adding namespace took 7.860245ms I0428 15:37:45.946346 3647 namespace.go:98] [openshift-backplane-tam] adding namespace I0428 15:37:45.946882 3647 namespace.go:102] [openshift-infra] adding namespace took 7.853391ms I0428 15:37:45.946902 3647 namespace.go:98] [openshift-node] adding namespace I0428 15:37:45.947502 3647 namespace.go:102] [openshift-kube-storage-version-migrator-operator] adding namespace took 7.979234ms I0428 15:37:45.948091 3647 namespace.go:102] [openshift-etcd] adding namespace took 8.066158ms I0428 15:37:45.948109 3647 namespace.go:98] [openshift-insights] adding namespace I0428 15:37:45.948623 3647 namespace.go:102] [kube-node-lease] adding namespace took 8.149285ms I0428 15:37:45.948641 3647 namespace.go:98] [openshift-backplane] adding namespace I0428 15:37:45.949181 3647 namespace.go:102] [openshift-config-operator] adding namespace took 8.119301ms I0428 15:37:45.949197 3647 namespace.go:98] [openshift-cluster-machine-approver] adding namespace I0428 15:37:45.949674 3647 namespace.go:102] [openshift-cluster-version] adding namespace took 7.356401ms I0428 15:37:45.950292 3647 namespace.go:102] [dedicated-admin] adding namespace took 7.45939ms I0428 15:37:45.950812 3647 namespace.go:102] [openshift-network-operator] adding namespace took 7.462521ms I0428 15:37:45.950833 3647 namespace.go:98] [kube-system] adding namespace I0428 15:37:45.951405 3647 namespace.go:102] [openshift-service-ca-operator] adding namespace took 7.449171ms I0428 15:37:45.951422 3647 namespace.go:98] [openshift-network-node-identity] adding namespace I0428 15:37:45.951892 3647 namespace.go:102] [openshift-ovn-kubernetes] adding namespace took 7.397811ms I0428 15:37:45.951908 3647 namespace.go:98] [openshift-backplane-lpsre] adding namespace I0428 15:37:45.952486 3647 namespace.go:102] [openshift-config] adding namespace took 7.496872ms I0428 15:37:45.953076 3647 namespace.go:102] [openshift-kube-controller-manager] adding namespace took 7.620104ms I0428 15:37:45.953096 3647 namespace.go:98] [openshift-image-registry] adding namespace I0428 15:37:45.953609 3647 namespace.go:102] [openshift-backplane-mobb] adding namespace took 7.681764ms I0428 15:37:45.954206 3647 namespace.go:102] [openshift-backplane-tam] adding namespace took 7.855501ms I0428 15:37:45.954734 3647 namespace.go:102] [openshift-node] adding namespace took 7.824647ms I0428 15:37:45.955441 3647 namespace.go:102] [openshift-insights] adding namespace took 7.32397ms I0428 15:37:45.955460 3647 namespace.go:98] [openshift-dns-operator] adding namespace I0428 15:37:45.955978 3647 namespace.go:102] [openshift-backplane] adding namespace took 7.330802ms I0428 15:37:45.955993 3647 namespace.go:98] [openshift-operator-lifecycle-manager] adding namespace I0428 15:37:45.956508 3647 namespace.go:102] [openshift-cluster-machine-approver] adding namespace took 7.30636ms I0428 15:37:45.957073 3647 namespace.go:102] [kube-system] adding namespace took 6.232932ms I0428 15:37:45.957587 3647 namespace.go:102] [openshift-network-node-identity] adding namespace took 6.157214ms I0428 15:37:45.958210 3647 namespace.go:102] [openshift-backplane-lpsre] adding namespace took 6.296033ms I0428 15:37:45.958790 3647 namespace.go:102] [openshift-image-registry] adding namespace took 5.686858ms I0428 15:37:45.959357 3647 namespace.go:102] [openshift-dns-operator] adding namespace took 3.890769ms I0428 15:37:45.959938 3647 namespace.go:102] [openshift-operator-lifecycle-manager] adding namespace took 3.938801ms W0428 15:37:45.960035 3647 master.go:458] Unable to parse local node ip-10-0-1-106.ec2.internal chassis-id annotation. Chassis may be removed during sync I0428 15:37:45.960048 3647 master.go:481] Removing stale chassis with ID/Name: fc46d8bd-6618-4c55-a52b-039a7d5ef213, hostname: ip-10-0-1-106.ec2.internal I0428 15:37:45.960501 3647 master.go:548] Adding or Updating local node "ip-10-0-1-106.ec2.internal" for network "default" I0428 15:37:45.962048 3647 zone_ic_handler.go:195] Time taken to create transit switch: 526.485µs I0428 15:37:45.962070 3647 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-0-9.ec2.internal for the network default E0428 15:37:45.962098 3647 obj_retry.go:575] Failed to create *v1.Node ip-10-0-2-7.ec2.internal, error: adding or updating remote node IC resources ip-10-0-2-7.ec2.internal failed, err - ensuring transit switch for remote zone node ip-10-0-2-7.ec2.internal for the network default failed : err - failed to create/update transit switch transit_switch: error in transact with ops [{Op:wait Table:Logical_Switch Row:map[] Rows:[map[name:transit_switch]] Columns:[name] Mutations:[] Timeout:0xc0099f1838 Where:[where column name == transit_switch] Until:!= Durable: Comment: Lock: UUID: UUIDName:} {Op:insert Table:Logical_Switch Row:map[name:transit_switch other_config:{GoMap:map[interconn-ts:transit_switch mcast_flood_unregistered:true mcast_querier:false mcast_snoop:true requested-tnl-key:16711683]}] Rows:[] Columns:[] Mutations:[] Timeout: Where:[] Until: Durable: Comment: Lock: UUID: UUIDName:u4263010236}] results [{Count:0 Error:timed out Details:"where" clause test failed UUID:{GoUUID:} Rows:[]} {Count:0 Error: Details: UUID:{GoUUID:} Rows:[]}] and errors [timed out: "where" clause test failed]: 1 ovsdb operations failed I0428 15:37:45.962174 3647 event.go:377] Event(v1.ObjectReference{Kind:"Node", Namespace:"", Name:"ip-10-0-2-7.ec2.internal", UID:"4ce02079-fef7-4aca-8635-88b8b194d30a", APIVersion:"v1", ResourceVersion:"8742", FieldPath:""}): type: 'Warning' reason: 'ErrorAddingResource' adding or updating remote node IC resources ip-10-0-2-7.ec2.internal failed, err - ensuring transit switch for remote zone node ip-10-0-2-7.ec2.internal for the network default failed : err - failed to create/update transit switch transit_switch: error in transact with ops [{Op:wait Table:Logical_Switch Row:map[] Rows:[map[name:transit_switch]] Columns:[name] Mutations:[] Timeout:0xc0099f1838 Where:[where column name == transit_switch] Until:!= Durable: Comment: Lock: UUID: UUIDName:} {Op:insert Table:Logical_Switch Row:map[name:transit_switch other_config:{GoMap:map[interconn-ts:transit_switch mcast_flood_unregistered:true mcast_querier:false mcast_snoop:true requested-tnl-key:16711683]}] Rows:[] Columns:[] Mutations:[] Timeout: Where:[] Until: Durable: Comment: Lock: UUID: UUIDName:u4263010236}] results [{Count:0 Error:timed out Details:"where" clause test failed UUID:{GoUUID:} Rows:[]} {Count:0 Error: Details: UUID:{GoUUID:} Rows:[]}] and errors [timed out: "where" clause test failed]: 1 ovsdb operations failed I0428 15:37:45.963324 3647 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-0-9.ec2.internal" on network "default" took: 1.821012ms I0428 15:37:45.964822 3647 zone_ic_handler.go:203] Creating interconnect resources for local zone node ip-10-0-1-106.ec2.internal for the network default I0428 15:37:45.964989 3647 zone_ic_handler.go:195] Time taken to create transit switch: 152.169µs I0428 15:37:45.965653 3647 default_network_controller.go:823] Node add failed for ip-10-0-1-106.ec2.internal, will try again later: [k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-106.ec2.internal, error getting gateway config for node ip-10-0-1-106.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-106.ec2.internal", failed to update chassis to local for local node ip-10-0-1-106.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-1-106.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-106.ec2.internal] E0428 15:37:45.965693 3647 obj_retry.go:575] Failed to create *v1.Node ip-10-0-1-106.ec2.internal, error: [k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-106.ec2.internal, error getting gateway config for node ip-10-0-1-106.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-106.ec2.internal", failed to update chassis to local for local node ip-10-0-1-106.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-1-106.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-106.ec2.internal] I0428 15:37:45.965762 3647 services_controller.go:192] Starting controller ovn-lb-controller for network=default I0428 15:37:45.965764 3647 event.go:377] Event(v1.ObjectReference{Kind:"Node", Namespace:"", Name:"ip-10-0-1-106.ec2.internal", UID:"fd6b73d3-805d-4a23-95f0-9141f4bb6178", APIVersion:"v1", ResourceVersion:"9543", FieldPath:""}): type: 'Warning' reason: 'ErrorAddingResource' [k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-106.ec2.internal, error getting gateway config for node ip-10-0-1-106.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-106.ec2.internal", failed to update chassis to local for local node ip-10-0-1-106.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-1-106.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-106.ec2.internal] I0428 15:37:45.965795 3647 services_controller.go:200] Waiting for node tracker handler to sync for network=default I0428 15:37:45.965808 3647 shared_informer.go:349] "Waiting for caches to sync" controller="node-tracker-controller" I0428 15:37:45.965834 3647 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-9.ec2.internal in network "default" I0428 15:37:45.965887 3647 node_tracker.go:169] Node ip-10-0-0-9.ec2.internal switch + router changed, syncing services in network "default" I0428 15:37:45.965902 3647 services_controller.go:574] Full service sync requested for network=default I0428 15:37:45.965916 3647 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-106.ec2.internal in network "default" I0428 15:37:45.965926 3647 node_tracker.go:235] Node ip-10-0-1-106.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-106.ec2.internal" I0428 15:37:45.965933 3647 node_tracker.go:169] Node ip-10-0-1-106.ec2.internal switch + router changed, syncing services in network "default" I0428 15:37:45.965937 3647 services_controller.go:574] Full service sync requested for network=default I0428 15:37:45.965942 3647 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-7.ec2.internal in network "default" I0428 15:37:45.965969 3647 node_tracker.go:169] Node ip-10-0-2-7.ec2.internal switch + router changed, syncing services in network "default" I0428 15:37:45.965976 3647 services_controller.go:574] Full service sync requested for network=default I0428 15:37:45.986651 3647 iptables.go:108] Creating table: filter chain: FORWARD I0428 15:37:45.993468 3647 iptables.go:108] Creating table: filter chain: OUTPUT I0428 15:37:46.000543 3647 gateway_shared_intf.go:1816] Gateway Creation Complete I0428 15:37:46.000557 3647 kube.go:131] Setting annotations map[k8s.ovn.org/gateway-mtu-support: k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-1-106.ec2.internal","mac-address":"0a:ff:d0:6a:d3:f7","ip-addresses":["10.0.1.106/24"],"ip-address":"10.0.1.106/24","next-hops":["10.0.1.1"],"next-hop":"10.0.1.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:fc46d8bd-6618-4c55-a52b-039a7d5ef213 k8s.ovn.org/node-encap-ips:["10.0.1.106"] k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.1.106/24"} k8s.ovn.org/zone-name:ip-10-0-1-106.ec2.internal] on node ip-10-0-1-106.ec2.internal I0428 15:37:46.017875 3647 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-106.ec2.internal in network "default" I0428 15:37:46.017898 3647 master.go:548] Adding or Updating local node "ip-10-0-1-106.ec2.internal" for network "default" I0428 15:37:46.017934 3647 node_tracker.go:169] Node ip-10-0-1-106.ec2.internal switch + router changed, syncing services in network "default" I0428 15:37:46.017945 3647 services_controller.go:574] Full service sync requested for network=default I0428 15:37:46.022231 3647 default_node_network_controller.go:519] Node connection status = connected W0428 15:37:46.022694 3647 gateway.go:207] Unable to fetch podIPs for pod openshift-network-diagnostics/network-check-target-w6lhl: pod openshift-network-diagnostics/network-check-target-w6lhl: no pod IPs found W0428 15:37:46.022736 3647 gateway.go:207] Unable to fetch podIPs for pod openshift-multus/network-metrics-daemon-xdscj: pod openshift-multus/network-metrics-daemon-xdscj: no pod IPs found I0428 15:37:46.023852 3647 zone_ic_handler.go:203] Creating interconnect resources for local zone node ip-10-0-1-106.ec2.internal for the network default I0428 15:37:46.024052 3647 zone_ic_handler.go:195] Time taken to create transit switch: 187.524µs I0428 15:37:46.066266 3647 shared_informer.go:356] "Caches are synced" controller="node-tracker-controller" I0428 15:37:46.066286 3647 services_controller.go:208] Setting up event handlers for services for network=default I0428 15:37:46.066399 3647 services_controller.go:218] Setting up event handlers for endpoint slices for network=default I0428 15:37:46.066481 3647 services_controller.go:233] Waiting for service and endpoint handlers to sync for network=default I0428 15:37:46.066502 3647 shared_informer.go:349] "Waiting for caches to sync" controller="ovn-lb-controller" I0428 15:37:46.167605 3647 shared_informer.go:356] "Caches are synced" controller="ovn-lb-controller" I0428 15:37:46.167629 3647 repair.go:60] Starting repairing loop for services I0428 15:37:46.167745 3647 repair.go:131] Deleted 0 stale service LBs I0428 15:37:46.167770 3647 repair.go:137] Deleted 0 stale Chassis Template Vars I0428 15:37:46.167788 3647 repair.go:62] Finished repairing loop for services: 162.226µs I0428 15:37:46.167811 3647 services_controller.go:363] Controller cache of 0 load balancers initialized for 0 services for network=default I0428 15:37:46.167821 3647 services_controller.go:254] Starting workers for network=default I0428 15:37:46.168603 3647 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-w6lhl] creating logical port openshift-network-diagnostics_network-check-target-w6lhl for pod on switch ip-10-0-1-106.ec2.internal I0428 15:37:46.168754 3647 kube.go:256] Updating pod openshift-network-diagnostics/network-check-target-w6lhl W0428 15:37:46.169925 3647 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog W0428 15:37:46.174338 3647 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0428 15:37:46.186086 3647 pod.go:62] [openshift-network-diagnostics/network-check-target-w6lhl] pod update took 17.4144ms I0428 15:37:46.186110 3647 base_network_controller_pods.go:951] [default/openshift-network-diagnostics/network-check-target-w6lhl] addLogicalPort annotation time took 17.440286ms I0428 15:37:46.187094 3647 pods.go:271] [openshift-network-diagnostics/network-check-target-w6lhl] addLogicalPort took 18.508277ms, libovsdb time 775.201µs I0428 15:37:46.187174 3647 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-xdscj] creating logical port openshift-multus_network-metrics-daemon-xdscj for pod on switch ip-10-0-1-106.ec2.internal I0428 15:37:46.187235 3647 kube.go:256] Updating pod openshift-multus/network-metrics-daemon-xdscj I0428 15:37:46.204869 3647 pod.go:62] [openshift-multus/network-metrics-daemon-xdscj] pod update took 17.65487ms I0428 15:37:46.204894 3647 base_network_controller_pods.go:951] [default/openshift-multus/network-metrics-daemon-xdscj] addLogicalPort annotation time took 17.682811ms I0428 15:37:46.205871 3647 pods.go:271] [openshift-multus/network-metrics-daemon-xdscj] addLogicalPort took 18.702854ms, libovsdb time 736.811µs I0428 15:37:46.206359 3647 admin_network_policy_controller.go:227] Starting controller default-network-controller I0428 15:37:46.206384 3647 shared_informer.go:349] "Waiting for caches to sync" controller="default-network-controller" I0428 15:37:46.206398 3647 shared_informer.go:356] "Caches are synced" controller="default-network-controller" I0428 15:37:46.206402 3647 admin_network_policy_controller.go:237] Repairing Admin Network Policies I0428 15:37:46.206594 3647 repair.go:29] Repairing admin network policies took 186.603µs I0428 15:37:46.206725 3647 repair.go:92] Repairing baseline admin network policies took 120.687µs I0428 15:37:46.207218 3647 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-pprof for network default I0428 15:37:46.207239 3647 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-pprof added to peer address sets [] I0428 15:37:46.209020 3647 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-pprof resources completed, update namespace loglevel I0428 15:37:46.209054 3647 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-webhook for network default I0428 15:37:46.209067 3647 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-webhook added to peer address sets [] I0428 15:37:46.209603 3647 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-webhook resources completed, update namespace loglevel I0428 15:37:46.209623 3647 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles for network default I0428 15:37:46.209636 3647 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles added to peer address sets [] I0428 15:37:46.211907 3647 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles resources completed, update namespace loglevel I0428 15:37:46.211930 3647 base_network_controller_policy.go:1179] Adding network policy openshift-operators/default-allow-all for network default I0428 15:37:46.211941 3647 base_network_controller_policy.go:1054] Policy openshift-operators/default-allow-all added to peer address sets [] I0428 15:37:46.213894 3647 base_network_controller_policy.go:1232] Create network policy openshift-operators/default-allow-all resources completed, update namespace loglevel I0428 15:37:46.213921 3647 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range for network default I0428 15:37:46.213929 3647 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range added to peer address sets [] I0428 15:37:46.215682 3647 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range resources completed, update namespace loglevel I0428 15:37:46.215720 3647 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range for network default I0428 15:37:46.215729 3647 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range added to peer address sets [] I0428 15:37:46.216270 3647 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range resources completed, update namespace loglevel I0428 15:37:46.216293 3647 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/default-deny for network default I0428 15:37:46.216299 3647 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/default-deny added to peer address sets [] I0428 15:37:46.218420 3647 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/default-deny resources completed, update namespace loglevel I0428 15:37:46.218445 3647 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/default-deny-all-traffic for network default I0428 15:37:46.218454 3647 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/default-deny-all-traffic added to peer address sets [] I0428 15:37:46.220025 3647 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/default-deny-all-traffic resources completed, update namespace loglevel I0428 15:37:46.220045 3647 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/default-deny for network default I0428 15:37:46.220057 3647 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/default-deny added to peer address sets [] I0428 15:37:46.220364 3647 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/default-deny resources completed, update namespace loglevel I0428 15:37:46.220389 3647 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-to-dns for network default I0428 15:37:46.221168 3647 pod_selector_address_set.go:213] Created shared address set for pod selector LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},} I0428 15:37:46.221182 3647 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0428 15:37:46.221943 3647 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-to-dns resources completed, update namespace loglevel I0428 15:37:46.221965 3647 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/catalog-operator for network default I0428 15:37:46.221975 3647 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/catalog-operator added to peer address sets [] I0428 15:37:46.223200 3647 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/catalog-operator resources completed, update namespace loglevel I0428 15:37:46.223218 3647 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-metrics for network default I0428 15:37:46.223226 3647 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-metrics added to peer address sets [] I0428 15:37:46.223723 3647 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-metrics resources completed, update namespace loglevel I0428 15:37:46.223740 3647 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics for network default I0428 15:37:46.223747 3647 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics added to peer address sets [] I0428 15:37:46.224245 3647 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics resources completed, update namespace loglevel I0428 15:37:46.224267 3647 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-version/default-deny for network default I0428 15:37:46.224276 3647 base_network_controller_policy.go:1054] Policy openshift-cluster-version/default-deny added to peer address sets [] I0428 15:37:46.225525 3647 base_network_controller_policy.go:1232] Create network policy openshift-cluster-version/default-deny resources completed, update namespace loglevel I0428 15:37:46.225542 3647 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-olm for network default I0428 15:37:46.225551 3647 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-olm added to peer address sets [] I0428 15:37:46.226161 3647 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-olm resources completed, update namespace loglevel I0428 15:37:46.226185 3647 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-all-egress for network default I0428 15:37:46.226196 3647 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-all-egress added to peer address sets [] I0428 15:37:46.226734 3647 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-all-egress resources completed, update namespace loglevel I0428 15:37:46.226756 3647 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/olm-operator for network default I0428 15:37:46.226770 3647 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/olm-operator added to peer address sets [] I0428 15:37:46.227871 3647 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/olm-operator resources completed, update namespace loglevel I0428 15:37:46.227895 3647 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-to-dns for network default I0428 15:37:46.227919 3647 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0428 15:37:46.228614 3647 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-to-dns resources completed, update namespace loglevel I0428 15:37:46.228635 3647 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server for network default I0428 15:37:46.228644 3647 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server added to peer address sets [] I0428 15:37:46.229210 3647 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server resources completed, update namespace loglevel I0428 15:37:46.229228 3647 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/marketplace-operator for network default I0428 15:37:46.229238 3647 base_network_controller_policy.go:1054] Policy openshift-marketplace/marketplace-operator added to peer address sets [] I0428 15:37:46.231609 3647 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/marketplace-operator resources completed, update namespace loglevel I0428 15:37:46.231626 3647 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/package-server-manager for network default I0428 15:37:46.231636 3647 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/package-server-manager added to peer address sets [] I0428 15:37:46.232750 3647 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/package-server-manager resources completed, update namespace loglevel I0428 15:37:46.232773 3647 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/packageserver for network default I0428 15:37:46.232787 3647 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/packageserver added to peer address sets [] I0428 15:37:46.234043 3647 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/packageserver resources completed, update namespace loglevel I0428 15:37:46.234061 3647 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-egress-to-api-server for network default I0428 15:37:46.234069 3647 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-egress-to-api-server added to peer address sets [] I0428 15:37:46.234611 3647 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-egress-to-api-server resources completed, update namespace loglevel I0428 15:37:46.234634 3647 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-egress-to-api-server for network default I0428 15:37:46.234646 3647 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-egress-to-api-server added to peer address sets [] I0428 15:37:46.235224 3647 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-egress-to-api-server resources completed, update namespace loglevel I0428 15:37:46.235247 3647 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-insights for network default I0428 15:37:46.235260 3647 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-insights added to peer address sets [] I0428 15:37:46.236337 3647 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-insights resources completed, update namespace loglevel I0428 15:37:46.236360 3647 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/default-deny-all for network default I0428 15:37:46.236367 3647 base_network_controller_policy.go:1054] Policy openshift-marketplace/default-deny-all added to peer address sets [] I0428 15:37:46.236669 3647 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/default-deny-all resources completed, update namespace loglevel I0428 15:37:46.236688 3647 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/unpack-bundles for network default I0428 15:37:46.236713 3647 base_network_controller_policy.go:1054] Policy openshift-marketplace/unpack-bundles added to peer address sets [] I0428 15:37:46.237563 3647 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/unpack-bundles resources completed, update namespace loglevel I0428 15:37:46.237582 3647 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-egress for network default I0428 15:37:46.237591 3647 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-egress added to peer address sets [] I0428 15:37:46.238370 3647 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-egress resources completed, update namespace loglevel I0428 15:37:46.238390 3647 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-all-egress for network default I0428 15:37:46.238402 3647 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-all-egress added to peer address sets [] I0428 15:37:46.239038 3647 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-all-egress resources completed, update namespace loglevel I0428 15:37:46.245735 3647 egressfirewall.go:374] Starting EgressFirewall controller I0428 15:37:46.245770 3647 controller.go:133] Adding controller egress-firewall-controller event handlers I0428 15:37:46.245788 3647 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller" I0428 15:37:46.245798 3647 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller" I0428 15:37:46.245810 3647 controller.go:133] Adding controller egress-firewall-controller-node event handlers I0428 15:37:46.245850 3647 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller-node" I0428 15:37:46.245868 3647 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller-node" I0428 15:37:46.246162 3647 controller.go:157] Starting controller egress-firewall-controller with 1 workers I0428 15:37:46.246178 3647 controller.go:157] Starting controller egress-firewall-controller-node with 1 workers I0428 15:37:46.246183 3647 controller.go:157] Starting controller egress-firewall-controller-NAD with 1 workers I0428 15:37:46.246191 3647 egressqos.go:193] Setting up event handlers for EgressQoS I0428 15:37:46.246204 3647 egressfirewall.go:723] Syncing node "ip-10-0-2-7.ec2.internal" for egress firewall I0428 15:37:46.246215 3647 egressfirewall.go:723] Syncing node "ip-10-0-0-9.ec2.internal" for egress firewall I0428 15:37:46.246219 3647 egressfirewall.go:723] Syncing node "ip-10-0-1-106.ec2.internal" for egress firewall I0428 15:37:46.246342 3647 egressqos.go:245] Starting EgressQoS Controller I0428 15:37:46.246360 3647 shared_informer.go:349] "Waiting for caches to sync" controller="egressqosnodes" I0428 15:37:46.246373 3647 shared_informer.go:356] "Caches are synced" controller="egressqosnodes" I0428 15:37:46.246384 3647 shared_informer.go:349] "Waiting for caches to sync" controller="egressqospods" I0428 15:37:46.246392 3647 shared_informer.go:356] "Caches are synced" controller="egressqospods" I0428 15:37:46.246402 3647 shared_informer.go:349] "Waiting for caches to sync" controller="egressqos" I0428 15:37:46.246409 3647 shared_informer.go:356] "Caches are synced" controller="egressqos" I0428 15:37:46.246416 3647 egressqos.go:259] Repairing EgressQoSes I0428 15:37:46.246422 3647 egressqos.go:399] Starting repairing loop for egressqos I0428 15:37:46.246508 3647 egressqos.go:401] Finished repairing loop for egressqos: 86.956µs I0428 15:37:46.246533 3647 egressservice_zone.go:132] Setting up event handlers for Egress Services I0428 15:37:46.246563 3647 egressqos.go:1007] Processing sync for EgressQoS node ip-10-0-1-106.ec2.internal I0428 15:37:46.246606 3647 egressqos.go:1010] Finished syncing EgressQoS node ip-10-0-1-106.ec2.internal : 44.275µs I0428 15:37:46.246677 3647 egressservice_zone.go:210] Starting Egress Services Controller I0428 15:37:46.246688 3647 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0428 15:37:46.246694 3647 shared_informer.go:356] "Caches are synced" controller="egressservices" I0428 15:37:46.246713 3647 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0428 15:37:46.246718 3647 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0428 15:37:46.246724 3647 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0428 15:37:46.246729 3647 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0428 15:37:46.246734 3647 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_nodes" I0428 15:37:46.246738 3647 shared_informer.go:356] "Caches are synced" controller="egressservices_nodes" I0428 15:37:46.246742 3647 egressservice_zone.go:228] Repairing Egress Services I0428 15:37:46.247009 3647 master_controller.go:88] Starting Admin Policy Based Route Controller I0428 15:37:46.247017 3647 external_controller.go:278] Starting Admin Policy Based Route Controller I0428 15:37:46.247043 3647 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-9.ec2.internal I0428 15:37:46.247057 3647 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-9.ec2.internal: 16.888µs I0428 15:37:46.247064 3647 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-106.ec2.internal I0428 15:37:46.247068 3647 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-106.ec2.internal: 4.037µs I0428 15:37:46.247072 3647 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-7.ec2.internal I0428 15:37:46.247081 3647 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-7.ec2.internal: 8.364µs I0428 15:37:46.247212 3647 default_network_controller.go:585] Completing all the Watchers took 334.230042ms I0428 15:37:46.247224 3647 default_network_controller.go:589] Starting unidling controllers I0428 15:37:46.247236 3647 unidle.go:46] Registering OVN SB ControllerEvent handler I0428 15:37:46.247242 3647 unidle.go:63] Populating Initial ContollerEvent events I0428 15:37:46.247251 3647 unidle.go:79] Setting up event handlers for services I0428 15:37:46.247295 3647 ovnkube.go:545] Waiting for OVN northbound database changes to be processed by ovn-controller I0428 15:37:46.522329 3647 default_node_network_controller.go:519] Node connection status = connected I0428 15:37:46.960237 3647 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-9.ec2.internal in network "default" I0428 15:37:46.960301 3647 node_tracker.go:169] Node ip-10-0-0-9.ec2.internal switch + router changed, syncing services in network "default" I0428 15:37:46.960321 3647 services_controller.go:574] Full service sync requested for network=default I0428 15:37:46.960329 3647 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-9.ec2.internal I0428 15:37:46.960335 3647 egressfirewall.go:723] Syncing node "ip-10-0-0-9.ec2.internal" for egress firewall I0428 15:37:46.960342 3647 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-0-9.ec2.internal. Updating no re-route policies I0428 15:37:46.960346 3647 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-9.ec2.internal: 23.301µs W0428 15:37:46.960462 3647 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics W0428 15:37:46.960587 3647 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0428 15:37:47.022919 3647 default_node_network_controller.go:519] Node connection status = connected I0428 15:37:47.522982 3647 default_node_network_controller.go:519] Node connection status = connected I0428 15:37:48.047465 3647 default_node_network_controller.go:519] Node connection status = connected I0428 15:37:48.115274 3647 bridgeconfig.go:537] Gateway is ready I0428 15:37:48.115292 3647 gateway_shared_intf.go:1733] Creating Gateway Openflow Manager I0428 15:37:48.127587 3647 ovnkube.go:550] Finished waiting for OVN northbound database changes to be processed by ovn-controller I0428 15:37:48.127850 3647 node_ip_handler_linux.go:247] Node primary address changed to 10.0.1.106. Updating OVN encap IP. I0428 15:37:48.133519 3647 node_ip_handler_linux.go:542] Will not update encap IP 10.0.1.106 - it is already configured I0428 15:37:48.133531 3647 node_ip_handler_linux.go:485] Node address changed to map[10.0.1.106/24:{} 172.20.0.1/32:{}]. Updating annotations. I0428 15:37:48.133729 3647 kube.go:131] Setting annotations map[k8s.ovn.org/host-cidrs:["10.0.1.106/24","172.20.0.1/32"] k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-1-106.ec2.internal","mac-address":"0a:ff:d0:6a:d3:f7","ip-addresses":["10.0.1.106/24"],"ip-address":"10.0.1.106/24","next-hops":["10.0.1.1"],"next-hop":"10.0.1.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:fc46d8bd-6618-4c55-a52b-039a7d5ef213 k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.1.106/24"}] on node ip-10-0-1-106.ec2.internal I0428 15:37:48.150852 3647 egressfirewall.go:723] Syncing node "ip-10-0-1-106.ec2.internal" for egress firewall I0428 15:37:48.150857 3647 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-106.ec2.internal I0428 15:37:48.150877 3647 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-106.ec2.internal: 26.23µs I0428 15:37:48.150906 3647 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-106.ec2.internal in network "default" I0428 15:37:48.150914 3647 master.go:548] Adding or Updating local node "ip-10-0-1-106.ec2.internal" for network "default" I0428 15:37:48.150958 3647 node_tracker.go:169] Node ip-10-0-1-106.ec2.internal switch + router changed, syncing services in network "default" I0428 15:37:48.150974 3647 services_controller.go:574] Full service sync requested for network=default W0428 15:37:48.151065 3647 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog W0428 15:37:48.151414 3647 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0428 15:37:48.152570 3647 gateway_shared_intf.go:2023] Setting OVN Masquerade route with source: 10.0.1.106 I0428 15:37:48.152784 3647 kube.go:131] Setting annotations map[k8s.ovn.org/node-masquerade-subnet:{"ipv4":"169.254.0.0/17","ipv6":"fd69::/112"}] on node ip-10-0-1-106.ec2.internal I0428 15:37:48.155388 3647 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-1-106.ec2.internal. Updating no re-route policies I0428 15:37:48.155466 3647 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-w6lhl I0428 15:37:48.155472 3647 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/konnectivity-agent-knjft I0428 15:37:48.155482 3647 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-w6lhl I0428 15:37:48.155480 3647 obj_retry.go:334] Retry object setup: *v1.Pod openshift-image-registry/node-ca-8cb28 I0428 15:37:48.155491 3647 obj_retry.go:334] Retry object setup: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-859kp I0428 15:37:48.155497 3647 obj_retry.go:398] Adding new object: *v1.Pod openshift-image-registry/node-ca-8cb28 I0428 15:37:48.155496 3647 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/multus-xcp5b I0428 15:37:48.155504 3647 obj_retry.go:398] Adding new object: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-859kp I0428 15:37:48.155507 3647 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/multus-xcp5b I0428 15:37:48.155506 3647 obj_retry.go:334] Retry object setup: *v1.Pod openshift-dns/node-resolver-qv85l I0428 15:37:48.155510 3647 obj_retry.go:334] Retry object setup: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-1-106.ec2.internal I0428 15:37:48.155513 3647 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-w6lhl] creating logical port openshift-network-diagnostics_network-check-target-w6lhl for pod on switch ip-10-0-1-106.ec2.internal I0428 15:37:48.155520 3647 obj_retry.go:418] Retry successful for *v1.Pod openshift-ovn-kubernetes/ovnkube-node-859kp after 0 failed attempt(s) I0428 15:37:48.155525 3647 obj_retry.go:398] Adding new object: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-1-106.ec2.internal I0428 15:37:48.155486 3647 obj_retry.go:398] Adding new object: *v1.Pod kube-system/konnectivity-agent-knjft I0428 15:37:48.155526 3647 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-1-106.ec2.internal I0428 15:37:48.155539 3647 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-operator/iptables-alerter-md5mj I0428 15:37:48.155542 3647 obj_retry.go:334] Retry object setup: *v1.Pod openshift-cluster-node-tuning-operator/tuned-254dz I0428 15:37:48.155542 3647 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/multus-additional-cni-plugins-s6zhb I0428 15:37:48.155552 3647 obj_retry.go:398] Adding new object: *v1.Pod openshift-cluster-node-tuning-operator/tuned-254dz I0428 15:37:48.155553 3647 obj_retry.go:418] Retry successful for *v1.Pod kube-system/konnectivity-agent-knjft after 0 failed attempt(s) I0428 15:37:48.155549 3647 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-operator/iptables-alerter-md5mj I0428 15:37:48.155558 3647 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/multus-additional-cni-plugins-s6zhb I0428 15:37:48.155561 3647 obj_retry.go:334] Retry object setup: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-vv2nb I0428 15:37:48.155566 3647 obj_retry.go:418] Retry successful for *v1.Pod openshift-cluster-node-tuning-operator/tuned-254dz after 0 failed attempt(s) I0428 15:37:48.155564 3647 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-xdscj I0428 15:37:48.155576 3647 obj_retry.go:418] Retry successful for *v1.Pod openshift-network-operator/iptables-alerter-md5mj after 0 failed attempt(s) I0428 15:37:48.155577 3647 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/multus-additional-cni-plugins-s6zhb after 0 failed attempt(s) I0428 15:37:48.155572 3647 obj_retry.go:398] Adding new object: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-vv2nb I0428 15:37:48.155584 3647 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-xdscj I0428 15:37:48.155588 3647 obj_retry.go:398] Adding new object: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-1-106.ec2.internal I0428 15:37:48.155600 3647 obj_retry.go:418] Retry successful for *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-vv2nb after 0 failed attempt(s) I0428 15:37:48.155611 3647 obj_retry.go:418] Retry successful for *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-1-106.ec2.internal after 0 failed attempt(s) I0428 15:37:48.155525 3647 obj_retry.go:398] Adding new object: *v1.Pod openshift-dns/node-resolver-qv85l I0428 15:37:48.155620 3647 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-xdscj] creating logical port openshift-multus_network-metrics-daemon-xdscj for pod on switch ip-10-0-1-106.ec2.internal I0428 15:37:48.155527 3647 obj_retry.go:418] Retry successful for *v1.Pod openshift-image-registry/node-ca-8cb28 after 0 failed attempt(s) I0428 15:37:48.155632 3647 obj_retry.go:418] Retry successful for *v1.Pod openshift-dns/node-resolver-qv85l after 0 failed attempt(s) I0428 15:37:48.155552 3647 obj_retry.go:418] Retry successful for *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-1-106.ec2.internal after 0 failed attempt(s) I0428 15:37:48.155531 3647 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/multus-xcp5b after 0 failed attempt(s) I0428 15:37:48.156313 3647 pods.go:271] [openshift-network-diagnostics/network-check-target-w6lhl] addLogicalPort took 807.011µs, libovsdb time 233.766µs I0428 15:37:48.156327 3647 obj_retry.go:418] Retry successful for *v1.Pod openshift-network-diagnostics/network-check-target-w6lhl after 0 failed attempt(s) I0428 15:37:48.156377 3647 pods.go:271] [openshift-multus/network-metrics-daemon-xdscj] addLogicalPort took 767.463µs, libovsdb time 256.197µs I0428 15:37:48.156384 3647 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/network-metrics-daemon-xdscj after 0 failed attempt(s) I0428 15:37:48.175339 3647 gateway_shared_intf.go:1799] Creating Gateway Node Port Watcher I0428 15:37:48.187406 3647 iptables.go:108] Creating table: mangle chain: OUTPUT I0428 15:37:48.191028 3647 iptables.go:108] Creating table: nat chain: OUTPUT I0428 15:37:48.245526 3647 iptables.go:108] Creating table: nat chain: PREROUTING I0428 15:37:48.299095 3647 iptables.go:108] Creating table: filter chain: FORWARD I0428 15:37:48.316399 3647 gateway_shared_intf.go:2124] Ensuring IP Neighbor entry for: 169.254.0.1 W0428 15:37:48.316546 3647 gateway_shared_intf.go:2130] Failed to remove IP neighbor entry for ip 169.254.0.1, on iface br-ex: failed to delete neighbour entry 169.254.0.1 : no such file or directory I0428 15:37:48.316596 3647 gateway_shared_intf.go:2124] Ensuring IP Neighbor entry for: 169.254.0.4 W0428 15:37:48.316682 3647 gateway_shared_intf.go:2130] Failed to remove IP neighbor entry for ip 169.254.0.4, on iface br-ex: failed to delete neighbour entry 169.254.0.4 : no such file or directory I0428 15:37:48.316746 3647 gateway_init.go:325] Gateway and management port readiness took 2.298946422s I0428 15:37:48.316773 3647 controller.go:133] Adding controller [node-nad-controller NAD controller] event handlers I0428 15:37:48.316815 3647 shared_informer.go:349] "Waiting for caches to sync" controller="[node-nad-controller NAD controller]" I0428 15:37:48.316827 3647 shared_informer.go:356] "Caches are synced" controller="[node-nad-controller NAD controller]" I0428 15:37:48.316836 3647 nad_controller.go:541] [node-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0428 15:37:48.316891 3647 controller.go:157] Starting controller [node-nad-controller NAD controller] with 1 workers I0428 15:37:48.316972 3647 nad_controller.go:580] [node-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 53.685µs I0428 15:37:48.317215 3647 network_controller.go:373] [node-nad-controller network controller]: syncing all networks I0428 15:37:48.317237 3647 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 14.099µs I0428 15:37:48.317246 3647 network_controller.go:384] [node-nad-controller network controller]: finished syncing all networks. Time taken: 31.734µs I0428 15:37:48.317251 3647 controller.go:157] Starting controller [node-nad-controller network controller] with 1 workers I0428 15:37:48.317261 3647 nad_controller.go:379] [node-nad-controller NAD controller]: started I0428 15:37:48.317266 3647 default_node_network_controller.go:899] Starting the default node network controller I0428 15:37:48.317270 3647 gateway_init.go:335] Initializing Gateway Functionality for gateway Start I0428 15:37:48.317354 3647 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 15.595µs I0428 15:37:48.317446 3647 default_node_network_controller.go:1460] MTU (9001) of network interface br-ex is big enough to deal with Geneve header overhead (sum 8959). I0428 15:37:48.317455 3647 default_node_network_controller.go:935] Waiting for gateway and management port readiness... I0428 15:37:48.317488 3647 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc0008893d0 0xc00032f880 0xc00032f8f0 0xc00032f960 0xc00032f9d0 0xc00032fa40 0xc00032fab0 0xc000c19220 0xc00032fb20 0xc00032fc00 0xc000646370 0xc0064795e0 0xc00032fc70 map[0x2ff9080:0xc00299ef00 0x2ff95c0:0xc00299f080 0x2ffa2e0:0xc00299ee40 0x2ffa580:0xc00299eea0 0x2ffad60:0xc00299ef60 0x2ffb000:0xc00299efc0 0x2ffb2a0:0xc00299f020 0x2ffba80:0xc00299f0e0 0x2ffbd20:0xc00299f140 0x3026980:0xc001114720 0x3026d20:0xc0029086c0 0x3027800:0xc000c52f60 0x3029160:0xc001c88000 0x30298a0:0xc002908720 0x302a720:0xc0011146c0] 0xc00016ad90 0} I0428 15:37:48.317546 3647 gateway.go:151] Starting gateway service sync I0428 15:37:48.318040 3647 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ITP I0428 15:37:48.320326 3647 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-NODEPORT I0428 15:37:48.344516 3647 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-EXTERNALIP I0428 15:37:48.345823 3647 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ETP I0428 15:37:48.347005 3647 gateway_iptables.go:474] Recreating iptables rules for table: mangle, chain: OVN-KUBE-ITP I0428 15:37:48.381261 3647 gateway.go:168] Gateway service sync done. Time taken: 63.703405ms I0428 15:37:48.381684 3647 healthcheck.go:145] Opening healthcheck "openshift-ingress/router-default" on port 30931 I0428 15:37:48.382135 3647 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc0008893d0 0xc00032f880 0xc00032f8f0 0xc00032f960 0xc00032f9d0 0xc00032fa40 0xc00032fab0 0xc000c19220 0xc00032fb20 0xc00032fc00 0xc000646370 0xc0064795e0 0xc00032fc70 map[0x2ff9080:0xc00299ef00 0x2ff95c0:0xc00299f080 0x2ffa2e0:0xc00299ee40 0x2ffa580:0xc00299eea0 0x2ffad60:0xc00299ef60 0x2ffb000:0xc00299efc0 0x2ffb2a0:0xc00299f020 0x2ffba80:0xc00299f0e0 0x2ffbd20:0xc00299f140 0x3026980:0xc001114720 0x3026d20:0xc0029086c0 0x3027800:0xc000c52f60 0x3029160:0xc001c88000 0x30298a0:0xc002908720 0x302a720:0xc0011146c0] 0xc00016ad90 0} I0428 15:37:48.382607 3647 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc0008893d0 0xc00032f880 0xc00032f8f0 0xc00032f960 0xc00032f9d0 0xc00032fa40 0xc00032fab0 0xc000c19220 0xc00032fb20 0xc00032fc00 0xc000646370 0xc0064795e0 0xc00032fc70 map[0x2ff9080:0xc00299ef00 0x2ff95c0:0xc00299f080 0x2ffa2e0:0xc00299ee40 0x2ffa580:0xc00299eea0 0x2ffad60:0xc00299ef60 0x2ffb000:0xc00299efc0 0x2ffb2a0:0xc00299f020 0x2ffba80:0xc00299f0e0 0x2ffbd20:0xc00299f140 0x3026980:0xc001114720 0x3026d20:0xc0029086c0 0x3027800:0xc000c52f60 0x3029160:0xc001c88000 0x30298a0:0xc002908720 0x302a720:0xc0011146c0] 0xc00016ad90 0} I0428 15:37:48.382642 3647 gateway.go:532] Reconciling gateway with updates I0428 15:37:48.388256 3647 gateway.go:339] Spawning Conntrack Rule Check Thread I0428 15:37:48.388335 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0428 15:37:48.388334 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0428 15:37:48.388352 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0428 15:37:48.388354 3647 default_node_network_controller.go:944] Gateway and management port readiness took 70.891581ms I0428 15:37:48.388357 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0428 15:37:48.388355 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0428 15:37:48.388366 3647 port_claim.go:71] Svc openshift-ingress/router-default: port 32194 is already open, no action needed I0428 15:37:48.388372 3647 port_claim.go:71] Svc openshift-ingress/router-default: port 31728 is already open, no action needed I0428 15:37:48.388372 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0428 15:37:48.388375 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0428 15:37:48.388390 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0428 15:37:48.388400 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0428 15:37:48.388398 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0428 15:37:48.388418 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0428 15:37:48.388425 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0428 15:37:48.388428 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0428 15:37:48.388440 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0428 15:37:48.388447 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0428 15:37:48.388441 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0428 15:37:48.388455 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0428 15:37:48.388450 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0428 15:37:48.388454 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0428 15:37:48.388462 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0428 15:37:48.388462 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0428 15:37:48.388465 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0428 15:37:48.388471 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0428 15:37:48.388481 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/packageserver I0428 15:37:48.388489 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/packageserver I0428 15:37:48.388502 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0428 15:37:48.388510 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0428 15:37:48.388515 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0428 15:37:48.388516 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0428 15:37:48.388515 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0428 15:37:48.388525 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0428 15:37:48.388527 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0428 15:37:48.388532 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0428 15:37:48.388534 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0428 15:37:48.388537 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0428 15:37:48.388542 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0428 15:37:48.388536 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0428 15:37:48.388549 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0428 15:37:48.388554 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0428 15:37:48.388527 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0428 15:37:48.388559 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0428 15:37:48.388586 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0428 15:37:48.388586 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0428 15:37:48.388601 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0428 15:37:48.388550 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0428 15:37:48.388604 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0428 15:37:48.388611 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0428 15:37:48.388524 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0428 15:37:48.388603 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0428 15:37:48.388625 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0428 15:37:48.388625 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0428 15:37:48.388631 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0428 15:37:48.388640 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0428 15:37:48.388643 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0428 15:37:48.388645 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0428 15:37:48.388646 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0428 15:37:48.388649 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0428 15:37:48.388611 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0428 15:37:48.388653 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0428 15:37:48.388658 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0428 15:37:48.388662 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0428 15:37:48.388671 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0428 15:37:48.388643 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0428 15:37:48.388686 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0428 15:37:48.388640 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0428 15:37:48.388696 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0428 15:37:48.388489 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0428 15:37:48.388501 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0428 15:37:48.388732 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0428 15:37:48.388532 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0428 15:37:48.388767 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0428 15:37:48.388736 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0428 15:37:48.388809 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0428 15:37:48.388620 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0428 15:37:48.388824 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0428 15:37:48.388517 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0428 15:37:48.388634 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0428 15:37:48.388630 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0428 15:37:48.388856 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift I0428 15:37:48.388867 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift I0428 15:37:48.388874 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0428 15:37:48.388857 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0428 15:37:48.388880 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0428 15:37:48.388661 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0428 15:37:48.388914 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0428 15:37:48.388656 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0428 15:37:48.388656 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0428 15:37:48.388969 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0428 15:37:48.388723 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0428 15:37:48.388993 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0428 15:37:48.388831 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0428 15:37:48.388443 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0428 15:37:48.388646 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0428 15:37:48.389031 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0428 15:37:48.389135 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0428 15:37:48.389149 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0428 15:37:48.389157 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0428 15:37:48.389176 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0428 15:37:48.389181 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0428 15:37:48.389198 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0428 15:37:48.389211 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/kubernetes I0428 15:37:48.389212 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0428 15:37:48.389222 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/kubernetes I0428 15:37:48.389159 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0428 15:37:48.389227 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0428 15:37:48.389231 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0428 15:37:48.389235 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0428 15:37:48.389241 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0428 15:37:48.389248 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0428 15:37:48.389525 3647 node_ip_handler_linux.go:160] Node IP manager is running W0428 15:37:48.426410 3647 port_linux.go:245] Missing or unable to find route entry for subnet 10.128.0.0/14 via gateway 10.130.0.1 on link ovn-k8s-mp0 with MTU: 0 W0428 15:37:48.426626 3647 port_linux.go:245] Missing or unable to find route entry for subnet 169.254.0.3/32 via gateway 10.130.0.1 on link ovn-k8s-mp0 with MTU: 0 W0428 15:37:48.426875 3647 port_linux.go:261] Missing arp entry for MAC/IP binding (0a:58:0a:82:00:01/10.130.0.1) on link ovn-k8s-mp0 I0428 15:37:48.434875 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0428 15:37:48.434900 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0428 15:37:48.434908 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0428 15:37:48.434914 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0428 15:37:48.434920 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0428 15:37:48.434925 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0428 15:37:48.434930 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0428 15:37:48.434936 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0428 15:37:48.434941 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0428 15:37:48.434946 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0428 15:37:48.434952 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0428 15:37:48.434959 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0428 15:37:48.434965 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0428 15:37:48.434970 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0428 15:37:48.434976 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0428 15:37:48.434981 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0428 15:37:48.434986 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0428 15:37:48.434990 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0428 15:37:48.435004 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0428 15:37:48.435024 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0428 15:37:48.474254 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0428 15:37:48.474280 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0428 15:37:48.474287 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0428 15:37:50.418877 3647 default_node_network_controller.go:1380] Starting node controller node sync W0428 15:37:50.418933 3647 egressip_healthcheck.go:76] Health checking using insecure connection I0428 15:37:50.418971 3647 egressip_healthcheck.go:109] Starting Egress IP Health Server on 10.130.0.2:9107 I0428 15:37:50.430557 3647 default_node_network_controller.go:1422] Node controller node sync done. Time taken: 11.662798ms I0428 15:37:50.430672 3647 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-9.ec2.internal", IP: 10.0.0.9 to PMTUD blocking rules I0428 15:37:50.430686 3647 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-9.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0428 15:37:50.430725 3647 default_node_network_controller.go:1293] Adding remote node "ip-10-0-2-7.ec2.internal", IP: 10.0.2.7 to PMTUD blocking rules I0428 15:37:50.430740 3647 default_node_network_controller.go:1293] Adding remote node "ip-10-0-2-7.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0428 15:37:50.444246 3647 healthcheck_node.go:124] "Starting node proxy healthz server" address="0.0.0.0:10256" I0428 15:37:50.444514 3647 egressservice_node.go:93] Setting up event handlers for Egress Services I0428 15:37:50.444661 3647 egressservice_node.go:183] Starting Egress Services Controller I0428 15:37:50.444680 3647 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0428 15:37:50.444690 3647 shared_informer.go:356] "Caches are synced" controller="egressservices" I0428 15:37:50.444713 3647 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0428 15:37:50.444719 3647 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0428 15:37:50.444724 3647 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0428 15:37:50.444729 3647 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0428 15:37:50.444734 3647 egressservice_node.go:197] Repairing Egress Services I0428 15:37:50.454376 3647 node_controller.go:43] Starting Admin Policy Based Route Node Controller I0428 15:37:50.454388 3647 external_controller.go:278] Starting Admin Policy Based Route Controller I0428 15:37:50.454495 3647 default_node_network_controller.go:1077] Egress IP for secondary host network is disabled I0428 15:37:50.454504 3647 link_network_manager.go:119] Link manager is running I0428 15:37:50.454514 3647 default_node_network_controller.go:1098] Default node network controller initialized and ready. I0428 15:37:50.454749 3647 ovspinning_linux.go:50] OVS CPU affinity pinning disabled I0428 15:37:50.454883 3647 vrf_manager.go:128] VRF manager is running I0428 15:37:50.455146 3647 node_controller_manager.go:437] Removing flows to drop GARP I0428 15:37:50.455155 3647 gateway.go:532] Reconciling gateway with updates I0428 15:37:50.484562 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0428 15:37:50.484569 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0428 15:37:50.484588 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0428 15:37:50.484594 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0428 15:37:50.484602 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0428 15:37:50.484609 3647 port_claim.go:71] Svc openshift-ingress/router-default: port 32194 is already open, no action needed I0428 15:37:50.484604 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0428 15:37:50.484618 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0428 15:37:50.484621 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0428 15:37:50.484622 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0428 15:37:50.484628 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/kubernetes I0428 15:37:50.484630 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0428 15:37:50.484641 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0428 15:37:50.484641 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/kubernetes I0428 15:37:50.484643 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0428 15:37:50.484645 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0428 15:37:50.484656 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0428 15:37:50.484657 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0428 15:37:50.484590 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0428 15:37:50.484664 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0428 15:37:50.484665 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0428 15:37:50.484665 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0428 15:37:50.484672 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0428 15:37:50.484674 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0428 15:37:50.484681 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0428 15:37:50.484685 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0428 15:37:50.484694 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0428 15:37:50.484694 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0428 15:37:50.484722 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0428 15:37:50.484619 3647 port_claim.go:71] Svc openshift-ingress/router-default: port 31728 is already open, no action needed I0428 15:37:50.484732 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0428 15:37:50.484741 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/exporter I0428 15:37:50.484744 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift I0428 15:37:50.484650 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0428 15:37:50.484754 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift I0428 15:37:50.484751 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/exporter I0428 15:37:50.484747 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0428 15:37:50.484755 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0428 15:37:50.484759 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0428 15:37:50.484767 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/exporter after 0 failed attempt(s) I0428 15:37:50.484757 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0428 15:37:50.484774 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0428 15:37:50.484776 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0428 15:37:50.484780 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0428 15:37:50.484783 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/packageserver I0428 15:37:50.484784 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0428 15:37:50.484789 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0428 15:37:50.484791 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0428 15:37:50.484795 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/packageserver I0428 15:37:50.484776 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0428 15:37:50.484797 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0428 15:37:50.484800 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0428 15:37:50.484810 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0428 15:37:50.484810 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0428 15:37:50.484814 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0428 15:37:50.484818 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0428 15:37:50.484823 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0428 15:37:50.484576 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0428 15:37:50.484829 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0428 15:37:50.484826 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0428 15:37:50.484837 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0428 15:37:50.484839 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0428 15:37:50.484844 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0428 15:37:50.484845 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0428 15:37:50.484851 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0428 15:37:50.484854 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0428 15:37:50.484857 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0428 15:37:50.484759 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0428 15:37:50.484863 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0428 15:37:50.484860 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0428 15:37:50.484875 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0428 15:37:50.484899 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0428 15:37:50.484776 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0428 15:37:50.484777 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0428 15:37:50.484979 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0428 15:37:50.484994 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0428 15:37:50.484994 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0428 15:37:50.484997 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0428 15:37:50.485005 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0428 15:37:50.485003 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0428 15:37:50.485008 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0428 15:37:50.485008 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0428 15:37:50.485015 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0428 15:37:50.485020 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0428 15:37:50.485021 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0428 15:37:50.485032 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0428 15:37:50.485033 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0428 15:37:50.485038 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0428 15:37:50.485040 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0428 15:37:50.485040 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0428 15:37:50.485046 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0428 15:37:50.485047 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0428 15:37:50.485016 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0428 15:37:50.485052 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0428 15:37:50.485056 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0428 15:37:50.485062 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0428 15:37:50.485062 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0428 15:37:50.485061 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0428 15:37:50.485061 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0428 15:37:50.485038 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0428 15:37:50.485024 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0428 15:37:50.485094 3647 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0428 15:37:50.485104 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0428 15:37:50.485041 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0428 15:37:50.485050 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0428 15:37:50.485198 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0428 15:37:50.485014 3647 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0428 15:37:50.523127 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0428 15:37:50.523142 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0428 15:37:50.523152 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0428 15:37:50.523158 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0428 15:37:50.523162 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0428 15:37:50.523167 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0428 15:37:50.523171 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0428 15:37:50.523176 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0428 15:37:50.523181 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0428 15:37:50.523186 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0428 15:37:50.523190 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0428 15:37:50.523195 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0428 15:37:50.523202 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0428 15:37:50.523206 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0428 15:37:50.523210 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0428 15:37:50.523215 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0428 15:37:50.561858 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0428 15:37:50.561875 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0428 15:37:50.561884 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0428 15:37:50.561889 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0428 15:37:50.561895 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0428 15:37:50.561900 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0428 15:37:50.561923 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0428 15:37:50.561949 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0428 15:37:50.561955 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0428 15:37:50.561960 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0428 15:37:50.561965 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0428 15:37:50.561970 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0428 15:37:50.561975 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0428 15:37:50.561980 3647 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0428 15:37:57.282325 3647 default_network_controller.go:1025] Node "ip-10-0-2-7.ec2.internal" in remote zone "ip-10-0-2-7.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0428 15:37:57.283325 3647 zone_ic_handler.go:195] Time taken to create transit switch: 368.858µs I0428 15:37:57.283346 3647 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-2-7.ec2.internal for the network default I0428 15:37:57.284567 3647 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-2-7.ec2.internal" on network "default" took: 1.628717ms W0428 15:38:03.236249 3647 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0428 15:38:03.236266 3647 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0428 15:38:03.236264 3647 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0428 15:38:03.236271 3647 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0428 15:38:03.236276 3647 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0428 15:38:03.252432 3647 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0428 15:38:03.255288 3647 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0428 15:38:03.441752 3647 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-9.ec2.internal I0428 15:38:03.441783 3647 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-9.ec2.internal: 39.751µs W0428 15:38:03.492318 3647 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-image-registry/image-registry-796c59fd79-92mqs IP address from the namespace address-set, err: pod openshift-image-registry/image-registry-796c59fd79-92mqs: no pod IPs found W0428 15:38:03.511116 3647 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0428 15:38:03.991495 3647 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-106.ec2.internal I0428 15:38:03.991527 3647 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-106.ec2.internal: 41.923µs I0428 15:38:04.168081 3647 base_network_controller_pods.go:487] [default/openshift-insights/insights-runtime-extractor-whzzz] creating logical port openshift-insights_insights-runtime-extractor-whzzz for pod on switch ip-10-0-1-106.ec2.internal I0428 15:38:04.168215 3647 kube.go:256] Updating pod openshift-insights/insights-runtime-extractor-whzzz I0428 15:38:04.185118 3647 base_network_controller_pods.go:487] [default/openshift-ingress-canary/ingress-canary-d4f79] creating logical port openshift-ingress-canary_ingress-canary-d4f79 for pod on switch ip-10-0-1-106.ec2.internal I0428 15:38:04.185193 3647 kube.go:256] Updating pod openshift-ingress-canary/ingress-canary-d4f79 I0428 15:38:04.205107 3647 pod.go:62] [openshift-insights/insights-runtime-extractor-whzzz] pod update took 36.926845ms I0428 15:38:04.205127 3647 base_network_controller_pods.go:951] [default/openshift-insights/insights-runtime-extractor-whzzz] addLogicalPort annotation time took 36.952008ms I0428 15:38:04.205977 3647 pods.go:271] [openshift-insights/insights-runtime-extractor-whzzz] addLogicalPort took 37.913145ms, libovsdb time 637.3µs I0428 15:38:04.230164 3647 pod.go:62] [openshift-ingress-canary/ingress-canary-d4f79] pod update took 44.990765ms I0428 15:38:04.230189 3647 base_network_controller_pods.go:951] [default/openshift-ingress-canary/ingress-canary-d4f79] addLogicalPort annotation time took 45.019599ms I0428 15:38:04.231147 3647 pods.go:271] [openshift-ingress-canary/ingress-canary-d4f79] addLogicalPort took 46.041402ms, libovsdb time 703.821µs I0428 15:38:04.271465 3647 base_network_controller_pods.go:487] [default/openshift-dns/dns-default-7rtsv] creating logical port openshift-dns_dns-default-7rtsv for pod on switch ip-10-0-1-106.ec2.internal I0428 15:38:04.271558 3647 kube.go:256] Updating pod openshift-dns/dns-default-7rtsv I0428 15:38:04.271599 3647 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-7d9d855f7f-vmhtv] creating logical port openshift-image-registry_image-registry-7d9d855f7f-vmhtv for pod on switch ip-10-0-1-106.ec2.internal I0428 15:38:04.271667 3647 kube.go:256] Updating pod openshift-image-registry/image-registry-7d9d855f7f-vmhtv I0428 15:38:04.310131 3647 pod.go:62] [openshift-dns/dns-default-7rtsv] pod update took 38.604286ms I0428 15:38:04.310152 3647 base_network_controller_pods.go:951] [default/openshift-dns/dns-default-7rtsv] addLogicalPort annotation time took 38.630778ms I0428 15:38:04.311233 3647 pods.go:271] [openshift-dns/dns-default-7rtsv] addLogicalPort took 39.778338ms, libovsdb time 828.322µs I0428 15:38:04.317289 3647 pod.go:62] [openshift-image-registry/image-registry-7d9d855f7f-vmhtv] pod update took 45.635226ms I0428 15:38:04.317305 3647 base_network_controller_pods.go:951] [default/openshift-image-registry/image-registry-7d9d855f7f-vmhtv] addLogicalPort annotation time took 45.66425ms I0428 15:38:04.318153 3647 pods.go:271] [openshift-image-registry/image-registry-7d9d855f7f-vmhtv] addLogicalPort took 46.56372ms, libovsdb time 626.568µs I0428 15:38:04.547308 3647 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dggfm8] creating logical port openshift-deployment-validation-operator_99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dggfm8 for pod on switch ip-10-0-1-106.ec2.internal I0428 15:38:04.547549 3647 kube.go:256] Updating pod openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dggfm8 I0428 15:38:04.550755 3647 cni.go:452] [openshift-insights/insights-runtime-extractor-whzzz d0384da407f63c6d7aaf6c4588573378c55fd1422318af47517acb77886370ab network default NAD default NAD key ] ADD starting CNI request [openshift-insights/insights-runtime-extractor-whzzz d0384da407f63c6d7aaf6c4588573378c55fd1422318af47517acb77886370ab network default NAD default NAD key ] I0428 15:38:04.572360 3647 pod.go:62] [openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dggfm8] pod update took 24.848027ms I0428 15:38:04.572387 3647 base_network_controller_pods.go:951] [default/openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dggfm8] addLogicalPort annotation time took 24.883488ms I0428 15:38:04.573516 3647 pods.go:271] [openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dggfm8] addLogicalPort took 26.231311ms, libovsdb time 842.93µs I0428 15:38:04.586922 3647 cni.go:452] [openshift-ingress-canary/ingress-canary-d4f79 0d9682696e024a6f6f7a374ded35a30a1a28dddd43c5ae89f745c18d13417956 network default NAD default NAD key ] ADD starting CNI request [openshift-ingress-canary/ingress-canary-d4f79 0d9682696e024a6f6f7a374ded35a30a1a28dddd43c5ae89f745c18d13417956 network default NAD default NAD key ] I0428 15:38:04.590257 3647 helper_linux.go:556] ConfigureOVS: namespace: openshift-insights, podName: insights-runtime-extractor-whzzz, hostIfaceName: d0384da407f63c6, network: default, NAD default, SandboxID: "d0384da407f63c6d7aaf6c4588573378c55fd1422318af47517acb77886370ab", PCI device ID: , UID: "5b813b7d-d76e-4665-ac22-1d2688ef9685", MAC: 0a:58:0a:82:00:05, IPs: [10.130.0.5/23] I0428 15:38:04.596339 3647 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress-canary, podName: ingress-canary-d4f79, hostIfaceName: 0d9682696e024a6, network: default, NAD default, SandboxID: "0d9682696e024a6f6f7a374ded35a30a1a28dddd43c5ae89f745c18d13417956", PCI device ID: , UID: "c7c808c9-bff7-431e-bbb5-104b8b9e3908", MAC: 0a:58:0a:82:00:06, IPs: [10.130.0.6/23] I0428 15:38:04.639826 3647 cni.go:473] [openshift-insights/insights-runtime-extractor-whzzz d0384da407f63c6d7aaf6c4588573378c55fd1422318af47517acb77886370ab network default NAD default NAD key default] ADD finished CNI request [openshift-insights/insights-runtime-extractor-whzzz d0384da407f63c6d7aaf6c4588573378c55fd1422318af47517acb77886370ab network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"46:f5:81:6a:43:ce\",\"name\":\"d0384da407f63c6\"},{\"mac\":\"0a:58:0a:82:00:05\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8354b33e-7bc4-42e8-a02f-f9d49bf46192\"}],\"ips\":[{\"address\":\"10.130.0.5/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0428 15:38:04.656108 3647 cni.go:473] [openshift-ingress-canary/ingress-canary-d4f79 0d9682696e024a6f6f7a374ded35a30a1a28dddd43c5ae89f745c18d13417956 network default NAD default NAD key default] ADD finished CNI request [openshift-ingress-canary/ingress-canary-d4f79 0d9682696e024a6f6f7a374ded35a30a1a28dddd43c5ae89f745c18d13417956 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"e2:6e:5d:5c:10:c7\",\"name\":\"0d9682696e024a6\"},{\"mac\":\"0a:58:0a:82:00:06\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/edfe348d-b692-47ef-958d-80b8fdf5d8b0\"}],\"ips\":[{\"address\":\"10.130.0.6/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0428 15:38:04.659468 3647 cni.go:452] [openshift-dns/dns-default-7rtsv 25db9206b4e9e60904b8349b457d308ba2b1f3960fc6bc52e32e9b82bcc0a0be network default NAD default NAD key ] ADD starting CNI request [openshift-dns/dns-default-7rtsv 25db9206b4e9e60904b8349b457d308ba2b1f3960fc6bc52e32e9b82bcc0a0be network default NAD default NAD key ] I0428 15:38:04.669813 3647 helper_linux.go:556] ConfigureOVS: namespace: openshift-dns, podName: dns-default-7rtsv, hostIfaceName: 25db9206b4e9e60, network: default, NAD default, SandboxID: "25db9206b4e9e60904b8349b457d308ba2b1f3960fc6bc52e32e9b82bcc0a0be", PCI device ID: , UID: "934619da-2edb-4e07-8d75-d1cbcdc45b20", MAC: 0a:58:0a:82:00:07, IPs: [10.130.0.7/23] I0428 15:38:04.677031 3647 cni.go:452] [openshift-image-registry/image-registry-7d9d855f7f-vmhtv a64b5e7d907ea6bff901c86a4ff0c9c2d3df66138f8f7aefe91a93983f8783e4 network default NAD default NAD key ] ADD starting CNI request [openshift-image-registry/image-registry-7d9d855f7f-vmhtv a64b5e7d907ea6bff901c86a4ff0c9c2d3df66138f8f7aefe91a93983f8783e4 network default NAD default NAD key ] I0428 15:38:04.688183 3647 helper_linux.go:556] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-7d9d855f7f-vmhtv, hostIfaceName: a64b5e7d907ea6b, network: default, NAD default, SandboxID: "a64b5e7d907ea6bff901c86a4ff0c9c2d3df66138f8f7aefe91a93983f8783e4", PCI device ID: , UID: "d234d8e6-2c41-49f9-b9b1-06211ba027a3", MAC: 0a:58:0a:82:00:08, IPs: [10.130.0.8/23] I0428 15:38:04.720361 3647 cni.go:473] [openshift-dns/dns-default-7rtsv 25db9206b4e9e60904b8349b457d308ba2b1f3960fc6bc52e32e9b82bcc0a0be network default NAD default NAD key default] ADD finished CNI request [openshift-dns/dns-default-7rtsv 25db9206b4e9e60904b8349b457d308ba2b1f3960fc6bc52e32e9b82bcc0a0be network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"de:89:79:91:08:e4\",\"name\":\"25db9206b4e9e60\"},{\"mac\":\"0a:58:0a:82:00:07\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2a559327-8373-44fa-bf94-92d83dbb1738\"}],\"ips\":[{\"address\":\"10.130.0.7/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0428 15:38:04.735854 3647 cni.go:473] [openshift-image-registry/image-registry-7d9d855f7f-vmhtv a64b5e7d907ea6bff901c86a4ff0c9c2d3df66138f8f7aefe91a93983f8783e4 network default NAD default NAD key default] ADD finished CNI request [openshift-image-registry/image-registry-7d9d855f7f-vmhtv a64b5e7d907ea6bff901c86a4ff0c9c2d3df66138f8f7aefe91a93983f8783e4 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"9a:c5:5a:f8:c8:73\",\"name\":\"a64b5e7d907ea6b\"},{\"mac\":\"0a:58:0a:82:00:08\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b1993e7f-7cbe-41b1-95c4-970329b60bdb\"}],\"ips\":[{\"address\":\"10.130.0.8/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0428 15:38:04.915245 3647 cni.go:452] [openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dggfm8 917af8f2df049a7628f59b3e35f00a84d2bafa98c1e6f72cdda2ea31f2f04572 network default NAD default NAD key ] ADD starting CNI request [openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dggfm8 917af8f2df049a7628f59b3e35f00a84d2bafa98c1e6f72cdda2ea31f2f04572 network default NAD default NAD key ] I0428 15:38:04.925866 3647 helper_linux.go:556] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: 99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dggfm8, hostIfaceName: 917af8f2df049a7, network: default, NAD default, SandboxID: "917af8f2df049a7628f59b3e35f00a84d2bafa98c1e6f72cdda2ea31f2f04572", PCI device ID: , UID: "2147751c-1e00-4a2d-87ff-f84527fc1af1", MAC: 0a:58:0a:82:00:09, IPs: [10.130.0.9/23] I0428 15:38:04.973489 3647 cni.go:473] [openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dggfm8 917af8f2df049a7628f59b3e35f00a84d2bafa98c1e6f72cdda2ea31f2f04572 network default NAD default NAD key default] ADD finished CNI request [openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dggfm8 917af8f2df049a7628f59b3e35f00a84d2bafa98c1e6f72cdda2ea31f2f04572 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"62:36:53:7f:f6:57\",\"name\":\"917af8f2df049a7\"},{\"mac\":\"0a:58:0a:82:00:09\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f0f741b2-95bd-4c97-b956-6b47a94792ce\"}],\"ips\":[{\"address\":\"10.130.0.9/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0428 15:38:05.208426 3647 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0428 15:38:05.242981 3647 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0428 15:38:05.757827 3647 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-bcvmz] creating logical port openshift-monitoring_prometheus-operator-admission-webhook-59b79769fb-bcvmz for pod on switch ip-10-0-1-106.ec2.internal I0428 15:38:05.757948 3647 kube.go:256] Updating pod openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-bcvmz I0428 15:38:05.777089 3647 pod.go:62] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-bcvmz] pod update took 19.173984ms I0428 15:38:05.777119 3647 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-bcvmz] addLogicalPort annotation time took 19.210332ms I0428 15:38:05.778358 3647 pods.go:271] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-bcvmz] addLogicalPort took 20.551488ms, libovsdb time 931.948µs I0428 15:38:07.437220 3647 cni.go:452] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-bcvmz a69fcb5453a7c26f3bd0e2c0315a186a4633ff58a86dacb5cc4a168c9b3fcc7e network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-bcvmz a69fcb5453a7c26f3bd0e2c0315a186a4633ff58a86dacb5cc4a168c9b3fcc7e network default NAD default NAD key ] I0428 15:38:07.448018 3647 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-operator-admission-webhook-59b79769fb-bcvmz, hostIfaceName: a69fcb5453a7c26, network: default, NAD default, SandboxID: "a69fcb5453a7c26f3bd0e2c0315a186a4633ff58a86dacb5cc4a168c9b3fcc7e", PCI device ID: , UID: "dc23a9fd-ce24-4a87-bb64-2cf64a9fec53", MAC: 0a:58:0a:82:00:0a, IPs: [10.130.0.10/23] I0428 15:38:07.496263 3647 cni.go:473] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-bcvmz a69fcb5453a7c26f3bd0e2c0315a186a4633ff58a86dacb5cc4a168c9b3fcc7e network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-bcvmz a69fcb5453a7c26f3bd0e2c0315a186a4633ff58a86dacb5cc4a168c9b3fcc7e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"c6:a0:bd:4d:ab:eb\",\"name\":\"a69fcb5453a7c26\"},{\"mac\":\"0a:58:0a:82:00:0a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/020c7608-4f3a-4387-8093-cf4f446ed5fd\"}],\"ips\":[{\"address\":\"10.130.0.10/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0428 15:38:09.266510 3647 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook I0428 15:38:10.848042 3647 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-operator-76b589d668-q88gs] creating logical port openshift-monitoring_prometheus-operator-76b589d668-q88gs for pod on switch ip-10-0-1-106.ec2.internal I0428 15:38:10.848155 3647 kube.go:256] Updating pod openshift-monitoring/prometheus-operator-76b589d668-q88gs I0428 15:38:10.867285 3647 pod.go:62] [openshift-monitoring/prometheus-operator-76b589d668-q88gs] pod update took 19.162266ms I0428 15:38:10.867306 3647 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-operator-76b589d668-q88gs] addLogicalPort annotation time took 19.18894ms I0428 15:38:10.868626 3647 pods.go:271] [openshift-monitoring/prometheus-operator-76b589d668-q88gs] addLogicalPort took 20.607169ms, libovsdb time 995.999µs I0428 15:38:11.949549 3647 base_network_controller_pods.go:487] [default/openshift-console/console-957cdcb8f-n8rgx] creating logical port openshift-console_console-957cdcb8f-n8rgx for pod on switch ip-10-0-1-106.ec2.internal I0428 15:38:11.949667 3647 kube.go:256] Updating pod openshift-console/console-957cdcb8f-n8rgx I0428 15:38:11.966788 3647 pod.go:62] [openshift-console/console-957cdcb8f-n8rgx] pod update took 17.152456ms I0428 15:38:11.966815 3647 base_network_controller_pods.go:951] [default/openshift-console/console-957cdcb8f-n8rgx] addLogicalPort annotation time took 17.184265ms I0428 15:38:11.968202 3647 pods.go:271] [openshift-console/console-957cdcb8f-n8rgx] addLogicalPort took 18.665719ms, libovsdb time 1.01517ms I0428 15:38:12.525458 3647 cni.go:452] [openshift-console/console-957cdcb8f-n8rgx 0ac0c0d5ecb6ea0f667ae43a032fa663d2ba436214e3e0f8220b44c21a203bea network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-957cdcb8f-n8rgx 0ac0c0d5ecb6ea0f667ae43a032fa663d2ba436214e3e0f8220b44c21a203bea network default NAD default NAD key ] I0428 15:38:12.525985 3647 cni.go:452] [openshift-monitoring/prometheus-operator-76b589d668-q88gs fea01d59a64c3d688df686ef88f119a8f750adfd6407eadc04d8c5749d2770d8 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-operator-76b589d668-q88gs fea01d59a64c3d688df686ef88f119a8f750adfd6407eadc04d8c5749d2770d8 network default NAD default NAD key ] I0428 15:38:12.538188 3647 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-operator-76b589d668-q88gs, hostIfaceName: fea01d59a64c3d6, network: default, NAD default, SandboxID: "fea01d59a64c3d688df686ef88f119a8f750adfd6407eadc04d8c5749d2770d8", PCI device ID: , UID: "eb5b1a6e-8d88-42eb-b410-681dca1a2b23", MAC: 0a:58:0a:82:00:0b, IPs: [10.130.0.11/23] I0428 15:38:12.539015 3647 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-957cdcb8f-n8rgx, hostIfaceName: 0ac0c0d5ecb6ea0, network: default, NAD default, SandboxID: "0ac0c0d5ecb6ea0f667ae43a032fa663d2ba436214e3e0f8220b44c21a203bea", PCI device ID: , UID: "76305703-d07b-4161-baab-0949355bc8d5", MAC: 0a:58:0a:82:00:0c, IPs: [10.130.0.12/23] I0428 15:38:12.587102 3647 cni.go:473] [openshift-monitoring/prometheus-operator-76b589d668-q88gs fea01d59a64c3d688df686ef88f119a8f750adfd6407eadc04d8c5749d2770d8 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-operator-76b589d668-q88gs fea01d59a64c3d688df686ef88f119a8f750adfd6407eadc04d8c5749d2770d8 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"3e:d2:12:b5:3f:f1\",\"name\":\"fea01d59a64c3d6\"},{\"mac\":\"0a:58:0a:82:00:0b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/da0c1477-5290-479c-9e38-f98f3ea50740\"}],\"ips\":[{\"address\":\"10.130.0.11/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0428 15:38:12.600692 3647 cni.go:473] [openshift-console/console-957cdcb8f-n8rgx 0ac0c0d5ecb6ea0f667ae43a032fa663d2ba436214e3e0f8220b44c21a203bea network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-957cdcb8f-n8rgx 0ac0c0d5ecb6ea0f667ae43a032fa663d2ba436214e3e0f8220b44c21a203bea network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"e2:f3:fc:6a:66:63\",\"name\":\"0ac0c0d5ecb6ea0\"},{\"mac\":\"0a:58:0a:82:00:0c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/cef81822-87df-4f58-8250-6ac714e2b772\"}],\"ips\":[{\"address\":\"10.130.0.12/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0428 15:38:13.478693 3647 util.go:959] Failed to build node endpoints for node ip-10-0-1-106.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0428 15:38:13.478812 3647 util.go:959] Failed to build node endpoints for node ip-10-0-1-106.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0428 15:38:16.280871 3647 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0428 15:38:16.292192 3647 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0428 15:38:17.252849 3647 util.go:959] Failed to build node endpoints for node ip-10-0-1-106.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0428 15:38:17.252870 3647 util.go:959] Failed to build node endpoints for node ip-10-0-1-106.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0428 15:38:17.252876 3647 util.go:959] Failed to build node endpoints for node ip-10-0-1-106.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0428 15:38:17.252887 3647 util.go:959] Failed to build node endpoints for node ip-10-0-1-106.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0428 15:38:17.252893 3647 util.go:959] Failed to build node endpoints for node ip-10-0-1-106.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0428 15:38:17.252897 3647 util.go:959] Failed to build node endpoints for node ip-10-0-1-106.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0428 15:38:18.213024 3647 util.go:959] Failed to build node endpoints for node ip-10-0-1-106.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0428 15:38:18.213046 3647 util.go:959] Failed to build node endpoints for node ip-10-0-1-106.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0428 15:38:18.213052 3647 util.go:959] Failed to build node endpoints for node ip-10-0-1-106.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default I0428 15:38:18.443290 3647 base_network_controller_pods.go:487] [default/openshift-monitoring/openshift-state-metrics-b8587444b-jdh5v] creating logical port openshift-monitoring_openshift-state-metrics-b8587444b-jdh5v for pod on switch ip-10-0-1-106.ec2.internal I0428 15:38:18.443399 3647 kube.go:256] Updating pod openshift-monitoring/openshift-state-metrics-b8587444b-jdh5v I0428 15:38:18.470174 3647 pod.go:62] [openshift-monitoring/openshift-state-metrics-b8587444b-jdh5v] pod update took 26.803861ms I0428 15:38:18.470193 3647 base_network_controller_pods.go:951] [default/openshift-monitoring/openshift-state-metrics-b8587444b-jdh5v] addLogicalPort annotation time took 26.828491ms I0428 15:38:18.471308 3647 pods.go:271] [openshift-monitoring/openshift-state-metrics-b8587444b-jdh5v] addLogicalPort took 28.041443ms, libovsdb time 849.188µs I0428 15:38:19.536120 3647 base_network_controller_pods.go:487] [default/openshift-monitoring/alertmanager-main-1] creating logical port openshift-monitoring_alertmanager-main-1 for pod on switch ip-10-0-1-106.ec2.internal I0428 15:38:19.536216 3647 kube.go:256] Updating pod openshift-monitoring/alertmanager-main-1 I0428 15:38:19.565740 3647 pod.go:62] [openshift-monitoring/alertmanager-main-1] pod update took 29.557409ms I0428 15:38:19.565768 3647 base_network_controller_pods.go:951] [default/openshift-monitoring/alertmanager-main-1] addLogicalPort annotation time took 29.59194ms I0428 15:38:19.567051 3647 pods.go:271] [openshift-monitoring/alertmanager-main-1] addLogicalPort took 30.941978ms, libovsdb time 905.053µs I0428 15:38:19.949773 3647 cni.go:452] [openshift-monitoring/alertmanager-main-1 8091c7450c2c694aad1af097eb28ae8aa74525bcecd1e103573b799b1626e1aa network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/alertmanager-main-1 8091c7450c2c694aad1af097eb28ae8aa74525bcecd1e103573b799b1626e1aa network default NAD default NAD key ] I0428 15:38:19.959065 3647 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: alertmanager-main-1, hostIfaceName: 8091c7450c2c694, network: default, NAD default, SandboxID: "8091c7450c2c694aad1af097eb28ae8aa74525bcecd1e103573b799b1626e1aa", PCI device ID: , UID: "9404d230-659d-4ebc-947d-fb631b9d11be", MAC: 0a:58:0a:82:00:0e, IPs: [10.130.0.14/23] I0428 15:38:20.008001 3647 cni.go:473] [openshift-monitoring/alertmanager-main-1 8091c7450c2c694aad1af097eb28ae8aa74525bcecd1e103573b799b1626e1aa network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/alertmanager-main-1 8091c7450c2c694aad1af097eb28ae8aa74525bcecd1e103573b799b1626e1aa network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"f6:d0:bd:4d:2f:f3\",\"name\":\"8091c7450c2c694\"},{\"mac\":\"0a:58:0a:82:00:0e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a295f070-11d2-4218-ba36-f039dccba4b3\"}],\"ips\":[{\"address\":\"10.130.0.14/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0428 15:38:20.300342 3647 cni.go:452] [openshift-monitoring/openshift-state-metrics-b8587444b-jdh5v c54b99d67c27d415cfe17ec2714039804a22ab5780c49e3f6187b60a2c39c86a network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/openshift-state-metrics-b8587444b-jdh5v c54b99d67c27d415cfe17ec2714039804a22ab5780c49e3f6187b60a2c39c86a network default NAD default NAD key ] I0428 15:38:20.310054 3647 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: openshift-state-metrics-b8587444b-jdh5v, hostIfaceName: c54b99d67c27d41, network: default, NAD default, SandboxID: "c54b99d67c27d415cfe17ec2714039804a22ab5780c49e3f6187b60a2c39c86a", PCI device ID: , UID: "d47295ae-4c29-405f-9261-fbad7be9f647", MAC: 0a:58:0a:82:00:0d, IPs: [10.130.0.13/23] I0428 15:38:20.358471 3647 cni.go:473] [openshift-monitoring/openshift-state-metrics-b8587444b-jdh5v c54b99d67c27d415cfe17ec2714039804a22ab5780c49e3f6187b60a2c39c86a network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/openshift-state-metrics-b8587444b-jdh5v c54b99d67c27d415cfe17ec2714039804a22ab5780c49e3f6187b60a2c39c86a network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"3a:df:44:97:ec:2f\",\"name\":\"c54b99d67c27d41\"},{\"mac\":\"0a:58:0a:82:00:0d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/11fc1947-644d-4bdd-a778-2e5d96195327\"}],\"ips\":[{\"address\":\"10.130.0.13/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0428 15:38:20.489757 3647 base_network_controller_pods.go:487] [default/openshift-monitoring/thanos-querier-5f7b998cbb-wnwvq] creating logical port openshift-monitoring_thanos-querier-5f7b998cbb-wnwvq for pod on switch ip-10-0-1-106.ec2.internal I0428 15:38:20.489893 3647 kube.go:256] Updating pod openshift-monitoring/thanos-querier-5f7b998cbb-wnwvq I0428 15:38:20.514559 3647 pod.go:62] [openshift-monitoring/thanos-querier-5f7b998cbb-wnwvq] pod update took 24.702029ms I0428 15:38:20.514581 3647 base_network_controller_pods.go:951] [default/openshift-monitoring/thanos-querier-5f7b998cbb-wnwvq] addLogicalPort annotation time took 24.729725ms I0428 15:38:20.515683 3647 pods.go:271] [openshift-monitoring/thanos-querier-5f7b998cbb-wnwvq] addLogicalPort took 25.947722ms, libovsdb time 799.368µs I0428 15:38:20.878080 3647 cni.go:452] [openshift-monitoring/thanos-querier-5f7b998cbb-wnwvq ee19df0339a1ec0a4a79c03248b6639976195c4c9d9ad2cb3afa6e3714aa1b46 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/thanos-querier-5f7b998cbb-wnwvq ee19df0339a1ec0a4a79c03248b6639976195c4c9d9ad2cb3afa6e3714aa1b46 network default NAD default NAD key ] I0428 15:38:20.888163 3647 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: thanos-querier-5f7b998cbb-wnwvq, hostIfaceName: ee19df0339a1ec0, network: default, NAD default, SandboxID: "ee19df0339a1ec0a4a79c03248b6639976195c4c9d9ad2cb3afa6e3714aa1b46", PCI device ID: , UID: "4298eefa-5e65-4b40-8da8-7bed7bd74273", MAC: 0a:58:0a:82:00:0f, IPs: [10.130.0.15/23] I0428 15:38:20.942102 3647 cni.go:473] [openshift-monitoring/thanos-querier-5f7b998cbb-wnwvq ee19df0339a1ec0a4a79c03248b6639976195c4c9d9ad2cb3afa6e3714aa1b46 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/thanos-querier-5f7b998cbb-wnwvq ee19df0339a1ec0a4a79c03248b6639976195c4c9d9ad2cb3afa6e3714aa1b46 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ee:24:83:c7:7b:e0\",\"name\":\"ee19df0339a1ec0\"},{\"mac\":\"0a:58:0a:82:00:0f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7a908239-05f1-43dc-9b06-04ccc359aca7\"}],\"ips\":[{\"address\":\"10.130.0.15/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0428 15:38:22.341650 3647 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0428 15:38:22.341671 3647 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0428 15:38:22.341677 3647 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0428 15:38:23.034395 3647 base_network_controller_pods.go:487] [default/openshift-monitoring/metrics-server-7d968657fc-v7z67] creating logical port openshift-monitoring_metrics-server-7d968657fc-v7z67 for pod on switch ip-10-0-1-106.ec2.internal I0428 15:38:23.034522 3647 kube.go:256] Updating pod openshift-monitoring/metrics-server-7d968657fc-v7z67 I0428 15:38:23.055460 3647 pod.go:62] [openshift-monitoring/metrics-server-7d968657fc-v7z67] pod update took 20.979743ms I0428 15:38:23.055483 3647 base_network_controller_pods.go:951] [default/openshift-monitoring/metrics-server-7d968657fc-v7z67] addLogicalPort annotation time took 21.011375ms I0428 15:38:23.056762 3647 pods.go:271] [openshift-monitoring/metrics-server-7d968657fc-v7z67] addLogicalPort took 22.388568ms, libovsdb time 889.74µs I0428 15:38:23.105579 3647 base_network_controller_pods.go:487] [default/openshift-monitoring/monitoring-plugin-5cbb48bf6b-8xgkq] creating logical port openshift-monitoring_monitoring-plugin-5cbb48bf6b-8xgkq for pod on switch ip-10-0-1-106.ec2.internal I0428 15:38:23.105721 3647 kube.go:256] Updating pod openshift-monitoring/monitoring-plugin-5cbb48bf6b-8xgkq I0428 15:38:23.129166 3647 pod.go:62] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-8xgkq] pod update took 23.485488ms I0428 15:38:23.129186 3647 base_network_controller_pods.go:951] [default/openshift-monitoring/monitoring-plugin-5cbb48bf6b-8xgkq] addLogicalPort annotation time took 23.512504ms I0428 15:38:23.130510 3647 pods.go:271] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-8xgkq] addLogicalPort took 24.951412ms, libovsdb time 998.181µs W0428 15:38:23.317910 3647 util.go:959] Failed to build node endpoints for node ip-10-0-1-106.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0428 15:38:23.317936 3647 util.go:959] Failed to build node endpoints for node ip-10-0-1-106.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0428 15:38:23.483783 3647 cni.go:452] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-8xgkq 1ec36c9b0ca509938356ff85c92a461becebf5cee0d728be3005f7f181ef0147 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/monitoring-plugin-5cbb48bf6b-8xgkq 1ec36c9b0ca509938356ff85c92a461becebf5cee0d728be3005f7f181ef0147 network default NAD default NAD key ] I0428 15:38:23.485632 3647 cni.go:452] [openshift-monitoring/metrics-server-7d968657fc-v7z67 5de9601fb3f89a0a0c0914793b956db2474fa6baf7f4add7e93bcc4a0c117326 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/metrics-server-7d968657fc-v7z67 5de9601fb3f89a0a0c0914793b956db2474fa6baf7f4add7e93bcc4a0c117326 network default NAD default NAD key ] I0428 15:38:23.495887 3647 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: monitoring-plugin-5cbb48bf6b-8xgkq, hostIfaceName: 1ec36c9b0ca5099, network: default, NAD default, SandboxID: "1ec36c9b0ca509938356ff85c92a461becebf5cee0d728be3005f7f181ef0147", PCI device ID: , UID: "afa6823f-b536-434c-ab1f-2e8cbf8739f3", MAC: 0a:58:0a:82:00:11, IPs: [10.130.0.17/23] I0428 15:38:23.497843 3647 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: metrics-server-7d968657fc-v7z67, hostIfaceName: 5de9601fb3f89a0, network: default, NAD default, SandboxID: "5de9601fb3f89a0a0c0914793b956db2474fa6baf7f4add7e93bcc4a0c117326", PCI device ID: , UID: "cb289215-402a-4675-aed5-a4452bc83acf", MAC: 0a:58:0a:82:00:10, IPs: [10.130.0.16/23] I0428 15:38:23.545435 3647 cni.go:473] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-8xgkq 1ec36c9b0ca509938356ff85c92a461becebf5cee0d728be3005f7f181ef0147 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/monitoring-plugin-5cbb48bf6b-8xgkq 1ec36c9b0ca509938356ff85c92a461becebf5cee0d728be3005f7f181ef0147 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"26:9a:93:f5:38:f8\",\"name\":\"1ec36c9b0ca5099\"},{\"mac\":\"0a:58:0a:82:00:11\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/92af8415-006d-4ab4-8dcf-d3881edde45c\"}],\"ips\":[{\"address\":\"10.130.0.17/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0428 15:38:23.563243 3647 cni.go:473] [openshift-monitoring/metrics-server-7d968657fc-v7z67 5de9601fb3f89a0a0c0914793b956db2474fa6baf7f4add7e93bcc4a0c117326 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/metrics-server-7d968657fc-v7z67 5de9601fb3f89a0a0c0914793b956db2474fa6baf7f4add7e93bcc4a0c117326 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"26:40:e5:f6:b3:30\",\"name\":\"5de9601fb3f89a0\"},{\"mac\":\"0a:58:0a:82:00:10\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1b8c33f6-a205-45cf-9c2a-4f8e5cc3706f\"}],\"ips\":[{\"address\":\"10.130.0.16/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0428 15:38:23.809367 3647 base_network_controller_pods.go:487] [default/openshift-monitoring/telemeter-client-64cddff44c-tnc74] creating logical port openshift-monitoring_telemeter-client-64cddff44c-tnc74 for pod on switch ip-10-0-1-106.ec2.internal I0428 15:38:23.809469 3647 kube.go:256] Updating pod openshift-monitoring/telemeter-client-64cddff44c-tnc74 I0428 15:38:23.836548 3647 pod.go:62] [openshift-monitoring/telemeter-client-64cddff44c-tnc74] pod update took 27.107851ms I0428 15:38:23.836575 3647 base_network_controller_pods.go:951] [default/openshift-monitoring/telemeter-client-64cddff44c-tnc74] addLogicalPort annotation time took 27.139009ms I0428 15:38:23.837799 3647 pods.go:271] [openshift-monitoring/telemeter-client-64cddff44c-tnc74] addLogicalPort took 28.448654ms, libovsdb time 879.237µs I0428 15:38:24.174255 3647 cni.go:452] [openshift-monitoring/telemeter-client-64cddff44c-tnc74 f0ed62042b03b1d34f37cc164bf56efde3a0f2ad9da7dc16b55f345ead345913 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/telemeter-client-64cddff44c-tnc74 f0ed62042b03b1d34f37cc164bf56efde3a0f2ad9da7dc16b55f345ead345913 network default NAD default NAD key ] I0428 15:38:24.185623 3647 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: telemeter-client-64cddff44c-tnc74, hostIfaceName: f0ed62042b03b1d, network: default, NAD default, SandboxID: "f0ed62042b03b1d34f37cc164bf56efde3a0f2ad9da7dc16b55f345ead345913", PCI device ID: , UID: "18db2359-20ca-430e-a919-6b9a0485e96e", MAC: 0a:58:0a:82:00:12, IPs: [10.130.0.18/23] I0428 15:38:24.245622 3647 cni.go:473] [openshift-monitoring/telemeter-client-64cddff44c-tnc74 f0ed62042b03b1d34f37cc164bf56efde3a0f2ad9da7dc16b55f345ead345913 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/telemeter-client-64cddff44c-tnc74 f0ed62042b03b1d34f37cc164bf56efde3a0f2ad9da7dc16b55f345ead345913 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"82:ac:0d:df:3f:0e\",\"name\":\"f0ed62042b03b1d\"},{\"mac\":\"0a:58:0a:82:00:12\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f37cf66f-d043-4f88-851b-341b07d70757\"}],\"ips\":[{\"address\":\"10.130.0.18/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0428 15:38:24.321161 3647 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads I0428 15:38:24.883437 3647 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-k8s-0] creating logical port openshift-monitoring_prometheus-k8s-0 for pod on switch ip-10-0-1-106.ec2.internal I0428 15:38:24.883553 3647 kube.go:256] Updating pod openshift-monitoring/prometheus-k8s-0 I0428 15:38:24.913144 3647 pod.go:62] [openshift-monitoring/prometheus-k8s-0] pod update took 29.634101ms I0428 15:38:24.913168 3647 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-k8s-0] addLogicalPort annotation time took 29.667216ms I0428 15:38:24.914366 3647 pods.go:271] [openshift-monitoring/prometheus-k8s-0] addLogicalPort took 30.948206ms, libovsdb time 863.461µs W0428 15:38:25.774876 3647 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-console/console-56b489d6fc-lccqq IP address from the namespace address-set, err: pod openshift-console/console-56b489d6fc-lccqq: no pod IPs found W0428 15:38:26.205935 3647 util.go:959] Failed to build node endpoints for node ip-10-0-1-106.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0428 15:38:28.356211 3647 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0428 15:38:28.356232 3647 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0428 15:38:28.356237 3647 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0428 15:38:28.544143 3647 cni.go:452] [openshift-monitoring/prometheus-k8s-0 a6f4536597d70449602d080960670c09aca34ff8307cae44e135230aefaab8bb network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-k8s-0 a6f4536597d70449602d080960670c09aca34ff8307cae44e135230aefaab8bb network default NAD default NAD key ] I0428 15:38:28.558286 3647 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-k8s-0, hostIfaceName: a6f4536597d7044, network: default, NAD default, SandboxID: "a6f4536597d70449602d080960670c09aca34ff8307cae44e135230aefaab8bb", PCI device ID: , UID: "17215cee-8191-4f8f-9a10-5c22fb82c0dd", MAC: 0a:58:0a:82:00:13, IPs: [10.130.0.19/23] I0428 15:38:28.562929 3647 cni.go:452] [openshift-multus/network-metrics-daemon-xdscj 97a875d3fd491911597e6579ad0deb9c4600e903773a1cc80d65763af3f0d2f9 network default NAD default NAD key ] ADD starting CNI request [openshift-multus/network-metrics-daemon-xdscj 97a875d3fd491911597e6579ad0deb9c4600e903773a1cc80d65763af3f0d2f9 network default NAD default NAD key ] I0428 15:38:28.563578 3647 cni.go:452] [openshift-network-diagnostics/network-check-target-w6lhl 531b59ff5c289449d55a5e568d4011701864ed968b8b884e527bf2729fd68b61 network default NAD default NAD key ] ADD starting CNI request [openshift-network-diagnostics/network-check-target-w6lhl 531b59ff5c289449d55a5e568d4011701864ed968b8b884e527bf2729fd68b61 network default NAD default NAD key ] I0428 15:38:28.603412 3647 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-target-w6lhl, hostIfaceName: 531b59ff5c28944, network: default, NAD default, SandboxID: "531b59ff5c289449d55a5e568d4011701864ed968b8b884e527bf2729fd68b61", PCI device ID: , UID: "e697422d-c481-44b9-a83b-3cd6ab29bd4e", MAC: 0a:58:0a:82:00:03, IPs: [10.130.0.3/23] I0428 15:38:28.603811 3647 helper_linux.go:556] ConfigureOVS: namespace: openshift-multus, podName: network-metrics-daemon-xdscj, hostIfaceName: 97a875d3fd49191, network: default, NAD default, SandboxID: "97a875d3fd491911597e6579ad0deb9c4600e903773a1cc80d65763af3f0d2f9", PCI device ID: , UID: "503abab7-471d-4beb-a962-59b0b17f84f1", MAC: 0a:58:0a:82:00:04, IPs: [10.130.0.4/23] I0428 15:38:28.648144 3647 cni.go:473] [openshift-monitoring/prometheus-k8s-0 a6f4536597d70449602d080960670c09aca34ff8307cae44e135230aefaab8bb network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-k8s-0 a6f4536597d70449602d080960670c09aca34ff8307cae44e135230aefaab8bb network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"8e:2c:7b:cf:73:2d\",\"name\":\"a6f4536597d7044\"},{\"mac\":\"0a:58:0a:82:00:13\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/694922b5-a458-4116-9183-4912c5b794e3\"}],\"ips\":[{\"address\":\"10.130.0.19/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0428 15:38:28.675905 3647 cni.go:473] [openshift-network-diagnostics/network-check-target-w6lhl 531b59ff5c289449d55a5e568d4011701864ed968b8b884e527bf2729fd68b61 network default NAD default NAD key default] ADD finished CNI request [openshift-network-diagnostics/network-check-target-w6lhl 531b59ff5c289449d55a5e568d4011701864ed968b8b884e527bf2729fd68b61 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"56:ff:62:b5:82:e6\",\"name\":\"531b59ff5c28944\"},{\"mac\":\"0a:58:0a:82:00:03\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/69524f9b-5e0e-40bc-96b3-3fda72cf729e\"}],\"ips\":[{\"address\":\"10.130.0.3/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0428 15:38:28.697917 3647 cni.go:473] [openshift-multus/network-metrics-daemon-xdscj 97a875d3fd491911597e6579ad0deb9c4600e903773a1cc80d65763af3f0d2f9 network default NAD default NAD key default] ADD finished CNI request [openshift-multus/network-metrics-daemon-xdscj 97a875d3fd491911597e6579ad0deb9c4600e903773a1cc80d65763af3f0d2f9 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"5a:33:92:80:1a:2a\",\"name\":\"97a875d3fd49191\"},{\"mac\":\"0a:58:0a:82:00:04\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/55216952-7dac-4cfa-af09-dd10df714db8\"}],\"ips\":[{\"address\":\"10.130.0.4/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0428 15:38:28.765373 3647 obj_retry.go:498] Detected object openshift-image-registry/image-registry-7fc7c49d95-dgxkp of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:38:28.765955 3647 obj_retry.go:498] Detected object openshift-image-registry/image-registry-7fc7c49d95-dgxkp of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:38:29.119770 3647 obj_retry.go:492] Detected object openshift-image-registry/image-registry-7fc7c49d95-dgxkp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:38:29.119858 3647 obj_retry.go:492] Detected object openshift-image-registry/image-registry-7fc7c49d95-dgxkp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:38:29.132148 3647 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-7fc7c49d95-dgxkp I0428 15:38:29.132180 3647 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-7fc7c49d95-dgxkp W0428 15:38:29.342856 3647 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0428 15:38:29.342882 3647 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0428 15:38:29.440304 3647 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0428 15:38:29.514435 3647 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0428 15:38:29.605086 3647 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0428 15:38:29.605102 3647 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0428 15:38:29.687917 3647 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-console/console-69f694b5b7-j8xh7 IP address from the namespace address-set, err: pod openshift-console/console-69f694b5b7-j8xh7: no pod IPs found W0428 15:38:29.693497 3647 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-console/console-69f694b5b7-9ddmp IP address from the namespace address-set, err: pod openshift-console/console-69f694b5b7-9ddmp: no pod IPs found I0428 15:38:30.349122 3647 cni.go:452] [openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dggfm8 917af8f2df049a7628f59b3e35f00a84d2bafa98c1e6f72cdda2ea31f2f04572 network default NAD default NAD key ] DEL starting CNI request [openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dggfm8 917af8f2df049a7628f59b3e35f00a84d2bafa98c1e6f72cdda2ea31f2f04572 network default NAD default NAD key ] I0428 15:38:30.419025 3647 cni.go:473] [openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dggfm8 917af8f2df049a7628f59b3e35f00a84d2bafa98c1e6f72cdda2ea31f2f04572 network default NAD default NAD key default] DEL finished CNI request [openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dggfm8 917af8f2df049a7628f59b3e35f00a84d2bafa98c1e6f72cdda2ea31f2f04572 network default NAD default NAD key default], result "{}", err W0428 15:38:30.520915 3647 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server I0428 15:38:31.423268 3647 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dggfm8 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:38:31.423302 3647 pods.go:176] Deleting pod: openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dggfm8 I0428 15:38:31.424591 3647 pods.go:236] Attempting to release IPs for pod: openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dggfm8, ips: 10.130.0.9 I0428 15:38:31.424618 3647 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dggfm8 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:38:31.424638 3647 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dggfm8 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 15:38:31.424647 3647 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles to delete 1 local pods... I0428 15:38:31.424856 3647 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dggfm8 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 15:38:31.424883 3647 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... W0428 15:38:32.350850 3647 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0428 15:38:32.449883 3647 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dggfm8 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:38:32.449913 3647 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dggfm8 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:38:32.449933 3647 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dggfm8 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:38:32.449946 3647 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dggfm8 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed W0428 15:38:32.989382 3647 util.go:959] Failed to build node endpoints for node ip-10-0-1-106.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0428 15:38:32.989409 3647 util.go:959] Failed to build node endpoints for node ip-10-0-1-106.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0428 15:38:34.772271 3647 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0428 15:38:34.787993 3647 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0428 15:38:41.373338 3647 cni.go:452] [openshift-console/console-957cdcb8f-n8rgx 0ac0c0d5ecb6ea0f667ae43a032fa663d2ba436214e3e0f8220b44c21a203bea network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-957cdcb8f-n8rgx 0ac0c0d5ecb6ea0f667ae43a032fa663d2ba436214e3e0f8220b44c21a203bea network default NAD default NAD key ] I0428 15:38:41.463795 3647 cni.go:473] [openshift-console/console-957cdcb8f-n8rgx 0ac0c0d5ecb6ea0f667ae43a032fa663d2ba436214e3e0f8220b44c21a203bea network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-957cdcb8f-n8rgx 0ac0c0d5ecb6ea0f667ae43a032fa663d2ba436214e3e0f8220b44c21a203bea network default NAD default NAD key default], result "{}", err I0428 15:38:41.509151 3647 obj_retry.go:498] Detected object openshift-console/console-957cdcb8f-n8rgx of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:38:41.509179 3647 pods.go:176] Deleting pod: openshift-console/console-957cdcb8f-n8rgx I0428 15:38:41.510558 3647 pods.go:236] Attempting to release IPs for pod: openshift-console/console-957cdcb8f-n8rgx, ips: 10.130.0.12 I0428 15:38:41.510587 3647 obj_retry.go:498] Detected object openshift-console/console-957cdcb8f-n8rgx of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:38:41.521938 3647 base_network_controller_pods.go:487] [default/openshift-console/console-5d45f7999c-zgjjj] creating logical port openshift-console_console-5d45f7999c-zgjjj for pod on switch ip-10-0-1-106.ec2.internal I0428 15:38:41.522083 3647 kube.go:256] Updating pod openshift-console/console-5d45f7999c-zgjjj I0428 15:38:41.537349 3647 pod.go:62] [openshift-console/console-5d45f7999c-zgjjj] pod update took 15.294074ms I0428 15:38:41.537375 3647 base_network_controller_pods.go:951] [default/openshift-console/console-5d45f7999c-zgjjj] addLogicalPort annotation time took 15.323169ms I0428 15:38:41.538676 3647 pods.go:271] [openshift-console/console-5d45f7999c-zgjjj] addLogicalPort took 16.750246ms, libovsdb time 980.526µs I0428 15:38:41.883808 3647 cni.go:452] [openshift-console/console-5d45f7999c-zgjjj 83d64fca4ec9a189748c01975b5eaa951a6a29d267fbb332239746d0becf355a network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-5d45f7999c-zgjjj 83d64fca4ec9a189748c01975b5eaa951a6a29d267fbb332239746d0becf355a network default NAD default NAD key ] I0428 15:38:41.893293 3647 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-5d45f7999c-zgjjj, hostIfaceName: 83d64fca4ec9a18, network: default, NAD default, SandboxID: "83d64fca4ec9a189748c01975b5eaa951a6a29d267fbb332239746d0becf355a", PCI device ID: , UID: "bb905c8f-8b49-4fa4-9af2-3fa5a88f4b5c", MAC: 0a:58:0a:82:00:14, IPs: [10.130.0.20/23] I0428 15:38:41.941777 3647 cni.go:473] [openshift-console/console-5d45f7999c-zgjjj 83d64fca4ec9a189748c01975b5eaa951a6a29d267fbb332239746d0becf355a network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-5d45f7999c-zgjjj 83d64fca4ec9a189748c01975b5eaa951a6a29d267fbb332239746d0becf355a network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"fa:90:ec:58:87:c7\",\"name\":\"83d64fca4ec9a18\"},{\"mac\":\"0a:58:0a:82:00:14\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/be75da18-f759-4b1c-b30b-9c7bc41a4d77\"}],\"ips\":[{\"address\":\"10.130.0.20/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0428 15:38:42.411183 3647 obj_retry.go:492] Detected object openshift-console/console-957cdcb8f-n8rgx of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:38:42.411210 3647 obj_retry.go:492] Detected object openshift-console/console-957cdcb8f-n8rgx of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:38:42.413972 3647 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-957cdcb8f-n8rgx I0428 15:38:42.414008 3647 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-957cdcb8f-n8rgx W0428 15:38:46.194278 3647 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-service-ca-operator/metrics W0428 15:38:49.997584 3647 util.go:959] Failed to build node endpoints for node ip-10-0-1-106.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0428 15:38:49.997604 3647 util.go:959] Failed to build node endpoints for node ip-10-0-1-106.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0428 15:38:49.997610 3647 util.go:959] Failed to build node endpoints for node ip-10-0-1-106.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0428 15:38:50.350416 3647 util.go:959] Failed to build node endpoints for node ip-10-0-1-106.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server W0428 15:38:51.238299 3647 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-insights/metrics I0428 15:38:51.647973 3647 obj_retry.go:498] Detected object openshift-image-registry/image-registry-796c59fd79-njh7k of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:38:51.648494 3647 obj_retry.go:498] Detected object openshift-image-registry/image-registry-796c59fd79-njh7k of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:38:52.428546 3647 obj_retry.go:498] Detected object openshift-console/console-56b489d6fc-b8dxl of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:38:52.429109 3647 obj_retry.go:498] Detected object openshift-console/console-56b489d6fc-b8dxl of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0428 15:38:52.464472 3647 util.go:959] Failed to build node endpoints for node ip-10-0-1-106.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0428 15:38:52.485324 3647 obj_retry.go:492] Detected object openshift-image-registry/image-registry-796c59fd79-njh7k of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:38:52.485340 3647 obj_retry.go:492] Detected object openshift-image-registry/image-registry-796c59fd79-njh7k of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:38:52.489171 3647 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-796c59fd79-njh7k I0428 15:38:52.489194 3647 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-796c59fd79-njh7k I0428 15:38:53.234850 3647 obj_retry.go:492] Detected object openshift-console/console-56b489d6fc-b8dxl of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:38:53.234870 3647 obj_retry.go:492] Detected object openshift-console/console-56b489d6fc-b8dxl of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:38:53.239687 3647 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-56b489d6fc-b8dxl I0428 15:38:53.239731 3647 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-56b489d6fc-b8dxl W0428 15:39:00.382489 3647 util.go:959] Failed to build node endpoints for node ip-10-0-1-106.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0428 15:39:00.382511 3647 util.go:959] Failed to build node endpoints for node ip-10-0-1-106.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0428 15:39:03.360269 3647 util.go:959] Failed to build node endpoints for node ip-10-0-1-106.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0428 15:39:16.320358 3647 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics I0428 15:39:17.834875 3647 obj_retry.go:498] Detected object openshift-console/console-957cdcb8f-6cq6s of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:39:17.835423 3647 obj_retry.go:498] Detected object openshift-console/console-957cdcb8f-6cq6s of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:39:18.576014 3647 obj_retry.go:492] Detected object openshift-console/console-957cdcb8f-6cq6s of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:39:18.576036 3647 obj_retry.go:492] Detected object openshift-console/console-957cdcb8f-6cq6s of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:39:18.580996 3647 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-957cdcb8f-6cq6s I0428 15:39:18.581022 3647 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-957cdcb8f-6cq6s I0428 15:39:19.358575 3647 obj_retry.go:498] Detected object openshift-ingress/router-default-5f578f8577-xnnpr of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:39:19.359109 3647 obj_retry.go:498] Detected object openshift-ingress/router-default-5f578f8577-xnnpr of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:39:20.328626 3647 obj_retry.go:492] Detected object openshift-ingress/router-default-5f578f8577-xnnpr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:39:20.328650 3647 obj_retry.go:492] Detected object openshift-ingress/router-default-5f578f8577-xnnpr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:39:20.337572 3647 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/router-default-5f578f8577-xnnpr I0428 15:39:20.337594 3647 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/router-default-5f578f8577-xnnpr W0428 15:39:25.624583 3647 util.go:959] Failed to build node endpoints for node ip-10-0-1-106.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0428 15:39:25.624602 3647 util.go:959] Failed to build node endpoints for node ip-10-0-1-106.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0428 15:39:33.166112 3647 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0428 15:39:40.951588 3647 cni.go:452] [openshift-monitoring/alertmanager-main-1 8091c7450c2c694aad1af097eb28ae8aa74525bcecd1e103573b799b1626e1aa network default NAD default NAD key ] DEL starting CNI request [openshift-monitoring/alertmanager-main-1 8091c7450c2c694aad1af097eb28ae8aa74525bcecd1e103573b799b1626e1aa network default NAD default NAD key ] I0428 15:39:41.028872 3647 cni.go:473] [openshift-monitoring/alertmanager-main-1 8091c7450c2c694aad1af097eb28ae8aa74525bcecd1e103573b799b1626e1aa network default NAD default NAD key default] DEL finished CNI request [openshift-monitoring/alertmanager-main-1 8091c7450c2c694aad1af097eb28ae8aa74525bcecd1e103573b799b1626e1aa network default NAD default NAD key default], result "{}", err I0428 15:39:41.096555 3647 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:39:41.096592 3647 pods.go:176] Deleting pod: openshift-monitoring/alertmanager-main-1 I0428 15:39:41.097902 3647 pods.go:236] Attempting to release IPs for pod: openshift-monitoring/alertmanager-main-1, ips: 10.130.0.14 I0428 15:39:41.097929 3647 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:39:41.663651 3647 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:39:41.663695 3647 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:39:41.671221 3647 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-1 I0428 15:39:41.671250 3647 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-1 I0428 15:39:41.745087 3647 base_network_controller_pods.go:487] [default/openshift-monitoring/alertmanager-main-1] creating logical port openshift-monitoring_alertmanager-main-1 for pod on switch ip-10-0-1-106.ec2.internal I0428 15:39:41.745209 3647 kube.go:256] Updating pod openshift-monitoring/alertmanager-main-1 I0428 15:39:41.840146 3647 pod.go:62] [openshift-monitoring/alertmanager-main-1] pod update took 94.975573ms I0428 15:39:41.840170 3647 base_network_controller_pods.go:951] [default/openshift-monitoring/alertmanager-main-1] addLogicalPort annotation time took 95.005957ms I0428 15:39:41.841326 3647 pods.go:271] [openshift-monitoring/alertmanager-main-1] addLogicalPort took 96.26108ms, libovsdb time 852.344µs I0428 15:39:42.098962 3647 cni.go:452] [openshift-monitoring/alertmanager-main-1 a2f1cc39f453436178f17e8e00c014d928de4ca030e325a11bfbffcadc768a94 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/alertmanager-main-1 a2f1cc39f453436178f17e8e00c014d928de4ca030e325a11bfbffcadc768a94 network default NAD default NAD key ] I0428 15:39:42.109787 3647 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: alertmanager-main-1, hostIfaceName: a2f1cc39f453436, network: default, NAD default, SandboxID: "a2f1cc39f453436178f17e8e00c014d928de4ca030e325a11bfbffcadc768a94", PCI device ID: , UID: "88b68008-9454-4a84-b7b7-29ed093ae593", MAC: 0a:58:0a:82:00:15, IPs: [10.130.0.21/23] I0428 15:39:42.169811 3647 cni.go:473] [openshift-monitoring/alertmanager-main-1 a2f1cc39f453436178f17e8e00c014d928de4ca030e325a11bfbffcadc768a94 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/alertmanager-main-1 a2f1cc39f453436178f17e8e00c014d928de4ca030e325a11bfbffcadc768a94 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"8e:1d:98:26:07:43\",\"name\":\"a2f1cc39f453436\"},{\"mac\":\"0a:58:0a:82:00:15\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c0d16efb-536f-4bb5-9114-5de640af731e\"}],\"ips\":[{\"address\":\"10.130.0.21/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0428 15:39:44.913481 3647 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:39:44.914050 3647 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:39:45.688084 3647 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:39:45.688107 3647 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:39:45.693537 3647 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-1 I0428 15:39:45.693561 3647 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-1 W0428 15:40:02.154354 3647 util.go:959] Failed to build node endpoints for node ip-10-0-1-106.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0428 15:40:02.154374 3647 util.go:959] Failed to build node endpoints for node ip-10-0-1-106.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0428 15:40:02.154380 3647 util.go:959] Failed to build node endpoints for node ip-10-0-1-106.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0428 15:40:03.445784 3647 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:40:03.446359 3647 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:40:03.778675 3647 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:40:03.778695 3647 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:40:03.785731 3647 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-0 I0428 15:40:03.785757 3647 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-0 I0428 15:40:05.662254 3647 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29623180-zgjmh of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:40:05.662786 3647 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29623180-zgjmh of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:40:06.547944 3647 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29623180-zgjmh of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:40:06.547971 3647 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29623180-zgjmh of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:40:10.363539 3647 cni.go:452] [openshift-console/console-5d45f7999c-zgjjj 83d64fca4ec9a189748c01975b5eaa951a6a29d267fbb332239746d0becf355a network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-5d45f7999c-zgjjj 83d64fca4ec9a189748c01975b5eaa951a6a29d267fbb332239746d0becf355a network default NAD default NAD key ] I0428 15:40:10.452346 3647 cni.go:473] [openshift-console/console-5d45f7999c-zgjjj 83d64fca4ec9a189748c01975b5eaa951a6a29d267fbb332239746d0becf355a network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-5d45f7999c-zgjjj 83d64fca4ec9a189748c01975b5eaa951a6a29d267fbb332239746d0becf355a network default NAD default NAD key default], result "{}", err I0428 15:40:10.501304 3647 obj_retry.go:498] Detected object openshift-console/console-5d45f7999c-zgjjj of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:40:10.501339 3647 pods.go:176] Deleting pod: openshift-console/console-5d45f7999c-zgjjj I0428 15:40:10.502520 3647 pods.go:236] Attempting to release IPs for pod: openshift-console/console-5d45f7999c-zgjjj, ips: 10.130.0.20 I0428 15:40:10.502548 3647 obj_retry.go:498] Detected object openshift-console/console-5d45f7999c-zgjjj of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:40:10.510525 3647 base_network_controller_pods.go:487] [default/openshift-console/console-7ff7689755-sjpnz] creating logical port openshift-console_console-7ff7689755-sjpnz for pod on switch ip-10-0-1-106.ec2.internal I0428 15:40:10.510643 3647 kube.go:256] Updating pod openshift-console/console-7ff7689755-sjpnz I0428 15:40:10.525076 3647 pod.go:62] [openshift-console/console-7ff7689755-sjpnz] pod update took 14.471329ms I0428 15:40:10.525099 3647 base_network_controller_pods.go:951] [default/openshift-console/console-7ff7689755-sjpnz] addLogicalPort annotation time took 14.499673ms I0428 15:40:10.526444 3647 pods.go:271] [openshift-console/console-7ff7689755-sjpnz] addLogicalPort took 15.932507ms, libovsdb time 958.052µs I0428 15:40:10.794497 3647 obj_retry.go:492] Detected object openshift-console/console-5d45f7999c-zgjjj of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:40:10.794523 3647 obj_retry.go:492] Detected object openshift-console/console-5d45f7999c-zgjjj of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:40:10.799824 3647 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-5d45f7999c-zgjjj I0428 15:40:10.799851 3647 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-5d45f7999c-zgjjj I0428 15:40:10.875110 3647 cni.go:452] [openshift-console/console-7ff7689755-sjpnz 76f75bddc22de23a7adea41a4fee190d24e97d7606f819aa3155dce3952dfd0d network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-7ff7689755-sjpnz 76f75bddc22de23a7adea41a4fee190d24e97d7606f819aa3155dce3952dfd0d network default NAD default NAD key ] I0428 15:40:10.885068 3647 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-7ff7689755-sjpnz, hostIfaceName: 76f75bddc22de23, network: default, NAD default, SandboxID: "76f75bddc22de23a7adea41a4fee190d24e97d7606f819aa3155dce3952dfd0d", PCI device ID: , UID: "f7b967da-3edb-486e-9516-1063052f4ca2", MAC: 0a:58:0a:82:00:16, IPs: [10.130.0.22/23] I0428 15:40:10.933344 3647 cni.go:473] [openshift-console/console-7ff7689755-sjpnz 76f75bddc22de23a7adea41a4fee190d24e97d7606f819aa3155dce3952dfd0d network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-7ff7689755-sjpnz 76f75bddc22de23a7adea41a4fee190d24e97d7606f819aa3155dce3952dfd0d network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"86:be:8c:36:0d:70\",\"name\":\"76f75bddc22de23\"},{\"mac\":\"0a:58:0a:82:00:16\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e1849ecb-d86f-410f-9c0f-260819cbe56d\"}],\"ips\":[{\"address\":\"10.130.0.22/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0428 15:40:21.086121 3647 obj_retry.go:498] Detected object openshift-console/console-5d45f7999c-wmx4k of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:40:21.086638 3647 obj_retry.go:498] Detected object openshift-console/console-5d45f7999c-wmx4k of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:40:21.587096 3647 obj_retry.go:492] Detected object openshift-console/console-5d45f7999c-wmx4k of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:40:21.587119 3647 obj_retry.go:492] Detected object openshift-console/console-5d45f7999c-wmx4k of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:40:21.593098 3647 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-5d45f7999c-wmx4k I0428 15:40:21.593126 3647 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-5d45f7999c-wmx4k W0428 15:40:21.851921 3647 util.go:959] Failed to build node endpoints for node ip-10-0-1-106.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0428 15:40:48.160826 3647 cni.go:452] [openshift-monitoring/prometheus-k8s-0 a6f4536597d70449602d080960670c09aca34ff8307cae44e135230aefaab8bb network default NAD default NAD key ] DEL starting CNI request [openshift-monitoring/prometheus-k8s-0 a6f4536597d70449602d080960670c09aca34ff8307cae44e135230aefaab8bb network default NAD default NAD key ] I0428 15:40:48.229882 3647 cni.go:473] [openshift-monitoring/prometheus-k8s-0 a6f4536597d70449602d080960670c09aca34ff8307cae44e135230aefaab8bb network default NAD default NAD key default] DEL finished CNI request [openshift-monitoring/prometheus-k8s-0 a6f4536597d70449602d080960670c09aca34ff8307cae44e135230aefaab8bb network default NAD default NAD key default], result "{}", err I0428 15:40:48.294675 3647 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:40:48.294724 3647 pods.go:176] Deleting pod: openshift-monitoring/prometheus-k8s-0 I0428 15:40:48.295900 3647 pods.go:236] Attempting to release IPs for pod: openshift-monitoring/prometheus-k8s-0, ips: 10.130.0.19 I0428 15:40:48.295924 3647 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:40:48.951983 3647 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:40:48.952007 3647 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:40:48.959872 3647 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-0 I0428 15:40:48.959971 3647 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-0 I0428 15:40:49.006192 3647 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-k8s-0] creating logical port openshift-monitoring_prometheus-k8s-0 for pod on switch ip-10-0-1-106.ec2.internal I0428 15:40:49.006327 3647 kube.go:256] Updating pod openshift-monitoring/prometheus-k8s-0 I0428 15:40:49.031810 3647 pod.go:62] [openshift-monitoring/prometheus-k8s-0] pod update took 25.517673ms I0428 15:40:49.031840 3647 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-k8s-0] addLogicalPort annotation time took 25.554513ms I0428 15:40:49.033311 3647 pods.go:271] [openshift-monitoring/prometheus-k8s-0] addLogicalPort took 27.134591ms, libovsdb time 1.065511ms I0428 15:40:49.364642 3647 cni.go:452] [openshift-monitoring/prometheus-k8s-0 576df64b1ce96e17d4a535b6df5480c205db89c4af25b7054664495aaaee4b03 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-k8s-0 576df64b1ce96e17d4a535b6df5480c205db89c4af25b7054664495aaaee4b03 network default NAD default NAD key ] I0428 15:40:49.374580 3647 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-k8s-0, hostIfaceName: 576df64b1ce96e1, network: default, NAD default, SandboxID: "576df64b1ce96e17d4a535b6df5480c205db89c4af25b7054664495aaaee4b03", PCI device ID: , UID: "0d0ea599-68d6-4bb2-9602-fe59a43f7715", MAC: 0a:58:0a:82:00:17, IPs: [10.130.0.23/23] I0428 15:40:49.422312 3647 cni.go:473] [openshift-monitoring/prometheus-k8s-0 576df64b1ce96e17d4a535b6df5480c205db89c4af25b7054664495aaaee4b03 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-k8s-0 576df64b1ce96e17d4a535b6df5480c205db89c4af25b7054664495aaaee4b03 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"46:a6:5d:94:e6:67\",\"name\":\"576df64b1ce96e1\"},{\"mac\":\"0a:58:0a:82:00:17\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d1f0fc9a-064b-4228-a224-f87712bc513e\"}],\"ips\":[{\"address\":\"10.130.0.23/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0428 15:41:50.232326 3647 util.go:959] Failed to build node endpoints for node ip-10-0-1-106.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0428 15:41:50.232349 3647 util.go:959] Failed to build node endpoints for node ip-10-0-1-106.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0428 15:42:45.595613 3647 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=5 I0428 15:42:59.407284 3647 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=7 I0428 15:43:04.480859 3647 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=7 I0428 15:43:06.410442 3647 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=56 I0428 15:43:32.604356 3647 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=7 W0428 15:43:39.338005 3647 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0428 15:43:50.391532 3647 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... W0428 15:43:50.391596 3647 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-deployment-validation-operator/deployment-validation-operator-65d68cbd6c-8j7mw IP address from the namespace address-set, err: pod openshift-deployment-validation-operator/deployment-validation-operator-65d68cbd6c-8j7mw: no pod IPs found I0428 15:43:52.433377 3647 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=483 I0428 15:44:14.489857 3647 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=7 I0428 15:44:57.409075 3647 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=12 I0428 15:45:02.599382 3647 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=9 I0428 15:45:47.761696 3647 namespace.go:98] [tsf] adding namespace I0428 15:45:47.762536 3647 namespace.go:102] [tsf] adding namespace took 791.727µs I0428 15:45:47.775534 3647 namespace.go:142] [tsf] updating namespace I0428 15:45:47.791691 3647 namespace.go:142] [tsf] updating namespace I0428 15:45:51.427792 3647 namespace.go:98] [cert-manager-operator] adding namespace I0428 15:45:51.428663 3647 namespace.go:102] [cert-manager-operator] adding namespace took 848.362µs I0428 15:45:51.435919 3647 namespace.go:98] [konflux-ui] adding namespace I0428 15:45:51.436720 3647 namespace.go:102] [konflux-ui] adding namespace took 759.381µs I0428 15:45:51.437057 3647 namespace.go:98] [rhtpa-operator] adding namespace I0428 15:45:51.437089 3647 namespace.go:98] [tsf-tpa] adding namespace I0428 15:45:51.437754 3647 namespace.go:102] [rhtpa-operator] adding namespace took 683.547µs I0428 15:45:51.438320 3647 namespace.go:102] [tsf-tpa] adding namespace took 1.217747ms I0428 15:45:51.451865 3647 namespace.go:98] [rhbk-operator] adding namespace I0428 15:45:51.452002 3647 namespace.go:98] [tsf-keycloak] adding namespace I0428 15:45:51.452066 3647 namespace.go:98] [tsf-tas] adding namespace I0428 15:45:51.452506 3647 namespace.go:102] [rhbk-operator] adding namespace took 622.819µs I0428 15:45:51.452997 3647 namespace.go:102] [tsf-keycloak] adding namespace took 975.721µs I0428 15:45:51.453019 3647 namespace.go:98] [konflux-operator] adding namespace I0428 15:45:51.453520 3647 namespace.go:102] [konflux-operator] adding namespace took 493.334µs I0428 15:45:51.454032 3647 namespace.go:102] [tsf-tas] adding namespace took 1.950439ms I0428 15:45:51.470639 3647 namespace.go:142] [cert-manager-operator] updating namespace I0428 15:45:51.494419 3647 namespace.go:142] [cert-manager-operator] updating namespace I0428 15:45:51.505651 3647 namespace.go:142] [konflux-ui] updating namespace I0428 15:45:51.507891 3647 namespace.go:142] [rhtpa-operator] updating namespace I0428 15:45:51.507977 3647 namespace.go:142] [rhbk-operator] updating namespace I0428 15:45:51.511233 3647 namespace.go:142] [tsf-tpa] updating namespace I0428 15:45:51.514848 3647 namespace.go:142] [konflux-ui] updating namespace I0428 15:45:51.519859 3647 namespace.go:142] [tsf-keycloak] updating namespace I0428 15:45:51.533488 3647 namespace.go:142] [konflux-ui] updating namespace I0428 15:45:51.533535 3647 namespace.go:142] [tsf-tas] updating namespace I0428 15:45:51.533660 3647 namespace.go:142] [konflux-operator] updating namespace I0428 15:45:51.533673 3647 namespace.go:142] [cert-manager-operator] updating namespace I0428 15:45:51.541668 3647 namespace.go:142] [tsf-tpa] updating namespace I0428 15:45:51.552576 3647 namespace.go:142] [tsf-tpa] updating namespace I0428 15:45:51.560905 3647 namespace.go:142] [rhbk-operator] updating namespace I0428 15:45:51.571671 3647 namespace.go:142] [rhbk-operator] updating namespace I0428 15:45:51.584463 3647 namespace.go:142] [tsf-keycloak] updating namespace I0428 15:45:51.595221 3647 namespace.go:142] [tsf-keycloak] updating namespace I0428 15:45:51.599452 3647 namespace.go:142] [tsf-tas] updating namespace I0428 15:45:51.609307 3647 namespace.go:142] [tsf-tas] updating namespace I0428 15:45:51.616569 3647 namespace.go:142] [konflux-operator] updating namespace I0428 15:45:51.628172 3647 namespace.go:142] [konflux-operator] updating namespace I0428 15:45:51.634127 3647 namespace.go:142] [rhtpa-operator] updating namespace I0428 15:45:51.642340 3647 namespace.go:142] [rhtpa-operator] updating namespace I0428 15:45:52.792881 3647 namespace.go:142] [tsf-tpa] updating namespace I0428 15:45:52.806921 3647 namespace.go:142] [tsf-keycloak] updating namespace I0428 15:45:52.972390 3647 base_network_controller_pods.go:487] [default/tsf/test-tsf-subscriptions] creating logical port tsf_test-tsf-subscriptions for pod on switch ip-10-0-1-106.ec2.internal I0428 15:45:52.972502 3647 kube.go:256] Updating pod tsf/test-tsf-subscriptions I0428 15:45:52.998151 3647 pod.go:62] [tsf/test-tsf-subscriptions] pod update took 25.67672ms I0428 15:45:52.998173 3647 base_network_controller_pods.go:951] [default/tsf/test-tsf-subscriptions] addLogicalPort annotation time took 25.703057ms I0428 15:45:52.999388 3647 pods.go:271] [tsf/test-tsf-subscriptions] addLogicalPort took 27.011191ms, libovsdb time 900.88µs I0428 15:45:53.345350 3647 cni.go:452] [tsf/test-tsf-subscriptions abc59cd6f3ad0133a5b534c3c89e1c78955b3eba3746469591c4e0035fde106b network default NAD default NAD key ] ADD starting CNI request [tsf/test-tsf-subscriptions abc59cd6f3ad0133a5b534c3c89e1c78955b3eba3746469591c4e0035fde106b network default NAD default NAD key ] I0428 15:45:53.355044 3647 helper_linux.go:556] ConfigureOVS: namespace: tsf, podName: test-tsf-subscriptions, hostIfaceName: abc59cd6f3ad013, network: default, NAD default, SandboxID: "abc59cd6f3ad0133a5b534c3c89e1c78955b3eba3746469591c4e0035fde106b", PCI device ID: , UID: "02fe4c42-73f2-4eaf-ae18-adcbc882859a", MAC: 0a:58:0a:82:00:18, IPs: [10.130.0.24/23] I0428 15:45:53.403172 3647 cni.go:473] [tsf/test-tsf-subscriptions abc59cd6f3ad0133a5b534c3c89e1c78955b3eba3746469591c4e0035fde106b network default NAD default NAD key default] ADD finished CNI request [tsf/test-tsf-subscriptions abc59cd6f3ad0133a5b534c3c89e1c78955b3eba3746469591c4e0035fde106b network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ea:5b:aa:9c:23:ea\",\"name\":\"abc59cd6f3ad013\"},{\"mac\":\"0a:58:0a:82:00:18\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/90dc01f0-701a-4d61-a449-70fa5c7b9811\"}],\"ips\":[{\"address\":\"10.130.0.24/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0428 15:45:53.914036 3647 base_network_controller_pods.go:487] [default/openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01586jksk] creating logical port openshift-marketplace_26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01586jksk for pod on switch ip-10-0-1-106.ec2.internal I0428 15:45:53.914165 3647 kube.go:256] Updating pod openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01586jksk I0428 15:45:53.927514 3647 pod.go:62] [openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01586jksk] pod update took 13.381033ms I0428 15:45:53.927538 3647 base_network_controller_pods.go:951] [default/openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01586jksk] addLogicalPort annotation time took 13.410945ms I0428 15:45:53.928809 3647 pods.go:271] [openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01586jksk] addLogicalPort took 14.788689ms, libovsdb time 923.246µs I0428 15:45:54.003424 3647 base_network_controller_pods.go:487] [default/openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dgh4wr] creating logical port openshift-marketplace_2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dgh4wr for pod on switch ip-10-0-1-106.ec2.internal I0428 15:45:54.003541 3647 kube.go:256] Updating pod openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dgh4wr I0428 15:45:54.016085 3647 pod.go:62] [openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dgh4wr] pod update took 12.57168ms I0428 15:45:54.016107 3647 base_network_controller_pods.go:951] [default/openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dgh4wr] addLogicalPort annotation time took 12.596992ms I0428 15:45:54.017251 3647 pods.go:271] [openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dgh4wr] addLogicalPort took 13.842158ms, libovsdb time 831.847µs I0428 15:45:54.273428 3647 cni.go:452] [openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01586jksk 1bbf78b294c1d39175fa52886993c51b2123e41eb2725ea58394ac1e31fd0862 network default NAD default NAD key ] ADD starting CNI request [openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01586jksk 1bbf78b294c1d39175fa52886993c51b2123e41eb2725ea58394ac1e31fd0862 network default NAD default NAD key ] I0428 15:45:54.283869 3647 helper_linux.go:556] ConfigureOVS: namespace: openshift-marketplace, podName: 26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01586jksk, hostIfaceName: 1bbf78b294c1d39, network: default, NAD default, SandboxID: "1bbf78b294c1d39175fa52886993c51b2123e41eb2725ea58394ac1e31fd0862", PCI device ID: , UID: "43f7a7c0-06c7-46ee-8694-93df31a90442", MAC: 0a:58:0a:82:00:19, IPs: [10.130.0.25/23] I0428 15:45:54.333518 3647 cni.go:473] [openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01586jksk 1bbf78b294c1d39175fa52886993c51b2123e41eb2725ea58394ac1e31fd0862 network default NAD default NAD key default] ADD finished CNI request [openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01586jksk 1bbf78b294c1d39175fa52886993c51b2123e41eb2725ea58394ac1e31fd0862 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"b2:28:2e:ad:81:87\",\"name\":\"1bbf78b294c1d39\"},{\"mac\":\"0a:58:0a:82:00:19\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d17283af-c72b-4a19-a1a3-2498560d945f\"}],\"ips\":[{\"address\":\"10.130.0.25/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0428 15:45:54.366365 3647 cni.go:452] [openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dgh4wr f52d293b6fac9fb9f8acbbaee5f7dfd905e4df3a21eebc37d8ee56be898255e4 network default NAD default NAD key ] ADD starting CNI request [openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dgh4wr f52d293b6fac9fb9f8acbbaee5f7dfd905e4df3a21eebc37d8ee56be898255e4 network default NAD default NAD key ] I0428 15:45:54.377531 3647 helper_linux.go:556] ConfigureOVS: namespace: openshift-marketplace, podName: 2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dgh4wr, hostIfaceName: f52d293b6fac9fb, network: default, NAD default, SandboxID: "f52d293b6fac9fb9f8acbbaee5f7dfd905e4df3a21eebc37d8ee56be898255e4", PCI device ID: , UID: "3b8624e4-d57a-4c3b-b4cf-a305002c154d", MAC: 0a:58:0a:82:00:1a, IPs: [10.130.0.26/23] I0428 15:45:54.429275 3647 cni.go:473] [openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dgh4wr f52d293b6fac9fb9f8acbbaee5f7dfd905e4df3a21eebc37d8ee56be898255e4 network default NAD default NAD key default] ADD finished CNI request [openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dgh4wr f52d293b6fac9fb9f8acbbaee5f7dfd905e4df3a21eebc37d8ee56be898255e4 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"06:8d:f8:d7:96:9e\",\"name\":\"f52d293b6fac9fb\"},{\"mac\":\"0a:58:0a:82:00:1a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a483e7bb-a2a5-46bc-8fc3-0c6ac8b0f40b\"}],\"ips\":[{\"address\":\"10.130.0.26/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0428 15:45:57.016166 3647 base_network_controller_pods.go:487] [default/openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51ngx6w] creating logical port openshift-marketplace_79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51ngx6w for pod on switch ip-10-0-1-106.ec2.internal I0428 15:45:57.016305 3647 kube.go:256] Updating pod openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51ngx6w I0428 15:45:57.028906 3647 pod.go:62] [openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51ngx6w] pod update took 12.631892ms I0428 15:45:57.028934 3647 base_network_controller_pods.go:951] [default/openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51ngx6w] addLogicalPort annotation time took 12.666884ms I0428 15:45:57.030393 3647 pods.go:271] [openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51ngx6w] addLogicalPort took 14.247128ms, libovsdb time 1.088588ms I0428 15:45:57.385807 3647 cni.go:452] [openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51ngx6w 36d21276224dff2c7921c622d3ea5e3d8b5f021032e911b5ca73cf7a20abed33 network default NAD default NAD key ] ADD starting CNI request [openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51ngx6w 36d21276224dff2c7921c622d3ea5e3d8b5f021032e911b5ca73cf7a20abed33 network default NAD default NAD key ] I0428 15:45:57.397608 3647 helper_linux.go:556] ConfigureOVS: namespace: openshift-marketplace, podName: 79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51ngx6w, hostIfaceName: 36d21276224dff2, network: default, NAD default, SandboxID: "36d21276224dff2c7921c622d3ea5e3d8b5f021032e911b5ca73cf7a20abed33", PCI device ID: , UID: "77d464c4-c950-4819-a2e7-4c83092b2bd8", MAC: 0a:58:0a:82:00:1b, IPs: [10.130.0.27/23] I0428 15:45:57.451024 3647 cni.go:473] [openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51ngx6w 36d21276224dff2c7921c622d3ea5e3d8b5f021032e911b5ca73cf7a20abed33 network default NAD default NAD key default] ADD finished CNI request [openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51ngx6w 36d21276224dff2c7921c622d3ea5e3d8b5f021032e911b5ca73cf7a20abed33 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"de:69:ba:50:6d:46\",\"name\":\"36d21276224dff2\"},{\"mac\":\"0a:58:0a:82:00:1b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/826b3f44-7493-4816-9715-f9649fe6b40b\"}],\"ips\":[{\"address\":\"10.130.0.27/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0428 15:45:58.180959 3647 cni.go:452] [openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01586jksk 1bbf78b294c1d39175fa52886993c51b2123e41eb2725ea58394ac1e31fd0862 network default NAD default NAD key ] DEL starting CNI request [openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01586jksk 1bbf78b294c1d39175fa52886993c51b2123e41eb2725ea58394ac1e31fd0862 network default NAD default NAD key ] I0428 15:45:58.249843 3647 cni.go:473] [openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01586jksk 1bbf78b294c1d39175fa52886993c51b2123e41eb2725ea58394ac1e31fd0862 network default NAD default NAD key default] DEL finished CNI request [openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01586jksk 1bbf78b294c1d39175fa52886993c51b2123e41eb2725ea58394ac1e31fd0862 network default NAD default NAD key default], result "{}", err I0428 15:45:58.291495 3647 obj_retry.go:498] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01586jksk of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:45:58.291536 3647 pods.go:176] Deleting pod: openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01586jksk I0428 15:45:58.293197 3647 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01586jksk, ips: 10.130.0.25 I0428 15:45:58.293226 3647 obj_retry.go:498] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01586jksk of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:45:58.293245 3647 obj_retry.go:498] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01586jksk of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 15:45:58.293252 3647 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0428 15:45:58.293517 3647 obj_retry.go:498] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01586jksk of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 15:45:58.293533 3647 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0428 15:45:58.414289 3647 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=132 I0428 15:45:58.822413 3647 base_network_controller_pods.go:487] [default/openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2kdg2m] creating logical port openshift-marketplace_a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2kdg2m for pod on switch ip-10-0-1-106.ec2.internal I0428 15:45:58.822560 3647 kube.go:256] Updating pod openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2kdg2m I0428 15:45:58.839532 3647 pod.go:62] [openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2kdg2m] pod update took 17.009374ms I0428 15:45:58.839559 3647 base_network_controller_pods.go:951] [default/openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2kdg2m] addLogicalPort annotation time took 17.041956ms I0428 15:45:58.841027 3647 pods.go:271] [openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2kdg2m] addLogicalPort took 18.637147ms, libovsdb time 1.077993ms I0428 15:45:59.200905 3647 cni.go:452] [openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2kdg2m b78b7825965ccb38f1868a66f395ec3cccd9d5c744ffe7bd19abf09cf1cc823c network default NAD default NAD key ] ADD starting CNI request [openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2kdg2m b78b7825965ccb38f1868a66f395ec3cccd9d5c744ffe7bd19abf09cf1cc823c network default NAD default NAD key ] I0428 15:45:59.211128 3647 helper_linux.go:556] ConfigureOVS: namespace: openshift-marketplace, podName: a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2kdg2m, hostIfaceName: b78b7825965ccb3, network: default, NAD default, SandboxID: "b78b7825965ccb38f1868a66f395ec3cccd9d5c744ffe7bd19abf09cf1cc823c", PCI device ID: , UID: "ae6358ec-3815-4e7e-b516-a3a022814654", MAC: 0a:58:0a:82:00:1c, IPs: [10.130.0.28/23] I0428 15:45:59.260723 3647 cni.go:473] [openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2kdg2m b78b7825965ccb38f1868a66f395ec3cccd9d5c744ffe7bd19abf09cf1cc823c network default NAD default NAD key default] ADD finished CNI request [openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2kdg2m b78b7825965ccb38f1868a66f395ec3cccd9d5c744ffe7bd19abf09cf1cc823c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"7a:2c:9f:68:8a:a6\",\"name\":\"b78b7825965ccb3\"},{\"mac\":\"0a:58:0a:82:00:1c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/90a9a9f1-355e-432d-8827-e46f809994e3\"}],\"ips\":[{\"address\":\"10.130.0.28/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0428 15:45:59.322806 3647 obj_retry.go:492] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01586jksk of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:45:59.322834 3647 obj_retry.go:492] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01586jksk of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:45:59.322847 3647 obj_retry.go:492] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01586jksk of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:45:59.322856 3647 obj_retry.go:492] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01586jksk of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:00.197547 3647 cni.go:452] [openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dgh4wr f52d293b6fac9fb9f8acbbaee5f7dfd905e4df3a21eebc37d8ee56be898255e4 network default NAD default NAD key ] DEL starting CNI request [openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dgh4wr f52d293b6fac9fb9f8acbbaee5f7dfd905e4df3a21eebc37d8ee56be898255e4 network default NAD default NAD key ] I0428 15:46:00.266823 3647 cni.go:473] [openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dgh4wr f52d293b6fac9fb9f8acbbaee5f7dfd905e4df3a21eebc37d8ee56be898255e4 network default NAD default NAD key default] DEL finished CNI request [openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dgh4wr f52d293b6fac9fb9f8acbbaee5f7dfd905e4df3a21eebc37d8ee56be898255e4 network default NAD default NAD key default], result "{}", err I0428 15:46:00.327054 3647 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dgh4wr of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:46:00.327084 3647 pods.go:176] Deleting pod: openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dgh4wr I0428 15:46:00.328587 3647 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dgh4wr, ips: 10.130.0.26 I0428 15:46:00.328615 3647 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dgh4wr of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:46:00.328631 3647 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dgh4wr of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 15:46:00.328638 3647 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0428 15:46:00.328873 3647 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dgh4wr of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 15:46:00.328891 3647 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0428 15:46:00.455345 3647 base_network_controller_pods.go:487] [default/openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d76sw5q] creating logical port openshift-marketplace_09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d76sw5q for pod on switch ip-10-0-1-106.ec2.internal I0428 15:46:00.455436 3647 kube.go:256] Updating pod openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d76sw5q I0428 15:46:00.473151 3647 pod.go:62] [openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d76sw5q] pod update took 17.744061ms I0428 15:46:00.473179 3647 base_network_controller_pods.go:951] [default/openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d76sw5q] addLogicalPort annotation time took 17.77541ms I0428 15:46:00.474460 3647 pods.go:271] [openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d76sw5q] addLogicalPort took 19.129984ms, libovsdb time 849.54µs I0428 15:46:00.552167 3647 base_network_controller_pods.go:487] [default/openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136mvngw] creating logical port openshift-marketplace_03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136mvngw for pod on switch ip-10-0-1-106.ec2.internal I0428 15:46:00.552292 3647 kube.go:256] Updating pod openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136mvngw I0428 15:46:00.571259 3647 pod.go:62] [openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136mvngw] pod update took 18.993974ms I0428 15:46:00.571287 3647 base_network_controller_pods.go:951] [default/openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136mvngw] addLogicalPort annotation time took 19.023403ms I0428 15:46:00.572425 3647 pods.go:271] [openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136mvngw] addLogicalPort took 20.276408ms, libovsdb time 828.255µs I0428 15:46:00.608581 3647 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=9 I0428 15:46:00.815889 3647 cni.go:452] [openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d76sw5q 1d8489f0a47679881dfb6448548090a6148cfd2ee65ee65b8d08b393a509f092 network default NAD default NAD key ] ADD starting CNI request [openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d76sw5q 1d8489f0a47679881dfb6448548090a6148cfd2ee65ee65b8d08b393a509f092 network default NAD default NAD key ] I0428 15:46:00.826460 3647 helper_linux.go:556] ConfigureOVS: namespace: openshift-marketplace, podName: 09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d76sw5q, hostIfaceName: 1d8489f0a476798, network: default, NAD default, SandboxID: "1d8489f0a47679881dfb6448548090a6148cfd2ee65ee65b8d08b393a509f092", PCI device ID: , UID: "235ec15b-155f-4a45-94d1-7d7b1ffe82bd", MAC: 0a:58:0a:82:00:1d, IPs: [10.130.0.29/23] I0428 15:46:00.874767 3647 cni.go:473] [openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d76sw5q 1d8489f0a47679881dfb6448548090a6148cfd2ee65ee65b8d08b393a509f092 network default NAD default NAD key default] ADD finished CNI request [openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d76sw5q 1d8489f0a47679881dfb6448548090a6148cfd2ee65ee65b8d08b393a509f092 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"3a:7b:f9:c7:46:31\",\"name\":\"1d8489f0a476798\"},{\"mac\":\"0a:58:0a:82:00:1d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/801a40cf-814d-4729-9bd3-32ff21434063\"}],\"ips\":[{\"address\":\"10.130.0.29/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0428 15:46:00.918846 3647 cni.go:452] [openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136mvngw f0386257ef43ec5a3d05ff1e894403a9aef5e1fb9df4ba9e83c612bfb13a9659 network default NAD default NAD key ] ADD starting CNI request [openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136mvngw f0386257ef43ec5a3d05ff1e894403a9aef5e1fb9df4ba9e83c612bfb13a9659 network default NAD default NAD key ] I0428 15:46:00.929687 3647 helper_linux.go:556] ConfigureOVS: namespace: openshift-marketplace, podName: 03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136mvngw, hostIfaceName: f0386257ef43ec5, network: default, NAD default, SandboxID: "f0386257ef43ec5a3d05ff1e894403a9aef5e1fb9df4ba9e83c612bfb13a9659", PCI device ID: , UID: "fd7e97c8-44f7-4dcc-bee9-26e39743c227", MAC: 0a:58:0a:82:00:1e, IPs: [10.130.0.30/23] I0428 15:46:00.980636 3647 cni.go:473] [openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136mvngw f0386257ef43ec5a3d05ff1e894403a9aef5e1fb9df4ba9e83c612bfb13a9659 network default NAD default NAD key default] ADD finished CNI request [openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136mvngw f0386257ef43ec5a3d05ff1e894403a9aef5e1fb9df4ba9e83c612bfb13a9659 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"fe:7e:63:19:d0:98\",\"name\":\"f0386257ef43ec5\"},{\"mac\":\"0a:58:0a:82:00:1e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/14e2cc84-0426-4e98-893e-ef9a7cafb130\"}],\"ips\":[{\"address\":\"10.130.0.30/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0428 15:46:01.360553 3647 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dgh4wr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:01.360582 3647 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dgh4wr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:01.360615 3647 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dgh4wr of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:01.360630 3647 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dgh4wr of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:02.240196 3647 cni.go:452] [openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51ngx6w 36d21276224dff2c7921c622d3ea5e3d8b5f021032e911b5ca73cf7a20abed33 network default NAD default NAD key ] DEL starting CNI request [openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51ngx6w 36d21276224dff2c7921c622d3ea5e3d8b5f021032e911b5ca73cf7a20abed33 network default NAD default NAD key ] I0428 15:46:02.319770 3647 cni.go:473] [openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51ngx6w 36d21276224dff2c7921c622d3ea5e3d8b5f021032e911b5ca73cf7a20abed33 network default NAD default NAD key default] DEL finished CNI request [openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51ngx6w 36d21276224dff2c7921c622d3ea5e3d8b5f021032e911b5ca73cf7a20abed33 network default NAD default NAD key default], result "{}", err I0428 15:46:03.893319 3647 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51ngx6w of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:46:03.893353 3647 pods.go:176] Deleting pod: openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51ngx6w I0428 15:46:03.894804 3647 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51ngx6w, ips: 10.130.0.27 I0428 15:46:03.894836 3647 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51ngx6w of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:46:03.894859 3647 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51ngx6w of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 15:46:03.894870 3647 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0428 15:46:03.895048 3647 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51ngx6w of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 15:46:03.895065 3647 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0428 15:46:04.930416 3647 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51ngx6w of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:04.930439 3647 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51ngx6w of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:04.930463 3647 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51ngx6w of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:04.930479 3647 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51ngx6w of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:05.258582 3647 cni.go:452] [openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d76sw5q 1d8489f0a47679881dfb6448548090a6148cfd2ee65ee65b8d08b393a509f092 network default NAD default NAD key ] DEL starting CNI request [openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d76sw5q 1d8489f0a47679881dfb6448548090a6148cfd2ee65ee65b8d08b393a509f092 network default NAD default NAD key ] I0428 15:46:05.258910 3647 cni.go:452] [openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2kdg2m b78b7825965ccb38f1868a66f395ec3cccd9d5c744ffe7bd19abf09cf1cc823c network default NAD default NAD key ] DEL starting CNI request [openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2kdg2m b78b7825965ccb38f1868a66f395ec3cccd9d5c744ffe7bd19abf09cf1cc823c network default NAD default NAD key ] I0428 15:46:05.331492 3647 cni.go:473] [openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d76sw5q 1d8489f0a47679881dfb6448548090a6148cfd2ee65ee65b8d08b393a509f092 network default NAD default NAD key default] DEL finished CNI request [openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d76sw5q 1d8489f0a47679881dfb6448548090a6148cfd2ee65ee65b8d08b393a509f092 network default NAD default NAD key default], result "{}", err I0428 15:46:05.368181 3647 cni.go:473] [openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2kdg2m b78b7825965ccb38f1868a66f395ec3cccd9d5c744ffe7bd19abf09cf1cc823c network default NAD default NAD key default] DEL finished CNI request [openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2kdg2m b78b7825965ccb38f1868a66f395ec3cccd9d5c744ffe7bd19abf09cf1cc823c network default NAD default NAD key default], result "{}", err I0428 15:46:07.614664 3647 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=9 I0428 15:46:07.643973 3647 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d76sw5q of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:46:07.644005 3647 pods.go:176] Deleting pod: openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d76sw5q I0428 15:46:07.645615 3647 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d76sw5q, ips: 10.130.0.29 I0428 15:46:07.645640 3647 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d76sw5q of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:46:07.645659 3647 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d76sw5q of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 15:46:07.645669 3647 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0428 15:46:07.645872 3647 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d76sw5q of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 15:46:07.645892 3647 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0428 15:46:07.746465 3647 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2kdg2m of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:46:07.746530 3647 pods.go:176] Deleting pod: openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2kdg2m I0428 15:46:07.747963 3647 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2kdg2m, ips: 10.130.0.28 I0428 15:46:07.747996 3647 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2kdg2m of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:46:07.748020 3647 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2kdg2m of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 15:46:07.748029 3647 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0428 15:46:07.748227 3647 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2kdg2m of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 15:46:07.748241 3647 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0428 15:46:08.697329 3647 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d76sw5q of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:08.697358 3647 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d76sw5q of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:08.697372 3647 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d76sw5q of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:08.697391 3647 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d76sw5q of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:08.846438 3647 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2kdg2m of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:08.846467 3647 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2kdg2m of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:08.846483 3647 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2kdg2m of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:08.846494 3647 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2kdg2m of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:10.289642 3647 cni.go:452] [openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136mvngw f0386257ef43ec5a3d05ff1e894403a9aef5e1fb9df4ba9e83c612bfb13a9659 network default NAD default NAD key ] DEL starting CNI request [openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136mvngw f0386257ef43ec5a3d05ff1e894403a9aef5e1fb9df4ba9e83c612bfb13a9659 network default NAD default NAD key ] I0428 15:46:10.364284 3647 cni.go:473] [openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136mvngw f0386257ef43ec5a3d05ff1e894403a9aef5e1fb9df4ba9e83c612bfb13a9659 network default NAD default NAD key default] DEL finished CNI request [openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136mvngw f0386257ef43ec5a3d05ff1e894403a9aef5e1fb9df4ba9e83c612bfb13a9659 network default NAD default NAD key default], result "{}", err I0428 15:46:10.478280 3647 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136mvngw of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:46:10.478319 3647 pods.go:176] Deleting pod: openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136mvngw I0428 15:46:10.479696 3647 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136mvngw, ips: 10.130.0.30 I0428 15:46:10.479736 3647 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136mvngw of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:46:10.479754 3647 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136mvngw of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 15:46:10.479763 3647 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0428 15:46:10.479953 3647 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136mvngw of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 15:46:10.479968 3647 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0428 15:46:11.493946 3647 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136mvngw of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:11.493970 3647 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136mvngw of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:11.493982 3647 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136mvngw of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:11.493990 3647 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136mvngw of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:13.711370 3647 namespace.go:98] [cert-manager] adding namespace I0428 15:46:13.712297 3647 namespace.go:102] [cert-manager] adding namespace took 905.972µs I0428 15:46:13.739985 3647 namespace.go:142] [cert-manager] updating namespace I0428 15:46:13.756512 3647 namespace.go:142] [cert-manager] updating namespace W0428 15:46:19.238551 3647 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service cert-manager/cert-manager-webhook W0428 15:46:19.238573 3647 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service cert-manager/cert-manager-webhook W0428 15:46:19.274610 3647 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0428 15:46:21.884399 3647 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0428 15:46:21.891655 3647 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0428 15:46:21.929791 3647 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0428 15:46:21.937888 3647 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0428 15:46:22.408309 3647 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=60 I0428 15:46:30.107372 3647 namespace.go:142] [openshift-operators] updating namespace I0428 15:46:30.115514 3647 namespace.go:142] [openshift-operators] updating namespace W0428 15:46:36.094383 3647 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0428 15:46:37.479257 3647 namespace.go:142] [openshift-operators] updating namespace I0428 15:46:38.622600 3647 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=9 I0428 15:46:44.975983 3647 namespace.go:98] [trusted-artifact-signer] adding namespace I0428 15:46:44.976885 3647 namespace.go:102] [trusted-artifact-signer] adding namespace took 876.55µs I0428 15:46:45.004161 3647 namespace.go:142] [trusted-artifact-signer] updating namespace I0428 15:46:45.025037 3647 namespace.go:142] [trusted-artifact-signer] updating namespace W0428 15:46:45.406660 3647 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0428 15:46:45.484917 3647 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=9 I0428 15:46:50.497654 3647 cni.go:452] [tsf/test-tsf-subscriptions abc59cd6f3ad0133a5b534c3c89e1c78955b3eba3746469591c4e0035fde106b network default NAD default NAD key ] DEL starting CNI request [tsf/test-tsf-subscriptions abc59cd6f3ad0133a5b534c3c89e1c78955b3eba3746469591c4e0035fde106b network default NAD default NAD key ] I0428 15:46:50.579308 3647 cni.go:473] [tsf/test-tsf-subscriptions abc59cd6f3ad0133a5b534c3c89e1c78955b3eba3746469591c4e0035fde106b network default NAD default NAD key default] DEL finished CNI request [tsf/test-tsf-subscriptions abc59cd6f3ad0133a5b534c3c89e1c78955b3eba3746469591c4e0035fde106b network default NAD default NAD key default], result "{}", err I0428 15:46:50.661574 3647 obj_retry.go:498] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:46:50.661606 3647 pods.go:176] Deleting pod: tsf/test-tsf-subscriptions I0428 15:46:50.662820 3647 pods.go:236] Attempting to release IPs for pod: tsf/test-tsf-subscriptions, ips: 10.130.0.24 I0428 15:46:50.662845 3647 obj_retry.go:498] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:46:50.681089 3647 obj_retry.go:492] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:50.681108 3647 obj_retry.go:492] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:50.691399 3647 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-subscriptions I0428 15:46:50.691418 3647 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-subscriptions I0428 15:46:51.992001 3647 base_network_controller_pods.go:487] [default/tsf/test-tsf-infrastructure] creating logical port tsf_test-tsf-infrastructure for pod on switch ip-10-0-1-106.ec2.internal I0428 15:46:51.992102 3647 kube.go:256] Updating pod tsf/test-tsf-infrastructure I0428 15:46:52.010243 3647 pod.go:62] [tsf/test-tsf-infrastructure] pod update took 18.166475ms I0428 15:46:52.010265 3647 base_network_controller_pods.go:951] [default/tsf/test-tsf-infrastructure] addLogicalPort annotation time took 18.191763ms I0428 15:46:52.011465 3647 pods.go:271] [tsf/test-tsf-infrastructure] addLogicalPort took 19.479033ms, libovsdb time 887.405µs I0428 15:46:52.363399 3647 cni.go:452] [tsf/test-tsf-infrastructure 060d1974af4a0e9d2661e3b243ed629bec9a01bbb2973e7e59de03f27df588ca network default NAD default NAD key ] ADD starting CNI request [tsf/test-tsf-infrastructure 060d1974af4a0e9d2661e3b243ed629bec9a01bbb2973e7e59de03f27df588ca network default NAD default NAD key ] I0428 15:46:52.372786 3647 helper_linux.go:556] ConfigureOVS: namespace: tsf, podName: test-tsf-infrastructure, hostIfaceName: 060d1974af4a0e9, network: default, NAD default, SandboxID: "060d1974af4a0e9d2661e3b243ed629bec9a01bbb2973e7e59de03f27df588ca", PCI device ID: , UID: "c974ac8e-2522-4ed1-99ee-3cde2be1d540", MAC: 0a:58:0a:82:00:1f, IPs: [10.130.0.31/23] I0428 15:46:52.421243 3647 cni.go:473] [tsf/test-tsf-infrastructure 060d1974af4a0e9d2661e3b243ed629bec9a01bbb2973e7e59de03f27df588ca network default NAD default NAD key default] ADD finished CNI request [tsf/test-tsf-infrastructure 060d1974af4a0e9d2661e3b243ed629bec9a01bbb2973e7e59de03f27df588ca network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"46:5e:82:10:fe:94\",\"name\":\"060d1974af4a0e9\"},{\"mac\":\"0a:58:0a:82:00:1f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b1faf7cf-5666-4fcc-92ec-864d1b030600\"}],\"ips\":[{\"address\":\"10.130.0.31/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0428 15:46:53.213361 3647 namespace.go:98] [openshift-pipelines] adding namespace I0428 15:46:53.214194 3647 namespace.go:102] [openshift-pipelines] adding namespace took 806.704µs I0428 15:46:53.232750 3647 namespace.go:142] [openshift-pipelines] updating namespace I0428 15:46:53.940695 3647 namespace.go:142] [cert-manager] updating namespace I0428 15:46:54.142869 3647 namespace.go:142] [cert-manager-operator] updating namespace I0428 15:46:54.547828 3647 namespace.go:142] [konflux-operator] updating namespace I0428 15:46:54.746770 3647 namespace.go:142] [konflux-ui] updating namespace I0428 15:46:54.895642 3647 base_network_controller_pods.go:487] [default/tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-bb4zt] creating logical port tsf-tpa_tpa-pgsql-bee-5df6ff8bdf-bb4zt for pod on switch ip-10-0-1-106.ec2.internal I0428 15:46:54.895777 3647 kube.go:256] Updating pod tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-bb4zt I0428 15:46:54.906777 3647 pod.go:62] [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-bb4zt] pod update took 11.030594ms I0428 15:46:54.906803 3647 base_network_controller_pods.go:951] [default/tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-bb4zt] addLogicalPort annotation time took 11.066014ms I0428 15:46:54.908509 3647 pods.go:271] [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-bb4zt] addLogicalPort took 12.886658ms, libovsdb time 1.034674ms I0428 15:46:54.947943 3647 namespace.go:142] [open-cluster-management-2pv84nia58gje8norbmmjqtj26oprnd8] updating namespace I0428 15:46:54.995322 3647 base_network_controller_pods.go:487] [default/tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-n4fb2] creating logical port tsf-keycloak_keycloak-pgsql-bee-744f6fc4d9-n4fb2 for pod on switch ip-10-0-1-106.ec2.internal I0428 15:46:54.995436 3647 kube.go:256] Updating pod tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-n4fb2 I0428 15:46:55.007432 3647 pod.go:62] [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-n4fb2] pod update took 12.018057ms I0428 15:46:55.007461 3647 base_network_controller_pods.go:951] [default/tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-n4fb2] addLogicalPort annotation time took 12.051304ms I0428 15:46:55.008806 3647 pods.go:271] [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-n4fb2] addLogicalPort took 13.506589ms, libovsdb time 863.54µs I0428 15:46:55.350517 3647 namespace.go:142] [rhbk-operator] updating namespace I0428 15:46:55.552515 3647 namespace.go:142] [rhtpa-operator] updating namespace I0428 15:46:55.751966 3647 namespace.go:142] [trusted-artifact-signer] updating namespace I0428 15:46:55.952089 3647 namespace.go:142] [tsf] updating namespace I0428 15:46:56.155106 3647 namespace.go:142] [tsf-keycloak] updating namespace I0428 15:46:56.355198 3647 namespace.go:142] [tsf-tas] updating namespace I0428 15:46:56.559085 3647 namespace.go:142] [tsf-tpa] updating namespace I0428 15:46:56.690093 3647 namespace.go:142] [cert-manager] updating namespace I0428 15:46:56.702502 3647 namespace.go:142] [cert-manager-operator] updating namespace I0428 15:46:56.716083 3647 namespace.go:142] [default] updating namespace I0428 15:46:56.728947 3647 namespace.go:142] [konflux-operator] updating namespace I0428 15:46:56.739124 3647 namespace.go:142] [konflux-ui] updating namespace I0428 15:46:56.753980 3647 namespace.go:142] [open-cluster-management-2pv84nia58gje8norbmmjqtj26oprnd8] updating namespace I0428 15:46:56.769180 3647 namespace.go:142] [openshift] updating namespace I0428 15:46:56.786147 3647 namespace.go:142] [rhbk-operator] updating namespace I0428 15:46:56.807186 3647 namespace.go:142] [rhtpa-operator] updating namespace I0428 15:46:56.829995 3647 namespace.go:142] [trusted-artifact-signer] updating namespace I0428 15:46:56.847656 3647 namespace.go:142] [tsf] updating namespace I0428 15:46:56.865144 3647 namespace.go:142] [tsf-keycloak] updating namespace I0428 15:46:56.878822 3647 namespace.go:142] [tsf-tas] updating namespace I0428 15:46:56.915899 3647 namespace.go:142] [tsf-tpa] updating namespace I0428 15:46:57.014100 3647 namespace.go:142] [cert-manager] updating namespace I0428 15:46:57.104053 3647 namespace.go:142] [cert-manager-operator] updating namespace I0428 15:46:57.181119 3647 namespace.go:142] [default] updating namespace I0428 15:46:57.265260 3647 namespace.go:142] [konflux-operator] updating namespace I0428 15:46:57.359225 3647 namespace.go:142] [konflux-ui] updating namespace I0428 15:46:57.449227 3647 namespace.go:142] [open-cluster-management-2pv84nia58gje8norbmmjqtj26oprnd8] updating namespace I0428 15:46:57.554205 3647 namespace.go:142] [openshift] updating namespace I0428 15:46:57.654463 3647 namespace.go:142] [rhbk-operator] updating namespace I0428 15:46:57.748182 3647 namespace.go:142] [rhtpa-operator] updating namespace I0428 15:46:57.846144 3647 namespace.go:142] [trusted-artifact-signer] updating namespace I0428 15:46:57.946127 3647 namespace.go:142] [tsf] updating namespace I0428 15:46:58.054306 3647 namespace.go:142] [tsf-keycloak] updating namespace I0428 15:46:58.150319 3647 namespace.go:142] [tsf-tas] updating namespace I0428 15:46:58.249301 3647 namespace.go:142] [tsf-tpa] updating namespace I0428 15:46:58.662388 3647 namespace.go:142] [openshift-pipelines] updating namespace I0428 15:47:05.475683 3647 cni.go:452] [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-bb4zt af345c31d5aad853cbdd4d3b22503c54aa82b14d319db563303119a8de7e5781 network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-bb4zt af345c31d5aad853cbdd4d3b22503c54aa82b14d319db563303119a8de7e5781 network default NAD default NAD key ] I0428 15:47:05.485841 3647 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: tpa-pgsql-bee-5df6ff8bdf-bb4zt, hostIfaceName: af345c31d5aad85, network: default, NAD default, SandboxID: "af345c31d5aad853cbdd4d3b22503c54aa82b14d319db563303119a8de7e5781", PCI device ID: , UID: "ee6c10d4-50aa-45f3-988a-efd8a6788468", MAC: 0a:58:0a:82:00:20, IPs: [10.130.0.32/23] I0428 15:47:05.537009 3647 cni.go:473] [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-bb4zt af345c31d5aad853cbdd4d3b22503c54aa82b14d319db563303119a8de7e5781 network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-bb4zt af345c31d5aad853cbdd4d3b22503c54aa82b14d319db563303119a8de7e5781 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"5a:11:54:b0:3a:68\",\"name\":\"af345c31d5aad85\"},{\"mac\":\"0a:58:0a:82:00:20\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/69ceaf8e-5541-46a8-a533-e35e4991462b\"}],\"ips\":[{\"address\":\"10.130.0.32/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0428 15:47:05.559899 3647 cni.go:452] [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-n4fb2 08ad03a3ff630139fbc3244ac79549237d620177aed26bf2379a64c43e0347b4 network default NAD default NAD key ] ADD starting CNI request [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-n4fb2 08ad03a3ff630139fbc3244ac79549237d620177aed26bf2379a64c43e0347b4 network default NAD default NAD key ] I0428 15:47:05.569379 3647 helper_linux.go:556] ConfigureOVS: namespace: tsf-keycloak, podName: keycloak-pgsql-bee-744f6fc4d9-n4fb2, hostIfaceName: 08ad03a3ff63013, network: default, NAD default, SandboxID: "08ad03a3ff630139fbc3244ac79549237d620177aed26bf2379a64c43e0347b4", PCI device ID: , UID: "820d7a1e-98e8-4ac2-917a-279b2f895c2f", MAC: 0a:58:0a:82:00:21, IPs: [10.130.0.33/23] I0428 15:47:05.620864 3647 cni.go:473] [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-n4fb2 08ad03a3ff630139fbc3244ac79549237d620177aed26bf2379a64c43e0347b4 network default NAD default NAD key default] ADD finished CNI request [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-n4fb2 08ad03a3ff630139fbc3244ac79549237d620177aed26bf2379a64c43e0347b4 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"1e:4d:4e:6c:c1:8a\",\"name\":\"08ad03a3ff63013\"},{\"mac\":\"0a:58:0a:82:00:21\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e5a06ccd-b687-431b-b58e-8919ecd1f7bf\"}],\"ips\":[{\"address\":\"10.130.0.33/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0428 15:47:06.603345 3647 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-tpa/tpa-pgsql W0428 15:47:06.630935 3647 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-keycloak/keycloak-pgsql W0428 15:47:10.210679 3647 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0428 15:47:10.210714 3647 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0428 15:47:10.210722 3647 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0428 15:47:10.488010 3647 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0428 15:47:10.488027 3647 util.go:870] Failed to build global endpoints for port TCP/https-webhook: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0428 15:47:10.488032 3647 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0428 15:47:10.488036 3647 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook I0428 15:47:10.590089 3647 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=10 W0428 15:47:13.506057 3647 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0428 15:47:13.506078 3647 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0428 15:47:13.506083 3647 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0428 15:47:27.593719 3647 util.go:959] Failed to build node endpoints for node ip-10-0-1-106.ec2.internal port 5432: empty IP address endpoints for service tsf-keycloak/keycloak-pgsql W0428 15:47:27.618991 3647 util.go:959] Failed to build node endpoints for node ip-10-0-1-106.ec2.internal port 5432: empty IP address endpoints for service tsf-tpa/tpa-pgsql I0428 15:47:31.325662 3647 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-triggers-webhook-6b5b45569f-lkltw] creating logical port openshift-pipelines_tekton-triggers-webhook-6b5b45569f-lkltw for pod on switch ip-10-0-1-106.ec2.internal I0428 15:47:31.325766 3647 kube.go:256] Updating pod openshift-pipelines/tekton-triggers-webhook-6b5b45569f-lkltw I0428 15:47:31.338376 3647 pod.go:62] [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-lkltw] pod update took 12.628352ms I0428 15:47:31.338401 3647 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-triggers-webhook-6b5b45569f-lkltw] addLogicalPort annotation time took 12.65695ms I0428 15:47:31.339685 3647 pods.go:271] [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-lkltw] addLogicalPort took 14.036939ms, libovsdb time 917.147µs I0428 15:47:31.686359 3647 cni.go:452] [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-lkltw deb1b20c442257329a1ca1cc632f1c1040a922ecca19e9bfc093d4d437984fde network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-lkltw deb1b20c442257329a1ca1cc632f1c1040a922ecca19e9bfc093d4d437984fde network default NAD default NAD key ] I0428 15:47:31.708665 3647 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-triggers-webhook-6b5b45569f-lkltw, hostIfaceName: deb1b20c4422573, network: default, NAD default, SandboxID: "deb1b20c442257329a1ca1cc632f1c1040a922ecca19e9bfc093d4d437984fde", PCI device ID: , UID: "59157e61-d06d-479c-b310-24749c73b02b", MAC: 0a:58:0a:82:00:22, IPs: [10.130.0.34/23] I0428 15:47:31.758587 3647 cni.go:473] [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-lkltw deb1b20c442257329a1ca1cc632f1c1040a922ecca19e9bfc093d4d437984fde network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-lkltw deb1b20c442257329a1ca1cc632f1c1040a922ecca19e9bfc093d4d437984fde network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"66:67:86:10:6b:d0\",\"name\":\"deb1b20c4422573\"},{\"mac\":\"0a:58:0a:82:00:22\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3852bf45-6c58-4b48-94c1-e2449682b26b\"}],\"ips\":[{\"address\":\"10.130.0.34/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0428 15:47:36.327000 3647 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors I0428 15:47:36.751652 3647 cni.go:452] [tsf/test-tsf-infrastructure 060d1974af4a0e9d2661e3b243ed629bec9a01bbb2973e7e59de03f27df588ca network default NAD default NAD key ] DEL starting CNI request [tsf/test-tsf-infrastructure 060d1974af4a0e9d2661e3b243ed629bec9a01bbb2973e7e59de03f27df588ca network default NAD default NAD key ] I0428 15:47:36.842040 3647 cni.go:473] [tsf/test-tsf-infrastructure 060d1974af4a0e9d2661e3b243ed629bec9a01bbb2973e7e59de03f27df588ca network default NAD default NAD key default] DEL finished CNI request [tsf/test-tsf-infrastructure 060d1974af4a0e9d2661e3b243ed629bec9a01bbb2973e7e59de03f27df588ca network default NAD default NAD key default], result "{}", err I0428 15:47:36.891482 3647 obj_retry.go:498] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:47:36.891514 3647 pods.go:176] Deleting pod: tsf/test-tsf-infrastructure I0428 15:47:36.892786 3647 pods.go:236] Attempting to release IPs for pod: tsf/test-tsf-infrastructure, ips: 10.130.0.31 I0428 15:47:36.892819 3647 obj_retry.go:498] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:47:36.912491 3647 obj_retry.go:492] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:47:36.912510 3647 obj_retry.go:492] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:47:36.915497 3647 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-infrastructure I0428 15:47:36.915518 3647 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-infrastructure I0428 15:47:37.931871 3647 base_network_controller_pods.go:487] [default/tsf/test-tsf-iam] creating logical port tsf_test-tsf-iam for pod on switch ip-10-0-1-106.ec2.internal I0428 15:47:37.931974 3647 kube.go:256] Updating pod tsf/test-tsf-iam I0428 15:47:37.949217 3647 pod.go:62] [tsf/test-tsf-iam] pod update took 17.268307ms I0428 15:47:37.949244 3647 base_network_controller_pods.go:951] [default/tsf/test-tsf-iam] addLogicalPort annotation time took 17.301754ms I0428 15:47:37.950587 3647 pods.go:271] [tsf/test-tsf-iam] addLogicalPort took 18.736815ms, libovsdb time 965.498µs I0428 15:47:38.291080 3647 cni.go:452] [tsf/test-tsf-iam 0dcf2449fad6bfe0a7415da8965e88d8ca712443c594cd017c28de97fcca1935 network default NAD default NAD key ] ADD starting CNI request [tsf/test-tsf-iam 0dcf2449fad6bfe0a7415da8965e88d8ca712443c594cd017c28de97fcca1935 network default NAD default NAD key ] I0428 15:47:38.300447 3647 helper_linux.go:556] ConfigureOVS: namespace: tsf, podName: test-tsf-iam, hostIfaceName: 0dcf2449fad6bfe, network: default, NAD default, SandboxID: "0dcf2449fad6bfe0a7415da8965e88d8ca712443c594cd017c28de97fcca1935", PCI device ID: , UID: "29246694-a624-42a3-b526-34f841fa8e51", MAC: 0a:58:0a:82:00:23, IPs: [10.130.0.35/23] I0428 15:47:38.351661 3647 cni.go:473] [tsf/test-tsf-iam 0dcf2449fad6bfe0a7415da8965e88d8ca712443c594cd017c28de97fcca1935 network default NAD default NAD key default] ADD finished CNI request [tsf/test-tsf-iam 0dcf2449fad6bfe0a7415da8965e88d8ca712443c594cd017c28de97fcca1935 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ce:a1:3e:c2:f8:aa\",\"name\":\"0dcf2449fad6bfe\"},{\"mac\":\"0a:58:0a:82:00:23\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/19812e4b-8320-433f-85a8-60ed0741562c\"}],\"ips\":[{\"address\":\"10.130.0.35/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0428 15:47:38.654641 3647 base_network_controller_policy.go:1179] Adding network policy tsf-keycloak/keycloak-network-policy for network default I0428 15:47:38.655312 3647 pod_selector_address_set.go:213] Created shared address set for pod selector tsf-keycloak_LS{ML:{app: keycloak,app.kubernetes.io/instance: keycloak,app.kubernetes.io/managed-by: keycloak-operator,},} I0428 15:47:38.655333 3647 base_network_controller_policy.go:1054] Policy tsf-keycloak/keycloak-network-policy added to peer address sets [tsf-keycloak_LS{ML:{app: keycloak,app.kubernetes.io/instance: keycloak,app.kubernetes.io/managed-by: keycloak-operator,},}] I0428 15:47:38.657519 3647 base_network_controller_policy.go:1232] Create network policy tsf-keycloak/keycloak-network-policy resources completed, update namespace loglevel I0428 15:47:39.032219 3647 base_network_controller_pods.go:487] [default/tsf-keycloak/keycloak-0] creating logical port tsf-keycloak_keycloak-0 for pod on switch ip-10-0-1-106.ec2.internal I0428 15:47:39.032351 3647 kube.go:256] Updating pod tsf-keycloak/keycloak-0 I0428 15:47:39.048240 3647 pod.go:62] [tsf-keycloak/keycloak-0] pod update took 15.926007ms I0428 15:47:39.048264 3647 base_network_controller_pods.go:951] [default/tsf-keycloak/keycloak-0] addLogicalPort annotation time took 15.953759ms I0428 15:47:39.049434 3647 pods.go:271] [tsf-keycloak/keycloak-0] addLogicalPort took 17.231277ms, libovsdb time 858.003µs I0428 15:47:39.429551 3647 cni.go:452] [tsf-keycloak/keycloak-0 3690ef6c65770b1a5ec47239aa804ef97675fc42ba709be1097d7eac4f823041 network default NAD default NAD key ] ADD starting CNI request [tsf-keycloak/keycloak-0 3690ef6c65770b1a5ec47239aa804ef97675fc42ba709be1097d7eac4f823041 network default NAD default NAD key ] I0428 15:47:39.439526 3647 helper_linux.go:556] ConfigureOVS: namespace: tsf-keycloak, podName: keycloak-0, hostIfaceName: 3690ef6c65770b1, network: default, NAD default, SandboxID: "3690ef6c65770b1a5ec47239aa804ef97675fc42ba709be1097d7eac4f823041", PCI device ID: , UID: "24c12d60-0e2c-4687-8fec-65c5e4354d5b", MAC: 0a:58:0a:82:00:24, IPs: [10.130.0.36/23] I0428 15:47:39.488115 3647 cni.go:473] [tsf-keycloak/keycloak-0 3690ef6c65770b1a5ec47239aa804ef97675fc42ba709be1097d7eac4f823041 network default NAD default NAD key default] ADD finished CNI request [tsf-keycloak/keycloak-0 3690ef6c65770b1a5ec47239aa804ef97675fc42ba709be1097d7eac4f823041 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"5e:1a:58:3e:83:64\",\"name\":\"3690ef6c65770b1\"},{\"mac\":\"0a:58:0a:82:00:24\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/85e8b6d7-4146-45ad-8b10-4eeda0cff495\"}],\"ips\":[{\"address\":\"10.130.0.36/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0428 15:47:46.815748 3647 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0428 15:47:46.815790 3647 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0428 15:47:46.816746 3647 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0428 15:47:46.816763 3647 util.go:870] Failed to build global endpoints for port TCP/management: empty IP address endpoints for service tsf-keycloak/keycloak-service I0428 15:47:54.268919 3647 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0428 15:48:07.957479 3647 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-retention-policy-agent-585694b54-vs8s2] creating logical port openshift-pipelines_tekton-results-retention-policy-agent-585694b54-vs8s2 for pod on switch ip-10-0-1-106.ec2.internal I0428 15:48:07.957575 3647 kube.go:256] Updating pod openshift-pipelines/tekton-results-retention-policy-agent-585694b54-vs8s2 I0428 15:48:07.970998 3647 pod.go:62] [openshift-pipelines/tekton-results-retention-policy-agent-585694b54-vs8s2] pod update took 13.445167ms I0428 15:48:07.971021 3647 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-results-retention-policy-agent-585694b54-vs8s2] addLogicalPort annotation time took 13.47191ms I0428 15:48:07.972484 3647 pods.go:271] [openshift-pipelines/tekton-results-retention-policy-agent-585694b54-vs8s2] addLogicalPort took 15.019281ms, libovsdb time 1.066654ms I0428 15:48:08.454494 3647 cni.go:452] [openshift-pipelines/tekton-results-retention-policy-agent-585694b54-vs8s2 2039f186d53836c7bd87f62c78d3313b0ac4b2d150601403c3a6a37796ee31e4 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-results-retention-policy-agent-585694b54-vs8s2 2039f186d53836c7bd87f62c78d3313b0ac4b2d150601403c3a6a37796ee31e4 network default NAD default NAD key ] I0428 15:48:08.466093 3647 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-retention-policy-agent-585694b54-vs8s2, hostIfaceName: 2039f186d53836c, network: default, NAD default, SandboxID: "2039f186d53836c7bd87f62c78d3313b0ac4b2d150601403c3a6a37796ee31e4", PCI device ID: , UID: "0594a6a5-41c4-4ba0-9913-399d81e900ed", MAC: 0a:58:0a:82:00:25, IPs: [10.130.0.37/23] I0428 15:48:08.522881 3647 cni.go:473] [openshift-pipelines/tekton-results-retention-policy-agent-585694b54-vs8s2 2039f186d53836c7bd87f62c78d3313b0ac4b2d150601403c3a6a37796ee31e4 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-results-retention-policy-agent-585694b54-vs8s2 2039f186d53836c7bd87f62c78d3313b0ac4b2d150601403c3a6a37796ee31e4 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"06:73:89:68:f8:6d\",\"name\":\"2039f186d53836c\"},{\"mac\":\"0a:58:0a:82:00:25\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e1a79a17-224d-49a7-abf6-c764646153f3\"}],\"ips\":[{\"address\":\"10.130.0.37/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0428 15:48:09.648039 3647 util.go:959] Failed to build node endpoints for node ip-10-0-1-106.ec2.internal port 8443: empty IP address endpoints for service tsf-keycloak/keycloak-service W0428 15:48:09.648059 3647 util.go:959] Failed to build node endpoints for node ip-10-0-1-106.ec2.internal port 9000: empty IP address endpoints for service tsf-keycloak/keycloak-service W0428 15:48:09.648144 3647 util.go:959] Failed to build node endpoints for node ip-10-0-1-106.ec2.internal port 8443: empty IP address endpoints for service tsf-keycloak/keycloak W0428 15:48:09.648152 3647 util.go:959] Failed to build node endpoints for node ip-10-0-1-106.ec2.internal port 8080: empty IP address endpoints for service tsf-keycloak/keycloak W0428 15:48:13.797482 3647 util.go:870] Failed to build global endpoints for port TCP/server: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0428 15:48:13.797500 3647 util.go:870] Failed to build global endpoints for port TCP/prometheus: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0428 15:48:13.797506 3647 util.go:870] Failed to build global endpoints for port TCP/profiling: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service I0428 15:48:15.409739 3647 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=8 I0428 15:48:18.911452 3647 base_network_controller_pods.go:487] [default/tsf-keycloak/tsf-iam-m7t22] creating logical port tsf-keycloak_tsf-iam-m7t22 for pod on switch ip-10-0-1-106.ec2.internal I0428 15:48:18.911543 3647 kube.go:256] Updating pod tsf-keycloak/tsf-iam-m7t22 I0428 15:48:18.928262 3647 pod.go:62] [tsf-keycloak/tsf-iam-m7t22] pod update took 16.744489ms I0428 15:48:18.928283 3647 base_network_controller_pods.go:951] [default/tsf-keycloak/tsf-iam-m7t22] addLogicalPort annotation time took 16.769202ms I0428 15:48:18.929486 3647 pods.go:271] [tsf-keycloak/tsf-iam-m7t22] addLogicalPort took 18.050956ms, libovsdb time 837.993µs I0428 15:48:19.274529 3647 cni.go:452] [tsf-keycloak/tsf-iam-m7t22 3824a279ac85cd61727ebaef2952ed3694bc4ad25f79241abce60d234228d4f5 network default NAD default NAD key ] ADD starting CNI request [tsf-keycloak/tsf-iam-m7t22 3824a279ac85cd61727ebaef2952ed3694bc4ad25f79241abce60d234228d4f5 network default NAD default NAD key ] I0428 15:48:19.283663 3647 helper_linux.go:556] ConfigureOVS: namespace: tsf-keycloak, podName: tsf-iam-m7t22, hostIfaceName: 3824a279ac85cd6, network: default, NAD default, SandboxID: "3824a279ac85cd61727ebaef2952ed3694bc4ad25f79241abce60d234228d4f5", PCI device ID: , UID: "cc2a4cc4-9286-405e-bc12-aedcddb36e39", MAC: 0a:58:0a:82:00:26, IPs: [10.130.0.38/23] I0428 15:48:19.335101 3647 cni.go:473] [tsf-keycloak/tsf-iam-m7t22 3824a279ac85cd61727ebaef2952ed3694bc4ad25f79241abce60d234228d4f5 network default NAD default NAD key default] ADD finished CNI request [tsf-keycloak/tsf-iam-m7t22 3824a279ac85cd61727ebaef2952ed3694bc4ad25f79241abce60d234228d4f5 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ae:41:c0:94:18:3f\",\"name\":\"3824a279ac85cd6\"},{\"mac\":\"0a:58:0a:82:00:26\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7b3dea26-ab1e-4ade-9425-3dc46b50d477\"}],\"ips\":[{\"address\":\"10.130.0.38/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0428 15:48:20.460309 3647 namespace.go:142] [openshift-pipelines] updating namespace I0428 15:48:29.090523 3647 cni.go:452] [tsf/test-tsf-iam 0dcf2449fad6bfe0a7415da8965e88d8ca712443c594cd017c28de97fcca1935 network default NAD default NAD key ] DEL starting CNI request [tsf/test-tsf-iam 0dcf2449fad6bfe0a7415da8965e88d8ca712443c594cd017c28de97fcca1935 network default NAD default NAD key ] I0428 15:48:29.175233 3647 cni.go:473] [tsf/test-tsf-iam 0dcf2449fad6bfe0a7415da8965e88d8ca712443c594cd017c28de97fcca1935 network default NAD default NAD key default] DEL finished CNI request [tsf/test-tsf-iam 0dcf2449fad6bfe0a7415da8965e88d8ca712443c594cd017c28de97fcca1935 network default NAD default NAD key default], result "{}", err I0428 15:48:29.233793 3647 obj_retry.go:498] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:48:29.233906 3647 pods.go:176] Deleting pod: tsf/test-tsf-iam I0428 15:48:29.235226 3647 pods.go:236] Attempting to release IPs for pod: tsf/test-tsf-iam, ips: 10.130.0.35 I0428 15:48:29.235263 3647 obj_retry.go:498] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:48:29.269431 3647 obj_retry.go:492] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:48:29.269461 3647 obj_retry.go:492] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:48:29.275111 3647 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0428 15:48:29.275189 3647 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam I0428 15:48:29.999619 3647 base_network_controller_pods.go:487] [default/tsf-tas/test-tsf-tas] creating logical port tsf-tas_test-tsf-tas for pod on switch ip-10-0-1-106.ec2.internal I0428 15:48:29.999747 3647 kube.go:256] Updating pod tsf-tas/test-tsf-tas I0428 15:48:30.012408 3647 pod.go:62] [tsf-tas/test-tsf-tas] pod update took 12.709262ms I0428 15:48:30.012431 3647 base_network_controller_pods.go:951] [default/tsf-tas/test-tsf-tas] addLogicalPort annotation time took 12.737726ms I0428 15:48:30.013739 3647 pods.go:271] [tsf-tas/test-tsf-tas] addLogicalPort took 14.141302ms, libovsdb time 851.644µs I0428 15:48:30.373781 3647 cni.go:452] [tsf-tas/test-tsf-tas ca84de3aa3c602e63fe619bfeba74292414977dcf4306de10c3a672d64cc209d network default NAD default NAD key ] ADD starting CNI request [tsf-tas/test-tsf-tas ca84de3aa3c602e63fe619bfeba74292414977dcf4306de10c3a672d64cc209d network default NAD default NAD key ] I0428 15:48:30.384630 3647 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: test-tsf-tas, hostIfaceName: ca84de3aa3c602e, network: default, NAD default, SandboxID: "ca84de3aa3c602e63fe619bfeba74292414977dcf4306de10c3a672d64cc209d", PCI device ID: , UID: "27afad00-72dc-4384-be10-03110ff0d560", MAC: 0a:58:0a:82:00:27, IPs: [10.130.0.39/23] I0428 15:48:30.440836 3647 cni.go:473] [tsf-tas/test-tsf-tas ca84de3aa3c602e63fe619bfeba74292414977dcf4306de10c3a672d64cc209d network default NAD default NAD key default] ADD finished CNI request [tsf-tas/test-tsf-tas ca84de3aa3c602e63fe619bfeba74292414977dcf4306de10c3a672d64cc209d network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"12:1b:9b:19:bd:f3\",\"name\":\"ca84de3aa3c602e\"},{\"mac\":\"0a:58:0a:82:00:27\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9807c496-b797-4c95-a98f-12460174899f\"}],\"ips\":[{\"address\":\"10.130.0.39/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0428 15:48:31.977388 3647 base_network_controller_pods.go:487] [default/tsf-tas/trillian-logsigner-6c5897fc57-db8kh] creating logical port tsf-tas_trillian-logsigner-6c5897fc57-db8kh for pod on switch ip-10-0-1-106.ec2.internal I0428 15:48:31.977484 3647 kube.go:256] Updating pod tsf-tas/trillian-logsigner-6c5897fc57-db8kh I0428 15:48:32.037744 3647 kube.go:256] Updating pod tsf-tas/trillian-logsigner-6c5897fc57-db8kh I0428 15:48:32.068338 3647 pod.go:62] [tsf-tas/trillian-logsigner-6c5897fc57-db8kh] pod update took 90.881768ms I0428 15:48:32.068369 3647 base_network_controller_pods.go:951] [default/tsf-tas/trillian-logsigner-6c5897fc57-db8kh] addLogicalPort annotation time took 90.916115ms I0428 15:48:32.069793 3647 pods.go:271] [tsf-tas/trillian-logsigner-6c5897fc57-db8kh] addLogicalPort took 92.420143ms, libovsdb time 944.32µs I0428 15:48:32.945604 3647 cni.go:452] [tsf-tas/trillian-logsigner-6c5897fc57-db8kh 49674b6d457569a4e2c95d6f49d21b680e811d6c49370da51669a04a57383e84 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/trillian-logsigner-6c5897fc57-db8kh 49674b6d457569a4e2c95d6f49d21b680e811d6c49370da51669a04a57383e84 network default NAD default NAD key ] I0428 15:48:32.957126 3647 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: trillian-logsigner-6c5897fc57-db8kh, hostIfaceName: 49674b6d457569a, network: default, NAD default, SandboxID: "49674b6d457569a4e2c95d6f49d21b680e811d6c49370da51669a04a57383e84", PCI device ID: , UID: "8665918d-d5ad-45c3-9d8f-72385f0196b3", MAC: 0a:58:0a:82:00:28, IPs: [10.130.0.40/23] I0428 15:48:33.010113 3647 cni.go:473] [tsf-tas/trillian-logsigner-6c5897fc57-db8kh 49674b6d457569a4e2c95d6f49d21b680e811d6c49370da51669a04a57383e84 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/trillian-logsigner-6c5897fc57-db8kh 49674b6d457569a4e2c95d6f49d21b680e811d6c49370da51669a04a57383e84 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ee:36:d4:8b:66:3d\",\"name\":\"49674b6d457569a\"},{\"mac\":\"0a:58:0a:82:00:28\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2e740d12-b359-48ec-87b4-a391b795687a\"}],\"ips\":[{\"address\":\"10.130.0.40/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0428 15:48:34.625134 3647 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/fulcio-server W0428 15:48:34.625157 3647 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/fulcio-server I0428 15:48:38.154792 3647 cni.go:452] [tsf-keycloak/tsf-iam-m7t22 3824a279ac85cd61727ebaef2952ed3694bc4ad25f79241abce60d234228d4f5 network default NAD default NAD key ] DEL starting CNI request [tsf-keycloak/tsf-iam-m7t22 3824a279ac85cd61727ebaef2952ed3694bc4ad25f79241abce60d234228d4f5 network default NAD default NAD key ] I0428 15:48:38.235802 3647 cni.go:473] [tsf-keycloak/tsf-iam-m7t22 3824a279ac85cd61727ebaef2952ed3694bc4ad25f79241abce60d234228d4f5 network default NAD default NAD key default] DEL finished CNI request [tsf-keycloak/tsf-iam-m7t22 3824a279ac85cd61727ebaef2952ed3694bc4ad25f79241abce60d234228d4f5 network default NAD default NAD key default], result "{}", err I0428 15:48:38.295084 3647 obj_retry.go:498] Detected object tsf-keycloak/tsf-iam-m7t22 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:48:38.295113 3647 pods.go:176] Deleting pod: tsf-keycloak/tsf-iam-m7t22 I0428 15:48:38.296739 3647 pods.go:236] Attempting to release IPs for pod: tsf-keycloak/tsf-iam-m7t22, ips: 10.130.0.38 I0428 15:48:38.296766 3647 obj_retry.go:498] Detected object tsf-keycloak/tsf-iam-m7t22 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:48:38.392440 3647 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-pjxxl] creating logical port openshift-pipelines_pipelines-as-code-controller-79b6fbbc5c-pjxxl for pod on switch ip-10-0-1-106.ec2.internal I0428 15:48:38.392534 3647 kube.go:256] Updating pod openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-pjxxl I0428 15:48:38.406545 3647 pod.go:62] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-pjxxl] pod update took 14.032621ms I0428 15:48:38.406569 3647 base_network_controller_pods.go:951] [default/openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-pjxxl] addLogicalPort annotation time took 14.06401ms I0428 15:48:38.407760 3647 pods.go:271] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-pjxxl] addLogicalPort took 15.340594ms, libovsdb time 803.091µs I0428 15:48:38.485209 3647 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-bzqfx] creating logical port openshift-pipelines_pipelines-as-code-watcher-7947cf58bc-bzqfx for pod on switch ip-10-0-1-106.ec2.internal I0428 15:48:38.485312 3647 kube.go:256] Updating pod openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-bzqfx I0428 15:48:38.499769 3647 pod.go:62] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-bzqfx] pod update took 14.479505ms I0428 15:48:38.499796 3647 base_network_controller_pods.go:951] [default/openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-bzqfx] addLogicalPort annotation time took 14.508253ms I0428 15:48:38.501067 3647 pods.go:271] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-bzqfx] addLogicalPort took 15.874763ms, libovsdb time 910.859µs I0428 15:48:38.589351 3647 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-as-code-webhook-68965d5857-89jg9] creating logical port openshift-pipelines_pipelines-as-code-webhook-68965d5857-89jg9 for pod on switch ip-10-0-1-106.ec2.internal I0428 15:48:38.589462 3647 kube.go:256] Updating pod openshift-pipelines/pipelines-as-code-webhook-68965d5857-89jg9 I0428 15:48:38.602647 3647 pod.go:62] [openshift-pipelines/pipelines-as-code-webhook-68965d5857-89jg9] pod update took 13.207651ms I0428 15:48:38.602671 3647 base_network_controller_pods.go:951] [default/openshift-pipelines/pipelines-as-code-webhook-68965d5857-89jg9] addLogicalPort annotation time took 13.248414ms I0428 15:48:38.604136 3647 pods.go:271] [openshift-pipelines/pipelines-as-code-webhook-68965d5857-89jg9] addLogicalPort took 14.803825ms, libovsdb time 986.783µs I0428 15:48:38.756546 3647 cni.go:452] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-pjxxl 3389f85e7f8127d99bd678f8a960ccd5ffb34d327a5f9189fd789ae7ab2588d3 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-pjxxl 3389f85e7f8127d99bd678f8a960ccd5ffb34d327a5f9189fd789ae7ab2588d3 network default NAD default NAD key ] I0428 15:48:38.785901 3647 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-as-code-controller-79b6fbbc5c-pjxxl, hostIfaceName: 3389f85e7f8127d, network: default, NAD default, SandboxID: "3389f85e7f8127d99bd678f8a960ccd5ffb34d327a5f9189fd789ae7ab2588d3", PCI device ID: , UID: "674e30c5-73bf-4925-a949-a0145963fe6b", MAC: 0a:58:0a:82:00:29, IPs: [10.130.0.41/23] I0428 15:48:38.842022 3647 cni.go:473] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-pjxxl 3389f85e7f8127d99bd678f8a960ccd5ffb34d327a5f9189fd789ae7ab2588d3 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-pjxxl 3389f85e7f8127d99bd678f8a960ccd5ffb34d327a5f9189fd789ae7ab2588d3 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ba:b7:9c:32:7f:68\",\"name\":\"3389f85e7f8127d\"},{\"mac\":\"0a:58:0a:82:00:29\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/26508cb2-48c3-4b5a-86e8-a38c6a0ec063\"}],\"ips\":[{\"address\":\"10.130.0.41/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0428 15:48:38.858002 3647 cni.go:452] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-bzqfx 5c96d351317b7c4fb2df53a937f523ca4d3fafd9b5cb4e5ab920ec96c8d4ce75 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-bzqfx 5c96d351317b7c4fb2df53a937f523ca4d3fafd9b5cb4e5ab920ec96c8d4ce75 network default NAD default NAD key ] I0428 15:48:38.867928 3647 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-as-code-watcher-7947cf58bc-bzqfx, hostIfaceName: 5c96d351317b7c4, network: default, NAD default, SandboxID: "5c96d351317b7c4fb2df53a937f523ca4d3fafd9b5cb4e5ab920ec96c8d4ce75", PCI device ID: , UID: "4465198f-341c-491a-aa93-cada3f07c98f", MAC: 0a:58:0a:82:00:2a, IPs: [10.130.0.42/23] I0428 15:48:38.924070 3647 cni.go:473] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-bzqfx 5c96d351317b7c4fb2df53a937f523ca4d3fafd9b5cb4e5ab920ec96c8d4ce75 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-bzqfx 5c96d351317b7c4fb2df53a937f523ca4d3fafd9b5cb4e5ab920ec96c8d4ce75 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"66:3b:b0:4b:6a:35\",\"name\":\"5c96d351317b7c4\"},{\"mac\":\"0a:58:0a:82:00:2a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3bc73c7a-36aa-4493-8e19-17d862c73523\"}],\"ips\":[{\"address\":\"10.130.0.42/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0428 15:48:38.957056 3647 cni.go:452] [openshift-pipelines/pipelines-as-code-webhook-68965d5857-89jg9 5052fb523a478f6439606c8ad1e4c5641b1b7553edefd38cfc6f292658f1ac5b network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/pipelines-as-code-webhook-68965d5857-89jg9 5052fb523a478f6439606c8ad1e4c5641b1b7553edefd38cfc6f292658f1ac5b network default NAD default NAD key ] I0428 15:48:38.967247 3647 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-as-code-webhook-68965d5857-89jg9, hostIfaceName: 5052fb523a478f6, network: default, NAD default, SandboxID: "5052fb523a478f6439606c8ad1e4c5641b1b7553edefd38cfc6f292658f1ac5b", PCI device ID: , UID: "0b4cfd34-dfbc-4c42-8df3-15c4464f5482", MAC: 0a:58:0a:82:00:2b, IPs: [10.130.0.43/23] I0428 15:48:39.020762 3647 cni.go:473] [openshift-pipelines/pipelines-as-code-webhook-68965d5857-89jg9 5052fb523a478f6439606c8ad1e4c5641b1b7553edefd38cfc6f292658f1ac5b network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/pipelines-as-code-webhook-68965d5857-89jg9 5052fb523a478f6439606c8ad1e4c5641b1b7553edefd38cfc6f292658f1ac5b network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"e2:d1:e7:07:42:8d\",\"name\":\"5052fb523a478f6\"},{\"mac\":\"0a:58:0a:82:00:2b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/cd2acf68-e289-413a-a7ea-73f607a80359\"}],\"ips\":[{\"address\":\"10.130.0.43/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0428 15:48:39.175250 3647 obj_retry.go:492] Detected object tsf-keycloak/tsf-iam-m7t22 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:48:39.175283 3647 obj_retry.go:492] Detected object tsf-keycloak/tsf-iam-m7t22 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0428 15:48:42.178682 3647 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logsigner W0428 15:48:44.194685 3647 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher W0428 15:48:44.974428 3647 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logserver W0428 15:48:47.989448 3647 util.go:870] Failed to build global endpoints for port TCP/trillian-mysql: empty IP address endpoints for service tsf-tas/trillian-mysql I0428 15:48:49.149635 3647 obj_retry.go:498] Detected object tsf-tas/segment-backup-installation-p7x8d-22qzd of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:48:49.150235 3647 obj_retry.go:498] Detected object tsf-tas/segment-backup-installation-p7x8d-22qzd of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:48:50.032089 3647 obj_retry.go:492] Detected object tsf-tas/segment-backup-installation-p7x8d-22qzd of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:48:50.032116 3647 obj_retry.go:492] Detected object tsf-tas/segment-backup-installation-p7x8d-22qzd of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0428 15:48:55.207291 3647 util.go:959] Failed to build node endpoints for node ip-10-0-1-106.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher W0428 15:49:15.320023 3647 util.go:959] Failed to build node endpoints for node ip-10-0-1-106.ec2.internal port 8091: empty IP address endpoints for service tsf-tas/trillian-logsigner I0428 15:49:19.400297 3647 cni.go:452] [tsf-tas/test-tsf-tas ca84de3aa3c602e63fe619bfeba74292414977dcf4306de10c3a672d64cc209d network default NAD default NAD key ] DEL starting CNI request [tsf-tas/test-tsf-tas ca84de3aa3c602e63fe619bfeba74292414977dcf4306de10c3a672d64cc209d network default NAD default NAD key ] I0428 15:49:19.479553 3647 cni.go:473] [tsf-tas/test-tsf-tas ca84de3aa3c602e63fe619bfeba74292414977dcf4306de10c3a672d64cc209d network default NAD default NAD key default] DEL finished CNI request [tsf-tas/test-tsf-tas ca84de3aa3c602e63fe619bfeba74292414977dcf4306de10c3a672d64cc209d network default NAD default NAD key default], result "{}", err I0428 15:49:19.529394 3647 obj_retry.go:498] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:49:19.529435 3647 pods.go:176] Deleting pod: tsf-tas/test-tsf-tas I0428 15:49:19.530834 3647 pods.go:236] Attempting to release IPs for pod: tsf-tas/test-tsf-tas, ips: 10.130.0.39 I0428 15:49:19.530863 3647 obj_retry.go:498] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:49:19.548475 3647 obj_retry.go:492] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:49:19.548494 3647 obj_retry.go:492] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:49:19.553913 3647 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tas/test-tsf-tas I0428 15:49:19.553946 3647 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tas/test-tsf-tas I0428 15:49:28.019413 3647 obj_retry.go:498] Detected object tsf-tas/rekor-createtree-job-2ngdk-wk5k9 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:49:28.019976 3647 obj_retry.go:498] Detected object tsf-tas/rekor-createtree-job-2ngdk-wk5k9 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:49:28.348382 3647 obj_retry.go:498] Detected object tsf/patch-tekton-config-2rx2d of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:49:28.348985 3647 obj_retry.go:498] Detected object tsf/patch-tekton-config-2rx2d of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:49:28.905725 3647 obj_retry.go:492] Detected object tsf-tas/rekor-createtree-job-2ngdk-wk5k9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:49:28.905748 3647 obj_retry.go:492] Detected object tsf-tas/rekor-createtree-job-2ngdk-wk5k9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:49:29.377442 3647 obj_retry.go:492] Detected object tsf/patch-tekton-config-2rx2d of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:49:29.377467 3647 obj_retry.go:492] Detected object tsf/patch-tekton-config-2rx2d of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:49:33.374872 3647 obj_retry.go:498] Detected object tsf-tas/ctlog-createtree-job-9pz4t-55l6n of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:49:33.375407 3647 obj_retry.go:498] Detected object tsf-tas/ctlog-createtree-job-9pz4t-55l6n of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:49:34.249405 3647 obj_retry.go:492] Detected object tsf-tas/ctlog-createtree-job-9pz4t-55l6n of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:49:34.249424 3647 obj_retry.go:492] Detected object tsf-tas/ctlog-createtree-job-9pz4t-55l6n of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:49:35.605056 3647 base_network_controller_pods.go:487] [default/tsf-tas/rekor-server-7fddcfd5bb-q6sp7] creating logical port tsf-tas_rekor-server-7fddcfd5bb-q6sp7 for pod on switch ip-10-0-1-106.ec2.internal I0428 15:49:35.605163 3647 kube.go:256] Updating pod tsf-tas/rekor-server-7fddcfd5bb-q6sp7 I0428 15:49:35.616966 3647 pod.go:62] [tsf-tas/rekor-server-7fddcfd5bb-q6sp7] pod update took 11.829126ms I0428 15:49:35.616987 3647 base_network_controller_pods.go:951] [default/tsf-tas/rekor-server-7fddcfd5bb-q6sp7] addLogicalPort annotation time took 11.858943ms I0428 15:49:35.618217 3647 pods.go:271] [tsf-tas/rekor-server-7fddcfd5bb-q6sp7] addLogicalPort took 13.182593ms, libovsdb time 884.704µs W0428 15:49:36.259141 3647 util.go:870] Failed to build global endpoints for port TCP/resp: empty IP address endpoints for service tsf-tas/rekor-redis I0428 15:49:36.941752 3647 base_network_controller_pods.go:487] [default/tsf-tas/ctlog-79c5ccfdb4-m4kc4] creating logical port tsf-tas_ctlog-79c5ccfdb4-m4kc4 for pod on switch ip-10-0-1-106.ec2.internal I0428 15:49:36.941844 3647 kube.go:256] Updating pod tsf-tas/ctlog-79c5ccfdb4-m4kc4 W0428 15:49:36.947760 3647 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-search-ui I0428 15:49:36.959271 3647 pod.go:62] [tsf-tas/ctlog-79c5ccfdb4-m4kc4] pod update took 17.454676ms I0428 15:49:36.959294 3647 base_network_controller_pods.go:951] [default/tsf-tas/ctlog-79c5ccfdb4-m4kc4] addLogicalPort annotation time took 17.484532ms I0428 15:49:36.960471 3647 pods.go:271] [tsf-tas/ctlog-79c5ccfdb4-m4kc4] addLogicalPort took 18.738633ms, libovsdb time 832.148µs I0428 15:49:37.902751 3647 cni.go:452] [tsf-tas/ctlog-79c5ccfdb4-m4kc4 8c005278fa186db0bfb154f51a1010e9b0092a79d905f637144258e69dc7d936 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/ctlog-79c5ccfdb4-m4kc4 8c005278fa186db0bfb154f51a1010e9b0092a79d905f637144258e69dc7d936 network default NAD default NAD key ] I0428 15:49:37.912146 3647 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: ctlog-79c5ccfdb4-m4kc4, hostIfaceName: 8c005278fa186db, network: default, NAD default, SandboxID: "8c005278fa186db0bfb154f51a1010e9b0092a79d905f637144258e69dc7d936", PCI device ID: , UID: "9b798372-0023-418f-8df9-1486c9818bb0", MAC: 0a:58:0a:82:00:2d, IPs: [10.130.0.45/23] I0428 15:49:37.963680 3647 cni.go:473] [tsf-tas/ctlog-79c5ccfdb4-m4kc4 8c005278fa186db0bfb154f51a1010e9b0092a79d905f637144258e69dc7d936 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/ctlog-79c5ccfdb4-m4kc4 8c005278fa186db0bfb154f51a1010e9b0092a79d905f637144258e69dc7d936 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"f6:9b:d6:e2:e0:8e\",\"name\":\"8c005278fa186db\"},{\"mac\":\"0a:58:0a:82:00:2d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/404e5601-2606-4d8e-9e16-845fa76ab651\"}],\"ips\":[{\"address\":\"10.130.0.45/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0428 15:49:38.672366 3647 cni.go:452] [tsf-tas/rekor-server-7fddcfd5bb-q6sp7 1d3513b1bf6f0a5d796d3ebe76aa91f375696d1068e11debfd5da4fe9632a607 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/rekor-server-7fddcfd5bb-q6sp7 1d3513b1bf6f0a5d796d3ebe76aa91f375696d1068e11debfd5da4fe9632a607 network default NAD default NAD key ] I0428 15:49:38.682331 3647 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: rekor-server-7fddcfd5bb-q6sp7, hostIfaceName: 1d3513b1bf6f0a5, network: default, NAD default, SandboxID: "1d3513b1bf6f0a5d796d3ebe76aa91f375696d1068e11debfd5da4fe9632a607", PCI device ID: , UID: "a28592c8-6745-43bc-ae7c-730c15dfde7e", MAC: 0a:58:0a:82:00:2c, IPs: [10.130.0.44/23] I0428 15:49:38.732505 3647 cni.go:473] [tsf-tas/rekor-server-7fddcfd5bb-q6sp7 1d3513b1bf6f0a5d796d3ebe76aa91f375696d1068e11debfd5da4fe9632a607 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/rekor-server-7fddcfd5bb-q6sp7 1d3513b1bf6f0a5d796d3ebe76aa91f375696d1068e11debfd5da4fe9632a607 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"22:e6:d3:98:e3:dd\",\"name\":\"1d3513b1bf6f0a5\"},{\"mac\":\"0a:58:0a:82:00:2c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/479a0cdf-4bb4-4790-9482-0c90b8f4fda5\"}],\"ips\":[{\"address\":\"10.130.0.44/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0428 15:49:39.514634 3647 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/ctlog I0428 15:49:40.433365 3647 obj_retry.go:498] Detected object tsf/tsf-tekton-configuration-5wjr8 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:49:40.433926 3647 obj_retry.go:498] Detected object tsf/tsf-tekton-configuration-5wjr8 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:49:41.453911 3647 obj_retry.go:492] Detected object tsf/tsf-tekton-configuration-5wjr8 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:49:41.453936 3647 obj_retry.go:492] Detected object tsf/tsf-tekton-configuration-5wjr8 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:49:41.485935 3647 obj_retry.go:492] Detected object tsf/patch-tekton-config-2rx2d of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:49:41.485966 3647 obj_retry.go:492] Detected object tsf/patch-tekton-config-2rx2d of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:49:41.490323 3647 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/patch-tekton-config-2rx2d I0428 15:49:41.490350 3647 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/patch-tekton-config-2rx2d W0428 15:49:41.527224 3647 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-server I0428 15:49:42.111287 3647 base_network_controller_pods.go:487] [default/tsf-tpa/pre-install-tsf-tpa] creating logical port tsf-tpa_pre-install-tsf-tpa for pod on switch ip-10-0-1-106.ec2.internal I0428 15:49:42.111411 3647 kube.go:256] Updating pod tsf-tpa/pre-install-tsf-tpa I0428 15:49:42.124171 3647 pod.go:62] [tsf-tpa/pre-install-tsf-tpa] pod update took 12.807664ms I0428 15:49:42.124198 3647 base_network_controller_pods.go:951] [default/tsf-tpa/pre-install-tsf-tpa] addLogicalPort annotation time took 12.841161ms I0428 15:49:42.125540 3647 pods.go:271] [tsf-tpa/pre-install-tsf-tpa] addLogicalPort took 14.274144ms, libovsdb time 973.124µs I0428 15:49:42.471733 3647 cni.go:452] [tsf-tpa/pre-install-tsf-tpa 101c66c023ecbdc6312481ada53a35d8b9a3d64ab1b4274e99d542464b750951 network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/pre-install-tsf-tpa 101c66c023ecbdc6312481ada53a35d8b9a3d64ab1b4274e99d542464b750951 network default NAD default NAD key ] I0428 15:49:42.480902 3647 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: pre-install-tsf-tpa, hostIfaceName: 101c66c023ecbdc, network: default, NAD default, SandboxID: "101c66c023ecbdc6312481ada53a35d8b9a3d64ab1b4274e99d542464b750951", PCI device ID: , UID: "ac0d19e1-3ca1-4d84-ab40-5571dee7b89b", MAC: 0a:58:0a:82:00:2e, IPs: [10.130.0.46/23] I0428 15:49:42.532905 3647 cni.go:473] [tsf-tpa/pre-install-tsf-tpa 101c66c023ecbdc6312481ada53a35d8b9a3d64ab1b4274e99d542464b750951 network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/pre-install-tsf-tpa 101c66c023ecbdc6312481ada53a35d8b9a3d64ab1b4274e99d542464b750951 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"12:81:65:f8:15:2b\",\"name\":\"101c66c023ecbdc\"},{\"mac\":\"0a:58:0a:82:00:2e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8eb7bf82-15da-4099-b419-4fdf87dd189b\"}],\"ips\":[{\"address\":\"10.130.0.46/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0428 15:49:45.553861 3647 cni.go:452] [tsf-tpa/pre-install-tsf-tpa 101c66c023ecbdc6312481ada53a35d8b9a3d64ab1b4274e99d542464b750951 network default NAD default NAD key ] DEL starting CNI request [tsf-tpa/pre-install-tsf-tpa 101c66c023ecbdc6312481ada53a35d8b9a3d64ab1b4274e99d542464b750951 network default NAD default NAD key ] I0428 15:49:45.639790 3647 cni.go:473] [tsf-tpa/pre-install-tsf-tpa 101c66c023ecbdc6312481ada53a35d8b9a3d64ab1b4274e99d542464b750951 network default NAD default NAD key default] DEL finished CNI request [tsf-tpa/pre-install-tsf-tpa 101c66c023ecbdc6312481ada53a35d8b9a3d64ab1b4274e99d542464b750951 network default NAD default NAD key default], result "{}", err I0428 15:49:45.690569 3647 obj_retry.go:498] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:49:45.690602 3647 pods.go:176] Deleting pod: tsf-tpa/pre-install-tsf-tpa I0428 15:49:45.691851 3647 pods.go:236] Attempting to release IPs for pod: tsf-tpa/pre-install-tsf-tpa, ips: 10.130.0.46 I0428 15:49:45.691886 3647 obj_retry.go:498] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:49:45.708105 3647 obj_retry.go:492] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:49:45.708123 3647 obj_retry.go:492] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:49:45.713425 3647 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/pre-install-tsf-tpa I0428 15:49:45.713457 3647 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/pre-install-tsf-tpa I0428 15:49:45.901292 3647 base_network_controller_pods.go:487] [default/tsf-tpa/test-tsf-tpa] creating logical port tsf-tpa_test-tsf-tpa for pod on switch ip-10-0-1-106.ec2.internal I0428 15:49:45.901375 3647 kube.go:256] Updating pod tsf-tpa/test-tsf-tpa I0428 15:49:45.914369 3647 pod.go:62] [tsf-tpa/test-tsf-tpa] pod update took 13.013785ms I0428 15:49:45.914396 3647 base_network_controller_pods.go:951] [default/tsf-tpa/test-tsf-tpa] addLogicalPort annotation time took 13.046159ms I0428 15:49:45.915641 3647 pods.go:271] [tsf-tpa/test-tsf-tpa] addLogicalPort took 14.363995ms, libovsdb time 867.084µs I0428 15:49:46.265799 3647 cni.go:452] [tsf-tpa/test-tsf-tpa ac07f003c5db39b90a171b7c44113973e57ec6479f2ba4339c358cef78c4199f network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/test-tsf-tpa ac07f003c5db39b90a171b7c44113973e57ec6479f2ba4339c358cef78c4199f network default NAD default NAD key ] I0428 15:49:46.275406 3647 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: test-tsf-tpa, hostIfaceName: ac07f003c5db39b, network: default, NAD default, SandboxID: "ac07f003c5db39b90a171b7c44113973e57ec6479f2ba4339c358cef78c4199f", PCI device ID: , UID: "a46b8346-11df-4779-9f28-28a72cc6a252", MAC: 0a:58:0a:82:00:2f, IPs: [10.130.0.47/23] I0428 15:49:46.326723 3647 cni.go:473] [tsf-tpa/test-tsf-tpa ac07f003c5db39b90a171b7c44113973e57ec6479f2ba4339c358cef78c4199f network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/test-tsf-tpa ac07f003c5db39b90a171b7c44113973e57ec6479f2ba4339c358cef78c4199f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"42:c2:38:6f:06:08\",\"name\":\"ac07f003c5db39b\"},{\"mac\":\"0a:58:0a:82:00:2f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/941f43dc-463f-430b-9e57-945659bdfaa4\"}],\"ips\":[{\"address\":\"10.130.0.47/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0428 15:49:50.524839 3647 util.go:959] Failed to build node endpoints for node ip-10-0-1-106.ec2.internal port 6962: empty IP address endpoints for service tsf-tas/ctlog I0428 15:49:51.500782 3647 obj_retry.go:498] Detected object tsf-tpa/create-db-txz9b of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:49:51.501406 3647 obj_retry.go:498] Detected object tsf-tpa/create-db-txz9b of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:49:52.525683 3647 obj_retry.go:492] Detected object tsf-tpa/create-db-txz9b of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:49:52.525728 3647 obj_retry.go:492] Detected object tsf-tpa/create-db-txz9b of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0428 15:49:52.537866 3647 util.go:959] Failed to build node endpoints for node ip-10-0-1-106.ec2.internal port 3000: empty IP address endpoints for service tsf-tas/rekor-server I0428 15:49:57.190764 3647 obj_retry.go:498] Detected object tsf-tpa/migrate-db-cfwnr of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:49:57.191333 3647 obj_retry.go:498] Detected object tsf-tpa/migrate-db-cfwnr of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:49:58.214264 3647 obj_retry.go:492] Detected object tsf-tpa/migrate-db-cfwnr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:49:58.214294 3647 obj_retry.go:492] Detected object tsf-tpa/migrate-db-cfwnr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:50:00.218285 3647 obj_retry.go:498] Detected object tsf-tpa/create-importers-cgccv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:50:00.218798 3647 obj_retry.go:498] Detected object tsf-tpa/create-importers-cgccv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:50:01.247926 3647 obj_retry.go:492] Detected object tsf-tpa/create-importers-cgccv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:50:01.247949 3647 obj_retry.go:492] Detected object tsf-tpa/create-importers-cgccv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:50:03.483096 3647 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=8 I0428 15:50:04.234260 3647 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29623190-qm4qv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:50:04.234884 3647 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29623190-qm4qv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:50:05.254546 3647 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29623190-qm4qv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:50:05.254569 3647 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29623190-qm4qv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:50:06.601752 3647 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=6 W0428 15:50:08.447622 3647 util.go:870] Failed to build global endpoints for port TCP/endpoint: empty IP address endpoints for service tsf-tpa/server I0428 15:50:11.608823 3647 obj_retry.go:498] Detected object tsf-tas/tuf-repository-init-4sjvv-w8b87 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:50:11.609424 3647 obj_retry.go:498] Detected object tsf-tas/tuf-repository-init-4sjvv-w8b87 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:50:12.639373 3647 obj_retry.go:492] Detected object tsf-tas/tuf-repository-init-4sjvv-w8b87 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:50:12.639400 3647 obj_retry.go:492] Detected object tsf-tas/tuf-repository-init-4sjvv-w8b87 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0428 15:50:28.560906 3647 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/tuf I0428 15:50:29.816489 3647 cni.go:452] [tsf-tpa/test-tsf-tpa ac07f003c5db39b90a171b7c44113973e57ec6479f2ba4339c358cef78c4199f network default NAD default NAD key ] DEL starting CNI request [tsf-tpa/test-tsf-tpa ac07f003c5db39b90a171b7c44113973e57ec6479f2ba4339c358cef78c4199f network default NAD default NAD key ] I0428 15:50:29.905534 3647 cni.go:473] [tsf-tpa/test-tsf-tpa ac07f003c5db39b90a171b7c44113973e57ec6479f2ba4339c358cef78c4199f network default NAD default NAD key default] DEL finished CNI request [tsf-tpa/test-tsf-tpa ac07f003c5db39b90a171b7c44113973e57ec6479f2ba4339c358cef78c4199f network default NAD default NAD key default], result "{}", err I0428 15:50:29.956297 3647 obj_retry.go:498] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:50:29.956325 3647 pods.go:176] Deleting pod: tsf-tpa/test-tsf-tpa I0428 15:50:29.958024 3647 pods.go:236] Attempting to release IPs for pod: tsf-tpa/test-tsf-tpa, ips: 10.130.0.47 I0428 15:50:29.958054 3647 obj_retry.go:498] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:50:29.995051 3647 obj_retry.go:492] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:50:29.995082 3647 obj_retry.go:492] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:50:30.000513 3647 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/test-tsf-tpa I0428 15:50:30.000542 3647 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/test-tsf-tpa I0428 15:50:30.708181 3647 namespace.go:98] [image-controller] adding namespace I0428 15:50:30.709150 3647 namespace.go:102] [image-controller] adding namespace took 948.731µs I0428 15:50:30.713314 3647 namespace.go:98] [build-service] adding namespace I0428 15:50:30.714114 3647 namespace.go:102] [build-service] adding namespace took 782.686µs I0428 15:50:30.714245 3647 namespace.go:98] [integration-service] adding namespace I0428 15:50:30.714917 3647 namespace.go:102] [integration-service] adding namespace took 655.851µs I0428 15:50:30.727181 3647 namespace.go:98] [default-tenant] adding namespace I0428 15:50:30.727746 3647 namespace.go:102] [default-tenant] adding namespace took 550.5µs I0428 15:50:30.767331 3647 namespace.go:142] [image-controller] updating namespace I0428 15:50:30.798173 3647 namespace.go:142] [image-controller] updating namespace I0428 15:50:30.811025 3647 namespace.go:142] [build-service] updating namespace I0428 15:50:30.833334 3647 namespace.go:142] [build-service] updating namespace I0428 15:50:30.841104 3647 namespace.go:142] [integration-service] updating namespace I0428 15:50:30.865443 3647 namespace.go:142] [integration-service] updating namespace I0428 15:50:30.879232 3647 namespace.go:142] [default-tenant] updating namespace I0428 15:50:30.895259 3647 namespace.go:142] [default-tenant] updating namespace I0428 15:50:31.003022 3647 namespace.go:142] [build-service] updating namespace I0428 15:50:31.041440 3647 namespace.go:142] [integration-service] updating namespace I0428 15:50:31.094106 3647 namespace.go:98] [release-service] adding namespace I0428 15:50:31.095000 3647 namespace.go:102] [release-service] adding namespace took 875.476µs I0428 15:50:31.152607 3647 namespace.go:142] [konflux-ui] updating namespace I0428 15:50:31.178024 3647 namespace.go:142] [release-service] updating namespace I0428 15:50:31.204202 3647 namespace.go:142] [release-service] updating namespace I0428 15:50:31.311197 3647 namespace.go:98] [konflux-info] adding namespace I0428 15:50:31.312123 3647 namespace.go:102] [konflux-info] adding namespace took 905.336µs I0428 15:50:31.409326 3647 namespace.go:142] [konflux-info] updating namespace I0428 15:50:31.449424 3647 namespace.go:98] [namespace-lister] adding namespace I0428 15:50:31.450251 3647 namespace.go:102] [namespace-lister] adding namespace took 810.229µs I0428 15:50:31.476147 3647 namespace.go:142] [konflux-info] updating namespace I0428 15:50:31.508041 3647 namespace.go:98] [enterprise-contract-service] adding namespace I0428 15:50:31.508841 3647 namespace.go:102] [enterprise-contract-service] adding namespace took 785.017µs I0428 15:50:31.543254 3647 namespace.go:142] [namespace-lister] updating namespace I0428 15:50:31.571276 3647 namespace.go:142] [image-controller] updating namespace I0428 15:50:31.745313 3647 namespace.go:142] [default-tenant] updating namespace I0428 15:50:31.899162 3647 namespace.go:142] [namespace-lister] updating namespace I0428 15:50:32.061353 3647 namespace.go:98] [segment-bridge] adding namespace I0428 15:50:32.062223 3647 namespace.go:102] [segment-bridge] adding namespace took 849.567µs I0428 15:50:32.165443 3647 namespace.go:142] [enterprise-contract-service] updating namespace I0428 15:50:32.165500 3647 namespace.go:98] [konflux-cli] adding namespace I0428 15:50:32.166278 3647 namespace.go:102] [konflux-cli] adding namespace took 764.41µs I0428 15:50:32.314265 3647 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-from-konfluxui for network default I0428 15:50:32.314919 3647 pod_selector_address_set.go:213] Created shared address set for pod selector namespace-lister_LS{ML:{app: proxy,},} I0428 15:50:32.314936 3647 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-from-konfluxui added to peer address sets [namespace-lister_LS{ML:{app: proxy,},}] I0428 15:50:32.317377 3647 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-from-konfluxui resources completed, update namespace loglevel I0428 15:50:32.336010 3647 namespace.go:142] [enterprise-contract-service] updating namespace I0428 15:50:32.499485 3647 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-to-apiserver for network default I0428 15:50:32.499508 3647 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-to-apiserver added to peer address sets [] I0428 15:50:32.500419 3647 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-to-apiserver resources completed, update namespace loglevel I0428 15:50:32.515022 3647 namespace.go:142] [segment-bridge] updating namespace I0428 15:50:32.685426 3647 namespace.go:142] [segment-bridge] updating namespace I0428 15:50:32.916297 3647 namespace.go:142] [konflux-cli] updating namespace I0428 15:50:32.952227 3647 namespace.go:142] [konflux-cli] updating namespace I0428 15:50:33.373443 3647 namespace.go:142] [build-service] updating namespace I0428 15:50:33.437354 3647 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=543 I0428 15:50:33.502696 3647 base_network_controller_pods.go:487] [default/image-controller/image-controller-controller-manager-6f859968fc-9nxmp] creating logical port image-controller_image-controller-controller-manager-6f859968fc-9nxmp for pod on switch ip-10-0-1-106.ec2.internal I0428 15:50:33.502800 3647 kube.go:256] Updating pod image-controller/image-controller-controller-manager-6f859968fc-9nxmp I0428 15:50:33.545369 3647 pod.go:62] [image-controller/image-controller-controller-manager-6f859968fc-9nxmp] pod update took 42.594895ms I0428 15:50:33.545390 3647 base_network_controller_pods.go:951] [default/image-controller/image-controller-controller-manager-6f859968fc-9nxmp] addLogicalPort annotation time took 42.619398ms I0428 15:50:33.546554 3647 pods.go:271] [image-controller/image-controller-controller-manager-6f859968fc-9nxmp] addLogicalPort took 43.872503ms, libovsdb time 769.646µs I0428 15:50:33.757179 3647 namespace.go:142] [default-tenant] updating namespace I0428 15:50:33.835780 3647 cni.go:452] [image-controller/image-controller-controller-manager-6f859968fc-9nxmp 3ffc83fd65b02734a6cc51028008eabc8dcc6de07046442f64c31e7f5028de68 network default NAD default NAD key ] ADD starting CNI request [image-controller/image-controller-controller-manager-6f859968fc-9nxmp 3ffc83fd65b02734a6cc51028008eabc8dcc6de07046442f64c31e7f5028de68 network default NAD default NAD key ] I0428 15:50:33.845150 3647 helper_linux.go:556] ConfigureOVS: namespace: image-controller, podName: image-controller-controller-manager-6f859968fc-9nxmp, hostIfaceName: 3ffc83fd65b0273, network: default, NAD default, SandboxID: "3ffc83fd65b02734a6cc51028008eabc8dcc6de07046442f64c31e7f5028de68", PCI device ID: , UID: "d2083583-624d-4c0f-bc7f-4d448ae848d4", MAC: 0a:58:0a:82:00:30, IPs: [10.130.0.48/23] I0428 15:50:33.896958 3647 cni.go:473] [image-controller/image-controller-controller-manager-6f859968fc-9nxmp 3ffc83fd65b02734a6cc51028008eabc8dcc6de07046442f64c31e7f5028de68 network default NAD default NAD key default] ADD finished CNI request [image-controller/image-controller-controller-manager-6f859968fc-9nxmp 3ffc83fd65b02734a6cc51028008eabc8dcc6de07046442f64c31e7f5028de68 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"06:ae:af:6d:50:05\",\"name\":\"3ffc83fd65b0273\"},{\"mac\":\"0a:58:0a:82:00:30\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/26ed4234-aae1-47d6-b194-f18a10bbe626\"}],\"ips\":[{\"address\":\"10.130.0.48/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0428 15:50:34.078123 3647 namespace.go:142] [image-controller] updating namespace I0428 15:50:34.344091 3647 namespace.go:142] [integration-service] updating namespace I0428 15:50:34.561463 3647 namespace.go:142] [release-service] updating namespace I0428 15:50:34.754417 3647 base_network_controller_pods.go:487] [default/konflux-ui/proxy-9f756bb87-klgvn] creating logical port konflux-ui_proxy-9f756bb87-klgvn for pod on switch ip-10-0-1-106.ec2.internal I0428 15:50:34.754542 3647 kube.go:256] Updating pod konflux-ui/proxy-9f756bb87-klgvn I0428 15:50:34.851497 3647 namespace.go:142] [build-service] updating namespace I0428 15:50:34.886254 3647 kube.go:256] Updating pod konflux-ui/proxy-9f756bb87-klgvn I0428 15:50:35.013390 3647 namespace.go:142] [default-tenant] updating namespace I0428 15:50:35.056026 3647 pod.go:62] [konflux-ui/proxy-9f756bb87-klgvn] pod update took 301.535278ms I0428 15:50:35.056051 3647 base_network_controller_pods.go:951] [default/konflux-ui/proxy-9f756bb87-klgvn] addLogicalPort annotation time took 301.564729ms I0428 15:50:35.057386 3647 pods.go:271] [konflux-ui/proxy-9f756bb87-klgvn] addLogicalPort took 302.985731ms, libovsdb time 967.312µs I0428 15:50:35.131206 3647 namespace.go:142] [image-controller] updating namespace I0428 15:50:35.196511 3647 namespace.go:142] [integration-service] updating namespace I0428 15:50:35.256375 3647 namespace.go:142] [release-service] updating namespace I0428 15:50:35.448388 3647 namespace.go:142] [build-service] updating namespace I0428 15:50:35.732272 3647 namespace.go:142] [default-tenant] updating namespace I0428 15:50:35.844376 3647 namespace.go:142] [image-controller] updating namespace I0428 15:50:35.924585 3647 namespace.go:142] [integration-service] updating namespace I0428 15:50:35.988287 3647 namespace.go:142] [release-service] updating namespace W0428 15:50:36.878519 3647 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service image-controller/image-controller-controller-manager-metrics-service I0428 15:50:37.237620 3647 namespace.go:142] [enterprise-contract-service] updating namespace W0428 15:50:37.297528 3647 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service namespace-lister/namespace-lister I0428 15:50:37.437105 3647 namespace.go:142] [konflux-cli] updating namespace I0428 15:50:37.636824 3647 namespace.go:142] [konflux-info] updating namespace I0428 15:50:37.839799 3647 namespace.go:142] [namespace-lister] updating namespace I0428 15:50:38.039235 3647 namespace.go:142] [segment-bridge] updating namespace I0428 15:50:38.158865 3647 namespace.go:142] [enterprise-contract-service] updating namespace I0428 15:50:38.172928 3647 namespace.go:142] [konflux-cli] updating namespace I0428 15:50:38.188800 3647 namespace.go:142] [konflux-info] updating namespace I0428 15:50:38.205517 3647 namespace.go:142] [namespace-lister] updating namespace I0428 15:50:38.220643 3647 namespace.go:142] [segment-bridge] updating namespace I0428 15:50:38.266406 3647 namespace.go:142] [enterprise-contract-service] updating namespace I0428 15:50:38.371518 3647 namespace.go:142] [konflux-cli] updating namespace I0428 15:50:38.465038 3647 namespace.go:142] [konflux-info] updating namespace I0428 15:50:38.567203 3647 namespace.go:142] [namespace-lister] updating namespace I0428 15:50:38.663626 3647 namespace.go:142] [segment-bridge] updating namespace W0428 15:50:41.353895 3647 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service I0428 15:50:42.607686 3647 cni.go:452] [konflux-ui/proxy-9f756bb87-klgvn 51a7441ba70fb63357cea019fbdeb6171e74dd2bd47048aa24942623f325f81e network default NAD default NAD key ] ADD starting CNI request [konflux-ui/proxy-9f756bb87-klgvn 51a7441ba70fb63357cea019fbdeb6171e74dd2bd47048aa24942623f325f81e network default NAD default NAD key ] I0428 15:50:42.616945 3647 helper_linux.go:556] ConfigureOVS: namespace: konflux-ui, podName: proxy-9f756bb87-klgvn, hostIfaceName: 51a7441ba70fb63, network: default, NAD default, SandboxID: "51a7441ba70fb63357cea019fbdeb6171e74dd2bd47048aa24942623f325f81e", PCI device ID: , UID: "8383caca-a76d-42f9-a1e3-9002059dad61", MAC: 0a:58:0a:82:00:31, IPs: [10.130.0.49/23] I0428 15:50:42.667989 3647 cni.go:473] [konflux-ui/proxy-9f756bb87-klgvn 51a7441ba70fb63357cea019fbdeb6171e74dd2bd47048aa24942623f325f81e network default NAD default NAD key default] ADD finished CNI request [konflux-ui/proxy-9f756bb87-klgvn 51a7441ba70fb63357cea019fbdeb6171e74dd2bd47048aa24942623f325f81e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"2e:6c:fe:de:c7:fe\",\"name\":\"51a7441ba70fb63\"},{\"mac\":\"0a:58:0a:82:00:31\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2c5d5222-c3b0-4a47-9a8d-9538e7bcd1f6\"}],\"ips\":[{\"address\":\"10.130.0.49/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0428 15:50:43.337382 3647 util.go:870] Failed to build global endpoints for port TCP/dex: empty IP address endpoints for service konflux-ui/dex W0428 15:50:47.862245 3647 util.go:959] Failed to build node endpoints for node ip-10-0-1-106.ec2.internal port 8443: empty IP address endpoints for service image-controller/image-controller-controller-manager-metrics-service W0428 15:50:47.927297 3647 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0428 15:50:47.927315 3647 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0428 15:50:48.387065 3647 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0428 15:50:48.388626 3647 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0428 15:50:48.454300 3647 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0428 15:50:48.454317 3647 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0428 15:50:48.474386 3647 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service W0428 15:50:48.476088 3647 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service release-service/release-service-webhook-service W0428 15:50:48.498651 3647 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0428 15:50:48.498687 3647 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service I0428 15:50:59.753321 3647 obj_retry.go:498] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-5qf9p of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:50:59.753923 3647 obj_retry.go:498] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-5qf9p of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:51:00.451288 3647 obj_retry.go:492] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-5qf9p of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:51:00.451310 3647 obj_retry.go:492] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-5qf9p of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:51:00.460281 3647 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod integration-service/integration-service-controller-manager-7b666bf6d8-5qf9p I0428 15:51:00.460313 3647 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod integration-service/integration-service-controller-manager-7b666bf6d8-5qf9p I0428 15:51:22.598060 3647 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=9 W0428 15:51:25.443606 3647 util.go:959] Failed to build node endpoints for node ip-10-0-1-106.ec2.internal port 8080: empty IP address endpoints for service konflux-ui/proxy W0428 15:51:25.443628 3647 util.go:959] Failed to build node endpoints for node ip-10-0-1-106.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/proxy W0428 15:51:25.443638 3647 util.go:959] Failed to build node endpoints for node ip-10-0-1-106.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/proxy W0428 15:51:25.443643 3647 util.go:959] Failed to build node endpoints for node ip-10-0-1-106.ec2.internal port 8080: empty IP address endpoints for service konflux-ui/proxy W0428 15:51:28.001466 3647 util.go:959] Failed to build node endpoints for node ip-10-0-1-106.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/proxy W0428 15:51:28.001491 3647 util.go:959] Failed to build node endpoints for node ip-10-0-1-106.ec2.internal port 8080: empty IP address endpoints for service konflux-ui/proxy I0428 15:51:35.412778 3647 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=82 I0428 15:51:57.465741 3647 obj_retry.go:498] Detected object konflux-ui/proxy-c8b546c9d-qbnsp of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:51:57.466336 3647 obj_retry.go:498] Detected object konflux-ui/proxy-c8b546c9d-qbnsp of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:51:57.763952 3647 obj_retry.go:492] Detected object konflux-ui/proxy-c8b546c9d-qbnsp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:51:57.763975 3647 obj_retry.go:492] Detected object konflux-ui/proxy-c8b546c9d-qbnsp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:51:57.770629 3647 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/proxy-c8b546c9d-qbnsp I0428 15:51:57.770648 3647 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/proxy-c8b546c9d-qbnsp I0428 15:52:02.939120 3647 obj_retry.go:498] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:52:02.939735 3647 obj_retry.go:498] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:52:02.955060 3647 obj_retry.go:492] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:52:02.955109 3647 obj_retry.go:492] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:52:02.964957 3647 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/test-tsf-konflux I0428 15:52:02.964986 3647 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/test-tsf-konflux I0428 15:52:25.492369 3647 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=9 I0428 15:52:32.870979 3647 namespace.go:98] [managed-tenant-v8l0n] adding namespace I0428 15:52:32.871978 3647 namespace.go:102] [managed-tenant-v8l0n] adding namespace took 974.059µs I0428 15:52:32.894107 3647 namespace.go:142] [managed-tenant-v8l0n] updating namespace I0428 15:52:32.916216 3647 namespace.go:142] [managed-tenant-v8l0n] updating namespace I0428 15:52:34.056603 3647 namespace.go:142] [managed-tenant-v8l0n] updating namespace I0428 15:52:34.193902 3647 namespace.go:142] [managed-tenant-v8l0n] updating namespace I0428 15:52:34.295946 3647 namespace.go:142] [managed-tenant-v8l0n] updating namespace I0428 15:53:16.617830 3647 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=9 I0428 15:53:24.410308 3647 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=68 I0428 15:53:24.411207 3647 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=171 I0428 15:53:26.606602 3647 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=10 I0428 15:53:37.628166 3647 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=7 I0428 15:54:09.611035 3647 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=9 I0428 15:54:21.592565 3647 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=8 I0428 15:54:51.416505 3647 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=155 I0428 15:55:27.510426 3647 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=9 I0428 15:57:06.412300 3647 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=12 I0428 15:57:29.485265 3647 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=9 I0428 15:57:43.604640 3647 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=8 I0428 15:58:55.600223 3647 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=9 I0428 15:59:50.440376 3647 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=89 I0428 15:59:52.045280 3647 obj_retry.go:492] Detected object tsf-tpa/create-db-txz9b of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:59:52.045333 3647 obj_retry.go:492] Detected object tsf-tpa/create-db-txz9b of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:59:52.050007 3647 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-db-txz9b I0428 15:59:52.050036 3647 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-db-txz9b I0428 15:59:58.028124 3647 obj_retry.go:492] Detected object tsf-tpa/migrate-db-cfwnr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:59:58.028171 3647 obj_retry.go:492] Detected object tsf-tpa/migrate-db-cfwnr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:59:58.030693 3647 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/migrate-db-cfwnr I0428 15:59:58.030738 3647 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/migrate-db-cfwnr I0428 16:00:01.027017 3647 obj_retry.go:492] Detected object tsf-tpa/create-importers-cgccv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 16:00:01.027039 3647 obj_retry.go:492] Detected object tsf-tpa/create-importers-cgccv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 16:00:01.029878 3647 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-importers-cgccv I0428 16:00:01.029909 3647 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-importers-cgccv I0428 16:00:04.512620 3647 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29623200-9j7ns of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 16:00:04.513187 3647 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29623200-9j7ns of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 16:00:05.538898 3647 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29623200-9j7ns of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 16:00:05.538948 3647 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29623200-9j7ns of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 16:00:44.608916 3647 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=8 I0428 16:00:45.613659 3647 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=7 I0428 16:01:17.594738 3647 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=6 I0428 16:01:30.415329 3647 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=10 I0428 16:01:39.495412 3647 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=10 I0428 16:01:50.413441 3647 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=10 I0428 16:01:58.620866 3647 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=9 I0428 16:01:58.631858 3647 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=9 I0428 16:03:14.413600 3647 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=43 I0428 16:03:42.512768 3647 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=8 I0428 16:03:50.487591 3647 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=8 I0428 16:04:14.606908 3647 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=7 I0428 16:04:20.419642 3647 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=11 I0428 16:04:37.602447 3647 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=6 I0428 16:05:18.443386 3647 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=36 I0428 16:06:21.597161 3647 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=5 I0428 16:06:34.417547 3647 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=6 I0428 16:06:42.414712 3647 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=11