+ . /ovnkube-lib/ovnkube-lib.sh ++ set -x ++ K8S_NODE=ip-10-0-2-216.ec2.internal ++ [[ -n ip-10-0-2-216.ec2.internal ]] ++ [[ -f /env/ip-10-0-2-216.ec2.internal ]] ++ northd_pidfile=/var/run/ovn/ovn-northd.pid ++ controller_pidfile=/var/run/ovn/ovn-controller.pid ++ controller_logfile=/var/log/ovn/acl-audit-log.log ++ vswitch_dbsock=/var/run/openvswitch/db.sock ++ nbdb_pidfile=/var/run/ovn/ovnnb_db.pid ++ nbdb_sock=/var/run/ovn/ovnnb_db.sock ++ nbdb_ctl=/var/run/ovn/ovnnb_db.ctl ++ sbdb_pidfile=/var/run/ovn/ovnsb_db.pid ++ sbdb_sock=/var/run/ovn/ovnsb_db.sock ++ sbdb_ctl=/var/run/ovn/ovnsb_db.ctl + start-ovnkube-node 4 29103 29105 + local log_level=4 + local metrics_port=29103 + local ovn_metrics_port=29105 + ovn_advertised_udn_isolation_mode_flag= + openflow_probe_flag= + [[ 3 -ne 3 ]] + ovs-vsctl br-exists br-ex + add_garp_drop_flow br-ex + local bridge=br-ex + local cookie=0x0305 + local priority=499 ++ ovs-vsctl list-ports br-ex + for port_name in $(ovs-vsctl list-ports "$bridge") + [[ ens5 == *to-br-int ]] + cni-bin-copy + . /host/etc/os-release ++ NAME='Red Hat Enterprise Linux CoreOS' ++ VERSION='9.6.20260318-0 (Plow)' ++ ID=rhel ++ ID_LIKE=fedora ++ VERSION_ID=9.6 ++ PLATFORM_ID=platform:el9 ++ PRETTY_NAME='Red Hat Enterprise Linux CoreOS 9.6.20260318-0 (Plow)' ++ ANSI_COLOR='0;31' ++ LOGO=fedora-logo-icon ++ CPE_NAME=cpe:/o:redhat:enterprise_linux:9::baseos ++ HOME_URL=https://www.redhat.com/ ++ DOCUMENTATION_URL=https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9 ++ BUG_REPORT_URL=https://issues.redhat.com/ ++ REDHAT_BUGZILLA_PRODUCT='Red Hat Enterprise Linux 9' ++ REDHAT_BUGZILLA_PRODUCT_VERSION=9.6 ++ REDHAT_SUPPORT_PRODUCT='Red Hat Enterprise Linux' ++ REDHAT_SUPPORT_PRODUCT_VERSION=9.6 ++ OSTREE_VERSION=9.6.20260318-0 ++ VARIANT=CoreOS ++ VARIANT_ID=coreos ++ OPENSHIFT_VERSION=4.21 + rhelmajor= + case "${ID}" in ++ echo 9.6 ++ cut -f 1 -d . + rhelmajor=9 + sourcedir=/usr/libexec/cni/ + case "${rhelmajor}" in + sourcedir=/usr/libexec/cni/rhel9 + cp -f /usr/libexec/cni/rhel9/ovn-k8s-cni-overlay /cni-bin-dir/ ++ date '+%m%d %H:%M:%S.%N' + echo 'I0406 02:56:32.823354835 - disable conntrack on geneve port' I0406 02:56:32.823354835 - disable conntrack on geneve port + iptables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + iptables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK ++ date '+%m%d %H:%M:%S.%N' I0406 02:56:32.841080731 - starting ovnkube-node + echo 'I0406 02:56:32.841080731 - starting ovnkube-node' + egress_features_enable_flag='--enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true' + init_ovnkube_controller='--init-ovnkube-controller ip-10-0-2-216.ec2.internal' + multi_external_gateway_enable_flag=--enable-multi-external-gateway=true + gateway_interface=br-ex + enable_multicast_flag=--enable-multicast + OVN_NODE_MODE=full + '[' full == dpu-host ']' + '[' shared == shared ']' + gateway_mode_flags='--gateway-mode shared --gateway-interface br-ex' + export_network_flows_flags= + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + gw_interface_flag= + '[' -d /sys/class/net/br-ex1 ']' + node_mgmt_port_netdev_flags= + [[ -n '' ]] + [[ -n '' ]] + multi_network_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag=--enable-network-segmentation + route_advertisements_enable_flag= + [[ false == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag= + [[ true == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag=--enable-preconfigured-udn-addresses + network_observability_enabled_flag= + [[ false == \t\r\u\e ]] + multi_network_policy_enabled_flag= + [[ false == \t\r\u\e ]] + admin_network_policy_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + admin_network_policy_enabled_flag=--enable-admin-network-policy + dns_name_resolver_enabled_flag= + [[ false == \t\r\u\e ]] + ip_forwarding_flag= + '[' '' == Global ']' + ip_forwarding_flag=--disable-forwarding + sysctl -w net.ipv4.ip_forward=0 net.ipv4.ip_forward = 0 + sysctl -w net.ipv6.conf.all.forwarding=0 net.ipv6.conf.all.forwarding = 0 + [[ '' != '' ]] + [[ '' != '' ]] + NETWORK_NODE_IDENTITY_ENABLE= + [[ true == \t\r\u\e ]] + NETWORK_NODE_IDENTITY_ENABLE=' --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h ' + ovn_v4_join_subnet_opt= + [[ '' != '' ]] + ovn_v6_join_subnet_opt= + [[ '' != '' ]] + ovn_v4_masquerade_subnet_opt= + [[ 169.254.0.0/17 != '' ]] + ovn_v4_masquerade_subnet_opt='--gateway-v4-masquerade-subnet 169.254.0.0/17' + ovn_v6_masquerade_subnet_opt= + [[ fd69::/112 != '' ]] + ovn_v6_masquerade_subnet_opt='--gateway-v6-masquerade-subnet fd69::/112' + ovn_v4_transit_switch_subnet_opt= + [[ '' != '' ]] + ovn_v6_transit_switch_subnet_opt= + [[ '' != '' ]] + exec /usr/bin/ovnkube --init-ovnkube-controller ip-10-0-2-216.ec2.internal --init-node ip-10-0-2-216.ec2.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4 --inactivity-probe=180000 --gateway-mode shared --gateway-interface br-ex --metrics-bind-address 127.0.0.1:29103 --ovn-metrics-bind-address 127.0.0.1:29105 --metrics-enable-pprof --metrics-enable-config-duration --export-ovs-metrics --disable-snat-multiple-gws --enable-multi-network --enable-network-segmentation --enable-preconfigured-udn-addresses --enable-admin-network-policy --enable-multicast --zone ip-10-0-2-216.ec2.internal --enable-interconnect --acl-logging-rate-limit 20 --disable-forwarding --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h --gateway-v4-masquerade-subnet 169.254.0.0/17 --gateway-v6-masquerade-subnet fd69::/112 --enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true --enable-multi-external-gateway=true I0406 02:56:32.880041 3776 config.go:2459] Parsed config file /run/ovnkube-config/ovnkube.conf I0406 02:56:32.880100 3776 config.go:2460] Parsed config: {Default:{MTU:8901 RoutableMTU:0 ConntrackZone:64000 HostMasqConntrackZone:0 OVNMasqConntrackZone:0 HostNodePortConntrackZone:0 ReassemblyConntrackZone:0 EncapType:geneve EncapIP: EffectiveEncapIP: EncapPort:6081 InactivityProbe:100000 OpenFlowProbe:0 OfctrlWaitBeforeClear:0 MonitorAll:true OVSDBTxnTimeout:1m40s LFlowCacheEnable:true LFlowCacheLimit:0 LFlowCacheLimitKb:1048576 RawClusterSubnets:10.128.0.0/14/23 ClusterSubnets:[] EnableUDPAggregation:true Zone:global RawUDNAllowedDefaultServices:default/kubernetes,openshift-dns/dns-default UDNAllowedDefaultServices:[]} Logging:{File: CNIFile: LibovsdbFile: Level:4 LogFileMaxSize:100 LogFileMaxBackups:5 LogFileMaxAge:5 ACLLoggingRateLimit:20} Monitoring:{RawNetFlowTargets: RawSFlowTargets: RawIPFIXTargets: NetFlowTargets:[] SFlowTargets:[] IPFIXTargets:[]} IPFIX:{Sampling:400 CacheActiveTimeout:60 CacheMaxFlows:0} CNI:{ConfDir:/etc/cni/net.d Plugin:ovn-k8s-cni-overlay} OVNKubernetesFeature:{EnableAdminNetworkPolicy:false EnableEgressIP:false EgressIPReachabiltyTotalTimeout:1 EnableEgressFirewall:false EnableEgressQoS:false EnableEgressService:false EgressIPNodeHealthCheckPort:9107 EnableMultiNetwork:false EnableNetworkSegmentation:true EnableNetworkConnect:false EnablePreconfiguredUDNAddresses:true EnableRouteAdvertisements:false EnableMultiNetworkPolicy:false EnableStatelessNetPol:false EnableInterconnect:false EnableMultiExternalGateway:false EnablePersistentIPs:false EnableDNSNameResolver:false EnableServiceTemplateSupport:false EnableObservability:false EnableNetworkQoS:false AdvertisedUDNIsolationMode:strict} Kubernetes:{BootstrapKubeconfig: CertDir: CertDuration:10m0s Kubeconfig: CACert: CAData:[] APIServer:https://api.kx-7042996710.hypershift.local:443 Token: TokenFile: CompatServiceCIDR: RawServiceCIDRs:172.30.0.0/16 ServiceCIDRs:[] OVNConfigNamespace:openshift-ovn-kubernetes OVNEmptyLbEvents:false RawNoHostSubnetNodes: NoHostSubnetNodes: HostNetworkNamespace:openshift-host-network DisableRequestedChassis:false PlatformType:AWS HealthzBindAddress:0.0.0.0:10256 CompatMetricsBindAddress: CompatOVNMetricsBindAddress: CompatMetricsEnablePprof:false DNSServiceNamespace:openshift-dns DNSServiceName:dns-default} Metrics:{BindAddress: OVNMetricsBindAddress: ExportOVSMetrics:false EnablePprof:false NodeServerPrivKey: NodeServerCert: EnableConfigDuration:false EnableScaleMetrics:false} OvnNorth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnnb_db.db exec:} OvnSouth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnsb_db.db exec:} Gateway:{Mode:shared Interface: GatewayAcceleratedInterface: EgressGWInterface: NextHop: VLANID:0 NodeportEnable:true DisableSNATMultipleGWs:false V4JoinSubnet:100.64.0.0/16 V6JoinSubnet:fd98::/64 V4MasqueradeSubnet:169.254.169.0/29 V6MasqueradeSubnet:fd69::/125 MasqueradeIPs:{V4OVNMasqueradeIP:169.254.169.1 V6OVNMasqueradeIP:fd69::1 V4HostMasqueradeIP:169.254.169.2 V6HostMasqueradeIP:fd69::2 V4HostETPLocalMasqueradeIP:169.254.169.3 V6HostETPLocalMasqueradeIP:fd69::3 V4DummyNextHopMasqueradeIP:169.254.169.4 V6DummyNextHopMasqueradeIP:fd69::4 V4OVNServiceHairpinMasqueradeIP:169.254.169.5 V6OVNServiceHairpinMasqueradeIP:fd69::5} DisablePacketMTUCheck:false RouterSubnet: SingleNode:false DisableForwarding:false AllowNoUplink:false EphemeralPortRange:} MasterHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} ClusterMgrHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} HybridOverlay:{Enabled:false RawClusterSubnets: ClusterSubnets:[] VXLANPort:4789} OvnKubeNode:{Mode:full MgmtPortNetdev: MgmtPortDPResourceName:} ClusterManager:{V4TransitSubnet:100.88.0.0/16 V6TransitSubnet:fd97::/64} OvsPaths:{RunDir:/var/run/openvswitch/}} I0406 02:56:32.881644 3776 kube.go:423] Waiting for certificate I0406 02:56:32.881686 3776 certificate_manager.go:422] "Certificate rotation is enabled" logger="kubernetes.io/kube-apiserver-client" I0406 02:56:32.881732 3776 certificate_manager.go:566] "Rotating certificates" logger="kubernetes.io/kube-apiserver-client" I0406 02:56:32.882892 3776 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0406 02:56:32.909078 3776 reflector.go:358] "Starting reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0406 02:56:32.909101 3776 reflector.go:404] "Listing and watching" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0406 02:56:32.911660 3776 reflector.go:436] "Caches populated" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0406 02:56:32.913940 3776 csr.go:274] "Certificate signing request is approved, waiting to be issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-xfhb9" I0406 02:56:32.921610 3776 csr.go:270] "Certificate signing request is issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-xfhb9" I0406 02:56:32.921701 3776 reflector.go:364] "Stopping reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0406 02:56:33.882351 3776 kube.go:430] Certificate found I0406 02:56:33.882935 3776 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0406 02:56:33.883849 3776 metrics.go:541] Starting metrics server at address "127.0.0.1:29103" I0406 02:56:33.930359 3776 certificate_manager.go:715] "Certificate rotation deadline determined" logger="kubernetes.io/kube-apiserver-client" expiration="2026-04-07 02:51:32 +0000 UTC" deadline="2026-04-06 19:49:28.138037326 +0000 UTC" I0406 02:56:33.930544 3776 certificate_manager.go:431] "Waiting for next certificate rotation" logger="kubernetes.io/kube-apiserver-client" sleep="16h52m54.207510127s" I0406 02:56:33.961681 3776 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0406 02:56:33.970050 3776 metrics.go:541] Starting metrics server at address "127.0.0.1:29105" I0406 02:56:33.972031 3776 node_controller_manager.go:338] Starting the node network controller manager, Mode: full I0406 02:56:33.972090 3776 factory.go:537] Starting watch factory I0406 02:56:33.972181 3776 reflector.go:358] "Starting reflector" type="*v1.Service" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0406 02:56:33.972199 3776 reflector.go:404] "Listing and watching" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0406 02:56:33.972216 3776 reflector.go:358] "Starting reflector" type="*v1.Node" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0406 02:56:33.972242 3776 reflector.go:404] "Listing and watching" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0406 02:56:33.972226 3776 reflector.go:358] "Starting reflector" type="*v1.Pod" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0406 02:56:33.972265 3776 reflector.go:404] "Listing and watching" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0406 02:56:33.972326 3776 reflector.go:358] "Starting reflector" type="*v1.NetworkPolicy" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0406 02:56:33.972375 3776 reflector.go:404] "Listing and watching" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0406 02:56:33.972483 3776 reflector.go:358] "Starting reflector" type="*v1.Namespace" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0406 02:56:33.972506 3776 reflector.go:404] "Listing and watching" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0406 02:56:33.972834 3776 reflector.go:358] "Starting reflector" type="*v1.EndpointSlice" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0406 02:56:33.972922 3776 reflector.go:404] "Listing and watching" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0406 02:56:33.980493 3776 controller_manager.go:368] Starting the ovnkube controller I0406 02:56:33.980504 3776 controller_manager.go:373] Waiting up to 5m0s for NBDB zone to match: ip-10-0-2-216.ec2.internal I0406 02:56:33.980549 3776 controller_manager.go:393] NBDB zone sync took: 39.105µs I0406 02:56:33.980556 3776 factory.go:537] Starting watch factory I0406 02:56:33.993676 3776 reflector.go:436] "Caches populated" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0406 02:56:33.994683 3776 ovn_db.go:368] Found OVN DB Pod running on this node. Registering OVN DB Metrics I0406 02:56:33.995744 3776 reflector.go:436] "Caches populated" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0406 02:56:33.996480 3776 reflector.go:436] "Caches populated" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0406 02:56:33.996707 3776 reflector.go:436] "Caches populated" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0406 02:56:34.000087 3776 reflector.go:436] "Caches populated" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0406 02:56:34.000638 3776 ovn_northd.go:121] Registering OVN northd metrics I0406 02:56:34.001856 3776 ovn_db.go:330] /var/run/openvswitch/ovnnb_db.sock getting info failed: stat /var/run/openvswitch/ovnnb_db.sock: no such file or directory I0406 02:56:34.001912 3776 ovn_db.go:327] ovnnb_db.sock found at /var/run/ovn/ I0406 02:56:34.008449 3776 reflector.go:436] "Caches populated" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0406 02:56:34.020398 3776 ovn_db.go:415] Found db is standalone, don't register db_cluster metrics I0406 02:56:34.077539 3776 factory.go:1931] *v1.Service informer cache synced successfully I0406 02:56:34.077582 3776 factory.go:1931] *v1.EndpointSlice informer cache synced successfully I0406 02:56:34.077587 3776 factory.go:1931] *v1.Pod informer cache synced successfully I0406 02:56:34.077590 3776 factory.go:1931] *v1.NetworkPolicy informer cache synced successfully I0406 02:56:34.077593 3776 factory.go:1931] *v1.Namespace informer cache synced successfully I0406 02:56:34.077596 3776 factory.go:1931] *v1.Node informer cache synced successfully I0406 02:56:34.077704 3776 reflector.go:358] "Starting reflector" type="*v1alpha1.BaselineAdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0406 02:56:34.077731 3776 reflector.go:404] "Listing and watching" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0406 02:56:34.077800 3776 reflector.go:358] "Starting reflector" type="*v1alpha1.AdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0406 02:56:34.077886 3776 reflector.go:404] "Listing and watching" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0406 02:56:34.080622 3776 reflector.go:436] "Caches populated" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0406 02:56:34.080626 3776 reflector.go:436] "Caches populated" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0406 02:56:34.082641 3776 factory.go:1931] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0406 02:56:34.082651 3776 factory.go:1931] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0406 02:56:34.082709 3776 reflector.go:358] "Starting reflector" type="*v1.EgressIP" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0406 02:56:34.082720 3776 reflector.go:404] "Listing and watching" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0406 02:56:34.084933 3776 reflector.go:436] "Caches populated" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0406 02:56:34.086909 3776 factory.go:1931] *v1.Service informer cache synced successfully I0406 02:56:34.086919 3776 factory.go:1931] *v1.EndpointSlice informer cache synced successfully I0406 02:56:34.086924 3776 factory.go:1931] *v1.Pod informer cache synced successfully I0406 02:56:34.086929 3776 factory.go:1931] *v1.NetworkPolicy informer cache synced successfully I0406 02:56:34.086933 3776 factory.go:1931] *v1.Namespace informer cache synced successfully I0406 02:56:34.086937 3776 factory.go:1931] *v1.Node informer cache synced successfully I0406 02:56:34.088058 3776 factory.go:1931] *v1.EgressIP informer cache synced successfully I0406 02:56:34.088117 3776 reflector.go:358] "Starting reflector" type="*v1.EgressFirewall" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0406 02:56:34.088150 3776 reflector.go:404] "Listing and watching" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0406 02:56:34.090723 3776 reflector.go:436] "Caches populated" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0406 02:56:34.092673 3776 factory.go:1931] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0406 02:56:34.092686 3776 factory.go:1931] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0406 02:56:34.093884 3776 factory.go:1931] *v1.EgressFirewall informer cache synced successfully I0406 02:56:34.093944 3776 reflector.go:358] "Starting reflector" type="*v1.EgressQoS" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0406 02:56:34.093956 3776 reflector.go:404] "Listing and watching" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0406 02:56:34.096341 3776 reflector.go:436] "Caches populated" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0406 02:56:34.098187 3776 factory.go:1931] *v1.EgressIP informer cache synced successfully I0406 02:56:34.099335 3776 factory.go:1931] *v1.EgressQoS informer cache synced successfully I0406 02:56:34.099393 3776 reflector.go:358] "Starting reflector" type="*v1.EgressService" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0406 02:56:34.099405 3776 reflector.go:404] "Listing and watching" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0406 02:56:34.101762 3776 reflector.go:436] "Caches populated" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0406 02:56:34.103657 3776 factory.go:1931] *v1.EgressFirewall informer cache synced successfully I0406 02:56:34.104799 3776 factory.go:1931] *v1.EgressService informer cache synced successfully I0406 02:56:34.104858 3776 reflector.go:358] "Starting reflector" type="*v1.AdminPolicyBasedExternalRoute" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0406 02:56:34.104869 3776 reflector.go:404] "Listing and watching" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0406 02:56:34.107269 3776 reflector.go:436] "Caches populated" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0406 02:56:34.109183 3776 factory.go:1931] *v1.EgressQoS informer cache synced successfully I0406 02:56:34.110269 3776 factory.go:1931] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0406 02:56:34.110432 3776 reflector.go:358] "Starting reflector" type="*v1.NetworkAttachmentDefinition" resyncPeriod="0s" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0406 02:56:34.110444 3776 reflector.go:404] "Listing and watching" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0406 02:56:34.112644 3776 reflector.go:436] "Caches populated" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0406 02:56:34.114510 3776 factory.go:1931] *v1.EgressService informer cache synced successfully I0406 02:56:34.115597 3776 factory.go:1931] *v1.NetworkAttachmentDefinition informer cache synced successfully I0406 02:56:34.115655 3776 reflector.go:358] "Starting reflector" type="*v1.ClusterUserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0406 02:56:34.115667 3776 reflector.go:404] "Listing and watching" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0406 02:56:34.115716 3776 reflector.go:358] "Starting reflector" type="*v1.UserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0406 02:56:34.115735 3776 reflector.go:404] "Listing and watching" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0406 02:56:34.118225 3776 reflector.go:436] "Caches populated" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0406 02:56:34.118279 3776 reflector.go:436] "Caches populated" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0406 02:56:34.120138 3776 factory.go:1931] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0406 02:56:34.121556 3776 factory.go:1931] *v1.UserDefinedNetwork informer cache synced successfully I0406 02:56:34.121563 3776 factory.go:1931] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0406 02:56:34.121570 3776 factory.go:649] Watch Factory start up complete, took: 149.482413ms I0406 02:56:34.121633 3776 default_node_network_controller.go:170] Enable node proxy healthz server on 0.0.0.0:10256 I0406 02:56:34.126119 3776 factory.go:1931] *v1.NetworkAttachmentDefinition informer cache synced successfully I0406 02:56:34.131386 3776 factory.go:1931] *v1.UserDefinedNetwork informer cache synced successfully I0406 02:56:34.131396 3776 factory.go:1931] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0406 02:56:34.131403 3776 factory.go:649] Watch Factory start up complete, took: 150.846372ms I0406 02:56:34.131411 3776 controller_manager.go:404] Waiting up to 5m0s for a node to have "ip-10-0-2-216.ec2.internal" zone I0406 02:56:34.152380 3776 default_node_network_controller.go:709] Initializing the default node network controller I0406 02:56:34.210626 3776 udn_isolation.go:101] Starting UDN host isolation manager I0406 02:56:34.217451 3776 udn_isolation.go:112] Found kubelet cgroup path: system.slice/kubelet.service I0406 02:56:34.269152 3776 controller.go:133] Adding controller udn-host-isolation-manager event handlers I0406 02:56:34.269262 3776 shared_informer.go:349] "Waiting for caches to sync" controller="udn-host-isolation-manager" I0406 02:56:34.269286 3776 shared_informer.go:356] "Caches are synced" controller="udn-host-isolation-manager" I0406 02:56:34.304342 3776 controller.go:157] Starting controller udn-host-isolation-manager with 1 workers I0406 02:56:34.304410 3776 default_node_network_controller.go:808] Node ip-10-0-2-216.ec2.internal ready for ovn initialization with subnet 10.129.0.0/23 I0406 02:56:34.310075 3776 kube.go:131] Setting annotations map[k8s.ovn.org/node-encap-ips:["10.0.2.216"] k8s.ovn.org/zone-name:ip-10-0-2-216.ec2.internal] on node ip-10-0-2-216.ec2.internal I0406 02:56:34.332003 3776 gateway_init.go:234] Initializing Gateway Functionality for Gateway PreStart I0406 02:56:34.338922 3776 helper_linux.go:93] Provided gateway interface "br-ex", found as index: 5 I0406 02:56:34.339088 3776 helper_linux.go:118] Found default gateway interface br-ex 10.0.2.1 I0406 02:56:34.339352 3776 gateway_init.go:263] Preparing Gateway I0406 02:56:34.339364 3776 gateway_shared_intf.go:1658] Creating new gateway I0406 02:56:34.381803 3776 controller_manager.go:427] Waiting for node in zone sync took: 250.377952ms I0406 02:56:34.387058 3776 controller_manager.go:608] Switching to transit router for layer2 networks I0406 02:56:34.387075 3776 kube.go:131] Setting annotations map[k8s.ovn.org/layer2-topology-version:2.0] on node ip-10-0-2-216.ec2.internal I0406 02:56:34.404310 3776 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0406 02:56:34.410701 3776 controller_manager.go:288] SCTP support detected in OVN I0406 02:56:34.411500 3776 services_controller.go:69] Creating services controller for network=default I0406 02:56:34.411637 3776 udn_enabled_svc.go:80] Waiting for service informer to sync I0406 02:56:34.411762 3776 controller.go:133] Adding controller [zone-nad-controller NAD controller] event handlers I0406 02:56:34.411840 3776 shared_informer.go:349] "Waiting for caches to sync" controller="[zone-nad-controller NAD controller]" I0406 02:56:34.411855 3776 shared_informer.go:356] "Caches are synced" controller="[zone-nad-controller NAD controller]" I0406 02:56:34.411864 3776 nad_controller.go:241] [zone-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0406 02:56:34.412051 3776 controller.go:157] Starting controller [zone-nad-controller NAD controller] with 1 workers I0406 02:56:34.412090 3776 network_controller.go:246] [zone-nad-controller network controller]: syncing all networks I0406 02:56:34.412148 3776 network_controller.go:275] [zone-nad-controller network controller]: finished syncing network default, took 44.318µs I0406 02:56:34.412160 3776 network_controller.go:257] [zone-nad-controller network controller]: finished syncing all networks. Time taken: 68.191µs I0406 02:56:34.412160 3776 nad_controller.go:280] [zone-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 74.569µs I0406 02:56:34.412165 3776 controller.go:157] Starting controller [zone-nad-controller network controller] with 1 workers I0406 02:56:34.412173 3776 nad_controller.go:173] [zone-nad-controller NAD controller]: started I0406 02:56:34.412181 3776 default_network_controller.go:321] Starting the default network controller I0406 02:56:34.412186 3776 udn_enabled_svc.go:95] Performing full resync I0406 02:56:34.412263 3776 network_controller.go:275] [zone-nad-controller network controller]: finished syncing network default, took 15.771µs I0406 02:56:34.412709 3776 udn_enabled_svc.go:99] Waiting for handler to sync I0406 02:56:34.412725 3776 udn_enabled_svc.go:104] Starting worker I0406 02:56:34.420964 3776 default_network_controller.go:388] Cleaning External Gateway ECMP routes I0406 02:56:34.421048 3776 repair.go:33] Syncing exgw routes took 68.444µs I0406 02:56:34.421067 3776 default_network_controller.go:407] Starting all the Watchers... I0406 02:56:34.421218 3776 namespace.go:98] [openshift-controller-manager-operator] adding namespace I0406 02:56:34.421233 3776 namespace.go:98] [openshift-package-operator] adding namespace I0406 02:56:34.421244 3776 namespace.go:98] [openshift-ingress] adding namespace I0406 02:56:34.421250 3776 namespace.go:98] [openshift-machine-config-operator] adding namespace I0406 02:56:34.421259 3776 namespace.go:98] [openshift-backplane-mcs-tier-two] adding namespace I0406 02:56:34.421263 3776 namespace.go:98] [openshift-config] adding namespace I0406 02:56:34.421270 3776 namespace.go:98] [openshift-cluster-version] adding namespace I0406 02:56:34.421276 3776 namespace.go:98] [openshift-ingress-operator] adding namespace I0406 02:56:34.421285 3776 namespace.go:98] [openshift-infra] adding namespace I0406 02:56:34.421306 3776 namespace.go:98] [openshift-config-managed] adding namespace I0406 02:56:34.421328 3776 namespace.go:98] [openshift-route-controller-manager] adding namespace I0406 02:56:34.421341 3776 namespace.go:98] [openshift-network-diagnostics] adding namespace I0406 02:56:34.421369 3776 namespace.go:98] [openshift-dns] adding namespace I0406 02:56:34.421391 3776 namespace.go:98] [openshift-kube-controller-manager-operator] adding namespace I0406 02:56:34.421402 3776 namespace.go:98] [openshift-cluster-csi-drivers] adding namespace I0406 02:56:34.421887 3776 namespace.go:102] [openshift-controller-manager-operator] adding namespace took 655.749µs I0406 02:56:34.421902 3776 namespace.go:98] [openshift-backplane-lpsre] adding namespace I0406 02:56:34.422361 3776 namespace.go:102] [openshift-backplane-lpsre] adding namespace took 454.202µs I0406 02:56:34.422373 3776 namespace.go:98] [openshift-apiserver] adding namespace I0406 02:56:34.422849 3776 namespace.go:102] [openshift-apiserver] adding namespace took 469.052µs I0406 02:56:34.422868 3776 namespace.go:98] [openshift-network-operator] adding namespace I0406 02:56:34.423866 3776 namespace.go:102] [openshift-package-operator] adding namespace took 2.62156ms I0406 02:56:34.423883 3776 namespace.go:98] [openshift-insights] adding namespace I0406 02:56:34.424483 3776 namespace.go:102] [openshift-ingress] adding namespace took 3.230149ms I0406 02:56:34.424499 3776 namespace.go:98] [openshift-backplane-tam] adding namespace I0406 02:56:34.425065 3776 namespace.go:102] [openshift-machine-config-operator] adding namespace took 3.80395ms I0406 02:56:34.425082 3776 namespace.go:98] [openshift-kube-scheduler] adding namespace I0406 02:56:34.425507 3776 namespace.go:102] [openshift-backplane-mcs-tier-two] adding namespace took 4.234156ms I0406 02:56:34.425527 3776 namespace.go:98] [openshift-operator-lifecycle-manager] adding namespace I0406 02:56:34.426049 3776 namespace.go:102] [openshift-config] adding namespace took 4.778287ms I0406 02:56:34.426062 3776 namespace.go:98] [openshift-machine-api] adding namespace I0406 02:56:34.426598 3776 namespace.go:102] [openshift-cluster-version] adding namespace took 5.322491ms I0406 02:56:34.426608 3776 namespace.go:98] [openshift-backplane] adding namespace I0406 02:56:34.427084 3776 namespace.go:102] [openshift-ingress-operator] adding namespace took 5.798429ms I0406 02:56:34.427096 3776 namespace.go:98] [openshift-kube-controller-manager] adding namespace I0406 02:56:34.427705 3776 namespace.go:102] [openshift-infra] adding namespace took 6.411443ms I0406 02:56:34.427719 3776 namespace.go:98] [openshift-kube-storage-version-migrator] adding namespace I0406 02:56:34.428249 3776 namespace.go:102] [openshift-config-managed] adding namespace took 6.905137ms I0406 02:56:34.428271 3776 namespace.go:98] [openshift-controller-manager] adding namespace I0406 02:56:34.428465 3776 iptables.go:108] Creating table: filter chain: FORWARD W0406 02:56:34.428720 3776 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-9jhwf: pod openshift-network-diagnostics/network-check-target-9jhwf: no pod IPs found W0406 02:56:34.428741 3776 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-nc244: pod openshift-network-diagnostics/network-check-target-nc244: no pod IPs found I0406 02:56:34.428745 3776 namespace.go:102] [openshift-route-controller-manager] adding namespace took 7.406341ms I0406 02:56:34.428758 3776 namespace.go:98] [openshift-console-user-settings] adding namespace I0406 02:56:34.429383 3776 namespace.go:102] [openshift-network-diagnostics] adding namespace took 8.036391ms I0406 02:56:34.429398 3776 namespace.go:98] [openshift-backplane-csm] adding namespace I0406 02:56:34.429953 3776 namespace.go:102] [openshift-dns] adding namespace took 8.5615ms I0406 02:56:34.429972 3776 namespace.go:98] [openshift-node] adding namespace I0406 02:56:34.430530 3776 namespace.go:102] [openshift-kube-controller-manager-operator] adding namespace took 9.134084ms I0406 02:56:34.430545 3776 namespace.go:98] [openshift-ingress-canary] adding namespace I0406 02:56:34.431009 3776 namespace.go:102] [openshift-cluster-csi-drivers] adding namespace took 9.600665ms I0406 02:56:34.431028 3776 namespace.go:98] [openshift-console] adding namespace I0406 02:56:34.431479 3776 namespace.go:102] [openshift-network-operator] adding namespace took 8.60613ms I0406 02:56:34.431494 3776 namespace.go:98] [openshift-deployment-validation-operator] adding namespace I0406 02:56:34.431900 3776 namespace.go:102] [openshift-insights] adding namespace took 8.010951ms I0406 02:56:34.431915 3776 namespace.go:98] [openshift-cluster-samples-operator] adding namespace I0406 02:56:34.432301 3776 namespace.go:102] [openshift-backplane-tam] adding namespace took 7.796552ms I0406 02:56:34.432313 3776 namespace.go:98] [openshift-backplane-cee] adding namespace I0406 02:56:34.432727 3776 namespace.go:102] [openshift-kube-scheduler] adding namespace took 7.640182ms I0406 02:56:34.432749 3776 namespace.go:98] [openshift-cloud-controller-manager] adding namespace I0406 02:56:34.433156 3776 namespace.go:102] [openshift-operator-lifecycle-manager] adding namespace took 7.620717ms I0406 02:56:34.433172 3776 namespace.go:98] [openshift-operators] adding namespace I0406 02:56:34.433532 3776 namespace.go:102] [openshift-machine-api] adding namespace took 7.464448ms I0406 02:56:34.433543 3776 namespace.go:98] [openshift-apiserver-operator] adding namespace I0406 02:56:34.433938 3776 namespace.go:102] [openshift-backplane] adding namespace took 7.323773ms I0406 02:56:34.433956 3776 namespace.go:98] [kube-system] adding namespace I0406 02:56:34.434385 3776 namespace.go:102] [openshift-kube-controller-manager] adding namespace took 7.28251ms I0406 02:56:34.434404 3776 namespace.go:98] [openshift-backplane-srep-ro] adding namespace I0406 02:56:34.434834 3776 namespace.go:102] [openshift-kube-storage-version-migrator] adding namespace took 7.107337ms I0406 02:56:34.434853 3776 namespace.go:98] [openshift-must-gather-operator] adding namespace I0406 02:56:34.435252 3776 namespace.go:102] [openshift-controller-manager] adding namespace took 6.974517ms I0406 02:56:34.435268 3776 namespace.go:98] [openshift-config-operator] adding namespace I0406 02:56:34.435490 3776 iptables.go:108] Creating table: filter chain: OUTPUT I0406 02:56:34.435658 3776 namespace.go:102] [openshift-console-user-settings] adding namespace took 6.894192ms I0406 02:56:34.435674 3776 namespace.go:98] [openshift-backplane-managed-scripts] adding namespace I0406 02:56:34.436071 3776 namespace.go:102] [openshift-backplane-csm] adding namespace took 6.667041ms I0406 02:56:34.436085 3776 namespace.go:98] [openshift-dns-operator] adding namespace I0406 02:56:34.436561 3776 namespace.go:102] [openshift-node] adding namespace took 6.582094ms I0406 02:56:34.436577 3776 namespace.go:98] [dedicated-admin] adding namespace I0406 02:56:34.436963 3776 namespace.go:102] [openshift-ingress-canary] adding namespace took 6.411013ms I0406 02:56:34.436981 3776 namespace.go:98] [openshift-marketplace] adding namespace I0406 02:56:34.437423 3776 namespace.go:102] [openshift-console] adding namespace took 6.375378ms I0406 02:56:34.437447 3776 namespace.go:98] [openshift-authentication-operator] adding namespace I0406 02:56:34.437849 3776 namespace.go:102] [openshift-deployment-validation-operator] adding namespace took 6.347353ms I0406 02:56:34.438408 3776 namespace.go:102] [openshift-cluster-samples-operator] adding namespace took 6.4886ms I0406 02:56:34.438425 3776 namespace.go:98] [openshift-kube-apiserver] adding namespace I0406 02:56:34.438850 3776 namespace.go:102] [openshift-backplane-cee] adding namespace took 6.532635ms I0406 02:56:34.438875 3776 namespace.go:98] [openshift-cloud-network-config-controller] adding namespace I0406 02:56:34.439306 3776 namespace.go:102] [openshift-cloud-controller-manager] adding namespace took 6.549038ms I0406 02:56:34.439321 3776 namespace.go:98] [openshift-operators-redhat] adding namespace I0406 02:56:34.439679 3776 namespace.go:102] [openshift-operators] adding namespace took 6.5021ms I0406 02:56:34.439692 3776 namespace.go:98] [openshift-host-network] adding namespace W0406 02:56:34.440542 3776 base_network_controller_namespace.go:409] Failed to get IPs for pod kube-system/global-pull-secret-syncer-gzf4p: pod kube-system/global-pull-secret-syncer-gzf4p: no pod IPs found I0406 02:56:34.440568 3776 namespace.go:102] [openshift-apiserver-operator] adding namespace took 7.019942ms I0406 02:56:34.440580 3776 namespace.go:98] [open-cluster-management-agent-addon] adding namespace I0406 02:56:34.441054 3776 namespace.go:102] [kube-system] adding namespace took 7.092915ms I0406 02:56:34.441071 3776 namespace.go:98] [openshift-kube-storage-version-migrator-operator] adding namespace I0406 02:56:34.441492 3776 namespace.go:102] [openshift-backplane-srep-ro] adding namespace took 7.081533ms I0406 02:56:34.441508 3776 namespace.go:98] [openshift-backplane-cse] adding namespace I0406 02:56:34.441910 3776 namespace.go:102] [openshift-must-gather-operator] adding namespace took 7.05069ms I0406 02:56:34.441922 3776 namespace.go:98] [openshift-cluster-machine-approver] adding namespace I0406 02:56:34.442338 3776 namespace.go:102] [openshift-config-operator] adding namespace took 7.063272ms I0406 02:56:34.442352 3776 namespace.go:98] [openshift-etcd] adding namespace I0406 02:56:34.442745 3776 namespace.go:102] [openshift-backplane-managed-scripts] adding namespace took 7.063758ms I0406 02:56:34.442760 3776 namespace.go:98] [kube-node-lease] adding namespace I0406 02:56:34.442846 3776 gateway_shared_intf.go:1787] Gateway Creation Complete I0406 02:56:34.442861 3776 kube.go:131] Setting annotations map[k8s.ovn.org/gateway-mtu-support: k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-2-216.ec2.internal","mac-address":"0e:3e:78:df:3c:53","ip-addresses":["10.0.2.216/24"],"ip-address":"10.0.2.216/24","next-hops":["10.0.2.1"],"next-hop":"10.0.2.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:97cd1d65-bc3b-4947-b8e4-edc37074663b k8s.ovn.org/node-encap-ips:["10.0.2.216"] k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.2.216/24"} k8s.ovn.org/zone-name:ip-10-0-2-216.ec2.internal] on node ip-10-0-2-216.ec2.internal I0406 02:56:34.443219 3776 namespace.go:102] [openshift-dns-operator] adding namespace took 7.127906ms I0406 02:56:34.443236 3776 namespace.go:98] [openshift-customer-monitoring] adding namespace I0406 02:56:34.443599 3776 namespace.go:102] [dedicated-admin] adding namespace took 7.016689ms I0406 02:56:34.443611 3776 namespace.go:98] [kube-public] adding namespace I0406 02:56:34.443986 3776 namespace.go:102] [openshift-marketplace] adding namespace took 6.999208ms I0406 02:56:34.444001 3776 namespace.go:98] [openshift-network-console] adding namespace I0406 02:56:34.444459 3776 namespace.go:102] [openshift-authentication-operator] adding namespace took 7.004878ms I0406 02:56:34.444472 3776 namespace.go:98] [openshift-backplane-srep] adding namespace I0406 02:56:34.444865 3776 namespace.go:102] [openshift-kube-apiserver] adding namespace took 6.433218ms I0406 02:56:34.444880 3776 namespace.go:98] [openshift-cluster-node-tuning-operator] adding namespace I0406 02:56:34.445301 3776 namespace.go:102] [openshift-cloud-network-config-controller] adding namespace took 6.420128ms I0406 02:56:34.445319 3776 namespace.go:98] [openshift-network-node-identity] adding namespace I0406 02:56:34.445724 3776 namespace.go:102] [openshift-operators-redhat] adding namespace took 6.398378ms I0406 02:56:34.445734 3776 namespace.go:98] [openshift-user-workload-monitoring] adding namespace I0406 02:56:34.446151 3776 namespace.go:102] [openshift-host-network] adding namespace took 6.452287ms I0406 02:56:34.446169 3776 namespace.go:98] [openshift-service-ca] adding namespace I0406 02:56:34.446586 3776 namespace.go:102] [open-cluster-management-agent-addon] adding namespace took 6.000488ms I0406 02:56:34.446600 3776 namespace.go:98] [openshift-service-ca-operator] adding namespace I0406 02:56:34.446961 3776 namespace.go:102] [openshift-kube-storage-version-migrator-operator] adding namespace took 5.885152ms I0406 02:56:34.446973 3776 namespace.go:98] [openshift-cloud-credential-operator] adding namespace I0406 02:56:34.447376 3776 namespace.go:102] [openshift-backplane-cse] adding namespace took 5.862666ms I0406 02:56:34.447388 3776 namespace.go:98] [openshift-kube-scheduler-operator] adding namespace I0406 02:56:34.447781 3776 namespace.go:102] [openshift-cluster-machine-approver] adding namespace took 5.854292ms I0406 02:56:34.447793 3776 namespace.go:98] [openshift-kube-apiserver-operator] adding namespace I0406 02:56:34.448238 3776 namespace.go:102] [openshift-etcd] adding namespace took 5.879772ms I0406 02:56:34.448660 3776 namespace.go:102] [kube-node-lease] adding namespace took 5.895627ms I0406 02:56:34.448672 3776 namespace.go:98] [openshift-image-registry] adding namespace I0406 02:56:34.449049 3776 namespace.go:102] [openshift-customer-monitoring] adding namespace took 5.808913ms I0406 02:56:34.449058 3776 namespace.go:98] [openshift-cluster-storage-operator] adding namespace I0406 02:56:34.449502 3776 namespace.go:102] [kube-public] adding namespace took 5.884963ms I0406 02:56:34.449512 3776 namespace.go:98] [openshift-multus] adding namespace I0406 02:56:34.449921 3776 namespace.go:102] [openshift-network-console] adding namespace took 5.913689ms I0406 02:56:34.449938 3776 namespace.go:98] [openshift-backplane-mobb] adding namespace I0406 02:56:34.450324 3776 namespace.go:102] [openshift-backplane-srep] adding namespace took 5.845521ms I0406 02:56:34.450339 3776 namespace.go:98] [openshift-console-operator] adding namespace I0406 02:56:34.450737 3776 namespace.go:102] [openshift-cluster-node-tuning-operator] adding namespace took 5.850151ms I0406 02:56:34.451150 3776 namespace.go:102] [openshift-network-node-identity] adding namespace took 5.824204ms I0406 02:56:34.451660 3776 namespace.go:102] [openshift-user-workload-monitoring] adding namespace took 5.920544ms I0406 02:56:34.451673 3776 namespace.go:98] [default] adding namespace I0406 02:56:34.452241 3776 namespace.go:102] [openshift-service-ca] adding namespace took 6.066766ms I0406 02:56:34.452256 3776 namespace.go:98] [openshift-authentication] adding namespace I0406 02:56:34.452846 3776 namespace.go:102] [openshift-service-ca-operator] adding namespace took 6.237584ms I0406 02:56:34.453341 3776 namespace.go:102] [openshift-cloud-credential-operator] adding namespace took 6.362325ms I0406 02:56:34.453924 3776 namespace.go:102] [openshift-kube-scheduler-operator] adding namespace took 6.528856ms I0406 02:56:34.454498 3776 namespace.go:102] [openshift-kube-apiserver-operator] adding namespace took 6.697128ms I0406 02:56:34.455099 3776 namespace.go:102] [openshift-image-registry] adding namespace took 6.420904ms I0406 02:56:34.455116 3776 namespace.go:98] [openshift] adding namespace W0406 02:56:34.455677 3776 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-multus/network-metrics-daemon-h8vhx: pod openshift-multus/network-metrics-daemon-h8vhx: no pod IPs found W0406 02:56:34.455709 3776 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-multus/network-metrics-daemon-tg4jk: pod openshift-multus/network-metrics-daemon-tg4jk: no pod IPs found I0406 02:56:34.455712 3776 namespace.go:102] [openshift-cluster-storage-operator] adding namespace took 6.648181ms I0406 02:56:34.455725 3776 namespace.go:98] [openshift-monitoring] adding namespace I0406 02:56:34.456341 3776 namespace.go:102] [openshift-multus] adding namespace took 6.824249ms I0406 02:56:34.456356 3776 namespace.go:98] [openshift-logging] adding namespace I0406 02:56:34.456769 3776 namespace.go:102] [openshift-backplane-mobb] adding namespace took 6.823766ms I0406 02:56:34.456784 3776 namespace.go:98] [open-cluster-management-2pgcu9mfas1m46ulnh3binnrqpenib9u] adding namespace I0406 02:56:34.457203 3776 namespace.go:102] [openshift-console-operator] adding namespace took 6.856898ms I0406 02:56:34.457219 3776 namespace.go:98] [openshift-ovn-kubernetes] adding namespace I0406 02:56:34.457755 3776 namespace.go:102] [default] adding namespace took 6.07626ms I0406 02:56:34.458320 3776 namespace.go:102] [openshift-authentication] adding namespace took 6.056034ms I0406 02:56:34.458960 3776 namespace.go:102] [openshift] adding namespace took 3.829846ms I0406 02:56:34.459500 3776 namespace.go:102] [openshift-monitoring] adding namespace took 3.769844ms I0406 02:56:34.460035 3776 namespace.go:102] [openshift-logging] adding namespace took 3.673678ms I0406 02:56:34.460542 3776 namespace.go:102] [open-cluster-management-2pgcu9mfas1m46ulnh3binnrqpenib9u] adding namespace took 3.752899ms I0406 02:56:34.460936 3776 namespace.go:102] [openshift-ovn-kubernetes] adding namespace took 3.712276ms I0406 02:56:34.461424 3776 zone_ic_handler.go:186] Time taken to create transit switch: 374.69µs I0406 02:56:34.461587 3776 master.go:532] Adding or Updating local node "ip-10-0-2-216.ec2.internal" for network "default" I0406 02:56:34.461641 3776 obj_retry.go:544] Failed to create *v1.Node ip-10-0-1-4.ec2.internal, error: adding or updating remote node chassis ip-10-0-1-4.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-1-4.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-1-4.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-4.ec2.internal I0406 02:56:34.462197 3776 zone_ic_handler.go:260] Creating interconnect resources for remote zone node ip-10-0-0-56.ec2.internal for the network default I0406 02:56:34.462906 3776 default_node_network_controller.go:518] Node connection status = connected I0406 02:56:34.463637 3776 zone_ic_handler.go:265] Creating Interconnect resources for node "ip-10-0-0-56.ec2.internal" on network "default" took: 1.415482ms W0406 02:56:34.472825 3776 gateway.go:197] Unable to fetch podIPs for pod openshift-network-diagnostics/network-check-target-nc244: pod openshift-network-diagnostics/network-check-target-nc244: no pod IPs found W0406 02:56:34.472847 3776 gateway.go:197] Unable to fetch podIPs for pod openshift-multus/network-metrics-daemon-h8vhx: pod openshift-multus/network-metrics-daemon-h8vhx: no pod IPs found W0406 02:56:34.472853 3776 gateway.go:197] Unable to fetch podIPs for pod kube-system/global-pull-secret-syncer-gzf4p: pod kube-system/global-pull-secret-syncer-gzf4p: no pod IPs found I0406 02:56:34.474065 3776 zone_ic_handler.go:194] Creating interconnect resources for local zone node ip-10-0-2-216.ec2.internal for the network default I0406 02:56:34.475035 3776 services_controller.go:192] Starting controller ovn-lb-controller for network=default I0406 02:56:34.475062 3776 services_controller.go:200] Waiting for node tracker handler to sync for network=default I0406 02:56:34.475076 3776 shared_informer.go:349] "Waiting for caches to sync" controller="node-tracker-controller" I0406 02:56:34.475103 3776 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-56.ec2.internal in network "default" I0406 02:56:34.475186 3776 node_tracker.go:169] Node ip-10-0-0-56.ec2.internal switch + router changed, syncing services in network "default" I0406 02:56:34.475199 3776 services_controller.go:574] Full service sync requested for network=default I0406 02:56:34.475210 3776 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-4.ec2.internal in network "default" I0406 02:56:34.475223 3776 node_tracker.go:235] Node ip-10-0-1-4.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-4.ec2.internal" I0406 02:56:34.475234 3776 node_tracker.go:169] Node ip-10-0-1-4.ec2.internal switch + router changed, syncing services in network "default" I0406 02:56:34.475240 3776 services_controller.go:574] Full service sync requested for network=default I0406 02:56:34.475247 3776 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-216.ec2.internal in network "default" I0406 02:56:34.475276 3776 node_tracker.go:169] Node ip-10-0-2-216.ec2.internal switch + router changed, syncing services in network "default" I0406 02:56:34.475284 3776 services_controller.go:574] Full service sync requested for network=default I0406 02:56:34.575912 3776 shared_informer.go:356] "Caches are synced" controller="node-tracker-controller" I0406 02:56:34.575939 3776 services_controller.go:208] Setting up event handlers for services for network=default I0406 02:56:34.576017 3776 services_controller.go:218] Setting up event handlers for endpoint slices for network=default I0406 02:56:34.576063 3776 services_controller.go:233] Waiting for service and endpoint handlers to sync for network=default I0406 02:56:34.576081 3776 shared_informer.go:349] "Waiting for caches to sync" controller="ovn-lb-controller" I0406 02:56:34.676547 3776 shared_informer.go:356] "Caches are synced" controller="ovn-lb-controller" I0406 02:56:34.676574 3776 repair.go:60] Starting repairing loop for services I0406 02:56:34.676665 3776 repair.go:131] Deleted 0 stale service LBs I0406 02:56:34.676684 3776 repair.go:137] Deleted 0 stale Chassis Template Vars I0406 02:56:34.676700 3776 repair.go:62] Finished repairing loop for services: 128.78µs I0406 02:56:34.676724 3776 services_controller.go:363] Controller cache of 0 load balancers initialized for 0 services for network=default I0406 02:56:34.676731 3776 services_controller.go:254] Starting workers for network=default W0406 02:56:34.676876 3776 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0406 02:56:34.677257 3776 obj_retry.go:464] Detected object kube-system/ecr-credential-refresh-initial-zkdct of type *v1.Pod in terminal state (e.g. completed) during add event: will remove it I0406 02:56:34.677321 3776 base_network_controller_pods.go:487] [default/kube-system/global-pull-secret-syncer-gzf4p] creating logical port kube-system_global-pull-secret-syncer-gzf4p for pod on switch ip-10-0-2-216.ec2.internal I0406 02:56:34.677344 3776 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-h8vhx] creating logical port openshift-multus_network-metrics-daemon-h8vhx for pod on switch ip-10-0-2-216.ec2.internal I0406 02:56:34.677429 3776 obj_retry.go:544] Failed to create *v1.Pod openshift-network-diagnostics/network-check-target-9jhwf, error: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-9jhwf: suppressed error logged: pod openshift-network-diagnostics/network-check-target-9jhwf: no pod IPs found I0406 02:56:34.677456 3776 kube.go:256] Updating pod kube-system/global-pull-secret-syncer-gzf4p I0406 02:56:34.677491 3776 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-nc244] creating logical port openshift-network-diagnostics_network-check-target-nc244 for pod on switch ip-10-0-2-216.ec2.internal I0406 02:56:34.677607 3776 kube.go:256] Updating pod openshift-multus/network-metrics-daemon-h8vhx I0406 02:56:34.677649 3776 kube.go:256] Updating pod openshift-network-diagnostics/network-check-target-nc244 W0406 02:56:34.682210 3776 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0406 02:56:34.682389 3776 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0406 02:56:34.692633 3776 pod.go:62] [kube-system/global-pull-secret-syncer-gzf4p] pod update took 15.225811ms I0406 02:56:34.692657 3776 base_network_controller_pods.go:937] [default/kube-system/global-pull-secret-syncer-gzf4p] addLogicalPort annotation time took 15.256043ms I0406 02:56:34.693627 3776 pods.go:271] [kube-system/global-pull-secret-syncer-gzf4p] addLogicalPort took 16.313953ms, libovsdb time 767.402µs I0406 02:56:34.693694 3776 obj_retry.go:544] Failed to create *v1.Pod openshift-multus/network-metrics-daemon-tg4jk, error: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-tg4jk: suppressed error logged: pod openshift-multus/network-metrics-daemon-tg4jk: no pod IPs found I0406 02:56:34.693824 3776 pod.go:62] [openshift-multus/network-metrics-daemon-h8vhx] pod update took 16.427574ms I0406 02:56:34.693843 3776 base_network_controller_pods.go:937] [default/openshift-multus/network-metrics-daemon-h8vhx] addLogicalPort annotation time took 16.448895ms I0406 02:56:34.694267 3776 pod.go:62] [openshift-network-diagnostics/network-check-target-nc244] pod update took 16.664206ms I0406 02:56:34.694284 3776 base_network_controller_pods.go:937] [default/openshift-network-diagnostics/network-check-target-nc244] addLogicalPort annotation time took 16.711915ms I0406 02:56:34.695109 3776 pods.go:271] [openshift-multus/network-metrics-daemon-h8vhx] addLogicalPort took 17.798676ms, libovsdb time 1.050758ms I0406 02:56:34.695373 3776 pods.go:271] [openshift-network-diagnostics/network-check-target-nc244] addLogicalPort took 17.93625ms, libovsdb time 667.523µs I0406 02:56:34.695957 3776 admin_network_policy_controller.go:227] Starting controller default-network-controller I0406 02:56:34.695985 3776 shared_informer.go:349] "Waiting for caches to sync" controller="default-network-controller" I0406 02:56:34.696003 3776 shared_informer.go:356] "Caches are synced" controller="default-network-controller" I0406 02:56:34.696017 3776 admin_network_policy_controller.go:237] Repairing Admin Network Policies I0406 02:56:34.696229 3776 repair.go:29] Repairing admin network policies took 206.333µs I0406 02:56:34.696330 3776 repair.go:92] Repairing baseline admin network policies took 92.308µs I0406 02:56:34.696758 3776 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/default-deny for network default I0406 02:56:34.696772 3776 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/default-deny added to peer address sets [] I0406 02:56:34.698194 3776 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/default-deny resources completed, update namespace loglevel I0406 02:56:34.698214 3776 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-version/default-deny for network default I0406 02:56:34.698222 3776 base_network_controller_policy.go:1054] Policy openshift-cluster-version/default-deny added to peer address sets [] I0406 02:56:34.699468 3776 base_network_controller_policy.go:1232] Create network policy openshift-cluster-version/default-deny resources completed, update namespace loglevel I0406 02:56:34.699486 3776 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-insights for network default I0406 02:56:34.699504 3776 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-insights added to peer address sets [] I0406 02:56:34.701772 3776 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-insights resources completed, update namespace loglevel I0406 02:56:34.701795 3776 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-egress-to-api-server for network default I0406 02:56:34.701804 3776 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-egress-to-api-server added to peer address sets [] I0406 02:56:34.703607 3776 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-egress-to-api-server resources completed, update namespace loglevel I0406 02:56:34.703631 3776 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-to-dns for network default I0406 02:56:34.704537 3776 pod_selector_address_set.go:212] Created shared address set for pod selector LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},} I0406 02:56:34.704552 3776 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0406 02:56:34.705342 3776 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-to-dns resources completed, update namespace loglevel I0406 02:56:34.705362 3776 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/default-deny-all for network default I0406 02:56:34.705370 3776 base_network_controller_policy.go:1054] Policy openshift-marketplace/default-deny-all added to peer address sets [] I0406 02:56:34.706961 3776 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/default-deny-all resources completed, update namespace loglevel I0406 02:56:34.706980 3776 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/marketplace-operator for network default I0406 02:56:34.706991 3776 base_network_controller_policy.go:1054] Policy openshift-marketplace/marketplace-operator added to peer address sets [] I0406 02:56:34.707950 3776 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/marketplace-operator resources completed, update namespace loglevel I0406 02:56:34.707969 3776 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/default-deny-all-traffic for network default I0406 02:56:34.707976 3776 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/default-deny-all-traffic added to peer address sets [] I0406 02:56:34.709208 3776 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/default-deny-all-traffic resources completed, update namespace loglevel I0406 02:56:34.709230 3776 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/olm-operator for network default I0406 02:56:34.709242 3776 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/olm-operator added to peer address sets [] I0406 02:56:34.710194 3776 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/olm-operator resources completed, update namespace loglevel I0406 02:56:34.710212 3776 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/package-server-manager for network default I0406 02:56:34.710237 3776 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/package-server-manager added to peer address sets [] I0406 02:56:34.711259 3776 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/package-server-manager resources completed, update namespace loglevel I0406 02:56:34.711280 3776 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-egress for network default I0406 02:56:34.711290 3776 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-egress added to peer address sets [] I0406 02:56:34.712946 3776 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-egress resources completed, update namespace loglevel I0406 02:56:34.712966 3776 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-webhook for network default I0406 02:56:34.712974 3776 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-webhook added to peer address sets [] I0406 02:56:34.713464 3776 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-webhook resources completed, update namespace loglevel I0406 02:56:34.713484 3776 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range for network default I0406 02:56:34.713492 3776 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range added to peer address sets [] I0406 02:56:34.713934 3776 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range resources completed, update namespace loglevel I0406 02:56:34.713952 3776 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-egress-to-api-server for network default I0406 02:56:34.713962 3776 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-egress-to-api-server added to peer address sets [] I0406 02:56:34.714492 3776 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-egress-to-api-server resources completed, update namespace loglevel I0406 02:56:34.714508 3776 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics for network default I0406 02:56:34.714518 3776 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics added to peer address sets [] I0406 02:56:34.715003 3776 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics resources completed, update namespace loglevel I0406 02:56:34.715019 3776 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-to-dns for network default I0406 02:56:34.715040 3776 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0406 02:56:34.715759 3776 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-to-dns resources completed, update namespace loglevel I0406 02:56:34.715778 3776 base_network_controller_policy.go:1179] Adding network policy openshift-operators/default-allow-all for network default I0406 02:56:34.715797 3776 base_network_controller_policy.go:1054] Policy openshift-operators/default-allow-all added to peer address sets [] I0406 02:56:34.717609 3776 base_network_controller_policy.go:1232] Create network policy openshift-operators/default-allow-all resources completed, update namespace loglevel I0406 02:56:34.717645 3776 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server for network default I0406 02:56:34.717663 3776 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server added to peer address sets [] I0406 02:56:34.718229 3776 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server resources completed, update namespace loglevel I0406 02:56:34.718248 3776 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-metrics for network default I0406 02:56:34.718256 3776 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-metrics added to peer address sets [] I0406 02:56:34.718761 3776 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-metrics resources completed, update namespace loglevel I0406 02:56:34.718779 3776 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/default-deny for network default I0406 02:56:34.718785 3776 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/default-deny added to peer address sets [] I0406 02:56:34.719084 3776 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/default-deny resources completed, update namespace loglevel I0406 02:56:34.719110 3776 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/unpack-bundles for network default I0406 02:56:34.719134 3776 base_network_controller_policy.go:1054] Policy openshift-marketplace/unpack-bundles added to peer address sets [] I0406 02:56:34.720016 3776 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/unpack-bundles resources completed, update namespace loglevel I0406 02:56:34.720043 3776 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/catalog-operator for network default I0406 02:56:34.720061 3776 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/catalog-operator added to peer address sets [] I0406 02:56:34.721287 3776 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/catalog-operator resources completed, update namespace loglevel I0406 02:56:34.721306 3776 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/packageserver for network default I0406 02:56:34.721321 3776 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/packageserver added to peer address sets [] I0406 02:56:34.722540 3776 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/packageserver resources completed, update namespace loglevel I0406 02:56:34.722559 3776 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-all-egress for network default I0406 02:56:34.722567 3776 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-all-egress added to peer address sets [] I0406 02:56:34.723069 3776 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-all-egress resources completed, update namespace loglevel I0406 02:56:34.723088 3776 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles for network default I0406 02:56:34.723103 3776 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles added to peer address sets [] I0406 02:56:34.724039 3776 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles resources completed, update namespace loglevel I0406 02:56:34.724058 3776 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range for network default I0406 02:56:34.724065 3776 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range added to peer address sets [] I0406 02:56:34.724547 3776 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range resources completed, update namespace loglevel I0406 02:56:34.724567 3776 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-all-egress for network default I0406 02:56:34.724575 3776 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-all-egress added to peer address sets [] I0406 02:56:34.725037 3776 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-all-egress resources completed, update namespace loglevel I0406 02:56:34.725056 3776 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-olm for network default I0406 02:56:34.725063 3776 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-olm added to peer address sets [] I0406 02:56:34.725566 3776 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-olm resources completed, update namespace loglevel I0406 02:56:34.725584 3776 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-pprof for network default I0406 02:56:34.725599 3776 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-pprof added to peer address sets [] I0406 02:56:34.726067 3776 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-pprof resources completed, update namespace loglevel I0406 02:56:34.726379 3776 obj_retry.go:464] Detected object kube-system/ecr-credential-refresh-initial-zkdct of type *factory.egressIPPod in terminal state (e.g. completed) during add event: will remove it I0406 02:56:34.732503 3776 egressfirewall.go:319] Starting EgressFirewall controller I0406 02:56:34.732539 3776 controller.go:133] Adding controller egress-firewall-controller-node event handlers I0406 02:56:34.732569 3776 controller.go:133] Adding controller egress-firewall-controller event handlers I0406 02:56:34.732568 3776 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller-node" I0406 02:56:34.732589 3776 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller-node" I0406 02:56:34.732619 3776 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller" I0406 02:56:34.732632 3776 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller" I0406 02:56:34.732909 3776 controller.go:157] Starting controller egress-firewall-controller with 1 workers I0406 02:56:34.732926 3776 controller.go:157] Starting controller egress-firewall-controller-node with 1 workers I0406 02:56:34.732940 3776 egressqos.go:193] Setting up event handlers for EgressQoS I0406 02:56:34.733023 3776 egressfirewall.go:628] Syncing node "ip-10-0-0-56.ec2.internal" for egress firewall I0406 02:56:34.733039 3776 egressfirewall.go:628] Syncing node "ip-10-0-1-4.ec2.internal" for egress firewall I0406 02:56:34.733044 3776 egressfirewall.go:628] Syncing node "ip-10-0-2-216.ec2.internal" for egress firewall I0406 02:56:34.733102 3776 egressqos.go:245] Starting EgressQoS Controller I0406 02:56:34.733113 3776 shared_informer.go:349] "Waiting for caches to sync" controller="egressqosnodes" I0406 02:56:34.733120 3776 shared_informer.go:356] "Caches are synced" controller="egressqosnodes" I0406 02:56:34.733145 3776 shared_informer.go:349] "Waiting for caches to sync" controller="egressqospods" I0406 02:56:34.733151 3776 shared_informer.go:356] "Caches are synced" controller="egressqospods" I0406 02:56:34.733156 3776 shared_informer.go:349] "Waiting for caches to sync" controller="egressqos" I0406 02:56:34.733160 3776 shared_informer.go:356] "Caches are synced" controller="egressqos" I0406 02:56:34.733164 3776 egressqos.go:259] Repairing EgressQoSes I0406 02:56:34.733168 3776 egressqos.go:399] Starting repairing loop for egressqos I0406 02:56:34.733258 3776 egressqos.go:401] Finished repairing loop for egressqos: 89.377µs I0406 02:56:34.733309 3776 egressservice_zone.go:132] Setting up event handlers for Egress Services I0406 02:56:34.733363 3776 egressqos.go:1007] Processing sync for EgressQoS node ip-10-0-2-216.ec2.internal I0406 02:56:34.733409 3776 egressqos.go:1010] Finished syncing EgressQoS node ip-10-0-2-216.ec2.internal : 46.658µs I0406 02:56:34.733520 3776 egressservice_zone.go:210] Starting Egress Services Controller I0406 02:56:34.733532 3776 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0406 02:56:34.733541 3776 shared_informer.go:356] "Caches are synced" controller="egressservices" I0406 02:56:34.733547 3776 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0406 02:56:34.733551 3776 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0406 02:56:34.733605 3776 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0406 02:56:34.733616 3776 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0406 02:56:34.733623 3776 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_nodes" I0406 02:56:34.733628 3776 shared_informer.go:356] "Caches are synced" controller="egressservices_nodes" I0406 02:56:34.733631 3776 egressservice_zone.go:228] Repairing Egress Services I0406 02:56:34.733937 3776 master_controller.go:88] Starting Admin Policy Based Route Controller I0406 02:56:34.733947 3776 external_controller.go:278] Starting Admin Policy Based Route Controller I0406 02:56:34.733960 3776 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-56.ec2.internal I0406 02:56:34.733980 3776 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-56.ec2.internal: 22.003µs I0406 02:56:34.733987 3776 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-4.ec2.internal I0406 02:56:34.733991 3776 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-4.ec2.internal: 4.178µs I0406 02:56:34.733995 3776 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-216.ec2.internal I0406 02:56:34.734000 3776 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-216.ec2.internal: 3.848µs I0406 02:56:34.734088 3776 default_network_controller.go:568] Completing all the Watchers took 313.015009ms I0406 02:56:34.734101 3776 default_network_controller.go:572] Starting unidling controllers I0406 02:56:34.734107 3776 unidle.go:46] Registering OVN SB ControllerEvent handler I0406 02:56:34.734112 3776 unidle.go:63] Populating Initial ContollerEvent events I0406 02:56:34.734210 3776 unidle.go:79] Setting up event handlers for services I0406 02:56:34.734281 3776 ovnkube.go:544] Waiting for OVN northbound database changes to be processed by ovn-controller I0406 02:56:34.963674 3776 default_node_network_controller.go:518] Node connection status = connected I0406 02:56:35.462828 3776 default_node_network_controller.go:518] Node connection status = connected I0406 02:56:35.962875 3776 default_node_network_controller.go:518] Node connection status = connected I0406 02:56:36.463650 3776 default_node_network_controller.go:518] Node connection status = connected I0406 02:56:36.490347 3776 bridgeconfig.go:537] Gateway is ready I0406 02:56:36.490369 3776 gateway_shared_intf.go:1704] Creating Gateway Openflow Manager I0406 02:56:36.504815 3776 node_ip_handler_linux.go:247] Node primary address changed to 10.0.2.216. Updating OVN encap IP. I0406 02:56:36.505150 3776 ovnkube.go:549] Finished waiting for OVN northbound database changes to be processed by ovn-controller I0406 02:56:36.510869 3776 node_ip_handler_linux.go:542] Will not update encap IP 10.0.2.216 - it is already configured I0406 02:56:36.510885 3776 node_ip_handler_linux.go:485] Node address changed to map[10.0.2.216/24:{} 172.20.0.1/32:{}]. Updating annotations. I0406 02:56:36.511108 3776 kube.go:131] Setting annotations map[k8s.ovn.org/host-cidrs:["10.0.2.216/24","172.20.0.1/32"] k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-2-216.ec2.internal","mac-address":"0e:3e:78:df:3c:53","ip-addresses":["10.0.2.216/24"],"ip-address":"10.0.2.216/24","next-hops":["10.0.2.1"],"next-hop":"10.0.2.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:97cd1d65-bc3b-4947-b8e4-edc37074663b k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.2.216/24"}] on node ip-10-0-2-216.ec2.internal I0406 02:56:36.537795 3776 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-216.ec2.internal I0406 02:56:36.537822 3776 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-216.ec2.internal: 33.302µs I0406 02:56:36.537795 3776 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-216.ec2.internal in network "default" I0406 02:56:36.537867 3776 egressfirewall.go:628] Syncing node "ip-10-0-2-216.ec2.internal" for egress firewall I0406 02:56:36.537901 3776 node_tracker.go:169] Node ip-10-0-2-216.ec2.internal switch + router changed, syncing services in network "default" I0406 02:56:36.537921 3776 services_controller.go:574] Full service sync requested for network=default I0406 02:56:36.537807 3776 master.go:532] Adding or Updating local node "ip-10-0-2-216.ec2.internal" for network "default" W0406 02:56:36.538154 3776 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog W0406 02:56:36.538235 3776 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics W0406 02:56:36.538506 3776 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0406 02:56:36.538993 3776 gateway_shared_intf.go:1994] Setting OVN Masquerade route with source: 10.0.2.216 I0406 02:56:36.539068 3776 kube.go:131] Setting annotations map[k8s.ovn.org/node-masquerade-subnet:{"ipv4":"169.254.0.0/17","ipv6":"fd69::/112"}] on node ip-10-0-2-216.ec2.internal I0406 02:56:36.543144 3776 default_network_controller.go:1067] Egress IP detected IP address change for node ip-10-0-2-216.ec2.internal. Updating no re-route policies I0406 02:56:36.543176 3776 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/konnectivity-agent-b5fjf I0406 02:56:36.543193 3776 obj_retry.go:364] Adding new object: *v1.Pod kube-system/konnectivity-agent-b5fjf I0406 02:56:36.543232 3776 obj_retry.go:300] Retry object setup: *v1.Pod openshift-image-registry/node-ca-2zxwv I0406 02:56:36.543244 3776 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-9jhwf I0406 02:56:36.543250 3776 obj_retry.go:364] Adding new object: *v1.Pod openshift-image-registry/node-ca-2zxwv I0406 02:56:36.543247 3776 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/multus-additional-cni-plugins-sc6q6 I0406 02:56:36.543258 3776 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-h8vhx I0406 02:56:36.543266 3776 obj_retry.go:300] Retry object setup: *v1.Pod openshift-cluster-node-tuning-operator/tuned-l8jh6 I0406 02:56:36.543273 3776 obj_retry.go:384] Retry successful for *v1.Pod openshift-image-registry/node-ca-2zxwv after 0 failed attempt(s) I0406 02:56:36.543276 3776 obj_retry.go:364] Adding new object: *v1.Pod openshift-cluster-node-tuning-operator/tuned-l8jh6 I0406 02:56:36.543275 3776 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-h8vhx I0406 02:56:36.543278 3776 obj_retry.go:300] Retry object setup: *v1.Pod openshift-dns/node-resolver-q4rct I0406 02:56:36.543237 3776 obj_retry.go:384] Retry successful for *v1.Pod kube-system/konnectivity-agent-b5fjf after 0 failed attempt(s) I0406 02:56:36.543286 3776 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-nc244 I0406 02:56:36.543287 3776 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-216.ec2.internal I0406 02:56:36.543299 3776 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-tg4jk I0406 02:56:36.543296 3776 obj_retry.go:300] Retry object setup: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-216.ec2.internal I0406 02:56:36.543308 3776 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-tg4jk I0406 02:56:36.543309 3776 obj_retry.go:364] Adding new object: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-216.ec2.internal I0406 02:56:36.543312 3776 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/multus-plgr9 I0406 02:56:36.543314 3776 obj_retry.go:364] Adding new object: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-216.ec2.internal I0406 02:56:36.543322 3776 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/multus-plgr9 I0406 02:56:36.543324 3776 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-operator/iptables-alerter-khrzs I0406 02:56:36.543336 3776 obj_retry.go:384] Retry successful for *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-216.ec2.internal after 0 failed attempt(s) I0406 02:56:36.543334 3776 obj_retry.go:384] Retry successful for *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-216.ec2.internal after 0 failed attempt(s) I0406 02:56:36.543336 3776 obj_retry.go:375] Retry add failed for *v1.Pod openshift-multus/network-metrics-daemon-tg4jk, will try again later: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-tg4jk: suppressed error logged: pod openshift-multus/network-metrics-daemon-tg4jk: no pod IPs found I0406 02:56:36.543292 3776 obj_retry.go:384] Retry successful for *v1.Pod openshift-cluster-node-tuning-operator/tuned-l8jh6 after 0 failed attempt(s) I0406 02:56:36.543346 3776 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/multus-plgr9 after 0 failed attempt(s) I0406 02:56:36.543272 3776 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/global-pull-secret-syncer-gzf4p I0406 02:56:36.543340 3776 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-h8vhx] creating logical port openshift-multus_network-metrics-daemon-h8vhx for pod on switch ip-10-0-2-216.ec2.internal I0406 02:56:36.543348 3776 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-operator/iptables-alerter-khrzs I0406 02:56:36.543266 3776 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/multus-additional-cni-plugins-sc6q6 I0406 02:56:36.543378 3776 obj_retry.go:384] Retry successful for *v1.Pod openshift-network-operator/iptables-alerter-khrzs after 0 failed attempt(s) I0406 02:56:36.543289 3776 obj_retry.go:364] Adding new object: *v1.Pod openshift-dns/node-resolver-q4rct I0406 02:56:36.543404 3776 obj_retry.go:384] Retry successful for *v1.Pod openshift-dns/node-resolver-q4rct after 0 failed attempt(s) I0406 02:56:36.543302 3776 obj_retry.go:300] Retry object setup: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-xgcrf I0406 02:56:36.543425 3776 obj_retry.go:364] Adding new object: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-xgcrf I0406 02:56:36.543444 3776 obj_retry.go:384] Retry successful for *v1.Pod openshift-ovn-kubernetes/ovnkube-node-xgcrf after 0 failed attempt(s) I0406 02:56:36.543445 3776 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/multus-additional-cni-plugins-sc6q6 after 0 failed attempt(s) I0406 02:56:36.543303 3776 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-nc244 I0406 02:56:36.543255 3776 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-9jhwf I0406 02:56:36.543482 3776 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-nc244] creating logical port openshift-network-diagnostics_network-check-target-nc244 for pod on switch ip-10-0-2-216.ec2.internal I0406 02:56:36.543299 3776 obj_retry.go:300] Retry object setup: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-qml4n I0406 02:56:36.543494 3776 obj_retry.go:375] Retry add failed for *v1.Pod openshift-network-diagnostics/network-check-target-9jhwf, will try again later: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-9jhwf: suppressed error logged: pod openshift-network-diagnostics/network-check-target-9jhwf: no pod IPs found I0406 02:56:36.543498 3776 obj_retry.go:364] Adding new object: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-qml4n I0406 02:56:36.543358 3776 obj_retry.go:364] Adding new object: *v1.Pod kube-system/global-pull-secret-syncer-gzf4p I0406 02:56:36.543512 3776 obj_retry.go:384] Retry successful for *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-qml4n after 0 failed attempt(s) I0406 02:56:36.543559 3776 base_network_controller_pods.go:487] [default/kube-system/global-pull-secret-syncer-gzf4p] creating logical port kube-system_global-pull-secret-syncer-gzf4p for pod on switch ip-10-0-2-216.ec2.internal I0406 02:56:36.544061 3776 pods.go:271] [openshift-multus/network-metrics-daemon-h8vhx] addLogicalPort took 752.52µs, libovsdb time 262.548µs I0406 02:56:36.544075 3776 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/network-metrics-daemon-h8vhx after 0 failed attempt(s) I0406 02:56:36.544377 3776 pods.go:271] [openshift-network-diagnostics/network-check-target-nc244] addLogicalPort took 901.408µs, libovsdb time 342.813µs I0406 02:56:36.544395 3776 obj_retry.go:384] Retry successful for *v1.Pod openshift-network-diagnostics/network-check-target-nc244 after 0 failed attempt(s) I0406 02:56:36.544435 3776 pods.go:271] [kube-system/global-pull-secret-syncer-gzf4p] addLogicalPort took 911.045µs, libovsdb time 268.023µs I0406 02:56:36.544445 3776 obj_retry.go:384] Retry successful for *v1.Pod kube-system/global-pull-secret-syncer-gzf4p after 0 failed attempt(s) I0406 02:56:36.568949 3776 gateway_shared_intf.go:1770] Creating Gateway Node Port Watcher I0406 02:56:36.582609 3776 iptables.go:108] Creating table: mangle chain: OUTPUT I0406 02:56:36.586238 3776 iptables.go:108] Creating table: nat chain: OUTPUT I0406 02:56:36.612272 3776 iptables.go:108] Creating table: nat chain: PREROUTING I0406 02:56:36.654861 3776 iptables.go:108] Creating table: filter chain: FORWARD I0406 02:56:36.673910 3776 gateway_shared_intf.go:2095] Ensuring IP Neighbor entry for: 169.254.0.1 W0406 02:56:36.674035 3776 gateway_shared_intf.go:2101] Failed to remove IP neighbor entry for ip 169.254.0.1, on iface br-ex: failed to delete neighbour entry 169.254.0.1 : no such file or directory I0406 02:56:36.674089 3776 gateway_shared_intf.go:2095] Ensuring IP Neighbor entry for: 169.254.0.4 W0406 02:56:36.674218 3776 gateway_shared_intf.go:2101] Failed to remove IP neighbor entry for ip 169.254.0.4, on iface br-ex: failed to delete neighbour entry 169.254.0.4 : no such file or directory I0406 02:56:36.674277 3776 gateway_init.go:325] Gateway and management port readiness took 2.216000048s I0406 02:56:36.674313 3776 controller.go:133] Adding controller [node-nad-controller NAD controller] event handlers I0406 02:56:36.674352 3776 shared_informer.go:349] "Waiting for caches to sync" controller="[node-nad-controller NAD controller]" I0406 02:56:36.674368 3776 shared_informer.go:356] "Caches are synced" controller="[node-nad-controller NAD controller]" I0406 02:56:36.674377 3776 nad_controller.go:241] [node-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0406 02:56:36.674455 3776 controller.go:157] Starting controller [node-nad-controller NAD controller] with 1 workers I0406 02:56:36.674535 3776 nad_controller.go:280] [node-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 49.533µs I0406 02:56:36.674845 3776 network_controller.go:246] [node-nad-controller network controller]: syncing all networks I0406 02:56:36.674873 3776 network_controller.go:275] [node-nad-controller network controller]: finished syncing network default, took 18.141µs I0406 02:56:36.674882 3776 network_controller.go:257] [node-nad-controller network controller]: finished syncing all networks. Time taken: 38.423µs I0406 02:56:36.674889 3776 controller.go:157] Starting controller [node-nad-controller network controller] with 1 workers I0406 02:56:36.674900 3776 nad_controller.go:173] [node-nad-controller NAD controller]: started I0406 02:56:36.674907 3776 default_node_network_controller.go:898] Starting the default node network controller I0406 02:56:36.674913 3776 gateway_init.go:335] Initializing Gateway Functionality for gateway Start I0406 02:56:36.675004 3776 network_controller.go:275] [node-nad-controller network controller]: finished syncing network default, took 20.615µs I0406 02:56:36.675093 3776 default_node_network_controller.go:1449] MTU (9001) of network interface br-ex is big enough to deal with Geneve header overhead (sum 8959). I0406 02:56:36.675105 3776 default_node_network_controller.go:934] Waiting for gateway and management port readiness... I0406 02:56:36.675150 3776 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc0009e1320 0xc0002d3420 0xc0002d3500 0xc0002d3570 0xc0002d35e0 0xc0002d3650 0xc0002d3730 0xc00072bb30 0xc0002d37a0 0xc0002d3810 0xc0008f03c0 0xc0063ff2d0 0xc0002d3880 map[0x2f8f0c0:0xc002e890e0 0x2f8f600:0xc002e89260 0x2f90080:0xc002e89020 0x2f90320:0xc002e89080 0x2f90b00:0xc002e89140 0x2f90da0:0xc002e891a0 0x2f91040:0xc002e89200 0x2f91820:0xc002e892c0 0x2f91ac0:0xc002e89320 0x2fbc3e0:0xc000ab8b40 0x2fbc780:0xc000ab8ba0 0x2fbd260:0xc000bdc300 0x2fbebc0:0xc001bf4000 0x2fbf6a0:0xc000ab8cc0 0x2fc0520:0xc000ab8ae0] 0xc00016aa80 0} I0406 02:56:36.675214 3776 gateway.go:151] Starting gateway service sync I0406 02:56:36.675693 3776 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ITP I0406 02:56:36.678155 3776 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-NODEPORT I0406 02:56:36.687531 3776 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-EXTERNALIP I0406 02:56:36.688759 3776 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ETP I0406 02:56:36.690139 3776 gateway_iptables.go:474] Recreating iptables rules for table: mangle, chain: OVN-KUBE-ITP I0406 02:56:36.727148 3776 gateway.go:168] Gateway service sync done. Time taken: 51.922587ms I0406 02:56:36.727712 3776 healthcheck.go:145] Opening healthcheck "openshift-ingress/router-default" on port 30647 I0406 02:56:36.727976 3776 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc0009e1320 0xc0002d3420 0xc0002d3500 0xc0002d3570 0xc0002d35e0 0xc0002d3650 0xc0002d3730 0xc00072bb30 0xc0002d37a0 0xc0002d3810 0xc0008f03c0 0xc0063ff2d0 0xc0002d3880 map[0x2f8f0c0:0xc002e890e0 0x2f8f600:0xc002e89260 0x2f90080:0xc002e89020 0x2f90320:0xc002e89080 0x2f90b00:0xc002e89140 0x2f90da0:0xc002e891a0 0x2f91040:0xc002e89200 0x2f91820:0xc002e892c0 0x2f91ac0:0xc002e89320 0x2fbc3e0:0xc000ab8b40 0x2fbc780:0xc000ab8ba0 0x2fbd260:0xc000bdc300 0x2fbebc0:0xc001bf4000 0x2fbf6a0:0xc000ab8cc0 0x2fc0520:0xc000ab8ae0] 0xc00016aa80 0} I0406 02:56:36.728508 3776 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc0009e1320 0xc0002d3420 0xc0002d3500 0xc0002d3570 0xc0002d35e0 0xc0002d3650 0xc0002d3730 0xc00072bb30 0xc0002d37a0 0xc0002d3810 0xc0008f03c0 0xc0063ff2d0 0xc0002d3880 map[0x2f8f0c0:0xc002e890e0 0x2f8f600:0xc002e89260 0x2f90080:0xc002e89020 0x2f90320:0xc002e89080 0x2f90b00:0xc002e89140 0x2f90da0:0xc002e891a0 0x2f91040:0xc002e89200 0x2f91820:0xc002e892c0 0x2f91ac0:0xc002e89320 0x2fbc3e0:0xc000ab8b40 0x2fbc780:0xc000ab8ba0 0x2fbd260:0xc000bdc300 0x2fbebc0:0xc001bf4000 0x2fbf6a0:0xc000ab8cc0 0x2fc0520:0xc000ab8ae0] 0xc00016aa80 0} I0406 02:56:36.728549 3776 gateway.go:519] Reconciling gateway with updates I0406 02:56:36.734429 3776 gateway.go:326] Spawning Conntrack Rule Check Thread I0406 02:56:36.734545 3776 default_node_network_controller.go:943] Gateway and management port readiness took 59.43057ms I0406 02:56:36.734550 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0406 02:56:36.734559 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0406 02:56:36.734569 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0406 02:56:36.734576 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0406 02:56:36.734586 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0406 02:56:36.734596 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0406 02:56:36.734601 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0406 02:56:36.734611 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0406 02:56:36.734613 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0406 02:56:36.734619 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0406 02:56:36.734630 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0406 02:56:36.734623 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0406 02:56:36.734636 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0406 02:56:36.734634 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0406 02:56:36.734646 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0406 02:56:36.734649 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0406 02:56:36.734657 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-insights/exporter I0406 02:56:36.734673 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-insights/exporter I0406 02:56:36.734684 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-insights/exporter after 0 failed attempt(s) I0406 02:56:36.734684 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0406 02:56:36.734689 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0406 02:56:36.734684 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0406 02:56:36.734705 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0406 02:56:36.734709 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0406 02:56:36.734702 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0406 02:56:36.734716 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0406 02:56:36.734720 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0406 02:56:36.734721 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0406 02:56:36.734726 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift I0406 02:56:36.734728 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0406 02:56:36.734733 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0406 02:56:36.734740 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift I0406 02:56:36.734744 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0406 02:56:36.734748 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0406 02:56:36.734751 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0406 02:56:36.734751 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0406 02:56:36.734756 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0406 02:56:36.734758 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0406 02:56:36.734759 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0406 02:56:36.734763 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0406 02:56:36.734770 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0406 02:56:36.734775 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0406 02:56:36.734776 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0406 02:56:36.734782 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0406 02:56:36.734787 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0406 02:56:36.734790 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0406 02:56:36.734789 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0406 02:56:36.734796 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0406 02:56:36.734799 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0406 02:56:36.734802 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0406 02:56:36.734801 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0406 02:56:36.734804 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0406 02:56:36.734813 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0406 02:56:36.734813 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0406 02:56:36.734816 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0406 02:56:36.734821 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0406 02:56:36.734706 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0406 02:56:36.734749 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0406 02:56:36.734832 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0406 02:56:36.734723 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0406 02:56:36.734803 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0406 02:56:36.734857 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0406 02:56:36.734875 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0406 02:56:36.734786 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0406 02:56:36.734885 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0406 02:56:36.734789 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0406 02:56:36.734547 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0406 02:56:36.734717 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/packageserver I0406 02:56:36.734841 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0406 02:56:36.734776 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/kubernetes I0406 02:56:36.734923 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0406 02:56:36.734937 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/kubernetes I0406 02:56:36.734943 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0406 02:56:36.734955 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0406 02:56:36.734707 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0406 02:56:36.734993 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0406 02:56:36.735008 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0406 02:56:36.734730 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0406 02:56:36.735017 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0406 02:56:36.735027 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0406 02:56:36.735006 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0406 02:56:36.735036 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0406 02:56:36.734943 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0406 02:56:36.734756 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0406 02:56:36.735066 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0406 02:56:36.734729 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0406 02:56:36.735103 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0406 02:56:36.735108 3776 port_claim.go:71] Svc openshift-ingress/router-default: port 32685 is already open, no action needed I0406 02:56:36.735115 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0406 02:56:36.734763 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0406 02:56:36.735139 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0406 02:56:36.735149 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0406 02:56:36.734771 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0406 02:56:36.735163 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0406 02:56:36.734790 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0406 02:56:36.735180 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0406 02:56:36.734792 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0406 02:56:36.735162 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0406 02:56:36.735212 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0406 02:56:36.735155 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0406 02:56:36.734826 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0406 02:56:36.734847 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0406 02:56:36.734911 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0406 02:56:36.734837 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0406 02:56:36.734708 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0406 02:56:36.735298 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0406 02:56:36.734929 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/packageserver I0406 02:56:36.735305 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0406 02:56:36.734927 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0406 02:56:36.735321 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0406 02:56:36.734724 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0406 02:56:36.734765 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0406 02:56:36.735350 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0406 02:56:36.735119 3776 port_claim.go:71] Svc openshift-ingress/router-default: port 32404 is already open, no action needed I0406 02:56:36.735751 3776 node_ip_handler_linux.go:160] Node IP manager is running W0406 02:56:36.778504 3776 port_linux.go:245] Missing or unable to find route entry for subnet 10.128.0.0/14 via gateway 10.129.0.1 on link ovn-k8s-mp0 with MTU: 0 W0406 02:56:36.778647 3776 port_linux.go:245] Missing or unable to find route entry for subnet 169.254.0.3/32 via gateway 10.129.0.1 on link ovn-k8s-mp0 with MTU: 0 W0406 02:56:36.778771 3776 port_linux.go:261] Missing arp entry for MAC/IP binding (0a:58:0a:81:00:01/10.129.0.1) on link ovn-k8s-mp0 I0406 02:56:36.787543 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0406 02:56:36.787573 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0406 02:56:36.787583 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0406 02:56:36.827442 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0406 02:56:36.827473 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0406 02:56:36.827482 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0406 02:56:36.827488 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0406 02:56:36.827494 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0406 02:56:36.827501 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0406 02:56:36.827508 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0406 02:56:36.827513 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0406 02:56:36.827518 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0406 02:56:36.827524 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0406 02:56:36.827530 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0406 02:56:36.827535 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0406 02:56:36.827540 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0406 02:56:36.827545 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0406 02:56:36.827551 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0406 02:56:36.827557 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0406 02:56:36.827563 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0406 02:56:36.827569 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0406 02:56:36.827590 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0406 02:56:36.866937 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0406 02:56:37.141847 3776 egressfirewall.go:628] Syncing node "ip-10-0-1-4.ec2.internal" for egress firewall I0406 02:56:37.141907 3776 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-4.ec2.internal I0406 02:56:37.141922 3776 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-4.ec2.internal: 14.559µs I0406 02:56:37.141943 3776 default_network_controller.go:1008] Node "ip-10-0-1-4.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0406 02:56:37.629534 3776 default_network_controller.go:1008] Node "ip-10-0-1-4.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0406 02:56:38.236506 3776 default_node_network_controller.go:1369] Starting node controller node sync W0406 02:56:38.236535 3776 egressip_healthcheck.go:76] Health checking using insecure connection I0406 02:56:38.236593 3776 egressip_healthcheck.go:109] Starting Egress IP Health Server on 10.129.0.2:9107 I0406 02:56:38.248753 3776 default_node_network_controller.go:1411] Node controller node sync done. Time taken: 12.221565ms I0406 02:56:38.248872 3776 default_node_network_controller.go:1282] Adding remote node "ip-10-0-0-56.ec2.internal", IP: 10.0.0.56 to PMTUD blocking rules I0406 02:56:38.248895 3776 default_node_network_controller.go:1282] Adding remote node "ip-10-0-0-56.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0406 02:56:38.255325 3776 healthcheck_node.go:124] "Starting node proxy healthz server" address="0.0.0.0:10256" I0406 02:56:38.255574 3776 egressservice_node.go:93] Setting up event handlers for Egress Services I0406 02:56:38.255694 3776 egressservice_node.go:183] Starting Egress Services Controller I0406 02:56:38.255711 3776 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0406 02:56:38.255722 3776 shared_informer.go:356] "Caches are synced" controller="egressservices" I0406 02:56:38.255728 3776 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0406 02:56:38.255733 3776 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0406 02:56:38.255739 3776 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0406 02:56:38.255743 3776 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0406 02:56:38.255747 3776 egressservice_node.go:197] Repairing Egress Services I0406 02:56:38.266530 3776 node_controller.go:43] Starting Admin Policy Based Route Node Controller I0406 02:56:38.266551 3776 external_controller.go:278] Starting Admin Policy Based Route Controller I0406 02:56:38.266685 3776 default_node_network_controller.go:1076] Egress IP for secondary host network is disabled I0406 02:56:38.266700 3776 link_network_manager.go:117] Link manager is running I0406 02:56:38.266708 3776 default_node_network_controller.go:1087] Default node network controller initialized and ready. I0406 02:56:38.266788 3776 ovspinning_linux.go:42] OVS CPU affinity pinning disabled I0406 02:56:38.267041 3776 vrf_manager.go:128] VRF manager is running I0406 02:56:38.267145 3776 node_controller_manager.go:437] Removing flows to drop GARP I0406 02:56:38.267157 3776 gateway.go:519] Reconciling gateway with updates I0406 02:56:38.291054 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0406 02:56:38.291060 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0406 02:56:38.291075 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0406 02:56:38.291075 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0406 02:56:38.291080 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0406 02:56:38.291085 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0406 02:56:38.291094 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0406 02:56:38.291097 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0406 02:56:38.291097 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0406 02:56:38.291104 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0406 02:56:38.291109 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0406 02:56:38.291110 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0406 02:56:38.291113 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0406 02:56:38.291136 3776 port_claim.go:71] Svc openshift-ingress/router-default: port 32685 is already open, no action needed I0406 02:56:38.291111 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0406 02:56:38.291142 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0406 02:56:38.291147 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0406 02:56:38.291146 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0406 02:56:38.291158 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0406 02:56:38.291158 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0406 02:56:38.291075 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0406 02:56:38.291164 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/packageserver I0406 02:56:38.291174 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0406 02:56:38.291175 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0406 02:56:38.291178 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift I0406 02:56:38.291070 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0406 02:56:38.291187 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0406 02:56:38.291191 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift I0406 02:56:38.291193 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0406 02:56:38.291195 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0406 02:56:38.291197 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0406 02:56:38.291199 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0406 02:56:38.291201 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/kubernetes I0406 02:56:38.291102 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0406 02:56:38.291208 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0406 02:56:38.291213 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0406 02:56:38.291216 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0406 02:56:38.291222 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0406 02:56:38.291179 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0406 02:56:38.291225 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-insights/exporter I0406 02:56:38.291222 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0406 02:56:38.291230 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0406 02:56:38.291232 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0406 02:56:38.291237 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0406 02:56:38.291236 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0406 02:56:38.291242 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0406 02:56:38.291244 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0406 02:56:38.291249 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0406 02:56:38.291249 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0406 02:56:38.291256 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0406 02:56:38.291259 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0406 02:56:38.291260 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0406 02:56:38.291268 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0406 02:56:38.291268 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0406 02:56:38.291274 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0406 02:56:38.291212 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/kubernetes I0406 02:56:38.291274 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0406 02:56:38.291292 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0406 02:56:38.291138 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0406 02:56:38.291211 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0406 02:56:38.291274 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0406 02:56:38.291176 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0406 02:56:38.291147 3776 port_claim.go:71] Svc openshift-ingress/router-default: port 32404 is already open, no action needed I0406 02:56:38.291423 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0406 02:56:38.291101 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0406 02:56:38.291446 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0406 02:56:38.291463 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0406 02:56:38.291260 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0406 02:56:38.291485 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0406 02:56:38.291159 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0406 02:56:38.291506 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0406 02:56:38.291231 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0406 02:56:38.291243 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0406 02:56:38.291238 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-insights/exporter I0406 02:56:38.291559 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-insights/exporter after 0 failed attempt(s) I0406 02:56:38.291181 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/packageserver I0406 02:56:38.291089 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0406 02:56:38.291590 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0406 02:56:38.291048 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0406 02:56:38.291598 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0406 02:56:38.291089 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0406 02:56:38.291139 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0406 02:56:38.291657 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0406 02:56:38.291150 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0406 02:56:38.291162 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0406 02:56:38.291689 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0406 02:56:38.291165 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0406 02:56:38.291167 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0406 02:56:38.291731 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0406 02:56:38.291169 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0406 02:56:38.291168 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0406 02:56:38.291771 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0406 02:56:38.291177 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0406 02:56:38.291178 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0406 02:56:38.291798 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0406 02:56:38.291179 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0406 02:56:38.291822 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0406 02:56:38.291190 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0406 02:56:38.291847 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0406 02:56:38.291187 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0406 02:56:38.291203 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0406 02:56:38.291878 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0406 02:56:38.291883 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0406 02:56:38.291209 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0406 02:56:38.291889 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0406 02:56:38.291226 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0406 02:56:38.291244 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0406 02:56:38.291909 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0406 02:56:38.291049 3776 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0406 02:56:38.291921 3776 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0406 02:56:38.291259 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0406 02:56:38.333150 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0406 02:56:38.333186 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0406 02:56:38.333196 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0406 02:56:38.333206 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0406 02:56:38.333217 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0406 02:56:38.333224 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0406 02:56:38.333233 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0406 02:56:38.376333 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0406 02:56:38.376351 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0406 02:56:38.376338 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0406 02:56:38.376369 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0406 02:56:38.376376 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0406 02:56:38.376383 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0406 02:56:38.376390 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0406 02:56:38.376396 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0406 02:56:38.376405 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0406 02:56:38.376413 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0406 02:56:38.376421 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0406 02:56:38.376428 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0406 02:56:38.376434 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0406 02:56:38.376440 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0406 02:56:38.376445 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0406 02:56:38.376451 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0406 02:56:38.376456 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0406 02:56:38.376463 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0406 02:56:38.376473 3776 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0406 02:56:47.983819 3776 default_network_controller.go:1008] Node "ip-10-0-1-4.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0406 02:56:48.946771 3776 default_network_controller.go:1008] Node "ip-10-0-1-4.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false W0406 02:56:48.969796 3776 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0406 02:56:49.048293 3776 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0406 02:56:49.053093 3776 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-image-registry/image-registry-6f4c66dc87-wmz7l IP address from the namespace address-set, err: pod openshift-image-registry/image-registry-6f4c66dc87-wmz7l: no pod IPs found I0406 02:56:49.118676 3776 default_network_controller.go:1008] Node "ip-10-0-1-4.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0406 02:56:50.073237 3776 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-4.ec2.internal in network "default" I0406 02:56:50.073292 3776 default_network_controller.go:1008] Node "ip-10-0-1-4.ec2.internal" in remote zone "ip-10-0-1-4.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0406 02:56:50.073294 3776 node_tracker.go:235] Node ip-10-0-1-4.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-4.ec2.internal" I0406 02:56:50.073316 3776 node_tracker.go:169] Node ip-10-0-1-4.ec2.internal switch + router changed, syncing services in network "default" I0406 02:56:50.073335 3776 services_controller.go:574] Full service sync requested for network=default W0406 02:56:50.073809 3776 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0406 02:56:50.073820 3776 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0406 02:56:50.198161 3776 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-4.ec2.internal in network "default" I0406 02:56:50.198227 3776 node_tracker.go:169] Node ip-10-0-1-4.ec2.internal switch + router changed, syncing services in network "default" I0406 02:56:50.198240 3776 services_controller.go:574] Full service sync requested for network=default I0406 02:56:50.198165 3776 default_network_controller.go:1008] Node "ip-10-0-1-4.ec2.internal" in remote zone "ip-10-0-1-4.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false W0406 02:56:50.198489 3776 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0406 02:56:50.198562 3776 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0406 02:56:50.198983 3776 zone_ic_handler.go:260] Creating interconnect resources for remote zone node ip-10-0-1-4.ec2.internal for the network default I0406 02:56:50.200192 3776 zone_ic_handler.go:265] Creating Interconnect resources for node "ip-10-0-1-4.ec2.internal" on network "default" took: 1.22279ms I0406 02:56:50.598888 3776 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-216.ec2.internal I0406 02:56:50.598916 3776 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-216.ec2.internal: 34.092µs I0406 02:56:50.629918 3776 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-85b66fd5f-c4fgm] creating logical port openshift-image-registry_image-registry-85b66fd5f-c4fgm for pod on switch ip-10-0-2-216.ec2.internal I0406 02:56:50.630098 3776 kube.go:256] Updating pod openshift-image-registry/image-registry-85b66fd5f-c4fgm I0406 02:56:50.653211 3776 pod.go:62] [openshift-image-registry/image-registry-85b66fd5f-c4fgm] pod update took 23.154764ms I0406 02:56:50.653235 3776 base_network_controller_pods.go:937] [default/openshift-image-registry/image-registry-85b66fd5f-c4fgm] addLogicalPort annotation time took 23.184228ms I0406 02:56:50.653683 3776 base_network_controller_pods.go:487] [default/openshift-console/downloads-744b9bb959-wrgjl] creating logical port openshift-console_downloads-744b9bb959-wrgjl for pod on switch ip-10-0-2-216.ec2.internal I0406 02:56:50.653770 3776 kube.go:256] Updating pod openshift-console/downloads-744b9bb959-wrgjl I0406 02:56:50.654443 3776 pods.go:271] [openshift-image-registry/image-registry-85b66fd5f-c4fgm] addLogicalPort took 24.545897ms, libovsdb time 929.73µs I0406 02:56:50.669737 3776 base_network_controller_pods.go:487] [default/openshift-dns/dns-default-7lv5p] creating logical port openshift-dns_dns-default-7lv5p for pod on switch ip-10-0-2-216.ec2.internal I0406 02:56:50.669818 3776 kube.go:256] Updating pod openshift-dns/dns-default-7lv5p I0406 02:56:50.672920 3776 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-5bc8cc747b-pbhgg] creating logical port openshift-ingress_router-default-5bc8cc747b-pbhgg for pod on switch ip-10-0-2-216.ec2.internal I0406 02:56:50.673004 3776 kube.go:256] Updating pod openshift-ingress/router-default-5bc8cc747b-pbhgg I0406 02:56:50.678529 3776 pod.go:62] [openshift-console/downloads-744b9bb959-wrgjl] pod update took 24.78303ms I0406 02:56:50.678551 3776 base_network_controller_pods.go:937] [default/openshift-console/downloads-744b9bb959-wrgjl] addLogicalPort annotation time took 24.806831ms I0406 02:56:50.678587 3776 base_network_controller_pods.go:487] [default/openshift-insights/insights-runtime-extractor-kl82h] creating logical port openshift-insights_insights-runtime-extractor-kl82h for pod on switch ip-10-0-2-216.ec2.internal I0406 02:56:50.678701 3776 kube.go:256] Updating pod openshift-insights/insights-runtime-extractor-kl82h I0406 02:56:50.679568 3776 pods.go:271] [openshift-console/downloads-744b9bb959-wrgjl] addLogicalPort took 25.908076ms, libovsdb time 765.642µs I0406 02:56:50.692052 3776 pod.go:62] [openshift-dns/dns-default-7lv5p] pod update took 22.25314ms I0406 02:56:50.692076 3776 base_network_controller_pods.go:937] [default/openshift-dns/dns-default-7lv5p] addLogicalPort annotation time took 22.283957ms I0406 02:56:50.693282 3776 pods.go:271] [openshift-dns/dns-default-7lv5p] addLogicalPort took 23.557263ms, libovsdb time 910.527µs I0406 02:56:50.697273 3776 pod.go:62] [openshift-ingress/router-default-5bc8cc747b-pbhgg] pod update took 24.291323ms I0406 02:56:50.697294 3776 base_network_controller_pods.go:937] [default/openshift-ingress/router-default-5bc8cc747b-pbhgg] addLogicalPort annotation time took 24.318035ms I0406 02:56:50.697358 3776 pod.go:62] [openshift-insights/insights-runtime-extractor-kl82h] pod update took 18.689576ms I0406 02:56:50.697381 3776 base_network_controller_pods.go:937] [default/openshift-insights/insights-runtime-extractor-kl82h] addLogicalPort annotation time took 18.71624ms I0406 02:56:50.698558 3776 pods.go:271] [openshift-ingress/router-default-5bc8cc747b-pbhgg] addLogicalPort took 25.649347ms, libovsdb time 988.841µs I0406 02:56:50.698867 3776 pods.go:271] [openshift-insights/insights-runtime-extractor-kl82h] addLogicalPort took 20.293935ms, libovsdb time 1.166377ms I0406 02:56:50.754937 3776 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-4.ec2.internal in network "default" I0406 02:56:50.754996 3776 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-4.ec2.internal I0406 02:56:50.754995 3776 egressfirewall.go:628] Syncing node "ip-10-0-1-4.ec2.internal" for egress firewall I0406 02:56:50.755012 3776 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-4.ec2.internal: 21.522µs I0406 02:56:50.755038 3776 node_tracker.go:169] Node ip-10-0-1-4.ec2.internal switch + router changed, syncing services in network "default" I0406 02:56:50.755042 3776 default_node_network_controller.go:1282] Adding remote node "ip-10-0-1-4.ec2.internal", IP: 10.0.1.4 to PMTUD blocking rules I0406 02:56:50.755057 3776 services_controller.go:574] Full service sync requested for network=default I0406 02:56:50.755056 3776 default_node_network_controller.go:1282] Adding remote node "ip-10-0-1-4.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0406 02:56:50.755061 3776 base_network_controller_pods.go:487] [default/openshift-ingress-canary/ingress-canary-dtmg6] creating logical port openshift-ingress-canary_ingress-canary-dtmg6 for pod on switch ip-10-0-2-216.ec2.internal I0406 02:56:50.755147 3776 kube.go:256] Updating pod openshift-ingress-canary/ingress-canary-dtmg6 W0406 02:56:50.755498 3776 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0406 02:56:50.755517 3776 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0406 02:56:50.761419 3776 default_network_controller.go:1067] Egress IP detected IP address change for node ip-10-0-1-4.ec2.internal. Updating no re-route policies I0406 02:56:50.769253 3776 pod.go:62] [openshift-ingress-canary/ingress-canary-dtmg6] pod update took 14.143014ms I0406 02:56:50.769273 3776 base_network_controller_pods.go:937] [default/openshift-ingress-canary/ingress-canary-dtmg6] addLogicalPort annotation time took 14.166504ms I0406 02:56:50.770211 3776 pods.go:271] [openshift-ingress-canary/ingress-canary-dtmg6] addLogicalPort took 15.161031ms, libovsdb time 697.362µs I0406 02:56:50.774442 3776 default_node_network_controller.go:1282] Adding remote node "ip-10-0-1-4.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0406 02:56:50.774456 3776 default_node_network_controller.go:1282] Adding remote node "ip-10-0-1-4.ec2.internal", IP: 10.0.1.4 to PMTUD blocking rules I0406 02:56:51.045895 3776 cni.go:403] [openshift-image-registry/image-registry-85b66fd5f-c4fgm 0ef3cd2acbea47842ed8b4b66018903a4ee81bd171e6f97fb43531353cb410f5 network default NAD default] ADD starting CNI request [openshift-image-registry/image-registry-85b66fd5f-c4fgm 0ef3cd2acbea47842ed8b4b66018903a4ee81bd171e6f97fb43531353cb410f5 network default NAD default] I0406 02:56:51.071893 3776 cni.go:403] [openshift-console/downloads-744b9bb959-wrgjl d7681c195f8e304052152cce3604e0368ae4c14c1d1b95b2352fd87f1c32d719 network default NAD default] ADD starting CNI request [openshift-console/downloads-744b9bb959-wrgjl d7681c195f8e304052152cce3604e0368ae4c14c1d1b95b2352fd87f1c32d719 network default NAD default] I0406 02:56:51.077477 3776 helper_linux.go:450] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-85b66fd5f-c4fgm, hostIfaceName: 0ef3cd2acbea478, network: default, NAD default, SandboxID: "0ef3cd2acbea47842ed8b4b66018903a4ee81bd171e6f97fb43531353cb410f5", PCI device ID: , UID: "12bba2b0-3598-44f1-8115-39cff82bda65", MAC: 0a:58:0a:81:00:06, IPs: [10.129.0.6/23] I0406 02:56:51.082914 3776 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: downloads-744b9bb959-wrgjl, hostIfaceName: d7681c195f8e304, network: default, NAD default, SandboxID: "d7681c195f8e304052152cce3604e0368ae4c14c1d1b95b2352fd87f1c32d719", PCI device ID: , UID: "a74bdba4-a25c-4c7a-9e72-e7054da63c82", MAC: 0a:58:0a:81:00:07, IPs: [10.129.0.7/23] I0406 02:56:51.094416 3776 cni.go:403] [openshift-dns/dns-default-7lv5p 96d88cab39a0afa5ea64ced5f64d0abbad61983ea8a13d92ada19c9012bc506b network default NAD default] ADD starting CNI request [openshift-dns/dns-default-7lv5p 96d88cab39a0afa5ea64ced5f64d0abbad61983ea8a13d92ada19c9012bc506b network default NAD default] I0406 02:56:51.114610 3776 cni.go:403] [openshift-ingress/router-default-5bc8cc747b-pbhgg bb695ea093a58d23eb81e5f376d133756965d6cde39882cd195ff632f090d2a5 network default NAD default] ADD starting CNI request [openshift-ingress/router-default-5bc8cc747b-pbhgg bb695ea093a58d23eb81e5f376d133756965d6cde39882cd195ff632f090d2a5 network default NAD default] I0406 02:56:51.114882 3776 helper_linux.go:450] ConfigureOVS: namespace: openshift-dns, podName: dns-default-7lv5p, hostIfaceName: 96d88cab39a0afa, network: default, NAD default, SandboxID: "96d88cab39a0afa5ea64ced5f64d0abbad61983ea8a13d92ada19c9012bc506b", PCI device ID: , UID: "163caab3-b0dc-409d-a4bc-fcb083b006d3", MAC: 0a:58:0a:81:00:08, IPs: [10.129.0.8/23] I0406 02:56:51.142556 3776 helper_linux.go:450] ConfigureOVS: namespace: openshift-ingress, podName: router-default-5bc8cc747b-pbhgg, hostIfaceName: bb695ea093a58d2, network: default, NAD default, SandboxID: "bb695ea093a58d23eb81e5f376d133756965d6cde39882cd195ff632f090d2a5", PCI device ID: , UID: "c586ebc8-d4d4-4434-96a5-2ee3e8914862", MAC: 0a:58:0a:81:00:09, IPs: [10.129.0.9/23] I0406 02:56:51.156213 3776 cni.go:403] [openshift-insights/insights-runtime-extractor-kl82h 038bec2d8cfbcb6887f9fd26ca5f913a7d25fe316cdf35015a5ca123701bbfbb network default NAD default] ADD starting CNI request [openshift-insights/insights-runtime-extractor-kl82h 038bec2d8cfbcb6887f9fd26ca5f913a7d25fe316cdf35015a5ca123701bbfbb network default NAD default] I0406 02:56:51.167565 3776 cni.go:403] [openshift-ingress-canary/ingress-canary-dtmg6 dd49adabdc78307472a675072c7dd7ecadaef4b34a84a58a7fd24b1c16e298de network default NAD default] ADD starting CNI request [openshift-ingress-canary/ingress-canary-dtmg6 dd49adabdc78307472a675072c7dd7ecadaef4b34a84a58a7fd24b1c16e298de network default NAD default] I0406 02:56:51.168046 3776 helper_linux.go:450] ConfigureOVS: namespace: openshift-insights, podName: insights-runtime-extractor-kl82h, hostIfaceName: 038bec2d8cfbcb6, network: default, NAD default, SandboxID: "038bec2d8cfbcb6887f9fd26ca5f913a7d25fe316cdf35015a5ca123701bbfbb", PCI device ID: , UID: "fc143840-81b7-42ab-9a99-caa3107528ce", MAC: 0a:58:0a:81:00:0a, IPs: [10.129.0.10/23] I0406 02:56:51.182837 3776 cni.go:424] [openshift-image-registry/image-registry-85b66fd5f-c4fgm 0ef3cd2acbea47842ed8b4b66018903a4ee81bd171e6f97fb43531353cb410f5 network default NAD default] ADD finished CNI request [openshift-image-registry/image-registry-85b66fd5f-c4fgm 0ef3cd2acbea47842ed8b4b66018903a4ee81bd171e6f97fb43531353cb410f5 network default NAD default], result "{\"interfaces\":[{\"mac\":\"12:9a:78:b9:af:13\",\"name\":\"0ef3cd2acbea478\"},{\"mac\":\"0a:58:0a:81:00:06\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e117aa91-5d32-43ab-a3d4-e2ffbbd0fb28\"}],\"ips\":[{\"address\":\"10.129.0.6/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0406 02:56:51.182916 3776 cni.go:424] [openshift-console/downloads-744b9bb959-wrgjl d7681c195f8e304052152cce3604e0368ae4c14c1d1b95b2352fd87f1c32d719 network default NAD default] ADD finished CNI request [openshift-console/downloads-744b9bb959-wrgjl d7681c195f8e304052152cce3604e0368ae4c14c1d1b95b2352fd87f1c32d719 network default NAD default], result "{\"interfaces\":[{\"mac\":\"06:f3:82:28:fe:0a\",\"name\":\"d7681c195f8e304\"},{\"mac\":\"0a:58:0a:81:00:07\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/abe8a406-c7ba-4f74-bf7d-33705a7276fe\"}],\"ips\":[{\"address\":\"10.129.0.7/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0406 02:56:51.188183 3776 helper_linux.go:450] ConfigureOVS: namespace: openshift-ingress-canary, podName: ingress-canary-dtmg6, hostIfaceName: dd49adabdc78307, network: default, NAD default, SandboxID: "dd49adabdc78307472a675072c7dd7ecadaef4b34a84a58a7fd24b1c16e298de", PCI device ID: , UID: "dee4bd50-ead5-41a1-8843-33db555bc6ee", MAC: 0a:58:0a:81:00:0b, IPs: [10.129.0.11/23] I0406 02:56:51.189982 3776 cni.go:424] [openshift-dns/dns-default-7lv5p 96d88cab39a0afa5ea64ced5f64d0abbad61983ea8a13d92ada19c9012bc506b network default NAD default] ADD finished CNI request [openshift-dns/dns-default-7lv5p 96d88cab39a0afa5ea64ced5f64d0abbad61983ea8a13d92ada19c9012bc506b network default NAD default], result "{\"interfaces\":[{\"mac\":\"42:ba:01:a4:42:01\",\"name\":\"96d88cab39a0afa\"},{\"mac\":\"0a:58:0a:81:00:08\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/30827591-6fe3-49d2-993f-05dd11d096f0\"}],\"ips\":[{\"address\":\"10.129.0.8/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0406 02:56:51.209736 3776 cni.go:424] [openshift-ingress/router-default-5bc8cc747b-pbhgg bb695ea093a58d23eb81e5f376d133756965d6cde39882cd195ff632f090d2a5 network default NAD default] ADD finished CNI request [openshift-ingress/router-default-5bc8cc747b-pbhgg bb695ea093a58d23eb81e5f376d133756965d6cde39882cd195ff632f090d2a5 network default NAD default], result "{\"interfaces\":[{\"mac\":\"0a:7a:03:60:25:3f\",\"name\":\"bb695ea093a58d2\"},{\"mac\":\"0a:58:0a:81:00:09\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1f72f7b7-23c3-408d-ade2-8bae732b7597\"}],\"ips\":[{\"address\":\"10.129.0.9/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0406 02:56:51.233881 3776 cni.go:424] [openshift-insights/insights-runtime-extractor-kl82h 038bec2d8cfbcb6887f9fd26ca5f913a7d25fe316cdf35015a5ca123701bbfbb network default NAD default] ADD finished CNI request [openshift-insights/insights-runtime-extractor-kl82h 038bec2d8cfbcb6887f9fd26ca5f913a7d25fe316cdf35015a5ca123701bbfbb network default NAD default], result "{\"interfaces\":[{\"mac\":\"0e:68:e9:5d:54:8f\",\"name\":\"038bec2d8cfbcb6\"},{\"mac\":\"0a:58:0a:81:00:0a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8af28aeb-f9c9-44c0-af06-f1551d026d56\"}],\"ips\":[{\"address\":\"10.129.0.10/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0406 02:56:51.251807 3776 cni.go:424] [openshift-ingress-canary/ingress-canary-dtmg6 dd49adabdc78307472a675072c7dd7ecadaef4b34a84a58a7fd24b1c16e298de network default NAD default] ADD finished CNI request [openshift-ingress-canary/ingress-canary-dtmg6 dd49adabdc78307472a675072c7dd7ecadaef4b34a84a58a7fd24b1c16e298de network default NAD default], result "{\"interfaces\":[{\"mac\":\"e2:29:1a:35:59:b0\",\"name\":\"dd49adabdc78307\"},{\"mac\":\"0a:58:0a:81:00:0b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/05ac9dbd-260c-43f9-92f5-bb105bf36f8f\"}],\"ips\":[{\"address\":\"10.129.0.11/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0406 02:56:51.280296 3776 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-operator-admission-webhook-7464b9f894-mf8bw] creating logical port openshift-monitoring_prometheus-operator-admission-webhook-7464b9f894-mf8bw for pod on switch ip-10-0-2-216.ec2.internal I0406 02:56:51.280392 3776 kube.go:256] Updating pod openshift-monitoring/prometheus-operator-admission-webhook-7464b9f894-mf8bw I0406 02:56:51.301251 3776 pod.go:62] [openshift-monitoring/prometheus-operator-admission-webhook-7464b9f894-mf8bw] pod update took 20.883975ms I0406 02:56:51.301275 3776 base_network_controller_pods.go:937] [default/openshift-monitoring/prometheus-operator-admission-webhook-7464b9f894-mf8bw] addLogicalPort annotation time took 20.91298ms I0406 02:56:51.302502 3776 pods.go:271] [openshift-monitoring/prometheus-operator-admission-webhook-7464b9f894-mf8bw] addLogicalPort took 22.224733ms, libovsdb time 933.995µs W0406 02:56:51.569760 3776 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0406 02:56:51.569781 3776 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0406 02:56:51.569785 3776 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default I0406 02:56:51.699520 3776 cni.go:403] [openshift-monitoring/prometheus-operator-admission-webhook-7464b9f894-mf8bw c52da776e8cf772807fbb517e99627e6d601180d8d9bd82abe971bdaa9136c9a network default NAD default] ADD starting CNI request [openshift-monitoring/prometheus-operator-admission-webhook-7464b9f894-mf8bw c52da776e8cf772807fbb517e99627e6d601180d8d9bd82abe971bdaa9136c9a network default NAD default] I0406 02:56:51.711505 3776 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-operator-admission-webhook-7464b9f894-mf8bw, hostIfaceName: c52da776e8cf772, network: default, NAD default, SandboxID: "c52da776e8cf772807fbb517e99627e6d601180d8d9bd82abe971bdaa9136c9a", PCI device ID: , UID: "4f700b18-1577-4eb0-a612-5dc9253c2091", MAC: 0a:58:0a:81:00:0c, IPs: [10.129.0.12/23] I0406 02:56:51.763874 3776 cni.go:424] [openshift-monitoring/prometheus-operator-admission-webhook-7464b9f894-mf8bw c52da776e8cf772807fbb517e99627e6d601180d8d9bd82abe971bdaa9136c9a network default NAD default] ADD finished CNI request [openshift-monitoring/prometheus-operator-admission-webhook-7464b9f894-mf8bw c52da776e8cf772807fbb517e99627e6d601180d8d9bd82abe971bdaa9136c9a network default NAD default], result "{\"interfaces\":[{\"mac\":\"1e:96:8e:11:f7:70\",\"name\":\"c52da776e8cf772\"},{\"mac\":\"0a:58:0a:81:00:0c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2bdf6e0e-6039-4076-81e0-296491546586\"}],\"ips\":[{\"address\":\"10.129.0.12/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0406 02:56:51.848734 3776 util.go:959] Failed to build node endpoints for node ip-10-0-2-216.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-default W0406 02:56:51.848767 3776 util.go:959] Failed to build node endpoints for node ip-10-0-2-216.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-default W0406 02:56:51.872928 3776 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0406 02:56:52.852859 3776 util.go:959] Failed to build node endpoints for node ip-10-0-2-216.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-internal-default W0406 02:56:52.852884 3776 util.go:959] Failed to build node endpoints for node ip-10-0-2-216.ec2.internal port 1936: empty IP address endpoints for service openshift-ingress/router-internal-default W0406 02:56:52.852890 3776 util.go:959] Failed to build node endpoints for node ip-10-0-2-216.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-internal-default W0406 02:56:52.855479 3776 util.go:959] Failed to build node endpoints for node ip-10-0-2-216.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-default W0406 02:56:52.855502 3776 util.go:959] Failed to build node endpoints for node ip-10-0-2-216.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-default I0406 02:56:53.001179 3776 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0406 02:56:53.595798 3776 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf5653908jjng] creating logical port openshift-deployment-validation-operator_ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf5653908jjng for pod on switch ip-10-0-2-216.ec2.internal I0406 02:56:53.595915 3776 kube.go:256] Updating pod openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf5653908jjng I0406 02:56:53.608778 3776 pod.go:62] [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf5653908jjng] pod update took 12.896825ms I0406 02:56:53.608811 3776 base_network_controller_pods.go:937] [default/openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf5653908jjng] addLogicalPort annotation time took 12.936064ms I0406 02:56:53.610627 3776 pods.go:271] [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf5653908jjng] addLogicalPort took 14.843267ms, libovsdb time 1.431246ms I0406 02:56:53.915920 3776 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-4.ec2.internal I0406 02:56:53.916046 3776 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-4.ec2.internal: 134.477µs I0406 02:56:54.325799 3776 cni.go:403] [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf5653908jjng d87bd4e286d0e668497644a6a3dc9e66f2bf6246307501b10b59fb4d07ea5b28 network default NAD default] ADD starting CNI request [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf5653908jjng d87bd4e286d0e668497644a6a3dc9e66f2bf6246307501b10b59fb4d07ea5b28 network default NAD default] I0406 02:56:54.339019 3776 helper_linux.go:450] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf5653908jjng, hostIfaceName: d87bd4e286d0e66, network: default, NAD default, SandboxID: "d87bd4e286d0e668497644a6a3dc9e66f2bf6246307501b10b59fb4d07ea5b28", PCI device ID: , UID: "68f7151b-dba6-4b99-80eb-aa134e25a58e", MAC: 0a:58:0a:81:00:0d, IPs: [10.129.0.13/23] I0406 02:56:54.408308 3776 cni.go:424] [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf5653908jjng d87bd4e286d0e668497644a6a3dc9e66f2bf6246307501b10b59fb4d07ea5b28 network default NAD default] ADD finished CNI request [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf5653908jjng d87bd4e286d0e668497644a6a3dc9e66f2bf6246307501b10b59fb4d07ea5b28 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ae:80:c1:6f:b3:80\",\"name\":\"d87bd4e286d0e66\"},{\"mac\":\"0a:58:0a:81:00:0d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/19d07cc9-665c-4406-8e36-8774b317caeb\"}],\"ips\":[{\"address\":\"10.129.0.13/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0406 02:56:54.461222 3776 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook I0406 02:56:54.693290 3776 base_network_controller_pods.go:487] [default/openshift-console/console-69796cc9d4-djt8q] creating logical port openshift-console_console-69796cc9d4-djt8q for pod on switch ip-10-0-2-216.ec2.internal I0406 02:56:54.693389 3776 kube.go:256] Updating pod openshift-console/console-69796cc9d4-djt8q I0406 02:56:54.707664 3776 pod.go:62] [openshift-console/console-69796cc9d4-djt8q] pod update took 14.304581ms I0406 02:56:54.707693 3776 base_network_controller_pods.go:937] [default/openshift-console/console-69796cc9d4-djt8q] addLogicalPort annotation time took 14.338316ms I0406 02:56:54.708941 3776 pods.go:271] [openshift-console/console-69796cc9d4-djt8q] addLogicalPort took 15.66656ms, libovsdb time 937.768µs W0406 02:56:54.874256 3776 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0406 02:56:54.874277 3776 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0406 02:56:54.874288 3776 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default I0406 02:56:55.095453 3776 cni.go:403] [openshift-console/console-69796cc9d4-djt8q 360bff00044787ebebba06684163270d35e342c0d0b6c8d43ba17e6de2e20392 network default NAD default] ADD starting CNI request [openshift-console/console-69796cc9d4-djt8q 360bff00044787ebebba06684163270d35e342c0d0b6c8d43ba17e6de2e20392 network default NAD default] I0406 02:56:55.106541 3776 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-69796cc9d4-djt8q, hostIfaceName: 360bff00044787e, network: default, NAD default, SandboxID: "360bff00044787ebebba06684163270d35e342c0d0b6c8d43ba17e6de2e20392", PCI device ID: , UID: "5007d9bb-211a-4b57-8c99-76178d1a64e7", MAC: 0a:58:0a:81:00:0e, IPs: [10.129.0.14/23] I0406 02:56:55.162428 3776 cni.go:424] [openshift-console/console-69796cc9d4-djt8q 360bff00044787ebebba06684163270d35e342c0d0b6c8d43ba17e6de2e20392 network default NAD default] ADD finished CNI request [openshift-console/console-69796cc9d4-djt8q 360bff00044787ebebba06684163270d35e342c0d0b6c8d43ba17e6de2e20392 network default NAD default], result "{\"interfaces\":[{\"mac\":\"1e:20:a6:4a:07:1b\",\"name\":\"360bff00044787e\"},{\"mac\":\"0a:58:0a:81:00:0e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b1ca2bfc-dafd-475a-81ce-e752e0e1699e\"}],\"ips\":[{\"address\":\"10.129.0.14/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0406 02:56:55.304388 3776 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0406 02:56:55.346212 3776 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-operator-847b8ff89d-x84sh] creating logical port openshift-monitoring_prometheus-operator-847b8ff89d-x84sh for pod on switch ip-10-0-2-216.ec2.internal I0406 02:56:55.346380 3776 kube.go:256] Updating pod openshift-monitoring/prometheus-operator-847b8ff89d-x84sh I0406 02:56:55.359118 3776 pod.go:62] [openshift-monitoring/prometheus-operator-847b8ff89d-x84sh] pod update took 12.81494ms I0406 02:56:55.359225 3776 base_network_controller_pods.go:937] [default/openshift-monitoring/prometheus-operator-847b8ff89d-x84sh] addLogicalPort annotation time took 12.927119ms I0406 02:56:55.360537 3776 pods.go:271] [openshift-monitoring/prometheus-operator-847b8ff89d-x84sh] addLogicalPort took 14.344992ms, libovsdb time 967.19µs I0406 02:56:56.343270 3776 cni.go:403] [openshift-monitoring/prometheus-operator-847b8ff89d-x84sh 9ec10eb7e0361fe011a39dc377e1a67e14c531fdfd9412aab3228e0118f4c756 network default NAD default] ADD starting CNI request [openshift-monitoring/prometheus-operator-847b8ff89d-x84sh 9ec10eb7e0361fe011a39dc377e1a67e14c531fdfd9412aab3228e0118f4c756 network default NAD default] I0406 02:56:56.354253 3776 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-operator-847b8ff89d-x84sh, hostIfaceName: 9ec10eb7e0361fe, network: default, NAD default, SandboxID: "9ec10eb7e0361fe011a39dc377e1a67e14c531fdfd9412aab3228e0118f4c756", PCI device ID: , UID: "2005e620-5443-4eee-8df8-8aff09aba8e5", MAC: 0a:58:0a:81:00:0f, IPs: [10.129.0.15/23] I0406 02:56:56.410520 3776 cni.go:424] [openshift-monitoring/prometheus-operator-847b8ff89d-x84sh 9ec10eb7e0361fe011a39dc377e1a67e14c531fdfd9412aab3228e0118f4c756 network default NAD default] ADD finished CNI request [openshift-monitoring/prometheus-operator-847b8ff89d-x84sh 9ec10eb7e0361fe011a39dc377e1a67e14c531fdfd9412aab3228e0118f4c756 network default NAD default], result "{\"interfaces\":[{\"mac\":\"3a:c8:fe:88:19:8b\",\"name\":\"9ec10eb7e0361fe\"},{\"mac\":\"0a:58:0a:81:00:0f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ede8a839-7c01-46e8-8227-afafee1a443d\"}],\"ips\":[{\"address\":\"10.129.0.15/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0406 02:56:59.309316 3776 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0406 02:56:59.309337 3776 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0406 02:56:59.309343 3776 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0406 02:56:59.363162 3776 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0406 02:56:59.913918 3776 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0406 02:57:00.469502 3776 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0406 02:57:01.491075 3776 util.go:959] Failed to build node endpoints for node ip-10-0-2-216.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0406 02:57:01.491098 3776 util.go:959] Failed to build node endpoints for node ip-10-0-2-216.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0406 02:57:01.491104 3776 util.go:959] Failed to build node endpoints for node ip-10-0-2-216.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0406 02:57:01.491136 3776 util.go:959] Failed to build node endpoints for node ip-10-0-2-216.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0406 02:57:01.491145 3776 util.go:959] Failed to build node endpoints for node ip-10-0-2-216.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0406 02:57:01.491153 3776 util.go:959] Failed to build node endpoints for node ip-10-0-2-216.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default I0406 02:57:01.771456 3776 base_network_controller_pods.go:487] [default/openshift-monitoring/openshift-state-metrics-77dc8fdc5c-6f8gb] creating logical port openshift-monitoring_openshift-state-metrics-77dc8fdc5c-6f8gb for pod on switch ip-10-0-2-216.ec2.internal I0406 02:57:01.771549 3776 kube.go:256] Updating pod openshift-monitoring/openshift-state-metrics-77dc8fdc5c-6f8gb I0406 02:57:01.785629 3776 pod.go:62] [openshift-monitoring/openshift-state-metrics-77dc8fdc5c-6f8gb] pod update took 14.104287ms I0406 02:57:01.785653 3776 base_network_controller_pods.go:937] [default/openshift-monitoring/openshift-state-metrics-77dc8fdc5c-6f8gb] addLogicalPort annotation time took 14.134223ms I0406 02:57:01.786861 3776 pods.go:271] [openshift-monitoring/openshift-state-metrics-77dc8fdc5c-6f8gb] addLogicalPort took 15.425776ms, libovsdb time 890.836µs I0406 02:57:02.947332 3776 base_network_controller_pods.go:487] [default/openshift-monitoring/alertmanager-main-1] creating logical port openshift-monitoring_alertmanager-main-1 for pod on switch ip-10-0-2-216.ec2.internal I0406 02:57:02.947467 3776 kube.go:256] Updating pod openshift-monitoring/alertmanager-main-1 I0406 02:57:02.967801 3776 pod.go:62] [openshift-monitoring/alertmanager-main-1] pod update took 20.36764ms I0406 02:57:02.967828 3776 base_network_controller_pods.go:937] [default/openshift-monitoring/alertmanager-main-1] addLogicalPort annotation time took 20.400102ms I0406 02:57:02.969185 3776 pods.go:271] [openshift-monitoring/alertmanager-main-1] addLogicalPort took 21.8701ms, libovsdb time 996.668µs W0406 02:57:04.936171 3776 util.go:959] Failed to build node endpoints for node ip-10-0-2-216.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0406 02:57:04.936192 3776 util.go:959] Failed to build node endpoints for node ip-10-0-2-216.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0406 02:57:04.936198 3776 util.go:959] Failed to build node endpoints for node ip-10-0-2-216.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default I0406 02:57:05.001845 3776 base_network_controller_pods.go:487] [default/openshift-monitoring/thanos-querier-556c699878-zxgtw] creating logical port openshift-monitoring_thanos-querier-556c699878-zxgtw for pod on switch ip-10-0-2-216.ec2.internal I0406 02:57:05.001960 3776 kube.go:256] Updating pod openshift-monitoring/thanos-querier-556c699878-zxgtw I0406 02:57:05.035315 3776 pod.go:62] [openshift-monitoring/thanos-querier-556c699878-zxgtw] pod update took 33.394932ms I0406 02:57:05.035341 3776 base_network_controller_pods.go:937] [default/openshift-monitoring/thanos-querier-556c699878-zxgtw] addLogicalPort annotation time took 33.430011ms I0406 02:57:05.036724 3776 pods.go:271] [openshift-monitoring/thanos-querier-556c699878-zxgtw] addLogicalPort took 34.899141ms, libovsdb time 1.083429ms W0406 02:57:05.929511 3776 util.go:959] Failed to build node endpoints for node ip-10-0-2-216.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0406 02:57:06.326254 3776 base_network_controller_pods.go:487] [default/openshift-monitoring/metrics-server-78f5f7f777-nwp7t] creating logical port openshift-monitoring_metrics-server-78f5f7f777-nwp7t for pod on switch ip-10-0-2-216.ec2.internal I0406 02:57:06.326357 3776 kube.go:256] Updating pod openshift-monitoring/metrics-server-78f5f7f777-nwp7t I0406 02:57:06.341907 3776 pod.go:62] [openshift-monitoring/metrics-server-78f5f7f777-nwp7t] pod update took 15.572409ms I0406 02:57:06.341931 3776 base_network_controller_pods.go:937] [default/openshift-monitoring/metrics-server-78f5f7f777-nwp7t] addLogicalPort annotation time took 15.603256ms I0406 02:57:06.343233 3776 pods.go:271] [openshift-monitoring/metrics-server-78f5f7f777-nwp7t] addLogicalPort took 16.9989ms, libovsdb time 940.06µs W0406 02:57:06.391853 3776 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0406 02:57:06.391881 3776 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0406 02:57:06.391899 3776 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0406 02:57:06.558919 3776 base_network_controller_pods.go:487] [default/openshift-monitoring/monitoring-plugin-76957fdd44-fkbpn] creating logical port openshift-monitoring_monitoring-plugin-76957fdd44-fkbpn for pod on switch ip-10-0-2-216.ec2.internal I0406 02:57:06.559041 3776 kube.go:256] Updating pod openshift-monitoring/monitoring-plugin-76957fdd44-fkbpn I0406 02:57:06.571437 3776 pod.go:62] [openshift-monitoring/monitoring-plugin-76957fdd44-fkbpn] pod update took 12.42513ms I0406 02:57:06.571461 3776 base_network_controller_pods.go:937] [default/openshift-monitoring/monitoring-plugin-76957fdd44-fkbpn] addLogicalPort annotation time took 12.467631ms I0406 02:57:06.573437 3776 pods.go:271] [openshift-monitoring/monitoring-plugin-76957fdd44-fkbpn] addLogicalPort took 14.53924ms, libovsdb time 1.592585ms W0406 02:57:07.524141 3776 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads W0406 02:57:07.680982 3776 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-console/console-7788bfb9bb-vs4ct IP address from the namespace address-set, err: pod openshift-console/console-7788bfb9bb-vs4ct: no pod IPs found I0406 02:57:08.191444 3776 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-k8s-1] creating logical port openshift-monitoring_prometheus-k8s-1 for pod on switch ip-10-0-2-216.ec2.internal I0406 02:57:08.191556 3776 kube.go:256] Updating pod openshift-monitoring/prometheus-k8s-1 I0406 02:57:08.218394 3776 pod.go:62] [openshift-monitoring/prometheus-k8s-1] pod update took 26.874293ms I0406 02:57:08.218423 3776 base_network_controller_pods.go:937] [default/openshift-monitoring/prometheus-k8s-1] addLogicalPort annotation time took 26.910338ms I0406 02:57:08.220033 3776 pods.go:271] [openshift-monitoring/prometheus-k8s-1] addLogicalPort took 28.609515ms, libovsdb time 1.159133ms I0406 02:57:09.166446 3776 cni.go:403] [openshift-monitoring/alertmanager-main-1 587698c74a82ec20ebc0a579ab6dab7b133e6b86b2f0310848d7db268d5f1e19 network default NAD default] ADD starting CNI request [openshift-monitoring/alertmanager-main-1 587698c74a82ec20ebc0a579ab6dab7b133e6b86b2f0310848d7db268d5f1e19 network default NAD default] I0406 02:57:09.173783 3776 cni.go:403] [openshift-monitoring/thanos-querier-556c699878-zxgtw abf9e657b0b966a8f6e58b386c3766b21921857574f0f1bdbbf5a1ffcd256793 network default NAD default] ADD starting CNI request [openshift-monitoring/thanos-querier-556c699878-zxgtw abf9e657b0b966a8f6e58b386c3766b21921857574f0f1bdbbf5a1ffcd256793 network default NAD default] I0406 02:57:09.177912 3776 cni.go:403] [openshift-monitoring/openshift-state-metrics-77dc8fdc5c-6f8gb 8fabf971b34d3a213e1e0314e119e3325d18ac82a93f105dbb8f821072e3dfb3 network default NAD default] ADD starting CNI request [openshift-monitoring/openshift-state-metrics-77dc8fdc5c-6f8gb 8fabf971b34d3a213e1e0314e119e3325d18ac82a93f105dbb8f821072e3dfb3 network default NAD default] I0406 02:57:09.177917 3776 cni.go:403] [openshift-monitoring/metrics-server-78f5f7f777-nwp7t 977efd03a45e9d156b4f1188b78307b7aed8a48b8659f4184bd461a4cf72a80d network default NAD default] ADD starting CNI request [openshift-monitoring/metrics-server-78f5f7f777-nwp7t 977efd03a45e9d156b4f1188b78307b7aed8a48b8659f4184bd461a4cf72a80d network default NAD default] I0406 02:57:09.179233 3776 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: alertmanager-main-1, hostIfaceName: 587698c74a82ec2, network: default, NAD default, SandboxID: "587698c74a82ec20ebc0a579ab6dab7b133e6b86b2f0310848d7db268d5f1e19", PCI device ID: , UID: "8d228c61-dea8-405a-ba54-b2bc5b69db6e", MAC: 0a:58:0a:81:00:11, IPs: [10.129.0.17/23] I0406 02:57:09.179944 3776 cni.go:403] [openshift-monitoring/prometheus-k8s-1 e997191d1c771335e0dec1638571757f55910d5f582f940e408462b57f315eab network default NAD default] ADD starting CNI request [openshift-monitoring/prometheus-k8s-1 e997191d1c771335e0dec1638571757f55910d5f582f940e408462b57f315eab network default NAD default] I0406 02:57:09.186724 3776 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: thanos-querier-556c699878-zxgtw, hostIfaceName: abf9e657b0b966a, network: default, NAD default, SandboxID: "abf9e657b0b966a8f6e58b386c3766b21921857574f0f1bdbbf5a1ffcd256793", PCI device ID: , UID: "94af7658-9aef-4cc6-8ddf-61992dfd0ebd", MAC: 0a:58:0a:81:00:12, IPs: [10.129.0.18/23] I0406 02:57:09.189110 3776 cni.go:403] [openshift-monitoring/monitoring-plugin-76957fdd44-fkbpn 493ed92a2d6e9d9769cf9dc691c5d82628c83f9455abfa46cdecdc2131c75a15 network default NAD default] ADD starting CNI request [openshift-monitoring/monitoring-plugin-76957fdd44-fkbpn 493ed92a2d6e9d9769cf9dc691c5d82628c83f9455abfa46cdecdc2131c75a15 network default NAD default] I0406 02:57:09.200205 3776 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: metrics-server-78f5f7f777-nwp7t, hostIfaceName: 977efd03a45e9d1, network: default, NAD default, SandboxID: "977efd03a45e9d156b4f1188b78307b7aed8a48b8659f4184bd461a4cf72a80d", PCI device ID: , UID: "2ea143a2-0c71-4218-a188-990283520bfb", MAC: 0a:58:0a:81:00:13, IPs: [10.129.0.19/23] I0406 02:57:09.201217 3776 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: openshift-state-metrics-77dc8fdc5c-6f8gb, hostIfaceName: 8fabf971b34d3a2, network: default, NAD default, SandboxID: "8fabf971b34d3a213e1e0314e119e3325d18ac82a93f105dbb8f821072e3dfb3", PCI device ID: , UID: "bfc4df26-a508-4257-94a1-4406eefc62d7", MAC: 0a:58:0a:81:00:10, IPs: [10.129.0.16/23] I0406 02:57:09.203083 3776 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-k8s-1, hostIfaceName: e997191d1c77133, network: default, NAD default, SandboxID: "e997191d1c771335e0dec1638571757f55910d5f582f940e408462b57f315eab", PCI device ID: , UID: "b34af8cd-b93d-4d32-a5d3-8db755142a09", MAC: 0a:58:0a:81:00:15, IPs: [10.129.0.21/23] I0406 02:57:09.206256 3776 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: monitoring-plugin-76957fdd44-fkbpn, hostIfaceName: 493ed92a2d6e9d9, network: default, NAD default, SandboxID: "493ed92a2d6e9d9769cf9dc691c5d82628c83f9455abfa46cdecdc2131c75a15", PCI device ID: , UID: "d12530ec-6cdb-46cf-bf00-19faca7c9359", MAC: 0a:58:0a:81:00:14, IPs: [10.129.0.20/23] I0406 02:57:09.269462 3776 cni.go:424] [openshift-monitoring/alertmanager-main-1 587698c74a82ec20ebc0a579ab6dab7b133e6b86b2f0310848d7db268d5f1e19 network default NAD default] ADD finished CNI request [openshift-monitoring/alertmanager-main-1 587698c74a82ec20ebc0a579ab6dab7b133e6b86b2f0310848d7db268d5f1e19 network default NAD default], result "{\"interfaces\":[{\"mac\":\"8a:65:7d:9d:66:de\",\"name\":\"587698c74a82ec2\"},{\"mac\":\"0a:58:0a:81:00:11\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/16964ccb-7194-4582-9a4d-3a5eb7396533\"}],\"ips\":[{\"address\":\"10.129.0.17/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0406 02:57:09.390275 3776 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0406 02:57:09.390299 3776 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0406 02:57:09.432188 3776 util.go:959] Failed to build node endpoints for node ip-10-0-2-216.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0406 02:57:09.432214 3776 util.go:959] Failed to build node endpoints for node ip-10-0-2-216.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry I0406 02:57:09.485939 3776 cni.go:424] [openshift-monitoring/monitoring-plugin-76957fdd44-fkbpn 493ed92a2d6e9d9769cf9dc691c5d82628c83f9455abfa46cdecdc2131c75a15 network default NAD default] ADD finished CNI request [openshift-monitoring/monitoring-plugin-76957fdd44-fkbpn 493ed92a2d6e9d9769cf9dc691c5d82628c83f9455abfa46cdecdc2131c75a15 network default NAD default], result "{\"interfaces\":[{\"mac\":\"e6:9f:6d:28:3f:f4\",\"name\":\"493ed92a2d6e9d9\"},{\"mac\":\"0a:58:0a:81:00:14\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/eef8fe32-ff83-454e-a9f2-a0d48f5ff10b\"}],\"ips\":[{\"address\":\"10.129.0.20/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0406 02:57:09.488868 3776 cni.go:424] [openshift-monitoring/thanos-querier-556c699878-zxgtw abf9e657b0b966a8f6e58b386c3766b21921857574f0f1bdbbf5a1ffcd256793 network default NAD default] ADD finished CNI request [openshift-monitoring/thanos-querier-556c699878-zxgtw abf9e657b0b966a8f6e58b386c3766b21921857574f0f1bdbbf5a1ffcd256793 network default NAD default], result "{\"interfaces\":[{\"mac\":\"82:a5:29:22:ce:a9\",\"name\":\"abf9e657b0b966a\"},{\"mac\":\"0a:58:0a:81:00:12\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f918ea04-5c08-43a3-91b3-505cf6e8ec0d\"}],\"ips\":[{\"address\":\"10.129.0.18/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0406 02:57:09.491363 3776 cni.go:424] [openshift-monitoring/metrics-server-78f5f7f777-nwp7t 977efd03a45e9d156b4f1188b78307b7aed8a48b8659f4184bd461a4cf72a80d network default NAD default] ADD finished CNI request [openshift-monitoring/metrics-server-78f5f7f777-nwp7t 977efd03a45e9d156b4f1188b78307b7aed8a48b8659f4184bd461a4cf72a80d network default NAD default], result "{\"interfaces\":[{\"mac\":\"fa:44:63:6e:01:81\",\"name\":\"977efd03a45e9d1\"},{\"mac\":\"0a:58:0a:81:00:13\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a481a054-e37e-4157-ac3a-04dab2bd766a\"}],\"ips\":[{\"address\":\"10.129.0.19/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0406 02:57:09.529261 3776 cni.go:424] [openshift-monitoring/openshift-state-metrics-77dc8fdc5c-6f8gb 8fabf971b34d3a213e1e0314e119e3325d18ac82a93f105dbb8f821072e3dfb3 network default NAD default] ADD finished CNI request [openshift-monitoring/openshift-state-metrics-77dc8fdc5c-6f8gb 8fabf971b34d3a213e1e0314e119e3325d18ac82a93f105dbb8f821072e3dfb3 network default NAD default], result "{\"interfaces\":[{\"mac\":\"82:71:46:78:94:a9\",\"name\":\"8fabf971b34d3a2\"},{\"mac\":\"0a:58:0a:81:00:10\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b76b4457-0323-4125-a1b5-a72444c5b4d0\"}],\"ips\":[{\"address\":\"10.129.0.16/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0406 02:57:09.529371 3776 cni.go:424] [openshift-monitoring/prometheus-k8s-1 e997191d1c771335e0dec1638571757f55910d5f582f940e408462b57f315eab network default NAD default] ADD finished CNI request [openshift-monitoring/prometheus-k8s-1 e997191d1c771335e0dec1638571757f55910d5f582f940e408462b57f315eab network default NAD default], result "{\"interfaces\":[{\"mac\":\"3e:a1:96:6b:2f:42\",\"name\":\"e997191d1c77133\"},{\"mac\":\"0a:58:0a:81:00:15\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/99c83dd3-2368-4c3a-8205-04aab9072cb2\"}],\"ips\":[{\"address\":\"10.129.0.21/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0406 02:57:10.036608 3776 util.go:959] Failed to build node endpoints for node ip-10-0-2-216.ec2.internal port 8080: empty IP address endpoints for service openshift-console/downloads W0406 02:57:10.389798 3776 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0406 02:57:11.396253 3776 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0406 02:57:11.396276 3776 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0406 02:57:11.396282 3776 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0406 02:57:11.396287 3776 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0406 02:57:12.859259 3776 util.go:959] Failed to build node endpoints for node ip-10-0-2-216.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0406 02:57:12.896496 3776 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-image-registry/image-registry-6f4c66dc87-r5smh IP address from the namespace address-set, err: pod openshift-image-registry/image-registry-6f4c66dc87-r5smh: no pod IPs found I0406 02:57:14.466601 3776 cni.go:403] [openshift-multus/network-metrics-daemon-h8vhx df6bffab988afcdb1e01302eec086cfde75df4312369f9530f81ba6430167fd2 network default NAD default] ADD starting CNI request [openshift-multus/network-metrics-daemon-h8vhx df6bffab988afcdb1e01302eec086cfde75df4312369f9530f81ba6430167fd2 network default NAD default] I0406 02:57:14.478332 3776 helper_linux.go:450] ConfigureOVS: namespace: openshift-multus, podName: network-metrics-daemon-h8vhx, hostIfaceName: df6bffab988afcd, network: default, NAD default, SandboxID: "df6bffab988afcdb1e01302eec086cfde75df4312369f9530f81ba6430167fd2", PCI device ID: , UID: "73d07508-3749-462a-b4b5-d2f0dd3a58b8", MAC: 0a:58:0a:81:00:03, IPs: [10.129.0.3/23] I0406 02:57:14.535004 3776 cni.go:424] [openshift-multus/network-metrics-daemon-h8vhx df6bffab988afcdb1e01302eec086cfde75df4312369f9530f81ba6430167fd2 network default NAD default] ADD finished CNI request [openshift-multus/network-metrics-daemon-h8vhx df6bffab988afcdb1e01302eec086cfde75df4312369f9530f81ba6430167fd2 network default NAD default], result "{\"interfaces\":[{\"mac\":\"fa:62:0b:7a:7b:d3\",\"name\":\"df6bffab988afcd\"},{\"mac\":\"0a:58:0a:81:00:03\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/fbce4af7-7373-4df5-a865-5b64c5f3f66c\"}],\"ips\":[{\"address\":\"10.129.0.3/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0406 02:57:14.693085 3776 obj_retry.go:464] Detected object openshift-image-registry/image-registry-68b596bb54-4w84r of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0406 02:57:14.693664 3776 obj_retry.go:464] Detected object openshift-image-registry/image-registry-68b596bb54-4w84r of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0406 02:57:14.774823 3776 cni.go:403] [openshift-network-diagnostics/network-check-target-nc244 7f0048c64087b5b4d167349d997023f538c43b2727562b7bfd2ecfe766bef449 network default NAD default] ADD starting CNI request [openshift-network-diagnostics/network-check-target-nc244 7f0048c64087b5b4d167349d997023f538c43b2727562b7bfd2ecfe766bef449 network default NAD default] I0406 02:57:14.786529 3776 helper_linux.go:450] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-target-nc244, hostIfaceName: 7f0048c64087b5b, network: default, NAD default, SandboxID: "7f0048c64087b5b4d167349d997023f538c43b2727562b7bfd2ecfe766bef449", PCI device ID: , UID: "1cfa4b33-d319-4bd9-b7d4-0c51c665610b", MAC: 0a:58:0a:81:00:05, IPs: [10.129.0.5/23] I0406 02:57:14.843590 3776 cni.go:424] [openshift-network-diagnostics/network-check-target-nc244 7f0048c64087b5b4d167349d997023f538c43b2727562b7bfd2ecfe766bef449 network default NAD default] ADD finished CNI request [openshift-network-diagnostics/network-check-target-nc244 7f0048c64087b5b4d167349d997023f538c43b2727562b7bfd2ecfe766bef449 network default NAD default], result "{\"interfaces\":[{\"mac\":\"3e:e4:28:8f:fc:18\",\"name\":\"7f0048c64087b5b\"},{\"mac\":\"0a:58:0a:81:00:05\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/472ba220-7140-4e5e-a807-65ec338ce1b7\"}],\"ips\":[{\"address\":\"10.129.0.5/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0406 02:57:14.991967 3776 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0406 02:57:14.991989 3776 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0406 02:57:14.991996 3776 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0406 02:57:15.036402 3776 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0406 02:57:15.077909 3776 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0406 02:57:15.077932 3776 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0406 02:57:15.567304 3776 obj_retry.go:458] Detected object openshift-image-registry/image-registry-68b596bb54-4w84r of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 02:57:15.567418 3776 obj_retry.go:458] Detected object openshift-image-registry/image-registry-68b596bb54-4w84r of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 02:57:15.570182 3776 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-68b596bb54-4w84r I0406 02:57:15.570212 3776 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-68b596bb54-4w84r I0406 02:57:22.673973 3776 cni.go:403] [kube-system/global-pull-secret-syncer-gzf4p b52b16d08e012ad36584cd695625f68b662e7824846c60dedcba88cf35e7c231 network default NAD default] ADD starting CNI request [kube-system/global-pull-secret-syncer-gzf4p b52b16d08e012ad36584cd695625f68b662e7824846c60dedcba88cf35e7c231 network default NAD default] I0406 02:57:22.684244 3776 helper_linux.go:450] ConfigureOVS: namespace: kube-system, podName: global-pull-secret-syncer-gzf4p, hostIfaceName: b52b16d08e012ad, network: default, NAD default, SandboxID: "b52b16d08e012ad36584cd695625f68b662e7824846c60dedcba88cf35e7c231", PCI device ID: , UID: "eaa4b671-99aa-424e-abc0-14f1315c35d0", MAC: 0a:58:0a:81:00:04, IPs: [10.129.0.4/23] I0406 02:57:22.741837 3776 cni.go:424] [kube-system/global-pull-secret-syncer-gzf4p b52b16d08e012ad36584cd695625f68b662e7824846c60dedcba88cf35e7c231 network default NAD default] ADD finished CNI request [kube-system/global-pull-secret-syncer-gzf4p b52b16d08e012ad36584cd695625f68b662e7824846c60dedcba88cf35e7c231 network default NAD default], result "{\"interfaces\":[{\"mac\":\"8e:6d:8f:87:57:8b\",\"name\":\"b52b16d08e012ad\"},{\"mac\":\"0a:58:0a:81:00:04\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e536aaa4-3506-4d9a-a537-bd48e2438ff2\"}],\"ips\":[{\"address\":\"10.129.0.4/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0406 02:57:24.051855 3776 util.go:959] Failed to build node endpoints for node ip-10-0-2-216.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/thanos-querier W0406 02:57:24.051972 3776 util.go:959] Failed to build node endpoints for node ip-10-0-2-216.ec2.internal port 9094: empty IP address endpoints for service openshift-monitoring/thanos-querier W0406 02:57:24.052003 3776 util.go:959] Failed to build node endpoints for node ip-10-0-2-216.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/thanos-querier W0406 02:57:24.052029 3776 util.go:959] Failed to build node endpoints for node ip-10-0-2-216.ec2.internal port 9093: empty IP address endpoints for service openshift-monitoring/thanos-querier I0406 02:57:25.080342 3776 cni.go:403] [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf5653908jjng d87bd4e286d0e668497644a6a3dc9e66f2bf6246307501b10b59fb4d07ea5b28 network default NAD default] DEL starting CNI request [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf5653908jjng d87bd4e286d0e668497644a6a3dc9e66f2bf6246307501b10b59fb4d07ea5b28 network default NAD default] I0406 02:57:25.161000 3776 cni.go:424] [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf5653908jjng d87bd4e286d0e668497644a6a3dc9e66f2bf6246307501b10b59fb4d07ea5b28 network default NAD default] DEL finished CNI request [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf5653908jjng d87bd4e286d0e668497644a6a3dc9e66f2bf6246307501b10b59fb4d07ea5b28 network default NAD default], result "{}", err I0406 02:57:25.220993 3776 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf5653908jjng of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0406 02:57:25.221035 3776 pods.go:176] Deleting pod: openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf5653908jjng I0406 02:57:25.222510 3776 pods.go:236] Attempting to release IPs for pod: openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf5653908jjng, ips: 10.129.0.13 I0406 02:57:25.222546 3776 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf5653908jjng of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0406 02:57:25.222572 3776 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf5653908jjng of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0406 02:57:25.222587 3776 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles to delete 1 local pods... I0406 02:57:25.222829 3776 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf5653908jjng of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0406 02:57:25.222845 3776 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... I0406 02:57:26.250559 3776 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf5653908jjng of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 02:57:26.250590 3776 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf5653908jjng of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 02:57:26.250606 3776 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf5653908jjng of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0406 02:57:26.250620 3776 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf5653908jjng of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0406 02:57:26.583565 3776 obj_retry.go:464] Detected object openshift-console/console-69796cc9d4-zzk9f of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0406 02:57:26.584160 3776 obj_retry.go:464] Detected object openshift-console/console-69796cc9d4-zzk9f of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0406 02:57:27.483214 3776 obj_retry.go:458] Detected object openshift-console/console-69796cc9d4-zzk9f of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 02:57:27.483238 3776 obj_retry.go:458] Detected object openshift-console/console-69796cc9d4-zzk9f of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 02:57:27.488561 3776 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-69796cc9d4-zzk9f I0406 02:57:27.488587 3776 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-69796cc9d4-zzk9f I0406 02:57:28.780646 3776 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0406 02:57:28.792664 3776 namespace.go:142] [openshift-deployment-validation-operator] updating namespace W0406 02:57:29.630990 3776 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-insights/metrics W0406 02:57:31.403985 3776 util.go:959] Failed to build node endpoints for node ip-10-0-2-216.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server W0406 02:57:31.404013 3776 util.go:959] Failed to build node endpoints for node ip-10-0-2-216.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server W0406 02:57:33.150401 3776 util.go:959] Failed to build node endpoints for node ip-10-0-2-216.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0406 02:57:33.150426 3776 util.go:959] Failed to build node endpoints for node ip-10-0-2-216.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0406 02:57:33.390816 3776 util.go:959] Failed to build node endpoints for node ip-10-0-2-216.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0406 02:57:33.390839 3776 util.go:959] Failed to build node endpoints for node ip-10-0-2-216.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0406 02:57:33.390846 3776 util.go:959] Failed to build node endpoints for node ip-10-0-2-216.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0406 02:57:33.390866 3776 util.go:959] Failed to build node endpoints for node ip-10-0-2-216.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0406 02:57:33.390873 3776 util.go:959] Failed to build node endpoints for node ip-10-0-2-216.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0406 02:57:33.390880 3776 util.go:959] Failed to build node endpoints for node ip-10-0-2-216.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0406 02:57:34.826359 3776 obj_retry.go:464] Detected object openshift-console/console-7788bfb9bb-pb4sx of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0406 02:57:34.826932 3776 obj_retry.go:464] Detected object openshift-console/console-7788bfb9bb-pb4sx of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0406 02:57:35.674910 3776 obj_retry.go:458] Detected object openshift-console/console-7788bfb9bb-pb4sx of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 02:57:35.674935 3776 obj_retry.go:458] Detected object openshift-console/console-7788bfb9bb-pb4sx of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 02:57:35.680531 3776 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-7788bfb9bb-pb4sx I0406 02:57:35.680561 3776 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-7788bfb9bb-pb4sx W0406 02:57:36.012866 3776 util.go:959] Failed to build node endpoints for node ip-10-0-2-216.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server W0406 02:57:43.387954 3776 util.go:959] Failed to build node endpoints for node ip-10-0-2-216.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0406 02:57:43.387976 3776 util.go:959] Failed to build node endpoints for node ip-10-0-2-216.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0406 02:57:43.387983 3776 util.go:959] Failed to build node endpoints for node ip-10-0-2-216.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0406 02:57:53.755057 3776 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-service-ca-operator/metrics W0406 02:57:54.073676 3776 util.go:959] Failed to build node endpoints for node ip-10-0-2-216.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0406 02:57:58.766194 3776 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics I0406 02:58:02.733927 3776 cni.go:403] [openshift-console/console-69796cc9d4-djt8q 360bff00044787ebebba06684163270d35e342c0d0b6c8d43ba17e6de2e20392 network default NAD default] DEL starting CNI request [openshift-console/console-69796cc9d4-djt8q 360bff00044787ebebba06684163270d35e342c0d0b6c8d43ba17e6de2e20392 network default NAD default] I0406 02:58:02.813945 3776 cni.go:424] [openshift-console/console-69796cc9d4-djt8q 360bff00044787ebebba06684163270d35e342c0d0b6c8d43ba17e6de2e20392 network default NAD default] DEL finished CNI request [openshift-console/console-69796cc9d4-djt8q 360bff00044787ebebba06684163270d35e342c0d0b6c8d43ba17e6de2e20392 network default NAD default], result "{}", err I0406 02:58:02.885564 3776 obj_retry.go:464] Detected object openshift-console/console-69796cc9d4-djt8q of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0406 02:58:02.885605 3776 pods.go:176] Deleting pod: openshift-console/console-69796cc9d4-djt8q I0406 02:58:02.886960 3776 pods.go:236] Attempting to release IPs for pod: openshift-console/console-69796cc9d4-djt8q, ips: 10.129.0.14 I0406 02:58:02.886988 3776 obj_retry.go:464] Detected object openshift-console/console-69796cc9d4-djt8q of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0406 02:58:03.240877 3776 obj_retry.go:458] Detected object openshift-console/console-69796cc9d4-djt8q of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 02:58:03.240914 3776 obj_retry.go:458] Detected object openshift-console/console-69796cc9d4-djt8q of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 02:58:03.244306 3776 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-69796cc9d4-djt8q I0406 02:58:03.244346 3776 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-69796cc9d4-djt8q I0406 02:58:04.338778 3776 obj_retry.go:464] Detected object openshift-ingress/router-default-566948cd75-l6vhx of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0406 02:58:04.339426 3776 obj_retry.go:464] Detected object openshift-ingress/router-default-566948cd75-l6vhx of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0406 02:58:04.785506 3776 obj_retry.go:458] Detected object openshift-ingress/router-default-566948cd75-l6vhx of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 02:58:04.785534 3776 obj_retry.go:458] Detected object openshift-ingress/router-default-566948cd75-l6vhx of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 02:58:04.790498 3776 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/router-default-566948cd75-l6vhx I0406 02:58:04.790532 3776 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/router-default-566948cd75-l6vhx I0406 02:58:13.619995 3776 cni.go:403] [openshift-monitoring/alertmanager-main-1 587698c74a82ec20ebc0a579ab6dab7b133e6b86b2f0310848d7db268d5f1e19 network default NAD default] DEL starting CNI request [openshift-monitoring/alertmanager-main-1 587698c74a82ec20ebc0a579ab6dab7b133e6b86b2f0310848d7db268d5f1e19 network default NAD default] I0406 02:58:13.698977 3776 cni.go:424] [openshift-monitoring/alertmanager-main-1 587698c74a82ec20ebc0a579ab6dab7b133e6b86b2f0310848d7db268d5f1e19 network default NAD default] DEL finished CNI request [openshift-monitoring/alertmanager-main-1 587698c74a82ec20ebc0a579ab6dab7b133e6b86b2f0310848d7db268d5f1e19 network default NAD default], result "{}", err I0406 02:58:13.770800 3776 obj_retry.go:464] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0406 02:58:13.770839 3776 pods.go:176] Deleting pod: openshift-monitoring/alertmanager-main-1 I0406 02:58:13.772372 3776 pods.go:236] Attempting to release IPs for pod: openshift-monitoring/alertmanager-main-1, ips: 10.129.0.17 I0406 02:58:13.772408 3776 obj_retry.go:464] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0406 02:58:14.293939 3776 obj_retry.go:458] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 02:58:14.293972 3776 obj_retry.go:458] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 02:58:14.300894 3776 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-1 I0406 02:58:14.300939 3776 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-1 I0406 02:58:14.340887 3776 base_network_controller_pods.go:487] [default/openshift-monitoring/alertmanager-main-1] creating logical port openshift-monitoring_alertmanager-main-1 for pod on switch ip-10-0-2-216.ec2.internal I0406 02:58:14.341032 3776 kube.go:256] Updating pod openshift-monitoring/alertmanager-main-1 I0406 02:58:14.362536 3776 pod.go:62] [openshift-monitoring/alertmanager-main-1] pod update took 21.547257ms I0406 02:58:14.362560 3776 base_network_controller_pods.go:937] [default/openshift-monitoring/alertmanager-main-1] addLogicalPort annotation time took 21.575947ms I0406 02:58:14.363838 3776 pods.go:271] [openshift-monitoring/alertmanager-main-1] addLogicalPort took 22.967093ms, libovsdb time 965.443µs I0406 02:58:14.761892 3776 cni.go:403] [openshift-monitoring/alertmanager-main-1 a1dad503ef99ca921da5cc293664612fba364f272ce0862469dd83221dcdde84 network default NAD default] ADD starting CNI request [openshift-monitoring/alertmanager-main-1 a1dad503ef99ca921da5cc293664612fba364f272ce0862469dd83221dcdde84 network default NAD default] I0406 02:58:14.771773 3776 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: alertmanager-main-1, hostIfaceName: a1dad503ef99ca9, network: default, NAD default, SandboxID: "a1dad503ef99ca921da5cc293664612fba364f272ce0862469dd83221dcdde84", PCI device ID: , UID: "09f137b2-53bc-4f4d-b147-85b711764717", MAC: 0a:58:0a:81:00:16, IPs: [10.129.0.22/23] I0406 02:58:14.826016 3776 cni.go:424] [openshift-monitoring/alertmanager-main-1 a1dad503ef99ca921da5cc293664612fba364f272ce0862469dd83221dcdde84 network default NAD default] ADD finished CNI request [openshift-monitoring/alertmanager-main-1 a1dad503ef99ca921da5cc293664612fba364f272ce0862469dd83221dcdde84 network default NAD default], result "{\"interfaces\":[{\"mac\":\"d2:e1:85:d6:1a:d2\",\"name\":\"a1dad503ef99ca9\"},{\"mac\":\"0a:58:0a:81:00:16\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/85ea56b4-dc88-4b55-b07a-9d9976df4590\"}],\"ips\":[{\"address\":\"10.129.0.22/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0406 02:58:17.854320 3776 cni.go:403] [openshift-monitoring/prometheus-k8s-1 e997191d1c771335e0dec1638571757f55910d5f582f940e408462b57f315eab network default NAD default] DEL starting CNI request [openshift-monitoring/prometheus-k8s-1 e997191d1c771335e0dec1638571757f55910d5f582f940e408462b57f315eab network default NAD default] I0406 02:58:17.935872 3776 cni.go:424] [openshift-monitoring/prometheus-k8s-1 e997191d1c771335e0dec1638571757f55910d5f582f940e408462b57f315eab network default NAD default] DEL finished CNI request [openshift-monitoring/prometheus-k8s-1 e997191d1c771335e0dec1638571757f55910d5f582f940e408462b57f315eab network default NAD default], result "{}", err I0406 02:58:18.015931 3776 obj_retry.go:464] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0406 02:58:18.015969 3776 pods.go:176] Deleting pod: openshift-monitoring/prometheus-k8s-1 I0406 02:58:18.017259 3776 pods.go:236] Attempting to release IPs for pod: openshift-monitoring/prometheus-k8s-1, ips: 10.129.0.21 I0406 02:58:18.017288 3776 obj_retry.go:464] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0406 02:58:18.329663 3776 obj_retry.go:458] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 02:58:18.329691 3776 obj_retry.go:458] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 02:58:18.337754 3776 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-1 I0406 02:58:18.337829 3776 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-1 I0406 02:58:18.380724 3776 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-k8s-1] creating logical port openshift-monitoring_prometheus-k8s-1 for pod on switch ip-10-0-2-216.ec2.internal I0406 02:58:18.380840 3776 kube.go:256] Updating pod openshift-monitoring/prometheus-k8s-1 I0406 02:58:18.404544 3776 pod.go:62] [openshift-monitoring/prometheus-k8s-1] pod update took 23.741357ms I0406 02:58:18.404569 3776 base_network_controller_pods.go:937] [default/openshift-monitoring/prometheus-k8s-1] addLogicalPort annotation time took 23.772804ms I0406 02:58:18.406085 3776 pods.go:271] [openshift-monitoring/prometheus-k8s-1] addLogicalPort took 25.378279ms, libovsdb time 1.144647ms I0406 02:58:18.800366 3776 cni.go:403] [openshift-monitoring/prometheus-k8s-1 eb1414db99c1e5cda45a220e6aa5b5b6c2452b6c4c9ad483185e092743538c93 network default NAD default] ADD starting CNI request [openshift-monitoring/prometheus-k8s-1 eb1414db99c1e5cda45a220e6aa5b5b6c2452b6c4c9ad483185e092743538c93 network default NAD default] I0406 02:58:18.810055 3776 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-k8s-1, hostIfaceName: eb1414db99c1e5c, network: default, NAD default, SandboxID: "eb1414db99c1e5cda45a220e6aa5b5b6c2452b6c4c9ad483185e092743538c93", PCI device ID: , UID: "33585c7b-35b9-4ebe-8522-8b073c75eeca", MAC: 0a:58:0a:81:00:17, IPs: [10.129.0.23/23] I0406 02:58:18.860499 3776 cni.go:424] [openshift-monitoring/prometheus-k8s-1 eb1414db99c1e5cda45a220e6aa5b5b6c2452b6c4c9ad483185e092743538c93 network default NAD default] ADD finished CNI request [openshift-monitoring/prometheus-k8s-1 eb1414db99c1e5cda45a220e6aa5b5b6c2452b6c4c9ad483185e092743538c93 network default NAD default], result "{\"interfaces\":[{\"mac\":\"2e:ef:8b:f1:2f:1d\",\"name\":\"eb1414db99c1e5c\"},{\"mac\":\"0a:58:0a:81:00:17\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0e990536-21f5-4df8-a85c-ef0b3c3eb3d0\"}],\"ips\":[{\"address\":\"10.129.0.23/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0406 02:58:19.945403 3776 base_network_controller_pods.go:487] [default/openshift-console/console-59f5544487-mfh6j] creating logical port openshift-console_console-59f5544487-mfh6j for pod on switch ip-10-0-2-216.ec2.internal I0406 02:58:19.945553 3776 kube.go:256] Updating pod openshift-console/console-59f5544487-mfh6j I0406 02:58:19.965000 3776 pod.go:62] [openshift-console/console-59f5544487-mfh6j] pod update took 19.473462ms I0406 02:58:19.965091 3776 base_network_controller_pods.go:937] [default/openshift-console/console-59f5544487-mfh6j] addLogicalPort annotation time took 19.568716ms I0406 02:58:19.966583 3776 pods.go:271] [openshift-console/console-59f5544487-mfh6j] addLogicalPort took 21.197504ms, libovsdb time 1.131178ms I0406 02:58:20.339922 3776 cni.go:403] [openshift-console/console-59f5544487-mfh6j 6764ad9590b4afb52871c23b92c5ab36c0e9afe3ec503c953634ce5ec438d3e9 network default NAD default] ADD starting CNI request [openshift-console/console-59f5544487-mfh6j 6764ad9590b4afb52871c23b92c5ab36c0e9afe3ec503c953634ce5ec438d3e9 network default NAD default] I0406 02:58:20.350510 3776 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-59f5544487-mfh6j, hostIfaceName: 6764ad9590b4afb, network: default, NAD default, SandboxID: "6764ad9590b4afb52871c23b92c5ab36c0e9afe3ec503c953634ce5ec438d3e9", PCI device ID: , UID: "9d070ff6-b511-4917-ba72-3e5284cf6cdc", MAC: 0a:58:0a:81:00:18, IPs: [10.129.0.24/23] I0406 02:58:20.406977 3776 cni.go:424] [openshift-console/console-59f5544487-mfh6j 6764ad9590b4afb52871c23b92c5ab36c0e9afe3ec503c953634ce5ec438d3e9 network default NAD default] ADD finished CNI request [openshift-console/console-59f5544487-mfh6j 6764ad9590b4afb52871c23b92c5ab36c0e9afe3ec503c953634ce5ec438d3e9 network default NAD default], result "{\"interfaces\":[{\"mac\":\"d2:f7:f0:fc:7b:35\",\"name\":\"6764ad9590b4afb\"},{\"mac\":\"0a:58:0a:81:00:18\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/800b4262-733a-43eb-8da0-35f23065ec64\"}],\"ips\":[{\"address\":\"10.129.0.24/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0406 02:58:30.405918 3776 util.go:959] Failed to build node endpoints for node ip-10-0-2-216.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0406 02:58:44.786871 3776 util.go:959] Failed to build node endpoints for node ip-10-0-2-216.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0406 02:58:44.786902 3776 util.go:959] Failed to build node endpoints for node ip-10-0-2-216.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0406 02:58:44.786911 3776 util.go:959] Failed to build node endpoints for node ip-10-0-2-216.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0406 02:58:45.236066 3776 obj_retry.go:464] Detected object openshift-console/console-85cd798b89-xmrlz of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0406 02:58:45.236636 3776 obj_retry.go:464] Detected object openshift-console/console-85cd798b89-xmrlz of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0406 02:58:45.775779 3776 obj_retry.go:458] Detected object openshift-console/console-85cd798b89-xmrlz of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 02:58:45.775814 3776 obj_retry.go:458] Detected object openshift-console/console-85cd798b89-xmrlz of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 02:58:45.781543 3776 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-85cd798b89-xmrlz I0406 02:58:45.781575 3776 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-85cd798b89-xmrlz I0406 02:58:46.073269 3776 obj_retry.go:464] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0406 02:58:46.073857 3776 obj_retry.go:464] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0406 02:58:46.811564 3776 obj_retry.go:458] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 02:58:46.811598 3776 obj_retry.go:458] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 02:58:46.817915 3776 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-0 I0406 02:58:46.817939 3776 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-0 I0406 02:58:55.839241 3776 obj_retry.go:464] Detected object openshift-console/console-85cd798b89-4n5hg of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0406 02:58:55.839831 3776 obj_retry.go:464] Detected object openshift-console/console-85cd798b89-4n5hg of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0406 02:58:55.988454 3776 obj_retry.go:458] Detected object openshift-console/console-85cd798b89-4n5hg of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 02:58:55.988481 3776 obj_retry.go:458] Detected object openshift-console/console-85cd798b89-4n5hg of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 02:58:55.993736 3776 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-85cd798b89-4n5hg I0406 02:58:55.993774 3776 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-85cd798b89-4n5hg W0406 02:59:19.657666 3776 util.go:959] Failed to build node endpoints for node ip-10-0-2-216.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0406 02:59:19.657693 3776 util.go:959] Failed to build node endpoints for node ip-10-0-2-216.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0406 02:59:20.972525 3776 obj_retry.go:464] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0406 02:59:20.973113 3776 obj_retry.go:464] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0406 02:59:21.955981 3776 obj_retry.go:458] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 02:59:21.956015 3776 obj_retry.go:458] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 02:59:21.963481 3776 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-0 I0406 02:59:21.963509 3776 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-0 I0406 03:00:05.358849 3776 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29590740-fvg78 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0406 03:00:05.359427 3776 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29590740-fvg78 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0406 03:00:06.263997 3776 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29590740-fvg78 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:00:06.264022 3776 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29590740-fvg78 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:00:52.294994 3776 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0406 03:00:56.701200 3776 cni.go:403] [openshift-console/console-59f5544487-mfh6j 6764ad9590b4afb52871c23b92c5ab36c0e9afe3ec503c953634ce5ec438d3e9 network default NAD default] DEL starting CNI request [openshift-console/console-59f5544487-mfh6j 6764ad9590b4afb52871c23b92c5ab36c0e9afe3ec503c953634ce5ec438d3e9 network default NAD default] I0406 03:00:56.781386 3776 cni.go:424] [openshift-console/console-59f5544487-mfh6j 6764ad9590b4afb52871c23b92c5ab36c0e9afe3ec503c953634ce5ec438d3e9 network default NAD default] DEL finished CNI request [openshift-console/console-59f5544487-mfh6j 6764ad9590b4afb52871c23b92c5ab36c0e9afe3ec503c953634ce5ec438d3e9 network default NAD default], result "{}", err I0406 03:00:56.827361 3776 obj_retry.go:464] Detected object openshift-console/console-59f5544487-mfh6j of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0406 03:00:56.827409 3776 pods.go:176] Deleting pod: openshift-console/console-59f5544487-mfh6j I0406 03:00:56.828914 3776 pods.go:236] Attempting to release IPs for pod: openshift-console/console-59f5544487-mfh6j, ips: 10.129.0.24 I0406 03:00:56.828946 3776 obj_retry.go:464] Detected object openshift-console/console-59f5544487-mfh6j of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0406 03:00:56.836195 3776 base_network_controller_pods.go:487] [default/openshift-console/console-6674554896-wrsbk] creating logical port openshift-console_console-6674554896-wrsbk for pod on switch ip-10-0-2-216.ec2.internal I0406 03:00:56.836320 3776 kube.go:256] Updating pod openshift-console/console-6674554896-wrsbk I0406 03:00:56.849994 3776 pod.go:62] [openshift-console/console-6674554896-wrsbk] pod update took 13.69935ms I0406 03:00:56.850022 3776 base_network_controller_pods.go:937] [default/openshift-console/console-6674554896-wrsbk] addLogicalPort annotation time took 13.734007ms I0406 03:00:56.851400 3776 pods.go:271] [openshift-console/console-6674554896-wrsbk] addLogicalPort took 15.218152ms, libovsdb time 1.017085ms I0406 03:00:56.996059 3776 obj_retry.go:458] Detected object openshift-console/console-59f5544487-mfh6j of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:00:56.996085 3776 obj_retry.go:458] Detected object openshift-console/console-59f5544487-mfh6j of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:00:57.001990 3776 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-59f5544487-mfh6j I0406 03:00:57.002027 3776 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-59f5544487-mfh6j I0406 03:00:57.196077 3776 cni.go:403] [openshift-console/console-6674554896-wrsbk 360978daf234a1b120c3e29ac137881a9384af1d4806417da5d287fb4bcdeb8b network default NAD default] ADD starting CNI request [openshift-console/console-6674554896-wrsbk 360978daf234a1b120c3e29ac137881a9384af1d4806417da5d287fb4bcdeb8b network default NAD default] I0406 03:00:57.205626 3776 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-6674554896-wrsbk, hostIfaceName: 360978daf234a1b, network: default, NAD default, SandboxID: "360978daf234a1b120c3e29ac137881a9384af1d4806417da5d287fb4bcdeb8b", PCI device ID: , UID: "ce2dc482-436c-4783-8e79-07770685307d", MAC: 0a:58:0a:81:00:19, IPs: [10.129.0.25/23] I0406 03:00:57.256645 3776 cni.go:424] [openshift-console/console-6674554896-wrsbk 360978daf234a1b120c3e29ac137881a9384af1d4806417da5d287fb4bcdeb8b network default NAD default] ADD finished CNI request [openshift-console/console-6674554896-wrsbk 360978daf234a1b120c3e29ac137881a9384af1d4806417da5d287fb4bcdeb8b network default NAD default], result "{\"interfaces\":[{\"mac\":\"36:a7:9d:e5:e4:96\",\"name\":\"360978daf234a1b\"},{\"mac\":\"0a:58:0a:81:00:19\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/331e68b0-fcea-4b1e-9e00-6e3572492277\"}],\"ips\":[{\"address\":\"10.129.0.25/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0406 03:01:07.703856 3776 obj_retry.go:464] Detected object openshift-console/console-59f5544487-t98g7 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0406 03:01:07.704460 3776 obj_retry.go:464] Detected object openshift-console/console-59f5544487-t98g7 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0406 03:01:08.062426 3776 util.go:959] Failed to build node endpoints for node ip-10-0-2-216.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0406 03:01:08.366086 3776 obj_retry.go:458] Detected object openshift-console/console-59f5544487-t98g7 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:01:08.366149 3776 obj_retry.go:458] Detected object openshift-console/console-59f5544487-t98g7 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:01:08.371280 3776 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-59f5544487-t98g7 I0406 03:01:08.371299 3776 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-59f5544487-t98g7 I0406 03:01:39.761360 3776 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-4.ec2.internal I0406 03:01:39.761404 3776 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-4.ec2.internal: 51.282µs I0406 03:01:39.761473 3776 egressfirewall.go:628] Syncing node "ip-10-0-1-4.ec2.internal" for egress firewall I0406 03:02:23.103940 3776 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=7 W0406 03:02:33.811683 3776 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0406 03:02:37.093289 3776 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=7 I0406 03:02:42.085416 3776 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=8 I0406 03:02:42.998884 3776 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=53 I0406 03:02:44.858703 3776 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... W0406 03:02:44.858773 3776 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-deployment-validation-operator/deployment-validation-operator-847f6f7b8-wjtqn IP address from the namespace address-set, err: pod openshift-deployment-validation-operator/deployment-validation-operator-847f6f7b8-wjtqn: no pod IPs found I0406 03:03:32.082973 3776 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=8 I0406 03:03:33.110483 3776 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=8 I0406 03:03:44.098509 3776 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=8 I0406 03:04:32.115205 3776 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=8 I0406 03:04:38.121254 3776 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=10 I0406 03:05:12.087981 3776 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=10 I0406 03:05:30.121290 3776 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=9 I0406 03:05:34.999589 3776 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=71 I0406 03:06:01.305786 3776 namespace.go:98] [tsf] adding namespace I0406 03:06:01.306647 3776 namespace.go:102] [tsf] adding namespace took 832.81µs I0406 03:06:01.324174 3776 namespace.go:142] [tsf] updating namespace I0406 03:06:01.334371 3776 namespace.go:142] [tsf] updating namespace I0406 03:06:04.000040 3776 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=136 I0406 03:06:05.192282 3776 namespace.go:98] [tsf-keycloak] adding namespace I0406 03:06:05.192417 3776 namespace.go:98] [rhbk-operator] adding namespace I0406 03:06:05.193250 3776 namespace.go:102] [tsf-keycloak] adding namespace took 946.994µs I0406 03:06:05.193838 3776 namespace.go:102] [rhbk-operator] adding namespace took 1.399504ms I0406 03:06:05.198027 3776 namespace.go:98] [rhtpa-operator] adding namespace I0406 03:06:05.198043 3776 namespace.go:98] [konflux-ui] adding namespace I0406 03:06:05.198283 3776 namespace.go:98] [tsf-tpa] adding namespace I0406 03:06:05.198810 3776 namespace.go:102] [rhtpa-operator] adding namespace took 767.783µs I0406 03:06:05.199419 3776 namespace.go:102] [konflux-ui] adding namespace took 1.362202ms I0406 03:06:05.199927 3776 namespace.go:102] [tsf-tpa] adding namespace took 1.628814ms I0406 03:06:05.201296 3776 namespace.go:98] [cert-manager-operator] adding namespace I0406 03:06:05.201357 3776 namespace.go:98] [tsf-tas] adding namespace I0406 03:06:05.201523 3776 namespace.go:98] [konflux-operator] adding namespace I0406 03:06:05.201886 3776 namespace.go:102] [cert-manager-operator] adding namespace took 575.143µs I0406 03:06:05.202419 3776 namespace.go:102] [tsf-tas] adding namespace took 1.048308ms I0406 03:06:05.202893 3776 namespace.go:102] [konflux-operator] adding namespace took 1.357577ms I0406 03:06:05.211670 3776 namespace.go:142] [tsf-keycloak] updating namespace I0406 03:06:05.227186 3776 namespace.go:142] [tsf-keycloak] updating namespace I0406 03:06:05.234316 3776 namespace.go:142] [rhbk-operator] updating namespace I0406 03:06:05.234346 3776 namespace.go:142] [rhtpa-operator] updating namespace I0406 03:06:05.239490 3776 namespace.go:142] [tsf-tpa] updating namespace I0406 03:06:05.243381 3776 namespace.go:142] [konflux-ui] updating namespace I0406 03:06:05.249469 3776 namespace.go:142] [tsf-tas] updating namespace I0406 03:06:05.249480 3776 namespace.go:142] [cert-manager-operator] updating namespace I0406 03:06:05.249643 3776 namespace.go:142] [rhbk-operator] updating namespace I0406 03:06:05.251072 3776 namespace.go:142] [konflux-operator] updating namespace I0406 03:06:05.251187 3776 namespace.go:142] [tsf-keycloak] updating namespace I0406 03:06:05.284303 3776 namespace.go:142] [rhtpa-operator] updating namespace I0406 03:06:05.295451 3776 namespace.go:142] [rhtpa-operator] updating namespace I0406 03:06:05.307809 3776 namespace.go:142] [tsf-tpa] updating namespace I0406 03:06:05.321849 3776 namespace.go:142] [tsf-tpa] updating namespace I0406 03:06:05.325844 3776 namespace.go:142] [cert-manager-operator] updating namespace I0406 03:06:05.339524 3776 namespace.go:142] [cert-manager-operator] updating namespace I0406 03:06:05.345174 3776 namespace.go:142] [tsf-tas] updating namespace I0406 03:06:05.356478 3776 namespace.go:142] [tsf-tas] updating namespace I0406 03:06:05.370743 3776 namespace.go:142] [konflux-operator] updating namespace I0406 03:06:05.381530 3776 namespace.go:142] [konflux-operator] updating namespace I0406 03:06:05.388197 3776 namespace.go:142] [konflux-ui] updating namespace I0406 03:06:05.398109 3776 namespace.go:142] [konflux-ui] updating namespace I0406 03:06:05.418685 3776 namespace.go:142] [rhbk-operator] updating namespace I0406 03:06:06.352187 3776 namespace.go:142] [tsf-tpa] updating namespace I0406 03:06:06.367593 3776 namespace.go:142] [tsf-keycloak] updating namespace I0406 03:06:07.965294 3776 base_network_controller_pods.go:487] [default/openshift-marketplace/f62a5f2c325d61e6132a2c4083bd440f62039be4175a2a270da764adebm6qdv] creating logical port openshift-marketplace_f62a5f2c325d61e6132a2c4083bd440f62039be4175a2a270da764adebm6qdv for pod on switch ip-10-0-2-216.ec2.internal I0406 03:06:07.965429 3776 kube.go:256] Updating pod openshift-marketplace/f62a5f2c325d61e6132a2c4083bd440f62039be4175a2a270da764adebm6qdv I0406 03:06:07.979272 3776 pod.go:62] [openshift-marketplace/f62a5f2c325d61e6132a2c4083bd440f62039be4175a2a270da764adebm6qdv] pod update took 13.868056ms I0406 03:06:07.979300 3776 base_network_controller_pods.go:937] [default/openshift-marketplace/f62a5f2c325d61e6132a2c4083bd440f62039be4175a2a270da764adebm6qdv] addLogicalPort annotation time took 13.904361ms I0406 03:06:07.980523 3776 pods.go:271] [openshift-marketplace/f62a5f2c325d61e6132a2c4083bd440f62039be4175a2a270da764adebm6qdv] addLogicalPort took 15.253388ms, libovsdb time 866.39µs I0406 03:06:08.051343 3776 base_network_controller_pods.go:487] [default/openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece98wnv] creating logical port openshift-marketplace_53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece98wnv for pod on switch ip-10-0-2-216.ec2.internal I0406 03:06:08.051464 3776 kube.go:256] Updating pod openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece98wnv I0406 03:06:08.064922 3776 pod.go:62] [openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece98wnv] pod update took 13.484255ms I0406 03:06:08.064945 3776 base_network_controller_pods.go:937] [default/openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece98wnv] addLogicalPort annotation time took 13.511909ms I0406 03:06:08.066178 3776 pods.go:271] [openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece98wnv] addLogicalPort took 14.853276ms, libovsdb time 880.446µs I0406 03:06:08.326312 3776 cni.go:403] [openshift-marketplace/f62a5f2c325d61e6132a2c4083bd440f62039be4175a2a270da764adebm6qdv 141a5309db2354ab019e73002b6a4f2839ed77bc40f190c97646abbcefae8b04 network default NAD default] ADD starting CNI request [openshift-marketplace/f62a5f2c325d61e6132a2c4083bd440f62039be4175a2a270da764adebm6qdv 141a5309db2354ab019e73002b6a4f2839ed77bc40f190c97646abbcefae8b04 network default NAD default] I0406 03:06:08.336649 3776 helper_linux.go:450] ConfigureOVS: namespace: openshift-marketplace, podName: f62a5f2c325d61e6132a2c4083bd440f62039be4175a2a270da764adebm6qdv, hostIfaceName: 141a5309db2354a, network: default, NAD default, SandboxID: "141a5309db2354ab019e73002b6a4f2839ed77bc40f190c97646abbcefae8b04", PCI device ID: , UID: "3359adc7-ed01-4534-8e6c-110de561846e", MAC: 0a:58:0a:81:00:1a, IPs: [10.129.0.26/23] I0406 03:06:08.387680 3776 cni.go:424] [openshift-marketplace/f62a5f2c325d61e6132a2c4083bd440f62039be4175a2a270da764adebm6qdv 141a5309db2354ab019e73002b6a4f2839ed77bc40f190c97646abbcefae8b04 network default NAD default] ADD finished CNI request [openshift-marketplace/f62a5f2c325d61e6132a2c4083bd440f62039be4175a2a270da764adebm6qdv 141a5309db2354ab019e73002b6a4f2839ed77bc40f190c97646abbcefae8b04 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ae:16:83:bc:04:88\",\"name\":\"141a5309db2354a\"},{\"mac\":\"0a:58:0a:81:00:1a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5f6bb925-9b08-4c62-a54e-e56b32fc490e\"}],\"ips\":[{\"address\":\"10.129.0.26/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0406 03:06:08.416748 3776 cni.go:403] [openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece98wnv 4824f24c512d7f182f9f43faeb65cd1bce802f33785416d3a0ca093c54e47c68 network default NAD default] ADD starting CNI request [openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece98wnv 4824f24c512d7f182f9f43faeb65cd1bce802f33785416d3a0ca093c54e47c68 network default NAD default] I0406 03:06:08.427546 3776 helper_linux.go:450] ConfigureOVS: namespace: openshift-marketplace, podName: 53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece98wnv, hostIfaceName: 4824f24c512d7f1, network: default, NAD default, SandboxID: "4824f24c512d7f182f9f43faeb65cd1bce802f33785416d3a0ca093c54e47c68", PCI device ID: , UID: "bbe93a5c-286f-4eda-81f6-e4b330b28749", MAC: 0a:58:0a:81:00:1b, IPs: [10.129.0.27/23] I0406 03:06:08.480689 3776 cni.go:424] [openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece98wnv 4824f24c512d7f182f9f43faeb65cd1bce802f33785416d3a0ca093c54e47c68 network default NAD default] ADD finished CNI request [openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece98wnv 4824f24c512d7f182f9f43faeb65cd1bce802f33785416d3a0ca093c54e47c68 network default NAD default], result "{\"interfaces\":[{\"mac\":\"a6:a1:be:e5:b0:ed\",\"name\":\"4824f24c512d7f1\"},{\"mac\":\"0a:58:0a:81:00:1b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/271c8ebd-0c8d-45d8-9348-5f2fa41f5ba7\"}],\"ips\":[{\"address\":\"10.129.0.27/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0406 03:06:11.010974 3776 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=539 I0406 03:06:11.032375 3776 base_network_controller_pods.go:487] [default/openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1wbszj] creating logical port openshift-marketplace_3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1wbszj for pod on switch ip-10-0-2-216.ec2.internal I0406 03:06:11.032499 3776 kube.go:256] Updating pod openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1wbszj I0406 03:06:11.051712 3776 pod.go:62] [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1wbszj] pod update took 19.241602ms I0406 03:06:11.051736 3776 base_network_controller_pods.go:937] [default/openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1wbszj] addLogicalPort annotation time took 19.26984ms I0406 03:06:11.052968 3776 pods.go:271] [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1wbszj] addLogicalPort took 20.606192ms, libovsdb time 896.679µs I0406 03:06:11.386303 3776 base_network_controller_pods.go:487] [default/openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5crq6l] creating logical port openshift-marketplace_925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5crq6l for pod on switch ip-10-0-2-216.ec2.internal I0406 03:06:11.386412 3776 kube.go:256] Updating pod openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5crq6l I0406 03:06:11.393803 3776 cni.go:403] [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1wbszj 0d51bb9a80cded66d3bdb85fec604eccba12d8063fd7c4ee9a4be35b8752a4ee network default NAD default] ADD starting CNI request [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1wbszj 0d51bb9a80cded66d3bdb85fec604eccba12d8063fd7c4ee9a4be35b8752a4ee network default NAD default] I0406 03:06:11.405225 3776 helper_linux.go:450] ConfigureOVS: namespace: openshift-marketplace, podName: 3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1wbszj, hostIfaceName: 0d51bb9a80cded6, network: default, NAD default, SandboxID: "0d51bb9a80cded66d3bdb85fec604eccba12d8063fd7c4ee9a4be35b8752a4ee", PCI device ID: , UID: "da4565bb-76a6-477b-89ce-19aa68954bac", MAC: 0a:58:0a:81:00:1c, IPs: [10.129.0.28/23] I0406 03:06:11.406355 3776 pod.go:62] [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5crq6l] pod update took 19.97564ms I0406 03:06:11.406377 3776 base_network_controller_pods.go:937] [default/openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5crq6l] addLogicalPort annotation time took 20.004677ms I0406 03:06:11.407768 3776 pods.go:271] [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5crq6l] addLogicalPort took 21.485211ms, libovsdb time 979.025µs I0406 03:06:11.459068 3776 cni.go:424] [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1wbszj 0d51bb9a80cded66d3bdb85fec604eccba12d8063fd7c4ee9a4be35b8752a4ee network default NAD default] ADD finished CNI request [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1wbszj 0d51bb9a80cded66d3bdb85fec604eccba12d8063fd7c4ee9a4be35b8752a4ee network default NAD default], result "{\"interfaces\":[{\"mac\":\"2e:6e:69:fe:87:2c\",\"name\":\"0d51bb9a80cded6\"},{\"mac\":\"0a:58:0a:81:00:1c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/730d2696-a4ef-4ced-908e-90cec14bb911\"}],\"ips\":[{\"address\":\"10.129.0.28/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0406 03:06:11.749554 3776 cni.go:403] [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5crq6l a4c5d9bee54284945ee02fa9df0ca6daa559164d316f995ca9e3fa5236c90eef network default NAD default] ADD starting CNI request [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5crq6l a4c5d9bee54284945ee02fa9df0ca6daa559164d316f995ca9e3fa5236c90eef network default NAD default] I0406 03:06:11.760415 3776 helper_linux.go:450] ConfigureOVS: namespace: openshift-marketplace, podName: 925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5crq6l, hostIfaceName: a4c5d9bee542849, network: default, NAD default, SandboxID: "a4c5d9bee54284945ee02fa9df0ca6daa559164d316f995ca9e3fa5236c90eef", PCI device ID: , UID: "e2ad1893-dd37-4b4d-b288-31ab882c76b7", MAC: 0a:58:0a:81:00:1d, IPs: [10.129.0.29/23] I0406 03:06:11.812361 3776 cni.go:424] [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5crq6l a4c5d9bee54284945ee02fa9df0ca6daa559164d316f995ca9e3fa5236c90eef network default NAD default] ADD finished CNI request [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5crq6l a4c5d9bee54284945ee02fa9df0ca6daa559164d316f995ca9e3fa5236c90eef network default NAD default], result "{\"interfaces\":[{\"mac\":\"7e:59:78:c6:63:87\",\"name\":\"a4c5d9bee542849\"},{\"mac\":\"0a:58:0a:81:00:1d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/810e25ca-9bf1-4827-b06c-dc8608c4d6ab\"}],\"ips\":[{\"address\":\"10.129.0.29/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0406 03:06:11.995975 3776 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=11 I0406 03:06:12.321796 3776 cni.go:403] [openshift-marketplace/f62a5f2c325d61e6132a2c4083bd440f62039be4175a2a270da764adebm6qdv 141a5309db2354ab019e73002b6a4f2839ed77bc40f190c97646abbcefae8b04 network default NAD default] DEL starting CNI request [openshift-marketplace/f62a5f2c325d61e6132a2c4083bd440f62039be4175a2a270da764adebm6qdv 141a5309db2354ab019e73002b6a4f2839ed77bc40f190c97646abbcefae8b04 network default NAD default] I0406 03:06:12.397870 3776 cni.go:424] [openshift-marketplace/f62a5f2c325d61e6132a2c4083bd440f62039be4175a2a270da764adebm6qdv 141a5309db2354ab019e73002b6a4f2839ed77bc40f190c97646abbcefae8b04 network default NAD default] DEL finished CNI request [openshift-marketplace/f62a5f2c325d61e6132a2c4083bd440f62039be4175a2a270da764adebm6qdv 141a5309db2354ab019e73002b6a4f2839ed77bc40f190c97646abbcefae8b04 network default NAD default], result "{}", err I0406 03:06:12.445787 3776 obj_retry.go:464] Detected object openshift-marketplace/f62a5f2c325d61e6132a2c4083bd440f62039be4175a2a270da764adebm6qdv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0406 03:06:12.445831 3776 pods.go:176] Deleting pod: openshift-marketplace/f62a5f2c325d61e6132a2c4083bd440f62039be4175a2a270da764adebm6qdv I0406 03:06:12.447251 3776 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/f62a5f2c325d61e6132a2c4083bd440f62039be4175a2a270da764adebm6qdv, ips: 10.129.0.26 I0406 03:06:12.447278 3776 obj_retry.go:464] Detected object openshift-marketplace/f62a5f2c325d61e6132a2c4083bd440f62039be4175a2a270da764adebm6qdv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0406 03:06:12.447301 3776 obj_retry.go:464] Detected object openshift-marketplace/f62a5f2c325d61e6132a2c4083bd440f62039be4175a2a270da764adebm6qdv of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0406 03:06:12.447309 3776 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0406 03:06:12.447479 3776 obj_retry.go:464] Detected object openshift-marketplace/f62a5f2c325d61e6132a2c4083bd440f62039be4175a2a270da764adebm6qdv of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0406 03:06:12.447498 3776 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0406 03:06:13.334687 3776 cni.go:403] [openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece98wnv 4824f24c512d7f182f9f43faeb65cd1bce802f33785416d3a0ca093c54e47c68 network default NAD default] DEL starting CNI request [openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece98wnv 4824f24c512d7f182f9f43faeb65cd1bce802f33785416d3a0ca093c54e47c68 network default NAD default] I0406 03:06:13.410955 3776 cni.go:424] [openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece98wnv 4824f24c512d7f182f9f43faeb65cd1bce802f33785416d3a0ca093c54e47c68 network default NAD default] DEL finished CNI request [openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece98wnv 4824f24c512d7f182f9f43faeb65cd1bce802f33785416d3a0ca093c54e47c68 network default NAD default], result "{}", err I0406 03:06:13.461607 3776 obj_retry.go:464] Detected object openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece98wnv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0406 03:06:13.461646 3776 pods.go:176] Deleting pod: openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece98wnv I0406 03:06:13.463181 3776 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece98wnv, ips: 10.129.0.27 I0406 03:06:13.463216 3776 obj_retry.go:464] Detected object openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece98wnv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0406 03:06:13.463243 3776 obj_retry.go:464] Detected object openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece98wnv of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0406 03:06:13.463254 3776 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0406 03:06:13.463428 3776 obj_retry.go:464] Detected object openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece98wnv of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0406 03:06:13.463440 3776 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0406 03:06:13.470273 3776 obj_retry.go:458] Detected object openshift-marketplace/f62a5f2c325d61e6132a2c4083bd440f62039be4175a2a270da764adebm6qdv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:06:13.470303 3776 obj_retry.go:458] Detected object openshift-marketplace/f62a5f2c325d61e6132a2c4083bd440f62039be4175a2a270da764adebm6qdv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:06:13.470319 3776 obj_retry.go:458] Detected object openshift-marketplace/f62a5f2c325d61e6132a2c4083bd440f62039be4175a2a270da764adebm6qdv of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:06:13.470336 3776 obj_retry.go:458] Detected object openshift-marketplace/f62a5f2c325d61e6132a2c4083bd440f62039be4175a2a270da764adebm6qdv of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:06:14.161494 3776 base_network_controller_pods.go:487] [default/openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaflcdc8] creating logical port openshift-marketplace_30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaflcdc8 for pod on switch ip-10-0-2-216.ec2.internal I0406 03:06:14.161596 3776 kube.go:256] Updating pod openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaflcdc8 I0406 03:06:14.173817 3776 pod.go:62] [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaflcdc8] pod update took 12.250747ms I0406 03:06:14.173873 3776 base_network_controller_pods.go:937] [default/openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaflcdc8] addLogicalPort annotation time took 12.280958ms I0406 03:06:14.175378 3776 pods.go:271] [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaflcdc8] addLogicalPort took 13.899246ms, libovsdb time 1.080728ms I0406 03:06:14.271012 3776 base_network_controller_pods.go:487] [default/openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7j4l96] creating logical port openshift-marketplace_18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7j4l96 for pod on switch ip-10-0-2-216.ec2.internal I0406 03:06:14.271141 3776 kube.go:256] Updating pod openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7j4l96 I0406 03:06:14.287209 3776 pod.go:62] [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7j4l96] pod update took 16.122615ms I0406 03:06:14.287237 3776 base_network_controller_pods.go:937] [default/openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7j4l96] addLogicalPort annotation time took 16.158528ms I0406 03:06:14.288676 3776 pods.go:271] [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7j4l96] addLogicalPort took 17.683075ms, libovsdb time 1.044585ms I0406 03:06:14.485579 3776 obj_retry.go:458] Detected object openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece98wnv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:06:14.485612 3776 obj_retry.go:458] Detected object openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece98wnv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:06:14.485635 3776 obj_retry.go:458] Detected object openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece98wnv of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:06:14.485657 3776 obj_retry.go:458] Detected object openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ece98wnv of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:06:14.725819 3776 cni.go:403] [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaflcdc8 2980816813adb0c80ee5b6aad7e87f52598e8c304d3c72ff6902b760bbca823e network default NAD default] ADD starting CNI request [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaflcdc8 2980816813adb0c80ee5b6aad7e87f52598e8c304d3c72ff6902b760bbca823e network default NAD default] I0406 03:06:14.725839 3776 cni.go:403] [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7j4l96 94976be6b34cd7d339e041e1bca89663663df6068fb59beb0807f11b006e5340 network default NAD default] ADD starting CNI request [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7j4l96 94976be6b34cd7d339e041e1bca89663663df6068fb59beb0807f11b006e5340 network default NAD default] I0406 03:06:14.739642 3776 helper_linux.go:450] ConfigureOVS: namespace: openshift-marketplace, podName: 30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaflcdc8, hostIfaceName: 2980816813adb0c, network: default, NAD default, SandboxID: "2980816813adb0c80ee5b6aad7e87f52598e8c304d3c72ff6902b760bbca823e", PCI device ID: , UID: "2579589a-7057-43f3-a1d4-a59339ae05ca", MAC: 0a:58:0a:81:00:1e, IPs: [10.129.0.30/23] I0406 03:06:14.740721 3776 helper_linux.go:450] ConfigureOVS: namespace: openshift-marketplace, podName: 18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7j4l96, hostIfaceName: 94976be6b34cd7d, network: default, NAD default, SandboxID: "94976be6b34cd7d339e041e1bca89663663df6068fb59beb0807f11b006e5340", PCI device ID: , UID: "781fbd12-e774-405a-a1c4-4e761e24e317", MAC: 0a:58:0a:81:00:1f, IPs: [10.129.0.31/23] I0406 03:06:14.793610 3776 cni.go:424] [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaflcdc8 2980816813adb0c80ee5b6aad7e87f52598e8c304d3c72ff6902b760bbca823e network default NAD default] ADD finished CNI request [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaflcdc8 2980816813adb0c80ee5b6aad7e87f52598e8c304d3c72ff6902b760bbca823e network default NAD default], result "{\"interfaces\":[{\"mac\":\"8a:3c:a4:c3:bb:e2\",\"name\":\"2980816813adb0c\"},{\"mac\":\"0a:58:0a:81:00:1e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/79963401-b0bc-46e3-a79b-b9601b8737bf\"}],\"ips\":[{\"address\":\"10.129.0.30/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0406 03:06:14.813995 3776 cni.go:424] [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7j4l96 94976be6b34cd7d339e041e1bca89663663df6068fb59beb0807f11b006e5340 network default NAD default] ADD finished CNI request [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7j4l96 94976be6b34cd7d339e041e1bca89663663df6068fb59beb0807f11b006e5340 network default NAD default], result "{\"interfaces\":[{\"mac\":\"4e:07:5a:bf:f5:4b\",\"name\":\"94976be6b34cd7d\"},{\"mac\":\"0a:58:0a:81:00:1f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e90a240c-87d9-47f7-8818-fe5add32e277\"}],\"ips\":[{\"address\":\"10.129.0.31/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0406 03:06:15.350487 3776 cni.go:403] [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1wbszj 0d51bb9a80cded66d3bdb85fec604eccba12d8063fd7c4ee9a4be35b8752a4ee network default NAD default] DEL starting CNI request [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1wbszj 0d51bb9a80cded66d3bdb85fec604eccba12d8063fd7c4ee9a4be35b8752a4ee network default NAD default] I0406 03:06:15.422405 3776 cni.go:424] [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1wbszj 0d51bb9a80cded66d3bdb85fec604eccba12d8063fd7c4ee9a4be35b8752a4ee network default NAD default] DEL finished CNI request [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1wbszj 0d51bb9a80cded66d3bdb85fec604eccba12d8063fd7c4ee9a4be35b8752a4ee network default NAD default], result "{}", err I0406 03:06:15.469797 3776 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1wbszj of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0406 03:06:15.469835 3776 pods.go:176] Deleting pod: openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1wbszj I0406 03:06:15.471309 3776 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1wbszj, ips: 10.129.0.28 I0406 03:06:15.471336 3776 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1wbszj of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0406 03:06:15.471359 3776 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1wbszj of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0406 03:06:15.471367 3776 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0406 03:06:15.471537 3776 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1wbszj of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0406 03:06:15.471555 3776 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0406 03:06:16.496002 3776 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1wbszj of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:06:16.496032 3776 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1wbszj of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:06:16.496044 3776 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1wbszj of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:06:16.496061 3776 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1wbszj of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:06:17.373446 3776 cni.go:403] [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5crq6l a4c5d9bee54284945ee02fa9df0ca6daa559164d316f995ca9e3fa5236c90eef network default NAD default] DEL starting CNI request [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5crq6l a4c5d9bee54284945ee02fa9df0ca6daa559164d316f995ca9e3fa5236c90eef network default NAD default] I0406 03:06:17.447514 3776 cni.go:424] [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5crq6l a4c5d9bee54284945ee02fa9df0ca6daa559164d316f995ca9e3fa5236c90eef network default NAD default] DEL finished CNI request [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5crq6l a4c5d9bee54284945ee02fa9df0ca6daa559164d316f995ca9e3fa5236c90eef network default NAD default], result "{}", err I0406 03:06:17.505144 3776 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5crq6l of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0406 03:06:17.505180 3776 pods.go:176] Deleting pod: openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5crq6l I0406 03:06:17.506451 3776 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5crq6l, ips: 10.129.0.29 I0406 03:06:17.506486 3776 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5crq6l of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0406 03:06:17.506512 3776 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5crq6l of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0406 03:06:17.506523 3776 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0406 03:06:17.506704 3776 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5crq6l of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0406 03:06:17.506721 3776 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0406 03:06:18.380402 3776 cni.go:403] [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7j4l96 94976be6b34cd7d339e041e1bca89663663df6068fb59beb0807f11b006e5340 network default NAD default] DEL starting CNI request [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7j4l96 94976be6b34cd7d339e041e1bca89663663df6068fb59beb0807f11b006e5340 network default NAD default] I0406 03:06:18.456146 3776 cni.go:424] [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7j4l96 94976be6b34cd7d339e041e1bca89663663df6068fb59beb0807f11b006e5340 network default NAD default] DEL finished CNI request [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7j4l96 94976be6b34cd7d339e041e1bca89663663df6068fb59beb0807f11b006e5340 network default NAD default], result "{}", err I0406 03:06:18.511531 3776 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7j4l96 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0406 03:06:18.511588 3776 pods.go:176] Deleting pod: openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7j4l96 I0406 03:06:18.513157 3776 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7j4l96, ips: 10.129.0.31 I0406 03:06:18.513186 3776 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7j4l96 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0406 03:06:18.513206 3776 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7j4l96 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0406 03:06:18.513214 3776 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0406 03:06:18.513396 3776 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7j4l96 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0406 03:06:18.513414 3776 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0406 03:06:18.533714 3776 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5crq6l of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:06:18.533737 3776 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5crq6l of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:06:18.533750 3776 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5crq6l of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:06:18.533764 3776 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5crq6l of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:06:19.387196 3776 cni.go:403] [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaflcdc8 2980816813adb0c80ee5b6aad7e87f52598e8c304d3c72ff6902b760bbca823e network default NAD default] DEL starting CNI request [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaflcdc8 2980816813adb0c80ee5b6aad7e87f52598e8c304d3c72ff6902b760bbca823e network default NAD default] I0406 03:06:19.463183 3776 cni.go:424] [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaflcdc8 2980816813adb0c80ee5b6aad7e87f52598e8c304d3c72ff6902b760bbca823e network default NAD default] DEL finished CNI request [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaflcdc8 2980816813adb0c80ee5b6aad7e87f52598e8c304d3c72ff6902b760bbca823e network default NAD default], result "{}", err I0406 03:06:19.533886 3776 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaflcdc8 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0406 03:06:19.533945 3776 pods.go:176] Deleting pod: openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaflcdc8 I0406 03:06:19.535254 3776 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaflcdc8, ips: 10.129.0.30 I0406 03:06:19.535284 3776 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaflcdc8 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0406 03:06:19.535300 3776 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaflcdc8 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0406 03:06:19.535308 3776 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0406 03:06:19.535471 3776 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaflcdc8 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0406 03:06:19.535486 3776 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0406 03:06:19.567690 3776 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7j4l96 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:06:19.567718 3776 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7j4l96 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:06:19.567734 3776 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7j4l96 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:06:19.567746 3776 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7j4l96 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:06:20.559597 3776 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaflcdc8 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:06:20.559622 3776 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaflcdc8 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:06:20.559639 3776 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaflcdc8 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:06:20.559647 3776 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaflcdc8 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed W0406 03:06:24.715651 3776 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0406 03:06:25.021841 3776 base_network_controller_pods.go:487] [default/rhbk-operator/rhbk-operator-8468f77bc5-kkdjt] creating logical port rhbk-operator_rhbk-operator-8468f77bc5-kkdjt for pod on switch ip-10-0-2-216.ec2.internal I0406 03:06:25.021986 3776 kube.go:256] Updating pod rhbk-operator/rhbk-operator-8468f77bc5-kkdjt I0406 03:06:25.040621 3776 pod.go:62] [rhbk-operator/rhbk-operator-8468f77bc5-kkdjt] pod update took 18.658423ms I0406 03:06:25.040649 3776 base_network_controller_pods.go:937] [default/rhbk-operator/rhbk-operator-8468f77bc5-kkdjt] addLogicalPort annotation time took 18.69197ms I0406 03:06:25.041970 3776 pods.go:271] [rhbk-operator/rhbk-operator-8468f77bc5-kkdjt] addLogicalPort took 20.147476ms, libovsdb time 921.192µs I0406 03:06:25.403460 3776 cni.go:403] [rhbk-operator/rhbk-operator-8468f77bc5-kkdjt 8e337111a18d030451cae630cd0f48356fe294dbb7be3d9aea1825e41c30d9af network default NAD default] ADD starting CNI request [rhbk-operator/rhbk-operator-8468f77bc5-kkdjt 8e337111a18d030451cae630cd0f48356fe294dbb7be3d9aea1825e41c30d9af network default NAD default] I0406 03:06:25.412984 3776 helper_linux.go:450] ConfigureOVS: namespace: rhbk-operator, podName: rhbk-operator-8468f77bc5-kkdjt, hostIfaceName: 8e337111a18d030, network: default, NAD default, SandboxID: "8e337111a18d030451cae630cd0f48356fe294dbb7be3d9aea1825e41c30d9af", PCI device ID: , UID: "571fd986-bf85-4cb7-b479-503080667079", MAC: 0a:58:0a:81:00:20, IPs: [10.129.0.32/23] I0406 03:06:25.465055 3776 cni.go:424] [rhbk-operator/rhbk-operator-8468f77bc5-kkdjt 8e337111a18d030451cae630cd0f48356fe294dbb7be3d9aea1825e41c30d9af network default NAD default] ADD finished CNI request [rhbk-operator/rhbk-operator-8468f77bc5-kkdjt 8e337111a18d030451cae630cd0f48356fe294dbb7be3d9aea1825e41c30d9af network default NAD default], result "{\"interfaces\":[{\"mac\":\"be:51:e6:51:b6:c9\",\"name\":\"8e337111a18d030\"},{\"mac\":\"0a:58:0a:81:00:20\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/79fdc27b-e588-4e65-8fbe-e5f9826c1e43\"}],\"ips\":[{\"address\":\"10.129.0.32/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0406 03:06:29.734267 3776 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0406 03:06:31.002341 3776 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=51 I0406 03:06:52.103472 3776 namespace.go:142] [openshift-operators] updating namespace I0406 03:06:52.120728 3776 namespace.go:142] [openshift-operators] updating namespace I0406 03:06:54.274363 3776 namespace.go:98] [cert-manager] adding namespace I0406 03:06:54.275277 3776 namespace.go:102] [cert-manager] adding namespace took 886.477µs I0406 03:06:54.322816 3776 namespace.go:142] [cert-manager] updating namespace I0406 03:06:54.336340 3776 namespace.go:142] [cert-manager] updating namespace I0406 03:06:55.468459 3776 base_network_controller_pods.go:487] [default/cert-manager/cert-manager-webhook-fd44f5cbb-mfc6v] creating logical port cert-manager_cert-manager-webhook-fd44f5cbb-mfc6v for pod on switch ip-10-0-2-216.ec2.internal I0406 03:06:55.468571 3776 kube.go:256] Updating pod cert-manager/cert-manager-webhook-fd44f5cbb-mfc6v I0406 03:06:55.488162 3776 pod.go:62] [cert-manager/cert-manager-webhook-fd44f5cbb-mfc6v] pod update took 19.624163ms I0406 03:06:55.488188 3776 base_network_controller_pods.go:937] [default/cert-manager/cert-manager-webhook-fd44f5cbb-mfc6v] addLogicalPort annotation time took 19.656594ms I0406 03:06:55.489432 3776 pods.go:271] [cert-manager/cert-manager-webhook-fd44f5cbb-mfc6v] addLogicalPort took 20.99099ms, libovsdb time 901.323µs I0406 03:06:55.840083 3776 cni.go:403] [cert-manager/cert-manager-webhook-fd44f5cbb-mfc6v 0e17a7fcecfeec9ece5de12a9d676487ffbc2ae4b251f046ce9b31960fb36295 network default NAD default] ADD starting CNI request [cert-manager/cert-manager-webhook-fd44f5cbb-mfc6v 0e17a7fcecfeec9ece5de12a9d676487ffbc2ae4b251f046ce9b31960fb36295 network default NAD default] I0406 03:06:55.849892 3776 helper_linux.go:450] ConfigureOVS: namespace: cert-manager, podName: cert-manager-webhook-fd44f5cbb-mfc6v, hostIfaceName: 0e17a7fcecfeec9, network: default, NAD default, SandboxID: "0e17a7fcecfeec9ece5de12a9d676487ffbc2ae4b251f046ce9b31960fb36295", PCI device ID: , UID: "72f50b3d-f20b-4394-b7aa-5891bba5260b", MAC: 0a:58:0a:81:00:21, IPs: [10.129.0.33/23] I0406 03:06:55.906726 3776 cni.go:424] [cert-manager/cert-manager-webhook-fd44f5cbb-mfc6v 0e17a7fcecfeec9ece5de12a9d676487ffbc2ae4b251f046ce9b31960fb36295 network default NAD default] ADD finished CNI request [cert-manager/cert-manager-webhook-fd44f5cbb-mfc6v 0e17a7fcecfeec9ece5de12a9d676487ffbc2ae4b251f046ce9b31960fb36295 network default NAD default], result "{\"interfaces\":[{\"mac\":\"62:40:f4:6d:aa:99\",\"name\":\"0e17a7fcecfeec9\"},{\"mac\":\"0a:58:0a:81:00:21\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a8f38371-e4dc-4fd9-9b55-9a2b1c6b0ebf\"}],\"ips\":[{\"address\":\"10.129.0.33/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0406 03:06:59.602258 3776 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service cert-manager/cert-manager-webhook W0406 03:06:59.602278 3776 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service cert-manager/cert-manager-webhook W0406 03:07:00.892531 3776 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0406 03:07:00.932544 3776 namespace.go:98] [trusted-artifact-signer] adding namespace I0406 03:07:00.933605 3776 namespace.go:102] [trusted-artifact-signer] adding namespace took 1.032097ms I0406 03:07:00.959102 3776 namespace.go:142] [trusted-artifact-signer] updating namespace I0406 03:07:00.988714 3776 namespace.go:142] [trusted-artifact-signer] updating namespace I0406 03:07:04.655713 3776 namespace.go:142] [openshift-operators] updating namespace I0406 03:07:05.146459 3776 base_network_controller_pods.go:487] [default/openshift-operators/tekton-operator-webhook-668b6fdd5d-dzn7c] creating logical port openshift-operators_tekton-operator-webhook-668b6fdd5d-dzn7c for pod on switch ip-10-0-2-216.ec2.internal I0406 03:07:05.146569 3776 kube.go:256] Updating pod openshift-operators/tekton-operator-webhook-668b6fdd5d-dzn7c I0406 03:07:05.168964 3776 pod.go:62] [openshift-operators/tekton-operator-webhook-668b6fdd5d-dzn7c] pod update took 22.418332ms I0406 03:07:05.168990 3776 base_network_controller_pods.go:937] [default/openshift-operators/tekton-operator-webhook-668b6fdd5d-dzn7c] addLogicalPort annotation time took 22.450472ms I0406 03:07:05.170382 3776 pods.go:271] [openshift-operators/tekton-operator-webhook-668b6fdd5d-dzn7c] addLogicalPort took 23.938091ms, libovsdb time 974.82µs I0406 03:07:05.516686 3776 cni.go:403] [openshift-operators/tekton-operator-webhook-668b6fdd5d-dzn7c 4fe9e50fdbf6361df2521bcdfdb212a1e60a75e3e93d5d9f6e3e3a74926d5f10 network default NAD default] ADD starting CNI request [openshift-operators/tekton-operator-webhook-668b6fdd5d-dzn7c 4fe9e50fdbf6361df2521bcdfdb212a1e60a75e3e93d5d9f6e3e3a74926d5f10 network default NAD default] I0406 03:07:05.526793 3776 helper_linux.go:450] ConfigureOVS: namespace: openshift-operators, podName: tekton-operator-webhook-668b6fdd5d-dzn7c, hostIfaceName: 4fe9e50fdbf6361, network: default, NAD default, SandboxID: "4fe9e50fdbf6361df2521bcdfdb212a1e60a75e3e93d5d9f6e3e3a74926d5f10", PCI device ID: , UID: "5bb4ee95-3044-4f19-803d-daf7887bb0af", MAC: 0a:58:0a:81:00:22, IPs: [10.129.0.34/23] I0406 03:07:05.582147 3776 cni.go:424] [openshift-operators/tekton-operator-webhook-668b6fdd5d-dzn7c 4fe9e50fdbf6361df2521bcdfdb212a1e60a75e3e93d5d9f6e3e3a74926d5f10 network default NAD default] ADD finished CNI request [openshift-operators/tekton-operator-webhook-668b6fdd5d-dzn7c 4fe9e50fdbf6361df2521bcdfdb212a1e60a75e3e93d5d9f6e3e3a74926d5f10 network default NAD default], result "{\"interfaces\":[{\"mac\":\"2a:78:66:2c:26:0e\",\"name\":\"4fe9e50fdbf6361\"},{\"mac\":\"0a:58:0a:81:00:22\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d7d5ae09-eb40-4f75-94db-7864e4c7ec5c\"}],\"ips\":[{\"address\":\"10.129.0.34/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0406 03:07:05.613702 3776 util.go:959] Failed to build node endpoints for node ip-10-0-2-216.ec2.internal port 10250: empty IP address endpoints for service cert-manager/cert-manager-webhook W0406 03:07:05.613726 3776 util.go:959] Failed to build node endpoints for node ip-10-0-2-216.ec2.internal port 9402: empty IP address endpoints for service cert-manager/cert-manager-webhook I0406 03:07:06.975261 3776 base_network_controller_pods.go:487] [default/cert-manager/cert-manager-59ccb4bb6b-4728l] creating logical port cert-manager_cert-manager-59ccb4bb6b-4728l for pod on switch ip-10-0-2-216.ec2.internal I0406 03:07:06.975359 3776 kube.go:256] Updating pod cert-manager/cert-manager-59ccb4bb6b-4728l I0406 03:07:06.991304 3776 pod.go:62] [cert-manager/cert-manager-59ccb4bb6b-4728l] pod update took 15.968131ms I0406 03:07:06.991338 3776 base_network_controller_pods.go:937] [default/cert-manager/cert-manager-59ccb4bb6b-4728l] addLogicalPort annotation time took 16.010464ms I0406 03:07:06.992784 3776 pods.go:271] [cert-manager/cert-manager-59ccb4bb6b-4728l] addLogicalPort took 17.537281ms, libovsdb time 1.060401ms I0406 03:07:07.681628 3776 cni.go:403] [cert-manager/cert-manager-59ccb4bb6b-4728l bc7dea7998a59ab8b8533798ba4f258fec103d45786d3f7cf4980406af8f633f network default NAD default] ADD starting CNI request [cert-manager/cert-manager-59ccb4bb6b-4728l bc7dea7998a59ab8b8533798ba4f258fec103d45786d3f7cf4980406af8f633f network default NAD default] I0406 03:07:07.691798 3776 helper_linux.go:450] ConfigureOVS: namespace: cert-manager, podName: cert-manager-59ccb4bb6b-4728l, hostIfaceName: bc7dea7998a59ab, network: default, NAD default, SandboxID: "bc7dea7998a59ab8b8533798ba4f258fec103d45786d3f7cf4980406af8f633f", PCI device ID: , UID: "1555e3db-3eb9-45ea-abe9-4d1161ebc25b", MAC: 0a:58:0a:81:00:23, IPs: [10.129.0.35/23] I0406 03:07:07.745177 3776 cni.go:424] [cert-manager/cert-manager-59ccb4bb6b-4728l bc7dea7998a59ab8b8533798ba4f258fec103d45786d3f7cf4980406af8f633f network default NAD default] ADD finished CNI request [cert-manager/cert-manager-59ccb4bb6b-4728l bc7dea7998a59ab8b8533798ba4f258fec103d45786d3f7cf4980406af8f633f network default NAD default], result "{\"interfaces\":[{\"mac\":\"b6:86:4c:31:b5:08\",\"name\":\"bc7dea7998a59ab\"},{\"mac\":\"0a:58:0a:81:00:23\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/85ad06a0-d682-4f38-9b8e-957087fb6517\"}],\"ips\":[{\"address\":\"10.129.0.35/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0406 03:07:14.930440 3776 obj_retry.go:464] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0406 03:07:14.930984 3776 obj_retry.go:464] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0406 03:07:14.956152 3776 obj_retry.go:458] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:07:14.956184 3776 obj_retry.go:458] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:07:14.960641 3776 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-subscriptions I0406 03:07:14.960666 3776 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-subscriptions I0406 03:07:23.133977 3776 namespace.go:98] [openshift-pipelines] adding namespace I0406 03:07:23.134798 3776 namespace.go:102] [openshift-pipelines] adding namespace took 796.487µs I0406 03:07:23.152060 3776 namespace.go:142] [openshift-pipelines] updating namespace I0406 03:07:23.842859 3776 namespace.go:142] [cert-manager] updating namespace I0406 03:07:24.044631 3776 namespace.go:142] [cert-manager-operator] updating namespace I0406 03:07:24.450068 3776 namespace.go:142] [konflux-operator] updating namespace I0406 03:07:24.645608 3776 namespace.go:142] [konflux-ui] updating namespace W0406 03:07:24.833204 3776 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-keycloak/keycloak-pgsql I0406 03:07:24.847523 3776 namespace.go:142] [open-cluster-management-2pgcu9mfas1m46ulnh3binnrqpenib9u] updating namespace W0406 03:07:24.858894 3776 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-tpa/tpa-pgsql I0406 03:07:25.249884 3776 namespace.go:142] [rhbk-operator] updating namespace I0406 03:07:25.451202 3776 namespace.go:142] [rhtpa-operator] updating namespace I0406 03:07:25.649206 3776 namespace.go:142] [trusted-artifact-signer] updating namespace I0406 03:07:25.852034 3776 namespace.go:142] [tsf] updating namespace I0406 03:07:26.010067 3776 namespace.go:142] [tsf-keycloak] updating namespace I0406 03:07:26.208382 3776 namespace.go:142] [tsf-tas] updating namespace I0406 03:07:26.409479 3776 namespace.go:142] [tsf-tpa] updating namespace I0406 03:07:26.533228 3776 namespace.go:142] [cert-manager] updating namespace I0406 03:07:26.540495 3776 namespace.go:142] [cert-manager-operator] updating namespace I0406 03:07:26.550457 3776 namespace.go:142] [default] updating namespace I0406 03:07:26.557920 3776 namespace.go:142] [konflux-operator] updating namespace I0406 03:07:26.567293 3776 namespace.go:142] [konflux-ui] updating namespace I0406 03:07:26.574795 3776 namespace.go:142] [open-cluster-management-2pgcu9mfas1m46ulnh3binnrqpenib9u] updating namespace I0406 03:07:26.583308 3776 namespace.go:142] [openshift] updating namespace I0406 03:07:26.590508 3776 namespace.go:142] [rhbk-operator] updating namespace I0406 03:07:26.600010 3776 namespace.go:142] [rhtpa-operator] updating namespace I0406 03:07:26.608775 3776 namespace.go:142] [trusted-artifact-signer] updating namespace I0406 03:07:26.629042 3776 namespace.go:142] [tsf] updating namespace I0406 03:07:26.651433 3776 namespace.go:142] [tsf-keycloak] updating namespace I0406 03:07:26.671372 3776 namespace.go:142] [tsf-tas] updating namespace I0406 03:07:26.688539 3776 namespace.go:142] [tsf-tpa] updating namespace I0406 03:07:26.792470 3776 namespace.go:142] [cert-manager] updating namespace I0406 03:07:26.891578 3776 namespace.go:142] [cert-manager-operator] updating namespace I0406 03:07:26.996029 3776 namespace.go:142] [default] updating namespace I0406 03:07:27.099073 3776 namespace.go:142] [konflux-operator] updating namespace I0406 03:07:27.192268 3776 namespace.go:142] [konflux-ui] updating namespace I0406 03:07:27.295381 3776 namespace.go:142] [open-cluster-management-2pgcu9mfas1m46ulnh3binnrqpenib9u] updating namespace I0406 03:07:27.392589 3776 namespace.go:142] [openshift] updating namespace I0406 03:07:27.496697 3776 namespace.go:142] [rhbk-operator] updating namespace I0406 03:07:27.596777 3776 namespace.go:142] [rhtpa-operator] updating namespace I0406 03:07:27.696210 3776 namespace.go:142] [trusted-artifact-signer] updating namespace I0406 03:07:27.792587 3776 namespace.go:142] [tsf] updating namespace I0406 03:07:27.891352 3776 namespace.go:142] [tsf-keycloak] updating namespace I0406 03:07:27.994222 3776 namespace.go:142] [tsf-tas] updating namespace I0406 03:07:28.091523 3776 namespace.go:142] [tsf-tpa] updating namespace I0406 03:07:28.310077 3776 namespace.go:142] [openshift-pipelines] updating namespace I0406 03:07:37.227329 3776 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-4vmz4] creating logical port openshift-pipelines_tekton-pipelines-controller-84d6d58f5c-4vmz4 for pod on switch ip-10-0-2-216.ec2.internal I0406 03:07:37.227436 3776 kube.go:256] Updating pod openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-4vmz4 I0406 03:07:37.246520 3776 pod.go:62] [openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-4vmz4] pod update took 19.119582ms I0406 03:07:37.246545 3776 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-4vmz4] addLogicalPort annotation time took 19.152507ms I0406 03:07:37.247881 3776 pods.go:271] [openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-4vmz4] addLogicalPort took 20.570524ms, libovsdb time 943.925µs I0406 03:07:37.522472 3776 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-pipelines-webhook-584c448755-j47bq] creating logical port openshift-pipelines_tekton-pipelines-webhook-584c448755-j47bq for pod on switch ip-10-0-2-216.ec2.internal I0406 03:07:37.522574 3776 kube.go:256] Updating pod openshift-pipelines/tekton-pipelines-webhook-584c448755-j47bq I0406 03:07:37.536938 3776 pod.go:62] [openshift-pipelines/tekton-pipelines-webhook-584c448755-j47bq] pod update took 14.39316ms I0406 03:07:37.536963 3776 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-pipelines-webhook-584c448755-j47bq] addLogicalPort annotation time took 14.424191ms I0406 03:07:37.538253 3776 pods.go:271] [openshift-pipelines/tekton-pipelines-webhook-584c448755-j47bq] addLogicalPort took 15.796656ms, libovsdb time 837.787µs I0406 03:07:37.589046 3776 cni.go:403] [openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-4vmz4 36e94d673f49b3cf2b9253f6f6511e7b09f6595b43485da9604ae698c2e9142c network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-4vmz4 36e94d673f49b3cf2b9253f6f6511e7b09f6595b43485da9604ae698c2e9142c network default NAD default] I0406 03:07:37.599554 3776 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-controller-84d6d58f5c-4vmz4, hostIfaceName: 36e94d673f49b3c, network: default, NAD default, SandboxID: "36e94d673f49b3cf2b9253f6f6511e7b09f6595b43485da9604ae698c2e9142c", PCI device ID: , UID: "96f4271a-788e-4a2e-b9a0-8d8982a17039", MAC: 0a:58:0a:81:00:24, IPs: [10.129.0.36/23] I0406 03:07:37.652782 3776 cni.go:424] [openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-4vmz4 36e94d673f49b3cf2b9253f6f6511e7b09f6595b43485da9604ae698c2e9142c network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-4vmz4 36e94d673f49b3cf2b9253f6f6511e7b09f6595b43485da9604ae698c2e9142c network default NAD default], result "{\"interfaces\":[{\"mac\":\"e2:6a:a0:e5:07:ab\",\"name\":\"36e94d673f49b3c\"},{\"mac\":\"0a:58:0a:81:00:24\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d04ebd24-d88f-46b4-82a1-ef047b2dfc31\"}],\"ips\":[{\"address\":\"10.129.0.36/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0406 03:07:37.888231 3776 cni.go:403] [openshift-pipelines/tekton-pipelines-webhook-584c448755-j47bq d085ab9679cc3d71f4c9c9018b219865006526439a4c9b5bb4641fdc38a33377 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-pipelines-webhook-584c448755-j47bq d085ab9679cc3d71f4c9c9018b219865006526439a4c9b5bb4641fdc38a33377 network default NAD default] I0406 03:07:37.898484 3776 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-webhook-584c448755-j47bq, hostIfaceName: d085ab9679cc3d7, network: default, NAD default, SandboxID: "d085ab9679cc3d71f4c9c9018b219865006526439a4c9b5bb4641fdc38a33377", PCI device ID: , UID: "5f9d1d65-a593-4a32-b502-819300ed0336", MAC: 0a:58:0a:81:00:25, IPs: [10.129.0.37/23] I0406 03:07:37.950008 3776 cni.go:424] [openshift-pipelines/tekton-pipelines-webhook-584c448755-j47bq d085ab9679cc3d71f4c9c9018b219865006526439a4c9b5bb4641fdc38a33377 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-webhook-584c448755-j47bq d085ab9679cc3d71f4c9c9018b219865006526439a4c9b5bb4641fdc38a33377 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ae:37:6b:e4:09:cf\",\"name\":\"d085ab9679cc3d7\"},{\"mac\":\"0a:58:0a:81:00:25\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/eaf7f517-983a-425d-acc9-b87a384bc48b\"}],\"ips\":[{\"address\":\"10.129.0.37/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0406 03:07:41.947279 3776 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0406 03:07:41.947308 3776 util.go:870] Failed to build global endpoints for port TCP/https-webhook: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0406 03:07:41.947313 3776 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0406 03:07:41.947317 3776 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0406 03:07:42.074704 3776 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0406 03:07:42.074736 3776 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0406 03:07:42.074746 3776 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0406 03:07:42.202534 3776 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0406 03:07:42.202557 3776 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0406 03:07:42.202563 3776 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0406 03:07:52.843854 3776 util.go:959] Failed to build node endpoints for node ip-10-0-2-216.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0406 03:07:52.843880 3776 util.go:959] Failed to build node endpoints for node ip-10-0-2-216.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0406 03:07:52.843886 3776 util.go:959] Failed to build node endpoints for node ip-10-0-2-216.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0406 03:07:52.871204 3776 util.go:959] Failed to build node endpoints for node ip-10-0-2-216.ec2.internal port 8443: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0406 03:07:52.871225 3776 util.go:959] Failed to build node endpoints for node ip-10-0-2-216.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0406 03:07:52.871231 3776 util.go:959] Failed to build node endpoints for node ip-10-0-2-216.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0406 03:07:52.871236 3776 util.go:959] Failed to build node endpoints for node ip-10-0-2-216.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook I0406 03:08:04.271862 3776 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-triggers-controller-68564d75ff-qjw4f] creating logical port openshift-pipelines_tekton-triggers-controller-68564d75ff-qjw4f for pod on switch ip-10-0-2-216.ec2.internal I0406 03:08:04.272013 3776 kube.go:256] Updating pod openshift-pipelines/tekton-triggers-controller-68564d75ff-qjw4f I0406 03:08:04.286432 3776 pod.go:62] [openshift-pipelines/tekton-triggers-controller-68564d75ff-qjw4f] pod update took 14.448553ms I0406 03:08:04.286459 3776 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-triggers-controller-68564d75ff-qjw4f] addLogicalPort annotation time took 14.493994ms I0406 03:08:04.287876 3776 pods.go:271] [openshift-pipelines/tekton-triggers-controller-68564d75ff-qjw4f] addLogicalPort took 16.031019ms, libovsdb time 1.032115ms I0406 03:08:04.372967 3776 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-lnx7h] creating logical port openshift-pipelines_tekton-triggers-webhook-55d94ff6c6-lnx7h for pod on switch ip-10-0-2-216.ec2.internal I0406 03:08:04.373061 3776 kube.go:256] Updating pod openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-lnx7h I0406 03:08:04.387492 3776 pod.go:62] [openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-lnx7h] pod update took 14.453603ms I0406 03:08:04.387516 3776 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-lnx7h] addLogicalPort annotation time took 14.484298ms I0406 03:08:04.388741 3776 pods.go:271] [openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-lnx7h] addLogicalPort took 15.79617ms, libovsdb time 844.958µs I0406 03:08:04.635182 3776 cni.go:403] [openshift-pipelines/tekton-triggers-controller-68564d75ff-qjw4f 156d90d464282d6cede14f40413f8f02b616dec8de7f75345dcb66371daa5b8b network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-triggers-controller-68564d75ff-qjw4f 156d90d464282d6cede14f40413f8f02b616dec8de7f75345dcb66371daa5b8b network default NAD default] I0406 03:08:04.645228 3776 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-triggers-controller-68564d75ff-qjw4f, hostIfaceName: 156d90d464282d6, network: default, NAD default, SandboxID: "156d90d464282d6cede14f40413f8f02b616dec8de7f75345dcb66371daa5b8b", PCI device ID: , UID: "8aa2a689-a30d-4547-b95d-0baadddbc4e2", MAC: 0a:58:0a:81:00:26, IPs: [10.129.0.38/23] I0406 03:08:04.699999 3776 cni.go:424] [openshift-pipelines/tekton-triggers-controller-68564d75ff-qjw4f 156d90d464282d6cede14f40413f8f02b616dec8de7f75345dcb66371daa5b8b network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-triggers-controller-68564d75ff-qjw4f 156d90d464282d6cede14f40413f8f02b616dec8de7f75345dcb66371daa5b8b network default NAD default], result "{\"interfaces\":[{\"mac\":\"4e:63:ba:14:f8:9f\",\"name\":\"156d90d464282d6\"},{\"mac\":\"0a:58:0a:81:00:26\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/772c5436-a928-486a-b6e2-bba20c5cc29d\"}],\"ips\":[{\"address\":\"10.129.0.38/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0406 03:08:04.736690 3776 cni.go:403] [openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-lnx7h 5f712162f6f1e26e0324833227429a6460c6d0c62bbfc0511ad95f2f3fd01ab9 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-lnx7h 5f712162f6f1e26e0324833227429a6460c6d0c62bbfc0511ad95f2f3fd01ab9 network default NAD default] I0406 03:08:04.747246 3776 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-triggers-webhook-55d94ff6c6-lnx7h, hostIfaceName: 5f712162f6f1e26, network: default, NAD default, SandboxID: "5f712162f6f1e26e0324833227429a6460c6d0c62bbfc0511ad95f2f3fd01ab9", PCI device ID: , UID: "eda4b0a2-5132-4234-a4ba-f3c9b5f38559", MAC: 0a:58:0a:81:00:27, IPs: [10.129.0.39/23] I0406 03:08:04.798229 3776 cni.go:424] [openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-lnx7h 5f712162f6f1e26e0324833227429a6460c6d0c62bbfc0511ad95f2f3fd01ab9 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-lnx7h 5f712162f6f1e26e0324833227429a6460c6d0c62bbfc0511ad95f2f3fd01ab9 network default NAD default], result "{\"interfaces\":[{\"mac\":\"c2:49:87:d4:3d:de\",\"name\":\"5f712162f6f1e26\"},{\"mac\":\"0a:58:0a:81:00:27\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5c2e00a3-2e1a-46de-bfef-fb7dd4053aeb\"}],\"ips\":[{\"address\":\"10.129.0.39/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0406 03:08:07.223017 3776 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors I0406 03:08:10.163576 3776 obj_retry.go:464] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0406 03:08:10.164264 3776 obj_retry.go:464] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0406 03:08:10.204666 3776 obj_retry.go:458] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:08:10.204694 3776 obj_retry.go:458] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:08:10.207239 3776 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-infrastructure I0406 03:08:10.207255 3776 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-infrastructure W0406 03:08:18.073904 3776 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0406 03:08:18.073931 3776 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0406 03:08:18.074580 3776 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service I0406 03:08:25.164441 3776 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0406 03:08:39.113305 3776 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=6 I0406 03:08:39.484057 3776 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0406 03:08:39.484607 3776 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0406 03:08:40.175072 3776 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:08:40.175102 3776 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:08:40.178682 3776 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-keycloak/keycloak-0 I0406 03:08:40.178707 3776 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-keycloak/keycloak-0 I0406 03:08:40.670161 3776 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-retention-policy-agent-55cbb85cb8-lhwnh] creating logical port openshift-pipelines_tekton-results-retention-policy-agent-55cbb85cb8-lhwnh for pod on switch ip-10-0-2-216.ec2.internal I0406 03:08:40.670262 3776 kube.go:256] Updating pod openshift-pipelines/tekton-results-retention-policy-agent-55cbb85cb8-lhwnh I0406 03:08:40.683955 3776 pod.go:62] [openshift-pipelines/tekton-results-retention-policy-agent-55cbb85cb8-lhwnh] pod update took 13.716792ms I0406 03:08:40.683992 3776 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-results-retention-policy-agent-55cbb85cb8-lhwnh] addLogicalPort annotation time took 13.757896ms I0406 03:08:40.685508 3776 pods.go:271] [openshift-pipelines/tekton-results-retention-policy-agent-55cbb85cb8-lhwnh] addLogicalPort took 15.380861ms, libovsdb time 1.09557ms I0406 03:08:41.037044 3776 cni.go:403] [openshift-pipelines/tekton-results-retention-policy-agent-55cbb85cb8-lhwnh 6129a52e1d635cc8e01357b793b6e0769c0218cd759b36da5cd0508bfbeb330d network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-results-retention-policy-agent-55cbb85cb8-lhwnh 6129a52e1d635cc8e01357b793b6e0769c0218cd759b36da5cd0508bfbeb330d network default NAD default] I0406 03:08:41.046498 3776 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-retention-policy-agent-55cbb85cb8-lhwnh, hostIfaceName: 6129a52e1d635cc, network: default, NAD default, SandboxID: "6129a52e1d635cc8e01357b793b6e0769c0218cd759b36da5cd0508bfbeb330d", PCI device ID: , UID: "93dbc287-8013-4230-8545-a9a68affefd6", MAC: 0a:58:0a:81:00:28, IPs: [10.129.0.40/23] I0406 03:08:41.101779 3776 cni.go:424] [openshift-pipelines/tekton-results-retention-policy-agent-55cbb85cb8-lhwnh 6129a52e1d635cc8e01357b793b6e0769c0218cd759b36da5cd0508bfbeb330d network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-results-retention-policy-agent-55cbb85cb8-lhwnh 6129a52e1d635cc8e01357b793b6e0769c0218cd759b36da5cd0508bfbeb330d network default NAD default], result "{\"interfaces\":[{\"mac\":\"ba:e7:cb:1f:f7:59\",\"name\":\"6129a52e1d635cc\"},{\"mac\":\"0a:58:0a:81:00:28\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/79dc61c6-eb81-405a-8348-91652c2ea092\"}],\"ips\":[{\"address\":\"10.129.0.40/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0406 03:08:42.196997 3776 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0406 03:08:42.197025 3776 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0406 03:08:42.199568 3776 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0406 03:08:45.412315 3776 util.go:870] Failed to build global endpoints for port TCP/server: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0406 03:08:45.412337 3776 util.go:870] Failed to build global endpoints for port TCP/prometheus: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0406 03:08:45.412343 3776 util.go:870] Failed to build global endpoints for port TCP/profiling: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service I0406 03:08:57.317724 3776 base_network_controller_pods.go:487] [default/openshift-pipelines/tkn-cli-serve-784b566968-d2k9v] creating logical port openshift-pipelines_tkn-cli-serve-784b566968-d2k9v for pod on switch ip-10-0-2-216.ec2.internal I0406 03:08:57.317849 3776 kube.go:256] Updating pod openshift-pipelines/tkn-cli-serve-784b566968-d2k9v I0406 03:08:57.336568 3776 pod.go:62] [openshift-pipelines/tkn-cli-serve-784b566968-d2k9v] pod update took 18.743257ms I0406 03:08:57.336594 3776 base_network_controller_pods.go:937] [default/openshift-pipelines/tkn-cli-serve-784b566968-d2k9v] addLogicalPort annotation time took 18.775659ms I0406 03:08:57.337870 3776 pods.go:271] [openshift-pipelines/tkn-cli-serve-784b566968-d2k9v] addLogicalPort took 20.162967ms, libovsdb time 862.665µs I0406 03:08:57.675747 3776 cni.go:403] [openshift-pipelines/tkn-cli-serve-784b566968-d2k9v 418dbd3e76617212ad1dbef996d1275bf040730564387b7bad7584036a7cab4d network default NAD default] ADD starting CNI request [openshift-pipelines/tkn-cli-serve-784b566968-d2k9v 418dbd3e76617212ad1dbef996d1275bf040730564387b7bad7584036a7cab4d network default NAD default] I0406 03:08:57.685359 3776 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tkn-cli-serve-784b566968-d2k9v, hostIfaceName: 418dbd3e7661721, network: default, NAD default, SandboxID: "418dbd3e76617212ad1dbef996d1275bf040730564387b7bad7584036a7cab4d", PCI device ID: , UID: "618876e7-4608-4a7c-a5e2-6a4b6d02c44c", MAC: 0a:58:0a:81:00:29, IPs: [10.129.0.41/23] I0406 03:08:57.739060 3776 cni.go:424] [openshift-pipelines/tkn-cli-serve-784b566968-d2k9v 418dbd3e76617212ad1dbef996d1275bf040730564387b7bad7584036a7cab4d network default NAD default] ADD finished CNI request [openshift-pipelines/tkn-cli-serve-784b566968-d2k9v 418dbd3e76617212ad1dbef996d1275bf040730564387b7bad7584036a7cab4d network default NAD default], result "{\"interfaces\":[{\"mac\":\"ee:3f:0e:c3:33:14\",\"name\":\"418dbd3e7661721\"},{\"mac\":\"0a:58:0a:81:00:29\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d3c1b4a0-926a-4005-8a61-670b2505d8ab\"}],\"ips\":[{\"address\":\"10.129.0.41/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0406 03:09:08.878421 3776 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-x8fn9] creating logical port openshift-pipelines_pipelines-as-code-controller-79b6fbbc5c-x8fn9 for pod on switch ip-10-0-2-216.ec2.internal I0406 03:09:08.878517 3776 kube.go:256] Updating pod openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-x8fn9 I0406 03:09:08.895768 3776 pod.go:62] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-x8fn9] pod update took 17.273375ms I0406 03:09:08.895798 3776 base_network_controller_pods.go:937] [default/openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-x8fn9] addLogicalPort annotation time took 17.309302ms I0406 03:09:08.897066 3776 pods.go:271] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-x8fn9] addLogicalPort took 18.66723ms, libovsdb time 827.871µs I0406 03:09:09.241078 3776 cni.go:403] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-x8fn9 7a8d61bf74284f9f46a3d2ff5f9fb8793c9fa0c1eb0cc797ea8184689823c415 network default NAD default] ADD starting CNI request [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-x8fn9 7a8d61bf74284f9f46a3d2ff5f9fb8793c9fa0c1eb0cc797ea8184689823c415 network default NAD default] I0406 03:09:09.251071 3776 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-as-code-controller-79b6fbbc5c-x8fn9, hostIfaceName: 7a8d61bf74284f9, network: default, NAD default, SandboxID: "7a8d61bf74284f9f46a3d2ff5f9fb8793c9fa0c1eb0cc797ea8184689823c415", PCI device ID: , UID: "e418a0ca-232e-4fbe-85a0-0dddd824c175", MAC: 0a:58:0a:81:00:2a, IPs: [10.129.0.42/23] I0406 03:09:09.305057 3776 cni.go:424] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-x8fn9 7a8d61bf74284f9f46a3d2ff5f9fb8793c9fa0c1eb0cc797ea8184689823c415 network default NAD default] ADD finished CNI request [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-x8fn9 7a8d61bf74284f9f46a3d2ff5f9fb8793c9fa0c1eb0cc797ea8184689823c415 network default NAD default], result "{\"interfaces\":[{\"mac\":\"02:df:2c:d4:ea:5e\",\"name\":\"7a8d61bf74284f9\"},{\"mac\":\"0a:58:0a:81:00:2a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/824ecd7f-413d-4393-8041-a0ed21bc40fb\"}],\"ips\":[{\"address\":\"10.129.0.42/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0406 03:09:11.336597 3776 util.go:870] Failed to build global endpoints for port TCP/http-listener: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0406 03:09:11.336627 3776 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0406 03:09:11.365888 3776 util.go:959] Failed to build node endpoints for node ip-10-0-2-216.ec2.internal port 8082: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0406 03:09:11.365913 3776 util.go:959] Failed to build node endpoints for node ip-10-0-2-216.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller I0406 03:09:12.505270 3776 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0406 03:09:12.505826 3776 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0406 03:09:12.546312 3776 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher I0406 03:09:18.531692 3776 obj_retry.go:464] Detected object tsf-keycloak/tsf-iam-8qvwb of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0406 03:09:18.532286 3776 obj_retry.go:464] Detected object tsf-keycloak/tsf-iam-8qvwb of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0406 03:09:19.558782 3776 obj_retry.go:458] Detected object tsf-keycloak/tsf-iam-8qvwb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:09:19.558808 3776 obj_retry.go:458] Detected object tsf-keycloak/tsf-iam-8qvwb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:09:19.910148 3776 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0406 03:09:19.910710 3776 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0406 03:09:20.407434 3776 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:09:20.407461 3776 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:09:20.414170 3776 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-keycloak/keycloak-0 I0406 03:09:20.414200 3776 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-keycloak/keycloak-0 W0406 03:09:21.424668 3776 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0406 03:09:21.424697 3776 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0406 03:09:21.424718 3776 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service I0406 03:10:03.943317 3776 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29590750-xk4lg of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0406 03:10:03.943909 3776 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29590750-xk4lg of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0406 03:10:04.819402 3776 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29590750-xk4lg of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:10:04.819435 3776 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29590750-xk4lg of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:10:06.123970 3776 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=6 I0406 03:10:12.619903 3776 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:10:12.619937 3776 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:10:12.624641 3776 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0406 03:10:12.624668 3776 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam I0406 03:10:21.845195 3776 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0406 03:10:21.845768 3776 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0406 03:10:21.865363 3776 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:10:21.865389 3776 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:10:21.871255 3776 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0406 03:10:21.871287 3776 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam I0406 03:10:23.489233 3776 base_network_controller_pods.go:487] [default/tsf-tas/segment-backup-installation-2xr7j-krl4t] creating logical port tsf-tas_segment-backup-installation-2xr7j-krl4t for pod on switch ip-10-0-2-216.ec2.internal I0406 03:10:23.489332 3776 kube.go:256] Updating pod tsf-tas/segment-backup-installation-2xr7j-krl4t I0406 03:10:23.507566 3776 pod.go:62] [tsf-tas/segment-backup-installation-2xr7j-krl4t] pod update took 18.254881ms I0406 03:10:23.507597 3776 base_network_controller_pods.go:937] [default/tsf-tas/segment-backup-installation-2xr7j-krl4t] addLogicalPort annotation time took 18.289159ms I0406 03:10:23.509018 3776 pods.go:271] [tsf-tas/segment-backup-installation-2xr7j-krl4t] addLogicalPort took 19.804826ms, libovsdb time 976.169µs I0406 03:10:23.850827 3776 cni.go:403] [tsf-tas/segment-backup-installation-2xr7j-krl4t 9d12d5463fca3a4956e6aced9706ebb02c3ee01c69d0daa103606d0e179be537 network default NAD default] ADD starting CNI request [tsf-tas/segment-backup-installation-2xr7j-krl4t 9d12d5463fca3a4956e6aced9706ebb02c3ee01c69d0daa103606d0e179be537 network default NAD default] I0406 03:10:23.860170 3776 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: segment-backup-installation-2xr7j-krl4t, hostIfaceName: 9d12d5463fca3a4, network: default, NAD default, SandboxID: "9d12d5463fca3a4956e6aced9706ebb02c3ee01c69d0daa103606d0e179be537", PCI device ID: , UID: "7b6295ea-3598-4efe-8e02-6c83e2b029ac", MAC: 0a:58:0a:81:00:2b, IPs: [10.129.0.43/23] I0406 03:10:23.916706 3776 cni.go:424] [tsf-tas/segment-backup-installation-2xr7j-krl4t 9d12d5463fca3a4956e6aced9706ebb02c3ee01c69d0daa103606d0e179be537 network default NAD default] ADD finished CNI request [tsf-tas/segment-backup-installation-2xr7j-krl4t 9d12d5463fca3a4956e6aced9706ebb02c3ee01c69d0daa103606d0e179be537 network default NAD default], result "{\"interfaces\":[{\"mac\":\"96:24:ed:d3:01:36\",\"name\":\"9d12d5463fca3a4\"},{\"mac\":\"0a:58:0a:81:00:2b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/40667c89-8659-4f27-9404-efc0c9ada14c\"}],\"ips\":[{\"address\":\"10.129.0.43/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0406 03:10:26.921305 3776 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/fulcio-server W0406 03:10:26.921334 3776 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/fulcio-server I0406 03:10:27.262682 3776 base_network_controller_pods.go:487] [default/tsf-tas/trillian-db-54b67ffc97-kkk99] creating logical port tsf-tas_trillian-db-54b67ffc97-kkk99 for pod on switch ip-10-0-2-216.ec2.internal I0406 03:10:27.262881 3776 kube.go:256] Updating pod tsf-tas/trillian-db-54b67ffc97-kkk99 I0406 03:10:27.275443 3776 pod.go:62] [tsf-tas/trillian-db-54b67ffc97-kkk99] pod update took 12.591604ms I0406 03:10:27.275474 3776 base_network_controller_pods.go:937] [default/tsf-tas/trillian-db-54b67ffc97-kkk99] addLogicalPort annotation time took 12.629491ms I0406 03:10:27.276913 3776 pods.go:271] [tsf-tas/trillian-db-54b67ffc97-kkk99] addLogicalPort took 14.254525ms, libovsdb time 1.010484ms I0406 03:10:30.086565 3776 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=8 W0406 03:10:31.781242 3776 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logserver W0406 03:10:31.807485 3776 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logsigner I0406 03:10:32.569368 3776 cni.go:403] [tsf-tas/trillian-db-54b67ffc97-kkk99 0bf78029e68fe9bf1ef72996615415b09ea9ed6491689cee37faf87b642a9002 network default NAD default] ADD starting CNI request [tsf-tas/trillian-db-54b67ffc97-kkk99 0bf78029e68fe9bf1ef72996615415b09ea9ed6491689cee37faf87b642a9002 network default NAD default] I0406 03:10:32.578857 3776 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: trillian-db-54b67ffc97-kkk99, hostIfaceName: 0bf78029e68fe9b, network: default, NAD default, SandboxID: "0bf78029e68fe9bf1ef72996615415b09ea9ed6491689cee37faf87b642a9002", PCI device ID: , UID: "eaa25466-cb4a-41cd-8ee5-3706d90b5cfd", MAC: 0a:58:0a:81:00:2c, IPs: [10.129.0.44/23] I0406 03:10:32.635832 3776 cni.go:424] [tsf-tas/trillian-db-54b67ffc97-kkk99 0bf78029e68fe9bf1ef72996615415b09ea9ed6491689cee37faf87b642a9002 network default NAD default] ADD finished CNI request [tsf-tas/trillian-db-54b67ffc97-kkk99 0bf78029e68fe9bf1ef72996615415b09ea9ed6491689cee37faf87b642a9002 network default NAD default], result "{\"interfaces\":[{\"mac\":\"6a:e3:ff:22:dc:5c\",\"name\":\"0bf78029e68fe9b\"},{\"mac\":\"0a:58:0a:81:00:2c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9448e865-455a-4620-b572-21226cbf83a5\"}],\"ips\":[{\"address\":\"10.129.0.44/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0406 03:10:35.833934 3776 util.go:870] Failed to build global endpoints for port TCP/trillian-mysql: empty IP address endpoints for service tsf-tas/trillian-mysql I0406 03:10:36.825467 3776 cni.go:403] [tsf-tas/segment-backup-installation-2xr7j-krl4t 9d12d5463fca3a4956e6aced9706ebb02c3ee01c69d0daa103606d0e179be537 network default NAD default] DEL starting CNI request [tsf-tas/segment-backup-installation-2xr7j-krl4t 9d12d5463fca3a4956e6aced9706ebb02c3ee01c69d0daa103606d0e179be537 network default NAD default] I0406 03:10:36.913496 3776 cni.go:424] [tsf-tas/segment-backup-installation-2xr7j-krl4t 9d12d5463fca3a4956e6aced9706ebb02c3ee01c69d0daa103606d0e179be537 network default NAD default] DEL finished CNI request [tsf-tas/segment-backup-installation-2xr7j-krl4t 9d12d5463fca3a4956e6aced9706ebb02c3ee01c69d0daa103606d0e179be537 network default NAD default], result "{}", err I0406 03:10:36.964107 3776 obj_retry.go:464] Detected object tsf-tas/segment-backup-installation-2xr7j-krl4t of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0406 03:10:36.964170 3776 pods.go:176] Deleting pod: tsf-tas/segment-backup-installation-2xr7j-krl4t I0406 03:10:36.965664 3776 pods.go:236] Attempting to release IPs for pod: tsf-tas/segment-backup-installation-2xr7j-krl4t, ips: 10.129.0.43 I0406 03:10:36.965703 3776 obj_retry.go:464] Detected object tsf-tas/segment-backup-installation-2xr7j-krl4t of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0406 03:10:37.835470 3776 obj_retry.go:458] Detected object tsf-tas/segment-backup-installation-2xr7j-krl4t of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:10:37.835499 3776 obj_retry.go:458] Detected object tsf-tas/segment-backup-installation-2xr7j-krl4t of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0406 03:10:46.863292 3776 util.go:959] Failed to build node endpoints for node ip-10-0-2-216.ec2.internal port 3306: empty IP address endpoints for service tsf-tas/trillian-mysql I0406 03:10:58.106739 3776 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=10 I0406 03:11:03.003152 3776 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=55 I0406 03:11:05.090431 3776 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=10 I0406 03:11:11.126994 3776 obj_retry.go:464] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0406 03:11:11.127578 3776 obj_retry.go:464] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0406 03:11:11.147179 3776 obj_retry.go:458] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:11:11.147210 3776 obj_retry.go:458] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:11:11.153251 3776 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tas/test-tsf-tas I0406 03:11:11.153277 3776 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tas/test-tsf-tas I0406 03:11:11.822699 3776 base_network_controller_pods.go:487] [default/tsf/patch-tekton-config-fjvd8] creating logical port tsf_patch-tekton-config-fjvd8 for pod on switch ip-10-0-2-216.ec2.internal I0406 03:11:11.822799 3776 kube.go:256] Updating pod tsf/patch-tekton-config-fjvd8 I0406 03:11:11.835199 3776 pod.go:62] [tsf/patch-tekton-config-fjvd8] pod update took 12.428823ms I0406 03:11:11.835223 3776 base_network_controller_pods.go:937] [default/tsf/patch-tekton-config-fjvd8] addLogicalPort annotation time took 12.458714ms I0406 03:11:11.836646 3776 pods.go:271] [tsf/patch-tekton-config-fjvd8] addLogicalPort took 13.965809ms, libovsdb time 948.869µs I0406 03:11:12.187948 3776 cni.go:403] [tsf/patch-tekton-config-fjvd8 65da19b340bb0c1f15233dcf28cdfc55fc2fbcc54642221dabcbc015cff5f0ed network default NAD default] ADD starting CNI request [tsf/patch-tekton-config-fjvd8 65da19b340bb0c1f15233dcf28cdfc55fc2fbcc54642221dabcbc015cff5f0ed network default NAD default] I0406 03:11:12.197104 3776 helper_linux.go:450] ConfigureOVS: namespace: tsf, podName: patch-tekton-config-fjvd8, hostIfaceName: 65da19b340bb0c1, network: default, NAD default, SandboxID: "65da19b340bb0c1f15233dcf28cdfc55fc2fbcc54642221dabcbc015cff5f0ed", PCI device ID: , UID: "a1f90560-c140-43e8-8c8a-264cb21076c3", MAC: 0a:58:0a:81:00:2d, IPs: [10.129.0.45/23] I0406 03:11:12.252391 3776 cni.go:424] [tsf/patch-tekton-config-fjvd8 65da19b340bb0c1f15233dcf28cdfc55fc2fbcc54642221dabcbc015cff5f0ed network default NAD default] ADD finished CNI request [tsf/patch-tekton-config-fjvd8 65da19b340bb0c1f15233dcf28cdfc55fc2fbcc54642221dabcbc015cff5f0ed network default NAD default], result "{\"interfaces\":[{\"mac\":\"a2:48:73:2b:b0:2a\",\"name\":\"65da19b340bb0c1\"},{\"mac\":\"0a:58:0a:81:00:2d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2e167faa-6158-4544-ac8f-c2b6c0100c82\"}],\"ips\":[{\"address\":\"10.129.0.45/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0406 03:11:16.145445 3776 obj_retry.go:464] Detected object tsf-tas/ctlog-createtree-job-2nggf-z5ztv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0406 03:11:16.146015 3776 obj_retry.go:464] Detected object tsf-tas/ctlog-createtree-job-2nggf-z5ztv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0406 03:11:17.041332 3776 obj_retry.go:458] Detected object tsf-tas/ctlog-createtree-job-2nggf-z5ztv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:11:17.041361 3776 obj_retry.go:458] Detected object tsf-tas/ctlog-createtree-job-2nggf-z5ztv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:11:20.081152 3776 cni.go:403] [tsf/patch-tekton-config-fjvd8 65da19b340bb0c1f15233dcf28cdfc55fc2fbcc54642221dabcbc015cff5f0ed network default NAD default] DEL starting CNI request [tsf/patch-tekton-config-fjvd8 65da19b340bb0c1f15233dcf28cdfc55fc2fbcc54642221dabcbc015cff5f0ed network default NAD default] I0406 03:11:20.157952 3776 cni.go:424] [tsf/patch-tekton-config-fjvd8 65da19b340bb0c1f15233dcf28cdfc55fc2fbcc54642221dabcbc015cff5f0ed network default NAD default] DEL finished CNI request [tsf/patch-tekton-config-fjvd8 65da19b340bb0c1f15233dcf28cdfc55fc2fbcc54642221dabcbc015cff5f0ed network default NAD default], result "{}", err I0406 03:11:20.211057 3776 obj_retry.go:464] Detected object tsf/patch-tekton-config-fjvd8 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0406 03:11:20.211103 3776 pods.go:176] Deleting pod: tsf/patch-tekton-config-fjvd8 I0406 03:11:20.212762 3776 pods.go:236] Attempting to release IPs for pod: tsf/patch-tekton-config-fjvd8, ips: 10.129.0.45 I0406 03:11:20.212801 3776 obj_retry.go:464] Detected object tsf/patch-tekton-config-fjvd8 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0406 03:11:21.091877 3776 obj_retry.go:458] Detected object tsf/patch-tekton-config-fjvd8 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:11:21.091910 3776 obj_retry.go:458] Detected object tsf/patch-tekton-config-fjvd8 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0406 03:11:21.194841 3776 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/ctlog I0406 03:11:21.198720 3776 base_network_controller_pods.go:487] [default/tsf/tsf-tekton-configuration-fhf59] creating logical port tsf_tsf-tekton-configuration-fhf59 for pod on switch ip-10-0-2-216.ec2.internal I0406 03:11:21.198814 3776 kube.go:256] Updating pod tsf/tsf-tekton-configuration-fhf59 I0406 03:11:21.210615 3776 pod.go:62] [tsf/tsf-tekton-configuration-fhf59] pod update took 11.8181ms I0406 03:11:21.210646 3776 base_network_controller_pods.go:937] [default/tsf/tsf-tekton-configuration-fhf59] addLogicalPort annotation time took 11.855756ms I0406 03:11:21.212043 3776 pods.go:271] [tsf/tsf-tekton-configuration-fhf59] addLogicalPort took 13.327085ms, libovsdb time 959.881µs I0406 03:11:21.561869 3776 cni.go:403] [tsf/tsf-tekton-configuration-fhf59 581d09ee754e44e1ecfd00216bb168802b4b7453c6bdf62ab8d9d882e897d2e3 network default NAD default] ADD starting CNI request [tsf/tsf-tekton-configuration-fhf59 581d09ee754e44e1ecfd00216bb168802b4b7453c6bdf62ab8d9d882e897d2e3 network default NAD default] I0406 03:11:21.571337 3776 helper_linux.go:450] ConfigureOVS: namespace: tsf, podName: tsf-tekton-configuration-fhf59, hostIfaceName: 581d09ee754e44e, network: default, NAD default, SandboxID: "581d09ee754e44e1ecfd00216bb168802b4b7453c6bdf62ab8d9d882e897d2e3", PCI device ID: , UID: "51e18ea2-dc47-40c3-aeb2-ddc4cdf327fc", MAC: 0a:58:0a:81:00:2e, IPs: [10.129.0.46/23] I0406 03:11:21.627200 3776 cni.go:424] [tsf/tsf-tekton-configuration-fhf59 581d09ee754e44e1ecfd00216bb168802b4b7453c6bdf62ab8d9d882e897d2e3 network default NAD default] ADD finished CNI request [tsf/tsf-tekton-configuration-fhf59 581d09ee754e44e1ecfd00216bb168802b4b7453c6bdf62ab8d9d882e897d2e3 network default NAD default], result "{\"interfaces\":[{\"mac\":\"5a:d0:f6:54:bd:e8\",\"name\":\"581d09ee754e44e\"},{\"mac\":\"0a:58:0a:81:00:2e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9edb6e25-9946-4340-a463-b191b42d2f56\"}],\"ips\":[{\"address\":\"10.129.0.46/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0406 03:11:26.118245 3776 cni.go:403] [tsf/tsf-tekton-configuration-fhf59 581d09ee754e44e1ecfd00216bb168802b4b7453c6bdf62ab8d9d882e897d2e3 network default NAD default] DEL starting CNI request [tsf/tsf-tekton-configuration-fhf59 581d09ee754e44e1ecfd00216bb168802b4b7453c6bdf62ab8d9d882e897d2e3 network default NAD default] I0406 03:11:26.201986 3776 obj_retry.go:464] Detected object tsf-tas/rekor-createtree-job-lfsnf-khdt2 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0406 03:11:26.206624 3776 obj_retry.go:464] Detected object tsf-tas/rekor-createtree-job-lfsnf-khdt2 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0406 03:11:26.219114 3776 cni.go:424] [tsf/tsf-tekton-configuration-fhf59 581d09ee754e44e1ecfd00216bb168802b4b7453c6bdf62ab8d9d882e897d2e3 network default NAD default] DEL finished CNI request [tsf/tsf-tekton-configuration-fhf59 581d09ee754e44e1ecfd00216bb168802b4b7453c6bdf62ab8d9d882e897d2e3 network default NAD default], result "{}", err I0406 03:11:26.273880 3776 obj_retry.go:464] Detected object tsf/tsf-tekton-configuration-fhf59 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0406 03:11:26.273927 3776 pods.go:176] Deleting pod: tsf/tsf-tekton-configuration-fhf59 I0406 03:11:26.275351 3776 pods.go:236] Attempting to release IPs for pod: tsf/tsf-tekton-configuration-fhf59, ips: 10.129.0.46 I0406 03:11:26.275386 3776 obj_retry.go:464] Detected object tsf/tsf-tekton-configuration-fhf59 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0406 03:11:27.082997 3776 obj_retry.go:458] Detected object tsf-tas/rekor-createtree-job-lfsnf-khdt2 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:11:27.083027 3776 obj_retry.go:458] Detected object tsf-tas/rekor-createtree-job-lfsnf-khdt2 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:11:27.297186 3776 obj_retry.go:458] Detected object tsf/tsf-tekton-configuration-fhf59 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:11:27.297211 3776 obj_retry.go:458] Detected object tsf/tsf-tekton-configuration-fhf59 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:11:27.328991 3776 obj_retry.go:458] Detected object tsf/patch-tekton-config-fjvd8 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:11:27.329019 3776 obj_retry.go:458] Detected object tsf/patch-tekton-config-fjvd8 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:11:27.333437 3776 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/patch-tekton-config-fjvd8 I0406 03:11:27.333472 3776 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/patch-tekton-config-fjvd8 I0406 03:11:31.222852 3776 obj_retry.go:464] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0406 03:11:31.223459 3776 obj_retry.go:464] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0406 03:11:31.241320 3776 obj_retry.go:458] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:11:31.241345 3776 obj_retry.go:458] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:11:31.246451 3776 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/pre-install-tsf-tpa I0406 03:11:31.246474 3776 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/pre-install-tsf-tpa W0406 03:11:33.134682 3776 util.go:870] Failed to build global endpoints for port TCP/resp: empty IP address endpoints for service tsf-tas/rekor-redis I0406 03:11:33.481040 3776 base_network_controller_pods.go:487] [default/tsf-tas/rekor-server-6df69f77c7-6c7fk] creating logical port tsf-tas_rekor-server-6df69f77c7-6c7fk for pod on switch ip-10-0-2-216.ec2.internal I0406 03:11:33.481150 3776 kube.go:256] Updating pod tsf-tas/rekor-server-6df69f77c7-6c7fk I0406 03:11:33.496188 3776 pod.go:62] [tsf-tas/rekor-server-6df69f77c7-6c7fk] pod update took 15.073934ms I0406 03:11:33.496216 3776 base_network_controller_pods.go:937] [default/tsf-tas/rekor-server-6df69f77c7-6c7fk] addLogicalPort annotation time took 15.108167ms I0406 03:11:33.497591 3776 pods.go:271] [tsf-tas/rekor-server-6df69f77c7-6c7fk] addLogicalPort took 16.566217ms, libovsdb time 958.025µs W0406 03:11:34.253418 3776 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-search-ui I0406 03:11:37.096695 3776 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=10 I0406 03:11:37.268776 3776 obj_retry.go:464] Detected object tsf-tpa/create-db-k882r of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0406 03:11:37.269397 3776 obj_retry.go:464] Detected object tsf-tpa/create-db-k882r of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0406 03:11:38.293435 3776 obj_retry.go:458] Detected object tsf-tpa/create-db-k882r of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:11:38.293462 3776 obj_retry.go:458] Detected object tsf-tpa/create-db-k882r of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:11:39.543734 3776 cni.go:403] [tsf-tas/rekor-server-6df69f77c7-6c7fk dabdfd0aff9d0cba3ec323eb15df03d449de67cf58c548aa828d91b1424ef8e9 network default NAD default] ADD starting CNI request [tsf-tas/rekor-server-6df69f77c7-6c7fk dabdfd0aff9d0cba3ec323eb15df03d449de67cf58c548aa828d91b1424ef8e9 network default NAD default] I0406 03:11:39.553513 3776 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: rekor-server-6df69f77c7-6c7fk, hostIfaceName: dabdfd0aff9d0cb, network: default, NAD default, SandboxID: "dabdfd0aff9d0cba3ec323eb15df03d449de67cf58c548aa828d91b1424ef8e9", PCI device ID: , UID: "f3a0fa2d-81ad-48fb-a60e-02ca1314fcb2", MAC: 0a:58:0a:81:00:2f, IPs: [10.129.0.47/23] I0406 03:11:39.606082 3776 cni.go:424] [tsf-tas/rekor-server-6df69f77c7-6c7fk dabdfd0aff9d0cba3ec323eb15df03d449de67cf58c548aa828d91b1424ef8e9 network default NAD default] ADD finished CNI request [tsf-tas/rekor-server-6df69f77c7-6c7fk dabdfd0aff9d0cba3ec323eb15df03d449de67cf58c548aa828d91b1424ef8e9 network default NAD default], result "{\"interfaces\":[{\"mac\":\"3a:5c:5f:00:a4:21\",\"name\":\"dabdfd0aff9d0cb\"},{\"mac\":\"0a:58:0a:81:00:2f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9bc828f5-ea22-4c3f-b1f4-671c6e0df031\"}],\"ips\":[{\"address\":\"10.129.0.47/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0406 03:11:41.015144 3776 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=499 I0406 03:11:41.283201 3776 obj_retry.go:464] Detected object tsf-tpa/migrate-db-b9dtx of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0406 03:11:41.283737 3776 obj_retry.go:464] Detected object tsf-tpa/migrate-db-b9dtx of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0406 03:11:41.433422 3776 obj_retry.go:458] Detected object tsf-tpa/migrate-db-b9dtx of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:11:41.433446 3776 obj_retry.go:458] Detected object tsf-tpa/migrate-db-b9dtx of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0406 03:11:42.213911 3776 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-server I0406 03:11:44.311660 3776 obj_retry.go:464] Detected object tsf-tpa/create-importers-2z247 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0406 03:11:44.312242 3776 obj_retry.go:464] Detected object tsf-tpa/create-importers-2z247 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0406 03:11:44.564723 3776 base_network_controller_pods.go:487] [default/tsf-tpa/server-84fb47c457-rssdr] creating logical port tsf-tpa_server-84fb47c457-rssdr for pod on switch ip-10-0-2-216.ec2.internal I0406 03:11:44.564818 3776 kube.go:256] Updating pod tsf-tpa/server-84fb47c457-rssdr I0406 03:11:44.577694 3776 pod.go:62] [tsf-tpa/server-84fb47c457-rssdr] pod update took 12.897723ms I0406 03:11:44.577727 3776 base_network_controller_pods.go:937] [default/tsf-tpa/server-84fb47c457-rssdr] addLogicalPort annotation time took 12.937545ms I0406 03:11:44.579207 3776 pods.go:271] [tsf-tpa/server-84fb47c457-rssdr] addLogicalPort took 14.498337ms, libovsdb time 935.163µs I0406 03:11:44.642927 3776 obj_retry.go:458] Detected object tsf-tpa/create-importers-2z247 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:11:44.642953 3776 obj_retry.go:458] Detected object tsf-tpa/create-importers-2z247 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:11:46.023552 3776 base_network_controller_pods.go:487] [default/tsf-tpa/importer-5b5b6dd8fb-zd8n6] creating logical port tsf-tpa_importer-5b5b6dd8fb-zd8n6 for pod on switch ip-10-0-2-216.ec2.internal I0406 03:11:46.023673 3776 kube.go:256] Updating pod tsf-tpa/importer-5b5b6dd8fb-zd8n6 I0406 03:11:46.036729 3776 pod.go:62] [tsf-tpa/importer-5b5b6dd8fb-zd8n6] pod update took 13.084546ms I0406 03:11:46.036755 3776 base_network_controller_pods.go:937] [default/tsf-tpa/importer-5b5b6dd8fb-zd8n6] addLogicalPort annotation time took 13.116948ms I0406 03:11:46.038116 3776 pods.go:271] [tsf-tpa/importer-5b5b6dd8fb-zd8n6] addLogicalPort took 14.585421ms, libovsdb time 866.335µs I0406 03:11:50.670111 3776 cni.go:403] [tsf-tpa/importer-5b5b6dd8fb-zd8n6 bc9aaa744b2b1e7fe0ecda73b61fae2f848da8e2e7255e136a95ec9a96a35c7a network default NAD default] ADD starting CNI request [tsf-tpa/importer-5b5b6dd8fb-zd8n6 bc9aaa744b2b1e7fe0ecda73b61fae2f848da8e2e7255e136a95ec9a96a35c7a network default NAD default] I0406 03:11:50.679815 3776 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: importer-5b5b6dd8fb-zd8n6, hostIfaceName: bc9aaa744b2b1e7, network: default, NAD default, SandboxID: "bc9aaa744b2b1e7fe0ecda73b61fae2f848da8e2e7255e136a95ec9a96a35c7a", PCI device ID: , UID: "05af9597-e8c5-4674-884b-c5c6df0991c0", MAC: 0a:58:0a:81:00:31, IPs: [10.129.0.49/23] I0406 03:11:50.730103 3776 cni.go:403] [tsf-tpa/server-84fb47c457-rssdr 871b95c128cd7707fae38b4fca95bc799cb3dfc1192a2ee6887283bc8ec012be network default NAD default] ADD starting CNI request [tsf-tpa/server-84fb47c457-rssdr 871b95c128cd7707fae38b4fca95bc799cb3dfc1192a2ee6887283bc8ec012be network default NAD default] I0406 03:11:50.736310 3776 cni.go:424] [tsf-tpa/importer-5b5b6dd8fb-zd8n6 bc9aaa744b2b1e7fe0ecda73b61fae2f848da8e2e7255e136a95ec9a96a35c7a network default NAD default] ADD finished CNI request [tsf-tpa/importer-5b5b6dd8fb-zd8n6 bc9aaa744b2b1e7fe0ecda73b61fae2f848da8e2e7255e136a95ec9a96a35c7a network default NAD default], result "{\"interfaces\":[{\"mac\":\"22:03:13:2a:2c:57\",\"name\":\"bc9aaa744b2b1e7\"},{\"mac\":\"0a:58:0a:81:00:31\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8ddb7719-5c43-4791-8caa-21cab6e8f060\"}],\"ips\":[{\"address\":\"10.129.0.49/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0406 03:11:50.740571 3776 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: server-84fb47c457-rssdr, hostIfaceName: 871b95c128cd770, network: default, NAD default, SandboxID: "871b95c128cd7707fae38b4fca95bc799cb3dfc1192a2ee6887283bc8ec012be", PCI device ID: , UID: "45fae549-08f8-48f4-a50e-1bf887fc1c7a", MAC: 0a:58:0a:81:00:30, IPs: [10.129.0.48/23] I0406 03:11:50.798625 3776 cni.go:424] [tsf-tpa/server-84fb47c457-rssdr 871b95c128cd7707fae38b4fca95bc799cb3dfc1192a2ee6887283bc8ec012be network default NAD default] ADD finished CNI request [tsf-tpa/server-84fb47c457-rssdr 871b95c128cd7707fae38b4fca95bc799cb3dfc1192a2ee6887283bc8ec012be network default NAD default], result "{\"interfaces\":[{\"mac\":\"62:13:79:4a:77:a5\",\"name\":\"871b95c128cd770\"},{\"mac\":\"0a:58:0a:81:00:30\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e7c7e8c3-8b02-43c3-954a-005cc2ced8ea\"}],\"ips\":[{\"address\":\"10.129.0.48/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0406 03:11:53.239416 3776 util.go:959] Failed to build node endpoints for node ip-10-0-2-216.ec2.internal port 3000: empty IP address endpoints for service tsf-tas/rekor-server W0406 03:11:54.289531 3776 util.go:870] Failed to build global endpoints for port TCP/endpoint: empty IP address endpoints for service tsf-tpa/server W0406 03:11:55.329669 3776 util.go:959] Failed to build node endpoints for node ip-10-0-2-216.ec2.internal port 8080: empty IP address endpoints for service tsf-tpa/server I0406 03:11:57.373764 3776 obj_retry.go:464] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0406 03:11:57.374349 3776 obj_retry.go:464] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0406 03:11:57.394781 3776 obj_retry.go:458] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:11:57.394812 3776 obj_retry.go:458] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:11:57.401651 3776 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/test-tsf-tpa I0406 03:11:57.401681 3776 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/test-tsf-tpa I0406 03:11:58.001445 3776 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=71 I0406 03:11:58.114610 3776 namespace.go:98] [integration-service] adding namespace I0406 03:11:58.115532 3776 namespace.go:102] [integration-service] adding namespace took 897.926µs I0406 03:11:58.119719 3776 namespace.go:98] [default-tenant] adding namespace I0406 03:11:58.120502 3776 namespace.go:102] [default-tenant] adding namespace took 766.416µs I0406 03:11:58.121160 3776 namespace.go:98] [image-controller] adding namespace I0406 03:11:58.121343 3776 namespace.go:98] [build-service] adding namespace I0406 03:11:58.121877 3776 namespace.go:102] [image-controller] adding namespace took 699.536µs I0406 03:11:58.122537 3776 namespace.go:102] [build-service] adding namespace took 1.177965ms I0406 03:11:58.163157 3776 namespace.go:142] [integration-service] updating namespace I0406 03:11:58.183849 3776 namespace.go:142] [integration-service] updating namespace I0406 03:11:58.183861 3776 namespace.go:142] [default-tenant] updating namespace I0406 03:11:58.196742 3776 namespace.go:142] [default-tenant] updating namespace I0406 03:11:58.209987 3776 namespace.go:142] [image-controller] updating namespace I0406 03:11:58.227914 3776 namespace.go:142] [image-controller] updating namespace I0406 03:11:58.230203 3776 namespace.go:142] [build-service] updating namespace I0406 03:11:58.245858 3776 namespace.go:142] [build-service] updating namespace I0406 03:11:58.338403 3776 namespace.go:142] [build-service] updating namespace I0406 03:11:58.353553 3776 namespace.go:142] [integration-service] updating namespace I0406 03:11:58.388090 3776 namespace.go:98] [release-service] adding namespace I0406 03:11:58.388898 3776 namespace.go:102] [release-service] adding namespace took 782.892µs I0406 03:11:58.411447 3776 namespace.go:142] [release-service] updating namespace I0406 03:11:58.414831 3776 namespace.go:142] [konflux-ui] updating namespace I0406 03:11:58.426786 3776 namespace.go:142] [release-service] updating namespace I0406 03:11:58.498991 3776 namespace.go:98] [konflux-info] adding namespace I0406 03:11:58.499916 3776 namespace.go:102] [konflux-info] adding namespace took 900.2µs I0406 03:11:58.526909 3776 namespace.go:98] [namespace-lister] adding namespace I0406 03:11:58.527775 3776 namespace.go:102] [namespace-lister] adding namespace took 840.927µs I0406 03:11:58.548901 3776 namespace.go:142] [konflux-info] updating namespace I0406 03:11:58.593969 3776 namespace.go:98] [enterprise-contract-service] adding namespace I0406 03:11:58.594024 3776 namespace.go:142] [image-controller] updating namespace I0406 03:11:58.594837 3776 namespace.go:102] [enterprise-contract-service] adding namespace took 842.049µs I0406 03:11:58.615891 3776 namespace.go:142] [konflux-info] updating namespace I0406 03:11:58.645671 3776 namespace.go:142] [namespace-lister] updating namespace I0406 03:11:58.715755 3776 namespace.go:142] [enterprise-contract-service] updating namespace I0406 03:11:58.724642 3776 namespace.go:142] [namespace-lister] updating namespace I0406 03:11:58.741726 3776 namespace.go:142] [default-tenant] updating namespace I0406 03:11:58.778923 3776 namespace.go:142] [enterprise-contract-service] updating namespace I0406 03:11:58.794906 3776 namespace.go:98] [segment-bridge] adding namespace I0406 03:11:58.795801 3776 namespace.go:102] [segment-bridge] adding namespace took 870.539µs I0406 03:11:58.849683 3776 namespace.go:142] [openshift-pipelines] updating namespace I0406 03:11:58.877915 3776 namespace.go:142] [segment-bridge] updating namespace I0406 03:11:58.931771 3776 namespace.go:142] [segment-bridge] updating namespace I0406 03:11:59.095637 3776 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-from-konfluxui for network default I0406 03:11:59.096304 3776 pod_selector_address_set.go:212] Created shared address set for pod selector namespace-lister_LS{ML:{app: proxy,},} I0406 03:11:59.096322 3776 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-from-konfluxui added to peer address sets [namespace-lister_LS{ML:{app: proxy,},}] I0406 03:11:59.099034 3776 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-from-konfluxui resources completed, update namespace loglevel I0406 03:11:59.211860 3776 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-to-apiserver for network default I0406 03:11:59.211890 3776 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-to-apiserver added to peer address sets [] I0406 03:11:59.212808 3776 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-to-apiserver resources completed, update namespace loglevel I0406 03:11:59.717779 3776 namespace.go:142] [build-service] updating namespace I0406 03:12:01.000337 3776 namespace.go:142] [default-tenant] updating namespace I0406 03:12:01.419566 3776 namespace.go:142] [enterprise-contract-service] updating namespace I0406 03:12:01.665491 3776 namespace.go:142] [image-controller] updating namespace I0406 03:12:01.878669 3776 namespace.go:142] [integration-service] updating namespace I0406 03:12:02.086546 3776 namespace.go:142] [konflux-info] updating namespace I0406 03:12:02.271295 3776 namespace.go:142] [namespace-lister] updating namespace I0406 03:12:02.463556 3776 namespace.go:142] [release-service] updating namespace I0406 03:12:02.582107 3776 namespace.go:142] [build-service] updating namespace I0406 03:12:02.597718 3776 namespace.go:142] [default-tenant] updating namespace I0406 03:12:02.610576 3776 namespace.go:142] [enterprise-contract-service] updating namespace I0406 03:12:02.622091 3776 namespace.go:142] [image-controller] updating namespace I0406 03:12:02.636829 3776 namespace.go:142] [integration-service] updating namespace I0406 03:12:02.659756 3776 namespace.go:142] [konflux-info] updating namespace I0406 03:12:02.676223 3776 namespace.go:142] [namespace-lister] updating namespace I0406 03:12:02.694190 3776 namespace.go:142] [release-service] updating namespace I0406 03:12:02.757451 3776 namespace.go:142] [build-service] updating namespace I0406 03:12:02.821042 3776 namespace.go:142] [default-tenant] updating namespace I0406 03:12:02.986614 3776 namespace.go:142] [enterprise-contract-service] updating namespace I0406 03:12:03.188807 3776 namespace.go:142] [image-controller] updating namespace I0406 03:12:03.288875 3776 namespace.go:142] [integration-service] updating namespace I0406 03:12:03.411449 3776 namespace.go:142] [konflux-info] updating namespace W0406 03:12:03.430544 3776 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service namespace-lister/namespace-lister I0406 03:12:03.515864 3776 namespace.go:142] [namespace-lister] updating namespace I0406 03:12:03.615032 3776 namespace.go:142] [release-service] updating namespace I0406 03:12:04.861926 3776 namespace.go:142] [segment-bridge] updating namespace I0406 03:12:04.993739 3776 namespace.go:142] [segment-bridge] updating namespace I0406 03:12:05.093249 3776 namespace.go:142] [segment-bridge] updating namespace W0406 03:12:05.430004 3776 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service image-controller/image-controller-controller-manager-metrics-service W0406 03:12:05.448478 3776 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service W0406 03:12:11.477194 3776 util.go:870] Failed to build global endpoints for port TCP/dex: empty IP address endpoints for service konflux-ui/dex I0406 03:12:11.501685 3776 obj_retry.go:464] Detected object tsf-tas/tuf-repository-init-pvsbd-6v2t2 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0406 03:12:11.502318 3776 obj_retry.go:464] Detected object tsf-tas/tuf-repository-init-pvsbd-6v2t2 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0406 03:12:12.368781 3776 obj_retry.go:458] Detected object tsf-tas/tuf-repository-init-pvsbd-6v2t2 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:12:12.368812 3776 obj_retry.go:458] Detected object tsf-tas/tuf-repository-init-pvsbd-6v2t2 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0406 03:12:12.502434 3776 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0406 03:12:12.502492 3776 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0406 03:12:12.527620 3776 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0406 03:12:12.528473 3776 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0406 03:12:15.386152 3776 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0406 03:12:15.386176 3776 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0406 03:12:15.411730 3776 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service W0406 03:12:15.411781 3776 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service release-service/release-service-webhook-service W0406 03:12:16.519469 3776 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0406 03:12:16.519496 3776 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0406 03:12:20.412798 3776 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/tuf I0406 03:12:22.103387 3776 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=10 I0406 03:12:23.839108 3776 obj_retry.go:464] Detected object integration-service/integration-service-controller-manager-7884d6c5d6-dmk2n of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0406 03:12:23.839724 3776 obj_retry.go:464] Detected object integration-service/integration-service-controller-manager-7884d6c5d6-dmk2n of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0406 03:12:24.560236 3776 obj_retry.go:458] Detected object integration-service/integration-service-controller-manager-7884d6c5d6-dmk2n of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:12:24.560268 3776 obj_retry.go:458] Detected object integration-service/integration-service-controller-manager-7884d6c5d6-dmk2n of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:12:24.570183 3776 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod integration-service/integration-service-controller-manager-7884d6c5d6-dmk2n I0406 03:12:24.570201 3776 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod integration-service/integration-service-controller-manager-7884d6c5d6-dmk2n I0406 03:12:48.001076 3776 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=10 I0406 03:12:58.090459 3776 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=8 I0406 03:13:24.436232 3776 obj_retry.go:464] Detected object konflux-ui/proxy-5bd6d778cf-lnqzn of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0406 03:13:24.436819 3776 obj_retry.go:464] Detected object konflux-ui/proxy-5bd6d778cf-lnqzn of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0406 03:13:24.770751 3776 obj_retry.go:458] Detected object konflux-ui/proxy-5bd6d778cf-lnqzn of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:13:24.770781 3776 obj_retry.go:458] Detected object konflux-ui/proxy-5bd6d778cf-lnqzn of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:13:24.776592 3776 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/proxy-5bd6d778cf-lnqzn I0406 03:13:24.776620 3776 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/proxy-5bd6d778cf-lnqzn I0406 03:13:29.934789 3776 obj_retry.go:464] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0406 03:13:29.935486 3776 obj_retry.go:464] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0406 03:13:29.970792 3776 obj_retry.go:458] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:13:29.970821 3776 obj_retry.go:458] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:13:29.976272 3776 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/test-tsf-konflux I0406 03:13:29.976301 3776 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/test-tsf-konflux I0406 03:13:48.118709 3776 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=9 I0406 03:14:00.103055 3776 obj_retry.go:464] Detected object konflux-operator/konflux-operator-controller-manager-5bc5d8c98f-qbl86 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0406 03:14:00.103716 3776 obj_retry.go:464] Detected object konflux-operator/konflux-operator-controller-manager-5bc5d8c98f-qbl86 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0406 03:14:01.101116 3776 obj_retry.go:458] Detected object konflux-operator/konflux-operator-controller-manager-5bc5d8c98f-qbl86 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:14:01.101171 3776 obj_retry.go:458] Detected object konflux-operator/konflux-operator-controller-manager-5bc5d8c98f-qbl86 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:14:01.106695 3776 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-operator/konflux-operator-controller-manager-5bc5d8c98f-qbl86 I0406 03:14:01.106729 3776 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-operator/konflux-operator-controller-manager-5bc5d8c98f-qbl86 I0406 03:14:15.323076 3776 obj_retry.go:464] Detected object image-controller/image-controller-controller-manager-fc44cc754-8css2 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0406 03:14:15.323670 3776 obj_retry.go:464] Detected object image-controller/image-controller-controller-manager-fc44cc754-8css2 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0406 03:14:16.175398 3776 obj_retry.go:458] Detected object image-controller/image-controller-controller-manager-fc44cc754-8css2 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:14:16.175431 3776 obj_retry.go:458] Detected object image-controller/image-controller-controller-manager-fc44cc754-8css2 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:14:16.180590 3776 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod image-controller/image-controller-controller-manager-fc44cc754-8css2 I0406 03:14:16.180619 3776 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod image-controller/image-controller-controller-manager-fc44cc754-8css2 I0406 03:14:26.124532 3776 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=8 I0406 03:14:58.006926 3776 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=121 I0406 03:15:06.117096 3776 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=8 I0406 03:16:00.002862 3776 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=180 I0406 03:16:18.126929 3776 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=7 I0406 03:17:17.005979 3776 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=35 I0406 03:17:29.109314 3776 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=7 I0406 03:18:20.089342 3776 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=9 I0406 03:18:24.111014 3776 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=5 I0406 03:19:05.004971 3776 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=8 I0406 03:20:04.130026 3776 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29590760-7flnq of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0406 03:20:04.130633 3776 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29590760-7flnq of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0406 03:20:05.004287 3776 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29590760-7flnq of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:20:05.004347 3776 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29590760-7flnq of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:20:33.018648 3776 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=179 I0406 03:20:39.004395 3776 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=20 I0406 03:20:53.093397 3776 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=11 I0406 03:20:55.093079 3776 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=7 I0406 03:21:28.099384 3776 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=11 I0406 03:21:32.119971 3776 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=8 I0406 03:21:35.127626 3776 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=8 I0406 03:21:38.045805 3776 obj_retry.go:458] Detected object tsf-tpa/create-db-k882r of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:21:38.045852 3776 obj_retry.go:458] Detected object tsf-tpa/create-db-k882r of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:21:38.049703 3776 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-db-k882r I0406 03:21:38.049724 3776 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-db-k882r I0406 03:21:41.029598 3776 obj_retry.go:458] Detected object tsf-tpa/migrate-db-b9dtx of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:21:41.029652 3776 obj_retry.go:458] Detected object tsf-tpa/migrate-db-b9dtx of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:21:41.033371 3776 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/migrate-db-b9dtx I0406 03:21:41.033395 3776 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/migrate-db-b9dtx I0406 03:21:44.029182 3776 obj_retry.go:458] Detected object tsf-tpa/create-importers-2z247 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:21:44.029232 3776 obj_retry.go:458] Detected object tsf-tpa/create-importers-2z247 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0406 03:21:44.032960 3776 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-importers-2z247 I0406 03:21:44.032983 3776 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-importers-2z247 I0406 03:21:58.122230 3776 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=8 I0406 03:22:03.422169 3776 namespace.go:98] [default-tenant-managed] adding namespace I0406 03:22:03.423107 3776 namespace.go:102] [default-tenant-managed] adding namespace took 909.091µs I0406 03:22:03.474792 3776 namespace.go:142] [default-tenant-managed] updating namespace I0406 03:22:03.495149 3776 namespace.go:142] [default-tenant-managed] updating namespace I0406 03:22:04.562339 3776 namespace.go:142] [default-tenant-managed] updating namespace I0406 03:22:04.692410 3776 namespace.go:142] [default-tenant-managed] updating namespace I0406 03:22:04.794577 3776 namespace.go:142] [default-tenant-managed] updating namespace I0406 03:22:15.005701 3776 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=7