stats: metric expiry (#40395)

Change-Id: If3a45283b13cfda7d4f9a7bb661a1573f552ed7e
Commit Message: Introduce mark and sweep eviction of stale metrics in a
stats scope.

Additional Description: The intended use case is the high cardinality
metrics generated from the request data (e.g. [Istio standard
metrics](https://istio.io/latest/docs/reference/config/metrics/)). This
in combination with the cardinality bounds (future PR) would ensure
bounded metric resource usage. The algorithm works as follows:

1. An "evictable" scope is allocated by a filter.
2. A delta stats sink is configured, e.g. OTLP.
3. At every flush interval, a scope metric that is used (e.g. has
observed a data point) is marked as unused. A metric that has not been
used is deleted from the central caches.
4. A notification is sent to all workers to purge scope stale metrics
from their thread-local caches.
5. Once all workers complete, the unused metrics are purged from the
allocator.

There are several edge conditions that need to be explained to validate
correctness of this algorithm:

1. A worker attempting to use a stale metric after (3) but before (4)
might have its data lost. It will not be lost if 1) the same metric is
recreated in the central cache by another worker since all metrics are
uniquely indexed in the allocators; or 2) we implement deferred
allocator deletions to await for the flush operation.

2. A worker should not use a stored stale metric after (4). This
requires that workers to not store the metrics by reference (hence, this
solution will not work for most xDS metrics). Thread local cache
references are always deleted before the storage is deleted.

3. Histograms are handled slightly different because the parent
histogram needs to be "merged" to observe usage, and clearing the usage
requires updating all "children" histograms. Because we do this during
flush, merging is always done first.

4. A metric that is re-created after eviction would continue having its
start time set as the original metric. This is a limitation of Envoy
since it does not store the metric start times, but it is not an issue
with delta aggregation in OTLP. Delta is the recommended protocol for
handling high cardinality or sparse metric data. We could add start_time
in a follow-up.

Risk Level: low, requires explicit usage
Testing: unit and a load test with Istio Proxy
Docs Changes: none
Release Notes: none

---------

Signed-off-by: Kuat Yessenov <[email protected]>

Author: kyessenov

api/envoy/config/bootstrap/v3/bootstrap.proto

@@ -41,7 +41,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE;
 // <config_overview_bootstrap>` for more detail.
 
 // Bootstrap :ref:`configuration overview <config_overview_bootstrap>`.
-// [#next-free-field: 42]
+// [#next-free-field: 43]
 message Bootstrap {
   option (udpa.annotations.versioning).previous_message_type =
       "envoy.config.bootstrap.v2.Bootstrap";
@@ -230,6 +230,14 @@ message Bootstrap {
     bool stats_flush_on_admin = 29 [(validate.rules).bool = {const: true}];
   }
 
+  oneof stats_eviction {
+    // Optional duration to perform metric eviction. At every interval, during the stats flush
+    // the unused metrics are removed from the worker caches and the used metrics
+    // are marked as unused. Must be a multiple of the ``stats_flush_interval``.
+    google.protobuf.Duration stats_eviction_interval = 42
+        [(validate.rules).duration = {gte {nanos: 1000000}}];
+  }
+
   // Optional watchdog configuration.
   // This is for a single watchdog configuration for the entire system.
   // Deprecated in favor of ``watchdogs`` which has finer granularity.

changelogs/current.yaml

@@ -145,6 +145,12 @@ removed_config_or_runtime:
     Removed runtime guard ``envoy.reloadable_features.proxy_104`` and legacy code paths.
 
 new_features:
+- area: stats
+  change: |
+    Added support to remove unused metrics from memory for extensions that
+    support evictable metrics. This is done :ref:`periodically
+    <envoy_v3_api_field_config.bootstrap.v3.Bootstrap.stats_eviction_interval>`
+    during the metric flush.
 - area: quic
   change: |
     Added new option to support :ref:`base64 encoded server ID

envoy/server/configuration.h

@@ -89,6 +89,11 @@ class StatsConfig {
    * @return true if deferred creation of stats is enabled.
    */
   virtual bool enableDeferredCreationStats() const PURE;
+
+  /**
+   * @return uint32_t a multiple of the flush interval to perform stats eviction, or 0 if disabled.
+   */
+  virtual uint32_t evictOnFlush() const PURE;
 };
 
 /**

envoy/stats/scope.h

@@ -71,17 +71,21 @@ class Scope : public std::enable_shared_from_this<Scope> {
    * See also scopeFromStatName, which is preferred.
    *
    * @param name supplies the scope's namespace prefix.
+   * @param evictable whether unused metrics can be deleted from the scope caches. This requires
+   * that the metrics are not stored by reference.
    */
-  virtual ScopeSharedPtr createScope(const std::string& name) PURE;
+  virtual ScopeSharedPtr createScope(const std::string& name, bool evictable = false) PURE;
 
   /**
    * Allocate a new scope. NOTE: The implementation should correctly handle overlapping scopes
    * that point to the same reference counted backing stats. This allows a new scope to be
    * gracefully swapped in while an old scope with the same name is being destroyed.
    *
    * @param name supplies the scope's namespace prefix.
+   * @param evictable whether unused metrics can be deleted from the scope caches. This requires
+   * that the metrics are not stored by reference.
    */
-  virtual ScopeSharedPtr scopeFromStatName(StatName name) PURE;
+  virtual ScopeSharedPtr scopeFromStatName(StatName name, bool evictable = false) PURE;
 
   /**
    * Creates a Counter from the stat name. Tag extraction will be performed on the name.

envoy/stats/stats.h

@@ -93,6 +93,11 @@ class Metric : public RefcountInterface {
    */
   virtual bool used() const PURE;
 
+  /**
+   * Clear any indicator on whether this metric has been updated.
+   */
+  virtual void markUnused() PURE;
+
   /**
    * Indicates whether this metric is hidden.
    */

envoy/stats/store.h

@@ -117,6 +117,11 @@ class Store {
   virtual void forEachHistogram(SizeFn f_size, StatFn<ParentHistogram> f_stat) const PURE;
   virtual void forEachScope(SizeFn f_size, StatFn<const Scope> f_stat) const PURE;
 
+  /**
+   * Delete unused metrics from all the evictable scope caches, and mark the rest as unused.
+   */
+  virtual void evictUnused() PURE;
+
   /**
    * @return a null counter that will ignore increments and always return 0.
    */
@@ -172,7 +177,9 @@ class Store {
   /**
    * @return a scope of the given name.
    */
-  ScopeSharedPtr createScope(const std::string& name) { return rootScope()->createScope(name); }
+  ScopeSharedPtr createScope(const std::string& name, bool evictable = false) {
+    return rootScope()->createScope(name, evictable);
+  }
 
   /**
    * Extracts tags from the name and appends them to the provided StatNameTagVector.

source/common/stats/allocator_impl.cc

@@ -85,6 +85,7 @@ template <class BaseClass> class StatsSharedImpl : public MetricImpl<BaseClass>
   // Metric
   SymbolTable& symbolTable() final { return alloc_.symbolTable(); }
   bool used() const override { return flags_ & Metric::Flags::Used; }
+  void markUnused() override { flags_ &= ~Metric::Flags::Used; }
   bool hidden() const override { return flags_ & Metric::Flags::Hidden; }
 
   // RefcountInterface

source/common/stats/histogram_impl.h

@@ -111,6 +111,7 @@ class HistogramImpl : public HistogramImplHelper {
   void recordValue(uint64_t value) override { parent_.deliverHistogramToSinks(*this, value); }
 
   bool used() const override { return true; }
+  void markUnused() override {}
   bool hidden() const override { return false; }
   SymbolTable& symbolTable() final { return parent_.symbolTable(); }
 
@@ -132,6 +133,7 @@ class NullHistogramImpl : public HistogramImplHelper {
   ~NullHistogramImpl() override { MetricImpl::clear(symbol_table_); }
 
   bool used() const override { return false; }
+  void markUnused() override {}
   bool hidden() const override { return false; }
   SymbolTable& symbolTable() override { return symbol_table_; }
 

source/common/stats/isolated_store_impl.cc

@@ -63,12 +63,12 @@ ConstScopeSharedPtr IsolatedStoreImpl::constRootScope() const {
 
 IsolatedStoreImpl::~IsolatedStoreImpl() = default;
 
-ScopeSharedPtr IsolatedScopeImpl::createScope(const std::string& name) {
+ScopeSharedPtr IsolatedScopeImpl::createScope(const std::string& name, bool) {
   StatNameManagedStorage stat_name_storage(Utility::sanitizeStatsName(name), symbolTable());
-  return scopeFromStatName(stat_name_storage.statName());
+  return scopeFromStatName(stat_name_storage.statName(), false);
 }
 
-ScopeSharedPtr IsolatedScopeImpl::scopeFromStatName(StatName name) {
+ScopeSharedPtr IsolatedScopeImpl::scopeFromStatName(StatName name, bool) {
   SymbolTable::StoragePtr prefix_name_storage = symbolTable().join({prefix(), name});
   ScopeSharedPtr scope = store_.makeScope(StatName(prefix_name_storage.get()));
   addScopeToStore(scope);

source/common/stats/isolated_store_impl.h

@@ -203,6 +203,10 @@ class IsolatedStoreImpl : public Store {
     }
   }
 
+  void evictUnused() override {
+    // Do nothing. Eviction is only supported on the thread local stores.
+  }
+
   void forEachSinkedCounter(SizeFn f_size, StatFn<Counter> f_stat) const override {
     forEachCounter(f_size, f_stat);
   }
@@ -295,8 +299,8 @@ class IsolatedScopeImpl : public Scope {
                                        StatNameTagVectorOptConstRef tags) override {
     return store_.counters_.get(prefix(), name, tags, symbolTable());
   }
-  ScopeSharedPtr createScope(const std::string& name) override;
-  ScopeSharedPtr scopeFromStatName(StatName name) override;
+  ScopeSharedPtr createScope(const std::string& name, bool evictable) override;
+  ScopeSharedPtr scopeFromStatName(StatName name, bool evictable) override;
   Gauge& gaugeFromStatNameWithTags(const StatName& name, StatNameTagVectorOptConstRef tags,
                                    Gauge::ImportMode import_mode) override {
     Gauge& gauge = store_.gauges_.get(prefix(), name, tags, symbolTable(), import_mode);