[core][gcs] Add a bounded TableName label to GCS Store count and latency metrics

doc/source/ray-observability/reference/system-metrics.md

@@ -48,6 +48,12 @@ These variables must be set in the Ray node and Dashboard Reporter process envir
 * - `ray_placement_groups`
   - `State`
   - Current number of placement groups by state. The State label (e.g., PENDING, CREATED, REMOVED) describes the state of the placement group. See [rpc::PlacementGroupTable](https://github.com/ray-project/ray/blob/e85355b9b593742b4f5cb72cab92051980fa73d3/src/ray/protobuf/gcs.proto#L517) for more information.
+* - `ray_gcs_storage_operation_count_total`
+  - `Operation`, `TableName`
+  - Number of operations the GCS invoked on its storage backend, broken down by operation (`Put`, `Get`, `GetAll`, `MultiGet`, `Delete`, `BatchDelete`, `GetKeys`, `Exists`, `GetNextJobID`) and by the GCS table it addressed. `TableName` is bounded to the names of Ray's internal `TablePrefix` enum, plus `JobCounter` for `GetNextJobID` and `OTHER` for a table name outside that enum. The values the GCS produces in practice are `JOB`, `ACTOR`, `ACTOR_TASK_SPEC`, `PLACEMENT_GROUP`, `NODE`, `WORKERS` and `KV`; the remaining enum names are legacy and no current code path emits them. Keys, internal-KV namespaces and any other user-supplied string never appear in labels. Recorded when the operation is issued, so this leads `ray_gcs_storage_operation_latency_ms` by the number of operations in flight. Exported only when the GCS storage backend is in-memory or RocksDB.
+* - `ray_gcs_storage_operation_latency_ms`
+  - `Operation`, `TableName`
+  - Time to invoke an operation on the GCS storage backend, with the same label domain as `ray_gcs_storage_operation_count_total`. Recorded when the operation completes.
 * - OpenTelemetry: `ray_gcs_redis_request_payload_bytes`; OpenCensus: `ray_gcs_redis_request_payload_bytes_total` (plus the deprecated unsuffixed gauge when `RAY_EXPORT_COUNTER_AS_GAUGE=true`)
   - `Command`, `TableName`
   - Application bytes in Redis command arguments accepted for sending by the GCS Redis client, by command and GCS table. Exported only when the GCS storage backend is Redis. Includes the verb, Redis key, field names, and values; excludes RESP framing, TLS, and TCP/IP overhead. Recorded on the first successful submission of each logical command; retries are not counted again. `Command` is the actual verb normalized to uppercase ASCII and truncated to 16 bytes; this limits label length, not the number of distinct labels, so production verbs must remain code-controlled. `TableName` is a GCS table, `NONE`, or `ALL`.

src/ray/asio/BUILD.bazel

@@ -62,6 +62,7 @@ ray_cc_library(
     hdrs = ["io_service_pool.h"],
     visibility = [
         "//src/ray/gcs/store_client:__pkg__",
+        "//src/ray/gcs/store_client/tests:__pkg__",
         "//src/ray/pubsub/tests:__pkg__",
         "//src/ray/stats:__pkg__",
     ],

src/ray/gcs/metrics.h

@@ -152,22 +152,35 @@ inline ray::stats::Gauge GetGcsActorByStateGaugeMetric() {
   };
 }
 
+// TableName is a bounded internal table name -- see NormalizeTableNameLabel in
+// store_client/table_name_label.h. Keys, internal-KV namespaces and any other
+// user-supplied string are never labels. Note that the tag_keys list below is
+// the tag allowlist on both metric stacks: OpenCensus turns it into the view's
+// column set (metric.cc, RegisterAsView) and OpenTelemetry seeds its attribute
+// map from it and drops anything undeclared (metric.cc, Metric::Record), so a
+// tag recorded without being declared here is exported by neither.
+
 inline ray::stats::Histogram GetGcsStorageOperationLatencyInMsHistogramMetric() {
   return ray::stats::Histogram{
       /*name=*/"gcs_storage_operation_latency_ms",
-      /*description=*/"Time to invoke an operation on Gcs storage",
+      /*description=*/
+      "Time to invoke an operation on GCS storage, broken down by operation and "
+      "by the GCS table it addressed. Recorded when the operation completes.",
       /*unit=*/"",
       /*boundaries=*/{0.1, 1, 10, 100, 1000, 10000},
-      /*tag_keys=*/{"Operation"},
+      /*tag_keys=*/{"Operation", "TableName"},
   };
 }
 
 inline ray::stats::Count GetGcsStorageOperationCountCounterMetric() {
   return ray::stats::Count{
       /*name=*/"gcs_storage_operation_count",
-      /*description=*/"Number of operations invoked on Gcs storage",
+      /*description=*/
+      "Number of operations invoked on GCS storage, broken down by operation and "
+      "by the GCS table it addressed. Recorded when the operation is issued, so "
+      "this leads gcs_storage_operation_latency_ms by the number in flight.",
       /*unit=*/"",
-      /*tag_keys=*/{"Operation"},
+      /*tag_keys=*/{"Operation", "TableName"},
   };
 }
 

src/ray/gcs/store_client/BUILD.bazel

@@ -71,6 +71,16 @@ ray_cc_library(
     ],
 )
 
+ray_cc_library(
+    name = "table_name_label",
+    srcs = ["table_name_label.cc"],
+    hdrs = ["table_name_label.h"],
+    visibility = [":__subpackages__"],
+    deps = [
+        "//src/ray/protobuf:gcs_cc_proto",
+    ],
+)
+
 ray_cc_library(
     name = "in_memory_store_client",
     srcs = ["in_memory_store_client.cc"],
@@ -98,6 +108,7 @@ ray_cc_library(
     ],
     deps = [
         ":store_client",
+        ":table_name_label",
         "//src/ray/util:clock",
     ],
 )

src/ray/gcs/store_client/observable_store_client.cc

@@ -14,134 +14,160 @@
 
 #include "ray/gcs/store_client/observable_store_client.h"
 
+#include <chrono>
 #include <string>
+#include <string_view>
 #include <utility>
 #include <vector>
 
-#include "absl/time/time.h"
+#include "ray/gcs/store_client/table_name_label.h"
 
 namespace ray {
 namespace gcs {
 
+namespace {
+
+// Both tags for one recording site. `table` must already be normalized (see
+// NormalizeTableNameLabel) so that TableName stays inside a bounded domain.
+std::vector<std::pair<std::string_view, std::string>> MetricTags(
+    std::string_view operation, std::string_view table) {
+  return {{"Operation", std::string(operation)}, {"TableName", std::string(table)}};
+}
+
+// Milliseconds elapsed since `start`. Durations are measured on the monotonic
+// clock, per the ClockInterface contract: Now()/NowUnixNanos() is wall time and
+// can jump with NTP -- backwards, which would record a negative latency, or
+// forwards, which would inflate one -- while SteadyNow() never goes backwards.
+double ElapsedMs(ClockInterface &clock, SteadyTimePoint start) {
+  return std::chrono::duration<double, std::milli>(clock.SteadyNow() - start).count();
+}
+
+}  // namespace
+
 void ObservableStoreClient::AsyncPut(const std::string &table_name,
                                      const std::string &key,
                                      std::string data,
                                      bool overwrite,
                                      Postable<void(bool)> callback) {
-  auto start = clock_.NowUnixNanos();
-  storage_operation_count_counter_.Record(1, {{"Operation", "Put"}});
+  const auto start = clock_.SteadyNow();
+  const std::string table = NormalizeTableNameLabel(table_name);
+  storage_operation_count_counter_.Record(1, MetricTags("Put", table));
   delegate_->AsyncPut(table_name,
                       key,
                       std::move(data),
                       overwrite,
-                      std::move(callback).OnInvocation([this, start]() {
-                        auto end = clock_.NowUnixNanos();
+                      std::move(callback).OnInvocation([this, start, table]() {
                         storage_operation_latency_in_ms_histogram_.Record(
-                            absl::ToDoubleMilliseconds(absl::Nanoseconds(end - start)),
-                            {{"Operation", "Put"}});
+                            ElapsedMs(clock_, start), MetricTags("Put", table));
                       }));
 }
 
 void ObservableStoreClient::AsyncGet(
     const std::string &table_name,
     const std::string &key,
     ToPostable<rpc::OptionalItemCallback<std::string>> callback) {
-  auto start = clock_.NowUnixNanos();
-  storage_operation_count_counter_.Record(1, {{"Operation", "Get"}});
-  delegate_->AsyncGet(table_name, key, std::move(callback).OnInvocation([this, start]() {
-    auto end = clock_.NowUnixNanos();
-    storage_operation_latency_in_ms_histogram_.Record(
-        absl::ToDoubleMilliseconds(absl::Nanoseconds(end - start)),
-        {{"Operation", "Get"}});
-  }));
+  const auto start = clock_.SteadyNow();
+  const std::string table = NormalizeTableNameLabel(table_name);
+  storage_operation_count_counter_.Record(1, MetricTags("Get", table));
+  delegate_->AsyncGet(
+      table_name, key, std::move(callback).OnInvocation([this, start, table]() {
+        storage_operation_latency_in_ms_histogram_.Record(ElapsedMs(clock_, start),
+                                                          MetricTags("Get", table));
+      }));
 }
 
 void ObservableStoreClient::AsyncGetAll(
     const std::string &table_name,
     Postable<void(absl::flat_hash_map<std::string, std::string>)> callback) {
-  auto start = clock_.NowUnixNanos();
-  storage_operation_count_counter_.Record(1, {{"Operation", "GetAll"}});
-  delegate_->AsyncGetAll(table_name, std::move(callback).OnInvocation([this, start]() {
-    auto end = clock_.NowUnixNanos();
-    storage_operation_latency_in_ms_histogram_.Record(
-        absl::ToDoubleMilliseconds(absl::Nanoseconds(end - start)),
-        {{"Operation", "GetAll"}});
-  }));
+  const auto start = clock_.SteadyNow();
+  const std::string table = NormalizeTableNameLabel(table_name);
+  storage_operation_count_counter_.Record(1, MetricTags("GetAll", table));
+  delegate_->AsyncGetAll(table_name,
+                         std::move(callback).OnInvocation([this, start, table]() {
+                           storage_operation_latency_in_ms_histogram_.Record(
+                               ElapsedMs(clock_, start), MetricTags("GetAll", table));
+                         }));
 }
 
 void ObservableStoreClient::AsyncMultiGet(
     const std::string &table_name,
     const std::vector<std::string> &keys,
     Postable<void(absl::flat_hash_map<std::string, std::string>)> callback) {
-  auto start = clock_.NowUnixNanos();
-  storage_operation_count_counter_.Record(1, {{"Operation", "MultiGet"}});
+  const auto start = clock_.SteadyNow();
+  const std::string table = NormalizeTableNameLabel(table_name);
+  storage_operation_count_counter_.Record(1, MetricTags("MultiGet", table));
   delegate_->AsyncMultiGet(
-      table_name, keys, std::move(callback).OnInvocation([this, start]() {
-        auto end = clock_.NowUnixNanos();
-        storage_operation_latency_in_ms_histogram_.Record(
-            absl::ToDoubleMilliseconds(absl::Nanoseconds(end - start)),
-            {{"Operation", "MultiGet"}});
+      table_name, keys, std::move(callback).OnInvocation([this, start, table]() {
+        storage_operation_latency_in_ms_histogram_.Record(ElapsedMs(clock_, start),
+                                                          MetricTags("MultiGet", table));
       }));
 }
 
 void ObservableStoreClient::AsyncDelete(const std::string &table_name,
                                         const std::string &key,
                                         Postable<void(bool)> callback) {
-  auto start = clock_.NowUnixNanos();
-  storage_operation_count_counter_.Record(1, {{"Operation", "Delete"}});
+  const auto start = clock_.SteadyNow();
+  const std::string table = NormalizeTableNameLabel(table_name);
+  storage_operation_count_counter_.Record(1, MetricTags("Delete", table));
   delegate_->AsyncDelete(
-      table_name, key, std::move(callback).OnInvocation([this, start]() {
-        auto end = clock_.NowUnixNanos();
-        storage_operation_latency_in_ms_histogram_.Record(
-            absl::ToDoubleMilliseconds(absl::Nanoseconds(end - start)),
-            {{"Operation", "Delete"}});
+      table_name, key, std::move(callback).OnInvocation([this, start, table]() {
+        storage_operation_latency_in_ms_histogram_.Record(ElapsedMs(clock_, start),
+                                                          MetricTags("Delete", table));
       }));
 }
 
 void ObservableStoreClient::AsyncBatchDelete(const std::string &table_name,
                                              const std::vector<std::string> &keys,
                                              Postable<void(int64_t)> callback) {
-  auto start = clock_.NowUnixNanos();
-  storage_operation_count_counter_.Record(1, {{"Operation", "BatchDelete"}});
+  const auto start = clock_.SteadyNow();
+  const std::string table = NormalizeTableNameLabel(table_name);
+  storage_operation_count_counter_.Record(1, MetricTags("BatchDelete", table));
   delegate_->AsyncBatchDelete(
-      table_name, keys, std::move(callback).OnInvocation([this, start]() {
-        auto end = clock_.NowUnixNanos();
+      table_name, keys, std::move(callback).OnInvocation([this, start, table]() {
         storage_operation_latency_in_ms_histogram_.Record(
-            absl::ToDoubleMilliseconds(absl::Nanoseconds(end - start)),
-            {{"Operation", "BatchDelete"}});
+            ElapsedMs(clock_, start), MetricTags("BatchDelete", table));
       }));
 }
 
 void ObservableStoreClient::AsyncGetNextJobID(Postable<void(int)> callback) {
-  delegate_->AsyncGetNextJobID(std::move(callback));
+  const auto start = clock_.SteadyNow();
+  // This operation addresses a plain counter key rather than a GCS table, so it
+  // carries the explicit kJobCounterTable sentinel. It is labeled like every
+  // other site because both metric stacks export a declared tag that a site
+  // does not record with an empty value rather than omitting it, and an
+  // unexplained TableName="" is worse on a dashboard than a named sentinel.
+  storage_operation_count_counter_.Record(1,
+                                          MetricTags("GetNextJobID", kJobCounterTable));
+  delegate_->AsyncGetNextJobID(std::move(callback).OnInvocation([this, start]() {
+    storage_operation_latency_in_ms_histogram_.Record(
+        ElapsedMs(clock_, start), MetricTags("GetNextJobID", kJobCounterTable));
+  }));
 }
 
 void ObservableStoreClient::AsyncGetKeys(
     const std::string &table_name,
     const std::string &prefix,
     Postable<void(std::vector<std::string>)> callback) {
-  auto start = clock_.NowUnixNanos();
-  storage_operation_count_counter_.Record(1, {{"Operation", "GetKeys"}});
+  const auto start = clock_.SteadyNow();
+  const std::string table = NormalizeTableNameLabel(table_name);
+  storage_operation_count_counter_.Record(1, MetricTags("GetKeys", table));
   delegate_->AsyncGetKeys(
-      table_name, prefix, std::move(callback).OnInvocation([this, start]() {
-        auto end = clock_.NowUnixNanos();
-        storage_operation_latency_in_ms_histogram_.Record(
-            absl::ToDoubleMilliseconds(absl::Nanoseconds(end - start)),
-            {{"Operation", "GetKeys"}});
+      table_name, prefix, std::move(callback).OnInvocation([this, start, table]() {
+        storage_operation_latency_in_ms_histogram_.Record(ElapsedMs(clock_, start),
+                                                          MetricTags("GetKeys", table));
       }));
 }
 
 void ObservableStoreClient::AsyncExists(const std::string &table_name,
                                         const std::string &key,
                                         Postable<void(bool)> callback) {
-  auto start = clock_.NowUnixNanos();
-  storage_operation_count_counter_.Record(1, {{"Operation", "Exists"}});
+  const auto start = clock_.SteadyNow();
+  const std::string table = NormalizeTableNameLabel(table_name);
+  storage_operation_count_counter_.Record(1, MetricTags("Exists", table));
   delegate_->AsyncExists(
-      table_name, key, std::move(callback).OnInvocation([this, start]() {
-        auto end = clock_.NowUnixNanos();
-        storage_operation_latency_in_ms_histogram_.Record(
-            absl::ToDoubleMilliseconds(absl::Nanoseconds(end - start)),
-            {{"Operation", "Exists"}});
+      table_name, key, std::move(callback).OnInvocation([this, start, table]() {
+        storage_operation_latency_in_ms_histogram_.Record(ElapsedMs(clock_, start),
+                                                          MetricTags("Exists", table));
       }));
 }
 

src/ray/gcs/store_client/table_name_label.cc

@@ -0,0 +1,31 @@
+// Copyright 2026 The Ray Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "ray/gcs/store_client/table_name_label.h"
+
+#include <string>
+
+#include "src/ray/protobuf/gcs.pb.h"
+
+namespace ray::gcs {
+
+std::string NormalizeTableNameLabel(const std::string &table_name) {
+  rpc::TablePrefix prefix;
+  if (!rpc::TablePrefix_Parse(table_name, &prefix)) {
+    return std::string(kUnknownTable);
+  }
+  return table_name;
+}
+
+}  // namespace ray::gcs

src/ray/gcs/store_client/table_name_label.h

@@ -0,0 +1,47 @@
+// Copyright 2026 The Ray Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <string>
+#include <string_view>
+
+namespace ray::gcs {
+
+/// TableName label value for a table name that is not an rpc::TablePrefix
+/// constant. Reaching this in production means a StoreClient caller invented a
+/// table name; the label collapses rather than growing the exported series
+/// count.
+inline constexpr std::string_view kUnknownTable = "OTHER";
+
+/// TableName label value for AsyncGetNextJobID, which addresses a plain
+/// counter key rather than a GCS table and so has no TablePrefix. Named
+/// explicitly instead of collapsing into kUnknownTable, and spelled to match
+/// the label the Redis layer uses for the underlying INCRBY.
+inline constexpr std::string_view kJobCounterTable = "JobCounter";
+
+/// Maps a GCS table name onto a closed label domain: the names of the
+/// rpc::TablePrefix enum, plus kUnknownTable.
+///
+/// Every table name a StoreClient sees today is produced by TablePrefix_Name,
+/// so this normally returns its argument unchanged. What it adds is that the
+/// domain is bounded by construction rather than by convention: StoreClient
+/// takes an arbitrary std::string, so without this a future caller could turn a
+/// job or actor id into unbounded label cardinality.
+///
+/// \param table_name The table name a StoreClient method was called with.
+/// \return The matching enum name, or kUnknownTable.
+std::string NormalizeTableNameLabel(const std::string &table_name);
+
+}  // namespace ray::gcs