feat: multi-backend gRPC addresses via static pick_first resolver

.github/workflows/e2e.yaml

@@ -47,6 +47,8 @@ jobs:
         case:
           - name: gRPC
             path: test/e2e/case/grpc/e2e.yaml
+          - name: gRPC multi-backend
+            path: test/e2e/case/grpc-multi-backend/e2e.yaml
           - name: Kafka
             path: test/e2e/case/kafka/e2e.yaml
     steps:

.github/workflows/skywalking-go.yaml

@@ -39,6 +39,9 @@ jobs:
         run: make test
       - name: Test Race
         run: make test-race
+      - name: Test Race (reporter packages)
+        # Codex: run the full multi-backend packages under -race, not only TestRace*.
+        run: go test -race -count=1 -timeout 180s ./plugins/core/reporter/ ./plugins/core/reporter/grpc/
       - name: Lint
         run: make lint
 

agent/reporter/imports.go

@@ -25,12 +25,15 @@ import (
 	_ "crypto/x509"
 	_ "fmt"
 	_ "io"
+	_ "math/rand"
+	_ "net"
 	_ "os"
 	_ "runtime"
 	_ "runtime/pprof"
 	_ "strconv"
 	_ "strings"
 	_ "sync"
+	_ "sync/atomic"
 	_ "time"
 
 	// imports the logs and profiles for reporter

docs/en/advanced-features/grpc-backend-troubleshooting.md

@@ -0,0 +1,81 @@
+# gRPC Backend Connectivity Troubleshooting
+
+## Multi-address failover
+
+Configure comma-separated backends:
+
+```yaml
+reporter:
+  grpc:
+    backend_service: oap-a:11800,oap-b:11800
+```
+
+Use `host:port` entries with numeric ports from 1 to 65535 and brackets around
+IPv6 addresses (`[::1]:11800`). The agent trims whitespace, ignores empty entries,
+and removes duplicates. Unlike the Python and Node.js parsers, Go rejects a
+malformed list instead of silently dropping its invalid entries. A list that
+normalizes to one address uses the existing single-address connection behavior.
+
+For multiple addresses, the agent creates one gRPC channel and shuffles its
+static endpoint list once. Native `pick_first` selects the first reachable
+backend in that order and keeps using it until its transport fails. This follows
+the Node.js and Python failover model and distributes agents across backends.
+Resolver refreshes preserve the shuffled order.
+
+The configured list stays fixed. Go's TCP dialer resolves hostname entries when
+opening a connection; there is no periodic DNS refresh of the list. Python's
+grpcio implementation instead expands multi-address hostnames to IP addresses
+once at channel creation.
+
+Reporters wait for a READY channel before taking the next queued batch. Buffers
+are bounded, so new reports are dropped when an outage fills them. A failed
+batch is discarded and **never replayed**: the backend may already have accepted
+it. Automatic gRPC retries apply only to the idempotent
+`reportInstanceProperties` RPC on `UNAVAILABLE`, with at most three attempts.
+Instance properties are refreshed every ten heartbeat intervals so a standby
+also learns the metadata after a silent backend switch. A properties-report
+failure does not suppress the heartbeat.
+
+RPC deadlines and stream cancellation bound individual operations. RPC errors,
+including authentication rejection or resource exhaustion, do not recreate the
+channel or rotate a connected backend. TCP keepalive helps detect failed
+transports; the agent does not enable aggressive HTTP/2 keepalive pings, which
+can conflict with OAP's gRPC ping policy. A half-open connection may remain READY
+until the transport detects the failure, so an RPC deadline alone does not
+guarantee immediate failover.
+
+The `grpc-multi-backend` E2E case discovers the active collector with a unique
+probe, stops it, and checks trace and log growth on the standby. Unit tests also
+cover failover, queue retention, failed acknowledgements, and send panics.
+`ConnectionManager.ResolvedBackendAddresses()` exposes the published dial order
+for tests and diagnostics.
+
+## HTTP(S) proxy
+
+Multi-address connections bypass `HTTP_PROXY` / `HTTPS_PROXY`. Provide a direct
+network route to the OAP ports. The single-address path retains its existing
+proxy behavior.
+
+## TLS / SNI
+
+The first configured endpoint after normalization supplies the channel's fixed
+HTTP/2 `:authority`, including its port. TLS uses that endpoint's host for
+certificate verification and SNI where applicable. Shuffling or failing over
+does not change this identity. An explicit server-name override in transport
+credentials takes precedence.
+
+Every backend must present a certificate valid for this common identity. With
+`oap.example.com:11800,10.0.0.2:11800`, both servers need a certificate covering
+`oap.example.com`. With an IP address first, certificates must cover that IP in
+their SANs; a hostname later in the list does not replace it. Put the intended
+common hostname first when using DNS certificates.
+
+See [Transport Layer Security (TLS)](./grpc-tls.md).
+
+## Authentication failures
+
+`UNAUTHENTICATED` / `PERMISSION_DENIED` are not retryable in the multi-backend
+service configuration. A bad shared credential will not usefully rotate across
+backends. The connection interceptor emits an authentication diagnostic at most
+once per 30 seconds and keeps the existing channel. Check
+`reporter.grpc.authentication` and the OAP authentication configuration.

docs/menu.yml

@@ -42,6 +42,8 @@ catalog:
           path: /en/agent/plugin-configurations
         - name: Transport Layer Security (TLS)
           path: /en/advanced-features/grpc-tls
+        - name: gRPC Backend Troubleshooting
+          path: /en/advanced-features/grpc-backend-troubleshooting
         - name: Kafka Reporter
           path: /en/advanced-features/kafka-reporter
         - name: Manual APIs

plugins/core/reporter/auth_failure_log.go

@@ -0,0 +1,116 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package reporter
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/metadata"
+	"google.golang.org/grpc/status"
+
+	"github.com/apache/skywalking-go/plugins/core/operator"
+)
+
+const authFailureLogInterval = 30 * time.Second
+
+// authFailureLogger rate-limits UNAUTHENTICATED / PERMISSION_DENIED logs so a
+// bad credential does not spam (and does not encourage treating auth as failover).
+type authFailureLogger struct {
+	logger operator.LogOperator
+	mu     sync.Mutex
+	last   time.Time
+}
+
+func (a *authFailureLogger) note(err error) {
+	if a == nil || a.logger == nil || err == nil {
+		return
+	}
+	st, ok := status.FromError(err)
+	if !ok {
+		return
+	}
+	switch st.Code() {
+	case codes.Unauthenticated, codes.PermissionDenied:
+	default:
+		return
+	}
+	a.mu.Lock()
+	defer a.mu.Unlock()
+	now := time.Now()
+	if !a.last.IsZero() && now.Sub(a.last) < authFailureLogInterval {
+		return
+	}
+	a.last = now
+	a.logger.Errorf("backend rejected authentication/authorization (%s); "+
+		"not treating this as multi-backend failover — check reporter.grpc.authentication",
+		st.Code())
+}
+
+func (a *authFailureLogger) unaryInterceptor() grpc.UnaryClientInterceptor {
+	return func(ctx context.Context, method string, req, reply interface{},
+		cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
+		err := invoker(ctx, method, req, reply, cc, opts...)
+		a.note(err)
+		return err
+	}
+}
+
+func (a *authFailureLogger) streamInterceptor() grpc.StreamClientInterceptor {
+	return func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn,
+		method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) {
+		stream, err := streamer(ctx, desc, cc, method, opts...)
+		a.note(err)
+		if err != nil || stream == nil {
+			return stream, err
+		}
+		return &authWatchingClientStream{ClientStream: stream, auth: a}, nil
+	}
+}
+
+// authWatchingClientStream notes auth failures on Recv/Send, not only stream open.
+type authWatchingClientStream struct {
+	grpc.ClientStream
+	auth *authFailureLogger
+}
+
+func (s *authWatchingClientStream) SendMsg(m interface{}) error {
+	err := s.ClientStream.SendMsg(m)
+	s.auth.note(err)
+	return err
+}
+
+func (s *authWatchingClientStream) RecvMsg(m interface{}) error {
+	err := s.ClientStream.RecvMsg(m)
+	s.auth.note(err)
+	return err
+}
+
+func (s *authWatchingClientStream) CloseSend() error {
+	err := s.ClientStream.CloseSend()
+	s.auth.note(err)
+	return err
+}
+
+func (s *authWatchingClientStream) Header() (metadata.MD, error) {
+	md, err := s.ClientStream.Header()
+	s.auth.note(err)
+	return md, err
+}