diff --git a/.github/workflows/images.yaml b/.github/workflows/images.yaml
index 4e4ec0549b..1a3528f44c 100644
--- a/.github/workflows/images.yaml
+++ b/.github/workflows/images.yaml
@@ -102,6 +102,13 @@ jobs:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ - name: Record build provenance
+ id: provenance
+ run: |
+ git_commit=$(git rev-parse HEAD)
+ build_time=$(date -u +%Y-%m-%dT%H:%M:%SZ)
+ printf 'git_commit=%s\nbuild_time=%s\n' "$git_commit" "$build_time" >> "$GITHUB_OUTPUT"
+
- uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4.4.0
with:
platforms: linux/amd64,linux/arm64
@@ -140,3 +147,5 @@ jobs:
build-args: |
VERSION=${{ needs.resolve.outputs.tag }}
CONTAINER_REGISTRY=${{ env.CONTAINER_REGISTRY }}
+ GIT_COMMIT=${{ steps.provenance.outputs.git_commit }}
+ BUILD_TIME=${{ steps.provenance.outputs.build_time }}
diff --git a/Makefile b/Makefile
index 700ec0ed1f..98a81b0b3c 100644
--- a/Makefile
+++ b/Makefile
@@ -23,6 +23,7 @@ GO_PACKAGE_DIRS=$(shell $(GOCMD) list -tags e2e -f '{{.Dir}}' $(GO_PACKAGE_PATTE
CONTAINER_ENGINE ?= podman
CONTAINER_REGISTRY ?= ghcr.io/azure
+RACER_OBJECT_IMAGE ?= $(CONTAINER_REGISTRY)/racer-object:$(VERSION_TAG)
# Unified install namespace for all unbounded components. Each component's
# *_NAMESPACE var derives from this by default, so overriding UNBOUNDED_NAMESPACE
@@ -237,7 +238,7 @@ REACT_DEV ?= false
##@ General
-all: kubectl-unbounded forge relctl machina machine-ops-controller token-refresher unbounded-operator unbounded-net-controller unbounded-net-node unbounded-net-routeplan-debug unping unroute gantry ## Build all binaries (default)
+all: kubectl-unbounded forge relctl machina machine-ops-controller token-refresher unbounded-operator unbounded-net-controller unbounded-net-node unbounded-net-routeplan-debug unping unroute gantry racer-object ## Build all binaries (default)
help: ## Show this help
@echo ""
@@ -295,6 +296,9 @@ help: ## Show this help
@echo " unbounded-net-routeplan-debug Build net routeplan debug tool"
@echo " unping Build unping health-check utility"
@echo " unroute Build unroute eBPF inspection utility"
+ @echo " racer-object Test and build the S3 read adapter"
+ @echo " racer-object-build Build the S3 read adapter without tests"
+ @echo " racer-object-test Test the S3 read adapter"
@echo ""
@echo "Container Images (local, single-arch):"
@echo " image-inventory-all-local Build all local inventory container images"
@@ -312,6 +316,7 @@ help: ## Show this help
@echo " image-unbounded-operator-local Build unbounded-operator image"
@echo " image-unbounded-operator-push Build and push unbounded-operator image"
@echo " image-playpen-local Build playpen image"
+ @echo " image-racer-object-local Build the S3 read adapter image locally"
@echo " image-net-controller-local Build unbounded-net-controller image"
@echo " image-net-controller-push Build and push unbounded-net-controller image"
@echo " image-net-node-local Build unbounded-net-node image"
@@ -818,6 +823,27 @@ unroute-build: ## Build the unroute utility binary (no lint/test)
unroute: test unroute-build ## Build the unroute utility (implies test)
+##@ Racer Object (S3 read adapter)
+
+.PHONY: racer-object racer-object-build racer-object-test image-racer-object-local
+racer-object: racer-object-test racer-object-build ## Test and build the S3 read adapter
+
+racer-object-build: ## Build the S3 read adapter without lint/test
+ @mkdir -p bin
+ $(GOBUILD) -trimpath -ldflags '$(STAMP_LDFLAGS)' -o bin/racer-object ./cmd/racer-object
+
+racer-object-test: ## Test the S3 read adapter
+ timeout --signal=TERM --kill-after=10s 300s $(GOTEST) -timeout=5m ./cmd/racer-object/... ./internal/racer/object/...
+
+image-racer-object-local: ## Build the S3 read adapter image locally (single-arch)
+ $(CONTAINER_ENGINE) build \
+ --build-arg VERSION=$(VERSION) \
+ --build-arg GIT_COMMIT=$(GIT_COMMIT) \
+ --build-arg BUILD_TIME=$(BUILD_TIME) \
+ -t racer-object:$(VERSION_TAG) -t $(RACER_OBJECT_IMAGE) \
+ -f ./images/racer-object/Containerfile .
+ $(call trivy-maybe,$(RACER_OBJECT_IMAGE))
+
##@ Gantry (peer-to-peer OCI distribution)
gantry-build: ## Build the gantry binary (no lint/test)
diff --git a/NOTICE b/NOTICE
index f5328ee602..3240d075fa 100644
--- a/NOTICE
+++ b/NOTICE
@@ -631,6 +631,13 @@ notices:
license:
- name: BSD 3-Clause License
link: https://go.googlesource.com/mod
+ - dependency: golang.org/x/net
+ ecosystem: go
+ copyright:
+ - Copyright 2009 The Go Authors.
+ license:
+ - name: BSD 3-Clause License
+ link: https://go.googlesource.com/net
- dependency: golang.org/x/sync
ecosystem: go
copyright:
diff --git a/cmd/racer-object/config.go b/cmd/racer-object/config.go
new file mode 100644
index 0000000000..0302f4f047
--- /dev/null
+++ b/cmd/racer-object/config.go
@@ -0,0 +1,46 @@
+// Copyright (c) Microsoft Corporation.
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+ "net/http"
+
+ "github.com/aws/aws-sdk-go-v2/aws"
+ awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http"
+ "github.com/aws/aws-sdk-go-v2/service/s3"
+)
+
+type originS3Client struct {
+ *s3.Client
+ pool originConnectionPool
+}
+
+func (c *originS3Client) Close() error {
+ c.pool.close()
+ return nil
+}
+
+func newOriginS3Client(cfg aws.Config, o options) (*originS3Client, error) {
+ owned := &originS3Client{}
+
+ client := s3.NewFromConfig(cfg, func(s *s3.Options) {
+ s.UsePathStyle = o.pathStyle
+ if o.endpoint != "" {
+ s.BaseEndpoint = aws.String(o.endpoint)
+ }
+
+ if buildable, ok := s.HTTPClient.(*awshttp.BuildableClient); ok {
+ s.HTTPClient = buildable.WithTransportOptions(func(tr *http.Transport) {
+ tr.DialContext = owned.pool.dial(tr.DialContext)
+ })
+ }
+ })
+ if err := validateEndpoint(aws.ToString(client.Options().BaseEndpoint)); err != nil {
+ return nil, safeDiagnostic("invalid effective S3 endpoint: must be an HTTP(S) origin without credentials, query, fragment, or path prefix, with a valid port", "config", err)
+ }
+
+ owned.Client = client
+
+ return owned, nil
+}
diff --git a/cmd/racer-object/config_test.go b/cmd/racer-object/config_test.go
new file mode 100644
index 0000000000..dbfa559341
--- /dev/null
+++ b/cmd/racer-object/config_test.go
@@ -0,0 +1,153 @@
+// Copyright (c) Microsoft Corporation.
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+ "errors"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/aws/aws-sdk-go-v2/aws"
+)
+
+func isolateOriginAWSConfig(t *testing.T) string {
+ t.Helper()
+
+ dir := t.TempDir()
+ for key, value := range map[string]string{
+ "AWS_CONFIG_FILE": filepath.Join(dir, "config"), "AWS_SHARED_CREDENTIALS_FILE": filepath.Join(dir, "credentials"),
+ "AWS_PROFILE": "", "AWS_DEFAULT_PROFILE": "", "AWS_EC2_METADATA_DISABLED": "true",
+ "AWS_ACCESS_KEY_ID": "test-access", "AWS_SECRET_ACCESS_KEY": "test-secret", "AWS_SESSION_TOKEN": "test-session",
+ "AWS_ENDPOINT_URL": "", "AWS_ENDPOINT_URL_S3": "", "AWS_IGNORE_CONFIGURED_ENDPOINT_URLS": "false",
+ "AWS_CA_BUNDLE": "",
+ } {
+ t.Setenv(key, value)
+ }
+
+ return dir
+}
+
+func TestOriginEffectiveEndpoint(t *testing.T) {
+ for _, source := range []string{"global-env", "service-env", "global-shared", "service-shared"} {
+ t.Run(source, func(t *testing.T) {
+ for _, endpoint := range []string{
+ "", "https://s3.example", "https://s3.example/", "http://localhost:9000",
+ "https://private-user:private-password@s3.example", "https://s3.example/private-path",
+ "https://s3.example?private-query=secret", "https://s3.example#private-fragment", "https://s3.example:99999",
+ "https://s3.example/%zz",
+ } {
+ t.Run(endpoint, func(t *testing.T) {
+ dir := isolateOriginAWSConfig(t)
+ setOriginEndpointSource(t, dir, source, endpoint)
+
+ o := options{region: "us-east-1", requestTimeout: time.Minute, pathStyle: true}
+
+ cfg, err := originAWSConfig(t.Context(), o)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ client, err := newOriginS3Client(cfg, o)
+ if validateEndpoint(endpoint) != nil {
+ if err == nil || !strings.HasPrefix(err.Error(), "invalid effective S3 endpoint:") || strings.Contains(err.Error(), endpoint) {
+ t.Fatalf("error = %v, want redacted endpoint error", err)
+ }
+
+ cause := errors.Unwrap(err)
+ if cause == nil || cause.Error() != validateEndpoint(endpoint).Error() || !errors.Is(err, cause) || !strings.Contains(err.Error(), "category=config") {
+ t.Fatalf("endpoint diagnostic cause or category lost: %v", err)
+ }
+
+ if err := runOrigin(t.Context(), o); err == nil || !strings.HasPrefix(err.Error(), "invalid effective S3 endpoint:") {
+ t.Fatalf("startup error = %v, want endpoint rejection before directory/socket setup", err)
+ }
+
+ return
+ }
+
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if got := aws.ToString(client.Options().BaseEndpoint); got != endpoint || !client.Options().UsePathStyle {
+ t.Fatalf("endpoint = %q, want %q with path style", got, endpoint)
+ }
+ })
+ }
+ })
+ }
+}
+
+func setOriginEndpointSource(t *testing.T, dir, source, endpoint string) {
+ t.Helper()
+
+ switch source {
+ case "global-env":
+ t.Setenv("AWS_ENDPOINT_URL", endpoint)
+ case "service-env":
+ t.Setenv("AWS_ENDPOINT_URL_S3", endpoint)
+ case "global-shared", "service-shared":
+ text := "[default]\nendpoint_url = " + endpoint + "\n"
+ if source == "service-shared" {
+ text = "[default]\nservices = local\n[services local]\ns3 =\n endpoint_url = " + endpoint + "\n"
+ }
+
+ if err := os.WriteFile(filepath.Join(dir, "config"), []byte(text), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ default:
+ t.Fatal("unknown endpoint source")
+ }
+}
+
+func TestOriginEndpointOverride(t *testing.T) {
+ for _, source := range []string{"global-env", "service-env", "global-shared", "service-shared"} {
+ t.Run(source, func(t *testing.T) {
+ dir := isolateOriginAWSConfig(t)
+ setOriginEndpointSource(t, dir, source, "https://private-user:private-password@s3.example/private-path?secret=value")
+
+ o := options{region: "us-east-1", requestTimeout: time.Minute, endpoint: "https://override.example"}
+
+ cfg, err := originAWSConfig(t.Context(), o)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ client, err := newOriginS3Client(cfg, o)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if got := aws.ToString(client.Options().BaseEndpoint); got != o.endpoint {
+ t.Fatalf("endpoint = %q, want flag override %q", got, o.endpoint)
+ }
+ })
+ }
+}
+
+func TestOriginServiceEndpointPrecedence(t *testing.T) {
+ isolateOriginAWSConfig(t)
+ t.Setenv("AWS_ENDPOINT_URL", "https://private-user:private-password@s3.example/private-path")
+ t.Setenv("AWS_ENDPOINT_URL_S3", "https://service.example")
+
+ o := options{region: "us-east-1", requestTimeout: time.Minute}
+
+ cfg, err := originAWSConfig(t.Context(), o)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ client, err := newOriginS3Client(cfg, o)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer closeResource(client)
+
+ if got := aws.ToString(client.Options().BaseEndpoint); got != "https://service.example" {
+ t.Fatalf("endpoint = %q, want service environment override", got)
+ }
+}
diff --git a/cmd/racer-object/deadline_test.go b/cmd/racer-object/deadline_test.go
new file mode 100644
index 0000000000..eb6674e907
--- /dev/null
+++ b/cmd/racer-object/deadline_test.go
@@ -0,0 +1,119 @@
+// Copyright (c) Microsoft Corporation.
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+ "errors"
+ "net/http"
+ "net/http/httptest"
+ "sync"
+ "testing"
+ "time"
+)
+
+// deadlineRecorder records write deadlines set through http.ResponseController.
+type deadlineRecorder struct {
+ *httptest.ResponseRecorder
+
+ mu sync.Mutex
+ deadlines []time.Time
+}
+
+func (d *deadlineRecorder) SetWriteDeadline(t time.Time) error {
+ d.mu.Lock()
+ defer d.mu.Unlock()
+
+ d.deadlines = append(d.deadlines, t)
+
+ return nil
+}
+
+func (d *deadlineRecorder) last(t *testing.T) time.Time {
+ t.Helper()
+
+ d.mu.Lock()
+ defer d.mu.Unlock()
+
+ if len(d.deadlines) == 0 {
+ t.Fatal("write deadline was never set")
+ }
+
+ return d.deadlines[len(d.deadlines)-1]
+}
+
+// clearingHandler mimics Object.WriteTo, which clears the write deadline
+// before returning.
+func clearingHandler(t *testing.T, then func()) http.Handler {
+ return http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ if err := http.NewResponseController(w).SetWriteDeadline(time.Time{}); err != nil {
+ t.Errorf("clear deadline: %v", err)
+ }
+
+ then()
+ })
+}
+
+func assertRearmed(t *testing.T, got, before, after time.Time, bound time.Duration) {
+ t.Helper()
+
+ if got.IsZero() {
+ t.Fatal("write deadline left cleared after handler returned")
+ }
+
+ if got.Before(before.Add(bound)) || got.After(after.Add(bound)) {
+ t.Fatalf("write deadline %v outside [%v, %v]", got, before.Add(bound), after.Add(bound))
+ }
+}
+
+func TestSidecarHandlerRearmsWriteDeadlineAfterReturn(t *testing.T) {
+ const requestTimeout = 2 * time.Second
+
+ w := &deadlineRecorder{ResponseRecorder: httptest.NewRecorder()}
+ h := sidecarHTTPHandler(clearingHandler(t, func() {}), requestTimeout)
+
+ before := time.Now()
+
+ h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/bucket/key", nil))
+
+ assertRearmed(t, w.last(t), before, time.Now(), requestTimeout+sidecarWriteGrace)
+}
+
+func TestSidecarHandlerRearmsWriteDeadlineAfterPanic(t *testing.T) {
+ const requestTimeout = 2 * time.Second
+
+ w := &deadlineRecorder{ResponseRecorder: httptest.NewRecorder()}
+ h := sidecarHTTPHandler(clearingHandler(t, func() { panic("boom") }), requestTimeout)
+
+ before := time.Now()
+
+ func() {
+ defer func() {
+ if value := recover(); !errors.Is(asError(value), http.ErrAbortHandler) {
+ t.Errorf("recovered %v, want http.ErrAbortHandler", value)
+ }
+ }()
+
+ h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/bucket/key", nil))
+ }()
+
+ assertRearmed(t, w.last(t), before, time.Now(), requestTimeout+sidecarWriteGrace)
+}
+
+func TestSidecarHandlerIgnoresUnsupportedWriteDeadline(t *testing.T) {
+ w := httptest.NewRecorder()
+ h := sidecarHTTPHandler(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.WriteHeader(http.StatusNoContent)
+ }), time.Second)
+
+ h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/bucket/key", nil))
+
+ if w.Code != http.StatusNoContent {
+ t.Fatalf("status = %d, want %d", w.Code, http.StatusNoContent)
+ }
+}
+
+func asError(value any) error {
+ err, _ := value.(error)
+ return err
+}
diff --git a/cmd/racer-object/diagnostics.go b/cmd/racer-object/diagnostics.go
new file mode 100644
index 0000000000..620666d217
--- /dev/null
+++ b/cmd/racer-object/diagnostics.go
@@ -0,0 +1,179 @@
+// Copyright (c) Microsoft Corporation.
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+ "bytes"
+ "context"
+ "crypto/tls"
+ "crypto/x509"
+ "errors"
+ "fmt"
+ "io"
+ "log"
+ "net"
+ "net/http"
+ "os"
+ "syscall"
+
+ "github.com/Azure/unbounded/internal/racer/object"
+ "github.com/Azure/unbounded/pkg/racersdk"
+)
+
+// diagnosticError retains the cause without printing paths, URLs, or credentials.
+type diagnosticError struct {
+ message string
+ category string
+ cause error
+}
+
+func (e *diagnosticError) Error() string { return e.message + " (category=" + e.category + ")" }
+func (e *diagnosticError) Unwrap() error { return e.cause }
+
+// message and fallback must be fixed strings, never external error text.
+func safeDiagnostic(message, fallback string, cause error) error {
+ return &diagnosticError{message: message, category: diagnosticCategory(cause, fallback), cause: cause}
+}
+
+func diagnosticCategory(err error, fallback string) string {
+ // A TLS or certificate failure is a configuration error even though the
+ // origin maps it to ErrUnavailable, so check it before the sentinels.
+ if tlsFailure(err) {
+ return "tls"
+ }
+
+ for _, entry := range []struct {
+ cause error
+ category string
+ }{
+ {context.Canceled, "canceled"},
+ {context.DeadlineExceeded, "timeout"},
+ {os.ErrDeadlineExceeded, "timeout"},
+ {os.ErrPermission, "permission"},
+ {syscall.EADDRINUSE, "addressinuse"},
+ {syscall.EADDRNOTAVAIL, "addressunavailable"},
+ {syscall.ECONNREFUSED, "connectionrefused"},
+ {syscall.ECONNRESET, "connectionreset"},
+ {syscall.ENOSPC, "nospace"},
+ {syscall.EROFS, "readonly"},
+ {syscall.EMFILE, "resourceexhausted"},
+ {syscall.ENFILE, "resourceexhausted"},
+ {syscall.ENOTDIR, "notdirectory"},
+ {syscall.ELOOP, "symlink"},
+ {os.ErrNotExist, "notfound"},
+ {os.ErrExist, "exists"},
+ {net.ErrClosed, "closed"},
+ {racersdk.ErrUnauthorized, "unauthorized"},
+ {racersdk.ErrForbidden, "forbidden"},
+ {racersdk.ErrNotFound, "notfound"},
+ {racersdk.ErrVersionMismatch, "versionmismatch"},
+ {racersdk.ErrRangeNotSatisfiable, "range"},
+ {racersdk.ErrInvalidRequest, "invalid"},
+ {racersdk.ErrUnavailable, "unavailable"},
+ {object.ErrMetadataMismatch, "mismatch"},
+ {os.ErrInvalid, "invalid"},
+ } {
+ if errors.Is(err, entry.cause) {
+ return entry.category
+ }
+ }
+
+ var networkError net.Error
+ if errors.As(err, &networkError) && networkError.Timeout() {
+ return "timeout"
+ }
+
+ var dnsError *net.DNSError
+ if errors.As(err, &dnsError) {
+ return "dns"
+ }
+
+ if category := object.InvalidResponseCategory(err); category != "" {
+ return category
+ }
+
+ return fallback
+}
+
+func tlsFailure(err error) bool {
+ var (
+ verification *tls.CertificateVerificationError
+ authority x509.UnknownAuthorityError
+ hostname x509.HostnameError
+ invalid x509.CertificateInvalidError
+ header tls.RecordHeaderError
+ alert tls.AlertError
+ )
+
+ return errors.As(err, &verification) || errors.As(err, &authority) || errors.As(err, &hostname) ||
+ errors.As(err, &invalid) || errors.As(err, &header) || errors.As(err, &alert)
+}
+
+func logSidecarStartup(address net.Addr) {
+ // Only a bound loopback TCP address is safe to include in the log.
+ if tcp, ok := address.(*net.TCPAddr); ok && tcp.IP.IsLoopback() && tcp.Zone == "" && tcp.Port > 0 && tcp.Port <= 65535 {
+ log.Printf("racer-object: mode=sidecar event=listening address=%s", tcp.AddrPort())
+ return
+ }
+
+ log.Print("racer-object: mode=sidecar event=listening")
+}
+
+func diagnoseSidecarPanic() {
+ if value := recover(); value != nil {
+ if value != http.ErrAbortHandler {
+ log.Print("racer-object: mode=sidecar event=request_failed category=panic")
+ }
+
+ // Abort the response without letting net/http format the panic value.
+ panic(http.ErrAbortHandler)
+ }
+}
+
+// net/http diagnostics contain peer addresses and may contain private text.
+type safeHTTPLogWriter struct{}
+
+func (safeHTTPLogWriter) Write(p []byte) (int, error) {
+ category := "server"
+
+ switch {
+ case bytes.HasPrefix(p, []byte("http: panic serving ")):
+ category = "panic"
+ case bytes.HasPrefix(p, []byte("http: Accept error:")):
+ category = "accept"
+ case bytes.HasPrefix(p, []byte("http: TLS handshake error")):
+ category = "tls"
+ }
+
+ log.Printf("racer-object: mode=sidecar event=server_error category=%s", category)
+
+ return len(p), nil
+}
+
+func diagnoseOrigin(origin racersdk.Origin) racersdk.Origin {
+ return func(ctx context.Context, request racersdk.OriginRequest) (racersdk.Metadata, io.ReadCloser, error) {
+ defer func() {
+ if recover() != nil {
+ log.Print("racer-object: mode=origin event=request_failed category=panic")
+ panic(http.ErrAbortHandler)
+ }
+ }()
+
+ metadata, body, err := origin(ctx, request)
+ if err != nil {
+ category := diagnosticCategory(err, "upstream")
+ logOutcome("origin", category, "racer-object: mode=origin event=request_failed category="+category)
+ }
+
+ return metadata, body, err
+ }
+}
+
+// logSidecarFailure logs a sidecar 5xx. Only fixed fields are printed: the
+// method is one of GET or HEAD, and the bucket was validated by the sidecar.
+func logSidecarFailure(failure object.SidecarFailure) {
+ category := diagnosticCategory(failure.Err, "upstream")
+ logOutcome("sidecar", category, fmt.Sprintf("racer-object: mode=sidecar event=response_failed method=%s status=%d category=%s bucket=%q",
+ failure.Method, failure.Status, category, failure.Bucket))
+}
diff --git a/cmd/racer-object/diagnostics_category_test.go b/cmd/racer-object/diagnostics_category_test.go
new file mode 100644
index 0000000000..87f7da4a8f
--- /dev/null
+++ b/cmd/racer-object/diagnostics_category_test.go
@@ -0,0 +1,72 @@
+// Copyright (c) Microsoft Corporation.
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+ "crypto/tls"
+ "crypto/x509"
+ "errors"
+ "fmt"
+ "net"
+ "strings"
+ "syscall"
+ "testing"
+ "time"
+
+ "github.com/Azure/unbounded/internal/racer/object"
+ "github.com/Azure/unbounded/pkg/racersdk"
+)
+
+func TestDiagnosticCategoryTLS(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ err error
+ }{
+ {"verification", &tls.CertificateVerificationError{Err: errors.New("private")}},
+ {"authority", x509.UnknownAuthorityError{}},
+ {"hostname", x509.HostnameError{Host: "private"}},
+ {"invalid", x509.CertificateInvalidError{Reason: x509.Expired}},
+ {"record-header", tls.RecordHeaderError{Msg: "private"}},
+ {"alert", tls.AlertError(42)},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ // Mirror the origin wrapping: ErrUnavailable plus the send failure.
+ err := fmt.Errorf("%w: %w", racersdk.ErrUnavailable, &net.OpError{Op: "dial", Err: tc.err})
+ if got := diagnosticCategory(err, "fallback"); got != "tls" {
+ t.Fatalf("category = %q, want tls", got)
+ }
+ })
+ }
+
+ plain := fmt.Errorf("%w: %w", racersdk.ErrUnavailable, &net.OpError{Op: "dial", Err: syscall.ECONNREFUSED})
+ if got := diagnosticCategory(plain, "fallback"); got != "connectionrefused" {
+ t.Fatalf("non-TLS send failure category = %q", got)
+ }
+}
+
+func TestLogSidecarFailure(t *testing.T) {
+ useTestLogLimiter(t, time.Now)
+
+ output := captureDiagnosticLog(t)
+
+ logSidecarFailure(object.SidecarFailure{
+ Method: "GET", Status: 503, Bucket: "my-bucket",
+ Err: fmt.Errorf("%s: %w", diagnosticSecret, racersdk.ErrUnavailable),
+ })
+ logSidecarFailure(object.SidecarFailure{Method: "HEAD", Status: 502, Bucket: "b", Err: object.ErrMetadataMismatch})
+
+ got := output.String()
+ for _, want := range []string{
+ `mode=sidecar event=response_failed method=GET status=503 category=unavailable bucket="my-bucket"`,
+ `method=HEAD status=502 category=mismatch bucket="b"`,
+ } {
+ if !strings.Contains(got, want) {
+ t.Fatalf("missing %q in %q", want, got)
+ }
+ }
+
+ if strings.Contains(got, diagnosticSecret) {
+ t.Fatal("sidecar failure log leaked cause text")
+ }
+}
diff --git a/cmd/racer-object/diagnostics_test.go b/cmd/racer-object/diagnostics_test.go
new file mode 100644
index 0000000000..445b14a101
--- /dev/null
+++ b/cmd/racer-object/diagnostics_test.go
@@ -0,0 +1,301 @@
+// Copyright (c) Microsoft Corporation.
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "log"
+ "net"
+ "net/http"
+ "os"
+ "path/filepath"
+ "strings"
+ "sync"
+ "syscall"
+ "testing"
+ "time"
+
+ "github.com/Azure/unbounded/pkg/racersdk"
+)
+
+const diagnosticSecret = "private-diagnostic-sentinel"
+
+func TestSafeDiagnostic(t *testing.T) {
+ for _, tt := range []struct {
+ cause error
+ category string
+ }{
+ {os.ErrPermission, "permission"},
+ {&os.PathError{Op: "open", Path: diagnosticSecret, Err: os.ErrPermission}, "permission"},
+ {syscall.EADDRINUSE, "addressinuse"},
+ {syscall.ENOTDIR, "notdirectory"},
+ {syscall.ENOSPC, "nospace"},
+ {context.Canceled, "canceled"},
+ {context.DeadlineExceeded, "timeout"},
+ {net.ErrClosed, "closed"},
+ {racersdk.ErrForbidden, "forbidden"},
+ {racersdk.ErrUnavailable, "unavailable"},
+ {&net.DNSError{Name: diagnosticSecret, Err: diagnosticSecret}, "dns"},
+ {&net.DNSError{Name: diagnosticSecret, IsTimeout: true}, "timeout"},
+ {errors.New(diagnosticSecret), "config"},
+ } {
+ t.Run(tt.category, func(t *testing.T) {
+ cause := fmt.Errorf("%s: %w", diagnosticSecret, tt.cause)
+
+ err := safeDiagnostic("setup failed", "config", cause)
+ if !errors.Is(err, cause) || !errors.Is(err, tt.cause) {
+ t.Fatal("lost diagnostic cause")
+ }
+
+ if got := fmt.Sprintf("%s %v %+v", err, err, err); strings.Contains(got, diagnosticSecret) || !strings.Contains(got, "category="+tt.category) {
+ t.Fatalf("unsafe or missing diagnostic category: %s", got)
+ }
+ })
+ }
+}
+
+type diagnosticOutput struct{ err error }
+
+func (w diagnosticOutput) Write([]byte) (int, error) { return 0, w.err }
+
+func TestOutputDiagnosticCause(t *testing.T) {
+ cause := errors.New(diagnosticSecret)
+
+ err := writeHelp(diagnosticOutput{err: cause}, "help")
+ if !errors.Is(err, cause) || strings.Contains(err.Error(), diagnosticSecret) {
+ t.Fatal("output cause lost or exposed")
+ }
+}
+
+func TestListenAndServeDiagnosticCauses(t *testing.T) {
+ l, err := listenSidecar(context.Background(), "127.0.0.1:0")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer l.Close()
+
+ other, err := listenSidecar(context.Background(), l.Addr().String())
+ if other != nil {
+ other.Close()
+ }
+
+ if !errors.Is(err, syscall.EADDRINUSE) || !strings.Contains(err.Error(), "category=addressinuse") {
+ t.Fatalf("listen diagnostic: %v", err)
+ }
+
+ l.Close()
+
+ err = serveSidecar(context.Background(), l, http.NotFoundHandler(), time.Second)
+ if !errors.Is(err, net.ErrClosed) || !strings.Contains(err.Error(), "category=closed") {
+ t.Fatalf("server diagnostic: %v", err)
+ }
+}
+
+func TestAWSConfigDiagnosticCause(t *testing.T) {
+ dir := t.TempDir()
+
+ configPath := filepath.Join(dir, diagnosticSecret)
+ if err := os.WriteFile(configPath, []byte("[default]\nmax_attempts = "+diagnosticSecret+"\n"), 0o600); err != nil {
+ t.Fatal(err)
+ }
+
+ for key, value := range map[string]string{
+ "AWS_CONFIG_FILE": configPath, "AWS_SHARED_CREDENTIALS_FILE": filepath.Join(dir, "credentials"),
+ "AWS_PROFILE": "", "AWS_DEFAULT_PROFILE": "", "AWS_EC2_METADATA_DISABLED": "true",
+ } {
+ t.Setenv(key, value)
+ }
+
+ _, err := originAWSConfig(context.Background(), options{region: "us-east-1", requestTimeout: time.Second})
+ if err == nil || errors.Unwrap(err) == nil || !strings.Contains(err.Error(), "category=config") || strings.Contains(err.Error(), diagnosticSecret) {
+ t.Fatalf("AWS config diagnostic: %v", err)
+ }
+}
+
+type diagnosticLog struct {
+ mu sync.Mutex
+ buffer bytes.Buffer
+}
+
+func (b *diagnosticLog) Write(p []byte) (int, error) {
+ b.mu.Lock()
+ defer b.mu.Unlock()
+
+ return b.buffer.Write(p)
+}
+
+func (b *diagnosticLog) String() string {
+ b.mu.Lock()
+ defer b.mu.Unlock()
+
+ return b.buffer.String()
+}
+
+func captureDiagnosticLog(t *testing.T) *diagnosticLog {
+ t.Helper()
+
+ var output diagnosticLog
+
+ previous := log.Writer()
+
+ log.SetOutput(&output)
+ t.Cleanup(func() { log.SetOutput(previous) })
+
+ return &output
+}
+
+func TestSidecarStartupAndPanicDiagnostics(t *testing.T) {
+ for _, abort := range []bool{false, true} {
+ t.Run(fmt.Sprint(abort), func(t *testing.T) {
+ output := captureDiagnosticLog(t)
+ handler := http.HandlerFunc(func(http.ResponseWriter, *http.Request) {
+ if abort {
+ panic(http.ErrAbortHandler)
+ }
+
+ panic(diagnosticSecret)
+ })
+ address, cancel, done := startTestSidecar(t, handler, time.Second)
+
+ client := &http.Client{Timeout: 2 * time.Second}
+ defer client.CloseIdleConnections()
+
+ response, err := client.Get(address + "/" + diagnosticSecret)
+ if response != nil {
+ response.Body.Close()
+ }
+
+ cancel()
+ waitSidecar(t, done)
+
+ if err == nil {
+ t.Fatal("panicked response did not abort")
+ }
+
+ got := output.String()
+ if !strings.Contains(got, "mode=sidecar event=listening address="+strings.TrimPrefix(address, "http://")) {
+ t.Fatalf("missing actual bound startup address: %s", got)
+ }
+
+ if strings.Contains(got, diagnosticSecret) || strings.Contains(got, "goroutine") || strings.Contains(got, "category=panic") == abort {
+ t.Fatalf("unsafe or incorrect panic log: %s", got)
+ }
+ })
+ }
+}
+
+func TestHTTPLogSanitization(t *testing.T) {
+ output := captureDiagnosticLog(t)
+
+ logger := log.New(safeHTTPLogWriter{}, "", 0)
+ for _, prefix := range []string{"http: panic serving ", "http: Accept error:", "http: TLS handshake error", "unknown "} {
+ logger.Print(prefix + diagnosticSecret)
+ }
+
+ got := output.String()
+ if strings.Contains(got, diagnosticSecret) {
+ t.Fatal("HTTP log exposed private text")
+ }
+
+ for _, category := range []string{"panic", "accept", "tls", "server"} {
+ if !strings.Contains(got, "category="+category) {
+ t.Fatalf("missing %s category: %s", category, got)
+ }
+ }
+}
+
+func TestOriginRuntimeDiagnostics(t *testing.T) {
+ output := captureDiagnosticLog(t)
+ origin := diagnoseOrigin(func(context.Context, racersdk.OriginRequest) (racersdk.Metadata, io.ReadCloser, error) {
+ return racersdk.Metadata{Size: 42}, nil, nil
+ })
+
+ metadata, _, err := origin(context.Background(), racersdk.OriginRequest{})
+ if err != nil || metadata.Size != 42 || output.String() != "" {
+ t.Fatal("successful origin request changed or logged")
+ }
+
+ // Forbidden is an expected outcome logged only at debug level.
+ cause := fmt.Errorf("%s: %w", diagnosticSecret, racersdk.ErrUnavailable)
+ origin = diagnoseOrigin(func(context.Context, racersdk.OriginRequest) (racersdk.Metadata, io.ReadCloser, error) {
+ return racersdk.Metadata{}, nil, cause
+ })
+
+ _, _, err = origin(context.Background(), racersdk.OriginRequest{})
+ if !errors.Is(err, cause) || !strings.Contains(output.String(), "mode=origin event=request_failed category=unavailable") || strings.Contains(output.String(), diagnosticSecret) {
+ t.Fatal("origin cause lost or diagnostic unsafe")
+ }
+
+ origin = diagnoseOrigin(func(context.Context, racersdk.OriginRequest) (racersdk.Metadata, io.ReadCloser, error) {
+ panic(diagnosticSecret)
+ })
+
+ defer func() {
+ if recover() != http.ErrAbortHandler {
+ t.Error("origin panic not passed to SDK recovery safely")
+ }
+
+ if !strings.Contains(output.String(), "mode=origin event=request_failed category=panic") || strings.Contains(output.String(), diagnosticSecret) {
+ t.Error("origin panic log missing or unsafe")
+ }
+ }()
+
+ _, _, _ = origin(context.Background(), racersdk.OriginRequest{})
+}
+
+type privateDiagnosticAddress struct{}
+
+func (privateDiagnosticAddress) Network() string { return diagnosticSecret }
+func (privateDiagnosticAddress) String() string { return diagnosticSecret }
+
+func TestStartupAddressSanitization(t *testing.T) {
+ output := captureDiagnosticLog(t)
+
+ for _, address := range []net.Addr{
+ privateDiagnosticAddress{},
+ &net.TCPAddr{IP: net.ParseIP("192.0.2.1"), Port: 1234},
+ &net.TCPAddr{IP: net.ParseIP("::1"), Port: 1234, Zone: diagnosticSecret},
+ &net.TCPAddr{IP: net.ParseIP("127.0.0.1"), Port: 0},
+ } {
+ logSidecarStartup(address)
+ }
+
+ if got := output.String(); strings.Contains(got, "address=") || strings.Contains(got, diagnosticSecret) {
+ t.Fatalf("logged unsafe address: %s", got)
+ }
+
+ logSidecarStartup(&net.TCPAddr{IP: net.ParseIP("::1"), Port: 1234})
+
+ if !strings.Contains(output.String(), "address=[::1]:1234") {
+ t.Fatal("IPv6 loopback address not logged")
+ }
+}
+
+type privateDiagnosticPanic struct{ formatted *bool }
+
+func (p privateDiagnosticPanic) String() string {
+ *p.formatted = true
+ return diagnosticSecret
+}
+
+func TestPanicValueNeverFormatted(t *testing.T) {
+ output := captureDiagnosticLog(t)
+ formatted := false
+
+ defer func() {
+ if recover() != http.ErrAbortHandler || formatted || strings.Contains(output.String(), diagnosticSecret) {
+ t.Error("panic value inspected or exposed")
+ }
+ }()
+
+ func() {
+ defer diagnoseSidecarPanic()
+
+ panic(privateDiagnosticPanic{formatted: &formatted})
+ }()
+}
diff --git a/cmd/racer-object/headers_test.go b/cmd/racer-object/headers_test.go
new file mode 100644
index 0000000000..58bf85835b
--- /dev/null
+++ b/cmd/racer-object/headers_test.go
@@ -0,0 +1,168 @@
+// Copyright (c) Microsoft Corporation.
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+ "bufio"
+ "fmt"
+ "io"
+ "net"
+ "net/http"
+ "strconv"
+ "strings"
+ "sync/atomic"
+ "testing"
+ "time"
+)
+
+func TestSidecarHeaderLimit(t *testing.T) {
+ var calls atomic.Int32
+
+ handler := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ calls.Add(1)
+ w.WriteHeader(http.StatusNoContent)
+ })
+
+ address, cancel, done := startTestSidecar(t, handler, time.Second)
+
+ defer func() {
+ cancel()
+ waitSidecar(t, done)
+ }()
+
+ for _, shape := range []struct {
+ name, uri, host, headers string
+ }{
+ {"single", "/bucket/key", "localhost", ""},
+ {"repeated", "/bucket/key", "localhost", strings.Repeat("X-Part: "+strings.Repeat("a", 100)+"\r\n", 100)},
+ {"empty", "/bucket/key", "localhost", strings.Repeat("X-Empty: \r\n", 100)},
+ {"long-name", "/bucket/key", "localhost", strings.Repeat("K", 1024) + ": value\r\n"},
+ {"uri", "/bucket/" + strings.Repeat("k", 1024), "localhost", ""},
+ {"host", "/bucket/key", strings.Repeat("h", 1024), ""},
+ } {
+ for _, size := range []int{16 << 10, (32 << 10) - 1, 32 << 10, (32 << 10) + 1, 33 << 10, 40 << 10} {
+ t.Run(fmt.Sprintf("%s/%d", shape.name, size), func(t *testing.T) {
+ padding := size - len(shape.uri) - len(shape.host) - len(shape.headers) - len("X-Pad: \r\n")
+ request := "GET " + shape.uri + " HTTP/1.1\r\nHost: " + shape.host + "\r\n" +
+ shape.headers + "X-Pad: " + strings.Repeat("p", padding) + "\r\n\r\n"
+
+ conn, err := net.DialTimeout("tcp", strings.TrimPrefix(address, "http://"), time.Second)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer conn.Close()
+
+ if err := conn.SetDeadline(time.Now().Add(3 * time.Second)); err != nil {
+ t.Fatal(err)
+ }
+
+ before := calls.Load()
+
+ if _, err := io.WriteString(conn, request); err != nil {
+ t.Fatal(err)
+ }
+
+ response, err := http.ReadResponse(bufio.NewReader(conn), nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer response.Body.Close()
+
+ wantStatus, wantCalls := http.StatusNoContent, int32(1)
+ if size > 32<<10 {
+ wantStatus, wantCalls = http.StatusRequestHeaderFieldsTooLarge, 0
+ }
+
+ if response.StatusCode != wantStatus {
+ t.Errorf("status = %d, want %d", response.StatusCode, wantStatus)
+ }
+
+ if got := calls.Load() - before; got != wantCalls {
+ t.Errorf("sidecar handler calls = %d, want %d", got, wantCalls)
+ }
+ })
+ }
+ }
+}
+
+func TestSidecarHeaderLimitWireFormat(t *testing.T) {
+ var calls atomic.Int32
+
+ handler := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ calls.Add(1)
+ w.WriteHeader(http.StatusNoContent)
+ })
+
+ address, cancel, done := startTestSidecar(t, handler, time.Second)
+
+ defer func() {
+ cancel()
+ waitSidecar(t, done)
+ }()
+
+ const wantXML = "RequestHeaderSectionTooLargeRequest Header Fields Too Large"
+
+ for _, method := range []string{http.MethodGet, http.MethodHead} {
+ for _, size := range []int{(32 << 10) + 1, 33 << 10} {
+ t.Run(fmt.Sprintf("%s/%d", method, size), func(t *testing.T) {
+ const uri, host, headers = "/bucket/private-key", "localhost", "Connection: close\r\n"
+
+ padding := size - len(uri) - len(host) - len(headers) - len("X-Pad: \r\n")
+ request := method + " " + uri + " HTTP/1.1\r\nHost: " + host + "\r\n" +
+ headers + "X-Pad: " + strings.Repeat("p", padding) + "\r\n\r\n"
+
+ conn, err := net.DialTimeout("tcp", strings.TrimPrefix(address, "http://"), time.Second)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer conn.Close()
+
+ if err := conn.SetDeadline(time.Now().Add(3 * time.Second)); err != nil {
+ t.Fatal(err)
+ }
+
+ if _, err := io.WriteString(conn, request); err != nil {
+ t.Fatal(err)
+ }
+
+ reader := bufio.NewReader(conn)
+
+ response, err := http.ReadResponse(reader, &http.Request{Method: method})
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer response.Body.Close()
+
+ if response.StatusCode != http.StatusRequestHeaderFieldsTooLarge ||
+ response.Header.Get("Content-Type") != "application/xml" ||
+ response.Header.Get("Content-Length") != strconv.Itoa(len(wantXML)) {
+ t.Errorf("error response: %d %v", response.StatusCode, response.Header)
+ }
+
+ body, err := io.ReadAll(response.Body)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ wantBody := wantXML
+ if method == http.MethodHead {
+ wantBody = ""
+ }
+
+ if string(body) != wantBody {
+ t.Errorf("body = %q, want %q", body, wantBody)
+ }
+
+ // ReadResponse suppresses HEAD bodies, so also check the remaining wire bytes.
+ if extra, err := io.ReadAll(reader); err != nil || len(extra) != 0 {
+ t.Errorf("unexpected trailing wire bytes %q, error %v", extra, err)
+ }
+ })
+ }
+ }
+
+ if got := calls.Load(); got != 0 {
+ t.Errorf("oversized headers reached the sidecar handler %d times", got)
+ }
+}
diff --git a/cmd/racer-object/image_workflow_test.go b/cmd/racer-object/image_workflow_test.go
new file mode 100644
index 0000000000..cd1c95ef5a
--- /dev/null
+++ b/cmd/racer-object/image_workflow_test.go
@@ -0,0 +1,117 @@
+// Copyright (c) Microsoft Corporation.
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+ "testing"
+ "time"
+
+ "sigs.k8s.io/yaml"
+)
+
+func TestImageWorkflowProvenance(t *testing.T) {
+ data, err := os.ReadFile(filepath.Join("..", "..", ".github", "workflows", "images.yaml"))
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ var workflow struct {
+ Jobs map[string]struct {
+ Steps []struct {
+ Name, ID, Uses, Run string
+ With map[string]string
+ }
+ }
+ }
+ if err := yaml.Unmarshal(data, &workflow); err != nil {
+ t.Fatal(err)
+ }
+
+ checkout, provenance, build := -1, -1, -1
+
+ var script string
+
+ for i, step := range workflow.Jobs["build"].Steps {
+ switch {
+ case strings.HasPrefix(step.Uses, "actions/checkout@"):
+ checkout = i
+ case step.ID == "provenance":
+ provenance, script = i, step.Run
+ case step.Name == "Build and push":
+ build = i
+
+ if step.With["context"] != "." {
+ t.Fatal("image must use the checkout whose provenance was recorded")
+ }
+
+ for _, arg := range []string{
+ "VERSION=${{ needs.resolve.outputs.tag }}",
+ "CONTAINER_REGISTRY=${{ env.CONTAINER_REGISTRY }}",
+ "GIT_COMMIT=${{ steps.provenance.outputs.git_commit }}",
+ "BUILD_TIME=${{ steps.provenance.outputs.build_time }}",
+ } {
+ if !strings.Contains(step.With["build-args"], arg+"\n") {
+ t.Errorf("missing build argument %s", arg)
+ }
+ }
+ }
+ }
+
+ if checkout < 0 || provenance <= checkout || build <= provenance || script == "" {
+ t.Fatal("expected checkout, provenance script, then image build")
+ }
+
+ for _, tc := range []struct {
+ name, setup string
+ wantError bool
+ }{
+ {name: "checked out revision"},
+ {name: "git failure", setup: "git() { return 17; }; ", wantError: true},
+ {name: "clock failure", setup: "date() { return 18; }; ", wantError: true},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ outputPath := filepath.Join(t.TempDir(), "outputs")
+ cmd := exec.Command("timeout", "--signal=TERM", "--kill-after=10s", "30s", "bash", "-e", "-c", tc.setup+script)
+
+ cmd.Env = append(os.Environ(), "GITHUB_OUTPUT="+outputPath)
+
+ output, err := cmd.CombinedOutput()
+ if (err != nil) != tc.wantError {
+ t.Fatalf("provenance error = %v, want error %t: %s", err, tc.wantError, output)
+ }
+
+ if tc.wantError {
+ if data, err := os.ReadFile(outputPath); err == nil && len(data) != 0 {
+ t.Fatalf("published incomplete provenance: %s", data)
+ }
+
+ return
+ }
+
+ data, err := os.ReadFile(outputPath)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ head, err := exec.Command("timeout", "--signal=TERM", "--kill-after=10s", "30s", "git", "rev-parse", "HEAD").Output()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ lines := strings.Split(strings.TrimSpace(string(data)), "\n")
+ if len(lines) != 2 || lines[0] != "git_commit="+strings.TrimSpace(string(head)) {
+ t.Fatalf("wrong checkout provenance: %s", data)
+ }
+
+ stamp := strings.TrimPrefix(lines[1], "build_time=")
+ if _, err := time.Parse(time.RFC3339, stamp); err != nil || !strings.HasSuffix(stamp, "Z") {
+ t.Fatalf("invalid UTC build time: %q", lines[1])
+ }
+ })
+ }
+}
diff --git a/cmd/racer-object/listener.go b/cmd/racer-object/listener.go
new file mode 100644
index 0000000000..9ecf64d987
--- /dev/null
+++ b/cmd/racer-object/listener.go
@@ -0,0 +1,82 @@
+// Copyright (c) Microsoft Corporation.
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+ "errors"
+ "io"
+ "net"
+ "sync"
+)
+
+// limitSidecarListener retains TCP ReaderFrom for net/http's Unix-to-TCP splice
+// path. Embedding only net.Conn, as netutil.LimitListener does, hides that method.
+func limitSidecarListener(listener net.Listener, limit int) net.Listener {
+ return &sidecarLimitedListener{Listener: listener, slots: make(chan struct{}, limit), done: make(chan struct{})}
+}
+
+type sidecarLimitedListener struct {
+ net.Listener
+ slots chan struct{}
+ done chan struct{}
+ closeOnce sync.Once
+}
+
+func (l *sidecarLimitedListener) Accept() (net.Conn, error) {
+ select {
+ case <-l.done:
+ return nil, net.ErrClosed
+ case l.slots <- struct{}{}:
+ }
+
+ c, err := l.Listener.Accept()
+ if err != nil {
+ <-l.slots
+ return nil, err
+ }
+
+ select {
+ case <-l.done:
+ err := c.Close()
+
+ <-l.slots
+
+ return nil, errors.Join(net.ErrClosed, err)
+ default:
+ }
+
+ limited := &sidecarLimitedConn{Conn: c, release: func() { <-l.slots }}
+ if reader, ok := c.(io.ReaderFrom); ok {
+ return &sidecarLimitedReaderConn{sidecarLimitedConn: limited, reader: reader}, nil
+ }
+
+ return limited, nil
+}
+
+func (l *sidecarLimitedListener) Close() error {
+ l.closeOnce.Do(func() { close(l.done) })
+ return l.Listener.Close()
+}
+
+type sidecarLimitedConn struct {
+ net.Conn
+ releaseOnce sync.Once
+ release func()
+}
+
+func (c *sidecarLimitedConn) Close() error {
+ err := c.Conn.Close()
+ c.releaseOnce.Do(c.release)
+
+ return err
+}
+
+type sidecarLimitedReaderConn struct {
+ *sidecarLimitedConn
+ reader io.ReaderFrom
+}
+
+func (c *sidecarLimitedReaderConn) ReadFrom(r io.Reader) (int64, error) {
+ return c.reader.ReadFrom(r)
+}
diff --git a/cmd/racer-object/listener_test.go b/cmd/racer-object/listener_test.go
new file mode 100644
index 0000000000..074f324000
--- /dev/null
+++ b/cmd/racer-object/listener_test.go
@@ -0,0 +1,314 @@
+// Copyright (c) Microsoft Corporation.
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "io"
+ "net"
+ "net/http"
+ "strconv"
+ "sync"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "golang.org/x/net/netutil"
+
+ "github.com/Azure/unbounded/pkg/racersdk"
+ "github.com/Azure/unbounded/pkg/racersdk/racersdktest"
+)
+
+type sidecarObservedListener struct {
+ net.Listener
+ bytes atomic.Int64
+ connections atomic.Int64
+}
+
+func (l *sidecarObservedListener) Accept() (net.Conn, error) {
+ c, err := l.Listener.Accept()
+ if err != nil {
+ return nil, err
+ }
+
+ l.connections.Add(1)
+
+ return &sidecarObservedConn{TCPConn: c.(*net.TCPConn), bytes: &l.bytes}, nil
+}
+
+type sidecarObservedConn struct {
+ *net.TCPConn
+ bytes *atomic.Int64
+}
+
+func (c *sidecarObservedConn) ReadFrom(r io.Reader) (int64, error) {
+ lr, ok := r.(*io.LimitedReader)
+ if !ok || lr.N <= 0 || lr.N > 256<<10 {
+ return 0, errors.New("unbounded source")
+ }
+
+ if _, ok := lr.R.(*net.UnixConn); !ok {
+ return 0, errors.New("wrapped Unix source")
+ }
+
+ n, err := c.TCPConn.ReadFrom(r)
+ c.bytes.Add(n)
+
+ return n, err
+}
+
+// Exercise the real SDK, Unix sockets, serveSidecar listener, net/http response,
+// and TCPConn.ReadFrom. Counting only ResponseWriter.ReadFrom misses the bug.
+func sidecarStreamFixture(tb testing.TB, size int, hideReaderFrom bool) (*http.Client, string, *sidecarObservedListener, []byte) {
+ tb.Helper()
+
+ data := bytes.Repeat([]byte("sidecar!"), size/8)
+
+ metadata := racersdk.Metadata{Size: int64(size), ETag: `"version"`, ExpiresAt: time.Now().Add(time.Hour).Truncate(time.Millisecond)}
+
+ sdk := racersdktest.NewClient(tb, func(_ context.Context, request racersdk.OriginRequest) (racersdk.Metadata, io.ReadCloser, error) {
+ if request.Head {
+ return metadata, nil, nil
+ }
+
+ return metadata, io.NopCloser(bytes.NewReader(data)), nil
+ })
+
+ l, err := listenSidecar(tb.Context(), "127.0.0.1:0")
+ if err != nil {
+ tb.Fatal(err)
+ }
+
+ observed := &sidecarObservedListener{Listener: l}
+
+ var listener net.Listener = observed
+ if hideReaderFrom {
+ listener = netutil.LimitListener(listener, 128)
+ }
+
+ ctx, cancel := context.WithCancel(tb.Context())
+ done := make(chan error, 1)
+ handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ v, err := sdk.Get(r.Context(), racersdk.Request{})
+ if err != nil {
+ tb.Error(err)
+ panic(http.ErrAbortHandler)
+ }
+ defer v.Close()
+
+ w.Header().Set("Content-Length", strconv.Itoa(size))
+ w.Header().Set("Content-Type", "application/octet-stream")
+
+ if n, err := v.WriteTo(w); err != nil || n != int64(size) {
+ tb.Errorf("transfer: %d, %v", n, err)
+ panic(http.ErrAbortHandler)
+ }
+ })
+
+ go func() { done <- serveSidecar(ctx, listener, handler, 10*time.Second) }()
+
+ tb.Cleanup(func() {
+ cancel()
+
+ select {
+ case err := <-done:
+ if err != nil {
+ tb.Error(err)
+ }
+ case <-time.After(7 * time.Second):
+ tb.Error("sidecar did not stop")
+ }
+ })
+
+ transport := &http.Transport{}
+ tb.Cleanup(transport.CloseIdleConnections)
+
+ return &http.Client{Transport: transport, Timeout: 10 * time.Second}, "http://" + l.Addr().String(), observed, data
+}
+
+func TestSidecarHTTPReaderFrom(t *testing.T) {
+ client, address, observed, data := sidecarStreamFixture(t, 1<<20, false)
+ for range 2 {
+ response, err := client.Get(address)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ got, err := io.ReadAll(response.Body)
+ response.Body.Close()
+
+ if err != nil || !bytes.Equal(got, data) {
+ t.Fatalf("integrity: %d, %v", len(got), err)
+ }
+ }
+
+ if got := observed.bytes.Load(); got < int64(len(data)) {
+ t.Fatalf("underlying TCP ReaderFrom bytes = %d, want at least %d", got, len(data))
+ }
+
+ if got := observed.connections.Load(); got != 1 {
+ t.Fatalf("keepalive connections = %d", got)
+ }
+}
+
+// This local benchmark includes fake origin/protocol work and HTTP draining,
+// not a production dataplane. The hidden variant reproduces the old limiter.
+func BenchmarkSidecarHTTPReaderFrom(b *testing.B) {
+ for _, hidden := range []bool{true, false} {
+ b.Run(map[bool]string{true: "hidden", false: "preserved"}[hidden], func(b *testing.B) {
+ client, address, observed, _ := sidecarStreamFixture(b, 16<<20, hidden)
+ consume := func() {
+ response, err := client.Get(address)
+ if err != nil {
+ b.Fatal(err)
+ }
+
+ n, err := io.Copy(io.Discard, response.Body)
+ response.Body.Close()
+
+ if err != nil || n != 16<<20 {
+ b.Fatalf("transfer: %d, %v", n, err)
+ }
+ }
+ consume()
+ b.ReportAllocs()
+ b.SetBytes(16 << 20)
+ b.ResetTimer()
+
+ for range b.N {
+ consume()
+ }
+
+ b.StopTimer()
+
+ if !hidden && observed.bytes.Load() == 0 {
+ b.Fatal("TCP ReaderFrom was not reached")
+ }
+ })
+ }
+}
+
+type (
+ sidecarAcceptResult struct {
+ conn net.Conn
+ err error
+ }
+ sidecarScriptListener struct {
+ results chan sidecarAcceptResult
+ done chan struct{}
+ once sync.Once
+ }
+)
+
+func (l *sidecarScriptListener) Accept() (net.Conn, error) {
+ select {
+ case r := <-l.results:
+ return r.conn, r.err
+ case <-l.done:
+ return nil, net.ErrClosed
+ }
+}
+func (l *sidecarScriptListener) Close() error { l.once.Do(func() { close(l.done) }); return nil }
+func (l *sidecarScriptListener) Addr() net.Addr { return &net.TCPAddr{} }
+
+func TestSidecarLimitedListenerLifecycle(t *testing.T) {
+ base := &sidecarScriptListener{results: make(chan sidecarAcceptResult, 1), done: make(chan struct{})}
+ l := limitSidecarListener(base, 1).(*sidecarLimitedListener)
+
+ t.Cleanup(func() { _ = l.Close() })
+
+ sentinel := errors.New("accept failed")
+ base.results <- sidecarAcceptResult{err: sentinel}
+
+ if _, err := l.Accept(); !errors.Is(err, sentinel) || len(l.slots) != 0 {
+ t.Fatalf("accept failure: %v", err)
+ }
+
+ left, right := net.Pipe()
+ defer right.Close()
+
+ base.results <- sidecarAcceptResult{conn: left}
+
+ c, err := l.Accept()
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer c.Close()
+
+ if _, ok := c.(io.ReaderFrom); ok {
+ t.Fatal("invented ReaderFrom")
+ }
+
+ if err := c.SetDeadline(time.Now().Add(-time.Second)); err != nil {
+ t.Fatal(err)
+ }
+
+ if _, err := c.Read(make([]byte, 1)); err == nil {
+ t.Fatal("read deadline lost")
+ }
+
+ if _, err := c.Write([]byte{1}); err == nil {
+ t.Fatal("write deadline lost")
+ }
+
+ accepted := make(chan error, 1)
+
+ base.results <- sidecarAcceptResult{err: sentinel}
+
+ go func() { _, err := l.Accept(); accepted <- err }()
+
+ select {
+ case <-accepted:
+ t.Fatal("limit bypassed")
+ case <-time.After(20 * time.Millisecond):
+ }
+
+ var closes sync.WaitGroup
+ for range 2 {
+ closes.Go(func() { _ = c.Close() })
+ }
+
+ closes.Wait()
+
+ select {
+ case err := <-accepted:
+ if !errors.Is(err, sentinel) {
+ t.Fatal(err)
+ }
+ case <-time.After(time.Second):
+ t.Fatal("slot not released")
+ }
+
+ if len(l.slots) != 0 {
+ t.Fatal("slot leaked")
+ }
+
+ for _, full := range []bool{false, true} {
+ t.Run(strconv.FormatBool(full), func(t *testing.T) {
+ base := &sidecarScriptListener{results: make(chan sidecarAcceptResult), done: make(chan struct{})}
+
+ l := limitSidecarListener(base, 1).(*sidecarLimitedListener)
+ if full {
+ l.slots <- struct{}{}
+ }
+
+ done := make(chan error, 1)
+
+ go func() { _, err := l.Accept(); done <- err }()
+
+ _ = l.Close()
+
+ select {
+ case err := <-done:
+ if !errors.Is(err, net.ErrClosed) {
+ t.Fatal(err)
+ }
+ case <-time.After(time.Second):
+ t.Fatal("close did not unblock Accept")
+ }
+ })
+ }
+}
diff --git a/cmd/racer-object/logging.go b/cmd/racer-object/logging.go
new file mode 100644
index 0000000000..7ee85fba69
--- /dev/null
+++ b/cmd/racer-object/logging.go
@@ -0,0 +1,148 @@
+// Copyright (c) Microsoft Corporation.
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+ "fmt"
+ "log"
+ "sync"
+ "sync/atomic"
+ "time"
+
+ "github.com/aws/aws-sdk-go-v2/aws"
+ "github.com/aws/smithy-go/logging"
+)
+
+const (
+ logLevelInfo = "info"
+ logLevelDebug = "debug"
+
+ // Each event and category may log diagnosticBurst warn lines per
+ // diagnosticWindow; the next line after the window reports the rest.
+ diagnosticBurst = 10
+ diagnosticWindow = time.Minute
+)
+
+// diagnosticDebug is set once from --log-level before serving starts.
+var diagnosticDebug atomic.Bool
+
+// expectedCategories are outcomes caused by the client or by normal object
+// churn, not by a fault in racer-object or its upstream.
+var expectedCategories = map[string]bool{
+ "canceled": true, "notfound": true, "versionmismatch": true, "forbidden": true,
+ "unauthorized": true, "invalid": true, "range": true,
+}
+
+var diagnosticLimiter = newLogLimiter(diagnosticBurst, diagnosticWindow, time.Now)
+
+// logOutcome logs a request outcome. Expected categories are logged only at
+// debug level. Other categories are rate limited per event and category.
+// line must contain only fixed or validated fields.
+func logOutcome(event, category, line string) {
+ if expectedCategories[category] {
+ if diagnosticDebug.Load() {
+ log.Print(line + " level=debug")
+ }
+
+ return
+ }
+
+ allowed, suppressed := diagnosticLimiter.allow(event + "/" + category)
+ if !allowed {
+ return
+ }
+
+ if suppressed > 0 {
+ line += fmt.Sprintf(" suppressed=%d", suppressed)
+ }
+
+ log.Print(line + " level=warn")
+}
+
+type logLimiter struct {
+ mu sync.Mutex
+ burst int
+ window time.Duration
+ now func() time.Time
+ keys map[string]*logLimit
+}
+
+type logLimit struct {
+ start time.Time
+ count int
+ suppressed int
+}
+
+func newLogLimiter(burst int, window time.Duration, now func() time.Time) *logLimiter {
+ return &logLimiter{burst: burst, window: window, now: now, keys: map[string]*logLimit{}}
+}
+
+// allow reports whether to log, and how many lines were dropped for the key
+// since the last allowed line. Keys are fixed categories, so the map is bounded.
+func (l *logLimiter) allow(key string) (bool, int) {
+ l.mu.Lock()
+ defer l.mu.Unlock()
+
+ now := l.now()
+
+ limit := l.keys[key]
+ if limit == nil || now.Sub(limit.start) >= l.window {
+ suppressed := 0
+ if limit != nil {
+ suppressed = limit.suppressed
+ }
+
+ l.keys[key] = &logLimit{start: now, count: 1}
+
+ return true, suppressed
+ }
+
+ if limit.count >= l.burst {
+ limit.suppressed++
+
+ return false, 0
+ }
+
+ limit.count++
+
+ return true, 0
+}
+
+// awsLogger records that the SDK emitted a warning, or a debug line when
+// debug is set, on the process log (stderr). It writes only fixed fields: the
+// SDK's format string and arguments are never printed, because they can carry
+// upstream error text, object keys, and URLs (for example, the retry
+// middleware includes the original error in its messages).
+func awsLogger(debug bool) logging.Logger {
+ return logging.LoggerFunc(func(classification logging.Classification, _ string, _ ...any) {
+ level := "warn"
+
+ switch classification {
+ case logging.Warn:
+ case logging.Debug:
+ if !debug {
+ return
+ }
+
+ level = "debug"
+ default:
+ return
+ }
+
+ log.Print("racer-object: mode=origin event=aws_sdk level=" + level)
+ })
+}
+
+// awsLogMode enables only retry and deprecation logs in debug mode. awsLogger
+// reduces them to fixed fields, so they show only that the SDK retried or
+// warned, not why. Request
+// and response logging is never enabled: it prints headers (including the
+// SigV4 Authorization header), bodies, and signing details.
+func awsLogMode(debug bool) aws.ClientLogMode {
+ if !debug {
+ return 0
+ }
+
+ return aws.LogRetries | aws.LogDeprecatedUsage
+}
diff --git a/cmd/racer-object/logging_test.go b/cmd/racer-object/logging_test.go
new file mode 100644
index 0000000000..a2d163fcca
--- /dev/null
+++ b/cmd/racer-object/logging_test.go
@@ -0,0 +1,196 @@
+// Copyright (c) Microsoft Corporation.
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/aws/smithy-go/logging"
+
+ "github.com/Azure/unbounded/internal/racer/object"
+ "github.com/Azure/unbounded/pkg/racersdk"
+)
+
+func useTestLogLimiter(t *testing.T, now func() time.Time) {
+ t.Helper()
+
+ previous := diagnosticLimiter
+ diagnosticLimiter = newLogLimiter(2, time.Minute, now)
+
+ t.Cleanup(func() { diagnosticLimiter = previous })
+}
+
+func setDiagnosticDebug(t *testing.T, debug bool) {
+ t.Helper()
+
+ previous := diagnosticDebug.Load()
+ diagnosticDebug.Store(debug)
+
+ t.Cleanup(func() { diagnosticDebug.Store(previous) })
+}
+
+func TestLogLimiter(t *testing.T) {
+ now := time.Unix(0, 0)
+ limiter := newLogLimiter(2, time.Minute, func() time.Time { return now })
+
+ for i, want := range []bool{true, true, false, false} {
+ if allowed, suppressed := limiter.allow("a"); allowed != want || suppressed != 0 {
+ t.Fatalf("call %d = %v/%d", i, allowed, suppressed)
+ }
+ }
+
+ if allowed, _ := limiter.allow("b"); !allowed {
+ t.Fatal("keys share a budget")
+ }
+
+ now = now.Add(time.Minute)
+
+ if allowed, suppressed := limiter.allow("a"); !allowed || suppressed != 2 {
+ t.Fatalf("after window = %v/%d, want true/2", allowed, suppressed)
+ }
+
+ if allowed, suppressed := limiter.allow("a"); !allowed || suppressed != 0 {
+ t.Fatalf("suppressed count repeated: %v/%d", allowed, suppressed)
+ }
+}
+
+func TestOriginExpectedOutcomesLevel(t *testing.T) {
+ useTestLogLimiter(t, time.Now)
+
+ for _, cause := range []error{racersdk.ErrNotFound, racersdk.ErrForbidden, racersdk.ErrVersionMismatch, context.Canceled, racersdk.ErrInvalidRequest} {
+ for _, debug := range []bool{false, true} {
+ t.Run(fmt.Sprintf("%v/%v", cause, debug), func(t *testing.T) {
+ output := captureDiagnosticLog(t)
+ setDiagnosticDebug(t, debug)
+
+ origin := diagnoseOrigin(func(context.Context, racersdk.OriginRequest) (racersdk.Metadata, io.ReadCloser, error) {
+ return racersdk.Metadata{}, nil, fmt.Errorf("%s: %w", diagnosticSecret, cause)
+ })
+ _, _, _ = origin(context.Background(), racersdk.OriginRequest{}) //nolint:errcheck // Only the log matters.
+
+ got := output.String()
+ if !debug && got != "" {
+ t.Fatalf("expected outcome logged at info: %q", got)
+ }
+
+ if debug && (!strings.Contains(got, "level=debug") || strings.Contains(got, "level=warn")) {
+ t.Fatalf("expected outcome not logged at debug: %q", got)
+ }
+
+ if strings.Contains(got, diagnosticSecret) {
+ t.Fatal("unsafe diagnostic")
+ }
+ })
+ }
+ }
+}
+
+func TestOutcomeRateLimit(t *testing.T) {
+ now := time.Unix(0, 0)
+
+ useTestLogLimiter(t, func() time.Time { return now })
+ setDiagnosticDebug(t, false)
+
+ output := captureDiagnosticLog(t)
+ failure := object.SidecarFailure{Method: "GET", Status: 503, Bucket: "b", Err: racersdk.ErrUnavailable}
+
+ for range 5 {
+ logSidecarFailure(failure)
+ }
+
+ logSidecarFailure(object.SidecarFailure{Method: "GET", Status: 502, Bucket: "b", Err: object.ErrMetadataMismatch})
+
+ if got := strings.Count(output.String(), "category=unavailable"); got != 2 {
+ t.Fatalf("unavailable lines = %d, want 2: %q", got, output.String())
+ }
+
+ if !strings.Contains(output.String(), "category=mismatch") {
+ t.Fatal("other category was rate limited")
+ }
+
+ now = now.Add(time.Minute)
+
+ logSidecarFailure(failure)
+
+ if !strings.Contains(output.String(), "category=unavailable bucket=\"b\" suppressed=3 level=warn") {
+ t.Fatalf("missing suppressed summary: %q", output.String())
+ }
+}
+
+func TestSidecarVersionMismatchIsDebug(t *testing.T) {
+ useTestLogLimiter(t, time.Now)
+ setDiagnosticDebug(t, false)
+
+ output := captureDiagnosticLog(t)
+
+ logSidecarFailure(object.SidecarFailure{Method: "GET", Status: 503, Bucket: "b", Err: racersdk.ErrVersionMismatch})
+
+ if output.String() != "" {
+ t.Fatalf("version mismatch logged at info: %q", output.String())
+ }
+}
+
+func TestAWSLogger(t *testing.T) {
+ const sentinel = "PRIVATE-SENTINEL"
+
+ args := []any{
+ "private/" + sentinel + "/key",
+ "https://bucket.example/" + sentinel + "?X-Amz-Signature=" + sentinel,
+ errors.New("AccessDenied: " + sentinel + "\nlevel=forged"),
+ }
+
+ for _, debug := range []bool{false, true} {
+ t.Run(fmt.Sprint(debug), func(t *testing.T) {
+ output := captureDiagnosticLog(t)
+ logger := awsLogger(debug)
+
+ logger.Logf(logging.Warn, "retry %s %s %v", args...)
+ logger.Logf(logging.Debug, "unretryable "+sentinel+" %s %s %v", args...)
+ logger.Logf(logging.Classification("OTHER"), "ignored %s", sentinel)
+
+ got := output.String()
+ if strings.Contains(got, sentinel) || strings.Contains(got, "forged") {
+ t.Fatalf("SDK message text leaked: %q", got)
+ }
+
+ if strings.Count(got, "event=aws_sdk level=warn\n") != 1 {
+ t.Fatalf("warning missing: %q", got)
+ }
+
+ if strings.Contains(got, "event=aws_sdk level=debug\n") != debug || strings.Count(got, "\n") != map[bool]int{false: 1, true: 2}[debug] {
+ t.Fatalf("debug=%v filtering wrong: %q", debug, got)
+ }
+ })
+ }
+}
+
+func TestAWSLogMode(t *testing.T) {
+ if awsLogMode(false) != 0 {
+ t.Fatal("info level enabled SDK client logs")
+ }
+
+ mode := awsLogMode(true)
+ if !mode.IsRetries() || mode.IsRequest() || mode.IsRequestWithBody() || mode.IsResponse() || mode.IsResponseWithBody() || mode.IsSigning() {
+ t.Fatalf("debug log mode %v exposes requests or signing", mode)
+ }
+}
+
+func TestOriginAWSConfigLogging(t *testing.T) {
+ for _, level := range []string{logLevelInfo, logLevelDebug} {
+ cfg, err := originAWSConfig(t.Context(), options{region: "us-east-1", requestTimeout: time.Second, logLevel: level})
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if cfg.Logger == nil || cfg.ClientLogMode != awsLogMode(level == logLevelDebug) {
+ t.Fatalf("%s: logger %v mode %v", level, cfg.Logger, cfg.ClientLogMode)
+ }
+ }
+}
diff --git a/cmd/racer-object/main.go b/cmd/racer-object/main.go
new file mode 100644
index 0000000000..d4d4e44e61
--- /dev/null
+++ b/cmd/racer-object/main.go
@@ -0,0 +1,528 @@
+// Copyright (c) Microsoft Corporation.
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "flag"
+ "fmt"
+ "io"
+ "log"
+ "math"
+ "net"
+ "net/http"
+ "net/url"
+ "os"
+ "os/signal"
+ "strconv"
+ "strings"
+ "syscall"
+ "time"
+
+ "github.com/aws/aws-sdk-go-v2/aws"
+ awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http"
+ "github.com/aws/aws-sdk-go-v2/config"
+ "golang.org/x/sys/unix"
+
+ "github.com/Azure/unbounded/internal/racer/object"
+ "github.com/Azure/unbounded/internal/version"
+ "github.com/Azure/unbounded/pkg/racersdk"
+)
+
+// maxOriginRequestTimeout matches the Racer SDK's fixed origin operation deadline.
+const maxOriginRequestTimeout = time.Minute
+
+const originMaxConcurrentRequests = 64
+
+// The socket write deadline starts before the handler deadline. Allow time to
+// send a timeout response after the handler's context expires.
+const sidecarWriteGrace = 5 * time.Second
+
+const sidecarMaxHeaderBytes = 32 << 10
+
+type bucketsFlag []string
+
+func (b *bucketsFlag) String() string { return strings.Join(*b, ",") }
+func (b *bucketsFlag) Set(value string) error {
+ *b = append(*b, value)
+ return nil
+}
+
+type options struct {
+ mode string
+ cache string
+ namespace string
+ buckets bucketsFlag
+ region string
+ endpoint string
+ pathStyle bool
+ metadataTTL time.Duration
+ requestTimeout time.Duration
+ listen string
+ logLevel string
+}
+
+func main() {
+ ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+ defer stop()
+
+ if err := run(ctx, os.Args[1:], os.Stdout); err != nil {
+ fmt.Fprintln(os.Stderr, "racer-object:", err)
+ os.Exit(1)
+ }
+}
+
+func parseOptions(args []string, output io.Writer) (options, error) {
+ var o options
+ if len(args) == 0 {
+ return o, errors.New("expected origin or sidecar (use --help)")
+ }
+
+ switch args[0] {
+ case "--help", "-h":
+ return o, writeHelp(output, "Usage: racer-object [flags]\nUse racer-object --help for flags.\n")
+ case "--version":
+ return o, writeHelp(output, version.String()+"\n")
+ case "origin", "sidecar":
+ o.mode = args[0]
+ default:
+ return o, errors.New("unknown command (use --help)")
+ }
+
+ fs := flag.NewFlagSet(o.mode, flag.ContinueOnError)
+ fs.SetOutput(io.Discard)
+
+ var helpErr error
+
+ fs.Usage = func() {
+ var defaults bytes.Buffer
+
+ fs.SetOutput(&defaults)
+ fs.PrintDefaults()
+ fs.SetOutput(io.Discard)
+
+ helpErr = writeHelp(output, "Usage: racer-object "+o.mode+" [flags]\n"+defaults.String())
+ }
+ cache := fs.String("cache", "racer-object", "Racer cache name")
+ fs.StringVar(&o.namespace, "namespace", "", "Required stable upstream identity; use the same value in origin and sidecar")
+ fs.Var(&o.buckets, "bucket", "Allowed bucket (repeatable; omitted allows all buckets)")
+
+ fs.StringVar(&o.logLevel, "log-level", logLevelInfo, "Diagnostic log level: info or debug; debug also logs expected outcomes")
+
+ showVersion := fs.Bool("version", false, "Print version")
+ if o.mode == "origin" {
+ fs.StringVar(&o.region, "region", "", "AWS region (defaults to AWS configuration and environment)")
+ fs.StringVar(&o.endpoint, "endpoint", "", "Optional HTTP(S) upstream origin, without a path or credentials")
+ fs.BoolVar(&o.pathStyle, "path-style", true, "Use path-style upstream addressing")
+ fs.DurationVar(&o.metadataTTL, "metadata-ttl", 30*time.Second, "Metadata lifetime; zero expires immediately")
+ fs.DurationVar(&o.requestTimeout, "request-timeout", maxOriginRequestTimeout, "Positive per-attempt S3 request timeout, at most 1m")
+ } else {
+ fs.StringVar(&o.listen, "listen", "127.0.0.1:8080", "Loopback IP literal and port")
+ fs.DurationVar(&o.requestTimeout, "request-timeout", 5*time.Minute, "Positive sidecar request timeout")
+ }
+
+ if err := fs.Parse(args[1:]); err != nil {
+ if errors.Is(err, flag.ErrHelp) {
+ return o, helpErr
+ }
+
+ return o, errors.New("invalid flags (use --help)")
+ }
+
+ if fs.NArg() != 0 {
+ return o, errors.New("unexpected positional arguments")
+ }
+
+ if *showVersion {
+ return o, writeHelp(output, version.String()+"\n")
+ }
+
+ o.cache = *cache
+
+ validationClient, err := racersdk.NewClient(racersdk.ClientConfig{Cache: o.cache})
+ if err != nil {
+ return o, errors.New("invalid --cache")
+ }
+
+ closeResource(validationClient)
+
+ if _, err := object.NewRequest(o.namespace, "validation", "validation", ""); err != nil {
+ return o, errors.New("--namespace must be a valid stable upstream identity")
+ }
+
+ for _, bucket := range o.buckets {
+ if _, err := object.NewRequest(o.namespace, bucket, "validation", ""); err != nil {
+ return o, errors.New("invalid --bucket")
+ }
+ }
+
+ if o.logLevel != logLevelInfo && o.logLevel != logLevelDebug {
+ return o, errors.New("--log-level must be info or debug")
+ }
+
+ if o.requestTimeout <= 0 {
+ return o, errors.New("--request-timeout must be positive")
+ }
+
+ if o.mode == "origin" {
+ if o.requestTimeout > maxOriginRequestTimeout {
+ return o, errors.New("--request-timeout must not exceed 1m in origin mode")
+ }
+
+ if o.metadataTTL < 0 {
+ return o, errors.New("--metadata-ttl must not be negative")
+ }
+
+ if err := validateEndpoint(o.endpoint); err != nil {
+ return o, err
+ }
+ } else {
+ if o.requestTimeout > math.MaxInt64-sidecarWriteGrace {
+ return o, errors.New("--request-timeout is too large to allow sidecar write grace")
+ }
+
+ if err := validateListen(o.listen); err != nil {
+ return o, err
+ }
+ }
+
+ return o, nil
+}
+
+func writeHelp(output io.Writer, text string) error {
+ if _, err := io.WriteString(output, text); err != nil {
+ return safeDiagnostic("could not write CLI output", "output", err)
+ }
+
+ return flag.ErrHelp
+}
+
+func closeResource(c io.Closer) {
+ if err := c.Close(); err != nil {
+ return
+ }
+}
+
+func validateEndpoint(value string) error {
+ if value == "" {
+ return nil
+ }
+
+ u, err := url.Parse(value)
+ if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Hostname() == "" ||
+ u.User != nil || u.Opaque != "" || u.RawQuery != "" || u.ForceQuery || strings.Contains(value, "#") ||
+ (u.EscapedPath() != "" && u.EscapedPath() != "/") {
+ return errors.New("--endpoint must be an HTTP(S) origin without credentials, query, fragment, or path prefix")
+ }
+
+ if strings.HasSuffix(u.Host, ":") {
+ return errors.New("invalid --endpoint port")
+ }
+
+ if port := u.Port(); port != "" {
+ n, err := strconv.Atoi(port)
+ if err != nil || n < 1 || n > 65535 {
+ return errors.New("invalid --endpoint port")
+ }
+ }
+
+ return nil
+}
+
+func validateListen(value string) error {
+ host, port, err := net.SplitHostPort(value)
+
+ ip := net.ParseIP(host)
+ if err != nil || ip == nil || !ip.IsLoopback() {
+ return errors.New("--listen must use a loopback IP literal and port")
+ }
+
+ if port == "" || strings.IndexFunc(port, func(r rune) bool { return r < '0' || r > '9' }) != -1 {
+ return errors.New("--listen port must be between 0 and 65535")
+ }
+
+ n, err := strconv.Atoi(port)
+ if err != nil || n < 0 || n > 65535 {
+ return errors.New("--listen port must be between 0 and 65535")
+ }
+
+ return nil
+}
+
+func run(ctx context.Context, args []string, output io.Writer) error {
+ o, err := parseOptions(args, output)
+ if errors.Is(err, flag.ErrHelp) {
+ return nil
+ }
+
+ if err != nil {
+ return err
+ }
+
+ if ctx.Err() != nil {
+ return nil
+ }
+
+ diagnosticDebug.Store(o.logLevel == logLevelDebug)
+
+ if o.mode == "origin" {
+ return runOrigin(ctx, o)
+ }
+
+ return runSidecar(ctx, o)
+}
+
+func originAWSConfig(ctx context.Context, o options) (aws.Config, error) {
+ httpClient := awshttp.NewBuildableClient().WithTimeout(o.requestTimeout).
+ WithTransportOptions(func(transport *http.Transport) {
+ // Keep room for the SDK's four separate metadata requests too.
+ transport.MaxIdleConnsPerHost = originMaxConcurrentRequests + 4
+ transport.MaxIdleConns = max(transport.MaxIdleConns, transport.MaxIdleConnsPerHost)
+ })
+
+ load := []func(*config.LoadOptions) error{
+ config.WithHTTPClient(httpClient),
+ config.WithLogger(awsLogger(o.logLevel == logLevelDebug)),
+ config.WithClientLogMode(awsLogMode(o.logLevel == logLevelDebug)),
+ // S3-compatible backends may reject the SDK's default optional checksums.
+ config.WithRequestChecksumCalculation(aws.RequestChecksumCalculationWhenRequired),
+ config.WithResponseChecksumValidation(aws.ResponseChecksumValidationWhenRequired),
+ }
+ if o.region != "" {
+ load = append(load, config.WithRegion(o.region))
+ }
+
+ cfg, err := config.LoadDefaultConfig(ctx, load...)
+ if err != nil {
+ return aws.Config{}, safeDiagnostic("could not load AWS configuration", "config", err)
+ }
+
+ if strings.TrimSpace(cfg.Region) == "" {
+ return aws.Config{}, errors.New("AWS region is required; set --region or AWS_REGION")
+ }
+
+ return cfg, nil
+}
+
+func runOrigin(ctx context.Context, o options) error {
+ cfg, err := originAWSConfig(ctx, o)
+ if err != nil {
+ return err
+ }
+
+ client, err := newOriginS3Client(cfg, o)
+ if err != nil {
+ return err
+ }
+ defer closeResource(client)
+
+ origin, err := object.NewOrigin(client, object.OriginConfig{
+ Namespace: o.namespace, Buckets: o.buckets, MetadataTTL: o.metadataTTL,
+ })
+ if err != nil {
+ return safeDiagnostic("could not configure origin", "config", err)
+ }
+
+ root, err := os.Open("/")
+ if err != nil {
+ return safeDiagnostic("could not open origin directory root", "filesystem", err)
+ }
+ defer closeResource(root)
+
+ if err := prepareOriginDirectory(root, []string{"run", "racer", o.cache, "origin"}); err != nil {
+ return safeDiagnostic("could not prepare safe origin directory", "filesystem", err)
+ }
+
+ log.Print("racer-object: mode=origin event=starting")
+
+ err = racersdk.ServeOrigin(ctx, racersdk.OriginConfig{
+ Cache: o.cache, RecoverStaleSocket: true,
+ MaxConcurrentRequests: originMaxConcurrentRequests,
+ }, diagnoseOrigin(origin))
+ if ctx.Err() != nil {
+ return nil
+ }
+
+ if err != nil {
+ return safeDiagnostic("origin server failed", "server", err)
+ }
+
+ return nil
+}
+
+// Walk from a trusted descriptor without following symlinks or changing existing permissions.
+func prepareOriginDirectory(parent *os.File, parts []string) error {
+ info, err := parent.Stat()
+ if err != nil {
+ return err
+ }
+
+ stat, ok := info.Sys().(*syscall.Stat_t)
+ if !ok || !info.IsDir() || info.Mode().Perm()&0o022 != 0 ||
+ (stat.Uid != 0 && stat.Uid != uint32(os.Geteuid())) {
+ return os.ErrPermission
+ }
+
+ if len(parts) == 0 {
+ if stat.Uid != uint32(os.Geteuid()) {
+ return os.ErrPermission
+ }
+
+ return nil
+ }
+
+ part := parts[0]
+ if part == "" || part == "." || part == ".." || strings.Contains(part, "/") {
+ return os.ErrInvalid
+ }
+
+ fd, err := unix.Openat(int(parent.Fd()), part, unix.O_RDONLY|unix.O_DIRECTORY|unix.O_NOFOLLOW|unix.O_CLOEXEC, 0)
+ if errors.Is(err, unix.ENOENT) {
+ if err := unix.Mkdirat(int(parent.Fd()), part, 0o755); err != nil && !errors.Is(err, unix.EEXIST) {
+ return err
+ }
+
+ fd, err = unix.Openat(int(parent.Fd()), part, unix.O_RDONLY|unix.O_DIRECTORY|unix.O_NOFOLLOW|unix.O_CLOEXEC, 0)
+ }
+
+ if err != nil {
+ return err
+ }
+
+ child := os.NewFile(uintptr(fd), part)
+ defer closeResource(child)
+
+ return prepareOriginDirectory(child, parts[1:])
+}
+
+func runSidecar(ctx context.Context, o options) error {
+ client, err := racersdk.NewClient(racersdk.ClientConfig{Cache: o.cache})
+ if err != nil {
+ return safeDiagnostic("could not configure Racer client", "config", err)
+ }
+ defer closeResource(client)
+
+ handler, err := object.NewSidecar(client, object.SidecarConfig{Namespace: o.namespace, Buckets: o.buckets, OnFailure: logSidecarFailure})
+ if err != nil {
+ return safeDiagnostic("could not configure sidecar", "config", err)
+ }
+
+ listener, err := listenSidecar(ctx, o.listen)
+ if err != nil {
+ return err
+ }
+
+ return serveSidecar(ctx, listener, handler, o.requestTimeout)
+}
+
+func listenSidecar(ctx context.Context, address string) (net.Listener, error) {
+ if err := validateListen(address); err != nil {
+ return nil, err
+ }
+
+ var lc net.ListenConfig
+
+ l, err := lc.Listen(ctx, "tcp", address)
+ if err != nil {
+ return nil, safeDiagnostic("could not bind sidecar listener", "listen", err)
+ }
+
+ return l, nil
+}
+
+func serveSidecar(ctx context.Context, listener net.Listener, handler http.Handler, requestTimeout time.Duration) error {
+ return serveSidecarWithGrace(ctx, listener, handler, requestTimeout, 5*time.Second)
+}
+
+func serveSidecarWithGrace(ctx context.Context, listener net.Listener, handler http.Handler, requestTimeout, shutdownGrace time.Duration) error {
+ lifetime, cancel := context.WithCancel(context.WithoutCancel(ctx))
+ defer cancel()
+
+ server := &http.Server{
+ ReadHeaderTimeout: min(5*time.Second, requestTimeout),
+ ReadTimeout: requestTimeout, WriteTimeout: requestTimeout + sidecarWriteGrace, IdleTimeout: 30 * time.Second,
+ MaxHeaderBytes: sidecarMaxHeaderBytes, ErrorLog: log.New(safeHTTPLogWriter{}, "", 0),
+ BaseContext: func(net.Listener) context.Context { return lifetime },
+ Handler: sidecarHTTPHandler(handler, requestTimeout),
+ }
+ defer closeResource(server)
+ defer closeResource(listener)
+
+ logSidecarStartup(listener.Addr())
+
+ finished := make(chan error, 1)
+
+ go func() { finished <- server.Serve(limitSidecarListener(listener, 128)) }()
+
+ select {
+ case err := <-finished:
+ if ctx.Err() != nil {
+ return nil
+ }
+
+ if err != nil {
+ return safeDiagnostic("sidecar server failed", "server", err)
+ }
+
+ return nil
+ case <-ctx.Done():
+ shutdownCtx, stop := context.WithTimeout(context.Background(), shutdownGrace)
+ defer stop()
+
+ if err := server.Shutdown(shutdownCtx); err != nil {
+ cancel()
+ closeResource(server)
+ }
+
+ <-finished
+
+ return nil
+ }
+}
+
+// sidecarHTTPHandler bounds each request by requestTimeout and keeps the
+// response flush bounded by requestTimeout plus sidecarWriteGrace.
+func sidecarHTTPHandler(handler http.Handler, requestTimeout time.Duration) http.Handler {
+ return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ defer diagnoseSidecarPanic()
+
+ // Object.WriteTo clears the write deadline when it returns, but
+ // net/http still flushes buffered bytes after the handler. Re-arm
+ // the server's write bound so a stalled reader cannot hold the
+ // connection past the request timeout plus grace.
+ writeDeadline := time.Now().Add(requestTimeout + sidecarWriteGrace)
+ defer rearmSidecarWriteDeadline(w, writeDeadline)
+
+ if sidecarRequestHeaderBytes(r) > sidecarMaxHeaderBytes {
+ object.WriteSidecarHeaderTooLarge(w, r)
+ return
+ }
+
+ requestCtx, done := context.WithTimeout(r.Context(), requestTimeout)
+ defer done()
+
+ handler.ServeHTTP(w, r.WithContext(requestCtx))
+ })
+}
+
+// rearmSidecarWriteDeadline restores the per-request write deadline after the
+// handler returns. Writers without deadline support are left unchanged.
+func rearmSidecarWriteDeadline(w http.ResponseWriter, deadline time.Time) {
+ _ = http.NewResponseController(w).SetWriteDeadline(deadline) //nolint:errcheck // ErrNotSupported or a closed connection leaves nothing to bound.
+}
+
+// net/http allows parser slop above MaxHeaderBytes. Match Racer's parsed-field
+// accounting to enforce the application bound before calling the sidecar.
+func sidecarRequestHeaderBytes(r *http.Request) int {
+ size := len(r.RequestURI) + len(r.Host)
+ for key, values := range r.Header {
+ for _, value := range values {
+ size += len(key) + len(value) + 4
+ }
+ }
+
+ return size
+}
diff --git a/cmd/racer-object/main_test.go b/cmd/racer-object/main_test.go
new file mode 100644
index 0000000000..c963fdd2fb
--- /dev/null
+++ b/cmd/racer-object/main_test.go
@@ -0,0 +1,695 @@
+// Copyright (c) Microsoft Corporation.
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "flag"
+ "io"
+ "math"
+ "net"
+ "net/http"
+ "os"
+ "path/filepath"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/aws/aws-sdk-go-v2/aws"
+ awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http"
+
+ "github.com/Azure/unbounded/internal/racer/object"
+ "github.com/Azure/unbounded/internal/version"
+ "github.com/Azure/unbounded/pkg/racersdk"
+ "github.com/Azure/unbounded/pkg/racersdk/racersdktest"
+)
+
+type failingOutput struct{}
+
+func (failingOutput) Write([]byte) (int, error) {
+ return 0, errors.New("private-output-error")
+}
+
+func TestHelpOutputFailure(t *testing.T) {
+ for _, args := range [][]string{{"--help"}, {"--version"}, {"origin", "--help"}, {"sidecar", "--help"}, {"origin", "--version"}, {"sidecar", "--version"}} {
+ err := run(context.Background(), args, failingOutput{})
+ if err == nil || strings.Contains(err.Error(), "private-output-error") {
+ t.Fatal("output failure was lost or leaked")
+ }
+ }
+}
+
+func TestOptionsDefaults(t *testing.T) {
+ for _, mode := range []string{"origin", "sidecar"} {
+ t.Run(mode, func(t *testing.T) {
+ o, err := parseOptions([]string{mode, "--namespace", "store"}, io.Discard)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if o.cache != "racer-object" || o.namespace != "store" || len(o.buckets) != 0 || o.logLevel != "info" {
+ t.Fatalf("unexpected shared defaults: %+v", o)
+ }
+
+ if mode == "origin" {
+ if o.requestTimeout != time.Minute || o.metadataTTL != 30*time.Second || !o.pathStyle || o.endpoint != "" || o.region != "" {
+ t.Fatalf("unexpected origin defaults: %+v", o)
+ }
+ } else if o.listen != "127.0.0.1:8080" || o.requestTimeout != 5*time.Minute {
+ t.Fatalf("unexpected sidecar defaults: %+v", o)
+ }
+ })
+ }
+
+ o, err := parseOptions([]string{"origin", "--namespace=store", "--bucket=one", "--bucket=two", "--cache=custom", "--metadata-ttl=0", "--path-style=false", "--region=us-west-2", "--endpoint=https://s3.example/", "--request-timeout=2s"}, io.Discard)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if strings.Join(o.buckets, ",") != "one,two" || o.cache != "custom" || o.metadataTTL != 0 || o.pathStyle || o.region != "us-west-2" || o.requestTimeout != 2*time.Second {
+ t.Fatalf("flags not applied: %+v", o)
+ }
+}
+
+func TestRequestTimeoutFlag(t *testing.T) {
+ for _, tt := range []struct {
+ mode string
+ value string
+ want time.Duration
+ wantErr string
+ }{
+ {mode: "origin", value: "1ns", want: time.Nanosecond},
+ {mode: "origin", value: "30s", want: 30 * time.Second},
+ {mode: "origin", value: "1m", want: time.Minute},
+ {mode: "origin", value: "61s", wantErr: "--request-timeout must not exceed 1m in origin mode"},
+ {mode: "origin", value: "2m", wantErr: "--request-timeout must not exceed 1m in origin mode"},
+ {mode: "sidecar", value: "5m", want: 5 * time.Minute},
+ {mode: "sidecar", value: "10m", want: 10 * time.Minute},
+ {mode: "sidecar", value: "1ns", want: time.Nanosecond},
+ {mode: "sidecar", value: (time.Duration(math.MaxInt64) - sidecarWriteGrace - time.Nanosecond).String(), want: time.Duration(math.MaxInt64) - sidecarWriteGrace - time.Nanosecond},
+ {mode: "sidecar", value: (time.Duration(math.MaxInt64) - sidecarWriteGrace).String(), want: time.Duration(math.MaxInt64) - sidecarWriteGrace},
+ {mode: "sidecar", value: (time.Duration(math.MaxInt64) - sidecarWriteGrace + time.Nanosecond).String(), wantErr: "--request-timeout is too large to allow sidecar write grace"},
+ {mode: "sidecar", value: time.Duration(math.MaxInt64).String(), wantErr: "--request-timeout is too large to allow sidecar write grace"},
+ {mode: "sidecar", value: "0", wantErr: "--request-timeout must be positive"},
+ {mode: "sidecar", value: "-1ns", wantErr: "--request-timeout must be positive"},
+ } {
+ t.Run(tt.mode+"/"+tt.value, func(t *testing.T) {
+ o, err := parseOptions([]string{tt.mode, "--namespace=store", "--request-timeout=" + tt.value}, io.Discard)
+ if tt.wantErr != "" {
+ if err == nil || err.Error() != tt.wantErr {
+ t.Fatalf("error = %v, want %q", err, tt.wantErr)
+ }
+
+ return
+ }
+
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if o.requestTimeout != tt.want {
+ t.Fatalf("request timeout = %v, want %v", o.requestTimeout, tt.want)
+ }
+ })
+ }
+}
+
+func TestCacheFlag(t *testing.T) {
+ for _, mode := range []string{"origin", "sidecar"} {
+ t.Run(mode, func(t *testing.T) {
+ for _, name := range []string{"custom", "a-b.c0", strings.Repeat("a", 63) + "." + strings.Repeat("b", 18)} {
+ o, err := parseOptions([]string{mode, "--namespace=store", "--cache=" + name}, io.Discard)
+ if err != nil || o.cache != name {
+ t.Fatalf("valid cache %q: options=%+v err=%v", name, o, err)
+ }
+ }
+
+ for _, name := range []string{"", "../bad", "Upper", strings.Repeat("a", 64), strings.Repeat("a", 63) + "." + strings.Repeat("b", 19)} {
+ _, err := parseOptions([]string{mode, "--namespace=store", "--cache=" + name}, io.Discard)
+ if err == nil || err.Error() != "invalid --cache" {
+ t.Fatalf("invalid cache %q: %v", name, err)
+ }
+ }
+
+ _, err := parseOptions([]string{mode, "--namespace=store", "--volume=custom", "--cache=custom"}, io.Discard)
+ if err == nil || err.Error() != "invalid flags (use --help)" {
+ t.Fatalf("unsupported flag was not rejected: %v", err)
+ }
+
+ var output bytes.Buffer
+
+ _, err = parseOptions([]string{mode, "--help"}, &output)
+ if !errors.Is(err, flag.ErrHelp) || !strings.Contains(output.String(), "-cache") || strings.Contains(output.String(), "-volume") {
+ t.Fatalf("unexpected help: %q, %v", output.String(), err)
+ }
+ })
+ }
+}
+
+func TestOptionsLogLevel(t *testing.T) {
+ for _, mode := range []string{"origin", "sidecar"} {
+ o, err := parseOptions([]string{mode, "--namespace=store", "--log-level=debug"}, io.Discard)
+ if err != nil || o.logLevel != "debug" {
+ t.Fatalf("%s: log level %q, err %v", mode, o.logLevel, err)
+ }
+ }
+}
+
+func TestOptionsErrors(t *testing.T) {
+ for _, args := range [][]string{
+ nil,
+ {"unknown"},
+ {"origin"},
+ {"sidecar"},
+ {"origin", "--namespace=has space"},
+ {"sidecar", "--namespace=store", "--cache=../bad"},
+ {"origin", "--namespace=store", "--bucket="},
+ {"sidecar", "--namespace=store", "--bucket=../bad"},
+ {"origin", "--namespace=store", "--request-timeout=0"},
+ {"sidecar", "--namespace=store", "--request-timeout=-1s"},
+ {"origin", "--namespace=store", "--metadata-ttl=-1s"},
+ {"origin", "--namespace=store", "--metadata-ttl=bad"},
+ {"origin", "--namespace=store", "--listen=127.0.0.1:0"},
+ {"sidecar", "--namespace=store", "--region=us-east-1"},
+ {"sidecar", "--namespace=store", "extra"},
+ {"sidecar", "--namespace=store", "--listen=0.0.0.0:8080"},
+ {"origin", "--namespace=store", "--log-level=warn"},
+ {"sidecar", "--namespace=store", "--log-level="},
+ } {
+ t.Run(strings.Join(args, " "), func(t *testing.T) {
+ if _, err := parseOptions(args, io.Discard); err == nil {
+ t.Fatal("accepted invalid flags")
+ }
+ })
+ }
+}
+
+func TestHelpVersionAndRedaction(t *testing.T) {
+ for _, args := range [][]string{{"--help"}, {"-h"}, {"origin", "--help"}, {"sidecar", "--help"}, {"--version"}, {"origin", "--version"}, {"sidecar", "--version"}} {
+ var output bytes.Buffer
+ if err := run(context.Background(), args, &output); err != nil {
+ t.Fatal(err)
+ }
+
+ if output.Len() == 0 {
+ t.Fatal("missing help or version output")
+ }
+
+ if args[len(args)-1] == "--version" && !strings.Contains(output.String(), version.String()) {
+ t.Fatal("version metadata missing")
+ }
+ }
+
+ for _, args := range [][]string{
+ {"origin", "--namespace=store", "--endpoint=https://private-token@upstream/"},
+ {"origin", "--namespace=store", "--request-timeout=private-token"},
+ {"sidecar", "--namespace=store", "--private-token"},
+ } {
+ var output bytes.Buffer
+
+ err := run(context.Background(), args, &output)
+ if err == nil || strings.Contains(err.Error()+output.String(), "private-token") {
+ t.Fatal("argument leaked or invalid argument accepted")
+ }
+ }
+
+ if _, err := parseOptions([]string{"origin", "--help"}, io.Discard); !errors.Is(err, flag.ErrHelp) {
+ t.Fatal("help not recognized")
+ }
+}
+
+func TestEndpointValidation(t *testing.T) {
+ for _, value := range []string{"", "https://s3.example", "http://localhost:9000/", "http://127.0.0.1:80", "https://[::1]:443"} {
+ if err := validateEndpoint(value); err != nil {
+ t.Errorf("rejected %q: %v", value, err)
+ }
+ }
+
+ for _, value := range []string{
+ "s3.example", "ftp://s3.example", "https://", "https://user:password@s3.example", "https://s3.example/prefix", "https://s3.example/%2f",
+ "https://s3.example?token=value", "https://s3.example?", "https://s3.example#fragment", "https://s3.example#", "https://s3.example:0", "https://s3.example:65536", "https://s3.example:abc", "https://s3.example:",
+ } {
+ if err := validateEndpoint(value); err == nil {
+ t.Errorf("accepted %q", value)
+ }
+ }
+}
+
+func TestLoopbackValidation(t *testing.T) {
+ for _, value := range []string{"127.0.0.1:0", "127.1.2.3:65535", "[::1]:8080", "[::ffff:127.0.0.1]:80"} {
+ if err := validateListen(value); err != nil {
+ t.Errorf("rejected %q: %v", value, err)
+ }
+ }
+
+ for _, value := range []string{"localhost:8080", ":8080", "0.0.0.0:8080", "[::]:8080", "192.0.2.1:8080", "127.0.0.1", "127.0.0.1:", "127.0.0.1:http", "127.0.0.1:-1", "127.0.0.1:+80", "127.0.0.1:65536", "[::1%lo]:80"} {
+ if _, err := listenSidecar(context.Background(), value); err == nil {
+ t.Errorf("accepted %q", value)
+ }
+ }
+}
+
+func TestAWSRegionAndCredentialChain(t *testing.T) {
+ dir := t.TempDir()
+ for key, value := range map[string]string{
+ "AWS_CONFIG_FILE": filepath.Join(dir, "config"), "AWS_SHARED_CREDENTIALS_FILE": filepath.Join(dir, "credentials"),
+ "AWS_PROFILE": "", "AWS_DEFAULT_PROFILE": "", "AWS_REGION": "", "AWS_DEFAULT_REGION": "", "AWS_EC2_METADATA_DISABLED": "true",
+ "AWS_ACCESS_KEY_ID": "test-access", "AWS_SECRET_ACCESS_KEY": "test-secret", "AWS_SESSION_TOKEN": "test-session",
+ } {
+ t.Setenv(key, value)
+ }
+
+ o := options{requestTimeout: time.Second}
+ if _, err := originAWSConfig(context.Background(), o); err == nil {
+ t.Fatal("accepted missing region")
+ }
+
+ t.Setenv("AWS_REGION", "us-east-1")
+
+ cfg, err := originAWSConfig(context.Background(), o)
+ if err != nil || cfg.Region != "us-east-1" {
+ t.Fatalf("environment region: %v", err)
+ }
+
+ creds, err := cfg.Credentials.Retrieve(context.Background())
+ if err != nil || creds.AccessKeyID != "test-access" || creds.SessionToken != "test-session" {
+ t.Fatal("default credential chain not used")
+ }
+
+ o.region = "us-west-2"
+
+ cfg, err = originAWSConfig(context.Background(), o)
+ if err != nil || cfg.Region != "us-west-2" {
+ t.Fatalf("explicit region: %v", err)
+ }
+
+ if cfg.HTTPClient.(*awshttp.BuildableClient).GetTimeout() != time.Second {
+ t.Fatal("upstream HTTP timeout missing")
+ }
+
+ o.region = ""
+
+ t.Setenv("AWS_REGION", "")
+ t.Setenv("AWS_DEFAULT_REGION", "eu-west-1")
+
+ cfg, err = originAWSConfig(context.Background(), o)
+ if err != nil || cfg.Region != "eu-west-1" {
+ t.Fatalf("default environment region: %v", err)
+ }
+
+ badConfig := filepath.Join(dir, "bad-config")
+ if err := os.WriteFile(badConfig, []byte("[default]\nmax_attempts = not-a-number\n"), 0o600); err != nil {
+ t.Fatal(err)
+ }
+
+ t.Setenv("AWS_CONFIG_FILE", badConfig)
+
+ if _, err := originAWSConfig(context.Background(), o); err == nil {
+ t.Fatal("accepted invalid AWS configuration")
+ }
+}
+
+func TestOriginAWSConfigRequiredChecksums(t *testing.T) {
+ dir := t.TempDir()
+ for key, value := range map[string]string{
+ "AWS_CONFIG_FILE": filepath.Join(dir, "config"), "AWS_SHARED_CREDENTIALS_FILE": filepath.Join(dir, "credentials"),
+ "AWS_PROFILE": "", "AWS_DEFAULT_PROFILE": "", "AWS_EC2_METADATA_DISABLED": "true",
+ "AWS_ACCESS_KEY_ID": "test-access", "AWS_SECRET_ACCESS_KEY": "test-secret", "AWS_SESSION_TOKEN": "test-session",
+ "AWS_REQUEST_CHECKSUM_CALCULATION": "when_supported", "AWS_RESPONSE_CHECKSUM_VALIDATION": "when_supported",
+ } {
+ t.Setenv(key, value)
+ }
+
+ o := options{region: "us-east-1", requestTimeout: time.Second}
+
+ cfg, err := originAWSConfig(context.Background(), o)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if cfg.RequestChecksumCalculation != aws.RequestChecksumCalculationWhenRequired {
+ t.Errorf("request checksum calculation = %v, want WhenRequired", cfg.RequestChecksumCalculation)
+ }
+
+ if cfg.ResponseChecksumValidation != aws.ResponseChecksumValidationWhenRequired {
+ t.Errorf("response checksum validation = %v, want WhenRequired", cfg.ResponseChecksumValidation)
+ }
+}
+
+func TestPrepareOriginDirectory(t *testing.T) {
+ for _, scenario := range []string{"create", "existing", "symlink", "writable", "file", "traversal"} {
+ t.Run(scenario, func(t *testing.T) {
+ dir := t.TempDir()
+ if err := os.Chmod(dir, 0o700); err != nil {
+ t.Fatal(err)
+ }
+
+ parent, err := os.Open(dir)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer parent.Close()
+
+ path := filepath.Join(dir, "cache")
+ parts := []string{"cache", "origin"}
+
+ switch scenario {
+ case "existing", "writable":
+ if err := os.Mkdir(path, 0o700); err != nil {
+ t.Fatal(err)
+ }
+
+ if scenario == "writable" {
+ if err := os.Chmod(path, 0o777); err != nil {
+ t.Fatal(err)
+ }
+ }
+ case "symlink":
+ if err := os.Symlink(t.TempDir(), path); err != nil {
+ t.Fatal(err)
+ }
+ case "file":
+ if err := os.WriteFile(path, nil, 0o600); err != nil {
+ t.Fatal(err)
+ }
+ case "traversal":
+ parts = []string{"..", "escape"}
+ }
+
+ err = prepareOriginDirectory(parent, parts)
+ if scenario != "create" && scenario != "existing" {
+ if err == nil {
+ t.Fatal("accepted unsafe directory")
+ }
+
+ return
+ }
+
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if err := prepareOriginDirectory(parent, parts); err != nil {
+ t.Fatal("not idempotent", err)
+ }
+
+ info, err := os.Stat(filepath.Join(path, "origin"))
+ if err != nil || !info.IsDir() || info.Mode().Perm()&0o022 != 0 {
+ t.Fatal("unsafe created directory")
+ }
+
+ if scenario == "existing" {
+ info, err := os.Stat(path)
+ if err != nil || info.Mode().Perm() != 0o700 {
+ t.Fatal("changed existing permissions")
+ }
+ }
+ })
+ }
+}
+
+func startTestSidecar(t *testing.T, handler http.Handler, timeout time.Duration) (string, context.CancelFunc, <-chan error) {
+ t.Helper()
+
+ ctx, cancel := context.WithCancel(context.Background())
+ t.Cleanup(cancel)
+
+ l, err := listenSidecar(ctx, "127.0.0.1:0")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ t.Cleanup(func() { _ = l.Close() })
+
+ done := make(chan error, 1)
+
+ go func() { done <- serveSidecar(ctx, l, handler, timeout) }()
+
+ return "http://" + l.Addr().String(), cancel, done
+}
+
+func waitSidecar(t *testing.T, done <-chan error) {
+ t.Helper()
+
+ select {
+ case err := <-done:
+ if err != nil {
+ t.Fatal(err)
+ }
+ case <-time.After(7 * time.Second):
+ t.Fatal("sidecar did not stop")
+ }
+}
+
+func TestSidecarLoopbackAndShutdown(t *testing.T) {
+ client, err := racersdk.NewClient(racersdk.ClientConfig{Cache: "test"})
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer client.Close()
+
+ handler, err := object.NewSidecar(client, object.SidecarConfig{Namespace: "store"})
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ address, cancel, done := startTestSidecar(t, handler, time.Second)
+ httpClient := &http.Client{Timeout: 2 * time.Second}
+
+ request, err := http.NewRequestWithContext(context.Background(), http.MethodPost, address+"/bucket/key", nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ response, err := httpClient.Do(request)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ response.Body.Close()
+
+ if response.StatusCode != http.StatusMethodNotAllowed {
+ t.Fatalf("status %d", response.StatusCode)
+ }
+
+ cancel()
+ waitSidecar(t, done)
+
+ conn, err := net.DialTimeout("tcp", strings.TrimPrefix(address, "http://"), time.Second)
+ if err == nil {
+ conn.Close()
+ t.Fatal("listener remained open")
+ }
+}
+
+func TestSidecarTimeoutResponse(t *testing.T) {
+ handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ <-r.Context().Done()
+ w.WriteHeader(http.StatusServiceUnavailable)
+ })
+
+ address, cancel, done := startTestSidecar(t, handler, 100*time.Millisecond)
+
+ defer func() {
+ cancel()
+ waitSidecar(t, done)
+ }()
+
+ client := &http.Client{Timeout: 3 * time.Second}
+ defer client.CloseIdleConnections()
+
+ response, err := client.Get(address + "/bucket/key")
+ if err != nil {
+ t.Fatalf("timeout response failed: %v", err)
+ }
+ defer response.Body.Close()
+
+ if response.StatusCode != http.StatusServiceUnavailable {
+ t.Fatalf("status = %d, want %d", response.StatusCode, http.StatusServiceUnavailable)
+ }
+}
+
+func TestSidecarRequestCancellation(t *testing.T) {
+ for _, shutdown := range []bool{false, true} {
+ t.Run(map[bool]string{false: "timeout", true: "shutdown"}[shutdown], func(t *testing.T) {
+ entered := make(chan struct{})
+ released := make(chan time.Time, 1)
+ handler := http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) {
+ close(entered)
+ <-r.Context().Done()
+
+ released <- time.Now()
+ })
+
+ requestTimeout := 100 * time.Millisecond
+ if shutdown {
+ requestTimeout = time.Minute
+ }
+
+ const grace = 250 * time.Millisecond
+
+ address, cancel, done, listenerClosed := startTestSidecarWithGrace(t, handler, requestTimeout, grace)
+ requestDone := make(chan struct{})
+
+ go func() {
+ defer close(requestDone)
+
+ client := &http.Client{Timeout: 3 * time.Second}
+
+ resp, err := client.Get(address)
+ if err == nil {
+ resp.Body.Close()
+ }
+ }()
+
+ select {
+ case <-entered:
+ case <-time.After(2 * time.Second):
+ t.Fatal("request did not start")
+ }
+
+ shutdownStarted := time.Now()
+
+ if shutdown {
+ cancel()
+ waitSidecarBarrier(t, listenerClosed, "listener did not close")
+ }
+
+ select {
+ case canceledAt := <-released:
+ if shutdown && canceledAt.Sub(shutdownStarted) < grace {
+ t.Fatal("request canceled before shutdown grace expired")
+ }
+ case <-time.After(2 * time.Second):
+ t.Fatal("request context not canceled")
+ }
+
+ cancel()
+ waitSidecar(t, done)
+
+ select {
+ case <-requestDone:
+ case <-time.After(4 * time.Second):
+ t.Fatal("HTTP request did not stop")
+ }
+ })
+ }
+}
+
+func TestSidecarListenerErrors(t *testing.T) {
+ l, err := listenSidecar(context.Background(), "127.0.0.1:0")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer l.Close()
+
+ if other, err := listenSidecar(context.Background(), l.Addr().String()); err == nil {
+ other.Close()
+ t.Fatal("bound occupied port")
+ }
+
+ l.Close()
+
+ if err := serveSidecar(context.Background(), l, http.NotFoundHandler(), time.Second); err == nil {
+ t.Fatal("lost listener failure")
+ }
+
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel()
+
+ if err := run(ctx, []string{"sidecar", "--namespace=store"}, io.Discard); err != nil {
+ t.Fatal(err)
+ }
+}
+
+type closeSignalReader struct {
+ *io.PipeReader
+ closed chan struct{}
+ closedAt time.Time
+ once sync.Once
+}
+
+func (r *closeSignalReader) Close() error {
+ err := r.PipeReader.Close()
+ r.once.Do(func() {
+ r.closedAt = time.Now()
+ close(r.closed)
+ })
+
+ return err
+}
+
+func TestSidecarShutdownReleasesSDKStream(t *testing.T) {
+ metadata := racersdk.Metadata{Size: 8, ETag: `"version"`, ExpiresAt: time.Now().Add(time.Minute).Truncate(time.Millisecond)}
+ reader, writer := io.Pipe()
+
+ body := &closeSignalReader{PipeReader: reader, closed: make(chan struct{})}
+ defer reader.Close()
+ defer writer.Close()
+
+ opened := make(chan struct{})
+
+ client := racersdktest.NewClient(t, func(_ context.Context, request racersdk.OriginRequest) (racersdk.Metadata, io.ReadCloser, error) {
+ if request.Head {
+ return metadata, nil, nil
+ }
+
+ close(opened)
+
+ return metadata, body, nil
+ })
+
+ handler, err := object.NewSidecar(client, object.SidecarConfig{Namespace: "store"})
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ const grace = 250 * time.Millisecond
+
+ address, cancel, done, listenerClosed := startTestSidecarWithGrace(t, handler, time.Minute, grace)
+ requestDone := make(chan struct{})
+
+ go func() {
+ defer close(requestDone)
+
+ httpClient := &http.Client{Timeout: 5 * time.Second}
+
+ response, err := httpClient.Get(address + "/bucket/key")
+ if err == nil {
+ _, _ = io.Copy(io.Discard, response.Body)
+ response.Body.Close()
+ }
+ }()
+
+ select {
+ case <-opened:
+ case <-time.After(3 * time.Second):
+ t.Fatal("SDK stream not opened")
+ }
+
+ shutdownStarted := time.Now()
+
+ cancel()
+ waitSidecarBarrier(t, listenerClosed, "listener did not close")
+
+ select {
+ case <-body.closed:
+ if body.closedAt.Sub(shutdownStarted) < grace {
+ t.Fatal("SDK stream released before shutdown grace expired")
+ }
+ case <-time.After(3 * time.Second):
+ t.Fatal("origin stream body not closed")
+ }
+
+ waitSidecar(t, done)
+
+ select {
+ case <-requestDone:
+ case <-time.After(6 * time.Second):
+ t.Fatal("stream request not released")
+ }
+}
diff --git a/cmd/racer-object/packaging_test.go b/cmd/racer-object/packaging_test.go
new file mode 100644
index 0000000000..a9019358c5
--- /dev/null
+++ b/cmd/racer-object/packaging_test.go
@@ -0,0 +1,105 @@
+// Copyright (c) Microsoft Corporation.
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+ "os"
+ "os/exec"
+ "path/filepath"
+ "regexp"
+ "strings"
+ "testing"
+)
+
+func TestContainerBuilderImage(t *testing.T) {
+ builder := regexp.MustCompile(`(?m)^FROM --platform=\$BUILDPLATFORM (docker\.io/library/golang:\S+) AS builder$`)
+ images := make(map[string]string)
+
+ for _, name := range []string{"gantry", "racer-object"} {
+ path := filepath.Join("..", "..", "images", name, "Containerfile")
+
+ data, err := os.ReadFile(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ match := builder.FindSubmatch(data)
+ if len(match) != 2 {
+ t.Fatalf("%s: missing Go builder image", path)
+ }
+
+ images[name] = string(match[1])
+ }
+
+ if images["racer-object"] != images["gantry"] {
+ t.Fatalf("racer-object builder = %q, want %q to match gantry", images["racer-object"], images["gantry"])
+ }
+}
+
+func TestLocalImageTrivyDryRun(t *testing.T) {
+ for _, trivy := range []string{"", " ", "1", "0"} {
+ t.Run("TRIVY="+trivy, func(t *testing.T) {
+ cmd := exec.Command("timeout", "--signal=TERM", "--kill-after=10s", "30s",
+ "make", "--no-print-directory", "-n", "image-racer-object-local",
+ "VERSION=test", "GIT_COMMIT=test", "BUILD_TIME=test", "CONTAINER_ENGINE=podman",
+ "RACER_OBJECT_IMAGE=example.test/racer-object:custom", "TRIVY="+trivy,
+ "TRIVY_IMAGE=aquasec/trivy:test", "TRIVY_SEVERITY=CRITICAL,HIGH", "TRIVY_EXIT_CODE=1")
+ cmd.Dir = filepath.Join("..", "..")
+
+ output, err := cmd.CombinedOutput()
+ if err != nil {
+ t.Fatalf("make dry run: %v\n%s", err, output)
+ }
+
+ text := string(output)
+ build := strings.Index(text, "podman build")
+
+ save := strings.Index(text, "podman image save -o $tmp example.test/racer-object:custom")
+ if build < 0 || !strings.Contains(text, "-t example.test/racer-object:custom") {
+ t.Fatalf("missing image build or tag:\n%s", text)
+ }
+
+ if strings.TrimSpace(trivy) == "" {
+ if strings.Contains(text, "image save") || strings.Contains(text, "podman run") {
+ t.Fatalf("scan enabled for blank TRIVY:\n%s", text)
+ }
+ } else if save <= build || !strings.Contains(text, "podman run --rm") ||
+ !strings.Contains(text, "aquasec/trivy:test image --severity CRITICAL,HIGH --exit-code 1") {
+ t.Fatalf("missing scan after build:\n%s", text)
+ }
+ })
+ }
+}
+
+func TestLocalImageTrivyFailure(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ engine string
+ scan string
+ wantError bool
+ wantScan bool
+ }{
+ {name: "success", engine: "true", scan: "true", wantScan: true},
+ {name: "scan failure", engine: "true", scan: "false", wantError: true, wantScan: true},
+ {name: "build failure", engine: "false", scan: "true", wantError: true},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ cmd := exec.Command("timeout", "--signal=TERM", "--kill-after=10s", "30s",
+ "make", "--no-print-directory", "-s", "image-racer-object-local",
+ "VERSION=test", "GIT_COMMIT=test", "BUILD_TIME=test", "TRIVY=1",
+ "RACER_OBJECT_IMAGE=example.test/racer-object:custom", "CONTAINER_ENGINE="+tc.engine,
+ "TRIVY_SCAN_CMD=printf 'scanned %s\\n' '$(1)'; "+tc.scan)
+ cmd.Dir = filepath.Join("..", "..")
+
+ output, err := cmd.CombinedOutput()
+ if (err != nil) != tc.wantError {
+ t.Fatalf("make error = %v, want error %t\n%s", err, tc.wantError, output)
+ }
+
+ if scanned := strings.Contains(string(output), "scanned example.test/racer-object:custom"); scanned != tc.wantScan {
+ t.Fatalf("scan ran = %t, want %t\n%s", scanned, tc.wantScan, output)
+ }
+ })
+ }
+}
diff --git a/cmd/racer-object/shutdown_test.go b/cmd/racer-object/shutdown_test.go
new file mode 100644
index 0000000000..939fa2aa38
--- /dev/null
+++ b/cmd/racer-object/shutdown_test.go
@@ -0,0 +1,196 @@
+// Copyright (c) Microsoft Corporation.
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+ "context"
+ "io"
+ "net"
+ "net/http"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+)
+
+type sidecarShutdownListener struct {
+ net.Listener
+ closed chan struct{}
+ once sync.Once
+}
+
+func (l *sidecarShutdownListener) Close() error {
+ err := l.Listener.Close()
+ l.once.Do(func() { close(l.closed) })
+
+ return err
+}
+
+func startTestSidecarWithGrace(t *testing.T, handler http.Handler, requestTimeout, grace time.Duration) (string, context.CancelFunc, <-chan error, <-chan struct{}) {
+ t.Helper()
+
+ ctx, cancel := context.WithCancel(context.Background())
+ t.Cleanup(cancel)
+
+ l, err := listenSidecar(ctx, "127.0.0.1:0")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ t.Cleanup(func() { _ = l.Close() })
+
+ listener := &sidecarShutdownListener{Listener: l, closed: make(chan struct{})}
+
+ done := make(chan error, 1)
+ go func() { done <- serveSidecarWithGrace(ctx, listener, handler, requestTimeout, grace) }()
+
+ return "http://" + l.Addr().String(), cancel, done, listener.closed
+}
+
+func waitSidecarBarrier(t *testing.T, barrier <-chan struct{}, failure string) {
+ t.Helper()
+
+ select {
+ case <-barrier:
+ case <-time.After(2 * time.Second):
+ t.Fatal(failure)
+ }
+}
+
+func TestSidecarShutdownDrainsActiveRequest(t *testing.T) {
+ entered := make(chan struct{})
+ release := make(chan struct{})
+
+ var releaseOnce sync.Once
+
+ releaseRequest := func() { releaseOnce.Do(func() { close(release) }) }
+ t.Cleanup(releaseRequest)
+
+ requestContexts := make(chan context.Context, 1)
+
+ handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ requestContexts <- r.Context()
+
+ close(entered)
+
+ select {
+ case <-release:
+ _, _ = io.WriteString(w, "finished after signal")
+ case <-r.Context().Done():
+ w.WriteHeader(http.StatusServiceUnavailable)
+ }
+ })
+
+ address, cancel, done, listenerClosed := startTestSidecarWithGrace(t, handler, time.Minute, 5*time.Second)
+
+ client := &http.Client{Timeout: 7 * time.Second}
+ defer client.CloseIdleConnections()
+
+ responseDone := make(chan struct{})
+ go func() {
+ defer close(responseDone)
+
+ response, err := client.Get(address)
+ if err != nil {
+ t.Error(err)
+ return
+ }
+ defer response.Body.Close()
+
+ body, err := io.ReadAll(response.Body)
+ if err != nil || response.StatusCode != http.StatusOK || string(body) != "finished after signal" {
+ t.Errorf("response after signal: status=%d body=%q err=%v", response.StatusCode, body, err)
+ }
+ }()
+
+ waitSidecarBarrier(t, entered, "request did not start")
+
+ requestCtx := <-requestContexts
+
+ cancel()
+ waitSidecarBarrier(t, listenerClosed, "shutdown did not close listener")
+
+ if err := requestCtx.Err(); err != nil {
+ t.Fatalf("active request canceled before draining: %v", err)
+ }
+
+ conn, err := net.DialTimeout("tcp", strings.TrimPrefix(address, "http://"), time.Second)
+ if err == nil {
+ conn.Close()
+ t.Fatal("shutdown accepted a new connection")
+ }
+
+ select {
+ case err := <-done:
+ t.Fatalf("server returned before active request drained: %v", err)
+ default:
+ }
+
+ releaseRequest()
+ waitSidecarBarrier(t, responseDone, "active request did not finish after signal")
+ waitSidecar(t, done)
+}
+
+func TestSidecarShutdownForceClosesStuckRequest(t *testing.T) {
+ entered := make(chan struct{})
+ release := make(chan struct{})
+ handlerDone := make(chan struct{})
+
+ requestContexts := make(chan context.Context, 1)
+
+ handler := http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) {
+ defer close(handlerDone)
+
+ requestContexts <- r.Context()
+
+ close(entered)
+ <-release
+ })
+
+ t.Cleanup(func() { close(release) })
+
+ const grace = 250 * time.Millisecond
+
+ address, cancel, done, listenerClosed := startTestSidecarWithGrace(t, handler, time.Minute, grace)
+
+ requestDone := make(chan struct{})
+ go func() {
+ defer close(requestDone)
+
+ client := &http.Client{Timeout: 3 * time.Second}
+ defer client.CloseIdleConnections()
+
+ response, err := client.Get(address)
+ if err == nil {
+ response.Body.Close()
+ t.Error("stuck request unexpectedly completed")
+ }
+ }()
+
+ waitSidecarBarrier(t, entered, "request did not start")
+
+ requestCtx := <-requestContexts
+
+ shutdownStarted := time.Now()
+
+ cancel()
+ waitSidecarBarrier(t, listenerClosed, "shutdown did not close listener")
+ waitSidecar(t, done)
+
+ if time.Since(shutdownStarted) < grace {
+ t.Fatal("server stopped before shutdown grace expired")
+ }
+
+ if requestCtx.Err() != context.Canceled {
+ t.Fatalf("request context after grace: %v", requestCtx.Err())
+ }
+
+ waitSidecarBarrier(t, requestDone, "force-close did not release HTTP client")
+
+ select {
+ case <-handlerDone:
+ t.Fatal("handler finished without being released")
+ default:
+ }
+}
diff --git a/cmd/racer-object/transport.go b/cmd/racer-object/transport.go
new file mode 100644
index 0000000000..021b614d52
--- /dev/null
+++ b/cmd/racer-object/transport.go
@@ -0,0 +1,69 @@
+// Copyright (c) Microsoft Corporation.
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+ "context"
+ "net"
+ "sync"
+)
+
+// BuildableClient exposes only transport copies, not its live pool. Track the
+// connections opened by its final transport so shutdown closes the real pool.
+type originConnectionPool struct {
+ mu sync.Mutex
+ connections map[*originConnection]struct{}
+ closed bool
+}
+
+func (p *originConnectionPool) dial(next func(context.Context, string, string) (net.Conn, error)) func(context.Context, string, string) (net.Conn, error) {
+ return func(ctx context.Context, network, address string) (net.Conn, error) {
+ conn, err := next(ctx, network, address)
+ if err != nil {
+ return nil, err
+ }
+
+ p.mu.Lock()
+ defer p.mu.Unlock()
+
+ if p.closed {
+ closeResource(conn)
+ return nil, net.ErrClosed
+ }
+
+ if p.connections == nil {
+ p.connections = make(map[*originConnection]struct{})
+ }
+
+ tracked := &originConnection{Conn: conn, pool: p}
+ p.connections[tracked] = struct{}{}
+
+ return tracked, nil
+ }
+}
+
+func (p *originConnectionPool) close() {
+ p.mu.Lock()
+ p.closed = true
+ connections := p.connections
+ p.connections = nil
+ p.mu.Unlock()
+
+ for conn := range connections {
+ closeResource(conn)
+ }
+}
+
+type originConnection struct {
+ net.Conn
+ pool *originConnectionPool
+}
+
+func (c *originConnection) Close() error {
+ c.pool.mu.Lock()
+ delete(c.pool.connections, c)
+ c.pool.mu.Unlock()
+
+ return c.Conn.Close()
+}
diff --git a/cmd/racer-object/transport_test.go b/cmd/racer-object/transport_test.go
new file mode 100644
index 0000000000..d0e4261af3
--- /dev/null
+++ b/cmd/racer-object/transport_test.go
@@ -0,0 +1,573 @@
+// Copyright (c) Microsoft Corporation.
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+ "context"
+ "encoding/pem"
+ "errors"
+ "io"
+ "net"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "path/filepath"
+ "strconv"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/aws/aws-sdk-go-v2/aws"
+ awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http"
+ "github.com/aws/aws-sdk-go-v2/service/s3"
+)
+
+func testOriginHTTPClient(t *testing.T) *awshttp.BuildableClient {
+ t.Helper()
+
+ isolateOriginAWSConfig(t)
+
+ return configuredOriginHTTPClient(t)
+}
+
+func configuredOriginHTTPClient(t *testing.T) *awshttp.BuildableClient {
+ t.Helper()
+
+ cfg, err := originAWSConfig(t.Context(), options{region: "us-east-1", requestTimeout: time.Minute})
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ s3client, err := newOriginS3Client(cfg, options{pathStyle: true})
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ t.Cleanup(func() { closeResource(s3client) })
+
+ client, ok := s3client.Options().HTTPClient.(*awshttp.BuildableClient)
+ if !ok {
+ t.Fatalf("HTTP client type = %T", cfg.HTTPClient)
+ }
+
+ return client
+}
+
+func TestOriginHTTPTransport(t *testing.T) {
+ client := testOriginHTTPClient(t)
+
+ transport := client.GetTransport()
+
+ defaults := http.DefaultTransport.(*http.Transport)
+ if transport == defaults {
+ t.Fatal("origin shares the default transport")
+ }
+
+ if transport.MaxIdleConnsPerHost < 68 || transport.MaxIdleConns < transport.MaxIdleConnsPerHost {
+ t.Fatalf("idle pool cannot retain origin concurrency: per-host=%d total=%d", transport.MaxIdleConnsPerHost, transport.MaxIdleConns)
+ }
+
+ if client.GetTimeout() != time.Minute || transport.DialContext == nil || transport.Proxy == nil ||
+ transport.TLSHandshakeTimeout != defaults.TLSHandshakeTimeout || transport.IdleConnTimeout != defaults.IdleConnTimeout ||
+ transport.ExpectContinueTimeout != defaults.ExpectContinueTimeout || transport.ForceAttemptHTTP2 != defaults.ForceAttemptHTTP2 {
+ t.Fatal("request timeout or default transport behavior changed")
+ }
+
+ other := testOriginHTTPClient(t).GetTransport()
+ if other == transport {
+ t.Fatal("origin clients share a transport")
+ }
+
+ defaultLimit := defaults.MaxIdleConnsPerHost
+
+ transport.MaxIdleConnsPerHost++
+ if defaults.MaxIdleConnsPerHost != defaultLimit || other.MaxIdleConnsPerHost == transport.MaxIdleConnsPerHost {
+ t.Fatal("transport configuration is not isolated")
+ }
+}
+
+func TestOriginHTTPTransportInvalidDefault(t *testing.T) {
+ defaults := http.DefaultTransport
+
+ t.Cleanup(func() { http.DefaultTransport = defaults })
+
+ http.DefaultTransport = nil
+
+ isolateOriginAWSConfig(t)
+
+ if _, err := originAWSConfig(t.Context(), options{region: "us-east-1", requestTimeout: time.Minute}); err != nil {
+ t.Fatalf("SDK client must not depend on http.DefaultTransport: %v", err)
+ }
+}
+
+func TestOriginHTTPTransportCustomCA(t *testing.T) {
+ server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.WriteHeader(http.StatusNoContent)
+ }))
+ defer server.Close()
+
+ for _, source := range []string{"env", "shared"} {
+ t.Run(source, func(t *testing.T) {
+ dir := isolateOriginAWSConfig(t)
+
+ bundle := filepath.Join(dir, "ca.pem")
+ if err := os.WriteFile(bundle, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: server.Certificate().Raw}), 0o600); err != nil {
+ t.Fatal(err)
+ }
+
+ if source == "env" {
+ t.Setenv("AWS_CA_BUNDLE", bundle)
+ } else if err := os.WriteFile(filepath.Join(dir, "config"), []byte("[default]\nca_bundle = "+bundle+"\n"), 0o600); err != nil {
+ t.Fatal(err)
+ }
+
+ client := configuredOriginHTTPClient(t)
+
+ request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, server.URL, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ response, err := client.Do(request)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ closeResource(response.Body)
+
+ if response.StatusCode != http.StatusNoContent {
+ t.Fatalf("status = %d", response.StatusCode)
+ }
+ })
+ }
+}
+
+func TestOriginHTTPTransportInvalidCA(t *testing.T) {
+ dir := isolateOriginAWSConfig(t)
+ path := filepath.Join(dir, "private-bundle.pem")
+ t.Setenv("AWS_CA_BUNDLE", path)
+
+ for _, contents := range []string{"", "not a certificate"} {
+ if contents != "" {
+ if err := os.WriteFile(path, []byte(contents), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ _, err := originAWSConfig(t.Context(), options{region: "us-east-1", requestTimeout: time.Minute})
+
+ category := "config"
+ if contents == "" {
+ category = "notfound"
+
+ if !errors.Is(err, os.ErrNotExist) {
+ t.Fatalf("missing CA file cause lost: %v", err)
+ }
+ }
+
+ if err == nil || !strings.Contains(err.Error(), "category="+category) || errors.Unwrap(err) == nil ||
+ !strings.HasPrefix(err.Error(), "could not load AWS configuration") || strings.Contains(err.Error(), path) || strings.Contains(err.Error(), "private-bundle") {
+ t.Fatalf("error = %v, want redacted CA configuration error", err)
+ }
+ }
+}
+
+func TestOriginHTTPRedirects(t *testing.T) {
+ client := testOriginHTTPClient(t)
+ for _, code := range []int{301, 302, 303, 307, 308} {
+ t.Run(strconv.Itoa(code), func(t *testing.T) {
+ var calls atomic.Int32
+
+ target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ calls.Add(1)
+
+ if r.Header.Get("X-Amz-Security-Token") != "" {
+ t.Error("session token forwarded to another host")
+ }
+
+ if r.Method != http.MethodHead {
+ t.Errorf("redirect changed method to %s", r.Method)
+ }
+
+ w.WriteHeader(http.StatusNoContent)
+ }))
+ defer target.Close()
+
+ source := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.Header.Get("X-Amz-Security-Token") != "test-session" {
+ t.Error("session token missing from initial request")
+ }
+
+ http.Redirect(w, r, target.URL, code)
+ }))
+ defer source.Close()
+
+ request, err := http.NewRequestWithContext(t.Context(), http.MethodHead, source.URL, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ request.Header.Set("X-Amz-Security-Token", "test-session")
+
+ response, err := client.Do(request)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ closeResource(response.Body)
+
+ if code == 307 || code == 308 {
+ if response.StatusCode != http.StatusNoContent || calls.Load() != 1 {
+ t.Fatalf("redirect not followed: status=%d calls=%d", response.StatusCode, calls.Load())
+ }
+ } else if response.StatusCode != code || calls.Load() != 0 {
+ t.Fatalf("unsafe redirect followed: status=%d calls=%d", response.StatusCode, calls.Load())
+ }
+ })
+ }
+}
+
+func TestOriginHTTPPoolClose(t *testing.T) {
+ isolateOriginAWSConfig(t)
+
+ cfg, err := originAWSConfig(t.Context(), options{region: "us-east-1", requestTimeout: time.Minute})
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ client, err := newOriginS3Client(cfg, options{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer closeResource(client)
+
+ closed := make(chan struct{}, 1)
+ server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.WriteHeader(http.StatusNoContent)
+ }))
+ server.Config.ConnState = func(_ net.Conn, state http.ConnState) {
+ if state == http.StateClosed {
+ closed <- struct{}{}
+ }
+ }
+
+ server.Start()
+ defer server.Close()
+
+ request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, server.URL, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ response, err := client.Options().HTTPClient.Do(request)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ closeResource(response.Body)
+
+ select {
+ case <-closed:
+ t.Fatal("connection closed before pool cleanup")
+ default:
+ }
+
+ closeResource(client)
+
+ select {
+ case <-closed:
+ case <-time.After(5 * time.Second):
+ t.Fatal("cleanup did not close the active transport's idle connection")
+ }
+}
+
+func TestOriginHTTPPoolClosePreservesSharedCredentials(t *testing.T) {
+ isolateOriginAWSConfig(t)
+
+ for _, key := range []string{
+ "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN",
+ "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", "AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE",
+ "AWS_WEB_IDENTITY_TOKEN_FILE", "AWS_ROLE_ARN",
+ } {
+ t.Setenv(key, "")
+ }
+
+ t.Setenv("AWS_CONTAINER_AUTHORIZATION_TOKEN", "test-container-token")
+
+ var credentialRequests atomic.Int32
+
+ credentials := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ credentialRequests.Add(1)
+
+ if r.Header.Get("Authorization") != "test-container-token" {
+ t.Error("credential endpoint authorization missing")
+ }
+
+ w.Header().Set("Content-Type", "application/json")
+ _, _ = io.WriteString(w, `{"AccessKeyId":"test-access","SecretAccessKey":"test-secret","Token":"test-session"}`)
+ }))
+ defer credentials.Close()
+
+ t.Setenv("AWS_CONTAINER_CREDENTIALS_FULL_URI", credentials.URL)
+
+ upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.Header.Get("X-Amz-Security-Token") != "test-session" || !strings.Contains(r.Header.Get("Authorization"), "Credential=test-access/") {
+ t.Error("S3 request not signed with endpoint credentials")
+ }
+
+ w.Header().Set("Content-Length", "0")
+ w.WriteHeader(http.StatusOK)
+ }))
+ defer upstream.Close()
+
+ o := options{region: "us-east-1", requestTimeout: time.Second, endpoint: upstream.URL, pathStyle: true}
+
+ cfg, err := originAWSConfig(t.Context(), o)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ first, err := newOriginS3Client(cfg, o)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer closeResource(first)
+
+ second, err := newOriginS3Client(cfg, o)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer closeResource(second)
+
+ request := &s3.HeadObjectInput{Bucket: aws.String("bucket"), Key: aws.String("key")}
+ if _, err := first.HeadObject(t.Context(), request); err != nil {
+ t.Fatal(err)
+ }
+
+ closeResource(first)
+
+ cache, ok := cfg.Credentials.(*aws.CredentialsCache)
+ if !ok {
+ t.Fatalf("credentials type = %T, want SDK cache", cfg.Credentials)
+ }
+
+ cache.Invalidate()
+
+ if _, err := second.HeadObject(t.Context(), request); err != nil {
+ t.Fatal("closing one S3 client broke shared credentials:", err)
+ }
+
+ if got := credentialRequests.Load(); got != 2 {
+ t.Fatalf("credential requests = %d, want initial retrieval and refresh", got)
+ }
+}
+
+func TestOriginConnectionPoolDial(t *testing.T) {
+ t.Run("failure", func(t *testing.T) {
+ var pool originConnectionPool
+
+ dial := pool.dial(func(context.Context, string, string) (net.Conn, error) {
+ return nil, net.ErrClosed
+ })
+ if _, err := dial(t.Context(), "tcp", "unused"); !errors.Is(err, net.ErrClosed) {
+ t.Fatalf("dial error = %v", err)
+ }
+
+ pool.close()
+ })
+ t.Run("close-during-dial", func(t *testing.T) {
+ var pool originConnectionPool
+
+ local, remote := net.Pipe()
+ defer closeResource(remote)
+
+ dial := pool.dial(func(context.Context, string, string) (net.Conn, error) {
+ pool.close()
+ return local, nil
+ })
+ if _, err := dial(t.Context(), "tcp", "unused"); !errors.Is(err, net.ErrClosed) {
+ t.Fatalf("dial after close = %v", err)
+ }
+
+ if _, err := remote.Read(make([]byte, 1)); !errors.Is(err, io.EOF) {
+ t.Fatalf("late connection not closed: %v", err)
+ }
+ })
+ t.Run("forget-closed-connection", func(t *testing.T) {
+ var pool originConnectionPool
+
+ local, remote := net.Pipe()
+ defer closeResource(remote)
+
+ dial := pool.dial(func(context.Context, string, string) (net.Conn, error) {
+ return local, nil
+ })
+
+ conn, err := dial(t.Context(), "tcp", "unused")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if len(pool.connections) != 1 {
+ t.Fatal("connection not tracked")
+ }
+
+ closeResource(conn)
+
+ if len(pool.connections) != 0 {
+ t.Fatal("closed connection retained")
+ }
+
+ pool.close()
+ pool.close()
+ })
+}
+
+func TestOriginHTTPSameHostRedirect(t *testing.T) {
+ client := testOriginHTTPClient(t)
+
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path == "/" {
+ http.Redirect(w, r, "/target", http.StatusTemporaryRedirect)
+ return
+ }
+
+ if r.Header.Get("X-Amz-Security-Token") != "test-session" {
+ t.Error("same-host redirect lost session token")
+ }
+
+ w.WriteHeader(http.StatusNoContent)
+ }))
+ defer server.Close()
+
+ request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, server.URL, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ request.Header.Set("X-Amz-Security-Token", "test-session")
+
+ response, err := client.Do(request)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ closeResource(response.Body)
+
+ if response.StatusCode != http.StatusNoContent {
+ t.Fatalf("status = %d", response.StatusCode)
+ }
+}
+
+func TestOriginHTTPRequestTimeout(t *testing.T) {
+ client := testOriginHTTPClient(t).WithTimeout(20 * time.Millisecond)
+
+ server := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) {
+ <-r.Context().Done()
+ }))
+ defer server.Close()
+
+ request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, server.URL, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ response, err := client.Do(request)
+ if response != nil {
+ closeResource(response.Body)
+ }
+
+ if !errors.Is(err, context.DeadlineExceeded) {
+ t.Fatalf("request timeout = %v", err)
+ }
+}
+
+func TestOriginHTTPConnectionReuse(t *testing.T) {
+ const concurrency = 64
+
+ client := testOriginHTTPClient(t)
+ ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second)
+
+ var connections atomic.Int32
+
+ entered := make(chan struct{}, concurrency)
+ release := make(chan struct{}, concurrency)
+ server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ entered <- struct{}{}
+
+ select {
+ case <-release:
+ case <-r.Context().Done():
+ return
+ }
+
+ w.Header().Set("Content-Length", "1")
+ _, _ = io.WriteString(w, "x")
+ }))
+ server.Config.ConnState = func(_ net.Conn, state http.ConnState) {
+ if state == http.StateNew {
+ connections.Add(1)
+ }
+ }
+
+ server.Start()
+ defer server.Close()
+ defer cancel()
+
+ for burst := range 2 {
+ var workers sync.WaitGroup
+
+ results := make(chan error, concurrency)
+ for range concurrency {
+ workers.Go(func() {
+ request, err := http.NewRequestWithContext(ctx, http.MethodGet, server.URL, nil)
+ if err != nil {
+ results <- err
+ return
+ }
+
+ response, err := client.Do(request)
+ if err != nil {
+ results <- err
+ return
+ }
+
+ _, err = io.Copy(io.Discard, response.Body)
+
+ closeErr := response.Body.Close()
+ if err == nil {
+ err = closeErr
+ }
+
+ results <- err
+ })
+ }
+
+ for range concurrency {
+ select {
+ case <-entered:
+ case <-ctx.Done():
+ t.Fatal("concurrent requests did not all arrive:", ctx.Err())
+ }
+ }
+
+ for range concurrency {
+ release <- struct{}{}
+ }
+
+ workers.Wait()
+
+ for range concurrency {
+ if err := <-results; err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ if got := connections.Load(); got != concurrency {
+ t.Fatalf("burst %d opened %d total connections, want %d with keep-alive reuse", burst+1, got, concurrency)
+ }
+ }
+}
diff --git a/cmd/unbounded-net-controller/node_auth_informers_test.go b/cmd/unbounded-net-controller/node_auth_informers_test.go
index 190f592c10..3b0494eec1 100644
--- a/cmd/unbounded-net-controller/node_auth_informers_test.go
+++ b/cmd/unbounded-net-controller/node_auth_informers_test.go
@@ -12,12 +12,14 @@ import (
"os"
"path/filepath"
"strings"
+ "sync/atomic"
"testing"
"time"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
+ "k8s.io/apimachinery/pkg/watch"
k8sfake "k8s.io/client-go/kubernetes/fake"
k8stesting "k8s.io/client-go/testing"
"k8s.io/client-go/tools/cache"
@@ -61,12 +63,46 @@ func waitNodeAuthCondition(t *testing.T, condition func() bool) {
}
}
+func nodeAuthTestWatchBarrier(client *k8sfake.Clientset) func() bool {
+ reactors := append([]k8stesting.WatchReactor(nil), client.WatchReactionChain...)
+ registered := make([]*atomic.Bool, 0, 2)
+
+ for _, resource := range []string{"pods", "serviceaccounts"} {
+ ready := &atomic.Bool{}
+ registered = append(registered, ready)
+
+ client.PrependWatchReactor(resource, func(action k8stesting.Action) (bool, watch.Interface, error) {
+ for _, reactor := range reactors {
+ if !reactor.Handles(action) {
+ continue
+ }
+
+ handled, watcher, err := reactor.React(action)
+ if handled {
+ if err == nil && watcher != nil {
+ ready.Store(true)
+ }
+
+ return handled, watcher, err
+ }
+ }
+
+ return false, nil, nil
+ })
+ }
+
+ return func() bool {
+ return registered[0].Load() && registered[1].Load()
+ }
+}
+
func startNodeAuthTestCaches(t *testing.T, client *k8sfake.Clientset) (*nodeAuthInformers, context.CancelFunc) {
t.Helper()
ctx, cancel := context.WithCancel(t.Context())
t.Cleanup(cancel)
+ watchesReady := nodeAuthTestWatchBarrier(client)
caches := newNodeAuthInformers(ctx, client, "unbounded-system", 0)
caches.start()
t.Cleanup(func() {
@@ -75,10 +111,125 @@ func startNodeAuthTestCaches(t *testing.T, client *k8sfake.Clientset) (*nodeAuth
caches.saFactory.Shutdown()
})
waitNodeAuthCondition(t, caches.ready)
+ // Initial cache sync can finish before watches start. The fake client cannot
+ // replay a deletion that happens before its watch is registered.
+ waitNodeAuthCondition(t, watchesReady)
return caches, cancel
}
+func TestNodeAuthTestWatchBarrier(t *testing.T) {
+ for _, resource := range []string{"pods", "serviceaccounts"} {
+ t.Run(resource, func(t *testing.T) {
+ client := k8sfake.NewClientset()
+ entered := make(chan struct{})
+ release := make(chan struct{})
+
+ ctx, cancel := context.WithCancel(t.Context())
+ defer cancel()
+
+ client.PrependWatchReactor(resource, func(k8stesting.Action) (bool, watch.Interface, error) {
+ close(entered)
+
+ select {
+ case <-release:
+ return false, nil, nil
+ case <-ctx.Done():
+ return true, nil, ctx.Err()
+ }
+ })
+ ready := nodeAuthTestWatchBarrier(client)
+
+ other := "pods"
+ if resource == other {
+ other = "serviceaccounts"
+ }
+
+ otherWatch, err := client.InvokesWatch(k8stesting.NewWatchAction(corev1.SchemeGroupVersion.WithResource(other), "unbounded-system", metav1.ListOptions{}))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer otherWatch.Stop()
+
+ done := make(chan error, 1)
+
+ go func() {
+ watcher, err := client.InvokesWatch(k8stesting.NewWatchAction(corev1.SchemeGroupVersion.WithResource(resource), "unbounded-system", metav1.ListOptions{}))
+ if watcher != nil {
+ watcher.Stop()
+ }
+
+ done <- err
+ }()
+
+ select {
+ case <-entered:
+ case <-time.After(5 * time.Second):
+ t.Fatal("watch reactor was not entered")
+ }
+
+ if ready() {
+ t.Error("watch barrier passed before registration completed")
+ }
+
+ close(release)
+
+ select {
+ case err := <-done:
+ if err != nil {
+ t.Fatal(err)
+ }
+ case <-time.After(5 * time.Second):
+ t.Fatal("watch registration did not finish")
+ }
+
+ if !ready() {
+ t.Fatal("watch barrier did not pass after both registrations")
+ }
+ })
+ }
+}
+
+func TestNodeAuthTestWatchBarrierFailedRegistration(t *testing.T) {
+ client := k8sfake.NewClientset()
+ fail := true
+
+ client.PrependWatchReactor("pods", func(k8stesting.Action) (bool, watch.Interface, error) {
+ if fail {
+ return true, nil, errors.New("watch unavailable")
+ }
+
+ return false, nil, nil
+ })
+ ready := nodeAuthTestWatchBarrier(client)
+
+ saWatch, err := client.CoreV1().ServiceAccounts("unbounded-system").Watch(t.Context(), metav1.ListOptions{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer saWatch.Stop()
+
+ if _, err := client.CoreV1().Pods("unbounded-system").Watch(t.Context(), metav1.ListOptions{}); err == nil {
+ t.Fatal("expected watch registration failure")
+ }
+
+ if ready() {
+ t.Fatal("failed watch registration passed the barrier")
+ }
+
+ fail = false
+
+ podWatch, err := client.CoreV1().Pods("unbounded-system").Watch(t.Context(), metav1.ListOptions{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer podWatch.Stop()
+
+ if !ready() {
+ t.Fatal("successful retry did not pass the barrier")
+ }
+}
+
func assertNodeAuthProbes(t *testing.T, caches *nodeAuthInformers, client *k8sfake.Clientset, verifier serviceAccountTokenVerifier, ready bool) {
t.Helper()
diff --git a/designs/racer-object.md b/designs/racer-object.md
new file mode 100644
index 0000000000..6c7dc6b6f9
--- /dev/null
+++ b/designs/racer-object.md
@@ -0,0 +1,299 @@
+# Racer object gateway (`racer-object`)
+
+## Summary
+
+`racer-object` lets normal programs read S3 objects through a Racer cache.
+Programs talk plain HTTP in the S3 style. They do not need to know about
+Racer's Unix socket API.
+
+One binary has two modes:
+
+- `racer-object origin` fills the cache. Racer calls it on a cache miss. It
+ reads the object from S3 (or an S3-compatible store) and hands the bytes to
+ Racer.
+- `racer-object sidecar` serves the cache. It listens on a loopback HTTP port
+ next to the app, asks Racer for the object, and streams it back.
+
+```text
+app --HTTP--> sidecar --client socket--> Racer --origin socket--> origin --> S3
+```
+
+Both modes use `pkg/racersdk`. Racer serves the client socket under
+`/run/racer//`. The origin process uses `racersdk.ServeOrigin` to
+create and serve `/run/racer//origin/socket`; Racer connects to it.
+The SDK creates the origin socket with mode `0600`, owned by the origin's
+effective user. Racer must run as that user or have permission to bypass
+the socket's file permissions.
+
+Code:
+
+- `cmd/racer-object/main.go`: flags, setup, and process lifetime.
+- `cmd/racer-object/listener.go`: the sidecar's connection limit.
+- `internal/racer/object/origin.go`: cache keys and the S3 origin.
+- `internal/racer/object/sidecar.go`: the HTTP handler.
+
+## Cache keys
+
+Each object is named by a small JSON document:
+
+```json
+{"schema":1,"namespace":"...","bucket":"...","key":"...","versionId":"..."}
+```
+
+The Racer key is the SHA-256 hash of that JSON. The JSON itself travels with
+the request as metadata, so the origin knows what to fetch.
+
+- `--namespace` names the upstream store. It is required and must match in
+ the origin and the sidecar. Two stores with different data must use
+ different namespaces, or they would share cache entries.
+- Object keys are used as given. No path cleaning, no case folding, no
+ unescaping beyond the single URL decode in the sidecar. `a/../b` and `b`
+ are different objects, as they are in S3.
+- The origin rebuilds the JSON from its parts and checks that both the bytes
+ and the hash match what Racer sent. This means one object has exactly one
+ key, and a client cannot ask the origin to fetch something under a key
+ that belongs to a different object.
+- Limits: namespace up to 256 bytes with no spaces or control characters,
+ bucket names use `[A-Za-z0-9._-]` up to 255 bytes, object keys are valid
+ UTF-8 up to 1024 bytes.
+
+## Origin
+
+On a miss Racer first asks for metadata, then for pages.
+
+**Metadata.** The origin sends S3 a HEAD request. It returns the size, the
+strong ETag, the content type, and an expiry time of now plus
+`--metadata-ttl` (default 30s). At expiry Racer must stop serving that version
+from cache; the TTL is not just an admission hint. Zero expires metadata
+immediately. Supported content types are normalized for the SDK metadata
+format; missing or unsupported values become `application/octet-stream`.
+The raw HEAD and GET content types must still match.
+Delete markers are reported as not found. S3 itself answers a delete marker
+with an error: 404 when it is the current version, and 405
+`MethodNotAllowed` when its version id is requested explicitly; the origin
+maps both to not found. A successful reply flagged as a delete marker is
+refused as an invalid response. The origin also refuses objects with a
+`Content-Encoding` other than identity, because the SDK has no way to pass
+that header on.
+
+**Pages.** Racer asks for pages of 16 MiB. The origin sends a ranged GET with
+`If-Match` set to the ETag from the metadata. When the request names no
+version but HEAD returned a version id, the GET is pinned to that version id,
+so an overwrite with identical bytes cannot fail the version check. It then checks the reply: ETag,
+length, `Content-Range`, content type, version id, and encoding must all
+match. The body goes straight to the SDK, which fails the read if the body is
+short or too long.
+
+**Changed objects.** If the object changed between the HEAD and the GET, the
+ETag no longer matches. If Racer pinned an ETag, the origin returns
+`ErrVersionMismatch`. If Racer did not pin one (a bootstrap read), the ETag
+came from the origin's own HEAD, so the origin returns `ErrUnavailable`
+instead: a retry sees the new version, and Racer never gets a 412 for an ETag
+it did not ask for. The origin checks this itself and does not rely on the
+store honoring `If-Match`. A pinned object that has since been deleted is also
+a version mismatch, not "not found".
+
+**Errors.** S3 error codes and HTTP status codes are mapped to `racersdk`
+errors (not found, forbidden, unauthorized, version mismatch, unavailable).
+Network operation and DNS failures map to unavailable when no API or HTTP
+status classification applies. Cancellation and deadlines take precedence.
+The error message is a fixed "S3 request failed". The upstream error is kept
+as the wrapped cause, so `errors.Is` still works, but its text is not shown.
+
+**Setup.**
+
+- Credentials and region come from the normal AWS SDK config chain.
+ `--region` overrides the configured region. A region must be available
+ through `--region` or AWS configuration, such as `AWS_REGION` or
+ `AWS_DEFAULT_REGION`. `--endpoint` and `--path-style` (default on) support
+ S3-compatible stores. The effective endpoint, including one from AWS
+ environment or shared configuration, must be an HTTP(S) origin without
+ credentials, a path prefix, query, or fragment. `--endpoint` takes precedence.
+- The origin uses the AWS SDK's HTTP transport so SDK transport settings,
+ including configured CA bundles, still apply. Its idle connection pool
+ has room for 64 page requests and four metadata requests.
+- Checksums are sent only when S3 requires them. Some S3-compatible stores
+ reject the optional ones.
+- `--request-timeout` is capped at one minute, which is the SDK's own limit
+ for an origin call.
+- Before it creates its socket, the origin walks the path to
+ `/run/racer//origin` one directory at a time without following
+ symlinks. Missing directories are created with mode `0755`, subject to
+ umask; existing permissions are not changed. It refuses any directory
+ that is group- or world-writable.
+ Ancestors must be owned by root or the process's effective UID; the final
+ `origin` directory must be owned by the effective UID. This stops another
+ local user from swapping the socket path.
+- The origin does no caching of its own. Racer owns all caching.
+
+## Sidecar
+
+The sidecar is a small, read-only S3 endpoint for one app.
+
+**Listening.** `--listen` must be a loopback IP literal (default
+`127.0.0.1:8080`). The sidecar does not check SigV4 signatures, and it does
+not forward them. Access control is the origin's job and the bucket
+allowlist's. Keeping the port on loopback means only local processes can
+reach it.
+
+**Requests.** Only GET and HEAD are allowed, using path-style URLs
+(`/bucket/key`).
+
+- The path is split into bucket and key before it is decoded, and decoded
+ once.
+- Query: only `versionId` and a method-matching `x-id` (`GetObject` or
+ `HeadObject`). Malformed encoding, empty values, and repeated parameters
+ return 400; for example, `?acl` and `?versionId=` return 400.
+ Well-formed unsupported options, such as `?partNumber=1`, return 501.
+- Headers: `If-Match`, `If-None-Match`, and a single byte `Range` are
+ supported. A single `x-amz-checksum-mode: ENABLED` is accepted for GET and
+ HEAD as a request for available checksums. The sidecar has no S3 checksum
+ metadata, so it returns no checksum headers, including for ranges and empty
+ objects. The default AWS SDK accepts this response without checksum
+ validation. Empty, repeated, or unknown checksum modes return 501.
+ `If-Modified-Since`, `If-Unmodified-Since`, `If-Range`, and unknown `x-amz-*`
+ headers still return 501, including owner checks, SSE-C, and requester-pays.
+ Signing headers such as `x-amz-date` and `x-amz-content-sha256` are ignored.
+
+**Flow.** The sidecar parses the range first. Open and suffix ranges are
+supported and clamped to the size. The `bytes` unit is case-insensitive. As
+RFC 9110 section 14.2 allows and S3 does, a `Range` that cannot be used is
+ignored and the full object is served with 200. This covers unknown units,
+malformed values, multi-range sets, a last position before the first, and
+repeated `Range` headers. Only a valid single byte range that the object
+cannot satisfy returns 416: one that starts at or past the end, a zero-length
+suffix, or any byte range on an empty object. Conditions are checked before
+the range: `If-Match` (strong compare, 412), then `If-None-Match` (weak
+compare, 304).
+
+A GET with no byte range or an ignored one, an open range (`bytes=N-`), or a
+closed range (`bytes=N-M`) does not need the size up front, so it skips Stat:
+
+1. GET asks Racer for the range without an ETag pin.
+2. The conditions and the range are checked against the metadata that Racer
+ returns with the read. Racer serves every page of a read from that one
+ version, so the headers always describe the body. On a failed condition
+ the read is closed without sending any of the body.
+3. The SDK does not clamp a range that ends past the object, and fails with
+ `ErrRangeNotSatisfiable` instead. The sidecar then falls back to the Stat
+ flow below, which clamps the range or returns 416 with the size in
+ `Content-Range`.
+4. The body is written with `WriteTo`.
+
+HEAD, suffix ranges (`bytes=-N`), and that fallback need the size first:
+
+1. Stat the object in Racer.
+2. Check the conditions and the range against the Stat metadata.
+3. HEAD returns 200 with headers, even for a ranged HEAD. This matches S3.
+4. GET asks Racer for the range, pinned to the ETag from step 1. If the size
+ or content type changed, it returns 502. An internal pin failure returns
+ 503, not 412: only a failed client `If-Match` condition returns 412. A
+ direct read is never pinned, so a GET without conditions never returns 412.
+5. The body is written with `WriteTo`.
+
+**Concurrency limits.** The listener accepts up to 128 connections. Behind it,
+SDK calls wait in fixed lanes, and a call fails at once with `ErrUnavailable`,
+which the sidecar returns as 503, when its lane's queue is full. Calls that
+do queue fail the same way if they wait too long.
+
+- GET reads use the SDK's bulk lane: 64 concurrent transfers
+ (`ClientConfig.MaxConnections`, left at its default) and a queue of 128.
+ That is more than the listener admits, so direct GETs, including parallel
+ ranged GETs from S3 transfer tools, are bounded by the listener.
+- Stat uses a separate lane with 4 connections and a queue of 16 that cannot
+ be configured. HEAD, suffix ranges, and closed ranges that end past the
+ object all Stat first, so about 20 of them in the Stat phase at once is the
+ ceiling: more get an immediate 503. Size HEAD-heavy clients for this.
+
+**Copying with splice.** For HTTP/1, `WriteTo` moves bytes from the Racer
+socket to the TCP socket with `splice()`, so the data does not pass through
+user space. This only works if the TCP connection still exposes
+`io.ReaderFrom`. `netutil.LimitListener` wraps connections and hides it, so
+`listener.go` has its own connection limit (128) that keeps it.
+
+**Errors.** Application errors use S3's XML shape with fixed text:
+`......`. SDK errors map to
+400, 403, 404, 416, 503 (internal pin failure, unavailable, canceled, timed
+out, or closed), and 502 for anything else. Once a non-empty body has started, a failure
+aborts the connection instead of adding an error to the end. The client then
+sees a short read, not a body with XML stuck on the end. The SDK holds back
+the last byte until Racer confirms the range is complete, so a broken read
+can never look like a finished one.
+
+Go's HTTP parser can reject a request before the application runs, including
+malformed HTTP and headers beyond its bounded parser limit. Those responses
+use Go's error format, not S3 XML, and may not follow the application's
+body-free HEAD error behavior. Raising the parser limit only moves this
+boundary; it does not make every parser error an application XML error.
+
+**Timeouts and limits.**
+
+- `--request-timeout` (default 5m) bounds each request. The write timeout is
+ that plus 5s, so the handler's own deadline fires first and the client gets
+ a clean error. The SDK clears the write deadline when a streamed GET body
+ finishes, so the sidecar re-arms it to the request start plus that bound
+ after the handler returns. The final net/http flush to a client that has
+ stopped reading therefore still times out.
+- Header read timeout is the smaller of 5s and the request timeout. Header
+ size is capped at 32 KiB. Idle connections close after 30s.
+- Shutdown stops accepting connections and lets open requests drain for up
+ to 5s without canceling them first. After that, it cancels remaining
+ requests and closes their connections.
+
+## Diagnostics
+
+Logs go to stderr. They contain only fixed categories and validated fields,
+never upstream error text, URLs, keys, queries, or credentials.
+
+- Origin mode logs `event=request_failed category=` per failed
+ origin request.
+- Sidecar mode logs `event=response_failed method= status=<5xx>
+ category= bucket=` for each 5xx caused by the cache or
+ upstream. The bucket is logged because it is operator configuration and has
+ passed the allowlist. The key is not logged, because keys may carry tenant
+ or user data.
+- Categories that need a closer look:
+ - `tls`: a certificate or TLS failure. The request still fails as
+ `ErrUnavailable` (503), but this is usually a configuration error.
+ - `redirect`: an S3 3xx, usually a wrong region or addressing style.
+ - `status`: an S3 status with no Racer meaning, such as 400 or 416.
+ A 405 is not here: it is a delete marker and maps to not found.
+ - `validation`: an S3 response that failed HEAD or GET validation.
+ - `upstream`: an unclassified failure.
+ - `mismatch`: the sidecar saw a pinned read return different metadata.
+ - `redirect`, `status`, `validation`, and `upstream` all map to 502.
+ The categories exist only in origin logs. The sidecar sees the 502 that
+ Racer returns, which it logs as `upstream`.
+- Expected outcomes (`notfound`, `forbidden`, `unauthorized`, `canceled`,
+ `versionmismatch`, `invalid`, `range`) are logged only with
+ `--log-level=debug`.
+- Other lines are logged at `level=warn`. Each event and category can log 10
+ lines per minute. The first line after that minute adds `suppressed=`.
+- The AWS SDK logger writes one `event=aws_sdk level=warn` line to stderr per
+ SDK warning. With `--log-level=debug`, it also writes `level=debug` lines for
+ SDK debug, retry, and deprecation logs. Only these fixed fields are written:
+ SDK message text is dropped, because it can include upstream error messages,
+ object keys, and URLs. Request, response, and signing logs are never enabled, because they
+ print the SigV4 `Authorization` header and bodies.
+
+`--log-level` accepts `info` (the default) or `debug` in both modes.
+
+## Bucket allowlist
+
+`--bucket` may be repeated in both modes. If set, other buckets are refused:
+403 in the sidecar, `ErrForbidden` in the origin. If not set, all buckets are
+allowed. The origin checks on its own too, so the rule holds for any client
+of the cache, not only this sidecar.
+
+## Packaging and tests
+
+`images/racer-object/Containerfile` builds one image with the binary at
+`/usr/local/bin/racer-object`. The pod picks the mode with its command.
+Publication is experimental: use the generic image workflow with a manual
+dispatch for `racer-object` or an `images/racer-object/` tag. This
+does not add the image to the regular release or nightly publication sets.
+
+Tests use a fake S3 server (`origin_test.go`) and `httptest` with
+`racersdktest` (`sidecar_test.go`). `main_test.go` and `listener_test.go`
+cover flags, the listen address, the directory checks, and the connection
+limit.
diff --git a/go.mod b/go.mod
index 6210ee5638..b36db6d541 100644
--- a/go.mod
+++ b/go.mod
@@ -81,6 +81,7 @@ require (
github.com/vishvananda/netns v0.0.5
golang.org/x/crypto v0.57.0
golang.org/x/mod v0.41.0
+ golang.org/x/net v0.60.0
golang.org/x/sync v0.23.0
golang.org/x/sys v0.48.0
golang.org/x/term v0.46.0
@@ -328,7 +329,6 @@ require (
go.yaml.in/yaml/v2 v2.4.4 // indirect
go.yaml.in/yaml/v3 v3.0.5 // indirect
golang.org/x/exp v0.0.0-20260824195058-e88cd73687aa // indirect
- golang.org/x/net v0.60.0 // indirect
golang.org/x/oauth2 v0.36.0 // indirect
golang.org/x/telemetry v0.0.0-20260908163034-4bcc4b2ee518 // indirect
golang.org/x/text v0.42.0 // indirect
diff --git a/images/racer-object/Containerfile b/images/racer-object/Containerfile
new file mode 100644
index 0000000000..80f31b7cb0
--- /dev/null
+++ b/images/racer-object/Containerfile
@@ -0,0 +1,33 @@
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+FROM --platform=$BUILDPLATFORM docker.io/library/golang:1.27.2-trixie AS builder
+WORKDIR /src
+COPY go.mod go.sum ./
+RUN go mod download
+COPY . .
+ARG TARGETOS
+ARG TARGETARCH
+ARG VERSION=dev
+ARG GIT_COMMIT=unknown
+ARG BUILD_TIME=unknown
+RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \
+ go build -trimpath \
+ -ldflags "-X github.com/Azure/unbounded/internal/version.Version=${VERSION} -X github.com/Azure/unbounded/internal/version.GitCommit=${GIT_COMMIT} -X github.com/Azure/unbounded/internal/version.BuildTime=${BUILD_TIME}" \
+ -o /out/racer-object ./cmd/racer-object
+
+FROM ubuntu:noble
+ARG VERSION=dev
+ARG GIT_COMMIT=unknown
+LABEL org.opencontainers.image.title="racer-object" \
+ org.opencontainers.image.licenses="Apache-2.0" \
+ org.opencontainers.image.version="${VERSION}" \
+ org.opencontainers.image.revision="${GIT_COMMIT}"
+RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends \
+ ca-certificates \
+ && apt-get clean && rm -rf /var/lib/apt/lists/*
+COPY --from=builder /out/racer-object /usr/local/bin/racer-object
+COPY LICENSE NOTICE /usr/share/licenses/racer-object/
+# Racer's node-local sockets are root-owned with mode 0600.
+USER 0:0
+ENTRYPOINT ["/usr/local/bin/racer-object"]
diff --git a/internal/gantry/mirror/rediscover_test.go b/internal/gantry/mirror/rediscover_test.go
index cb16068b77..ff582d8dc6 100644
--- a/internal/gantry/mirror/rediscover_test.go
+++ b/internal/gantry/mirror/rediscover_test.go
@@ -5,6 +5,7 @@ package mirror_test
import (
"bytes"
+ "context"
"fmt"
"io"
"net/http"
@@ -156,20 +157,36 @@ func TestMirror_Rediscover_ColdExhaustedFlushesHeadersBeforeLateProvider(t *test
dialer.Put(lateAddr, d, body)
dht := fakes.NewDHT()
- coldStart := &stubColdStart{err: mirror.ErrColdStartExhausted}
+ gate := make(chan struct{})
+
+ var once sync.Once
+
+ release := func() { once.Do(func() { close(gate) }) }
+ coldStart := &gatedRediscoverColdStart{
+ stubColdStart: &stubColdStart{err: mirror.ErrColdStartExhausted},
+ entered: make(chan struct{}),
+ release: gate,
+ }
m := mirror.New(cfg, fakes.NewCache(), oc,
mirror.WithLiveStreamThrough(),
mirror.WithDiscovery(dht, dialer),
mirror.WithColdStart(coldStart),
mirror.WithPeerBudgets(time.Second, time.Second, 20),
- mirror.WithPeerRediscover(5*time.Millisecond, 3*time.Second),
+ mirror.WithPeerRediscover(5*time.Second, 10*time.Millisecond),
)
srv := httptest.NewServer(m.Handler())
t.Cleanup(srv.Close)
+ // Unblock cold-start before closing the server, including on assertion failures.
+ t.Cleanup(release)
- // Returns as soon as the headers are flushed; no provider exists yet.
- resp, err := http.Get(srv.URL + "/v2/r/blobs/" + d.String())
+ transport := &http.Transport{ResponseHeaderTimeout: 2 * time.Second}
+ t.Cleanup(transport.CloseIdleConnections)
+
+ client := &http.Client{Transport: transport, Timeout: 10 * time.Second}
+
+ // Headers must arrive while cold-start is still blocked and no provider exists.
+ resp, err := client.Get(srv.URL + "/v2/r/blobs/" + d.String())
if err != nil {
t.Fatal(err)
}
@@ -188,18 +205,16 @@ func TestMirror_Rediscover_ColdExhaustedFlushesHeadersBeforeLateProvider(t *test
t.Fatalf("peer calls before advertise = %d, want 0", got)
}
- // Headers are flushed before round 0. Wait for cold-start so the late
- // provider cannot satisfy the initial lookup and skip the exhausted path.
- deadline := time.Now().Add(5 * time.Second)
- for atomic.LoadInt32(&coldStart.calls) == 0 {
- if time.Now().After(deadline) {
- t.Fatal("cold-start was not called before provider advertisement")
- }
-
- time.Sleep(time.Millisecond)
+ // A header flush precedes round 0, so it does not prove cold-start has run.
+ // Wait for its entry before making a provider visible to later rounds.
+ select {
+ case <-coldStart.entered:
+ case <-time.After(5 * time.Second):
+ t.Fatal("cold-start did not begin after headers were flushed")
}
dht.Inject(d, ifaces.Provider{NodeID: "late-seed", Addr: lateAddr})
+ release()
got, err := io.ReadAll(resp.Body)
if err != nil {
@@ -215,6 +230,25 @@ func TestMirror_Rediscover_ColdExhaustedFlushesHeadersBeforeLateProvider(t *test
}
}
+type gatedRediscoverColdStart struct {
+ *stubColdStart
+
+ entered chan struct{}
+ release <-chan struct{}
+ once sync.Once
+}
+
+func (s *gatedRediscoverColdStart) Resolve(ctx context.Context, d digest.Digest, kind ifaces.OriginRefKind, upstream, repo string, size int64) (*mirror.ColdStartResolution, error) {
+ s.once.Do(func() { close(s.entered) })
+
+ select {
+ case <-s.release:
+ return s.stubColdStart.Resolve(ctx, d, kind, upstream, repo, size)
+ case <-ctx.Done():
+ return nil, ctx.Err()
+ }
+}
+
// The re-discovery loop is silent whenever a round never reaches a peer: no
// provider means no peer fetch line, and cold-start is suppressed after round
// 0. A round 0 that resolves without providers therefore emits nothing at all
diff --git a/internal/racer/object/diagnostics.go b/internal/racer/object/diagnostics.go
new file mode 100644
index 0000000000..e26050b078
--- /dev/null
+++ b/internal/racer/object/diagnostics.go
@@ -0,0 +1,94 @@
+// Copyright (c) Microsoft Corporation.
+// SPDX-License-Identifier: Apache-2.0
+
+package object
+
+import (
+ "errors"
+ "net/http"
+ "net/url"
+ "strings"
+)
+
+// invalidS3Response refines errInvalidS3Response for diagnostics only. It
+// still matches errInvalidS3Response, so the Racer and sidecar mapping (502)
+// is unchanged.
+type invalidS3Response struct{ kind string }
+
+// Error matches errInvalidS3Response so messages stay unchanged.
+func (e *invalidS3Response) Error() string { return errInvalidS3Response.Error() }
+
+func (e *invalidS3Response) Is(target error) bool { return target == errInvalidS3Response }
+
+var (
+ // errS3Redirect is a 3xx, typically a wrong region or addressing style.
+ errS3Redirect error = &invalidS3Response{kind: "redirect"}
+ // errS3Status is an HTTP status with no Racer meaning, such as 400 or 416.
+ errS3Status error = &invalidS3Response{kind: "status"}
+ // errS3Validation is a response that failed HEAD or GET validation.
+ errS3Validation error = &invalidS3Response{kind: "validation"}
+)
+
+func invalidS3Status(code int) error {
+ if code >= 300 && code < 400 {
+ return errS3Redirect
+ }
+
+ return errS3Status
+}
+
+// InvalidResponseCategory returns a fixed diagnostic category for an invalid S3
+// response: "redirect", "status", "validation", or "upstream" when the cause is
+// unknown. It returns "" when err is not an invalid S3 response. The result
+// never contains upstream text.
+func InvalidResponseCategory(err error) string {
+ var invalid *invalidS3Response
+ if errors.As(err, &invalid) {
+ return invalid.kind
+ }
+
+ if errors.Is(err, errInvalidS3Response) {
+ return "upstream"
+ }
+
+ return ""
+}
+
+// ErrMetadataMismatch reports that a pinned read returned metadata that
+// differs from the metadata selected by Stat.
+var ErrMetadataMismatch = errors.New("racer-object: sidecar metadata mismatch")
+
+// SidecarFailure describes a 5xx returned by the sidecar. It carries the
+// bucket but not the key or query: the bucket is operator configuration and
+// is already validated against the allowlist, while keys may hold tenant or
+// user data and queries may hold presigned credentials. Err is the
+// classified cause; callers must not log its text, which may hold upstream
+// details. Use errors.Is or InvalidResponseCategory instead.
+type SidecarFailure struct {
+ Method string
+ Status int
+ Bucket string
+ Err error
+}
+
+func (s *sidecar) sdkFailure(r *http.Request, err error) sidecarFailure {
+ return s.report(r, sidecarSDKFailure(err), err)
+}
+
+func (s *sidecar) report(r *http.Request, failure sidecarFailure, err error) sidecarFailure {
+ if s.onFailure != nil && failure >= 500 {
+ s.onFailure(SidecarFailure{Method: r.Method, Status: int(failure), Bucket: sidecarBucket(r), Err: err})
+ }
+
+ return failure
+}
+
+// sidecarBucket repeats the request parser's split; it is only called after
+// the request has been accepted, so the bucket is valid and allowed.
+func sidecarBucket(r *http.Request) string {
+ path, _ := strings.CutPrefix(r.URL.EscapedPath(), "/")
+ bucket, _, _ := strings.Cut(path, "/")
+ bucket, _ = url.PathUnescape(bucket) //nolint:errcheck // EscapedPath guarantees valid escapes.
+
+ return bucket
+}
diff --git a/internal/racer/object/diagnostics_test.go b/internal/racer/object/diagnostics_test.go
new file mode 100644
index 0000000000..a7aff93d94
--- /dev/null
+++ b/internal/racer/object/diagnostics_test.go
@@ -0,0 +1,182 @@
+// Copyright (c) Microsoft Corporation.
+// SPDX-License-Identifier: Apache-2.0
+
+package object
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "sync/atomic"
+ "testing"
+
+ "github.com/Azure/unbounded/pkg/racersdk"
+)
+
+func TestInvalidResponseCategory(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ err error
+ want string
+ }{
+ {"redirect-301", classifyS3Error(originStatusError{301}, false), "redirect"},
+ {"redirect-307", classifyS3Error(originStatusError{307}, false), "redirect"},
+ {"status-400", classifyS3Error(originStatusError{400}, false), "status"},
+ {"status-416", classifyS3Error(originStatusError{416}, false), "status"},
+ {"validation", errS3Validation, "validation"},
+ {"unknown", classifyS3Error(errors.New("private"), false), "upstream"},
+ {"not-invalid", classifyS3Error(originStatusError{404}, false), ""},
+ {"delete-marker-405", classifyS3Error(originStatusError{405}, false), ""},
+ {"nil", nil, ""},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ if got := InvalidResponseCategory(tc.err); got != tc.want {
+ t.Fatalf("category = %q, want %q", got, tc.want)
+ }
+
+ if tc.want != "" && (!errors.Is(tc.err, errInvalidS3Response) || sidecarSDKFailure(tc.err) != http.StatusBadGateway) {
+ t.Fatal("sub-category changed the error mapping")
+ }
+ })
+ }
+
+ // A delete marker's 405 is not found, which diagnostics log as notfound.
+ if err := classifyS3Error(originStatusError{405}, false); !errors.Is(err, racersdk.ErrNotFound) || errors.Is(err, errInvalidS3Response) {
+ t.Fatalf("405 = %v, want not found", err)
+ }
+
+ if errors.Is(errS3Status, racersdk.ErrUnavailable) {
+ t.Fatal("status sub-category matched an unrelated sentinel")
+ }
+}
+
+func TestInvalidHeadIsValidation(t *testing.T) {
+ _, err := (&objectOrigin{}).headMetadata(nil, Object{})
+ if InvalidResponseCategory(err) != "validation" {
+ t.Fatalf("head validation category = %q", InvalidResponseCategory(err))
+ }
+}
+
+func sidecarReportingHandler(t *testing.T, origin racersdk.Origin, reports *[]SidecarFailure) http.Handler {
+ t.Helper()
+
+ h, err := NewSidecar(testSDKClient(t, origin), SidecarConfig{
+ Namespace: "test", Buckets: []string{"my-bucket"},
+ OnFailure: func(f SidecarFailure) { *reports = append(*reports, f) },
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ return h
+}
+
+func TestSidecarReportsServerFailures(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ err error
+ status int
+ report bool
+ want error
+ }{
+ // Only Racer wire errors cross to the sidecar; S3 detail stays in origin logs.
+ {"unavailable", racersdk.ErrUnavailable, 503, true, racersdk.ErrUnavailable},
+ {"version", racersdk.ErrVersionMismatch, 503, true, racersdk.ErrVersionMismatch},
+ {"invalid", errS3Redirect, 502, true, nil},
+ {"notfound", racersdk.ErrNotFound, 404, false, nil},
+ {"forbidden", racersdk.ErrForbidden, 403, false, nil},
+ {"bad-request", racersdk.ErrInvalidRequest, 400, false, nil},
+ } {
+ for _, method := range []string{"GET", "HEAD"} {
+ t.Run(tc.name+method, func(t *testing.T) {
+ var reports []SidecarFailure
+
+ h := sidecarReportingHandler(t, func(context.Context, racersdk.OriginRequest) (racersdk.Metadata, io.ReadCloser, error) {
+ return racersdk.Metadata{}, nil, fmt.Errorf("private: %w", tc.err)
+ }, &reports)
+
+ w := sidecarTestRequest(h, method, "/my-bucket/secret-key?versionId=v", nil)
+ if w.Code != tc.status {
+ t.Fatalf("status %d want %d", w.Code, tc.status)
+ }
+
+ if !tc.report {
+ if len(reports) != 0 {
+ t.Fatalf("client error reported: %+v", reports)
+ }
+
+ return
+ }
+
+ if len(reports) != 1 {
+ t.Fatalf("reports = %+v", reports)
+ }
+
+ got := reports[0]
+ if got.Method != method || got.Status != tc.status || got.Bucket != "my-bucket" || got.Err == nil ||
+ (tc.want != nil && !errors.Is(got.Err, tc.want)) {
+ t.Fatalf("report = %+v", got)
+ }
+ })
+ }
+ }
+}
+
+func TestSidecarReportsMetadataMismatch(t *testing.T) {
+ var (
+ reports []SidecarFailure
+ calls atomic.Int32
+ )
+
+ m := sidecarTestMetadata(t, 4, `"same"`)
+ h := sidecarReportingHandler(t, func(_ context.Context, r racersdk.OriginRequest) (racersdk.Metadata, io.ReadCloser, error) {
+ if calls.Add(1) == 1 {
+ return m, nil, nil
+ }
+
+ changed := m
+ changed.ContentType = "application/octet-stream"
+
+ if r.Head {
+ return changed, nil, nil
+ }
+
+ return changed, io.NopCloser(strings.NewReader("xxxx")), nil
+ }, &reports)
+
+ // A suffix range takes the Stat path, whose read is pinned and checked.
+ w := sidecarTestRequest(h, "GET", "/my-bucket/key", http.Header{"Range": {"bytes=-4"}})
+ if w.Code != http.StatusBadGateway || len(reports) != 1 || !errors.Is(reports[0].Err, ErrMetadataMismatch) || reports[0].Bucket != "my-bucket" {
+ t.Fatalf("status %d reports %+v", w.Code, reports)
+ }
+}
+
+func TestSidecarNilOnFailure(t *testing.T) {
+ h, _ := sidecarTestHandler(t, func(context.Context, racersdk.OriginRequest) (racersdk.Metadata, io.ReadCloser, error) {
+ return racersdk.Metadata{}, nil, racersdk.ErrUnavailable
+ })
+
+ if w := sidecarTestRequest(h, "GET", "/bucket/key", nil); w.Code != 503 {
+ t.Fatalf("status %d", w.Code)
+ }
+}
+
+func TestSidecarReportsCanceledRequest(t *testing.T) {
+ var reports []SidecarFailure
+
+ h := sidecarReportingHandler(t, sidecarTestOrigin(t, "data", `"tag"`, nil), &reports)
+
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel()
+
+ w := httptest.NewRecorder()
+ h.ServeHTTP(w, httptest.NewRequest("GET", "/my-bucket/key", nil).WithContext(ctx))
+
+ if w.Code != 503 || len(reports) != 1 || !errors.Is(reports[0].Err, context.Canceled) {
+ t.Fatalf("status %d reports %+v", w.Code, reports)
+ }
+}
diff --git a/internal/racer/object/origin.go b/internal/racer/object/origin.go
new file mode 100644
index 0000000000..1e897651f1
--- /dev/null
+++ b/internal/racer/object/origin.go
@@ -0,0 +1,576 @@
+// Copyright (c) Microsoft Corporation.
+// SPDX-License-Identifier: Apache-2.0
+
+// Package object adapts S3 objects to racersdk: an Origin that reads
+// exact object versions from S3 and a sidecar HTTP handler that serves them
+// to pods through a racersdk client.
+package object
+
+import (
+ "bytes"
+ "context"
+ "crypto/sha256"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "net"
+ "reflect"
+ "strings"
+ "time"
+ "unicode"
+ "unicode/utf8"
+
+ "github.com/aws/aws-sdk-go-v2/aws"
+ "github.com/aws/aws-sdk-go-v2/service/s3"
+ "github.com/aws/smithy-go"
+
+ "github.com/Azure/unbounded/pkg/racersdk"
+)
+
+// Object identifies an exact upstream object within a configured namespace.
+type Object struct {
+ Schema int `json:"schema"`
+ Namespace string `json:"namespace"`
+ Bucket string `json:"bucket"`
+ Key string `json:"key"`
+ VersionID string `json:"versionId,omitempty"`
+}
+
+// NewRequest hashes canonical schema-1 JSON and carries it as origin metadata.
+// Keys and version IDs are never cleaned, decoded, or normalized.
+func NewRequest(namespace, bucket, key, versionID string) (racersdk.Request, error) {
+ object := Object{Schema: 1, Namespace: namespace, Bucket: bucket, Key: key, VersionID: versionID}
+ if !validObject(object) {
+ return racersdk.Request{}, racersdk.ErrInvalidRequest
+ }
+
+ canonical, err := json.Marshal(object)
+ if err != nil {
+ return racersdk.Request{}, fmt.Errorf("%w: %w", racersdk.ErrInvalidRequest, err)
+ }
+
+ // JSON leaves DEL literal, but SDK metadata rejects it. Escape it before hashing.
+ canonical = bytes.ReplaceAll(canonical, []byte{0x7f}, []byte(`\u007f`))
+
+ if len(canonical) > 8<<10 {
+ return racersdk.Request{}, racersdk.ErrInvalidRequest
+ }
+
+ return racersdk.Request{Key: sha256.Sum256(canonical), Metadata: string(canonical)}, nil
+}
+
+func validNamespace(value string) bool {
+ return value != "" && len(value) <= 256 && utf8.ValidString(value) &&
+ strings.IndexFunc(value, func(r rune) bool { return unicode.IsSpace(r) || unicode.IsControl(r) }) == -1
+}
+
+func validBucket(value string) bool {
+ if value == "" || len(value) > 255 || value == "." || value == ".." {
+ return false
+ }
+
+ // Include legacy S3-compatible bucket names, but not paths or endpoint URLs.
+ for _, c := range value {
+ if (c < 'a' || c > 'z') && (c < 'A' || c > 'Z') && (c < '0' || c > '9') && c != '.' && c != '-' && c != '_' {
+ return false
+ }
+ }
+
+ return true
+}
+
+func validObject(object Object) bool {
+ return object.Schema == 1 && validNamespace(object.Namespace) && validBucket(object.Bucket) &&
+ object.Key != "" && len(object.Key) <= 1024 && utf8.ValidString(object.Key) &&
+ len(object.VersionID) <= 1024 && utf8.ValidString(object.VersionID)
+}
+
+var (
+ errInvalidNamespace = errors.New("invalid namespace")
+ errInvalidBucket = errors.New("invalid bucket")
+)
+
+// scope binds requests to one namespace and an optional bucket allowlist.
+// An empty allowlist admits every bucket.
+type scope struct {
+ namespace string
+ buckets map[string]struct{}
+}
+
+// newScope validates and copies the configured namespace and buckets.
+func newScope(namespace string, buckets []string) (scope, error) {
+ if !validNamespace(namespace) {
+ return scope{}, errInvalidNamespace
+ }
+
+ s := scope{namespace: namespace, buckets: make(map[string]struct{}, len(buckets))}
+ for _, bucket := range buckets {
+ if !validBucket(bucket) {
+ return scope{}, errInvalidBucket
+ }
+
+ s.buckets[bucket] = struct{}{}
+ }
+
+ return s, nil
+}
+
+func (s scope) allows(bucket string) bool {
+ if len(s.buckets) == 0 {
+ return true
+ }
+
+ _, ok := s.buckets[bucket]
+
+ return ok
+}
+
+// decodeObject checks both the canonical bytes and their hash so alternate JSON
+// spellings cannot give the same upstream object a different cache identity.
+func decodeObject(request racersdk.OriginRequest, s scope) (Object, error) {
+ raw := request.Metadata
+
+ var object Object
+ if err := json.Unmarshal([]byte(raw), &object); err != nil || !validObject(object) {
+ return Object{}, racersdk.ErrInvalidRequest
+ }
+
+ canonical, err := NewRequest(object.Namespace, object.Bucket, object.Key, object.VersionID)
+ if err != nil || canonical.Metadata != raw || canonical.Key != request.Key {
+ return Object{}, racersdk.ErrInvalidRequest
+ }
+
+ if object.Namespace != s.namespace || !s.allows(object.Bucket) {
+ return Object{}, racersdk.ErrForbidden
+ }
+
+ return object, nil
+}
+
+// S3Client is the read-only subset of the AWS v2 S3 client used by the origin.
+type S3Client interface {
+ HeadObject(context.Context, *s3.HeadObjectInput, ...func(*s3.Options)) (*s3.HeadObjectOutput, error)
+ GetObject(context.Context, *s3.GetObjectInput, ...func(*s3.Options)) (*s3.GetObjectOutput, error)
+}
+
+// OriginConfig binds requests to one upstream namespace and optional buckets.
+type OriginConfig struct {
+ Namespace string
+ Buckets []string
+ // MetadataTTL sets when Racer must stop serving this version from cache.
+ // Zero expires metadata immediately.
+ MetadataTTL time.Duration
+}
+
+// NewOrigin serves full metadata and conditional page reads without caching.
+// The client must return bodies whose Close interrupts Read, as the AWS client does.
+func NewOrigin(client S3Client, config OriginConfig) (racersdk.Origin, error) {
+ if client == nil || config.MetadataTTL < 0 {
+ return nil, racersdk.ErrInvalidRequest
+ }
+
+ value := reflect.ValueOf(client)
+ switch value.Kind() {
+ case reflect.Chan, reflect.Func, reflect.Map, reflect.Pointer, reflect.Slice:
+ if value.IsNil() {
+ return nil, racersdk.ErrInvalidRequest
+ }
+ }
+
+ bound, err := newScope(config.Namespace, config.Buckets)
+ if err != nil {
+ return nil, racersdk.ErrInvalidRequest
+ }
+
+ origin := &objectOrigin{client: client, scope: bound, ttl: config.MetadataTTL}
+
+ return origin.read, nil
+}
+
+type objectOrigin struct {
+ client S3Client
+ scope scope
+ ttl time.Duration
+}
+
+func (o *objectOrigin) read(ctx context.Context, request racersdk.OriginRequest) (racersdk.Metadata, io.ReadCloser, error) {
+ if err := ctx.Err(); err != nil {
+ return racersdk.Metadata{}, nil, classifyS3Error(err, false)
+ }
+
+ object, err := decodeObject(request, o.scope)
+ if err != nil {
+ return racersdk.Metadata{}, nil, err
+ }
+
+ input := objectHeadInput(request, object)
+
+ head, metadata, err := o.readHead(ctx, request, object, input)
+ if err != nil {
+ return metadata, nil, err
+ }
+
+ if request.Head {
+ return metadata, nil, nil
+ }
+
+ body, err := o.readPage(ctx, request, input, head, metadata)
+
+ return metadata, body, err
+}
+
+// readHead rechecks the pin on every page, including upstreams that ignore If-Match.
+func (o *objectOrigin) readHead(ctx context.Context, request racersdk.OriginRequest, object Object, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, racersdk.Metadata, error) {
+ pin, pinned := request.ETag, request.ETag != ""
+
+ head, err := o.client.HeadObject(ctx, input)
+ if err != nil {
+ return nil, racersdk.Metadata{}, classifyS3Error(err, pinned)
+ }
+
+ metadata, err := o.headMetadata(head, object)
+ if err != nil {
+ return nil, racersdk.Metadata{}, err
+ }
+
+ if pinned && metadata.ETag != pin {
+ return head, metadata, racersdk.ErrVersionMismatch
+ }
+
+ return head, metadata, nil
+}
+
+// objectHeadInput omits unspecified versions and pins rather than sending empty ones.
+func objectHeadInput(request racersdk.OriginRequest, object Object) *s3.HeadObjectInput {
+ input := &s3.HeadObjectInput{Bucket: aws.String(object.Bucket), Key: aws.String(object.Key)}
+ if object.VersionID != "" {
+ input.VersionId = aws.String(object.VersionID)
+ }
+
+ if request.ETag != "" {
+ input.IfMatch = aws.String(request.ETag)
+ }
+
+ return input
+}
+
+// readPage resolves the range against the fresh HEAD and pins GET to that version.
+func (o *objectOrigin) readPage(ctx context.Context, request racersdk.OriginRequest, input *s3.HeadObjectInput, head *s3.HeadObjectOutput, metadata racersdk.Metadata) (io.ReadCloser, error) {
+ if request.Offset < 0 || request.Offset%racersdk.PageSize != 0 || request.Length <= 0 || request.Length > racersdk.PageSize {
+ return nil, racersdk.ErrInvalidRequest
+ }
+
+ // The SDK handles empty objects and pages outside the object.
+ if request.Offset >= metadata.Size {
+ return nil, nil
+ }
+
+ first := request.Offset
+ last := first + min(request.Length, metadata.Size-first) - 1
+
+ // Pin GET to the version HEAD saw, so an overwrite with identical bytes
+ // (same ETag, new version) between HEAD and GET cannot fail the version
+ // comparison in validateObjectPage. Unversioned buckets return no version
+ // ID from HEAD, so none is sent.
+ version := input.VersionId
+ if version == nil && aws.ToString(head.VersionId) != "" {
+ version = head.VersionId
+ }
+
+ get := &s3.GetObjectInput{
+ Bucket: input.Bucket, Key: input.Key, VersionId: version,
+ IfMatch: aws.String(metadata.ETag), Range: aws.String(fmt.Sprintf("bytes=%d-%d", first, last)),
+ }
+
+ output, err := o.client.GetObject(ctx, get)
+ if err != nil {
+ return responseBody(output), unpinnedRace(request, classifyS3Error(err, true))
+ }
+
+ // Transfer even a rejected body to the SDK, which closes it exactly once.
+ // Do not limit the reader: the SDK must detect short and excess bodies.
+ return responseBody(output), unpinnedRace(request, validateObjectPage(output, head, metadata, first, last))
+}
+
+// unpinnedRace reports a HEAD/GET mismatch as retryable when Racer did not pin
+// an ETag. The ETag GET fails to match came from this origin's own HEAD, so
+// ErrVersionMismatch (412 to Racer) would name a version the caller never asked
+// for. The object changed underneath the read; a retry observes the new version.
+func unpinnedRace(request racersdk.OriginRequest, err error) error {
+ if request.ETag != "" || !errors.Is(err, racersdk.ErrVersionMismatch) {
+ return err
+ }
+
+ // Keep only the upstream cause: wrapping err itself would leave
+ // ErrVersionMismatch reachable through errors.Is.
+ var upstream privateS3Error
+ if errors.As(err, &upstream) {
+ return fmt.Errorf("%w: object changed during read: %w", racersdk.ErrUnavailable, upstream)
+ }
+
+ return fmt.Errorf("%w: object changed during read", racersdk.ErrUnavailable)
+}
+
+func (o *objectOrigin) headMetadata(head *s3.HeadObjectOutput, object Object) (racersdk.Metadata, error) {
+ if head == nil || head.ContentLength == nil || *head.ContentLength < 0 || aws.ToBool(head.DeleteMarker) ||
+ aws.ToString(head.ContentRange) != "" || (object.VersionID != "" && aws.ToString(head.VersionId) != object.VersionID) {
+ return racersdk.Metadata{}, errS3Validation
+ }
+
+ // SDK metadata cannot carry Content-Encoding to the reader.
+ if encoding := aws.ToString(head.ContentEncoding); encoding != "" && !strings.EqualFold(encoding, "identity") {
+ return racersdk.Metadata{}, errS3Validation
+ }
+
+ tag := aws.ToString(head.ETag)
+ if !validS3ETag(tag) {
+ return racersdk.Metadata{}, errS3Validation
+ }
+
+ metadata := racersdk.Metadata{
+ Size: *head.ContentLength, ETag: tag,
+ ExpiresAt: time.Now().Add(o.ttl).Truncate(time.Millisecond), ContentType: normalizeS3ContentType(aws.ToString(head.ContentType)),
+ }
+
+ return metadata, nil
+}
+
+var errInvalidS3Response = errors.New("racer-object: invalid S3 response")
+
+// normalizeS3ContentType follows racersdk/internal/wire.ValidateContentType.
+// Unsupported values become absent metadata so the sidecar can use its fallback.
+func normalizeS3ContentType(value string) string {
+ if len(value) > 256 || strings.TrimSpace(value) != value {
+ return ""
+ }
+
+ for i := range len(value) {
+ if value[i] < 0x20 || value[i] > 0x7e {
+ return ""
+ }
+ }
+
+ rest := value
+ token := func() string {
+ i := 0
+ for i < len(rest) && (rest[i] >= 'a' && rest[i] <= 'z' || rest[i] >= 'A' && rest[i] <= 'Z' ||
+ rest[i] >= '0' && rest[i] <= '9' || strings.ContainsRune("!#$%&'*+-.^_`|~", rune(rest[i]))) {
+ i++
+ }
+
+ part := rest[:i]
+ rest = rest[i:]
+
+ return part
+ }
+
+ consume := func(b byte) bool {
+ if len(rest) == 0 || rest[0] != b {
+ return false
+ }
+
+ rest = rest[1:]
+
+ return true
+ }
+ if token() == "" || !consume('/') || token() == "" {
+ return ""
+ }
+
+ var parameters []string
+
+ for rest != "" {
+ rest = strings.TrimLeft(rest, " ")
+
+ if !consume(';') {
+ return ""
+ }
+
+ rest = strings.TrimLeft(rest, " ")
+
+ name := token()
+ if name == "" {
+ return ""
+ }
+
+ for _, old := range parameters {
+ if strings.EqualFold(old, name) {
+ return ""
+ }
+ }
+
+ parameters = append(parameters, name)
+ rest = strings.TrimLeft(rest, " ")
+
+ if !consume('=') {
+ return ""
+ }
+
+ rest = strings.TrimLeft(rest, " ")
+
+ if consume('"') {
+ for {
+ if rest == "" {
+ return ""
+ }
+
+ b := rest[0]
+ rest = rest[1:]
+
+ if b == '"' {
+ break
+ }
+
+ if b == '\\' {
+ if rest == "" {
+ return ""
+ }
+
+ rest = rest[1:]
+ }
+ }
+ } else if token() == "" {
+ return ""
+ }
+ }
+
+ return value
+}
+
+func validS3ETag(tag string) bool {
+ if len(tag) < 2 || len(tag) > 8<<10 || tag[0] != '"' || tag[len(tag)-1] != '"' {
+ return false
+ }
+
+ for i := 1; i < len(tag)-1; i++ {
+ if tag[i] != 0x21 && (tag[i] < 0x23 || tag[i] > 0x7e) {
+ return false
+ }
+ }
+
+ return true
+}
+
+func responseBody(output *s3.GetObjectOutput) io.ReadCloser {
+ if output == nil {
+ return nil
+ }
+
+ return output.Body
+}
+
+func validateObjectPage(output *s3.GetObjectOutput, head *s3.HeadObjectOutput, metadata racersdk.Metadata, first, last int64) error {
+ if output == nil {
+ return errS3Validation
+ }
+
+ tag := aws.ToString(output.ETag)
+ if !validS3ETag(tag) {
+ return errS3Validation
+ }
+
+ if tag != metadata.ETag {
+ return racersdk.ErrVersionMismatch
+ }
+
+ // Compare raw content types so normalization cannot hide a HEAD/GET mismatch.
+ if output.Body == nil || output.ContentLength == nil || *output.ContentLength != int64(last-first+1) ||
+ aws.ToString(output.ContentRange) != fmt.Sprintf("bytes %d-%d/%d", first, last, metadata.Size) ||
+ aws.ToString(output.ContentType) != aws.ToString(head.ContentType) || aws.ToString(output.VersionId) != aws.ToString(head.VersionId) ||
+ !strings.EqualFold(aws.ToString(output.ContentEncoding), aws.ToString(head.ContentEncoding)) || aws.ToBool(output.DeleteMarker) {
+ return errS3Validation
+ }
+
+ return nil
+}
+
+func classifyS3Error(err error, pinned bool) error {
+ classification := s3Error(err)
+
+ // A missing pinned object means the selected version is no longer available.
+ if pinned && classification == racersdk.ErrNotFound {
+ classification = racersdk.ErrVersionMismatch
+ }
+
+ return fmt.Errorf("%w: %w", classification, privateS3Error{err})
+}
+
+// Keep upstream details out of messages while preserving the cause for errors.Is.
+type privateS3Error struct{ cause error }
+
+func (e privateS3Error) Error() string { return "S3 request failed" }
+func (e privateS3Error) Unwrap() error { return e.cause }
+
+// s3Error prefers cancellation and known API codes over HTTP status codes.
+func s3Error(err error) error {
+ var (
+ api smithy.APIError
+ status interface{ HTTPStatusCode() int }
+ )
+
+ switch {
+ case errors.Is(err, context.Canceled):
+ return context.Canceled
+ case errors.Is(err, context.DeadlineExceeded):
+ return context.DeadlineExceeded
+ case errors.As(err, &api):
+ if classification := s3APIError(api.ErrorCode()); classification != errInvalidS3Response {
+ return classification
+ }
+ }
+
+ // The AWS SDK wraps send failures in a synthetic response with status 0.
+ // Only a real HTTP status takes precedence over the transport cause.
+ if errors.As(err, &status) && status.HTTPStatusCode() != 0 {
+ return s3HTTPError(status.HTTPStatusCode())
+ }
+
+ var (
+ connection interface{ ConnectionError() bool }
+ op *net.OpError
+ dns *net.DNSError
+ )
+
+ if (errors.As(err, &connection) && connection.ConnectionError()) || errors.As(err, &op) || errors.As(err, &dns) {
+ return racersdk.ErrUnavailable
+ }
+
+ return errInvalidS3Response
+}
+
+func s3APIError(code string) error {
+ switch code {
+ // S3 answers a GET or HEAD of a delete marker's version ID with
+ // MethodNotAllowed (405), so a delete marker reads as not found.
+ case "NoSuchKey", "NoSuchBucket", "NoSuchVersion", "NotFound", "MethodNotAllowed":
+ return racersdk.ErrNotFound
+ case "AccessDenied", "InvalidAccessKeyId", "SignatureDoesNotMatch", "ExpiredToken", "InvalidToken", "TokenRefreshRequired":
+ return racersdk.ErrForbidden
+ case "PreconditionFailed":
+ return racersdk.ErrVersionMismatch
+ case "SlowDown", "ServiceUnavailable", "InternalError", "RequestTimeout", "Throttling", "ThrottlingException":
+ return racersdk.ErrUnavailable
+ default:
+ return errInvalidS3Response
+ }
+}
+
+func s3HTTPError(code int) error {
+ switch {
+ case code == 401:
+ return racersdk.ErrUnauthorized
+ case code == 403:
+ return racersdk.ErrForbidden
+ case code == 404 || code == 405:
+ // 405 is S3's answer for a delete marker requested by version ID.
+ return racersdk.ErrNotFound
+ case code == 412:
+ return racersdk.ErrVersionMismatch
+ case code == 408 || code == 429 || code >= 500:
+ return racersdk.ErrUnavailable
+ default:
+ // An upstream 416 contradicts the range resolved from HEAD, not caller input.
+ return invalidS3Status(code)
+ }
+}
diff --git a/internal/racer/object/origin_client_test.go b/internal/racer/object/origin_client_test.go
new file mode 100644
index 0000000000..f07e358ead
--- /dev/null
+++ b/internal/racer/object/origin_client_test.go
@@ -0,0 +1,102 @@
+// Copyright (c) Microsoft Corporation.
+// SPDX-License-Identifier: Apache-2.0
+
+package object
+
+import (
+ "context"
+ "errors"
+ "testing"
+
+ "github.com/aws/aws-sdk-go-v2/service/s3"
+
+ "github.com/Azure/unbounded/pkg/racersdk"
+)
+
+type originFuncClient func()
+
+func (originFuncClient) HeadObject(context.Context, *s3.HeadObjectInput, ...func(*s3.Options)) (*s3.HeadObjectOutput, error) {
+ panic("constructor must not call HeadObject")
+}
+
+func (originFuncClient) GetObject(context.Context, *s3.GetObjectInput, ...func(*s3.Options)) (*s3.GetObjectOutput, error) {
+ panic("constructor must not call GetObject")
+}
+
+type originMapClient map[string]string
+
+func (originMapClient) HeadObject(context.Context, *s3.HeadObjectInput, ...func(*s3.Options)) (*s3.HeadObjectOutput, error) {
+ panic("constructor must not call HeadObject")
+}
+
+func (originMapClient) GetObject(context.Context, *s3.GetObjectInput, ...func(*s3.Options)) (*s3.GetObjectOutput, error) {
+ panic("constructor must not call GetObject")
+}
+
+type originSliceClient []byte
+
+func (originSliceClient) HeadObject(context.Context, *s3.HeadObjectInput, ...func(*s3.Options)) (*s3.HeadObjectOutput, error) {
+ panic("constructor must not call HeadObject")
+}
+
+func (originSliceClient) GetObject(context.Context, *s3.GetObjectInput, ...func(*s3.Options)) (*s3.GetObjectOutput, error) {
+ panic("constructor must not call GetObject")
+}
+
+type originChanClient chan struct{}
+
+func (originChanClient) HeadObject(context.Context, *s3.HeadObjectInput, ...func(*s3.Options)) (*s3.HeadObjectOutput, error) {
+ panic("constructor must not call HeadObject")
+}
+
+func (originChanClient) GetObject(context.Context, *s3.GetObjectInput, ...func(*s3.Options)) (*s3.GetObjectOutput, error) {
+ panic("constructor must not call GetObject")
+}
+
+type originValueClient int
+
+func (originValueClient) HeadObject(context.Context, *s3.HeadObjectInput, ...func(*s3.Options)) (*s3.HeadObjectOutput, error) {
+ panic("constructor must not call HeadObject")
+}
+
+func (originValueClient) GetObject(context.Context, *s3.GetObjectInput, ...func(*s3.Options)) (*s3.GetObjectOutput, error) {
+ panic("constructor must not call GetObject")
+}
+
+func TestNewOriginClientKinds(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ client S3Client
+ wantNil bool
+ }{
+ {name: "nil-interface", wantNil: true},
+ {name: "nil-pointer", client: (*originS3Stub)(nil), wantNil: true},
+ {name: "nil-function", client: originFuncClient(nil), wantNil: true},
+ {name: "nil-map", client: originMapClient(nil), wantNil: true},
+ {name: "nil-slice", client: originSliceClient(nil), wantNil: true},
+ {name: "nil-channel", client: originChanClient(nil), wantNil: true},
+ {name: "pointer", client: &originS3Stub{}},
+ {name: "function", client: originFuncClient(func() {})},
+ {name: "empty-map", client: originMapClient{}},
+ {name: "empty-slice", client: originSliceClient{}},
+ {name: "channel", client: make(originChanClient)},
+ {name: "zero-value", client: originValueClient(0)},
+ {name: "value", client: originValueClient(1)},
+ {name: "struct", client: struct{ S3Client }{S3Client: &originS3Stub{}}},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ origin, err := NewOrigin(tc.client, OriginConfig{Namespace: "store"})
+ if tc.wantNil {
+ if origin != nil || !errors.Is(err, racersdk.ErrInvalidRequest) {
+ t.Fatalf("nil client: origin nil = %t, error = %v", origin == nil, err)
+ }
+
+ return
+ }
+
+ if origin == nil || err != nil {
+ t.Fatalf("nonnil client: origin nil = %t, error = %v", origin == nil, err)
+ }
+ })
+ }
+}
diff --git a/internal/racer/object/origin_test.go b/internal/racer/object/origin_test.go
new file mode 100644
index 0000000000..b2037597c5
--- /dev/null
+++ b/internal/racer/object/origin_test.go
@@ -0,0 +1,1591 @@
+// Copyright (c) Microsoft Corporation.
+// SPDX-License-Identifier: Apache-2.0
+
+package object
+
+import (
+ "context"
+ "crypto/sha256"
+ "errors"
+ "fmt"
+ "io"
+ "math"
+ "net"
+ "net/http"
+ "net/http/httptest"
+ "net/url"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "syscall"
+ "testing"
+ "time"
+
+ "github.com/aws/aws-sdk-go-v2/aws"
+ "github.com/aws/aws-sdk-go-v2/config"
+ "github.com/aws/aws-sdk-go-v2/credentials"
+ "github.com/aws/aws-sdk-go-v2/service/s3"
+ "github.com/aws/smithy-go"
+
+ "github.com/Azure/unbounded/pkg/racersdk"
+ "github.com/Azure/unbounded/pkg/racersdk/racersdktest"
+)
+
+func TestNewRequestCanonicalIdentity(t *testing.T) {
+ want := `{"schema":1,"namespace":"store","bucket":"bucket","key":"a/../b%2Fc +雪","versionId":"v+/="}`
+
+ request, err := NewRequest("store", "bucket", "a/../b%2Fc +雪", "v+/=")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if request.Key != racersdk.Key(sha256.Sum256([]byte(want))) || request.Metadata != want || request.Authorization != "" {
+ t.Fatal("canonical identity changed")
+ }
+
+ for _, fields := range [][4]string{
+ {"other", "bucket", "a/../b%2Fc +雪", "v+/="},
+ {"store", "other", "a/../b%2Fc +雪", "v+/="},
+ {"store", "bucket", "b%2Fc +雪", "v+/="},
+ {"store", "bucket", "a/../b/c +雪", "v+/="},
+ {"store", "bucket", "a/../b%2Fc +雪", ""},
+ } {
+ other, err := NewRequest(fields[0], fields[1], fields[2], fields[3])
+ if err != nil || other.Key == request.Key {
+ t.Fatalf("identity collision: %v", err)
+ }
+ }
+
+ plain, err := NewRequest("store", "bucket", "key", "")
+ if err != nil || strings.Contains(plain.Metadata, "versionId") {
+ t.Fatal("empty version must be omitted", err)
+ }
+}
+
+func TestNewRequestDEL(t *testing.T) {
+ for _, tc := range []struct {
+ name, key, version, fields string
+ }{
+ {"key", "a\x7fb", "", `"key":"a\u007fb"`},
+ {"version", "key", "v\x7f", `"key":"key","versionId":"v\u007f"`},
+ {"literal-escape", `a\u007fb`, "", `"key":"a\\u007fb"`},
+ {"backslash-and-DEL", "\\\x7f\x7f", "", `"key":"\\\u007f\u007f"`},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ request, err := NewRequest("store", "bucket", tc.key, tc.version)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ want := `{"schema":1,"namespace":"store","bucket":"bucket",` + tc.fields + `}`
+ if request.Metadata != want || request.Key != racersdk.Key(sha256.Sum256([]byte(want))) {
+ t.Fatalf("canonical metadata = %q, want %q", request.Metadata, want)
+ }
+
+ object, err := decodeObject(racersdk.OriginRequest{Request: request}, scope{namespace: "store"})
+ if err != nil || object.Key != tc.key || object.VersionID != tc.version {
+ t.Fatalf("decoded object = %+v, error = %v", object, err)
+ }
+ })
+ }
+
+ if _, err := NewRequest("store", "bucket", strings.Repeat("\x7f", 1024), ""); err != nil {
+ t.Fatal("rejected maximum-length DEL key", err)
+ }
+
+ _, err := NewRequest("store", "bucket", strings.Repeat("\x7f", 1024), strings.Repeat("\x7f", 1024))
+ originAssertError(t, err, racersdk.ErrInvalidRequest)
+}
+
+func TestDecodeObjectDEL(t *testing.T) {
+ canonical := `{"schema":1,"namespace":"store","bucket":"bucket","key":"a\u007fb","versionId":"v\u007f"}`
+ for name, raw := range map[string]string{
+ "literal-all": strings.ReplaceAll(canonical, `\u007f`, "\x7f"),
+ "literal-key": strings.Replace(canonical, `a\u007fb`, "a\x7fb", 1),
+ "literal-version": strings.Replace(canonical, `v\u007f`, "v\x7f", 1),
+ "uppercase": strings.ReplaceAll(canonical, `\u007f`, `\u007F`),
+ "short-escape": strings.ReplaceAll(canonical, `\u007f`, `\u07f`),
+ "invalid-escape": strings.ReplaceAll(canonical, `\u007f`, `\x7f`),
+ } {
+ t.Run(name, func(t *testing.T) {
+ request := racersdk.Request{Key: sha256.Sum256([]byte(raw)), Metadata: raw}
+ _, err := decodeObject(racersdk.OriginRequest{Request: request}, scope{namespace: "store"})
+ originAssertError(t, err, racersdk.ErrInvalidRequest)
+
+ client := originClient(t, &originS3Stub{head: func(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
+ t.Error("noncanonical DEL identity reached S3")
+ return nil, nil
+ }}, OriginConfig{Namespace: "store"})
+ _, err = client.Stat(t.Context(), request)
+ originAssertError(t, err, racersdk.ErrInvalidRequest)
+ })
+ }
+}
+
+func TestNewRequestValidation(t *testing.T) {
+ for _, fields := range [][4]string{
+ {"", "bucket", "key", ""},
+ {"with space", "bucket", "key", ""},
+ {"store\n", "bucket", "key", ""},
+ {"\xff", "bucket", "key", ""},
+ {"store", "", "key", ""},
+ {"store", "https://evil", "key", ""},
+ {"store", "../bucket", "key", ""},
+ {"store", "bucket", "", ""},
+ {"store", "bucket", strings.Repeat("k", 1025), ""},
+ {"store", "bucket", "\xff", ""},
+ {"store", "bucket", "key", "\xff"},
+ {"store", "bucket", "key", strings.Repeat("v", 1025)},
+ {strings.Repeat("n", 257), "bucket", "key", ""},
+ {"store\u00a0", "bucket", "key", ""},
+ {"store\x00", "bucket", "key", ""},
+ {"store", ".", "key", ""},
+ {"store", "..", "key", ""},
+ {"store", strings.Repeat("b", 256), "key", ""},
+ {"store", "buckét", "key", ""},
+ } {
+ if _, err := NewRequest(fields[0], fields[1], fields[2], fields[3]); err == nil {
+ t.Fatalf("accepted invalid fields %q", fields)
+ }
+ }
+
+ for _, key := range []string{"/", ".", "../", "a//b", "a\x00b", "line\nbreak", strings.Repeat("k", 1024)} {
+ if _, err := NewRequest("store", "bucket", key, "null"); err != nil {
+ t.Fatalf("rejected exact key %q: %v", key, err)
+ }
+ }
+
+ for _, fields := range [][4]string{
+ {strings.Repeat("n", 256), strings.Repeat("b", 255), strings.Repeat("k", 1024), strings.Repeat("v", 1024)},
+ {"雪", "Legacy_Bucket-9.example", "key", "v\x00\n雪"},
+ } {
+ if _, err := NewRequest(fields[0], fields[1], fields[2], fields[3]); err != nil {
+ t.Fatalf("rejected valid boundary fields %q: %v", fields, err)
+ }
+ }
+
+ // Both strings fit the object limits, but escaping exceeds the SDK field cap.
+ _, err := NewRequest("store", "bucket", strings.Repeat("\x00", 1024), strings.Repeat("\x00", 1024))
+ originAssertError(t, err, racersdk.ErrInvalidRequest)
+}
+
+type originS3Stub struct {
+ head func(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error)
+ get func(context.Context, *s3.GetObjectInput) (*s3.GetObjectOutput, error)
+}
+
+func (s *originS3Stub) HeadObject(ctx context.Context, in *s3.HeadObjectInput, _ ...func(*s3.Options)) (*s3.HeadObjectOutput, error) {
+ return s.head(ctx, in)
+}
+
+func (s *originS3Stub) GetObject(ctx context.Context, in *s3.GetObjectInput, _ ...func(*s3.Options)) (*s3.GetObjectOutput, error) {
+ return s.get(ctx, in)
+}
+
+func originHead(size int64) *s3.HeadObjectOutput {
+ return &s3.HeadObjectOutput{ContentLength: aws.Int64(size), ETag: aws.String(`"opaque-multipart-2"`), ContentType: aws.String("application/test")}
+}
+
+func originPage(body io.ReadCloser, first, last, total int64) *s3.GetObjectOutput {
+ return &s3.GetObjectOutput{
+ Body: body, ContentLength: aws.Int64(last - first + 1), ETag: aws.String(`"opaque-multipart-2"`),
+ ContentType: aws.String("application/test"), ContentRange: aws.String(fmt.Sprintf("bytes %d-%d/%d", first, last, total)),
+ }
+}
+
+func originClient(t *testing.T, stub S3Client, config OriginConfig) *racersdk.Client {
+ t.Helper()
+
+ origin, err := NewOrigin(stub, config)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ return testSDKClient(t, origin)
+}
+
+// testSDKClient starts an in-process racersdk client that is closed with the test.
+func testSDKClient(t *testing.T, origin racersdk.Origin) *racersdk.Client {
+ t.Helper()
+
+ return racersdktest.NewClient(t, origin)
+}
+
+// testS3Client returns a real, signing AWS S3 client that talks to server
+// with default checksum settings, path-style addressing, and no retries.
+func testS3Client(t *testing.T, server *httptest.Server) *s3.Client {
+ t.Helper()
+ t.Setenv("AWS_RESPONSE_CHECKSUM_VALIDATION", "")
+
+ cfg, err := config.LoadDefaultConfig(t.Context(),
+ config.WithRegion("us-east-1"),
+ config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider("placeholder", "placeholder", "")),
+ config.WithSharedConfigFiles([]string{}), config.WithSharedCredentialsFiles([]string{}),
+ config.WithHTTPClient(server.Client()), config.WithRetryMaxAttempts(1),
+ )
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ return s3.NewFromConfig(cfg, func(o *s3.Options) {
+ o.BaseEndpoint = aws.String(server.URL)
+ o.UsePathStyle = true
+ })
+}
+
+func objectRequest(t *testing.T, version string) racersdk.Request {
+ t.Helper()
+
+ request, err := NewRequest("store", "bucket", "a/../exact%2F +雪", version)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ return request
+}
+
+func originAssertError(t *testing.T, err, want error) {
+ t.Helper()
+
+ if want == errInvalidS3Response {
+ if err == nil || sidecarSDKFailure(err) != http.StatusBadGateway {
+ t.Fatalf("error = %v, want unclassified origin failure", err)
+ }
+
+ return
+ }
+
+ if !errors.Is(err, want) {
+ t.Fatalf("error = %v, want %v", err, want)
+ }
+}
+
+func originReadValue(t *testing.T, client *racersdk.Client, request racersdk.Request, options ...racersdk.ReadOptions) ([]byte, error) {
+ t.Helper()
+
+ value, err := client.Get(t.Context(), request, options...)
+ if err != nil {
+ return nil, err
+ }
+ defer value.Close()
+
+ return io.ReadAll(value)
+}
+
+func TestOriginDELBridge(t *testing.T) {
+ const key, version = "a/\x7f/\\u007f/雪", "v\x7f"
+
+ stub := &originS3Stub{
+ head: func(_ context.Context, in *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
+ if aws.ToString(in.Key) != key || aws.ToString(in.VersionId) != version {
+ t.Error("HEAD changed DEL identity")
+ }
+
+ out := originHead(5)
+ out.VersionId = aws.String(version)
+
+ return out, nil
+ },
+ get: func(_ context.Context, in *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
+ if aws.ToString(in.Key) != key || aws.ToString(in.VersionId) != version {
+ t.Error("GET changed DEL identity")
+ }
+
+ out := originPage(io.NopCloser(strings.NewReader("hello")), 0, 4, 5)
+ out.VersionId = aws.String(version)
+
+ return out, nil
+ },
+ }
+ client := originClient(t, stub, OriginConfig{Namespace: "store"})
+
+ request, err := NewRequest("store", "bucket", key, version)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ metadata, err := client.Stat(t.Context(), request)
+ if err != nil || metadata.Size != 5 {
+ t.Fatal("DEL HEAD", metadata, err)
+ }
+
+ got, err := originReadValue(t, client, request)
+ if err != nil || string(got) != "hello" {
+ t.Fatalf("DEL GET = %q, error = %v", got, err)
+ }
+}
+
+func TestOriginBridgeHeadBootstrapPinnedAndEmpty(t *testing.T) {
+ for _, version := range []string{"", "version+/="} {
+ t.Run(version, func(t *testing.T) {
+ var heads, gets atomic.Int32
+
+ stub := &originS3Stub{}
+ stub.head = func(ctx context.Context, in *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
+ heads.Add(1)
+
+ if (in.VersionId == nil) != (version == "") {
+ t.Error("HEAD changed absent version to an empty parameter")
+ }
+
+ if ctx.Err() != nil || aws.ToString(in.Bucket) != "bucket" || aws.ToString(in.Key) != "a/../exact%2F +雪" || aws.ToString(in.VersionId) != version {
+ t.Error("HEAD changed exact identity")
+ }
+
+ head := originHead(int64(racersdk.PageSize) + 3)
+ head.VersionId = aws.String(version)
+
+ return head, nil
+ }
+ stub.get = func(_ context.Context, in *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
+ gets.Add(1)
+
+ if (in.VersionId == nil) != (version == "") {
+ t.Error("GET changed absent version to an empty parameter")
+ }
+
+ if aws.ToString(in.IfMatch) != `"opaque-multipart-2"` || aws.ToString(in.VersionId) != version || aws.ToString(in.Key) != "a/../exact%2F +雪" || aws.ToString(in.Bucket) != "bucket" || aws.ToString(in.Range) != "bytes=16777216-16777218" {
+ t.Error("GET was not pinned to exact identity/range")
+ }
+
+ out := originPage(io.NopCloser(strings.NewReader("end")), int64(racersdk.PageSize), int64(racersdk.PageSize)+2, int64(racersdk.PageSize)+3)
+ out.VersionId = aws.String(version)
+
+ return out, nil
+ }
+ client := originClient(t, stub, OriginConfig{Namespace: "store", MetadataTTL: 30 * time.Second})
+ request := objectRequest(t, version)
+ before := time.Now()
+
+ metadata, err := client.Stat(t.Context(), request)
+ if err != nil || metadata.Size != racersdk.PageSize+3 || metadata.ContentType != "application/test" || metadata.ExpiresAt.Before(before.Add(29*time.Second)) || metadata.ExpiresAt.Nanosecond()%int(time.Millisecond) != 0 {
+ t.Fatal("HEAD metadata", metadata, err)
+ }
+
+ got, err := originReadValue(t, client, request, racersdk.ReadOptions{Offset: racersdk.PageSize, Length: 3, ETag: metadata.ETag})
+ if err != nil || string(got) != "end" || heads.Load() != 3 || gets.Load() != 1 {
+ t.Fatalf("pinned page = %q, %v; calls %d/%d", got, err, heads.Load(), gets.Load())
+ }
+ })
+ }
+
+ for _, data := range []string{"", "hello"} {
+ t.Run("bootstrap/"+data, func(t *testing.T) {
+ stub := &originS3Stub{head: func(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
+ return originHead(int64(len(data))), nil
+ }}
+ stub.get = func(_ context.Context, in *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
+ if data == "" || aws.ToString(in.IfMatch) != `"opaque-multipart-2"` || aws.ToString(in.Range) != "bytes=0-4" {
+ t.Error("bad bootstrap GET")
+ }
+
+ return originPage(io.NopCloser(strings.NewReader(data)), 0, int64(len(data))-1, int64(len(data))), nil
+ }
+ client := originClient(t, stub, OriginConfig{Namespace: "store"})
+
+ got, err := originReadValue(t, client, objectRequest(t, ""))
+ if err != nil || string(got) != data {
+ t.Fatalf("bootstrap = %q, %v", got, err)
+ }
+
+ metadata, err := client.Stat(t.Context(), objectRequest(t, ""))
+ if err != nil || metadata.ExpiresAt.After(time.Now()) || metadata.ExpiresAt.Before(time.Now().Add(-time.Second)) {
+ t.Fatal("zero TTL was defaulted", metadata, err)
+ }
+ })
+ }
+}
+
+func TestOriginStrictIdentityBridge(t *testing.T) {
+ canonical := objectRequest(t, "").Metadata
+ for name, raw := range map[string]string{
+ "absent": "", "null": "null", "unknown": strings.TrimSuffix(canonical, "}") + `,"endpoint":"https://evil"}`,
+ "duplicate": strings.Replace(canonical, `"schema":1`, `"schema":1,"schema":1`, 1),
+ "case": strings.Replace(canonical, "schema", "Schema", 1), "whitespace": strings.Replace(canonical, `"schema":1`, `"schema": 1`, 1),
+ "schema": strings.Replace(canonical, `"schema":1`, `"schema":2`, 1),
+ "trailing": canonical + "{}", "empty-version": strings.TrimSuffix(canonical, "}") + `,"versionId":""}`,
+ "escaped": strings.Replace(canonical, "store", `\u0073tore`, 1),
+ } {
+ t.Run(name, func(t *testing.T) {
+ stub := &originS3Stub{head: func(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
+ t.Error("invalid identity reached S3")
+ return nil, nil
+ }}
+ client := originClient(t, stub, OriginConfig{Namespace: "store"})
+
+ _, err := client.Stat(t.Context(), racersdk.Request{Key: sha256.Sum256([]byte(raw)), Metadata: raw})
+ originAssertError(t, err, racersdk.ErrInvalidRequest)
+ })
+ }
+
+ for _, mode := range []string{"key", "namespace", "bucket"} {
+ t.Run(mode, func(t *testing.T) {
+ stub := &originS3Stub{head: func(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
+ t.Error("forbidden request reached S3")
+ return nil, nil
+ }}
+ config := OriginConfig{Namespace: "store"}
+ request := objectRequest(t, "")
+ want := racersdk.ErrForbidden
+
+ switch mode {
+ case "key":
+ request.Key[0] ^= 1
+ want = racersdk.ErrInvalidRequest
+ case "namespace":
+ config.Namespace = "other"
+ case "bucket":
+ config.Buckets = []string{"other"}
+ }
+
+ client := originClient(t, stub, config)
+ _, err := client.Stat(t.Context(), request)
+ originAssertError(t, err, want)
+ })
+ }
+}
+
+func TestOriginConfigValidation(t *testing.T) {
+ for _, config := range []OriginConfig{{}, {Namespace: "bad space"}, {Namespace: "store", MetadataTTL: -1}, {Namespace: "store", Buckets: []string{""}}, {Namespace: "store", Buckets: []string{"https://evil"}}} {
+ _, err := NewOrigin(&originS3Stub{}, config)
+ originAssertError(t, err, racersdk.ErrInvalidRequest)
+ }
+
+ for _, client := range []S3Client{nil, (*originS3Stub)(nil)} {
+ _, err := NewOrigin(client, OriginConfig{Namespace: "store"})
+ originAssertError(t, err, racersdk.ErrInvalidRequest)
+ }
+}
+
+func TestOriginInvalidHeadMetadata(t *testing.T) {
+ for _, mode := range []string{"nil", "length-nil", "negative", "tag-nil", "weak-tag", "type", "range", "deleted", "version"} {
+ t.Run(mode, func(t *testing.T) {
+ stub := &originS3Stub{head: func(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
+ head := originHead(5)
+
+ switch mode {
+ case "nil":
+ return nil, nil
+ case "length-nil":
+ head.ContentLength = nil
+ case "negative":
+ head.ContentLength = aws.Int64(-1)
+ case "tag-nil":
+ head.ETag = nil
+ case "weak-tag":
+ head.ETag = aws.String(`W/"weak"`)
+ case "type":
+ head.ContentType = aws.String("bad\r\nheader")
+ case "range":
+ head.ContentRange = aws.String("bytes 0-4/5")
+ case "deleted":
+ head.DeleteMarker = aws.Bool(true)
+ }
+
+ return head, nil
+ }}
+ client := originClient(t, stub, OriginConfig{Namespace: "store"})
+
+ version := ""
+ if mode == "version" {
+ version = "requested"
+ }
+
+ metadata, err := client.Stat(t.Context(), objectRequest(t, version))
+ if mode == "type" {
+ if err != nil || metadata.ContentType != "" || metadata.Size != 5 {
+ t.Fatalf("unsupported content type was not omitted: %+v, %v", metadata, err)
+ }
+
+ return
+ }
+
+ originAssertError(t, err, errInvalidS3Response)
+ })
+ }
+}
+
+type originTrackedBody struct {
+ io.Reader
+ closed atomic.Int32
+}
+
+func (b *originTrackedBody) Close() error { b.closed.Add(1); return nil }
+
+func TestOriginRejectsGetMetadataAndClosesBodies(t *testing.T) {
+ for _, mode := range []string{"length", "nil-length", "range", "total", "etag", "missing-etag", "weak-etag", "type", "version", "encoding", "deleted", "nil", "nil-body", "error-body", "short", "long"} {
+ t.Run(mode, func(t *testing.T) {
+ body := &originTrackedBody{Reader: strings.NewReader("hello")}
+ stub := &originS3Stub{head: func(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) { return originHead(5), nil }}
+ stub.get = func(context.Context, *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
+ out := originPage(body, 0, 4, 5)
+
+ switch mode {
+ case "length":
+ out.ContentLength = aws.Int64(4)
+ case "nil-length":
+ out.ContentLength = nil
+ case "range":
+ out.ContentRange = nil
+ case "total":
+ out.ContentRange = aws.String("bytes 0-4/6")
+ case "etag":
+ out.ETag = aws.String(`"changed"`)
+ case "missing-etag":
+ out.ETag = nil
+ case "weak-etag":
+ out.ETag = aws.String(`W/"opaque-multipart-2"`)
+ case "type":
+ out.ContentType = aws.String("text/plain")
+ case "version":
+ out.VersionId = aws.String("unexpected")
+ case "encoding":
+ out.ContentEncoding = aws.String("gzip")
+ case "deleted":
+ out.DeleteMarker = aws.Bool(true)
+ case "nil":
+ return nil, nil
+ case "nil-body":
+ out.Body = nil
+ case "error-body":
+ return out, &smithy.GenericAPIError{Code: "AccessDenied"}
+ case "short":
+ body.Reader = strings.NewReader("hell")
+ case "long":
+ body.Reader = strings.NewReader("hello!")
+ }
+
+ return out, nil
+ }
+ client := originClient(t, stub, OriginConfig{Namespace: "store"})
+
+ got, err := originReadValue(t, client, objectRequest(t, ""))
+ if mode == "short" || mode == "long" {
+ if err == nil || len(got) >= 5 {
+ t.Fatalf("bad body accepted: %q %v", got, err)
+ }
+ } else {
+ want := errInvalidS3Response
+ if mode == "etag" {
+ // The request is unpinned, so a changed GET ETag is an overwrite race.
+ want = racersdk.ErrUnavailable
+ }
+
+ if mode == "error-body" {
+ want = racersdk.ErrForbidden
+ }
+
+ originAssertError(t, err, want)
+ }
+
+ if mode != "nil" && mode != "nil-body" {
+ deadline := time.Now().Add(time.Second)
+ for body.closed.Load() == 0 && time.Now().Before(deadline) {
+ time.Sleep(time.Millisecond)
+ }
+
+ if body.closed.Load() != 1 {
+ t.Fatal("body not closed exactly once", body.closed.Load())
+ }
+ }
+ })
+ }
+}
+
+type originStatusError struct{ code int }
+
+func (e originStatusError) Error() string { return "private upstream URL/credential" }
+func (e originStatusError) HTTPStatusCode() int { return e.code }
+
+func TestOriginErrorClassification(t *testing.T) {
+ for _, tt := range []struct {
+ err error
+ want error
+ }{
+ {context.Canceled, context.Canceled},
+ {context.DeadlineExceeded, context.DeadlineExceeded},
+ {&smithy.GenericAPIError{Code: "NoSuchKey"}, racersdk.ErrNotFound},
+ {&smithy.GenericAPIError{Code: "NoSuchVersion"}, racersdk.ErrNotFound},
+ {&smithy.GenericAPIError{Code: "NoSuchBucket"}, racersdk.ErrNotFound},
+ {&smithy.GenericAPIError{Code: "NotFound"}, racersdk.ErrNotFound},
+ {&smithy.GenericAPIError{Code: "MethodNotAllowed"}, racersdk.ErrNotFound},
+ {&smithy.GenericAPIError{Code: "AccessDenied"}, racersdk.ErrForbidden},
+ {&smithy.GenericAPIError{Code: "InvalidAccessKeyId"}, racersdk.ErrForbidden},
+ {&smithy.GenericAPIError{Code: "SignatureDoesNotMatch"}, racersdk.ErrForbidden},
+ {&smithy.GenericAPIError{Code: "ExpiredToken"}, racersdk.ErrForbidden},
+ {&smithy.GenericAPIError{Code: "InvalidToken"}, racersdk.ErrForbidden},
+ {&smithy.GenericAPIError{Code: "TokenRefreshRequired"}, racersdk.ErrForbidden},
+ {&smithy.GenericAPIError{Code: "PreconditionFailed"}, racersdk.ErrVersionMismatch},
+ {&smithy.GenericAPIError{Code: "SlowDown"}, racersdk.ErrUnavailable},
+ {&smithy.GenericAPIError{Code: "ServiceUnavailable"}, racersdk.ErrUnavailable},
+ {&smithy.GenericAPIError{Code: "InternalError"}, racersdk.ErrUnavailable},
+ {&smithy.GenericAPIError{Code: "RequestTimeout"}, racersdk.ErrUnavailable},
+ {&smithy.GenericAPIError{Code: "Throttling"}, racersdk.ErrUnavailable},
+ {&smithy.GenericAPIError{Code: "ThrottlingException"}, racersdk.ErrUnavailable},
+ {&smithy.GenericAPIError{Code: "Unknown"}, errInvalidS3Response},
+ {originStatusError{401}, racersdk.ErrUnauthorized},
+ {originStatusError{403}, racersdk.ErrForbidden},
+ {originStatusError{404}, racersdk.ErrNotFound},
+ {originStatusError{405}, racersdk.ErrNotFound},
+ {originStatusError{406}, errInvalidS3Response},
+ {originStatusError{412}, racersdk.ErrVersionMismatch},
+ {originStatusError{416}, errInvalidS3Response},
+ {originStatusError{408}, racersdk.ErrUnavailable},
+ {originStatusError{499}, errInvalidS3Response},
+ {originStatusError{429}, racersdk.ErrUnavailable},
+ {originStatusError{500}, racersdk.ErrUnavailable},
+ {originStatusError{600}, racersdk.ErrUnavailable},
+ {errors.New("private upstream URL/credential"), errInvalidS3Response},
+ } {
+ for _, pinned := range []bool{false, true} {
+ want := tt.want
+ if pinned && want == racersdk.ErrNotFound {
+ want = racersdk.ErrVersionMismatch
+ }
+
+ err := classifyS3Error(fmt.Errorf("wrapped: %w", tt.err), pinned)
+ originAssertError(t, err, want)
+
+ if strings.Contains(err.Error(), "private") || !errors.Is(err, tt.err) {
+ t.Fatal("unsafe or lost cause", err)
+ }
+ }
+ }
+}
+
+func TestOriginErrorClassificationPrecedence(t *testing.T) {
+ for _, tt := range []struct {
+ name string
+ err error
+ want error
+ }{
+ {"cancellation", errors.Join(context.Canceled, context.DeadlineExceeded, &smithy.GenericAPIError{Code: "AccessDenied"}, originStatusError{500}), context.Canceled},
+ {"deadline", errors.Join(context.DeadlineExceeded, &smithy.GenericAPIError{Code: "NoSuchKey"}, originStatusError{500}), context.DeadlineExceeded},
+ {"known-api", errors.Join(&smithy.GenericAPIError{Code: "AccessDenied"}, originStatusError{404}), racersdk.ErrForbidden},
+ {"unknown-api", errors.Join(&smithy.GenericAPIError{Code: "Unknown"}, originStatusError{404}), racersdk.ErrNotFound},
+ {"unknown-api-and-status", errors.Join(&smithy.GenericAPIError{Code: "Unknown"}, originStatusError{416}), errInvalidS3Response},
+ {"transport-cancellation", &net.OpError{Op: "dial", Net: "tcp", Err: context.Canceled}, context.Canceled},
+ {"transport-deadline", &net.OpError{Op: "dial", Net: "tcp", Err: context.DeadlineExceeded}, context.DeadlineExceeded},
+ {"transport-known-api", errors.Join(&net.OpError{Err: syscall.ECONNRESET}, &smithy.GenericAPIError{Code: "AccessDenied"}), racersdk.ErrForbidden},
+ {"transport-status", errors.Join(&net.OpError{Err: syscall.ECONNRESET}, originStatusError{404}), racersdk.ErrNotFound},
+ {"transport-invalid-status", errors.Join(&net.OpError{Err: syscall.ECONNRESET}, originStatusError{416}), errInvalidS3Response},
+ {"transport-unknown-api", errors.Join(&net.OpError{Err: syscall.ECONNRESET}, &smithy.GenericAPIError{Code: "Unknown"}), racersdk.ErrUnavailable},
+ } {
+ t.Run(tt.name, func(t *testing.T) {
+ for _, pinned := range []bool{false, true} {
+ want := tt.want
+ if pinned && want == racersdk.ErrNotFound {
+ want = racersdk.ErrVersionMismatch
+ }
+
+ err := classifyS3Error(tt.err, pinned)
+ originAssertError(t, err, want)
+
+ if !errors.Is(err, tt.err) || strings.Contains(err.Error(), "private") {
+ t.Fatal("unsafe or lost cause", err)
+ }
+ }
+ })
+ }
+}
+
+func TestOriginTransportErrorClassification(t *testing.T) {
+ for _, tt := range []struct {
+ name string
+ err error
+ want error
+ }{
+ {"dns", &net.DNSError{Err: "no such host", Name: "private.invalid", IsNotFound: true}, racersdk.ErrUnavailable},
+ {"dial-dns", &net.OpError{Op: "dial", Net: "tcp", Err: &net.DNSError{Err: "no such host", Name: "private.invalid", IsNotFound: true}}, racersdk.ErrUnavailable},
+ {"refused", &net.OpError{Op: "dial", Net: "tcp", Err: syscall.ECONNREFUSED}, racersdk.ErrUnavailable},
+ {"reset", &net.OpError{Op: "read", Net: "tcp", Err: syscall.ECONNRESET}, racersdk.ErrUnavailable},
+ {"timeout", &net.OpError{Op: "read", Net: "tcp", Err: context.DeadlineExceeded}, context.DeadlineExceeded},
+ {"non-transport", errors.New("private request configuration failure"), errInvalidS3Response},
+ } {
+ t.Run(tt.name, func(t *testing.T) {
+ for _, pinned := range []bool{false, true} {
+ wrapped := &smithy.OperationError{ServiceID: "S3", OperationName: "GetObject", Err: &url.Error{Op: "Get", URL: "https://private.invalid/secret", Err: tt.err}}
+ err := classifyS3Error(wrapped, pinned)
+ originAssertError(t, err, tt.want)
+
+ if !errors.Is(err, wrapped) || !errors.Is(err, tt.err) || strings.Contains(err.Error(), "private") || strings.Contains(err.Error(), "secret") {
+ t.Fatal("unsafe or lost cause", err)
+ }
+
+ wantStatus := http.StatusServiceUnavailable
+ if tt.want == errInvalidS3Response {
+ wantStatus = http.StatusBadGateway
+ }
+
+ if got := int(sidecarSDKFailure(err)); got != wantStatus {
+ t.Fatalf("sidecar status = %d, want %d", got, wantStatus)
+ }
+ }
+ })
+ }
+}
+
+func TestOriginReadGuards(t *testing.T) {
+ origin := &objectOrigin{}
+
+ for _, tt := range []struct {
+ name string
+ cause error
+ want error
+ }{
+ {"invalid-identity", nil, racersdk.ErrInvalidRequest},
+ {"canceled-before-identity", context.Canceled, context.Canceled},
+ {"deadline-before-identity", context.DeadlineExceeded, context.DeadlineExceeded},
+ } {
+ t.Run(tt.name, func(t *testing.T) {
+ ctx := t.Context()
+
+ switch tt.cause {
+ case context.Canceled:
+ canceled, cancel := context.WithCancel(ctx)
+ cancel()
+
+ ctx = canceled
+ case context.DeadlineExceeded:
+ expired, cancel := context.WithDeadline(ctx, time.Now().Add(-time.Second))
+ defer cancel()
+
+ ctx = expired
+ }
+
+ metadata, body, err := origin.read(ctx, racersdk.OriginRequest{})
+ originAssertError(t, err, tt.want)
+
+ if metadata != (racersdk.Metadata{}) || body != nil || (tt.cause != nil && !errors.Is(err, tt.cause)) {
+ t.Fatal("invalid request exposed metadata/body or lost cause", metadata, body, err)
+ }
+ })
+ }
+
+ // The SDK normally prevents this request; keep the defensive page guard tested.
+ body, err := origin.readPage(t.Context(), racersdk.OriginRequest{}, nil, nil, racersdk.Metadata{})
+ originAssertError(t, err, racersdk.ErrInvalidRequest)
+
+ if body != nil {
+ t.Fatal("missing range returned a body")
+ }
+}
+
+func TestOriginHeadMaximumSizeAndTTLPrecision(t *testing.T) {
+ origin := &objectOrigin{ttl: time.Nanosecond}
+
+ metadata, err := origin.headMetadata(originHead(math.MaxInt64), Object{})
+ if err != nil || metadata.Size != math.MaxInt64 || metadata.ExpiresAt.Nanosecond()%int(time.Millisecond) != 0 {
+ t.Fatal("maximum full size or TTL precision", metadata, err)
+ }
+}
+
+func TestOriginPageRangeBounds(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ size, offset, length int64
+ want error
+ }{
+ {"empty", 0, 0, racersdk.PageSize, nil},
+ {"past-end", 5, racersdk.PageSize, racersdk.PageSize, nil},
+ {"at-end", racersdk.PageSize, racersdk.PageSize, racersdk.PageSize, nil},
+ {"negative", 5, -racersdk.PageSize, racersdk.PageSize, racersdk.ErrInvalidRequest},
+ {"unaligned", 5, 1, racersdk.PageSize, racersdk.ErrInvalidRequest},
+ {"zero-length", 5, 0, 0, racersdk.ErrInvalidRequest},
+ {"negative-length", 5, 0, -1, racersdk.ErrInvalidRequest},
+ {"oversized", 5, 0, racersdk.PageSize + 1, racersdk.ErrInvalidRequest},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ origin := &objectOrigin{}
+
+ body, err := origin.readPage(t.Context(), racersdk.OriginRequest{Offset: tc.offset, Length: tc.length}, nil, nil, racersdk.Metadata{Size: tc.size})
+ if body != nil || !errors.Is(err, tc.want) {
+ t.Fatalf("body=%v error=%v, want nil body and %v", body, err, tc.want)
+ }
+ })
+ }
+
+ first := int64(math.MaxInt64 / racersdk.PageSize * racersdk.PageSize)
+ head := originHead(math.MaxInt64)
+ origin := &objectOrigin{client: &originS3Stub{
+ get: func(_ context.Context, input *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
+ if got, want := aws.ToString(input.Range), fmt.Sprintf("bytes=%d-%d", first, int64(math.MaxInt64-1)); got != want {
+ t.Fatalf("range=%s, want %s", got, want)
+ }
+
+ return originPage(io.NopCloser(strings.NewReader("")), first, math.MaxInt64-1, math.MaxInt64), nil
+ },
+ }}
+
+ body, err := origin.readPage(t.Context(), racersdk.OriginRequest{Offset: first, Length: racersdk.PageSize}, &s3.HeadObjectInput{}, head, racersdk.Metadata{
+ Size: math.MaxInt64, ETag: aws.ToString(head.ETag), ContentType: aws.ToString(head.ContentType),
+ })
+ if err != nil || body == nil {
+ t.Fatalf("overflow-safe range: body=%v error=%v", body, err)
+ }
+ defer body.Close()
+}
+
+func TestOriginAWSClientBridge(t *testing.T) {
+ var heads, gets atomic.Int32
+
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != "/bucket/a/../exact%2F +雪" || r.URL.Query().Get("versionId") != "v+/=" || !strings.HasPrefix(r.Header.Get("Authorization"), "AWS4-HMAC-SHA256 ") {
+ t.Error("AWS client changed object identity or omitted signing")
+ }
+
+ w.Header().Set("ETag", `"opaque-multipart-2"`)
+ w.Header().Set("Content-Type", "application/test")
+ w.Header().Set("Content-Length", "5")
+ w.Header().Set("x-amz-version-id", "v+/=")
+
+ switch r.Method {
+ case http.MethodHead:
+ heads.Add(1)
+ case http.MethodGet:
+ gets.Add(1)
+
+ if r.Header.Get("Range") != "bytes=0-4" || r.Header.Get("If-Match") != `"opaque-multipart-2"` {
+ t.Error("AWS GET missing range/pin")
+ }
+
+ w.Header().Set("Content-Range", "bytes 0-4/5")
+ w.WriteHeader(http.StatusPartialContent)
+
+ if _, err := io.WriteString(w, "hello"); err != nil {
+ t.Error(err)
+ }
+ default:
+ t.Error("unexpected S3 method")
+ w.WriteHeader(http.StatusMethodNotAllowed)
+ }
+ }))
+ t.Cleanup(server.Close)
+
+ client := originClient(t, testS3Client(t, server), OriginConfig{Namespace: "store", Buckets: []string{"bucket"}})
+
+ got, err := originReadValue(t, client, objectRequest(t, "v+/="))
+ if err != nil || string(got) != "hello" || heads.Load() != 1 || gets.Load() != 1 {
+ t.Fatalf("AWS bridge = %q, %v, calls %d/%d", got, err, heads.Load(), gets.Load())
+ }
+}
+
+func TestNormalizeS3ContentType(t *testing.T) {
+ for _, value := range []string{
+ "", "text/plain", "application/a+b; charset=utf-8",
+ `text/plain ; a = ""; b="escaped\"quote"`, `text/plain;a="a b"`,
+ "application/" + strings.Repeat("x", 244),
+ "a/!#$%&'*+-.^_`|~", `text/plain;name*=utf-8''file`,
+ } {
+ if got := normalizeS3ContentType(value); got != value {
+ t.Fatalf("normalized supported type %q to %q", value, got)
+ }
+ }
+
+ for _, value := range []string{
+ "application/" + strings.Repeat("x", 245), " text/plain", "text/plain ",
+ "text/\x7f", "text/\xff", "text/\tplain", "text/plain\x00", "bad\r\nheader",
+ "text", "/plain", "text/", "text/plain extra", "text/plain;", "text/plain;=x",
+ "text/plain;a", "text/plain;a=", "text/plain;a=x;A=y", "text/plain;a=x;a=x",
+ `text/plain;a="`, `text/plain;a="x\`,
+ } {
+ if got := normalizeS3ContentType(value); got != "" {
+ t.Fatalf("unsupported type %q survived as %q", value, got)
+ }
+ }
+}
+
+func TestOriginAWSClientContentType(t *testing.T) {
+ boundary := "application/" + strings.Repeat("x", 244)
+ for _, tc := range []struct {
+ name, head, get, want string
+ mismatch bool
+ }{
+ {"absent", "", "", "", false},
+ {"supported", "text/plain", "text/plain", "text/plain", false},
+ {"parameters", `text/plain ; a = ""; b="escaped\"quote"`, `text/plain ; a = ""; b="escaped\"quote"`, `text/plain ; a = ""; b="escaped\"quote"`, false},
+ {"quoted-space", `text/plain;a="a b"`, `text/plain;a="a b"`, `text/plain;a="a b"`, false},
+ {"suffix", "application/a+b; charset=utf-8", "application/a+b; charset=utf-8", "application/a+b; charset=utf-8", false},
+ {"boundary", boundary, boundary, boundary, false},
+ {"too-long", boundary + "x", boundary + "x", "", false},
+ {"non-ascii", `text/plain;name="café"`, `text/plain;name="café"`, "", false},
+ {"tab", "text/plain;\tcharset=utf-8", "text/plain;\tcharset=utf-8", "", false},
+ {"missing-slash", "text", "text", "", false},
+ {"missing-type", "/plain", "/plain", "", false},
+ {"missing-subtype", "text/", "text/", "", false},
+ {"extra-token", "text/plain extra", "text/plain extra", "", false},
+ {"trailing-semicolon", "text/plain;", "text/plain;", "", false},
+ {"missing-name", "text/plain;=x", "text/plain;=x", "", false},
+ {"missing-equals", "text/plain;a", "text/plain;a", "", false},
+ {"missing-value", "text/plain;a=", "text/plain;a=", "", false},
+ {"duplicate-parameter", "text/plain;a=x;A=x", "text/plain;a=x;A=x", "", false},
+ {"unclosed-quote", `text/plain;a="`, `text/plain;a="`, "", false},
+ {"unfinished-escape", `text/plain;a="x\`, `text/plain;a="x\`, "", false},
+ {"supported-mismatch", "text/plain", "application/test", "text/plain", true},
+ {"unsupported-mismatch", "text", "other", "", true},
+ {"unsupported-to-absent", "text", "", "", true},
+ {"absent-to-unsupported", "", "text", "", true},
+ {"supported-to-unsupported", "text/plain", "text", "text/plain", true},
+ {"unsupported-to-supported", "text", "text/plain", "", true},
+ } {
+ for _, operation := range []string{"HEAD", "bootstrap", "pinned"} {
+ t.Run(tc.name+"/"+operation, func(t *testing.T) {
+ var heads, gets atomic.Int32
+
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("ETag", `"opaque-multipart-2"`)
+ w.Header().Set("Content-Length", "5")
+ w.Header().Set("Content-Type", tc.head)
+
+ switch r.Method {
+ case http.MethodHead:
+ heads.Add(1)
+
+ if operation == "pinned" && r.Header.Get("If-Match") != `"opaque-multipart-2"` {
+ t.Error("pinned HEAD omitted If-Match")
+ }
+ case http.MethodGet:
+ gets.Add(1)
+
+ if r.Header.Get("Range") != "bytes=0-4" || r.Header.Get("If-Match") != `"opaque-multipart-2"` {
+ t.Error("GET missing range/pin")
+ }
+
+ w.Header().Set("Content-Type", tc.get)
+ w.Header().Set("Content-Range", "bytes 0-4/5")
+ w.WriteHeader(http.StatusPartialContent)
+
+ if _, err := io.WriteString(w, "hello"); err != nil {
+ t.Error(err)
+ }
+ default:
+ t.Error("unexpected S3 method")
+ w.WriteHeader(http.StatusMethodNotAllowed)
+ }
+ }))
+ t.Cleanup(server.Close)
+ client := originClient(t, testS3Client(t, server), OriginConfig{Namespace: "store"})
+
+ request := objectRequest(t, "")
+ if operation == "HEAD" {
+ metadata, err := client.Stat(t.Context(), request)
+ if err != nil || metadata.ContentType != tc.want || metadata.Size != 5 || heads.Load() != 1 || gets.Load() != 0 {
+ t.Fatalf("HEAD metadata = %+v, error = %v, calls = %d/%d", metadata, err, heads.Load(), gets.Load())
+ }
+
+ return
+ }
+
+ options := racersdk.ReadOptions{}
+ if operation == "pinned" {
+ options.ETag = `"opaque-multipart-2"`
+ }
+
+ value, err := client.Get(t.Context(), request, options)
+
+ var got []byte
+
+ if err == nil {
+ defer value.Close()
+
+ if value.Metadata().ContentType != tc.want {
+ t.Fatalf("GET content type = %q, want %q", value.Metadata().ContentType, tc.want)
+ }
+
+ got, err = io.ReadAll(value)
+ }
+
+ if tc.mismatch {
+ originAssertError(t, err, errInvalidS3Response)
+
+ if len(got) != 0 {
+ t.Fatalf("mismatched content type exposed bytes: %q", got)
+ }
+ } else if err != nil || string(got) != "hello" {
+ t.Fatalf("GET = %q, error = %v", got, err)
+ }
+
+ if gets.Load() != 1 || heads.Load() == 0 {
+ t.Fatalf("unexpected S3 calls = %d/%d", heads.Load(), gets.Load())
+ }
+ })
+ }
+ }
+}
+
+func TestOriginPinnedHeadAndRaceFailures(t *testing.T) {
+ for _, mode := range []string{"changed-head", "missing-head", "head-precondition", "get-precondition", "missing-get", "get-range"} {
+ t.Run(mode, func(t *testing.T) {
+ var gets atomic.Int32
+
+ stub := &originS3Stub{}
+ stub.head = func(_ context.Context, in *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
+ if aws.ToString(in.IfMatch) != `"opaque-multipart-2"` {
+ t.Error("pinned HEAD omitted If-Match")
+ }
+
+ switch mode {
+ case "changed-head":
+ head := originHead(5)
+ head.ETag = aws.String(`"new-version"`)
+
+ return head, nil
+ case "missing-head":
+ return nil, &smithy.GenericAPIError{Code: "NoSuchKey"}
+ case "head-precondition":
+ return nil, &smithy.GenericAPIError{Code: "PreconditionFailed"}
+ }
+
+ return originHead(5), nil
+ }
+ stub.get = func(context.Context, *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
+ gets.Add(1)
+
+ switch mode {
+ case "missing-get":
+ return nil, &smithy.GenericAPIError{Code: "NoSuchKey"}
+ case "get-range":
+ return nil, originStatusError{416}
+ default:
+ return nil, &smithy.GenericAPIError{Code: "PreconditionFailed"}
+ }
+ }
+ client := originClient(t, stub, OriginConfig{Namespace: "store"})
+
+ _, err := originReadValue(t, client, objectRequest(t, ""), racersdk.ReadOptions{ETag: `"opaque-multipart-2"`})
+
+ want := racersdk.ErrVersionMismatch
+ if mode == "get-range" {
+ want = errInvalidS3Response
+ }
+
+ originAssertError(t, err, want)
+
+ if strings.HasSuffix(mode, "head") || mode == "head-precondition" {
+ if gets.Load() != 0 {
+ t.Fatal("GET after failed pin")
+ }
+ }
+ })
+ }
+}
+
+func TestOriginGetPinnedToHeadVersion(t *testing.T) {
+ for _, tt := range []struct {
+ name, requested, headVersion, wantGet string
+ }{
+ {"versioned-bucket-unpinned", "", "v-head", "v-head"},
+ {"unversioned-bucket", "", "", ""},
+ {"explicit-version", "v-req", "v-req", "v-req"},
+ } {
+ t.Run(tt.name, func(t *testing.T) {
+ stub := &originS3Stub{
+ head: func(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
+ head := originHead(5)
+ if tt.headVersion != "" {
+ head.VersionId = aws.String(tt.headVersion)
+ }
+
+ return head, nil
+ },
+ get: func(_ context.Context, in *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
+ if tt.wantGet == "" && in.VersionId != nil {
+ t.Errorf("GET sent version %q for an unversioned object", aws.ToString(in.VersionId))
+ }
+
+ if aws.ToString(in.VersionId) != tt.wantGet {
+ t.Errorf("GET version = %q, want %q", aws.ToString(in.VersionId), tt.wantGet)
+ }
+
+ out := originPage(io.NopCloser(strings.NewReader("hello")), 0, 4, 5)
+ if tt.headVersion != "" {
+ out.VersionId = aws.String(tt.headVersion)
+ }
+
+ return out, nil
+ },
+ }
+ client := originClient(t, stub, OriginConfig{Namespace: "store"})
+
+ got, err := originReadValue(t, client, objectRequest(t, tt.requested))
+ if err != nil || string(got) != "hello" {
+ t.Fatalf("GET = %q, error = %v", got, err)
+ }
+ })
+ }
+}
+
+func TestOriginUnpinnedRaceIsUnavailable(t *testing.T) {
+ for _, mode := range []string{"get-precondition", "get-status-412", "missing-get", "changed-get-etag", "get-forbidden"} {
+ for _, pinned := range []bool{false, true} {
+ t.Run(fmt.Sprintf("%s/pinned=%t", mode, pinned), func(t *testing.T) {
+ stub := &originS3Stub{
+ head: func(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
+ return originHead(5), nil
+ },
+ get: func(context.Context, *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
+ switch mode {
+ case "get-precondition":
+ return nil, &smithy.GenericAPIError{Code: "PreconditionFailed"}
+ case "get-status-412":
+ return nil, originStatusError{412}
+ case "missing-get":
+ return nil, &smithy.GenericAPIError{Code: "NoSuchKey"}
+ case "get-forbidden":
+ return nil, &smithy.GenericAPIError{Code: "AccessDenied"}
+ }
+
+ out := originPage(io.NopCloser(strings.NewReader("hello")), 0, 4, 5)
+ out.ETag = aws.String(`"new-version"`)
+
+ return out, nil
+ },
+ }
+ client := originClient(t, stub, OriginConfig{Namespace: "store"})
+
+ var options []racersdk.ReadOptions
+ if pinned {
+ options = append(options, racersdk.ReadOptions{ETag: `"opaque-multipart-2"`})
+ }
+
+ _, err := originReadValue(t, client, objectRequest(t, ""), options...)
+
+ want := racersdk.ErrUnavailable
+ if pinned {
+ want = racersdk.ErrVersionMismatch
+ }
+
+ if mode == "get-forbidden" {
+ want = racersdk.ErrForbidden
+ }
+
+ originAssertError(t, err, want)
+
+ if !pinned && errors.Is(err, racersdk.ErrVersionMismatch) {
+ t.Fatal("unpinned race reported a version mismatch", err)
+ }
+ })
+ }
+ }
+}
+
+func TestUnpinnedRace(t *testing.T) {
+ unpinned := racersdk.OriginRequest{}
+ pinned := racersdk.OriginRequest{ETag: `"tag"`}
+
+ if unpinnedRace(unpinned, nil) != nil || unpinnedRace(pinned, nil) != nil {
+ t.Fatal("nil error changed")
+ }
+
+ cause := &smithy.GenericAPIError{Code: "PreconditionFailed", Message: "private"}
+ wrapped := classifyS3Error(cause, true)
+
+ err := unpinnedRace(unpinned, wrapped)
+ if !errors.Is(err, racersdk.ErrUnavailable) || errors.Is(err, racersdk.ErrVersionMismatch) || !errors.Is(err, cause) ||
+ strings.Contains(err.Error(), "private") {
+ t.Fatal("unpinned S3 race", err)
+ }
+
+ err = unpinnedRace(unpinned, racersdk.ErrVersionMismatch)
+ if !errors.Is(err, racersdk.ErrUnavailable) || errors.Is(err, racersdk.ErrVersionMismatch) {
+ t.Fatal("unpinned validation race", err)
+ }
+
+ if err := unpinnedRace(pinned, wrapped); err != wrapped {
+ t.Fatal("pinned mismatch changed", err)
+ }
+
+ if err := unpinnedRace(unpinned, errInvalidS3Response); err != errInvalidS3Response {
+ t.Fatal("unrelated error changed", err)
+ }
+}
+
+// TestOriginAWSClientDeleteMarker drives the real AWS client against S3's
+// delete-marker replies: HEAD has no body, so only the 405 status classifies
+// it, while GET carries a MethodNotAllowed error code.
+func TestOriginAWSClientDeleteMarker(t *testing.T) {
+ var gets atomic.Int32
+
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Query().Get("versionId") != "marker" {
+ t.Errorf("request version = %q", r.URL.Query().Get("versionId"))
+ }
+
+ w.Header().Set("x-amz-delete-marker", "true")
+ w.Header().Set("x-amz-version-id", "marker")
+
+ if r.Method == http.MethodGet {
+ gets.Add(1)
+ w.Header().Set("Content-Type", "application/xml")
+ w.WriteHeader(http.StatusMethodNotAllowed)
+ _, _ = io.WriteString(w, `MethodNotAllowedprivate`)
+
+ return
+ }
+
+ w.WriteHeader(http.StatusMethodNotAllowed)
+ }))
+ t.Cleanup(server.Close)
+
+ origin, err := NewOrigin(testS3Client(t, server), OriginConfig{Namespace: "store"})
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ request := objectRequest(t, "marker")
+
+ client := testSDKClient(t, origin)
+
+ _, err = client.Stat(t.Context(), request)
+ originAssertError(t, err, racersdk.ErrNotFound)
+
+ // GET error bodies are classified by code; drive GetObject directly since
+ // the origin stops after the failed HEAD.
+ _, err = testS3Client(t, server).GetObject(t.Context(), &s3.GetObjectInput{
+ Bucket: aws.String("bucket"), Key: aws.String("key"), VersionId: aws.String("marker"),
+ })
+ originAssertError(t, classifyS3Error(err, false), racersdk.ErrNotFound)
+
+ if gets.Load() != 1 {
+ t.Fatal("GET count", gets.Load())
+ }
+}
+
+func TestOriginUpstreamErrorsBridge(t *testing.T) {
+ for _, tt := range []struct {
+ code string
+ want error
+ }{
+ {"NoSuchKey", racersdk.ErrNotFound},
+ {"MethodNotAllowed", racersdk.ErrNotFound},
+ {"AccessDenied", racersdk.ErrForbidden},
+ {"SlowDown", racersdk.ErrUnavailable},
+ {"InvalidRange", errInvalidS3Response},
+ } {
+ t.Run(tt.code, func(t *testing.T) {
+ stub := &originS3Stub{head: func(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
+ return nil, &smithy.GenericAPIError{Code: tt.code, Message: "private"}
+ }}
+ client := originClient(t, stub, OriginConfig{Namespace: "store"})
+ _, err := client.Stat(t.Context(), objectRequest(t, ""))
+ originAssertError(t, err, tt.want)
+ })
+ }
+}
+
+func TestOriginMultipageAndCopiedAllowlist(t *testing.T) {
+ var heads, gets atomic.Int32
+
+ size := int64(racersdk.PageSize) + 3
+ stub := &originS3Stub{}
+ stub.head = func(_ context.Context, in *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
+ n := heads.Add(1)
+ if n > 1 && aws.ToString(in.IfMatch) != `"opaque-multipart-2"` {
+ t.Error("continuation HEAD not pinned")
+ }
+
+ return originHead(size), nil
+ }
+ stub.get = func(_ context.Context, in *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
+ gets.Add(1)
+
+ if aws.ToString(in.IfMatch) != `"opaque-multipart-2"` {
+ t.Error("GET not pinned")
+ }
+
+ if aws.ToString(in.Range) == fmt.Sprintf("bytes=0-%d", racersdk.PageSize-1) {
+ return originPage(io.NopCloser(io.LimitReader(originRepeatedByte('x'), int64(racersdk.PageSize))), 0, int64(racersdk.PageSize)-1, size), nil
+ }
+
+ if aws.ToString(in.Range) != fmt.Sprintf("bytes=%d-%d", racersdk.PageSize, size-1) {
+ t.Error("wrong final range")
+ }
+
+ return originPage(io.NopCloser(strings.NewReader("end")), int64(racersdk.PageSize), size-1, size), nil
+ }
+ buckets := []string{"bucket"}
+ client := originClient(t, stub, OriginConfig{Namespace: "store", Buckets: buckets})
+ buckets[0] = "changed"
+
+ value, err := client.Get(t.Context(), objectRequest(t, ""))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer value.Close()
+
+ if n, err := io.CopyN(io.Discard, value, int64(racersdk.PageSize)); err != nil || n != int64(racersdk.PageSize) {
+ t.Fatal("first page", n, err)
+ }
+
+ got, err := io.ReadAll(value)
+ if err != nil || string(got) != "end" || heads.Load() != 2 || gets.Load() != 2 {
+ t.Fatal("continuation", string(got), err, heads.Load(), gets.Load())
+ }
+}
+
+type originRepeatedByte byte
+
+func (b originRepeatedByte) Read(p []byte) (int, error) {
+ for i := range p {
+ p[i] = byte(b)
+ }
+
+ return len(p), nil
+}
+
+type originBlockedBody struct {
+ entered chan struct{}
+ closed chan struct{}
+ readOnce sync.Once
+ closeOnce sync.Once
+ closes atomic.Int32
+}
+
+func (b *originBlockedBody) Read([]byte) (int, error) {
+ b.readOnce.Do(func() { close(b.entered) })
+ <-b.closed
+
+ return 0, context.Canceled
+}
+
+func (b *originBlockedBody) Close() error {
+ b.closes.Add(1)
+ b.closeOnce.Do(func() { close(b.closed) })
+
+ return nil
+}
+
+func TestOriginBodyCancellationBridge(t *testing.T) {
+ body := &originBlockedBody{entered: make(chan struct{}), closed: make(chan struct{})}
+ stub := &originS3Stub{
+ head: func(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) { return originHead(5), nil },
+ get: func(context.Context, *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
+ return originPage(body, 0, 4, 5), nil
+ },
+ }
+ client := originClient(t, stub, OriginConfig{Namespace: "store"})
+
+ ctx, cancel := context.WithCancel(t.Context())
+ defer cancel()
+
+ value, err := client.Get(ctx, objectRequest(t, ""))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer value.Close()
+
+ select {
+ case <-body.entered:
+ case <-time.After(time.Second):
+ t.Fatal("body read not started")
+ }
+
+ cancel()
+
+ select {
+ case <-body.closed:
+ case <-time.After(time.Second):
+ t.Fatal("canceled body was not closed")
+ }
+
+ if body.closes.Load() != 1 {
+ t.Fatal("close count", body.closes.Load())
+ }
+
+ _, err = io.ReadAll(value)
+ if err == nil {
+ t.Fatal("canceled read succeeded")
+ }
+}
+
+func TestOriginHeadCancellationBridge(t *testing.T) {
+ entered, canceled := make(chan struct{}), make(chan struct{})
+ stub := &originS3Stub{head: func(ctx context.Context, _ *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
+ close(entered)
+ <-ctx.Done()
+ close(canceled)
+
+ return nil, ctx.Err()
+ }}
+ client := originClient(t, stub, OriginConfig{Namespace: "store"})
+
+ ctx, cancel := context.WithCancel(t.Context())
+ defer cancel()
+
+ done := make(chan error, 1)
+ request := objectRequest(t, "")
+
+ go func() { _, err := client.Stat(ctx, request); done <- err }()
+
+ select {
+ case <-entered:
+ case <-time.After(time.Second):
+ t.Fatal("HEAD not started")
+ }
+
+ cancel()
+
+ select {
+ case <-canceled:
+ case <-time.After(time.Second):
+ t.Fatal("context not passed upstream")
+ }
+
+ select {
+ case err := <-done:
+ if !errors.Is(err, context.Canceled) {
+ t.Fatal("cancellation cause", err)
+ }
+ case <-time.After(time.Second):
+ t.Fatal("Stat did not stop")
+ }
+}
+
+func TestOriginRangeResolvedAgainstPinnedHead(t *testing.T) {
+ var heads atomic.Int32
+
+ stub := &originS3Stub{
+ head: func(_ context.Context, in *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
+ if heads.Add(1) == 1 {
+ return originHead(int64(racersdk.PageSize) + 3), nil
+ }
+
+ if aws.ToString(in.IfMatch) != `"opaque-multipart-2"` {
+ t.Error("missing HEAD pin")
+ }
+
+ return originHead(5), nil
+ },
+ get: func(context.Context, *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
+ t.Error("unsatisfiable page reached S3 GET")
+ return nil, nil
+ },
+ }
+ client := originClient(t, stub, OriginConfig{Namespace: "store"})
+ _, err := originReadValue(t, client, objectRequest(t, ""), racersdk.ReadOptions{Offset: racersdk.PageSize, Length: 3})
+ // The origin returns 416 with the new size. The SDK rejects that size because
+ // it differs from the selected snapshot for the same ETag.
+ originAssertError(t, err, errInvalidS3Response)
+
+ if heads.Load() != 2 {
+ t.Fatal("did not resolve page against second HEAD", heads.Load())
+ }
+}
+
+func TestOriginContentEncoding(t *testing.T) {
+ for _, encoding := range []string{"", "identity", "Identity", "IDENTITY", "iDeNtItY", "gzip", "br", "gzip, identity"} {
+ for _, operation := range []string{"HEAD", "bootstrap"} {
+ t.Run(operation+"/"+encoding, func(t *testing.T) {
+ var gets atomic.Int32
+
+ stub := &originS3Stub{
+ head: func(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
+ head := originHead(5)
+ head.ContentEncoding = aws.String(encoding)
+
+ return head, nil
+ },
+ get: func(context.Context, *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
+ gets.Add(1)
+
+ output := originPage(io.NopCloser(strings.NewReader("hello")), 0, 4, 5)
+ output.ContentEncoding = aws.String(encoding)
+
+ return output, nil
+ },
+ }
+ client := originClient(t, stub, OriginConfig{Namespace: "store"})
+ request := objectRequest(t, "")
+
+ var (
+ err error
+ got []byte
+ )
+
+ if operation == "HEAD" {
+ _, err = client.Stat(t.Context(), request)
+ } else {
+ got, err = originReadValue(t, client, request)
+ }
+
+ if encoding == "gzip" || encoding == "br" || encoding == "gzip, identity" {
+ originAssertError(t, err, errInvalidS3Response)
+
+ if gets.Load() != 0 || len(got) != 0 {
+ t.Fatal("unsupported encoding reached GET or exposed bytes")
+ }
+
+ return
+ }
+
+ if err != nil {
+ t.Fatal("unencoded object rejected", err)
+ }
+
+ if operation == "HEAD" {
+ if gets.Load() != 0 {
+ t.Fatal("HEAD issued GET")
+ }
+ } else if string(got) != "hello" || gets.Load() != 1 {
+ t.Fatalf("bootstrap = %q, GET calls = %d", got, gets.Load())
+ }
+ })
+ }
+ }
+}
+
+func TestOriginGetContentEncoding(t *testing.T) {
+ for _, tc := range []struct {
+ head, get string
+ accepted bool
+ }{
+ {"identity", "Identity", true},
+ {"Identity", "identity", true},
+ {"IDENTITY", "iDeNtItY", true},
+ {"Identity", "gzip", false},
+ {"identity", "GZIP", false},
+ {"Identity", "br", false},
+ {"Identity", "identity, gzip", false},
+ {"", "Identity", false},
+ {"Identity", "", false},
+ } {
+ t.Run(tc.head+"/"+tc.get, func(t *testing.T) {
+ var gets atomic.Int32
+
+ stub := &originS3Stub{
+ head: func(context.Context, *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
+ head := originHead(5)
+ head.ContentEncoding = aws.String(tc.head)
+
+ return head, nil
+ },
+ get: func(context.Context, *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
+ gets.Add(1)
+
+ output := originPage(io.NopCloser(strings.NewReader("hello")), 0, 4, 5)
+ output.ContentEncoding = aws.String(tc.get)
+
+ return output, nil
+ },
+ }
+ client := originClient(t, stub, OriginConfig{Namespace: "store"})
+
+ got, err := originReadValue(t, client, objectRequest(t, ""))
+ if tc.accepted {
+ if err != nil || string(got) != "hello" {
+ t.Fatalf("GET = %q, error = %v", got, err)
+ }
+ } else {
+ originAssertError(t, err, errInvalidS3Response)
+
+ if len(got) != 0 {
+ t.Fatal("rejected encoding exposed bytes")
+ }
+ }
+
+ if gets.Load() != 1 {
+ t.Fatal("expected one GET", gets.Load())
+ }
+ })
+ }
+}
diff --git a/internal/racer/object/origin_transport_test.go b/internal/racer/object/origin_transport_test.go
new file mode 100644
index 0000000000..86cc0b1c4e
--- /dev/null
+++ b/internal/racer/object/origin_transport_test.go
@@ -0,0 +1,210 @@
+// Copyright (c) Microsoft Corporation.
+// SPDX-License-Identifier: Apache-2.0
+
+package object
+
+import (
+ "context"
+ "crypto/x509"
+ "errors"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "syscall"
+ "testing"
+ "time"
+
+ "github.com/aws/aws-sdk-go-v2/aws"
+ "github.com/aws/aws-sdk-go-v2/credentials"
+ "github.com/aws/aws-sdk-go-v2/service/s3"
+ "github.com/aws/smithy-go"
+ smithyhttp "github.com/aws/smithy-go/transport/http"
+
+ "github.com/Azure/unbounded/pkg/racersdk"
+)
+
+type originConnectionError bool
+
+func (e originConnectionError) Error() string { return "private connection failure" }
+func (e originConnectionError) ConnectionError() bool { return bool(e) }
+
+func TestOriginSendErrorPrecedence(t *testing.T) {
+ send := &smithyhttp.RequestSendError{Err: io.EOF}
+ for _, tc := range []struct {
+ name string
+ status int
+ cause error
+ want error
+ }{
+ {"send", 0, send, racersdk.ErrUnavailable},
+ {"connection", 0, originConnectionError(true), racersdk.ErrUnavailable},
+ {"not-connection", 0, originConnectionError(false), errInvalidS3Response},
+ {"unmarked-eof", 0, io.EOF, errInvalidS3Response},
+ {"configuration", 0, errors.New("private configuration failure"), errInvalidS3Response},
+ {"canceled", 0, &smithyhttp.RequestSendError{Err: context.Canceled}, context.Canceled},
+ {"deadline", 0, &smithyhttp.RequestSendError{Err: context.DeadlineExceeded}, context.DeadlineExceeded},
+ {"canceled-api-status", 403, errors.Join(context.Canceled, send, &smithy.GenericAPIError{Code: "NoSuchKey"}), context.Canceled},
+ {"deadline-api-status", 403, errors.Join(context.DeadlineExceeded, send, &smithy.GenericAPIError{Code: "NoSuchKey"}), context.DeadlineExceeded},
+ {"known-api", 0, errors.Join(send, &smithy.GenericAPIError{Code: "AccessDenied"}), racersdk.ErrForbidden},
+ {"known-api-status", 403, errors.Join(send, &smithy.GenericAPIError{Code: "NoSuchKey"}), racersdk.ErrNotFound},
+ {"unknown-api", 0, errors.Join(send, &smithy.GenericAPIError{Code: "Unknown"}), racersdk.ErrUnavailable},
+ {"unauthorized", 401, send, racersdk.ErrUnauthorized},
+ {"forbidden", 403, send, racersdk.ErrForbidden},
+ {"not-found", 404, send, racersdk.ErrNotFound},
+ {"delete-marker-version", 405, send, racersdk.ErrNotFound},
+ {"not-acceptable", 406, send, errInvalidS3Response},
+ {"precondition", 412, send, racersdk.ErrVersionMismatch},
+ {"invalid-range", 416, send, errInvalidS3Response},
+ {"server-error", 500, send, racersdk.ErrUnavailable},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ upstream := &smithyhttp.ResponseError{
+ Response: &smithyhttp.Response{Response: &http.Response{StatusCode: tc.status}},
+ Err: tc.cause,
+ }
+ assertOriginTransportClassification(t, upstream, tc.cause, tc.want)
+ })
+ }
+}
+
+func assertOriginTransportClassification(t *testing.T, upstream, cause, want error) {
+ t.Helper()
+
+ for _, pinned := range []bool{false, true} {
+ classification := want
+ if pinned && classification == racersdk.ErrNotFound {
+ classification = racersdk.ErrVersionMismatch
+ }
+
+ err := classifyS3Error(upstream, pinned)
+ if !errors.Is(err, classification) || !errors.Is(err, upstream) || !errors.Is(err, cause) {
+ t.Fatalf("classification or cause lost: %v, want %v", err, classification)
+ }
+
+ if got, want := err.Error(), classification.Error()+": S3 request failed"; got != want {
+ t.Fatalf("upstream details not redacted: %q, want %q", got, want)
+ }
+
+ if classification == racersdk.ErrUnavailable && sidecarSDKFailure(err) != http.StatusServiceUnavailable {
+ t.Fatal("transport failure did not map to HTTP 503")
+ }
+
+ if classification == errInvalidS3Response && sidecarSDKFailure(err) != http.StatusBadGateway {
+ t.Fatal("nontransport failure did not map to HTTP 502")
+ }
+ }
+}
+
+func TestOriginAWSClientTransportFailures(t *testing.T) {
+ for _, mode := range []string{"eof", "tls", "refused", "missing-region", "credentials"} {
+ t.Run(mode, func(t *testing.T) {
+ handler := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ if mode != "eof" {
+ t.Error("unexpected request reached S3")
+ w.WriteHeader(http.StatusInternalServerError)
+
+ return
+ }
+
+ conn, _, err := w.(http.Hijacker).Hijack()
+ if err != nil {
+ t.Error(err)
+ return
+ }
+
+ if err := conn.Close(); err != nil {
+ t.Error(err)
+ }
+ })
+
+ var server *httptest.Server
+ if mode == "tls" {
+ server = httptest.NewTLSServer(handler)
+ } else {
+ server = httptest.NewServer(handler)
+ }
+
+ t.Cleanup(server.Close)
+
+ if mode == "refused" {
+ server.Close()
+ }
+
+ transport := http.DefaultTransport.(*http.Transport).Clone()
+ transport.Proxy = nil
+ t.Cleanup(transport.CloseIdleConnections)
+
+ credentialFailure := errors.New("private credential configuration failure")
+
+ options := s3.Options{
+ Region: "us-east-1", BaseEndpoint: aws.String(server.URL), UsePathStyle: true,
+ Credentials: credentials.NewStaticCredentialsProvider("placeholder", "placeholder", ""),
+ RetryMaxAttempts: 1, HTTPClient: &http.Client{Transport: transport, Timeout: 5 * time.Second},
+ }
+ if mode == "missing-region" {
+ options.Region = ""
+ }
+
+ if mode == "credentials" {
+ options.Credentials = aws.CredentialsProviderFunc(func(context.Context) (aws.Credentials, error) {
+ return aws.Credentials{}, credentialFailure
+ })
+ }
+
+ client := s3.New(options)
+
+ for _, operation := range []string{"HEAD", "GET"} {
+ t.Run(operation, func(t *testing.T) {
+ ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second)
+ defer cancel()
+
+ var upstream error
+ if operation == "HEAD" {
+ _, upstream = client.HeadObject(ctx, &s3.HeadObjectInput{Bucket: aws.String("bucket"), Key: aws.String("private-key")})
+ } else {
+ _, upstream = client.GetObject(ctx, &s3.GetObjectInput{Bucket: aws.String("bucket"), Key: aws.String("private-key")})
+ }
+
+ if upstream == nil {
+ t.Fatal("expected real AWS SDK failure")
+ }
+
+ want, cause := racersdk.ErrUnavailable, upstream
+
+ var send *smithyhttp.RequestSendError
+
+ if mode == "missing-region" || mode == "credentials" {
+ want = errInvalidS3Response
+
+ if errors.As(upstream, &send) {
+ t.Fatal("configuration failure reached transport", upstream)
+ }
+ } else {
+ var status interface{ HTTPStatusCode() int }
+ if !errors.As(upstream, &status) || status.HTTPStatusCode() != 0 || !errors.As(upstream, &send) {
+ t.Fatal("expected SDK status-zero response wrapping RequestSendError", upstream)
+ }
+
+ cause = send.Err
+ }
+
+ switch mode {
+ case "eof":
+ cause = io.EOF
+ case "refused":
+ cause = syscall.ECONNREFUSED
+ case "tls":
+ var certificate x509.UnknownAuthorityError
+ if !errors.As(upstream, &certificate) {
+ t.Fatal("expected certificate verification failure", upstream)
+ }
+ case "credentials":
+ cause = credentialFailure
+ }
+
+ assertOriginTransportClassification(t, upstream, cause, want)
+ })
+ }
+ })
+ }
+}
diff --git a/internal/racer/object/sidecar.go b/internal/racer/object/sidecar.go
new file mode 100644
index 0000000000..d29968b388
--- /dev/null
+++ b/internal/racer/object/sidecar.go
@@ -0,0 +1,649 @@
+// Copyright (c) Microsoft Corporation.
+// SPDX-License-Identifier: Apache-2.0
+
+package object
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "math"
+ "net"
+ "net/http"
+ "net/url"
+ "strconv"
+ "strings"
+
+ "github.com/Azure/unbounded/pkg/racersdk"
+)
+
+// SidecarConfig selects the object namespace and optional bucket allowlist.
+type SidecarConfig struct {
+ Namespace string
+ Buckets []string
+ // OnFailure, when set, is called for each 5xx the sidecar returns because
+ // of an SDK or upstream failure. Nil disables reporting.
+ OnFailure func(SidecarFailure)
+}
+
+// sidecarClient is the part of [racersdk.Client] the sidecar uses.
+type sidecarClient interface {
+ Stat(ctx context.Context, request racersdk.Request) (racersdk.Metadata, error)
+ Get(ctx context.Context, request racersdk.Request, options ...racersdk.ReadOptions) (*racersdk.Object, error)
+}
+
+type sidecar struct {
+ client sidecarClient
+ scope scope
+ onFailure func(SidecarFailure)
+}
+
+// NewSidecar serves a read-only, trusted pod endpoint. The caller owns the client
+// and must bind the server to loopback. SigV4 headers are ignored, not verified or
+// forwarded. Presigned query parameters and unsupported S3 operations are rejected.
+// Known auth/plumbing x-amz headers and optional checksum mode ENABLED are
+// accepted, but no checksum metadata is returned. Owner checks, SSE-C, and
+// requester-pays are unsupported and return NotImplemented.
+// Install this handler directly: a path-cleaning mux changes object identities.
+func NewSidecar(client *racersdk.Client, config SidecarConfig) (http.Handler, error) {
+ if client == nil {
+ return nil, errors.New("racer-object: missing sidecar client")
+ }
+
+ bound, err := newScope(config.Namespace, config.Buckets)
+ if errors.Is(err, errInvalidNamespace) {
+ return nil, errors.New("racer-object: invalid sidecar namespace")
+ }
+
+ if err != nil {
+ return nil, errors.New("racer-object: invalid sidecar bucket")
+ }
+
+ return &sidecar{client: client, scope: bound, onFailure: config.OnFailure}, nil
+}
+
+func (s *sidecar) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+ if failure := s.serve(w, r); failure != 0 {
+ failure.write(w, r)
+ }
+}
+
+// serve returns failures only before headers are committed, so XML cannot be
+// appended to an object body. A failed committed stream must abort instead.
+func (s *sidecar) serve(w http.ResponseWriter, r *http.Request) sidecarFailure {
+ if r.Method != http.MethodGet && r.Method != http.MethodHead {
+ w.Header().Set("Allow", "GET, HEAD")
+ return http.StatusMethodNotAllowed
+ }
+
+ request, version, failure := s.request(r)
+ if failure != 0 {
+ return failure
+ }
+
+ match, ok := sidecarCondition(r.Header, "If-Match")
+ if !ok {
+ return http.StatusBadRequest
+ }
+
+ none, ok := sidecarCondition(r.Header, "If-None-Match")
+ if !ok {
+ return http.StatusBadRequest
+ }
+
+ conditions := sidecarConditions{match, none}
+
+ // A GET that needs no size up front reads directly, so whole-object, open,
+ // and closed range reads never wait in the SDK's small Stat lane.
+ if r.Method == http.MethodGet {
+ if options, ok := sidecarDirectRange(r.Header); ok {
+ return s.direct(w, r, request, version, conditions, options)
+ }
+ }
+
+ return s.statThenRead(w, r, request, version, conditions)
+}
+
+// statThenRead serves HEAD, suffix ranges, and ranges that may need clamping
+// from Stat metadata and pins the read to it.
+func (s *sidecar) statThenRead(w http.ResponseWriter, r *http.Request, request racersdk.Request, version string, conditions sidecarConditions) sidecarFailure {
+ m, err := s.client.Stat(r.Context(), request)
+ if err != nil {
+ return s.sdkFailure(r, err)
+ }
+
+ if failure, done := conditions.evaluate(w, m.ETag); done {
+ return failure
+ }
+
+ offset, length, partial, status := sidecarRange(r.Header, uint64(m.Size))
+ if status != 0 {
+ return sidecarRangeFailure(w, status, m.Size)
+ }
+
+ read := sidecarRead{m, version, offset, length, partial}
+ if r.Method == http.MethodHead {
+ read.partial = false
+ read.writeHeaders(w)
+
+ return 0
+ }
+
+ return s.stream(w, r, request, read)
+}
+
+// direct reads without Stat and takes every response header from the read's
+// own metadata, which covers every page of the read. The read is unpinned, so
+// client conditions are evaluated against the version Racer actually serves
+// and a plain GET can never fail with 412.
+func (s *sidecar) direct(w http.ResponseWriter, r *http.Request, request racersdk.Request, version string, conditions sidecarConditions, options racersdk.ReadOptions) sidecarFailure {
+ v, err := s.client.Get(r.Context(), request, options)
+ if errors.Is(err, racersdk.ErrRangeNotSatisfiable) {
+ // The SDK does not clamp a range past the end. Stat supplies the size
+ // to clamp it, or to report it in a 416 response.
+ return s.statThenRead(w, r, request, version, conditions)
+ }
+
+ if err != nil {
+ return s.sdkFailure(r, err)
+ }
+
+ defer func() { _ = v.Close() }() //nolint:errcheck // Close releases admission; its error cannot change the response.
+
+ m := v.Metadata()
+ if failure, done := conditions.evaluate(w, m.ETag); done {
+ return failure
+ }
+
+ offset, length, partial, status := sidecarRange(r.Header, uint64(m.Size))
+ if status != 0 {
+ return sidecarRangeFailure(w, status, m.Size)
+ }
+
+ // Racer accepted the requested range, so it must equal the selection.
+ if offset != uint64(options.Offset) || options.Length != 0 && length != uint64(options.Length) {
+ return s.report(r, http.StatusBadGateway, ErrMetadataMismatch)
+ }
+
+ return s.writeBody(w, r, sidecarRead{m, version, offset, length, partial}, v)
+}
+
+// sidecarDirectRange returns the read for a GET that does not need the size
+// up front: no byte range, an ignored one, or a valid "bytes=N-" or "bytes=N-M"
+// range. Suffix and unsatisfiable ranges keep the Stat path.
+func sidecarDirectRange(h http.Header) (racersdk.ReadOptions, bool) {
+ // With an effectively unbounded size, offsets and lengths are exact.
+ offset, length, partial, status := sidecarRange(h, math.MaxInt64)
+ if status != 0 {
+ return racersdk.ReadOptions{}, false
+ }
+
+ if !partial {
+ return racersdk.ReadOptions{}, true
+ }
+
+ _, spec, _ := strings.Cut(strings.Trim(h.Get("Range"), " \t"), "=")
+ switch {
+ case strings.HasPrefix(spec, "-"):
+ return racersdk.ReadOptions{}, false
+ case strings.HasSuffix(spec, "-"):
+ return racersdk.ReadOptions{Offset: int64(offset)}, true
+ }
+
+ return racersdk.ReadOptions{Offset: int64(offset), Length: int64(length)}, true
+}
+
+func sidecarRangeFailure(w http.ResponseWriter, status int, size int64) sidecarFailure {
+ if status == http.StatusRequestedRangeNotSatisfiable {
+ w.Header().Set("Content-Range", fmt.Sprintf("bytes */%d", size))
+ }
+
+ return sidecarFailure(status)
+}
+
+// sidecarConditions holds the parsed If-Match and If-None-Match headers.
+type sidecarConditions struct {
+ match, none sidecarTags
+}
+
+// evaluate applies the conditions to tag. done is true when the response is
+// decided, either a failure or an already written 304.
+func (c sidecarConditions) evaluate(w http.ResponseWriter, tag string) (sidecarFailure, bool) {
+ if c.match.present && !c.match.matches(tag, false) {
+ return http.StatusPreconditionFailed, true
+ }
+
+ if c.none.present && c.none.matches(tag, true) {
+ w.Header().Set("ETag", tag)
+ w.WriteHeader(http.StatusNotModified)
+
+ return 0, true
+ }
+
+ return 0, false
+}
+
+func (s *sidecar) stream(w http.ResponseWriter, r *http.Request, request racersdk.Request, read sidecarRead) sidecarFailure {
+ v, err := s.client.Get(r.Context(), request, racersdk.ReadOptions{
+ ETag: read.metadata.ETag, Offset: int64(read.offset), Length: int64(read.length),
+ })
+ if err != nil {
+ return s.sdkFailure(r, err)
+ }
+
+ defer func() { _ = v.Close() }() //nolint:errcheck // Close releases admission; its error cannot change the response.
+
+ // An ETag pin must not change the headers selected by Stat.
+ if m := v.Metadata(); m.Size != read.metadata.Size || m.ContentType != read.metadata.ContentType {
+ return s.report(r, http.StatusBadGateway, ErrMetadataMismatch)
+ }
+
+ return s.writeBody(w, r, read, v)
+}
+
+func (s *sidecar) writeBody(w http.ResponseWriter, r *http.Request, read sidecarRead, v *racersdk.Object) sidecarFailure {
+ // Empty success is not committed until the SDK has validated Complete.
+ // Nothing is written yet, so the failure is reported like any SDK error.
+ if read.length == 0 {
+ if _, err := v.WriteTo(io.Discard); err != nil {
+ return s.sdkFailure(r, err)
+ }
+ }
+
+ read.writeHeaders(w)
+
+ if read.length != 0 {
+ if _, err := v.WriteTo(w); err != nil {
+ panic(http.ErrAbortHandler)
+ }
+ }
+
+ return 0
+}
+
+func (s *sidecar) request(r *http.Request) (racersdk.Request, string, sidecarFailure) {
+ if !sidecarSupportedHeaders(r.Header) {
+ return racersdk.Request{}, "", http.StatusNotImplemented
+ }
+ // Split before decoding so an escaped slash is key data, not a bucket boundary.
+ // Decode each part exactly once; never clean dot segments or repeated slashes.
+ path, ok := strings.CutPrefix(r.URL.EscapedPath(), "/")
+
+ bucket, key, found := strings.Cut(path, "/")
+ if !ok || !found || bucket == "" || key == "" {
+ return racersdk.Request{}, "", http.StatusBadRequest
+ }
+
+ // EscapedPath always supplies valid escapes, including when RawPath is invalid.
+ bucket, _ = url.PathUnescape(bucket) //nolint:errcheck // EscapedPath guarantees valid escapes.
+ key, _ = url.PathUnescape(key) //nolint:errcheck // EscapedPath guarantees valid escapes.
+
+ if !s.scope.allows(bucket) {
+ return racersdk.Request{}, "", http.StatusForbidden
+ }
+
+ version, failure := sidecarVersion(r)
+ if failure != 0 {
+ return racersdk.Request{}, "", failure
+ }
+
+ request, err := NewRequest(s.scope.namespace, bucket, key, version)
+ if err != nil {
+ return racersdk.Request{}, "", http.StatusBadRequest
+ }
+
+ return request, version, 0
+}
+
+// sidecarVersion rejects query options that would silently change S3 semantics.
+func sidecarVersion(r *http.Request) (string, sidecarFailure) {
+ query, err := url.ParseQuery(r.URL.RawQuery)
+ if err != nil {
+ return "", http.StatusBadRequest
+ }
+
+ for name, values := range query {
+ if len(values) != 1 || values[0] == "" {
+ return "", http.StatusBadRequest
+ }
+
+ switch name {
+ case "versionId":
+ case "x-id":
+ want := "GetObject"
+ if r.Method == http.MethodHead {
+ want = "HeadObject"
+ }
+
+ if values[0] != want {
+ return "", http.StatusNotImplemented
+ }
+ default:
+ return "", http.StatusNotImplemented
+ }
+ }
+
+ version := query.Get("versionId")
+ // net/http trims edge OWS in headers, which would change the echoed version.
+ if strings.Trim(version, " \t") != version {
+ return "", http.StatusBadRequest
+ }
+
+ for _, c := range []byte(version) {
+ if c < 0x20 || c == 0x7f {
+ return "", http.StatusBadRequest
+ }
+ }
+
+ return version, 0
+}
+
+// sidecarRead keeps the selected metadata and range together for HEAD and GET.
+type sidecarRead struct {
+ metadata racersdk.Metadata
+ version string
+ offset, length uint64
+ partial bool
+}
+
+func (read sidecarRead) writeHeaders(w http.ResponseWriter) {
+ h := w.Header()
+ h.Set("ETag", read.metadata.ETag)
+ h.Set("Accept-Ranges", "bytes")
+ h.Set("Content-Length", strconv.FormatUint(read.length, 10))
+
+ contentType := read.metadata.ContentType
+ if contentType == "" {
+ contentType = "application/octet-stream"
+ }
+
+ h.Set("Content-Type", contentType)
+
+ if read.version != "" {
+ h.Set("x-amz-version-id", read.version)
+ }
+
+ status := http.StatusOK
+ if read.partial {
+ status = http.StatusPartialContent
+
+ h.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", read.offset, read.offset+read.length-1, read.metadata.Size))
+ }
+
+ w.WriteHeader(status)
+}
+
+// sidecarFailure is an HTTP status with one fixed S3 code. Zero means success.
+// Messages come only from StatusText, never from requests or upstream errors.
+type sidecarFailure int
+
+var sidecarCodes = map[sidecarFailure]string{
+ http.StatusBadRequest: "InvalidArgument",
+ http.StatusForbidden: "AccessDenied",
+ http.StatusNotFound: "NoSuchKey",
+ http.StatusMethodNotAllowed: "MethodNotAllowed",
+ http.StatusPreconditionFailed: "PreconditionFailed",
+ http.StatusRequestedRangeNotSatisfiable: "InvalidRange",
+ http.StatusRequestHeaderFieldsTooLarge: "RequestHeaderSectionTooLarge",
+ http.StatusInternalServerError: "InternalError",
+ http.StatusNotImplemented: "NotImplemented",
+ http.StatusBadGateway: "BadGateway",
+ http.StatusServiceUnavailable: "ServiceUnavailable",
+}
+
+// WriteSidecarHeaderTooLarge rejects oversized headers with the sidecar's S3 error format.
+// HEAD returns the same headers as GET without a body.
+func WriteSidecarHeaderTooLarge(w http.ResponseWriter, r *http.Request) {
+ sidecarFailure(http.StatusRequestHeaderFieldsTooLarge).write(w, r)
+}
+
+func (f sidecarFailure) write(w http.ResponseWriter, r *http.Request) {
+ // Both fields are fixed ASCII text without XML metacharacters.
+ body := "" + sidecarCodes[f] + "" + http.StatusText(int(f)) + ""
+
+ w.Header().Set("Content-Type", "application/xml")
+ w.Header().Set("Content-Length", strconv.Itoa(len(body)))
+ w.WriteHeader(int(f))
+
+ if r.Method != http.MethodHead {
+ if _, err := w.Write([]byte(body)); err != nil {
+ return
+ }
+ }
+}
+
+func sidecarSDKFailure(err error) sidecarFailure {
+ switch {
+ case errors.Is(err, racersdk.ErrInvalidRequest):
+ return http.StatusBadRequest
+ case errors.Is(err, racersdk.ErrUnauthorized), errors.Is(err, racersdk.ErrForbidden):
+ return http.StatusForbidden
+ case errors.Is(err, racersdk.ErrNotFound):
+ return http.StatusNotFound
+ case errors.Is(err, racersdk.ErrVersionMismatch):
+ // SDK pins select our representation, not the client's condition.
+ return http.StatusServiceUnavailable
+ case errors.Is(err, racersdk.ErrRangeNotSatisfiable):
+ return http.StatusRequestedRangeNotSatisfiable
+ case errors.Is(err, racersdk.ErrUnavailable), errors.Is(err, context.Canceled), errors.Is(err, context.DeadlineExceeded), errors.Is(err, net.ErrClosed):
+ return http.StatusServiceUnavailable
+ }
+
+ return http.StatusBadGateway
+}
+
+func sidecarSupportedHeaders(h http.Header) bool {
+ var checksumMode bool
+
+ for name, values := range h {
+ name = strings.ToLower(name)
+ switch name {
+ case "if-modified-since", "if-unmodified-since", "if-range":
+ return false
+ case "x-amz-checksum-mode":
+ if checksumMode || len(values) != 1 || strings.Trim(values[0], " \t") != "ENABLED" {
+ return false
+ }
+ // ENABLED asks for available checksums; it does not require one.
+ checksumMode = true
+
+ continue
+ case "x-amz-date", "x-amz-content-sha256", "x-amz-security-token", "x-amz-region-set", "x-amz-s3session-token", "x-amz-user-agent":
+ // These only carry ignored credentials, signing data, or SDK telemetry.
+ continue
+ }
+ // Unknown S3 headers may change authorization or object-read semantics.
+ // Reject even empty values rather than silently dropping those requirements.
+ if strings.HasPrefix(name, "x-amz-") {
+ return false
+ }
+ }
+
+ return true
+}
+
+type sidecarEntityTag struct {
+ value string
+ weak bool
+}
+
+type sidecarTags struct {
+ present, wildcard bool
+ tags []sidecarEntityTag
+}
+
+func sidecarCondition(h http.Header, name string) (sidecarTags, bool) {
+ values, present := h[http.CanonicalHeaderKey(name)]
+
+ result := sidecarTags{present: present}
+ if !present {
+ return result, true
+ }
+
+ s := strings.Trim(strings.Join(values, ","), " \t")
+ if s == "*" {
+ result.wildcard = true
+ return result, true
+ }
+
+ // Ignore empty list elements, but only recognize a wildcard on its own.
+ for s = strings.TrimLeft(s, ", \t"); s != ""; s = strings.TrimLeft(s, ", \t") {
+ var weak, quoted, closed bool
+
+ s, weak = strings.CutPrefix(s, "W/")
+
+ s, quoted = strings.CutPrefix(s, `"`)
+ if !quoted {
+ return result, false
+ }
+
+ // Commas inside a quoted tag are literal; backslashes are not escapes.
+ var value string
+
+ value, s, closed = strings.Cut(s, `"`)
+ if !closed {
+ return result, false
+ }
+
+ for _, c := range []byte(value) {
+ if c < 0x21 || c == 0x7f {
+ return result, false
+ }
+ }
+
+ result.tags = append(result.tags, sidecarEntityTag{value: `"` + value + `"`, weak: weak})
+
+ s = strings.TrimLeft(s, " \t")
+ if s != "" && s[0] != ',' {
+ return result, false
+ }
+ }
+
+ return result, len(result.tags) != 0
+}
+
+func (c sidecarTags) matches(tag string, weak bool) bool {
+ if c.wildcard {
+ return true
+ }
+
+ for _, candidate := range c.tags {
+ if candidate.value == tag && (weak || !candidate.weak) {
+ return true
+ }
+ }
+
+ return false
+}
+
+// sidecarRange selects the bytes to serve. Following RFC 9110 section 14.2 and
+// S3, a Range header that is repeated, malformed, multi-range, reversed, or in
+// an unknown unit is ignored and the full object is served. Only a valid single
+// byte range that the object cannot satisfy returns 416.
+func sidecarRange(h http.Header, size uint64) (offset, length uint64, partial bool, status int) {
+ values, present := h["Range"]
+ if !present {
+ return 0, size, false, 0
+ }
+
+ if len(values) != 1 {
+ return 0, size, false, 0
+ }
+
+ unit, s, ok := strings.Cut(strings.Trim(values[0], " \t"), "=")
+ if !ok || unit == "" {
+ return 0, size, false, 0
+ }
+
+ for _, c := range unit {
+ if (c < 'a' || c > 'z') && (c < 'A' || c > 'Z') && (c < '0' || c > '9') && !strings.ContainsRune("!#$%&'*+-.^_`|~", c) {
+ return 0, size, false, 0
+ }
+ }
+
+ if !strings.EqualFold(unit, "bytes") {
+ return 0, size, false, 0
+ }
+
+ first, last, ok := strings.Cut(s, "-")
+ if !ok || first == "" && last == "" {
+ return 0, size, false, 0
+ }
+
+ if first == "" {
+ suffix, valid := sidecarRangeNumber(last)
+ if !valid {
+ return 0, size, false, 0
+ }
+
+ if suffix == 0 || size == 0 {
+ return 0, 0, false, http.StatusRequestedRangeNotSatisfiable
+ }
+
+ length = min(suffix, size)
+
+ return size - length, length, true, 0
+ }
+
+ start, ok := sidecarRangeNumber(first)
+ if !ok {
+ return 0, size, false, 0
+ }
+
+ end := uint64(0)
+ if last != "" {
+ end, ok = sidecarRangeNumber(last)
+ // Compare the digits, since clamping can make reversed bounds equal.
+ if !ok || sidecarDecimalLess(last, first) {
+ return 0, size, false, 0
+ }
+ }
+
+ if start >= size {
+ return 0, 0, false, http.StatusRequestedRangeNotSatisfiable
+ }
+
+ if last == "" || end >= size {
+ end = size - 1
+ }
+
+ return start, end - start + 1, true, 0
+}
+
+// Reject signs and whitespace rather than accepting anything but decimal digits.
+func sidecarRangeNumber(s string) (uint64, bool) {
+ for _, c := range []byte(s) {
+ if c < '0' || c > '9' {
+ return 0, false
+ }
+ }
+
+ // Object sizes are int64, so clamping cannot change the selected range and
+ // keeps every offset and length convertible to int64.
+ n, err := strconv.ParseInt(s, 10, 64)
+ if errors.Is(err, strconv.ErrRange) {
+ if _, err := strconv.ParseUint(s, 10, 64); err != nil {
+ return 0, false
+ }
+
+ n = math.MaxInt64
+ } else if err != nil || n < 0 {
+ return 0, false
+ }
+
+ return uint64(n), true
+}
+
+// sidecarDecimalLess reports whether decimal digit string a is less than b,
+// ignoring leading zeros.
+func sidecarDecimalLess(a, b string) bool {
+ a = strings.TrimLeft(a, "0")
+ b = strings.TrimLeft(b, "0")
+
+ if len(a) != len(b) {
+ return len(a) < len(b)
+ }
+
+ return a < b
+}
diff --git a/internal/racer/object/sidecar_test.go b/internal/racer/object/sidecar_test.go
new file mode 100644
index 0000000000..7885a1c5be
--- /dev/null
+++ b/internal/racer/object/sidecar_test.go
@@ -0,0 +1,1558 @@
+// Copyright (c) Microsoft Corporation.
+// SPDX-License-Identifier: Apache-2.0
+
+package object
+
+import (
+ "context"
+ "encoding/xml"
+ "errors"
+ "fmt"
+ "io"
+ "math"
+ "net"
+ "net/http"
+ "net/http/httptest"
+ "net/url"
+ "strconv"
+ "strings"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/aws/aws-sdk-go-v2/aws"
+ "github.com/aws/aws-sdk-go-v2/service/s3"
+ "github.com/aws/smithy-go"
+
+ "github.com/Azure/unbounded/pkg/racersdk"
+)
+
+func sidecarTestMetadata(t *testing.T, size int, tag string) racersdk.Metadata {
+ t.Helper()
+
+ return racersdk.Metadata{Size: int64(size), ETag: tag, ExpiresAt: time.Unix(0, 0), ContentType: "text/plain"}
+}
+
+func sidecarTestHandler(t *testing.T, origin racersdk.Origin) (http.Handler, *racersdk.Client) {
+ t.Helper()
+
+ client := testSDKClient(t, origin)
+
+ h, err := NewSidecar(client, SidecarConfig{Namespace: "test", Buckets: []string{"bucket"}})
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ return h, client
+}
+
+func sidecarTestOrigin(t *testing.T, data, tag string, calls *atomic.Int32) racersdk.Origin {
+ t.Helper()
+ m := sidecarTestMetadata(t, len(data), tag)
+
+ return func(_ context.Context, r racersdk.OriginRequest) (racersdk.Metadata, io.ReadCloser, error) {
+ if calls != nil {
+ calls.Add(1)
+ }
+
+ if r.Authorization != "" {
+ t.Error("incoming authorization was forwarded")
+ }
+
+ if r.Head {
+ return m, nil, nil
+ }
+
+ // Direct reads are unpinned; reads after Stat must pin its version.
+ if r.ETag != "" && r.ETag != m.ETag {
+ t.Error("read was pinned to the wrong version")
+ }
+
+ if r.Offset >= m.Size {
+ return m, nil, nil
+ }
+
+ return m, io.NopCloser(strings.NewReader(data[r.Offset:min(r.Offset+r.Length, m.Size)])), nil
+ }
+}
+
+func sidecarTestRequest(h http.Handler, method, target string, headers http.Header) *httptest.ResponseRecorder {
+ r := httptest.NewRequest(method, target, nil)
+ if headers != nil {
+ r.Header = headers
+ }
+
+ w := httptest.NewRecorder()
+ h.ServeHTTP(w, r)
+
+ return w
+}
+
+func TestSidecarReads(t *testing.T) {
+ var calls atomic.Int32
+
+ h, _ := sidecarTestHandler(t, sidecarTestOrigin(t, "0123456789", `"opaque,tag\\value"`, &calls))
+ for _, tc := range []struct {
+ name, method, rangeValue, body, contentRange string
+ status, contentLength int
+ }{
+ {"full", "GET", "", "0123456789", "", 200, 10},
+ {"head", "HEAD", "", "", "", 200, 10},
+ {"bounded", "GET", "bytes=2-5", "2345", "bytes 2-5/10", 206, 4},
+ {"uppercase unit", "GET", "BYTES=2-5", "2345", "bytes 2-5/10", 206, 4},
+ {"mixed case unit", "GET", "bYtEs=7-", "789", "bytes 7-9/10", 206, 3},
+ {"unknown unit", "GET", "items=0-1", "0123456789", "", 200, 10},
+ {"unknown unit value", "GET", "items=not-a-byte-range", "0123456789", "", 200, 10},
+ {"unknown unit beyond size", "GET", "items=999-", "0123456789", "", 200, 10},
+ {"unknown unit head", "HEAD", "items=0-1", "", "", 200, 10},
+ {"open", "GET", "bytes=7-", "789", "bytes 7-9/10", 206, 3},
+ {"suffix", "GET", "bytes=-3", "789", "bytes 7-9/10", 206, 3},
+ {"clamped", "GET", "bytes=8-999", "89", "bytes 8-9/10", 206, 2},
+ {"large suffix", "GET", "bytes=-999", "0123456789", "bytes 0-9/10", 206, 10},
+ {"one byte", "GET", "bytes=0-0", "0", "bytes 0-0/10", 206, 1},
+ {"head range", "HEAD", "bytes=2-5", "", "", 200, 4},
+ {"head suffix range", "HEAD", "bytes=-3", "", "", 200, 3},
+ {"head mixed case unit", "HEAD", "ByTeS=-3", "", "", 200, 3},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ before := calls.Load()
+
+ headers := http.Header{"Authorization": {"AWS4-HMAC-SHA256 incoming-secret"}}
+ if tc.rangeValue != "" {
+ headers.Set("Range", tc.rangeValue)
+ }
+
+ w := sidecarTestRequest(h, tc.method, "/bucket/key?versionId=version%2B1", headers)
+ if w.Code != tc.status || w.Body.String() != tc.body {
+ t.Fatalf("response: %d %q", w.Code, w.Body.String())
+ }
+
+ if w.Header().Get("ETag") != `"opaque,tag\\value"` || w.Header().Get("Content-Range") != tc.contentRange || w.Header().Get("Content-Type") != "text/plain" || w.Header().Get("Accept-Ranges") != "bytes" || w.Header().Get("x-amz-version-id") != "version+1" {
+ t.Fatalf("headers: %v", w.Header())
+ }
+
+ if tc.method == "HEAD" && calls.Load()-before != 1 {
+ t.Fatal("HEAD opened a stream")
+ }
+
+ if w.Header().Get("Content-Length") != strconv.Itoa(tc.contentLength) {
+ t.Fatal("wrong content length")
+ }
+ })
+ }
+}
+
+func TestSidecarConditions(t *testing.T) {
+ var calls atomic.Int32
+
+ tag := `"a,b\c"`
+
+ h, _ := sidecarTestHandler(t, sidecarTestOrigin(t, "data", tag, &calls))
+ for _, tc := range []struct {
+ name, match, none string
+ status int
+ }{
+ {"match", tag, "", 200},
+ {"match list", `"other", ` + tag, "", 200},
+ {"match wildcard", "*", "", 200},
+ {"match absent", `"other"`, "", 412},
+ {"weak match fails", "W/" + tag, "", 412},
+ {"none", "", tag, 304},
+ {"none weak", "", "W/" + tag, 304},
+ {"none list", "", `"other", W/` + tag, 304},
+ {"none wildcard", "", "*", 304},
+ {"none absent", "", `"other"`, 200},
+ {"match precedence", `"other"`, "*", 412},
+ {"both", "*", tag, 304},
+ {"unquoted", "bare", "", 400},
+ {"unterminated", `"x`, "", 400},
+ {"bad list", tag + " suffix", "", 400},
+ {"wildcard list", "*, " + tag, "", 400},
+ {"space in tag", `"a b"`, "", 400},
+ {"empty list", ",,", "", 400},
+ {"empty tag valid", `""`, "", 412},
+ {"empty list elements", ", " + tag + ",,", "", 200},
+ } {
+ for _, method := range []string{"GET", "HEAD"} {
+ t.Run(tc.name+method, func(t *testing.T) {
+ before := calls.Load()
+
+ headers := http.Header{}
+ if tc.match != "" {
+ headers.Set("If-Match", tc.match)
+ }
+
+ if tc.none != "" {
+ headers.Set("If-None-Match", tc.none)
+ }
+
+ w := sidecarTestRequest(h, method, "/bucket/key", headers)
+ if w.Code != tc.status {
+ t.Fatalf("status %d want %d: %s", w.Code, tc.status, w.Body.String())
+ }
+
+ if method == "HEAD" || tc.status == 304 {
+ if w.Body.Len() != 0 {
+ t.Fatal("unexpected response body")
+ }
+ }
+
+ if tc.status == 304 && w.Header().Get("ETag") != tag {
+ t.Fatal("missing 304 ETag")
+ }
+
+ if tc.status != 200 && calls.Load()-before > 1 {
+ t.Fatal("condition opened a stream")
+ }
+ })
+ }
+ }
+
+ w := sidecarTestRequest(h, "GET", "/bucket/key", http.Header{"If-Match": {`"other"`, tag}})
+ if w.Code != 200 {
+ t.Fatal("multiple header lines were not treated as a list")
+ }
+
+ w = sidecarTestRequest(h, "GET", "/bucket/key", http.Header{"If-None-Match": {""}})
+ if w.Code != 400 {
+ t.Fatal("empty condition accepted")
+ }
+}
+
+func TestSidecarRejectedRequests(t *testing.T) {
+ var calls atomic.Int32
+
+ h, _ := sidecarTestHandler(t, sidecarTestOrigin(t, "data", `"v"`, &calls))
+ for _, tc := range []struct {
+ method, target string
+ headers http.Header
+ status int
+ }{
+ {"PUT", "/bucket/key", nil, 405},
+ {"POST", "/bucket/key", nil, 405},
+ {"DELETE", "/bucket/key", nil, 405},
+ {"OPTIONS", "/bucket/key", nil, 405},
+ {"GET", "/", nil, 400},
+ {"GET", "/bucket", nil, 400},
+ {"GET", "/bucket/", nil, 400},
+ {"GET", "/bucket%2Fkey", nil, 400},
+ {"GET", "/other/key", nil, 403},
+ {"GET", "/bucket/key?list-type=2", nil, 501},
+ {"GET", "/bucket/key?acl", nil, 400},
+ {"GET", "/bucket/key?partNumber=1", nil, 501},
+ {"GET", "/bucket/key?response-content-type=text/plain", nil, 501},
+ {"GET", "/bucket/key?X-Amz-Signature=secret", nil, 501},
+ {"GET", "/bucket/key?versionId=", nil, 400},
+ {"GET", "/bucket/key?versionId=1&versionId=2", nil, 400},
+ {"GET", "/bucket/key?versionId=%zz", nil, 400},
+ {"GET", "/bucket/key?versionId=a;b", nil, 400},
+ {"GET", "/bucket/key?versionId=a%0D%0AX-Injected%3Ayes", nil, 400},
+ {"GET", "/bucket/key?versionId=a%00", nil, 400},
+ {"GET", "/bucket/key?versionId=a%7F", nil, 400},
+ {"GET", "/bucket/key?versionId=%FF", nil, 400},
+ {"GET", "/bucket/" + strings.Repeat("k", 1025), nil, 400},
+ {"GET", "/bucket/%FF", nil, 400},
+ {"GET", "/bucket/key?x-id=ListObjects", nil, 501},
+ {"GET", "/bucket/key?x-id=HeadObject", nil, 501},
+ {"HEAD", "/bucket/key?x-id=GetObject", nil, 501},
+ {"GET", "/bucket/key", http.Header{"If-Modified-Since": {"Tue, 06 Oct 2026 00:00:00 GMT"}}, 501},
+ {"GET", "/bucket/key", http.Header{"If-Unmodified-Since": {""}}, 501},
+ {"GET", "/bucket/key", http.Header{"If-Range": {`"v"`}}, 501},
+ } {
+ t.Run(tc.method+tc.target+strconv.Itoa(tc.status), func(t *testing.T) {
+ before := calls.Load()
+
+ w := sidecarTestRequest(h, tc.method, tc.target, tc.headers)
+ if w.Code != tc.status {
+ t.Fatalf("status %d want %d", w.Code, tc.status)
+ }
+
+ if calls.Load() != before {
+ t.Fatal("invalid request reached Racer")
+ }
+
+ if tc.status == 405 && w.Header().Get("Allow") != "GET, HEAD" {
+ t.Fatal("missing Allow")
+ }
+
+ if tc.method == "HEAD" {
+ if w.Body.Len() != 0 {
+ t.Fatal("HEAD error body")
+ }
+
+ return
+ }
+
+ var parsed struct {
+ XMLName xml.Name
+ Code string
+ Message string
+ }
+ if err := xml.Unmarshal(w.Body.Bytes(), &parsed); err != nil || parsed.XMLName.Local != "Error" || parsed.Code == "" || parsed.Message == "" {
+ t.Fatalf("invalid error: %s", w.Body.String())
+ }
+
+ if strings.Contains(w.Body.String(), "secret") || strings.Contains(w.Body.String(), "bucket") {
+ t.Fatal("error leaked request data")
+ }
+ })
+ }
+}
+
+func TestSidecarInvalidRanges(t *testing.T) {
+ h, _ := sidecarTestHandler(t, sidecarTestOrigin(t, "data", `"v"`, nil))
+ for _, tc := range []struct {
+ value string
+ status int
+ }{
+ // Valid single byte ranges that the object cannot satisfy.
+ {"bytes=4-", 416},
+ {"BYTES=4-", 416},
+ {"bytes=4-4", 416},
+ {"bytes=-0", 416},
+ {"bytes=999-1000", 416},
+ {"bytes=9223372036854775807-", 416},
+ // Valid ranges that are satisfiable stay partial.
+ {"Bytes=0-1", 206},
+ {"bytes=3-3", 206},
+ // Invalid or unsupported ranges are ignored per RFC 9110 and S3.
+ {"bytes=2-1", 200},
+ {"bytes=0-1,2-3", 200},
+ {"BYTES=0-1,2-3", 200},
+ {"bytes=0-1, 2-3", 200},
+ {"items=0-1", 200},
+ {"items=4-", 200},
+ {"bytes", 200},
+ {"=0-1", 200},
+ {"bytes =0-1", 200},
+ {"by tes=0-1", 200},
+ {"bytés=0-1", 200},
+ {"bytes=", 200},
+ {"bytes=-", 200},
+ {"ByTeS=-", 200},
+ {"bytes=+1-2", 200},
+ {"bytes=0-+1", 200},
+ {"bytes=0-1-2", 200},
+ {"bytes= 0-1", 200},
+ {"bytes=18446744073709551616-", 200},
+ {"bytes=-bad", 200},
+ {"bytes=-18446744073709551616", 200},
+ {"bytes=5-4", 200},
+ // Reversed ranges stay ignored even when both bounds clamp to MaxInt64.
+ {"bytes=9223372036854775808-9223372036854775807", 200},
+ {"bytes=18446744073709551615-9223372036854775808", 200},
+ {"bytes=0009223372036854775808-9223372036854775807", 200},
+ {"bytes=18446744073709551615-018446744073709551614", 200},
+ // Non-reversed huge ranges still clamp and stay unsatisfiable.
+ {"bytes=9223372036854775807-9223372036854775808", 416},
+ {"bytes=9223372036854775808-9223372036854775808", 416},
+ {"bytes=09223372036854775808-18446744073709551615", 416},
+ {"bytes=00004-0004", 416},
+ {"bytes=002-0003", 206},
+ {"", 200},
+ } {
+ t.Run(tc.value, func(t *testing.T) {
+ for _, method := range []string{"GET", "HEAD"} {
+ t.Run(method, func(t *testing.T) {
+ w := sidecarTestRequest(h, method, "/bucket/key", http.Header{"Range": {tc.value}})
+
+ want := tc.status
+ if method == "HEAD" && want == 206 {
+ want = 200
+ }
+
+ if w.Code != want {
+ t.Fatalf("status %d want %d", w.Code, want)
+ }
+
+ switch tc.status {
+ case 416:
+ if w.Header().Get("Content-Range") != "bytes */4" {
+ t.Fatal("missing unsatisfied range size")
+ }
+ case 200:
+ if w.Header().Get("Content-Range") != "" || w.Header().Get("Content-Length") != "4" {
+ t.Fatalf("ignored range headers: %v", w.Header())
+ }
+
+ if method == "GET" && w.Body.String() != "data" {
+ t.Fatalf("ignored range body %q", w.Body.String())
+ }
+ }
+
+ if method == "HEAD" && w.Body.Len() != 0 {
+ t.Fatal("HEAD body")
+ }
+ })
+ }
+ })
+ }
+
+ for _, method := range []string{"GET", "HEAD"} {
+ w := sidecarTestRequest(h, method, "/bucket/key", http.Header{"Range": {"bytes=0-1", "bytes=2-3"}})
+ if w.Code != 200 || w.Header().Get("Content-Range") != "" || w.Header().Get("Content-Length") != "4" {
+ t.Fatalf("%s multiple Range headers: %d %v", method, w.Code, w.Header())
+ }
+
+ if method == "GET" && w.Body.String() != "data" {
+ t.Fatalf("multiple Range headers body %q", w.Body.String())
+ }
+ }
+}
+
+func TestSidecarExactIdentity(t *testing.T) {
+ for _, key := range []string{"a//b/../c", "/leading", "./relative", "a%2Fb", "a+b c", "雪/☃", "trailing/", "a?b#c", "a\x7fb"} {
+ t.Run(key, func(t *testing.T) {
+ want, err := NewRequest("test", "bucket", key, "version+/%")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ origin := sidecarTestOrigin(t, "body", `"v"`, nil)
+ h, _ := sidecarTestHandler(t, func(ctx context.Context, r racersdk.OriginRequest) (racersdk.Metadata, io.ReadCloser, error) {
+ if r.Key != want.Key || r.Metadata != want.Metadata {
+ t.Error("exact identity changed")
+ }
+
+ return origin(ctx, r)
+ })
+
+ w := sidecarTestRequest(h, "GET", "/bucket/"+url.PathEscape(key)+"?versionId="+url.QueryEscape("version+/%"), nil)
+ if w.Code != 200 || w.Body.String() != "body" {
+ t.Fatalf("response: %d %s", w.Code, w.Body.String())
+ }
+ })
+ }
+}
+
+func TestSidecarVersionRejectsEdgeOWSBeforeLookup(t *testing.T) {
+ // A nil client ensures invalid versions cannot reach Stat or any cache lookup.
+ h := &sidecar{scope: scope{namespace: "test"}}
+
+ for _, version := range []string{"%20v", "v%20", "%20v%20", "%20", "+++", "+v", "v+", "%09v", "v%09", "%20%09v%09%20", "v%09v"} {
+ for _, method := range []string{"GET", "HEAD"} {
+ t.Run(method+version, func(t *testing.T) {
+ w := sidecarTestRequest(h, method, "/bucket/key?versionId="+version, nil)
+ if w.Code != http.StatusBadRequest || w.Header().Get("x-amz-version-id") != "" {
+ t.Fatalf("invalid version response: %d %v", w.Code, w.Header())
+ }
+
+ if method == "HEAD" {
+ if w.Body.Len() != 0 {
+ t.Fatal("HEAD error body")
+ }
+ } else if !strings.Contains(w.Body.String(), "InvalidArgument") {
+ t.Fatalf("wrong error: %s", w.Body.String())
+ }
+ })
+ }
+ }
+}
+
+func TestSidecarVersionExactWireHeader(t *testing.T) {
+ for _, version := range []string{"v", "null", "v v", "version+/=%", "雪", "\u00a0v\u00a0", ""} {
+ t.Run(version, func(t *testing.T) {
+ want, err := NewRequest("test", "bucket", "key", version)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ origin := sidecarTestOrigin(t, "body", `"v"`, nil)
+ h, _ := sidecarTestHandler(t, func(ctx context.Context, r racersdk.OriginRequest) (racersdk.Metadata, io.ReadCloser, error) {
+ if r.Key != want.Key || r.Metadata != want.Metadata {
+ t.Error("exact version identity changed")
+ }
+
+ return origin(ctx, r)
+ })
+
+ server := httptest.NewServer(h)
+ defer server.Close()
+
+ target := server.URL + "/bucket/key"
+ if version != "" {
+ target += "?versionId=" + url.QueryEscape(version)
+ }
+
+ for _, method := range []string{"GET", "HEAD"} {
+ t.Run(method, func(t *testing.T) {
+ r, err := http.NewRequestWithContext(t.Context(), method, target, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ response, err := server.Client().Do(r)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer response.Body.Close()
+
+ body, err := io.ReadAll(response.Body)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if response.StatusCode != http.StatusOK || response.Header.Get("x-amz-version-id") != version {
+ t.Fatalf("wire response: %d %v; want version %q", response.StatusCode, response.Header, version)
+ }
+
+ wantBody := "body"
+ if method == "HEAD" {
+ wantBody = ""
+ }
+
+ if string(body) != wantBody {
+ t.Fatalf("body = %q, want %q", body, wantBody)
+ }
+ })
+ }
+ })
+ }
+}
+
+func TestSidecarSDKFailures(t *testing.T) {
+ for _, tc := range []struct {
+ err error
+ status int
+ code string
+ }{
+ {racersdk.ErrInvalidRequest, 400, "InvalidArgument"},
+ {racersdk.ErrUnauthorized, 403, "AccessDenied"},
+ {racersdk.ErrForbidden, 403, "AccessDenied"},
+ {racersdk.ErrNotFound, 404, "NoSuchKey"},
+ {racersdk.ErrVersionMismatch, 503, "ServiceUnavailable"},
+ {errInvalidS3Response, 502, "BadGateway"},
+ {racersdk.ErrUnavailable, 503, "ServiceUnavailable"},
+ {context.Canceled, 503, "ServiceUnavailable"},
+ {context.DeadlineExceeded, 503, "ServiceUnavailable"},
+ } {
+ for _, method := range []string{"GET", "HEAD"} {
+ t.Run(tc.err.Error()+method, func(t *testing.T) {
+ h, _ := sidecarTestHandler(t, func(context.Context, racersdk.OriginRequest) (racersdk.Metadata, io.ReadCloser, error) {
+ return racersdk.Metadata{}, nil, fmt.Errorf("private-upstream-secret: %w", tc.err)
+ })
+
+ w := sidecarTestRequest(h, method, "/bucket/key", nil)
+ if w.Code != tc.status {
+ t.Fatalf("status %d want %d", w.Code, tc.status)
+ }
+
+ if method == "HEAD" {
+ if w.Body.Len() != 0 {
+ t.Fatal("HEAD error body")
+ }
+
+ return
+ }
+
+ if !strings.Contains(w.Body.String(), ""+tc.code+"") || strings.Contains(w.Body.String(), "private") {
+ t.Fatalf("error: %s", w.Body.String())
+ }
+ })
+ }
+ }
+}
+
+func TestSidecarPinnedVersionFailure(t *testing.T) {
+ // Only suffix ranges, and ranges that need clamping, Stat first and pin the read.
+ for _, rangeValue := range []string{"bytes=-2", "bytes=-99"} {
+ for _, tc := range []struct {
+ name, match, none string
+ status int
+ }{
+ {"unconditional", "", "", 503},
+ {"matching tag", `"old"`, "", 503},
+ {"wildcard", "*", "", 503},
+ {"list accepts replacement", `"old", "new"`, "", 503},
+ {"list with weak replacement", `"old", W/"new"`, "", 503},
+ {"none allows selected", "", `"new"`, 503},
+ {"failed match", `"new"`, "", 412},
+ {"failed weak match", `W/"old"`, "", 412},
+ {"failed list", `"new", W/"old"`, "", 412},
+ {"failed match precedence", `"new"`, "*", 412},
+ {"none wildcard", "", "*", 304},
+ {"none list", "*", `"new", W/"old"`, 304},
+ } {
+ t.Run(rangeValue+"/"+tc.name, func(t *testing.T) {
+ m := sidecarTestMetadata(t, 4, `"old"`)
+
+ var calls atomic.Int32
+
+ h, _ := sidecarTestHandler(t, func(_ context.Context, r racersdk.OriginRequest) (racersdk.Metadata, io.ReadCloser, error) {
+ if calls.Add(1) == 1 {
+ return m, nil, nil
+ }
+
+ if r.ETag != m.ETag {
+ t.Error("missing Stat pin")
+ }
+
+ return racersdk.Metadata{}, nil, fmt.Errorf("private pin: %w", racersdk.ErrVersionMismatch)
+ })
+
+ headers := http.Header{"Range": {rangeValue}}
+ if tc.match != "" {
+ headers.Set("If-Match", tc.match)
+ }
+
+ if tc.none != "" {
+ headers.Set("If-None-Match", tc.none)
+ }
+
+ w := sidecarTestRequest(h, "GET", "/bucket/key", headers)
+ if w.Code != tc.status {
+ t.Fatalf("status %d want %d: %s", w.Code, tc.status, w.Body.String())
+ }
+
+ if tc.status == 503 {
+ if calls.Load() < 2 || !strings.Contains(w.Body.String(), "ServiceUnavailable") || strings.Contains(w.Body.String(), "private") || w.Header().Get("ETag") != "" || w.Header().Get("Content-Range") != "" {
+ t.Fatalf("pin failure committed object or unsafe error: calls=%d headers=%v body=%q", calls.Load(), w.Header(), w.Body.String())
+ }
+ } else if calls.Load() != 1 {
+ t.Fatal("failed condition opened a stream")
+ }
+ })
+ }
+ }
+}
+
+func TestSidecarRejectsChangedPinnedMetadata(t *testing.T) {
+ for _, field := range []string{"size", "empty-size", "content-type"} {
+ t.Run(field, func(t *testing.T) {
+ m := sidecarTestMetadata(t, 4, `"same"`)
+ if field == "empty-size" {
+ m.Size = 0
+ }
+
+ var calls atomic.Int32
+
+ h, _ := sidecarTestHandler(t, func(_ context.Context, r racersdk.OriginRequest) (racersdk.Metadata, io.ReadCloser, error) {
+ if calls.Add(1) == 1 {
+ return m, nil, nil
+ }
+
+ changed := m
+ if field == "content-type" {
+ changed.ContentType = "application/octet-stream"
+ } else {
+ changed.Size++
+ }
+
+ if r.Head {
+ return changed, nil, nil
+ }
+
+ return changed, io.NopCloser(strings.NewReader(strings.Repeat("x", int(changed.Size)))), nil
+ })
+
+ if field == "empty-size" {
+ // A whole-object GET reads directly, so its headers come from
+ // the read itself and cannot disagree with the body.
+ w := sidecarTestRequest(h, "GET", "/bucket/key", nil)
+ if w.Code != 200 || w.Body.Len() != 0 || w.Header().Get("Content-Length") != "0" || calls.Load() != 1 {
+ t.Fatalf("direct read: calls=%d %d %v %q", calls.Load(), w.Code, w.Header(), w.Body.String())
+ }
+
+ return
+ }
+
+ w := sidecarTestRequest(h, "GET", "/bucket/key", http.Header{"Range": {"bytes=-2"}})
+ if w.Code != http.StatusBadGateway || !strings.Contains(w.Body.String(), "BadGateway") {
+ t.Fatalf("changed metadata committed object: %d %q", w.Code, w.Body.String())
+ }
+ })
+ }
+}
+
+func TestSidecarEmptyObject(t *testing.T) {
+ h, _ := sidecarTestHandler(t, sidecarTestOrigin(t, "", `"empty"`, nil))
+ for _, method := range []string{"GET", "HEAD"} {
+ w := sidecarTestRequest(h, method, "/bucket/key", nil)
+ if w.Code != 200 || w.Body.Len() != 0 || w.Header().Get("Content-Length") != "0" {
+ t.Fatalf("empty: %d %v %q", w.Code, w.Header(), w.Body.String())
+ }
+
+ w = sidecarTestRequest(h, method, "/bucket/key", http.Header{"Range": {"bytes=0-"}})
+ if w.Code != 416 || w.Header().Get("Content-Range") != "bytes */0" {
+ t.Fatalf("empty range: %d", w.Code)
+ }
+
+ w = sidecarTestRequest(h, method, "/bucket/key", http.Header{"Range": {"items=0-"}})
+ if w.Code != 200 || w.Body.Len() != 0 || w.Header().Get("Content-Length") != "0" || w.Header().Get("Content-Range") != "" {
+ t.Fatalf("empty unknown range: %d %v %q", w.Code, w.Header(), w.Body.String())
+ }
+
+ for _, value := range []string{"bytes=1-0", "bytes=0-1,2-3", "bytes=x-"} {
+ w = sidecarTestRequest(h, method, "/bucket/key", http.Header{"Range": {value}})
+ if w.Code != 200 || w.Body.Len() != 0 || w.Header().Get("Content-Length") != "0" || w.Header().Get("Content-Range") != "" {
+ t.Fatalf("empty invalid range %q: %d %v %q", value, w.Code, w.Header(), w.Body.String())
+ }
+ }
+ }
+}
+
+func TestSidecarLateFailureAborts(t *testing.T) {
+ m := sidecarTestMetadata(t, 1024*1024, `"v"`)
+ h, _ := sidecarTestHandler(t, func(_ context.Context, r racersdk.OriginRequest) (racersdk.Metadata, io.ReadCloser, error) {
+ if r.Head {
+ return m, nil, nil
+ }
+
+ return m, io.NopCloser(strings.NewReader(strings.Repeat("x", 512*1024))), nil
+ })
+ t.Run("handler abort", func(t *testing.T) {
+ defer func() {
+ if got := recover(); got != http.ErrAbortHandler {
+ t.Errorf("panic = %v", got)
+ }
+ }()
+
+ sidecarTestRequest(h, "GET", "/bucket/key", nil)
+ t.Error("failed stream returned normally")
+ })
+
+ server := httptest.NewServer(h)
+ t.Cleanup(server.Close)
+
+ req, err := http.NewRequestWithContext(t.Context(), "GET", server.URL+"/bucket/key", nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ res, err := server.Client().Do(req)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer res.Body.Close()
+
+ data, err := io.ReadAll(res.Body)
+ if err == nil || len(data) == int(m.Size) || strings.Contains(string(data), "") {
+ t.Fatalf("late failure became success or XML: bytes=%d err=%v", len(data), err)
+ }
+}
+
+func TestSidecarAWSClient(t *testing.T) {
+ getRequest, err := NewRequest("test", "bucket", "a//b/../c", "")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ headRequest, err := NewRequest("test", "bucket", "key", "")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ missingRequest, err := NewRequest("test", "bucket", "missing", "")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ origin := sidecarTestOrigin(t, "0123456789", `"v"`, nil)
+ h, _ := sidecarTestHandler(t, func(ctx context.Context, r racersdk.OriginRequest) (racersdk.Metadata, io.ReadCloser, error) {
+ if r.Key == missingRequest.Key {
+ return racersdk.Metadata{}, nil, racersdk.ErrNotFound
+ }
+
+ if r.Key != getRequest.Key && r.Key != headRequest.Key {
+ t.Error("signed SDK key identity changed")
+ }
+
+ return origin(ctx, r)
+ })
+
+ var signed atomic.Bool
+
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if strings.HasPrefix(r.Header.Get("Authorization"), "AWS4-HMAC-SHA256 ") {
+ signed.Store(true)
+ }
+
+ h.ServeHTTP(w, r)
+ }))
+ t.Cleanup(server.Close)
+ client := testS3Client(t, server)
+
+ res, err := client.GetObject(t.Context(), &s3.GetObjectInput{Bucket: aws.String("bucket"), Key: aws.String("a//b/../c"), Range: aws.String("bytes=2-4"), IfMatch: aws.String(`"v"`)})
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer res.Body.Close()
+
+ data, err := io.ReadAll(res.Body)
+ if err != nil || string(data) != "234" || aws.ToString(res.ContentRange) != "bytes 2-4/10" {
+ t.Fatalf("SDK response %q %v", data, err)
+ }
+
+ head, err := client.HeadObject(t.Context(), &s3.HeadObjectInput{Bucket: aws.String("bucket"), Key: aws.String("key")})
+ if err != nil || aws.ToInt64(head.ContentLength) != 10 {
+ t.Fatalf("SDK HEAD: %v", err)
+ }
+
+ if !signed.Load() {
+ t.Fatal("test did not send signed SDK requests")
+ }
+
+ _, err = client.GetObject(t.Context(), &s3.GetObjectInput{Bucket: aws.String("bucket"), Key: aws.String("missing")})
+
+ var apiError smithy.APIError
+ if !errors.As(err, &apiError) || apiError.ErrorCode() != "NoSuchKey" {
+ t.Fatalf("SDK error mapping: %v", err)
+ }
+
+ res, err = client.GetObject(t.Context(), &s3.GetObjectInput{Bucket: aws.String("bucket"), Key: aws.String("key"), ChecksumMode: "ENABLED"})
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer res.Body.Close()
+
+ data, err = io.ReadAll(res.Body)
+ if err != nil || string(data) != "0123456789" {
+ t.Fatalf("SDK explicit checksum response %q %v", data, err)
+ }
+}
+
+func TestSidecarAWSDefaultChecksum(t *testing.T) {
+ for _, tc := range []struct {
+ name, data, rangeValue, want string
+ }{
+ {"full", "data", "", "data"},
+ {"range", "data", "bytes=1-2", "at"},
+ {"empty", "", "", ""},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ h, _ := sidecarTestHandler(t, sidecarTestOrigin(t, tc.data, `"v"`, nil))
+
+ var requests atomic.Int32
+
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ requests.Add(1)
+
+ if got := r.Header.Values("X-Amz-Checksum-Mode"); len(got) != 1 || got[0] != "ENABLED" {
+ t.Errorf("SDK checksum mode = %q, want ENABLED", got)
+ }
+
+ h.ServeHTTP(w, r)
+ }))
+ t.Cleanup(server.Close)
+ client := testS3Client(t, server)
+
+ input := &s3.GetObjectInput{Bucket: aws.String("bucket"), Key: aws.String("key")}
+ if tc.rangeValue != "" {
+ input.Range = aws.String(tc.rangeValue)
+ }
+
+ res, err := client.GetObject(t.Context(), input)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer res.Body.Close()
+
+ data, err := io.ReadAll(res.Body)
+ if err != nil || string(data) != tc.want || requests.Load() != 1 {
+ t.Fatalf("SDK read = %q, error = %v, requests = %d", data, err, requests.Load())
+ }
+
+ if res.ChecksumCRC32 != nil || res.ChecksumCRC32C != nil || res.ChecksumCRC64NVME != nil || res.ChecksumSHA1 != nil || res.ChecksumSHA256 != nil || res.ChecksumType != "" {
+ t.Fatal("invented checksum metadata")
+ }
+ })
+ }
+}
+
+func TestSidecarChecksumMode(t *testing.T) {
+ var calls atomic.Int32
+
+ h, _ := sidecarTestHandler(t, sidecarTestOrigin(t, "data", `"v"`, &calls))
+
+ for _, tc := range []struct {
+ name string
+ values []string
+ status int
+ }{
+ {"enabled", []string{"ENABLED"}, 200},
+ {"whitespace", []string{" \tENABLED\t "}, 200},
+ {"empty", []string{""}, 501},
+ {"no values", nil, 501},
+ {"lowercase", []string{"enabled"}, 501},
+ {"disabled", []string{"DISABLED"}, 501},
+ {"unknown", []string{"private-mode"}, 501},
+ {"repeated", []string{"ENABLED", "ENABLED"}, 501},
+ {"conflicting", []string{"ENABLED", "DISABLED"}, 501},
+ {"combined", []string{"ENABLED, ENABLED"}, 501},
+ } {
+ for _, method := range []string{"GET", "HEAD"} {
+ t.Run(tc.name+method, func(t *testing.T) {
+ for _, name := range []string{"x-amz-checksum-mode", "X-Amz-Checksum-Mode", "X-AMZ-CHECKSUM-MODE"} {
+ before := calls.Load()
+
+ w := sidecarTestRequest(h, method, "/bucket/key", http.Header{name: tc.values})
+ if w.Code != tc.status {
+ t.Fatalf("status = %d, want %d", w.Code, tc.status)
+ }
+
+ if tc.status != 200 && calls.Load() != before {
+ t.Fatal("invalid checksum mode reached Racer")
+ }
+
+ if method == "HEAD" && w.Body.Len() != 0 {
+ t.Fatal("HEAD returned a body")
+ }
+
+ for header := range w.Header() {
+ if strings.HasPrefix(strings.ToLower(header), "x-amz-checksum-") {
+ t.Fatalf("invented checksum header %s", header)
+ }
+ }
+ }
+ })
+ }
+ }
+
+ for _, extra := range []string{"x-amz-checksum-mode", "x-amz-expected-bucket-owner", "x-amz-server-side-encryption-customer-key", "x-amz-request-payer", "x-amz-unknown-operation"} {
+ for _, method := range []string{"GET", "HEAD"} {
+ t.Run("enabled with "+extra+method, func(t *testing.T) {
+ before := calls.Load()
+
+ w := sidecarTestRequest(h, method, "/bucket/key", http.Header{
+ "X-Amz-Checksum-Mode": {"ENABLED"}, extra: {"ENABLED"},
+ })
+ if w.Code != 501 || calls.Load() != before {
+ t.Fatalf("unsupported header accepted: status=%d calls=%d", w.Code, calls.Load()-before)
+ }
+ })
+ }
+ }
+}
+
+func TestSidecarConfigAndClosedClient(t *testing.T) {
+ h, client := sidecarTestHandler(t, sidecarTestOrigin(t, "data", `"v"`, nil))
+ for _, tc := range []struct {
+ config SidecarConfig
+ message string
+ }{
+ {SidecarConfig{}, "racer-object: invalid sidecar namespace"},
+ {SidecarConfig{Namespace: "test", Buckets: []string{""}}, "racer-object: invalid sidecar bucket"},
+ } {
+ if _, err := NewSidecar(client, tc.config); err == nil || err.Error() != tc.message {
+ t.Fatalf("invalid config error: %v, want %q", err, tc.message)
+ }
+ }
+
+ if _, err := NewSidecar(nil, SidecarConfig{Namespace: "test"}); err == nil || err.Error() != "racer-object: missing sidecar client" {
+ t.Fatalf("nil client error: %v", err)
+ }
+
+ if err := client.Close(); err != nil {
+ t.Fatal(err)
+ }
+
+ w := sidecarTestRequest(h, "GET", "/bucket/key", nil)
+ if w.Code != 503 {
+ t.Fatalf("closed client: %d", w.Code)
+ }
+}
+
+func TestSidecarCrossPageRange(t *testing.T) {
+ data := strings.Repeat("0123456789abcdef", int(racersdk.PageSize)/16+2)
+ h, _ := sidecarTestHandler(t, sidecarTestOrigin(t, data, `"pages"`, nil))
+ first := int(racersdk.PageSize) - 4
+
+ w := sidecarTestRequest(h, "GET", "/bucket/key", http.Header{"Range": {"bytes=" + strconv.Itoa(first) + "-" + strconv.Itoa(first+10)}})
+ if w.Code != 206 || w.Body.String() != data[first:first+11] {
+ t.Fatalf("cross-page response: %d %q", w.Code, w.Body.String())
+ }
+}
+
+func TestSidecarEmptyReadFailure(t *testing.T) {
+ m := sidecarTestMetadata(t, 0, `"empty"`)
+
+ var calls atomic.Int32
+
+ h, _ := sidecarTestHandler(t, func(_ context.Context, r racersdk.OriginRequest) (racersdk.Metadata, io.ReadCloser, error) {
+ calls.Add(1)
+
+ if r.Head {
+ return m, nil, nil
+ }
+
+ return racersdk.Metadata{}, nil, racersdk.ErrUnavailable
+ })
+
+ w := sidecarTestRequest(h, "GET", "/bucket/key", nil)
+ if calls.Load() < 1 || w.Code != 503 || !strings.Contains(w.Body.String(), "ServiceUnavailable") {
+ t.Fatalf("empty read failure became success: %d %s", w.Code, w.Body.String())
+ }
+}
+
+func TestSidecarCancellation(t *testing.T) {
+ entered := make(chan struct{})
+ m := sidecarTestMetadata(t, 10, `"v"`)
+ h, _ := sidecarTestHandler(t, func(ctx context.Context, r racersdk.OriginRequest) (racersdk.Metadata, io.ReadCloser, error) {
+ if r.Head {
+ return m, nil, nil
+ }
+
+ close(entered)
+ <-ctx.Done()
+
+ return racersdk.Metadata{}, nil, ctx.Err()
+ })
+
+ ctx, cancel := context.WithCancel(t.Context())
+ defer cancel()
+
+ r := httptest.NewRequestWithContext(ctx, "GET", "/bucket/key", nil)
+ done := make(chan struct{})
+
+ go func() { defer close(done); h.ServeHTTP(httptest.NewRecorder(), r) }()
+
+ select {
+ case <-entered:
+ case <-time.After(5 * time.Second):
+ t.Fatal("origin not entered")
+ }
+
+ cancel()
+
+ select {
+ case <-done:
+ case <-time.After(5 * time.Second):
+ t.Fatal("handler did not stop after cancellation")
+ }
+}
+
+func TestSidecarAllowlistSnapshot(t *testing.T) {
+ _, client := sidecarTestHandler(t, sidecarTestOrigin(t, "data", `"v"`, nil))
+ config := SidecarConfig{Namespace: "test", Buckets: []string{"bucket"}}
+
+ h, err := NewSidecar(client, config)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ config.Buckets[0] = "other"
+
+ w := sidecarTestRequest(h, "HEAD", "/bucket/key", nil)
+ if w.Code != 200 {
+ t.Fatal("caller mutation changed allowlist")
+ }
+
+ h, err = NewSidecar(client, SidecarConfig{Namespace: "test"})
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ w = sidecarTestRequest(h, "HEAD", "/other/key", nil)
+ if w.Code != 200 {
+ t.Fatal("empty allowlist rejected bucket")
+ }
+}
+
+func TestSidecarUnsupportedSemanticHeaders(t *testing.T) {
+ var calls atomic.Int32
+
+ h, _ := sidecarTestHandler(t, sidecarTestOrigin(t, "data", `"v"`, &calls))
+
+ for _, name := range []string{
+ "x-amz-expected-bucket-owner",
+ "x-amz-server-side-encryption-customer-algorithm",
+ "x-amz-server-side-encryption-customer-key",
+ "x-amz-server-side-encryption-customer-key-MD5",
+ "x-amz-server-side-encryption",
+ "x-amz-checksum-mode",
+ "x-amz-request-payer",
+ "x-amz-copy-source",
+ "x-amz-meta-custom",
+ "x-amz-unknown-operation",
+ "If-Modified-Since", "If-Unmodified-Since", "If-Range",
+ } {
+ for _, method := range []string{"GET", "HEAD"} {
+ for _, value := range []string{"private-semantic-value", ""} {
+ t.Run(name+method+value, func(t *testing.T) {
+ for _, spelling := range []string{name, http.CanonicalHeaderKey(name), strings.ToUpper(name)} {
+ w := sidecarTestRequest(h, method, "/bucket/key", http.Header{spelling: {value}})
+ if w.Code != 501 {
+ t.Fatalf("%s status %d", spelling, w.Code)
+ }
+
+ if method == "HEAD" {
+ if w.Body.Len() != 0 {
+ t.Fatal("HEAD error body")
+ }
+ } else if !strings.Contains(w.Body.String(), "NotImplemented") || strings.Contains(w.Body.String(), "private-semantic-value") {
+ t.Fatalf("unsafe or incorrect XML: %s", w.Body.String())
+ }
+ }
+ })
+ }
+ }
+ }
+
+ if calls.Load() != 0 {
+ t.Fatal("unsupported headers reached Racer")
+ }
+}
+
+func TestSidecarAuthAndPlumbingHeaders(t *testing.T) {
+ h, _ := sidecarTestHandler(t, sidecarTestOrigin(t, "data", `"v"`, nil))
+
+ headers := http.Header{
+ "Authorization": {"AWS4-HMAC-SHA256 ignored"},
+ "X-Amz-Date": {"20261006T140000Z"},
+ "X-Amz-Content-Sha256": {"UNSIGNED-PAYLOAD"},
+ "X-Amz-Security-Token": {"ignored-session-token"},
+ "X-Amz-Region-Set": {"us-east-1"},
+ "X-Amz-S3session-Token": {"ignored-express-token"},
+ "X-Amz-User-Agent": {"aws-sdk-test"},
+ "Amz-Sdk-Invocation-Id": {"test-invocation"},
+ "Amz-Sdk-Request": {"attempt=1; max=1"},
+ "User-Agent": {"aws-sdk-test"},
+ "Accept-Encoding": {"identity"},
+ }
+ for _, method := range []string{"GET", "HEAD"} {
+ w := sidecarTestRequest(h, method, "/bucket/key", headers)
+ if w.Code != 200 {
+ t.Fatalf("normal SDK headers rejected: %d %s", w.Code, w.Body.String())
+ }
+ }
+}
+
+func TestSidecarFailureWireFormat(t *testing.T) {
+ for _, tc := range []struct {
+ status int
+ code string
+ }{
+ {400, "InvalidArgument"},
+ {403, "AccessDenied"},
+ {404, "NoSuchKey"},
+ {405, "MethodNotAllowed"},
+ {412, "PreconditionFailed"},
+ {416, "InvalidRange"},
+ {431, "RequestHeaderSectionTooLarge"},
+ {500, "InternalError"},
+ {501, "NotImplemented"},
+ {502, "BadGateway"},
+ {503, "ServiceUnavailable"},
+ } {
+ for _, method := range []string{"GET", "HEAD"} {
+ t.Run(tc.code+method, func(t *testing.T) {
+ w := httptest.NewRecorder()
+ sidecarFailure(tc.status).write(w, httptest.NewRequest(method, "/", nil))
+
+ body := "" + tc.code + "" + http.StatusText(tc.status) + ""
+ if w.Code != tc.status || w.Header().Get("Content-Type") != "application/xml" || w.Header().Get("Content-Length") != strconv.Itoa(len(body)) {
+ t.Fatalf("error response: %d %v", w.Code, w.Header())
+ }
+
+ if method == "HEAD" {
+ body = ""
+ }
+
+ if w.Body.String() != body {
+ t.Fatalf("body %q, want %q", w.Body.String(), body)
+ }
+ })
+ }
+ }
+
+ for _, tc := range []struct {
+ err error
+ status sidecarFailure
+ }{
+ {errors.New("private"), 502},
+ {racersdk.ErrRangeNotSatisfiable, 416},
+ {context.Canceled, 503},
+ {context.DeadlineExceeded, 503},
+ {net.ErrClosed, 503},
+ {fmt.Errorf("wrapped: %w", racersdk.ErrNotFound), 404},
+ } {
+ if got := sidecarSDKFailure(tc.err); got != tc.status {
+ t.Errorf("mapping %v: %d, want %d", tc.err, got, tc.status)
+ }
+ }
+
+ w := &sidecarFailingWriter{ResponseRecorder: httptest.NewRecorder()}
+ sidecarFailure(400).write(w, httptest.NewRequest("GET", "/", nil))
+
+ if w.Code != 400 || w.writes != 1 {
+ t.Fatalf("failed error write: status=%d writes=%d", w.Code, w.writes)
+ }
+}
+
+type sidecarFailingWriter struct {
+ *httptest.ResponseRecorder
+ writes int
+}
+
+func (w *sidecarFailingWriter) Write([]byte) (int, error) {
+ w.writes++
+ return 0, io.ErrClosedPipe
+}
+
+func TestSidecarPathFallbackAndDefaultHeaders(t *testing.T) {
+ origin := sidecarTestOrigin(t, "data", `"v"`, nil)
+
+ h, _ := sidecarTestHandler(t, func(ctx context.Context, r racersdk.OriginRequest) (racersdk.Metadata, io.ReadCloser, error) {
+ want, err := NewRequest("test", "bucket", "a%zz", "")
+ if err != nil || r.Key != want.Key {
+ t.Error("invalid RawPath changed identity")
+ }
+
+ m, body, err := origin(ctx, r)
+ m.ContentType = ""
+
+ return m, body, err
+ })
+ for _, method := range []string{"GET", "HEAD"} {
+ r := httptest.NewRequest(method, "/bucket/a%25zz?x-id="+map[string]string{"GET": "GetObject", "HEAD": "HeadObject"}[method], nil)
+ r.URL.RawPath = "/%zz/bad"
+ w := httptest.NewRecorder()
+ h.ServeHTTP(w, r)
+
+ if w.Code != 200 || w.Header().Get("Content-Type") != "application/octet-stream" || w.Header().Get("Content-Length") != "4" || w.Header().Get("x-amz-version-id") != "" {
+ t.Fatalf("default headers: %d %v", w.Code, w.Header())
+ }
+ }
+}
+
+func TestSidecarEmptyStreamClosedBeforeCompletion(t *testing.T) {
+ _, client := sidecarTestHandler(t, sidecarTestOrigin(t, "", `"empty"`, nil))
+
+ request, err := NewRequest("test", "bucket", "key", "")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ m := sidecarTestMetadata(t, 0, `"empty"`)
+
+ v, err := client.Get(t.Context(), request, racersdk.ReadOptions{ETag: m.ETag})
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if err := v.Close(); err != nil {
+ t.Fatal(err)
+ }
+
+ w := httptest.NewRecorder()
+
+ var reports []SidecarFailure
+
+ s := &sidecar{onFailure: func(f SidecarFailure) { reports = append(reports, f) }}
+
+ failure := s.writeBody(w, httptest.NewRequest("GET", "/bucket/key", nil), sidecarRead{metadata: m}, v)
+
+ if len(reports) != 1 || reports[0].Status != 503 || reports[0].Bucket != "bucket" || reports[0].Err == nil {
+ t.Fatalf("empty failure reports = %+v", reports)
+ }
+
+ if failure != 503 || len(w.Header()) != 0 || w.Body.Len() != 0 || w.Flushed {
+ t.Fatalf("empty failure committed success: failure=%d headers=%v body=%q", failure, w.Header(), w.Body.String())
+ }
+}
+
+type sidecarDeadlineWriter struct {
+ *httptest.ResponseRecorder
+ deadline time.Time
+ changes int
+ statuses []int
+}
+
+func (w *sidecarDeadlineWriter) SetWriteDeadline(deadline time.Time) error {
+ w.deadline = deadline
+ w.changes++
+
+ return nil
+}
+
+func (w *sidecarDeadlineWriter) WriteHeader(status int) {
+ w.statuses = append(w.statuses, status)
+ w.ResponseRecorder.WriteHeader(status)
+}
+
+func TestSidecarEmptyStreamPreservesHTTPDeadline(t *testing.T) {
+ for _, outcome := range []string{"complete", "closed", "canceled"} {
+ t.Run(outcome, func(t *testing.T) {
+ _, client := sidecarTestHandler(t, sidecarTestOrigin(t, "", `"empty"`, nil))
+
+ request, err := NewRequest("test", "bucket", "key", "")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ ctx, cancel := context.WithCancel(t.Context())
+ defer cancel()
+
+ m := sidecarTestMetadata(t, 0, `"empty"`)
+
+ v, err := client.Get(ctx, request, racersdk.ReadOptions{ETag: m.ETag})
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer v.Close()
+
+ switch outcome {
+ case "closed":
+ if err := v.Close(); err != nil {
+ t.Fatal(err)
+ }
+ case "canceled":
+ cancel()
+ }
+
+ deadline := time.Now().Add(time.Minute)
+ w := &sidecarDeadlineWriter{ResponseRecorder: httptest.NewRecorder(), deadline: deadline}
+
+ var reports []SidecarFailure
+
+ s := &sidecar{onFailure: func(f SidecarFailure) { reports = append(reports, f) }}
+
+ failure := s.writeBody(w, httptest.NewRequest("GET", "/bucket/key", nil), sidecarRead{metadata: m}, v)
+
+ if want := map[bool]int{true: 0, false: 1}[outcome == "complete"]; len(reports) != want || want == 1 && reports[0].Status != 503 {
+ t.Fatalf("reports = %+v, want %d", reports, want)
+ }
+
+ if w.changes != 0 || !w.deadline.Equal(deadline) {
+ t.Fatalf("empty validation changed HTTP deadline: changes=%d deadline=%v", w.changes, w.deadline)
+ }
+
+ if outcome == "complete" {
+ if failure != 0 || len(w.statuses) != 1 || w.statuses[0] != 200 || w.Header().Get("Content-Length") != "0" || w.Header().Get("ETag") != m.ETag || w.Body.Len() != 0 {
+ t.Fatalf("empty success: failure=%d statuses=%v headers=%v body=%q", failure, w.statuses, w.Header(), w.Body.String())
+ }
+ } else if failure != 503 || len(w.statuses) != 0 || len(w.Header()) != 0 || w.Body.Len() != 0 {
+ t.Fatalf("empty failure committed response: failure=%d statuses=%v headers=%v body=%q", failure, w.statuses, w.Header(), w.Body.String())
+ }
+ })
+ }
+}
+
+func TestSidecarRangeClampsHugeNumbers(t *testing.T) {
+ const huge = "18446744073709551615"
+
+ for _, tc := range []struct {
+ header string
+ offset, length uint64
+ status int
+ }{
+ {"bytes=2-" + huge, 2, 8, 0},
+ {"bytes=-" + huge, 0, 10, 0},
+ {"bytes=" + huge + "-", 0, 0, http.StatusRequestedRangeNotSatisfiable},
+ {"bytes=0-18446744073709551616", 0, 10, 0},
+ } {
+ h := http.Header{"Range": {tc.header}}
+
+ offset, length, _, status := sidecarRange(h, 10)
+ if offset != tc.offset || length != tc.length || status != tc.status {
+ t.Fatalf("%s: offset=%d length=%d status=%d", tc.header, offset, length, status)
+ }
+ }
+
+ if n, ok := sidecarRangeNumber(huge); !ok || n != math.MaxInt64 {
+ t.Fatalf("clamp: %d %v", n, ok)
+ }
+}
+
+func TestSidecarDirectReadsSkipStat(t *testing.T) {
+ const data, tag = "0123456789", `"v"`
+
+ client := &sidecarCountingClient{Client: testSDKClient(t, sidecarTestOrigin(t, data, tag, nil))}
+ h := sidecarCountingHandler(t, client)
+
+ for _, tc := range []struct {
+ name, method, rangeValue, match, none, body, contentRange string
+ status, stats int
+ }{
+ {"whole", "GET", "", "", "", data, "", 200, 0},
+ {"open", "GET", "bytes=3-", "", "", "3456789", "bytes 3-9/10", 206, 0},
+ {"open last byte", "GET", "bytes=9-", "", "", "9", "bytes 9-9/10", 206, 0},
+ {"unknown unit", "GET", "items=0-1", "", "", data, "", 200, 0},
+ {"match", "GET", "", tag, "", data, "", 200, 0},
+ {"match open", "GET", "bytes=8-", "*", "", "89", "bytes 8-9/10", 206, 0},
+ {"match fails", "GET", "", `"other"`, "", "", "", 412, 0},
+ {"none matches", "GET", "", "", tag, "", "", 304, 0},
+ {"none matches open", "GET", "bytes=2-", "", "W/" + tag, "", "", 304, 0},
+ {"none differs", "GET", "", "", `"other"`, data, "", 200, 0},
+ {"open at size", "GET", "bytes=10-", "", "", "", "bytes */10", 416, 1},
+ {"open past size", "GET", "bytes=99-", "", "", "", "bytes */10", 416, 1},
+ {"closed", "GET", "bytes=1-2", "", "", "12", "bytes 1-2/10", 206, 0},
+ {"closed to end", "GET", "bytes=1-9", "", "", "123456789", "bytes 1-9/10", 206, 0},
+ {"closed match fails", "GET", "bytes=1-2", `"other"`, "", "", "", 412, 0},
+ {"closed clamped", "GET", "bytes=8-99", "", "", "89", "bytes 8-9/10", 206, 1},
+ {"closed past size", "GET", "bytes=10-12", "", "", "", "bytes */10", 416, 1},
+ {"suffix", "GET", "bytes=-2", "", "", "89", "bytes 8-9/10", 206, 1},
+ {"head", "HEAD", "", "", "", "", "", 200, 1},
+ {"head open", "HEAD", "bytes=3-", "", "", "", "", 200, 1},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ client.stats.Store(0)
+
+ headers := http.Header{}
+ if tc.rangeValue != "" {
+ headers.Set("Range", tc.rangeValue)
+ }
+
+ if tc.match != "" {
+ headers.Set("If-Match", tc.match)
+ }
+
+ if tc.none != "" {
+ headers.Set("If-None-Match", tc.none)
+ }
+
+ w := sidecarTestRequest(h, tc.method, "/bucket/key", headers)
+ if w.Code != tc.status || w.Header().Get("Content-Range") != tc.contentRange {
+ t.Fatalf("response: %d %v %q", w.Code, w.Header(), w.Body.String())
+ }
+
+ if tc.status < 300 && w.Body.String() != tc.body {
+ t.Fatalf("body %q want %q", w.Body.String(), tc.body)
+ }
+
+ if tc.status == 304 && (w.Body.Len() != 0 || w.Header().Get("ETag") != tag) {
+ t.Fatalf("304: %v %q", w.Header(), w.Body.String())
+ }
+
+ if tc.status < 300 && w.Header().Get("ETag") != tag {
+ t.Fatalf("missing ETag: %v", w.Header())
+ }
+
+ if got := int(client.stats.Load()); got != tc.stats {
+ t.Fatalf("Stat calls %d want %d", got, tc.stats)
+ }
+ })
+ }
+}
+
+// An unpinned direct read cannot fail its own pin, so a plain GET never
+// returns 412 even when Racer reports a version mismatch.
+func TestSidecarDirectReadVersionMismatchIsNotPrecondition(t *testing.T) {
+ h, _ := sidecarTestHandler(t, func(context.Context, racersdk.OriginRequest) (racersdk.Metadata, io.ReadCloser, error) {
+ return racersdk.Metadata{}, nil, racersdk.ErrVersionMismatch
+ })
+
+ for _, rangeValue := range []string{"", "bytes=2-"} {
+ headers := http.Header{}
+ if rangeValue != "" {
+ headers.Set("Range", rangeValue)
+ }
+
+ if w := sidecarTestRequest(h, "GET", "/bucket/key", headers); w.Code == 412 {
+ t.Fatalf("%q: plain GET returned 412", rangeValue)
+ }
+ }
+}
+
+// Whole-object, open, and in-bounds closed range GETs do not use the SDK's Stat lane, so more
+// concurrent reads than that lane admits (four connections plus a queue of 16)
+// all succeed while every read is held open at once.
+func TestSidecarConcurrentDirectReads(t *testing.T) {
+ const n = 48
+
+ entered := make(chan struct{}, n)
+ release := make(chan struct{})
+ client := &sidecarCountingClient{
+ Client: testSDKClient(t, sidecarTestOrigin(t, "0123456789", `"v"`, nil)),
+ afterGet: func() {
+ entered <- struct{}{}
+
+ <-release
+ },
+ }
+ h := sidecarCountingHandler(t, client)
+
+ codes := make(chan int, n)
+ for i := range n {
+ go func() {
+ headers := http.Header{}
+
+ switch i % 3 {
+ case 1:
+ headers.Set("Range", "bytes=4-")
+ case 2:
+ headers.Set("Range", "bytes=2-5")
+ }
+
+ codes <- sidecarTestRequest(h, "GET", "/bucket/key", headers).Code
+ }()
+ }
+
+ timeout := time.After(30 * time.Second)
+
+ for range n {
+ select {
+ case <-entered:
+ case code := <-codes:
+ close(release)
+ t.Fatalf("request finished early with %d", code)
+ case <-timeout:
+ close(release)
+ t.Fatal("reads were not all in flight")
+ }
+ }
+
+ close(release)
+
+ for range n {
+ if code := <-codes; code != 200 && code != 206 {
+ t.Fatalf("concurrent GET: %d", code)
+ }
+ }
+
+ if client.stats.Load() != 0 {
+ t.Fatalf("Stat calls: %d", client.stats.Load())
+ }
+}
+
+// sidecarCountingClient counts Stat calls and can hold reads after Get.
+type sidecarCountingClient struct {
+ *racersdk.Client
+ stats atomic.Int32
+ afterGet func()
+}
+
+func (c *sidecarCountingClient) Stat(ctx context.Context, request racersdk.Request) (racersdk.Metadata, error) {
+ c.stats.Add(1)
+ return c.Client.Stat(ctx, request)
+}
+
+func (c *sidecarCountingClient) Get(ctx context.Context, request racersdk.Request, options ...racersdk.ReadOptions) (*racersdk.Object, error) {
+ v, err := c.Client.Get(ctx, request, options...)
+ if err == nil && c.afterGet != nil {
+ c.afterGet()
+ }
+
+ return v, err
+}
+
+func sidecarCountingHandler(t *testing.T, client *sidecarCountingClient) http.Handler {
+ t.Helper()
+
+ bound, err := newScope("test", []string{"bucket"})
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ return &sidecar{client: client, scope: bound}
+}
+
+func TestSidecarDecimalLess(t *testing.T) {
+ for _, tc := range []struct {
+ a, b string
+ want bool
+ }{
+ {"9223372036854775807", "9223372036854775808", true},
+ {"9223372036854775808", "9223372036854775807", false},
+ {"0009223372036854775808", "9223372036854775808", false},
+ {"9", "10", true},
+ {"010", "9", false},
+ {"0", "000", false},
+ {"", "0", false},
+ {"1", "2", true},
+ } {
+ if got := sidecarDecimalLess(tc.a, tc.b); got != tc.want {
+ t.Fatalf("%q < %q = %v", tc.a, tc.b, got)
+ }
+ }
+}