feat(racer-object): add S3 origin and HTTP sidecar

.github/workflows/images.yaml

@@ -102,6 +102,13 @@ jobs:
 
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
 
+      - name: Record build provenance
+        id: provenance
+        run: |
+          git_commit=$(git rev-parse HEAD)
+          build_time=$(date -u +%Y-%m-%dT%H:%M:%SZ)
+          printf 'git_commit=%s\nbuild_time=%s\n' "$git_commit" "$build_time" >> "$GITHUB_OUTPUT"
+
       - uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4.4.0
         with:
           platforms: linux/amd64,linux/arm64
@@ -140,3 +147,5 @@ jobs:
           build-args: |
             VERSION=${{ needs.resolve.outputs.tag }}
             CONTAINER_REGISTRY=${{ env.CONTAINER_REGISTRY }}
+            GIT_COMMIT=${{ steps.provenance.outputs.git_commit }}
+            BUILD_TIME=${{ steps.provenance.outputs.build_time }}

Makefile

@@ -23,6 +23,7 @@ GO_PACKAGE_DIRS=$(shell $(GOCMD) list -tags e2e -f '{{.Dir}}' $(GO_PACKAGE_PATTE
 
 CONTAINER_ENGINE ?= podman
 CONTAINER_REGISTRY ?= ghcr.io/azure
+RACER_OBJECT_IMAGE ?= $(CONTAINER_REGISTRY)/racer-object:$(VERSION_TAG)
 
 # Unified install namespace for all unbounded components. Each component's
 # *_NAMESPACE var derives from this by default, so overriding UNBOUNDED_NAMESPACE
@@ -237,7 +238,7 @@ REACT_DEV ?= false
 
 ##@ General
 
-all: kubectl-unbounded forge relctl machina machine-ops-controller token-refresher unbounded-operator unbounded-net-controller unbounded-net-node unbounded-net-routeplan-debug unping unroute gantry ## Build all binaries (default)
+all: kubectl-unbounded forge relctl machina machine-ops-controller token-refresher unbounded-operator unbounded-net-controller unbounded-net-node unbounded-net-routeplan-debug unping unroute gantry racer-object ## Build all binaries (default)
 
 help: ## Show this help
 	@echo ""
@@ -295,6 +296,9 @@ help: ## Show this help
 	@echo "  unbounded-net-routeplan-debug    Build net routeplan debug tool"
 	@echo "  unping                           Build unping health-check utility"
 	@echo "  unroute                          Build unroute eBPF inspection utility"
+	@echo "  racer-object                     Test and build the S3 read adapter"
+	@echo "  racer-object-build               Build the S3 read adapter without tests"
+	@echo "  racer-object-test                Test the S3 read adapter"
 	@echo ""
 	@echo "Container Images (local, single-arch):"
 	@echo "  image-inventory-all-local        Build all local inventory container images"
@@ -312,6 +316,7 @@ help: ## Show this help
 	@echo "  image-unbounded-operator-local   Build unbounded-operator image"
 	@echo "  image-unbounded-operator-push    Build and push unbounded-operator image"
 	@echo "  image-playpen-local              Build playpen image"
+	@echo "  image-racer-object-local         Build the S3 read adapter image locally"
 	@echo "  image-net-controller-local       Build unbounded-net-controller image"
 	@echo "  image-net-controller-push        Build and push unbounded-net-controller image"
 	@echo "  image-net-node-local             Build unbounded-net-node image"
@@ -818,6 +823,27 @@ unroute-build: ## Build the unroute utility binary (no lint/test)
 
 unroute: test unroute-build ## Build the unroute utility (implies test)
 
+##@ Racer Object (S3 read adapter)
+
+.PHONY: racer-object racer-object-build racer-object-test image-racer-object-local
+racer-object: racer-object-test racer-object-build ## Test and build the S3 read adapter
+
+racer-object-build: ## Build the S3 read adapter without lint/test
+	@mkdir -p bin
+	$(GOBUILD) -trimpath -ldflags '$(STAMP_LDFLAGS)' -o bin/racer-object ./cmd/racer-object
+
+racer-object-test: ## Test the S3 read adapter
+	timeout --signal=TERM --kill-after=10s 300s $(GOTEST) -timeout=5m ./cmd/racer-object/... ./internal/racer/object/...
+
+image-racer-object-local: ## Build the S3 read adapter image locally (single-arch)
+	$(CONTAINER_ENGINE) build \
+		--build-arg VERSION=$(VERSION) \
+		--build-arg GIT_COMMIT=$(GIT_COMMIT) \
+		--build-arg BUILD_TIME=$(BUILD_TIME) \
+		-t racer-object:$(VERSION_TAG) -t $(RACER_OBJECT_IMAGE) \
+		-f ./images/racer-object/Containerfile .
+	$(call trivy-maybe,$(RACER_OBJECT_IMAGE))
+
 ##@ Gantry (peer-to-peer OCI distribution)
 
 gantry-build: ## Build the gantry binary (no lint/test)

NOTICE

@@ -631,6 +631,13 @@ notices:
     license:
       - name: BSD 3-Clause License
         link: https://go.googlesource.com/mod
+  - dependency: golang.org/x/net
+    ecosystem: go
+    copyright:
+      - Copyright 2009 The Go Authors.
+    license:
+      - name: BSD 3-Clause License
+        link: https://go.googlesource.com/net
   - dependency: golang.org/x/sync
     ecosystem: go
     copyright:

cmd/racer-object/config.go

@@ -0,0 +1,46 @@
+// Copyright (c) Microsoft Corporation.
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+	"net/http"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+)
+
+type originS3Client struct {
+	*s3.Client
+	pool originConnectionPool
+}
+
+func (c *originS3Client) Close() error {
+	c.pool.close()
+	return nil
+}
+
+func newOriginS3Client(cfg aws.Config, o options) (*originS3Client, error) {
+	owned := &originS3Client{}
+
+	client := s3.NewFromConfig(cfg, func(s *s3.Options) {
+		s.UsePathStyle = o.pathStyle
+		if o.endpoint != "" {
+			s.BaseEndpoint = aws.String(o.endpoint)
+		}
+
+		if buildable, ok := s.HTTPClient.(*awshttp.BuildableClient); ok {
+			s.HTTPClient = buildable.WithTransportOptions(func(tr *http.Transport) {
+				tr.DialContext = owned.pool.dial(tr.DialContext)
+			})
+		}
+	})
+	if err := validateEndpoint(aws.ToString(client.Options().BaseEndpoint)); err != nil {
+		return nil, safeDiagnostic("invalid effective S3 endpoint: must be an HTTP(S) origin without credentials, query, fragment, or path prefix, with a valid port", "config", err)
+	}
+
+	owned.Client = client
+
+	return owned, nil
+}

cmd/racer-object/config_test.go

@@ -0,0 +1,153 @@
+// Copyright (c) Microsoft Corporation.
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+	"errors"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+)
+
+func isolateOriginAWSConfig(t *testing.T) string {
+	t.Helper()
+
+	dir := t.TempDir()
+	for key, value := range map[string]string{
+		"AWS_CONFIG_FILE": filepath.Join(dir, "config"), "AWS_SHARED_CREDENTIALS_FILE": filepath.Join(dir, "credentials"),
+		"AWS_PROFILE": "", "AWS_DEFAULT_PROFILE": "", "AWS_EC2_METADATA_DISABLED": "true",
+		"AWS_ACCESS_KEY_ID": "test-access", "AWS_SECRET_ACCESS_KEY": "test-secret", "AWS_SESSION_TOKEN": "test-session",
+		"AWS_ENDPOINT_URL": "", "AWS_ENDPOINT_URL_S3": "", "AWS_IGNORE_CONFIGURED_ENDPOINT_URLS": "false",
+		"AWS_CA_BUNDLE": "",
+	} {
+		t.Setenv(key, value)
+	}
+
+	return dir
+}
+
+func TestOriginEffectiveEndpoint(t *testing.T) {
+	for _, source := range []string{"global-env", "service-env", "global-shared", "service-shared"} {
+		t.Run(source, func(t *testing.T) {
+			for _, endpoint := range []string{
+				"", "https://s3.example", "https://s3.example/", "http://localhost:9000",
+				"https://private-user:private-password@s3.example", "https://s3.example/private-path",
+				"https://s3.example?private-query=secret", "https://s3.example#private-fragment", "https://s3.example:99999",
+				"https://s3.example/%zz",
+			} {
+				t.Run(endpoint, func(t *testing.T) {
+					dir := isolateOriginAWSConfig(t)
+					setOriginEndpointSource(t, dir, source, endpoint)
+
+					o := options{region: "us-east-1", requestTimeout: time.Minute, pathStyle: true}
+
+					cfg, err := originAWSConfig(t.Context(), o)
+					if err != nil {
+						t.Fatal(err)
+					}
+
+					client, err := newOriginS3Client(cfg, o)
+					if validateEndpoint(endpoint) != nil {
+						if err == nil || !strings.HasPrefix(err.Error(), "invalid effective S3 endpoint:") || strings.Contains(err.Error(), endpoint) {
+							t.Fatalf("error = %v, want redacted endpoint error", err)
+						}
+
+						cause := errors.Unwrap(err)
+						if cause == nil || cause.Error() != validateEndpoint(endpoint).Error() || !errors.Is(err, cause) || !strings.Contains(err.Error(), "category=config") {
+							t.Fatalf("endpoint diagnostic cause or category lost: %v", err)
+						}
+
+						if err := runOrigin(t.Context(), o); err == nil || !strings.HasPrefix(err.Error(), "invalid effective S3 endpoint:") {
+							t.Fatalf("startup error = %v, want endpoint rejection before directory/socket setup", err)
+						}
+
+						return
+					}
+
+					if err != nil {
+						t.Fatal(err)
+					}
+
+					if got := aws.ToString(client.Options().BaseEndpoint); got != endpoint || !client.Options().UsePathStyle {
+						t.Fatalf("endpoint = %q, want %q with path style", got, endpoint)
+					}
+				})
+			}
+		})
+	}
+}
+
+func setOriginEndpointSource(t *testing.T, dir, source, endpoint string) {
+	t.Helper()
+
+	switch source {
+	case "global-env":
+		t.Setenv("AWS_ENDPOINT_URL", endpoint)
+	case "service-env":
+		t.Setenv("AWS_ENDPOINT_URL_S3", endpoint)
+	case "global-shared", "service-shared":
+		text := "[default]\nendpoint_url = " + endpoint + "\n"
+		if source == "service-shared" {
+			text = "[default]\nservices = local\n[services local]\ns3 =\n  endpoint_url = " + endpoint + "\n"
+		}
+
+		if err := os.WriteFile(filepath.Join(dir, "config"), []byte(text), 0o600); err != nil {
+			t.Fatal(err)
+		}
+	default:
+		t.Fatal("unknown endpoint source")
+	}
+}
+
+func TestOriginEndpointOverride(t *testing.T) {
+	for _, source := range []string{"global-env", "service-env", "global-shared", "service-shared"} {
+		t.Run(source, func(t *testing.T) {
+			dir := isolateOriginAWSConfig(t)
+			setOriginEndpointSource(t, dir, source, "https://private-user:private-password@s3.example/private-path?secret=value")
+
+			o := options{region: "us-east-1", requestTimeout: time.Minute, endpoint: "https://override.example"}
+
+			cfg, err := originAWSConfig(t.Context(), o)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			client, err := newOriginS3Client(cfg, o)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if got := aws.ToString(client.Options().BaseEndpoint); got != o.endpoint {
+				t.Fatalf("endpoint = %q, want flag override %q", got, o.endpoint)
+			}
+		})
+	}
+}
+
+func TestOriginServiceEndpointPrecedence(t *testing.T) {
+	isolateOriginAWSConfig(t)
+	t.Setenv("AWS_ENDPOINT_URL", "https://private-user:private-password@s3.example/private-path")
+	t.Setenv("AWS_ENDPOINT_URL_S3", "https://service.example")
+
+	o := options{region: "us-east-1", requestTimeout: time.Minute}
+
+	cfg, err := originAWSConfig(t.Context(), o)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	client, err := newOriginS3Client(cfg, o)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer closeResource(client)
+
+	if got := aws.ToString(client.Options().BaseEndpoint); got != "https://service.example" {
+		t.Fatalf("endpoint = %q, want service environment override", got)
+	}
+}