Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions docs/metrics/DESIGN.md
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,7 @@ single object that `display()` and `to_json()` render.
class Report(msgspec.Struct, frozen=True):
version: str
git_sha: str | None
git_sha_source: str # which channel resolved git_sha: git | env | none
test_started_at: int
n_samples_issued: int
n_samples_completed: int
Expand Down
2 changes: 1 addition & 1 deletion docs/metrics/report_design.md
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ returns `{}` (or an all-null early-stopping map if the feature is enabled).

### `Report` (frozen `msgspec.Struct`)

Fields: `version`, `git_sha`, `test_started_at`,
Fields: `version`, `git_sha`, `git_sha_source`, `test_started_at`,
`n_samples_issued/completed/failed`, `duration_ns`, `state`, `complete`, the
five rollup dicts (`ttft`, `tpot`, `latency`, `input_sequence_lengths`,
`output_sequence_lengths`), `legacy_loadgen_window_duration_ns`, `qps`, `tps`,
Expand Down
18 changes: 11 additions & 7 deletions docs/utils/DESIGN.md
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,11 @@ benchmarking tool that lives here for convenience but does import from other

`logging.py` and `dataset_utils.py` stay lightweight and broadly reusable.
`version.py` is also small, but it intentionally imports
`inference_endpoint.__version__` and shells out to `git` to report build
metadata. `benchmark_httpclient.py` is exempt entirely: it is a standalone tool,
not a reusable helper.
`inference_endpoint.__version__` and resolves the source SHA to report build
metadata: a live `git` query of its own checkout, or, only outside a git
checkout, the `ENDPOINTS_GIT_SHA` env var. It is resolved once per process.
`benchmark_httpclient.py` is exempt entirely: it is a standalone tool, not a
reusable helper.

**`benchmark_httpclient.py` is a standalone tool**

Expand All @@ -58,7 +60,9 @@ uv run python -m inference_endpoint.utils.benchmark_httpclient --endpoint URL --

## Integration Points

| Consumer | Usage |
| ------------------ | -------------------------------------------- |
| `main.py` | Calls `setup_logging()` at startup |
| `commands/info.py` | Imports `__version__` for the `info` command |
| Consumer | Usage |
| ------------------------------- | ------------------------------------------------------------------------------------------------------ |
| `main.py` | Calls `setup_logging()` at startup |
| `commands/info.py` | Imports `__version__` for the `info` command |
| `commands/benchmark/execute.py` | `setup_benchmark` calls `get_version_info()` first, pinning provenance before the run |
| `metrics/report.py` | `Report.from_snapshot` calls `get_version_info()` to record `git_sha` / `git_sha_source` in the report |
20 changes: 16 additions & 4 deletions scripts/Dockerfile.dev
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# Development Dockerfile for the MLPerf Inference Endpoint Benchmarking System
# From project root:
# docker build -f scripts/Dockerfile.dev --build-arg USER_ID=$(id -u) --build-arg GROUP_ID=$(id -g) -t inference-endpoint-dev .
# From project root (GIT_SHA is the source commit the report prints; see the end of this file):
# docker build -f scripts/Dockerfile.dev --build-arg GIT_SHA=$(git rev-parse --short=7 HEAD)$(git diff --quiet HEAD -- src/inference_endpoint pyproject.toml uv.lock examples/09_Wan22_VideoGen_Example/accuracy || echo -dirty) --build-arg USER_ID=$(id -u) --build-arg GROUP_ID=$(id -g) -t inference-endpoint-dev .
# docker run -v $(pwd):/mnt/inference-endpoint -it --shm-size=512m inference-endpoint-dev bash
#
# VBench (WAN 2.2) accuracy scorer is ON by default (consistent with the DeepSeek-R1
Expand Down Expand Up @@ -130,12 +130,24 @@ RUN if [ "${PROVISION_ROUGE}" = "1" ]; then \
fi

# OCI image metadata so `docker inspect` self-identifies this image (the endpoints benchmark
# client, distinct from the lcb-service evaluator image). Declared last so a wording change
# only rebuilds this tiny config layer, not the apt/uv/DSR1/VBench stages above.
# client, distinct from the lcb-service evaluator image). Declared after the build stages
# so a wording change only rebuilds the trailing config layers, not the
# apt/uv/DSR1/VBench stages above.
# push_docker_image.sh additionally stamps dynamic source/revision/version/description
# annotations on the pushed manifest. The description is deliberately capability-neutral:
# PROVISION_DSR1 / PROVISION_VBENCH can be 0, so it names the evaluators without asserting
# they are present in every build.
LABEL org.opencontainers.image.title="inference-endpoint" \
org.opencontainers.image.description="MLPerf inference endpoint benchmarking client (the inference-endpoint CLI); DeepSeek-R1 and VBench accuracy evaluators are provisioned by default, toggled by PROVISION_DSR1 / PROVISION_VBENCH." \
org.opencontainers.image.source="https://github.com/mlcommons/endpoints"

# Runtime provenance. This image COPYs src/ without .git, so the report prints
# this value (source: env). A git checkout bind-mounted over the workdir (the
# `docker run -v` above) reports live git instead, or no SHA if git can't read
# it: this value describes the image, not the mounted code. Mount the whole
# checkout: code mounted without .git (a tarball, or src/ alone) is reported
# with this value. Declared last so a new SHA doesn't invalidate the cached
# layers above; empty or non-hex values are ignored. push_docker_image.sh passes
# it and tags the image with the same SHA.
ARG GIT_SHA=
ENV ENDPOINTS_GIT_SHA=${GIT_SHA}
13 changes: 12 additions & 1 deletion scripts/push_docker_image.sh
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,9 @@
# PROVISION_VBENCH passthrough build-arg (default: Dockerfile default = 1)
# IMAGE_DESCRIPTION GHCR package-page description (default: "endpoints client image (ref <ref>, commit <sha>)")
# GHCR_USER/GHCR_TOKEN optional registry login (fallback: GITHUB_ACTOR/GITHUB_TOKEN)
#
# Always passes --build-arg GIT_SHA=<short sha>[-dirty], which the image reports
# at runtime as its source commit.
set -euo pipefail

REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
Expand Down Expand Up @@ -153,7 +156,7 @@ else
ENDPOINTS_REF="$ORIGINAL_REF"
fi

SHORT_SHA="$(git rev-parse --short HEAD)"
SHORT_SHA="$(git rev-parse --short=7 HEAD)"
# Docker tags allow only [A-Za-z0-9_.-], must not start with '.' or '-', and cap at
# 128 chars. Map every other char (e.g. '/', '+', '~', '#') to '-' so a legitimate
# git ref name can't yield an invalid tag that fails `-t` only after the whole build.
Expand Down Expand Up @@ -213,6 +216,14 @@ BUILD_ARGS=()
[[ "$NO_CACHE" == "1" ]] && BUILD_ARGS+=(--no-cache)
[[ -n "${PROVISION_DSR1:-}" ]] && BUILD_ARGS+=(--build-arg "PROVISION_DSR1=${PROVISION_DSR1}")
[[ -n "${PROVISION_VBENCH:-}" ]] && BUILD_ARGS+=(--build-arg "PROVISION_VBENCH=${PROVISION_VBENCH}")
# Source SHA the image reports at runtime (Dockerfile ARG GIT_SHA -> ENV
# ENDPOINTS_GIT_SHA): the :$SHORT_SHA tag, formatted like live git's
# (--short=7). The -dirty check uses the runtime check's scope
# (utils/version._DIRTY_PATHSPEC); it is reachable only via --allow-dirty /
# --no-push, since a clean :SHA is otherwise required above.
GIT_SHA_VALUE="$SHORT_SHA"
git diff --quiet HEAD -- src/inference_endpoint pyproject.toml uv.lock examples/09_Wan22_VideoGen_Example/accuracy || GIT_SHA_VALUE="${GIT_SHA_VALUE}-dirty"
BUILD_ARGS+=(--build-arg "GIT_SHA=${GIT_SHA_VALUE}")

# Don't attach SLSA provenance attestations. buildx adds them by default on --push,
# which surfaces a spurious `unknown/unknown` entry on the GHCR package page and
Expand Down
10 changes: 10 additions & 0 deletions src/inference_endpoint/commands/benchmark/execute.py
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,7 @@
SessionResult,
)
from inference_endpoint.metrics.report import Report
from inference_endpoint.utils.version import get_version_info

if TYPE_CHECKING:
from inference_endpoint.async_utils.event_publisher import EventPublisherService
Expand Down Expand Up @@ -471,6 +472,15 @@ def setup_benchmark(
``audit_run_spec``, when set, overrides the issue count and sample order
for a compliance-audit phase (see ``commands/audit.py:run_audit``).
"""
# Pins source provenance for the process before any work runs; every report
# (main run and audit phases) records this value.
version_info = get_version_info()
logger.info(
"Endpoints %s, git SHA %s (source: %s)",
version_info["version"],
version_info["git_sha"] or "unknown",
version_info["git_sha_source"],
)
# Accuracy-only runs force single-stream (1 worker / 1 connection) for
# deterministic sample ordering. Bake it into the config here — before CPU
# affinity, report_dir/config.yaml persistence, and RuntimeSettings — so the
Expand Down
11 changes: 9 additions & 2 deletions src/inference_endpoint/metrics/report.py
Original file line number Diff line number Diff line change
Expand Up @@ -211,6 +211,10 @@ class Report(msgspec.Struct, frozen=True): # type: ignore[call-arg]

version: str
git_sha: str | None
# Which channel resolved git_sha: "git" (read from the running checkout),
# "env" (ENDPOINTS_GIT_SHA, an attestation by the image or launcher), or
# "none". See utils/version.resolve_git_sha.
git_sha_source: str
test_started_at: int
n_samples_issued: int
# Terminal responses; failed samples are a subset of this count.
Expand Down Expand Up @@ -429,6 +433,7 @@ def _series_dict(key: str) -> dict[str, Any]:
return cls(
version=str(version_info.get("version", "unknown")),
git_sha=version_info.get("git_sha"),
git_sha_source=str(version_info.get("git_sha_source", "none")),
test_started_at=0, # TODO: surface session_started_ns via snapshot
n_samples_issued=n_issued,
n_samples_completed=n_completed,
Expand Down Expand Up @@ -528,8 +533,10 @@ def display(
f"final snapshot received) — some async metrics may be missing.{newline}"
)
fn(f"Version: {self.version}{newline}")
if self.git_sha:
fn(f"Git SHA: {self.git_sha}{newline}")
# Always emitted, so an unresolved SHA reads "unknown (source: none)".
fn(
f"Git SHA: {self.git_sha or 'unknown'} (source: {self.git_sha_source}){newline}"
)
if self.run_config:
fn(f"Run config:{newline}")
for section, params in self.run_config.items():
Expand Down
Loading
Loading