Refine semantic inner-loop early stop

.github/workflows/benchmark.yml

@@ -0,0 +1,238 @@
+name: LanPaintBench LPIPS (ImageNet subset)
+
+on:
+  workflow_dispatch:
+    inputs:
+      base_ref:
+        description: "Base ref to benchmark against (commit SHA or branch). Leave empty to use repo default branch."
+        required: false
+        default: ""
+      subset:
+        description: "Number of ImageNet images to benchmark"
+        required: true
+        default: "1"
+      sample_steps:
+        description: "Outer diffusion steps (guided_diffusion_e2e)"
+        required: true
+        default: "20"
+      n_steps:
+        description: "LanPaint inner steps per outer step"
+        required: true
+        default: "5"
+      seed:
+        description: "Random seed"
+        required: true
+        default: "42"
+      shuffle_buffer:
+        description: "HF streaming shuffle buffer"
+        required: true
+        default: "1024"
+      semantic_threshold:
+        description: "Enable semantic early-stop if > 0 (head_on run)"
+        required: true
+        default: "0.01"
+      semantic_patience:
+        description: "Semantic early-stop patience (head_on run)"
+        required: true
+        default: "2"
+      min_seconds:
+        description: "Fail if a benchmark runtime is below this (0 to disable)"
+        required: true
+        default: "0"
+      max_case_mean_delta:
+        description: "Fail if mean per-case metric delta exceeds this (head_on - head_off)"
+        required: true
+        default: "0.01"
+      max_case_p95_delta:
+        description: "Fail if P95 per-case metric delta exceeds this (head_on - head_off)"
+        required: true
+        default: "0.02"
+      max_case_max_delta:
+        description: "Fail if max per-case metric delta exceeds this (head_on - head_off)"
+        required: true
+        default: "0.05"
+      gallery_topk:
+        description: "How many worst-case examples to render in the gallery"
+        required: true
+        default: "8"
+
+jobs:
+  benchmark:
+    name: Compare Base vs Head
+    runs-on: ubuntu-latest
+    timeout-minutes: 360
+    env:
+      HF_HOME: ~/.cache/huggingface
+      BASE_REF: ${{ inputs.base_ref || github.event.repository.default_branch }}
+      OMP_NUM_THREADS: "1"
+      MKL_NUM_THREADS: "1"
+      LANPAINTBENCH_REF: "ea5be5d"
+      CHECKPOINT_URL: "https://openaipublic.blob.core.windows.net/diffusion/jul-2021/256x256_diffusion_uncond.pt"
+    steps:
+      - name: Checkout Head (Current PR)
+        uses: actions/checkout@v4
+        with:
+          path: head
+          fetch-depth: 1
+
+      - name: Checkout Base (Baseline)
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ env.BASE_REF }}
+          path: base
+          fetch-depth: 1
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+          cache: 'pip'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu
+          pip install lpips datasets numpy pillow blobfile
+          # We don't need to install LanPaint as a package because we use sys.path in the script
+
+      - name: Cache HF + torch + checkpoints
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/huggingface
+            ~/.cache/torch
+            ~/.cache/lanpaintbench
+          key: ${{ runner.os }}-lpips-imagenet-${{ hashFiles('head/tests/benchmark_lpips.py', 'head/tests/compare_benchmarks.py', 'head/tests/render_benchmark_gallery.py') }}
+
+      - name: Clone LanPaintBench
+        run: |
+          git clone https://github.com/scraed/LanPaintBench.git bench
+          cd bench
+          git checkout "$LANPAINTBENCH_REF"
+
+      - name: Download ImageNet-256 checkpoint
+        run: |
+          CHECKPOINT_PATH="$HOME/.cache/lanpaintbench/checkpoints/256x256_diffusion_uncond.pt"
+          mkdir -p ~/.cache/lanpaintbench/checkpoints
+          if [ ! -f "$CHECKPOINT_PATH" ]; then
+            curl -L "$CHECKPOINT_URL" -o "$CHECKPOINT_PATH"
+          fi
+          ls -lh "$CHECKPOINT_PATH"
+
+      - name: Run Baseline Benchmark
+        run: |
+          CHECKPOINT_PATH="$HOME/.cache/lanpaintbench/checkpoints/256x256_diffusion_uncond.pt"
+          # Use the benchmark script from the HEAD branch to ensure test consistency
+          python head/tests/benchmark_lpips.py \
+            --mode guided_diffusion_e2e \
+            --src base/src \
+            --lanpaintbench bench \
+            --checkpoint "$CHECKPOINT_PATH" \
+            --output baseline.json \
+            --dataset benjamin-paine/imagenet-1k-256x256 \
+            --split train \
+            --subset "${{ inputs.subset }}" \
+            --seed "${{ inputs.seed }}" \
+            --shuffle-buffer "${{ inputs.shuffle_buffer }}" \
+            --no-shuffle \
+            --mask-type box \
+            --n-steps "${{ inputs.n_steps }}" \
+            --sample-steps "${{ inputs.sample_steps }}" \
+            --device cpu \
+            --min-seconds "${{ inputs.min_seconds }}" \
+            --semantic-threshold 0.0
+
+      - name: Run Head Benchmark (early-stop OFF)
+        run: |
+          CHECKPOINT_PATH="$HOME/.cache/lanpaintbench/checkpoints/256x256_diffusion_uncond.pt"
+          python head/tests/benchmark_lpips.py \
+            --mode guided_diffusion_e2e \
+            --src head/src \
+            --lanpaintbench bench \
+            --checkpoint "$CHECKPOINT_PATH" \
+            --output head_off.json \
+            --dataset benjamin-paine/imagenet-1k-256x256 \
+            --split train \
+            --subset "${{ inputs.subset }}" \
+            --seed "${{ inputs.seed }}" \
+            --shuffle-buffer "${{ inputs.shuffle_buffer }}" \
+            --no-shuffle \
+            --mask-type box \
+            --n-steps "${{ inputs.n_steps }}" \
+            --sample-steps "${{ inputs.sample_steps }}" \
+            --device cpu \
+            --min-seconds "${{ inputs.min_seconds }}" \
+            --semantic-threshold 0.0 \
+            --save-images-dir head_off_images
+
+      - name: Run Head Benchmark (early-stop ON)
+        run: |
+          CHECKPOINT_PATH="$HOME/.cache/lanpaintbench/checkpoints/256x256_diffusion_uncond.pt"
+          python head/tests/benchmark_lpips.py \
+            --mode guided_diffusion_e2e \
+            --src head/src \
+            --lanpaintbench bench \
+            --checkpoint "$CHECKPOINT_PATH" \
+            --output head_on.json \
+            --dataset benjamin-paine/imagenet-1k-256x256 \
+            --split train \
+            --subset "${{ inputs.subset }}" \
+            --seed "${{ inputs.seed }}" \
+            --shuffle-buffer "${{ inputs.shuffle_buffer }}" \
+            --no-shuffle \
+            --mask-type box \
+            --n-steps "${{ inputs.n_steps }}" \
+            --sample-steps "${{ inputs.sample_steps }}" \
+            --device cpu \
+            --min-seconds "${{ inputs.min_seconds }}" \
+            --semantic-threshold "${{ inputs.semantic_threshold }}" \
+            --semantic-patience "${{ inputs.semantic_patience }}" \
+            --save-images-dir head_on_images
+
+      - name: Compare baseline vs head_off
+        continue-on-error: true
+        run: |
+          python head/tests/compare_benchmarks.py --baseline baseline.json --current head_off.json --max-rel-diff 0.01 --max-abs-diff 0.0
+
+      - name: Compare head_off vs head_on
+        run: |
+          python head/tests/compare_benchmarks.py \
+            --baseline head_off.json \
+            --current head_on.json \
+            --max-rel-diff 0.01 \
+            --max-abs-diff 0.0 \
+            --case-metric lpips_inpaint_ctx \
+            --max-case-mean-delta "${{ inputs.max_case_mean_delta }}" \
+            --max-case-p95-delta "${{ inputs.max_case_p95_delta }}" \
+            --max-case-max-delta "${{ inputs.max_case_max_delta }}"
+
+      - name: Compare baseline vs head_on
+        continue-on-error: true
+        run: |
+          python head/tests/compare_benchmarks.py --baseline baseline.json --current head_on.json --max-rel-diff 0.01 --max-abs-diff 0.0
+
+      - name: Render worst-case gallery (head_off vs head_on)
+        if: ${{ always() }}
+        run: |
+          python head/tests/render_benchmark_gallery.py \
+            --baseline-json head_off.json \
+            --current-json head_on.json \
+            --baseline-images head_off_images \
+            --current-images head_on_images \
+            --out-dir gallery \
+            --metric lpips_inpaint_ctx \
+            --top-k "${{ inputs.gallery_topk }}"
+
+      - name: Upload benchmark JSON
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: lpips-benchmark-artifacts
+          path: |
+            baseline.json
+            head_off.json
+            head_on.json
+            head_off_images
+            head_on_images
+            gallery
+          if-no-files-found: warn