From fa5cb78cf6e08eb582b3b29c0871a1ce0da74d40 Mon Sep 17 00:00:00 2001 From: imbajin Date: Tue, 13 Oct 2026 02:35:44 +0800 Subject: [PATCH 01/14] fix(ci): submit trusted Maven dependency baselines - submit canonical master snapshots in a separate job - limit write permission to trusted push and dispatch - pin privileged actions and verify the event revision - document repository scope and baseline acceptance --- .github/workflows/submit-dependencies.yml | 95 +++++++++++++++++++++++ docs/ci.md | 24 ++++++ 2 files changed, 119 insertions(+) create mode 100644 .github/workflows/submit-dependencies.yml diff --git a/.github/workflows/submit-dependencies.yml b/.github/workflows/submit-dependencies.yml new file mode 100644 index 0000000000..18d534a484 --- /dev/null +++ b/.github/workflows/submit-dependencies.yml @@ -0,0 +1,95 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: Dependency baseline + +on: + push: + branches: [master] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: dependency-baseline-${{ github.repository }}-master + cancel-in-progress: false + +jobs: + validate: + if: >- + (github.repository == 'apache/hugegraph' || github.repository == 'hugegraph/hugegraph') && + github.ref == 'refs/heads/master' && + (github.event_name == 'push' || github.event_name == 'workflow_dispatch') + runs-on: ubuntu-latest + timeout-minutes: 10 + env: + PYTHONDONTWRITEBYTECODE: '1' + steps: + - name: Checkout event commit + uses: actions/checkout@v7 + with: + ref: ${{ github.sha }} + persist-credentials: false + - name: Test CI policy and dependency inventory + run: | + python3 -m unittest discover -s .github/scripts -p 'test_ci_policy.py' + python3 -m unittest discover -s install-dist/scripts/dependency -p 'test_*.py' + + submit: + needs: validate + if: >- + github.repository == 'apache/hugegraph' && + github.event.repository.fork == false && + github.ref == 'refs/heads/master' && + (github.event_name == 'push' || github.event_name == 'workflow_dispatch') + runs-on: ubuntu-latest + timeout-minutes: 60 + permissions: + contents: write + steps: + - name: Checkout event commit + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + persist-credentials: false + - name: Set up project JDK + uses: actions/setup-java@de7274f081f381c8f8158605e0321c36c376e2e6 # v6.0.1 + with: + java-version-file: .github/workflows/.java-version + distribution: 'temurin' + - name: Verify checkout and current master + env: + EXPECTED_SHA: ${{ github.sha }} + run: | + test "$(git rev-parse HEAD)" = "$EXPECTED_SHA" + test "$(git ls-remote origin refs/heads/master | cut -f1)" = "$EXPECTED_SHA" + - name: Install the complete Maven reactor + run: mvn -B -ntp install -Dmaven.test.skip=true -Dmaven.javadoc.skip=true --fail-at-end + - name: Refuse a stale master submission + env: + EXPECTED_SHA: ${{ github.sha }} + run: test "$(git ls-remote origin refs/heads/master | cut -f1)" = "$EXPECTED_SHA" + - name: Submit resolved reactor dependencies + uses: advanced-security/maven-dependency-submission-action@a64327a7329c9939cf675e458452febe1894a70c # v6.0.1 + with: + directory: '.' + ignore-maven-wrapper: true + maven-args: '-ntp -Dmaven.test.skip=true -Dmaven.javadoc.skip=true' + correlator: '.github/workflows/submit-dependencies.yml:submit' + snapshot-sha: ${{ github.sha }} + snapshot-ref: ${{ github.ref }} diff --git a/docs/ci.md b/docs/ci.md index a6dd9fc1e5..3505c53fef 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -105,3 +105,27 @@ Each separate job or matrix item requests a runner; steps within a job share one but [plan-dependent total and macOS concurrency limits](https://docs.github.com/en/actions/reference/limits#job-concurrency-limits-for-github-hosted-runners) still apply. Reducing job allocations and long executions lowers shared-pool demand; it does not guarantee a specific queue delay. + +## Dependency baseline + +The separate [dependency submission workflow](../.github/workflows/submit-dependencies.yml) +installs the complete reactor with Java 17, then uses the pinned official +[Maven dependency submission action](https://github.com/advanced-security/maven-dependency-submission-action/tree/a64327a7329c9939cf675e458452febe1894a70c) +to submit each module's resolved dependencies. It runs on master pushes or manual +dispatches. Only `apache/hugegraph` on `refs/heads/master` can run the submission +job, which alone has `contents: write`; the org repository runs read-only policy +and inventory validation. PRs, forks and other refs cannot run the writer. + +Checkout and snapshot use the event commit SHA. Master is checked before install +and again before invoking the submission action; a run already stale at either +check fails. Snapshots retain the checked event SHA even if master advances during +graph generation. A fixed correlator keeps successive master snapshots in the same stream. +The [snapshot API](https://docs.github.com/en/rest/dependency-graph/dependency-submission#create-a-snapshot-of-dependencies-for-a-repository) +requires Contents write permission; no additional metadata permission is needed. + +This producer supplements the existing dependency review and shipped-inventory +checks without changing their severity, license rules or inventory. Merging the +workflow does not prove that a canonical baseline exists: after the first ASF +master run, verify the submitted SHA/ref, non-empty module manifests and a real +dependency-review comparison against that baseline. Until then, baseline acceptance +remains open. No ASF repository setting is changed by this workflow. From eed65ac5c9a4a23d8925ff192c59dd39a9231a2d Mon Sep 17 00:00:00 2001 From: imbajin Date: Tue, 13 Oct 2026 04:45:12 +0800 Subject: [PATCH 02/14] fix(ci): preserve both master dependency baselines - allow the designated org master producer - keep fork and pull request events read-only - document isolated baseline acceptance checks --- .github/workflows/submit-dependencies.yml | 4 +- docs/ci.md | 54 +++++++++++++++++------ 2 files changed, 42 insertions(+), 16 deletions(-) diff --git a/.github/workflows/submit-dependencies.yml b/.github/workflows/submit-dependencies.yml index 18d534a484..45fc0557fa 100644 --- a/.github/workflows/submit-dependencies.yml +++ b/.github/workflows/submit-dependencies.yml @@ -53,8 +53,8 @@ jobs: submit: needs: validate if: >- - github.repository == 'apache/hugegraph' && - github.event.repository.fork == false && + ((github.repository == 'apache/hugegraph' && github.event.repository.fork == false) || + github.repository == 'hugegraph/hugegraph') && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') runs-on: ubuntu-latest diff --git a/docs/ci.md b/docs/ci.md index 3505c53fef..6da30987bf 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -109,23 +109,49 @@ it does not guarantee a specific queue delay. ## Dependency baseline The separate [dependency submission workflow](../.github/workflows/submit-dependencies.yml) -installs the complete reactor with Java 17, then uses the pinned official -[Maven dependency submission action](https://github.com/advanced-security/maven-dependency-submission-action/tree/a64327a7329c9939cf675e458452febe1894a70c) -to submit each module's resolved dependencies. It runs on master pushes or manual -dispatches. Only `apache/hugegraph` on `refs/heads/master` can run the submission -job, which alone has `contents: write`; the org repository runs read-only policy -and inventory validation. PRs, forks and other refs cannot run the writer. +installs the complete reactor with Java 17 in a `contents: read` job, then generates +each module's resolved graph with Ferstl 4.0.3. The repository's +[stdlib serializer](../.github/scripts/dependency-snapshot.py) preserves the +[official Maven parser 6.0.1](https://github.com/actions/maven-dependency-submission-action/tree/1f666f7ac2ab7b72cd5e22eaf7f23c8a581db1db) +package URLs, scopes and relationships. Every tracked POM must have a graph; empty +aggregator manifests are allowed, but a completely empty inventory fails. It runs on master pushes or manual +dispatches. Only `apache/hugegraph` (not a fork) and the explicitly allowed +`hugegraph/hugegraph` test fork on `refs/heads/master` can run the submission job, +which alone has `contents: write`. PRs, other repositories and other refs cannot +run the writer. Maven inherits no submission token, and checkout does not persist +credentials. A pinned `actions/upload-artifact` transfers only `snapshot.json`. +The fresh writer runner uses fixed inline stdlib Python to verify artifact/run +identity before downloading. It bounds ZIP and JSON sizes, checks the archive digest, +and reads only one regular `snapshot.json` member in memory. Duplicate members, +paths and symlinks fail; no archive members are extracted to disk. A signed storage +redirect is fetched without forwarding GitHub Authorization, and further redirects +are refused. The writer does not check out the repository, run Maven, or execute +downloaded code. The first-party upload action is permitted by the +[ASF Actions policy](https://infra.apache.org/github-actions-policy.html). Checkout and snapshot use the event commit SHA. Master is checked before install -and again before invoking the submission action; a run already stale at either -check fails. Snapshots retain the checked event SHA even if master advances during -graph generation. A fixed correlator keeps successive master snapshots in the same stream. +and again after graph generation and serialization; a stale run fails. The writer +checks the run attempt, successful producer, artifact ID/name, JSON schema, +detector/correlator and source SHA/ref. It verifies all manifest paths against the +event commit's source tree, rejects empty or mismatched inventory, and checks master +immediately before and after submission. Snapshots retain the checked event SHA +if master advances during submission; that run fails its final freshness check. + +If submission fails, use **Re-run all jobs** or start a fresh `workflow_dispatch` +on master. **Re-run failed jobs** keeps the older generation artifact, which the +writer rejects because the snapshot must be produced in the same run attempt. + +A fixed master-only correlator keeps successive baseline snapshots in the +same stream within each repository; ASF and org inventories are independent. The writer's +identity is separate from automatic PR submissions so they cannot replace its baseline. The [snapshot API](https://docs.github.com/en/rest/dependency-graph/dependency-submission#create-a-snapshot-of-dependencies-for-a-repository) -requires Contents write permission; no additional metadata permission is needed. +requires Contents write permission. The writer also has Actions read permission +to verify run and artifact metadata. Snapshot metadata identifies the repository +adapter rather than claiming that the combined generation/submission action ran. This producer supplements the existing dependency review and shipped-inventory checks without changing their severity, license rules or inventory. Merging the -workflow does not prove that a canonical baseline exists: after the first ASF -master run, verify the submitted SHA/ref, non-empty module manifests and a real -dependency-review comparison against that baseline. Until then, baseline acceptance -remains open. No ASF repository setting is changed by this workflow. +workflow does not prove that either repository has a baseline: verify each repository's +first master submission, including its SHA/ref, non-empty module manifests and a real +dependency-review comparison against that baseline. Until then, that repository's baseline +acceptance remains open. This workflow changes no repository settings. From 4d6c07df2c46c4eeae37c704c6ee59d676215c32 Mon Sep 17 00:00:00 2001 From: imbajin Date: Tue, 13 Oct 2026 09:26:19 +0800 Subject: [PATCH 03/14] fix(ci): use ASF-approved submission action - use the pinned GitHub-maintained upstream fork - preserve master-only writes and freshness guards - document the permitted publisher and source reference --- .github/workflows/submit-dependencies.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/submit-dependencies.yml b/.github/workflows/submit-dependencies.yml index 45fc0557fa..de40a11a27 100644 --- a/.github/workflows/submit-dependencies.yml +++ b/.github/workflows/submit-dependencies.yml @@ -85,7 +85,7 @@ jobs: EXPECTED_SHA: ${{ github.sha }} run: test "$(git ls-remote origin refs/heads/master | cut -f1)" = "$EXPECTED_SHA" - name: Submit resolved reactor dependencies - uses: advanced-security/maven-dependency-submission-action@a64327a7329c9939cf675e458452febe1894a70c # v6.0.1 + uses: actions/maven-dependency-submission-action@1f666f7ac2ab7b72cd5e22eaf7f23c8a581db1db # GitHub-maintained fork with: directory: '.' ignore-maven-wrapper: true From d9ca13cab2e7fd10b61a0304d41767149dc49d2d Mon Sep 17 00:00:00 2001 From: imbajin Date: Tue, 13 Oct 2026 12:26:09 +0800 Subject: [PATCH 04/14] fix(ci): isolate dependency snapshot token - build and serialize Maven graphs in a read-only job - validate a bounded JSON ZIP in a fresh data-only writer - preserve dependency identity and master freshness - document boundaries and add archive redirect regressions --- .github/scripts/dependency-snapshot.py | 154 ++++++++++ .github/scripts/test_dependency_snapshot.py | 302 ++++++++++++++++++++ .github/workflows/submit-dependencies.yml | 242 +++++++++++++++- 3 files changed, 687 insertions(+), 11 deletions(-) create mode 100644 .github/scripts/dependency-snapshot.py create mode 100644 .github/scripts/test_dependency_snapshot.py diff --git a/.github/scripts/dependency-snapshot.py b/.github/scripts/dependency-snapshot.py new file mode 100644 index 0000000000..ed71cef60a --- /dev/null +++ b/.github/scripts/dependency-snapshot.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Serialize native Ferstl 4.0.3 graphs; no token or submission code. + +Matches maven-dependency-submission-action 6.0.1's Maven parser/toolkit: +first classifier/type, test scope, shared PURL packages, direct precedence, +child edge order and cycle handling. See docs/ci.md for the producer boundary. +""" + +import datetime +import json +import os +from pathlib import Path +import subprocess +import sys +from urllib.parse import quote + +REPORT = 'maven-dependency-submission-action-depgraph.json' +DETECTOR = {'name': 'maven-dependency-submission-action', 'version': '6.0.1', + 'url': 'https://github.com/advanced-security/maven-dependency-submission-action'} +CORRELATOR = '.github/workflows/submit-dependencies.yml:submit' +SERIALIZER = 'Ferstl 4.0.3; repository stdlib adapter for Maven parser 6.0.1' + + +def unique_object(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ValueError(f'Duplicate JSON key: {key}') + result[key] = value + return result + + +def package_url(artifact): + # packageurl-js uses encodeURIComponent for components and URLSearchParams + # for qualifiers; notably '+' in versions and '/' in namespaces survive. + namespace = '/'.join(part for part in artifact['groupId'].split('/') if part) + name, version = artifact['artifactId'].strip(), artifact['version'].strip() + if not namespace or not name or not version: + raise ValueError('Incomplete Maven coordinates') + encoded_namespace = quote(namespace, safe="!~*'():/") + encoded_name = quote(name, safe="!~*'()") + encoded_version = quote(version, safe="!~*'():+") + purl = f'pkg:maven/{encoded_namespace}/{encoded_name}@{encoded_version}' + qualifiers = {} + for key, field in [('classifier', 'classifiers'), ('type', 'types')]: + values = artifact.get(field) or [] + if values and values[0].strip(): + qualifiers[key] = values[0].strip() + if qualifiers: + purl += '?' + '&'.join(f'{key}={quote(value, safe="*").replace("~", "%7E")}' + for key, value in sorted(qualifiers.items())) + return purl + + +def manifest(graph, pom): + if not isinstance(graph, dict) or not isinstance(graph.get('graphName'), str) or not graph['graphName'].strip(): + raise ValueError(f'Malformed graph name: {pom}') + artifacts, edges = graph.get('artifacts'), graph.get('dependencies') or [] + if not isinstance(artifacts, list) or not artifacts or not isinstance(edges, list): + raise ValueError(f'Empty or malformed graph: {pom}') + if graph.get('dependencies') is not None and not isinstance(graph['dependencies'], list): + raise ValueError(f'Malformed dependency list: {pom}') + ids, packages, scopes = {}, {}, {} + for artifact in artifacts: + if not isinstance(artifact, dict) or any(not isinstance(artifact.get(key), str) or not artifact[key].strip() + for key in ['id', 'groupId', 'artifactId', 'version']): + raise ValueError(f'Malformed artifact: {pom}') + for key in ['scopes', 'types', 'classifiers']: + values = artifact.get(key) + if values is not None and (not isinstance(values, list) or any(not isinstance(x, str) for x in values)): + raise ValueError(f'Malformed artifact {key}: {pom}') + identifier, purl = artifact['id'], package_url(artifact) + if identifier in ids: + raise ValueError(f'Duplicate artifact ID: {identifier}') + ids[identifier] = purl + packages.setdefault(purl, []) + scopes[purl] = 'development' if 'test' in (artifact.get('scopes') or []) else 'runtime' + children, incoming = {}, set() + for edge in edges: + if not isinstance(edge, dict) or any(not isinstance(edge.get(key), str) for key in ['from', 'to']): + raise ValueError(f'Malformed dependency edge: {pom}') + source, target = edge['from'], edge['to'] + if source not in ids or target not in ids: + raise ValueError(f'Unknown dependency edge: {pom}') + children.setdefault(source, []).append(target) + incoming.add(target) + # Aliased artifact IDs share one package and append all their child edges. + for source, targets in children.items(): + packages[ids[source]].extend(ids[target] for target in targets) + direct = dict.fromkeys(target for source in ids if source not in incoming + for target in children.get(source, [])) + resolved = {} + + def entry(purl, relationship): + return {'package_url': purl, 'relationship': relationship, + 'scope': scopes[purl], 'dependencies': packages[purl]} + + for identifier in direct: + purl = ids[identifier] + resolved[purl] = entry(purl, 'direct') + seen, pending = set(), list(reversed(packages[purl])) + while pending: + child = pending.pop() + if child in seen: + continue + seen.add(child) + resolved.setdefault(child, entry(child, 'indirect')) + pending.extend(reversed(packages[child])) + return {'name': graph['graphName'], 'file': {'source_location': pom}, 'resolved': resolved} + + +def create_snapshot(workspace, poms, sha, repository, run_id, run_attempt): + manifests = {} + for pom in poms: + report = workspace / Path(pom).parent / 'target' / REPORT + if report.is_symlink() or report.stat().st_size > 25 * 1024 * 1024: + raise ValueError(f'Invalid report: {pom}') + item = manifest(json.loads(report.read_text(), object_pairs_hook=unique_object), pom) + if item['name'] in manifests: + raise ValueError(f'Duplicate graph name: {item["name"]}') + manifests[item['name']] = item + if not manifests or not any(item['resolved'] for item in manifests.values()): + raise ValueError('Refusing empty dependency inventory') + return {'version': 0, 'sha': sha, 'ref': 'refs/heads/master', + 'job': {'id': run_id, 'correlator': CORRELATOR, + 'html_url': f'https://github.com/{repository}/actions/runs/{run_id}'}, + 'detector': DETECTOR, 'metadata': {'serializer': SERIALIZER, 'producer': 'generate', + 'run_attempt': run_attempt}, + 'scanned': datetime.datetime.now(datetime.timezone.utc).isoformat().replace('+00:00', 'Z'), + 'manifests': manifests} + + +if __name__ == '__main__': + poms = subprocess.check_output(['git', 'ls-files', '-z', '*pom.xml']).decode().split('\0') + poms = [pom for pom in poms if Path(pom).name == 'pom.xml'] + snapshot = create_snapshot(Path.cwd(), poms, os.environ['GITHUB_SHA'], + os.environ['GITHUB_REPOSITORY'], os.environ['GITHUB_RUN_ID'], + os.environ['GITHUB_RUN_ATTEMPT']) + Path(sys.argv[1]).write_text(json.dumps(snapshot, indent=2) + '\n') diff --git a/.github/scripts/test_dependency_snapshot.py b/.github/scripts/test_dependency_snapshot.py new file mode 100644 index 0000000000..18cb868c59 --- /dev/null +++ b/.github/scripts/test_dependency_snapshot.py @@ -0,0 +1,302 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Native-parser regression fixtures and local-only writer protocol controls.""" + +import copy +from contextlib import redirect_stdout +import datetime +import hashlib +import importlib.util +import io +import json +import os +from pathlib import Path +import tempfile +import textwrap +import unittest +import urllib.error +import warnings +import zipfile +from unittest.mock import patch +from urllib.parse import urlsplit + + +def load_module(path, name): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +producer = load_module(Path(__file__).with_name('dependency-snapshot.py'), 'producer') + + +def artifact(identifier, **fields): + return dict(id=identifier, groupId='org.example', artifactId=identifier, version='1.0', types=['jar'], **fields) + + +class SnapshotTest(unittest.TestCase): + + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + workflow = Path(__file__).resolve().parents[1] / 'workflows/submit-dependencies.yml' + # Only the fixed workflow program is loaded as code. Artifacts remain JSON data. + program = workflow.read_text().split(" python3 -I - <<'PY'\n", 1)[1].split('\n PY', 1)[0] + script = self.root / 'fixed-writer.py' + script.write_text(textwrap.dedent(program)) + self.writer = load_module(script, 'writer') + self.env = {'EXPECTED_REPOSITORY': 'apache/hugegraph', 'EXPECTED_SHA': 'a' * 40, + 'RUN_ID': '321', 'RUN_ATTEMPT': '2', 'ARTIFACT_ID': '789', 'RUNNER_TEMP': str(self.root), + 'SNAPSHOT_TOKEN': 'public-synthetic-sentinel'} + directory = self.root / 'dependency-snapshot' + directory.mkdir() + self.source = directory / 'snapshot.json' + graph = {'graphName': 'module', 'artifacts': [artifact('root'), artifact('one'), artifact('two')], + 'dependencies': [{'from': 'root', 'to': 'one'}, {'from': 'one', 'to': 'two'}]} + now = datetime.datetime.now(datetime.timezone.utc) + self.snapshot = {'version': 0, 'sha': self.env['EXPECTED_SHA'], 'ref': 'refs/heads/master', + 'job': {'id': '321', 'correlator': producer.CORRELATOR, + 'html_url': 'https://github.com/apache/hugegraph/actions/runs/321'}, + 'detector': producer.DETECTOR, 'metadata': {'serializer': producer.SERIALIZER, + 'producer': 'generate', 'run_attempt': '2'}, + 'scanned': now.isoformat(), 'manifests': {'module': producer.manifest(graph, 'pom.xml')}} + self.responses = { + 'actions/runs/321/attempts/2': {'id': 321, 'run_attempt': 2, 'head_sha': 'a' * 40, + 'head_branch': 'master', 'event': 'push', 'repository': {'full_name': 'apache/hugegraph'}, + 'path': '.github/workflows/submit-dependencies.yml', + 'run_started_at': (now - datetime.timedelta(minutes=2)).isoformat()}, + 'actions/runs/321/attempts/2/jobs?per_page=100': {'total_count': 3, + 'jobs': [{'name': 'generate', 'conclusion': 'success', 'status': 'completed', + 'run_id': 321, 'head_sha': 'a' * 40}]}, + 'actions/artifacts/789': {'id': 789, 'expired': False, 'name': 'dependency-snapshot-321-2', + 'created_at': now.isoformat(), + 'workflow_run': {'id': 321, 'head_sha': 'a' * 40, 'head_branch': 'master'}}, + 'git/trees/' + 'a' * 40 + '?recursive=1': {'truncated': False, + 'tree': [{'path': 'pom.xml', 'type': 'blob', 'mode': '100644'}]}, + 'git/ref/heads/master': {'object': {'sha': 'a' * 40}}, + 'dependency-graph/snapshots': {'result': 'SUCCESS'}, + } + self.posts = [] + self.blob_headers = [] + + def run_writer(self, snapshot=None, raw=None, archive=None, artifact_changes=None): + if archive is None: + payload = raw if raw is not None else json.dumps(snapshot or self.snapshot) + archive = self.zip_bytes([('snapshot.json', payload)]) + self.responses['actions/artifacts/789'].update(size_in_bytes=len(archive), + digest='sha256:' + hashlib.sha256(archive).hexdigest()) + self.responses['actions/artifacts/789'].update(artifact_changes or {}) + + def open_request(request, timeout): + url = urlsplit(request.full_url) + self.assertEqual('https', url.scheme) + if url.netloc == 'storage.blob.core.windows.net': + self.blob_headers.append(dict(request.header_items())) + self.assertIsNone(request.get_header('Authorization')) + response = io.BytesIO(archive) + response.status = 200 + return response + self.assertEqual('api.github.com', url.netloc) + self.assertTrue(url.path.startswith('/repos/apache/hugegraph/')) + self.assertEqual('Bearer public-synthetic-sentinel', request.get_header('Authorization')) + path = url.path.removeprefix('/repos/apache/hugegraph/') + ('?' + url.query if url.query else '') + if path == 'actions/artifacts/789/zip': + raise urllib.error.HTTPError(request.full_url, 302, 'Found', + {'Location': 'https://storage.blob.core.windows.net/archive.zip?sig=public'}, io.BytesIO()) + if request.data is not None: + self.assertEqual('dependency-graph/snapshots', path) + self.posts.append(json.loads(request.data)) + response = io.BytesIO(json.dumps(self.responses[path]).encode()) + response.status = 201 if request.data is not None else 200 + return response + + opener = self.writer.urllib.request.OpenerDirector() + opener.open = open_request + with patch.dict(os.environ, self.env), patch.object(self.writer.urllib.request, + 'build_opener', return_value=opener): + with redirect_stdout(io.StringIO()): + self.writer.main() + + @staticmethod + def zip_bytes(entries, mode=None): + output = io.BytesIO() + with warnings.catch_warnings(): + warnings.simplefilter('ignore', UserWarning) + with zipfile.ZipFile(output, 'w', compression=zipfile.ZIP_DEFLATED) as archive: + for name, data in entries: + entry = zipfile.ZipInfo(name) + entry.compress_type = zipfile.ZIP_DEFLATED + if mode is not None: + entry.create_system = 3 + entry.external_attr = mode << 16 + archive.writestr(entry, data) + return output.getvalue() + + def test_native_package_url_encoding(self): + # Expected strings were captured from the pinned action's actual parser. + cases = [ + (dict(groupId='org.图//组/', artifactId='测试/名@', version='1+2/slash~', + classifiers=['测试 c/+~*'], types=['jar']), + 'pkg:maven/org.%E5%9B%BE/%E7%BB%84/%E6%B5%8B%E8%AF%95%2F%E5%90%8D%40@1+2%2Fslash~' + '?classifier=%E6%B5%8B%E8%AF%95%20c%2F%2B%7E*&type=jar'), + (dict(groupId="g:!/x~*'()", artifactId="n:!~*'()", version="v:!~*'()+", + classifiers=[' c/+~* '], types=[' jar ', 'war']), + "pkg:maven/g:!/x~*'()/n%3A!~*'()@v:!~*'()+?classifier=c%2F%2B%7E*&type=jar"), + ] + for value, expected in cases: + self.assertEqual(expected, producer.package_url(value)) + + def test_native_cycle_alias_scope_and_direct_precedence(self): + graph = {'graphName': 'module', 'artifacts': [artifact('root'), artifact('one', scopes=['test']), + dict(artifact('one'), id='alias', scopes=['provided']), artifact('two'), artifact('three')], + 'dependencies': [{'from': 'root', 'to': 'one'}, {'from': 'one', 'to': 'two'}, + {'from': 'two', 'to': 'one'}, {'from': 'alias', 'to': 'three'}, + {'from': 'root', 'to': 'two'}]} + resolved = producer.manifest(graph, 'pom.xml')['resolved'] + one, two, three = [f'pkg:maven/org.example/{name}@1.0?type=jar' for name in ['one', 'two', 'three']] + self.assertEqual({'package_url': one, 'relationship': 'direct', 'scope': 'runtime', + 'dependencies': [two, three]}, resolved[one]) + self.assertEqual({'direct'}, {entry['relationship'] for entry in resolved.values()}) + self.assertEqual({}, producer.manifest({'graphName': 'parent', 'artifacts': [artifact('root')]}, + 'pom.xml')['resolved']) + + def test_generation_rejects_missing_empty_duplicate_and_malformed_graphs(self): + with self.assertRaises(FileNotFoundError): + producer.create_snapshot(self.root, ['pom.xml'], 'a' * 40, 'apache/hugegraph', '321', '2') + report = self.root / 'target' / producer.REPORT + report.parent.mkdir() + report.write_text(json.dumps({'graphName': 'parent', 'artifacts': [artifact('root')]})) + with self.assertRaisesRegex(ValueError, 'empty dependency inventory'): + producer.create_snapshot(self.root, ['pom.xml'], 'a' * 40, 'apache/hugegraph', '321', '2') + for graph in [{'graphName': 'bad', 'artifacts': [artifact('root'), artifact('root')]}, + {'graphName': 'bad', 'artifacts': [artifact('root')], 'dependencies': False}, + {'graphName': 'bad', 'artifacts': [artifact('root', scopes='test')]}]: + with self.assertRaises(ValueError): + producer.manifest(graph, 'pom.xml') + with self.assertRaises(ValueError): + json.loads('{"sha":"a","sha":"b"}', object_pairs_hook=producer.unique_object) + + def test_writer_preserves_snapshot_and_never_executes_data(self): + marker = self.root / 'must-not-exist' + name = f"__import__('pathlib').Path({str(marker)!r}).touch()" + self.snapshot['manifests'][name] = self.snapshot['manifests'].pop('module') + self.snapshot['manifests'][name]['name'] = name + self.run_writer() + self.assertEqual([self.snapshot], self.posts) + self.assertFalse(marker.exists()) + + def test_writer_accepts_master_path_suffix_and_rejects_other_refs(self): + run = self.responses['actions/runs/321/attempts/2'] + path = '.github/workflows/submit-dependencies.yml' + run['path'] = path + '@master' + self.run_writer() + self.assertEqual([self.snapshot], self.posts) + self.posts.clear() + for ref in ['main', 'feature', 'refs/heads/master', '']: + run['path'] = path + '@' + ref + with self.subTest(ref=ref), self.assertRaisesRegex(ValueError, 'Run identity'): + self.run_writer() + self.assertEqual([], self.posts) + + def test_writer_rejects_snapshot_changes_before_post(self): + mutations = [('sha', 'b' * 40), ('ref', 'refs/heads/feature'), ('version', False), + ('job', {}), ('detector', {}), ('metadata', {}), ('manifests', {}), + ('scanned', '2000-01-01T00:00:00Z'), ('scanned', '2999-01-01T00:00:00Z')] + for key, value in mutations: + with self.subTest(key=key, value=value), self.assertRaises((ValueError, KeyError)): + snapshot = copy.deepcopy(self.snapshot) + snapshot[key] = value + self.run_writer(snapshot) + for field, value in [('source_location', 'other/pom.xml'), ('resolved', {}), ('scope', 'optional'), + ('dependencies', ['pkg:maven/missing/name@1']), + ('package_url', 'pkg:maven/invalid%Q0/a@1')]: + snapshot = copy.deepcopy(self.snapshot) + manifest = snapshot['manifests']['module'] + target = manifest['file'] if field == 'source_location' else manifest if field == 'resolved' else next( + iter(manifest['resolved'].values())) + target[field] = value + with self.subTest(field=field), self.assertRaises(ValueError): + self.run_writer(snapshot) + with self.assertRaises(ValueError): + self.run_writer(raw=json.dumps(self.snapshot)[:-1] + ',"sha":"b"}') + self.assertEqual([], self.posts) + + def test_writer_rejects_failed_producer_wrong_artifact_and_stale_master(self): + for path, field, value in [('actions/runs/321/attempts/2', 'run_attempt', 1), + ('actions/artifacts/789', 'name', 'dependency-snapshot-321-1'), + ('actions/artifacts/789', 'expired', True), + ('actions/artifacts/789', 'workflow_run', {'id': 123}), + ('actions/runs/321/attempts/2/jobs?per_page=100', 'jobs', + [{'name': 'generate', 'conclusion': 'failure'}]), + ('git/trees/' + 'a' * 40 + '?recursive=1', 'truncated', True), + ('git/ref/heads/master', 'object', {'sha': 'b' * 40})]: + original = copy.deepcopy(self.responses[path]) + self.responses[path][field] = value + with self.subTest(path=path, field=field), self.assertRaises((ValueError, KeyError)): + self.run_writer() + self.responses[path] = original + self.assertEqual([], self.posts) + + def test_writer_rejects_zip_paths_duplicates_symlinks_and_bounds_without_writes(self): + body = json.dumps(self.snapshot) + fixtures = [self.zip_bytes([('snapshot.json', body), ('x/../../outside', 'must not write')]), + self.zip_bytes([('snapshot.json', body), ('snapshot.json', body)]), + self.zip_bytes([('snapshot.json', body)], 0o120777), + self.zip_bytes([('snapshot.json', '0' * (self.writer.LIMIT + 1))]), + b'0' * (self.writer.LIMIT + 1)] + fixtures += [self.zip_bytes([(path, body)]) for path in + ['../snapshot.json', 'x/../../snapshot.json', '/snapshot.json', 'x\\snapshot.json']] + before = set(self.root.rglob('*')) + for archive in fixtures: + with self.subTest(size=len(archive)), self.assertRaises(ValueError): + self.run_writer(archive=archive) + self.assertEqual(before, set(self.root.rglob('*'))) + self.assertFalse((self.root / 'outside').exists()) + self.assertEqual([], self.posts) + + def test_writer_download_has_no_cross_host_authorization_or_followup_redirect(self): + self.run_writer() + self.assertEqual([{}], self.blob_headers) + self.posts.clear() + for metadata in [{'digest': 'sha256:' + '0' * 64}, {'size_in_bytes': 1}]: + with self.assertRaisesRegex(ValueError, 'digest/size mismatch'): + self.run_writer(artifact_changes=metadata) + calls = [] + + def redirected(request, timeout): + calls.append(request) + if len(calls) == 1: + raise urllib.error.HTTPError(request.full_url, 302, 'Found', + {'Location': 'https://storage.blob.core.windows.net/archive.zip'}, io.BytesIO()) + self.assertIsNone(request.get_header('Authorization')) + raise urllib.error.HTTPError(request.full_url, 302, 'Found', + {'Location': 'https://other.example'}, io.BytesIO()) + + opener = self.writer.urllib.request.OpenerDirector() + opener.open = redirected + with patch.dict(os.environ, self.env), patch.object(self.writer.urllib.request, + 'build_opener', return_value=opener): + with self.assertRaises(urllib.error.HTTPError): + self.writer.download_snapshot('apache/hugegraph', '789', {}) + self.assertEqual(2, len(calls)) + self.assertEqual([], self.posts) + + +if __name__ == '__main__': + unittest.main() diff --git a/.github/workflows/submit-dependencies.yml b/.github/workflows/submit-dependencies.yml index de40a11a27..9b4a9edc46 100644 --- a/.github/workflows/submit-dependencies.yml +++ b/.github/workflows/submit-dependencies.yml @@ -48,9 +48,10 @@ jobs: - name: Test CI policy and dependency inventory run: | python3 -m unittest discover -s .github/scripts -p 'test_ci_policy.py' + python3 -m unittest discover -s .github/scripts -p 'test_dependency_snapshot.py' python3 -m unittest discover -s install-dist/scripts/dependency -p 'test_*.py' - submit: + generate: needs: validate if: >- ((github.repository == 'apache/hugegraph' && github.event.repository.fork == false) || @@ -60,7 +61,14 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 60 permissions: - contents: write + contents: read + env: + PYTHONDONTWRITEBYTECODE: '1' + INPUT_TOKEN: '' + GITHUB_TOKEN: '' + GH_TOKEN: '' + outputs: + artifact_id: ${{ steps.snapshot.outputs.artifact-id }} steps: - name: Checkout event commit uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -80,16 +88,228 @@ jobs: test "$(git ls-remote origin refs/heads/master | cut -f1)" = "$EXPECTED_SHA" - name: Install the complete Maven reactor run: mvn -B -ntp install -Dmaven.test.skip=true -Dmaven.javadoc.skip=true --fail-at-end - - name: Refuse a stale master submission + - name: Generate resolved reactor snapshot + run: | + mvn -B -ntp com.github.ferstl:depgraph-maven-plugin:4.0.3:graph \ + -DgraphFormat=json -DoutputFileName=maven-dependency-submission-action-depgraph.json \ + -Dmaven.test.skip=true -Dmaven.javadoc.skip=true + python3 .github/scripts/dependency-snapshot.py "$RUNNER_TEMP/snapshot.json" + - name: Refuse a stale generated snapshot env: EXPECTED_SHA: ${{ github.sha }} run: test "$(git ls-remote origin refs/heads/master | cut -f1)" = "$EXPECTED_SHA" - - name: Submit resolved reactor dependencies - uses: actions/maven-dependency-submission-action@1f666f7ac2ab7b72cd5e22eaf7f23c8a581db1db # GitHub-maintained fork + - name: Upload snapshot data only + id: snapshot + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: - directory: '.' - ignore-maven-wrapper: true - maven-args: '-ntp -Dmaven.test.skip=true -Dmaven.javadoc.skip=true' - correlator: '.github/workflows/submit-dependencies.yml:submit' - snapshot-sha: ${{ github.sha }} - snapshot-ref: ${{ github.ref }} + name: dependency-snapshot-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/snapshot.json + if-no-files-found: error + retention-days: 1 + + submit: + needs: [validate, generate] + if: >- + needs.validate.result == 'success' && needs.generate.result == 'success' && + ((github.repository == 'apache/hugegraph' && github.event.repository.fork == false) || + github.repository == 'hugegraph/hugegraph') && + github.ref == 'refs/heads/master' && + (github.event_name == 'push' || github.event_name == 'workflow_dispatch') + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: write + actions: read + steps: + - name: Validate and submit snapshot data + env: + SNAPSHOT_TOKEN: ${{ github.token }} + EXPECTED_REPOSITORY: ${{ github.repository }} + EXPECTED_SHA: ${{ github.sha }} + RUN_ID: ${{ github.run_id }} + RUN_ATTEMPT: ${{ github.run_attempt }} + ARTIFACT_ID: ${{ needs.generate.outputs.artifact_id }} + run: | + python3 -I - <<'PY' + import datetime + import hashlib + import io + import json + import os + import re + import stat + import urllib.error + import urllib.parse + import urllib.request + import zipfile + + LIMIT = 25 * 1024 * 1024 + + def require(condition, message): + if not condition: + raise ValueError(message) + + def unique_object(pairs): + result = {} + for key, value in pairs: + require(key not in result, 'Duplicate JSON key') + result[key] = value + return result + + class NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, *args, **kwargs): + return None + + def github_request(repository, path, data=None): + return urllib.request.Request('https://api.github.com/repos/' + repository + '/' + path, + data=None if data is None else json.dumps(data).encode(), + headers={'Authorization': 'Bearer ' + os.environ['SNAPSHOT_TOKEN'], + 'Accept': 'application/vnd.github+json', 'Content-Type': 'application/json', + 'X-GitHub-Api-Version': '2022-11-28'}) + + def bounded_body(response): + raw = response.read(LIMIT + 1) + require(len(raw) <= LIMIT, 'Response too large') + return raw + + def api(repository, path, data=None): + request = github_request(repository, path, data) + with urllib.request.build_opener(NoRedirect()).open(request, timeout=60) as response: + require(response.status == (200 if data is None else 201), 'Unexpected API status') + raw = bounded_body(response) + return json.loads(raw, object_pairs_hook=unique_object) + + def read_snapshot(raw): + require(0 < len(raw) <= LIMIT, 'Invalid ZIP size') + with zipfile.ZipFile(io.BytesIO(raw)) as archive: + members = archive.infolist() + require(len(members) == 1, 'ZIP must contain one snapshot.json') + member = members[0] + require(member.orig_filename == 'snapshot.json' and not member.is_dir() and + not (member.external_attr & 0x10) and + stat.S_IFMT(member.external_attr >> 16) in [0, stat.S_IFREG] and + not (member.flag_bits & 1) and + member.compress_type in [zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED] and + 0 < member.file_size <= LIMIT, 'Invalid ZIP member') + with archive.open(member) as stream: + payload = bounded_body(stream) + require(len(payload) == member.file_size, 'Incomplete ZIP member') + return json.loads(payload, object_pairs_hook=unique_object) + + def download_snapshot(repository, artifact_id, artifact): + opener = urllib.request.build_opener(NoRedirect()) + try: + response = opener.open(github_request(repository, f'actions/artifacts/{artifact_id}/zip'), timeout=60) + except urllib.error.HTTPError as error: + require(error.code == 302, 'Expected artifact redirect') + location = error.headers.get('Location', '') + error.close() + else: + response.close() + raise ValueError('Expected artifact redirect') + target = urllib.parse.urlsplit(location) + require(target.scheme == 'https' and target.hostname and + target.hostname.endswith(('.blob.core.windows.net', '.githubusercontent.com')) and + target.username is None and target.password is None and + target.port in [None, 443] and not target.fragment, 'Invalid artifact redirect') + # A separate request carries no GitHub Authorization to blob storage. + with opener.open(urllib.request.Request(location), timeout=60) as response: + require(response.status == 200, 'Artifact download failed') + raw = bounded_body(response) + require(len(raw) == artifact['size_in_bytes'] and + 'sha256:' + hashlib.sha256(raw).hexdigest() == artifact['digest'], 'Artifact digest/size mismatch') + return read_snapshot(raw) + + def timestamp(value): + parsed = datetime.datetime.fromisoformat(value.replace('Z', '+00:00')) + require(parsed.tzinfo is not None, 'Timestamp lacks timezone') + return parsed + + def main(): + repository, sha = os.environ['EXPECTED_REPOSITORY'], os.environ['EXPECTED_SHA'] + run_id, attempt, artifact_id = (os.environ[key] for key in ['RUN_ID', 'RUN_ATTEMPT', 'ARTIFACT_ID']) + require(repository in ['apache/hugegraph', 'hugegraph/hugegraph'], 'Repository refused') + require(re.fullmatch('[0-9a-f]{40}', sha), 'Invalid commit') + require(all(re.fullmatch('[1-9][0-9]*', x) for x in [run_id, attempt, artifact_id]), 'Invalid ID') + run = api(repository, f'actions/runs/{run_id}/attempts/{attempt}') + require(run['id'] == int(run_id) and run['run_attempt'] == int(attempt) and + run['head_sha'] == sha and run['head_branch'] == 'master' and + run['event'] in ['push', 'workflow_dispatch'] and + run['repository']['full_name'] == repository and + run['path'] in ['.github/workflows/submit-dependencies.yml', + '.github/workflows/submit-dependencies.yml@master'], 'Run identity') + jobs = api(repository, f'actions/runs/{run_id}/attempts/{attempt}/jobs?per_page=100') + producers = [job for job in jobs['jobs'] if job['name'] == 'generate'] + require(jobs['total_count'] <= 100 and len(producers) == 1 and + producers[0]['conclusion'] == 'success' and producers[0]['status'] == 'completed' and + producers[0]['run_id'] == int(run_id) and producers[0]['head_sha'] == sha, + 'Generation did not succeed for this commit') + artifact = api(repository, f'actions/artifacts/{artifact_id}') + require(artifact['id'] == int(artifact_id) and artifact['expired'] is False and + artifact['name'] == f'dependency-snapshot-{run_id}-{attempt}' and + artifact['workflow_run']['id'] == int(run_id) and + artifact['workflow_run']['head_sha'] == sha and + artifact['workflow_run']['head_branch'] == 'master' and + type(artifact['size_in_bytes']) is int and 0 < artifact['size_in_bytes'] <= LIMIT and + isinstance(artifact['digest'], str) and + re.fullmatch('sha256:[0-9a-f]{64}', artifact['digest']), 'Artifact identity') + snapshot = download_snapshot(repository, artifact_id, artifact) + require(isinstance(snapshot, dict) and set(snapshot) == + {'version', 'sha', 'ref', 'job', 'detector', 'metadata', 'scanned', 'manifests'}, 'Snapshot schema') + require(type(snapshot['version']) is int and snapshot['version'] == 0 and + snapshot['sha'] == sha and snapshot['ref'] == 'refs/heads/master', 'Snapshot source') + require(snapshot['job'] == {'id': run_id, + 'correlator': '.github/workflows/submit-dependencies.yml:submit', + 'html_url': f'https://github.com/{repository}/actions/runs/{run_id}'}, 'Snapshot job') + require(snapshot['detector'] == {'name': 'maven-dependency-submission-action', 'version': '6.0.1', + 'url': 'https://github.com/advanced-security/maven-dependency-submission-action'}, 'Detector') + require(snapshot['metadata'] == {'serializer': + 'Ferstl 4.0.3; repository stdlib adapter for Maven parser 6.0.1', + 'producer': 'generate', 'run_attempt': attempt}, 'Producer identity') + now = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(minutes=1) + require(timestamp(run['run_started_at']) <= timestamp(snapshot['scanned']) <= + timestamp(artifact['created_at']) + datetime.timedelta(minutes=1) and + timestamp(artifact['created_at']) <= now and timestamp(snapshot['scanned']) <= now, + 'Snapshot freshness') + tree = api(repository, f'git/trees/{sha}?recursive=1') + require(tree['truncated'] is False, 'Incomplete source tree') + poms = {node['path'] for node in tree['tree'] if node['type'] == 'blob' and + node['path'].split('/')[-1] == 'pom.xml' and node['mode'] != '120000'} + manifests, paths = snapshot['manifests'], set() + require(isinstance(manifests, dict) and manifests and poms, 'Empty inventory') + component = r"(?:[A-Za-z0-9._!~*'():+-]|%[0-9A-F]{2})+" + qualifier = r'(?:[A-Za-z0-9._*\-]|%[0-9A-F]{2})+' + purl_pattern = (r'pkg:maven/' + component + '(?:/' + component + ')+@' + component + + r'(?:\?classifier=' + qualifier + '(?:&type=' + qualifier + r')?|\?type=' + qualifier + ')?') + for name, manifest in manifests.items(): + require(isinstance(name, str) and name.strip() and isinstance(manifest, dict) and + set(manifest) == {'name', 'file', 'resolved'} and manifest['name'] == name and + isinstance(manifest['file'], dict) and set(manifest['file']) == {'source_location'}, + 'Manifest schema') + location = manifest['file']['source_location'] + require(isinstance(location, str) and location in poms and location not in paths, 'Manifest source') + paths.add(location) + resolved = manifest['resolved'] + require(isinstance(resolved, dict), 'Resolved schema') + for purl, package in resolved.items(): + require(isinstance(purl, str) and len(purl) <= 4096 and + re.fullmatch(purl_pattern, purl), 'Package URL') + require(isinstance(package, dict) and set(package) == + {'package_url', 'relationship', 'scope', 'dependencies'} and + package['package_url'] == purl and package['relationship'] in ['direct', 'indirect'] and + package['scope'] in ['runtime', 'development'] and + isinstance(package['dependencies'], list) and + all(isinstance(child, str) and child in resolved for child in package['dependencies']), + 'Package schema') + require(not resolved or any(p['relationship'] == 'direct' for p in resolved.values()), + 'Missing direct dependencies') + require(paths == poms and any(m['resolved'] for m in manifests.values()), 'Incomplete inventory') + require(api(repository, 'git/ref/heads/master')['object']['sha'] == sha, 'Stale master before submission') + result = api(repository, 'dependency-graph/snapshots', snapshot) + require(result['result'] == 'SUCCESS', 'Snapshot not accepted') + require(api(repository, 'git/ref/heads/master')['object']['sha'] == sha, 'Master advanced during submission') + print('Dependency baseline submitted for ' + sha) + + if __name__ == '__main__': + main() + PY From d78ad6c8f509aa1f69beed3d117969e679c69c72 Mon Sep 17 00:00:00 2001 From: imbajin Date: Tue, 13 Oct 2026 23:32:54 +0800 Subject: [PATCH 05/14] fix(ci): clarify dependency rerun recovery - document all-job reruns or fresh master dispatch - test dependency paths containing only indirect edges - verify direct edges take precedence over indirect paths --- .github/scripts/test_dependency_snapshot.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/.github/scripts/test_dependency_snapshot.py b/.github/scripts/test_dependency_snapshot.py index 18cb868c59..9c8eae3440 100644 --- a/.github/scripts/test_dependency_snapshot.py +++ b/.github/scripts/test_dependency_snapshot.py @@ -176,6 +176,27 @@ def test_native_cycle_alias_scope_and_direct_precedence(self): self.assertEqual({}, producer.manifest({'graphName': 'parent', 'artifacts': [artifact('root')]}, 'pom.xml')['resolved']) + def test_transitive_only_package_is_indirect(self): + graph = {'graphName': 'module', 'artifacts': [artifact('root'), artifact('parent'), artifact('leaf')], + 'dependencies': [{'from': 'root', 'to': 'parent'}, {'from': 'parent', 'to': 'leaf'}]} + parent = 'pkg:maven/org.example/parent@1.0?type=jar' + leaf = 'pkg:maven/org.example/leaf@1.0?type=jar' + self.assertEqual({ + parent: {'package_url': parent, 'relationship': 'direct', 'scope': 'runtime', 'dependencies': [leaf]}, + leaf: {'package_url': leaf, 'relationship': 'indirect', 'scope': 'runtime', 'dependencies': []}, + }, producer.manifest(graph, 'pom.xml')['resolved']) + + def test_direct_relationship_overrides_indirect_path(self): + graph = {'graphName': 'module', 'artifacts': [artifact('root'), artifact('parent'), artifact('shared')], + 'dependencies': [{'from': 'root', 'to': 'parent'}, {'from': 'parent', 'to': 'shared'}, + {'from': 'root', 'to': 'shared'}]} + parent = 'pkg:maven/org.example/parent@1.0?type=jar' + shared = 'pkg:maven/org.example/shared@1.0?type=jar' + self.assertEqual({ + parent: {'package_url': parent, 'relationship': 'direct', 'scope': 'runtime', 'dependencies': [shared]}, + shared: {'package_url': shared, 'relationship': 'direct', 'scope': 'runtime', 'dependencies': []}, + }, producer.manifest(graph, 'pom.xml')['resolved']) + def test_generation_rejects_missing_empty_duplicate_and_malformed_graphs(self): with self.assertRaises(FileNotFoundError): producer.create_snapshot(self.root, ['pom.xml'], 'a' * 40, 'apache/hugegraph', '321', '2') From 2fa78c13d1a2a21b9f27e61ce705d54341a8cbcb Mon Sep 17 00:00:00 2001 From: imbajin Date: Wed, 14 Oct 2026 04:35:48 +0800 Subject: [PATCH 06/14] fix(ci): scope dependency snapshot validation - select dependency checks for producer and workflow inputs - keep fixture tests in the existing entry job - verify mixed changes retain runtime coverage --- .github/scripts/ci-policy.py | 5 ++++- .github/scripts/test_ci_policy.py | 12 ++++++++++++ docs/ci.md | 2 ++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/.github/scripts/ci-policy.py b/.github/scripts/ci-policy.py index dd2fd27331..c6059742ca 100644 --- a/.github/scripts/ci-policy.py +++ b/.github/scripts/ci-policy.py @@ -43,7 +43,8 @@ "codeql-analysis.yml": {"security"}, "server-memory-ci.yml": {"server"}, "server-tests.yml": {"server", "hbase"}, "server-compatibility-ci.yml": {"macos", "macos_full", "hbase", "riscv64"}, - "riscv64-ci.yml": {"riscv64"}, "check-dependencies.yml": {"dependency_license"}}, + "riscv64-ci.yml": {"riscv64"}, "check-dependencies.yml": {"dependency_license"}, + "submit-dependencies.yml": {"dependency_license"}}, "toolchain": {"client-ci.yml": {"client"}, "client-go-ci.yml": {"go"}, "loader-ci.yml": {"loader"}, "tools-ci.yml": {"tools"}, "spark-connector-ci.yml": {"spark"}, "hubble-ci.yml": {"hubble"}, @@ -60,6 +61,8 @@ ".asf.yaml": set(), ".editorconfig": set(), ".licenserc.yaml": set(), ".github/PULL_REQUEST_TEMPLATE.md": set(), ".github/dependabot.yml": set(), ".github/workflows/.java-version": MAINLINE | {"macos_full", "hbase", "riscv64"}, + ".github/scripts/dependency-snapshot.py": {"dependency_license"}, + ".github/scripts/test_dependency_snapshot.py": set(), # Covered by preflight discovery. ".github/scripts/check-docker-images.sh": {"docker"}, ".github/scripts/docker-deployment.py": {"docker"}, ".github/scripts/test_docker_deployment.py": {"docker"}, diff --git a/.github/scripts/test_ci_policy.py b/.github/scripts/test_ci_policy.py index a60486ebec..83b75337cd 100644 --- a/.github/scripts/test_ci_policy.py +++ b/.github/scripts/test_ci_policy.py @@ -121,6 +121,18 @@ def test_shared_runtime_consumers_and_protocols(self): self.assertTrue({"pd", "store", "hstore", "cluster"}.issubset(policy.select( "server", ["hugegraph-pd/hg-pd-grpc/src/main/proto/common.proto"]))) + def test_dependency_snapshot_inputs_do_not_fan_out_to_runtime_suites(self): + for path in [".github/scripts/dependency-snapshot.py", ".github/workflows/submit-dependencies.yml"]: + self.assertEqual({"dependency_license"}, set(self.plan_for([path])["selected"])) + self.assertEqual({"dependency_license"}, self.coordinator_jobs(self.plan_for([path]))) + fixture = ".github/scripts/test_dependency_snapshot.py" + self.assertEqual([], self.plan_for([fixture])["expected"]) + runtime = "hugegraph-server/hugegraph-core/src/main/A.java" + self.assertEqual(self.plan_for([runtime])["selected"], self.plan_for([runtime, fixture])["selected"]) + # Classifier changes still receive conservative mainline validation. + self.assertEqual(policy.select("server", [".github/scripts/ci-policy.py"]), + policy.select("server", [".github/scripts/ci-policy.py", fixture])) + def test_noncore_platforms_only_follow_direct_inputs(self): for path in ["hugegraph-server/hugegraph-hbase/src/main/A.java", "hugegraph-server/hugegraph-dist/src/assembly/travis/install-hbase.sh", diff --git a/docs/ci.md b/docs/ci.md index 6da30987bf..a7beebcec4 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -155,3 +155,5 @@ workflow does not prove that either repository has a baseline: verify each repos first master submission, including its SHA/ref, non-empty module manifests and a real dependency-review comparison against that baseline. Until then, that repository's baseline acceptance remains open. This workflow changes no repository settings. + +Dependency snapshot producer and submission-workflow changes select dependency checks. Snapshot fixture tests run in the entry job; they do not select runtime suites. Changes to the CI classifier still receive conservative mainline validation. From 397414764253e10efb41a2050e6f0f26b43ab13c Mon Sep 17 00:00:00 2001 From: imbajin Date: Thu, 15 Oct 2026 01:54:43 +0800 Subject: [PATCH 07/14] fix(ci): verify dependency inventory - resolve an independent per-module Maven inventory - reject incomplete snapshots before token-bearing submission - cover inventory omissions and document acceptance gates --- .github/scripts/ci-policy.py | 1 + .github/scripts/dependency-inventory.py | 99 +++++++++++++ .github/scripts/test_ci_policy.py | 3 +- .github/scripts/test_dependency_snapshot.py | 148 +++++++++++++++++++- .github/workflows/submit-dependencies.yml | 125 ++++++++++++++--- docs/ci.md | 22 ++- 6 files changed, 364 insertions(+), 34 deletions(-) create mode 100644 .github/scripts/dependency-inventory.py diff --git a/.github/scripts/ci-policy.py b/.github/scripts/ci-policy.py index dfa19a952e..33a1d4ea1e 100644 --- a/.github/scripts/ci-policy.py +++ b/.github/scripts/ci-policy.py @@ -62,6 +62,7 @@ ".github/PULL_REQUEST_TEMPLATE.md": set(), ".github/dependabot.yml": set(), ".github/workflows/.java-version": MAINLINE | {"macos_full", "hbase", "riscv64"}, ".github/scripts/dependency-snapshot.py": {"dependency_license"}, + ".github/scripts/dependency-inventory.py": {"dependency_license"}, ".github/scripts/test_dependency_snapshot.py": set(), # Covered by preflight discovery. ".github/scripts/check-docker-images.sh": {"docker"}, ".github/scripts/docker-deployment.py": {"docker"}, diff --git a/.github/scripts/dependency-inventory.py b/.github/scripts/dependency-inventory.py new file mode 100644 index 0000000000..9010041017 --- /dev/null +++ b/.github/scripts/dependency-inventory.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Canonical inventory from independent Apache Maven dependency:tree JSON.""" + +import datetime +import importlib.util +import json +import os +from pathlib import Path +import subprocess +import sys + +spec = importlib.util.spec_from_file_location('snapshot', Path(__file__).with_name('dependency-snapshot.py')) +snapshot = importlib.util.module_from_spec(spec) +spec.loader.exec_module(snapshot) + +REPORT = 'dependency-inventory.json' +RESOLVER = 'org.apache.maven.plugins:maven-dependency-plugin:3.8.1:tree' +LIMIT = 25 * 1024 * 1024 +FIELDS = {'groupId', 'artifactId', 'version', 'type', 'scope', 'classifier', 'optional', 'children'} +SCOPES = {'compile', 'runtime', 'provided', 'system', 'test'} + + +def packages(tree): + result = {} + pending = [(tree, 0)] + while pending: + node, depth = pending.pop() + if not isinstance(node, dict) or set(node) - FIELDS: + raise ValueError('Invalid inventory node') + for field in ['groupId', 'artifactId', 'version', 'type']: + if not isinstance(node.get(field), str) or not node[field].strip(): + raise ValueError('Incomplete inventory coordinates') + if not isinstance(node.get('classifier'), str) or node.get('optional') not in ['true', 'false']: + raise ValueError('Invalid inventory classifier/optional') + scope = node.get('scope') + if (depth == 0 and scope != '') or (depth > 0 and scope not in SCOPES): + raise ValueError('Invalid inventory scope') + children = node.get('children', []) + if not isinstance(children, list): + raise ValueError('Invalid inventory children') + pending.extend((child, depth + 1) for child in children) + if depth == 0: + continue + purl = snapshot.package_url(dict(node, types=[node['type']], classifiers=[node['classifier']])) + value = {'scope': 'development' if scope == 'test' else 'runtime', + 'relationship': 'direct' if depth == 1 else 'indirect'} + # Maven may expose the same selected package along more than one path. + previous = result.get(purl) + if previous: + if previous['scope'] == 'runtime': + value['scope'] = 'runtime' + if previous['relationship'] == 'direct': + value['relationship'] = 'direct' + result[purl] = value + return result + + +def create_inventory(workspace, poms, sha, repository, run_id, run_attempt): + manifests = {} + for pom in poms: + if pom in manifests: + raise ValueError('Duplicate inventory POM') + report = workspace / Path(pom).parent / 'target' / REPORT + if report.is_symlink(): + raise ValueError('Invalid inventory report') + with report.open('rb') as stream: + raw = stream.read(LIMIT + 1) + if not raw or len(raw) > LIMIT: + raise ValueError('Invalid inventory report size') + manifests[pom] = packages(json.loads(raw, object_pairs_hook=snapshot.unique_object)) + if not manifests or not any(manifests.values()): + raise ValueError('Empty independent dependency inventory') + return {'version': 1, 'sha': sha, 'repository': repository, 'run_id': run_id, + 'run_attempt': run_attempt, 'producer': 'inventory', 'resolver': RESOLVER, + 'scanned': datetime.datetime.now(datetime.timezone.utc).isoformat().replace('+00:00', 'Z'), + 'manifests': manifests} + + +if __name__ == '__main__': + poms = subprocess.check_output(['git', 'ls-files', '-z', '*pom.xml']).decode().split('\0') + poms = [pom for pom in poms if Path(pom).name == 'pom.xml'] + inventory = create_inventory(Path.cwd(), poms, os.environ['GITHUB_SHA'], os.environ['GITHUB_REPOSITORY'], + os.environ['GITHUB_RUN_ID'], os.environ['GITHUB_RUN_ATTEMPT']) + Path(sys.argv[1]).write_text(json.dumps(inventory, indent=2) + '\n') diff --git a/.github/scripts/test_ci_policy.py b/.github/scripts/test_ci_policy.py index ca566d5d70..c43130b904 100644 --- a/.github/scripts/test_ci_policy.py +++ b/.github/scripts/test_ci_policy.py @@ -122,7 +122,8 @@ def test_shared_runtime_consumers_and_protocols(self): "server", ["hugegraph-pd/hg-pd-grpc/src/main/proto/common.proto"]))) def test_dependency_snapshot_inputs_do_not_fan_out_to_runtime_suites(self): - for path in [".github/scripts/dependency-snapshot.py", ".github/workflows/submit-dependencies.yml"]: + for path in [".github/scripts/dependency-snapshot.py", ".github/scripts/dependency-inventory.py", + ".github/workflows/submit-dependencies.yml"]: self.assertEqual({"dependency_license"}, set(self.plan_for([path])["selected"])) self.assertEqual({"dependency_license"}, self.coordinator_jobs(self.plan_for([path]))) fixture = ".github/scripts/test_dependency_snapshot.py" diff --git a/.github/scripts/test_dependency_snapshot.py b/.github/scripts/test_dependency_snapshot.py index 9c8eae3440..7885ed46d5 100644 --- a/.github/scripts/test_dependency_snapshot.py +++ b/.github/scripts/test_dependency_snapshot.py @@ -43,6 +43,7 @@ def load_module(path, name): producer = load_module(Path(__file__).with_name('dependency-snapshot.py'), 'producer') +inventory_producer = load_module(Path(__file__).with_name('dependency-inventory.py'), 'inventory_producer') def artifact(identifier, **fields): @@ -62,7 +63,8 @@ def setUp(self): script.write_text(textwrap.dedent(program)) self.writer = load_module(script, 'writer') self.env = {'EXPECTED_REPOSITORY': 'apache/hugegraph', 'EXPECTED_SHA': 'a' * 40, - 'RUN_ID': '321', 'RUN_ATTEMPT': '2', 'ARTIFACT_ID': '789', 'RUNNER_TEMP': str(self.root), + 'RUN_ID': '321', 'RUN_ATTEMPT': '2', 'ARTIFACT_ID': '789', 'INVENTORY_ARTIFACT_ID': '987', + 'RUNNER_TEMP': str(self.root), 'SNAPSHOT_TOKEN': 'public-synthetic-sentinel'} directory = self.root / 'dependency-snapshot' directory.mkdir() @@ -76,17 +78,27 @@ def setUp(self): 'detector': producer.DETECTOR, 'metadata': {'serializer': producer.SERIALIZER, 'producer': 'generate', 'run_attempt': '2'}, 'scanned': now.isoformat(), 'manifests': {'module': producer.manifest(graph, 'pom.xml')}} + self.inventory = {'version': 1, 'sha': 'a' * 40, 'repository': 'apache/hugegraph', 'run_id': '321', + 'run_attempt': '2', 'producer': 'inventory', 'resolver': inventory_producer.RESOLVER, + 'scanned': now.isoformat(), 'manifests': {'pom.xml': { + key: {'scope': value['scope'], 'relationship': value['relationship']} + for key, value in self.snapshot['manifests']['module']['resolved'].items()}}} self.responses = { 'actions/runs/321/attempts/2': {'id': 321, 'run_attempt': 2, 'head_sha': 'a' * 40, 'head_branch': 'master', 'event': 'push', 'repository': {'full_name': 'apache/hugegraph'}, 'path': '.github/workflows/submit-dependencies.yml', 'run_started_at': (now - datetime.timedelta(minutes=2)).isoformat()}, - 'actions/runs/321/attempts/2/jobs?per_page=100': {'total_count': 3, + 'actions/runs/321/attempts/2/jobs?per_page=100': {'total_count': 4, 'jobs': [{'name': 'generate', 'conclusion': 'success', 'status': 'completed', + 'run_id': 321, 'head_sha': 'a' * 40}, + {'name': 'inventory', 'conclusion': 'success', 'status': 'completed', 'run_id': 321, 'head_sha': 'a' * 40}]}, 'actions/artifacts/789': {'id': 789, 'expired': False, 'name': 'dependency-snapshot-321-2', 'created_at': now.isoformat(), 'workflow_run': {'id': 321, 'head_sha': 'a' * 40, 'head_branch': 'master'}}, + 'actions/artifacts/987': {'id': 987, 'expired': False, 'name': 'dependency-inventory-321-2', + 'created_at': now.isoformat(), + 'workflow_run': {'id': 321, 'head_sha': 'a' * 40, 'head_branch': 'master'}}, 'git/trees/' + 'a' * 40 + '?recursive=1': {'truncated': False, 'tree': [{'path': 'pom.xml', 'type': 'blob', 'mode': '100644'}]}, 'git/ref/heads/master': {'object': {'sha': 'a' * 40}}, @@ -95,13 +107,19 @@ def setUp(self): self.posts = [] self.blob_headers = [] - def run_writer(self, snapshot=None, raw=None, archive=None, artifact_changes=None): + def run_writer(self, snapshot=None, raw=None, archive=None, artifact_changes=None, inventory=None, + inventory_archive=None, inventory_artifact_changes=None): if archive is None: payload = raw if raw is not None else json.dumps(snapshot or self.snapshot) archive = self.zip_bytes([('snapshot.json', payload)]) self.responses['actions/artifacts/789'].update(size_in_bytes=len(archive), digest='sha256:' + hashlib.sha256(archive).hexdigest()) self.responses['actions/artifacts/789'].update(artifact_changes or {}) + if inventory_archive is None: + inventory_archive = self.zip_bytes([('inventory.json', json.dumps(inventory or self.inventory))]) + self.responses['actions/artifacts/987'].update(size_in_bytes=len(inventory_archive), + digest='sha256:' + hashlib.sha256(inventory_archive).hexdigest()) + self.responses['actions/artifacts/987'].update(inventory_artifact_changes or {}) def open_request(request, timeout): url = urlsplit(request.full_url) @@ -109,16 +127,17 @@ def open_request(request, timeout): if url.netloc == 'storage.blob.core.windows.net': self.blob_headers.append(dict(request.header_items())) self.assertIsNone(request.get_header('Authorization')) - response = io.BytesIO(archive) + response = io.BytesIO(inventory_archive if url.path == '/987.zip' else archive) response.status = 200 return response self.assertEqual('api.github.com', url.netloc) self.assertTrue(url.path.startswith('/repos/apache/hugegraph/')) self.assertEqual('Bearer public-synthetic-sentinel', request.get_header('Authorization')) path = url.path.removeprefix('/repos/apache/hugegraph/') + ('?' + url.query if url.query else '') - if path == 'actions/artifacts/789/zip': + if path in ['actions/artifacts/789/zip', 'actions/artifacts/987/zip']: raise urllib.error.HTTPError(request.full_url, 302, 'Found', - {'Location': 'https://storage.blob.core.windows.net/archive.zip?sig=public'}, io.BytesIO()) + {'Location': 'https://storage.blob.core.windows.net/' + path.split('/')[2] + '.zip?sig=public'}, + io.BytesIO()) if request.data is not None: self.assertEqual('dependency-graph/snapshots', path) self.posts.append(json.loads(request.data)) @@ -274,6 +293,67 @@ def test_writer_rejects_failed_producer_wrong_artifact_and_stale_master(self): self.responses[path] = original self.assertEqual([], self.posts) + def test_writer_rejects_incomplete_package_inventory_before_post(self): + resolved = self.snapshot['manifests']['module']['resolved'] + for key in list(resolved): + candidate = copy.deepcopy(self.snapshot) + packages = candidate['manifests']['module']['resolved'] + packages.pop(key) + for package in packages.values(): + package['dependencies'] = [child for child in package['dependencies'] if child != key] + with self.subTest(missing=key), self.assertRaises(ValueError): + self.run_writer(snapshot=candidate) + for field, value in [('scope', 'development'), ('relationship', 'direct')]: + candidate = copy.deepcopy(self.snapshot) + key = next(key for key, value in resolved.items() if value['relationship'] == 'indirect') + candidate['manifests']['module']['resolved'][key][field] = value + with self.subTest(field=field), self.assertRaisesRegex(ValueError, 'inventory mismatch'): + self.run_writer(snapshot=candidate) + candidate = copy.deepcopy(self.snapshot) + packages = candidate['manifests']['module']['resolved'] + key = next(key for key, value in packages.items() if value['relationship'] == 'indirect') + altered = key.replace('@1.0?', '@2.0?') + packages[altered] = packages.pop(key) + packages[altered]['package_url'] = altered + for package in packages.values(): + package['dependencies'] = [altered if child == key else child for child in package['dependencies']] + with self.assertRaisesRegex(ValueError, 'inventory mismatch'): + self.run_writer(snapshot=candidate) + self.assertEqual([], self.posts) + + def test_writer_rejects_foreign_or_incomplete_inventory(self): + for key, value in [('sha', 'b' * 40), ('repository', 'other/repo'), ('run_id', '123'), + ('run_attempt', '1'), ('producer', 'generate'), ('resolver', 'unknown'), + ('version', False), ('scanned', '2000-01-01T00:00:00Z'), ('manifests', {})]: + candidate = copy.deepcopy(self.inventory) + candidate[key] = value + with self.subTest(field=key), self.assertRaises(ValueError): + self.run_writer(inventory=candidate) + for fields in [{'expired': True}, {'name': 'dependency-inventory-321-1'}, + {'workflow_run': {'id': 321, 'head_sha': 'b' * 40, 'head_branch': 'master'}}, + {'digest': 'sha256:' + '0' * 64}]: + original = copy.deepcopy(self.responses['actions/artifacts/987']) + with self.subTest(fields=fields), self.assertRaises((ValueError, KeyError)): + self.run_writer(inventory_artifact_changes=fields) + self.responses['actions/artifacts/987'] = original + jobs = self.responses['actions/runs/321/attempts/2/jobs?per_page=100']['jobs'] + for field, value in [('conclusion', 'failure'), ('head_sha', 'b' * 40), ('run_id', 123)]: + original = jobs[1][field] + jobs[1][field] = value + with self.subTest(field=field), self.assertRaises(ValueError): + self.run_writer() + jobs[1][field] = original + self.assertEqual([], self.posts) + + def test_writer_rejects_inventory_zip_wrong_member_without_extraction(self): + before = set(self.root.rglob('*')) + for entries in [[('snapshot.json', '{}')], [('inventory.json', '{}'), ('inventory.json', '{}')], + [('../inventory.json', '{}')]]: + with self.subTest(entries=entries), self.assertRaises(ValueError): + self.run_writer(inventory_archive=self.zip_bytes(entries)) + self.assertEqual(before, set(self.root.rglob('*'))) + self.assertEqual([], self.posts) + def test_writer_rejects_zip_paths_duplicates_symlinks_and_bounds_without_writes(self): body = json.dumps(self.snapshot) fixtures = [self.zip_bytes([('snapshot.json', body), ('x/../../outside', 'must not write')]), @@ -293,7 +373,7 @@ def test_writer_rejects_zip_paths_duplicates_symlinks_and_bounds_without_writes( def test_writer_download_has_no_cross_host_authorization_or_followup_redirect(self): self.run_writer() - self.assertEqual([{}], self.blob_headers) + self.assertEqual([{}, {}], self.blob_headers) self.posts.clear() for metadata in [{'digest': 'sha256:' + '0' * 64}, {'size_in_bytes': 1}]: with self.assertRaisesRegex(ValueError, 'digest/size mismatch'): @@ -319,5 +399,59 @@ def redirected(request, timeout): self.assertEqual([], self.posts) +class InventoryTest(unittest.TestCase): + + @staticmethod + def node(name, scope='', children=None, **fields): + return dict(groupId='org.example', artifactId=name, version='1.0', type='jar', scope=scope, + classifier='', optional='false', children=children or [], **fields) + + def test_independent_tree_scopes_classifier_and_direct_precedence(self): + leaf = self.node('leaf', 'test') + parent = self.node('parent', 'compile', [leaf]) + classified = self.node('native', 'provided') + classified['classifier'] = 'linux-aarch_64' + tree = self.node('root', children=[parent, self.node('leaf', 'compile'), classified]) + prefix = 'pkg:maven/org.example/' + self.assertEqual({ + prefix + 'parent@1.0?type=jar': {'relationship': 'direct', 'scope': 'runtime'}, + prefix + 'leaf@1.0?type=jar': {'relationship': 'direct', 'scope': 'runtime'}, + prefix + 'native@1.0?classifier=linux-aarch_64&type=jar': + {'relationship': 'direct', 'scope': 'runtime'}, + }, inventory_producer.packages(tree)) + self.assertEqual({}, inventory_producer.packages(self.node('aggregate'))) + + def test_inventory_refuses_invalid_nodes(self): + for field, value in [('scope', 'compile'), ('children', {}), ('optional', None), + ('classifier', None), ('version', '')]: + node = self.node('root') + node[field] = value + with self.subTest(field=field), self.assertRaises(ValueError): + inventory_producer.packages(node) + with self.assertRaises(ValueError): + inventory_producer.packages(self.node('root', children=[self.node('bad', 'unknown')])) + + def test_inventory_requires_every_report_and_refuses_empty_or_duplicate_poms(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + target = root / 'target' + target.mkdir() + report = target / inventory_producer.REPORT + report.write_text(json.dumps(self.node('root', children=[self.node('leaf', 'test')]))) + result = inventory_producer.create_inventory(root, ['pom.xml'], 'a' * 40, 'apache/hugegraph', '1', '1') + self.assertEqual('development', next(iter(result['manifests']['pom.xml'].values()))['scope']) + with self.assertRaises(ValueError): + inventory_producer.create_inventory(root, ['pom.xml', 'pom.xml'], 'a' * 40, 'repo', '1', '1') + with self.assertRaises(FileNotFoundError): + inventory_producer.create_inventory(root, ['module/pom.xml'], 'a' * 40, 'repo', '1', '1') + report.write_text(json.dumps(self.node('root'))) + with self.assertRaisesRegex(ValueError, 'Empty independent'): + inventory_producer.create_inventory(root, ['pom.xml'], 'a' * 40, 'repo', '1', '1') + report.unlink() + report.symlink_to(target / 'outside') + with self.assertRaises(ValueError): + inventory_producer.create_inventory(root, ['pom.xml'], 'a' * 40, 'repo', '1', '1') + + if __name__ == '__main__': unittest.main() diff --git a/.github/workflows/submit-dependencies.yml b/.github/workflows/submit-dependencies.yml index 9b4a9edc46..c29d561ec5 100644 --- a/.github/workflows/submit-dependencies.yml +++ b/.github/workflows/submit-dependencies.yml @@ -107,10 +107,68 @@ jobs: if-no-files-found: error retention-days: 1 + inventory: + needs: validate + if: >- + ((github.repository == 'apache/hugegraph' && github.event.repository.fork == false) || + github.repository == 'hugegraph/hugegraph') && + github.ref == 'refs/heads/master' && + (github.event_name == 'push' || github.event_name == 'workflow_dispatch') + runs-on: ubuntu-latest + timeout-minutes: 60 + permissions: + contents: read + env: + PYTHONDONTWRITEBYTECODE: '1' + INPUT_TOKEN: '' + GITHUB_TOKEN: '' + GH_TOKEN: '' + outputs: + artifact_id: ${{ steps.inventory.outputs.artifact-id }} + steps: + - name: Checkout event commit + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + persist-credentials: false + - name: Set up project JDK + uses: actions/setup-java@de7274f081f381c8f8158605e0321c36c376e2e6 # v6.0.1 + with: + java-version-file: .github/workflows/.java-version + distribution: 'temurin' + - name: Verify checkout and current master + env: + EXPECTED_SHA: ${{ github.sha }} + run: | + test "$(git rev-parse HEAD)" = "$EXPECTED_SHA" + test "$(git ls-remote origin refs/heads/master | cut -f1)" = "$EXPECTED_SHA" + - name: Install the complete Maven reactor + run: mvn -B -ntp install -Dmaven.test.skip=true -Dmaven.javadoc.skip=true --fail-at-end + - name: Resolve independent dependency inventory + run: | + mvn -B -ntp org.apache.maven.plugins:maven-dependency-plugin:3.8.1:tree \ + -DoutputType=json -DoutputFile=target/dependency-inventory.json \ + -Dverbose=false -DappendOutput=false \ + -Dmaven.test.skip=true -Dmaven.javadoc.skip=true + python3 .github/scripts/dependency-inventory.py "$RUNNER_TEMP/inventory.json" + - name: Refuse a stale inventory + env: + EXPECTED_SHA: ${{ github.sha }} + run: test "$(git ls-remote origin refs/heads/master | cut -f1)" = "$EXPECTED_SHA" + - name: Upload inventory data only + id: inventory + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: dependency-inventory-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/inventory.json + if-no-files-found: error + retention-days: 1 + submit: - needs: [validate, generate] + needs: [validate, generate, inventory] if: >- needs.validate.result == 'success' && needs.generate.result == 'success' && + needs.inventory.result == 'success' && ((github.repository == 'apache/hugegraph' && github.event.repository.fork == false) || github.repository == 'hugegraph/hugegraph') && github.ref == 'refs/heads/master' && @@ -129,6 +187,7 @@ jobs: RUN_ID: ${{ github.run_id }} RUN_ATTEMPT: ${{ github.run_attempt }} ARTIFACT_ID: ${{ needs.generate.outputs.artifact_id }} + INVENTORY_ARTIFACT_ID: ${{ needs.inventory.outputs.artifact_id }} run: | python3 -I - <<'PY' import datetime @@ -179,13 +238,13 @@ jobs: raw = bounded_body(response) return json.loads(raw, object_pairs_hook=unique_object) - def read_snapshot(raw): + def read_snapshot(raw, member_name='snapshot.json'): require(0 < len(raw) <= LIMIT, 'Invalid ZIP size') with zipfile.ZipFile(io.BytesIO(raw)) as archive: members = archive.infolist() - require(len(members) == 1, 'ZIP must contain one snapshot.json') + require(len(members) == 1, 'ZIP must contain one data file') member = members[0] - require(member.orig_filename == 'snapshot.json' and not member.is_dir() and + require(member.orig_filename == member_name and not member.is_dir() and not (member.external_attr & 0x10) and stat.S_IFMT(member.external_attr >> 16) in [0, stat.S_IFREG] and not (member.flag_bits & 1) and @@ -196,7 +255,7 @@ jobs: require(len(payload) == member.file_size, 'Incomplete ZIP member') return json.loads(payload, object_pairs_hook=unique_object) - def download_snapshot(repository, artifact_id, artifact): + def download_snapshot(repository, artifact_id, artifact, member_name='snapshot.json'): opener = urllib.request.build_opener(NoRedirect()) try: response = opener.open(github_request(repository, f'actions/artifacts/{artifact_id}/zip'), timeout=60) @@ -218,7 +277,7 @@ jobs: raw = bounded_body(response) require(len(raw) == artifact['size_in_bytes'] and 'sha256:' + hashlib.sha256(raw).hexdigest() == artifact['digest'], 'Artifact digest/size mismatch') - return read_snapshot(raw) + return read_snapshot(raw, member_name) def timestamp(value): parsed = datetime.datetime.fromisoformat(value.replace('Z', '+00:00')) @@ -239,20 +298,28 @@ jobs: run['path'] in ['.github/workflows/submit-dependencies.yml', '.github/workflows/submit-dependencies.yml@master'], 'Run identity') jobs = api(repository, f'actions/runs/{run_id}/attempts/{attempt}/jobs?per_page=100') - producers = [job for job in jobs['jobs'] if job['name'] == 'generate'] - require(jobs['total_count'] <= 100 and len(producers) == 1 and - producers[0]['conclusion'] == 'success' and producers[0]['status'] == 'completed' and - producers[0]['run_id'] == int(run_id) and producers[0]['head_sha'] == sha, - 'Generation did not succeed for this commit') - artifact = api(repository, f'actions/artifacts/{artifact_id}') - require(artifact['id'] == int(artifact_id) and artifact['expired'] is False and - artifact['name'] == f'dependency-snapshot-{run_id}-{attempt}' and - artifact['workflow_run']['id'] == int(run_id) and - artifact['workflow_run']['head_sha'] == sha and - artifact['workflow_run']['head_branch'] == 'master' and - type(artifact['size_in_bytes']) is int and 0 < artifact['size_in_bytes'] <= LIMIT and - isinstance(artifact['digest'], str) and - re.fullmatch('sha256:[0-9a-f]{64}', artifact['digest']), 'Artifact identity') + require(jobs['total_count'] <= 100, 'Too many jobs') + for name in ['generate', 'inventory']: + producers = [job for job in jobs['jobs'] if job['name'] == name] + require(len(producers) == 1 and producers[0]['conclusion'] == 'success' and + producers[0]['status'] == 'completed' and producers[0]['run_id'] == int(run_id) and + producers[0]['head_sha'] == sha, 'Producer did not succeed for this commit') + inventory_id = os.environ['INVENTORY_ARTIFACT_ID'] + require(re.fullmatch('[1-9][0-9]*', inventory_id) and inventory_id != artifact_id, 'Inventory ID') + + def checked_artifact(identifier, kind): + item = api(repository, f'actions/artifacts/{identifier}') + require(item['id'] == int(identifier) and item['expired'] is False and + item['name'] == f'dependency-{kind}-{run_id}-{attempt}' and + item['workflow_run']['id'] == int(run_id) and item['workflow_run']['head_sha'] == sha and + item['workflow_run']['head_branch'] == 'master' and + type(item['size_in_bytes']) is int and 0 < item['size_in_bytes'] <= LIMIT and + isinstance(item['digest'], str) and re.fullmatch('sha256:[0-9a-f]{64}', item['digest']), + 'Artifact identity') + return item + + artifact = checked_artifact(artifact_id, 'snapshot') + inventory_artifact = checked_artifact(inventory_id, 'inventory') snapshot = download_snapshot(repository, artifact_id, artifact) require(isinstance(snapshot, dict) and set(snapshot) == {'version', 'sha', 'ref', 'job', 'detector', 'metadata', 'scanned', 'manifests'}, 'Snapshot schema') @@ -271,10 +338,25 @@ jobs: timestamp(artifact['created_at']) + datetime.timedelta(minutes=1) and timestamp(artifact['created_at']) <= now and timestamp(snapshot['scanned']) <= now, 'Snapshot freshness') + inventory = download_snapshot(repository, inventory_id, inventory_artifact, 'inventory.json') + require(isinstance(inventory, dict) and set(inventory) == + {'version', 'sha', 'repository', 'run_id', 'run_attempt', 'producer', 'resolver', + 'scanned', 'manifests'}, 'Inventory schema') + require(type(inventory['version']) is int and inventory['version'] == 1 and + inventory['sha'] == sha and inventory['repository'] == repository and + inventory['run_id'] == run_id and inventory['run_attempt'] == attempt and + inventory['producer'] == 'inventory' and inventory['resolver'] == + 'org.apache.maven.plugins:maven-dependency-plugin:3.8.1:tree', 'Inventory source') + require(timestamp(run['run_started_at']) <= timestamp(inventory['scanned']) <= + timestamp(inventory_artifact['created_at']) + datetime.timedelta(minutes=1) and + timestamp(inventory_artifact['created_at']) <= now and timestamp(inventory['scanned']) <= now, + 'Inventory freshness') tree = api(repository, f'git/trees/{sha}?recursive=1') require(tree['truncated'] is False, 'Incomplete source tree') poms = {node['path'] for node in tree['tree'] if node['type'] == 'blob' and node['path'].split('/')[-1] == 'pom.xml' and node['mode'] != '120000'} + require(isinstance(inventory['manifests'], dict) and set(inventory['manifests']) == poms, + 'Incomplete independent inventory') manifests, paths = snapshot['manifests'], set() require(isinstance(manifests, dict) and manifests and poms, 'Empty inventory') component = r"(?:[A-Za-z0-9._!~*'():+-]|%[0-9A-F]{2})+" @@ -303,6 +385,9 @@ jobs: 'Package schema') require(not resolved or any(p['relationship'] == 'direct' for p in resolved.values()), 'Missing direct dependencies') + observed = {purl: {'scope': package['scope'], 'relationship': package['relationship']} + for purl, package in resolved.items()} + require(observed == inventory['manifests'][location], 'Dependency inventory mismatch') require(paths == poms and any(m['resolved'] for m in manifests.values()), 'Incomplete inventory') require(api(repository, 'git/ref/heads/master')['object']['sha'] == sha, 'Stale master before submission') result = api(repository, 'dependency-graph/snapshots', snapshot) diff --git a/docs/ci.md b/docs/ci.md index de63f40411..118917926a 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -156,10 +156,20 @@ dispatches. Only `apache/hugegraph` (not a fork) and the explicitly allowed `hugegraph/hugegraph` test fork on `refs/heads/master` can run the submission job, which alone has `contents: write`. PRs, other repositories and other refs cannot run the writer. Maven inherits no submission token, and checkout does not persist -credentials. A pinned `actions/upload-artifact` transfers only `snapshot.json`. -The fresh writer runner uses fixed inline stdlib Python to verify artifact/run -identity before downloading. It bounds ZIP and JSON sizes, checks the archive digest, -and reads only one regular `snapshot.json` member in memory. Duplicate members, +credentials. An independent `contents: read` job resolves all module dependency +scopes with Apache Maven Dependency Plugin 3.8.1 `tree` JSON. Its +[stdlib inventory adapter](../.github/scripts/dependency-inventory.py) records each POM's +package URLs, scopes and direct/indirect relationships independently of the Ferstl +parser. The writer requires exact equality with that inventory, including empty +aggregators; a graph retaining one direct package while omitting another fails. +The global runtime/shipped jar filename inventory has a different scope and does +not replace this per-module check. + +A pinned `actions/upload-artifact` transfers `snapshot.json` and `inventory.json` +as separate artifacts. The fresh writer runner uses fixed inline stdlib Python to +verify both successful producer jobs and both artifact/run/attempt/SHA identities +before downloading. It bounds ZIP and JSON sizes, checks each archive digest, +and reads only its one expected regular JSON member in memory. Duplicate members, paths and symlinks fail; no archive members are extracted to disk. A signed storage redirect is fetched without forwarding GitHub Authorization, and further redirects are refused. The writer does not check out the repository, run Maven, or execute @@ -167,7 +177,7 @@ downloaded code. The first-party upload action is permitted by the [ASF Actions policy](https://infra.apache.org/github-actions-policy.html). Checkout and snapshot use the event commit SHA. Master is checked before install -and again after graph generation and serialization; a stale run fails. The writer +and again after generation in both read-only jobs; a stale run fails. The writer checks the run attempt, successful producer, artifact ID/name, JSON schema, detector/correlator and source SHA/ref. It verifies all manifest paths against the event commit's source tree, rejects empty or mismatched inventory, and checks master @@ -176,7 +186,7 @@ if master advances during submission; that run fails its final freshness check. If submission fails, use **Re-run all jobs** or start a fresh `workflow_dispatch` on master. **Re-run failed jobs** keeps the older generation artifact, which the -writer rejects because the snapshot must be produced in the same run attempt. +writer rejects because both reports must be produced in the same run attempt. A fixed master-only correlator keeps successive baseline snapshots in the same stream within each repository; ASF and org inventories are independent. The writer's From a75e16be16395b394698fae425e9dd6a12a52f91 Mon Sep 17 00:00:00 2001 From: imbajin Date: Thu, 15 Oct 2026 05:29:17 +0800 Subject: [PATCH 08/14] fix(ci): preserve mixed dependency scopes - classify mixed test and non-test scopes as runtime - verify scope parity through the existing inventory writer - document precedence without changing submission flow --- .github/scripts/dependency-snapshot.py | 9 +++++---- .github/scripts/test_dependency_snapshot.py | 17 +++++++++++++++++ docs/ci.md | 6 ++++-- 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/.github/scripts/dependency-snapshot.py b/.github/scripts/dependency-snapshot.py index ed71cef60a..55a38b2627 100644 --- a/.github/scripts/dependency-snapshot.py +++ b/.github/scripts/dependency-snapshot.py @@ -16,9 +16,10 @@ """Serialize native Ferstl 4.0.3 graphs; no token or submission code. -Matches maven-dependency-submission-action 6.0.1's Maven parser/toolkit: -first classifier/type, test scope, shared PURL packages, direct precedence, -child edge order and cycle handling. See docs/ci.md for the producer boundary. +Uses maven-dependency-submission-action 6.0.1's Maven parser/toolkit conventions: +first classifier/type, shared PURL packages, direct precedence, child edge order +and cycle handling. Non-test scopes take precedence over test to match the +independent inventory. See docs/ci.md for the producer boundary. """ import datetime @@ -89,7 +90,7 @@ def manifest(graph, pom): raise ValueError(f'Duplicate artifact ID: {identifier}') ids[identifier] = purl packages.setdefault(purl, []) - scopes[purl] = 'development' if 'test' in (artifact.get('scopes') or []) else 'runtime' + scopes[purl] = 'development' if set(artifact.get('scopes') or []) == {'test'} else 'runtime' children, incoming = {}, set() for edge in edges: if not isinstance(edge, dict) or any(not isinstance(edge.get(key), str) for key in ['from', 'to']): diff --git a/.github/scripts/test_dependency_snapshot.py b/.github/scripts/test_dependency_snapshot.py index 7885ed46d5..72dd80df67 100644 --- a/.github/scripts/test_dependency_snapshot.py +++ b/.github/scripts/test_dependency_snapshot.py @@ -195,6 +195,23 @@ def test_native_cycle_alias_scope_and_direct_precedence(self): self.assertEqual({}, producer.manifest({'graphName': 'parent', 'artifacts': [artifact('root')]}, 'pom.xml')['resolved']) + def test_native_mixed_scopes_match_independent_inventory(self): + for scope in ['compile', 'runtime', 'provided', 'system']: + for scopes in [[scope, 'test'], ['test', scope]]: + with self.subTest(scopes=scopes): + graph = {'graphName': 'module', 'artifacts': [ + artifact('root'), artifact('parent', scopes=['test']), artifact('one', scopes=scopes)], + 'dependencies': [{'from': 'root', 'to': 'parent'}, {'from': 'parent', 'to': 'one'}, + {'from': 'root', 'to': 'one'}]} + self.snapshot['manifests']['module'] = producer.manifest(graph, 'pom.xml') + tree = InventoryTest.node('root', children=[ + InventoryTest.node('parent', 'test', [InventoryTest.node('one', 'test')]), + InventoryTest.node('one', scope)]) + self.inventory['manifests']['pom.xml'] = inventory_producer.packages(tree) + self.posts.clear() + self.run_writer() + self.assertEqual([self.snapshot], self.posts) + def test_transitive_only_package_is_indirect(self): graph = {'graphName': 'module', 'artifacts': [artifact('root'), artifact('parent'), artifact('leaf')], 'dependencies': [{'from': 'root', 'to': 'parent'}, {'from': 'parent', 'to': 'leaf'}]} diff --git a/docs/ci.md b/docs/ci.md index 118917926a..0e80499ed8 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -150,8 +150,10 @@ installs the complete reactor with Java 17 in a `contents: read` job, then gener each module's resolved graph with Ferstl 4.0.3. The repository's [stdlib serializer](../.github/scripts/dependency-snapshot.py) preserves the [official Maven parser 6.0.1](https://github.com/actions/maven-dependency-submission-action/tree/1f666f7ac2ab7b72cd5e22eaf7f23c8a581db1db) -package URLs, scopes and relationships. Every tracked POM must have a graph; empty -aggregator manifests are allowed, but a completely empty inventory fails. It runs on master pushes or manual +package URLs and relationships. Non-test scopes take precedence when Ferstl merges +scopes onto one artifact; only test-only artifacts are development dependencies. +Every tracked POM must have a graph; empty aggregator manifests are allowed, but a +completely empty inventory fails. It runs on master pushes or manual dispatches. Only `apache/hugegraph` (not a fork) and the explicitly allowed `hugegraph/hugegraph` test fork on `refs/heads/master` can run the submission job, which alone has `contents: write`. PRs, other repositories and other refs cannot From e821d8e79e74666435d1d6fa73fa1432ebbd1cca Mon Sep 17 00:00:00 2001 From: imbajin Date: Thu, 15 Oct 2026 08:50:45 +0800 Subject: [PATCH 09/14] docs(ci): clarify baseline review reruns - require a successful baseline for the exact PR event base SHA - distinguish the same repository from other baseline streams - explain missing or skipped baselines are not fixed by review reruns --- docs/ci.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/ci.md b/docs/ci.md index 0e80499ed8..49b471c4c5 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -190,6 +190,12 @@ If submission fails, use **Re-run all jobs** or start a fresh `workflow_dispatch on master. **Re-run failed jobs** keeps the older generation artifact, which the writer rejects because both reports must be produced in the same run attempt. +If a PR `dependency-review` failure reports 0 base snapshots, rerun that job only +once the same repository's **Dependency baseline** run has succeeded for the event's +exact `pull_request.base.sha` (not the branch's merge base or a newer master tip). +If that SHA has no successful baseline, including a skipped run, rerunning the +review alone cannot supply it. + A fixed master-only correlator keeps successive baseline snapshots in the same stream within each repository; ASF and org inventories are independent. The writer's identity is separate from automatic PR submissions so they cannot replace its baseline. From 81a73d43bd7b6abbec5dfa205384849750c7ed29 Mon Sep 17 00:00:00 2001 From: imbajin Date: Fri, 16 Oct 2026 16:01:23 +0800 Subject: [PATCH 10/14] fix(ci): share dependency baseline build - reuse one cached reactor install for both resolvers - cancel superseded master runs within the same ref - document current-tip recovery and test producer guards --- .github/scripts/test_dependency_snapshot.py | 37 +++++++--- .github/workflows/submit-dependencies.yml | 77 +++++---------------- docs/ci.md | 28 ++++++-- 3 files changed, 66 insertions(+), 76 deletions(-) diff --git a/.github/scripts/test_dependency_snapshot.py b/.github/scripts/test_dependency_snapshot.py index 72dd80df67..bc1bcb7924 100644 --- a/.github/scripts/test_dependency_snapshot.py +++ b/.github/scripts/test_dependency_snapshot.py @@ -88,10 +88,8 @@ def setUp(self): 'head_branch': 'master', 'event': 'push', 'repository': {'full_name': 'apache/hugegraph'}, 'path': '.github/workflows/submit-dependencies.yml', 'run_started_at': (now - datetime.timedelta(minutes=2)).isoformat()}, - 'actions/runs/321/attempts/2/jobs?per_page=100': {'total_count': 4, + 'actions/runs/321/attempts/2/jobs?per_page=100': {'total_count': 3, 'jobs': [{'name': 'generate', 'conclusion': 'success', 'status': 'completed', - 'run_id': 321, 'head_sha': 'a' * 40}, - {'name': 'inventory', 'conclusion': 'success', 'status': 'completed', 'run_id': 321, 'head_sha': 'a' * 40}]}, 'actions/artifacts/789': {'id': 789, 'expired': False, 'name': 'dependency-snapshot-321-2', 'created_at': now.isoformat(), @@ -310,6 +308,32 @@ def test_writer_rejects_failed_producer_wrong_artifact_and_stale_master(self): self.responses[path] = original self.assertEqual([], self.posts) + def test_dependency_baseline_concurrency_isolates_nonmaster_dispatch(self): + workflow = (Path(__file__).resolve().parents[1] / "workflows/submit-dependencies.yml").read_text() + template = next(line.removeprefix(" group: ") for line in workflow.splitlines() + if line.startswith(" group: ")) + + def group(event, ref): + return template.replace("${{ github.repository }}", "apache/hugegraph").replace( + "${{ github.ref }}", ref).replace("${{ github.event_name }}", event) + + master = group("push", "refs/heads/master") + self.assertEqual(master, group("workflow_dispatch", "refs/heads/master")) + self.assertNotEqual(master, group("workflow_dispatch", "refs/heads/feature")) + + def test_writer_requires_one_successful_combined_producer(self): + path = 'actions/runs/321/attempts/2/jobs?per_page=100' + producer = self.responses[path]['jobs'][0] + cases = [[], [producer, producer]] + for field, value in [('conclusion', 'failure'), ('status', 'in_progress'), + ('run_id', 123), ('head_sha', 'b' * 40)]: + cases.append([dict(producer, **{field: value})]) + for jobs in cases: + with self.subTest(jobs=jobs), self.assertRaisesRegex(ValueError, 'Producer did not succeed'): + self.responses[path]['jobs'] = jobs + self.run_writer() + self.assertEqual([], self.posts) + def test_writer_rejects_incomplete_package_inventory_before_post(self): resolved = self.snapshot['manifests']['module']['resolved'] for key in list(resolved): @@ -353,13 +377,6 @@ def test_writer_rejects_foreign_or_incomplete_inventory(self): with self.subTest(fields=fields), self.assertRaises((ValueError, KeyError)): self.run_writer(inventory_artifact_changes=fields) self.responses['actions/artifacts/987'] = original - jobs = self.responses['actions/runs/321/attempts/2/jobs?per_page=100']['jobs'] - for field, value in [('conclusion', 'failure'), ('head_sha', 'b' * 40), ('run_id', 123)]: - original = jobs[1][field] - jobs[1][field] = value - with self.subTest(field=field), self.assertRaises(ValueError): - self.run_writer() - jobs[1][field] = original self.assertEqual([], self.posts) def test_writer_rejects_inventory_zip_wrong_member_without_extraction(self): diff --git a/.github/workflows/submit-dependencies.yml b/.github/workflows/submit-dependencies.yml index c29d561ec5..8085a0cb8a 100644 --- a/.github/workflows/submit-dependencies.yml +++ b/.github/workflows/submit-dependencies.yml @@ -26,8 +26,8 @@ permissions: contents: read concurrency: - group: dependency-baseline-${{ github.repository }}-master - cancel-in-progress: false + group: dependency-baseline-${{ github.repository }}-${{ github.ref }} + cancel-in-progress: true jobs: validate: @@ -69,6 +69,7 @@ jobs: GH_TOKEN: '' outputs: artifact_id: ${{ steps.snapshot.outputs.artifact-id }} + inventory_artifact_id: ${{ steps.inventory.outputs.artifact-id }} steps: - name: Checkout event commit uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -80,6 +81,7 @@ jobs: with: java-version-file: .github/workflows/.java-version distribution: 'temurin' + cache: maven - name: Verify checkout and current master env: EXPECTED_SHA: ${{ github.sha }} @@ -94,7 +96,14 @@ jobs: -DgraphFormat=json -DoutputFileName=maven-dependency-submission-action-depgraph.json \ -Dmaven.test.skip=true -Dmaven.javadoc.skip=true python3 .github/scripts/dependency-snapshot.py "$RUNNER_TEMP/snapshot.json" - - name: Refuse a stale generated snapshot + - name: Resolve independent dependency inventory + run: | + mvn -B -ntp org.apache.maven.plugins:maven-dependency-plugin:3.8.1:tree \ + -DoutputType=json -DoutputFile=target/dependency-inventory.json \ + -Dverbose=false -DappendOutput=false \ + -Dmaven.test.skip=true -Dmaven.javadoc.skip=true + python3 .github/scripts/dependency-inventory.py "$RUNNER_TEMP/inventory.json" + - name: Refuse stale dependency reports env: EXPECTED_SHA: ${{ github.sha }} run: test "$(git ls-remote origin refs/heads/master | cut -f1)" = "$EXPECTED_SHA" @@ -107,54 +116,6 @@ jobs: if-no-files-found: error retention-days: 1 - inventory: - needs: validate - if: >- - ((github.repository == 'apache/hugegraph' && github.event.repository.fork == false) || - github.repository == 'hugegraph/hugegraph') && - github.ref == 'refs/heads/master' && - (github.event_name == 'push' || github.event_name == 'workflow_dispatch') - runs-on: ubuntu-latest - timeout-minutes: 60 - permissions: - contents: read - env: - PYTHONDONTWRITEBYTECODE: '1' - INPUT_TOKEN: '' - GITHUB_TOKEN: '' - GH_TOKEN: '' - outputs: - artifact_id: ${{ steps.inventory.outputs.artifact-id }} - steps: - - name: Checkout event commit - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: ${{ github.sha }} - persist-credentials: false - - name: Set up project JDK - uses: actions/setup-java@de7274f081f381c8f8158605e0321c36c376e2e6 # v6.0.1 - with: - java-version-file: .github/workflows/.java-version - distribution: 'temurin' - - name: Verify checkout and current master - env: - EXPECTED_SHA: ${{ github.sha }} - run: | - test "$(git rev-parse HEAD)" = "$EXPECTED_SHA" - test "$(git ls-remote origin refs/heads/master | cut -f1)" = "$EXPECTED_SHA" - - name: Install the complete Maven reactor - run: mvn -B -ntp install -Dmaven.test.skip=true -Dmaven.javadoc.skip=true --fail-at-end - - name: Resolve independent dependency inventory - run: | - mvn -B -ntp org.apache.maven.plugins:maven-dependency-plugin:3.8.1:tree \ - -DoutputType=json -DoutputFile=target/dependency-inventory.json \ - -Dverbose=false -DappendOutput=false \ - -Dmaven.test.skip=true -Dmaven.javadoc.skip=true - python3 .github/scripts/dependency-inventory.py "$RUNNER_TEMP/inventory.json" - - name: Refuse a stale inventory - env: - EXPECTED_SHA: ${{ github.sha }} - run: test "$(git ls-remote origin refs/heads/master | cut -f1)" = "$EXPECTED_SHA" - name: Upload inventory data only id: inventory uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 @@ -165,10 +126,9 @@ jobs: retention-days: 1 submit: - needs: [validate, generate, inventory] + needs: [validate, generate] if: >- needs.validate.result == 'success' && needs.generate.result == 'success' && - needs.inventory.result == 'success' && ((github.repository == 'apache/hugegraph' && github.event.repository.fork == false) || github.repository == 'hugegraph/hugegraph') && github.ref == 'refs/heads/master' && @@ -187,7 +147,7 @@ jobs: RUN_ID: ${{ github.run_id }} RUN_ATTEMPT: ${{ github.run_attempt }} ARTIFACT_ID: ${{ needs.generate.outputs.artifact_id }} - INVENTORY_ARTIFACT_ID: ${{ needs.inventory.outputs.artifact_id }} + INVENTORY_ARTIFACT_ID: ${{ needs.generate.outputs.inventory_artifact_id }} run: | python3 -I - <<'PY' import datetime @@ -299,11 +259,10 @@ jobs: '.github/workflows/submit-dependencies.yml@master'], 'Run identity') jobs = api(repository, f'actions/runs/{run_id}/attempts/{attempt}/jobs?per_page=100') require(jobs['total_count'] <= 100, 'Too many jobs') - for name in ['generate', 'inventory']: - producers = [job for job in jobs['jobs'] if job['name'] == name] - require(len(producers) == 1 and producers[0]['conclusion'] == 'success' and - producers[0]['status'] == 'completed' and producers[0]['run_id'] == int(run_id) and - producers[0]['head_sha'] == sha, 'Producer did not succeed for this commit') + producers = [job for job in jobs['jobs'] if job['name'] == 'generate'] + require(len(producers) == 1 and producers[0]['conclusion'] == 'success' and + producers[0]['status'] == 'completed' and producers[0]['run_id'] == int(run_id) and + producers[0]['head_sha'] == sha, 'Producer did not succeed for this commit') inventory_id = os.environ['INVENTORY_ARTIFACT_ID'] require(re.fullmatch('[1-9][0-9]*', inventory_id) and inventory_id != artifact_id, 'Inventory ID') diff --git a/docs/ci.md b/docs/ci.md index 49b471c4c5..bf9306ef6d 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -146,7 +146,8 @@ it does not guarantee a specific queue delay. ## Dependency baseline The separate [dependency submission workflow](../.github/workflows/submit-dependencies.yml) -installs the complete reactor with Java 17 in a `contents: read` job, then generates +installs the complete reactor once with Java 17 in a `contents: read` job, reusing +`actions/setup-java`'s Maven cache, then generates each module's resolved graph with Ferstl 4.0.3. The repository's [stdlib serializer](../.github/scripts/dependency-snapshot.py) preserves the [official Maven parser 6.0.1](https://github.com/actions/maven-dependency-submission-action/tree/1f666f7ac2ab7b72cd5e22eaf7f23c8a581db1db) @@ -158,8 +159,9 @@ dispatches. Only `apache/hugegraph` (not a fork) and the explicitly allowed `hugegraph/hugegraph` test fork on `refs/heads/master` can run the submission job, which alone has `contents: write`. PRs, other repositories and other refs cannot run the writer. Maven inherits no submission token, and checkout does not persist -credentials. An independent `contents: read` job resolves all module dependency -scopes with Apache Maven Dependency Plugin 3.8.1 `tree` JSON. Its +credentials. The same read-only job separately resolves all module dependency +scopes with Apache Maven Dependency Plugin 3.8.1 `tree` JSON, sharing the installed +reactor while keeping the two resolvers and JSON reports independent. Its [stdlib inventory adapter](../.github/scripts/dependency-inventory.py) records each POM's package URLs, scopes and direct/indirect relationships independently of the Ferstl parser. The writer requires exact equality with that inventory, including empty @@ -169,7 +171,7 @@ not replace this per-module check. A pinned `actions/upload-artifact` transfers `snapshot.json` and `inventory.json` as separate artifacts. The fresh writer runner uses fixed inline stdlib Python to -verify both successful producer jobs and both artifact/run/attempt/SHA identities +verify the successful producer job and both artifact/run/attempt/SHA identities before downloading. It bounds ZIP and JSON sizes, checks each archive digest, and reads only its one expected regular JSON member in memory. Duplicate members, paths and symlinks fail; no archive members are extracted to disk. A signed storage @@ -179,7 +181,11 @@ downloaded code. The first-party upload action is permitted by the [ASF Actions policy](https://infra.apache.org/github-actions-policy.html). Checkout and snapshot use the event commit SHA. Master is checked before install -and again after generation in both read-only jobs; a stale run fails. The writer +and again after both reports are generated; a stale run fails. A newer master +run cancels the older master run because only the current master can submit. +Concurrency groups include the ref, so manual runs on other refs cannot cancel master. Every master +push still starts a baseline run without path filters, so unrelated changes also +can receive an exact-SHA baseline. The writer checks the run attempt, successful producer, artifact ID/name, JSON schema, detector/correlator and source SHA/ref. It verifies all manifest paths against the event commit's source tree, rejects empty or mismatched inventory, and checks master @@ -193,8 +199,16 @@ writer rejects because both reports must be produced in the same run attempt. If a PR `dependency-review` failure reports 0 base snapshots, rerun that job only once the same repository's **Dependency baseline** run has succeeded for the event's exact `pull_request.base.sha` (not the branch's merge base or a newer master tip). -If that SHA has no successful baseline, including a skipped run, rerunning the -review alone cannot supply it. +If that SHA has no successful baseline, including a skipped or superseded run, +rerunning the review alone cannot supply it. The producer supports only the current +master tip; cancelling an older run does not backfill its earlier PR events. + +To recover, wait until the current master has a successful baseline in the same +repository. Normally merge that master into the PR branch (or use Update branch) +and push the update to create a new PR event. Confirm that the new event's +`pull_request.base.sha` is the SHA with the successful baseline, then use that +new dependency-review run. Rerunning the old job retains its old event payload +and cannot move its base SHA to the submitted baseline. A fixed master-only correlator keeps successive baseline snapshots in the same stream within each repository; ASF and org inventories are independent. The writer's From bf4aa3fad410c104ec19ddd634b59a7767cae84d Mon Sep 17 00:00:00 2001 From: imbajin Date: Sun, 18 Oct 2026 05:11:56 +0800 Subject: [PATCH 11/14] refactor(ci): consolidate runners and builds - share dependency inventory and baseline reactor builds - fold Helm, Struct and RocksDB checks into existing jobs - retain failure checks, Javadoc and isolated snapshot writes - reserve merged-job budgets and cancel stale master runs - document runner reductions and focused manual validation --- .github/actions/helm-checks/action.yml | 458 ++++++++++++++++++ .../actions/rocksdb-compatibility/action.yml | 73 +++ .github/scripts/ci-policy.py | 3 + .github/scripts/test_ci_policy.py | 126 ++++- .github/scripts/test_dependency_snapshot.py | 38 +- .github/workflows/check-dependencies.yml | 105 +++- .github/workflows/helm-chart-ci.yml | 428 +--------------- .github/workflows/pd-store-ci.yml | 109 ++--- .github/workflows/server-ci.yml | 42 +- .github/workflows/submit-dependencies.yml | 124 +---- docs/ci.md | 70 ++- 11 files changed, 916 insertions(+), 660 deletions(-) create mode 100644 .github/actions/helm-checks/action.yml create mode 100644 .github/actions/rocksdb-compatibility/action.yml diff --git a/.github/actions/helm-checks/action.yml b/.github/actions/helm-checks/action.yml new file mode 100644 index 0000000000..937b447c53 --- /dev/null +++ b/.github/actions/helm-checks/action.yml @@ -0,0 +1,458 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: Helm checks +description: Lint, render, validate and package the HugeGraph Helm chart +runs: + using: composite + steps: + # azure/setup-helm is not on the ASF-approved actions allowlist, which + # fails the workflow at startup; install the pinned release directly. + - name: install helm + shell: 'bash -e {0}' + run: | + curl -fsSLo /tmp/helm.tgz https://get.helm.sh/helm-v3.16.2-linux-amd64.tar.gz + echo "9318379b847e333460d33d291d4c088156299a26cd93d570a7f5d0c36e50b5bb /tmp/helm.tgz" | sha256sum -c - + tar -xzf /tmp/helm.tgz -C /tmp + sudo install -m 0755 /tmp/linux-amd64/helm /usr/local/bin/helm + helm version + + - name: helm lint + shell: 'bash -e {0}' + run: | + helm lint helm/hugegraph + helm lint helm/hugegraph -f helm/hugegraph/values-single.yaml + helm lint helm/hugegraph -f helm/hugegraph/values-cluster.yaml + + - name: packaged chart regression + shell: 'bash -e {0}' + run: | + python3 -m unittest discover -s helm -p 'test_helm_package.py' -v + + - name: helm unittest + shell: 'bash -e {0}' + run: | + # v1.0.0 is the newest helm-unittest whose plugin.yaml Helm 3.16.2 can + # parse (v1.1.x adds a platformHooks field that 3.16.2 rejects). + # Pinned by the tag's commit so a moved tag cannot change what runs. + helm plugin install https://github.com/helm-unittest/helm-unittest.git \ + --version ddd5feaa9465e7a3591792524ce7f441d4c5157e + helm unittest helm/hugegraph + + - name: helm template + shell: 'bash -e {0}' + run: | + for preset in "" "-f helm/hugegraph/values-single.yaml" "-f helm/hugegraph/values-cluster.yaml"; do + # shellcheck disable=SC2086 # $preset intentionally splits into flags + helm template ci helm/hugegraph $preset > /dev/null + done + # Positive coverage for every hubble wrapper branch: pd mode with + # auth, direct mode with a custom port, and a TLS-less ingress with + # the explicit opt-in. + helm template ci helm/hugegraph \ + --set hubble.enabled=true \ + --set server.auth.enabled=true \ + --set server.auth.admin.existingSecret=ci-auth > /dev/null + helm template ci helm/hugegraph \ + --set hubble.enabled=true \ + --set hubble.allowWithoutServerAuth=true \ + --set hubble.mode=direct \ + --set hubble.port=9090 \ + --set hubble.persistence.enabled=true \ + --set hubble.ingress.enabled=true \ + --set hubble.ingress.allowPlainHttp=true > /dev/null + # PD sharding knobs rendered as -D system properties, including + # the values-file shape where the count arrives as a numeric string + helm template ci helm/hugegraph \ + --set pd.partition.defaultShardCount=3 \ + --set pd.partition.storeMaxShardCount=12 > /dev/null + helm template ci helm/hugegraph \ + --set-string pd.partition.defaultShardCount=3 > /dev/null + # The derived shard count and the raft whitelist and RPC-timeout flags must land in + # the PD JAVA_OPTS verbatim: shard count 3 on the default topology, + # 1 on the single-node preset, whitelist disabled in both. + helm template ci helm/hugegraph \ + | grep -qF 'value: "-Dpartition.default-shard-count=3 -Draft.ip-whitelist.enabled=false -Draft.rpc-timeout=3000"' + helm template ci helm/hugegraph -f helm/hugegraph/values-single.yaml \ + | grep -qF 'value: "-Dpartition.default-shard-count=1 -Draft.ip-whitelist.enabled=false -Draft.rpc-timeout=3000"' + # The stock distributed install must put every Server replica on + # the shared PD graph catalog, even without Hubble. Auth is on by + # default, so the chart-managed admin Secret must render too. + DEFAULT=$(helm template ci helm/hugegraph) + grep -qF "printf 'usePD=true\\n'" <<<"$DEFAULT" + grep -qF "printf 'pd.peers=%s\\n' \"\${HG_SERVER_PD_PEERS}\"" \ + <<<"$DEFAULT" + grep -qF 'value: "ci-hugegraph-pd-0.ci-hugegraph-pd.default.svc:8686,ci-hugegraph-pd-1.ci-hugegraph-pd.default.svc:8686,ci-hugegraph-pd-2.ci-hugegraph-pd.default.svc:8686"' \ + <<<"$DEFAULT" + grep -qE '^kind: Secret$' <<<"$DEFAULT" + grep -qF 'name: ci-admin' <<<"$DEFAULT" + grep -qF 'name: ci-auth-token' <<<"$DEFAULT" + # A YAML-coercible Secret key must render quoted, or the API + # server stores the key as a boolean and the pods cannot find it. + helm template ci helm/hugegraph --set-string server.auth.admin.key=on \ + | grep -qF '"on":' + # The install render must hash the generated credentials it writes: + # the rollout checksums equal sha256(name|key|sha256(value)) over + # the rendered Secret data, so the first upgrade, which reads the + # same values back through lookup, renders the same checksums and + # rolls nothing. + python3 - <<'PY' + import hashlib, base64, subprocess, yaml + out = subprocess.check_output(['helm', 'template', 'ci', 'helm/hugegraph']) + docs = [d for d in yaml.safe_load_all(out) if d] + h = lambda s: hashlib.sha256(s.encode()).hexdigest() + sec = {d['metadata']['name']: d['data'] for d in docs if d['kind'] == 'Secret'} + val = lambda n, k: base64.b64decode(sec[n][k]).decode() + pd = next(d for d in docs if d['kind'] == 'StatefulSet' and d['metadata']['name'] == 'ci-hugegraph-pd') + srv = next(d for d in docs if d['kind'] == 'Deployment' and d['metadata']['name'] == 'ci-hugegraph-server') + ann = lambda d, k: d['spec']['template']['metadata']['annotations'][k] + assert ann(pd, 'checksum/pd-auth') == h('ci-pd-auth|secret-key|' + h(val('ci-pd-auth', 'secret-key'))) + assert ann(srv, 'checksum/pd-auth') == ann(pd, 'checksum/pd-auth') + assert ann(srv, 'checksum/auth') == h('ci-admin|password|ci-auth-token|token_secret|' + + h(val('ci-admin', 'password')) + '|' + h(val('ci-auth-token', 'token_secret'))) + print('install-time checksums match the generated Secret values') + PY + + - name: reject invalid values + shell: 'bash -e {0}' + run: | + # Each case must fail to render; the schema and helpers are the + # contract. `! cmd` alone is NOT enforced under `set -e` (errexit + # ignores inverted commands), so every case checks explicitly. + must_fail() { + if helm template ci helm/hugegraph "$@" >/dev/null 2>&1; then + echo "expected render failure for: $*" >&2 + exit 1 + fi + } + # Kubernetes permits successThreshold > 1 only for readiness. + for component in pd store server hubble; do + for probe in startup liveness; do + must_fail --set "${component}.probes.${probe}.successThreshold=2" + done + helm template ci helm/hugegraph \ + --set "${component}.probes.readiness.successThreshold=2" > /dev/null + done + must_fail --set pd.replicas=100 + must_fail --set pd.pdb.minAvailable=3 + must_fail --set server.hpa.enabled=true + must_fail \ + --set server.hpa.enabled=true \ + --set server.hpa.minReplicas=2 \ + --set server.resources.requests.cpu=100m \ + --set server.pdb.enabled=true \ + --set server.pdb.minAvailable=2 + must_fail --set server.auth.enabled=true --set server.auth.admin.autoGenerate=false --set server.auth.token.autoGenerate=false + # pd.auth with every source disabled fails the schema anyOf; the + # unit framework at v1.0.0 cannot match schema aborts, so the case + # lives here. + must_fail --set pd.auth.autoGenerate=false + # Commons Configuration trims a properties value, so a credential + # with surrounding whitespace would be stored in the Secret and + # applied to the account without it. The schema rejects both ends; + # these are schema aborts, so they live here rather than in the + # unit suite. --set-string keeps the value a string. + must_fail --set-string "server.auth.admin.password=review-secret " + must_fail --set-string "$(printf 'server.auth.admin.password=review-secret\t')" + must_fail --set-string "pd.auth.value=review-secret " + # An inner space in the admin password fails too: the image + # entrypoint stores it with a backslash before each space, so the + # account would answer 401 to the configured value. The PD secret + # is not written by that encoder, so its inner space stays legal. + must_fail --set-string "server.auth.admin.password=review secret" + helm template ci helm/hugegraph \ + --set-string "pd.auth.value=review secret" >/dev/null + # The Server decodes Basic-auth credentials as ASCII, so a non-ASCII + # admin password would be applied and then answer 401 to every + # request; the schema refuses it before install, as it already + # does for the PD REST secret. Backslashes and tabs go the same way. + must_fail --set-string "server.auth.admin.password=密码abc" + must_fail --set-string "server.auth.admin.password=café" + # --set unescapes a single backslash, so the literal needs two. + must_fail --set-string 'server.auth.admin.password=review\\secret' + must_fail --set-string "$(printf 'server.auth.admin.password=review\tsecret')" + must_fail --set-string "pd.auth.value=密码abc" + # The Server splits the decoded credential on every colon (400). + must_fail --set-string 'server.auth.admin.password=review:secret' + helm template ci helm/hugegraph \ + --set-string "server.auth.admin.password=review-secret_42!" >/dev/null + # Auth defaults to on, so Hubble alone is valid; refuse Hubble only + # when authentication is explicitly disabled. + must_fail --set hubble.enabled=true --set server.auth.enabled=false + A=(--set hubble.enabled=true --set hubble.allowWithoutServerAuth=true) + must_fail "${A[@]}" --set hubble.port=0 + must_fail "${A[@]}" \ + --set hubble.persistence.enabled=true \ + --set hubble.persistence.size="" + must_fail "${A[@]}" --set hubble.service.nodePort=30080 + # A NodePort or LoadBalancer Hubble Service publishes the plain-HTTP + # UI directly; it needs the same explicit acknowledgement as PD. + must_fail "${A[@]}" --set hubble.service.type=NodePort + must_fail "${A[@]}" --set hubble.service.type=LoadBalancer + helm template ci helm/hugegraph "${A[@]}" \ + --set hubble.service.type=NodePort \ + --set hubble.service.allowInsecureExposure=true > /dev/null + must_fail "${A[@]}" --set hubble.mode=bogus + must_fail "${A[@]}" --set hubble.image.tag="" + must_fail "${A[@]}" --set hubble.ingress.enabled=true + # A NodePort or LoadBalancer Server Service publishes the plain-HTTP + # API, credentials and JWTs included; same acknowledgement. + must_fail --set server.service.type=NodePort + must_fail --set server.service.type=LoadBalancer + helm template ci helm/hugegraph \ + --set server.service.type=LoadBalancer \ + --set server.service.allowInsecureExposure=true > /dev/null + # A TLS-less Server Ingress needs the explicit plain-HTTP opt-in + must_fail --set server.ingress.enabled=true + helm template ci helm/hugegraph \ + --set server.ingress.enabled=true \ + --set server.ingress.allowPlainHttp=true > /dev/null + # Kubernetes refuses a StatefulSet that carries rollingUpdate + # options with updateStrategy OnDelete; the schema rejects the + # combination up front (a schema abort, so it lives here). + for component in pd store; do + must_fail \ + --set "${component}.updateStrategy.type=OnDelete" \ + --set "${component}.updateStrategy.rollingUpdate.partition=1" + helm template ci helm/hugegraph \ + --set "${component}.updateStrategy.type=OnDelete" > /dev/null + done + # maxUnavailable above 1 lets the controller disrupt two members + # of a three-member raft group or shard at once, and PDBs do not + # constrain controller rollouts; the schema accepts integer 1 only. + for component in pd store; do + must_fail --set "${component}.updateStrategy.rollingUpdate.maxUnavailable=2" + must_fail --set "${component}.updateStrategy.rollingUpdate.maxUnavailable=0" + must_fail --set "${component}.updateStrategy.rollingUpdate.maxUnavailable=-1" + must_fail --set-string "${component}.updateStrategy.rollingUpdate.maxUnavailable=67%" + must_fail --set-string "${component}.updateStrategy.rollingUpdate.maxUnavailable=foo" + must_fail --set-string "${component}.updateStrategy.rollingUpdate.maxUnavailable=1" + helm template ci helm/hugegraph \ + --set "${component}.updateStrategy.rollingUpdate.maxUnavailable=1" > /dev/null + done + # PD PDB must keep the Raft majority: floor(replicas/2)+1 + must_fail --set pd.replicas=5 --set pd.pdb.minAvailable=2 + must_fail --set pd.replicas=4 --set pd.pdb.minAvailable=2 + helm template ci helm/hugegraph \ + --set pd.replicas=5 --set pd.pdb.minAvailable=3 > /dev/null + helm template ci helm/hugegraph \ + --set pd.replicas=5 --set pd.pdb.minAvailable=4 > /dev/null + # PD sharding knobs: empty or a positive integer; an explicit + # shard count must be odd and stay within the store count + must_fail --set pd.partition.defaultShardCount=0 + must_fail --set pd.partition.defaultShardCount=-1 + must_fail --set-string pd.partition.defaultShardCount=abc + must_fail --set pd.partition.defaultShardCount=2 + must_fail --set pd.partition.defaultShardCount=5 + must_fail --set pd.partition.storeMaxShardCount=0 + # extraEnv must not override chart-managed variables + must_fail --set 'server.extraEnv[0].name=HG_SERVER_INIT_STORE_ENABLED' \ + --set 'server.extraEnv[0].value=true' + must_fail --set 'pd.extraEnv[0].name=HG_PD_RAFT_PEERS_LIST' \ + --set 'pd.extraEnv[0].value=x' + must_fail --set 'store.extraEnv[0].name=HG_STORE_PD_ADDRESS' \ + --set 'store.extraEnv[0].value=x' + # JAVA_OPTIONS is reserved: a preset value makes the start scripts + # skip auto heap sizing and drop the chart's JAVA_OPTS entirely + must_fail --set 'pd.extraEnv[0].name=JAVA_OPTIONS' \ + --set 'pd.extraEnv[0].value=-Xmx1g' + + - name: server credential wrapper behaviour + shell: bash + run: | + # An existingSecret is only seen when the container starts, so run + # the rendered Server wrapper against Secret values. A valid value + # passes the checks and stops at the missing properties file; an + # unsupported one stops earlier with its own message. + set -euo pipefail + helm template ci helm/hugegraph -s templates/server-deployment.yaml \ + --set server.auth.admin.existingSecret=ci-byo > /tmp/srv.yaml + python3 - <<'PY' + import yaml + d = yaml.safe_load(open('/tmp/srv.yaml')) + open('/tmp/srv.sh', 'w').write(d['spec']['template']['spec']['containers'][0]['args'][0]) + PY + mkdir -p /tmp/srv-run + run() { + (cd /tmp/srv-run && PASSWORD="$1" bash /tmp/srv.sh 2>&1) || true + } + expect() { + local out + out=$(run "$1") + if ! grep -qF -- "$2" <<<"$out"; then + printf 'password %q: expected "%s", got:\n%s\n' "$1" "$2" "$out" >&2 + exit 1 + fi + } + expect 'review-secret_42!' 'missing or unreadable ./conf/rest-server.properties' + expect 'ASCII inner space 123!?' 'must not contain spaces' + expect ' leading' 'must not contain spaces' + expect 'trailing ' 'must not contain spaces' + expect 'review:secret' 'must not contain a colon' + expect 'review\secret' 'must not contain backslashes' + expect "$(printf 'review\tsecret')" 'must be printable ASCII' + expect 'café' 'must be printable ASCII' + + - name: helm test hook behaviour + shell: bash + run: | + # The hook must fail when any one Server Pod cannot answer a + # graph-bound Gremlin query, even while /versions and /graphs + # answer 200 through the Service: a Server that started during a + # PD roll stays in exactly that state for its whole life. Run the + # rendered hook script against loopback fixtures. + set -euo pipefail + sudo sh -c 'echo "127.0.0.1 ci-hugegraph-server ci-hugegraph-server-headless" >> /etc/hosts' + helm template ci helm/hugegraph -s templates/tests/test-connection.yaml > /tmp/hook.yaml + python3 - <<'PY' + import yaml + d = yaml.safe_load(open('/tmp/hook.yaml')) + open('/tmp/hook.sh', 'w').write(d['spec']['containers'][0]['command'][2]) + PY + grep -q 'DEFAULT-hugegraph' /tmp/hook.sh + # getent stub: the headless lookup answers with the fixture IPs. + mkdir -p /tmp/stub + printf '#!/bin/sh\nfor ip in ${STUB_IPS}; do printf "%%s STREAM host\\n" "$ip"; done\n' > /tmp/stub/getent + chmod +x /tmp/stub/getent + cat > /tmp/fixture.py <<'PY' + import http.server, os, sys, threading, time + calls = {} + class H(http.server.BaseHTTPRequestHandler): + def _send(self, code, body): + self.send_response(code) + self.send_header('Content-Type', 'application/json') + self.send_header('Content-Length', str(len(body))) + self.end_headers() + self.wfile.write(body) + def do_GET(self): + self._send(200, b'{"versions":{}}') + def do_POST(self): + self.rfile.read(int(self.headers.get('Content-Length', 0))) + ip = self.server.server_address[0] + # broken-: every Gremlin call fails. flip-odd- / + # flip-even-: the odd or even calls fail, counted from + # the first call made while the file exists, so two Pods + # on opposite phases are never healthy in the same sweep. + odd = os.path.exists('/tmp/flip-odd-' + ip) + even = os.path.exists('/tmp/flip-even-' + ip) + n = calls[ip] = (calls.get(ip, 0) + 1) if (odd or even) else 0 + broken = (os.path.exists('/tmp/broken-' + ip) + or (odd and n % 2 == 1) + or (even and n % 2 == 0)) + if broken: + self._send(400, b'{"exception":"Could not rebind [graph]"}') + else: + self._send(200, b'{"result":{"data":[0]}}') + def log_message(self, *a): + pass + for ip in sys.argv[1:]: + s = http.server.HTTPServer((ip, 8080), H) + threading.Thread(target=s.serve_forever, daemon=True).start() + time.sleep(3600) + PY + python3 /tmp/fixture.py 127.0.0.1 127.0.0.2 127.0.0.3 & + sleep 1 + run_hook() { + PATH="/tmp/stub:${PATH}" PASSWORD=ci-password HG_TEST_DEADLINE_S=8 \ + STUB_IPS="$1" sh /tmp/hook.sh + } + # Healthy: three Pods, all Gremlin-bound. + run_hook "127.0.0.1 127.0.0.2 127.0.0.3" + # One Pod answers REST but not Gremlin: the hook must fail. + touch /tmp/broken-127.0.0.3 + if run_hook "127.0.0.1 127.0.0.2 127.0.0.3"; then + echo "expected the hook to fail on a broken-binding Pod" >&2 + exit 1 + fi + rm /tmp/broken-127.0.0.3 + # Fewer Pods than the replica floor: the hook must fail. + if run_hook "127.0.0.1"; then + echo "expected the hook to fail below the replica floor" >&2 + exit 1 + fi + # Alternating health: sweep 1 sees A ok, B broken; sweep 2 sees A + # broken, B ok; and so on. No single sweep is clean, so the hook + # must fail. A hook that remembered earlier successes per Pod would + # pass here after the second sweep. + touch /tmp/flip-even-127.0.0.1 /tmp/flip-odd-127.0.0.2 + if run_hook "127.0.0.1 127.0.0.2 127.0.0.3"; then + echo "expected the hook to fail when no sweep is completely healthy" >&2 + exit 1 + fi + rm /tmp/flip-even-127.0.0.1 /tmp/flip-odd-127.0.0.2 + # Dual-stack: each Pod publishes an IPv4 and an IPv6 address. Two + # Pods give four addresses, which must not satisfy a floor of + # three; three Pods must pass without the IPv6 addresses being + # queried (nothing listens on them, so a query would fail). + if run_hook "127.0.0.1 127.0.0.2 ::1 ::2"; then + echo "expected the hook to count Pods, not addresses, on dual-stack" >&2 + exit 1 + fi + run_hook "127.0.0.1 127.0.0.2 127.0.0.3 ::1 ::2 ::3" + # The auth-off render must carry no credentials. + if helm template ci helm/hugegraph -s templates/tests/test-connection.yaml \ + --set server.auth.enabled=false | grep -q 'Authorization'; then + echo "the auth-off hook render must not carry credentials" >&2 + exit 1 + fi + + - name: kubeconform + shell: bash + run: | + set -o pipefail + curl -sSLo /tmp/kc.tar.gz https://github.com/yannh/kubeconform/releases/download/v0.6.7/kubeconform-linux-amd64.tar.gz + echo "95f14e87aa28c09d5941f11bd024c1d02fdc0303ccaa23f61cef67bc92619d73 /tmp/kc.tar.gz" | sha256sum -c - + tar -xzf /tmp/kc.tar.gz -C /tmp + for preset in "" "-f helm/hugegraph/values-single.yaml" "-f helm/hugegraph/values-cluster.yaml"; do + # shellcheck disable=SC2086 # $preset intentionally splits into flags + helm template ci helm/hugegraph $preset | /tmp/kubeconform -strict -summary -kubernetes-version 1.23.0 + done + helm template ci helm/hugegraph \ + --set hubble.enabled=true \ + --set hubble.allowWithoutServerAuth=true \ + --set hubble.mode=direct \ + --set hubble.port=9090 \ + --set hubble.persistence.enabled=true \ + --set hubble.ingress.enabled=true \ + --set hubble.ingress.allowPlainHttp=true \ + | /tmp/kubeconform -strict -summary -kubernetes-version 1.23.0 + helm template ci helm/hugegraph \ + --set hubble.enabled=true \ + --set server.auth.enabled=true \ + --set server.auth.admin.existingSecret=ci-auth \ + | /tmp/kubeconform -strict -summary -kubernetes-version 1.23.0 + helm template ci helm/hugegraph \ + --set pd.partition.defaultShardCount=3 \ + --set pd.partition.storeMaxShardCount=12 \ + | /tmp/kubeconform -strict -summary -kubernetes-version 1.23.0 + + - name: legacy --reuse-values compatibility + shell: 'bash -e {0}' + run: | + # A release created before a field existed must still render. This has + # regressed five times, so it is guarded here rather than by review. + # `-f` is NOT equivalent: it merges over the new defaults, whereas + # --reuse-values discards them, so the fixture must become values.yaml. + cp -R helm/hugegraph /tmp/legacy + cp helm/hugegraph/testdata/values-pre-hardening.yaml /tmp/legacy/values.yaml + helm template legacy /tmp/legacy > /dev/null + helm template legacy /tmp/legacy --is-upgrade > /dev/null + + - name: helm package + shell: 'bash -e {0}' + run: | + helm package helm/hugegraph -d /tmp/chart diff --git a/.github/actions/rocksdb-compatibility/action.yml b/.github/actions/rocksdb-compatibility/action.yml new file mode 100644 index 0000000000..138dc76e46 --- /dev/null +++ b/.github/actions/rocksdb-compatibility/action.yml @@ -0,0 +1,73 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: RocksDB version compatibility +description: Compare exact before/after RocksDB versions on an existing distributed CI runner +runs: + using: composite + steps: + - name: Require exact comparison revisions + shell: bash + env: + BEFORE_SHA: ${{ github.event.pull_request.base.sha || github.event.before || github.sha }} + AFTER_SHA: ${{ github.event.pull_request.head.sha || github.event.after || github.sha }} + run: | + for revision in "$BEFORE_SHA" "$AFTER_SHA"; do + if [[ ! "$revision" =~ ^[0-9a-f]{40}$ || "$revision" =~ ^0+$ ]]; then + echo "Error: cannot determine the before/after revision for RocksDB compatibility" >&2 + exit 1 + fi + done + + - name: Checkout before + uses: actions/checkout@v7 + with: + ref: ${{ github.event.pull_request.base.sha || github.event.before || github.sha }} + path: .ci-rocksdb/before + persist-credentials: false + + - name: Checkout after + uses: actions/checkout@v7 + with: + repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} + ref: ${{ github.event.pull_request.head.sha || github.event.after || github.sha }} + path: .ci-rocksdb/after + persist-credentials: false + + - name: Install comparison JDK + uses: actions/setup-java@v6 + with: + java-version-file: .ci-rocksdb/after/.github/workflows/.java-version + distribution: 'zulu' + + - name: Test only changed RocksDB version pairs + shell: bash + run: | + bash .ci-rocksdb/after/hugegraph-server/hugegraph-test/src/test/rocksdb-compatibility/run.sh \ + "$GITHUB_WORKSPACE/.ci-rocksdb/before" "$GITHUB_WORKSPACE/.ci-rocksdb/after" \ + "$RUNNER_TEMP/rocksdb-compatibility" + + - name: Preserve comparison and synthetic-data failure evidence + if: always() + uses: actions/upload-artifact@v7 + with: + name: rocksdb-compatibility-${{ github.sha }} + path: | + ${{ runner.temp }}/rocksdb-compatibility/run.log + ${{ runner.temp }}/rocksdb-compatibility/models/ + ${{ runner.temp }}/rocksdb-compatibility/pairs.tsv + ${{ runner.temp }}/rocksdb-compatibility/cases/ diff --git a/.github/scripts/ci-policy.py b/.github/scripts/ci-policy.py index 33a1d4ea1e..adb5c27513 100644 --- a/.github/scripts/ci-policy.py +++ b/.github/scripts/ci-policy.py @@ -58,6 +58,8 @@ "toolchain": {"client": {"loader", "tools", "spark", "hubble"}, "loader": {"hubble"}}, } MAINTENANCE_INPUTS = { + ".github/actions/helm-checks/action.yml": {"helm"}, + ".github/actions/rocksdb-compatibility/action.yml": {"pd", "store", "hstore"}, ".asf.yaml": set(), ".editorconfig": set(), ".licenserc.yaml": set(), ".github/PULL_REQUEST_TEMPLATE.md": set(), ".github/dependabot.yml": set(), ".github/workflows/.java-version": MAINLINE | {"macos_full", "hbase", "riscv64"}, @@ -70,6 +72,7 @@ } PRESETS = { "mainline": MAINLINE, + "dependencies": {"dependency_license"}, "distributed": {"struct", "pd", "store", "hstore", "cluster", "distributed_smoke"}, "macos-arm": {"macos", "macos_full"}, "hbase": {"hbase"}, "riscv64": {"riscv64"}, "codeql": {"security"}, "full": set(MODULES["server"]), diff --git a/.github/scripts/test_ci_policy.py b/.github/scripts/test_ci_policy.py index c43130b904..76aea74dd9 100644 --- a/.github/scripts/test_ci_policy.py +++ b/.github/scripts/test_ci_policy.py @@ -328,6 +328,7 @@ def coordinator_jobs(self, plan, selection="success", java="success", result="su expression = re.sub(r"needs\.preflight\.outputs\.([\w]+)", lambda match: repr(outputs.get(match.group(1), "")), expression) expression = expression.replace("needs.preflight.result", repr(result)) + expression = expression.replace("needs.dependency_license.result", repr(result)) expression = expression.replace("!cancelled()", "not cancelled") expression = expression.replace("&&", "and").replace("||", "or") jobs[name] = bool(eval(expression, {"__builtins__": {}}, {"cancelled": cancelled})) @@ -338,7 +339,7 @@ def test_real_coordinator_keeps_selected_jobs_after_license_or_selftest_failure( preflight = re.search(r"^ preflight:\n(.*?)(?=^ [\w-]+:|\Z)", workflow, re.M | re.S).group(1) self.assertEqual("check-license", re.search(r"^ name: (.+)$", preflight, re.M).group(1)) full = self.plan_for([], "workflow_dispatch", {"inputs": {"test-set": "full"}}) - expected = {"build-server", "server-rocksdb", "commons", "pd_store", "cluster", "docker", "helm", + expected = {"build-server", "server-rocksdb", "commons", "pd_store", "cluster", "docker", "dependency_license", "compatibility", "codeql"} self.assertEqual(expected, self.coordinator_jobs(full)) self.assertEqual(expected, self.coordinator_jobs(full, result="failure")) @@ -356,7 +357,7 @@ def test_real_coordinator_blocks_selection_runtime_failure_and_cancellation(self with self.subTest(selection=state): self.assertEqual(set(), self.coordinator_jobs(full, selection=state, result="failure")) with self.subTest(java=state): - self.assertEqual({"helm"}, self.coordinator_jobs(full, java=state, result="failure")) + self.assertEqual(set(), self.coordinator_jobs(full, java=state, result="failure")) self.assertEqual(set(), self.coordinator_jobs(full, cancelled=True)) def test_contract_scripts_reach_their_actual_inner_jobs_and_steps(self): @@ -386,7 +387,7 @@ def test_contract_scripts_reach_their_actual_inner_jobs_and_steps(self): condition = re.search(r"^ if: (.+)$", struct, re.M).group(1) condition = condition.replace("inputs.run-struct", str(inputs["struct"])) condition = condition.replace("inputs.run-store", str(inputs["store"])) - self.assertTrue(eval(condition.replace("||", "or"), {"__builtins__": {}})) + self.assertTrue(eval(condition.replace("||", "or").replace("&&", "and").replace("!", "not "), {"__builtins__": {}})) step = next(step for step in struct.split("\n - ") if Path(script).name in step) self.assertNotRegex(step, r"(?m)^ if:") else: @@ -415,6 +416,125 @@ def test_contract_scripts_reach_their_actual_inner_jobs_and_steps(self): self.plan_for([runtime, policy.TRAVIS + "check-java-runtime-logs.sh"], "push", master)["selected"]) + @staticmethod + def workflow_expression(expression, values): + expression = expression.strip().removeprefix("${{").removesuffix("}}").strip() + expression = re.sub(r"\b(?:github|inputs|needs|steps)\.[\w.-]+", + lambda match: repr(values.get(match.group(), "")), expression) + expression = expression.replace("&&", "and").replace("||", "or") + expression = re.sub(r"!(?!=)", "not ", expression) + return eval(expression, {"__builtins__": {}}, {"true": True, "false": False, + "cancelled": lambda: False, "format": lambda pattern, value: pattern.format(value)}) + + def test_shared_dependency_install_preserves_inventory_build_arguments(self): + workflow = (Path(__file__).resolve().parents[1] / "workflows/check-dependencies.yml").read_text() + step = next(step for step in workflow.split("\n - ") if step.startswith("name: mvn install\n")) + code = textwrap.dedent(step.split(" run: |\n", 1)[1]) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + recorded = root / "arguments.jsonl" + maven = root / "mvn" + maven.write_text("#!/usr/bin/env python3\nimport json, os, sys\n" + "with open(os.environ['RECORDED_ARGS'], 'a') as output:\n" + " output.write(json.dumps(sys.argv[1:]) + '\\n')\n") + maven.chmod(0o755) + for inventory in [True, False]: + recorded.unlink(missing_ok=True) + subprocess.run(["bash", "-e", "-c", code], check=True, env={**os.environ, + "PATH": str(root) + os.pathsep + os.environ["PATH"], "RECORDED_ARGS": str(recorded), + "CHECK_INVENTORY": str(inventory).lower()}) + commands = [json.loads(line) for line in recorded.read_text().splitlines()] + with self.subTest(inventory=inventory): + self.assertEqual(1, len(commands)) + self.assertEqual("install", commands[0][0]) + self.assertIn("-Dmaven.test.skip=true", commands[0]) + self.assertEqual(not inventory, "-Dmaven.javadoc.skip=true" in commands[0]) + + def test_master_baseline_is_selected_only_for_trusted_full_runs(self): + workflow = (Path(__file__).resolve().parents[1] / "workflows/server-ci.yml").read_text() + expression = re.search(r"^ ENABLED: (.+)$", workflow, re.M).group(1) + values = {"github.repository": "apache/hugegraph", "github.event.repository.fork": False, + "github.ref": "refs/heads/master", "github.event_name": "push", "inputs.test-set": "mainline"} + for event in ["push", "workflow_dispatch"]: + with self.subTest(event=event): + self.assertTrue(self.workflow_expression(expression, dict(values, **{"github.event_name": event}))) + for field, value in [("github.event_name", "schedule"), ("github.event_name", "pull_request"), + ("github.event_name", "pull_request_target"), + ("github.ref", "refs/heads/release-1.7"), ("github.repository", "unknown/fork"), + ("github.event.repository.fork", True)]: + with self.subTest(field=field, value=value): + self.assertFalse(self.workflow_expression(expression, dict(values, **{field: value}))) + for preset in ["distributed", "codeql", "macos-arm"]: + self.assertFalse(self.workflow_expression(expression, dict(values, **{ + "github.event_name": "workflow_dispatch", "inputs.test-set": preset}))) + self.assertTrue(self.workflow_expression(expression, dict(values, **{ + "github.repository": "hugegraph/hugegraph", "github.event.repository.fork": True}))) + dependencies = self.plan_for([], "workflow_dispatch", {"inputs": {"test-set": "dependencies"}}) + self.assertEqual({"dependency_license"}, set(dependencies["selected"])) + self.assertTrue(self.workflow_expression(expression, dict(values, **{ + "github.event_name": "workflow_dispatch", "inputs.test-set": "dependencies"}))) + plan = self.plan_for(["README.md"], "push", {"ref": "refs/heads/master"}) + plan["dependency_baseline"] = True + self.assertEqual({"dependency_license", "dependency_baseline"}, self.coordinator_jobs(plan)) + self.assertEqual(set(), self.coordinator_jobs(plan, result="failure")) + + def test_new_master_push_replaces_queued_work_without_cancelling_manual_or_weekly_runs(self): + workflow = (Path(__file__).resolve().parents[1] / "workflows/server-ci.yml").read_text() + template = re.search(r"^ group: (.+)$", workflow, re.M).group(1) + cancellation = re.search(r"^ cancel-in-progress: (.+)$", workflow, re.M).group(1) + + def context(event, run, ref="refs/heads/master"): + return {"github.workflow": "HugeGraph-Server CI", "github.event_name": event, + "github.ref": ref, "github.run_id": run, "github.run_attempt": 1, + "github.event.pull_request.number": 42 if event == "pull_request" else ""} + + def group(values): + return re.sub(r"\$\{\{(.*?)\}\}", + lambda match: str(self.workflow_expression(match.group(1), values)), template) + + for event in ["push", "pull_request"]: + first, second = context(event, 100), context(event, 101) + self.assertEqual(group(first), group(second)) + self.assertTrue(self.workflow_expression(cancellation, first)) + for event in ["workflow_dispatch", "schedule"]: + first, second = context(event, 100), context(event, 101) + self.assertNotEqual(group(first), group(second)) + self.assertFalse(self.workflow_expression(cancellation, first)) + release = context("push", 100, "refs/heads/release-1.7") + self.assertFalse(self.workflow_expression(cancellation, release)) + self.assertNotEqual(group(release), group(context("push", 101, "refs/heads/release-1.7"))) + + def test_distributed_checks_keep_exactly_one_host_and_preserve_struct_fallback(self): + workflow = (Path(__file__).resolve().parents[1] / "workflows/pd-store-ci.yml").read_text() + jobs = dict(re.findall(r"^ ([\w-]+):\n(.*?)(?=^ [\w-]+:\n|\Z)", + workflow.split("\njobs:\n", 1)[1], re.M | re.S)) + for mask in range(16): + values = {"inputs.run-" + name: bool(mask & (1 << index)) + for index, name in enumerate(["struct", "pd", "store", "hstore"])} + active = {name: body for name, body in jobs.items() if self.workflow_expression( + re.search(r"^ if: (.+)$", body, re.M).group(1), values)} + hosts = [] + for name, body in active.items(): + for step in body.split("\n - "): + if "uses: ./.github/actions/rocksdb-compatibility" in step: + condition = re.search(r"^ if: (.+)$", step, re.M).group(1) + if self.workflow_expression(condition, values): + hosts.append(name) + with self.subTest(inputs=values): + expected = [name for name in ["pd", "store", "hstore"] if values["inputs.run-" + name]][:1] + self.assertEqual(expected, hosts) + self.assertEqual(values["inputs.run-struct"] and not values["inputs.run-store"], "struct" in active) + if values["inputs.run-struct"] or values["inputs.run-store"]: + host = active["store" if values["inputs.run-store"] else "struct"] + for script in ["test-stop-hugegraph-store.sh", "test-check-java-runtime-logs.sh", + "test-check-jacoco-report.sh", "test-codecov-upload-config.sh", + "test-tinkerpop-reports.sh"]: + self.assertIn(script, host) + if values["inputs.run-store"]: + step = next(step for step in host.split("\n - ") if script in step) + self.assertIn("if: ${{ !cancelled() }}", step) + self.assertIn("Run hugegraph-struct test", host) + def test_coordinator_reads_and_validates_real_java_input(self): workflow = (Path(__file__).resolve().parents[1] / "workflows/server-ci.yml").read_text() code = workflow.split("python3 - <<'PYTHON'\n", 1)[1].split("\n PYTHON", 1)[0] diff --git a/.github/scripts/test_dependency_snapshot.py b/.github/scripts/test_dependency_snapshot.py index bc1bcb7924..95e0bf820e 100644 --- a/.github/scripts/test_dependency_snapshot.py +++ b/.github/scripts/test_dependency_snapshot.py @@ -86,10 +86,10 @@ def setUp(self): self.responses = { 'actions/runs/321/attempts/2': {'id': 321, 'run_attempt': 2, 'head_sha': 'a' * 40, 'head_branch': 'master', 'event': 'push', 'repository': {'full_name': 'apache/hugegraph'}, - 'path': '.github/workflows/submit-dependencies.yml', + 'path': '.github/workflows/server-ci.yml', 'run_started_at': (now - datetime.timedelta(minutes=2)).isoformat()}, 'actions/runs/321/attempts/2/jobs?per_page=100': {'total_count': 3, - 'jobs': [{'name': 'generate', 'conclusion': 'success', 'status': 'completed', + 'jobs': [{'name': 'dependency_license / dependency-check', 'conclusion': 'success', 'status': 'completed', 'run_id': 321, 'head_sha': 'a' * 40}]}, 'actions/artifacts/789': {'id': 789, 'expired': False, 'name': 'dependency-snapshot-321-2', 'created_at': now.isoformat(), @@ -258,7 +258,7 @@ def test_writer_preserves_snapshot_and_never_executes_data(self): def test_writer_accepts_master_path_suffix_and_rejects_other_refs(self): run = self.responses['actions/runs/321/attempts/2'] - path = '.github/workflows/submit-dependencies.yml' + path = '.github/workflows/server-ci.yml' run['path'] = path + '@master' self.run_writer() self.assertEqual([self.snapshot], self.posts) @@ -298,7 +298,7 @@ def test_writer_rejects_failed_producer_wrong_artifact_and_stale_master(self): ('actions/artifacts/789', 'expired', True), ('actions/artifacts/789', 'workflow_run', {'id': 123}), ('actions/runs/321/attempts/2/jobs?per_page=100', 'jobs', - [{'name': 'generate', 'conclusion': 'failure'}]), + [{'name': 'dependency_license / dependency-check', 'conclusion': 'failure'}]), ('git/trees/' + 'a' * 40 + '?recursive=1', 'truncated', True), ('git/ref/heads/master', 'object', {'sha': 'b' * 40})]: original = copy.deepcopy(self.responses[path]) @@ -308,18 +308,24 @@ def test_writer_rejects_failed_producer_wrong_artifact_and_stale_master(self): self.responses[path] = original self.assertEqual([], self.posts) - def test_dependency_baseline_concurrency_isolates_nonmaster_dispatch(self): - workflow = (Path(__file__).resolve().parents[1] / "workflows/submit-dependencies.yml").read_text() - template = next(line.removeprefix(" group: ") for line in workflow.splitlines() - if line.startswith(" group: ")) - - def group(event, ref): - return template.replace("${{ github.repository }}", "apache/hugegraph").replace( - "${{ github.ref }}", ref).replace("${{ github.event_name }}", event) - - master = group("push", "refs/heads/master") - self.assertEqual(master, group("workflow_dispatch", "refs/heads/master")) - self.assertNotEqual(master, group("workflow_dispatch", "refs/heads/feature")) + def test_writer_accepts_trusted_coordinator_events_and_rejects_pull_requests(self): + run = self.responses['actions/runs/321/attempts/2'] + for event in ['push', 'workflow_dispatch']: + run['event'] = event + self.posts.clear() + self.run_writer() + self.assertEqual([self.snapshot], self.posts) + for field, value in [('event', 'schedule'), ('event', 'pull_request'), ('event', 'pull_request_target'), + ('path', '.github/workflows/submit-dependencies.yml')]: + original = copy.deepcopy(run) + run['event'] = 'push' + run[field] = value + self.posts.clear() + with self.subTest(field=field, value=value), self.assertRaisesRegex(ValueError, 'Run identity'): + self.run_writer() + self.assertEqual([], self.posts) + run.clear() + run.update(original) def test_writer_requires_one_successful_combined_producer(self): path = 'actions/runs/321/attempts/2/jobs?per_page=100' diff --git a/.github/workflows/check-dependencies.yml b/.github/workflows/check-dependencies.yml index 71da915301..1f7b693b20 100644 --- a/.github/workflows/check-dependencies.yml +++ b/.github/workflows/check-dependencies.yml @@ -3,61 +3,140 @@ name: "3rd-party" on: workflow_call: + inputs: + submit-baseline: + type: boolean + default: false + check-inventory: + type: boolean + default: true + outputs: + artifact_id: + value: ${{ jobs.dependency-check.outputs.artifact_id }} + inventory_artifact_id: + value: ${{ jobs.dependency-check.outputs.inventory_artifact_id }} permissions: contents: read jobs: dependency-check: + name: dependency-check runs-on: ubuntu-24.04 + # Preserve the existing inventory job budget; baseline-only builds keep their 60-minute limit. + timeout-minutes: ${{ inputs.check-inventory && 360 || 60 }} + outputs: + artifact_id: ${{ steps.snapshot.outputs.artifact-id }} + inventory_artifact_id: ${{ steps.inventory.outputs.artifact-id }} env: USE_STAGE: 'false' # Whether to include the stage repository. SCRIPT_DEPENDENCY: install-dist/scripts/dependency PYTHONDONTWRITEBYTECODE: '1' # Keep generated Python caches out of source style checks. + INPUT_TOKEN: '' + GITHUB_TOKEN: '' + GH_TOKEN: '' steps: - name: Checkout source - uses: actions/checkout@v7 + id: checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + persist-credentials: false - name: Set up project JDK - uses: actions/setup-java@v6 + uses: actions/setup-java@de7274f081f381c8f8158605e0321c36c376e2e6 # v6.0.1 with: java-version-file: .github/workflows/.java-version distribution: 'temurin' + cache: maven - name: use staged maven repo settings if: ${{ env.USE_STAGE == 'true' }} run: | - cp $HOME/.m2/settings.xml /tmp/settings.xml - mv -vf .github/configs/settings.xml $HOME/.m2/settings.xml + cp "$HOME/.m2/settings.xml" /tmp/settings.xml + mv -vf .github/configs/settings.xml "$HOME/.m2/settings.xml" - name: Test shared foundation checks + if: inputs.check-inventory run: python3 -m unittest discover -s install-dist/scripts/shared-foundation -p 'test_*.py' - name: Test dependency inventory producer and checker run: python3 -m unittest discover -s install-dist/scripts/dependency -p 'test_*.py' + - name: Verify baseline checkout and current master + if: inputs.submit-baseline + env: + EXPECTED_SHA: ${{ github.sha }} + run: | + test "$(git rev-parse HEAD)" = "$EXPECTED_SHA" + test "$(git ls-remote origin refs/heads/master | cut -f1)" = "$EXPECTED_SHA" + - name: mvn install + env: + CHECK_INVENTORY: ${{ inputs.check-inventory }} run: | - mvn install -Dmaven.test.skip=true -ntp --fail-at-end + args=(-Dmaven.test.skip=true -ntp --fail-at-end) + if [[ "$CHECK_INVENTORY" != 'true' ]]; then + args+=(-Dmaven.javadoc.skip=true) + fi + mvn install "${args[@]}" - name: Check shared foundation boundaries and shipped classes + if: inputs.check-inventory run: | REVISION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) python3 install-dist/scripts/shared-foundation/check_shared_foundation.py --revision "$REVISION" - name: generate current dependencies + if: inputs.check-inventory run: | - bash $SCRIPT_DEPENDENCY/regenerate_known_dependencies.sh "$PWD/target/dependency/current-dependencies.txt" + bash "$SCRIPT_DEPENDENCY/regenerate_known_dependencies.sh" "$PWD/target/dependency/current-dependencies.txt" - name: check third dependencies + if: inputs.check-inventory run: | - bash $SCRIPT_DEPENDENCY/check_dependencies.sh \ + bash "$SCRIPT_DEPENDENCY/check_dependencies.sh" \ "$SCRIPT_DEPENDENCY/known-dependencies.txt" "$PWD/target/dependency/current-dependencies.txt" - dependency-review: - if: github.event_name == 'pull_request' - runs-on: ubuntu-24.04 - steps: - - name: 'Checkout Repository' - uses: actions/checkout@v7 + - name: Generate resolved reactor snapshot + if: inputs.submit-baseline + run: | + mvn -B -ntp com.github.ferstl:depgraph-maven-plugin:4.0.3:graph \ + -DgraphFormat=json -DoutputFileName=maven-dependency-submission-action-depgraph.json \ + -Dmaven.test.skip=true -Dmaven.javadoc.skip=true + python3 .github/scripts/dependency-snapshot.py "$RUNNER_TEMP/snapshot.json" + - name: Resolve independent dependency inventory + if: inputs.submit-baseline + run: | + mvn -B -ntp org.apache.maven.plugins:maven-dependency-plugin:3.8.1:tree \ + -DoutputType=json -DoutputFile=target/dependency-inventory.json \ + -Dverbose=false -DappendOutput=false \ + -Dmaven.test.skip=true -Dmaven.javadoc.skip=true + python3 .github/scripts/dependency-inventory.py "$RUNNER_TEMP/inventory.json" + - name: Refuse stale dependency reports + if: inputs.submit-baseline + env: + EXPECTED_SHA: ${{ github.sha }} + run: test "$(git ls-remote origin refs/heads/master | cut -f1)" = "$EXPECTED_SHA" + - name: Upload snapshot data only + if: inputs.submit-baseline + id: snapshot + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: dependency-snapshot-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/snapshot.json + if-no-files-found: error + retention-days: 1 + + - name: Upload inventory data only + if: inputs.submit-baseline + id: inventory + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: dependency-inventory-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/inventory.json + if-no-files-found: error + retention-days: 1 + - name: 'Dependency Review' + if: ${{ !cancelled() && steps.checkout.outcome == 'success' && github.event_name == 'pull_request' }} uses: actions/dependency-review-action@v5 # Refer: https://github.com/actions/dependency-review-action with: diff --git a/.github/workflows/helm-chart-ci.yml b/.github/workflows/helm-chart-ci.yml index 6e425a3dca..8739c8ba20 100644 --- a/.github/workflows/helm-chart-ci.yml +++ b/.github/workflows/helm-chart-ci.yml @@ -31,429 +31,5 @@ jobs: with: persist-credentials: false - # azure/setup-helm is not on the ASF-approved actions allowlist, which - # fails the workflow at startup; install the pinned release directly. - - name: install helm - run: | - curl -fsSLo /tmp/helm.tgz https://get.helm.sh/helm-v3.16.2-linux-amd64.tar.gz - echo "9318379b847e333460d33d291d4c088156299a26cd93d570a7f5d0c36e50b5bb /tmp/helm.tgz" | sha256sum -c - - tar -xzf /tmp/helm.tgz -C /tmp - sudo install -m 0755 /tmp/linux-amd64/helm /usr/local/bin/helm - helm version - - - name: helm lint - run: | - helm lint helm/hugegraph - helm lint helm/hugegraph -f helm/hugegraph/values-single.yaml - helm lint helm/hugegraph -f helm/hugegraph/values-cluster.yaml - - - name: packaged chart regression - run: python3 -m unittest discover -s helm -p 'test_helm_package.py' -v - - - name: helm unittest - run: | - # v1.0.0 is the newest helm-unittest whose plugin.yaml Helm 3.16.2 can - # parse (v1.1.x adds a platformHooks field that 3.16.2 rejects). - # Pinned by the tag's commit so a moved tag cannot change what runs. - helm plugin install https://github.com/helm-unittest/helm-unittest.git \ - --version ddd5feaa9465e7a3591792524ce7f441d4c5157e - helm unittest helm/hugegraph - - - name: helm template - run: | - for preset in "" "-f helm/hugegraph/values-single.yaml" "-f helm/hugegraph/values-cluster.yaml"; do - # shellcheck disable=SC2086 # $preset intentionally splits into flags - helm template ci helm/hugegraph $preset > /dev/null - done - # Positive coverage for every hubble wrapper branch: pd mode with - # auth, direct mode with a custom port, and a TLS-less ingress with - # the explicit opt-in. - helm template ci helm/hugegraph \ - --set hubble.enabled=true \ - --set server.auth.enabled=true \ - --set server.auth.admin.existingSecret=ci-auth > /dev/null - helm template ci helm/hugegraph \ - --set hubble.enabled=true \ - --set hubble.allowWithoutServerAuth=true \ - --set hubble.mode=direct \ - --set hubble.port=9090 \ - --set hubble.persistence.enabled=true \ - --set hubble.ingress.enabled=true \ - --set hubble.ingress.allowPlainHttp=true > /dev/null - # PD sharding knobs rendered as -D system properties, including - # the values-file shape where the count arrives as a numeric string - helm template ci helm/hugegraph \ - --set pd.partition.defaultShardCount=3 \ - --set pd.partition.storeMaxShardCount=12 > /dev/null - helm template ci helm/hugegraph \ - --set-string pd.partition.defaultShardCount=3 > /dev/null - # The derived shard count and the raft whitelist and RPC-timeout flags must land in - # the PD JAVA_OPTS verbatim: shard count 3 on the default topology, - # 1 on the single-node preset, whitelist disabled in both. - helm template ci helm/hugegraph \ - | grep -qF 'value: "-Dpartition.default-shard-count=3 -Draft.ip-whitelist.enabled=false -Draft.rpc-timeout=3000"' - helm template ci helm/hugegraph -f helm/hugegraph/values-single.yaml \ - | grep -qF 'value: "-Dpartition.default-shard-count=1 -Draft.ip-whitelist.enabled=false -Draft.rpc-timeout=3000"' - # The stock distributed install must put every Server replica on - # the shared PD graph catalog, even without Hubble. Auth is on by - # default, so the chart-managed admin Secret must render too. - DEFAULT=$(helm template ci helm/hugegraph) - grep -qF "printf 'usePD=true\\n'" <<<"$DEFAULT" - grep -qF "printf 'pd.peers=%s\\n' \"\${HG_SERVER_PD_PEERS}\"" \ - <<<"$DEFAULT" - grep -qF 'value: "ci-hugegraph-pd-0.ci-hugegraph-pd.default.svc:8686,ci-hugegraph-pd-1.ci-hugegraph-pd.default.svc:8686,ci-hugegraph-pd-2.ci-hugegraph-pd.default.svc:8686"' \ - <<<"$DEFAULT" - grep -qE '^kind: Secret$' <<<"$DEFAULT" - grep -qF 'name: ci-admin' <<<"$DEFAULT" - grep -qF 'name: ci-auth-token' <<<"$DEFAULT" - # A YAML-coercible Secret key must render quoted, or the API - # server stores the key as a boolean and the pods cannot find it. - helm template ci helm/hugegraph --set-string server.auth.admin.key=on \ - | grep -qF '"on":' - # The install render must hash the generated credentials it writes: - # the rollout checksums equal sha256(name|key|sha256(value)) over - # the rendered Secret data, so the first upgrade, which reads the - # same values back through lookup, renders the same checksums and - # rolls nothing. - python3 - <<'PY' - import hashlib, base64, subprocess, yaml - out = subprocess.check_output(['helm', 'template', 'ci', 'helm/hugegraph']) - docs = [d for d in yaml.safe_load_all(out) if d] - h = lambda s: hashlib.sha256(s.encode()).hexdigest() - sec = {d['metadata']['name']: d['data'] for d in docs if d['kind'] == 'Secret'} - val = lambda n, k: base64.b64decode(sec[n][k]).decode() - pd = next(d for d in docs if d['kind'] == 'StatefulSet' and d['metadata']['name'] == 'ci-hugegraph-pd') - srv = next(d for d in docs if d['kind'] == 'Deployment' and d['metadata']['name'] == 'ci-hugegraph-server') - ann = lambda d, k: d['spec']['template']['metadata']['annotations'][k] - assert ann(pd, 'checksum/pd-auth') == h('ci-pd-auth|secret-key|' + h(val('ci-pd-auth', 'secret-key'))) - assert ann(srv, 'checksum/pd-auth') == ann(pd, 'checksum/pd-auth') - assert ann(srv, 'checksum/auth') == h('ci-admin|password|ci-auth-token|token_secret|' - + h(val('ci-admin', 'password')) + '|' + h(val('ci-auth-token', 'token_secret'))) - print('install-time checksums match the generated Secret values') - PY - - - name: reject invalid values - run: | - # Each case must fail to render; the schema and helpers are the - # contract. `! cmd` alone is NOT enforced under `set -e` (errexit - # ignores inverted commands), so every case checks explicitly. - must_fail() { - if helm template ci helm/hugegraph "$@" >/dev/null 2>&1; then - echo "expected render failure for: $*" >&2 - exit 1 - fi - } - # Kubernetes permits successThreshold > 1 only for readiness. - for component in pd store server hubble; do - for probe in startup liveness; do - must_fail --set "${component}.probes.${probe}.successThreshold=2" - done - helm template ci helm/hugegraph \ - --set "${component}.probes.readiness.successThreshold=2" > /dev/null - done - must_fail --set pd.replicas=100 - must_fail --set pd.pdb.minAvailable=3 - must_fail --set server.hpa.enabled=true - must_fail \ - --set server.hpa.enabled=true \ - --set server.hpa.minReplicas=2 \ - --set server.resources.requests.cpu=100m \ - --set server.pdb.enabled=true \ - --set server.pdb.minAvailable=2 - must_fail --set server.auth.enabled=true --set server.auth.admin.autoGenerate=false --set server.auth.token.autoGenerate=false - # pd.auth with every source disabled fails the schema anyOf; the - # unit framework at v1.0.0 cannot match schema aborts, so the case - # lives here. - must_fail --set pd.auth.autoGenerate=false - # Commons Configuration trims a properties value, so a credential - # with surrounding whitespace would be stored in the Secret and - # applied to the account without it. The schema rejects both ends; - # these are schema aborts, so they live here rather than in the - # unit suite. --set-string keeps the value a string. - must_fail --set-string "server.auth.admin.password=review-secret " - must_fail --set-string "$(printf 'server.auth.admin.password=review-secret\t')" - must_fail --set-string "pd.auth.value=review-secret " - # An inner space in the admin password fails too: the image - # entrypoint stores it with a backslash before each space, so the - # account would answer 401 to the configured value. The PD secret - # is not written by that encoder, so its inner space stays legal. - must_fail --set-string "server.auth.admin.password=review secret" - helm template ci helm/hugegraph \ - --set-string "pd.auth.value=review secret" >/dev/null - # The Server decodes Basic-auth credentials as ASCII, so a non-ASCII - # admin password would be applied and then answer 401 to every - # request; the schema refuses it before install, as it already - # does for the PD REST secret. Backslashes and tabs go the same way. - must_fail --set-string "server.auth.admin.password=密码abc" - must_fail --set-string "server.auth.admin.password=café" - # --set unescapes a single backslash, so the literal needs two. - must_fail --set-string 'server.auth.admin.password=review\\secret' - must_fail --set-string "$(printf 'server.auth.admin.password=review\tsecret')" - must_fail --set-string "pd.auth.value=密码abc" - # The Server splits the decoded credential on every colon (400). - must_fail --set-string 'server.auth.admin.password=review:secret' - helm template ci helm/hugegraph \ - --set-string "server.auth.admin.password=review-secret_42!" >/dev/null - # Auth defaults to on, so Hubble alone is valid; refuse Hubble only - # when authentication is explicitly disabled. - must_fail --set hubble.enabled=true --set server.auth.enabled=false - A=(--set hubble.enabled=true --set hubble.allowWithoutServerAuth=true) - must_fail "${A[@]}" --set hubble.port=0 - must_fail "${A[@]}" \ - --set hubble.persistence.enabled=true \ - --set hubble.persistence.size="" - must_fail "${A[@]}" --set hubble.service.nodePort=30080 - # A NodePort or LoadBalancer Hubble Service publishes the plain-HTTP - # UI directly; it needs the same explicit acknowledgement as PD. - must_fail "${A[@]}" --set hubble.service.type=NodePort - must_fail "${A[@]}" --set hubble.service.type=LoadBalancer - helm template ci helm/hugegraph "${A[@]}" \ - --set hubble.service.type=NodePort \ - --set hubble.service.allowInsecureExposure=true > /dev/null - must_fail "${A[@]}" --set hubble.mode=bogus - must_fail "${A[@]}" --set hubble.image.tag="" - must_fail "${A[@]}" --set hubble.ingress.enabled=true - # A NodePort or LoadBalancer Server Service publishes the plain-HTTP - # API, credentials and JWTs included; same acknowledgement. - must_fail --set server.service.type=NodePort - must_fail --set server.service.type=LoadBalancer - helm template ci helm/hugegraph \ - --set server.service.type=LoadBalancer \ - --set server.service.allowInsecureExposure=true > /dev/null - # A TLS-less Server Ingress needs the explicit plain-HTTP opt-in - must_fail --set server.ingress.enabled=true - helm template ci helm/hugegraph \ - --set server.ingress.enabled=true \ - --set server.ingress.allowPlainHttp=true > /dev/null - # Kubernetes refuses a StatefulSet that carries rollingUpdate - # options with updateStrategy OnDelete; the schema rejects the - # combination up front (a schema abort, so it lives here). - for component in pd store; do - must_fail \ - --set "${component}.updateStrategy.type=OnDelete" \ - --set "${component}.updateStrategy.rollingUpdate.partition=1" - helm template ci helm/hugegraph \ - --set "${component}.updateStrategy.type=OnDelete" > /dev/null - done - # maxUnavailable above 1 lets the controller disrupt two members - # of a three-member raft group or shard at once, and PDBs do not - # constrain controller rollouts; the schema accepts integer 1 only. - for component in pd store; do - must_fail --set "${component}.updateStrategy.rollingUpdate.maxUnavailable=2" - must_fail --set "${component}.updateStrategy.rollingUpdate.maxUnavailable=0" - must_fail --set "${component}.updateStrategy.rollingUpdate.maxUnavailable=-1" - must_fail --set-string "${component}.updateStrategy.rollingUpdate.maxUnavailable=67%" - must_fail --set-string "${component}.updateStrategy.rollingUpdate.maxUnavailable=foo" - must_fail --set-string "${component}.updateStrategy.rollingUpdate.maxUnavailable=1" - helm template ci helm/hugegraph \ - --set "${component}.updateStrategy.rollingUpdate.maxUnavailable=1" > /dev/null - done - # PD PDB must keep the Raft majority: floor(replicas/2)+1 - must_fail --set pd.replicas=5 --set pd.pdb.minAvailable=2 - must_fail --set pd.replicas=4 --set pd.pdb.minAvailable=2 - helm template ci helm/hugegraph \ - --set pd.replicas=5 --set pd.pdb.minAvailable=3 > /dev/null - helm template ci helm/hugegraph \ - --set pd.replicas=5 --set pd.pdb.minAvailable=4 > /dev/null - # PD sharding knobs: empty or a positive integer; an explicit - # shard count must be odd and stay within the store count - must_fail --set pd.partition.defaultShardCount=0 - must_fail --set pd.partition.defaultShardCount=-1 - must_fail --set-string pd.partition.defaultShardCount=abc - must_fail --set pd.partition.defaultShardCount=2 - must_fail --set pd.partition.defaultShardCount=5 - must_fail --set pd.partition.storeMaxShardCount=0 - # extraEnv must not override chart-managed variables - must_fail --set 'server.extraEnv[0].name=HG_SERVER_INIT_STORE_ENABLED' \ - --set 'server.extraEnv[0].value=true' - must_fail --set 'pd.extraEnv[0].name=HG_PD_RAFT_PEERS_LIST' \ - --set 'pd.extraEnv[0].value=x' - must_fail --set 'store.extraEnv[0].name=HG_STORE_PD_ADDRESS' \ - --set 'store.extraEnv[0].value=x' - # JAVA_OPTIONS is reserved: a preset value makes the start scripts - # skip auto heap sizing and drop the chart's JAVA_OPTS entirely - must_fail --set 'pd.extraEnv[0].name=JAVA_OPTIONS' \ - --set 'pd.extraEnv[0].value=-Xmx1g' - - - name: server credential wrapper behaviour - shell: bash - run: | - # An existingSecret is only seen when the container starts, so run - # the rendered Server wrapper against Secret values. A valid value - # passes the checks and stops at the missing properties file; an - # unsupported one stops earlier with its own message. - set -euo pipefail - helm template ci helm/hugegraph -s templates/server-deployment.yaml \ - --set server.auth.admin.existingSecret=ci-byo > /tmp/srv.yaml - python3 - <<'PY' - import yaml - d = yaml.safe_load(open('/tmp/srv.yaml')) - open('/tmp/srv.sh', 'w').write(d['spec']['template']['spec']['containers'][0]['args'][0]) - PY - mkdir -p /tmp/srv-run - run() { - (cd /tmp/srv-run && PASSWORD="$1" bash /tmp/srv.sh 2>&1) || true - } - expect() { - local out - out=$(run "$1") - if ! grep -qF -- "$2" <<<"$out"; then - printf 'password %q: expected "%s", got:\n%s\n' "$1" "$2" "$out" >&2 - exit 1 - fi - } - expect 'review-secret_42!' 'missing or unreadable ./conf/rest-server.properties' - expect 'ASCII inner space 123!?' 'must not contain spaces' - expect ' leading' 'must not contain spaces' - expect 'trailing ' 'must not contain spaces' - expect 'review:secret' 'must not contain a colon' - expect 'review\secret' 'must not contain backslashes' - expect "$(printf 'review\tsecret')" 'must be printable ASCII' - expect 'café' 'must be printable ASCII' - - - name: helm test hook behaviour - shell: bash - run: | - # The hook must fail when any one Server Pod cannot answer a - # graph-bound Gremlin query, even while /versions and /graphs - # answer 200 through the Service: a Server that started during a - # PD roll stays in exactly that state for its whole life. Run the - # rendered hook script against loopback fixtures. - set -euo pipefail - sudo sh -c 'echo "127.0.0.1 ci-hugegraph-server ci-hugegraph-server-headless" >> /etc/hosts' - helm template ci helm/hugegraph -s templates/tests/test-connection.yaml > /tmp/hook.yaml - python3 - <<'PY' - import yaml - d = yaml.safe_load(open('/tmp/hook.yaml')) - open('/tmp/hook.sh', 'w').write(d['spec']['containers'][0]['command'][2]) - PY - grep -q 'DEFAULT-hugegraph' /tmp/hook.sh - # getent stub: the headless lookup answers with the fixture IPs. - mkdir -p /tmp/stub - printf '#!/bin/sh\nfor ip in ${STUB_IPS}; do printf "%%s STREAM host\\n" "$ip"; done\n' > /tmp/stub/getent - chmod +x /tmp/stub/getent - cat > /tmp/fixture.py <<'PY' - import http.server, os, sys, threading, time - calls = {} - class H(http.server.BaseHTTPRequestHandler): - def _send(self, code, body): - self.send_response(code) - self.send_header('Content-Type', 'application/json') - self.send_header('Content-Length', str(len(body))) - self.end_headers() - self.wfile.write(body) - def do_GET(self): - self._send(200, b'{"versions":{}}') - def do_POST(self): - self.rfile.read(int(self.headers.get('Content-Length', 0))) - ip = self.server.server_address[0] - # broken-: every Gremlin call fails. flip-odd- / - # flip-even-: the odd or even calls fail, counted from - # the first call made while the file exists, so two Pods - # on opposite phases are never healthy in the same sweep. - odd = os.path.exists('/tmp/flip-odd-' + ip) - even = os.path.exists('/tmp/flip-even-' + ip) - n = calls[ip] = (calls.get(ip, 0) + 1) if (odd or even) else 0 - broken = (os.path.exists('/tmp/broken-' + ip) - or (odd and n % 2 == 1) - or (even and n % 2 == 0)) - if broken: - self._send(400, b'{"exception":"Could not rebind [graph]"}') - else: - self._send(200, b'{"result":{"data":[0]}}') - def log_message(self, *a): - pass - for ip in sys.argv[1:]: - s = http.server.HTTPServer((ip, 8080), H) - threading.Thread(target=s.serve_forever, daemon=True).start() - time.sleep(3600) - PY - python3 /tmp/fixture.py 127.0.0.1 127.0.0.2 127.0.0.3 & - sleep 1 - run_hook() { - PATH="/tmp/stub:${PATH}" PASSWORD=ci-password HG_TEST_DEADLINE_S=8 \ - STUB_IPS="$1" sh /tmp/hook.sh - } - # Healthy: three Pods, all Gremlin-bound. - run_hook "127.0.0.1 127.0.0.2 127.0.0.3" - # One Pod answers REST but not Gremlin: the hook must fail. - touch /tmp/broken-127.0.0.3 - if run_hook "127.0.0.1 127.0.0.2 127.0.0.3"; then - echo "expected the hook to fail on a broken-binding Pod" >&2 - exit 1 - fi - rm /tmp/broken-127.0.0.3 - # Fewer Pods than the replica floor: the hook must fail. - if run_hook "127.0.0.1"; then - echo "expected the hook to fail below the replica floor" >&2 - exit 1 - fi - # Alternating health: sweep 1 sees A ok, B broken; sweep 2 sees A - # broken, B ok; and so on. No single sweep is clean, so the hook - # must fail. A hook that remembered earlier successes per Pod would - # pass here after the second sweep. - touch /tmp/flip-even-127.0.0.1 /tmp/flip-odd-127.0.0.2 - if run_hook "127.0.0.1 127.0.0.2 127.0.0.3"; then - echo "expected the hook to fail when no sweep is completely healthy" >&2 - exit 1 - fi - rm /tmp/flip-even-127.0.0.1 /tmp/flip-odd-127.0.0.2 - # Dual-stack: each Pod publishes an IPv4 and an IPv6 address. Two - # Pods give four addresses, which must not satisfy a floor of - # three; three Pods must pass without the IPv6 addresses being - # queried (nothing listens on them, so a query would fail). - if run_hook "127.0.0.1 127.0.0.2 ::1 ::2"; then - echo "expected the hook to count Pods, not addresses, on dual-stack" >&2 - exit 1 - fi - run_hook "127.0.0.1 127.0.0.2 127.0.0.3 ::1 ::2 ::3" - # The auth-off render must carry no credentials. - if helm template ci helm/hugegraph -s templates/tests/test-connection.yaml \ - --set server.auth.enabled=false | grep -q 'Authorization'; then - echo "the auth-off hook render must not carry credentials" >&2 - exit 1 - fi - - - name: kubeconform - shell: bash - run: | - set -o pipefail - curl -sSLo /tmp/kc.tar.gz https://github.com/yannh/kubeconform/releases/download/v0.6.7/kubeconform-linux-amd64.tar.gz - echo "95f14e87aa28c09d5941f11bd024c1d02fdc0303ccaa23f61cef67bc92619d73 /tmp/kc.tar.gz" | sha256sum -c - - tar -xzf /tmp/kc.tar.gz -C /tmp - for preset in "" "-f helm/hugegraph/values-single.yaml" "-f helm/hugegraph/values-cluster.yaml"; do - # shellcheck disable=SC2086 # $preset intentionally splits into flags - helm template ci helm/hugegraph $preset | /tmp/kubeconform -strict -summary -kubernetes-version 1.23.0 - done - helm template ci helm/hugegraph \ - --set hubble.enabled=true \ - --set hubble.allowWithoutServerAuth=true \ - --set hubble.mode=direct \ - --set hubble.port=9090 \ - --set hubble.persistence.enabled=true \ - --set hubble.ingress.enabled=true \ - --set hubble.ingress.allowPlainHttp=true \ - | /tmp/kubeconform -strict -summary -kubernetes-version 1.23.0 - helm template ci helm/hugegraph \ - --set hubble.enabled=true \ - --set server.auth.enabled=true \ - --set server.auth.admin.existingSecret=ci-auth \ - | /tmp/kubeconform -strict -summary -kubernetes-version 1.23.0 - helm template ci helm/hugegraph \ - --set pd.partition.defaultShardCount=3 \ - --set pd.partition.storeMaxShardCount=12 \ - | /tmp/kubeconform -strict -summary -kubernetes-version 1.23.0 - - - name: legacy --reuse-values compatibility - run: | - # A release created before a field existed must still render. This has - # regressed five times, so it is guarded here rather than by review. - # `-f` is NOT equivalent: it merges over the new defaults, whereas - # --reuse-values discards them, so the fixture must become values.yaml. - cp -R helm/hugegraph /tmp/legacy - cp helm/hugegraph/testdata/values-pre-hardening.yaml /tmp/legacy/values.yaml - helm template legacy /tmp/legacy > /dev/null - helm template legacy /tmp/legacy --is-upgrade > /dev/null - - - name: helm package - run: helm package helm/hugegraph -d /tmp/chart + - name: Helm checks + uses: ./.github/actions/helm-checks diff --git a/.github/workflows/pd-store-ci.yml b/.github/workflows/pd-store-ci.yml index 8dcc74b8f7..7f638bd2bf 100644 --- a/.github/workflows/pd-store-ci.yml +++ b/.github/workflows/pd-store-ci.yml @@ -19,64 +19,9 @@ permissions: contents: read jobs: - rocksdb-compatibility: - if: inputs.run-pd || inputs.run-store || inputs.run-hstore - permissions: - contents: read - runs-on: ubuntu-24.04 - env: - BEFORE_SHA: ${{ github.event.pull_request.base.sha || github.event.before || github.sha }} - AFTER_SHA: ${{ github.event.pull_request.head.sha || github.event.after || github.sha }} - steps: - - name: Require exact comparison revisions - run: | - for revision in "$BEFORE_SHA" "$AFTER_SHA"; do - if [[ ! "$revision" =~ ^[0-9a-f]{40}$ || "$revision" =~ ^0+$ ]]; then - echo "Error: cannot determine the before/after revision for RocksDB compatibility" >&2 - exit 1 - fi - done - - - name: Checkout before - uses: actions/checkout@v7 - with: - ref: ${{ env.BEFORE_SHA }} - path: before - persist-credentials: false - - - name: Checkout after - uses: actions/checkout@v7 - with: - repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} - ref: ${{ env.AFTER_SHA }} - path: after - persist-credentials: false - - - name: Install project JDK - uses: actions/setup-java@v6 - with: - java-version-file: after/.github/workflows/.java-version - distribution: 'zulu' - - - name: Test only changed RocksDB version pairs - run: | - bash after/hugegraph-server/hugegraph-test/src/test/rocksdb-compatibility/run.sh \ - "$GITHUB_WORKSPACE/before" "$GITHUB_WORKSPACE/after" \ - "$RUNNER_TEMP/rocksdb-compatibility" - - - name: Preserve comparison and synthetic-data failure evidence - if: always() - uses: actions/upload-artifact@v7 - with: - name: rocksdb-compatibility-${{ github.sha }} - path: | - ${{ runner.temp }}/rocksdb-compatibility/run.log - ${{ runner.temp }}/rocksdb-compatibility/models/ - ${{ runner.temp }}/rocksdb-compatibility/pairs.tsv - ${{ runner.temp }}/rocksdb-compatibility/cases/ - + # Store already builds Struct; only allocate a separate runner without Store. struct: - if: inputs.run-struct || inputs.run-store + if: inputs.run-struct && !inputs.run-store runs-on: ubuntu-24.04 env: USE_STAGE: 'false' @@ -140,7 +85,8 @@ jobs: pd: if: inputs.run-pd - timeout-minutes: 30 + # Reserve time for the shared RocksDB compatibility check after PD tests. + timeout-minutes: 45 runs-on: ubuntu-24.04 env: # TODO: avoid duplicated env setup in pd & store @@ -338,9 +284,15 @@ jobs: disable_search: true fail_ci_if_error: false + # The first selected distributed job owns the shared compatibility check. + - name: Check RocksDB version compatibility + if: ${{ !cancelled() }} + uses: ./.github/actions/rocksdb-compatibility + store: if: inputs.run-store - timeout-minutes: 45 + # Preserve test time while also hosting Struct and compatibility checks. + timeout-minutes: 70 runs-on: ubuntu-24.04 env: USE_STAGE: 'false' # Whether to include the stage repository. @@ -568,6 +520,40 @@ jobs: disable_search: true fail_ci_if_error: false + - name: Verify Store stop failure propagation and PID retention + if: ${{ !cancelled() }} + run: bash hugegraph-server/hugegraph-dist/src/assembly/travis/test-stop-hugegraph-store.sh + + - name: Run Java runtime log checker tests + if: ${{ !cancelled() }} + run: bash "$TRAVIS_DIR/test-check-java-runtime-logs.sh" + + - name: Run JaCoCo report validator tests + if: ${{ !cancelled() }} + run: hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-jacoco-report.sh + + - name: Run Codecov upload configuration tests + if: ${{ !cancelled() }} + run: hugegraph-server/hugegraph-dist/src/assembly/travis/test-codecov-upload-config.sh + + - name: Check TinkerPop branch and report gates + if: ${{ !cancelled() }} + run: bash hugegraph-server/hugegraph-dist/src/assembly/travis/test-tinkerpop-reports.sh + + # Reuse the reactor build; -am also lets Struct run after a Store build failure. + - name: Run hugegraph-struct test + if: ${{ !cancelled() && hashFiles('hugegraph-struct/pom.xml') != '' }} + run: | + set -o pipefail + RUNTIME_LOG=$(mktemp "$RUNNER_TEMP/java-runtime.XXXXXX") + mvn -U -ntp -pl hugegraph-struct -am test 2>&1 | tee "$RUNTIME_LOG" + bash "$TRAVIS_DIR/check-java-runtime-logs.sh" "$RUNTIME_LOG" + + # The first selected distributed job owns the shared compatibility check. + - name: Check RocksDB version compatibility + if: ${{ !cancelled() && !inputs.run-pd }} + uses: ./.github/actions/rocksdb-compatibility + hstore: if: inputs.run-hstore runs-on: ubuntu-24.04 @@ -915,3 +901,8 @@ jobs: flags: hstore disable_search: true fail_ci_if_error: false + + # The first selected distributed job owns the shared compatibility check. + - name: Check RocksDB version compatibility + if: ${{ !cancelled() && !inputs.run-pd && !inputs.run-store }} + uses: ./.github/actions/rocksdb-compatibility diff --git a/.github/workflows/server-ci.yml b/.github/workflows/server-ci.yml index f21f1f73a1..c49aa3bd3c 100644 --- a/.github/workflows/server-ci.yml +++ b/.github/workflows/server-ci.yml @@ -13,12 +13,12 @@ on: description: Tests to run on the selected branch type: choice default: mainline - options: [mainline, distributed, macos-arm, hbase, riscv64, codeql, full] + options: [mainline, dependencies, distributed, macos-arm, hbase, riscv64, codeql, full] -# A new PR head replaces its old run; scheduled and manual runs stay independent. +# New PR heads and master pushes replace their old runs; scheduled and manual runs stay independent. concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }}-${{ github.run_attempt == 1 && 'head' || format('rerun-{0}', github.run_id) }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref == 'refs/heads/master' && github.ref) || github.run_id }}-${{ github.run_attempt == 1 && 'head' || format('rerun-{0}', github.run_id) }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && github.ref == 'refs/heads/master') }} permissions: contents: read @@ -28,10 +28,11 @@ jobs: preflight: name: check-license runs-on: ubuntu-24.04 - timeout-minutes: 15 + timeout-minutes: 20 env: PYTHONDONTWRITEBYTECODE: '1' outputs: + dependency_baseline: ${{ steps.baseline.outputs.enabled }} selection_result: ${{ steps.select.outcome }} java_result: ${{ steps.java.outcome }} java_versions: ${{ steps.java.outputs.versions }} @@ -69,6 +70,12 @@ jobs: --event-name "$GITHUB_EVENT_NAME" --event-path "$GITHUB_EVENT_PATH" --repository "$GITHUB_REPOSITORY" --output "$RUNNER_TEMP/ci-plan.json" + - name: Select trusted master baseline + id: baseline + env: + ENABLED: ${{ ((github.repository == 'apache/hugegraph' && github.event.repository.fork == false) || github.repository == 'hugegraph/hugegraph') && github.ref == 'refs/heads/master' && (github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && (inputs.test-set == 'mainline' || inputs.test-set == 'full' || inputs.test-set == 'dependencies'))) }} + run: echo "enabled=$ENABLED" >> "$GITHUB_OUTPUT" + - name: Read project Java runtime id: java if: ${{ !cancelled() && steps.checkout.outcome == 'success' }} @@ -99,6 +106,10 @@ jobs: if: ${{ !cancelled() && steps.checkout.outcome == 'success' }} run: python3 -m unittest discover -s .github/scripts -p 'test_*.py' + - name: Check Helm chart + if: ${{ !cancelled() && steps.checkout.outcome == 'success' && steps.select.outputs.helm == 'true' }} + uses: ./.github/actions/helm-checks + - name: Check license headers if: ${{ !cancelled() && steps.checkout.outcome == 'success' }} uses: apache/skywalking-eyes@main @@ -164,15 +175,24 @@ jobs: with: smoke-only: ${{ needs.preflight.outputs.distributed_smoke == 'true' && needs.preflight.outputs.docker != 'true' }} - helm: - needs: preflight - if: ${{ !cancelled() && needs.preflight.outputs.selection_result == 'success' && needs.preflight.outputs.helm == 'true' }} - uses: ./.github/workflows/helm-chart-ci.yml - dependency_license: needs: preflight - if: ${{ !cancelled() && needs.preflight.outputs.selection_result == 'success' && needs.preflight.outputs.java_result == 'success' && needs.preflight.outputs.dependency_license == 'true' }} + if: ${{ !cancelled() && needs.preflight.outputs.selection_result == 'success' && needs.preflight.outputs.java_result == 'success' && (needs.preflight.outputs.dependency_license == 'true' || (needs.preflight.outputs.dependency_baseline == 'true' && needs.preflight.result == 'success')) }} uses: ./.github/workflows/check-dependencies.yml + with: + check-inventory: ${{ needs.preflight.outputs.dependency_license == 'true' }} + submit-baseline: ${{ needs.preflight.result == 'success' && needs.preflight.outputs.dependency_baseline == 'true' }} + + dependency_baseline: + needs: [preflight, dependency_license] + if: ${{ !cancelled() && needs.preflight.result == 'success' && needs.preflight.outputs.dependency_baseline == 'true' && needs.dependency_license.result == 'success' }} + uses: ./.github/workflows/submit-dependencies.yml + permissions: + contents: write + actions: read + with: + artifact-id: ${{ needs.dependency_license.outputs.artifact_id }} + inventory-artifact-id: ${{ needs.dependency_license.outputs.inventory_artifact_id }} compatibility: needs: preflight diff --git a/.github/workflows/submit-dependencies.yml b/.github/workflows/submit-dependencies.yml index 8085a0cb8a..6746073abe 100644 --- a/.github/workflows/submit-dependencies.yml +++ b/.github/workflows/submit-dependencies.yml @@ -15,120 +15,24 @@ # limitations under the License. # -name: Dependency baseline +name: Dependency baseline submission on: - push: - branches: [master] - workflow_dispatch: + workflow_call: + inputs: + artifact-id: + required: true + type: string + inventory-artifact-id: + required: true + type: string permissions: contents: read -concurrency: - group: dependency-baseline-${{ github.repository }}-${{ github.ref }} - cancel-in-progress: true - jobs: - validate: - if: >- - (github.repository == 'apache/hugegraph' || github.repository == 'hugegraph/hugegraph') && - github.ref == 'refs/heads/master' && - (github.event_name == 'push' || github.event_name == 'workflow_dispatch') - runs-on: ubuntu-latest - timeout-minutes: 10 - env: - PYTHONDONTWRITEBYTECODE: '1' - steps: - - name: Checkout event commit - uses: actions/checkout@v7 - with: - ref: ${{ github.sha }} - persist-credentials: false - - name: Test CI policy and dependency inventory - run: | - python3 -m unittest discover -s .github/scripts -p 'test_ci_policy.py' - python3 -m unittest discover -s .github/scripts -p 'test_dependency_snapshot.py' - python3 -m unittest discover -s install-dist/scripts/dependency -p 'test_*.py' - - generate: - needs: validate - if: >- - ((github.repository == 'apache/hugegraph' && github.event.repository.fork == false) || - github.repository == 'hugegraph/hugegraph') && - github.ref == 'refs/heads/master' && - (github.event_name == 'push' || github.event_name == 'workflow_dispatch') - runs-on: ubuntu-latest - timeout-minutes: 60 - permissions: - contents: read - env: - PYTHONDONTWRITEBYTECODE: '1' - INPUT_TOKEN: '' - GITHUB_TOKEN: '' - GH_TOKEN: '' - outputs: - artifact_id: ${{ steps.snapshot.outputs.artifact-id }} - inventory_artifact_id: ${{ steps.inventory.outputs.artifact-id }} - steps: - - name: Checkout event commit - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: ${{ github.sha }} - persist-credentials: false - - name: Set up project JDK - uses: actions/setup-java@de7274f081f381c8f8158605e0321c36c376e2e6 # v6.0.1 - with: - java-version-file: .github/workflows/.java-version - distribution: 'temurin' - cache: maven - - name: Verify checkout and current master - env: - EXPECTED_SHA: ${{ github.sha }} - run: | - test "$(git rev-parse HEAD)" = "$EXPECTED_SHA" - test "$(git ls-remote origin refs/heads/master | cut -f1)" = "$EXPECTED_SHA" - - name: Install the complete Maven reactor - run: mvn -B -ntp install -Dmaven.test.skip=true -Dmaven.javadoc.skip=true --fail-at-end - - name: Generate resolved reactor snapshot - run: | - mvn -B -ntp com.github.ferstl:depgraph-maven-plugin:4.0.3:graph \ - -DgraphFormat=json -DoutputFileName=maven-dependency-submission-action-depgraph.json \ - -Dmaven.test.skip=true -Dmaven.javadoc.skip=true - python3 .github/scripts/dependency-snapshot.py "$RUNNER_TEMP/snapshot.json" - - name: Resolve independent dependency inventory - run: | - mvn -B -ntp org.apache.maven.plugins:maven-dependency-plugin:3.8.1:tree \ - -DoutputType=json -DoutputFile=target/dependency-inventory.json \ - -Dverbose=false -DappendOutput=false \ - -Dmaven.test.skip=true -Dmaven.javadoc.skip=true - python3 .github/scripts/dependency-inventory.py "$RUNNER_TEMP/inventory.json" - - name: Refuse stale dependency reports - env: - EXPECTED_SHA: ${{ github.sha }} - run: test "$(git ls-remote origin refs/heads/master | cut -f1)" = "$EXPECTED_SHA" - - name: Upload snapshot data only - id: snapshot - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: dependency-snapshot-${{ github.run_id }}-${{ github.run_attempt }} - path: ${{ runner.temp }}/snapshot.json - if-no-files-found: error - retention-days: 1 - - - name: Upload inventory data only - id: inventory - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: dependency-inventory-${{ github.run_id }}-${{ github.run_attempt }} - path: ${{ runner.temp }}/inventory.json - if-no-files-found: error - retention-days: 1 - submit: - needs: [validate, generate] if: >- - needs.validate.result == 'success' && needs.generate.result == 'success' && ((github.repository == 'apache/hugegraph' && github.event.repository.fork == false) || github.repository == 'hugegraph/hugegraph') && github.ref == 'refs/heads/master' && @@ -146,8 +50,8 @@ jobs: EXPECTED_SHA: ${{ github.sha }} RUN_ID: ${{ github.run_id }} RUN_ATTEMPT: ${{ github.run_attempt }} - ARTIFACT_ID: ${{ needs.generate.outputs.artifact_id }} - INVENTORY_ARTIFACT_ID: ${{ needs.generate.outputs.inventory_artifact_id }} + ARTIFACT_ID: ${{ inputs.artifact-id }} + INVENTORY_ARTIFACT_ID: ${{ inputs.inventory-artifact-id }} run: | python3 -I - <<'PY' import datetime @@ -255,11 +159,11 @@ jobs: run['head_sha'] == sha and run['head_branch'] == 'master' and run['event'] in ['push', 'workflow_dispatch'] and run['repository']['full_name'] == repository and - run['path'] in ['.github/workflows/submit-dependencies.yml', - '.github/workflows/submit-dependencies.yml@master'], 'Run identity') + run['path'] in ['.github/workflows/server-ci.yml', + '.github/workflows/server-ci.yml@master'], 'Run identity') jobs = api(repository, f'actions/runs/{run_id}/attempts/{attempt}/jobs?per_page=100') require(jobs['total_count'] <= 100, 'Too many jobs') - producers = [job for job in jobs['jobs'] if job['name'] == 'generate'] + producers = [job for job in jobs['jobs'] if job['name'] == 'dependency_license / dependency-check'] require(len(producers) == 1 and producers[0]['conclusion'] == 'success' and producers[0]['status'] == 'completed' and producers[0]['run_id'] == int(run_id) and producers[0]['head_sha'] == sha, 'Producer did not succeed for this commit') diff --git a/docs/ci.md b/docs/ci.md index bf9306ef6d..3fb3ebe01d 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -3,7 +3,8 @@ [HugeGraph-Server CI](../.github/workflows/server-ci.yml) is the single entry point for PRs, supported branch pushes, weekly coverage and manual validation. Its preflight job selects affected tests, reads the project Java version, checks CI policy and verifies -license headers and RAT. Selected suites run through reusable workflows in the same run. +license headers and RAT. Selected Helm checks run as steps in preflight; other selected suites +run through reusable workflows in the same run. CI results become advisory after [`.asf.yaml`](../.asf.yaml) is applied. It requests no required CI contexts and retains one approving review. This protection change takes effect after the configuration @@ -65,8 +66,10 @@ Contract scripts explicitly select their executing suites without adding runtime ### Documentation -Documentation-only ordinary PRs and pushes allocate just the preflight CI runner. This count -excludes the non-test [Auto PR Commenter](../.github/workflows/auto-pr-review.yml), which runs +Documentation-only ordinary PRs allocate just the preflight CI runner. On the two trusted +master repositories, pushes also run the shared dependency producer and isolated writer so +that the exact event commit has a dependency baseline; other documentation-only pushes use +only preflight. This count excludes the non-test [Auto PR Commenter](../.github/workflows/auto-pr-review.yml), which runs once when a PR opens. Scheduled and manual selections still run their requested suites. Plain documentation includes regular-file root prose, README/AGENTS files and supported prose @@ -80,6 +83,7 @@ Use **Actions → HugeGraph-Server CI → Run workflow**, select a branch and ch | Test set | Validation | | --- | --- | +| `dependencies` | Dependency inventory and PR review; on trusted master, generate and submit a baseline | | `mainline` (default) | Full mainline modules, deployment/dependency checks and CodeQL; ARM smoke | | `distributed` | Struct, PD, Store, HStore and Cluster, plus distributed smoke | | `macos-arm` | ARM smoke and full RocksDB core/API tests | @@ -120,18 +124,28 @@ Preflight verifies that the event merge commit's second parent matches the curre A stale PR head fails preflight. Target-branch advancement alone does not invalidate a run. Plans and selection summaries describe this run; previous results are never reused. -A new PR head cancels the previous PR run, including its called workflows. Manual and weekly -runs remain independent. Each job reports its own result; there are no separate post-test -aggregation runners or end-of-run metadata checks. - -Startup contracts run inside RocksDB validation, and Store stop contracts run inside Struct. +A new PR head or master push cancels the previous automatic run for that PR or master ref, +including its called workflows. Release/test branch pushes, manual runs, weekly runs and +explicit rerun attempts remain independent. Checks keep their real failed results; there +are no separate post-test aggregation runners or end-of-run metadata checks. + +Startup contracts run inside RocksDB validation. Struct tests and their shell contracts run +inside the selected Store job, reusing its reactor build; a Struct-only selection retains +its own runner. RocksDB version compatibility runs once inside the first selected PD, Store +or HStore job, in that order. These added steps still execute after ordinary host-job failures; +a host timeout or cancellation can stop them. Helm uses preflight, whose total budget is +20 minutes. PR dependency review runs as the last step of the existing dependency job even +if its build or inventory check failed. These mergers remove four runner allocations from +a complete mainline PR run, without dropping checks; heavy module suites remain separate. Docker's static checks, deployment contracts and current-image validation share one runner. The Docker job has a 90-minute total budget, with 25 minutes for deployment contracts and 45 minutes for image build/runtime validation. Distributed smoke builds current PD, Store and Server images and checks authentication, readiness and graph read/write. It does not substitute published images for the current commit. -PD, Store and Commons have total job budgets of 30, 45 and 20 minutes. Cluster has 120 minutes +PD and Store reserve 45 and 70 minutes including their merged checks, rather than +sharing the previous 30/45-minute limits. The dependency job retains the existing inventory +budget; baseline-only builds keep a 60-minute limit. Commons has 20 minutes. Cluster has 120 minutes for its two 45-minute test windows and build/diagnostics. Other budgets are in the workflows. TinkerPop suites keep their dedicated branch policy: the source, target or push/manual branch must start with `release-`, `test-` or `tinkerpop-`. Executed suites require non-empty reports @@ -145,17 +159,21 @@ it does not guarantee a specific queue delay. ## Dependency baseline -The separate [dependency submission workflow](../.github/workflows/submit-dependencies.yml) -installs the complete reactor once with Java 17 in a `contents: read` job, reusing -`actions/setup-java`'s Maven cache, then generates +The main CI coordinator selects the [shared dependency job](../.github/workflows/check-dependencies.yml) +for every trusted master push and manual `dependencies`, `mainline` or `full` run on master. +Other manual presets keep their requested scope. The job installs the complete reactor +once with Java 17 and `contents: read`, reusing `actions/setup-java`'s Maven cache. When the +path classifier also selects dependency inventory checks, those checks and baseline +production share this same installation instead of building the reactor twice. It generates each module's resolved graph with Ferstl 4.0.3. The repository's [stdlib serializer](../.github/scripts/dependency-snapshot.py) preserves the [official Maven parser 6.0.1](https://github.com/actions/maven-dependency-submission-action/tree/1f666f7ac2ab7b72cd5e22eaf7f23c8a581db1db) package URLs and relationships. Non-test scopes take precedence when Ferstl merges scopes onto one artifact; only test-only artifacts are development dependencies. Every tracked POM must have a graph; empty aggregator manifests are allowed, but a -completely empty inventory fails. It runs on master pushes or manual -dispatches. Only `apache/hugegraph` (not a fork) and the explicitly allowed +completely empty inventory fails. Fixture validation shares preflight and the dependency job; +there is no separate validation runner. The [reusable submission workflow](../.github/workflows/submit-dependencies.yml) +contains only the isolated writer. Only `apache/hugegraph` (not a fork) and the explicitly allowed `hugegraph/hugegraph` test fork on `refs/heads/master` can run the submission job, which alone has `contents: write`. PRs, other repositories and other refs cannot run the writer. Maven inherits no submission token, and checkout does not persist @@ -171,7 +189,8 @@ not replace this per-module check. A pinned `actions/upload-artifact` transfers `snapshot.json` and `inventory.json` as separate artifacts. The fresh writer runner uses fixed inline stdlib Python to -verify the successful producer job and both artifact/run/attempt/SHA identities +verify the successful `dependency_license / dependency-check` producer in the same +`server-ci.yml` run and both artifact/run/attempt/SHA identities before downloading. It bounds ZIP and JSON sizes, checks each archive digest, and reads only its one expected regular JSON member in memory. Duplicate members, paths and symlinks fail; no archive members are extracted to disk. A signed storage @@ -182,10 +201,12 @@ downloaded code. The first-party upload action is permitted by the Checkout and snapshot use the event commit SHA. Master is checked before install and again after both reports are generated; a stale run fails. A newer master -run cancels the older master run because only the current master can submit. -Concurrency groups include the ref, so manual runs on other refs cannot cancel master. Every master -push still starts a baseline run without path filters, so unrelated changes also -can receive an exact-SHA baseline. The writer +push cancels the older master run because only the current master can submit. +Manual and scheduled runs use separate concurrency groups and cannot cancel automatic +master pushes. Every trusted master push still selects the producer without path filters, +so unrelated changes also can receive an exact-SHA baseline. Such a push still requires +one complete reactor install; merging jobs removes duplicate builds, not that inherent cost. +The writer checks the run attempt, successful producer, artifact ID/name, JSON schema, detector/correlator and source SHA/ref. It verifies all manifest paths against the event commit's source tree, rejects empty or mismatched inventory, and checks master @@ -193,11 +214,13 @@ immediately before and after submission. Snapshots retain the checked event SHA if master advances during submission; that run fails its final freshness check. If submission fails, use **Re-run all jobs** or start a fresh `workflow_dispatch` -on master. **Re-run failed jobs** keeps the older generation artifact, which the +of **HugeGraph-Server CI**, choosing `dependencies` on master. **Re-run failed jobs** +keeps the older generation artifact, which the writer rejects because both reports must be produced in the same run attempt. If a PR `dependency-review` failure reports 0 base snapshots, rerun that job only -once the same repository's **Dependency baseline** run has succeeded for the event's +once the same repository's **HugeGraph-Server CI** run has successfully submitted the +baseline for the event's exact `pull_request.base.sha` (not the branch's merge base or a newer master tip). If that SHA has no successful baseline, including a skipped or superseded run, rerunning the review alone cannot supply it. The producer supports only the current @@ -225,4 +248,7 @@ first master submission, including its SHA/ref, non-empty module manifests and a dependency-review comparison against that baseline. Until then, that repository's baseline acceptance remains open. This workflow changes no repository settings. -Dependency snapshot producer and submission-workflow changes select dependency checks. Snapshot fixture tests run in the entry job; they do not select runtime suites. Changes to the CI classifier still receive conservative mainline validation. +Dependency snapshot producer and submission-workflow changes select dependency checks. +Snapshot fixture tests run in the entry job; they do not select runtime suites. Local Helm +and RocksDB compatibility action changes select their existing owners. Changes to the CI +classifier still receive conservative mainline validation. From ae4d8cc33072494c6824baadd229cad86e354819 Mon Sep 17 00:00:00 2001 From: imbajin Date: Tue, 20 Oct 2026 15:35:24 +0800 Subject: [PATCH 12/14] fix(ci): keep uploader artifacts out of RAT - run Store Maven checks before coverage upload - handle newly created refs without a before revision - disable informational Scorecard lookups and remove an unused wrapper - preserve test gates and runner allocations --- .../actions/rocksdb-compatibility/action.yml | 14 ++++++- .github/scripts/ci-policy.py | 2 +- .github/scripts/test_ci_policy.py | 40 +++++++++++++++++++ .github/workflows/check-dependencies.yml | 2 + .github/workflows/helm-chart-ci.yml | 35 ---------------- .github/workflows/pd-store-ci.yml | 21 +++++----- docs/ci.md | 9 ++++- 7 files changed, 75 insertions(+), 48 deletions(-) delete mode 100644 .github/workflows/helm-chart-ci.yml diff --git a/.github/actions/rocksdb-compatibility/action.yml b/.github/actions/rocksdb-compatibility/action.yml index 138dc76e46..f636c921d5 100644 --- a/.github/actions/rocksdb-compatibility/action.yml +++ b/.github/actions/rocksdb-compatibility/action.yml @@ -21,19 +21,28 @@ runs: using: composite steps: - name: Require exact comparison revisions + id: revisions shell: bash env: BEFORE_SHA: ${{ github.event.pull_request.base.sha || github.event.before || github.sha }} AFTER_SHA: ${{ github.event.pull_request.head.sha || github.event.after || github.sha }} run: | + if [[ "$GITHUB_EVENT_NAME" == push && "$BEFORE_SHA" == 0000000000000000000000000000000000000000 && + "$AFTER_SHA" =~ ^[0-9a-f]{40}$ && ! "$AFTER_SHA" =~ ^0+$ ]]; then + echo "::notice::Skipping RocksDB comparison for a newly created ref with no before revision" + echo "compare=false" >> "$GITHUB_OUTPUT" + exit 0 + fi for revision in "$BEFORE_SHA" "$AFTER_SHA"; do if [[ ! "$revision" =~ ^[0-9a-f]{40}$ || "$revision" =~ ^0+$ ]]; then echo "Error: cannot determine the before/after revision for RocksDB compatibility" >&2 exit 1 fi done + echo "compare=true" >> "$GITHUB_OUTPUT" - name: Checkout before + if: steps.revisions.outputs.compare == 'true' uses: actions/checkout@v7 with: ref: ${{ github.event.pull_request.base.sha || github.event.before || github.sha }} @@ -41,6 +50,7 @@ runs: persist-credentials: false - name: Checkout after + if: steps.revisions.outputs.compare == 'true' uses: actions/checkout@v7 with: repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} @@ -49,12 +59,14 @@ runs: persist-credentials: false - name: Install comparison JDK + if: steps.revisions.outputs.compare == 'true' uses: actions/setup-java@v6 with: java-version-file: .ci-rocksdb/after/.github/workflows/.java-version distribution: 'zulu' - name: Test only changed RocksDB version pairs + if: steps.revisions.outputs.compare == 'true' shell: bash run: | bash .ci-rocksdb/after/hugegraph-server/hugegraph-test/src/test/rocksdb-compatibility/run.sh \ @@ -62,7 +74,7 @@ runs: "$RUNNER_TEMP/rocksdb-compatibility" - name: Preserve comparison and synthetic-data failure evidence - if: always() + if: always() && steps.revisions.outputs.compare == 'true' uses: actions/upload-artifact@v7 with: name: rocksdb-compatibility-${{ github.sha }} diff --git a/.github/scripts/ci-policy.py b/.github/scripts/ci-policy.py index 2e6fbc2028..d3e75dad2e 100644 --- a/.github/scripts/ci-policy.py +++ b/.github/scripts/ci-policy.py @@ -39,7 +39,7 @@ WORKFLOWS = { "server": {"server-ci.yml": MAINLINE, "commons-ci.yml": {"commons"}, "pd-store-ci.yml": {"struct", "pd", "store", "hstore"}, "cluster-test-ci.yml": {"cluster"}, - "docker-build-ci.yml": {"docker"}, "helm-chart-ci.yml": {"helm"}, + "docker-build-ci.yml": {"docker"}, "codeql-analysis.yml": {"security"}, "server-memory-ci.yml": {"server"}, "server-tests.yml": {"server", "hbase"}, "server-compatibility-ci.yml": {"macos", "macos_full", "hbase", "riscv64"}, diff --git a/.github/scripts/test_ci_policy.py b/.github/scripts/test_ci_policy.py index 766b0a586c..6348e85a78 100644 --- a/.github/scripts/test_ci_policy.py +++ b/.github/scripts/test_ci_policy.py @@ -539,6 +539,46 @@ def test_distributed_checks_keep_exactly_one_host_and_preserve_struct_fallback(s self.assertIn("if: ${{ !cancelled() }}", step) self.assertIn("Run hugegraph-struct test", host) + def test_store_upload_cannot_contaminate_later_maven_checks(self): + workflow = (Path(__file__).resolve().parents[1] / "workflows/pd-store-ci.yml").read_text() + store = workflow.split("\n store:\n", 1)[1].split("\n hstore:\n", 1)[0] + upload = store.index(" - name: Upload coverage to Codecov\n") + self.assertLess(store.index(" - name: Run hugegraph-struct test\n"), upload) + self.assertNotIn("mvn ", store[upload:]) + self.assertIn("id: store-coverage", store) + condition = re.search(r"^ if: (.+)$", store[upload:], re.M).group(1) + self.assertIn("!cancelled()", condition) + for outcome in ["success", "failure", "skipped", ""]: + self.assertEqual(outcome == "success", self.workflow_expression(condition, + {"steps.store-coverage.outcome": outcome})) + + def test_rocksdb_created_ref_skips_comparison_but_rejects_invalid_revisions(self): + action = (Path(__file__).resolve().parents[1] / "actions/rocksdb-compatibility/action.yml").read_text() + steps = action.split("\n - ")[1:] + code = textwrap.dedent(steps[0].split(" run: |\n", 1)[1]) + zero, before, after = "0" * 40, "a" * 40, "b" * 40 + cases = [("push", before, after, "true"), ("pull_request", before, after, "true"), + ("push", zero, after, "false"), ("pull_request", zero, after, None), + ("push", "0" * 39, after, None), ("push", "bad", after, None), + ("push", zero, "bad", None), ("push", zero, zero, None), + ("push", before, zero, None), ("push", before, "", None)] + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / "output" + for event, base, head, compare in cases: + output.write_text("") + result = subprocess.run(["bash", "-e", "-c", code], capture_output=True, text=True, + env={**os.environ, "GITHUB_EVENT_NAME": event, "BEFORE_SHA": base, + "AFTER_SHA": head, "GITHUB_OUTPUT": str(output)}) + with self.subTest(event=event, before=base, after=head): + self.assertEqual(compare is not None, result.returncode == 0, result.stderr) + self.assertEqual("" if compare is None else f"compare={compare}\n", output.read_text()) + self.assertEqual(compare == "false", "::notice::" in result.stdout) + for step in steps[1:]: + condition = re.search(r"^ if: (.+)$", step, re.M).group(1) + enabled = self.workflow_expression(condition.replace("always()", "true"), + {"steps.revisions.outputs.compare": compare or ""}) + self.assertEqual(compare == "true", enabled, step.splitlines()[0]) + def test_coordinator_reads_and_validates_real_java_input(self): workflow = (Path(__file__).resolve().parents[1] / "workflows/server-ci.yml").read_text() code = workflow.split("python3 - <<'PYTHON'\n", 1)[1].split("\n PYTHON", 1)[0] diff --git a/.github/workflows/check-dependencies.yml b/.github/workflows/check-dependencies.yml index 1f7b693b20..18caa0332f 100644 --- a/.github/workflows/check-dependencies.yml +++ b/.github/workflows/check-dependencies.yml @@ -142,6 +142,8 @@ jobs: with: # TODO: reset critical to low before releasing fail-on-severity: critical + # No Scorecard gate is configured; avoid informational lookups for every package. + show-openssf-scorecard: false # Action will fail if dependencies don't match the list #allow-licenses: Apache-2.0, MIT #deny-licenses: GPL-3.0, AGPL-1.0, AGPL-3.0, LGPL-2.0, CC-BY-3.0 diff --git a/.github/workflows/helm-chart-ci.yml b/.github/workflows/helm-chart-ci.yml deleted file mode 100644 index 8739c8ba20..0000000000 --- a/.github/workflows/helm-chart-ci.yml +++ /dev/null @@ -1,35 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -name: "Helm Chart CI" - -on: - workflow_call: - -permissions: - contents: read - -jobs: - lint-and-render: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - - name: Helm checks - uses: ./.github/actions/helm-checks diff --git a/.github/workflows/pd-store-ci.yml b/.github/workflows/pd-store-ci.yml index be673d6fc9..e4dd1809ea 100644 --- a/.github/workflows/pd-store-ci.yml +++ b/.github/workflows/pd-store-ci.yml @@ -468,6 +468,7 @@ jobs: -DskipTests -Deditorconfig.skip=true -ntp - name: Validate aggregate coverage report + id: store-coverage run: | $TRAVIS_DIR/check-jacoco-report.sh \ --require-test-report \ @@ -519,15 +520,6 @@ jobs: tail -n 120 "$report" || true done - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v7 - with: - token: ${{ secrets.CODECOV_TOKEN }} - files: ${{ env.REPORT_FILE }} - flags: store - disable_search: true - fail_ci_if_error: false - - name: Verify Store stop failure propagation and PID retention if: ${{ !cancelled() }} run: bash hugegraph-server/hugegraph-dist/src/assembly/travis/test-stop-hugegraph-store.sh @@ -562,6 +554,17 @@ jobs: if: ${{ !cancelled() && !inputs.run-pd }} uses: ./.github/actions/rocksdb-compatibility + # Upload last: Codecov downloads unlicensed checksum files into the workspace. + - name: Upload coverage to Codecov + if: ${{ !cancelled() && steps.store-coverage.outcome == 'success' }} + uses: codecov/codecov-action@v7 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ${{ env.REPORT_FILE }} + flags: store + disable_search: true + fail_ci_if_error: false + hstore: if: inputs.run-hstore runs-on: ubuntu-24.04 diff --git a/docs/ci.md b/docs/ci.md index cc08f136a7..0b7f88918e 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -132,10 +132,15 @@ are no separate post-test aggregation runners or end-of-run metadata checks. Startup contracts run inside RocksDB validation. Struct tests and their shell contracts run inside the selected Store job, reusing its reactor build; a Struct-only selection retains its own runner. RocksDB version compatibility runs once inside the first selected PD, Store -or HStore job, in that order. These added steps still execute after ordinary host-job failures; +or HStore job, in that order. A push creating a branch has no previous revision; it +reports this explicitly and skips only the before/after compatibility comparison. These added steps +still execute after ordinary host-job failures; a host timeout or cancellation can stop them. Helm uses preflight, whose total budget is 20 minutes. PR dependency review runs as the last step of the existing dependency job even -if its build or inventory check failed. These mergers remove four runner allocations from +if its build or inventory check failed. Informational OpenSSF Scorecard display is disabled; +vulnerability checks and their failure threshold are unchanged. Store uploads Codecov data +after all Maven checks so downloaded checksum files cannot contaminate a later RAT scan. +These mergers remove four runner allocations from a complete mainline PR run, without dropping checks; heavy module suites remain separate. Docker's static checks, deployment contracts and current-image validation share one runner. The Docker job has a 90-minute total budget, with 25 minutes for deployment contracts and From 79cad7209551b4f42c6d1382f9bbadf4801c6771 Mon Sep 17 00:00:00 2001 From: imbajin Date: Tue, 20 Oct 2026 19:57:52 +0800 Subject: [PATCH 13/14] fix(ci): isolate manual tests from baselines - limit baseline production to master pushes and explicit requests - retain all manual mainline and full inventory checks - document the trigger policy and cover manual selection --- .github/scripts/test_ci_policy.py | 12 +++++++----- .github/workflows/server-ci.yml | 2 +- docs/ci.md | 7 ++++--- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/.github/scripts/test_ci_policy.py b/.github/scripts/test_ci_policy.py index 6348e85a78..5c1d84f2a7 100644 --- a/.github/scripts/test_ci_policy.py +++ b/.github/scripts/test_ci_policy.py @@ -454,23 +454,25 @@ def test_shared_dependency_install_preserves_inventory_build_arguments(self): self.assertIn("-Dmaven.test.skip=true", commands[0]) self.assertEqual(not inventory, "-Dmaven.javadoc.skip=true" in commands[0]) - def test_master_baseline_is_selected_only_for_trusted_full_runs(self): + def test_master_baseline_requires_push_or_explicit_dependency_run(self): workflow = (Path(__file__).resolve().parents[1] / "workflows/server-ci.yml").read_text() expression = re.search(r"^ ENABLED: (.+)$", workflow, re.M).group(1) values = {"github.repository": "apache/hugegraph", "github.event.repository.fork": False, "github.ref": "refs/heads/master", "github.event_name": "push", "inputs.test-set": "mainline"} - for event in ["push", "workflow_dispatch"]: - with self.subTest(event=event): - self.assertTrue(self.workflow_expression(expression, dict(values, **{"github.event_name": event}))) + self.assertTrue(self.workflow_expression(expression, values)) for field, value in [("github.event_name", "schedule"), ("github.event_name", "pull_request"), ("github.event_name", "pull_request_target"), ("github.ref", "refs/heads/release-1.7"), ("github.repository", "unknown/fork"), ("github.event.repository.fork", True)]: with self.subTest(field=field, value=value): self.assertFalse(self.workflow_expression(expression, dict(values, **{field: value}))) - for preset in ["distributed", "codeql", "macos-arm"]: + for preset in ["mainline", "full", "distributed", "codeql", "macos-arm", "hbase", "riscv64"]: self.assertFalse(self.workflow_expression(expression, dict(values, **{ "github.event_name": "workflow_dispatch", "inputs.test-set": preset}))) + for preset in ["mainline", "full"]: + plan = self.plan_for([], "workflow_dispatch", {"inputs": {"test-set": preset}}) + self.assertIn("dependency_license", self.coordinator_jobs(plan)) + self.assertNotIn("dependency_baseline", self.coordinator_jobs(plan)) self.assertTrue(self.workflow_expression(expression, dict(values, **{ "github.repository": "hugegraph/hugegraph", "github.event.repository.fork": True}))) dependencies = self.plan_for([], "workflow_dispatch", {"inputs": {"test-set": "dependencies"}}) diff --git a/.github/workflows/server-ci.yml b/.github/workflows/server-ci.yml index c49aa3bd3c..7995350eb1 100644 --- a/.github/workflows/server-ci.yml +++ b/.github/workflows/server-ci.yml @@ -73,7 +73,7 @@ jobs: - name: Select trusted master baseline id: baseline env: - ENABLED: ${{ ((github.repository == 'apache/hugegraph' && github.event.repository.fork == false) || github.repository == 'hugegraph/hugegraph') && github.ref == 'refs/heads/master' && (github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && (inputs.test-set == 'mainline' || inputs.test-set == 'full' || inputs.test-set == 'dependencies'))) }} + ENABLED: ${{ ((github.repository == 'apache/hugegraph' && github.event.repository.fork == false) || github.repository == 'hugegraph/hugegraph') && github.ref == 'refs/heads/master' && (github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && inputs.test-set == 'dependencies')) }} run: echo "enabled=$ENABLED" >> "$GITHUB_OUTPUT" - name: Read project Java runtime diff --git a/docs/ci.md b/docs/ci.md index 0b7f88918e..c818ad08f1 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -165,9 +165,10 @@ it does not guarantee a specific queue delay. ## Dependency baseline The main CI coordinator selects the [shared dependency job](../.github/workflows/check-dependencies.yml) -for every trusted master push and manual `dependencies`, `mainline` or `full` run on master. -Other manual presets keep their requested scope. The job installs the complete reactor -once with Java 17 and `contents: read`, reusing `actions/setup-java`'s Maven cache. When the +for baseline production on every trusted master push and manual `dependencies` run on master. +Manual `mainline` and `full` runs keep all requested tests and inventory checks without +producing a baseline, so master advancement cannot skip or fail those checks. The job installs +the complete reactor once with Java 17 and `contents: read`, reusing `actions/setup-java`'s Maven cache. When the path classifier also selects dependency inventory checks, those checks and baseline production share this same installation instead of building the reactor twice. It generates each module's resolved graph with Ferstl 4.0.3. The repository's From ce359c6bdad22b699445c005e9fc89582181c6cd Mon Sep 17 00:00:00 2001 From: imbajin Date: Wed, 21 Oct 2026 12:39:57 +0800 Subject: [PATCH 14/14] fix(ci): synchronize Arthas test commands - wait for authentication and its next native prompt - wait for version output before requesting interactive exit - preserve all authentication and cleanup assertions --- .../src/assembly/travis/test-arthas-cli.sh | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-arthas-cli.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-arthas-cli.sh index b4428ebff4..1402c67146 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-arthas-cli.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-arthas-cli.sh @@ -146,9 +146,9 @@ with tempfile.TemporaryDirectory(prefix="arthas-test-") as work: except OSError: pass # The slave closes when the helper exits. - def ready(self): + def ready(self, pattern=r"\[arthas@"): deadline = time.monotonic() + 15 - while "[arthas@" not in self.output: + while not re.search(pattern, self.output): self.drain() assert self.poll() is None and time.monotonic() < deadline, self.output time.sleep(0.02) @@ -338,8 +338,13 @@ with tempfile.TemporaryDirectory(prefix="arthas-test-") as work: assert "credentials rejected" in wrong and "agent left running" in wrong, wrong wait_port(True) quoted_password = "'" + password.replace("'", "'\\''") + "'" - os.write(interactive.master, - f'auth --username "operator name" {quoted_password}\nversion\nquit\n'.encode()) + # Wait for each native-client response before sending the next command; + # pasted lines can be consumed while the terminal is changing prompts. + os.write(interactive.master, f'auth --username "operator name" {quoted_password}\n'.encode()) + interactive.ready(r"(?s)Authentication result: true.*\[arthas@") + os.write(interactive.master, b"version\n") + interactive.ready(r"(?s)[\r\n]4\.1\.5\s*[\r\n].*\[arthas@") + os.write(interactive.master, b"quit\n") assert interactive.wait(45) == 0, interactive.output assert re.search(r"[\r\n]4\.1\.5\s*[\r\n]", interactive.output), interactive.output wait_port(False)