fix(ci): submit matching master dependency baselines

.github/scripts/ci-policy.py

@@ -43,7 +43,8 @@
                "codeql-analysis.yml": {"security"}, "server-memory-ci.yml": {"server"},
                "server-tests.yml": {"server", "hbase"},
                "server-compatibility-ci.yml": {"macos", "macos_full", "hbase", "riscv64"},
-               "riscv64-ci.yml": {"riscv64"}, "check-dependencies.yml": {"dependency_license"}},
+               "riscv64-ci.yml": {"riscv64"}, "check-dependencies.yml": {"dependency_license"},
+               "submit-dependencies.yml": {"dependency_license"}},
     "toolchain": {"client-ci.yml": {"client"}, "client-go-ci.yml": {"go"},
                   "loader-ci.yml": {"loader"}, "tools-ci.yml": {"tools"},
                   "spark-connector-ci.yml": {"spark"}, "hubble-ci.yml": {"hubble"},
@@ -60,6 +61,9 @@
     ".asf.yaml": set(), ".editorconfig": set(), ".licenserc.yaml": set(),
     ".github/PULL_REQUEST_TEMPLATE.md": set(), ".github/dependabot.yml": set(),
     ".github/workflows/.java-version": MAINLINE | {"macos_full", "hbase", "riscv64"},
+    ".github/scripts/dependency-snapshot.py": {"dependency_license"},
+    ".github/scripts/dependency-inventory.py": {"dependency_license"},
+    ".github/scripts/test_dependency_snapshot.py": set(),  # Covered by preflight discovery.
     ".github/scripts/check-docker-images.sh": {"docker"},
     ".github/scripts/docker-deployment.py": {"docker"},
     ".github/scripts/test_docker_deployment.py": {"docker"},

.github/scripts/dependency-inventory.py

@@ -0,0 +1,99 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Canonical inventory from independent Apache Maven dependency:tree JSON."""
+
+import datetime
+import importlib.util
+import json
+import os
+from pathlib import Path
+import subprocess
+import sys
+
+spec = importlib.util.spec_from_file_location('snapshot', Path(__file__).with_name('dependency-snapshot.py'))
+snapshot = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(snapshot)
+
+REPORT = 'dependency-inventory.json'
+RESOLVER = 'org.apache.maven.plugins:maven-dependency-plugin:3.8.1:tree'
+LIMIT = 25 * 1024 * 1024
+FIELDS = {'groupId', 'artifactId', 'version', 'type', 'scope', 'classifier', 'optional', 'children'}
+SCOPES = {'compile', 'runtime', 'provided', 'system', 'test'}
+
+
+def packages(tree):
+    result = {}
+    pending = [(tree, 0)]
+    while pending:
+        node, depth = pending.pop()
+        if not isinstance(node, dict) or set(node) - FIELDS:
+            raise ValueError('Invalid inventory node')
+        for field in ['groupId', 'artifactId', 'version', 'type']:
+            if not isinstance(node.get(field), str) or not node[field].strip():
+                raise ValueError('Incomplete inventory coordinates')
+        if not isinstance(node.get('classifier'), str) or node.get('optional') not in ['true', 'false']:
+            raise ValueError('Invalid inventory classifier/optional')
+        scope = node.get('scope')
+        if (depth == 0 and scope != '') or (depth > 0 and scope not in SCOPES):
+            raise ValueError('Invalid inventory scope')
+        children = node.get('children', [])
+        if not isinstance(children, list):
+            raise ValueError('Invalid inventory children')
+        pending.extend((child, depth + 1) for child in children)
+        if depth == 0:
+            continue
+        purl = snapshot.package_url(dict(node, types=[node['type']], classifiers=[node['classifier']]))
+        value = {'scope': 'development' if scope == 'test' else 'runtime',
+                 'relationship': 'direct' if depth == 1 else 'indirect'}
+        # Maven may expose the same selected package along more than one path.
+        previous = result.get(purl)
+        if previous:
+            if previous['scope'] == 'runtime':
+                value['scope'] = 'runtime'
+            if previous['relationship'] == 'direct':
+                value['relationship'] = 'direct'
+        result[purl] = value
+    return result
+
+
+def create_inventory(workspace, poms, sha, repository, run_id, run_attempt):
+    manifests = {}
+    for pom in poms:
+        if pom in manifests:
+            raise ValueError('Duplicate inventory POM')
+        report = workspace / Path(pom).parent / 'target' / REPORT
+        if report.is_symlink():
+            raise ValueError('Invalid inventory report')
+        with report.open('rb') as stream:
+            raw = stream.read(LIMIT + 1)
+        if not raw or len(raw) > LIMIT:
+            raise ValueError('Invalid inventory report size')
+        manifests[pom] = packages(json.loads(raw, object_pairs_hook=snapshot.unique_object))
+    if not manifests or not any(manifests.values()):
+        raise ValueError('Empty independent dependency inventory')
+    return {'version': 1, 'sha': sha, 'repository': repository, 'run_id': run_id,
+            'run_attempt': run_attempt, 'producer': 'inventory', 'resolver': RESOLVER,
+            'scanned': datetime.datetime.now(datetime.timezone.utc).isoformat().replace('+00:00', 'Z'),
+            'manifests': manifests}
+
+
+if __name__ == '__main__':
+    poms = subprocess.check_output(['git', 'ls-files', '-z', '*pom.xml']).decode().split('\0')
+    poms = [pom for pom in poms if Path(pom).name == 'pom.xml']
+    inventory = create_inventory(Path.cwd(), poms, os.environ['GITHUB_SHA'], os.environ['GITHUB_REPOSITORY'],
+                                 os.environ['GITHUB_RUN_ID'], os.environ['GITHUB_RUN_ATTEMPT'])
+    Path(sys.argv[1]).write_text(json.dumps(inventory, indent=2) + '\n')

.github/scripts/dependency-snapshot.py

@@ -0,0 +1,155 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Serialize native Ferstl 4.0.3 graphs; no token or submission code.
+
+Uses maven-dependency-submission-action 6.0.1's Maven parser/toolkit conventions:
+first classifier/type, shared PURL packages, direct precedence, child edge order
+and cycle handling. Non-test scopes take precedence over test to match the
+independent inventory. See docs/ci.md for the producer boundary.
+"""
+
+import datetime
+import json
+import os
+from pathlib import Path
+import subprocess
+import sys
+from urllib.parse import quote
+
+REPORT = 'maven-dependency-submission-action-depgraph.json'
+DETECTOR = {'name': 'maven-dependency-submission-action', 'version': '6.0.1',
+            'url': 'https://github.com/advanced-security/maven-dependency-submission-action'}
+CORRELATOR = '.github/workflows/submit-dependencies.yml:submit'
+SERIALIZER = 'Ferstl 4.0.3; repository stdlib adapter for Maven parser 6.0.1'
+
+
+def unique_object(pairs):
+    result = {}
+    for key, value in pairs:
+        if key in result:
+            raise ValueError(f'Duplicate JSON key: {key}')
+        result[key] = value
+    return result
+
+
+def package_url(artifact):
+    # packageurl-js uses encodeURIComponent for components and URLSearchParams
+    # for qualifiers; notably '+' in versions and '/' in namespaces survive.
+    namespace = '/'.join(part for part in artifact['groupId'].split('/') if part)
+    name, version = artifact['artifactId'].strip(), artifact['version'].strip()
+    if not namespace or not name or not version:
+        raise ValueError('Incomplete Maven coordinates')
+    encoded_namespace = quote(namespace, safe="!~*'():/")
+    encoded_name = quote(name, safe="!~*'()")
+    encoded_version = quote(version, safe="!~*'():+")
+    purl = f'pkg:maven/{encoded_namespace}/{encoded_name}@{encoded_version}'
+    qualifiers = {}
+    for key, field in [('classifier', 'classifiers'), ('type', 'types')]:
+        values = artifact.get(field) or []
+        if values and values[0].strip():
+            qualifiers[key] = values[0].strip()
+    if qualifiers:
+        purl += '?' + '&'.join(f'{key}={quote(value, safe="*").replace("~", "%7E")}'
+                               for key, value in sorted(qualifiers.items()))
+    return purl
+
+
+def manifest(graph, pom):
+    if not isinstance(graph, dict) or not isinstance(graph.get('graphName'), str) or not graph['graphName'].strip():
+        raise ValueError(f'Malformed graph name: {pom}')
+    artifacts, edges = graph.get('artifacts'), graph.get('dependencies') or []
+    if not isinstance(artifacts, list) or not artifacts or not isinstance(edges, list):
+        raise ValueError(f'Empty or malformed graph: {pom}')
+    if graph.get('dependencies') is not None and not isinstance(graph['dependencies'], list):
+        raise ValueError(f'Malformed dependency list: {pom}')
+    ids, packages, scopes = {}, {}, {}
+    for artifact in artifacts:
+        if not isinstance(artifact, dict) or any(not isinstance(artifact.get(key), str) or not artifact[key].strip()
+                                                for key in ['id', 'groupId', 'artifactId', 'version']):
+            raise ValueError(f'Malformed artifact: {pom}')
+        for key in ['scopes', 'types', 'classifiers']:
+            values = artifact.get(key)
+            if values is not None and (not isinstance(values, list) or any(not isinstance(x, str) for x in values)):
+                raise ValueError(f'Malformed artifact {key}: {pom}')
+        identifier, purl = artifact['id'], package_url(artifact)
+        if identifier in ids:
+            raise ValueError(f'Duplicate artifact ID: {identifier}')
+        ids[identifier] = purl
+        packages.setdefault(purl, [])
+        scopes[purl] = 'development' if set(artifact.get('scopes') or []) == {'test'} else 'runtime'
+    children, incoming = {}, set()
+    for edge in edges:
+        if not isinstance(edge, dict) or any(not isinstance(edge.get(key), str) for key in ['from', 'to']):
+            raise ValueError(f'Malformed dependency edge: {pom}')
+        source, target = edge['from'], edge['to']
+        if source not in ids or target not in ids:
+            raise ValueError(f'Unknown dependency edge: {pom}')
+        children.setdefault(source, []).append(target)
+        incoming.add(target)
+    # Aliased artifact IDs share one package and append all their child edges.
+    for source, targets in children.items():
+        packages[ids[source]].extend(ids[target] for target in targets)
+    direct = dict.fromkeys(target for source in ids if source not in incoming
+                           for target in children.get(source, []))
+    resolved = {}
+
+    def entry(purl, relationship):
+        return {'package_url': purl, 'relationship': relationship,
+                'scope': scopes[purl], 'dependencies': packages[purl]}
+
+    for identifier in direct:
+        purl = ids[identifier]
+        resolved[purl] = entry(purl, 'direct')
+        seen, pending = set(), list(reversed(packages[purl]))
+        while pending:
+            child = pending.pop()
+            if child in seen:
+                continue
+            seen.add(child)
+            resolved.setdefault(child, entry(child, 'indirect'))
+            pending.extend(reversed(packages[child]))
+    return {'name': graph['graphName'], 'file': {'source_location': pom}, 'resolved': resolved}
+
+
+def create_snapshot(workspace, poms, sha, repository, run_id, run_attempt):
+    manifests = {}
+    for pom in poms:
+        report = workspace / Path(pom).parent / 'target' / REPORT
+        if report.is_symlink() or report.stat().st_size > 25 * 1024 * 1024:
+            raise ValueError(f'Invalid report: {pom}')
+        item = manifest(json.loads(report.read_text(), object_pairs_hook=unique_object), pom)
+        if item['name'] in manifests:
+            raise ValueError(f'Duplicate graph name: {item["name"]}')
+        manifests[item['name']] = item
+    if not manifests or not any(item['resolved'] for item in manifests.values()):
+        raise ValueError('Refusing empty dependency inventory')
+    return {'version': 0, 'sha': sha, 'ref': 'refs/heads/master',
+            'job': {'id': run_id, 'correlator': CORRELATOR,
+                    'html_url': f'https://github.com/{repository}/actions/runs/{run_id}'},
+            'detector': DETECTOR, 'metadata': {'serializer': SERIALIZER, 'producer': 'generate',
+                                            'run_attempt': run_attempt},
+            'scanned': datetime.datetime.now(datetime.timezone.utc).isoformat().replace('+00:00', 'Z'),
+            'manifests': manifests}
+
+
+if __name__ == '__main__':
+    poms = subprocess.check_output(['git', 'ls-files', '-z', '*pom.xml']).decode().split('\0')
+    poms = [pom for pom in poms if Path(pom).name == 'pom.xml']
+    snapshot = create_snapshot(Path.cwd(), poms, os.environ['GITHUB_SHA'],
+                               os.environ['GITHUB_REPOSITORY'], os.environ['GITHUB_RUN_ID'],
+                               os.environ['GITHUB_RUN_ATTEMPT'])
+    Path(sys.argv[1]).write_text(json.dumps(snapshot, indent=2) + '\n')

.github/scripts/test_ci_policy.py

@@ -121,6 +121,19 @@ def test_shared_runtime_consumers_and_protocols(self):
         self.assertTrue({"pd", "store", "hstore", "cluster"}.issubset(policy.select(
             "server", ["hugegraph-pd/hg-pd-grpc/src/main/proto/common.proto"])))
 
+    def test_dependency_snapshot_inputs_do_not_fan_out_to_runtime_suites(self):
+        for path in [".github/scripts/dependency-snapshot.py", ".github/scripts/dependency-inventory.py",
+                     ".github/workflows/submit-dependencies.yml"]:
+            self.assertEqual({"dependency_license"}, set(self.plan_for([path])["selected"]))
+            self.assertEqual({"dependency_license"}, self.coordinator_jobs(self.plan_for([path])))
+        fixture = ".github/scripts/test_dependency_snapshot.py"
+        self.assertEqual([], self.plan_for([fixture])["expected"])
+        runtime = "hugegraph-server/hugegraph-core/src/main/A.java"
+        self.assertEqual(self.plan_for([runtime])["selected"], self.plan_for([runtime, fixture])["selected"])
+        # Classifier changes still receive conservative mainline validation.
+        self.assertEqual(policy.select("server", [".github/scripts/ci-policy.py"]),
+                         policy.select("server", [".github/scripts/ci-policy.py", fixture]))
+
     def test_noncore_platforms_only_follow_direct_inputs(self):
         for path in ["hugegraph-server/hugegraph-hbase/src/main/A.java",
                      "hugegraph-server/hugegraph-dist/src/assembly/travis/install-hbase.sh",