From 2c6b69e96bb297698a988f0d9653694f1c3c7b80 Mon Sep 17 00:00:00 2001 From: Hanasi Date: Tue, 3 Nov 2026 15:12:03 -0500 Subject: [PATCH 1/7] Use L2 Script from Docker repo --- .github/workflows/L2-tests.yml | 9 +- docs/rrd_l2_function_coverage_HLD.md | 41 +- .../generate_l2_coverage_report.py | 646 ------------------ 3 files changed, 40 insertions(+), 656 deletions(-) delete mode 100644 test/functional-tests/generate_l2_coverage_report.py diff --git a/.github/workflows/L2-tests.yml b/.github/workflows/L2-tests.yml index 8344ad6f9..72d1db57d 100644 --- a/.github/workflows/L2-tests.yml +++ b/.github/workflows/L2-tests.yml @@ -26,6 +26,12 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} ref: ${{ github.head_ref || github.ref }} + - name: Check out shared L2 coverage tooling + uses: actions/checkout@v4 + with: + repository: 'rdkcentral/docker-device-mgt-service-test' + path: docker-device-mgt-service-test + - name: Check out dependent repostiories uses: actions/checkout@v4 with: @@ -105,7 +111,8 @@ jobs: - name: Generate L2_Coverage.md if: (github.event_name == 'push' || github.event_name == 'pull_request') && steps.copy_tracefile.outcome == 'success' run: | - python3 remote_debugger/test/functional-tests/generate_l2_coverage_report.py \ + python3 docker-device-mgt-service-test/generate_l2_coverage_report.py \ + --component "Remote Debugger" \ --tracefile /tmp/coverage.info \ --src-dir remote_debugger/src \ --features-dir remote_debugger/test/functional-tests/features \ diff --git a/docs/rrd_l2_function_coverage_HLD.md b/docs/rrd_l2_function_coverage_HLD.md index a9abc59aa..8a77b02c4 100644 --- a/docs/rrd_l2_function_coverage_HLD.md +++ b/docs/rrd_l2_function_coverage_HLD.md @@ -183,7 +183,10 @@ The L2 suite terminates the daemon by signal during teardown in coverage mode. W ### 5.5 Coverage Report Generation -**Module:** `test/functional-tests/generate_l2_coverage_report.py` (as referenced by the workflow) +**Module:** `generate_l2_coverage_report.py` in the shared +`rdkcentral/docker-device-mgt-service-test` repository. The L2 workflow checks +out that repository alongside `remote_debugger` and invokes the shared script +with `--component "Remote Debugger"`. There is no module-local copy. **Purpose:** Convert the raw coverage trace and the feature/test inventory into a human-readable summary and actionable gap analysis. @@ -203,6 +206,22 @@ The L2 suite terminates the daemon by signal during teardown in coverage mode. W **Outputs:** - Markdown report with summary metrics, tables, and scenario coverage status +For local use, check out both repositories into the same parent directory and +run from that directory after collecting the tracefile: + +```bash +python3 docker-device-mgt-service-test/generate_l2_coverage_report.py \ + --component "Remote Debugger" \ + --tracefile /tmp/coverage.info \ + --src-dir remote_debugger/src \ + --features-dir remote_debugger/test/functional-tests/features \ + --tests-dir remote_debugger/test/functional-tests/tests \ + --output remote_debugger/test/functional-tests/L2_Coverage.md +``` + +The shared generator uses Python's standard library. See the shared repository's +README for reuse by other modules and report limitations. + ## 6. Functional Workflow ### 6.1 Build Flow @@ -225,10 +244,10 @@ The L2 suite terminates the daemon by signal during teardown in coverage mode. W 2. The Python report generator reads the trace file plus the feature/test directory metadata. 3. A Markdown report is produced with sections like: - coverage overview - - direct and indirect coverage counts + - source function hit and miss counts - mapped vs unmapped features - orphan features and orphan tests - - gap analysis by behavior group + - scenario-to-test count gaps 4. On push-based workflow execution, the report is committed to the branch for review. ## 7. Data and Interface Model @@ -256,13 +275,17 @@ The generator reads both and builds mappings to support consistent coverage narr The generated report documents the following: - total source-function estimate -- functions with direct L2 coverage -- functions with indirect coverage +- functions called at least once according to lcov - functions with no L2 coverage -- active vs disabled tests -- active feature scenarios and expected gaps -- per-behavior coverage tables -- feature/test gap summary by area such as static profile handling, dynamic profile handling, upload flow, and harmful command detection +- test-definition counts and disabled-test estimates +- feature scenario counts and expected count gaps +- per-source-file line and function coverage +- unmatched features/tests and scenario-to-test count gaps + +Static per-behavior coverage claims, RRD-specific infrastructure notes, and +hand-written priority recommendations are not generated by the shared script. +Keep such assessments in module-owned documentation and review them separately +from measured lcov coverage. Counts alone do not prove behavior coverage. ## 8. Error Handling and Boundary Conditions diff --git a/test/functional-tests/generate_l2_coverage_report.py b/test/functional-tests/generate_l2_coverage_report.py deleted file mode 100644 index 79d2b63ad..000000000 --- a/test/functional-tests/generate_l2_coverage_report.py +++ /dev/null @@ -1,646 +0,0 @@ -#!/usr/bin/env python3 -""" -Generate test/functional-tests/L2_Coverage.md. - -Inputs ------- - --tracefile lcov .info file produced by run_l2.sh - --src-dir directory containing the remotedebugger .c sources - --features-dir directory containing .feature files - --tests-dir directory containing test_*.py files - --output path for the generated markdown file - -Usage ------ - python3 generate_l2_coverage_report.py \ - --tracefile /tmp/l2_coverage/coverage.info \ - --src-dir src \ - --features-dir test/functional-tests/features \ - --tests-dir test/functional-tests/tests \ - --output test/functional-tests/L2_Coverage.md -""" - -import argparse -import datetime -import os -import re - - -# --------------------------------------------------------------------------- -# lcov tracefile parser -# --------------------------------------------------------------------------- - -def parse_tracefile(path): - """Return (per_file_dict, summary_dict) from an lcov .info file.""" - files = {} - cur = None - with open(path) as fh: - for raw in fh: - line = raw.rstrip() - if line.startswith('SF:'): - name = os.path.basename(line[3:]) - cur = {'name': name, 'lh': 0, 'lf': 0, - 'fnh': 0, 'fnf': 0, 'hit': set()} - elif line == 'end_of_record' and cur: - files[cur['name']] = cur - cur = None - elif cur: - if line.startswith('LH:'): cur['lh'] = int(line[3:]) - elif line.startswith('LF:'): cur['lf'] = int(line[3:]) - elif line.startswith('FNH:'): cur['fnh'] = int(line[4:]) - elif line.startswith('FNF:'): cur['fnf'] = int(line[4:]) - elif line.startswith('FNDA:'): - count, _, fname = line[5:].partition(',') - if int(count) > 0: - cur['hit'].add(fname) - - def tot(key): return sum(v[key] for v in files.values()) - summary = { - 'lines_pct': _pct(tot('lh'), tot('lf')), - 'lines_det': f"{tot('lh')} of {tot('lf')} lines", - } - return files, summary - - -def _pct(hit, found): - if found == 0: - return 'N/A' - return f'{hit / found * 100:.1f}%' - - -def _bar(hit, found, width=20): - if found == 0: - return '░' * width - filled = round(hit / found * width) - return '█' * filled + '░' * (width - filled) - - -# --------------------------------------------------------------------------- -# source function scanner -# --------------------------------------------------------------------------- - -_C_KEYWORDS = {'if', 'while', 'for', 'switch', 'else', 'return', 'sizeof', 'do'} - - -def _strip_c(code): - out, i, n = [], 0, len(code) - while i < n: - c = code[i] - if code.startswith('//', i): - j = code.find('\n', i) - i = n if j < 0 else j - elif code.startswith('/*', i): - j = code.find('*/', i + 2) - i = n if j < 0 else j + 2 - out.append(' ') - elif c in '"\'': - j = i + 1 - while j < n and code[j] != c: - j += 2 if code[j] == '\\' else 1 - i = j + 1 - out.append('""') - else: - out.append(c) - i += 1 - return ''.join(out) - - -def _c_function_names(path): - """Names of function definitions at brace depth 0; every #if branch counts.""" - with open(path, errors='replace') as fh: - text = _strip_c(fh.read()) - names, depth, buf = set(), 0, '' - for line in text.split('\n'): - if line.strip().startswith('#'): - continue - for ch in line: - if ch == '{': - if depth == 0: - m = re.search(r'([A-Za-z_]\w*)\s*\([^;{}]*\)\s*$', buf) - if m and m.group(1) not in _C_KEYWORDS: - names.add(m.group(1)) - buf = '' - depth += 1 - elif ch == '}': - depth -= 1 - if depth == 0: - buf = '' - elif depth == 0: - buf = '' if ch == ';' else buf + ch - if depth == 0: - buf += ' ' - return names - - -def scan_source_functions(src_dir): - """Return {c_file: set(function names)} for every .c file directly in src_dir.""" - result = {} - for fn in sorted(os.listdir(src_dir)): - p = os.path.join(src_dir, fn) - if fn.endswith('.c') and os.path.isfile(p): - result[fn] = _c_function_names(p) - return result - - -# --------------------------------------------------------------------------- -# test directory scanners -# --------------------------------------------------------------------------- - -def scan_features(features_dir): - """Return {filename: scenario_count}.""" - result = {} - for fn in sorted(os.listdir(features_dir)): - if fn.endswith('.feature'): - result[fn] = _count_scenarios(os.path.join(features_dir, fn)) - return result - - -def scan_tests(tests_dir): - """Return {filename: test_function_count}.""" - result = {} - for fn in sorted(os.listdir(tests_dir)): - if fn.startswith('test_') and fn.endswith('.py'): - result[fn] = _count_test_funcs(os.path.join(tests_dir, fn)) - return result - - -def _count_scenarios(path): - count = 0 - with open(path, errors='replace') as fh: - for line in fh: - if re.match(r'\s*Scenario(\s+Outline)?:', line): - count += 1 - return count - - -def _count_test_funcs(path): - count = 0 - with open(path, errors='replace') as fh: - for line in fh: - if re.match(r'^def test_', line): - count += 1 - return count - - -def scan_disabled_tests(tests_dir): - """Return (disabled_function_count, disabled_file_count). - - A test is disabled when its def is commented out or carries a skip marker; - a file is disabled when it has no active test functions left. - """ - funcs = files = 0 - for fn in sorted(os.listdir(tests_dir)): - if not (fn.startswith('test_') and fn.endswith('.py')): - continue - with open(os.path.join(tests_dir, fn), errors='replace') as fh: - text = fh.read() - active = len(re.findall(r'^def test_', text, re.M)) - skipped = len(re.findall( - r'^@pytest\.mark\.skip[^\n]*\n(?:@[^\n]*\n)*def test_', text, re.M)) - commented = len(re.findall(r'^\s*#\s*def test_', text, re.M)) - funcs += skipped + commented - if active - skipped == 0 and (commented or skipped or active == 0): - files += 1 - return funcs, files - - -# --------------------------------------------------------------------------- -# feature → test matching -# --------------------------------------------------------------------------- - -def build_mapping(features, tests): - """ - Return (pairs, orphan_features, orphan_tests). - pairs = list of (feature_file, scenarios, test_file_or_None, test_funcs_or_0) - """ - unmatched_tests = set(tests.keys()) - pairs = [] - for feat, scen in features.items(): - stem = feat.replace('.feature', '') - candidate = (stem if stem.startswith('test_') else 'test_' + stem) + '.py' - if candidate in tests: - pairs.append((feat, scen, candidate, tests[candidate])) - unmatched_tests.discard(candidate) - else: - pairs.append((feat, scen, None, 0)) - orphan_tests = {t: tests[t] for t in sorted(unmatched_tests)} - orphan_feats = [p[0] for p in pairs if p[2] is None] - return pairs, orphan_feats, orphan_tests - - -# --------------------------------------------------------------------------- -# static sections (priorities / recommendations — human judgment) -# --------------------------------------------------------------------------- - -_RECOMMENDATIONS = """\ -## 6. Gap Recommendations - -### Priority 1 — Must Fix - -| # | Gap | Impacted Modules | -|:---:|---|---| -| 1 | Implement missing `test_rrd_c_api_upload.py` scenarios (16 of 21 unimplemented) | `rrd_upload.c` | -| 2 | Add `rrd_profile_data.feature` for the existing `test_rrd_profile_data.py` | `rrdInterface.c` | -| 3 | Add WebCfg / MsgPack event L2 test (`rrd_webcfg_event.feature` + test) | `rrdMsgPackDecoder.c`, `rrdEventProcess.c` | - -### Priority 2 — Should Fix - -| # | Gap | Impacted Modules | -|:---:|---|---| -| 4 | Upload lock contention test | `rrd_upload.c` | -| 5 | Configuration fallback chain (RFC → DCM → dcm.properties) | `rrd_config.c` | -| 6 | Archive CPU throttle logic | `rrd_archive.c` | -| 7 | RDM download event with dynamic-profile cache miss | `rrdDynamic.c`, `rrdInterface.c` | -| 8 | Dynamic profile JSON parse failure | `rrdDynamic.c` | - -### Priority 3 — Nice to Have - -| # | Gap | Impacted Modules | -|:---:|---|---| -| 9 | RBUS registration / unregistration failure injection | `rrdInterface.c` | -| 10 | Message queue creation failure | `rrdMain.c` | -| 11 | Event thread creation failure | `rrdMain.c` | -| 12 | Directory creation / chdir failures | `rrdJsonParser.c` | -| 13 | `systemd-run` / `journalctl` execution failures | `rrdRunCmdThread.c` | -| 14 | Output file write errors | `rrdRunCmdThread.c` | -| 15 | Invalid deep sleep event type | `rrdDynamic.c` | -""" - - -def count_priorities(recommendations): - """Return (high, medium, low) row counts from the P1/P2/P3 recommendation tables.""" - counts = [] - for part in re.split(r'^### Priority \d', recommendations, flags=re.M)[1:4]: - counts.append(len(re.findall(r'^\|\s*\d+\s*\|', part, re.M))) - return tuple(counts) - - -_BEHAVIOR_DETAIL = """\ -## 4. Per-Behavior Coverage Detail - -> Legend — **YES**: tested by an L2 scenario | **NO**: no test exists | **PARTIAL**: subset covered - -### 4.1 Daemon Lifecycle - -| Behavior | Covered | -|---|:---:| -| RBUS subscription + event wait | YES | -| RFC enable → daemon starts | YES | -| RFC disable → daemon stops | YES | -| Single instance enforcement | YES | -| Message queue creation failure | NO | -| Event thread creation failure | NO | -| Signal handling / graceful shutdown | NO | -| Device info file read failure | NO | - -### 4.2 Static Profile Processing - -| Behavior | Covered | -|---|:---:| -| Config file exists check | YES | -| IssueType event trigger + message flow | YES | -| JSON parse success + command execution | YES | -| Upload report success / failure | YES | -| Category-only issue type | YES | -| Suffixed issue type | YES | -| Overlength suffix (negative) | YES | -| Background command execution | YES | -| Missing command in profile | YES | -| Corrupted / invalid JSON profile | YES | - -### 4.3 Dynamic Profile Processing - -| Behavior | Covered | -|---|:---:| -| Dynamic profile fallback (static miss) | YES | -| Dynamic subcategory | YES | -| Dynamic profile missing → RDM trigger | YES | -| Append mode (static + dynamic) | YES | -| Append when static not found | YES | -| RDM download event (cache miss) | NO | -| Dynamic profile JSON parse failure | NO | - -### 4.4 Harmful Command Detection - -| Behavior | Covered | -|---|:---:| -| Static profile harmful command abort | YES | -| Dynamic profile harmful command abort | YES | -| Macro replacement edge cases | NO | -| Background command modification | PARTIAL | - -### 4.5 Event Handling - -| Behavior | Covered | -|---|:---:| -| IssueType RBUS event | YES | -| Empty IssueType event | YES | -| Deep sleep event | YES | -| WebCfg event (MsgPack decode) | NO | -| WebCfg corrupted data | NO | -| Multiple simultaneous IssueType events | NO | -| Invalid deep sleep event type | NO | - -### 4.6 Upload & Archive - -| Behavior | Covered | -|---|:---:| -| Upload via shell script | YES | -| Upload + download validation | YES | -| C API `rrd_upload_orchestrate` (happy path) | YES | -| C API NULL parameters | NO | -| C API empty / non-existent directory | NO | -| C API config loading / MAC retrieval | NO | -| C API archive creation + cleanup | NO | -| Concurrent upload lock | NO | -| Archive CPU throttle | NO | - -### 4.7 Profile Data SET/GET (RBUS) - -| Behavior | Covered | -|---|:---:| -| `setProfileData` / `getProfileData` via RBUS | PARTIAL (no feature file) | -| Profile category load error | NO | -| Profile file write error | NO | -""" - -_SRC_MODULES = [ - 'rrdMain.c', 'rrdInterface.c', 'rrdEventProcess.c', 'rrdJsonParser.c', - 'rrdRunCmdThread.c', 'rrdCommandSanity.c', 'rrdDynamic.c', - 'rrdExecuteScript.c', 'rrdMsgPackDecoder.c', - 'rrd_config.c', 'rrd_sysinfo.c', 'rrd_logproc.c', - 'rrd_archive.c', 'rrd_upload.c', 'rrdIarmEvents.c', 'uploadRRDLogs.c', -] - - -# --------------------------------------------------------------------------- -# markdown builder -# --------------------------------------------------------------------------- - -def generate(pairs, orphan_feats, orphan_tests, features, files, summary, today, - src_funcs, disabled_funcs, disabled_files): - total_scenarios = sum(p[1] for p in pairs) - total_test_funcs = sum(p[3] for p in pairs) + sum(orphan_tests.values()) - total_features = len(pairs) - total_tests = len(pairs) - len(orphan_feats) + len(orphan_tests) - mapped = sum(1 for p in pairs if p[2]) - gap_pairs = [p for p in pairs if p[2] and p[3] < p[1]] - - # A source function is covered when lcov recorded at least one call to it. - fn_total = sum(len(v) for v in src_funcs.values()) - fn_hit = sum(len(src_funcs[m] & files[m]['hit']) for m in src_funcs if m in files) - fn_miss = fn_total - fn_hit - high, medium, low = count_priorities(_RECOMMENDATIONS) - - lines = [] - - # ── header ──────────────────────────────────────────────────────────────── - lines += [ - '# Remote Debugger L2 Coverage Report', - '', - f'**Generated:** {today} ', - '**Component:** `remotedebugger` (`src/`) ', - '**Test suite:** `test/functional-tests/` ', - '**Coverage tool:** lcov (source-level instrumentation via `--coverage`)', - '', - '---', - '', - ] - - # ── executive summary ───────────────────────────────────────────────────── - lines += [ - '## 1. Executive Summary', - '', - '**L2 Test Summary**', - '', - '| Metric | Value |', - '|---|:---:|', - f'| Feature files | {total_features} |', - f'| Feature scenarios | {total_scenarios} |', - f'| Pytest files | {total_tests} |', - f'| Test functions (`test_*`) | {total_test_funcs} |', - f'| Mapped feature/test pairs | {mapped} / {total_features} |', - f'| Unmatched test files | {len(orphan_tests)} |', - f'| Line coverage (lcov) | {summary["lines_pct"]} ({summary["lines_det"]}) |', - '', - '**L2 Test Coverage**', - '', - '| Metric | Value |', - '|---|:---:|', - f'| Source functions found | {fn_total} |', - f'| Functions hit | {fn_hit} |', - f'| Functions not hit | {fn_miss} |', - f'| Current functional coverage | {_pct(fn_hit, fn_total)} |', - '| Target L2 functional coverage | ~100% |', - '', - '> Function coverage = functions called at least once during the L2 run (lcov `FNDA` > 0),', - '> divided by all function definitions in `src/*.c`, including those compiled out by `#if`.', - '', - ] - - # ── feature ↔ test mapping ──────────────────────────────────────────────── - lines += [ - '---', - '', - '## 2. Feature ↔ Test Mapping', - '', - '### 2.1 Mapped Pairs', - '', - '| # | Feature File | Scenarios | Test File | Tests | Status |', - '|:---:|---|:---:|---|:---:|:---:|', - ] - idx = 1 - for feat, scen, test, tfuncs in pairs: - if test is None: - continue - delta = tfuncs - scen - status = 'PASS' if delta >= 0 else f'**GAP** ({abs(delta)} missing)' - lines.append(f'| {idx} | `{feat}` | {scen} | `{test}` | {tfuncs} | {status} |') - idx += 1 - - total_scen_mapped = sum(p[1] for p in pairs if p[2]) - total_funcs_mapped = sum(p[3] for p in pairs if p[2]) - lines += [ - f'| | **Totals** | **{total_scen_mapped}** | | **{total_funcs_mapped}** | |', - '', - ] - - over_pairs = [p for p in pairs if p[2] and p[3] > p[1]] - if over_pairs: - lines += [ - '### 2.2 Tests Exceeding Scenario Count', - '', - '> Extra functions handle setup/teardown split more granularly than feature scenarios.', - '', - '| Feature File | Scenarios | Test File | Tests | Extra |', - '|---|:---:|---|:---:|:---:|', - ] - for feat, scen, test, tfuncs in over_pairs: - lines.append(f'| `{feat}` | {scen} | `{test}` | {tfuncs} | +{tfuncs - scen} |') - lines.append('') - - if orphan_tests: - lines += [ - '### 2.3 Orphan Tests (test exists, no feature file)', - '', - '| Test File | Tests | Note |', - '|---|:---:|---|', - ] - for tf, cnt in orphan_tests.items(): - lines.append(f'| `{tf}` | {cnt} | **Missing `.feature` file** |') - lines.append('') - - if orphan_feats: - lines += [ - '### 2.4 Orphan Features (feature exists, no test file)', - '', - '| Feature File | Scenarios | Note |', - '|---|:---:|---|', - ] - for ff in orphan_feats: - lines.append(f'| `{ff}` | {features[ff]} | **Missing test file** |') - lines.append('') - - # ── per-module lcov coverage ─────────────────────────────────────────────── - lines += [ - '---', - '', - '## 3. Source Module Coverage (lcov)', - '', - '| Module | Lines | Functions | Coverage Bar |', - '|---|:---:|:---:|---|', - ] - for mod in _SRC_MODULES: - if mod in files: - d = files[mod] - if mod in src_funcs: - f_tot, f_hit = len(src_funcs[mod]), len(src_funcs[mod] & d['hit']) - else: - f_tot, f_hit = d['fnf'], d['fnh'] - lines.append( - f'| `{mod}` | {_pct(d["lh"], d["lf"])} ({d["lh"]}/{d["lf"]}) ' - f'| {_pct(f_hit, f_tot)} ({f_hit}/{f_tot}) ' - f'| `{_bar(d["lh"], d["lf"])}` |' - ) - else: - lines.append(f'| `{mod}` | — | — | no data |') - lines.append('') - - # ── per-behavior detail (static) ────────────────────────────────────────── - lines += ['---', '', _BEHAVIOR_DETAIL, ''] - - # ── gap analysis ────────────────────────────────────────────────────────── - lines += [ - '---', - '', - '## 5. Scenario-to-Test Gap Analysis', - '', - ] - if gap_pairs: - lines += [ - '| Feature File | Scenarios | Tests | Missing |', - '|---|:---:|:---:|:---:|', - ] - for feat, scen, test, tfuncs in gap_pairs: - lines.append(f'| `{feat}` | {scen} | {tfuncs} | **{scen - tfuncs}** |') - lines.append('') - else: - lines += ['All mapped feature files have sufficient test function coverage.', ''] - - if orphan_tests: - lines += [ - '**Orphan tests** (no feature file — behavior is tested but not documented):', - '', - ] - for tf in orphan_tests: - lines.append(f'- `{tf}`') - lines.append('') - - # ── recommendations (static) ───────────────────────────────────────────── - lines += ['---', '', _RECOMMENDATIONS, ''] - - # ── test infrastructure notes ──────────────────────────────────────────── - lines += [ - '---', - '', - '## 7. Test Infrastructure Notes', - '', - '| Interface | Tool | Used By |', - '|---|---|---|', - '| RBUS event trigger | `rbuscli set` | All event-based tests |', - '| RBUS profile data | `rbuscli set/get` | `test_rrd_profile_data.py` |', - '| Log scraping | `grep_rrdlogs()` in `helper_functions.py` | All tests |', - '| Process control | `pidof`, `kill`, `nohup` | Setup / teardown |', - '| Mock upload server | Mock xconf server | `test_rrd_debug_report_upload.py` |', - '| Mock upload script | `uploadSTBLogs.sh` | Upload tests |', - '', - '- **Runner:** `pytest`, sequential per file', - '- **Shared state:** tests within a file depend on prior test side effects', - '- **Coverage run:** `kill_rrd()` sends `SIGTERM` when `RRD_COVERAGE_MODE` is set so gcov flushes', - '', - ] - - # ── appendix ────────────────────────────────────────────────────────────── - lines += [ - '---', - '', - '## 8. Appendix: File Inventory', - '', - '### Feature Files', - '', - '| # | File | Scenarios |', - '|:---:|---|:---:|', - ] - for i, (feat, scen, *_) in enumerate(pairs, 1): - lines.append(f'| {i} | `{feat}` | {scen} |') - lines.append(f'| | **Total** | **{total_scenarios}** |') - lines.append('') - - lines += [ - '### Test Files', - '', - '| # | File | Tests |', - '|:---:|---|:---:|', - ] - all_tests = [(p[2], p[3]) for p in pairs if p[2]] + list(orphan_tests.items()) - all_tests.sort() - for i, (tf, cnt) in enumerate(all_tests, 1): - lines.append(f'| {i} | `{tf}` | {cnt} |') - lines.append(f'| | **Total** | **{total_test_funcs}** |') - lines.append('') - - return '\n'.join(lines) - - -# --------------------------------------------------------------------------- -# entry point -# --------------------------------------------------------------------------- - -def main(): - ap = argparse.ArgumentParser() - ap.add_argument('--tracefile', required=True) - ap.add_argument('--src-dir', required=True) - ap.add_argument('--features-dir', required=True) - ap.add_argument('--tests-dir', required=True) - ap.add_argument('--output', required=True) - args = ap.parse_args() - - features = scan_features(args.features_dir) - tests = scan_tests(args.tests_dir) - cov_files, summary = parse_tracefile(args.tracefile) - pairs, orphan_feats, orphan_tests = build_mapping(features, tests) - today = datetime.date.today().strftime('%Y-%m-%d') - - md = generate(pairs, orphan_feats, orphan_tests, features, cov_files, summary, today, - scan_source_functions(args.src_dir), - *scan_disabled_tests(args.tests_dir)) - - os.makedirs(os.path.dirname(os.path.abspath(args.output)), exist_ok=True) - with open(args.output, 'w', encoding='utf-8') as fh: - fh.write(md) - - print(f'Written: {args.output}') - - -if __name__ == '__main__': - main() From d7ad917e5531a4e8b042b3e72e4a55835578d150 Mon Sep 17 00:00:00 2001 From: Hanasi Date: Tue, 3 Nov 2026 15:15:30 -0500 Subject: [PATCH 2/7] test branch --- .github/workflows/L2-tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/L2-tests.yml b/.github/workflows/L2-tests.yml index 72d1db57d..7662adbc4 100644 --- a/.github/workflows/L2-tests.yml +++ b/.github/workflows/L2-tests.yml @@ -30,6 +30,7 @@ jobs: uses: actions/checkout@v4 with: repository: 'rdkcentral/docker-device-mgt-service-test' + ref: 'feature/l2docupdate' path: docker-device-mgt-service-test - name: Check out dependent repostiories From 6e9016895c835fff316d54c2563c83a954dd953b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 3 Nov 2026 20:38:14 +0000 Subject: [PATCH 3/7] ci: regenerate L2_Coverage.md [skip ci] --- test/functional-tests/L2_Coverage.md | 398 ++++++--------------------- 1 file changed, 83 insertions(+), 315 deletions(-) diff --git a/test/functional-tests/L2_Coverage.md b/test/functional-tests/L2_Coverage.md index 4e09e80bc..03afbe0db 100644 --- a/test/functional-tests/L2_Coverage.md +++ b/test/functional-tests/L2_Coverage.md @@ -1,339 +1,107 @@ # Remote Debugger L2 Coverage Report -**Generated:** 2026-10-28 -**Component:** `remotedebugger` (`src/`) -**Test suite:** `test/functional-tests/` -**Coverage tool:** lcov (source-level instrumentation via `--coverage`) - ---- +**Generated:** 2026-11-03 +**Component:** `Remote Debugger` +**Source directory:** `remote_debugger/src` +**Features:** `remote_debugger/test/functional-tests/features` +**Tests:** `remote_debugger/test/functional-tests/tests` +**Coverage tool:** lcov ## 1. Executive Summary -**L2 Test Summary** - | Metric | Value | |---|:---:| | Feature files | 22 | | Feature scenarios | 103 | | Pytest files | 23 | -| Test functions (`test_*`) | 118 | +| Test functions (`test_*`, including skipped) | 118 | +| Disabled test functions (skip markers or commented definitions) | 0 | +| Files with no enabled test functions | 0 | | Mapped feature/test pairs | 19 / 22 | | Unmatched test files | 4 | | Line coverage (lcov) | 49.7% (1508 of 3034 lines) | - -**L2 Test Coverage** - -| Metric | Value | -|---|:---:| | Source functions found | 128 | | Functions hit | 76 | | Functions not hit | 52 | | Current functional coverage | 59.4% | -| Target L2 functional coverage | ~100% | - -> Function coverage = functions called at least once during the L2 run (lcov `FNDA` > 0), -> divided by all function definitions in `src/*.c`, including those compiled out by `#if`. - ---- - -## 2. Feature ↔ Test Mapping - -### 2.1 Mapped Pairs - -| # | Feature File | Scenarios | Test File | Tests | Status | -|:---:|---|:---:|---|:---:|:---:| -| 1 | `rrd_append_report.feature` | 4 | `test_rrd_append_report.py` | 7 | PASS | -| 2 | `rrd_background_cmd_static_profile_report.feature` | 5 | `test_rrd_background_cmd_static_profile_report.py` | 5 | PASS | -| 3 | `rrd_c_api_upload.feature` | 21 | `test_rrd_c_api_upload.py` | 5 | **GAP** (16 missing) | -| 4 | `rrd_corrupted_static_profile_report.feature` | 4 | `test_rrd_corrupted_static_profile_report.py` | 4 | PASS | -| 5 | `rrd_debug_report_upload.feature` | 6 | `test_rrd_debug_report_upload.py` | 6 | PASS | -| 6 | `rrd_deepsleep_static_report.feature` | 2 | `test_rrd_deepsleep_static_report.py` | 5 | PASS | -| 7 | `rrd_dynamic_profile_missing_report.feature` | 4 | `test_rrd_dynamic_profile_missing_report.py` | 7 | PASS | -| 8 | `rrd_dynamic_profile_report.feature` | 5 | `test_rrd_dynamic_profile_report.py` | 9 | PASS | -| 9 | `rrd_empty_issuetype_event.feature` | 2 | `test_rrd_empty_issuetype_event.py` | 2 | PASS | -| 10 | `rrd_harmful_command_static_report.feature` | 5 | `test_rrd_harmful_command_static_report.py` | 5 | PASS | -| 11 | `rrd_single_instance.feature` | 1 | `test_rrd_single_instance.py` | 3 | PASS | -| 12 | `rrd_start_control.feature` | 2 | `test_rrd_start_control.py` | 1 | **GAP** (1 missing) | -| 13 | `rrd_start_subscribe_and_wait.feature` | 1 | `test_rrd_start_subscribe_and_wait.py` | 4 | PASS | -| 14 | `rrd_static_profile_category_report.feature` | 5 | `test_rrd_static_profile_category_report.py` | 5 | PASS | -| 15 | `rrd_static_profile_missing_command_report.feature` | 5 | `test_rrd_static_profile_missing_command_report.py` | 5 | PASS | -| 16 | `rrd_static_profile_report.feature` | 5 | `test_rrd_static_profile_report.py` | 5 | PASS | -| 17 | `test_rrd_dynamic_profile_harmful_report.feature` | 5 | `test_rrd_dynamic_profile_harmful_report.py` | 7 | PASS | -| 18 | `test_rrd_static_profile_report_with_suffix.feature` | 4 | `test_rrd_static_profile_report_with_suffix.py` | 5 | PASS | -| 19 | `test_rrd_static_profile_report_with_suffix_negative_case.feature` | 4 | `test_rrd_static_profile_report_with_suffix_negative_case.py` | 5 | PASS | -| | **Totals** | **90** | | **95** | | - -### 2.2 Tests Exceeding Scenario Count - -> Extra functions handle setup/teardown split more granularly than feature scenarios. - -| Feature File | Scenarios | Test File | Tests | Extra | -|---|:---:|---|:---:|:---:| -| `rrd_append_report.feature` | 4 | `test_rrd_append_report.py` | 7 | +3 | -| `rrd_deepsleep_static_report.feature` | 2 | `test_rrd_deepsleep_static_report.py` | 5 | +3 | -| `rrd_dynamic_profile_missing_report.feature` | 4 | `test_rrd_dynamic_profile_missing_report.py` | 7 | +3 | -| `rrd_dynamic_profile_report.feature` | 5 | `test_rrd_dynamic_profile_report.py` | 9 | +4 | -| `rrd_single_instance.feature` | 1 | `test_rrd_single_instance.py` | 3 | +2 | -| `rrd_start_subscribe_and_wait.feature` | 1 | `test_rrd_start_subscribe_and_wait.py` | 4 | +3 | -| `test_rrd_dynamic_profile_harmful_report.feature` | 5 | `test_rrd_dynamic_profile_harmful_report.py` | 7 | +2 | -| `test_rrd_static_profile_report_with_suffix.feature` | 4 | `test_rrd_static_profile_report_with_suffix.py` | 5 | +1 | -| `test_rrd_static_profile_report_with_suffix_negative_case.feature` | 4 | `test_rrd_static_profile_report_with_suffix_negative_case.py` | 5 | +1 | - -### 2.3 Orphan Tests (test exists, no feature file) - -| Test File | Tests | Note | -|---|:---:|---| -| `test_rrd_append_dynamic_profile_static_notfound.py` | 7 | **Missing `.feature` file** | -| `test_rrd_dynamic_profile_rdm_node_length_exceeded.py` | 6 | **Missing `.feature` file** | -| `test_rrd_dynamic_subcategory_report.py` | 7 | **Missing `.feature` file** | -| `test_rrd_profile_data.py` | 3 | **Missing `.feature` file** | - -### 2.4 Orphan Features (feature exists, no test file) -| Feature File | Scenarios | Note | -|---|:---:|---| -| `rrd_append_dynamic_profile_static_not_found.feature` | 4 | **Missing test file** | -| `rrd_dynamic_profile_node_length_exceeded.feature` | 4 | **Missing test file** | -| `rrd_dynamic_profile_subcategory_report.feature` | 5 | **Missing test file** | - ---- - -## 3. Source Module Coverage (lcov) - -| Module | Lines | Functions | Coverage Bar | -|---|:---:|:---:|---| -| `rrdMain.c` | 60.3% (38/63) | 75.0% (3/4) | `████████████░░░░░░░░` | -| `rrdInterface.c` | 44.0% (189/430) | 63.6% (14/22) | `█████████░░░░░░░░░░░` | -| `rrdEventProcess.c` | 68.8% (240/349) | 81.8% (9/11) | `██████████████░░░░░░` | -| `rrdJsonParser.c` | 81.8% (405/495) | 86.7% (13/15) | `████████████████░░░░` | -| `rrdRunCmdThread.c` | 55.2% (117/212) | 60.0% (6/10) | `███████████░░░░░░░░░` | -| `rrdCommandSanity.c` | 93.1% (67/72) | 100.0% (3/3) | `███████████████████░` | -| `rrdDynamic.c` | 40.4% (59/146) | 40.0% (2/5) | `████████░░░░░░░░░░░░` | -| `rrdExecuteScript.c` | 94.1% (16/17) | 100.0% (2/2) | `███████████████████░` | -| `rrdMsgPackDecoder.c` | 0.0% (0/280) | 0.0% (0/15) | `░░░░░░░░░░░░░░░░░░░░` | -| `rrd_config.c` | 42.8% (95/222) | 66.7% (6/9) | `█████████░░░░░░░░░░░` | -| `rrd_sysinfo.c` | 32.1% (36/112) | 33.3% (2/6) | `██████░░░░░░░░░░░░░░` | -| `rrd_logproc.c` | 48.6% (36/74) | 75.0% (3/4) | `██████████░░░░░░░░░░` | -| `rrd_archive.c` | 47.2% (126/267) | 72.7% (8/11) | `█████████░░░░░░░░░░░` | -| `rrd_upload.c` | 40.7% (37/91) | 60.0% (3/5) | `████████░░░░░░░░░░░░` | -| `rrdIarmEvents.c` | 8.6% (12/140) | 20.0% (1/5) | `██░░░░░░░░░░░░░░░░░░` | -| `uploadRRDLogs.c` | 54.7% (35/64) | 100.0% (1/1) | `███████████░░░░░░░░░` | - ---- - -## 4. Per-Behavior Coverage Detail - -> Legend — **YES**: tested by an L2 scenario | **NO**: no test exists | **PARTIAL**: subset covered - -### 4.1 Daemon Lifecycle - -| Behavior | Covered | +> Function coverage compares lcov FNDA counts with a heuristic scan of C function +> definitions under the source directory, including preprocessor branches. +> Test counts are a static inventory, not pass/fail results. Skip counts are +> estimates; conditional, class-level and module-level skips are not evaluated. + +## 2. Feature-to-Test Mapping + +| Feature File | Scenarios | Test File | Tests | Status | +|---|:---:|---|:---:|---| +| `rrd_append_dynamic_profile_static_not_found.feature` | 4 | - | 0 | Missing test file | +| `rrd_append_report.feature` | 4 | test_rrd_append_report.py | 7 | Count sufficient | +| `rrd_background_cmd_static_profile_report.feature` | 5 | test_rrd_background_cmd_static_profile_report.py | 5 | Count sufficient | +| `rrd_c_api_upload.feature` | 21 | test_rrd_c_api_upload.py | 5 | GAP (16 missing) | +| `rrd_corrupted_static_profile_report.feature` | 4 | test_rrd_corrupted_static_profile_report.py | 4 | Count sufficient | +| `rrd_debug_report_upload.feature` | 6 | test_rrd_debug_report_upload.py | 6 | Count sufficient | +| `rrd_deepsleep_static_report.feature` | 2 | test_rrd_deepsleep_static_report.py | 5 | Count sufficient | +| `rrd_dynamic_profile_missing_report.feature` | 4 | test_rrd_dynamic_profile_missing_report.py | 7 | Count sufficient | +| `rrd_dynamic_profile_node_length_exceeded.feature` | 4 | - | 0 | Missing test file | +| `rrd_dynamic_profile_report.feature` | 5 | test_rrd_dynamic_profile_report.py | 9 | Count sufficient | +| `rrd_dynamic_profile_subcategory_report.feature` | 5 | - | 0 | Missing test file | +| `rrd_empty_issuetype_event.feature` | 2 | test_rrd_empty_issuetype_event.py | 2 | Count sufficient | +| `rrd_harmful_command_static_report.feature` | 5 | test_rrd_harmful_command_static_report.py | 5 | Count sufficient | +| `rrd_single_instance.feature` | 1 | test_rrd_single_instance.py | 3 | Count sufficient | +| `rrd_start_control.feature` | 2 | test_rrd_start_control.py | 1 | GAP (1 missing) | +| `rrd_start_subscribe_and_wait.feature` | 1 | test_rrd_start_subscribe_and_wait.py | 4 | Count sufficient | +| `rrd_static_profile_category_report.feature` | 5 | test_rrd_static_profile_category_report.py | 5 | Count sufficient | +| `rrd_static_profile_missing_command_report.feature` | 5 | test_rrd_static_profile_missing_command_report.py | 5 | Count sufficient | +| `rrd_static_profile_report.feature` | 5 | test_rrd_static_profile_report.py | 5 | Count sufficient | +| `test_rrd_dynamic_profile_harmful_report.feature` | 5 | test_rrd_dynamic_profile_harmful_report.py | 7 | Count sufficient | +| `test_rrd_static_profile_report_with_suffix.feature` | 4 | test_rrd_static_profile_report_with_suffix.py | 5 | Count sufficient | +| `test_rrd_static_profile_report_with_suffix_negative_case.feature` | 4 | test_rrd_static_profile_report_with_suffix_negative_case.py | 5 | Count sufficient | + +> Matching uses the same relative directory and feature stem, with an optional +> test_ prefix. Scenario Outline rows and parametrized cases are not expanded. + +### Unmatched Test Files + +| Test File | Tests | |---|:---:| -| RBUS subscription + event wait | YES | -| RFC enable → daemon starts | YES | -| RFC disable → daemon stops | YES | -| Single instance enforcement | YES | -| Message queue creation failure | NO | -| Event thread creation failure | NO | -| Signal handling / graceful shutdown | NO | -| Device info file read failure | NO | +| `test_rrd_append_dynamic_profile_static_notfound.py` | 7 | +| `test_rrd_dynamic_profile_rdm_node_length_exceeded.py` | 6 | +| `test_rrd_dynamic_subcategory_report.py` | 7 | +| `test_rrd_profile_data.py` | 3 | -### 4.2 Static Profile Processing - -| Behavior | Covered | -|---|:---:| -| Config file exists check | YES | -| IssueType event trigger + message flow | YES | -| JSON parse success + command execution | YES | -| Upload report success / failure | YES | -| Category-only issue type | YES | -| Suffixed issue type | YES | -| Overlength suffix (negative) | YES | -| Background command execution | YES | -| Missing command in profile | YES | -| Corrupted / invalid JSON profile | YES | - -### 4.3 Dynamic Profile Processing - -| Behavior | Covered | -|---|:---:| -| Dynamic profile fallback (static miss) | YES | -| Dynamic subcategory | YES | -| Dynamic profile missing → RDM trigger | YES | -| Append mode (static + dynamic) | YES | -| Append when static not found | YES | -| RDM download event (cache miss) | NO | -| Dynamic profile JSON parse failure | NO | - -### 4.4 Harmful Command Detection - -| Behavior | Covered | -|---|:---:| -| Static profile harmful command abort | YES | -| Dynamic profile harmful command abort | YES | -| Macro replacement edge cases | NO | -| Background command modification | PARTIAL | - -### 4.5 Event Handling - -| Behavior | Covered | -|---|:---:| -| IssueType RBUS event | YES | -| Empty IssueType event | YES | -| Deep sleep event | YES | -| WebCfg event (MsgPack decode) | NO | -| WebCfg corrupted data | NO | -| Multiple simultaneous IssueType events | NO | -| Invalid deep sleep event type | NO | - -### 4.6 Upload & Archive - -| Behavior | Covered | -|---|:---:| -| Upload via shell script | YES | -| Upload + download validation | YES | -| C API `rrd_upload_orchestrate` (happy path) | YES | -| C API NULL parameters | NO | -| C API empty / non-existent directory | NO | -| C API config loading / MAC retrieval | NO | -| C API archive creation + cleanup | NO | -| Concurrent upload lock | NO | -| Archive CPU throttle | NO | - -### 4.7 Profile Data SET/GET (RBUS) - -| Behavior | Covered | -|---|:---:| -| `setProfileData` / `getProfileData` via RBUS | PARTIAL (no feature file) | -| Profile category load error | NO | -| Profile file write error | NO | - - ---- +## 3. Source Module Coverage (lcov) -## 5. Scenario-to-Test Gap Analysis +| Module | Lines | Functions | +|---|:---:|:---:| +| `rrdCommandSanity.c` | 93.1% (67/72) | 100.0% (3/3) | +| `rrdDynamic.c` | 40.4% (59/146) | 40.0% (2/5) | +| `rrdEventProcess.c` | 68.8% (240/349) | 81.8% (9/11) | +| `rrdExecuteScript.c` | 94.1% (16/17) | 100.0% (2/2) | +| `rrdIarmEvents.c` | 8.6% (12/140) | 20.0% (1/5) | +| `rrdInterface.c` | 44.0% (189/430) | 63.6% (14/22) | +| `rrdJsonParser.c` | 81.8% (405/495) | 86.7% (13/15) | +| `rrdMain.c` | 60.3% (38/63) | 75.0% (3/4) | +| `rrdMsgPackDecoder.c` | 0.0% (0/280) | 0.0% (0/15) | +| `rrdRunCmdThread.c` | 55.2% (117/212) | 60.0% (6/10) | +| `rrd_archive.c` | 47.2% (126/267) | 72.7% (8/11) | +| `rrd_config.c` | 42.8% (95/222) | 66.7% (6/9) | +| `rrd_logproc.c` | 48.6% (36/74) | 75.0% (3/4) | +| `rrd_sysinfo.c` | 32.1% (36/112) | 33.3% (2/6) | +| `rrd_upload.c` | 40.7% (37/91) | 60.0% (3/5) | +| `uploadRRDLogs.c` | 54.7% (35/64) | 100.0% (1/1) | + +> Only trace records matched to source files in this module contribute to line coverage. +> Ambiguous source paths are excluded; keep source-relative paths in the tracefile. + +## 4. Scenario-to-Test Gap Analysis | Feature File | Scenarios | Tests | Missing | |---|:---:|:---:|:---:| -| `rrd_c_api_upload.feature` | 21 | 5 | **16** | -| `rrd_start_control.feature` | 2 | 1 | **1** | - -**Orphan tests** (no feature file — behavior is tested but not documented): - -- `test_rrd_append_dynamic_profile_static_notfound.py` -- `test_rrd_dynamic_profile_rdm_node_length_exceeded.py` -- `test_rrd_dynamic_subcategory_report.py` -- `test_rrd_profile_data.py` - ---- - -## 6. Gap Recommendations - -### Priority 1 — Must Fix - -| # | Gap | Impacted Modules | -|:---:|---|---| -| 1 | Implement missing `test_rrd_c_api_upload.py` scenarios (16 of 21 unimplemented) | `rrd_upload.c` | -| 2 | Add `rrd_profile_data.feature` for the existing `test_rrd_profile_data.py` | `rrdInterface.c` | -| 3 | Add WebCfg / MsgPack event L2 test (`rrd_webcfg_event.feature` + test) | `rrdMsgPackDecoder.c`, `rrdEventProcess.c` | - -### Priority 2 — Should Fix - -| # | Gap | Impacted Modules | -|:---:|---|---| -| 4 | Upload lock contention test | `rrd_upload.c` | -| 5 | Configuration fallback chain (RFC → DCM → dcm.properties) | `rrd_config.c` | -| 6 | Archive CPU throttle logic | `rrd_archive.c` | -| 7 | RDM download event with dynamic-profile cache miss | `rrdDynamic.c`, `rrdInterface.c` | -| 8 | Dynamic profile JSON parse failure | `rrdDynamic.c` | - -### Priority 3 — Nice to Have - -| # | Gap | Impacted Modules | -|:---:|---|---| -| 9 | RBUS registration / unregistration failure injection | `rrdInterface.c` | -| 10 | Message queue creation failure | `rrdMain.c` | -| 11 | Event thread creation failure | `rrdMain.c` | -| 12 | Directory creation / chdir failures | `rrdJsonParser.c` | -| 13 | `systemd-run` / `journalctl` execution failures | `rrdRunCmdThread.c` | -| 14 | Output file write errors | `rrdRunCmdThread.c` | -| 15 | Invalid deep sleep event type | `rrdDynamic.c` | - - ---- - -## 7. Test Infrastructure Notes - -| Interface | Tool | Used By | -|---|---|---| -| RBUS event trigger | `rbuscli set` | All event-based tests | -| RBUS profile data | `rbuscli set/get` | `test_rrd_profile_data.py` | -| Log scraping | `grep_rrdlogs()` in `helper_functions.py` | All tests | -| Process control | `pidof`, `kill`, `nohup` | Setup / teardown | -| Mock upload server | Mock xconf server | `test_rrd_debug_report_upload.py` | -| Mock upload script | `uploadSTBLogs.sh` | Upload tests | - -- **Runner:** `pytest`, sequential per file -- **Shared state:** tests within a file depend on prior test side effects -- **Coverage run:** `kill_rrd()` sends `SIGTERM` when `RRD_COVERAGE_MODE` is set so gcov flushes - ---- - -## 8. Appendix: File Inventory - -### Feature Files - -| # | File | Scenarios | -|:---:|---|:---:| -| 1 | `rrd_append_dynamic_profile_static_not_found.feature` | 4 | -| 2 | `rrd_append_report.feature` | 4 | -| 3 | `rrd_background_cmd_static_profile_report.feature` | 5 | -| 4 | `rrd_c_api_upload.feature` | 21 | -| 5 | `rrd_corrupted_static_profile_report.feature` | 4 | -| 6 | `rrd_debug_report_upload.feature` | 6 | -| 7 | `rrd_deepsleep_static_report.feature` | 2 | -| 8 | `rrd_dynamic_profile_missing_report.feature` | 4 | -| 9 | `rrd_dynamic_profile_node_length_exceeded.feature` | 4 | -| 10 | `rrd_dynamic_profile_report.feature` | 5 | -| 11 | `rrd_dynamic_profile_subcategory_report.feature` | 5 | -| 12 | `rrd_empty_issuetype_event.feature` | 2 | -| 13 | `rrd_harmful_command_static_report.feature` | 5 | -| 14 | `rrd_single_instance.feature` | 1 | -| 15 | `rrd_start_control.feature` | 2 | -| 16 | `rrd_start_subscribe_and_wait.feature` | 1 | -| 17 | `rrd_static_profile_category_report.feature` | 5 | -| 18 | `rrd_static_profile_missing_command_report.feature` | 5 | -| 19 | `rrd_static_profile_report.feature` | 5 | -| 20 | `test_rrd_dynamic_profile_harmful_report.feature` | 5 | -| 21 | `test_rrd_static_profile_report_with_suffix.feature` | 4 | -| 22 | `test_rrd_static_profile_report_with_suffix_negative_case.feature` | 4 | -| | **Total** | **103** | - -### Test Files - -| # | File | Tests | -|:---:|---|:---:| -| 1 | `test_rrd_append_dynamic_profile_static_notfound.py` | 7 | -| 2 | `test_rrd_append_report.py` | 7 | -| 3 | `test_rrd_background_cmd_static_profile_report.py` | 5 | -| 4 | `test_rrd_c_api_upload.py` | 5 | -| 5 | `test_rrd_corrupted_static_profile_report.py` | 4 | -| 6 | `test_rrd_debug_report_upload.py` | 6 | -| 7 | `test_rrd_deepsleep_static_report.py` | 5 | -| 8 | `test_rrd_dynamic_profile_harmful_report.py` | 7 | -| 9 | `test_rrd_dynamic_profile_missing_report.py` | 7 | -| 10 | `test_rrd_dynamic_profile_rdm_node_length_exceeded.py` | 6 | -| 11 | `test_rrd_dynamic_profile_report.py` | 9 | -| 12 | `test_rrd_dynamic_subcategory_report.py` | 7 | -| 13 | `test_rrd_empty_issuetype_event.py` | 2 | -| 14 | `test_rrd_harmful_command_static_report.py` | 5 | -| 15 | `test_rrd_profile_data.py` | 3 | -| 16 | `test_rrd_single_instance.py` | 3 | -| 17 | `test_rrd_start_control.py` | 1 | -| 18 | `test_rrd_start_subscribe_and_wait.py` | 4 | -| 19 | `test_rrd_static_profile_category_report.py` | 5 | -| 20 | `test_rrd_static_profile_missing_command_report.py` | 5 | -| 21 | `test_rrd_static_profile_report.py` | 5 | -| 22 | `test_rrd_static_profile_report_with_suffix.py` | 5 | -| 23 | `test_rrd_static_profile_report_with_suffix_negative_case.py` | 5 | -| | **Total** | **118** | +| `rrd_append_dynamic_profile_static_not_found.feature` | 4 | 0 | 4 | +| `rrd_c_api_upload.feature` | 21 | 5 | 16 | +| `rrd_dynamic_profile_node_length_exceeded.feature` | 4 | 0 | 4 | +| `rrd_dynamic_profile_subcategory_report.feature` | 5 | 0 | 5 | +| `rrd_start_control.feature` | 2 | 1 | 1 | + +> Counts alone do not prove behavior coverage. Module-specific behavior analysis +> and recommendations belong in the module documentation. From 45f247638caea16ad881c52c45e35af3a07621c6 Mon Sep 17 00:00:00 2001 From: Hanasi Date: Tue, 3 Nov 2026 15:53:55 -0500 Subject: [PATCH 4/7] Update the additional details file l2_coverage_details.json --- .github/workflows/L2-tests.yml | 1 + docs/rrd_l2_function_coverage_HLD.md | 26 +++- .../functional-tests/l2_coverage_details.json | 144 ++++++++++++++++++ 3 files changed, 165 insertions(+), 6 deletions(-) create mode 100644 test/functional-tests/l2_coverage_details.json diff --git a/.github/workflows/L2-tests.yml b/.github/workflows/L2-tests.yml index 7662adbc4..0849c1f3f 100644 --- a/.github/workflows/L2-tests.yml +++ b/.github/workflows/L2-tests.yml @@ -114,6 +114,7 @@ jobs: run: | python3 docker-device-mgt-service-test/generate_l2_coverage_report.py \ --component "Remote Debugger" \ + --report-details remote_debugger/test/functional-tests/l2_coverage_details.json \ --tracefile /tmp/coverage.info \ --src-dir remote_debugger/src \ --features-dir remote_debugger/test/functional-tests/features \ diff --git a/docs/rrd_l2_function_coverage_HLD.md b/docs/rrd_l2_function_coverage_HLD.md index 8a77b02c4..93384c50c 100644 --- a/docs/rrd_l2_function_coverage_HLD.md +++ b/docs/rrd_l2_function_coverage_HLD.md @@ -186,7 +186,9 @@ The L2 suite terminates the daemon by signal during teardown in coverage mode. W **Module:** `generate_l2_coverage_report.py` in the shared `rdkcentral/docker-device-mgt-service-test` repository. The L2 workflow checks out that repository alongside `remote_debugger` and invokes the shared script -with `--component "Remote Debugger"`. There is no module-local copy. +with `--component "Remote Debugger"` and the module-owned +`test/functional-tests/l2_coverage_details.json` details file. There is no +module-local generator copy. **Purpose:** Convert the raw coverage trace and the feature/test inventory into a human-readable summary and actionable gap analysis. @@ -202,16 +204,20 @@ with `--component "Remote Debugger"`. There is no module-local copy. - feature directory - test directory - source tree metadata +- `test/functional-tests/l2_coverage_details.json` for display metadata, source + ordering, behavior assessments, recommendations, and infrastructure notes **Outputs:** - Markdown report with summary metrics, tables, and scenario coverage status -For local use, check out both repositories into the same parent directory and -run from that directory after collecting the tracefile: +For local use, check out both repositories into the same parent directory, using +`feature/l2docupdate` for `docker-device-mgt-service-test` to match the PR workflow. +Run from that parent directory after collecting the tracefile: ```bash python3 docker-device-mgt-service-test/generate_l2_coverage_report.py \ --component "Remote Debugger" \ + --report-details remote_debugger/test/functional-tests/l2_coverage_details.json \ --tracefile /tmp/coverage.info \ --src-dir remote_debugger/src \ --features-dir remote_debugger/test/functional-tests/features \ @@ -221,6 +227,8 @@ python3 docker-device-mgt-service-test/generate_l2_coverage_report.py \ The shared generator uses Python's standard library. See the shared repository's README for reuse by other modules and report limitations. +Update or remove the shared-tooling branch ref after these changes are available +on that repository's default branch. ## 6. Functional Workflow @@ -281,11 +289,17 @@ The generated report documents the following: - feature scenario counts and expected count gaps - per-source-file line and function coverage - unmatched features/tests and scenario-to-test count gaps +- excess tests, mapped-pair totals, source coverage bars, and full file inventories +- module-supplied behavior tables, priority recommendations, and infrastructure notes +The report retains all eight sections of the original `L2_Coverage.md`. Static per-behavior coverage claims, RRD-specific infrastructure notes, and -hand-written priority recommendations are not generated by the shared script. -Keep such assessments in module-owned documentation and review them separately -from measured lcov coverage. Counts alone do not prove behavior coverage. +hand-written priority recommendations are supplied by +`test/functional-tests/l2_coverage_details.json`, not inferred by the shared +script. The initial details preserve the existing report's assessments; module +owners must review them when behavior or tests change. Measured totals, mapping +tables, gaps, and inventories are recomputed on every run. Counts alone do not +prove behavior coverage. ## 8. Error Handling and Boundary Conditions diff --git a/test/functional-tests/l2_coverage_details.json b/test/functional-tests/l2_coverage_details.json new file mode 100644 index 000000000..9302bde70 --- /dev/null +++ b/test/functional-tests/l2_coverage_details.json @@ -0,0 +1,144 @@ +{ + "component_id": "remotedebugger", + "source_label": "src/", + "test_suite": "test/functional-tests/", + "target_coverage": "~100%", + "source_order": [ + "rrdMain.c", "rrdInterface.c", "rrdEventProcess.c", "rrdJsonParser.c", + "rrdRunCmdThread.c", "rrdCommandSanity.c", "rrdDynamic.c", + "rrdExecuteScript.c", "rrdMsgPackDecoder.c", + "rrd_config.c", "rrd_sysinfo.c", "rrd_logproc.c", + "rrd_archive.c", "rrd_upload.c", "rrdIarmEvents.c", "uploadRRDLogs.c" + ], + "behavior_detail": [ + "> Legend \u2014 **YES**: tested by an L2 scenario | **NO**: no test exists | **PARTIAL**: subset covered", + "", + "### 4.1 Daemon Lifecycle", + "", + "| Behavior | Covered |", + "|---|:---:|", + "| RBUS subscription + event wait | YES |", + "| RFC enable \u2192 daemon starts | YES |", + "| RFC disable \u2192 daemon stops | YES |", + "| Single instance enforcement | YES |", + "| Message queue creation failure | NO |", + "| Event thread creation failure | NO |", + "| Signal handling / graceful shutdown | NO |", + "| Device info file read failure | NO |", + "", + "### 4.2 Static Profile Processing", + "", + "| Behavior | Covered |", + "|---|:---:|", + "| Config file exists check | YES |", + "| IssueType event trigger + message flow | YES |", + "| JSON parse success + command execution | YES |", + "| Upload report success / failure | YES |", + "| Category-only issue type | YES |", + "| Suffixed issue type | YES |", + "| Overlength suffix (negative) | YES |", + "| Background command execution | YES |", + "| Missing command in profile | YES |", + "| Corrupted / invalid JSON profile | YES |", + "", + "### 4.3 Dynamic Profile Processing", + "", + "| Behavior | Covered |", + "|---|:---:|", + "| Dynamic profile fallback (static miss) | YES |", + "| Dynamic subcategory | YES |", + "| Dynamic profile missing \u2192 RDM trigger | YES |", + "| Append mode (static + dynamic) | YES |", + "| Append when static not found | YES |", + "| RDM download event (cache miss) | NO |", + "| Dynamic profile JSON parse failure | NO |", + "", + "### 4.4 Harmful Command Detection", + "", + "| Behavior | Covered |", + "|---|:---:|", + "| Static profile harmful command abort | YES |", + "| Dynamic profile harmful command abort | YES |", + "| Macro replacement edge cases | NO |", + "| Background command modification | PARTIAL |", + "", + "### 4.5 Event Handling", + "", + "| Behavior | Covered |", + "|---|:---:|", + "| IssueType RBUS event | YES |", + "| Empty IssueType event | YES |", + "| Deep sleep event | YES |", + "| WebCfg event (MsgPack decode) | NO |", + "| WebCfg corrupted data | NO |", + "| Multiple simultaneous IssueType events | NO |", + "| Invalid deep sleep event type | NO |", + "", + "### 4.6 Upload & Archive", + "", + "| Behavior | Covered |", + "|---|:---:|", + "| Upload via shell script | YES |", + "| Upload + download validation | YES |", + "| C API `rrd_upload_orchestrate` (happy path) | YES |", + "| C API NULL parameters | NO |", + "| C API empty / non-existent directory | NO |", + "| C API config loading / MAC retrieval | NO |", + "| C API archive creation + cleanup | NO |", + "| Concurrent upload lock | NO |", + "| Archive CPU throttle | NO |", + "", + "### 4.7 Profile Data SET/GET (RBUS)", + "", + "| Behavior | Covered |", + "|---|:---:|", + "| `setProfileData` / `getProfileData` via RBUS | PARTIAL (no feature file) |", + "| Profile category load error | NO |", + "| Profile file write error | NO |" + ], + "recommendations": [ + "### Priority 1 \u2014 Must Fix", + "", + "| # | Gap | Impacted Modules |", + "|:---:|---|---|", + "| 1 | Implement missing `test_rrd_c_api_upload.py` scenarios (16 of 21 unimplemented) | `rrd_upload.c` |", + "| 2 | Add `rrd_profile_data.feature` for the existing `test_rrd_profile_data.py` | `rrdInterface.c` |", + "| 3 | Add WebCfg / MsgPack event L2 test (`rrd_webcfg_event.feature` + test) | `rrdMsgPackDecoder.c`, `rrdEventProcess.c` |", + "", + "### Priority 2 \u2014 Should Fix", + "", + "| # | Gap | Impacted Modules |", + "|:---:|---|---|", + "| 4 | Upload lock contention test | `rrd_upload.c` |", + "| 5 | Configuration fallback chain (RFC \u2192 DCM \u2192 dcm.properties) | `rrd_config.c` |", + "| 6 | Archive CPU throttle logic | `rrd_archive.c` |", + "| 7 | RDM download event with dynamic-profile cache miss | `rrdDynamic.c`, `rrdInterface.c` |", + "| 8 | Dynamic profile JSON parse failure | `rrdDynamic.c` |", + "", + "### Priority 3 \u2014 Nice to Have", + "", + "| # | Gap | Impacted Modules |", + "|:---:|---|---|", + "| 9 | RBUS registration / unregistration failure injection | `rrdInterface.c` |", + "| 10 | Message queue creation failure | `rrdMain.c` |", + "| 11 | Event thread creation failure | `rrdMain.c` |", + "| 12 | Directory creation / chdir failures | `rrdJsonParser.c` |", + "| 13 | `systemd-run` / `journalctl` execution failures | `rrdRunCmdThread.c` |", + "| 14 | Output file write errors | `rrdRunCmdThread.c` |", + "| 15 | Invalid deep sleep event type | `rrdDynamic.c` |" + ], + "infrastructure_notes": [ + "| Interface | Tool | Used By |", + "|---|---|---|", + "| RBUS event trigger | `rbuscli set` | All event-based tests |", + "| RBUS profile data | `rbuscli set/get` | `test_rrd_profile_data.py` |", + "| Log scraping | `grep_rrdlogs()` in `helper_functions.py` | All tests |", + "| Process control | `pidof`, `kill`, `nohup` | Setup / teardown |", + "| Mock upload server | Mock xconf server | `test_rrd_debug_report_upload.py` |", + "| Mock upload script | `uploadSTBLogs.sh` | Upload tests |", + "", + "- **Runner:** `pytest`, sequential per file", + "- **Shared state:** tests within a file depend on prior test side effects", + "- **Coverage run:** `kill_rrd()` sends `SIGTERM` when `RRD_COVERAGE_MODE` is set so gcov flushes" + ] +} \ No newline at end of file From fccd7646f35c1013d2bec005fb540cf664725e1f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 3 Nov 2026 21:15:34 +0000 Subject: [PATCH 5/7] ci: regenerate L2_Coverage.md [skip ci] --- test/functional-tests/L2_Coverage.md | 399 ++++++++++++++++++++++----- 1 file changed, 324 insertions(+), 75 deletions(-) diff --git a/test/functional-tests/L2_Coverage.md b/test/functional-tests/L2_Coverage.md index 03afbe0db..73209c29a 100644 --- a/test/functional-tests/L2_Coverage.md +++ b/test/functional-tests/L2_Coverage.md @@ -1,107 +1,356 @@ # Remote Debugger L2 Coverage Report **Generated:** 2026-11-03 -**Component:** `Remote Debugger` -**Source directory:** `remote_debugger/src` -**Features:** `remote_debugger/test/functional-tests/features` -**Tests:** `remote_debugger/test/functional-tests/tests` -**Coverage tool:** lcov +**Component:** `remotedebugger` (`src/`) +**Test suite:** `test/functional-tests/` +**Coverage tool:** lcov (source-level instrumentation via `--coverage`) + +--- ## 1. Executive Summary +**L2 Test Coverage** + +| Metric | Value | +|---|:---:| +| Source functions found | 128 | +| Functions hit | 76 | +| Functions not hit | 52 | +| Current functional coverage | 59.4% | +| Target L2 functional coverage | ~100% | + +> Function coverage = functions called at least once during the L2 run (lcov `FNDA` > 0), +> divided by all function definitions in `src/*.c`, including those compiled out by `#if`. + +**L2 Test Summary** + | Metric | Value | |---|:---:| | Feature files | 22 | | Feature scenarios | 103 | | Pytest files | 23 | -| Test functions (`test_*`, including skipped) | 118 | -| Disabled test functions (skip markers or commented definitions) | 0 | -| Files with no enabled test functions | 0 | +| Test functions (`test_*`) | 118 | | Mapped feature/test pairs | 19 / 22 | | Unmatched test files | 4 | | Line coverage (lcov) | 49.7% (1508 of 3034 lines) | -| Source functions found | 128 | -| Functions hit | 76 | -| Functions not hit | 52 | -| Current functional coverage | 59.4% | -> Function coverage compares lcov FNDA counts with a heuristic scan of C function -> definitions under the source directory, including preprocessor branches. -> Test counts are a static inventory, not pass/fail results. Skip counts are +> Function definitions are discovered with a heuristic C source scan. +> Disabled test functions (skip markers or commented definitions): 0; +> files with no enabled test functions: 0. +> Test counts include skipped definitions and are not pass/fail results. Skip counts are > estimates; conditional, class-level and module-level skips are not evaluated. -## 2. Feature-to-Test Mapping - -| Feature File | Scenarios | Test File | Tests | Status | -|---|:---:|---|:---:|---| -| `rrd_append_dynamic_profile_static_not_found.feature` | 4 | - | 0 | Missing test file | -| `rrd_append_report.feature` | 4 | test_rrd_append_report.py | 7 | Count sufficient | -| `rrd_background_cmd_static_profile_report.feature` | 5 | test_rrd_background_cmd_static_profile_report.py | 5 | Count sufficient | -| `rrd_c_api_upload.feature` | 21 | test_rrd_c_api_upload.py | 5 | GAP (16 missing) | -| `rrd_corrupted_static_profile_report.feature` | 4 | test_rrd_corrupted_static_profile_report.py | 4 | Count sufficient | -| `rrd_debug_report_upload.feature` | 6 | test_rrd_debug_report_upload.py | 6 | Count sufficient | -| `rrd_deepsleep_static_report.feature` | 2 | test_rrd_deepsleep_static_report.py | 5 | Count sufficient | -| `rrd_dynamic_profile_missing_report.feature` | 4 | test_rrd_dynamic_profile_missing_report.py | 7 | Count sufficient | -| `rrd_dynamic_profile_node_length_exceeded.feature` | 4 | - | 0 | Missing test file | -| `rrd_dynamic_profile_report.feature` | 5 | test_rrd_dynamic_profile_report.py | 9 | Count sufficient | -| `rrd_dynamic_profile_subcategory_report.feature` | 5 | - | 0 | Missing test file | -| `rrd_empty_issuetype_event.feature` | 2 | test_rrd_empty_issuetype_event.py | 2 | Count sufficient | -| `rrd_harmful_command_static_report.feature` | 5 | test_rrd_harmful_command_static_report.py | 5 | Count sufficient | -| `rrd_single_instance.feature` | 1 | test_rrd_single_instance.py | 3 | Count sufficient | -| `rrd_start_control.feature` | 2 | test_rrd_start_control.py | 1 | GAP (1 missing) | -| `rrd_start_subscribe_and_wait.feature` | 1 | test_rrd_start_subscribe_and_wait.py | 4 | Count sufficient | -| `rrd_static_profile_category_report.feature` | 5 | test_rrd_static_profile_category_report.py | 5 | Count sufficient | -| `rrd_static_profile_missing_command_report.feature` | 5 | test_rrd_static_profile_missing_command_report.py | 5 | Count sufficient | -| `rrd_static_profile_report.feature` | 5 | test_rrd_static_profile_report.py | 5 | Count sufficient | -| `test_rrd_dynamic_profile_harmful_report.feature` | 5 | test_rrd_dynamic_profile_harmful_report.py | 7 | Count sufficient | -| `test_rrd_static_profile_report_with_suffix.feature` | 4 | test_rrd_static_profile_report_with_suffix.py | 5 | Count sufficient | -| `test_rrd_static_profile_report_with_suffix_negative_case.feature` | 4 | test_rrd_static_profile_report_with_suffix_negative_case.py | 5 | Count sufficient | +--- +## 2. Feature ↔ Test Mapping + +### 2.1 Mapped Pairs + +| # | Feature File | Scenarios | Test File | Tests | Status | +|:---:|---|:---:|---|:---:|:---:| +| 1 | `rrd_append_report.feature` | 4 | `test_rrd_append_report.py` | 7 | PASS | +| 2 | `rrd_background_cmd_static_profile_report.feature` | 5 | `test_rrd_background_cmd_static_profile_report.py` | 5 | PASS | +| 3 | `rrd_c_api_upload.feature` | 21 | `test_rrd_c_api_upload.py` | 5 | **GAP** (16 missing) | +| 4 | `rrd_corrupted_static_profile_report.feature` | 4 | `test_rrd_corrupted_static_profile_report.py` | 4 | PASS | +| 5 | `rrd_debug_report_upload.feature` | 6 | `test_rrd_debug_report_upload.py` | 6 | PASS | +| 6 | `rrd_deepsleep_static_report.feature` | 2 | `test_rrd_deepsleep_static_report.py` | 5 | PASS | +| 7 | `rrd_dynamic_profile_missing_report.feature` | 4 | `test_rrd_dynamic_profile_missing_report.py` | 7 | PASS | +| 8 | `rrd_dynamic_profile_report.feature` | 5 | `test_rrd_dynamic_profile_report.py` | 9 | PASS | +| 9 | `rrd_empty_issuetype_event.feature` | 2 | `test_rrd_empty_issuetype_event.py` | 2 | PASS | +| 10 | `rrd_harmful_command_static_report.feature` | 5 | `test_rrd_harmful_command_static_report.py` | 5 | PASS | +| 11 | `rrd_single_instance.feature` | 1 | `test_rrd_single_instance.py` | 3 | PASS | +| 12 | `rrd_start_control.feature` | 2 | `test_rrd_start_control.py` | 1 | **GAP** (1 missing) | +| 13 | `rrd_start_subscribe_and_wait.feature` | 1 | `test_rrd_start_subscribe_and_wait.py` | 4 | PASS | +| 14 | `rrd_static_profile_category_report.feature` | 5 | `test_rrd_static_profile_category_report.py` | 5 | PASS | +| 15 | `rrd_static_profile_missing_command_report.feature` | 5 | `test_rrd_static_profile_missing_command_report.py` | 5 | PASS | +| 16 | `rrd_static_profile_report.feature` | 5 | `test_rrd_static_profile_report.py` | 5 | PASS | +| 17 | `test_rrd_dynamic_profile_harmful_report.feature` | 5 | `test_rrd_dynamic_profile_harmful_report.py` | 7 | PASS | +| 18 | `test_rrd_static_profile_report_with_suffix.feature` | 4 | `test_rrd_static_profile_report_with_suffix.py` | 5 | PASS | +| 19 | `test_rrd_static_profile_report_with_suffix_negative_case.feature` | 4 | `test_rrd_static_profile_report_with_suffix_negative_case.py` | 5 | PASS | +| | **Totals** | **90** | | **95** | | + +> PASS means the test-definition count meets the scenario count, not that tests passed. > Matching uses the same relative directory and feature stem, with an optional > test_ prefix. Scenario Outline rows and parametrized cases are not expanded. -### Unmatched Test Files +### 2.2 Tests Exceeding Scenario Count -| Test File | Tests | -|---|:---:| -| `test_rrd_append_dynamic_profile_static_notfound.py` | 7 | -| `test_rrd_dynamic_profile_rdm_node_length_exceeded.py` | 6 | -| `test_rrd_dynamic_subcategory_report.py` | 7 | -| `test_rrd_profile_data.py` | 3 | +> Extra functions can include setup/teardown split more granularly than feature scenarios. + +| Feature File | Scenarios | Test File | Tests | Extra | +|---|:---:|---|:---:|:---:| +| `rrd_append_report.feature` | 4 | `test_rrd_append_report.py` | 7 | +3 | +| `rrd_deepsleep_static_report.feature` | 2 | `test_rrd_deepsleep_static_report.py` | 5 | +3 | +| `rrd_dynamic_profile_missing_report.feature` | 4 | `test_rrd_dynamic_profile_missing_report.py` | 7 | +3 | +| `rrd_dynamic_profile_report.feature` | 5 | `test_rrd_dynamic_profile_report.py` | 9 | +4 | +| `rrd_single_instance.feature` | 1 | `test_rrd_single_instance.py` | 3 | +2 | +| `rrd_start_subscribe_and_wait.feature` | 1 | `test_rrd_start_subscribe_and_wait.py` | 4 | +3 | +| `test_rrd_dynamic_profile_harmful_report.feature` | 5 | `test_rrd_dynamic_profile_harmful_report.py` | 7 | +2 | +| `test_rrd_static_profile_report_with_suffix.feature` | 4 | `test_rrd_static_profile_report_with_suffix.py` | 5 | +1 | +| `test_rrd_static_profile_report_with_suffix_negative_case.feature` | 4 | `test_rrd_static_profile_report_with_suffix_negative_case.py` | 5 | +1 | + +### 2.3 Orphan Tests (test exists, no feature file) + +| Test File | Tests | Note | +|---|:---:|---| +| `test_rrd_append_dynamic_profile_static_notfound.py` | 7 | **Missing `.feature` file** | +| `test_rrd_dynamic_profile_rdm_node_length_exceeded.py` | 6 | **Missing `.feature` file** | +| `test_rrd_dynamic_subcategory_report.py` | 7 | **Missing `.feature` file** | +| `test_rrd_profile_data.py` | 3 | **Missing `.feature` file** | + +### 2.4 Orphan Features (feature exists, no test file) + +| Feature File | Scenarios | Note | +|---|:---:|---| +| `rrd_append_dynamic_profile_static_not_found.feature` | 4 | **Missing test file** | +| `rrd_dynamic_profile_node_length_exceeded.feature` | 4 | **Missing test file** | +| `rrd_dynamic_profile_subcategory_report.feature` | 5 | **Missing test file** | + +--- ## 3. Source Module Coverage (lcov) -| Module | Lines | Functions | -|---|:---:|:---:| -| `rrdCommandSanity.c` | 93.1% (67/72) | 100.0% (3/3) | -| `rrdDynamic.c` | 40.4% (59/146) | 40.0% (2/5) | -| `rrdEventProcess.c` | 68.8% (240/349) | 81.8% (9/11) | -| `rrdExecuteScript.c` | 94.1% (16/17) | 100.0% (2/2) | -| `rrdIarmEvents.c` | 8.6% (12/140) | 20.0% (1/5) | -| `rrdInterface.c` | 44.0% (189/430) | 63.6% (14/22) | -| `rrdJsonParser.c` | 81.8% (405/495) | 86.7% (13/15) | -| `rrdMain.c` | 60.3% (38/63) | 75.0% (3/4) | -| `rrdMsgPackDecoder.c` | 0.0% (0/280) | 0.0% (0/15) | -| `rrdRunCmdThread.c` | 55.2% (117/212) | 60.0% (6/10) | -| `rrd_archive.c` | 47.2% (126/267) | 72.7% (8/11) | -| `rrd_config.c` | 42.8% (95/222) | 66.7% (6/9) | -| `rrd_logproc.c` | 48.6% (36/74) | 75.0% (3/4) | -| `rrd_sysinfo.c` | 32.1% (36/112) | 33.3% (2/6) | -| `rrd_upload.c` | 40.7% (37/91) | 60.0% (3/5) | -| `uploadRRDLogs.c` | 54.7% (35/64) | 100.0% (1/1) | +| Module | Lines | Functions | Coverage Bar | +|---|:---:|:---:|---| +| `rrdMain.c` | 60.3% (38/63) | 75.0% (3/4) | `████████████░░░░░░░░` | +| `rrdInterface.c` | 44.0% (189/430) | 63.6% (14/22) | `█████████░░░░░░░░░░░` | +| `rrdEventProcess.c` | 68.8% (240/349) | 81.8% (9/11) | `██████████████░░░░░░` | +| `rrdJsonParser.c` | 81.8% (405/495) | 86.7% (13/15) | `████████████████░░░░` | +| `rrdRunCmdThread.c` | 55.2% (117/212) | 60.0% (6/10) | `███████████░░░░░░░░░` | +| `rrdCommandSanity.c` | 93.1% (67/72) | 100.0% (3/3) | `███████████████████░` | +| `rrdDynamic.c` | 40.4% (59/146) | 40.0% (2/5) | `████████░░░░░░░░░░░░` | +| `rrdExecuteScript.c` | 94.1% (16/17) | 100.0% (2/2) | `███████████████████░` | +| `rrdMsgPackDecoder.c` | 0.0% (0/280) | 0.0% (0/15) | `░░░░░░░░░░░░░░░░░░░░` | +| `rrd_config.c` | 42.8% (95/222) | 66.7% (6/9) | `█████████░░░░░░░░░░░` | +| `rrd_sysinfo.c` | 32.1% (36/112) | 33.3% (2/6) | `██████░░░░░░░░░░░░░░` | +| `rrd_logproc.c` | 48.6% (36/74) | 75.0% (3/4) | `██████████░░░░░░░░░░` | +| `rrd_archive.c` | 47.2% (126/267) | 72.7% (8/11) | `█████████░░░░░░░░░░░` | +| `rrd_upload.c` | 40.7% (37/91) | 60.0% (3/5) | `████████░░░░░░░░░░░░` | +| `rrdIarmEvents.c` | 8.6% (12/140) | 20.0% (1/5) | `██░░░░░░░░░░░░░░░░░░` | +| `uploadRRDLogs.c` | 54.7% (35/64) | 100.0% (1/1) | `███████████░░░░░░░░░` | > Only trace records matched to source files in this module contribute to line coverage. > Ambiguous source paths are excluded; keep source-relative paths in the tracefile. -## 4. Scenario-to-Test Gap Analysis +--- + +## 4. Per-Behavior Coverage Detail + +> Module-supplied assessment; not inferred from lcov or test counts. + +> Legend — **YES**: tested by an L2 scenario | **NO**: no test exists | **PARTIAL**: subset covered + +### 4.1 Daemon Lifecycle + +| Behavior | Covered | +|---|:---:| +| RBUS subscription + event wait | YES | +| RFC enable → daemon starts | YES | +| RFC disable → daemon stops | YES | +| Single instance enforcement | YES | +| Message queue creation failure | NO | +| Event thread creation failure | NO | +| Signal handling / graceful shutdown | NO | +| Device info file read failure | NO | + +### 4.2 Static Profile Processing + +| Behavior | Covered | +|---|:---:| +| Config file exists check | YES | +| IssueType event trigger + message flow | YES | +| JSON parse success + command execution | YES | +| Upload report success / failure | YES | +| Category-only issue type | YES | +| Suffixed issue type | YES | +| Overlength suffix (negative) | YES | +| Background command execution | YES | +| Missing command in profile | YES | +| Corrupted / invalid JSON profile | YES | + +### 4.3 Dynamic Profile Processing + +| Behavior | Covered | +|---|:---:| +| Dynamic profile fallback (static miss) | YES | +| Dynamic subcategory | YES | +| Dynamic profile missing → RDM trigger | YES | +| Append mode (static + dynamic) | YES | +| Append when static not found | YES | +| RDM download event (cache miss) | NO | +| Dynamic profile JSON parse failure | NO | + +### 4.4 Harmful Command Detection + +| Behavior | Covered | +|---|:---:| +| Static profile harmful command abort | YES | +| Dynamic profile harmful command abort | YES | +| Macro replacement edge cases | NO | +| Background command modification | PARTIAL | + +### 4.5 Event Handling + +| Behavior | Covered | +|---|:---:| +| IssueType RBUS event | YES | +| Empty IssueType event | YES | +| Deep sleep event | YES | +| WebCfg event (MsgPack decode) | NO | +| WebCfg corrupted data | NO | +| Multiple simultaneous IssueType events | NO | +| Invalid deep sleep event type | NO | + +### 4.6 Upload & Archive + +| Behavior | Covered | +|---|:---:| +| Upload via shell script | YES | +| Upload + download validation | YES | +| C API `rrd_upload_orchestrate` (happy path) | YES | +| C API NULL parameters | NO | +| C API empty / non-existent directory | NO | +| C API config loading / MAC retrieval | NO | +| C API archive creation + cleanup | NO | +| Concurrent upload lock | NO | +| Archive CPU throttle | NO | + +### 4.7 Profile Data SET/GET (RBUS) + +| Behavior | Covered | +|---|:---:| +| `setProfileData` / `getProfileData` via RBUS | PARTIAL (no feature file) | +| Profile category load error | NO | +| Profile file write error | NO | + +--- + +## 5. Scenario-to-Test Gap Analysis | Feature File | Scenarios | Tests | Missing | |---|:---:|:---:|:---:| -| `rrd_append_dynamic_profile_static_not_found.feature` | 4 | 0 | 4 | -| `rrd_c_api_upload.feature` | 21 | 5 | 16 | -| `rrd_dynamic_profile_node_length_exceeded.feature` | 4 | 0 | 4 | -| `rrd_dynamic_profile_subcategory_report.feature` | 5 | 0 | 5 | -| `rrd_start_control.feature` | 2 | 1 | 1 | - -> Counts alone do not prove behavior coverage. Module-specific behavior analysis -> and recommendations belong in the module documentation. +| `rrd_c_api_upload.feature` | 21 | 5 | **16** | +| `rrd_start_control.feature` | 2 | 1 | **1** | + +**Orphan tests** (no feature file): + +- `test_rrd_append_dynamic_profile_static_notfound.py` +- `test_rrd_dynamic_profile_rdm_node_length_exceeded.py` +- `test_rrd_dynamic_subcategory_report.py` +- `test_rrd_profile_data.py` + +> Unmatched features are listed in section 2.4; counts alone do not prove behavior coverage. + +--- + +## 6. Gap Recommendations + +> Module-supplied recommendations; review when the implementation or tests change. + +### Priority 1 — Must Fix + +| # | Gap | Impacted Modules | +|:---:|---|---| +| 1 | Implement missing `test_rrd_c_api_upload.py` scenarios (16 of 21 unimplemented) | `rrd_upload.c` | +| 2 | Add `rrd_profile_data.feature` for the existing `test_rrd_profile_data.py` | `rrdInterface.c` | +| 3 | Add WebCfg / MsgPack event L2 test (`rrd_webcfg_event.feature` + test) | `rrdMsgPackDecoder.c`, `rrdEventProcess.c` | + +### Priority 2 — Should Fix + +| # | Gap | Impacted Modules | +|:---:|---|---| +| 4 | Upload lock contention test | `rrd_upload.c` | +| 5 | Configuration fallback chain (RFC → DCM → dcm.properties) | `rrd_config.c` | +| 6 | Archive CPU throttle logic | `rrd_archive.c` | +| 7 | RDM download event with dynamic-profile cache miss | `rrdDynamic.c`, `rrdInterface.c` | +| 8 | Dynamic profile JSON parse failure | `rrdDynamic.c` | + +### Priority 3 — Nice to Have + +| # | Gap | Impacted Modules | +|:---:|---|---| +| 9 | RBUS registration / unregistration failure injection | `rrdInterface.c` | +| 10 | Message queue creation failure | `rrdMain.c` | +| 11 | Event thread creation failure | `rrdMain.c` | +| 12 | Directory creation / chdir failures | `rrdJsonParser.c` | +| 13 | `systemd-run` / `journalctl` execution failures | `rrdRunCmdThread.c` | +| 14 | Output file write errors | `rrdRunCmdThread.c` | +| 15 | Invalid deep sleep event type | `rrdDynamic.c` | + +--- + +## 7. Test Infrastructure Notes + +| Interface | Tool | Used By | +|---|---|---| +| RBUS event trigger | `rbuscli set` | All event-based tests | +| RBUS profile data | `rbuscli set/get` | `test_rrd_profile_data.py` | +| Log scraping | `grep_rrdlogs()` in `helper_functions.py` | All tests | +| Process control | `pidof`, `kill`, `nohup` | Setup / teardown | +| Mock upload server | Mock xconf server | `test_rrd_debug_report_upload.py` | +| Mock upload script | `uploadSTBLogs.sh` | Upload tests | + +- **Runner:** `pytest`, sequential per file +- **Shared state:** tests within a file depend on prior test side effects +- **Coverage run:** `kill_rrd()` sends `SIGTERM` when `RRD_COVERAGE_MODE` is set so gcov flushes + +--- + +## 8. Appendix: File Inventory + +### Feature Files + +| # | File | Scenarios | +|:---:|---|:---:| +| 1 | `rrd_append_dynamic_profile_static_not_found.feature` | 4 | +| 2 | `rrd_append_report.feature` | 4 | +| 3 | `rrd_background_cmd_static_profile_report.feature` | 5 | +| 4 | `rrd_c_api_upload.feature` | 21 | +| 5 | `rrd_corrupted_static_profile_report.feature` | 4 | +| 6 | `rrd_debug_report_upload.feature` | 6 | +| 7 | `rrd_deepsleep_static_report.feature` | 2 | +| 8 | `rrd_dynamic_profile_missing_report.feature` | 4 | +| 9 | `rrd_dynamic_profile_node_length_exceeded.feature` | 4 | +| 10 | `rrd_dynamic_profile_report.feature` | 5 | +| 11 | `rrd_dynamic_profile_subcategory_report.feature` | 5 | +| 12 | `rrd_empty_issuetype_event.feature` | 2 | +| 13 | `rrd_harmful_command_static_report.feature` | 5 | +| 14 | `rrd_single_instance.feature` | 1 | +| 15 | `rrd_start_control.feature` | 2 | +| 16 | `rrd_start_subscribe_and_wait.feature` | 1 | +| 17 | `rrd_static_profile_category_report.feature` | 5 | +| 18 | `rrd_static_profile_missing_command_report.feature` | 5 | +| 19 | `rrd_static_profile_report.feature` | 5 | +| 20 | `test_rrd_dynamic_profile_harmful_report.feature` | 5 | +| 21 | `test_rrd_static_profile_report_with_suffix.feature` | 4 | +| 22 | `test_rrd_static_profile_report_with_suffix_negative_case.feature` | 4 | +| | **Total** | **103** | + +### Test Files + +| # | File | Tests | +|:---:|---|:---:| +| 1 | `test_rrd_append_dynamic_profile_static_notfound.py` | 7 | +| 2 | `test_rrd_append_report.py` | 7 | +| 3 | `test_rrd_background_cmd_static_profile_report.py` | 5 | +| 4 | `test_rrd_c_api_upload.py` | 5 | +| 5 | `test_rrd_corrupted_static_profile_report.py` | 4 | +| 6 | `test_rrd_debug_report_upload.py` | 6 | +| 7 | `test_rrd_deepsleep_static_report.py` | 5 | +| 8 | `test_rrd_dynamic_profile_harmful_report.py` | 7 | +| 9 | `test_rrd_dynamic_profile_missing_report.py` | 7 | +| 10 | `test_rrd_dynamic_profile_rdm_node_length_exceeded.py` | 6 | +| 11 | `test_rrd_dynamic_profile_report.py` | 9 | +| 12 | `test_rrd_dynamic_subcategory_report.py` | 7 | +| 13 | `test_rrd_empty_issuetype_event.py` | 2 | +| 14 | `test_rrd_harmful_command_static_report.py` | 5 | +| 15 | `test_rrd_profile_data.py` | 3 | +| 16 | `test_rrd_single_instance.py` | 3 | +| 17 | `test_rrd_start_control.py` | 1 | +| 18 | `test_rrd_start_subscribe_and_wait.py` | 4 | +| 19 | `test_rrd_static_profile_category_report.py` | 5 | +| 20 | `test_rrd_static_profile_missing_command_report.py` | 5 | +| 21 | `test_rrd_static_profile_report.py` | 5 | +| 22 | `test_rrd_static_profile_report_with_suffix.py` | 5 | +| 23 | `test_rrd_static_profile_report_with_suffix_negative_case.py` | 5 | +| | **Total** | **118** | From 330745f2f0e61712868a824692a3b5d5093d750b Mon Sep 17 00:00:00 2001 From: nhanasi Date: Wed, 4 Nov 2026 10:11:37 -0500 Subject: [PATCH 6/7] Update L2-tests.yml --- .github/workflows/L2-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/L2-tests.yml b/.github/workflows/L2-tests.yml index 0849c1f3f..a589b8831 100644 --- a/.github/workflows/L2-tests.yml +++ b/.github/workflows/L2-tests.yml @@ -30,7 +30,7 @@ jobs: uses: actions/checkout@v4 with: repository: 'rdkcentral/docker-device-mgt-service-test' - ref: 'feature/l2docupdate' + ref: 'develop' path: docker-device-mgt-service-test - name: Check out dependent repostiories From 9510ae6a908965809524b7eb80bef49f0265c2b3 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 4 Nov 2026 15:29:07 +0000 Subject: [PATCH 7/7] ci: regenerate L2_Coverage.md [skip ci] --- test/functional-tests/L2_Coverage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/functional-tests/L2_Coverage.md b/test/functional-tests/L2_Coverage.md index 73209c29a..60a0356f7 100644 --- a/test/functional-tests/L2_Coverage.md +++ b/test/functional-tests/L2_Coverage.md @@ -1,6 +1,6 @@ # Remote Debugger L2 Coverage Report -**Generated:** 2026-11-03 +**Generated:** 2026-11-04 **Component:** `remotedebugger` (`src/`) **Test suite:** `test/functional-tests/` **Coverage tool:** lcov (source-level instrumentation via `--coverage`)