Skip to content

Commit d16cb10

Browse files
davidslaterGitHub Ace
andauthored
test(conclude): expand conformance matrix for gh-aw-threat-detection#694 (#738)
* test(conclude): expand conformance matrix for gh-aw-threat-detection#694 Add test cases covering the full {result state} x {warn/strict} x {execution outcome} matrix for the `conclude` subcommand: success is unaffected by warn mode/executionFailed, threat_detected always warns in warn mode (mustFail excludes it), malformed/missing/unreadable result files are exercised across all warn-mode/executionFailed combinations to confirm mustFail scoping. conclude.go itself already implements the fixes called for in the issue (writes reason/conclusion on every fail path including strict missing-file, exports GH_AW_DETECTION_* consistently, and documents the mustFail divergence in specs/threat-detection-spec.md TD-20b/21a). The remaining divergence is entirely in gh-aw's actions/setup/sh/conclude_threat_detection.sh, which lives in the github/gh-aw repo and is out of scope for this repo. Co-authored-by: David Slater <12449447+davidslater@users.noreply.github.com> * test(conclude): fill remaining matrix gaps from review feedback Add strict-mode + executionFailed=true cases for malformed and unreadable result files, and assert success/GH_AW_DETECTION_CONCLUSION on the unreadable-file matrix (previously only reason/conclusion were checked), per copilot-pull-request-reviewer feedback on #738. Co-authored-by: David Slater <12449447+davidslater@users.noreply.github.com> --------- Co-authored-by: GitHub Ace <githubnext@users.noreply.github.com> Co-authored-by: David Slater <12449447+davidslater@users.noreply.github.com>
1 parent 5c123b3 commit d16cb10

1 file changed

Lines changed: 150 additions & 20 deletions

File tree

‎cmd/threat-detect/conclude_test.go‎

Lines changed: 150 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -153,6 +153,77 @@ func TestConcludeContract(t *testing.T) {
153153
wantSuccess: "false",
154154
wantReason: "parse_error",
155155
},
156+
// --- Remaining matrix combinations from
157+
// {RUN_DETECTION} x {result file present/absent/malformed} x {warn/strict} x
158+
// {execution outcome} (gh-aw-threat-detection#694). The cases above cover the
159+
// primary decision paths; these fill in the combinations that exercise
160+
// mustFail's scope (agent_failure/parse_error only, never threat_detected) and
161+
// confirm executionFailed is inert on the success path.
162+
{
163+
name: "success on safe verdict is unaffected by warn mode",
164+
runDetection: "true",
165+
warnMode: true,
166+
resultContent: safeVerdict,
167+
wantExit: concludeExitProceed,
168+
wantConclusion: "success",
169+
wantSuccess: "true",
170+
wantReason: "",
171+
},
172+
{
173+
name: "success on safe verdict is unaffected by executionFailed",
174+
runDetection: "true",
175+
warnMode: false,
176+
executionFailed: true,
177+
resultContent: safeVerdict,
178+
wantExit: concludeExitProceed,
179+
wantConclusion: "success",
180+
wantSuccess: "true",
181+
wantReason: "",
182+
},
183+
{
184+
name: "threat_detected warns even when execution failed (mustFail excludes threat_detected)",
185+
runDetection: "true",
186+
warnMode: true,
187+
executionFailed: true,
188+
resultContent: threatVerdict,
189+
wantExit: concludeExitProceed,
190+
wantConclusion: "warning",
191+
wantSuccess: "false",
192+
wantReason: "threat_detected",
193+
},
194+
{
195+
name: "malformed file warns when execution succeeded",
196+
runDetection: "true",
197+
warnMode: true,
198+
executionFailed: false,
199+
resultContent: "{not json",
200+
wantExit: concludeExitProceed,
201+
wantConclusion: "warning",
202+
wantSuccess: "false",
203+
wantReason: "parse_error",
204+
},
205+
{
206+
name: "missing file fails closed in strict mode regardless of execution outcome",
207+
runDetection: "true",
208+
warnMode: false,
209+
executionFailed: true,
210+
resultMissing: true,
211+
wantExit: concludeExitFail,
212+
wantConclusion: "failure",
213+
wantSuccess: "false",
214+
wantReason: "agent_failure",
215+
},
216+
{
217+
name: "malformed file fails closed in strict mode regardless of execution outcome",
218+
runDetection: "true",
219+
warnMode: false,
220+
executionFailed: true,
221+
resultContent: "{not json",
222+
wantExit: concludeExitFail,
223+
wantConclusion: "failure",
224+
wantSuccess: "false",
225+
wantReason: "parse_error",
226+
},
156227
}
157228

158229
for _, tt := range tests {
@@ -209,29 +280,88 @@ func TestConcludeContract(t *testing.T) {
209280
// TestConcludeUnreadableFileIsAgentFailure verifies that an IO error reading the
210281
// result file (here, the path is a directory) is classified as agent_failure
211282
// (ERR_SYSTEM), not parse_error — unreadable files are a system-side failure,
212-
// while parse_error is reserved for readable-but-malformed content.
283+
// while parse_error is reserved for readable-but-malformed content. It also
284+
// completes the {warn/strict} x {execution outcome} matrix for the unreadable-file
285+
// path (gh-aw-threat-detection#694).
213286
func TestConcludeUnreadableFileIsAgentFailure(t *testing.T) {
214-
dir := t.TempDir()
215-
// A directory at the result path yields a non-ErrNotExist *fs.PathError from
216-
// os.ReadFile, deterministically across platforms and regardless of euid.
217-
resultFile := filepath.Join(dir, "detection_result.json")
218-
if err := os.Mkdir(resultFile, 0o755); err != nil {
219-
t.Fatalf("Mkdir error = %v", err)
287+
tests := []struct {
288+
name string
289+
warnMode bool
290+
executionFailed bool
291+
wantExit int
292+
wantConclusion string
293+
}{
294+
{
295+
name: "strict mode fails closed",
296+
warnMode: false,
297+
wantExit: concludeExitFail,
298+
wantConclusion: "failure",
299+
},
300+
{
301+
name: "strict mode fails closed regardless of execution outcome",
302+
warnMode: false,
303+
executionFailed: true,
304+
wantExit: concludeExitFail,
305+
wantConclusion: "failure",
306+
},
307+
{
308+
name: "warn mode warns when execution succeeded",
309+
warnMode: true,
310+
wantExit: concludeExitProceed,
311+
wantConclusion: "warning",
312+
},
313+
{
314+
name: "warn mode fails closed when execution failed (mustFail)",
315+
warnMode: true,
316+
executionFailed: true,
317+
wantExit: concludeExitFail,
318+
wantConclusion: "failure",
319+
},
220320
}
221321

222-
var stdout bytes.Buffer
223-
c := &concluder{
224-
runDetection: "true",
225-
warnMode: false,
226-
githubOutput: filepath.Join(dir, "out"),
227-
githubEnv: filepath.Join(dir, "env"),
228-
stdout: &stdout,
229-
}
230-
if code := c.run(resultFile); code != concludeExitFail {
231-
t.Fatalf("exit code = %d, want %d (stdout: %s)", code, concludeExitFail, stdout.String())
232-
}
233-
if got := parseKV(t, filepath.Join(dir, "out"))["reason"]; got != "agent_failure" {
234-
t.Errorf("reason output = %q, want %q", got, "agent_failure")
322+
for _, tt := range tests {
323+
t.Run(tt.name, func(t *testing.T) {
324+
dir := t.TempDir()
325+
// A directory at the result path yields a non-ErrNotExist *fs.PathError
326+
// from os.ReadFile, deterministically across platforms and regardless of
327+
// euid.
328+
resultFile := filepath.Join(dir, "detection_result.json")
329+
if err := os.Mkdir(resultFile, 0o755); err != nil {
330+
t.Fatalf("Mkdir error = %v", err)
331+
}
332+
333+
var stdout bytes.Buffer
334+
c := &concluder{
335+
runDetection: "true",
336+
warnMode: tt.warnMode,
337+
executionFailed: tt.executionFailed,
338+
githubOutput: filepath.Join(dir, "out"),
339+
githubEnv: filepath.Join(dir, "env"),
340+
stdout: &stdout,
341+
}
342+
if code := c.run(resultFile); code != tt.wantExit {
343+
t.Fatalf("exit code = %d, want %d (stdout: %s)", code, tt.wantExit, stdout.String())
344+
}
345+
outputs := parseKV(t, filepath.Join(dir, "out"))
346+
if got := outputs["reason"]; got != "agent_failure" {
347+
t.Errorf("reason output = %q, want %q", got, "agent_failure")
348+
}
349+
if got := outputs["conclusion"]; got != tt.wantConclusion {
350+
t.Errorf("conclusion output = %q, want %q", got, tt.wantConclusion)
351+
}
352+
// success=false on every fail path, warn or strict — only a clean
353+
// verdict (or skip) sets success=true.
354+
if got := outputs["success"]; got != "false" {
355+
t.Errorf("success output = %q, want %q", got, "false")
356+
}
357+
env := parseKV(t, filepath.Join(dir, "env"))
358+
if got := env["GH_AW_DETECTION_REASON"]; got != "agent_failure" {
359+
t.Errorf("GH_AW_DETECTION_REASON = %q, want %q", got, "agent_failure")
360+
}
361+
if got := env["GH_AW_DETECTION_CONCLUSION"]; got != tt.wantConclusion {
362+
t.Errorf("GH_AW_DETECTION_CONCLUSION = %q, want %q", got, tt.wantConclusion)
363+
}
364+
})
235365
}
236366
}
237367

0 commit comments

Comments
 (0)