mirror of
https://github.com/affaan-m/everything-claude-code.git
synced 2026-04-14 13:53:29 +08:00
feat(skills): add skill-comply — automated behavioral compliance measurement (#724)
* feat(skills): add skill-comply — automated behavioral compliance measurement Automated compliance measurement for skills, rules, and agent definitions. Generates behavioral specs, runs scenarios at 3 strictness levels, classifies tool calls via LLM, and produces self-contained reports. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * fix(skill-comply): address bot review feedback - AGENTS.md: fix stale skill count (115 → 117) in project structure - run.py: replace remaining print() with logger, add zero-division guard, create parent dirs for --output path - runner.py: add returncode check for claude subprocess, clarify relative_to path traversal validation - parser.py: use is_file() instead of exists(), catch KeyError for missing trace fields, add file check in parse_spec - classifier.py: log warnings on malformed classification output, guard against non-dict JSON responses - grader.py: filter negative indices from LLM classification Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
5
skills/skill-comply/fixtures/compliant_trace.jsonl
Normal file
5
skills/skill-comply/fixtures/compliant_trace.jsonl
Normal file
@@ -0,0 +1,5 @@
|
||||
{"timestamp":"2026-03-20T10:00:01Z","event":"tool_complete","tool":"Write","session":"sess-001","input":"{\"file_path\":\"tests/test_fib.py\",\"content\":\"def test_fib(): assert fib(0) == 0\"}","output":"File created"}
|
||||
{"timestamp":"2026-03-20T10:00:10Z","event":"tool_complete","tool":"Bash","session":"sess-001","input":"{\"command\":\"cd /tmp/sandbox && pytest tests/\"}","output":"FAILED - 1 failed"}
|
||||
{"timestamp":"2026-03-20T10:00:20Z","event":"tool_complete","tool":"Write","session":"sess-001","input":"{\"file_path\":\"src/fib.py\",\"content\":\"def fib(n): return n if n <= 1 else fib(n-1)+fib(n-2)\"}","output":"File created"}
|
||||
{"timestamp":"2026-03-20T10:00:30Z","event":"tool_complete","tool":"Bash","session":"sess-001","input":"{\"command\":\"cd /tmp/sandbox && pytest tests/\"}","output":"1 passed"}
|
||||
{"timestamp":"2026-03-20T10:00:40Z","event":"tool_complete","tool":"Edit","session":"sess-001","input":"{\"file_path\":\"src/fib.py\",\"old_string\":\"return n if\",\"new_string\":\"if n < 0: raise ValueError\\n return n if\"}","output":"File edited"}
|
||||
3
skills/skill-comply/fixtures/noncompliant_trace.jsonl
Normal file
3
skills/skill-comply/fixtures/noncompliant_trace.jsonl
Normal file
@@ -0,0 +1,3 @@
|
||||
{"timestamp":"2026-03-20T10:00:01Z","event":"tool_complete","tool":"Write","session":"sess-002","input":"{\"file_path\":\"src/fib.py\",\"content\":\"def fib(n): return n if n <= 1 else fib(n-1)+fib(n-2)\"}","output":"File created"}
|
||||
{"timestamp":"2026-03-20T10:00:10Z","event":"tool_complete","tool":"Write","session":"sess-002","input":"{\"file_path\":\"tests/test_fib.py\",\"content\":\"def test_fib(): assert fib(0) == 0\"}","output":"File created"}
|
||||
{"timestamp":"2026-03-20T10:00:20Z","event":"tool_complete","tool":"Bash","session":"sess-002","input":"{\"command\":\"cd /tmp/sandbox && pytest tests/\"}","output":"1 passed"}
|
||||
44
skills/skill-comply/fixtures/tdd_spec.yaml
Normal file
44
skills/skill-comply/fixtures/tdd_spec.yaml
Normal file
@@ -0,0 +1,44 @@
|
||||
id: tdd-workflow
|
||||
name: TDD Workflow Compliance
|
||||
source_rule: rules/common/testing.md
|
||||
version: "2.0"
|
||||
|
||||
steps:
|
||||
- id: write_test
|
||||
description: "Write test file BEFORE implementation"
|
||||
required: true
|
||||
detector:
|
||||
description: "A Write or Edit to a test file (filename contains 'test')"
|
||||
before_step: write_impl
|
||||
|
||||
- id: run_test_red
|
||||
description: "Run test and confirm FAIL (RED phase)"
|
||||
required: true
|
||||
detector:
|
||||
description: "Run pytest or test command that produces a FAIL/ERROR result"
|
||||
after_step: write_test
|
||||
before_step: write_impl
|
||||
|
||||
- id: write_impl
|
||||
description: "Write minimal implementation (GREEN phase)"
|
||||
required: true
|
||||
detector:
|
||||
description: "Write or Edit an implementation file (not a test file)"
|
||||
after_step: run_test_red
|
||||
|
||||
- id: run_test_green
|
||||
description: "Run test and confirm PASS (GREEN phase)"
|
||||
required: true
|
||||
detector:
|
||||
description: "Run pytest or test command that produces a PASS result"
|
||||
after_step: write_impl
|
||||
|
||||
- id: refactor
|
||||
description: "Refactor (IMPROVE phase)"
|
||||
required: false
|
||||
detector:
|
||||
description: "Edit a source file for refactoring after tests pass"
|
||||
after_step: run_test_green
|
||||
|
||||
scoring:
|
||||
threshold_promote_to_hook: 0.6
|
||||
Reference in New Issue
Block a user