/** * Tests for the prompt workflow chain under `.pi/agent/prompts/`. * * Scope (static, no LLM): * - Frontmatter of every `c-*.md` recipe and `p-*.md` step is well-formed * and contains the keys the pi-subagents adapter requires. * - Every step named in a `chain:` resolves to an existing `p-*.md` file. * - Every step has `subagent: true` and `fork: true`, which `Doc/chain-commands.md` * calls out as required for `p-*.md` chain steps. * - The Doc pages describe the same recipes / sequences that exist on disk. * * These tests do not exercise the LLM. They are independent, deterministic, * and fast (run against the on-disk tree only). */ import { describe, it, before } from "node:test"; import assert from "node:assert/strict"; import { readFileSync, readdirSync } from "node:fs"; import { join, resolve, dirname } from "node:path"; import { fileURLToPath } from "node:url"; import { parseFrontmatter, parseChain } from "./support/frontmatter.ts"; // --------------------------------------------------------------------------- // Locating the prompts directory // --------------------------------------------------------------------------- const here = dirname(fileURLToPath(import.meta.url)); // test/ -> .pi/agent/prompts/ const PROMPTS_DIR = resolve(here, ".."); function readUtf8(path: string): string { return readFileSync(path, "utf8"); } function readDoc(path: string): { frontmatter: Record; body: string } { return parseFrontmatter(readUtf8(path)); } let chainFiles: string[] = []; let promptFiles: string[] = []; let docFiles: string[] = []; before(() => { chainFiles = readdirSync(PROMPTS_DIR) .filter((n) => n.startsWith("c-") && n.endsWith(".md")) .sort() .map((n) => join(PROMPTS_DIR, n)); promptFiles = readdirSync(PROMPTS_DIR) .filter((n) => n.startsWith("p-") && n.endsWith(".md")) .sort() .map((n) => join(PROMPTS_DIR, n)); docFiles = readdirSync(join(PROMPTS_DIR, "Doc")) .filter((n) => n.endsWith(".md")) .sort() .map((n) => join(PROMPTS_DIR, "Doc", n)); }); function readChain(name: string) { const path = join(PROMPTS_DIR, name); return { path, name, ...readDoc(path) }; } function readPrompt(step: string) { // Accept both `p-brainstorm` and `p-brainstorm.md`; the actual files // end in `.md` but the chain frontmatter references them without it. const fileName = step.endsWith(".md") ? step : `${step}.md`; const path = join(PROMPTS_DIR, fileName); return { path, step, ...readDoc(path) }; } // --------------------------------------------------------------------------- // Test data // --------------------------------------------------------------------------- const EXPECTED_CHAINS: Record = { "c-design-build.md": ["p-brainstorm", "p-plan", "p-test", "p-commit"], "c-fix-commit.md": ["p-summarize", "p-diagnose", "p-debug", "p-test"], "c-refactor-commit.md": ["p-refactor", "p-test", "p-review", "p-commit"], "c-security-loop.md": ["p-secaudit", "p-review", "p-test"], }; const PROMPTS_USED_IN_CHAINS = new Set( Object.values(EXPECTED_CHAINS).flat(), ); // --------------------------------------------------------------------------- // 1. Self-tests for the frontmatter/chain helpers (the test infra must work) // --------------------------------------------------------------------------- describe("support: frontmatter parser", () => { it("parses a simple key: value block", () => { const { frontmatter, body } = parseFrontmatter( '---\nfoo: bar\nbaz: "qux"\n---\nhello', ); assert.equal(frontmatter.foo, "bar"); assert.equal(frontmatter.baz, "qux"); assert.equal(body, "hello"); }); it("returns empty frontmatter when the file has no fence", () => { const { frontmatter } = parseFrontmatter("no fence here"); assert.deepEqual(frontmatter, {}); }); it("returns empty frontmatter when the fence is unterminated", () => { const { frontmatter } = parseFrontmatter("---\nfoo: bar\nstill going"); assert.deepEqual(frontmatter, {}); }); it("de-indents a block scalar and trims leading newlines", () => { const { frontmatter } = parseFrontmatter( "---\nbody: |\n line one\n line two\n line three\n---\n", ); assert.equal(frontmatter.body, "line one\nline two\nline three"); }); it("strips full-line comments and ignores empty lines between keys", () => { const { frontmatter } = parseFrontmatter( "---\nfoo: bar\n# full-line comment\nbaz: qux\n---\n", ); assert.equal(frontmatter.foo, "bar"); assert.equal(frontmatter.baz, "qux"); // Inline `#` after a value is left in the value (matches the // upstream `parseFrontmatter` in `pi-subagents`). This protects // against accidental whitespace stripping. const inline = parseFrontmatter("---\nfoo: bar # trailing\n---\n"); assert.equal(inline.frontmatter.foo, "bar # trailing"); }); it("normalizes CRLF line endings", () => { const { frontmatter, body } = parseFrontmatter( "---\r\nfoo: bar\r\n---\r\nbody", ); assert.equal(frontmatter.foo, "bar"); assert.equal(body, "body"); }); }); describe("support: chain parser", () => { it("splits on ' -> ' and trims", () => { assert.deepEqual(parseChain("a -> b -> c"), ["a", "b", "c"]); }); it("returns [] for undefined", () => { assert.deepEqual(parseChain(undefined), []); }); it("returns [] for an empty string", () => { assert.deepEqual(parseChain(""), []); }); it("drops empty parts from extra spaces", () => { assert.deepEqual(parseChain("a -> -> b"), ["a", "b"]); }); it("does not split on a single dash", () => { // A chain value should not use `-` as a separator; the actual // recipes use ` -> `. This protects against future drift. assert.deepEqual(parseChain("a-b-c"), ["a-b-c"]); }); }); // --------------------------------------------------------------------------- // 2. Happy path — the on-disk tree matches what we expect // --------------------------------------------------------------------------- describe("happy path: chain recipes are present and well-formed", () => { for (const [name, expectedSteps] of Object.entries(EXPECTED_CHAINS)) { it(`loads ${name}`, () => { assert.ok( chainFiles.includes(join(PROMPTS_DIR, name)), `missing chain recipe: ${name}`, ); }); it(`${name} has description, argument-hint, and chain fields`, () => { const { frontmatter } = readChain(name); assert.ok(frontmatter.description, "missing description"); assert.ok(frontmatter["argument-hint"], "missing argument-hint"); assert.ok(frontmatter.chain, "missing chain"); }); it(`${name}.chain parses to ${expectedSteps.join(" -> ")}`, () => { const { frontmatter } = readChain(name); assert.deepEqual(parseChain(frontmatter.chain), expectedSteps); }); it(`${name} contains a $@ placeholder for the user argument`, () => { const { body } = readChain(name); assert.ok( body.includes("$@"), "chain recipe body must use $@ for the user-supplied argument", ); }); } }); describe("happy path: every prompt step resolves to a real file", () => { for (const step of PROMPTS_USED_IN_CHAINS) { it(`step ${step} exists as a p-*.md file`, () => { assert.ok( promptFiles.includes(join(PROMPTS_DIR, `${step}.md`)), `chain references ${step} but no such p-*.md exists`, ); }); } }); describe("happy path: every p-*.md has the required chain-step shape", () => { for (const step of PROMPTS_USED_IN_CHAINS) { it(`${step} declares subagent: true`, () => { const { frontmatter } = readPrompt(step); assert.equal(frontmatter.subagent, "true"); }); it(`${step} declares fork: true`, () => { const { frontmatter } = readPrompt(step); assert.equal(frontmatter.fork, "true"); }); it(`${step} has description and argument-hint`, () => { const { frontmatter } = readPrompt(step); assert.ok(frontmatter.description, "missing description"); assert.ok(frontmatter["argument-hint"], "missing argument-hint"); }); it(`${step} body references $@`, () => { const { body } = readPrompt(step); assert.ok(body.includes("$@"), "p-* body must use $@ for arguments"); }); } }); // --------------------------------------------------------------------------- // 3. Edge cases — boundaries of the format // --------------------------------------------------------------------------- describe("edge cases: chain format boundaries", () => { it("every chain has at least one step", () => { for (const name of Object.keys(EXPECTED_CHAINS)) { const steps = parseChain(readChain(name).frontmatter.chain); assert.ok(steps.length > 0, `${name} chain is empty`); } }); it("no chain has a duplicate step", () => { for (const name of Object.keys(EXPECTED_CHAINS)) { const steps = parseChain(readChain(name).frontmatter.chain); assert.equal( new Set(steps).size, steps.length, `${name} contains a duplicate step`, ); } }); it("every chain step is lowercase kebab-case starting with 'p-'", () => { for (const name of Object.keys(EXPECTED_CHAINS)) { const steps = parseChain(readChain(name).frontmatter.chain); for (const step of steps) { assert.match(step, /^p-[a-z0-9][a-z0-9-]*$/, `${name} step '${step}' is not p-kebab-case`); } } }); it("chain frontmatter uses ' -> ' (single space) and not '->' or ' - >'", () => { // This is the format the pi-subagents adapter's `splitPromptChain` // actually understands. for (const name of Object.keys(EXPECTED_CHAINS)) { const raw = readChain(name).frontmatter.chain; assert.ok(raw, `${name} has no chain value`); assert.doesNotMatch(raw, /[^ ]->/, `${name} uses bare '->' which the adapter will not split on`); } }); it("every chain ends with a real prompt file, not a typo", () => { for (const name of Object.keys(EXPECTED_CHAINS)) { const last = parseChain(readChain(name).frontmatter.chain).at(-1); assert.ok(last && promptFiles.includes(join(PROMPTS_DIR, `${last}.md`)), `${name} last step '${last}' does not exist on disk`); } }); }); describe("edge cases: argument-hint style", () => { it("every argument-hint is non-empty", () => { for (const name of Object.keys(EXPECTED_CHAINS)) { const hint = readChain(name).frontmatter["argument-hint"]; assert.ok(hint && hint.length > 0, `${name} has empty argument-hint`); } for (const step of PROMPTS_USED_IN_CHAINS) { const hint = readPrompt(step).frontmatter["argument-hint"]; assert.ok(hint && hint.length > 0, `${step} has empty argument-hint`); } }); it("argument-hints are wrapped in [...] to match the documented convention", () => { // Doc/chain-commands.md and the existing recipes all use `[hint]`. const bracket = /^\[.+\]$/; for (const name of Object.keys(EXPECTED_CHAINS)) { const hint = readChain(name).frontmatter["argument-hint"]; assert.match(hint, bracket, `${name} argument-hint not wrapped in []`); } for (const step of PROMPTS_USED_IN_CHAINS) { const hint = readPrompt(step).frontmatter["argument-hint"]; assert.match(hint, bracket, `${step} argument-hint not wrapped in []`); } }); }); // --------------------------------------------------------------------------- // 4. Error cases — malformed / missing / inconsistent input // --------------------------------------------------------------------------- describe("error cases: malformed frontmatter", () => { it("rejects a chain file with no closing fence", () => { const { frontmatter } = parseFrontmatter( "---\ndescription: broken\nchain: a -> b\n", ); // Without a closing fence, the parser should return no frontmatter // (mirroring the real adapter), so the chain is empty. assert.deepEqual(frontmatter, {}); }); it("rejects a chain value with no separator at all", () => { assert.deepEqual(parseChain("only-one-step"), ["only-one-step"]); assert.equal(parseChain("only-one-step").length, 1); }); it("flags a chain step that does not exist on disk", () => { // Simulate what the adapter does: build a fake chain value and // confirm we'd catch the missing file. const fakeStep = "p-does-not-exist"; const steps = parseChain(`p-brainstorm -> ${fakeStep}`); assert.ok(steps.includes(fakeStep)); assert.ok( !promptFiles.includes(join(PROMPTS_DIR, `${fakeStep}.md`)), "sanity: the fake step should not exist on disk", ); }); }); describe("error cases: required keys", () => { it("every chain recipe has a non-empty description", () => { for (const name of Object.keys(EXPECTED_CHAINS)) { const d = readChain(name).frontmatter.description; assert.ok(d && d.trim().length > 0, `${name} has empty description`); } }); it("no p-* used in a chain is missing the subagent/fork flags", () => { // This is the failure mode Doc/chain-commands.md calls out: // "fork: true is required on each p-*.md. Check frontmatter." for (const step of PROMPTS_USED_IN_CHAINS) { const fm = readPrompt(step).frontmatter; assert.equal(fm.subagent, "true", `${step} missing subagent: true`); assert.equal(fm.fork, "true", `${step} missing fork: true`); } }); }); describe("error cases: cross-doc consistency", () => { // These tests catch the realistic drift bug: someone adds a new chain // recipe and forgets to add a row to the table in the Doc page. The // table is the canonical description, so we look for a row, not just // any mention of the name (the docs also reference recipes in // `/prompt-workflow ` examples, which would mask a missing row). // // chain-commands.md table uses the form | `c-design-build.md` | ... | // prompts.md table uses the form | **`c-design-build`** | ... | it("Doc/chain-commands.md has a table row for every chain recipe", () => { const text = readUtf8(join(PROMPTS_DIR, "Doc", "chain-commands.md")); for (const name of Object.keys(EXPECTED_CHAINS)) { // The chain-commands.md table uses the full filename in // backticks (e.g. `c-design-build.md`). const re = new RegExp(`^\\|\\s*\`${name}\`\\s*\\|`, "m"); assert.ok( re.test(text), `Doc/chain-commands.md is missing a table row for ${name}`, ); } }); it("Doc/chain-commands.md table rows show the actual chain sequence", () => { const text = readUtf8(join(PROMPTS_DIR, "Doc", "chain-commands.md")); for (const [name, steps] of Object.entries(EXPECTED_CHAINS)) { const re = new RegExp(`^\\|\\s*\`${name}\`\\s*\\|[^|]*\\|[^|]*\\|`, "m"); const row = text.match(re); assert.ok(row, `Doc/chain-commands.md is missing a row for ${name}`); const rowAscii = row[0].replace(/→/g, "->"); for (const step of steps) { assert.ok( rowAscii.includes(step), `Doc/chain-commands.md row for ${name} does not list ${step}`, ); } } }); it("Doc/prompts.md has a table row for every chain recipe", () => { const text = readUtf8(join(PROMPTS_DIR, "Doc", "prompts.md")); for (const name of Object.keys(EXPECTED_CHAINS)) { // The prompts.md table uses the base name in backticks // surrounded by `**` (e.g. `**`c-design-build`**`). const base = name.replace(/\.md$/, ""); const re = new RegExp(`^\\|\\s*\\*\\*\`${base}\`\\*\\*\\s*\\|`, "m"); assert.ok( re.test(text), `Doc/prompts.md is missing a table row for ${base}`, ); } }); it("Doc/prompts.md table rows show the actual chain sequence", () => { const text = readUtf8(join(PROMPTS_DIR, "Doc", "prompts.md")); for (const [name, steps] of Object.entries(EXPECTED_CHAINS)) { const base = name.replace(/\.md$/, ""); const re = new RegExp(`^\\|\\s*\\*\\*\`${base}\`\\*\\*\\s*\\|[^|]*\\|`, "m"); const row = text.match(re); assert.ok(row, `Doc/prompts.md is missing a row for ${base}`); const rowAscii = row[0].replace(/→/g, "->"); for (const step of steps) { assert.ok( rowAscii.includes(step), `Doc/prompts.md row for ${base} does not list ${step}`, ); } } }); it("Doc/chain-commands.md documents ' -> ' as the chain separator", () => { const text = readUtf8(join(PROMPTS_DIR, "Doc", "chain-commands.md")); assert.ok( text.includes("->") || text.includes("→"), "Doc/chain-commands.md should describe the chain arrow separator", ); }); it("Doc/prompts.md mentions every prompt used in any chain", () => { const text = readUtf8(join(PROMPTS_DIR, "Doc", "prompts.md")); for (const step of PROMPTS_USED_IN_CHAINS) { assert.ok( text.includes(step), `Doc/prompts.md does not mention prompt ${step}`, ); } }); }); // --------------------------------------------------------------------------- // 5. Directory-level sanity — catches accidental deletions // --------------------------------------------------------------------------- describe("directory sanity", () => { it("has at least one chain recipe", () => { assert.ok(chainFiles.length > 0, "no c-*.md files found"); }); it("has at least one p-*.md prompt", () => { assert.ok(promptFiles.length > 0, "no p-*.md files found"); }); it("has both Doc pages", () => { assert.ok(docFiles.some((p) => p.endsWith("chain-commands.md"))); assert.ok(docFiles.some((p) => p.endsWith("prompts.md"))); }); it("Doc/ has no extra markdown files we did not expect", () => { // If a new doc page is added, the cross-doc tests above should // grow with it. Catches accidental orphans in the other direction. const known = new Set(["chain-commands.md", "prompts.md"]); for (const p of docFiles) { const base = p.split(/[/\\]/).pop()!; assert.ok(known.has(base), `unexpected Doc file: ${base}`); } }); it("every file in the prompts root is a recognized c-*, p-*, or Doc entry", () => { const entries = readdirSync(PROMPTS_DIR); const allowed = (n: string) => n.startsWith("c-") || n.startsWith("p-") || n === "Doc" || n === "test"; for (const e of entries) { if (e.startsWith(".")) continue; // .pi-subagents, etc. assert.ok( allowed(e), `unexpected top-level entry in prompts dir: ${e}`, ); } }); });