493 lines
18 KiB
TypeScript
493 lines
18 KiB
TypeScript
/**
|
|
* Tests for the prompt workflow chain under `.pi/agent/prompts/`.
|
|
*
|
|
* Scope (static, no LLM):
|
|
* - Frontmatter of every `c-*.md` recipe and `p-*.md` step is well-formed
|
|
* and contains the keys the pi-subagents adapter requires.
|
|
* - Every step named in a `chain:` resolves to an existing `p-*.md` file.
|
|
* - Every step has `subagent: true` and `fork: true`, which `Doc/chain-commands.md`
|
|
* calls out as required for `p-*.md` chain steps.
|
|
* - The Doc pages describe the same recipes / sequences that exist on disk.
|
|
*
|
|
* These tests do not exercise the LLM. They are independent, deterministic,
|
|
* and fast (run against the on-disk tree only).
|
|
*/
|
|
|
|
import { describe, it, before } from "node:test";
|
|
import assert from "node:assert/strict";
|
|
import { readFileSync, readdirSync } from "node:fs";
|
|
import { join, resolve, dirname } from "node:path";
|
|
import { fileURLToPath } from "node:url";
|
|
|
|
import { parseFrontmatter, parseChain } from "./support/frontmatter.ts";
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Locating the prompts directory
|
|
// ---------------------------------------------------------------------------
|
|
|
|
const here = dirname(fileURLToPath(import.meta.url));
|
|
// test/ -> .pi/agent/prompts/
|
|
const PROMPTS_DIR = resolve(here, "..");
|
|
|
|
function readUtf8(path: string): string {
|
|
return readFileSync(path, "utf8");
|
|
}
|
|
|
|
function readDoc(path: string): { frontmatter: Record<string, string>; body: string } {
|
|
return parseFrontmatter(readUtf8(path));
|
|
}
|
|
|
|
let chainFiles: string[] = [];
|
|
let promptFiles: string[] = [];
|
|
let docFiles: string[] = [];
|
|
|
|
before(() => {
|
|
chainFiles = readdirSync(PROMPTS_DIR)
|
|
.filter((n) => n.startsWith("c-") && n.endsWith(".md"))
|
|
.sort()
|
|
.map((n) => join(PROMPTS_DIR, n));
|
|
promptFiles = readdirSync(PROMPTS_DIR)
|
|
.filter((n) => n.startsWith("p-") && n.endsWith(".md"))
|
|
.sort()
|
|
.map((n) => join(PROMPTS_DIR, n));
|
|
docFiles = readdirSync(join(PROMPTS_DIR, "Doc"))
|
|
.filter((n) => n.endsWith(".md"))
|
|
.sort()
|
|
.map((n) => join(PROMPTS_DIR, "Doc", n));
|
|
});
|
|
|
|
function readChain(name: string) {
|
|
const path = join(PROMPTS_DIR, name);
|
|
return { path, name, ...readDoc(path) };
|
|
}
|
|
|
|
function readPrompt(step: string) {
|
|
// Accept both `p-brainstorm` and `p-brainstorm.md`; the actual files
|
|
// end in `.md` but the chain frontmatter references them without it.
|
|
const fileName = step.endsWith(".md") ? step : `${step}.md`;
|
|
const path = join(PROMPTS_DIR, fileName);
|
|
return { path, step, ...readDoc(path) };
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Test data
|
|
// ---------------------------------------------------------------------------
|
|
|
|
const EXPECTED_CHAINS: Record<string, string[]> = {
|
|
"c-design-build.md": ["p-brainstorm", "p-plan", "p-test", "p-commit"],
|
|
"c-fix-commit.md": ["p-summarize", "p-diagnose", "p-debug", "p-test"],
|
|
"c-refactor-commit.md": ["p-refactor", "p-test", "p-review", "p-commit"],
|
|
"c-security-loop.md": ["p-secaudit", "p-review", "p-test"],
|
|
};
|
|
|
|
const PROMPTS_USED_IN_CHAINS = new Set<string>(
|
|
Object.values(EXPECTED_CHAINS).flat(),
|
|
);
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// 1. Self-tests for the frontmatter/chain helpers (the test infra must work)
|
|
// ---------------------------------------------------------------------------
|
|
|
|
describe("support: frontmatter parser", () => {
|
|
it("parses a simple key: value block", () => {
|
|
const { frontmatter, body } = parseFrontmatter(
|
|
'---\nfoo: bar\nbaz: "qux"\n---\nhello',
|
|
);
|
|
assert.equal(frontmatter.foo, "bar");
|
|
assert.equal(frontmatter.baz, "qux");
|
|
assert.equal(body, "hello");
|
|
});
|
|
|
|
it("returns empty frontmatter when the file has no fence", () => {
|
|
const { frontmatter } = parseFrontmatter("no fence here");
|
|
assert.deepEqual(frontmatter, {});
|
|
});
|
|
|
|
it("returns empty frontmatter when the fence is unterminated", () => {
|
|
const { frontmatter } = parseFrontmatter("---\nfoo: bar\nstill going");
|
|
assert.deepEqual(frontmatter, {});
|
|
});
|
|
|
|
it("de-indents a block scalar and trims leading newlines", () => {
|
|
const { frontmatter } = parseFrontmatter(
|
|
"---\nbody: |\n line one\n line two\n line three\n---\n",
|
|
);
|
|
assert.equal(frontmatter.body, "line one\nline two\nline three");
|
|
});
|
|
|
|
it("strips full-line comments and ignores empty lines between keys", () => {
|
|
const { frontmatter } = parseFrontmatter(
|
|
"---\nfoo: bar\n# full-line comment\nbaz: qux\n---\n",
|
|
);
|
|
assert.equal(frontmatter.foo, "bar");
|
|
assert.equal(frontmatter.baz, "qux");
|
|
// Inline `#` after a value is left in the value (matches the
|
|
// upstream `parseFrontmatter` in `pi-subagents`). This protects
|
|
// against accidental whitespace stripping.
|
|
const inline = parseFrontmatter("---\nfoo: bar # trailing\n---\n");
|
|
assert.equal(inline.frontmatter.foo, "bar # trailing");
|
|
});
|
|
|
|
it("normalizes CRLF line endings", () => {
|
|
const { frontmatter, body } = parseFrontmatter(
|
|
"---\r\nfoo: bar\r\n---\r\nbody",
|
|
);
|
|
assert.equal(frontmatter.foo, "bar");
|
|
assert.equal(body, "body");
|
|
});
|
|
});
|
|
|
|
describe("support: chain parser", () => {
|
|
it("splits on ' -> ' and trims", () => {
|
|
assert.deepEqual(parseChain("a -> b -> c"), ["a", "b", "c"]);
|
|
});
|
|
|
|
it("returns [] for undefined", () => {
|
|
assert.deepEqual(parseChain(undefined), []);
|
|
});
|
|
|
|
it("returns [] for an empty string", () => {
|
|
assert.deepEqual(parseChain(""), []);
|
|
});
|
|
|
|
it("drops empty parts from extra spaces", () => {
|
|
assert.deepEqual(parseChain("a -> -> b"), ["a", "b"]);
|
|
});
|
|
|
|
it("does not split on a single dash", () => {
|
|
// A chain value should not use `-` as a separator; the actual
|
|
// recipes use ` -> `. This protects against future drift.
|
|
assert.deepEqual(parseChain("a-b-c"), ["a-b-c"]);
|
|
});
|
|
});
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// 2. Happy path — the on-disk tree matches what we expect
|
|
// ---------------------------------------------------------------------------
|
|
|
|
describe("happy path: chain recipes are present and well-formed", () => {
|
|
for (const [name, expectedSteps] of Object.entries(EXPECTED_CHAINS)) {
|
|
it(`loads ${name}`, () => {
|
|
assert.ok(
|
|
chainFiles.includes(join(PROMPTS_DIR, name)),
|
|
`missing chain recipe: ${name}`,
|
|
);
|
|
});
|
|
|
|
it(`${name} has description, argument-hint, and chain fields`, () => {
|
|
const { frontmatter } = readChain(name);
|
|
assert.ok(frontmatter.description, "missing description");
|
|
assert.ok(frontmatter["argument-hint"], "missing argument-hint");
|
|
assert.ok(frontmatter.chain, "missing chain");
|
|
});
|
|
|
|
it(`${name}.chain parses to ${expectedSteps.join(" -> ")}`, () => {
|
|
const { frontmatter } = readChain(name);
|
|
assert.deepEqual(parseChain(frontmatter.chain), expectedSteps);
|
|
});
|
|
|
|
it(`${name} contains a $@ placeholder for the user argument`, () => {
|
|
const { body } = readChain(name);
|
|
assert.ok(
|
|
body.includes("$@"),
|
|
"chain recipe body must use $@ for the user-supplied argument",
|
|
);
|
|
});
|
|
}
|
|
});
|
|
|
|
describe("happy path: every prompt step resolves to a real file", () => {
|
|
for (const step of PROMPTS_USED_IN_CHAINS) {
|
|
it(`step ${step} exists as a p-*.md file`, () => {
|
|
assert.ok(
|
|
promptFiles.includes(join(PROMPTS_DIR, `${step}.md`)),
|
|
`chain references ${step} but no such p-*.md exists`,
|
|
);
|
|
});
|
|
}
|
|
});
|
|
|
|
describe("happy path: every p-*.md has the required chain-step shape", () => {
|
|
for (const step of PROMPTS_USED_IN_CHAINS) {
|
|
it(`${step} declares subagent: true`, () => {
|
|
const { frontmatter } = readPrompt(step);
|
|
assert.equal(frontmatter.subagent, "true");
|
|
});
|
|
|
|
it(`${step} declares fork: true`, () => {
|
|
const { frontmatter } = readPrompt(step);
|
|
assert.equal(frontmatter.fork, "true");
|
|
});
|
|
|
|
it(`${step} has description and argument-hint`, () => {
|
|
const { frontmatter } = readPrompt(step);
|
|
assert.ok(frontmatter.description, "missing description");
|
|
assert.ok(frontmatter["argument-hint"], "missing argument-hint");
|
|
});
|
|
|
|
it(`${step} body references $@`, () => {
|
|
const { body } = readPrompt(step);
|
|
assert.ok(body.includes("$@"), "p-* body must use $@ for arguments");
|
|
});
|
|
}
|
|
});
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// 3. Edge cases — boundaries of the format
|
|
// ---------------------------------------------------------------------------
|
|
|
|
describe("edge cases: chain format boundaries", () => {
|
|
it("every chain has at least one step", () => {
|
|
for (const name of Object.keys(EXPECTED_CHAINS)) {
|
|
const steps = parseChain(readChain(name).frontmatter.chain);
|
|
assert.ok(steps.length > 0, `${name} chain is empty`);
|
|
}
|
|
});
|
|
|
|
it("no chain has a duplicate step", () => {
|
|
for (const name of Object.keys(EXPECTED_CHAINS)) {
|
|
const steps = parseChain(readChain(name).frontmatter.chain);
|
|
assert.equal(
|
|
new Set(steps).size,
|
|
steps.length,
|
|
`${name} contains a duplicate step`,
|
|
);
|
|
}
|
|
});
|
|
|
|
it("every chain step is lowercase kebab-case starting with 'p-'", () => {
|
|
for (const name of Object.keys(EXPECTED_CHAINS)) {
|
|
const steps = parseChain(readChain(name).frontmatter.chain);
|
|
for (const step of steps) {
|
|
assert.match(step, /^p-[a-z0-9][a-z0-9-]*$/, `${name} step '${step}' is not p-kebab-case`);
|
|
}
|
|
}
|
|
});
|
|
|
|
it("chain frontmatter uses ' -> ' (single space) and not '->' or ' - >'", () => {
|
|
// This is the format the pi-subagents adapter's `splitPromptChain`
|
|
// actually understands.
|
|
for (const name of Object.keys(EXPECTED_CHAINS)) {
|
|
const raw = readChain(name).frontmatter.chain;
|
|
assert.ok(raw, `${name} has no chain value`);
|
|
assert.doesNotMatch(raw, /[^ ]->/, `${name} uses bare '->' which the adapter will not split on`);
|
|
}
|
|
});
|
|
|
|
it("every chain ends with a real prompt file, not a typo", () => {
|
|
for (const name of Object.keys(EXPECTED_CHAINS)) {
|
|
const last = parseChain(readChain(name).frontmatter.chain).at(-1);
|
|
assert.ok(last && promptFiles.includes(join(PROMPTS_DIR, `${last}.md`)),
|
|
`${name} last step '${last}' does not exist on disk`);
|
|
}
|
|
});
|
|
});
|
|
|
|
describe("edge cases: argument-hint style", () => {
|
|
it("every argument-hint is non-empty", () => {
|
|
for (const name of Object.keys(EXPECTED_CHAINS)) {
|
|
const hint = readChain(name).frontmatter["argument-hint"];
|
|
assert.ok(hint && hint.length > 0, `${name} has empty argument-hint`);
|
|
}
|
|
for (const step of PROMPTS_USED_IN_CHAINS) {
|
|
const hint = readPrompt(step).frontmatter["argument-hint"];
|
|
assert.ok(hint && hint.length > 0, `${step} has empty argument-hint`);
|
|
}
|
|
});
|
|
|
|
it("argument-hints are wrapped in [...] to match the documented convention", () => {
|
|
// Doc/chain-commands.md and the existing recipes all use `[hint]`.
|
|
const bracket = /^\[.+\]$/;
|
|
for (const name of Object.keys(EXPECTED_CHAINS)) {
|
|
const hint = readChain(name).frontmatter["argument-hint"];
|
|
assert.match(hint, bracket, `${name} argument-hint not wrapped in []`);
|
|
}
|
|
for (const step of PROMPTS_USED_IN_CHAINS) {
|
|
const hint = readPrompt(step).frontmatter["argument-hint"];
|
|
assert.match(hint, bracket, `${step} argument-hint not wrapped in []`);
|
|
}
|
|
});
|
|
});
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// 4. Error cases — malformed / missing / inconsistent input
|
|
// ---------------------------------------------------------------------------
|
|
|
|
describe("error cases: malformed frontmatter", () => {
|
|
it("rejects a chain file with no closing fence", () => {
|
|
const { frontmatter } = parseFrontmatter(
|
|
"---\ndescription: broken\nchain: a -> b\n",
|
|
);
|
|
// Without a closing fence, the parser should return no frontmatter
|
|
// (mirroring the real adapter), so the chain is empty.
|
|
assert.deepEqual(frontmatter, {});
|
|
});
|
|
|
|
it("rejects a chain value with no separator at all", () => {
|
|
assert.deepEqual(parseChain("only-one-step"), ["only-one-step"]);
|
|
assert.equal(parseChain("only-one-step").length, 1);
|
|
});
|
|
|
|
it("flags a chain step that does not exist on disk", () => {
|
|
// Simulate what the adapter does: build a fake chain value and
|
|
// confirm we'd catch the missing file.
|
|
const fakeStep = "p-does-not-exist";
|
|
const steps = parseChain(`p-brainstorm -> ${fakeStep}`);
|
|
assert.ok(steps.includes(fakeStep));
|
|
assert.ok(
|
|
!promptFiles.includes(join(PROMPTS_DIR, `${fakeStep}.md`)),
|
|
"sanity: the fake step should not exist on disk",
|
|
);
|
|
});
|
|
});
|
|
|
|
describe("error cases: required keys", () => {
|
|
it("every chain recipe has a non-empty description", () => {
|
|
for (const name of Object.keys(EXPECTED_CHAINS)) {
|
|
const d = readChain(name).frontmatter.description;
|
|
assert.ok(d && d.trim().length > 0, `${name} has empty description`);
|
|
}
|
|
});
|
|
|
|
it("no p-* used in a chain is missing the subagent/fork flags", () => {
|
|
// This is the failure mode Doc/chain-commands.md calls out:
|
|
// "fork: true is required on each p-*.md. Check frontmatter."
|
|
for (const step of PROMPTS_USED_IN_CHAINS) {
|
|
const fm = readPrompt(step).frontmatter;
|
|
assert.equal(fm.subagent, "true", `${step} missing subagent: true`);
|
|
assert.equal(fm.fork, "true", `${step} missing fork: true`);
|
|
}
|
|
});
|
|
});
|
|
|
|
describe("error cases: cross-doc consistency", () => {
|
|
// These tests catch the realistic drift bug: someone adds a new chain
|
|
// recipe and forgets to add a row to the table in the Doc page. The
|
|
// table is the canonical description, so we look for a row, not just
|
|
// any mention of the name (the docs also reference recipes in
|
|
// `/prompt-workflow <name>` examples, which would mask a missing row).
|
|
//
|
|
// chain-commands.md table uses the form | `c-design-build.md` | ... |
|
|
// prompts.md table uses the form | **`c-design-build`** | ... |
|
|
|
|
it("Doc/chain-commands.md has a table row for every chain recipe", () => {
|
|
const text = readUtf8(join(PROMPTS_DIR, "Doc", "chain-commands.md"));
|
|
for (const name of Object.keys(EXPECTED_CHAINS)) {
|
|
// The chain-commands.md table uses the full filename in
|
|
// backticks (e.g. `c-design-build.md`).
|
|
const re = new RegExp(`^\\|\\s*\`${name}\`\\s*\\|`, "m");
|
|
assert.ok(
|
|
re.test(text),
|
|
`Doc/chain-commands.md is missing a table row for ${name}`,
|
|
);
|
|
}
|
|
});
|
|
|
|
it("Doc/chain-commands.md table rows show the actual chain sequence", () => {
|
|
const text = readUtf8(join(PROMPTS_DIR, "Doc", "chain-commands.md"));
|
|
for (const [name, steps] of Object.entries(EXPECTED_CHAINS)) {
|
|
const re = new RegExp(`^\\|\\s*\`${name}\`\\s*\\|[^|]*\\|[^|]*\\|`, "m");
|
|
const row = text.match(re);
|
|
assert.ok(row, `Doc/chain-commands.md is missing a row for ${name}`);
|
|
const rowAscii = row[0].replace(/→/g, "->");
|
|
for (const step of steps) {
|
|
assert.ok(
|
|
rowAscii.includes(step),
|
|
`Doc/chain-commands.md row for ${name} does not list ${step}`,
|
|
);
|
|
}
|
|
}
|
|
});
|
|
|
|
it("Doc/prompts.md has a table row for every chain recipe", () => {
|
|
const text = readUtf8(join(PROMPTS_DIR, "Doc", "prompts.md"));
|
|
for (const name of Object.keys(EXPECTED_CHAINS)) {
|
|
// The prompts.md table uses the base name in backticks
|
|
// surrounded by `**` (e.g. `**`c-design-build`**`).
|
|
const base = name.replace(/\.md$/, "");
|
|
const re = new RegExp(`^\\|\\s*\\*\\*\`${base}\`\\*\\*\\s*\\|`, "m");
|
|
assert.ok(
|
|
re.test(text),
|
|
`Doc/prompts.md is missing a table row for ${base}`,
|
|
);
|
|
}
|
|
});
|
|
|
|
it("Doc/prompts.md table rows show the actual chain sequence", () => {
|
|
const text = readUtf8(join(PROMPTS_DIR, "Doc", "prompts.md"));
|
|
for (const [name, steps] of Object.entries(EXPECTED_CHAINS)) {
|
|
const base = name.replace(/\.md$/, "");
|
|
const re = new RegExp(`^\\|\\s*\\*\\*\`${base}\`\\*\\*\\s*\\|[^|]*\\|`, "m");
|
|
const row = text.match(re);
|
|
assert.ok(row, `Doc/prompts.md is missing a row for ${base}`);
|
|
const rowAscii = row[0].replace(/→/g, "->");
|
|
for (const step of steps) {
|
|
assert.ok(
|
|
rowAscii.includes(step),
|
|
`Doc/prompts.md row for ${base} does not list ${step}`,
|
|
);
|
|
}
|
|
}
|
|
});
|
|
|
|
it("Doc/chain-commands.md documents ' -> ' as the chain separator", () => {
|
|
const text = readUtf8(join(PROMPTS_DIR, "Doc", "chain-commands.md"));
|
|
assert.ok(
|
|
text.includes("->") || text.includes("→"),
|
|
"Doc/chain-commands.md should describe the chain arrow separator",
|
|
);
|
|
});
|
|
|
|
it("Doc/prompts.md mentions every prompt used in any chain", () => {
|
|
const text = readUtf8(join(PROMPTS_DIR, "Doc", "prompts.md"));
|
|
for (const step of PROMPTS_USED_IN_CHAINS) {
|
|
assert.ok(
|
|
text.includes(step),
|
|
`Doc/prompts.md does not mention prompt ${step}`,
|
|
);
|
|
}
|
|
});
|
|
});
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// 5. Directory-level sanity — catches accidental deletions
|
|
// ---------------------------------------------------------------------------
|
|
|
|
describe("directory sanity", () => {
|
|
it("has at least one chain recipe", () => {
|
|
assert.ok(chainFiles.length > 0, "no c-*.md files found");
|
|
});
|
|
|
|
it("has at least one p-*.md prompt", () => {
|
|
assert.ok(promptFiles.length > 0, "no p-*.md files found");
|
|
});
|
|
|
|
it("has both Doc pages", () => {
|
|
assert.ok(docFiles.some((p) => p.endsWith("chain-commands.md")));
|
|
assert.ok(docFiles.some((p) => p.endsWith("prompts.md")));
|
|
});
|
|
|
|
it("Doc/ has no extra markdown files we did not expect", () => {
|
|
// If a new doc page is added, the cross-doc tests above should
|
|
// grow with it. Catches accidental orphans in the other direction.
|
|
const known = new Set(["chain-commands.md", "prompts.md"]);
|
|
for (const p of docFiles) {
|
|
const base = p.split(/[/\\]/).pop()!;
|
|
assert.ok(known.has(base), `unexpected Doc file: ${base}`);
|
|
}
|
|
});
|
|
|
|
it("every file in the prompts root is a recognized c-*, p-*, or Doc entry", () => {
|
|
const entries = readdirSync(PROMPTS_DIR);
|
|
const allowed = (n: string) =>
|
|
n.startsWith("c-") || n.startsWith("p-") || n === "Doc" || n === "test";
|
|
for (const e of entries) {
|
|
if (e.startsWith(".")) continue; // .pi-subagents, etc.
|
|
assert.ok(
|
|
allowed(e),
|
|
`unexpected top-level entry in prompts dir: ${e}`,
|
|
);
|
|
}
|
|
});
|
|
});
|