commit pi sessions
This commit is contained in:
liph
@@ -0,0 +1,492 @@
|
||||
/**
|
||||
* Tests for the prompt workflow chain under `.pi/agent/prompts/`.
|
||||
*
|
||||
* Scope (static, no LLM):
|
||||
* - Frontmatter of every `c-*.md` recipe and `p-*.md` step is well-formed
|
||||
* and contains the keys the pi-subagents adapter requires.
|
||||
* - Every step named in a `chain:` resolves to an existing `p-*.md` file.
|
||||
* - Every step has `subagent: true` and `fork: true`, which `Doc/chain-commands.md`
|
||||
* calls out as required for `p-*.md` chain steps.
|
||||
* - The Doc pages describe the same recipes / sequences that exist on disk.
|
||||
*
|
||||
* These tests do not exercise the LLM. They are independent, deterministic,
|
||||
* and fast (run against the on-disk tree only).
|
||||
*/
|
||||
|
||||
import { describe, it, before } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { readFileSync, readdirSync } from "node:fs";
|
||||
import { join, resolve, dirname } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import { parseFrontmatter, parseChain } from "./support/frontmatter.ts";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Locating the prompts directory
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const here = dirname(fileURLToPath(import.meta.url));
|
||||
// test/ -> .pi/agent/prompts/
|
||||
const PROMPTS_DIR = resolve(here, "..");
|
||||
|
||||
function readUtf8(path: string): string {
|
||||
return readFileSync(path, "utf8");
|
||||
}
|
||||
|
||||
function readDoc(path: string): { frontmatter: Record<string, string>; body: string } {
|
||||
return parseFrontmatter(readUtf8(path));
|
||||
}
|
||||
|
||||
let chainFiles: string[] = [];
|
||||
let promptFiles: string[] = [];
|
||||
let docFiles: string[] = [];
|
||||
|
||||
before(() => {
|
||||
chainFiles = readdirSync(PROMPTS_DIR)
|
||||
.filter((n) => n.startsWith("c-") && n.endsWith(".md"))
|
||||
.sort()
|
||||
.map((n) => join(PROMPTS_DIR, n));
|
||||
promptFiles = readdirSync(PROMPTS_DIR)
|
||||
.filter((n) => n.startsWith("p-") && n.endsWith(".md"))
|
||||
.sort()
|
||||
.map((n) => join(PROMPTS_DIR, n));
|
||||
docFiles = readdirSync(join(PROMPTS_DIR, "Doc"))
|
||||
.filter((n) => n.endsWith(".md"))
|
||||
.sort()
|
||||
.map((n) => join(PROMPTS_DIR, "Doc", n));
|
||||
});
|
||||
|
||||
function readChain(name: string) {
|
||||
const path = join(PROMPTS_DIR, name);
|
||||
return { path, name, ...readDoc(path) };
|
||||
}
|
||||
|
||||
function readPrompt(step: string) {
|
||||
// Accept both `p-brainstorm` and `p-brainstorm.md`; the actual files
|
||||
// end in `.md` but the chain frontmatter references them without it.
|
||||
const fileName = step.endsWith(".md") ? step : `${step}.md`;
|
||||
const path = join(PROMPTS_DIR, fileName);
|
||||
return { path, step, ...readDoc(path) };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Test data
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const EXPECTED_CHAINS: Record<string, string[]> = {
|
||||
"c-design-build.md": ["p-brainstorm", "p-plan", "p-test", "p-commit"],
|
||||
"c-fix-commit.md": ["p-summarize", "p-diagnose", "p-debug", "p-test"],
|
||||
"c-refactor-commit.md": ["p-refactor", "p-test", "p-review", "p-commit"],
|
||||
"c-security-loop.md": ["p-secaudit", "p-review", "p-test"],
|
||||
};
|
||||
|
||||
const PROMPTS_USED_IN_CHAINS = new Set<string>(
|
||||
Object.values(EXPECTED_CHAINS).flat(),
|
||||
);
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 1. Self-tests for the frontmatter/chain helpers (the test infra must work)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("support: frontmatter parser", () => {
|
||||
it("parses a simple key: value block", () => {
|
||||
const { frontmatter, body } = parseFrontmatter(
|
||||
'---\nfoo: bar\nbaz: "qux"\n---\nhello',
|
||||
);
|
||||
assert.equal(frontmatter.foo, "bar");
|
||||
assert.equal(frontmatter.baz, "qux");
|
||||
assert.equal(body, "hello");
|
||||
});
|
||||
|
||||
it("returns empty frontmatter when the file has no fence", () => {
|
||||
const { frontmatter } = parseFrontmatter("no fence here");
|
||||
assert.deepEqual(frontmatter, {});
|
||||
});
|
||||
|
||||
it("returns empty frontmatter when the fence is unterminated", () => {
|
||||
const { frontmatter } = parseFrontmatter("---\nfoo: bar\nstill going");
|
||||
assert.deepEqual(frontmatter, {});
|
||||
});
|
||||
|
||||
it("de-indents a block scalar and trims leading newlines", () => {
|
||||
const { frontmatter } = parseFrontmatter(
|
||||
"---\nbody: |\n line one\n line two\n line three\n---\n",
|
||||
);
|
||||
assert.equal(frontmatter.body, "line one\nline two\nline three");
|
||||
});
|
||||
|
||||
it("strips full-line comments and ignores empty lines between keys", () => {
|
||||
const { frontmatter } = parseFrontmatter(
|
||||
"---\nfoo: bar\n# full-line comment\nbaz: qux\n---\n",
|
||||
);
|
||||
assert.equal(frontmatter.foo, "bar");
|
||||
assert.equal(frontmatter.baz, "qux");
|
||||
// Inline `#` after a value is left in the value (matches the
|
||||
// upstream `parseFrontmatter` in `pi-subagents`). This protects
|
||||
// against accidental whitespace stripping.
|
||||
const inline = parseFrontmatter("---\nfoo: bar # trailing\n---\n");
|
||||
assert.equal(inline.frontmatter.foo, "bar # trailing");
|
||||
});
|
||||
|
||||
it("normalizes CRLF line endings", () => {
|
||||
const { frontmatter, body } = parseFrontmatter(
|
||||
"---\r\nfoo: bar\r\n---\r\nbody",
|
||||
);
|
||||
assert.equal(frontmatter.foo, "bar");
|
||||
assert.equal(body, "body");
|
||||
});
|
||||
});
|
||||
|
||||
describe("support: chain parser", () => {
|
||||
it("splits on ' -> ' and trims", () => {
|
||||
assert.deepEqual(parseChain("a -> b -> c"), ["a", "b", "c"]);
|
||||
});
|
||||
|
||||
it("returns [] for undefined", () => {
|
||||
assert.deepEqual(parseChain(undefined), []);
|
||||
});
|
||||
|
||||
it("returns [] for an empty string", () => {
|
||||
assert.deepEqual(parseChain(""), []);
|
||||
});
|
||||
|
||||
it("drops empty parts from extra spaces", () => {
|
||||
assert.deepEqual(parseChain("a -> -> b"), ["a", "b"]);
|
||||
});
|
||||
|
||||
it("does not split on a single dash", () => {
|
||||
// A chain value should not use `-` as a separator; the actual
|
||||
// recipes use ` -> `. This protects against future drift.
|
||||
assert.deepEqual(parseChain("a-b-c"), ["a-b-c"]);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 2. Happy path — the on-disk tree matches what we expect
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("happy path: chain recipes are present and well-formed", () => {
|
||||
for (const [name, expectedSteps] of Object.entries(EXPECTED_CHAINS)) {
|
||||
it(`loads ${name}`, () => {
|
||||
assert.ok(
|
||||
chainFiles.includes(join(PROMPTS_DIR, name)),
|
||||
`missing chain recipe: ${name}`,
|
||||
);
|
||||
});
|
||||
|
||||
it(`${name} has description, argument-hint, and chain fields`, () => {
|
||||
const { frontmatter } = readChain(name);
|
||||
assert.ok(frontmatter.description, "missing description");
|
||||
assert.ok(frontmatter["argument-hint"], "missing argument-hint");
|
||||
assert.ok(frontmatter.chain, "missing chain");
|
||||
});
|
||||
|
||||
it(`${name}.chain parses to ${expectedSteps.join(" -> ")}`, () => {
|
||||
const { frontmatter } = readChain(name);
|
||||
assert.deepEqual(parseChain(frontmatter.chain), expectedSteps);
|
||||
});
|
||||
|
||||
it(`${name} contains a $@ placeholder for the user argument`, () => {
|
||||
const { body } = readChain(name);
|
||||
assert.ok(
|
||||
body.includes("$@"),
|
||||
"chain recipe body must use $@ for the user-supplied argument",
|
||||
);
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
describe("happy path: every prompt step resolves to a real file", () => {
|
||||
for (const step of PROMPTS_USED_IN_CHAINS) {
|
||||
it(`step ${step} exists as a p-*.md file`, () => {
|
||||
assert.ok(
|
||||
promptFiles.includes(join(PROMPTS_DIR, `${step}.md`)),
|
||||
`chain references ${step} but no such p-*.md exists`,
|
||||
);
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
describe("happy path: every p-*.md has the required chain-step shape", () => {
|
||||
for (const step of PROMPTS_USED_IN_CHAINS) {
|
||||
it(`${step} declares subagent: true`, () => {
|
||||
const { frontmatter } = readPrompt(step);
|
||||
assert.equal(frontmatter.subagent, "true");
|
||||
});
|
||||
|
||||
it(`${step} declares fork: true`, () => {
|
||||
const { frontmatter } = readPrompt(step);
|
||||
assert.equal(frontmatter.fork, "true");
|
||||
});
|
||||
|
||||
it(`${step} has description and argument-hint`, () => {
|
||||
const { frontmatter } = readPrompt(step);
|
||||
assert.ok(frontmatter.description, "missing description");
|
||||
assert.ok(frontmatter["argument-hint"], "missing argument-hint");
|
||||
});
|
||||
|
||||
it(`${step} body references $@`, () => {
|
||||
const { body } = readPrompt(step);
|
||||
assert.ok(body.includes("$@"), "p-* body must use $@ for arguments");
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 3. Edge cases — boundaries of the format
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("edge cases: chain format boundaries", () => {
|
||||
it("every chain has at least one step", () => {
|
||||
for (const name of Object.keys(EXPECTED_CHAINS)) {
|
||||
const steps = parseChain(readChain(name).frontmatter.chain);
|
||||
assert.ok(steps.length > 0, `${name} chain is empty`);
|
||||
}
|
||||
});
|
||||
|
||||
it("no chain has a duplicate step", () => {
|
||||
for (const name of Object.keys(EXPECTED_CHAINS)) {
|
||||
const steps = parseChain(readChain(name).frontmatter.chain);
|
||||
assert.equal(
|
||||
new Set(steps).size,
|
||||
steps.length,
|
||||
`${name} contains a duplicate step`,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("every chain step is lowercase kebab-case starting with 'p-'", () => {
|
||||
for (const name of Object.keys(EXPECTED_CHAINS)) {
|
||||
const steps = parseChain(readChain(name).frontmatter.chain);
|
||||
for (const step of steps) {
|
||||
assert.match(step, /^p-[a-z0-9][a-z0-9-]*$/, `${name} step '${step}' is not p-kebab-case`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it("chain frontmatter uses ' -> ' (single space) and not '->' or ' - >'", () => {
|
||||
// This is the format the pi-subagents adapter's `splitPromptChain`
|
||||
// actually understands.
|
||||
for (const name of Object.keys(EXPECTED_CHAINS)) {
|
||||
const raw = readChain(name).frontmatter.chain;
|
||||
assert.ok(raw, `${name} has no chain value`);
|
||||
assert.doesNotMatch(raw, /[^ ]->/, `${name} uses bare '->' which the adapter will not split on`);
|
||||
}
|
||||
});
|
||||
|
||||
it("every chain ends with a real prompt file, not a typo", () => {
|
||||
for (const name of Object.keys(EXPECTED_CHAINS)) {
|
||||
const last = parseChain(readChain(name).frontmatter.chain).at(-1);
|
||||
assert.ok(last && promptFiles.includes(join(PROMPTS_DIR, `${last}.md`)),
|
||||
`${name} last step '${last}' does not exist on disk`);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("edge cases: argument-hint style", () => {
|
||||
it("every argument-hint is non-empty", () => {
|
||||
for (const name of Object.keys(EXPECTED_CHAINS)) {
|
||||
const hint = readChain(name).frontmatter["argument-hint"];
|
||||
assert.ok(hint && hint.length > 0, `${name} has empty argument-hint`);
|
||||
}
|
||||
for (const step of PROMPTS_USED_IN_CHAINS) {
|
||||
const hint = readPrompt(step).frontmatter["argument-hint"];
|
||||
assert.ok(hint && hint.length > 0, `${step} has empty argument-hint`);
|
||||
}
|
||||
});
|
||||
|
||||
it("argument-hints are wrapped in [...] to match the documented convention", () => {
|
||||
// Doc/chain-commands.md and the existing recipes all use `[hint]`.
|
||||
const bracket = /^\[.+\]$/;
|
||||
for (const name of Object.keys(EXPECTED_CHAINS)) {
|
||||
const hint = readChain(name).frontmatter["argument-hint"];
|
||||
assert.match(hint, bracket, `${name} argument-hint not wrapped in []`);
|
||||
}
|
||||
for (const step of PROMPTS_USED_IN_CHAINS) {
|
||||
const hint = readPrompt(step).frontmatter["argument-hint"];
|
||||
assert.match(hint, bracket, `${step} argument-hint not wrapped in []`);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 4. Error cases — malformed / missing / inconsistent input
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("error cases: malformed frontmatter", () => {
|
||||
it("rejects a chain file with no closing fence", () => {
|
||||
const { frontmatter } = parseFrontmatter(
|
||||
"---\ndescription: broken\nchain: a -> b\n",
|
||||
);
|
||||
// Without a closing fence, the parser should return no frontmatter
|
||||
// (mirroring the real adapter), so the chain is empty.
|
||||
assert.deepEqual(frontmatter, {});
|
||||
});
|
||||
|
||||
it("rejects a chain value with no separator at all", () => {
|
||||
assert.deepEqual(parseChain("only-one-step"), ["only-one-step"]);
|
||||
assert.equal(parseChain("only-one-step").length, 1);
|
||||
});
|
||||
|
||||
it("flags a chain step that does not exist on disk", () => {
|
||||
// Simulate what the adapter does: build a fake chain value and
|
||||
// confirm we'd catch the missing file.
|
||||
const fakeStep = "p-does-not-exist";
|
||||
const steps = parseChain(`p-brainstorm -> ${fakeStep}`);
|
||||
assert.ok(steps.includes(fakeStep));
|
||||
assert.ok(
|
||||
!promptFiles.includes(join(PROMPTS_DIR, `${fakeStep}.md`)),
|
||||
"sanity: the fake step should not exist on disk",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("error cases: required keys", () => {
|
||||
it("every chain recipe has a non-empty description", () => {
|
||||
for (const name of Object.keys(EXPECTED_CHAINS)) {
|
||||
const d = readChain(name).frontmatter.description;
|
||||
assert.ok(d && d.trim().length > 0, `${name} has empty description`);
|
||||
}
|
||||
});
|
||||
|
||||
it("no p-* used in a chain is missing the subagent/fork flags", () => {
|
||||
// This is the failure mode Doc/chain-commands.md calls out:
|
||||
// "fork: true is required on each p-*.md. Check frontmatter."
|
||||
for (const step of PROMPTS_USED_IN_CHAINS) {
|
||||
const fm = readPrompt(step).frontmatter;
|
||||
assert.equal(fm.subagent, "true", `${step} missing subagent: true`);
|
||||
assert.equal(fm.fork, "true", `${step} missing fork: true`);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("error cases: cross-doc consistency", () => {
|
||||
// These tests catch the realistic drift bug: someone adds a new chain
|
||||
// recipe and forgets to add a row to the table in the Doc page. The
|
||||
// table is the canonical description, so we look for a row, not just
|
||||
// any mention of the name (the docs also reference recipes in
|
||||
// `/prompt-workflow <name>` examples, which would mask a missing row).
|
||||
//
|
||||
// chain-commands.md table uses the form | `c-design-build.md` | ... |
|
||||
// prompts.md table uses the form | **`c-design-build`** | ... |
|
||||
|
||||
it("Doc/chain-commands.md has a table row for every chain recipe", () => {
|
||||
const text = readUtf8(join(PROMPTS_DIR, "Doc", "chain-commands.md"));
|
||||
for (const name of Object.keys(EXPECTED_CHAINS)) {
|
||||
// The chain-commands.md table uses the full filename in
|
||||
// backticks (e.g. `c-design-build.md`).
|
||||
const re = new RegExp(`^\\|\\s*\`${name}\`\\s*\\|`, "m");
|
||||
assert.ok(
|
||||
re.test(text),
|
||||
`Doc/chain-commands.md is missing a table row for ${name}`,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("Doc/chain-commands.md table rows show the actual chain sequence", () => {
|
||||
const text = readUtf8(join(PROMPTS_DIR, "Doc", "chain-commands.md"));
|
||||
for (const [name, steps] of Object.entries(EXPECTED_CHAINS)) {
|
||||
const re = new RegExp(`^\\|\\s*\`${name}\`\\s*\\|[^|]*\\|[^|]*\\|`, "m");
|
||||
const row = text.match(re);
|
||||
assert.ok(row, `Doc/chain-commands.md is missing a row for ${name}`);
|
||||
const rowAscii = row[0].replace(/→/g, "->");
|
||||
for (const step of steps) {
|
||||
assert.ok(
|
||||
rowAscii.includes(step),
|
||||
`Doc/chain-commands.md row for ${name} does not list ${step}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it("Doc/prompts.md has a table row for every chain recipe", () => {
|
||||
const text = readUtf8(join(PROMPTS_DIR, "Doc", "prompts.md"));
|
||||
for (const name of Object.keys(EXPECTED_CHAINS)) {
|
||||
// The prompts.md table uses the base name in backticks
|
||||
// surrounded by `**` (e.g. `**`c-design-build`**`).
|
||||
const base = name.replace(/\.md$/, "");
|
||||
const re = new RegExp(`^\\|\\s*\\*\\*\`${base}\`\\*\\*\\s*\\|`, "m");
|
||||
assert.ok(
|
||||
re.test(text),
|
||||
`Doc/prompts.md is missing a table row for ${base}`,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("Doc/prompts.md table rows show the actual chain sequence", () => {
|
||||
const text = readUtf8(join(PROMPTS_DIR, "Doc", "prompts.md"));
|
||||
for (const [name, steps] of Object.entries(EXPECTED_CHAINS)) {
|
||||
const base = name.replace(/\.md$/, "");
|
||||
const re = new RegExp(`^\\|\\s*\\*\\*\`${base}\`\\*\\*\\s*\\|[^|]*\\|`, "m");
|
||||
const row = text.match(re);
|
||||
assert.ok(row, `Doc/prompts.md is missing a row for ${base}`);
|
||||
const rowAscii = row[0].replace(/→/g, "->");
|
||||
for (const step of steps) {
|
||||
assert.ok(
|
||||
rowAscii.includes(step),
|
||||
`Doc/prompts.md row for ${base} does not list ${step}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it("Doc/chain-commands.md documents ' -> ' as the chain separator", () => {
|
||||
const text = readUtf8(join(PROMPTS_DIR, "Doc", "chain-commands.md"));
|
||||
assert.ok(
|
||||
text.includes("->") || text.includes("→"),
|
||||
"Doc/chain-commands.md should describe the chain arrow separator",
|
||||
);
|
||||
});
|
||||
|
||||
it("Doc/prompts.md mentions every prompt used in any chain", () => {
|
||||
const text = readUtf8(join(PROMPTS_DIR, "Doc", "prompts.md"));
|
||||
for (const step of PROMPTS_USED_IN_CHAINS) {
|
||||
assert.ok(
|
||||
text.includes(step),
|
||||
`Doc/prompts.md does not mention prompt ${step}`,
|
||||
);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 5. Directory-level sanity — catches accidental deletions
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("directory sanity", () => {
|
||||
it("has at least one chain recipe", () => {
|
||||
assert.ok(chainFiles.length > 0, "no c-*.md files found");
|
||||
});
|
||||
|
||||
it("has at least one p-*.md prompt", () => {
|
||||
assert.ok(promptFiles.length > 0, "no p-*.md files found");
|
||||
});
|
||||
|
||||
it("has both Doc pages", () => {
|
||||
assert.ok(docFiles.some((p) => p.endsWith("chain-commands.md")));
|
||||
assert.ok(docFiles.some((p) => p.endsWith("prompts.md")));
|
||||
});
|
||||
|
||||
it("Doc/ has no extra markdown files we did not expect", () => {
|
||||
// If a new doc page is added, the cross-doc tests above should
|
||||
// grow with it. Catches accidental orphans in the other direction.
|
||||
const known = new Set(["chain-commands.md", "prompts.md"]);
|
||||
for (const p of docFiles) {
|
||||
const base = p.split(/[/\\]/).pop()!;
|
||||
assert.ok(known.has(base), `unexpected Doc file: ${base}`);
|
||||
}
|
||||
});
|
||||
|
||||
it("every file in the prompts root is a recognized c-*, p-*, or Doc entry", () => {
|
||||
const entries = readdirSync(PROMPTS_DIR);
|
||||
const allowed = (n: string) =>
|
||||
n.startsWith("c-") || n.startsWith("p-") || n === "Doc" || n === "test";
|
||||
for (const e of entries) {
|
||||
if (e.startsWith(".")) continue; // .pi-subagents, etc.
|
||||
assert.ok(
|
||||
allowed(e),
|
||||
`unexpected top-level entry in prompts dir: ${e}`,
|
||||
);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,9 @@
|
||||
---
|
||||
description: A test fixture prompt
|
||||
argument-hint: "[thing]"
|
||||
subagent: true
|
||||
fork: true
|
||||
body: |
|
||||
This is a multi-line
|
||||
block scalar body.
|
||||
---
|
||||
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"name": "pi-prompt-workflows-tests",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"description": "Static tests for the .pi/agent/prompts c-* and p-* workflow chain.",
|
||||
"scripts": {
|
||||
"test": "node --test --experimental-strip-types test/chain-workflow.test.ts",
|
||||
"test:file": "node --test --experimental-strip-types test/chain-workflow.test.ts"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,289 @@
|
||||
# Test design: add a rate-limit
|
||||
|
||||
> Deliverable for the `p-test` step of `c-design-build` (subject: "add a rate-limit").
|
||||
> Matches the existing `test/` conventions: `node --test` + `node:assert/strict`,
|
||||
> small independent tests, no LLM, no real clock by default.
|
||||
|
||||
## 1. Existing conventions
|
||||
|
||||
| Aspect | Convention in `test/` | Implication for rate-limit tests |
|
||||
|---|---|---|
|
||||
| Framework | `node:test` + `node:assert/strict` | Use the same; no jest/vitest. |
|
||||
| File layout | `test/<feature>.test.ts` next to `support/` | New file: `test/rate-limit.test.ts`. |
|
||||
| Imports | `import { describe, it, before } from "node:test";` | Identical. |
|
||||
| Assertions | `assert.equal`, `assert.deepEqual`, `assert.match` | Use strict asserts; avoid `assert.ok(x === y)`. |
|
||||
| Module type | `"type": "module"` in `package.json` | Use ESM, `.ts` extension in import specifiers. |
|
||||
| Test discovery | `node --test --experimental-strip-types <file>` | Same one-liner; no jest config. |
|
||||
| Time / randomness | Not yet used; tests are static and synchronous | Inject a `now()` function so we can advance the clock deterministically. |
|
||||
| Fixtures | `test/fixtures/valid-prompt.md` is a one-off example | Reuse the same pattern only if a non-trivial fixture is needed; prefer inline data. |
|
||||
| Support helpers | `test/support/frontmatter.ts` exposes `parseFrontmatter`, `parseChain` | Add `test/support/rate-limit.ts` for the SUT + clock injection; keep helpers pure. |
|
||||
|
||||
## 2. Test cases to add
|
||||
|
||||
Grouped by behavior. Each test is one `it(...)` and asserts one thing.
|
||||
|
||||
### Happy path
|
||||
- `allowRequest` returns `true` for the first request from a fresh key.
|
||||
- A burst up to the configured `limit` inside the window all return `true`.
|
||||
- A single request after the window has elapsed returns `true` again (full reset on window expiry).
|
||||
- Per-key isolation: a request from key `A` does not consume a slot for key `B`.
|
||||
- The limiter is independent across two distinct limiter instances (no shared global state).
|
||||
- `getStats(key)` reports `used`, `limit`, `remaining`, and `resetAt` matching the last operation.
|
||||
|
||||
### Edge cases
|
||||
- `limit = 0`: every request is rejected (degenerate but defined).
|
||||
- `limit = 1`: second request in the same window is rejected, the first is allowed.
|
||||
- Window boundary: request at exactly `windowMs` after the first is treated as a fresh window (off-by-one). Use a `now` function that returns the boundary value explicitly.
|
||||
- Two consecutive requests straddling the window: first allowed, second allowed, counts reset.
|
||||
- Large `limit` (e.g. `1_000_000`): behaves like an effectively unbounded limiter; one test with a high value is enough.
|
||||
- Empty / whitespace key string: rejected up front with a typed error, never reaches the bucket logic.
|
||||
- `now` going backwards (clock skew): does not grant extra capacity; refuses to decrement below zero.
|
||||
- `now` going forwards by exactly `windowMs + 1`: bucket fully reset; one extra request is allowed.
|
||||
|
||||
### Error cases
|
||||
- Missing required options (`limit` or `windowMs`): throws `TypeError` with a message that includes the missing key.
|
||||
- `limit` is negative or non-integer: throws `RangeError` (or `TypeError` — pick one and assert it consistently).
|
||||
- `windowMs` is zero or negative: throws `RangeError`.
|
||||
- Unknown option key (e.g. `limt` typo) is rejected to catch silent config bugs; throws with the unknown key in the message.
|
||||
- A custom `now` that throws propagates the error (no swallowed exceptions).
|
||||
- Concurrency: 100 synchronous calls to `allowRequest` for the same key with `limit = 50` result in exactly 50 `true` and 50 `false`, no off-by-one, no double-allow.
|
||||
|
||||
## 3. Test code
|
||||
|
||||
Target file: `test/rate-limit.test.ts`. The SUT is not yet in this repo; the
|
||||
tests assume it is implemented at `src/rate-limit.ts` with this minimal
|
||||
contract (declared here so the tests are runnable as soon as the SUT lands):
|
||||
|
||||
```ts
|
||||
// test/rate-limit.test.ts
|
||||
|
||||
import { describe, it } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
import {
|
||||
createRateLimiter,
|
||||
type RateLimiter,
|
||||
type RateLimiterOptions,
|
||||
} from "../src/rate-limit.ts";
|
||||
|
||||
// --- helpers -------------------------------------------------------------
|
||||
|
||||
function makeLimiter(
|
||||
overrides: Partial<RateLimiterOptions> = {},
|
||||
now: () => number = () => 0,
|
||||
): RateLimiter {
|
||||
return createRateLimiter(
|
||||
{
|
||||
limit: 3,
|
||||
windowMs: 1_000,
|
||||
...overrides,
|
||||
},
|
||||
now,
|
||||
);
|
||||
}
|
||||
|
||||
// --- happy path ----------------------------------------------------------
|
||||
|
||||
describe("rate-limit: happy path", () => {
|
||||
it("allows the first request from a fresh key", () => {
|
||||
const rl = makeLimiter();
|
||||
assert.equal(rl.allowRequest("user:1"), true);
|
||||
});
|
||||
|
||||
it("allows a burst up to the configured limit inside the window", () => {
|
||||
const rl = makeLimiter({ limit: 3 });
|
||||
assert.equal(rl.allowRequest("user:1"), true);
|
||||
assert.equal(rl.allowRequest("user:1"), true);
|
||||
assert.equal(rl.allowRequest("user:1"), true);
|
||||
});
|
||||
|
||||
it("rejects the (limit + 1)th request in the same window", () => {
|
||||
const rl = makeLimiter({ limit: 3 });
|
||||
for (let i = 0; i < 3; i++) rl.allowRequest("user:1");
|
||||
assert.equal(rl.allowRequest("user:1"), false);
|
||||
});
|
||||
|
||||
it("allows again after the window has fully elapsed", () => {
|
||||
let t = 0;
|
||||
const clock = () => t;
|
||||
const rl = makeLimiter({ limit: 1, windowMs: 1_000 }, clock);
|
||||
assert.equal(rl.allowRequest("user:1"), true);
|
||||
assert.equal(rl.allowRequest("user:1"), false);
|
||||
t = 1_001; // strictly past the window
|
||||
assert.equal(rl.allowRequest("user:1"), true);
|
||||
});
|
||||
|
||||
it("isolates buckets per key", () => {
|
||||
const rl = makeLimiter({ limit: 1 });
|
||||
assert.equal(rl.allowRequest("a"), true);
|
||||
assert.equal(rl.allowRequest("a"), false);
|
||||
assert.equal(rl.allowRequest("b"), true);
|
||||
assert.equal(rl.allowRequest("b"), false);
|
||||
});
|
||||
|
||||
it("keeps two limiter instances independent", () => {
|
||||
const a = makeLimiter({ limit: 1 });
|
||||
const b = makeLimiter({ limit: 1 });
|
||||
assert.equal(a.allowRequest("k"), true);
|
||||
assert.equal(b.allowRequest("k"), true);
|
||||
});
|
||||
|
||||
it("getStats reports remaining and resetAt matching the bucket state", () => {
|
||||
let t = 0;
|
||||
const rl = makeLimiter({ limit: 2, windowMs: 1_000 }, () => t);
|
||||
rl.allowRequest("user:1");
|
||||
const stats = rl.getStats("user:1");
|
||||
assert.equal(stats.used, 1);
|
||||
assert.equal(stats.remaining, 1);
|
||||
assert.equal(stats.resetAt, 1_000);
|
||||
});
|
||||
});
|
||||
|
||||
// --- edge cases ----------------------------------------------------------
|
||||
|
||||
describe("rate-limit: edge cases", () => {
|
||||
it("limit = 0 rejects every request", () => {
|
||||
const rl = makeLimiter({ limit: 0 });
|
||||
assert.equal(rl.allowRequest("k"), false);
|
||||
});
|
||||
|
||||
it("limit = 1 allows exactly one request then rejects", () => {
|
||||
const rl = makeLimiter({ limit: 1 });
|
||||
assert.equal(rl.allowRequest("k"), true);
|
||||
assert.equal(rl.allowRequest("k"), false);
|
||||
});
|
||||
|
||||
it("treats a request at exactly windowMs as a fresh window (off-by-one)", () => {
|
||||
let t = 0;
|
||||
const rl = makeLimiter({ limit: 1, windowMs: 1_000 }, () => t);
|
||||
assert.equal(rl.allowRequest("k"), true);
|
||||
t = 1_000; // exactly at the boundary
|
||||
assert.equal(rl.allowRequest("k"), true);
|
||||
});
|
||||
|
||||
it("does not grant extra capacity when the clock goes backwards", () => {
|
||||
let t = 1_000;
|
||||
const rl = makeLimiter({ limit: 1, windowMs: 1_000 }, () => t);
|
||||
assert.equal(rl.allowRequest("k"), true);
|
||||
t = 500; // clock skew backwards
|
||||
assert.equal(rl.allowRequest("k"), false);
|
||||
assert.equal(rl.getStats("k").used, 1);
|
||||
});
|
||||
|
||||
it("rejects an empty or whitespace key", () => {
|
||||
const rl = makeLimiter();
|
||||
assert.throws(() => rl.allowRequest(""), /key/);
|
||||
assert.throws(() => rl.allowRequest(" "), /key/);
|
||||
});
|
||||
|
||||
it("handles a very large limit without overflowing", () => {
|
||||
const rl = makeLimiter({ limit: 1_000_000 });
|
||||
for (let i = 0; i < 10; i++) {
|
||||
assert.equal(rl.allowRequest("k"), true);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// --- error cases ---------------------------------------------------------
|
||||
|
||||
describe("rate-limit: error cases", () => {
|
||||
it("throws when limit is missing", () => {
|
||||
// @ts-expect-error: limit intentionally omitted
|
||||
assert.throws(() => createRateLimiter({ windowMs: 1_000 }), /limit/);
|
||||
});
|
||||
|
||||
it("throws when windowMs is missing", () => {
|
||||
// @ts-expect-error: windowMs intentionally omitted
|
||||
assert.throws(() => createRateLimiter({ limit: 1 }), /windowMs/);
|
||||
});
|
||||
|
||||
it("throws on a negative limit", () => {
|
||||
assert.throws(() => makeLimiter({ limit: -1 }), /limit/);
|
||||
});
|
||||
|
||||
it("throws on a non-integer limit", () => {
|
||||
assert.throws(() => makeLimiter({ limit: 1.5 }), /limit/);
|
||||
});
|
||||
|
||||
it("throws on a zero or negative windowMs", () => {
|
||||
assert.throws(() => makeLimiter({ windowMs: 0 }), /windowMs/);
|
||||
assert.throws(() => makeLimiter({ windowMs: -1 }), /windowMs/);
|
||||
});
|
||||
|
||||
it("rejects an unknown option key", () => {
|
||||
assert.throws(
|
||||
() => makeLimiter({ limt: 3 } as unknown as Partial<RateLimiterOptions>),
|
||||
/limt/,
|
||||
);
|
||||
});
|
||||
|
||||
it("propagates errors from a custom now()", () => {
|
||||
const rl = createRateLimiter(
|
||||
{ limit: 1, windowMs: 1_000 },
|
||||
() => { throw new Error("clock broken"); },
|
||||
);
|
||||
assert.throws(() => rl.allowRequest("k"), /clock broken/);
|
||||
});
|
||||
|
||||
it("is exact under synchronous burst: exactly limit trues, then falses", () => {
|
||||
const rl = makeLimiter({ limit: 50 });
|
||||
let allowed = 0;
|
||||
let rejected = 0;
|
||||
for (let i = 0; i < 100; i++) {
|
||||
if (rl.allowRequest("k")) allowed++;
|
||||
else rejected++;
|
||||
}
|
||||
assert.equal(allowed, 50);
|
||||
assert.equal(rejected, 50);
|
||||
});
|
||||
});
|
||||
```
|
||||
|
||||
## 4. How to run
|
||||
|
||||
From the repository root:
|
||||
|
||||
```bash
|
||||
# all tests in this directory
|
||||
cd test && npm test
|
||||
|
||||
# only the new rate-limit tests
|
||||
cd test && node --test --experimental-strip-types rate-limit.test.ts
|
||||
|
||||
# a single test by name pattern
|
||||
cd test && node --test --experimental-strip-types \
|
||||
--test-name-pattern="rejects the \\(limit \\+ 1\\)th" rate-limit.test.ts
|
||||
```
|
||||
|
||||
Notes:
|
||||
- `--experimental-strip-types` is required because the existing `test/package.json`
|
||||
already targets Node's built-in TS stripping; no separate tsconfig needed.
|
||||
- These tests are pure and synchronous, so the whole file should finish in
|
||||
well under 100 ms once the SUT lands.
|
||||
|
||||
## 5. Coverage gaps (flagged, not refactored in this pass)
|
||||
|
||||
1. **SUT does not exist yet.** `src/rate-limit.ts` is assumed. Until it
|
||||
lands with the contract declared at the top of section 3, the test file
|
||||
will fail to import. This is the single largest blocker.
|
||||
2. **Async / distributed cases are not covered.** The contract above is
|
||||
synchronous. If the production limiter is async (e.g. backed by Redis
|
||||
with `INCR` + `EXPIRE`), the burst test will need to be reworked; a
|
||||
`before`/`after` test that injects an in-memory store would be the
|
||||
smallest refactor.
|
||||
3. **Concurrency under a real event loop is not exercised.** A single
|
||||
synchronous `for` loop does not stress the microtask queue. If the SUT
|
||||
becomes async, add a `Promise.all` of N concurrent calls and assert
|
||||
that the count of `true` results equals `limit`.
|
||||
4. **No metrics / observability surface is assumed.** If the limiter is
|
||||
later expected to emit counters or events, that needs an interface
|
||||
change first; tests would follow in a follow-up.
|
||||
5. **No per-route or weighted limits.** A real production limiter often
|
||||
supports per-route buckets or weighted costs per request. Out of scope
|
||||
here; would require a contract change.
|
||||
6. **No clock injection in the SUT contract yet.** The design assumes
|
||||
`createRateLimiter(opts, now?)`. If the chosen implementation uses a
|
||||
module-level `Date.now()`, the clock-dependent tests in this file
|
||||
cannot be written deterministically; the SUT should be refactored to
|
||||
accept a `now` parameter (smallest viable change) before these tests
|
||||
are added.
|
||||
@@ -0,0 +1,123 @@
|
||||
/**
|
||||
* Minimal YAML frontmatter parser tailored to the prompt workflow files
|
||||
* in this directory. Mirrors the subset of YAML that the existing `c-*.md`
|
||||
* and `p-*.md` files actually use:
|
||||
*
|
||||
* key: value -> string
|
||||
* key: "quoted value" -> string (quotes stripped)
|
||||
* key: -> block scalar (everything more-indented is
|
||||
* collected, de-indented, and returned as one
|
||||
* string with newlines)
|
||||
*
|
||||
* Comments (`# ...`) and blank lines are skipped. Quoted strings have
|
||||
* their quotes stripped, matching the convention used in the rest of the
|
||||
* repo (no nested quoting or escapes are needed by the fixtures).
|
||||
*/
|
||||
export interface ParsedDoc {
|
||||
frontmatter: Record<string, string>;
|
||||
body: string;
|
||||
}
|
||||
|
||||
function stripComment(line: string): string {
|
||||
const idx = line.indexOf("#");
|
||||
// Only treat `#` as a comment when it appears at start-of-line or
|
||||
// after whitespace; otherwise the `#` is part of a value (it never
|
||||
// is in this repo, but it keeps the parser robust).
|
||||
if (idx === -1) return line;
|
||||
const before = line.slice(0, idx);
|
||||
if (/^\s*$/.test(before)) return before;
|
||||
return line;
|
||||
}
|
||||
|
||||
function deIndentBlock(block: string): string {
|
||||
const lines = block.split("\n");
|
||||
const indents = lines
|
||||
.filter((l) => l.trim() !== "")
|
||||
.map((l) => l.match(/^[ \t]*/)?.[0].length ?? 0);
|
||||
const min = indents.length === 0 ? 0 : Math.min(...indents);
|
||||
if (min === 0) return block.replace(/^\n+/, "");
|
||||
return lines
|
||||
.map((l) => (l.length >= min ? l.slice(min) : l))
|
||||
.join("\n")
|
||||
.replace(/^\n+/, "");
|
||||
}
|
||||
|
||||
export function parseFrontmatter(content: string): ParsedDoc {
|
||||
const normalized = content.replace(/\r\n/g, "\n");
|
||||
if (!normalized.startsWith("---")) {
|
||||
return { frontmatter: {}, body: normalized };
|
||||
}
|
||||
|
||||
// The closing fence is `\n---` (optionally followed by whitespace/eol).
|
||||
const endIndex = normalized.indexOf("\n---", 3);
|
||||
if (endIndex === -1) {
|
||||
return { frontmatter: {}, body: normalized };
|
||||
}
|
||||
|
||||
const block = normalized.slice(3, endIndex);
|
||||
const body = normalized.slice(endIndex + 4).replace(/^\s+/, "");
|
||||
|
||||
const frontmatter: Record<string, string> = {};
|
||||
let currentKey: string | null = null;
|
||||
let currentBlock: string[] | null = null;
|
||||
let currentIndent = -1;
|
||||
|
||||
const flush = () => {
|
||||
if (currentKey === null || currentBlock === null) return;
|
||||
frontmatter[currentKey] = deIndentBlock(currentBlock.join("\n"));
|
||||
currentKey = null;
|
||||
currentBlock = null;
|
||||
currentIndent = -1;
|
||||
};
|
||||
|
||||
for (const rawLine of block.split("\n")) {
|
||||
const line = stripComment(rawLine);
|
||||
if (line.trim() === "") {
|
||||
// Preserve blank lines inside an open block scalar, otherwise
|
||||
// ignore them between keys.
|
||||
if (currentBlock !== null) currentBlock.push("");
|
||||
continue;
|
||||
}
|
||||
|
||||
const indent = line.match(/^[ \t]*/)?.[0].length ?? 0;
|
||||
if (currentKey !== null && currentBlock !== null && indent > currentIndent) {
|
||||
currentBlock.push(line);
|
||||
continue;
|
||||
}
|
||||
|
||||
flush();
|
||||
|
||||
const match = line.match(/^([A-Za-z][\w-]*):\s*(.*)$/);
|
||||
if (!match) continue;
|
||||
const key = match[1];
|
||||
const raw = match[2].trim();
|
||||
|
||||
if (raw === "" || raw === ">" || raw === "|-" || raw === ">-" || raw === "|") {
|
||||
currentKey = key;
|
||||
currentBlock = [];
|
||||
currentIndent = indent;
|
||||
} else {
|
||||
const quoted =
|
||||
(raw.startsWith('"') && raw.endsWith('"')) ||
|
||||
(raw.startsWith("'") && raw.endsWith("'"));
|
||||
frontmatter[key] = quoted ? raw.slice(1, -1) : raw;
|
||||
}
|
||||
}
|
||||
|
||||
flush();
|
||||
|
||||
return { frontmatter, body };
|
||||
}
|
||||
|
||||
/**
|
||||
* Split a `chain:` frontmatter value into individual step names.
|
||||
* Mirrors the upstream convention of ` -> ` as the delimiter and trims
|
||||
* whitespace from each step.
|
||||
*/
|
||||
export function parseChain(value: string | undefined): string[] {
|
||||
if (!value) return [];
|
||||
return value
|
||||
.split(" -> ")
|
||||
.map((s) => s.trim())
|
||||
.filter((s) => s.length > 0);
|
||||
}
|
||||
Reference in New Issue
Block a user