#ماژول ۳. ارزیابی (Evals) و تحلیل خطا

mkdir -p results/quick
blank=$(mktemp -d)        # an empty folder: no repo, no file names, no labels.json nearby
for f in fixtures/k8s/* fixtures/js/*; do
  codex exec --ephemeral --skip-git-repo-check -C "$blank" -m gpt-6-luna \
    "You are a security reviewer. List the security problems in the file in the <stdin> block, one short line each. Write NONE if there are none." \
    < "$f" > "results/quick/$(basename "$f").txt"
  echo "== $f"; cat "results/quick/$(basename "$f").txt"
done

 k   pass@k  pass^k
 1    0.94    0.94
 2    0.96    0.93
 3    0.97    0.93
 4    0.99    0.93
 5    1.00    0.93

sentinel/
  eval/
    review.schema.json   the shape Codex must answer in
    prompt.md            the reviewer instructions ({{KIND}} is filled in per file)
    run.ts               runs Codex over every fixture, N times
    score.ts             compares results with labels.json
    allowed.json         true-but-not-required labels (written after error analysis)
    judge.ts             binary judge for unlabelled findings
    agree.ts             judge versus your own labels
    human-labels.json    your verdicts on about 20 items
    promptfoo/           the same eval as a promptfoo config
  results/               add this to .gitignore

{
  "type": "object",
  "properties": {
    "findings": {
      "type": "array",
      "items": {
        "type": "object",
        "properties": {
          "label": {
            "type": "string",
            "enum": [
              "privileged-container", "hostpath-mount", "secret-in-plain-env",
              "mutable-image-tag", "runs-as-root", "rbac-wildcard",
              "host-network", "host-pid", "command-injection",
              "code-injection-eval", "ssrf", "jwt-not-verified"
            ]
          },
          "evidence": { "type": "string" }
        },
        "required": ["label", "evidence"],
        "additionalProperties": false
      }
    },
    "other_findings": {
      "type": "array",
      "items": {
        "type": "object",
        "properties": {
          "title": { "type": "string" },
          "evidence": { "type": "string" }
        },
        "required": ["title", "evidence"],
        "additionalProperties": false
      }
    }
  },
  "required": ["findings", "other_findings"],
  "additionalProperties": false
}

You are a security reviewer. The <stdin> block below is one {{KIND}}.
Review only that text. Do not run commands and do not read other files.

Report each problem from this list that is clearly present, using its exact id:

- privileged-container: a container sets securityContext.privileged: true
- hostpath-mount: a volume uses hostPath
- secret-in-plain-env: a secret value is written as a literal env value (not valueFrom)
- mutable-image-tag: an image has no tag or uses the tag "latest"
- runs-as-root: a pod or container explicitly sets runAsUser: 0
- rbac-wildcard: an RBAC rule uses "*" for apiGroups, resources or verbs
- host-network: the pod sets hostNetwork: true
- host-pid: the pod sets hostPID: true
- command-injection: user input reaches a shell command (exec, execSync, spawn with shell)
- code-injection-eval: user input reaches eval, new Function or vm
- ssrf: user input controls the URL of a server-side request
- jwt-not-verified: a JWT is decoded and trusted without signature verification

Put each one in "findings" with a short quote from the text as evidence.
Put any other real security problem in "other_findings" with a short title.
If nothing from the list is present, return an empty "findings" array. Do not guess.

// eval/run.ts — run Codex as the reviewer over every fixture, N times.
// Usage: node eval/run.ts --runs 5 [--model gpt-6-luna] [--out results/<dir>] [--only k8s/]
// Needs Node 22.18+ (runs .ts directly) and a signed-in `codex` on PATH.
import { spawn } from "node:child_process";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { parseArgs } from "node:util";

const { values: opt } = parseArgs({
  options: {
    runs: { type: "string", default: "5" },
    model: { type: "string", default: "gpt-6-luna" },
    out: { type: "string" },
    prompt: { type: "string", default: "eval/prompt.md" },
    only: { type: "string" },
  },
});

const ROOT = process.cwd(); // run from the sentinel repo root
const CODEX = process.env.CODEX_BIN ?? "codex";
const labels: Record<string, string[]> = JSON.parse(
  fs.readFileSync(path.join(ROOT, "fixtures/labels.json"), "utf8"),
);
const files = Object.keys(labels).filter((f) => !opt.only || f.includes(opt.only));
const stamp = new Date().toISOString().slice(0, 16).replace(/[:T]/g, "-");
const outDir = path.resolve(opt.out ?? `results/${stamp}-${opt.model}`);
const traceDir = path.join(outDir, "traces");
fs.mkdirSync(traceDir, { recursive: true });
const resultsPath = path.join(outDir, "results.jsonl");
const schemaPath = path.join(ROOT, "eval/review.schema.json");
const template = fs.readFileSync(path.resolve(opt.prompt), "utf8");

// Resume: skip (run, file) pairs that already have a good row. Failed rows are retried.
const done = new Set<string>();
if (fs.existsSync(resultsPath)) {
  for (const line of fs.readFileSync(resultsPath, "utf8").split("\n")) {
    if (!line.trim()) continue;
    const r = JSON.parse(line);
    if (r.ok) done.add(`${r.run}|${r.file}`);
  }
}

// The reviewer works in an empty folder and cannot read the repo at all,
// so it never sees file names like "k05-clean-deployment" or labels.json.
const blank = fs.mkdtempSync(path.join(os.tmpdir(), "sentinel-review-"));
const permissionArgs = [
  "-c", 'default_permissions="reviewer"',
  "-c", `permissions.reviewer.filesystem={":root"="read","${ROOT}"="deny"}`,
];

type Finding = { label: string; evidence: string };
type Row = {
  run: number; file: string; model: string; ok: boolean; ms: number;
  findings: Finding[]; other_findings: { title: string; evidence: string }[];
  usage: unknown; error?: string; contaminated?: boolean;
};

function reviewOnce(run: number, file: string): Promise<Row> {
  const kind = file.startsWith("k8s/") ? "Kubernetes manifest" : "JavaScript (Node.js) module";
  const tag = `run${run}-${file.replace(/[/.]/g, "_")}`;
  const lastPath = path.join(traceDir, `${tag}.last.json`);
  const tracePath = path.join(traceDir, `${tag}.jsonl`);
  const args = [
    "exec", "--json", "--ephemeral", "--skip-git-repo-check",
    "-C", blank, "-m", opt.model!, ...permissionArgs,
    "--output-schema", schemaPath, "-o", lastPath,
    template.replaceAll("{{KIND}}", kind),
  ];
  const started = Date.now();
  return new Promise((resolve) => {
    const child = spawn(CODEX, args, { stdio: ["pipe", "pipe", "pipe"] });
    const trace = fs.createWriteStream(tracePath);
    let usage: unknown = null;
    let pending = "";
    let stderr = "";
    child.stdout.on("data", (chunk: Buffer) => {
      trace.write(chunk);
      pending += chunk.toString();
      let nl: number;
      while ((nl = pending.indexOf("\n")) >= 0) {
        const line = pending.slice(0, nl);
        pending = pending.slice(nl + 1);
        try {
          const ev = JSON.parse(line);
          if (ev.type === "turn.completed") usage = ev.usage;
        } catch { /* not JSON: ignore */ }
      }
    });
    child.stderr.on("data", (chunk: Buffer) => (stderr += chunk.toString()));
    child.on("error", (err) => (stderr += String(err)));
    // The fixture goes in on stdin and becomes a <stdin> block after the prompt.
    child.stdin.end(fs.readFileSync(path.join(ROOT, "fixtures", file)));
    child.on("close", (code) => {
      trace.end();
      const base = { run, file, model: opt.model!, ms: Date.now() - started, usage };
      try {
        if (code !== 0) throw new Error(`codex exited ${code}: ${stderr.trim().split("\n").slice(-2).join(" | ")}`);
        const out = JSON.parse(fs.readFileSync(lastPath, "utf8"));
        const traceText = fs.readFileSync(tracePath, "utf8");
        resolve({
          ...base, ok: true, findings: out.findings ?? [], other_findings: out.other_findings ?? [],
          contaminated: traceText.includes("labels.json") || traceText.includes(ROOT),
        });
      } catch (err) {
        resolve({ ...base, ok: false, error: (err as Error).message, findings: [], other_findings: [] });
      }
    });
  });
}

let failuresInARow = 0;
for (let run = 1; run <= Number(opt.runs); run++) {
  for (const file of files) {
    if (done.has(`${run}|${file}`)) continue;
    const row = await reviewOnce(run, file);
    fs.appendFileSync(resultsPath, JSON.stringify(row) + "\n");
    const what = row.ok ? row.findings.map((f) => f.label).join(", ") || "(none)" : row.error;
    console.log(`${row.ok ? "ok " : "ERR"} run ${run} ${file}: ${what}${row.contaminated ? "  [CONTAMINATED]" : ""}`);
    failuresInARow = row.ok ? 0 : failuresInARow + 1;
    if (failuresInARow >= 3) {
      console.error(`3 failures in a row, stopping. Probably a usage limit. Re-run with --out ${outDir} later to resume.`);
      process.exit(1);
    }
  }
}
console.log(`results: ${resultsPath}`);

node eval/run.ts --runs 5            # about 70 calls; prints one line per trial

// eval/score.ts — score reviewer results against fixtures/labels.json.
// Usage: node eval/score.ts results/<dir>/results.jsonl [--runs 5] [--allowed eval/allowed.json] [--json summary.json]
import fs from "node:fs";
import { parseArgs } from "node:util";

const { values: opt, positionals } = parseArgs({
  allowPositionals: true,
  options: {
    labels: { type: "string", default: "fixtures/labels.json" },
    allowed: { type: "string" }, // file -> labels that are true but not required: neither TP nor FP
    runs: { type: "string" },    // expected runs; missing rows count as failed trials
    json: { type: "string" },    // write a machine-readable summary here
  },
});
if (positionals.length === 0) {
  console.error("usage: node eval/score.ts <results.jsonl> [more.jsonl] [--runs N] [--allowed file] [--json out]");
  process.exit(2);
}

type Row = { run: number; file: string; ok: boolean; findings?: { label: string }[]; other_findings?: unknown[] };
const labels: Record<string, string[]> = JSON.parse(fs.readFileSync(opt.labels!, "utf8"));
const allowed: Record<string, string[]> = opt.allowed ? JSON.parse(fs.readFileSync(opt.allowed, "utf8")) : {};

// Last row wins for each (run, file): a retried run replaces its failed attempt.
const rows = new Map<string, Row>();
for (const p of positionals) {
  for (const line of fs.readFileSync(p, "utf8").split("\n")) {
    if (!line.trim()) continue;
    const r: Row = JSON.parse(line);
    if (!(r.file in labels)) continue;
    rows.set(`${r.run}|${r.file}`, r);
  }
}
const seenRuns = [...rows.values()].map((r) => r.run);
const nRuns = opt.runs ? Number(opt.runs) : Math.max(0, ...seenRuns);
const files = Object.keys(labels);
const labelIds = [...new Set(Object.values(labels).flat())].sort();

const counts: Record<string, { tp: number; fp: number; fn: number }> = {};
const bump = (id: string, k: "tp" | "fp" | "fn") => ((counts[id] ??= { tp: 0, fp: 0, fn: 0 })[k] += 1);
for (const id of labelIds) counts[id] = { tp: 0, fp: 0, fn: 0 };

let failed = 0, missing = 0, cleanTrials = 0, cleanFlagged = 0, otherFindings = 0;
const passes: Record<string, number> = Object.fromEntries(files.map((f) => [f, 0]));

for (let run = 1; run <= nRuns; run++) {
  for (const file of files) {
    const row = rows.get(`${run}|${file}`);
    if (!row) missing++;
    else if (!row.ok) failed++;
    // A missing or failed trial predicts nothing: every expected label becomes a miss.
    const predicted = new Set(row?.ok ? (row.findings ?? []).map((f) => f.label) : []);
    const expected = new Set(labels[file]);
    const extra = new Set(allowed[file] ?? []);
    otherFindings += row?.ok ? (row.other_findings ?? []).length : 0;

    let fileFp = 0, fileFn = 0;
    for (const id of expected) {
      if (predicted.has(id)) bump(id, "tp");
      else { bump(id, "fn"); fileFn++; }
    }
    for (const id of predicted) {
      if (!expected.has(id) && !extra.has(id)) { bump(id, "fp"); fileFp++; }
    }
    // A trial passes when it finds every expected label and nothing outside expected + allowed.
    const ok = row?.ok === true && fileFn === 0 && fileFp === 0;
    if (ok) passes[file]++;
    if (expected.size === 0) {
      cleanTrials++;
      if (fileFp > 0) cleanFlagged++;
    }
  }
}

const ratio = (a: number, b: number) => (b === 0 ? null : a / b);
const f1 = (p: number | null, r: number | null) => (p === null || r === null || p + r === 0 ? null : (2 * p * r) / (p + r));
const fmt = (x: number | null) => (x === null ? "  -  " : x.toFixed(2).padStart(5));

// Unbiased estimators from n trials with c passes (Chen et al. 2021 for pass@k).
function choose(n: number, k: number): number {
  if (k < 0 || k > n) return 0;
  let r = 1;
  for (let i = 1; i <= k; i++) r = (r * (n - k + i)) / i;
  return r;
}
const passAtK = (n: number, c: number, k: number) => 1 - choose(n - c, k) / choose(n, k); // at least one of k passes
const passHatK = (n: number, c: number, k: number) => choose(c, k) / choose(n, k);         // all k pass

const perLabel = Object.entries(counts).sort(([a], [b]) => a.localeCompare(b)).map(([id, c]) => {
  const precision = ratio(c.tp, c.tp + c.fp);
  const recall = ratio(c.tp, c.tp + c.fn);
  return { id, ...c, precision, recall, f1: f1(precision, recall) };
});
const tot = perLabel.reduce((a, l) => ({ tp: a.tp + l.tp, fp: a.fp + l.fp, fn: a.fn + l.fn }), { tp: 0, fp: 0, fn: 0 });
const microP = ratio(tot.tp, tot.tp + tot.fp);
const microR = ratio(tot.tp, tot.tp + tot.fn);
const scored = perLabel.filter((l) => l.f1 !== null);
const macroF1 = scored.length ? scored.reduce((a, l) => a + (l.f1 as number), 0) / scored.length : null;
const ks = Array.from({ length: nRuns }, (_, i) => i + 1);
const passK = ks.map((k) => ({
  k,
  passAtK: files.reduce((a, f) => a + passAtK(nRuns, passes[f], k), 0) / files.length,
  passHatK: files.reduce((a, f) => a + passHatK(nRuns, passes[f], k), 0) / files.length,
}));

console.log(`runs ${nRuns} · fixtures ${files.length} · trials ${nRuns * files.length} · failed ${failed} · missing ${missing}\n`);
console.log("label                   TP  FP  FN  prec   rec    F1");
for (const l of perLabel) {
  console.log(`${l.id.padEnd(22)} ${String(l.tp).padStart(3)} ${String(l.fp).padStart(3)} ${String(l.fn).padStart(3)}  ${fmt(l.precision)} ${fmt(l.recall)} ${fmt(l.f1)}`);
}
console.log(`${"micro".padEnd(22)} ${String(tot.tp).padStart(3)} ${String(tot.fp).padStart(3)} ${String(tot.fn).padStart(3)}  ${fmt(microP)} ${fmt(microR)} ${fmt(f1(microP, microR))}`);
console.log(`macro F1 ${fmt(macroF1)}`);
console.log(`\nclean files flagged: ${cleanFlagged} of ${cleanTrials} trials · other_findings (unlabelled): ${otherFindings}`);
console.log("\nper fixture (passes / runs):");
for (const f of files) console.log(`  ${String(passes[f]).padStart(2)}/${nRuns}  ${f}`);
console.log("\n k   pass@k  pass^k");
for (const p of passK) console.log(`${String(p.k).padStart(2)}   ${fmt(p.passAtK)}   ${fmt(p.passHatK)}`);

if (opt.json) {
  const summary = {
    runs: nRuns, failed, missing, cleanFlagged, cleanTrials, otherFindings,
    micro: { ...tot, precision: microP, recall: microR, f1: f1(microP, microR) },
    macroF1, perLabel, passes, passK,
  };
  fs.writeFileSync(opt.json, JSON.stringify(summary, null, 2));
}

node eval/score.ts results/<your-dir>/results.jsonl --runs 5 --json results/<your-dir>/summary.json

{
  "k8s/k01-privileged.yaml": ["runs-as-root"],
  "k8s/k02-hostpath.yaml": ["runs-as-root"],
  "k8s/k03-secret-literal.yaml": ["runs-as-root"],
  "k8s/k07-hostnetwork.yaml": ["runs-as-root"]
}

node eval/judge.ts results/<your-dir>/results.jsonl --items-only   # writes items.jsonl

// eval/judge.ts — a binary LLM judge for one failure mode: "is this unlabelled finding real?"
// Usage: node eval/judge.ts results/<dir>/results.jsonl --items-only   (list the items, label them yourself first)
//        node eval/judge.ts results/<dir>/results.jsonl [--model gpt-6.1-sol]
// Writes results/<dir>/items.jsonl and judge.jsonl. Re-running skips items already judged.
import { spawnSync } from "node:child_process";
import crypto from "node:crypto";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { parseArgs } from "node:util";

const { values: opt, positionals } = parseArgs({
  allowPositionals: true,
  options: {
    model: { type: "string", default: "gpt-6.1-sol" },
    "items-only": { type: "boolean", default: false },
  },
});
const ROOT = process.cwd();
const CODEX = process.env.CODEX_BIN ?? "codex";
const resultsPath = path.resolve(positionals[0]);
const outPath = path.join(path.dirname(resultsPath), "judge.jsonl");
const labels: Record<string, string[]> = JSON.parse(fs.readFileSync("fixtures/labels.json", "utf8"));

// 1. Collect the items: every finding the labels do not account for, deduplicated per file.
type Item = { id: string; file: string; claim: string; evidence: string };
const items = new Map<string, Item>();
for (const line of fs.readFileSync(resultsPath, "utf8").split("\n")) {
  if (!line.trim()) continue;
  const r = JSON.parse(line);
  if (!r.ok) continue;
  const extra = [
    ...r.findings.filter((f: any) => !labels[r.file].includes(f.label)).map((f: any) => ({ claim: f.label, evidence: f.evidence })),
    ...r.other_findings.map((f: any) => ({ claim: f.title, evidence: f.evidence })),
  ];
  for (const e of extra) {
    const key = `${r.file}|${e.claim.toLowerCase().trim()}`;
    const id = crypto.createHash("sha256").update(key).digest("hex").slice(0, 8);
    if (!items.has(id)) items.set(id, { id, file: r.file, ...e });
  }
}

fs.writeFileSync(path.join(path.dirname(resultsPath), "items.jsonl"), [...items.values()].map((i) => JSON.stringify(i)).join("\n") + "\n");
console.log(`${items.size} unlabelled findings -> items.jsonl`);
if (opt["items-only"]) process.exit(0);

// 2. Judge each one. The judge sees only the file and the claim: not the labels, not the reviewer's other output.
const judged = new Set(
  fs.existsSync(outPath) ? fs.readFileSync(outPath, "utf8").split("\n").filter(Boolean).map((l) => JSON.parse(l).id) : [],
);
const schemaPath = path.join(os.tmpdir(), "sentinel-judge.schema.json");
fs.writeFileSync(schemaPath, JSON.stringify({
  type: "object",
  properties: {
    reasoning: { type: "string" },
    verdict: { type: "string", enum: ["valid", "invalid"] },
  },
  required: ["reasoning", "verdict"],
  additionalProperties: false,
}));
const blank = fs.mkdtempSync(path.join(os.tmpdir(), "sentinel-judge-"));

for (const item of items.values()) {
  if (judged.has(item.id)) continue;
  const prompt = [
    "You check one claim made by a security reviewer about the file in the <stdin> block.",
    `Claim: ${item.claim}`,
    `Reviewer's evidence: ${item.evidence}`,
    "",
    'Answer "valid" only if the claim describes a real, exploitable or policy-relevant security problem',
    "that is visible in this file as written. Style issues, missing best practices with no security impact,",
    'and problems that depend on code not shown are "invalid". Explain your reasoning first, in two sentences.',
  ].join("\n");
  const lastPath = path.join(blank, `${item.id}.json`);
  const r = spawnSync(CODEX, [
    "exec", "--ephemeral", "--skip-git-repo-check", "-C", blank, "-m", opt.model!,
    "-c", 'default_permissions="judge"',
    "-c", `permissions.judge.filesystem={":root"="read","${ROOT}"="deny"}`,
    "--output-schema", schemaPath, "-o", lastPath, prompt,
  ], { input: fs.readFileSync(path.join(ROOT, "fixtures", item.file)), encoding: "utf8" });
  if (r.status !== 0) {
    console.error(`judge failed on ${item.id}: ${r.stderr.trim().split("\n").pop()}`);
    process.exit(1);
  }
  const out = JSON.parse(fs.readFileSync(lastPath, "utf8"));
  fs.appendFileSync(outPath, JSON.stringify({ ...item, verdict: out.verdict, reasoning: out.reasoning }) + "\n");
  console.log(`${out.verdict.padEnd(7)} ${item.id} ${item.file}: ${item.claim}`);
}
console.log(`judge: ${outPath}`);

// eval/agree.ts — how well does the judge agree with you?
// Usage: node eval/agree.ts results/<dir>/judge.jsonl eval/human-labels.json
// human-labels.json: { "<item id>": "valid" | "invalid", ... } — about 20 items you labelled yourself, before reading the judge.
import fs from "node:fs";

const [judgePath, humanPath] = process.argv.slice(2);
if (!judgePath || !humanPath) {
  console.error("usage: node eval/agree.ts <judge.jsonl> <human-labels.json>");
  process.exit(2);
}
const human: Record<string, "valid" | "invalid"> = JSON.parse(fs.readFileSync(humanPath, "utf8"));
const judge = new Map<string, { verdict: string; file: string; claim: string }>();
for (const line of fs.readFileSync(judgePath, "utf8").split("\n")) {
  if (line.trim()) { const r = JSON.parse(line); judge.set(r.id, r); }
}

let tp = 0, fn = 0, tn = 0, fp = 0;
const disagreements: string[] = [];
for (const [id, mine] of Object.entries(human)) {
  const j = judge.get(id);
  if (!j) { console.warn(`no judge verdict for ${id}`); continue; }
  if (mine === "valid" && j.verdict === "valid") tp++;
  else if (mine === "valid") { fn++; disagreements.push(`  you: valid,   judge: invalid  ${id} ${j.file}: ${j.claim}`); }
  else if (j.verdict === "invalid") tn++;
  else { fp++; disagreements.push(`  you: invalid, judge: valid    ${id} ${j.file}: ${j.claim}`); }
}
const n = tp + fn + tn + fp;
const pct = (a: number, b: number) => (b === 0 ? "  n/a" : `${((100 * a) / b).toFixed(0).padStart(4)}%`);
console.log(`items compared: ${n}  (you said valid: ${tp + fn}, invalid: ${tn + fp})`);
console.log(`TPR (judge says valid when you do):     ${pct(tp, tp + fn)}`);
console.log(`TNR (judge says invalid when you do):   ${pct(tn, tn + fp)}`);
console.log(`raw agreement:                          ${pct(tp + tn, n)}`);
if (disagreements.length) console.log(`\nread these:\n${disagreements.join("\n")}`);

node eval/judge.ts results/<your-dir>/results.jsonl
node eval/agree.ts results/<your-dir>/judge.jsonl eval/human-labels.json

npm install --save-dev promptfoo@0.125.1 @openai/codex-sdk@0.166.0
mkdir -p eval/promptfoo/blank && echo '{ "type": "commonjs" }' > eval/promptfoo/package.json

// eval/promptfoo/prompt.js — builds the same prompt as eval/run.ts, with the fixture inlined.
const fs = require("node:fs");
const path = require("node:path");
const REPO = path.resolve(__dirname, "../..");

module.exports = async function ({ vars }) {
  const kind = vars.file.startsWith("k8s/") ? "Kubernetes manifest" : "JavaScript (Node.js) module";
  const template = fs.readFileSync(path.join(REPO, "eval/prompt.md"), "utf8").replaceAll("{{KIND}}", kind);
  const body = fs.readFileSync(path.join(REPO, "fixtures", vars.file), "utf8");
  return `${template}\n<stdin>\n${body}</stdin>\n`;
};

// eval/promptfoo/assert.js — same pass rule as eval/score.ts: every expected label, nothing else.
const labels = require("../../fixtures/labels.json");

module.exports = (output, context) => {
  let parsed;
  try {
    parsed = typeof output === "string" ? JSON.parse(output) : output;
  } catch {
    return { pass: false, score: 0, reason: "output is not JSON" };
  }
  const expected = new Set(labels[context.vars.file]);
  const predicted = new Set((parsed.findings || []).map((f) => f.label));
  const missed = [...expected].filter((l) => !predicted.has(l));
  const extra = [...predicted].filter((l) => !expected.has(l));
  const tp = expected.size - missed.length;
  const score = expected.size === 0 && predicted.size === 0 ? 1 : (2 * tp) / (expected.size + predicted.size);
  return {
    pass: missed.length === 0 && extra.length === 0,
    score,
    reason: `missed: ${missed.join(", ") || "none"} · extra: ${extra.join(", ") || "none"}`,
  };
};

# eval/promptfoo/promptfooconfig.yaml — promptfoo 0.125.1, Codex on your ChatGPT sign-in
description: sentinel reviewer on the 14 fixtures

prompts:
  - file://prompt.js

providers:
  - id: openai:codex-sdk
    config:
      model: gpt-6-luna
      sandbox_mode: read-only          # the provider's default is workspace-write
      working_dir: ./blank             # an empty folder: no repo, no labels.json
      skip_git_repo_check: true
      output_schema:
        type: object
        properties:
          findings:
            type: array
            items:
              type: object
              properties:
                label:
                  type: string
                  enum: [privileged-container, hostpath-mount, secret-in-plain-env, mutable-image-tag,
                         runs-as-root, rbac-wildcard, host-network, host-pid, command-injection,
                         code-injection-eval, ssrf, jwt-not-verified]
                evidence: { type: string }
              required: [label, evidence]
              additionalProperties: false
          other_findings:
            type: array
            items:
              type: object
              properties:
                title: { type: string }
                evidence: { type: string }
              required: [title, evidence]
              additionalProperties: false
        required: [findings, other_findings]
        additionalProperties: false

defaultTest:
  assert:
    - type: is-json
    - type: javascript
      value: file://assert.js

tests:
  - vars: { file: k8s/k01-privileged.yaml }
  - vars: { file: k8s/k02-hostpath.yaml }
  - vars: { file: k8s/k03-secret-literal.yaml }
  - vars: { file: k8s/k04-latest-root.yaml }
  - vars: { file: k8s/k05-clean-deployment.yaml }
  - vars: { file: k8s/k06-rbac-wildcard.yaml }
  - vars: { file: k8s/k07-hostnetwork.yaml }
  - vars: { file: k8s/k08-clean-netpol.yaml }
  - vars: { file: js/j01-cmd-injection.js }
  - vars: { file: js/j02-eval.js }
  - vars: { file: js/j03-ssrf.js }
  - vars: { file: js/j04-clean-execfile.js }
  - vars: { file: js/j05-jwt-decode.js }
  - vars: { file: js/j06-clean-path.js }

evaluateOptions:
  maxConcurrency: 1

cd eval/promptfoo
env -u OPENAI_API_KEY -u CODEX_API_KEY npx promptfoo validate -c promptfooconfig.yaml
env -u OPENAI_API_KEY -u CODEX_API_KEY npx promptfoo eval -c promptfooconfig.yaml --repeat 5 --no-cache -j 1 -o ../../results/promptfoo.json
npx promptfoo view