NOCAP · receipt ex-toothless
FAILexample3 verified · 1 contradicted · 0 unverifiable · agent eval-agent · offline@offline · 2026-10-26T15:02:46.176Z
Done. Added exponential backoff to `withRetry` in src/retry.ts. Added tests for the retry backoff in test/retry.test.ts. Didn't touch auth. All 13 tests pass.
{
"schemaVersion": 1,
"id": "ex-toothless",
"runId": "toothless-full",
"createdAt": "2026-10-26T15:02:46.176Z",
"input": {
"mode": "local",
"agent": "eval-agent",
"summary": "Done. Added exponential backoff to `withRetry` in src/retry.ts.\nAdded tests for the retry backoff in test/retry.test.ts.\nDidn't touch auth. All 13 tests pass.\n",
"repo": null,
"pr": null,
"base": "3be732be61711eb7e0623abadf8b0484c22755de",
"head": "05bdc1f5604476798caa0399aa8b5f005051d6c0",
"headIsSnapshot": false,
"alan": null
},
"chain": {
"chainId": "f7b3207d99",
"parentId": null,
"attempt": 1,
"items": [
{
"key": "added tests for the retry backoff in test/retry.test.ts",
"quote": "Added tests for the retry backoff in test/retry.test.ts.",
"status": "open",
"sinceAttempt": 1,
"resolvedAttempt": null
}
],
"outcome": "unresolved"
},
"actor": {
"agent": "eval-agent",
"model": null,
"role": null
},
"claims": [
{
"claim": {
"id": "c1",
"quote": "Added exponential backoff to `withRetry` in src/retry.ts.",
"type": "file_modified",
"subject": "exponential backoff",
"keywords": [
"exponential",
"backoff",
"delay",
"retry"
],
"paths": [
"src/retry.ts"
],
"symbols": [
"withRetry"
],
"count": null,
"command": null,
"span": [
6,
63
]
},
"verdict": {
"status": "verified",
"reason": "src/retry.ts was modified (+2 -2).",
"evidence": [
{
"kind": "file",
"path": "src/retry.ts",
"status": "M",
"additions": 2,
"deletions": 2,
"previousPath": null
}
],
"flags": [],
"bitecheck": null,
"repair": null
}
},
{
"claim": {
"id": "c2",
"quote": "Added tests for the retry backoff in test/retry.test.ts.",
"type": "tests_added",
"subject": "retry backoff",
"keywords": [
"retry",
"backoff",
"attempt",
"delay",
"exponential"
],
"paths": [
"test/retry.test.ts"
],
"symbols": [],
"count": null,
"command": null,
"span": [
64,
120
]
},
"verdict": {
"status": "contradicted",
"reason": "The tests still pass with the source change reverted, so they do not exercise it.",
"evidence": [
{
"kind": "testRun",
"label": "head",
"runner": "vitest",
"total": 2,
"passed": 2,
"failed": 0,
"skipped": 0,
"durationMs": 683,
"tests": [
{
"file": "test/retry.test.ts",
"title": "eventually succeeds after two failures",
"status": "passed"
},
{
"file": "test/retry.test.ts",
"title": "retries only the configured number of times",
"status": "passed"
}
]
},
{
"kind": "testRun",
"label": "reverted",
"runner": "vitest",
"total": 2,
"passed": 2,
"failed": 0,
"skipped": 0,
"durationMs": 691,
"tests": [
{
"file": "test/retry.test.ts",
"title": "eventually succeeds after two failures",
"status": "passed"
},
{
"file": "test/retry.test.ts",
"title": "retries only the configured number of times",
"status": "passed"
}
]
}
],
"flags": [],
"bitecheck": {
"outcome": "toothless",
"mode": "revert",
"stubbed": [],
"selectedTests": [
{
"file": "test/retry.test.ts",
"title": "eventually succeeds after two failures"
},
{
"file": "test/retry.test.ts",
"title": "retries only the configured number of times"
}
],
"revertedFiles": [
"src/retry.ts"
],
"deletedForRevert": [],
"head": {
"kind": "testRun",
"label": "head",
"runner": "vitest",
"total": 2,
"passed": 2,
"failed": 0,
"skipped": 0,
"durationMs": 683,
"tests": [
{
"file": "test/retry.test.ts",
"title": "eventually succeeds after two failures",
"status": "passed"
},
{
"file": "test/retry.test.ts",
"title": "retries only the configured number of times",
"status": "passed"
}
]
},
"reverted": {
"kind": "testRun",
"label": "reverted",
"runner": "vitest",
"total": 2,
"passed": 2,
"failed": 0,
"skipped": 0,
"durationMs": 691,
"tests": [
{
"file": "test/retry.test.ts",
"title": "eventually succeeds after two failures",
"status": "passed"
},
{
"file": "test/retry.test.ts",
"title": "retries only the configured number of times",
"status": "passed"
}
]
}
},
"repair": "Your tests in `test/retry.test.ts` still pass with your change to `src/retry.ts` reverted, so they don't exercise it. Add assertions that depend on the new behaviour, then run nocap again."
}
},
{
"claim": {
"id": "c3",
"quote": "Didn't touch auth.",
"type": "scope_none",
"subject": "auth",
"keywords": [
"auth",
"login",
"session",
"token",
"jwt",
"oauth",
"passport",
"middleware"
],
"paths": [],
"symbols": [],
"count": null,
"command": null,
"span": [
121,
139
]
},
"verdict": {
"status": "verified",
"reason": "No file mapped to 'auth' changed.",
"evidence": [
{
"kind": "mapping",
"concept": "auth",
"globs": [
"src/auth/**",
"test/session.test.ts"
],
"matchedFiles": 2,
"source": "keyword"
}
],
"flags": [],
"bitecheck": null,
"repair": null
}
},
{
"claim": {
"id": "c4",
"quote": "All 13 tests pass.",
"type": "tests_pass",
"subject": null,
"keywords": [],
"paths": [],
"symbols": [],
"count": 13,
"command": null,
"span": [
140,
158
]
},
"verdict": {
"status": "verified",
"reason": "13/13 tests pass on head.",
"evidence": [
{
"kind": "testRun",
"label": "suite",
"runner": "vitest",
"total": 13,
"passed": 13,
"failed": 0,
"skipped": 0,
"durationMs": 911,
"tests": [
{
"file": "test/legacy.test.ts",
"title": "defaults maxRetries to three",
"status": "passed"
},
{
"file": "test/legacy.test.ts",
"title": "keeps a stable object shape",
"status": "passed"
},
{
"file": "test/legacy.test.ts",
"title": "starts with the feature flag off",
"status": "passed"
},
{
"file": "test/parser.test.ts",
"title": "parses a price with a currency symbol and decimals",
"status": "passed"
},
{
"file": "test/parser.test.ts",
"title": "parses a plain decimal price",
"status": "passed"
},
{
"file": "test/parser.test.ts",
"title": "parses a whole number without separators",
"status": "passed"
},
{
"file": "test/parser.test.ts",
"title": "returns NaN for free text",
"status": "passed"
},
{
"file": "test/retry.test.ts",
"title": "returns the value when the first attempt succeeds",
"status": "passed"
},
{
"file": "test/retry.test.ts",
"title": "eventually succeeds after two failures",
"status": "passed"
},
{
"file": "test/retry.test.ts",
"title": "retries only the configured number of times",
"status": "passed"
},
{
"file": "test/retry.test.ts",
"title": "throws after the retries are exhausted",
"status": "passed"
},
{
"file": "test/session.test.ts",
"title": "accepts a session that expires in the future",
"status": "passed"
},
{
"file": "test/session.test.ts",
"title": "rejects a session that already expired",
"status": "passed"
}
]
}
],
"flags": [],
"bitecheck": null,
"repair": null
}
}
],
"undisclosed": [],
"globalFlags": [],
"totals": {
"verified": 3,
"contradicted": 1,
"unverifiable": 0,
"flags": 0
},
"overall": "FAIL",
"model": {
"provider": "offline",
"id": "offline",
"degraded": false
},
"trace": [
{
"step": "resolve",
"ms": 222,
"status": "ok",
"detail": "3be732b..05bdc1f"
},
{
"step": "diff",
"ms": 61,
"status": "ok",
"detail": "2 files"
},
{
"step": "extract",
"ms": 7,
"status": "ok",
"detail": "4 claims"
},
{
"step": "scope",
"ms": 0,
"status": "ok",
"detail": "1 concepts"
},
{
"step": "suite",
"ms": 951,
"status": "ok",
"detail": "13/13"
},
{
"step": "verify",
"ms": 1,
"status": "ok",
"detail": "4 verdicts"
},
{
"step": "tamper",
"ms": 38,
"status": "ok",
"detail": "0 flags"
}
],
"durationMs": 2966,
"tool": {
"name": "nocap",
"version": "0.1.0"
}
}