{
"customId": "code-review-run-009",
"strategy": "RPM",
"model": "gpt-4o",
"datasetColumns": ["diff", "toolFindings"],
"budget": 40,
"minibatchSize": 3,
"paretoSize": 4,
"evaluator": "{\"model\": \"gpt-4o-mini\", \"metric\": \"exact_match\", \"threshold\": 0.8}",
"dataset": [
{
"input": {
"diff": "@@ -24,6 +24,11 @@\n+ if (!user) {\n+ throw new Error('missing user');\n+ }\n+ if (!user.email) {\n+ return;\n+ }\n+ sendEmail(user.email);",
"toolFindings": "WARN: sendEmail runs without rate limiting."
},
"expectedOutput": "1. Highlight missing rate limiting. 2. Suggest retry/backoff strategy."
},
{
"input": {
"diff": "@@ -88,7 +88,12 @@\n- const auth = req.headers['Authorization'];\n+ const auth = req.headers['authorization'];\n+ if (!auth) {\n+ res.status(401).send('missing token');\n+ return;\n+ }",
"toolFindings": "WARN: continues execution after sending 401."
},
"expectedOutput": "Flag missing return, advise using early exit after response."
},
{
"input": {
"diff": "@@ -55,8 +55,15 @@\n+app.use(cors({origin: '*'}));",
"toolFindings": "WARN: CORS policy allows all origins."
},
"expectedOutput": "Explain CORS risk with wildcard origin, recommend allowlist."
},
{
"input": {
"diff": "@@ -12,6 +12,16 @@\n const query = `SELECT * FROM orders WHERE id = ${orderId}`;\n return db.execute(query);",
"toolFindings": "CRIT: SQL injection risk due to string interpolation."
},
"expectedOutput": "Explain SQL injection vector and recommend parameterized query."
},
{
"input": {
"diff": "@@ -45,9 +45,14 @@\n+ const token = jwt.sign({userId}, SECRET, {expiresIn: '365d'});",
"toolFindings": "WARN: token expiration set to 1 year."
},
"expectedOutput": "Recommend shorter token lifetime (e.g., 15 minutes) with refresh mechanism."
},
{
"input": {
"diff": "@@ -78,6 +78,10 @@\n+ const filePath = path.join(UPLOAD_DIR, userInput);\n+ return fs.readFileSync(filePath);",
"toolFindings": "CRIT: Path traversal vulnerability with unsanitized input."
},
"expectedOutput": "Identify path traversal risk, recommend path sanitization and validation."
},
{
"input": {
"diff": "@@ -92,7 +92,11 @@\n+ const result = eval(userFormula);",
"toolFindings": "CRIT: eval() with user input enables code injection."
},
"expectedOutput": "Flag eval() as critical security issue, recommend safe expression parser."
}
],
"prompt": "You are a senior engineer reviewing pull requests. Summarize the key risks, reference static analysis findings explicitly, and propose concrete fixes."
}