chore: updated how we store accuracy result

Instead of storing multiple documents per accuracy test run(one for each
prompt+model response), we will now be storing one document for accuracy
result and under that, all the prompt+model responses will be nested.

Author: himanshusinghs

package.json

@@ -31,7 +31,7 @@
     "generate": "./scripts/generate.sh",
     "test": "node --experimental-vm-modules node_modules/jest/bin/jest.js --coverage --testPathIgnorePatterns=/tests/accuracy/",
     "pre:test:accuracy": "npm run build:compile",
-    "test:accuracy": "sh ./scripts/run-accuracy-tests.sh"
+    "test:accuracy": "sh ./scripts/accuracy/run-accuracy-tests.sh"
   },
   "license": "Apache-2.0",
   "devDependencies": {

resources/test-summary-template.html

@@ -167,28 +167,28 @@
                 font-family: "Monaco", "Menlo", monospace;
                 font-size: 12px;
                 max-height: 400px;
+                max-width: 1300px;
                 overflow-y: auto;
             }
-            .accuracy-perfect {
-                background-color: #d4edda;
-                color: #155724;
+            .run-status {
+                text-transform: capitalize;
+            }
+            .chip {
                 padding: 2px 6px;
                 border-radius: 3px;
                 font-weight: bold;
             }
-            .accuracy-good {
+            .perfect {
+                background-color: #d4edda;
+                color: #155724;
+            }
+            .good {
                 background-color: #fff3cd;
                 color: #856404;
-                padding: 2px 6px;
-                border-radius: 3px;
-                font-weight: bold;
             }
-            .accuracy-poor {
+            .poor {
                 background-color: #f8d7da;
                 color: #721c24;
-                padding: 2px 6px;
-                border-radius: 3px;
-                font-weight: bold;
             }
             .tool-call {
                 background: #e9ecef;
@@ -303,13 +303,17 @@ <h1>📊 MongoDB MCP Server - Accuracy Test Summary</h1>
             <div class="header-info">
                 <h2>📊 Current Run Information</h2>
                 <div class="info-grid">
+                    <div class="info-item">
+                        <div class="info-label">Commit SHA</div>
+                        <div class="info-value">{{commitSHA}}</div>
+                    </div>
                     <div class="info-item">
                         <div class="info-label">Accuracy Run ID</div>
                         <div class="info-value">{{accuracyRunId}}</div>
                     </div>
                     <div class="info-item">
-                        <div class="info-label">Commit SHA</div>
-                        <div class="info-value">{{commitSHA}}</div>
+                        <div class="info-label">Accuracy Run Status</div>
+                        <div class="info-value">{{accuracyRunStatus}}</div>
                     </div>
                     <div class="info-item">
                         <div class="info-label">Run Created On</div>
@@ -347,13 +351,17 @@ <h2>📈 Test Results Summary</h2>
             <div class="header-info">
                 <h2>🔄 Baseline Comparison</h2>
                 <div class="info-grid">
+                    <div class="info-item">
+                        <div class="info-label">Baseline Commit SHA</div>
+                        <div class="info-value">{{baselineCommitSHA}}</div>
+                    </div>
                     <div class="info-item">
                         <div class="info-label">Baseline Accuracy Run ID</div>
                         <div class="info-value">{{baselineAccuracyRunId}}</div>
                     </div>
                     <div class="info-item">
-                        <div class="info-label">Baseline Commit SHA</div>
-                        <div class="info-value">{{baselineCommitSHA}}</div>
+                        <div class="info-label">Baseline Accuracy Run Status</div>
+                        <div class="info-value">{{baselineAccuracyRunStatus}}</div>
                     </div>
                     <div class="info-item">
                         <div class="info-label">Baseline Run Created On</div>

scripts/run-accuracy-tests.sh renamed to scripts/accuracy/run-accuracy-tests.sh

@@ -8,39 +8,38 @@ export MDB_ACCURACY_RUN_ID=$(npx uuid v4)
 # export MDB_AZURE_OPEN_AI_API_KEY=""
 # export MDB_AZURE_OPEN_AI_API_URL=""
 
-# For providing a mongodb based storage to store accuracy snapshots
+# For providing a mongodb based storage to store accuracy result
 # export MDB_ACCURACY_MDB_URL=""
 
 # By default we run all the tests under tests/accuracy folder unless a path is
 # specified in the command line. Such as:
 # npm run test:accuracy -- tests/accuracy/some-test.test.ts
 TEST_PATH_PATTERN="${1:-tests/accuracy}"
 shift || true
+echo "Running accuracy tests with MDB_ACCURACY_RUN_ID '$MDB_ACCURACY_RUN_ID' and TEST_PATH_PATTERN '$TEST_PATH_PATTERN'"
 node --experimental-vm-modules node_modules/jest/bin/jest.js --bail --testPathPatterns "$TEST_PATH_PATTERN" "$@"
 
 # Preserving the exit code from test run to correctly notify in the CI
 # environments when the tests fail.
 JEST_EXIT_CODE=$?
 
-# Each test run submits an accuracy snapshot entry with the accuracyRunStatus:
+# Each test run submits an accuracy result with the accuracyRunStatus:
 # "in-progress". When all the tests are done and jest exits with an exit code of
 # 0, we can safely mark accuracy run as finished otherwise failed.
 
 # This "outside-the-tests-status-update" is arising out of the fact that each
 # test suite stores their own accuracy run data in the storage and this setup
 # might lead to data inconsistency when the tests fail. To overcome that each
-# accuracy snapshot entry has a status which by default is "in-progress" and is
+# accuracy result entry has a status which by default is "in-progress" and is
 # updated when the tests either pass (all our accuracy tests are supposed to
 # pass unless some errors occurs during the test runs), or fail.
 
 # This is necessary when comparing one accuracy run with another as we wouldn't
 # want to compare against an incomplete run.
-if [ $JEST_EXIT_CODE -eq 0 ]; then
-  MDB_ACCURACY_RUN_STATUS="done" npx tsx scripts/update-accuracy-run-status.ts || echo "Warning: Failed to update accuracy run status to 'done'"
-  npx tsx scripts/generate-test-summary.ts || echo "Warning: Failed to generate test summary HTML report"
-else
-  MDB_ACCURACY_RUN_STATUS="failed" npx tsx scripts/update-accuracy-run-status.ts || echo "Warning: Failed to update accuracy run status to 'failed'"
-fi
+export MDB_ACCURACY_RUN_STATUS=$([ $JEST_EXIT_CODE -eq 0 ] && echo "done" || echo "failed")
+npx tsx scripts/accuracy/update-accuracy-run-status.ts || echo "Warning: Failed to update accuracy run status to '$MDB_ACCURACY_RUN_STATUS'"
 
+# This is optional but we do it anyways to generate a readable summary of report.
+npx tsx scripts/generate-test-summary.ts || echo "Warning: Failed to generate test summary HTML report"
 
 exit $JEST_EXIT_CODE

scripts/update-accuracy-run-status.ts renamed to scripts/accuracy/update-accuracy-run-status.ts

@@ -1,18 +1,21 @@
-import { getAccuracySnapshotStorage } from "../tests/accuracy/sdk/accuracy-snapshot-storage/get-snapshot-storage.js";
-import { AccuracyRunStatus } from "../tests/accuracy/sdk/accuracy-snapshot-storage/snapshot-storage.js";
+import { getAccuracyResultStorage } from "../../tests/accuracy/sdk/accuracy-result-storage/get-accuracy-result-storage.js";
+import { AccuracyRunStatus } from "../../tests/accuracy/sdk/accuracy-result-storage/result-storage.js";
+import { getCommitSHA } from "../../tests/accuracy/sdk/git-info.js";
 
 const envAccuracyRunId = process.env.MDB_ACCURACY_RUN_ID;
 const envAccuracyRunStatus = process.env.MDB_ACCURACY_RUN_STATUS;
+const commitSHA = await getCommitSHA();
 
 if (
     !envAccuracyRunId ||
+    !commitSHA ||
     (envAccuracyRunStatus !== AccuracyRunStatus.Done && envAccuracyRunStatus !== AccuracyRunStatus.Failed)
 ) {
     process.exit(1);
 }
 
 console.time(`Marked accuracy run id - ${envAccuracyRunId} as ${envAccuracyRunStatus} in`);
-const storage = await getAccuracySnapshotStorage();
-await storage.updateAccuracyRunStatus(envAccuracyRunId, envAccuracyRunStatus);
+const storage = getAccuracyResultStorage();
+await storage.updateRunStatus(commitSHA, envAccuracyRunId, envAccuracyRunStatus);
 await storage.close();
 console.timeEnd(`Marked accuracy run id - ${envAccuracyRunId} as ${envAccuracyRunStatus} in`);