defog-ai
diff --git a/Diff for: ‎.eslintrc.cjs
+21 b/Diff for: ‎.eslintrc.cjs
+21
diff --git a/Diff for: ‎.gitignore
+24 b/Diff for: ‎.gitignore
+24
diff --git a/Diff for: ‎README.md
+35 b/Diff for: ‎README.md
+35
diff --git a/Diff for: ‎index.html
+13 b/Diff for: ‎index.html
+13
@@ -0,0 +1,21 @@
+module.exports = {
+  root: true,
+  env: { browser: true, es2020: true },
+  extends: [
+    'eslint:recommended',
+    'plugin:react/recommended',
+    'plugin:react/jsx-runtime',
+    'plugin:react-hooks/recommended',
+  ],
+  ignorePatterns: ['dist', '.eslintrc.cjs'],
+  parserOptions: { ecmaVersion: 'latest', sourceType: 'module' },
+  settings: { react: { version: '18.2' } },
+  plugins: ['react-refresh'],
+  rules: {
+    'react/jsx-no-target-blank': 'off',
+    'react-refresh/only-export-components': [
+      'warn',
+      { allowConstantExport: true },
+    ],
+  },
+}
@@ -0,0 +1,24 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+lerna-debug.log*
+
+node_modules
+dist
+dist-ssr
+*.local
+
+# Editor directories and files
+.vscode/*
+!.vscode/extensions.json
+.idea
+.DS_Store
+*.suo
+*.ntvs*
+*.njsproj
+*.sln
+*.sw?
@@ -0,0 +1,35 @@
+# SQL-Eval Visualizer
+
+This repo helps visualize the results of sql-eval better, and checking where the model is performing suboptimally and what are the tokens where it is losing confidence.
+
+## Running the repo
+1. Make sure that this repo is contained within the `sql-eval` folder
+2. `cd` into the folder containing this repo, and install dependencies with `npm i`
+3. Run the repo with `npm run dev`
+4. Profit!
+
+## Adding data from a vLLM API run
+_Note: Currently, only the custom Defog implementation of the vLLM API server is supported. We hope to expand this to other runners in the future._
+
+Run sql-eval using the vLLM API runner, and with the `--logprobs` command line parameter enabled, like below.
+
+```bash
+python main.py \            
+  -db postgres \
+  -q "data/questions_gen_postgres.csv" "data/instruct_basic_postgres.csv" "data/instruct_advanced_postgres.csv" \
+  -o results/api_bs32_exp.csv results/api_basic_bs32_exp.csv results/api_advanced_bs32_exp.csv \
+  -g api \
+  -b 1 \
+  -f prompts/prompt.md \
+  --api_url "http://209.20.159.234:5000/generate" \
+  --api_type "vllm" \
+  -p 20 \
+  -c 0 \
+  --logprobs
+  ```
+
+## TODO
+
+- [ ] Let users run queries with a single click, and see results
+- [ ] Compare the results of 2 different runs, instead of just looking at results from a single run
+- [ ] Let users manually mark some squares as "almost correct", or "partially correct" in the UI, in order to differentiate responses that are _almost_ correct with those that are very off
@@ -0,0 +1,13 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <link rel="icon" type="image/svg+xml" href="/vite.svg" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Defog Eval Visualizer</title>
+  </head>
+  <body>
+    <div id="root"></div>
+    <script type="module" src="/src/main.jsx"></script>
+  </body>
+</html>