AI-powered marking (#1248)

* feat: v1 of AI-generated comments

* feat: added logging of inputs and outputs

* Update generate_ai_comments.ex

* feat: function to save outputs to database

* Format answers json before sending to LLM

* Add LLM Prompt to question params when submitting assessment xml file

* Add LLM Prompt to api response when grading view is open

* feat: added llm_prompt from qn to raw_prompt

* feat: enabling/disabling of LLM feature by course level

* feat: added llm_grading boolean field to course creation API

* feat: added api key storage in courses & edit api key/enable llm grading

* feat: encryption for llm_api_key

* feat: added final comment editing route

* feat: added logging of chosen comments

* fix: bugs when certain fields were missing

* feat: updated tests

* formatting

* fix: error handling when calling openai API

* fix: credo issues

* formatting

* Address some comments

* Fix formatting

* rm IO.inspect

* a

* Use case instead of if

* Streamlines generate_ai_comments to only send the selected question and its relevant info + use the correct llm_prompt

* Remove unncessary field

* default: false for llm_grading

* Add proper linking between ai_comments table and submissions. Return it to submission retrieval as well

* Resolve some migration comments

* Add llm_model and llm_api_url to the DB + schema

* Moves api key, api url, llm model and course prompt to course level

* Add encryption_key to env

* Do not hardcode formatting instructions

* Add Assessment level prompts to the XML

* Return some additional info for composing of prompts

* Remove un-used 'save comments'

* Fix existing assessment tests

* Fix generate_ai_comments test cases

* Fix bug preventing avengers from generating ai comments

* Fix up tests + error msgs

* Formatting

* some mix credo suggestions

* format

* Fix credo issue

* bug fix + credo fixes

* Fix tests

* format

* Modify test.exs

* Update lib/cadet_web/controllers/generate_ai_comments.ex

Co-authored-by: Copilot <[email protected]>

* Copilot feedback

* format

* Work on sentry comments

* Fix type

* Redate migrations to maintain total order

* Add newline at EOF

* Fix indent

* Fix capitalization

* Remove llmApiKey from any kind of storage on FE

* Remove indexes

* rm todo

* rm todo

* Re-format ai_comments to reference answer_id instead

* Abstract out + remove un-used field

* Add delimeter + bug fixes

* Mix format

* Switch to openAI module

* rm un-used

* Merge all prompts to :prompts to preserve abstraction

* rm

* Fix formatting

* Fix test

* Revert some dependency changes

* Update actions versions

* Improve encrypt + decrypt robustness

* Fix dialyzer

* Re-factor schema

---------

Co-authored-by: Eugene Oh Yun Zheng <[email protected]>
Co-authored-by: tkaixiang <[email protected]>

Author: 3 people

.github/workflows/cd.yml

@@ -29,7 +29,7 @@ jobs:
       OTP_VERSION: 27.3.3
     steps:
       - uses: rlespinasse/[email protected]
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
       - name: Cache deps
         uses: actions/cache@v4
         with:

.github/workflows/ci.yml

@@ -37,7 +37,7 @@ jobs:
         # needed because the postgres container does not provide a healthcheck
         options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
       - name: Cache deps
         uses: actions/cache@v4
         with:

.gitignore

@@ -118,3 +118,6 @@ erl_crash.dump
 
 # Generated lexer
 /src/source_lexer.erl
+
+# Ignore log files
+/log

config/dev.secrets.exs.example

@@ -98,13 +98,9 @@ config :cadet,
     # ws_endpoint_address: "ws://hostname:port"
   ]
 
-  config :openai,
-  # find it at https://platform.openai.com/account/api-keys
-  api_key: "the actual api key",
-  # For source academy deployment, leave this as empty string.Ingeneral could find it at https://platform.openai.com/account/org-settings under "Organization ID".
-  organization_key: "",
-  # optional, passed to [HTTPoison.Request](https://hexdocs.pm/httpoison/HTTPoison.Request.html) options
-  http_options: [recv_timeout: 170_0000]
+config :openai,
+  # Input your own AES-256 encryption key here for encrypting LLM API keys of 16, 24 or 32 bytes
+  encryption_key: "<ADD YOUR OWN KEY HERE>"
 
 # config :sentry,
 #   dsn: "https://public_key/sentry.io/somethingsomething"

config/test.exs

@@ -100,3 +100,7 @@ config :cadet, Oban,
   testing: :manual
 
 config :cadet, Cadet.Mailer, adapter: Bamboo.TestAdapter
+
+config :openai,
+  # Input your own AES-256 encryption key here for encrypting LLM API keys
+  encryption_key: "b4u7g0AyN3Tu2br9WSdZQjLMQ8bed/wgQWrH2x3qPdW8D55iv10+ySgs+bxDirWE"

lib/cadet/ai_comments.ex

@@ -0,0 +1,77 @@
+defmodule Cadet.AIComments do
+  @moduledoc """
+  Handles operations related to AI comments, including creation, updates, and retrieval.
+  """
+
+  import Ecto.Query
+  alias Cadet.Repo
+  alias Cadet.AIComments.AIComment
+
+  @doc """
+  Creates a new AI comment log entry.
+  """
+  def create_ai_comment(attrs \\ %{}) do
+    %AIComment{}
+    |> AIComment.changeset(attrs)
+    |> Repo.insert()
+  end
+
+  @doc """
+  Gets an AI comment by ID.
+  """
+  def get_ai_comment(id) do
+    case Repo.get(AIComment, id) do
+      nil -> {:error, :not_found}
+      comment -> {:ok, comment}
+    end
+  end
+
+  @doc """
+  Retrieves the latest AI comment for a specific submission and question.
+  Returns `nil` if no comment exists.
+  """
+  def get_latest_ai_comment(answer_id) do
+    Repo.one(
+      from(c in AIComment,
+        where: c.answer_id == ^answer_id,
+        order_by: [desc: c.inserted_at],
+        limit: 1
+      )
+    )
+  end
+
+  @doc """
+  Updates the final comment for a specific submission and question.
+  Returns the most recent comment entry for that submission/question.
+  """
+  def update_final_comment(answer_id, final_comment) do
+    comment = get_latest_ai_comment(answer_id)
+
+    case comment do
+      nil ->
+        {:error, :not_found}
+
+      _ ->
+        comment
+        |> AIComment.changeset(%{final_comment: final_comment})
+        |> Repo.update()
+    end
+  end
+
+  @doc """
+  Updates an existing AI comment with new attributes.
+  """
+  def update_ai_comment(id, attrs) do
+    id
+    |> get_ai_comment()
+    |> case do
+      {:error, :not_found} ->
+        {:error, :not_found}
+
+      {:ok, comment} ->
+        comment
+        |> AIComment.changeset(attrs)
+        |> Repo.update()
+    end
+  end
+end

lib/cadet/ai_comments/ai_comment.ex

@@ -0,0 +1,30 @@
+defmodule Cadet.AIComments.AIComment do
+  @moduledoc """
+  Defines the schema and changeset for AI comments.
+  """
+
+  use Ecto.Schema
+  import Ecto.Changeset
+
+  schema "ai_comment_logs" do
+    field(:raw_prompt, :string)
+    field(:answers_json, :string)
+    field(:response, :string)
+    field(:error, :string)
+    field(:final_comment, :string)
+
+    belongs_to(:answer, Cadet.Assessments.Answer)
+
+    timestamps()
+  end
+
+  @required_fields ~w(answer_id raw_prompt answers_json)a
+  @optional_fields ~w(response error final_comment)a
+
+  def changeset(ai_comment, attrs) do
+    ai_comment
+    |> cast(attrs, @required_fields ++ @optional_fields)
+    |> validate_required(@required_fields)
+    |> foreign_key_constraint(:answer_id)
+  end
+end

lib/cadet/assessments/answer.ex

@@ -10,6 +10,7 @@ defmodule Cadet.Assessments.Answer do
   alias Cadet.Assessments.Answer.AutogradingStatus
   alias Cadet.Assessments.AnswerTypes.{MCQAnswer, ProgrammingAnswer, VotingAnswer}
   alias Cadet.Assessments.{Question, QuestionType, Submission}
+  alias Cadet.AIComments.AIComment
 
   @type t :: %__MODULE__{}
 
@@ -29,6 +30,7 @@ defmodule Cadet.Assessments.Answer do
     belongs_to(:grader, CourseRegistration)
     belongs_to(:submission, Submission)
     belongs_to(:question, Question)
+    has_many(:ai_comments, AIComment, on_delete: :delete_all)
 
     timestamps()
   end

lib/cadet/assessments/assessment.ex

@@ -36,6 +36,7 @@ defmodule Cadet.Assessments.Assessment do
     field(:max_team_size, :integer, default: 1)
     field(:has_token_counter, :boolean, default: false)
     field(:has_voting_features, :boolean, default: false)
+    field(:llm_assessment_prompt, :string, default: nil)
 
     belongs_to(:config, AssessmentConfig)
     belongs_to(:course, Course)
@@ -46,7 +47,7 @@ defmodule Cadet.Assessments.Assessment do
 
   @required_fields ~w(title open_at close_at number course_id config_id max_team_size)a
   @optional_fields ~w(reading summary_short summary_long
-    is_published story cover_picture access password has_token_counter has_voting_features)a
+    is_published story cover_picture access password has_token_counter has_voting_features llm_assessment_prompt)a
   @optional_file_fields ~w(mission_pdf)a
 
   def changeset(assessment, params) do

lib/cadet/assessments/assessments.ex

@@ -3043,13 +3043,60 @@ defmodule Cadet.Assessments do
     }
   end
 
+  @spec get_answer(integer() | String.t()) ::
+          {:ok, Answer.t()} | {:error, {:bad_request, String.t()}}
+  def get_answer(id) when is_ecto_id(id) do
+    answer =
+      Answer
+      |> where(id: ^id)
+      # [a] are bindings (in SQL it is similar to FROM answers "AS a"),
+      # this line's alias is INNER JOIN ... "AS q"
+      |> join(:inner, [a], q in assoc(a, :question))
+      |> join(:inner, [_, q], ast in assoc(q, :assessment))
+      |> join(:inner, [..., ast], ac in assoc(ast, :config))
+      |> join(:left, [a, ...], g in assoc(a, :grader))
+      |> join(:left, [_, ..., g], gu in assoc(g, :user))
+      |> join(:inner, [a, ...], s in assoc(a, :submission))
+      |> join(:left, [_, ..., s], st in assoc(s, :student))
+      |> join(:left, [..., st], u in assoc(st, :user))
+      |> join(:left, [..., s, _, _], t in assoc(s, :team))
+      |> join(:left, [..., t], tm in assoc(t, :team_members))
+      |> join(:left, [..., tm], tms in assoc(tm, :student))
+      |> join(:left, [..., tms], tmu in assoc(tms, :user))
+      |> join(:left, [a, ...], ai in assoc(a, :ai_comments))
+      |> preload([_, q, ast, ac, g, gu, s, st, u, t, tm, tms, tmu, ai],
+        ai_comments: ai,
+        question: {q, assessment: {ast, config: ac}},
+        grader: {g, user: gu},
+        submission:
+          {s, student: {st, user: u}, team: {t, team_members: {tm, student: {tms, user: tmu}}}}
+      )
+      |> Repo.one()
+
+    if is_nil(answer) do
+      {:error, {:bad_request, "Answer not found."}}
+    else
+      if answer.question.type == :voting do
+        empty_contest_entries = Map.put(answer.question.question, :contest_entries, [])
+        empty_popular_leaderboard = Map.put(empty_contest_entries, :popular_leaderboard, [])
+        empty_contest_leaderboard = Map.put(empty_popular_leaderboard, :contest_leaderboard, [])
+        question = Map.put(answer.question, :question, empty_contest_leaderboard)
+        Map.put(answer, :question, question)
+      end
+
+      {:ok, answer}
+    end
+  end
+
   @spec get_answers_in_submission(integer() | String.t()) ::
           {:ok, {[Answer.t()], Assessment.t()}}
           | {:error, {:bad_request, String.t()}}
   def get_answers_in_submission(id) when is_ecto_id(id) do
-    answer_query =
+    base_query =
       Answer
       |> where(submission_id: ^id)
+      # [a] are bindings (in SQL it is similar to FROM answers "AS a"),
+      # this line's alias is INNER JOIN ... "AS q"
       |> join(:inner, [a], q in assoc(a, :question))
       |> join(:inner, [_, q], ast in assoc(q, :assessment))
       |> join(:inner, [..., ast], ac in assoc(ast, :config))
@@ -3062,15 +3109,17 @@ defmodule Cadet.Assessments do
       |> join(:left, [..., t], tm in assoc(t, :team_members))
       |> join(:left, [..., tm], tms in assoc(tm, :student))
       |> join(:left, [..., tms], tmu in assoc(tms, :user))
-      |> preload([_, q, ast, ac, g, gu, s, st, u, t, tm, tms, tmu],
+      |> join(:left, [a, ...], ai in assoc(a, :ai_comments))
+      |> preload([_, q, ast, ac, g, gu, s, st, u, t, tm, tms, tmu, ai],
         question: {q, assessment: {ast, config: ac}},
         grader: {g, user: gu},
         submission:
-          {s, student: {st, user: u}, team: {t, team_members: {tm, student: {tms, user: tmu}}}}
+          {s, student: {st, user: u}, team: {t, team_members: {tm, student: {tms, user: tmu}}}},
+        ai_comments: ai
       )
 
     answers =
-      answer_query
+      base_query
       |> Repo.all()
       |> Enum.sort_by(& &1.question.display_order)
       |> Enum.map(fn ans ->
@@ -3544,4 +3593,15 @@ defmodule Cadet.Assessments do
         end)
     end
   end
+
+  def get_llm_assessment_prompt(question_id) do
+    query =
+      from(q in Question,
+        where: q.id == ^question_id,
+        join: a in assoc(q, :assessment),
+        select: a.llm_assessment_prompt
+      )
+
+    Repo.one(query)
+  end
 end