AI-powered marking

.gitignore

@@ -118,3 +118,6 @@ erl_crash.dump
 
 # Generated lexer
 /src/source_lexer.erl
+
+# Ignore log files
+/log

config/dev.secrets.exs.example

@@ -99,12 +99,8 @@ config :cadet,
   ]
 
   config :openai,
-  # find it at https://platform.openai.com/account/api-keys
-  api_key: "the actual api key",
-  # For source academy deployment, leave this as empty string.Ingeneral could find it at https://platform.openai.com/account/org-settings under "Organization ID".
-  organization_key: "",
-  # optional, passed to [HTTPoison.Request](https://hexdocs.pm/httpoison/HTTPoison.Request.html) options
-  http_options: [recv_timeout: 170_0000]
+    # TODO: Input your own AES-256 encryption key here for encrypting LLM API keys
+    encryption_key: "<ADD YOUR OWN KEY HERE>"
 
 # config :sentry,
 #   dsn: "https://public_key/sentry.io/somethingsomething"

lib/cadet/ai_comments.ex

@@ -0,0 +1,107 @@
+defmodule Cadet.AIComments do
+  @moduledoc """
+  Handles operations related to AI comments, including creation, updates, and retrieval.
+  """
+
+  import Ecto.Query
+  alias Cadet.Repo
+  alias Cadet.AIComments.AIComment
+
+  @doc """
+  Creates a new AI comment log entry.
+  """
+  def create_ai_comment(attrs \\ %{}) do
+    %AIComment{}
+    |> AIComment.changeset(attrs)
+    |> Repo.insert()
+  end
+
+  @doc """
+  Gets an AI comment by ID.
+  """
+  def get_ai_comment(id) do
+    case Repo.get(AIComment, id) do
+      nil -> {:error, :not_found}
+      comment -> {:ok, comment}
+    end
+  end
+
+  @doc """
+  Retrieves an AI comment for a specific submission and question.
+  Returns `nil` if no comment exists.
+  """
+  def get_ai_comments_for_submission(submission_id, question_id) do
+    Repo.one(
+      from(c in AIComment,
+        where: c.submission_id == ^submission_id and c.question_id == ^question_id
+      )
+    )
+  end
+
+  @doc """
+  Retrieves the latest AI comment for a specific submission and question.
+  Returns `nil` if no comment exists.
+  """
+  def get_latest_ai_comment(submission_id, question_id) do
+    Repo.one(
+      from(c in AIComment,
+        where: c.submission_id == ^submission_id and c.question_id == ^question_id,
+        order_by: [desc: c.inserted_at],
+        limit: 1
+      )
+    )
+  end
+
+  @doc """
+  Updates the final comment for a specific submission and question.
+  Returns the most recent comment entry for that submission/question.
+  """
+  def update_final_comment(submission_id, question_id, final_comment) do
+    comment = get_latest_ai_comment(submission_id, question_id)
+
+    case comment do
+      nil ->
+        {:error, :not_found}
+
+      _ ->
+        comment
+        |> AIComment.changeset(%{final_comment: final_comment})
+        |> Repo.update()
+    end
+  end
+
+  @doc """
+  Updates an existing AI comment with new attributes.
+  """
+  def update_ai_comment(id, attrs) do
+    id
+    |> get_ai_comment()
+    |> case do
+      {:error, :not_found} ->
+        {:error, :not_found}
+
+      {:ok, comment} ->
+        comment
+        |> AIComment.changeset(attrs)
+        |> Repo.update()
+    end
+  end
+
+  @doc """
+  Updates the chosen comments for a specific submission and question.
+  Accepts an array of comments and replaces the existing array in the database.
+  """
+  def update_chosen_comments(submission_id, question_id, new_comments) do
+    comment = get_latest_ai_comment(submission_id, question_id)
+
+    case comment do
+      nil ->
+        {:error, :not_found}
+
+      _ ->
+        comment
+        |> AIComment.changeset(%{comment_chosen: new_comments})
+        |> Repo.update()
+    end
+  end
+end

lib/cadet/ai_comments/ai_comment.ex

@@ -0,0 +1,39 @@
+defmodule Cadet.AIComments.AIComment do
+  @moduledoc """
+  Defines the schema and changeset for AI comments.
+  """
+
+  use Ecto.Schema
+  import Ecto.Changeset
+
+  schema "ai_comment_logs" do
+    field(:raw_prompt, :string)
+    field(:answers_json, :string)
+    field(:response, :string)
+    field(:error, :string)
+    field(:comment_chosen, {:array, :string})
+    field(:final_comment, :string)
+
+    belongs_to(:submission, Cadet.Assessments.Submission)
+    belongs_to(:question, Cadet.Assessments.Question)
+
+    timestamps()
+  end
+
+  def changeset(ai_comment, attrs) do
+    ai_comment
+    |> cast(attrs, [
+      :submission_id,
+      :question_id,
+      :raw_prompt,
+      :answers_json,
+      :response,
+      :error,
+      :comment_chosen,
+      :final_comment
+    ])
+    |> validate_required([:submission_id, :question_id, :raw_prompt, :answers_json])
+    |> foreign_key_constraint(:submission_id)
+    |> foreign_key_constraint(:question_id)
+  end
+end

lib/cadet/assessments/answer.ex

@@ -29,6 +29,7 @@ defmodule Cadet.Assessments.Answer do
     belongs_to(:grader, CourseRegistration)
     belongs_to(:submission, Submission)
     belongs_to(:question, Question)
+    has_many(:ai_comments, through: [:submission, :ai_comments])
 
     timestamps()
   end

lib/cadet/assessments/assessment.ex

@@ -36,6 +36,7 @@ defmodule Cadet.Assessments.Assessment do
     field(:max_team_size, :integer, default: 1)
     field(:has_token_counter, :boolean, default: false)
     field(:has_voting_features, :boolean, default: false)
+    field(:llm_assessment_prompt, :string, default: nil)
 
     belongs_to(:config, AssessmentConfig)
     belongs_to(:course, Course)
@@ -46,7 +47,7 @@ defmodule Cadet.Assessments.Assessment do
 
   @required_fields ~w(title open_at close_at number course_id config_id max_team_size)a
   @optional_fields ~w(reading summary_short summary_long
-    is_published story cover_picture access password has_token_counter has_voting_features)a
+    is_published story cover_picture access password has_token_counter has_voting_features llm_assessment_prompt)a
   @optional_file_fields ~w(mission_pdf)a
 
   def changeset(assessment, params) do

lib/cadet/assessments/assessments.ex

@@ -2286,29 +2286,37 @@ defmodule Cadet.Assessments do
   @spec get_answers_in_submission(integer() | String.t()) ::
           {:ok, {[Answer.t()], Assessment.t()}}
           | {:error, {:bad_request, String.t()}}
-  def get_answers_in_submission(id) when is_ecto_id(id) do
-    answer_query =
+  def get_answers_in_submission(id, question_id \\ nil) when is_ecto_id(id) do
+    base_query =
       Answer
       |> where(submission_id: ^id)
-      |> join(:inner, [a], q in assoc(a, :question))
+      |> join(:inner, [a], q in assoc(a, :question))  # [a] are bindings (in SQL it is similar to FROM answers "AS a"), this line's alias is INNER JOIN ... "AS q"
       |> join(:inner, [_, q], ast in assoc(q, :assessment))
       |> join(:inner, [..., ast], ac in assoc(ast, :config))
       |> join(:left, [a, ...], g in assoc(a, :grader))
       |> join(:left, [_, ..., g], gu in assoc(g, :user))
       |> join(:inner, [a, ...], s in assoc(a, :submission))
-      |> join(:left, [_, ..., s], st in assoc(s, :student))
+      |> join(:left, [..., s], ai in assoc(s, :ai_comments))
+      |> join(:left, [_, ..., s, _], st in assoc(s, :student))
       |> join(:left, [..., st], u in assoc(st, :user))
-      |> join(:left, [..., s, _, _], t in assoc(s, :team))
+      |> join(:left, [..., s, _, _, _], t in assoc(s, :team))
       |> join(:left, [..., t], tm in assoc(t, :team_members))
       |> join(:left, [..., tm], tms in assoc(tm, :student))
       |> join(:left, [..., tms], tmu in assoc(tms, :user))
-      |> preload([_, q, ast, ac, g, gu, s, st, u, t, tm, tms, tmu],
+      |> preload([_, q, ast, ac, g, gu, s, ai, st, u, t, tm, tms, tmu],
+        ai_comments: ai,
         question: {q, assessment: {ast, config: ac}},
         grader: {g, user: gu},
         submission:
           {s, student: {st, user: u}, team: {t, team_members: {tm, student: {tms, user: tmu}}}}
       )
 
+    answer_query =
+      case question_id do
+        nil -> base_query
+        _ -> base_query |> where(question_id: ^question_id)
+      end
+
     answers =
       answer_query
       |> Repo.all()
@@ -2784,4 +2792,13 @@ defmodule Cadet.Assessments do
         end)
     end
   end
+
+  def get_llm_assessment_prompt(question_id) do
+     from(q in Question,
+    where: q.id == ^question_id,
+    join: a in assoc(q, :assessment),
+    select: a.llm_assessment_prompt
+  )
+  |> Repo.one()
+  end
 end

lib/cadet/assessments/question_types/programming_question.ex

@@ -13,13 +13,14 @@ defmodule Cadet.Assessments.QuestionTypes.ProgrammingQuestion do
     field(:template, :string)
     field(:postpend, :string, default: "")
     field(:solution, :string)
+    field(:llm_prompt, :string)
     embeds_many(:public, Testcase)
     embeds_many(:opaque, Testcase)
     embeds_many(:secret, Testcase)
   end
 
   @required_fields ~w(content template)a
-  @optional_fields ~w(solution prepend postpend)a
+  @optional_fields ~w(solution prepend postpend llm_prompt)a
 
   def changeset(question, params \\ %{}) do
     question

lib/cadet/assessments/submission.ex

@@ -25,6 +25,7 @@ defmodule Cadet.Assessments.Submission do
     belongs_to(:team, Team)
     belongs_to(:unsubmitted_by, CourseRegistration)
     has_many(:answers, Answer, on_delete: :delete_all)
+    has_many(:ai_comments, Cadet.AIComments.AIComment, on_delete: :delete_all)
 
     timestamps()
   end

lib/cadet/courses/course.ex

@@ -14,6 +14,11 @@ defmodule Cadet.Courses.Course do
           enable_achievements: boolean(),
           enable_sourcecast: boolean(),
           enable_stories: boolean(),
+          enable_llm_grading: boolean(),
+          llm_api_key: String.t() | nil,
+          llm_model: String.t() | nil,
+          llm_api_url: String.t() | nil,
+          llm_course_level_prompt: String.t() | nil,
           source_chapter: integer(),
           source_variant: String.t(),
           module_help_text: String.t(),
@@ -28,6 +33,11 @@ defmodule Cadet.Courses.Course do
     field(:enable_achievements, :boolean, default: true)
     field(:enable_sourcecast, :boolean, default: true)
     field(:enable_stories, :boolean, default: false)
+    field(:enable_llm_grading, :boolean, default: false)
+    field(:llm_api_key, :string, default: nil)
+    field(:llm_model, :string, default: nil)
+    field(:llm_api_url, :string, default: nil)
+    field(:llm_course_level_prompt, :string, default: nil)
     field(:source_chapter, :integer)
     field(:source_variant, :string)
     field(:module_help_text, :string)
@@ -42,13 +52,59 @@ defmodule Cadet.Courses.Course do
 
   @required_fields ~w(course_name viewable enable_game
     enable_achievements enable_sourcecast enable_stories source_chapter source_variant)a
-  @optional_fields ~w(course_short_name module_help_text)a
-
+  @optional_fields ~w(course_short_name module_help_text enable_llm_grading llm_api_key llm_model llm_api_url llm_course_level_prompt)a
+
+  @spec changeset(
+          {map(), map()}
+          | %{
+              :__struct__ => atom() | %{:__changeset__ => map(), optional(any()) => any()},
+              optional(atom()) => any()
+            },
+          %{optional(:__struct__) => none(), optional(atom() | binary()) => any()}
+        ) :: Ecto.Changeset.t()
   def changeset(course, params) do
     course
     |> cast(params, @required_fields ++ @optional_fields)
     |> validate_required(@required_fields)
     |> validate_sublanguage_combination(params)
+    |> put_encrypted_llm_api_key()
+  end
+
+  def put_encrypted_llm_api_key(changeset) do
+    if llm_api_key = get_change(changeset, :llm_api_key) do
+      if is_binary(llm_api_key) and llm_api_key != "" do
+        secret = Application.get_env(:openai, :encryption_key)
+
+        if is_binary(secret) and byte_size(secret) >= 16 do
+          # Use first 16 bytes for AES-128, 24 for AES-192, or 32 for AES-256
+          key = binary_part(secret, 0, min(32, byte_size(secret)))
+          # Use AES in GCM mode for encryption
+          iv = :crypto.strong_rand_bytes(16)
+
+          {ciphertext, tag} =
+            :crypto.crypto_one_time_aead(
+              :aes_gcm,
+              key,
+              iv,
+              llm_api_key,
+              "",
+              true
+            )
+
+          # Store both the IV, ciphertext and tag
+          encrypted = iv <> tag <> ciphertext
+          put_change(changeset, :llm_api_key, Base.encode64(encrypted))
+        else
+          add_error(changeset, :llm_api_key, "encryption key not configured properly")
+        end
+      else
+        # If empty string or nil is provided, don't encrypt but don't add error
+        changeset
+      end
+    else
+      # The key is not being changed, so we need to preserve the existing value
+      put_change(changeset, :llm_api_key, changeset.data.llm_api_key)
+    end
   end
 
   # Validates combination of Source chapter and variant

lib/cadet/jobs/xml_parser.ex

@@ -98,6 +98,7 @@ defmodule Cadet.Updater.XMLParser do
         reading: ~x"//READING/text()" |> transform_by(&process_charlist/1),
         summary_short: ~x"//WEBSUMMARY/text()" |> transform_by(&process_charlist/1),
         summary_long: ~x"./TEXT/text()" |> transform_by(&process_charlist/1),
+        llm_assessment_prompt: ~x"./LLM_ASSESSMENT_PROMPT/text()" |> transform_by(&process_charlist/1),
         password: ~x"//PASSWORD/text()"so |> transform_by(&process_charlist/1)
       )
       |> Map.put(:is_published, false)
@@ -202,7 +203,8 @@ defmodule Cadet.Updater.XMLParser do
         prepend: ~x"./SNIPPET/PREPEND/text()" |> transform_by(&process_charlist/1),
         template: ~x"./SNIPPET/TEMPLATE/text()" |> transform_by(&process_charlist/1),
         postpend: ~x"./SNIPPET/POSTPEND/text()" |> transform_by(&process_charlist/1),
-        solution: ~x"./SNIPPET/SOLUTION/text()" |> transform_by(&process_charlist/1)
+        solution: ~x"./SNIPPET/SOLUTION/text()" |> transform_by(&process_charlist/1),
+        llm_prompt: ~x"./LLM_GRADING_PROMPT/text()" |> transform_by(&process_charlist/1)
       ),
       entity
       |> xmap(

lib/cadet_web/admin_controllers/admin_courses_controller.ex

@@ -108,6 +108,8 @@ defmodule CadetWeb.AdminCoursesController do
       enable_achievements(:body, :boolean, "Enable achievements")
       enable_sourcecast(:body, :boolean, "Enable sourcecast")
       enable_stories(:body, :boolean, "Enable stories")
+      enable_llm_grading(:body, :boolean, "Enable LLM grading")
+      llm_api_key(:body, :string, "OpenAI API key for this course")
       sublanguage(:body, Schema.ref(:AdminSublanguage), "sublanguage object")
       module_help_text(:body, :string, "Module help text")
     end