diff --git a/.gitignore b/.gitignore index c6f53010c..80b3e3026 100644 --- a/.gitignore +++ b/.gitignore @@ -118,3 +118,6 @@ erl_crash.dump # Generated lexer /src/source_lexer.erl + +# Ignore log files +/log \ No newline at end of file diff --git a/lib/cadet/ai_comments.ex b/lib/cadet/ai_comments.ex new file mode 100644 index 000000000..76b059601 --- /dev/null +++ b/lib/cadet/ai_comments.ex @@ -0,0 +1,95 @@ +defmodule Cadet.AIComments do + @moduledoc """ + Handles operations related to AI comments, including creation, updates, and retrieval. + """ + + import Ecto.Query + alias Cadet.Repo + alias Cadet.AIComments.AIComment + + @doc """ + Creates a new AI comment log entry. + """ + def create_ai_comment(attrs \\ %{}) do + %AIComment{} + |> AIComment.changeset(attrs) + |> Repo.insert() + end + + @doc """ + Gets an AI comment by ID. + """ + def get_ai_comment!(id), do: Repo.get!(AIComment, id) + + @doc """ + Retrieves an AI comment for a specific submission and question. + Returns `nil` if no comment exists. + """ + def get_ai_comments_for_submission(submission_id, question_id) do + Repo.one( + from(c in AIComment, + where: c.submission_id == ^submission_id and c.question_id == ^question_id + ) + ) + end + + @doc """ + Retrieves the latest AI comment for a specific submission and question. + Returns `nil` if no comment exists. + """ + def get_latest_ai_comment(submission_id, question_id) do + Repo.one( + from(c in AIComment, + where: c.submission_id == ^submission_id and c.question_id == ^question_id, + order_by: [desc: c.inserted_at], + limit: 1 + ) + ) + end + + @doc """ + Updates the final comment for a specific submission and question. + Returns the most recent comment entry for that submission/question. + """ + def update_final_comment(submission_id, question_id, final_comment) do + comment = get_latest_ai_comment(submission_id, question_id) + + case comment do + nil -> + {:error, :not_found} + + _ -> + comment + |> AIComment.changeset(%{final_comment: final_comment}) + |> Repo.update() + end + end + + @doc """ + Updates an existing AI comment with new attributes. + """ + def update_ai_comment(id, attrs) do + id + |> get_ai_comment!() + |> AIComment.changeset(attrs) + |> Repo.update() + end + + @doc """ + Updates the chosen comments for a specific submission and question. + Accepts an array of comments and replaces the existing array in the database. + """ + def update_chosen_comments(submission_id, question_id, new_comments) do + comment = get_latest_ai_comment(submission_id, question_id) + + case comment do + nil -> + {:error, :not_found} + + _ -> + comment + |> AIComment.changeset(%{comment_chosen: new_comments}) + |> Repo.update() + end + end +end diff --git a/lib/cadet/ai_comments/ai_comment.ex b/lib/cadet/ai_comments/ai_comment.ex new file mode 100644 index 000000000..1f1fae478 --- /dev/null +++ b/lib/cadet/ai_comments/ai_comment.ex @@ -0,0 +1,36 @@ +defmodule Cadet.AIComments.AIComment do + @moduledoc """ + Defines the schema and changeset for AI comments. + """ + + use Ecto.Schema + import Ecto.Changeset + + schema "ai_comment_logs" do + field(:submission_id, :integer) + field(:question_id, :integer) + field(:raw_prompt, :string) + field(:answers_json, :string) + field(:response, :string) + field(:error, :string) + field(:comment_chosen, {:array, :string}) + field(:final_comment, :string) + + timestamps() + end + + def changeset(ai_comment, attrs) do + ai_comment + |> cast(attrs, [ + :submission_id, + :question_id, + :raw_prompt, + :answers_json, + :response, + :error, + :comment_chosen, + :final_comment + ]) + |> validate_required([:submission_id, :question_id, :raw_prompt, :answers_json]) + end +end diff --git a/lib/cadet/assessments/assessments.ex b/lib/cadet/assessments/assessments.ex index 244ebc742..70307d85d 100644 --- a/lib/cadet/assessments/assessments.ex +++ b/lib/cadet/assessments/assessments.ex @@ -2289,8 +2289,8 @@ defmodule Cadet.Assessments do @spec get_answers_in_submission(integer() | String.t()) :: {:ok, {[Answer.t()], Assessment.t()}} | {:error, {:bad_request, String.t()}} - def get_answers_in_submission(id) when is_ecto_id(id) do - answer_query = + def get_answers_in_submission(id, question_id \\ nil) when is_ecto_id(id) do + base_query = Answer |> where(submission_id: ^id) |> join(:inner, [a], q in assoc(a, :question)) @@ -2312,6 +2312,13 @@ defmodule Cadet.Assessments do {s, student: {st, user: u}, team: {t, team_members: {tm, student: {tms, user: tmu}}}} ) + answer_query = + if is_nil(question_id) do + base_query + else + base_query |> where(question_id: ^question_id) + end + answers = answer_query |> Repo.all() diff --git a/lib/cadet/assessments/question_types/programming_question.ex b/lib/cadet/assessments/question_types/programming_question.ex index a39625de3..245e8ef11 100644 --- a/lib/cadet/assessments/question_types/programming_question.ex +++ b/lib/cadet/assessments/question_types/programming_question.ex @@ -13,13 +13,14 @@ defmodule Cadet.Assessments.QuestionTypes.ProgrammingQuestion do field(:template, :string) field(:postpend, :string, default: "") field(:solution, :string) + field(:llm_prompt, :string) embeds_many(:public, Testcase) embeds_many(:opaque, Testcase) embeds_many(:secret, Testcase) end @required_fields ~w(content template)a - @optional_fields ~w(solution prepend postpend)a + @optional_fields ~w(solution prepend postpend llm_prompt)a def changeset(question, params \\ %{}) do question diff --git a/lib/cadet/courses/course.ex b/lib/cadet/courses/course.ex index c74d23bd7..cf6170cfb 100644 --- a/lib/cadet/courses/course.ex +++ b/lib/cadet/courses/course.ex @@ -14,6 +14,8 @@ defmodule Cadet.Courses.Course do enable_achievements: boolean(), enable_sourcecast: boolean(), enable_stories: boolean(), + enable_llm_grading: boolean(), + llm_api_key: String.t() | nil, source_chapter: integer(), source_variant: String.t(), module_help_text: String.t(), @@ -28,6 +30,8 @@ defmodule Cadet.Courses.Course do field(:enable_achievements, :boolean, default: true) field(:enable_sourcecast, :boolean, default: true) field(:enable_stories, :boolean, default: false) + field(:enable_llm_grading, :boolean) + field(:llm_api_key, :string) field(:source_chapter, :integer) field(:source_variant, :string) field(:module_help_text, :string) @@ -42,13 +46,59 @@ defmodule Cadet.Courses.Course do @required_fields ~w(course_name viewable enable_game enable_achievements enable_sourcecast enable_stories source_chapter source_variant)a - @optional_fields ~w(course_short_name module_help_text)a - + @optional_fields ~w(course_short_name module_help_text enable_llm_grading llm_api_key)a + + @spec changeset( + {map(), map()} + | %{ + :__struct__ => atom() | %{:__changeset__ => map(), optional(any()) => any()}, + optional(atom()) => any() + }, + %{optional(:__struct__) => none(), optional(atom() | binary()) => any()} + ) :: Ecto.Changeset.t() def changeset(course, params) do course |> cast(params, @required_fields ++ @optional_fields) |> validate_required(@required_fields) |> validate_sublanguage_combination(params) + |> put_encrypted_llm_api_key() + end + + def put_encrypted_llm_api_key(changeset) do + if llm_api_key = get_change(changeset, :llm_api_key) do + if is_binary(llm_api_key) and llm_api_key != "" do + secret = Application.get_env(:openai, :encryption_key) + + if is_binary(secret) and byte_size(secret) >= 16 do + # Use first 16 bytes for AES-128, 24 for AES-192, or 32 for AES-256 + key = binary_part(secret, 0, min(32, byte_size(secret))) + # Use AES in GCM mode for encryption + iv = :crypto.strong_rand_bytes(16) + + {ciphertext, tag} = + :crypto.crypto_one_time_aead( + :aes_gcm, + key, + iv, + llm_api_key, + "", + true + ) + + # Store both the IV, ciphertext and tag + encrypted = iv <> tag <> ciphertext + put_change(changeset, :llm_api_key, Base.encode64(encrypted)) + else + add_error(changeset, :llm_api_key, "encryption key not configured properly") + end + else + # If empty string or nil is provided, don't encrypt but don't add error + changeset + end + else + # The key is not being changed, so we need to preserve the existing value + put_change(changeset, :llm_api_key, changeset.data.llm_api_key) + end end # Validates combination of Source chapter and variant diff --git a/lib/cadet/jobs/xml_parser.ex b/lib/cadet/jobs/xml_parser.ex index b49128506..abf16c8cd 100644 --- a/lib/cadet/jobs/xml_parser.ex +++ b/lib/cadet/jobs/xml_parser.ex @@ -202,7 +202,8 @@ defmodule Cadet.Updater.XMLParser do prepend: ~x"./SNIPPET/PREPEND/text()" |> transform_by(&process_charlist/1), template: ~x"./SNIPPET/TEMPLATE/text()" |> transform_by(&process_charlist/1), postpend: ~x"./SNIPPET/POSTPEND/text()" |> transform_by(&process_charlist/1), - solution: ~x"./SNIPPET/SOLUTION/text()" |> transform_by(&process_charlist/1) + solution: ~x"./SNIPPET/SOLUTION/text()" |> transform_by(&process_charlist/1), + llm_prompt: ~x"./LLM_GRADING_PROMPT/text()" |> transform_by(&process_charlist/1) ), entity |> xmap( diff --git a/lib/cadet_web/admin_controllers/admin_courses_controller.ex b/lib/cadet_web/admin_controllers/admin_courses_controller.ex index 7220a4d80..09d7d8616 100644 --- a/lib/cadet_web/admin_controllers/admin_courses_controller.ex +++ b/lib/cadet_web/admin_controllers/admin_courses_controller.ex @@ -108,6 +108,8 @@ defmodule CadetWeb.AdminCoursesController do enable_achievements(:body, :boolean, "Enable achievements") enable_sourcecast(:body, :boolean, "Enable sourcecast") enable_stories(:body, :boolean, "Enable stories") + enable_llm_grading(:body, :boolean, "Enable LLM grading") + llm_api_key(:body, :string, "OpenAI API key for this course") sublanguage(:body, Schema.ref(:AdminSublanguage), "sublanguage object") module_help_text(:body, :string, "Module help text") end diff --git a/lib/cadet_web/controllers/courses_controller.ex b/lib/cadet_web/controllers/courses_controller.ex index e6555bd7a..c717e3f06 100644 --- a/lib/cadet_web/controllers/courses_controller.ex +++ b/lib/cadet_web/controllers/courses_controller.ex @@ -56,6 +56,8 @@ defmodule CadetWeb.CoursesController do enable_achievements(:body, :boolean, "Enable achievements", required: true) enable_sourcecast(:body, :boolean, "Enable sourcecast", required: true) enable_stories(:body, :boolean, "Enable stories", required: true) + enable_llm_grading(:body, :boolean, "Enable LLM grading", required: false) + llm_api_key(:body, :string, "OpenAI API key for this course", required: false) source_chapter(:body, :number, "Default source chapter", required: true) source_variant(:body, Schema.ref(:SourceVariant), "Default source variant name", @@ -97,6 +99,8 @@ defmodule CadetWeb.CoursesController do enable_achievements(:boolean, "Enable achievements", required: true) enable_sourcecast(:boolean, "Enable sourcecast", required: true) enable_stories(:boolean, "Enable stories", required: true) + enable_llm_grading(:boolean, "Enable LLM grading", required: false) + llm_api_key(:string, "OpenAI API key for this course", required: false) source_chapter(:integer, "Source Chapter number from 1 to 4", required: true) source_variant(Schema.ref(:SourceVariant), "Source Variant name", required: true) module_help_text(:string, "Module help text", required: true) @@ -111,6 +115,8 @@ defmodule CadetWeb.CoursesController do enable_achievements: true, enable_sourcecast: true, enable_stories: false, + enable_llm_grading: false, + llm_api_key: "sk-1234567890", source_chapter: 1, source_variant: "default", module_help_text: "Help text", diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex new file mode 100644 index 000000000..1c1503669 --- /dev/null +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -0,0 +1,442 @@ +defmodule CadetWeb.AICodeAnalysisController do + use CadetWeb, :controller + use PhoenixSwagger + require HTTPoison + require Logger + + alias Cadet.{Assessments, AIComments, Courses} + + @openai_api_url "https://api.openai.com/v1/chat/completions" + @model "gpt-4o" + # To set whether LLM grading is enabled across Source Academy + @default_llm_grading false + + # For logging outputs to both database and file + defp log_comment(submission_id, question_id, raw_prompt, answers_json, response, error \\ nil) do + # Log to database + attrs = %{ + submission_id: submission_id, + question_id: question_id, + raw_prompt: raw_prompt, + answers_json: answers_json, + response: response, + error: error, + inserted_at: NaiveDateTime.utc_now() + } + + # Check if a comment already exists for the given submission_id and question_id + case AIComments.get_latest_ai_comment(submission_id, question_id) do + nil -> + # If no existing comment, create a new one + case AIComments.create_ai_comment(attrs) do + {:ok, comment} -> + {:ok, comment} + + {:error, changeset} -> + Logger.error("Failed to log AI comment to database: #{inspect(changeset.errors)}") + {:error, changeset} + end + + existing_comment -> + # Convert the existing comment struct to a map before merging + updated_attrs = Map.merge(Map.from_struct(existing_comment), attrs) + + case AIComments.update_ai_comment(existing_comment.id, updated_attrs) do + {:ok, updated_comment} -> + {:ok, updated_comment} + + {:error, changeset} -> + Logger.error("Failed to update AI comment in database: #{inspect(changeset.errors)}") + {:error, changeset} + end + end + end + + @doc """ + Fetches the question details and answers based on submissionid and questionid and generates AI-generated comments. + """ + def generate_ai_comments(conn, %{ + "submissionid" => submission_id, + "questionid" => question_id, + "course_id" => course_id + }) + when is_ecto_id(submission_id) do + # Check if LLM grading is enabled for this course (default to @default_llm_grading if nil) + case Courses.get_course_config(course_id) do + {:ok, course} -> + if course.enable_llm_grading || @default_llm_grading do + Logger.info("LLM Api key: #{course.llm_api_key}") + # Get API key from course config or fall back to environment variable + decrypted_api_key = decrypt_llm_api_key(course.llm_api_key) + api_key = decrypted_api_key || Application.get_env(:openai, :api_key) + + if is_nil(api_key) do + conn + |> put_status(:internal_server_error) + |> json(%{"error" => "No OpenAI API key configured"}) + else + case Assessments.get_answers_in_submission(submission_id, question_id) do + {:ok, {answers, _assessment}} -> + analyze_code(conn, answers, submission_id, question_id, api_key) + + {:error, {status, message}} -> + conn + |> put_status(status) + |> text(message) + end + end + else + conn + |> put_status(:forbidden) + |> json(%{"error" => "LLM grading is not enabled for this course"}) + end + + {:error, {status, message}} -> + conn + |> put_status(status) + |> text(message) + end + end + + defp transform_answers(answers) do + Enum.map(answers, fn answer -> + %{ + id: answer.id, + comments: answer.comments, + autograding_status: answer.autograding_status, + autograding_results: answer.autograding_results, + code: answer.answer["code"], + question_id: answer.question_id, + question_content: answer.question["content"] + } + end) + end + + defp format_answers(json_string) do + {:ok, answers} = Jason.decode(json_string) + + Enum.map_join(answers, "\n\n", &format_answer/1) + end + + defp format_answer(answer) do + """ + **Question ID: #{answer["question"]["id"] || "N/A"}** + + **Question:** + #{answer["question"]["content"] || "N/A"} + + **Solution:** + ``` + #{answer["question"]["solution"] || "N/A"} + ``` + + **Answer:** + ``` + #{answer["answer"]["code"] || "N/A"} + ``` + + **Autograding Status:** #{answer["autograding_status"] || "N/A"} + **Autograding Results:** #{format_autograding_results(answer["autograding_results"])} + **Comments:** #{answer["comments"] || "None"} + """ + end + + defp format_autograding_results(nil), do: "N/A" + + defp format_autograding_results(results) when is_list(results) do + Enum.map_join(results, "; ", fn result -> + "Error: #{result["errorMessage"] || "N/A"}, Type: #{result["errorType"] || "N/A"}" + end) + end + + defp format_autograding_results(results), do: inspect(results) + + defp analyze_code(conn, answers, submission_id, question_id, api_key) do + answers_json = + answers + |> Enum.map(fn answer -> + question_data = + if answer.question do + %{ + id: answer.question_id, + content: Map.get(answer.question.question, "content"), + solution: Map.get(answer.question.question, "solution"), + llm_prompt: Map.get(answer.question.question, "llm_prompt") + } + else + %{ + id: nil, + content: nil, + llm_prompt: nil + } + end + + answer + |> Map.from_struct() + |> Map.take([ + :id, + :comments, + :autograding_status, + :autograding_results, + :answer + ]) + |> Map.put(:question, question_data) + end) + |> Jason.encode!() + |> format_answers() + + raw_prompt = """ + The code below is written in Source, a variant of JavaScript that comes with a rich set of built-in constants and functions. Below is a summary of some key built-in entities available in Source: + + Constants: + - Infinity: The special number value representing infinity. + - NaN: The special number value for "not a number." + - undefined: The special value for an undefined variable. + - math_PI: The constant π (approximately 3.14159). + - math_E: Euler's number (approximately 2.71828). + + Functions: + - __access_export__(exports, lookup_name): Searches for a name in an exports data structure. + - accumulate(f, initial, xs): Reduces a list by applying a binary function from right-to-left. + - append(xs, ys): Appends list ys to the end of list xs. + - char_at(s, i): Returns the character at index i of string s. + - display(v, s): Displays value v (optionally preceded by string s) in the console. + - filter(pred, xs): Returns a new list with elements of xs that satisfy the predicate pred. + - for_each(f, xs): Applies function f to each element of the list xs. + - get_time(): Returns the current time in milliseconds. + - is_list(xs): Checks whether xs is a proper list. + - length(xs): Returns the number of elements in list xs. + - list(...): Constructs a list from the provided values. + - map(f, xs): Applies function f to each element of list xs. + - math_abs(x): Returns the absolute value of x. + - math_ceil(x): Rounds x up to the nearest integer. + - math_floor(x): Rounds x down to the nearest integer. + - pair(x, y): A primitive function that makes a pair whose head (first component) is x and whose tail (second component) is y. + - head(xs): Returns the first element of pair xs. + - tail(xs): Returns the second element of pair xs. + - math_random(): Returns a random number between 0 (inclusive) and 1 (exclusive). + + (For a full list of built-in functions and constants, refer to the Source documentation.) + + Analyze the following submitted answers and provide detailed feedback on correctness, readability, efficiency, and possible improvements. Your evaluation should consider both standard JavaScript features and the additional built-in functions unique to Source. + + Provide between 3 and 5 concise comment suggestions, each under 200 words. + + Your output must include only the comment suggestions, separated exclusively by triple pipes ("|||") with no spaces before or after the pipes, and without any additional formatting, bullet points, or extra text. + + Comments and documentation in the code are not necessary for the code, do not penalise based on that, do not suggest to add comments as well. + + Follow the XP scoring guideline provided below in the question prompt, do not be too harsh! + + For example: "This is a good answer.|||This is a bad answer.|||This is a great answer." + """ + + # Get the llm_prompt from the first answer's question + llm_prompt = + answers + |> List.first() + |> Map.get(:question) + |> Map.get(:question) + |> Map.get("llm_prompt") + + # Combine prompts if llm_prompt exists + prompt = + if llm_prompt && llm_prompt != "" do + raw_prompt <> "Additional Instructions:\n\n" <> llm_prompt <> "\n\n" <> answers_json + else + raw_prompt <> "\n" <> answers_json + end + + input = + %{ + model: @model, + messages: [ + %{role: "system", content: "You are an expert software engineer and educator."}, + %{role: "user", content: prompt} + ], + temperature: 0.5 + } + |> Jason.encode!() + + headers = [ + {"Authorization", "Bearer #{api_key}"}, + {"Content-Type", "application/json"} + ] + + case HTTPoison.post(@openai_api_url, input, headers, timeout: 60_000, recv_timeout: 60_000) do + {:ok, %HTTPoison.Response{status_code: 200, body: body}} -> + case Jason.decode(body) do + {:ok, %{"choices" => [%{"message" => %{"content" => response}}]}} -> + log_comment(submission_id, question_id, prompt, answers_json, response) + comments_list = String.split(response, "|||") + + filtered_comments = + Enum.filter(comments_list, fn comment -> + String.trim(comment) != "" + end) + + json(conn, %{"comments" => filtered_comments}) + + {:error, _} -> + log_comment( + submission_id, + question_id, + prompt, + answers_json, + nil, + "Failed to parse response from OpenAI API" + ) + + json(conn, %{"error" => "Failed to parse response from OpenAI API"}) + end + + {:ok, %HTTPoison.Response{status_code: status, body: body}} -> + log_comment( + submission_id, + question_id, + prompt, + answers_json, + nil, + "API request failed with status #{status}" + ) + + conn + |> put_status(:internal_server_error) + |> json(%{"error" => "API request failed with status #{status}: #{body}"}) + + {:error, %HTTPoison.Error{reason: reason}} -> + log_comment(submission_id, question_id, prompt, answers_json, nil, reason) + json(conn, %{"error" => "HTTP request error: #{inspect(reason)}"}) + end + end + + @doc """ + Saves the final comment chosen for a submission. + """ + def save_final_comment(conn, %{ + "submissionid" => submission_id, + "questionid" => question_id, + "comment" => comment + }) do + case AIComments.update_final_comment(submission_id, question_id, comment) do + {:ok, _updated_comment} -> + json(conn, %{"status" => "success"}) + + {:error, changeset} -> + conn + |> put_status(:unprocessable_entity) + |> json(%{"error" => "Failed to save final comment"}) + end + end + + @doc """ + Saves the chosen comments for a submission and question. + Accepts an array of comments in the request body. + """ + def save_chosen_comments(conn, %{ + "submissionid" => submission_id, + "questionid" => question_id, + "comments" => comments + }) do + case AIComments.update_chosen_comments(submission_id, question_id, comments) do + {:ok, _updated_comment} -> + json(conn, %{"status" => "success"}) + + {:error, changeset} -> + conn + |> put_status(:unprocessable_entity) + |> json(%{"error" => "Failed to save chosen comments"}) + end + end + + swagger_path :generate_ai_comments do + post("/courses/{courseId}/admin/generate-comments/{submissionId}/{questionId}") + + summary("Generate AI comments for a given submission.") + + security([%{JWT: []}]) + + consumes("application/json") + produces("application/json") + + parameters do + courseId(:path, :integer, "course id", required: true) + submissionId(:path, :integer, "submission id", required: true) + questionId(:path, :integer, "question id", required: true) + end + + response(200, "OK", Schema.ref(:GenerateAIComments)) + response(400, "Invalid or missing parameter(s) or submission and/or question not found") + response(401, "Unauthorized") + response(403, "Forbidden") + response(403, "LLM grading is not enabled for this course") + end + + swagger_path :save_final_comment do + post("/courses/{courseId}/admin/save-final-comment/{submissionId}/{questionId}") + + summary("Save the final comment chosen for a submission.") + + security([%{JWT: []}]) + + consumes("application/json") + produces("application/json") + + parameters do + submissionId(:path, :integer, "submission id", required: true) + questionId(:path, :integer, "question id", required: true) + comment(:body, :string, "The final comment to save", required: true) + end + + response(200, "OK", Schema.ref(:SaveFinalComment)) + response(400, "Invalid or missing parameter(s)") + response(401, "Unauthorized") + response(403, "Forbidden") + end + + def swagger_definitions do + %{ + GenerateAIComments: + swagger_schema do + properties do + comments(:string, "AI-generated comments on the submission answers") + end + end, + SaveFinalComment: + swagger_schema do + properties do + status(:string, "Status of the operation") + end + end + } + end + + defp decrypt_llm_api_key(nil), do: nil + + defp decrypt_llm_api_key(encrypted_key) do + case Application.get_env(:openai, :encryption_key) do + secret when is_binary(secret) and byte_size(secret) >= 16 -> + key = binary_part(secret, 0, min(32, byte_size(secret))) + + case Base.decode64(encrypted_key) do + {:ok, decoded} -> + iv = binary_part(decoded, 0, 16) + tag = binary_part(decoded, 16, 16) + ciphertext = binary_part(decoded, 32, byte_size(decoded) - 32) + + case :crypto.crypto_one_time_aead(:aes_gcm, key, iv, ciphertext, "", tag, false) do + plain_text when is_binary(plain_text) -> plain_text + _ -> nil + end + + _ -> + Logger.error("Failed to decode encrypted key") + nil + end + + _ -> + Logger.error("Encryption key not configured properly") + nil + end + end +end diff --git a/lib/cadet_web/helpers/assessments_helpers.ex b/lib/cadet_web/helpers/assessments_helpers.ex index 967df1131..7e03e5d83 100644 --- a/lib/cadet_web/helpers/assessments_helpers.ex +++ b/lib/cadet_web/helpers/assessments_helpers.ex @@ -177,6 +177,7 @@ defmodule CadetWeb.AssessmentsHelpers do prepend: "prepend", solutionTemplate: "template", postpend: "postpend", + llm_prompt: "llm_prompt", testcases: build_testcases(all_testcases?) }) diff --git a/lib/cadet_web/router.ex b/lib/cadet_web/router.ex index 1ee304c01..c64631cf8 100644 --- a/lib/cadet_web/router.ex +++ b/lib/cadet_web/router.ex @@ -201,12 +201,30 @@ defmodule CadetWeb.Router do post("/grading/:submissionid/autograde", AdminGradingController, :autograde_submission) post("/grading/:submissionid/:questionid", AdminGradingController, :update) + post( + "/generate-comments/:submissionid/:questionid", + AICodeAnalysisController, + :generate_ai_comments + ) + post( "/grading/:submissionid/:questionid/autograde", AdminGradingController, :autograde_answer ) + post( + "/save-final-comment/:submissionid/:questionid", + AICodeAnalysisController, + :save_final_comment + ) + + post( + "/save-chosen-comments/:submissionid/:questionid", + AICodeAnalysisController, + :save_chosen_comments + ) + get("/users", AdminUserController, :index) get("/users/teamformation", AdminUserController, :get_students) put("/users", AdminUserController, :upsert_users_and_groups) diff --git a/lib/cadet_web/views/courses_view.ex b/lib/cadet_web/views/courses_view.ex index a6ae9c4fa..819f4aebd 100644 --- a/lib/cadet_web/views/courses_view.ex +++ b/lib/cadet_web/views/courses_view.ex @@ -12,6 +12,8 @@ defmodule CadetWeb.CoursesView do enableAchievements: :enable_achievements, enableSourcecast: :enable_sourcecast, enableStories: :enable_stories, + enableLlmGrading: :enable_llm_grading, + llmApiKey: :llm_api_key, sourceChapter: :source_chapter, sourceVariant: :source_variant, moduleHelpText: :module_help_text, diff --git a/priv/repo/migrations/20240320000000_add_llm_grading_access.exs b/priv/repo/migrations/20240320000000_add_llm_grading_access.exs new file mode 100644 index 000000000..c5f337eee --- /dev/null +++ b/priv/repo/migrations/20240320000000_add_llm_grading_access.exs @@ -0,0 +1,15 @@ +defmodule Cadet.Repo.Migrations.AddLLMGradingAccess do + use Ecto.Migration + + def up do + alter table(:courses) do + add(:enable_llm_grading, :boolean, null: true) + end + end + + def down do + alter table(:courses) do + remove(:enable_llm_grading) + end + end +end diff --git a/priv/repo/migrations/20240320000001_add_llm_api_key_to_courses.exs b/priv/repo/migrations/20240320000001_add_llm_api_key_to_courses.exs new file mode 100644 index 000000000..e752889b7 --- /dev/null +++ b/priv/repo/migrations/20240320000001_add_llm_api_key_to_courses.exs @@ -0,0 +1,15 @@ +defmodule Cadet.Repo.Migrations.AddLlmApiKeyToCourses do + use Ecto.Migration + + def up do + alter table(:courses) do + add(:llm_api_key, :string, null: true) + end + end + + def down do + alter table(:courses) do + remove(:llm_api_key) + end + end +end diff --git a/priv/repo/migrations/20250220103623_create_ai_comments.exs b/priv/repo/migrations/20250220103623_create_ai_comments.exs new file mode 100644 index 000000000..6338dccd8 --- /dev/null +++ b/priv/repo/migrations/20250220103623_create_ai_comments.exs @@ -0,0 +1,20 @@ +defmodule Cadet.Repo.Migrations.CreateAiCommentLogs do + use Ecto.Migration + + def change do + create table(:ai_comment_logs) do + add(:submission_id, :integer, null: false) + add(:question_id, :integer, null: false) + add(:raw_prompt, :text, null: false) + add(:answers_json, :text, null: false) + add(:response, :text) + add(:error, :text) + add(:comment_chosen, :text) + add(:final_comment, :text) + timestamps() + end + + create(index(:ai_comment_logs, [:submission_id])) + create(index(:ai_comment_logs, [:question_id])) + end +end diff --git a/priv/repo/migrations/20250402030934_increase_llm_api_key_length.exs b/priv/repo/migrations/20250402030934_increase_llm_api_key_length.exs new file mode 100644 index 000000000..ec3318eda --- /dev/null +++ b/priv/repo/migrations/20250402030934_increase_llm_api_key_length.exs @@ -0,0 +1,9 @@ +defmodule Cadet.Repo.Migrations.IncreaseLlmApiKeyLength do + use Ecto.Migration + + def change do + alter table(:courses) do + modify(:llm_api_key, :text) + end + end +end diff --git a/priv/repo/migrations/20250406053008_update_comment_chosen_to_array.exs b/priv/repo/migrations/20250406053008_update_comment_chosen_to_array.exs new file mode 100644 index 000000000..751cf9b7d --- /dev/null +++ b/priv/repo/migrations/20250406053008_update_comment_chosen_to_array.exs @@ -0,0 +1,17 @@ +defmodule Cadet.Repo.Migrations.UpdateCommentChosenToArray do + use Ecto.Migration + + def change do + alter table(:ai_comment_logs) do + add(:comment_chosen_temp, {:array, :string}, default: []) + end + + execute("UPDATE ai_comment_logs SET comment_chosen_temp = ARRAY[comment_chosen]") + + alter table(:ai_comment_logs) do + remove(:comment_chosen) + end + + rename(table(:ai_comment_logs), :comment_chosen_temp, to: :comment_chosen) + end +end diff --git a/test/cadet_web/controllers/ai_code_analysis_controller_test.exs b/test/cadet_web/controllers/ai_code_analysis_controller_test.exs new file mode 100644 index 000000000..c17e57174 --- /dev/null +++ b/test/cadet_web/controllers/ai_code_analysis_controller_test.exs @@ -0,0 +1,193 @@ +defmodule CadetWeb.AICodeAnalysisControllerTest do + use CadetWeb.ConnCase + alias Cadet.AIComments + alias Cadet.AIComments.AIComment + + setup do + # Clean up test files before each test + log_file = "log/ai_comments.csv" + File.rm(log_file) + :ok + end + + describe "generate_ai_comments" do + test "successfully logs comments to both database and file", %{conn: conn} do + # Test data + submission_id = 123 + question_id = 456 + raw_prompt = "Test prompt" + answers_json = ~s({"test": "data"}) + mock_response = "Comment 1|||Comment 2|||Comment 3" + + # Make the API call + response = + conn + |> post( + Routes.ai_code_analysis_path(conn, :generate_ai_comments, submission_id, question_id) + ) + |> json_response(200) + + # Verify database entry + comments = Repo.all(AIComment) + assert length(comments) > 0 + latest_comment = List.first(comments) + assert latest_comment.submission_id == submission_id + assert latest_comment.question_id == question_id + assert latest_comment.raw_prompt != nil + assert latest_comment.answers_json != nil + + # Verify CSV file + log_file = "log/ai_comments.csv" + assert File.exists?(log_file) + file_content = File.read!(log_file) + + # Check if CSV contains the required data + assert file_content =~ Integer.to_string(submission_id) + assert file_content =~ Integer.to_string(question_id) + end + + test "logs error when API call fails", %{conn: conn} do + # Test data with invalid submission_id to trigger error + submission_id = -1 + question_id = 456 + + # Make the API call that should fail + response = + conn + |> post( + Routes.ai_code_analysis_path(conn, :generate_ai_comments, submission_id, question_id) + ) + |> json_response(400) + + # Verify error is logged in database + comments = Repo.all(AIComment) + assert length(comments) > 0 + error_log = List.first(comments) + assert error_log.error != nil + assert error_log.submission_id == submission_id + assert error_log.question_id == question_id + + # Verify error is logged in CSV + log_file = "log/ai_comments.csv" + assert File.exists?(log_file) + file_content = File.read!(log_file) + assert file_content =~ Integer.to_string(submission_id) + assert file_content =~ Integer.to_string(question_id) + assert file_content =~ "error" + end + end + + describe "save_final_comment" do + test "successfully saves final comment", %{conn: conn} do + # First create a comment entry + submission_id = 123 + question_id = 456 + raw_prompt = "Test prompt" + answers_json = ~s({"test": "data"}) + response = "Comment 1|||Comment 2|||Comment 3" + + {:ok, _comment} = + AIComments.create_ai_comment(%{ + submission_id: submission_id, + question_id: question_id, + raw_prompt: raw_prompt, + answers_json: answers_json, + response: response + }) + + # Now save the final comment + final_comment = "This is the chosen final comment" + + response = + conn + |> post( + Routes.ai_code_analysis_path(conn, :save_final_comment, submission_id, question_id), + %{ + comment: final_comment + } + ) + |> json_response(200) + + assert response["status"] == "success" + + # Verify the comment was saved + comment = Repo.get_by(AIComment, submission_id: submission_id, question_id: question_id) + assert comment.final_comment == final_comment + end + + test "returns error when no comment exists", %{conn: conn} do + submission_id = 999 + question_id = 888 + final_comment = "This comment should not be saved" + + response = + conn + |> post( + Routes.ai_code_analysis_path(conn, :save_final_comment, submission_id, question_id), + %{ + comment: final_comment + } + ) + |> json_response(422) + + assert response["error"] == "Failed to save final comment" + end + end + + describe "save_chosen_comments" do + test "successfully saves chosen comments", %{conn: conn} do + # First create a comment entry + submission_id = 123 + question_id = 456 + raw_prompt = "Test prompt" + answers_json = ~s({"test": "data"}) + response = "Comment 1|||Comment 2|||Comment 3" + + {:ok, _comment} = + AIComments.create_ai_comment(%{ + submission_id: submission_id, + question_id: question_id, + raw_prompt: raw_prompt, + answers_json: answers_json, + response: response + }) + + # Now save the chosen comments + chosen_comments = ["Comment 1", "Comment 2"] + + response = + conn + |> post( + Routes.ai_code_analysis_path(conn, :save_chosen_comments, submission_id, question_id), + %{ + comments: chosen_comments + } + ) + |> json_response(200) + + assert response["status"] == "success" + + # Verify the chosen comments were saved + comment = Repo.get_by(AIComment, submission_id: submission_id, question_id: question_id) + assert comment.comment_chosen == chosen_comments + end + + test "returns error when no comment exists", %{conn: conn} do + submission_id = 999 + question_id = 888 + chosen_comments = ["Comment 1", "Comment 2"] + + response = + conn + |> post( + Routes.ai_code_analysis_path(conn, :save_chosen_comments, submission_id, question_id), + %{ + comments: chosen_comments + } + ) + |> json_response(422) + + assert response["error"] == "Failed to save chosen comments" + end + end +end