From 5cd6ac66206079d08b791e8a1bfa00df1071cc04 Mon Sep 17 00:00:00 2001 From: Arul Date: Sun, 2 Feb 2025 17:18:34 +0800 Subject: [PATCH 01/20] feat: v1 of AI-generated comments --- lib/cadet/assessments/assessments.ex | 11 +- .../controllers/generate_ai_comments.ex | 172 ++++++++++++++++++ lib/cadet_web/router.ex | 6 + 3 files changed, 187 insertions(+), 2 deletions(-) create mode 100644 lib/cadet_web/controllers/generate_ai_comments.ex diff --git a/lib/cadet/assessments/assessments.ex b/lib/cadet/assessments/assessments.ex index 244ebc742..70307d85d 100644 --- a/lib/cadet/assessments/assessments.ex +++ b/lib/cadet/assessments/assessments.ex @@ -2289,8 +2289,8 @@ defmodule Cadet.Assessments do @spec get_answers_in_submission(integer() | String.t()) :: {:ok, {[Answer.t()], Assessment.t()}} | {:error, {:bad_request, String.t()}} - def get_answers_in_submission(id) when is_ecto_id(id) do - answer_query = + def get_answers_in_submission(id, question_id \\ nil) when is_ecto_id(id) do + base_query = Answer |> where(submission_id: ^id) |> join(:inner, [a], q in assoc(a, :question)) @@ -2312,6 +2312,13 @@ defmodule Cadet.Assessments do {s, student: {st, user: u}, team: {t, team_members: {tm, student: {tms, user: tmu}}}} ) + answer_query = + if is_nil(question_id) do + base_query + else + base_query |> where(question_id: ^question_id) + end + answers = answer_query |> Repo.all() diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex new file mode 100644 index 000000000..85ede4a1c --- /dev/null +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -0,0 +1,172 @@ +defmodule CadetWeb.AICodeAnalysisController do + use CadetWeb, :controller + use PhoenixSwagger + require HTTPoison + + alias Cadet.Assessments + + @openai_api_url "https://api.groq.com/openai/v1/chat/completions" + @model "llama3-8b-8192" + @api_key "x" + + @doc """ + Fetches the question details and answers based on submissionid and questionid and generates AI-generated comments. + """ + def generate_ai_comments(conn, %{"submissionid" => submission_id, "questionid" => question_id}) + when is_ecto_id(submission_id) do + case Assessments.get_answers_in_submission(submission_id, question_id) do + {:ok, {answers, _assessment}} -> + analyze_code(conn, answers) + + {:error, {status, message}} -> + conn + |> put_status(status) + |> text(message) + end + end + + defp transform_answers(answers) do + Enum.map(answers, fn answer -> + %{ + id: answer.id, + comments: answer.comments, + autograding_status: answer.autograding_status, + autograding_results: answer.autograding_results, + code: answer.answer["code"], + question_id: answer.question_id, + question_content: answer.question["content"] + } + end) + end + + defp analyze_code(conn, answers) do + # Convert each struct into a map and select only the required fields + answers_json = + answers + |> Enum.map(fn answer -> + question_data = + if answer.question do + %{ + id: answer.question_id, + content: Map.get(answer.question.question, "content") + } + else + %{ + id: nil, + content: nil + } + end + + answer + |> Map.from_struct() + |> Map.take([ + :id, + :comments, + :autograding_status, + :autograding_results, + :answer + ]) + |> Map.put(:question, question_data) + end) + |> Jason.encode!() + + prompt = """ + The code below was written in JavaScript. + + Analyze the following submitted answers and provide feedback on correctness, readability, efficiency, and improvements: + + Provide minimum 3 comment suggestions and maximum 5 comment suggestions. Keep each comment suggestion concise and specific, less than 100 words. + + Only provide your comment suggestions in the output and nothing else. + + Your output should be in the following format. + + DO NOT start the output with |||. Separate each suggestion using |||. + + DO NOT add spaces before or after the |||. + + Only provide the comment suggestions and separate each comment suggestion by using triple pipes ("|||"). + + For example: "This is a good answer.|||This is a bad answer.|||This is a great answer." + + Do not provide any other information in the output, like "Here are the comment suggestions for the first answer" + + Do not include any bullet points, number lists, or any other formatting in your output. Just plain text comments, separated by triple pipes. + + #{answers_json} + """ + + body = + %{ + model: @model, + messages: [ + %{role: "system", content: "You are an expert software engineer and educator."}, + %{role: "user", content: prompt} + ], + temperature: 0.5 + } + |> Jason.encode!() + + headers = [ + {"Authorization", "Bearer #{@api_key}"}, + {"Content-Type", "application/json"} + ] + + case HTTPoison.post(@openai_api_url, body, headers) do + {:ok, %HTTPoison.Response{status_code: 200, body: body}} -> + case Jason.decode(body) do + {:ok, %{"choices" => [%{"message" => %{"content" => response}}]}} -> + IO.inspect(response, label: "DEBUG: Raw AI Response") + comments_list = String.split(response, "|||") + + filtered_comments = + Enum.filter(comments_list, fn comment -> + String.trim(comment) != "" + end) + + json(conn, %{"comments" => filtered_comments}) + + {:error, _} -> + json(conn, %{"error" => "Failed to parse response from OpenAI API"}) + end + + {:ok, %HTTPoison.Response{status_code: status, body: body}} -> + json(conn, %{"error" => "API request failed with status #{status}: #{body}"}) + + {:error, %HTTPoison.Error{reason: reason}} -> + json(conn, %{"error" => "HTTP request error: #{inspect(reason)}"}) + end + end + + swagger_path :generate_ai_comments do + post("/courses/{courseId}/admin/generate-comments/{submissionId}/{questionId}") + + summary("Generate AI comments for a given submission.") + + security([%{JWT: []}]) + + consumes("application/json") + produces("application/json") + + parameters do + submissionId(:path, :integer, "submission id", required: true) + questionId(:path, :integer, "question id", required: true) + end + + response(200, "OK", Schema.ref(:GenerateAIComments)) + response(400, "Invalid or missing parameter(s) or submission and/or question not found") + response(401, "Unauthorized") + response(403, "Forbidden") + end + + def swagger_definitions do + %{ + GenerateAIComments: + swagger_schema do + properties do + comments(:string, "AI-generated comments on the submission answers") + end + end + } + end +end diff --git a/lib/cadet_web/router.ex b/lib/cadet_web/router.ex index 1ee304c01..3e8c1a155 100644 --- a/lib/cadet_web/router.ex +++ b/lib/cadet_web/router.ex @@ -201,6 +201,12 @@ defmodule CadetWeb.Router do post("/grading/:submissionid/autograde", AdminGradingController, :autograde_submission) post("/grading/:submissionid/:questionid", AdminGradingController, :update) + post( + "/generate-comments/:submissionid/:questionid", + AICodeAnalysisController, + :generate_ai_comments + ) + post( "/grading/:submissionid/:questionid/autograde", AdminGradingController, From 853ba847208e2689d27a4e42af37f0dbbe27e6d7 Mon Sep 17 00:00:00 2001 From: Arul Date: Thu, 13 Feb 2025 11:35:14 +0800 Subject: [PATCH 02/20] feat: added logging of inputs and outputs --- .gitignore | 3 + .../controllers/generate_ai_comments.ex | 116 ++++++++++-------- 2 files changed, 71 insertions(+), 48 deletions(-) diff --git a/.gitignore b/.gitignore index c6f53010c..80b3e3026 100644 --- a/.gitignore +++ b/.gitignore @@ -118,3 +118,6 @@ erl_crash.dump # Generated lexer /src/source_lexer.erl + +# Ignore log files +/log \ No newline at end of file diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index 85ede4a1c..ac669a4f3 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -2,27 +2,46 @@ defmodule CadetWeb.AICodeAnalysisController do use CadetWeb, :controller use PhoenixSwagger require HTTPoison + require Logger alias Cadet.Assessments - @openai_api_url "https://api.groq.com/openai/v1/chat/completions" - @model "llama3-8b-8192" - @api_key "x" + @openai_api_url "https://api.openai.com/v1/chat/completions" + @model "gpt-4o" + @api_key Application.get_env(:openai, :api_key) + + + # For logging outputs to a file + defp log_to_csv(submission_id, question_id, input, student_submission, output, error \\ nil) do + log_file = "log/ai_comments.csv" + File.mkdir_p!("log") + + timestamp = NaiveDateTime.utc_now() |> NaiveDateTime.to_string() + input_str = Jason.encode!(input) |> String.replace("\"", "\"\"") + student_submission_str = Jason.encode!(student_submission) |> String.replace("\"", "\"\"") + output_str = Jason.encode!(output) |> String.replace("\"", "\"\"") + error_str = if is_nil(error), do: "", else: Jason.encode!(error) |> String.replace("\"", "\"\"") + + csv_row = "\"#{timestamp}\",\"#{submission_id}\",\"#{question_id}\",\"#{input_str}\",\"#{student_submission_str}\",\"#{output_str}\",\"#{error_str}\"\n" + + File.write!(log_file, csv_row, [:append]) + end + @doc """ Fetches the question details and answers based on submissionid and questionid and generates AI-generated comments. """ def generate_ai_comments(conn, %{"submissionid" => submission_id, "questionid" => question_id}) - when is_ecto_id(submission_id) do - case Assessments.get_answers_in_submission(submission_id, question_id) do - {:ok, {answers, _assessment}} -> - analyze_code(conn, answers) - - {:error, {status, message}} -> - conn - |> put_status(status) - |> text(message) - end + when is_ecto_id(submission_id) do + case Assessments.get_answers_in_submission(submission_id, question_id) do + {:ok, {answers, _assessment}} -> + analyze_code(conn, answers, submission_id, question_id) + + {:error, {status, message}} -> + conn + |> put_status(status) + |> text(message) + end end defp transform_answers(answers) do @@ -39,8 +58,7 @@ defmodule CadetWeb.AICodeAnalysisController do end) end - defp analyze_code(conn, answers) do - # Convert each struct into a map and select only the required fields + defp analyze_code(conn, answers, submission_id, question_id) do answers_json = answers |> Enum.map(fn answer -> @@ -48,7 +66,8 @@ defmodule CadetWeb.AICodeAnalysisController do if answer.question do %{ id: answer.question_id, - content: Map.get(answer.question.question, "content") + content: Map.get(answer.question.question, "content"), + solution: Map.get(answer.question.question, "solution") } else %{ @@ -56,7 +75,6 @@ defmodule CadetWeb.AICodeAnalysisController do content: nil } end - answer |> Map.from_struct() |> Map.take([ @@ -64,76 +82,78 @@ defmodule CadetWeb.AICodeAnalysisController do :comments, :autograding_status, :autograding_results, - :answer + :answer, ]) |> Map.put(:question, question_data) end) |> Jason.encode!() - prompt = """ - The code below was written in JavaScript. + raw_prompt = """ + The code below was written in JavaScript. - Analyze the following submitted answers and provide feedback on correctness, readability, efficiency, and improvements: + Analyze the following submitted answers and provide feedback on correctness, readability, efficiency, and improvements: - Provide minimum 3 comment suggestions and maximum 5 comment suggestions. Keep each comment suggestion concise and specific, less than 100 words. + Provide minimum 3 comment suggestions and maximum 5 comment suggestions. Keep each comment suggestion concise and specific, less than 200 words. - Only provide your comment suggestions in the output and nothing else. + Only provide your comment suggestions in the output and nothing else. - Your output should be in the following format. + Your output should be in the following format. - DO NOT start the output with |||. Separate each suggestion using |||. + DO NOT start the output with |||. Separate each suggestion using |||. - DO NOT add spaces before or after the |||. + DO NOT add spaces before or after the |||. - Only provide the comment suggestions and separate each comment suggestion by using triple pipes ("|||"). + Only provide the comment suggestions and separate each comment suggestion by using triple pipes ("|||"). - For example: "This is a good answer.|||This is a bad answer.|||This is a great answer." + For example: "This is a good answer.|||This is a bad answer.|||This is a great answer." - Do not provide any other information in the output, like "Here are the comment suggestions for the first answer" + Do not provide any other information in the output, like "Here are the comment suggestions for the first answer" - Do not include any bullet points, number lists, or any other formatting in your output. Just plain text comments, separated by triple pipes. + Do not include any bullet points, number lists, or any other formatting in your output. Just plain text comments, separated by triple pipes ("|||"). + """ - #{answers_json} - """ + prompt = raw_prompt <> "\n" <> answers_json - body = - %{ - model: @model, - messages: [ - %{role: "system", content: "You are an expert software engineer and educator."}, - %{role: "user", content: prompt} - ], - temperature: 0.5 - } - |> Jason.encode!() + + input = %{ + model: @model, + messages: [ + %{role: "system", content: "You are an expert software engineer and educator."}, + %{role: "user", content: prompt} + ], + temperature: 0.5 + } |> Jason.encode!() headers = [ {"Authorization", "Bearer #{@api_key}"}, {"Content-Type", "application/json"} ] - case HTTPoison.post(@openai_api_url, body, headers) do + + case HTTPoison.post(@openai_api_url, input, headers) do {:ok, %HTTPoison.Response{status_code: 200, body: body}} -> case Jason.decode(body) do {:ok, %{"choices" => [%{"message" => %{"content" => response}}]}} -> - IO.inspect(response, label: "DEBUG: Raw AI Response") + log_to_csv(submission_id, question_id, raw_prompt, answers_json, response) comments_list = String.split(response, "|||") - filtered_comments = - Enum.filter(comments_list, fn comment -> - String.trim(comment) != "" - end) + filtered_comments = Enum.filter(comments_list, fn comment -> + String.trim(comment) != "" + end) json(conn, %{"comments" => filtered_comments}) {:error, _} -> + log_to_csv(submission_id, question_id, raw_prompt, answers_json, nil, "Failed to parse response from OpenAI API") json(conn, %{"error" => "Failed to parse response from OpenAI API"}) end {:ok, %HTTPoison.Response{status_code: status, body: body}} -> + log_to_csv(submission_id, question_id, raw_prompt, answers_json, nil, "API request failed with status #{status}") json(conn, %{"error" => "API request failed with status #{status}: #{body}"}) {:error, %HTTPoison.Error{reason: reason}} -> + log_to_csv(submission_id, question_id, raw_prompt, answers_json, nil, reason) json(conn, %{"error" => "HTTP request error: #{inspect(reason)}"}) end end From 4c37d14ed886177aea10318238eca186314c1afe Mon Sep 17 00:00:00 2001 From: Arul Date: Fri, 21 Feb 2025 10:17:04 +0800 Subject: [PATCH 03/20] Update generate_ai_comments.ex --- .../controllers/generate_ai_comments.ex | 56 ++++++++++++------- 1 file changed, 37 insertions(+), 19 deletions(-) diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index ac669a4f3..8f16ad6d2 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -89,27 +89,45 @@ defmodule CadetWeb.AICodeAnalysisController do |> Jason.encode!() raw_prompt = """ - The code below was written in JavaScript. - - Analyze the following submitted answers and provide feedback on correctness, readability, efficiency, and improvements: - - Provide minimum 3 comment suggestions and maximum 5 comment suggestions. Keep each comment suggestion concise and specific, less than 200 words. - - Only provide your comment suggestions in the output and nothing else. - - Your output should be in the following format. - - DO NOT start the output with |||. Separate each suggestion using |||. - - DO NOT add spaces before or after the |||. - - Only provide the comment suggestions and separate each comment suggestion by using triple pipes ("|||"). + The code below is written in Source, a variant of JavaScript that comes with a rich set of built-in constants and functions. Below is a summary of some key built-in entities available in Source: + + Constants: + - Infinity: The special number value representing infinity. + - NaN: The special number value for "not a number." + - undefined: The special value for an undefined variable. + - math_PI: The constant π (approximately 3.14159). + - math_E: Euler's number (approximately 2.71828). + + Functions: + - __access_export__(exports, lookup_name): Searches for a name in an exports data structure. + - accumulate(f, initial, xs): Reduces a list by applying a binary function from right-to-left. + - append(xs, ys): Appends list ys to the end of list xs. + - char_at(s, i): Returns the character at index i of string s. + - display(v, s): Displays value v (optionally preceded by string s) in the console. + - filter(pred, xs): Returns a new list with elements of xs that satisfy the predicate pred. + - for_each(f, xs): Applies function f to each element of the list xs. + - get_time(): Returns the current time in milliseconds. + - is_list(xs): Checks whether xs is a proper list. + - length(xs): Returns the number of elements in list xs. + - list(...): Constructs a list from the provided values. + - map(f, xs): Applies function f to each element of list xs. + - math_abs(x): Returns the absolute value of x. + - math_ceil(x): Rounds x up to the nearest integer. + - math_floor(x): Rounds x down to the nearest integer. + - pair(x, y): A primitive function that makes a pair whose head (first component) is x and whose tail (second component) is y. + - head(xs): Returns the first element of pair xs. + - tail(xs): Returns the second element of pair xs. + - math_random(): Returns a random number between 0 (inclusive) and 1 (exclusive). + + (For a full list of built-in functions and constants, refer to the Source documentation.) + + Analyze the following submitted answers and provide detailed feedback on correctness, readability, efficiency, and possible improvements. Your evaluation should consider both standard JavaScript features and the additional built-in functions unique to Source. + + Provide between 3 and 5 concise comment suggestions, each under 200 words. + + Your output must include only the comment suggestions, separated exclusively by triple pipes ("|||") with no spaces before or after the pipes, and without any additional formatting, bullet points, or extra text. For example: "This is a good answer.|||This is a bad answer.|||This is a great answer." - - Do not provide any other information in the output, like "Here are the comment suggestions for the first answer" - - Do not include any bullet points, number lists, or any other formatting in your output. Just plain text comments, separated by triple pipes ("|||"). """ prompt = raw_prompt <> "\n" <> answers_json From 8192b3de4d06fdf6e569c5831ddc085eb9229124 Mon Sep 17 00:00:00 2001 From: Arul Date: Mon, 17 Mar 2025 17:35:35 +0800 Subject: [PATCH 04/20] feat: function to save outputs to database --- lib/cadet/ai_comments.ex | 50 +++++++ lib/cadet/ai_comments/ai_comment.ex | 23 ++++ .../controllers/generate_ai_comments.ex | 103 ++++++++++++--- .../20250220103623_create_ai_comments.exs | 20 +++ .../ai_code_analysis_controller_test.exs | 124 ++++++++++++++++++ 5 files changed, 304 insertions(+), 16 deletions(-) create mode 100644 lib/cadet/ai_comments.ex create mode 100644 lib/cadet/ai_comments/ai_comment.ex create mode 100644 priv/repo/migrations/20250220103623_create_ai_comments.exs create mode 100644 test/cadet_web/controllers/ai_code_analysis_controller_test.exs diff --git a/lib/cadet/ai_comments.ex b/lib/cadet/ai_comments.ex new file mode 100644 index 000000000..3f42c9004 --- /dev/null +++ b/lib/cadet/ai_comments.ex @@ -0,0 +1,50 @@ +defmodule Cadet.AIComments do + import Ecto.Query + alias Cadet.Repo + alias Cadet.AIComments.AIComment + + @doc """ + Creates a new AI comment log entry. + """ + def create_ai_comment(attrs \\ %{}) do + %AIComment{} + |> AIComment.changeset(attrs) + |> Repo.insert() + end + + @doc """ + Gets an AI comment by ID. + """ + def get_ai_comment!(id), do: Repo.get!(AIComment, id) + + @doc """ + Gets AI comments for a specific submission and question. + """ + def get_ai_comments_for_submission(submission_id, question_id) do + from(c in AIComment, + where: c.submission_id == ^submission_id and c.question_id == ^question_id, + order_by: [desc: c.inserted_at] + ) + |> Repo.all() + end + + @doc """ + Updates the final comment for a specific submission and question. + Returns the most recent comment entry for that submission/question. + """ + def update_final_comment(submission_id, question_id, final_comment) do + from(c in AIComment, + where: c.submission_id == ^submission_id and c.question_id == ^question_id, + order_by: [desc: c.inserted_at], + limit: 1 + ) + |> Repo.one() + |> case do + nil -> {:error, :not_found} + comment -> + comment + |> AIComment.changeset(%{final_comment: final_comment}) + |> Repo.update() + end + end +end diff --git a/lib/cadet/ai_comments/ai_comment.ex b/lib/cadet/ai_comments/ai_comment.ex new file mode 100644 index 000000000..575d1cc10 --- /dev/null +++ b/lib/cadet/ai_comments/ai_comment.ex @@ -0,0 +1,23 @@ +defmodule Cadet.AIComments.AIComment do + use Ecto.Schema + import Ecto.Changeset + + schema "ai_comment_logs" do + field :submission_id, :integer + field :question_id, :integer + field :raw_prompt, :string + field :answers_json, :string + field :response, :string + field :error, :string + field :comment_chosen, :string + field :final_comment, :string + + timestamps() + end + + def changeset(ai_comment, attrs) do + ai_comment + |> cast(attrs, [:submission_id, :question_id, :raw_prompt, :answers_json, :response, :error, :comment_chosen, :final_comment]) + |> validate_required([:submission_id, :question_id, :raw_prompt, :answers_json]) + end +end diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index 8f16ad6d2..a543d5518 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -5,28 +5,51 @@ defmodule CadetWeb.AICodeAnalysisController do require Logger alias Cadet.Assessments + alias Cadet.AIComments @openai_api_url "https://api.openai.com/v1/chat/completions" @model "gpt-4o" @api_key Application.get_env(:openai, :api_key) + # For logging outputs to both database and file + defp log_comment(submission_id, question_id, raw_prompt, answers_json, response, error \\ nil) do + # Log to database + attrs = %{ + submission_id: submission_id, + question_id: question_id, + raw_prompt: raw_prompt, + answers_json: answers_json, + response: response, + error: error, + inserted_at: NaiveDateTime.utc_now() + } - # For logging outputs to a file - defp log_to_csv(submission_id, question_id, input, student_submission, output, error \\ nil) do - log_file = "log/ai_comments.csv" - File.mkdir_p!("log") + case AIComments.create_ai_comment(attrs) do + {:ok, comment} -> {:ok, comment} + {:error, changeset} -> + Logger.error("Failed to log AI comment to database: #{inspect(changeset.errors)}") + {:error, changeset} + end - timestamp = NaiveDateTime.utc_now() |> NaiveDateTime.to_string() - input_str = Jason.encode!(input) |> String.replace("\"", "\"\"") - student_submission_str = Jason.encode!(student_submission) |> String.replace("\"", "\"\"") - output_str = Jason.encode!(output) |> String.replace("\"", "\"\"") - error_str = if is_nil(error), do: "", else: Jason.encode!(error) |> String.replace("\"", "\"\"") + # Log to file + try do + log_file = "log/ai_comments.csv" + File.mkdir_p!("log") - csv_row = "\"#{timestamp}\",\"#{submission_id}\",\"#{question_id}\",\"#{input_str}\",\"#{student_submission_str}\",\"#{output_str}\",\"#{error_str}\"\n" + timestamp = NaiveDateTime.utc_now() |> NaiveDateTime.to_string() + raw_prompt_str = Jason.encode!(raw_prompt) |> String.replace("\"", "\"\"") + answers_json_str = answers_json |> String.replace("\"", "\"\"") + response_str = if is_nil(response), do: "", else: response |> String.replace("\"", "\"\"") + error_str = if is_nil(error), do: "", else: error |> String.replace("\"", "\"\"") - File.write!(log_file, csv_row, [:append]) - end + csv_row = "\"#{timestamp}\",\"#{submission_id}\",\"#{question_id}\",\"#{raw_prompt_str}\",\"#{answers_json_str}\",\"#{response_str}\",\"#{error_str}\"\n" + File.write!(log_file, csv_row, [:append]) + rescue + e -> + Logger.error("Failed to log AI comment to file: #{inspect(e)}") + end + end @doc """ Fetches the question details and answers based on submissionid and questionid and generates AI-generated comments. @@ -127,7 +150,13 @@ defmodule CadetWeb.AICodeAnalysisController do Your output must include only the comment suggestions, separated exclusively by triple pipes ("|||") with no spaces before or after the pipes, and without any additional formatting, bullet points, or extra text. + Comments and documentation in the code are not necessary for the code, do not penalise based on that, do not suggest to add comments as well. + + Follow the XP scoring guideline provided below in the question prompt, do not be too harsh! + For example: "This is a good answer.|||This is a bad answer.|||This is a great answer." + + #Agent Role# You are a kind coding assistant and mentor. #General Instruction on comment style# There is a programming question, and you have to write a comment on the student's answer to the programming question. Note that your reply is addressed directly to the student, so prevent any sentence out of the desired comment in your response to this prompt. The comment includes feedback on the solution's correctness. Suggest improvement areas if necessary. If the answer is incorrect, declare why the answer is wrong, but only give general hints as suggestions and avoid explaining the Right solution. You should keep your tone friendly even if the answer is incorrect and you want to suggest improvements. If there are several problems in the solution, you have to mention all of them. The maximum length of your reply to this prompt can be 50 words. If the answer is correct and you don't have any suggestions, only write "Great job!". #Prequistic knowledge to solve the question# In this question, you're going to work with Runes. Predefined Runes include heart, circle, square, sail, rcross, nova, corner, and blank. You can access these Runes using their names. You can only use predeclared functions, including "show," "beside," "stack," "beside_frac," "stack_frack," "make_cross," "quarter_turn_left," "quarter_turn_right," "turn_upside_down." These functions are defined below: 1. [Function "show" renders the specified Rune in a tab as a basic drawing. Function prototype: show(rune: Rune): Rune Prototype Description: It takes a Rune parameter as input and returns the specified Rune. Example: "show(heart)" shows a heart shape rune.] 2. [Function "beside" makes a new Rune from two given Runes by placing the first on the left of the second, both occupying equal portions of the width of the result. Function prototype: beside(rune1: Rune, rune2: Rune): Rune Prototype Description: It takes two parameters of type Rune, rune1 and rune2, as input and returns a Rune. Example 1: "beside(r1, r2)", places r1 on the left of the r2. Example 2: "beside(stack(r1, r2), stack(r3, r4))" places the output of stack(r1, r2) on the left of output of stack(r3, r4). ] 3. [Function "stack" makes a new Rune from two given Runes by placing the first one on top of the second one, each occupying equal parts of the height of the result. Function prototype: stack(rune1: Rune, rune2: Rune): Rune Prototype Description: It takes two parameters of type Rune, rune1 and rune2, as input and returns a Rune. Example1: "stack(r1, r2)" places r1 on top of r2. Example 2: "Stack(beside(r1, r2), beside(r3, r4))" places output of beside(r1, r2) on top of the output of beside(r3, r4).] 4. [Function "beside_frack" Makes a new Rune from two given Runes by placing the first on the left of the second such that the first one occupies a frac portion of the width of the result and the second the rest. Function Prototype: beside_frac(frac: number, rune1: Rune, rune2: Rune): Rune Prototype Description: It takes a number between 0 and 1 as "frac" and two parameters of type Rune, "rune1" and "rune2," as input and returns a Rune parameter. Example 1: "beside_frac(1/2, heart, circle) places a heart on the left of the circle, and both occupy 1/2 of the plane." Example 2: "beside_frac(1/4, heart, circle) places a heart on the left of the circle. The heart occupies 1/4 of the plane, and the circle occupies 3/4 of the plane."] 5. [Function "stack_frack" Makes a new Rune from two given Runes by placing the first on top of the second such that the first one occupies a frac portion of the height of the result and the second the rest. Function Prototype:stack_frac(frac: number, rune1: Rune, rune2: Rune): Rune Prototype Description: It takes a number between 0 and 1 as "frac" and two parameters of type Rune, "rune1" and "rune2," as input and returns a Rune parameter. Example 1: "stack_frac(1/2, heart, circle) places a heart on top of the circle, and both occupy 1/2 of the plane." Example 2: "stack_frac(1/4, heart, circle) places a heart on top of the circle. The heart occupies 1/4 of the plane, and the circle occupies 3/4 of the plane."] 6. [Function "make_cross" makes a new Rune from a given Rune by arranging it into a square for copies of the given Rune in different orientations. Function Prototype: make_cross(rune: Rune): Rune Prototype Description: It takes a Rune parameter as input and returns a Rune parameter. Example: "make_cross(heart)" places a heart shape rune on the bottom-left, a 90-degree clockwise rotated heart on the top-left, a 180-degree clockwise rotated heart on the top-right, and a 270-degree clockwise rotated heart on the bottom-right. The final Rune consists of four runes.] 7. [Function "quarter_turn_left" Makes a new Rune from a given Rune by turning it a quarter-turn in an anti-clockwise direction. Function prototype: quarter_turn_right(rune: Rune): Rune Prototype Description: It takes a Rune parameter as input and returns a Rune parameter. Example 1: "quarter_turn_left(heart)" rotates the heart shape rune 90 degrees in an anti-clockwise direction. Example 2: "quarter_turn_left(stack(r1, r2))" rotates the output of stack(r1, r2) 90 degrees in an anti-clockwise direction. ] 8. [Function "quarter_turn_right" makes a new Rune from a given Rune by turning it a quarter-turn around the center in a clockwise direction. Function prototype: quarter_turn_right(rune: Rune): Rune Prototype Description: It takes a Rune parameter as input and returns a Rune parameter. Example 1: "quarter_turn_right(heart)" rotates the heart shape rune 90 degrees in a clockwise direction. Example 2: "quarter_turn_right(stack(r1, r2))" rotates the output of stack(r1, r2) 90 degrees in a clockwise direction. ] 9. [Function "turn_upside_down" makes a new Rune from a given Rune by turning it upside-down. Function prototype: turn_upside_down(rune: Rune): Rune Prototype Description: It takes a Rune parameter as input and returns a Rune parameter. Example 1: "turn_upside_down(heart)" rotates a heart shape rune 180 degrees in a clockwise direction. Example 2: "turn_upside_down(stack(r1, r2))" rotates the output of stack(r1, r2) 180 degrees in a clockwise direction.] You must only use the Runes and functions declared above and avoid importing any module in your program. You can pass the output of each function as input to another function. For example, consider beside(stack(r2, r1), stack(r3, r4)). First, the inner stack functions get executed. r2 goes to the left of r1, and r3 goes to the left of r4. Then the output Rune of each stack works as input of beside function. meaning output of stak(r2, r1) goes on top of output of stack(r3,r4). Avoid hard coding. #Programming question# Write a function hook that takes a fraction "frac" as an input and creates a 'hook' pattern. The fraction input determines the size of the base of the hook. The output rune: [Imagine a rectangle divided into two horizontal sections. Each section is the height of a square. Top Section: This section is simply a filled square. Bottom Section: The bottom section is also the size of a square. However, it's divided into two equal parts vertically. The left side of this square is filled (so it looks like a rectangle that's half the width of the square). The right side of this square is blank or empty. So, if you place these two sections on top of one another, you get: A full square on top. Directly below it, on the left side, you have a half-filled square (a rectangle), and on the right side, it's empty. This forms a "hook" rune, with the hook part facing to the left. The overall rune is a square with two times the height of the original squares used to create it. Examples: hook(1): It's simply a square rune. hook(0): A filled square at the top. An empty or blank space at the bottom of the same size as the square. hook(1/2): A full square on top. Below that, on the right side, there's another filled square that's half the width of the full square. On the left side, it's empty. hook(1/5): A full square on top. Below that, on the right side, there's a very thin filled rectangle (only 1/5 the width of the square). The rest (4/5) to the right is empty.] You will only need to use the square and blank primitive runes and transform them to get the hook. Implement your function in the code below: "function hook(frac) { // your answer here } // Test show(hook(1/5));" #Sample Solution and feedback# 1. "function hook(frac) { return stack(square, quarter_turn_right( stack_frac(frac, square, blank))); } // Test show(hook(1/5));" - Great job! 2. "function hook(frac) { return frac === 1 ? square : frac === 0 ? stack(square,blank) : stack(square,beside_frac(1-frac, blank, square)); } show(hook(1/5));" - Excellent work! 3."function hook(frac) { return stack(square, quarter_turn_left( stack_frac(1-frac, blank, square))); } show(hook(1/5));" -Great job! 4."function hook(frac) { // your answer here return stack_frac(1/2,square, beside_frac(1-frac,blank,square)); } // Test show(hook(1/5));" -Good job, However stack_frac(1 / 2, etc) could have been simplified by merely using stack. """ prompt = raw_prompt <> "\n" <> answers_json @@ -152,7 +181,7 @@ defmodule CadetWeb.AICodeAnalysisController do {:ok, %HTTPoison.Response{status_code: 200, body: body}} -> case Jason.decode(body) do {:ok, %{"choices" => [%{"message" => %{"content" => response}}]}} -> - log_to_csv(submission_id, question_id, raw_prompt, answers_json, response) + log_comment(submission_id, question_id, raw_prompt, answers_json, response) comments_list = String.split(response, "|||") filtered_comments = Enum.filter(comments_list, fn comment -> @@ -162,20 +191,34 @@ defmodule CadetWeb.AICodeAnalysisController do json(conn, %{"comments" => filtered_comments}) {:error, _} -> - log_to_csv(submission_id, question_id, raw_prompt, answers_json, nil, "Failed to parse response from OpenAI API") + log_comment(submission_id, question_id, raw_prompt, answers_json, nil, "Failed to parse response from OpenAI API") json(conn, %{"error" => "Failed to parse response from OpenAI API"}) end {:ok, %HTTPoison.Response{status_code: status, body: body}} -> - log_to_csv(submission_id, question_id, raw_prompt, answers_json, nil, "API request failed with status #{status}") + log_comment(submission_id, question_id, raw_prompt, answers_json, nil, "API request failed with status #{status}") json(conn, %{"error" => "API request failed with status #{status}: #{body}"}) {:error, %HTTPoison.Error{reason: reason}} -> - log_to_csv(submission_id, question_id, raw_prompt, answers_json, nil, reason) + log_comment(submission_id, question_id, raw_prompt, answers_json, nil, reason) json(conn, %{"error" => "HTTP request error: #{inspect(reason)}"}) end end + @doc """ + Saves the final comment chosen for a submission. + """ + def save_final_comment(conn, %{"submissionid" => submission_id, "questionid" => question_id, "comment" => comment}) do + case AIComments.update_final_comment(submission_id, question_id, comment) do + {:ok, _updated_comment} -> + json(conn, %{"status" => "success"}) + {:error, changeset} -> + conn + |> put_status(:unprocessable_entity) + |> json(%{"error" => "Failed to save final comment"}) + end + end + swagger_path :generate_ai_comments do post("/courses/{courseId}/admin/generate-comments/{submissionId}/{questionId}") @@ -197,6 +240,28 @@ defmodule CadetWeb.AICodeAnalysisController do response(403, "Forbidden") end + swagger_path :save_final_comment do + post("/courses/{courseId}/admin/save-final-comment/{submissionId}/{questionId}") + + summary("Save the final comment chosen for a submission.") + + security([%{JWT: []}]) + + consumes("application/json") + produces("application/json") + + parameters do + submissionId(:path, :integer, "submission id", required: true) + questionId(:path, :integer, "question id", required: true) + comment(:body, :string, "The final comment to save", required: true) + end + + response(200, "OK", Schema.ref(:SaveFinalComment)) + response(400, "Invalid or missing parameter(s)") + response(401, "Unauthorized") + response(403, "Forbidden") + end + def swagger_definitions do %{ GenerateAIComments: @@ -204,6 +269,12 @@ defmodule CadetWeb.AICodeAnalysisController do properties do comments(:string, "AI-generated comments on the submission answers") end + end, + SaveFinalComment: + swagger_schema do + properties do + status(:string, "Status of the operation") + end end } end diff --git a/priv/repo/migrations/20250220103623_create_ai_comments.exs b/priv/repo/migrations/20250220103623_create_ai_comments.exs new file mode 100644 index 000000000..6338dccd8 --- /dev/null +++ b/priv/repo/migrations/20250220103623_create_ai_comments.exs @@ -0,0 +1,20 @@ +defmodule Cadet.Repo.Migrations.CreateAiCommentLogs do + use Ecto.Migration + + def change do + create table(:ai_comment_logs) do + add(:submission_id, :integer, null: false) + add(:question_id, :integer, null: false) + add(:raw_prompt, :text, null: false) + add(:answers_json, :text, null: false) + add(:response, :text) + add(:error, :text) + add(:comment_chosen, :text) + add(:final_comment, :text) + timestamps() + end + + create(index(:ai_comment_logs, [:submission_id])) + create(index(:ai_comment_logs, [:question_id])) + end +end diff --git a/test/cadet_web/controllers/ai_code_analysis_controller_test.exs b/test/cadet_web/controllers/ai_code_analysis_controller_test.exs new file mode 100644 index 000000000..559c4ea9e --- /dev/null +++ b/test/cadet_web/controllers/ai_code_analysis_controller_test.exs @@ -0,0 +1,124 @@ +defmodule CadetWeb.AICodeAnalysisControllerTest do + use CadetWeb.ConnCase + alias Cadet.AIComments + alias Cadet.AIComments.AIComment + + setup do + # Clean up test files before each test + log_file = "log/ai_comments.csv" + File.rm(log_file) + :ok + end + + describe "generate_ai_comments" do + test "successfully logs comments to both database and file", %{conn: conn} do + # Test data + submission_id = 123 + question_id = 456 + raw_prompt = "Test prompt" + answers_json = ~s({"test": "data"}) + mock_response = "Comment 1|||Comment 2|||Comment 3" + + # Make the API call + response = + conn + |> post(Routes.ai_code_analysis_path(conn, :generate_ai_comments, submission_id, question_id)) + |> json_response(200) + + # Verify database entry + comments = Repo.all(AIComment) + assert length(comments) > 0 + latest_comment = List.first(comments) + assert latest_comment.submission_id == submission_id + assert latest_comment.question_id == question_id + assert latest_comment.raw_prompt != nil + assert latest_comment.answers_json != nil + + # Verify CSV file + log_file = "log/ai_comments.csv" + assert File.exists?(log_file) + file_content = File.read!(log_file) + + # Check if CSV contains the required data + assert file_content =~ Integer.to_string(submission_id) + assert file_content =~ Integer.to_string(question_id) + end + + test "logs error when API call fails", %{conn: conn} do + # Test data with invalid submission_id to trigger error + submission_id = -1 + question_id = 456 + + # Make the API call that should fail + response = + conn + |> post(Routes.ai_code_analysis_path(conn, :generate_ai_comments, submission_id, question_id)) + |> json_response(400) + + # Verify error is logged in database + comments = Repo.all(AIComment) + assert length(comments) > 0 + error_log = List.first(comments) + assert error_log.error != nil + assert error_log.submission_id == submission_id + assert error_log.question_id == question_id + + # Verify error is logged in CSV + log_file = "log/ai_comments.csv" + assert File.exists?(log_file) + file_content = File.read!(log_file) + assert file_content =~ Integer.to_string(submission_id) + assert file_content =~ Integer.to_string(question_id) + assert file_content =~ "error" + end + end + + describe "save_final_comment" do + test "successfully saves final comment", %{conn: conn} do + # First create a comment entry + submission_id = 123 + question_id = 456 + raw_prompt = "Test prompt" + answers_json = ~s({"test": "data"}) + response = "Comment 1|||Comment 2|||Comment 3" + + {:ok, _comment} = AIComments.create_ai_comment(%{ + submission_id: submission_id, + question_id: question_id, + raw_prompt: raw_prompt, + answers_json: answers_json, + response: response + }) + + # Now save the final comment + final_comment = "This is the chosen final comment" + response = + conn + |> post(Routes.ai_code_analysis_path(conn, :save_final_comment, submission_id, question_id), %{ + comment: final_comment + }) + |> json_response(200) + + assert response["status"] == "success" + + # Verify the comment was saved + comment = Repo.get_by(AIComment, submission_id: submission_id, question_id: question_id) + assert comment.final_comment == final_comment + end + + test "returns error when no comment exists", %{conn: conn} do + submission_id = 999 + question_id = 888 + final_comment = "This comment should not be saved" + + response = + conn + |> post(Routes.ai_code_analysis_path(conn, :save_final_comment, submission_id, question_id), %{ + comment: final_comment + }) + |> json_response(422) + + assert response["error"] == "Failed to save final comment" + end + end +end From 8a235b3251c5c80e01a3a738647e686dd055f360 Mon Sep 17 00:00:00 2001 From: Eugene Oh Yun Zheng Date: Tue, 18 Mar 2025 10:56:56 +0800 Subject: [PATCH 05/20] Format answers json before sending to LLM --- .../controllers/generate_ai_comments.ex | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index a543d5518..4557d52ed 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -81,6 +81,37 @@ defmodule CadetWeb.AICodeAnalysisController do end) end + def format_answers(json_string) do + {:ok, answers} = Jason.decode(json_string) + + answers + |> Enum.map(&format_answer/1) + |> Enum.join("\n\n") + end + + defp format_answer(answer) do + """ + **Question ID: #{answer["question"]["id"]}** + + **Question:** + #{answer["question"]["content"]} + + **Solution:** + ``` + #{answer["question"]["solution"]} + ``` + + **Answer:** + ``` + #{answer["answer"]["code"]} + ``` + + **Autograding Status:** #{answer["autograding_status"]} + **Autograding Results:** #{answer["autograding_results"]} + **Comments:** #{answer["comments"] || "None"} + """ + end + defp analyze_code(conn, answers, submission_id, question_id) do answers_json = answers @@ -110,6 +141,7 @@ defmodule CadetWeb.AICodeAnalysisController do |> Map.put(:question, question_data) end) |> Jason.encode!() + |> format_answers() raw_prompt = """ The code below is written in Source, a variant of JavaScript that comes with a rich set of built-in constants and functions. Below is a summary of some key built-in entities available in Source: From d384e06c9e13b451d045f53e6b3aadf308896694 Mon Sep 17 00:00:00 2001 From: Eugene Oh Yun Zheng Date: Tue, 18 Mar 2025 11:11:12 +0800 Subject: [PATCH 06/20] Add LLM Prompt to question params when submitting assessment xml file --- lib/cadet/assessments/question_types/programming_question.ex | 3 ++- lib/cadet/jobs/xml_parser.ex | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/cadet/assessments/question_types/programming_question.ex b/lib/cadet/assessments/question_types/programming_question.ex index a39625de3..245e8ef11 100644 --- a/lib/cadet/assessments/question_types/programming_question.ex +++ b/lib/cadet/assessments/question_types/programming_question.ex @@ -13,13 +13,14 @@ defmodule Cadet.Assessments.QuestionTypes.ProgrammingQuestion do field(:template, :string) field(:postpend, :string, default: "") field(:solution, :string) + field(:llm_prompt, :string) embeds_many(:public, Testcase) embeds_many(:opaque, Testcase) embeds_many(:secret, Testcase) end @required_fields ~w(content template)a - @optional_fields ~w(solution prepend postpend)a + @optional_fields ~w(solution prepend postpend llm_prompt)a def changeset(question, params \\ %{}) do question diff --git a/lib/cadet/jobs/xml_parser.ex b/lib/cadet/jobs/xml_parser.ex index b49128506..abf16c8cd 100644 --- a/lib/cadet/jobs/xml_parser.ex +++ b/lib/cadet/jobs/xml_parser.ex @@ -202,7 +202,8 @@ defmodule Cadet.Updater.XMLParser do prepend: ~x"./SNIPPET/PREPEND/text()" |> transform_by(&process_charlist/1), template: ~x"./SNIPPET/TEMPLATE/text()" |> transform_by(&process_charlist/1), postpend: ~x"./SNIPPET/POSTPEND/text()" |> transform_by(&process_charlist/1), - solution: ~x"./SNIPPET/SOLUTION/text()" |> transform_by(&process_charlist/1) + solution: ~x"./SNIPPET/SOLUTION/text()" |> transform_by(&process_charlist/1), + llm_prompt: ~x"./LLM_GRADING_PROMPT/text()" |> transform_by(&process_charlist/1) ), entity |> xmap( From 98feac2402403a71f6061ab811fe0a472eadb0cb Mon Sep 17 00:00:00 2001 From: Eugene Oh Yun Zheng Date: Tue, 18 Mar 2025 12:59:16 +0800 Subject: [PATCH 07/20] Add LLM Prompt to api response when grading view is open --- lib/cadet_web/helpers/assessments_helpers.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/cadet_web/helpers/assessments_helpers.ex b/lib/cadet_web/helpers/assessments_helpers.ex index 967df1131..7e03e5d83 100644 --- a/lib/cadet_web/helpers/assessments_helpers.ex +++ b/lib/cadet_web/helpers/assessments_helpers.ex @@ -177,6 +177,7 @@ defmodule CadetWeb.AssessmentsHelpers do prepend: "prepend", solutionTemplate: "template", postpend: "postpend", + llm_prompt: "llm_prompt", testcases: build_testcases(all_testcases?) }) From 7716d57732e71d678f227b6de07be69d4a5b1cc7 Mon Sep 17 00:00:00 2001 From: Arul Date: Wed, 19 Mar 2025 14:25:01 +0800 Subject: [PATCH 08/20] feat: added llm_prompt from qn to raw_prompt --- .../controllers/generate_ai_comments.ex | 36 ++++++++----- test/support/ai_comments_test_helpers.ex | 50 +++++++++++++++++++ 2 files changed, 74 insertions(+), 12 deletions(-) create mode 100644 test/support/ai_comments_test_helpers.ex diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index 4557d52ed..d96109da6 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -121,12 +121,14 @@ defmodule CadetWeb.AICodeAnalysisController do %{ id: answer.question_id, content: Map.get(answer.question.question, "content"), - solution: Map.get(answer.question.question, "solution") + solution: Map.get(answer.question.question, "solution"), + llm_prompt: Map.get(answer.question.question, "llm_prompt") } else %{ id: nil, - content: nil + content: nil, + llm_prompt: nil } end answer @@ -187,12 +189,22 @@ defmodule CadetWeb.AICodeAnalysisController do Follow the XP scoring guideline provided below in the question prompt, do not be too harsh! For example: "This is a good answer.|||This is a bad answer.|||This is a great answer." - - #Agent Role# You are a kind coding assistant and mentor. #General Instruction on comment style# There is a programming question, and you have to write a comment on the student's answer to the programming question. Note that your reply is addressed directly to the student, so prevent any sentence out of the desired comment in your response to this prompt. The comment includes feedback on the solution's correctness. Suggest improvement areas if necessary. If the answer is incorrect, declare why the answer is wrong, but only give general hints as suggestions and avoid explaining the Right solution. You should keep your tone friendly even if the answer is incorrect and you want to suggest improvements. If there are several problems in the solution, you have to mention all of them. The maximum length of your reply to this prompt can be 50 words. If the answer is correct and you don't have any suggestions, only write "Great job!". #Prequistic knowledge to solve the question# In this question, you're going to work with Runes. Predefined Runes include heart, circle, square, sail, rcross, nova, corner, and blank. You can access these Runes using their names. You can only use predeclared functions, including "show," "beside," "stack," "beside_frac," "stack_frack," "make_cross," "quarter_turn_left," "quarter_turn_right," "turn_upside_down." These functions are defined below: 1. [Function "show" renders the specified Rune in a tab as a basic drawing. Function prototype: show(rune: Rune): Rune Prototype Description: It takes a Rune parameter as input and returns the specified Rune. Example: "show(heart)" shows a heart shape rune.] 2. [Function "beside" makes a new Rune from two given Runes by placing the first on the left of the second, both occupying equal portions of the width of the result. Function prototype: beside(rune1: Rune, rune2: Rune): Rune Prototype Description: It takes two parameters of type Rune, rune1 and rune2, as input and returns a Rune. Example 1: "beside(r1, r2)", places r1 on the left of the r2. Example 2: "beside(stack(r1, r2), stack(r3, r4))" places the output of stack(r1, r2) on the left of output of stack(r3, r4). ] 3. [Function "stack" makes a new Rune from two given Runes by placing the first one on top of the second one, each occupying equal parts of the height of the result. Function prototype: stack(rune1: Rune, rune2: Rune): Rune Prototype Description: It takes two parameters of type Rune, rune1 and rune2, as input and returns a Rune. Example1: "stack(r1, r2)" places r1 on top of r2. Example 2: "Stack(beside(r1, r2), beside(r3, r4))" places output of beside(r1, r2) on top of the output of beside(r3, r4).] 4. [Function "beside_frack" Makes a new Rune from two given Runes by placing the first on the left of the second such that the first one occupies a frac portion of the width of the result and the second the rest. Function Prototype: beside_frac(frac: number, rune1: Rune, rune2: Rune): Rune Prototype Description: It takes a number between 0 and 1 as "frac" and two parameters of type Rune, "rune1" and "rune2," as input and returns a Rune parameter. Example 1: "beside_frac(1/2, heart, circle) places a heart on the left of the circle, and both occupy 1/2 of the plane." Example 2: "beside_frac(1/4, heart, circle) places a heart on the left of the circle. The heart occupies 1/4 of the plane, and the circle occupies 3/4 of the plane."] 5. [Function "stack_frack" Makes a new Rune from two given Runes by placing the first on top of the second such that the first one occupies a frac portion of the height of the result and the second the rest. Function Prototype:stack_frac(frac: number, rune1: Rune, rune2: Rune): Rune Prototype Description: It takes a number between 0 and 1 as "frac" and two parameters of type Rune, "rune1" and "rune2," as input and returns a Rune parameter. Example 1: "stack_frac(1/2, heart, circle) places a heart on top of the circle, and both occupy 1/2 of the plane." Example 2: "stack_frac(1/4, heart, circle) places a heart on top of the circle. The heart occupies 1/4 of the plane, and the circle occupies 3/4 of the plane."] 6. [Function "make_cross" makes a new Rune from a given Rune by arranging it into a square for copies of the given Rune in different orientations. Function Prototype: make_cross(rune: Rune): Rune Prototype Description: It takes a Rune parameter as input and returns a Rune parameter. Example: "make_cross(heart)" places a heart shape rune on the bottom-left, a 90-degree clockwise rotated heart on the top-left, a 180-degree clockwise rotated heart on the top-right, and a 270-degree clockwise rotated heart on the bottom-right. The final Rune consists of four runes.] 7. [Function "quarter_turn_left" Makes a new Rune from a given Rune by turning it a quarter-turn in an anti-clockwise direction. Function prototype: quarter_turn_right(rune: Rune): Rune Prototype Description: It takes a Rune parameter as input and returns a Rune parameter. Example 1: "quarter_turn_left(heart)" rotates the heart shape rune 90 degrees in an anti-clockwise direction. Example 2: "quarter_turn_left(stack(r1, r2))" rotates the output of stack(r1, r2) 90 degrees in an anti-clockwise direction. ] 8. [Function "quarter_turn_right" makes a new Rune from a given Rune by turning it a quarter-turn around the center in a clockwise direction. Function prototype: quarter_turn_right(rune: Rune): Rune Prototype Description: It takes a Rune parameter as input and returns a Rune parameter. Example 1: "quarter_turn_right(heart)" rotates the heart shape rune 90 degrees in a clockwise direction. Example 2: "quarter_turn_right(stack(r1, r2))" rotates the output of stack(r1, r2) 90 degrees in a clockwise direction. ] 9. [Function "turn_upside_down" makes a new Rune from a given Rune by turning it upside-down. Function prototype: turn_upside_down(rune: Rune): Rune Prototype Description: It takes a Rune parameter as input and returns a Rune parameter. Example 1: "turn_upside_down(heart)" rotates a heart shape rune 180 degrees in a clockwise direction. Example 2: "turn_upside_down(stack(r1, r2))" rotates the output of stack(r1, r2) 180 degrees in a clockwise direction.] You must only use the Runes and functions declared above and avoid importing any module in your program. You can pass the output of each function as input to another function. For example, consider beside(stack(r2, r1), stack(r3, r4)). First, the inner stack functions get executed. r2 goes to the left of r1, and r3 goes to the left of r4. Then the output Rune of each stack works as input of beside function. meaning output of stak(r2, r1) goes on top of output of stack(r3,r4). Avoid hard coding. #Programming question# Write a function hook that takes a fraction "frac" as an input and creates a 'hook' pattern. The fraction input determines the size of the base of the hook. The output rune: [Imagine a rectangle divided into two horizontal sections. Each section is the height of a square. Top Section: This section is simply a filled square. Bottom Section: The bottom section is also the size of a square. However, it's divided into two equal parts vertically. The left side of this square is filled (so it looks like a rectangle that's half the width of the square). The right side of this square is blank or empty. So, if you place these two sections on top of one another, you get: A full square on top. Directly below it, on the left side, you have a half-filled square (a rectangle), and on the right side, it's empty. This forms a "hook" rune, with the hook part facing to the left. The overall rune is a square with two times the height of the original squares used to create it. Examples: hook(1): It's simply a square rune. hook(0): A filled square at the top. An empty or blank space at the bottom of the same size as the square. hook(1/2): A full square on top. Below that, on the right side, there's another filled square that's half the width of the full square. On the left side, it's empty. hook(1/5): A full square on top. Below that, on the right side, there's a very thin filled rectangle (only 1/5 the width of the square). The rest (4/5) to the right is empty.] You will only need to use the square and blank primitive runes and transform them to get the hook. Implement your function in the code below: "function hook(frac) { // your answer here } // Test show(hook(1/5));" #Sample Solution and feedback# 1. "function hook(frac) { return stack(square, quarter_turn_right( stack_frac(frac, square, blank))); } // Test show(hook(1/5));" - Great job! 2. "function hook(frac) { return frac === 1 ? square : frac === 0 ? stack(square,blank) : stack(square,beside_frac(1-frac, blank, square)); } show(hook(1/5));" - Excellent work! 3."function hook(frac) { return stack(square, quarter_turn_left( stack_frac(1-frac, blank, square))); } show(hook(1/5));" -Great job! 4."function hook(frac) { // your answer here return stack_frac(1/2,square, beside_frac(1-frac,blank,square)); } // Test show(hook(1/5));" -Good job, However stack_frac(1 / 2, etc) could have been simplified by merely using stack. """ - - prompt = raw_prompt <> "\n" <> answers_json - + # Get the llm_prompt from the first answer's question + llm_prompt = + answers + |> List.first() + |> Map.get(:question) + |> Map.get(:question) + |> Map.get("llm_prompt") + + # Combine prompts if llm_prompt exists + prompt = + if llm_prompt && llm_prompt != "" do + raw_prompt <> "Additional Instructions:\n\n" <> llm_prompt <> "\n\n" <> answers_json + else + raw_prompt <> "\n" <> answers_json + end input = %{ model: @model, @@ -209,11 +221,11 @@ defmodule CadetWeb.AICodeAnalysisController do ] - case HTTPoison.post(@openai_api_url, input, headers) do + case HTTPoison.post(@openai_api_url, input, headers, timeout: 60_000, recv_timeout: 60_000) do {:ok, %HTTPoison.Response{status_code: 200, body: body}} -> case Jason.decode(body) do {:ok, %{"choices" => [%{"message" => %{"content" => response}}]}} -> - log_comment(submission_id, question_id, raw_prompt, answers_json, response) + log_comment(submission_id, question_id, prompt, answers_json, response) comments_list = String.split(response, "|||") filtered_comments = Enum.filter(comments_list, fn comment -> @@ -223,16 +235,16 @@ defmodule CadetWeb.AICodeAnalysisController do json(conn, %{"comments" => filtered_comments}) {:error, _} -> - log_comment(submission_id, question_id, raw_prompt, answers_json, nil, "Failed to parse response from OpenAI API") + log_comment(submission_id, question_id, prompt, answers_json, nil, "Failed to parse response from OpenAI API") json(conn, %{"error" => "Failed to parse response from OpenAI API"}) end {:ok, %HTTPoison.Response{status_code: status, body: body}} -> - log_comment(submission_id, question_id, raw_prompt, answers_json, nil, "API request failed with status #{status}") + log_comment(submission_id, question_id, prompt, answers_json, nil, "API request failed with status #{status}") json(conn, %{"error" => "API request failed with status #{status}: #{body}"}) {:error, %HTTPoison.Error{reason: reason}} -> - log_comment(submission_id, question_id, raw_prompt, answers_json, nil, reason) + log_comment(submission_id, question_id, prompt, answers_json, nil, reason) json(conn, %{"error" => "HTTP request error: #{inspect(reason)}"}) end end diff --git a/test/support/ai_comments_test_helpers.ex b/test/support/ai_comments_test_helpers.ex new file mode 100644 index 000000000..a359075b5 --- /dev/null +++ b/test/support/ai_comments_test_helpers.ex @@ -0,0 +1,50 @@ +defmodule Cadet.AICommentsTestHelpers do + @moduledoc """ + Helper functions for testing AI comments functionality. + """ + + alias Cadet.Repo + alias Cadet.AIComments.AIComment + import Ecto.Query + + @doc """ + Gets the latest AI comment from the database. + """ + def get_latest_comment do + AIComment + |> first(order_by: [desc: :inserted_at]) + |> Repo.one() + end + + @doc """ + Gets all AI comments for a specific submission and question. + """ + def get_comments_for_submission(submission_id, question_id) do + from(c in AIComment, + where: c.submission_id == ^submission_id and c.question_id == ^question_id, + order_by: [desc: c.inserted_at] + ) + |> Repo.all() + end + + @doc """ + Reads the CSV log file and returns its contents. + """ + def read_csv_log do + log_file = "log/ai_comments.csv" + if File.exists?(log_file) do + File.read!(log_file) + else + "" + end + end + + @doc """ + Cleans up test artifacts. + """ + def cleanup_test_artifacts do + log_file = "log/ai_comments.csv" + File.rm(log_file) + Repo.delete_all(AIComment) + end +end From df34dbd79920224ed0865a97f867906a54412c5b Mon Sep 17 00:00:00 2001 From: Arul Date: Wed, 19 Mar 2025 14:40:20 +0800 Subject: [PATCH 09/20] feat: enabling/disabling of LLM feature by course level --- lib/cadet/courses/course.ex | 4 ++- .../controllers/generate_ai_comments.ex | 27 ++++++++++++++++--- .../20240320000000_add_llm_grading_access.exs | 15 +++++++++++ 3 files changed, 41 insertions(+), 5 deletions(-) create mode 100644 priv/repo/migrations/20240320000000_add_llm_grading_access.exs diff --git a/lib/cadet/courses/course.ex b/lib/cadet/courses/course.ex index c74d23bd7..e3b3599ca 100644 --- a/lib/cadet/courses/course.ex +++ b/lib/cadet/courses/course.ex @@ -14,6 +14,7 @@ defmodule Cadet.Courses.Course do enable_achievements: boolean(), enable_sourcecast: boolean(), enable_stories: boolean(), + enable_llm_grading: boolean(), source_chapter: integer(), source_variant: String.t(), module_help_text: String.t(), @@ -28,6 +29,7 @@ defmodule Cadet.Courses.Course do field(:enable_achievements, :boolean, default: true) field(:enable_sourcecast, :boolean, default: true) field(:enable_stories, :boolean, default: false) + field(:enable_llm_grading, :boolean) field(:source_chapter, :integer) field(:source_variant, :string) field(:module_help_text, :string) @@ -42,7 +44,7 @@ defmodule Cadet.Courses.Course do @required_fields ~w(course_name viewable enable_game enable_achievements enable_sourcecast enable_stories source_chapter source_variant)a - @optional_fields ~w(course_short_name module_help_text)a + @optional_fields ~w(course_short_name module_help_text enable_llm_grading)a def changeset(course, params) do course diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index d96109da6..8d68e243a 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -6,10 +6,12 @@ defmodule CadetWeb.AICodeAnalysisController do alias Cadet.Assessments alias Cadet.AIComments + alias Cadet.Courses @openai_api_url "https://api.openai.com/v1/chat/completions" @model "gpt-4o" @api_key Application.get_env(:openai, :api_key) + @default_llm_grading false # For logging outputs to both database and file defp log_comment(submission_id, question_id, raw_prompt, answers_json, response, error \\ nil) do @@ -54,11 +56,26 @@ defmodule CadetWeb.AICodeAnalysisController do @doc """ Fetches the question details and answers based on submissionid and questionid and generates AI-generated comments. """ - def generate_ai_comments(conn, %{"submissionid" => submission_id, "questionid" => question_id}) + def generate_ai_comments(conn, %{"submissionid" => submission_id, "questionid" => question_id, "course_id" => course_id}) when is_ecto_id(submission_id) do - case Assessments.get_answers_in_submission(submission_id, question_id) do - {:ok, {answers, _assessment}} -> - analyze_code(conn, answers, submission_id, question_id) + # Check if LLM grading is enabled for this course (default to @default_llm_grading if nil) + case Courses.get_course_config(course_id) do + {:ok, course} -> + if course.enable_llm_grading == true || (course.enable_llm_grading == nil && @default_llm_grading == true) do + case Assessments.get_answers_in_submission(submission_id, question_id) do + {:ok, {answers, _assessment}} -> + analyze_code(conn, answers, submission_id, question_id) + + {:error, {status, message}} -> + conn + |> put_status(status) + |> text(message) + end + else + conn + |> put_status(:forbidden) + |> json(%{"error" => "LLM grading is not enabled for this course"}) + end {:error, {status, message}} -> conn @@ -274,6 +291,7 @@ defmodule CadetWeb.AICodeAnalysisController do produces("application/json") parameters do + courseId(:path, :integer, "course id", required: true) submissionId(:path, :integer, "submission id", required: true) questionId(:path, :integer, "question id", required: true) end @@ -282,6 +300,7 @@ defmodule CadetWeb.AICodeAnalysisController do response(400, "Invalid or missing parameter(s) or submission and/or question not found") response(401, "Unauthorized") response(403, "Forbidden") + response(403, "LLM grading is not enabled for this course") end swagger_path :save_final_comment do diff --git a/priv/repo/migrations/20240320000000_add_llm_grading_access.exs b/priv/repo/migrations/20240320000000_add_llm_grading_access.exs new file mode 100644 index 000000000..c5f337eee --- /dev/null +++ b/priv/repo/migrations/20240320000000_add_llm_grading_access.exs @@ -0,0 +1,15 @@ +defmodule Cadet.Repo.Migrations.AddLLMGradingAccess do + use Ecto.Migration + + def up do + alter table(:courses) do + add(:enable_llm_grading, :boolean, null: true) + end + end + + def down do + alter table(:courses) do + remove(:enable_llm_grading) + end + end +end From 0a25fa8e994fa795bbf65da61b62ee63f11aedb2 Mon Sep 17 00:00:00 2001 From: Arul Date: Wed, 19 Mar 2025 17:00:57 +0800 Subject: [PATCH 10/20] feat: added llm_grading boolean field to course creation API --- lib/cadet_web/controllers/courses_controller.ex | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/cadet_web/controllers/courses_controller.ex b/lib/cadet_web/controllers/courses_controller.ex index e6555bd7a..5b88be0dc 100644 --- a/lib/cadet_web/controllers/courses_controller.ex +++ b/lib/cadet_web/controllers/courses_controller.ex @@ -56,6 +56,7 @@ defmodule CadetWeb.CoursesController do enable_achievements(:body, :boolean, "Enable achievements", required: true) enable_sourcecast(:body, :boolean, "Enable sourcecast", required: true) enable_stories(:body, :boolean, "Enable stories", required: true) + enable_llm_grading(:body, :boolean, "Enable LLM grading", required: false) source_chapter(:body, :number, "Default source chapter", required: true) source_variant(:body, Schema.ref(:SourceVariant), "Default source variant name", @@ -97,6 +98,7 @@ defmodule CadetWeb.CoursesController do enable_achievements(:boolean, "Enable achievements", required: true) enable_sourcecast(:boolean, "Enable sourcecast", required: true) enable_stories(:boolean, "Enable stories", required: true) + enable_llm_grading(:boolean, "Enable LLM grading", required: false) source_chapter(:integer, "Source Chapter number from 1 to 4", required: true) source_variant(Schema.ref(:SourceVariant), "Source Variant name", required: true) module_help_text(:string, "Module help text", required: true) @@ -111,6 +113,7 @@ defmodule CadetWeb.CoursesController do enable_achievements: true, enable_sourcecast: true, enable_stories: false, + enable_llm_grading: false, source_chapter: 1, source_variant: "default", module_help_text: "Help text", From 2723f5ae87ac66ad8ba6decd2728658ae52b9128 Mon Sep 17 00:00:00 2001 From: Arul Date: Wed, 26 Mar 2025 11:39:05 +0800 Subject: [PATCH 11/20] feat: added api key storage in courses & edit api key/enable llm grading --- lib/cadet/courses/course.ex | 4 ++- .../admin_courses_controller.ex | 2 ++ .../controllers/courses_controller.ex | 3 ++ .../controllers/generate_ai_comments.ex | 30 ++++++++++++------- lib/cadet_web/views/courses_view.ex | 2 ++ ...40320000001_add_llm_api_key_to_courses.exs | 15 ++++++++++ 6 files changed, 44 insertions(+), 12 deletions(-) create mode 100644 priv/repo/migrations/20240320000001_add_llm_api_key_to_courses.exs diff --git a/lib/cadet/courses/course.ex b/lib/cadet/courses/course.ex index e3b3599ca..eb1fcb386 100644 --- a/lib/cadet/courses/course.ex +++ b/lib/cadet/courses/course.ex @@ -15,6 +15,7 @@ defmodule Cadet.Courses.Course do enable_sourcecast: boolean(), enable_stories: boolean(), enable_llm_grading: boolean(), + llm_api_key: String.t() | nil, source_chapter: integer(), source_variant: String.t(), module_help_text: String.t(), @@ -30,6 +31,7 @@ defmodule Cadet.Courses.Course do field(:enable_sourcecast, :boolean, default: true) field(:enable_stories, :boolean, default: false) field(:enable_llm_grading, :boolean) + field(:llm_api_key, :string) field(:source_chapter, :integer) field(:source_variant, :string) field(:module_help_text, :string) @@ -44,7 +46,7 @@ defmodule Cadet.Courses.Course do @required_fields ~w(course_name viewable enable_game enable_achievements enable_sourcecast enable_stories source_chapter source_variant)a - @optional_fields ~w(course_short_name module_help_text enable_llm_grading)a + @optional_fields ~w(course_short_name module_help_text enable_llm_grading llm_api_key)a def changeset(course, params) do course diff --git a/lib/cadet_web/admin_controllers/admin_courses_controller.ex b/lib/cadet_web/admin_controllers/admin_courses_controller.ex index 7220a4d80..09d7d8616 100644 --- a/lib/cadet_web/admin_controllers/admin_courses_controller.ex +++ b/lib/cadet_web/admin_controllers/admin_courses_controller.ex @@ -108,6 +108,8 @@ defmodule CadetWeb.AdminCoursesController do enable_achievements(:body, :boolean, "Enable achievements") enable_sourcecast(:body, :boolean, "Enable sourcecast") enable_stories(:body, :boolean, "Enable stories") + enable_llm_grading(:body, :boolean, "Enable LLM grading") + llm_api_key(:body, :string, "OpenAI API key for this course") sublanguage(:body, Schema.ref(:AdminSublanguage), "sublanguage object") module_help_text(:body, :string, "Module help text") end diff --git a/lib/cadet_web/controllers/courses_controller.ex b/lib/cadet_web/controllers/courses_controller.ex index 5b88be0dc..c717e3f06 100644 --- a/lib/cadet_web/controllers/courses_controller.ex +++ b/lib/cadet_web/controllers/courses_controller.ex @@ -57,6 +57,7 @@ defmodule CadetWeb.CoursesController do enable_sourcecast(:body, :boolean, "Enable sourcecast", required: true) enable_stories(:body, :boolean, "Enable stories", required: true) enable_llm_grading(:body, :boolean, "Enable LLM grading", required: false) + llm_api_key(:body, :string, "OpenAI API key for this course", required: false) source_chapter(:body, :number, "Default source chapter", required: true) source_variant(:body, Schema.ref(:SourceVariant), "Default source variant name", @@ -99,6 +100,7 @@ defmodule CadetWeb.CoursesController do enable_sourcecast(:boolean, "Enable sourcecast", required: true) enable_stories(:boolean, "Enable stories", required: true) enable_llm_grading(:boolean, "Enable LLM grading", required: false) + llm_api_key(:string, "OpenAI API key for this course", required: false) source_chapter(:integer, "Source Chapter number from 1 to 4", required: true) source_variant(Schema.ref(:SourceVariant), "Source Variant name", required: true) module_help_text(:string, "Module help text", required: true) @@ -114,6 +116,7 @@ defmodule CadetWeb.CoursesController do enable_sourcecast: true, enable_stories: false, enable_llm_grading: false, + llm_api_key: "sk-1234567890", source_chapter: 1, source_variant: "default", module_help_text: "Help text", diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index 8d68e243a..d8d53fd4a 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -10,7 +10,6 @@ defmodule CadetWeb.AICodeAnalysisController do @openai_api_url "https://api.openai.com/v1/chat/completions" @model "gpt-4o" - @api_key Application.get_env(:openai, :api_key) @default_llm_grading false # For logging outputs to both database and file @@ -62,14 +61,23 @@ defmodule CadetWeb.AICodeAnalysisController do case Courses.get_course_config(course_id) do {:ok, course} -> if course.enable_llm_grading == true || (course.enable_llm_grading == nil && @default_llm_grading == true) do - case Assessments.get_answers_in_submission(submission_id, question_id) do - {:ok, {answers, _assessment}} -> - analyze_code(conn, answers, submission_id, question_id) - - {:error, {status, message}} -> - conn - |> put_status(status) - |> text(message) + # Get API key from course config or fall back to environment variable + api_key = course.llm_api_key || Application.get_env(:openai, :api_key) + + if is_nil(api_key) do + conn + |> put_status(:internal_server_error) + |> json(%{"error" => "No OpenAI API key configured"}) + else + case Assessments.get_answers_in_submission(submission_id, question_id) do + {:ok, {answers, _assessment}} -> + analyze_code(conn, answers, submission_id, question_id, api_key) + + {:error, {status, message}} -> + conn + |> put_status(status) + |> text(message) + end end else conn @@ -129,7 +137,7 @@ defmodule CadetWeb.AICodeAnalysisController do """ end - defp analyze_code(conn, answers, submission_id, question_id) do + defp analyze_code(conn, answers, submission_id, question_id, api_key) do answers_json = answers |> Enum.map(fn answer -> @@ -233,7 +241,7 @@ defmodule CadetWeb.AICodeAnalysisController do } |> Jason.encode!() headers = [ - {"Authorization", "Bearer #{@api_key}"}, + {"Authorization", "Bearer #{api_key}"}, {"Content-Type", "application/json"} ] diff --git a/lib/cadet_web/views/courses_view.ex b/lib/cadet_web/views/courses_view.ex index a6ae9c4fa..819f4aebd 100644 --- a/lib/cadet_web/views/courses_view.ex +++ b/lib/cadet_web/views/courses_view.ex @@ -12,6 +12,8 @@ defmodule CadetWeb.CoursesView do enableAchievements: :enable_achievements, enableSourcecast: :enable_sourcecast, enableStories: :enable_stories, + enableLlmGrading: :enable_llm_grading, + llmApiKey: :llm_api_key, sourceChapter: :source_chapter, sourceVariant: :source_variant, moduleHelpText: :module_help_text, diff --git a/priv/repo/migrations/20240320000001_add_llm_api_key_to_courses.exs b/priv/repo/migrations/20240320000001_add_llm_api_key_to_courses.exs new file mode 100644 index 000000000..e752889b7 --- /dev/null +++ b/priv/repo/migrations/20240320000001_add_llm_api_key_to_courses.exs @@ -0,0 +1,15 @@ +defmodule Cadet.Repo.Migrations.AddLlmApiKeyToCourses do + use Ecto.Migration + + def up do + alter table(:courses) do + add(:llm_api_key, :string, null: true) + end + end + + def down do + alter table(:courses) do + remove(:llm_api_key) + end + end +end From 02f7ed178a2df244713462d89546ed36614fd245 Mon Sep 17 00:00:00 2001 From: Arul Date: Wed, 2 Apr 2025 16:32:10 +0800 Subject: [PATCH 12/20] feat: encryption for llm_api_key --- lib/cadet/courses/course.ex | 37 ++++++++++++++++ .../controllers/generate_ai_comments.ex | 42 +++++++++++++++++-- ...0402030934_increase_llm_api_key_length.exs | 9 ++++ 3 files changed, 85 insertions(+), 3 deletions(-) create mode 100644 priv/repo/migrations/20250402030934_increase_llm_api_key_length.exs diff --git a/lib/cadet/courses/course.ex b/lib/cadet/courses/course.ex index eb1fcb386..2261ba10d 100644 --- a/lib/cadet/courses/course.ex +++ b/lib/cadet/courses/course.ex @@ -48,11 +48,48 @@ defmodule Cadet.Courses.Course do enable_achievements enable_sourcecast enable_stories source_chapter source_variant)a @optional_fields ~w(course_short_name module_help_text enable_llm_grading llm_api_key)a + @spec changeset( + {map(), map()} + | %{ + :__struct__ => atom() | %{:__changeset__ => map(), optional(any()) => any()}, + optional(atom()) => any() + }, + %{optional(:__struct__) => none(), optional(atom() | binary()) => any()} + ) :: Ecto.Changeset.t() def changeset(course, params) do course |> cast(params, @required_fields ++ @optional_fields) |> validate_required(@required_fields) |> validate_sublanguage_combination(params) + |> put_encrypted_llm_api_key() + end + + def put_encrypted_llm_api_key(changeset) do + if llm_api_key = get_change(changeset, :llm_api_key) do + if is_binary(llm_api_key) and llm_api_key != "" do + secret = Application.get_env(:openai, :encryption_key) + if is_binary(secret) and byte_size(secret) >= 16 do + # Use first 16 bytes for AES-128, 24 for AES-192, or 32 for AES-256 + key = binary_part(secret, 0, min(32, byte_size(secret))) + # Use AES in GCM mode for encryption + iv = :crypto.strong_rand_bytes(16) + {ciphertext, tag} = :crypto.crypto_one_time_aead( + :aes_gcm, key, iv, llm_api_key, "", true + ) + # Store both the IV, ciphertext and tag + encrypted = iv <> tag <> ciphertext + put_change(changeset, :llm_api_key, Base.encode64(encrypted)) + else + add_error(changeset, :llm_api_key, "encryption key not configured properly") + end + else + # If empty string or nil is provided, don't encrypt but don't add error + changeset + end + else + # The key is not being changed, so we need to preserve the existing value + put_change(changeset, :llm_api_key, changeset.data.llm_api_key) + end end # Validates combination of Source chapter and variant diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index d8d53fd4a..6f0f2bf64 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -10,7 +10,7 @@ defmodule CadetWeb.AICodeAnalysisController do @openai_api_url "https://api.openai.com/v1/chat/completions" @model "gpt-4o" - @default_llm_grading false + @default_llm_grading false # To set whether LLM grading is enabled across Source Academy # For logging outputs to both database and file defp log_comment(submission_id, question_id, raw_prompt, answers_json, response, error \\ nil) do @@ -60,9 +60,11 @@ defmodule CadetWeb.AICodeAnalysisController do # Check if LLM grading is enabled for this course (default to @default_llm_grading if nil) case Courses.get_course_config(course_id) do {:ok, course} -> - if course.enable_llm_grading == true || (course.enable_llm_grading == nil && @default_llm_grading == true) do + if course.enable_llm_grading || @default_llm_grading do + Logger.info("LLM Api key: #{course.llm_api_key}") # Get API key from course config or fall back to environment variable - api_key = course.llm_api_key || Application.get_env(:openai, :api_key) + decrypted_api_key = decrypt_llm_api_key(course.llm_api_key) + api_key = decrypted_api_key || Application.get_env(:openai, :api_key) if is_nil(api_key) do conn @@ -349,4 +351,38 @@ defmodule CadetWeb.AICodeAnalysisController do end } end + + defp decrypt_llm_api_key(nil), do: nil + defp decrypt_llm_api_key(encrypted_key) do + try do + # Get the encryption key + secret = Application.get_env(:openai, :encryption_key) + + if is_binary(secret) and byte_size(secret) >= 16 do + # Use first 16 bytes for AES-128, 24 for AES-192, or 32 for AES-256 + key = binary_part(secret, 0, min(32, byte_size(secret))) + + # Decode the base64 string + decoded = Base.decode64!(encrypted_key) + + # Extract IV, tag and ciphertext + iv = binary_part(decoded, 0, 16) + tag = binary_part(decoded, 16, 16) + ciphertext = binary_part(decoded, 32, byte_size(decoded) - 32) + + # Decrypt + case :crypto.crypto_one_time_aead(:aes_gcm, key, iv, ciphertext, "", tag, false) do + plain_text when is_binary(plain_text) -> plain_text + _ -> nil + end + else + Logger.error("Encryption key not configured properly") + nil + end + rescue + e -> + Logger.error("Error decrypting LLM API key: #{inspect(e)}") + nil + end + end end diff --git a/priv/repo/migrations/20250402030934_increase_llm_api_key_length.exs b/priv/repo/migrations/20250402030934_increase_llm_api_key_length.exs new file mode 100644 index 000000000..295c83eea --- /dev/null +++ b/priv/repo/migrations/20250402030934_increase_llm_api_key_length.exs @@ -0,0 +1,9 @@ +defmodule Cadet.Repo.Migrations.IncreaseLlmApiKeyLength do + use Ecto.Migration + + def change do + alter table(:courses) do + modify :llm_api_key, :text + end + end +end From cb3498444727178d8776312cedb44840b9f49753 Mon Sep 17 00:00:00 2001 From: Arul Date: Sun, 6 Apr 2025 01:41:02 +0800 Subject: [PATCH 13/20] feat: added final comment editing route --- lib/cadet/ai_comments.ex | 29 +++++++++++-- .../controllers/generate_ai_comments.ex | 42 +++++++++---------- lib/cadet_web/router.ex | 6 +++ 3 files changed, 51 insertions(+), 26 deletions(-) diff --git a/lib/cadet/ai_comments.ex b/lib/cadet/ai_comments.ex index 3f42c9004..c0ddcde91 100644 --- a/lib/cadet/ai_comments.ex +++ b/lib/cadet/ai_comments.ex @@ -18,14 +18,27 @@ defmodule Cadet.AIComments do def get_ai_comment!(id), do: Repo.get!(AIComment, id) @doc """ - Gets AI comments for a specific submission and question. + Retrieves an AI comment for a specific submission and question. + Returns `nil` if no comment exists. """ def get_ai_comments_for_submission(submission_id, question_id) do + Repo.one( + from c in AIComment, + where: c.submission_id == ^submission_id and c.question_id == ^question_id + ) + end + + @doc """ + Retrieves the latest AI comment for a specific submission and question. + Returns `nil` if no comment exists. + """ + def get_latest_ai_comment(submission_id, question_id) do from(c in AIComment, where: c.submission_id == ^submission_id and c.question_id == ^question_id, - order_by: [desc: c.inserted_at] + order_by: [desc: c.inserted_at], + limit: 1 ) - |> Repo.all() + |> Repo.one() end @doc """ @@ -47,4 +60,14 @@ defmodule Cadet.AIComments do |> Repo.update() end end + + @doc """ + Updates an existing AI comment with new attributes. + """ + def update_ai_comment(id, attrs) do + id + |> get_ai_comment!() + |> AIComment.changeset(attrs) + |> Repo.update() + end end diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index 6f0f2bf64..652645b0a 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -25,30 +25,26 @@ defmodule CadetWeb.AICodeAnalysisController do inserted_at: NaiveDateTime.utc_now() } - case AIComments.create_ai_comment(attrs) do - {:ok, comment} -> {:ok, comment} - {:error, changeset} -> - Logger.error("Failed to log AI comment to database: #{inspect(changeset.errors)}") - {:error, changeset} - end - - # Log to file - try do - log_file = "log/ai_comments.csv" - File.mkdir_p!("log") - - timestamp = NaiveDateTime.utc_now() |> NaiveDateTime.to_string() - raw_prompt_str = Jason.encode!(raw_prompt) |> String.replace("\"", "\"\"") - answers_json_str = answers_json |> String.replace("\"", "\"\"") - response_str = if is_nil(response), do: "", else: response |> String.replace("\"", "\"\"") - error_str = if is_nil(error), do: "", else: error |> String.replace("\"", "\"\"") - - csv_row = "\"#{timestamp}\",\"#{submission_id}\",\"#{question_id}\",\"#{raw_prompt_str}\",\"#{answers_json_str}\",\"#{response_str}\",\"#{error_str}\"\n" + # Check if a comment already exists for the given submission_id and question_id + case AIComments.get_latest_ai_comment(submission_id, question_id) do + nil -> + # If no existing comment, create a new one + case AIComments.create_ai_comment(attrs) do + {:ok, comment} -> {:ok, comment} + {:error, changeset} -> + Logger.error("Failed to log AI comment to database: #{inspect(changeset.errors)}") + {:error, changeset} + end - File.write!(log_file, csv_row, [:append]) - rescue - e -> - Logger.error("Failed to log AI comment to file: #{inspect(e)}") + existing_comment -> + # If a comment exists, update it with the new data + updated_attrs = Map.merge(existing_comment, attrs) + case AIComments.update_ai_comment(existing_comment.id, updated_attrs) do + {:ok, updated_comment} -> {:ok, updated_comment} + {:error, changeset} -> + Logger.error("Failed to update AI comment in database: #{inspect(changeset.errors)}") + {:error, changeset} + end end end diff --git a/lib/cadet_web/router.ex b/lib/cadet_web/router.ex index 3e8c1a155..18aead900 100644 --- a/lib/cadet_web/router.ex +++ b/lib/cadet_web/router.ex @@ -213,6 +213,12 @@ defmodule CadetWeb.Router do :autograde_answer ) + post( + "/save-final-comment/:submissionid/:questionid", + AICodeAnalysisController, + :save_final_comment + ) + get("/users", AdminUserController, :index) get("/users/teamformation", AdminUserController, :get_students) put("/users", AdminUserController, :upsert_users_and_groups) From 09a7b093618ea0bfc09282cf38789dab8204c3ae Mon Sep 17 00:00:00 2001 From: Arul Date: Sun, 6 Apr 2025 13:56:43 +0800 Subject: [PATCH 14/20] feat: added logging of chosen comments --- lib/cadet/ai_comments.ex | 20 +++++++++++++++++++ lib/cadet/ai_comments/ai_comment.ex | 2 +- .../controllers/generate_ai_comments.ex | 16 +++++++++++++++ lib/cadet_web/router.ex | 6 ++++++ ...6053008_update_comment_chosen_to_array.exs | 17 ++++++++++++++++ 5 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 priv/repo/migrations/20250406053008_update_comment_chosen_to_array.exs diff --git a/lib/cadet/ai_comments.ex b/lib/cadet/ai_comments.ex index c0ddcde91..cbdae755c 100644 --- a/lib/cadet/ai_comments.ex +++ b/lib/cadet/ai_comments.ex @@ -70,4 +70,24 @@ defmodule Cadet.AIComments do |> AIComment.changeset(attrs) |> Repo.update() end + + @doc """ + Updates the chosen comments for a specific submission and question. + Accepts an array of comments and replaces the existing array in the database. + """ + def update_chosen_comments(submission_id, question_id, new_comments) do + from(c in AIComment, + where: c.submission_id == ^submission_id and c.question_id == ^question_id, + order_by: [desc: c.inserted_at], + limit: 1 + ) + |> Repo.one() + |> case do + nil -> {:error, :not_found} + comment -> + comment + |> AIComment.changeset(%{comment_chosen: new_comments}) + |> Repo.update() + end + end end diff --git a/lib/cadet/ai_comments/ai_comment.ex b/lib/cadet/ai_comments/ai_comment.ex index 575d1cc10..55c01802c 100644 --- a/lib/cadet/ai_comments/ai_comment.ex +++ b/lib/cadet/ai_comments/ai_comment.ex @@ -9,7 +9,7 @@ defmodule Cadet.AIComments.AIComment do field :answers_json, :string field :response, :string field :error, :string - field :comment_chosen, :string + field :comment_chosen, {:array, :string} field :final_comment, :string timestamps() diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index 652645b0a..45b5e830b 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -286,6 +286,22 @@ defmodule CadetWeb.AICodeAnalysisController do end end + @doc """ + Saves the chosen comments for a submission and question. + Accepts an array of comments in the request body. + """ + def save_chosen_comments(conn, %{"submissionid" => submission_id, "questionid" => question_id, "comments" => comments}) do + case AIComments.update_chosen_comments(submission_id, question_id, comments) do + {:ok, _updated_comment} -> + json(conn, %{"status" => "success"}) + + {:error, changeset} -> + conn + |> put_status(:unprocessable_entity) + |> json(%{"error" => "Failed to save chosen comments"}) + end + end + swagger_path :generate_ai_comments do post("/courses/{courseId}/admin/generate-comments/{submissionId}/{questionId}") diff --git a/lib/cadet_web/router.ex b/lib/cadet_web/router.ex index 18aead900..c64631cf8 100644 --- a/lib/cadet_web/router.ex +++ b/lib/cadet_web/router.ex @@ -219,6 +219,12 @@ defmodule CadetWeb.Router do :save_final_comment ) + post( + "/save-chosen-comments/:submissionid/:questionid", + AICodeAnalysisController, + :save_chosen_comments + ) + get("/users", AdminUserController, :index) get("/users/teamformation", AdminUserController, :get_students) put("/users", AdminUserController, :upsert_users_and_groups) diff --git a/priv/repo/migrations/20250406053008_update_comment_chosen_to_array.exs b/priv/repo/migrations/20250406053008_update_comment_chosen_to_array.exs new file mode 100644 index 000000000..112f8b4c2 --- /dev/null +++ b/priv/repo/migrations/20250406053008_update_comment_chosen_to_array.exs @@ -0,0 +1,17 @@ +defmodule Cadet.Repo.Migrations.UpdateCommentChosenToArray do + use Ecto.Migration + + def change do + alter table(:ai_comment_logs) do + add :comment_chosen_temp, {:array, :string}, default: [] + end + + execute "UPDATE ai_comment_logs SET comment_chosen_temp = ARRAY[comment_chosen]" + + alter table(:ai_comment_logs) do + remove :comment_chosen + end + + rename table(:ai_comment_logs), :comment_chosen_temp, to: :comment_chosen + end +end From ed44a7e7ca88d6634fae5b3412f164145a927800 Mon Sep 17 00:00:00 2001 From: Arul Date: Sun, 6 Apr 2025 14:30:24 +0800 Subject: [PATCH 15/20] fix: bugs when certain fields were missing --- .../controllers/generate_ai_comments.ex | 26 +++++++++++++------ 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index 45b5e830b..fcbf324ca 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -37,8 +37,8 @@ defmodule CadetWeb.AICodeAnalysisController do end existing_comment -> - # If a comment exists, update it with the new data - updated_attrs = Map.merge(existing_comment, attrs) + # Convert the existing comment struct to a map before merging + updated_attrs = Map.merge(Map.from_struct(existing_comment), attrs) case AIComments.update_ai_comment(existing_comment.id, updated_attrs) do {:ok, updated_comment} -> {:ok, updated_comment} {:error, changeset} -> @@ -114,27 +114,37 @@ defmodule CadetWeb.AICodeAnalysisController do defp format_answer(answer) do """ - **Question ID: #{answer["question"]["id"]}** + **Question ID: #{answer["question"]["id"] || "N/A"}** **Question:** - #{answer["question"]["content"]} + #{answer["question"]["content"] || "N/A"} **Solution:** ``` - #{answer["question"]["solution"]} + #{answer["question"]["solution"] || "N/A"} ``` **Answer:** ``` - #{answer["answer"]["code"]} + #{answer["answer"]["code"] || "N/A"} ``` - **Autograding Status:** #{answer["autograding_status"]} - **Autograding Results:** #{answer["autograding_results"]} + **Autograding Status:** #{answer["autograding_status"] || "N/A"} + **Autograding Results:** #{format_autograding_results(answer["autograding_results"])} **Comments:** #{answer["comments"] || "None"} """ end + defp format_autograding_results(nil), do: "N/A" + defp format_autograding_results(results) when is_list(results) do + results + |> Enum.map(fn result -> + "Error: #{result["errorMessage"] || "N/A"}, Type: #{result["errorType"] || "N/A"}" + end) + |> Enum.join("; ") + end + defp format_autograding_results(results), do: inspect(results) + defp analyze_code(conn, answers, submission_id, question_id, api_key) do answers_json = answers From 3715368fad436211aee3404784de325b4753fd96 Mon Sep 17 00:00:00 2001 From: Arul Date: Sun, 6 Apr 2025 14:36:59 +0800 Subject: [PATCH 16/20] feat: updated tests --- .../ai_code_analysis_controller_test.exs | 49 ++++++++++++++++++ test/support/ai_comments_test_helpers.ex | 50 ------------------- 2 files changed, 49 insertions(+), 50 deletions(-) delete mode 100644 test/support/ai_comments_test_helpers.ex diff --git a/test/cadet_web/controllers/ai_code_analysis_controller_test.exs b/test/cadet_web/controllers/ai_code_analysis_controller_test.exs index 559c4ea9e..54492c1ee 100644 --- a/test/cadet_web/controllers/ai_code_analysis_controller_test.exs +++ b/test/cadet_web/controllers/ai_code_analysis_controller_test.exs @@ -121,4 +121,53 @@ defmodule CadetWeb.AICodeAnalysisControllerTest do assert response["error"] == "Failed to save final comment" end end + + describe "save_chosen_comments" do + test "successfully saves chosen comments", %{conn: conn} do + # First create a comment entry + submission_id = 123 + question_id = 456 + raw_prompt = "Test prompt" + answers_json = ~s({"test": "data"}) + response = "Comment 1|||Comment 2|||Comment 3" + + {:ok, _comment} = AIComments.create_ai_comment(%{ + submission_id: submission_id, + question_id: question_id, + raw_prompt: raw_prompt, + answers_json: answers_json, + response: response + }) + + # Now save the chosen comments + chosen_comments = ["Comment 1", "Comment 2"] + response = + conn + |> post(Routes.ai_code_analysis_path(conn, :save_chosen_comments, submission_id, question_id), %{ + comments: chosen_comments + }) + |> json_response(200) + + assert response["status"] == "success" + + # Verify the chosen comments were saved + comment = Repo.get_by(AIComment, submission_id: submission_id, question_id: question_id) + assert comment.comment_chosen == chosen_comments + end + + test "returns error when no comment exists", %{conn: conn} do + submission_id = 999 + question_id = 888 + chosen_comments = ["Comment 1", "Comment 2"] + + response = + conn + |> post(Routes.ai_code_analysis_path(conn, :save_chosen_comments, submission_id, question_id), %{ + comments: chosen_comments + }) + |> json_response(422) + + assert response["error"] == "Failed to save chosen comments" + end + end end diff --git a/test/support/ai_comments_test_helpers.ex b/test/support/ai_comments_test_helpers.ex deleted file mode 100644 index a359075b5..000000000 --- a/test/support/ai_comments_test_helpers.ex +++ /dev/null @@ -1,50 +0,0 @@ -defmodule Cadet.AICommentsTestHelpers do - @moduledoc """ - Helper functions for testing AI comments functionality. - """ - - alias Cadet.Repo - alias Cadet.AIComments.AIComment - import Ecto.Query - - @doc """ - Gets the latest AI comment from the database. - """ - def get_latest_comment do - AIComment - |> first(order_by: [desc: :inserted_at]) - |> Repo.one() - end - - @doc """ - Gets all AI comments for a specific submission and question. - """ - def get_comments_for_submission(submission_id, question_id) do - from(c in AIComment, - where: c.submission_id == ^submission_id and c.question_id == ^question_id, - order_by: [desc: c.inserted_at] - ) - |> Repo.all() - end - - @doc """ - Reads the CSV log file and returns its contents. - """ - def read_csv_log do - log_file = "log/ai_comments.csv" - if File.exists?(log_file) do - File.read!(log_file) - else - "" - end - end - - @doc """ - Cleans up test artifacts. - """ - def cleanup_test_artifacts do - log_file = "log/ai_comments.csv" - File.rm(log_file) - Repo.delete_all(AIComment) - end -end From 5bfe276df8508e6487bebd873288a6ab819a4678 Mon Sep 17 00:00:00 2001 From: Arul Date: Sun, 6 Apr 2025 15:54:43 +0800 Subject: [PATCH 17/20] formatting --- lib/cadet/ai_comments.ex | 11 +- lib/cadet/ai_comments/ai_comment.ex | 27 +- lib/cadet/courses/course.ex | 15 +- .../controllers/generate_ai_comments.ex | 270 ++++++++++-------- ...0402030934_increase_llm_api_key_length.exs | 2 +- ...6053008_update_comment_chosen_to_array.exs | 8 +- .../ai_code_analysis_controller_test.exs | 76 +++-- 7 files changed, 247 insertions(+), 162 deletions(-) diff --git a/lib/cadet/ai_comments.ex b/lib/cadet/ai_comments.ex index cbdae755c..eaf91a344 100644 --- a/lib/cadet/ai_comments.ex +++ b/lib/cadet/ai_comments.ex @@ -23,8 +23,9 @@ defmodule Cadet.AIComments do """ def get_ai_comments_for_submission(submission_id, question_id) do Repo.one( - from c in AIComment, + from(c in AIComment, where: c.submission_id == ^submission_id and c.question_id == ^question_id + ) ) end @@ -53,7 +54,9 @@ defmodule Cadet.AIComments do ) |> Repo.one() |> case do - nil -> {:error, :not_found} + nil -> + {:error, :not_found} + comment -> comment |> AIComment.changeset(%{final_comment: final_comment}) @@ -83,7 +86,9 @@ defmodule Cadet.AIComments do ) |> Repo.one() |> case do - nil -> {:error, :not_found} + nil -> + {:error, :not_found} + comment -> comment |> AIComment.changeset(%{comment_chosen: new_comments}) diff --git a/lib/cadet/ai_comments/ai_comment.ex b/lib/cadet/ai_comments/ai_comment.ex index 55c01802c..0cefe9530 100644 --- a/lib/cadet/ai_comments/ai_comment.ex +++ b/lib/cadet/ai_comments/ai_comment.ex @@ -3,21 +3,30 @@ defmodule Cadet.AIComments.AIComment do import Ecto.Changeset schema "ai_comment_logs" do - field :submission_id, :integer - field :question_id, :integer - field :raw_prompt, :string - field :answers_json, :string - field :response, :string - field :error, :string - field :comment_chosen, {:array, :string} - field :final_comment, :string + field(:submission_id, :integer) + field(:question_id, :integer) + field(:raw_prompt, :string) + field(:answers_json, :string) + field(:response, :string) + field(:error, :string) + field(:comment_chosen, {:array, :string}) + field(:final_comment, :string) timestamps() end def changeset(ai_comment, attrs) do ai_comment - |> cast(attrs, [:submission_id, :question_id, :raw_prompt, :answers_json, :response, :error, :comment_chosen, :final_comment]) + |> cast(attrs, [ + :submission_id, + :question_id, + :raw_prompt, + :answers_json, + :response, + :error, + :comment_chosen, + :final_comment + ]) |> validate_required([:submission_id, :question_id, :raw_prompt, :answers_json]) end end diff --git a/lib/cadet/courses/course.ex b/lib/cadet/courses/course.ex index 2261ba10d..cf6170cfb 100644 --- a/lib/cadet/courses/course.ex +++ b/lib/cadet/courses/course.ex @@ -68,14 +68,23 @@ defmodule Cadet.Courses.Course do if llm_api_key = get_change(changeset, :llm_api_key) do if is_binary(llm_api_key) and llm_api_key != "" do secret = Application.get_env(:openai, :encryption_key) + if is_binary(secret) and byte_size(secret) >= 16 do # Use first 16 bytes for AES-128, 24 for AES-192, or 32 for AES-256 key = binary_part(secret, 0, min(32, byte_size(secret))) # Use AES in GCM mode for encryption iv = :crypto.strong_rand_bytes(16) - {ciphertext, tag} = :crypto.crypto_one_time_aead( - :aes_gcm, key, iv, llm_api_key, "", true - ) + + {ciphertext, tag} = + :crypto.crypto_one_time_aead( + :aes_gcm, + key, + iv, + llm_api_key, + "", + true + ) + # Store both the IV, ciphertext and tag encrypted = iv <> tag <> ciphertext put_change(changeset, :llm_api_key, Base.encode64(encrypted)) diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index fcbf324ca..48b81bc6b 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -10,7 +10,8 @@ defmodule CadetWeb.AICodeAnalysisController do @openai_api_url "https://api.openai.com/v1/chat/completions" @model "gpt-4o" - @default_llm_grading false # To set whether LLM grading is enabled across Source Academy + # To set whether LLM grading is enabled across Source Academy + @default_llm_grading false # For logging outputs to both database and file defp log_comment(submission_id, question_id, raw_prompt, answers_json, response, error \\ nil) do @@ -30,7 +31,9 @@ defmodule CadetWeb.AICodeAnalysisController do nil -> # If no existing comment, create a new one case AIComments.create_ai_comment(attrs) do - {:ok, comment} -> {:ok, comment} + {:ok, comment} -> + {:ok, comment} + {:error, changeset} -> Logger.error("Failed to log AI comment to database: #{inspect(changeset.errors)}") {:error, changeset} @@ -39,8 +42,11 @@ defmodule CadetWeb.AICodeAnalysisController do existing_comment -> # Convert the existing comment struct to a map before merging updated_attrs = Map.merge(Map.from_struct(existing_comment), attrs) + case AIComments.update_ai_comment(existing_comment.id, updated_attrs) do - {:ok, updated_comment} -> {:ok, updated_comment} + {:ok, updated_comment} -> + {:ok, updated_comment} + {:error, changeset} -> Logger.error("Failed to update AI comment in database: #{inspect(changeset.errors)}") {:error, changeset} @@ -51,43 +57,47 @@ defmodule CadetWeb.AICodeAnalysisController do @doc """ Fetches the question details and answers based on submissionid and questionid and generates AI-generated comments. """ - def generate_ai_comments(conn, %{"submissionid" => submission_id, "questionid" => question_id, "course_id" => course_id}) - when is_ecto_id(submission_id) do - # Check if LLM grading is enabled for this course (default to @default_llm_grading if nil) - case Courses.get_course_config(course_id) do - {:ok, course} -> - if course.enable_llm_grading || @default_llm_grading do - Logger.info("LLM Api key: #{course.llm_api_key}") - # Get API key from course config or fall back to environment variable - decrypted_api_key = decrypt_llm_api_key(course.llm_api_key) - api_key = decrypted_api_key || Application.get_env(:openai, :api_key) - - if is_nil(api_key) do - conn - |> put_status(:internal_server_error) - |> json(%{"error" => "No OpenAI API key configured"}) - else - case Assessments.get_answers_in_submission(submission_id, question_id) do - {:ok, {answers, _assessment}} -> - analyze_code(conn, answers, submission_id, question_id, api_key) - - {:error, {status, message}} -> - conn - |> put_status(status) - |> text(message) - end - end - else + def generate_ai_comments(conn, %{ + "submissionid" => submission_id, + "questionid" => question_id, + "course_id" => course_id + }) + when is_ecto_id(submission_id) do + # Check if LLM grading is enabled for this course (default to @default_llm_grading if nil) + case Courses.get_course_config(course_id) do + {:ok, course} -> + if course.enable_llm_grading || @default_llm_grading do + Logger.info("LLM Api key: #{course.llm_api_key}") + # Get API key from course config or fall back to environment variable + decrypted_api_key = decrypt_llm_api_key(course.llm_api_key) + api_key = decrypted_api_key || Application.get_env(:openai, :api_key) + + if is_nil(api_key) do conn - |> put_status(:forbidden) - |> json(%{"error" => "LLM grading is not enabled for this course"}) + |> put_status(:internal_server_error) + |> json(%{"error" => "No OpenAI API key configured"}) + else + case Assessments.get_answers_in_submission(submission_id, question_id) do + {:ok, {answers, _assessment}} -> + analyze_code(conn, answers, submission_id, question_id, api_key) + + {:error, {status, message}} -> + conn + |> put_status(status) + |> text(message) + end end - - {:error, {status, message}} -> + else conn - |> put_status(status) - |> text(message) - end + |> put_status(:forbidden) + |> json(%{"error" => "LLM grading is not enabled for this course"}) + end + + {:error, {status, message}} -> + conn + |> put_status(status) + |> text(message) + end end defp transform_answers(answers) do @@ -136,6 +146,7 @@ defmodule CadetWeb.AICodeAnalysisController do end defp format_autograding_results(nil), do: "N/A" + defp format_autograding_results(results) when is_list(results) do results |> Enum.map(fn result -> @@ -143,6 +154,7 @@ defmodule CadetWeb.AICodeAnalysisController do end) |> Enum.join("; ") end + defp format_autograding_results(results), do: inspect(results) defp analyze_code(conn, answers, submission_id, question_id, api_key) do @@ -164,6 +176,7 @@ defmodule CadetWeb.AICodeAnalysisController do llm_prompt: nil } end + answer |> Map.from_struct() |> Map.take([ @@ -171,89 +184,91 @@ defmodule CadetWeb.AICodeAnalysisController do :comments, :autograding_status, :autograding_results, - :answer, + :answer ]) |> Map.put(:question, question_data) end) |> Jason.encode!() |> format_answers() - raw_prompt = """ - The code below is written in Source, a variant of JavaScript that comes with a rich set of built-in constants and functions. Below is a summary of some key built-in entities available in Source: - - Constants: - - Infinity: The special number value representing infinity. - - NaN: The special number value for "not a number." - - undefined: The special value for an undefined variable. - - math_PI: The constant π (approximately 3.14159). - - math_E: Euler's number (approximately 2.71828). - - Functions: - - __access_export__(exports, lookup_name): Searches for a name in an exports data structure. - - accumulate(f, initial, xs): Reduces a list by applying a binary function from right-to-left. - - append(xs, ys): Appends list ys to the end of list xs. - - char_at(s, i): Returns the character at index i of string s. - - display(v, s): Displays value v (optionally preceded by string s) in the console. - - filter(pred, xs): Returns a new list with elements of xs that satisfy the predicate pred. - - for_each(f, xs): Applies function f to each element of the list xs. - - get_time(): Returns the current time in milliseconds. - - is_list(xs): Checks whether xs is a proper list. - - length(xs): Returns the number of elements in list xs. - - list(...): Constructs a list from the provided values. - - map(f, xs): Applies function f to each element of list xs. - - math_abs(x): Returns the absolute value of x. - - math_ceil(x): Rounds x up to the nearest integer. - - math_floor(x): Rounds x down to the nearest integer. - - pair(x, y): A primitive function that makes a pair whose head (first component) is x and whose tail (second component) is y. - - head(xs): Returns the first element of pair xs. - - tail(xs): Returns the second element of pair xs. - - math_random(): Returns a random number between 0 (inclusive) and 1 (exclusive). - - (For a full list of built-in functions and constants, refer to the Source documentation.) - - Analyze the following submitted answers and provide detailed feedback on correctness, readability, efficiency, and possible improvements. Your evaluation should consider both standard JavaScript features and the additional built-in functions unique to Source. - - Provide between 3 and 5 concise comment suggestions, each under 200 words. - - Your output must include only the comment suggestions, separated exclusively by triple pipes ("|||") with no spaces before or after the pipes, and without any additional formatting, bullet points, or extra text. - - Comments and documentation in the code are not necessary for the code, do not penalise based on that, do not suggest to add comments as well. - - Follow the XP scoring guideline provided below in the question prompt, do not be too harsh! - - For example: "This is a good answer.|||This is a bad answer.|||This is a great answer." - """ - # Get the llm_prompt from the first answer's question - llm_prompt = - answers - |> List.first() - |> Map.get(:question) - |> Map.get(:question) - |> Map.get("llm_prompt") - - # Combine prompts if llm_prompt exists - prompt = - if llm_prompt && llm_prompt != "" do - raw_prompt <> "Additional Instructions:\n\n" <> llm_prompt <> "\n\n" <> answers_json - else - raw_prompt <> "\n" <> answers_json - end + raw_prompt = """ + The code below is written in Source, a variant of JavaScript that comes with a rich set of built-in constants and functions. Below is a summary of some key built-in entities available in Source: + + Constants: + - Infinity: The special number value representing infinity. + - NaN: The special number value for "not a number." + - undefined: The special value for an undefined variable. + - math_PI: The constant π (approximately 3.14159). + - math_E: Euler's number (approximately 2.71828). + + Functions: + - __access_export__(exports, lookup_name): Searches for a name in an exports data structure. + - accumulate(f, initial, xs): Reduces a list by applying a binary function from right-to-left. + - append(xs, ys): Appends list ys to the end of list xs. + - char_at(s, i): Returns the character at index i of string s. + - display(v, s): Displays value v (optionally preceded by string s) in the console. + - filter(pred, xs): Returns a new list with elements of xs that satisfy the predicate pred. + - for_each(f, xs): Applies function f to each element of the list xs. + - get_time(): Returns the current time in milliseconds. + - is_list(xs): Checks whether xs is a proper list. + - length(xs): Returns the number of elements in list xs. + - list(...): Constructs a list from the provided values. + - map(f, xs): Applies function f to each element of list xs. + - math_abs(x): Returns the absolute value of x. + - math_ceil(x): Rounds x up to the nearest integer. + - math_floor(x): Rounds x down to the nearest integer. + - pair(x, y): A primitive function that makes a pair whose head (first component) is x and whose tail (second component) is y. + - head(xs): Returns the first element of pair xs. + - tail(xs): Returns the second element of pair xs. + - math_random(): Returns a random number between 0 (inclusive) and 1 (exclusive). + + (For a full list of built-in functions and constants, refer to the Source documentation.) + + Analyze the following submitted answers and provide detailed feedback on correctness, readability, efficiency, and possible improvements. Your evaluation should consider both standard JavaScript features and the additional built-in functions unique to Source. + + Provide between 3 and 5 concise comment suggestions, each under 200 words. + + Your output must include only the comment suggestions, separated exclusively by triple pipes ("|||") with no spaces before or after the pipes, and without any additional formatting, bullet points, or extra text. + + Comments and documentation in the code are not necessary for the code, do not penalise based on that, do not suggest to add comments as well. + + Follow the XP scoring guideline provided below in the question prompt, do not be too harsh! + + For example: "This is a good answer.|||This is a bad answer.|||This is a great answer." + """ + + # Get the llm_prompt from the first answer's question + llm_prompt = + answers + |> List.first() + |> Map.get(:question) + |> Map.get(:question) + |> Map.get("llm_prompt") + + # Combine prompts if llm_prompt exists + prompt = + if llm_prompt && llm_prompt != "" do + raw_prompt <> "Additional Instructions:\n\n" <> llm_prompt <> "\n\n" <> answers_json + else + raw_prompt <> "\n" <> answers_json + end - input = %{ - model: @model, - messages: [ - %{role: "system", content: "You are an expert software engineer and educator."}, - %{role: "user", content: prompt} - ], - temperature: 0.5 - } |> Jason.encode!() + input = + %{ + model: @model, + messages: [ + %{role: "system", content: "You are an expert software engineer and educator."}, + %{role: "user", content: prompt} + ], + temperature: 0.5 + } + |> Jason.encode!() headers = [ {"Authorization", "Bearer #{api_key}"}, {"Content-Type", "application/json"} ] - case HTTPoison.post(@openai_api_url, input, headers, timeout: 60_000, recv_timeout: 60_000) do {:ok, %HTTPoison.Response{status_code: 200, body: body}} -> case Jason.decode(body) do @@ -261,19 +276,36 @@ defmodule CadetWeb.AICodeAnalysisController do log_comment(submission_id, question_id, prompt, answers_json, response) comments_list = String.split(response, "|||") - filtered_comments = Enum.filter(comments_list, fn comment -> - String.trim(comment) != "" - end) + filtered_comments = + Enum.filter(comments_list, fn comment -> + String.trim(comment) != "" + end) json(conn, %{"comments" => filtered_comments}) {:error, _} -> - log_comment(submission_id, question_id, prompt, answers_json, nil, "Failed to parse response from OpenAI API") + log_comment( + submission_id, + question_id, + prompt, + answers_json, + nil, + "Failed to parse response from OpenAI API" + ) + json(conn, %{"error" => "Failed to parse response from OpenAI API"}) end {:ok, %HTTPoison.Response{status_code: status, body: body}} -> - log_comment(submission_id, question_id, prompt, answers_json, nil, "API request failed with status #{status}") + log_comment( + submission_id, + question_id, + prompt, + answers_json, + nil, + "API request failed with status #{status}" + ) + json(conn, %{"error" => "API request failed with status #{status}: #{body}"}) {:error, %HTTPoison.Error{reason: reason}} -> @@ -285,10 +317,15 @@ defmodule CadetWeb.AICodeAnalysisController do @doc """ Saves the final comment chosen for a submission. """ - def save_final_comment(conn, %{"submissionid" => submission_id, "questionid" => question_id, "comment" => comment}) do + def save_final_comment(conn, %{ + "submissionid" => submission_id, + "questionid" => question_id, + "comment" => comment + }) do case AIComments.update_final_comment(submission_id, question_id, comment) do {:ok, _updated_comment} -> json(conn, %{"status" => "success"}) + {:error, changeset} -> conn |> put_status(:unprocessable_entity) @@ -300,7 +337,11 @@ defmodule CadetWeb.AICodeAnalysisController do Saves the chosen comments for a submission and question. Accepts an array of comments in the request body. """ - def save_chosen_comments(conn, %{"submissionid" => submission_id, "questionid" => question_id, "comments" => comments}) do + def save_chosen_comments(conn, %{ + "submissionid" => submission_id, + "questionid" => question_id, + "comments" => comments + }) do case AIComments.update_chosen_comments(submission_id, question_id, comments) do {:ok, _updated_comment} -> json(conn, %{"status" => "success"}) @@ -375,6 +416,7 @@ defmodule CadetWeb.AICodeAnalysisController do end defp decrypt_llm_api_key(nil), do: nil + defp decrypt_llm_api_key(encrypted_key) do try do # Get the encryption key diff --git a/priv/repo/migrations/20250402030934_increase_llm_api_key_length.exs b/priv/repo/migrations/20250402030934_increase_llm_api_key_length.exs index 295c83eea..ec3318eda 100644 --- a/priv/repo/migrations/20250402030934_increase_llm_api_key_length.exs +++ b/priv/repo/migrations/20250402030934_increase_llm_api_key_length.exs @@ -3,7 +3,7 @@ defmodule Cadet.Repo.Migrations.IncreaseLlmApiKeyLength do def change do alter table(:courses) do - modify :llm_api_key, :text + modify(:llm_api_key, :text) end end end diff --git a/priv/repo/migrations/20250406053008_update_comment_chosen_to_array.exs b/priv/repo/migrations/20250406053008_update_comment_chosen_to_array.exs index 112f8b4c2..751cf9b7d 100644 --- a/priv/repo/migrations/20250406053008_update_comment_chosen_to_array.exs +++ b/priv/repo/migrations/20250406053008_update_comment_chosen_to_array.exs @@ -3,15 +3,15 @@ defmodule Cadet.Repo.Migrations.UpdateCommentChosenToArray do def change do alter table(:ai_comment_logs) do - add :comment_chosen_temp, {:array, :string}, default: [] + add(:comment_chosen_temp, {:array, :string}, default: []) end - execute "UPDATE ai_comment_logs SET comment_chosen_temp = ARRAY[comment_chosen]" + execute("UPDATE ai_comment_logs SET comment_chosen_temp = ARRAY[comment_chosen]") alter table(:ai_comment_logs) do - remove :comment_chosen + remove(:comment_chosen) end - rename table(:ai_comment_logs), :comment_chosen_temp, to: :comment_chosen + rename(table(:ai_comment_logs), :comment_chosen_temp, to: :comment_chosen) end end diff --git a/test/cadet_web/controllers/ai_code_analysis_controller_test.exs b/test/cadet_web/controllers/ai_code_analysis_controller_test.exs index 54492c1ee..c17e57174 100644 --- a/test/cadet_web/controllers/ai_code_analysis_controller_test.exs +++ b/test/cadet_web/controllers/ai_code_analysis_controller_test.exs @@ -22,7 +22,9 @@ defmodule CadetWeb.AICodeAnalysisControllerTest do # Make the API call response = conn - |> post(Routes.ai_code_analysis_path(conn, :generate_ai_comments, submission_id, question_id)) + |> post( + Routes.ai_code_analysis_path(conn, :generate_ai_comments, submission_id, question_id) + ) |> json_response(200) # Verify database entry @@ -52,7 +54,9 @@ defmodule CadetWeb.AICodeAnalysisControllerTest do # Make the API call that should fail response = conn - |> post(Routes.ai_code_analysis_path(conn, :generate_ai_comments, submission_id, question_id)) + |> post( + Routes.ai_code_analysis_path(conn, :generate_ai_comments, submission_id, question_id) + ) |> json_response(400) # Verify error is logged in database @@ -82,21 +86,26 @@ defmodule CadetWeb.AICodeAnalysisControllerTest do answers_json = ~s({"test": "data"}) response = "Comment 1|||Comment 2|||Comment 3" - {:ok, _comment} = AIComments.create_ai_comment(%{ - submission_id: submission_id, - question_id: question_id, - raw_prompt: raw_prompt, - answers_json: answers_json, - response: response - }) + {:ok, _comment} = + AIComments.create_ai_comment(%{ + submission_id: submission_id, + question_id: question_id, + raw_prompt: raw_prompt, + answers_json: answers_json, + response: response + }) # Now save the final comment final_comment = "This is the chosen final comment" + response = conn - |> post(Routes.ai_code_analysis_path(conn, :save_final_comment, submission_id, question_id), %{ - comment: final_comment - }) + |> post( + Routes.ai_code_analysis_path(conn, :save_final_comment, submission_id, question_id), + %{ + comment: final_comment + } + ) |> json_response(200) assert response["status"] == "success" @@ -113,9 +122,12 @@ defmodule CadetWeb.AICodeAnalysisControllerTest do response = conn - |> post(Routes.ai_code_analysis_path(conn, :save_final_comment, submission_id, question_id), %{ - comment: final_comment - }) + |> post( + Routes.ai_code_analysis_path(conn, :save_final_comment, submission_id, question_id), + %{ + comment: final_comment + } + ) |> json_response(422) assert response["error"] == "Failed to save final comment" @@ -131,21 +143,26 @@ defmodule CadetWeb.AICodeAnalysisControllerTest do answers_json = ~s({"test": "data"}) response = "Comment 1|||Comment 2|||Comment 3" - {:ok, _comment} = AIComments.create_ai_comment(%{ - submission_id: submission_id, - question_id: question_id, - raw_prompt: raw_prompt, - answers_json: answers_json, - response: response - }) + {:ok, _comment} = + AIComments.create_ai_comment(%{ + submission_id: submission_id, + question_id: question_id, + raw_prompt: raw_prompt, + answers_json: answers_json, + response: response + }) # Now save the chosen comments chosen_comments = ["Comment 1", "Comment 2"] + response = conn - |> post(Routes.ai_code_analysis_path(conn, :save_chosen_comments, submission_id, question_id), %{ - comments: chosen_comments - }) + |> post( + Routes.ai_code_analysis_path(conn, :save_chosen_comments, submission_id, question_id), + %{ + comments: chosen_comments + } + ) |> json_response(200) assert response["status"] == "success" @@ -162,9 +179,12 @@ defmodule CadetWeb.AICodeAnalysisControllerTest do response = conn - |> post(Routes.ai_code_analysis_path(conn, :save_chosen_comments, submission_id, question_id), %{ - comments: chosen_comments - }) + |> post( + Routes.ai_code_analysis_path(conn, :save_chosen_comments, submission_id, question_id), + %{ + comments: chosen_comments + } + ) |> json_response(422) assert response["error"] == "Failed to save chosen comments" From 17884fd155a307af80402f5bfb6084aab224eb1b Mon Sep 17 00:00:00 2001 From: Arul Date: Sun, 6 Apr 2025 18:40:48 +0800 Subject: [PATCH 18/20] fix: error handling when calling openai API --- lib/cadet_web/controllers/generate_ai_comments.ex | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index 48b81bc6b..9bd86e752 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -306,7 +306,9 @@ defmodule CadetWeb.AICodeAnalysisController do "API request failed with status #{status}" ) - json(conn, %{"error" => "API request failed with status #{status}: #{body}"}) + conn + |> put_status(:internal_server_error) + |> json(%{"error" => "API request failed with status #{status}: #{body}"}) {:error, %HTTPoison.Error{reason: reason}} -> log_comment(submission_id, question_id, prompt, answers_json, nil, reason) From f91cc92891513ef6042616160946fe1a82071297 Mon Sep 17 00:00:00 2001 From: Arul Date: Wed, 9 Apr 2025 12:56:06 +0800 Subject: [PATCH 19/20] fix: credo issues --- lib/cadet/ai_comments.ex | 43 +++++++-------- lib/cadet/ai_comments/ai_comment.ex | 4 ++ .../controllers/generate_ai_comments.ex | 52 +++++++------------ 3 files changed, 42 insertions(+), 57 deletions(-) diff --git a/lib/cadet/ai_comments.ex b/lib/cadet/ai_comments.ex index eaf91a344..58accc768 100644 --- a/lib/cadet/ai_comments.ex +++ b/lib/cadet/ai_comments.ex @@ -1,4 +1,8 @@ defmodule Cadet.AIComments do + @moduledoc """ + Handles operations related to AI comments, including creation, updates, and retrieval. + """ + import Ecto.Query alias Cadet.Repo alias Cadet.AIComments.AIComment @@ -34,12 +38,13 @@ defmodule Cadet.AIComments do Returns `nil` if no comment exists. """ def get_latest_ai_comment(submission_id, question_id) do - from(c in AIComment, - where: c.submission_id == ^submission_id and c.question_id == ^question_id, - order_by: [desc: c.inserted_at], - limit: 1 + Repo.one( + from(c in AIComment, + where: c.submission_id == ^submission_id and c.question_id == ^question_id, + order_by: [desc: c.inserted_at], + limit: 1 + ) ) - |> Repo.one() end @doc """ @@ -47,17 +52,11 @@ defmodule Cadet.AIComments do Returns the most recent comment entry for that submission/question. """ def update_final_comment(submission_id, question_id, final_comment) do - from(c in AIComment, - where: c.submission_id == ^submission_id and c.question_id == ^question_id, - order_by: [desc: c.inserted_at], - limit: 1 - ) - |> Repo.one() - |> case do - nil -> - {:error, :not_found} + comment = get_latest_ai_comment(submission_id, question_id) - comment -> + case comment do + nil -> {:error, :not_found} + _ -> comment |> AIComment.changeset(%{final_comment: final_comment}) |> Repo.update() @@ -79,17 +78,11 @@ defmodule Cadet.AIComments do Accepts an array of comments and replaces the existing array in the database. """ def update_chosen_comments(submission_id, question_id, new_comments) do - from(c in AIComment, - where: c.submission_id == ^submission_id and c.question_id == ^question_id, - order_by: [desc: c.inserted_at], - limit: 1 - ) - |> Repo.one() - |> case do - nil -> - {:error, :not_found} + comment = get_latest_ai_comment(submission_id, question_id) - comment -> + case comment do + nil -> {:error, :not_found} + _ -> comment |> AIComment.changeset(%{comment_chosen: new_comments}) |> Repo.update() diff --git a/lib/cadet/ai_comments/ai_comment.ex b/lib/cadet/ai_comments/ai_comment.ex index 0cefe9530..1f1fae478 100644 --- a/lib/cadet/ai_comments/ai_comment.ex +++ b/lib/cadet/ai_comments/ai_comment.ex @@ -1,4 +1,8 @@ defmodule Cadet.AIComments.AIComment do + @moduledoc """ + Defines the schema and changeset for AI comments. + """ + use Ecto.Schema import Ecto.Changeset diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index 9bd86e752..1c1503669 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -4,9 +4,7 @@ defmodule CadetWeb.AICodeAnalysisController do require HTTPoison require Logger - alias Cadet.Assessments - alias Cadet.AIComments - alias Cadet.Courses + alias Cadet.{Assessments, AIComments, Courses} @openai_api_url "https://api.openai.com/v1/chat/completions" @model "gpt-4o" @@ -114,12 +112,10 @@ defmodule CadetWeb.AICodeAnalysisController do end) end - def format_answers(json_string) do + defp format_answers(json_string) do {:ok, answers} = Jason.decode(json_string) - answers - |> Enum.map(&format_answer/1) - |> Enum.join("\n\n") + Enum.map_join(answers, "\n\n", &format_answer/1) end defp format_answer(answer) do @@ -148,11 +144,9 @@ defmodule CadetWeb.AICodeAnalysisController do defp format_autograding_results(nil), do: "N/A" defp format_autograding_results(results) when is_list(results) do - results - |> Enum.map(fn result -> + Enum.map_join(results, "; ", fn result -> "Error: #{result["errorMessage"] || "N/A"}, Type: #{result["errorType"] || "N/A"}" end) - |> Enum.join("; ") end defp format_autograding_results(results), do: inspect(results) @@ -420,35 +414,29 @@ defmodule CadetWeb.AICodeAnalysisController do defp decrypt_llm_api_key(nil), do: nil defp decrypt_llm_api_key(encrypted_key) do - try do - # Get the encryption key - secret = Application.get_env(:openai, :encryption_key) - - if is_binary(secret) and byte_size(secret) >= 16 do - # Use first 16 bytes for AES-128, 24 for AES-192, or 32 for AES-256 + case Application.get_env(:openai, :encryption_key) do + secret when is_binary(secret) and byte_size(secret) >= 16 -> key = binary_part(secret, 0, min(32, byte_size(secret))) - # Decode the base64 string - decoded = Base.decode64!(encrypted_key) + case Base.decode64(encrypted_key) do + {:ok, decoded} -> + iv = binary_part(decoded, 0, 16) + tag = binary_part(decoded, 16, 16) + ciphertext = binary_part(decoded, 32, byte_size(decoded) - 32) - # Extract IV, tag and ciphertext - iv = binary_part(decoded, 0, 16) - tag = binary_part(decoded, 16, 16) - ciphertext = binary_part(decoded, 32, byte_size(decoded) - 32) + case :crypto.crypto_one_time_aead(:aes_gcm, key, iv, ciphertext, "", tag, false) do + plain_text when is_binary(plain_text) -> plain_text + _ -> nil + end - # Decrypt - case :crypto.crypto_one_time_aead(:aes_gcm, key, iv, ciphertext, "", tag, false) do - plain_text when is_binary(plain_text) -> plain_text - _ -> nil + _ -> + Logger.error("Failed to decode encrypted key") + nil end - else + + _ -> Logger.error("Encryption key not configured properly") nil - end - rescue - e -> - Logger.error("Error decrypting LLM API key: #{inspect(e)}") - nil end end end From 81e5bf74d32dd4d6e2f95e1d9e0f2b1d558151b5 Mon Sep 17 00:00:00 2001 From: Arul Date: Wed, 9 Apr 2025 13:01:05 +0800 Subject: [PATCH 20/20] formatting --- lib/cadet/ai_comments.ex | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/cadet/ai_comments.ex b/lib/cadet/ai_comments.ex index 58accc768..76b059601 100644 --- a/lib/cadet/ai_comments.ex +++ b/lib/cadet/ai_comments.ex @@ -55,7 +55,9 @@ defmodule Cadet.AIComments do comment = get_latest_ai_comment(submission_id, question_id) case comment do - nil -> {:error, :not_found} + nil -> + {:error, :not_found} + _ -> comment |> AIComment.changeset(%{final_comment: final_comment}) @@ -81,7 +83,9 @@ defmodule Cadet.AIComments do comment = get_latest_ai_comment(submission_id, question_id) case comment do - nil -> {:error, :not_found} + nil -> + {:error, :not_found} + _ -> comment |> AIComment.changeset(%{comment_chosen: new_comments})