diff --git a/lib/cadet/assessments/answer.ex b/lib/cadet/assessments/answer.ex index e5e9d75b9..65f76a965 100644 --- a/lib/cadet/assessments/answer.ex +++ b/lib/cadet/assessments/answer.ex @@ -17,7 +17,7 @@ defmodule Cadet.Assessments.Answer do field(:xp, :integer, default: 0) field(:xp_adjustment, :integer, default: 0) field(:autograding_status, AutogradingStatus, default: :none) - field(:autograding_errors, {:array, :map}, default: []) + field(:autograding_results, {:array, :map}, default: []) field(:answer, :map) field(:type, QuestionType, virtual: true) field(:comment, :string) @@ -56,7 +56,7 @@ defmodule Cadet.Assessments.Answer do @spec autograding_changeset(%__MODULE__{} | Ecto.Changeset.t(), map()) :: Ecto.Changeset.t() def autograding_changeset(answer, params) do answer - |> cast(params, ~w(grade adjustment xp autograding_status autograding_errors)a) + |> cast(params, ~w(grade adjustment xp autograding_status autograding_results)a) |> validate_xp_grade_adjustment_total() end diff --git a/lib/cadet/assessments/autograding_status.ex b/lib/cadet/assessments/autograding_status.ex index f0b35f718..e3f8085aa 100644 --- a/lib/cadet/assessments/autograding_status.ex +++ b/lib/cadet/assessments/autograding_status.ex @@ -4,5 +4,6 @@ defenum(Cadet.Assessments.Answer.AutogradingStatus, :autograding_status, [ :none, :processing, :success, + # note that :failed refers to the autograder failing due to system errors :failed ]) diff --git a/lib/cadet/assessments/question_types/programming_question.ex b/lib/cadet/assessments/question_types/programming_question.ex index 806831af3..168950d33 100644 --- a/lib/cadet/assessments/question_types/programming_question.ex +++ b/lib/cadet/assessments/question_types/programming_question.ex @@ -4,20 +4,27 @@ defmodule Cadet.Assessments.QuestionTypes.ProgrammingQuestion do """ use Cadet, :model + alias Cadet.Assessments.QuestionTypes.Testcase + @primary_key false embedded_schema do field(:content, :string) - field(:solution_template, :string) + field(:prepend, :string, default: "") + field(:template, :string) + field(:postpend, :string, default: "") field(:solution, :string) - field(:autograder, {:array, :string}) + embeds_many(:public, Testcase) + embeds_many(:private, Testcase) end - @required_fields ~w(content solution_template)a - @optional_fields ~w(solution autograder)a + @required_fields ~w(content template)a + @optional_fields ~w(solution prepend postpend)a def changeset(question, params \\ %{}) do question |> cast(params, @required_fields ++ @optional_fields) + |> cast_embed(:public, with: &Testcase.changeset/2) + |> cast_embed(:private, with: &Testcase.changeset/2) |> validate_required(@required_fields) end end diff --git a/lib/cadet/assessments/question_types/programming_question_testcases.ex b/lib/cadet/assessments/question_types/programming_question_testcases.ex new file mode 100644 index 000000000..6aebafb23 --- /dev/null +++ b/lib/cadet/assessments/question_types/programming_question_testcases.ex @@ -0,0 +1,22 @@ +defmodule Cadet.Assessments.QuestionTypes.Testcase do + @moduledoc """ + The Assessments.QuestionTypes.Testcase entity represents a public/private testcase. + """ + use Cadet, :model + + @primary_key false + embedded_schema do + field(:program, :string) + field(:answer, :string) + field(:score, :integer) + end + + @required_fields ~w(program answer score)a + + def changeset(question, params \\ %{}) do + question + |> cast(params, @required_fields) + |> validate_required(@required_fields) + |> validate_number(:score, greater_than_or_equal_to: 0) + end +end diff --git a/lib/cadet/jobs/autograder/lambda_worker.ex b/lib/cadet/jobs/autograder/lambda_worker.ex index 18b468b30..f5ad1fa09 100644 --- a/lib/cadet/jobs/autograder/lambda_worker.ex +++ b/lib/cadet/jobs/autograder/lambda_worker.ex @@ -25,18 +25,9 @@ defmodule Cadet.Autograder.LambdaWorker do |> ExAws.Lambda.invoke(lambda_params, %{}) |> ExAws.request!() - # If the lambda crashes, results are in the format of: - # %{"errorMessage" => "${message}"} - if is_map(response) do - raise inspect(response) - else - result = - response - |> parse_response(lambda_params) - |> Map.put(:status, :success) + result = parse_response(response) - Que.add(ResultStoreWorker, %{answer_id: answer.id, result: result}) - end + Que.add(ResultStoreWorker, %{answer_id: answer.id, result: result}) end def on_failure(%{answer: answer = %Answer{}, question: %Question{}}, error) do @@ -55,8 +46,17 @@ defmodule Cadet.Autograder.LambdaWorker do result: %{ grade: 0, status: :failed, - errors: [ - %{"systemError" => "Autograder runtime error. Please contact a system administrator"} + result: [ + %{ + "resultType" => "error", + "errors" => [ + %{ + "errorType" => "systemError", + "errorMessage" => + "Autograder runtime error. Please contact a system administrator" + } + ] + } ] } } @@ -75,8 +75,11 @@ defmodule Cadet.Autograder.LambdaWorker do ) %{ - graderPrograms: question_content["autograder"], - studentProgram: answer.answer["code"], + prependProgram: Map.get(question_content, "prepend", ""), + studentProgram: Map.get(answer.answer, "code"), + postpendProgram: Map.get(question_content, "postpend", ""), + testcases: + Map.get(question_content, "public", []) ++ Map.get(question_content, "private", []), library: %{ chapter: question.grading_library.chapter, external: upcased_name_external, @@ -85,23 +88,24 @@ defmodule Cadet.Autograder.LambdaWorker do } end - def parse_response(response, %{graderPrograms: grader_programs}) do - response - |> Enum.zip(grader_programs) - |> Enum.reduce( - %{grade: 0, errors: []}, - fn {result, grader_program}, %{grade: grade, errors: errors} -> - if result["resultType"] == "pass" do - %{grade: grade + result["grade"], errors: errors} - else - error_result = %{ - grader_program: grader_program, - errors: result["errors"] + defp parse_response(response) when is_map(response) do + # If the lambda crashes, results are in the format of: + # %{"errorMessage" => "${message}"} + if Map.has_key?(response, "errorMessage") do + %{ + grade: 0, + status: :failed, + result: [ + %{ + "resultType" => "error", + "errors" => [ + %{"errorType" => "systemError", "errorMessage" => response["errorMessage"]} + ] } - - %{grade: grade, errors: errors ++ [error_result]} - end - end - ) + ] + } + else + %{grade: response["totalScore"], result: response["results"], status: :success} + end end end diff --git a/lib/cadet/jobs/autograder/result_store_worker.ex b/lib/cadet/jobs/autograder/result_store_worker.ex index a4bf7328f..e65e4df93 100644 --- a/lib/cadet/jobs/autograder/result_store_worker.ex +++ b/lib/cadet/jobs/autograder/result_store_worker.ex @@ -61,7 +61,7 @@ defmodule Cadet.Autograder.ResultStoreWorker do grade: result.grade, xp: xp, autograding_status: status, - autograding_errors: result.errors + autograding_results: result.result } answer diff --git a/lib/cadet/jobs/xml_parser.ex b/lib/cadet/jobs/xml_parser.ex index 524a1a318..bcb4f6bbd 100644 --- a/lib/cadet/jobs/xml_parser.ex +++ b/lib/cadet/jobs/xml_parser.ex @@ -247,15 +247,31 @@ defmodule Cadet.Updater.XMLParser do end defp process_question_entity_by_type(entity, "programming") do - entity - |> xpath( - ~x"."e, - content: ~x"./TEXT/text()" |> transform_by(&process_charlist/1), - solution_template: ~x"./SNIPPET/TEMPLATE/text()" |> transform_by(&process_charlist/1), - solution: ~x"./SNIPPET/SOLUTION/text()" |> transform_by(&process_charlist/1), - autograder: - ~x"./SNIPPET/GRADER/text()"l - |> transform_by(&Enum.map(&1, fn charlist -> process_charlist(charlist) end)) + Map.merge( + entity + |> xpath( + ~x"."e, + content: ~x"./TEXT/text()" |> transform_by(&process_charlist/1), + prepend: ~x"./SNIPPET/PREPEND/text()" |> transform_by(&process_charlist/1), + template: ~x"./SNIPPET/TEMPLATE/text()" |> transform_by(&process_charlist/1), + postpend: ~x"./SNIPPET/POSTPEND/text()" |> transform_by(&process_charlist/1), + solution: ~x"./SNIPPET/SOLUTION/text()" |> transform_by(&process_charlist/1) + ), + entity + |> xmap( + public: [ + ~x"./SNIPPET/TESTCASES/PUBLIC"l, + score: ~x"./@score"oi, + answer: ~x"./@answer" |> transform_by(&process_charlist/1), + program: ~x"./text()" |> transform_by(&process_charlist/1) + ], + private: [ + ~x"./SNIPPET/TESTCASES/PRIVATE"l, + score: ~x"./@score"oi, + answer: ~x"./@answer" |> transform_by(&process_charlist/1), + program: ~x"./text()" |> transform_by(&process_charlist/1) + ] + ) ) end diff --git a/lib/cadet_web/controllers/assessments_controller.ex b/lib/cadet_web/controllers/assessments_controller.ex index a605a5a54..31306d0da 100644 --- a/lib/cadet_web/controllers/assessments_controller.ex +++ b/lib/cadet_web/controllers/assessments_controller.ex @@ -193,9 +193,22 @@ defmodule CadetWeb.AssessmentsController do "The library used for this question" ) - solutionTemplate(:string, "Solution template for programming questions") + prepend(:string, "Prepend program for programming questions") + + template(:string, "Solution template for programming questions") + + postpend(:string, "Postpend program for programming questions") + + testcases( + Schema.new do + type(:array) + items(Schema.ref(:Testcase)) + end, + "Testcase programs for programming questions" + ) grader(Schema.ref(:GraderInfo)) + gradedAt(:string, "Last graded at", format: "date-time", required: false) xp(:integer, "Final XP given to this question. Only provided for students.") @@ -212,6 +225,18 @@ defmodule CadetWeb.AssessmentsController do "The max xp for this question", required: true ) + + autogradingStatus( + :string, + "One of none/processing/success/failed" + ) + + autogradingResults( + Schema.new do + type(:array) + items(Schema.ref(:AutogradingResult)) + end + ) end end, MCQChoice: @@ -261,6 +286,22 @@ defmodule CadetWeb.AssessmentsController do "The external library for this question" ) end + end, + Testcase: + swagger_schema do + properties do + answer(:string) + score(:integer) + program(:string) + end + end, + AutogradingResult: + swagger_schema do + properties do + resultType(:string, "One of pass/fail/error") + expected(:string) + actual(:string) + end end } end diff --git a/lib/cadet_web/views/assessments_view.ex b/lib/cadet_web/views/assessments_view.ex index 3352f8737..00b25bfb7 100644 --- a/lib/cadet_web/views/assessments_view.ex +++ b/lib/cadet_web/views/assessments_view.ex @@ -128,11 +128,47 @@ defmodule CadetWeb.AssessmentsView do grader: grader_builder(grader), gradedAt: graded_at_builder(grader), xp: &((&1.xp || 0) + (&1.xp_adjustment || 0)), - grade: &((&1.grade || 0) + (&1.adjustment || 0)) + grade: &((&1.grade || 0) + (&1.adjustment || 0)), + autogradingStatus: :autograding_status, + autogradingResults: build_results(%{results: answer.autograding_results}) }) end - def build_choice(%{choice: choice}) do + defp build_results(%{results: results}) do + case results do + nil -> nil + _ -> &Enum.map(&1.autograding_results, fn result -> build_result(result) end) + end + end + + defp build_result(result) do + transform_map_for_view(result, %{ + resultType: "resultType", + expected: "expected", + actual: "actual", + errorType: "errorType", + errors: build_errors(result["errors"]) + }) + end + + defp build_errors(errors) do + case errors do + nil -> nil + _ -> &Enum.map(&1["errors"], fn error -> build_error(error) end) + end + end + + defp build_error(error) do + transform_map_for_view(error, %{ + errorType: "errorType", + line: "line", + location: "location", + errorLine: "errorLine", + errorExplanation: "errorExplanation" + }) + end + + defp build_choice(choice) do transform_map_for_view(choice, %{ id: "choice_id", content: "content", @@ -140,18 +176,29 @@ defmodule CadetWeb.AssessmentsView do }) end + defp build_testcase(testcase) do + transform_map_for_view(testcase, %{ + answer: "answer", + score: "score", + program: "program" + }) + end + defp build_question_content_by_type(%{question: %{question: question, type: question_type}}) do case question_type do :programming -> transform_map_for_view(question, %{ content: "content", - solutionTemplate: "solution_template" + prepend: "prepend", + solutionTemplate: "template", + postpend: "postpend", + testcases: &Enum.map(&1["public"], fn testcase -> build_testcase(testcase) end) }) :mcq -> transform_map_for_view(question, %{ content: "content", - choices: &Enum.map(&1["choices"], fn choice -> build_choice(%{choice: choice}) end) + choices: &Enum.map(&1["choices"], fn choice -> build_choice(choice) end) }) end end diff --git a/priv/repo/migrations/20190318070229_alter_answers_table_autograding_errors.exs b/priv/repo/migrations/20190318070229_alter_answers_table_autograding_errors.exs new file mode 100644 index 000000000..f07d7d7a9 --- /dev/null +++ b/priv/repo/migrations/20190318070229_alter_answers_table_autograding_errors.exs @@ -0,0 +1,7 @@ +defmodule Cadet.Repo.Migrations.AlterAnswersTableAutogradingErrors do + use Ecto.Migration + + def change do + rename(table(:answers), :autograding_errors, to: :autograding_results) + end +end diff --git a/schema.png b/schema.png index c02fed7ec..2e0087262 100644 Binary files a/schema.png and b/schema.png differ diff --git a/test/cadet/assessments/assessments_test.exs b/test/cadet/assessments/assessments_test.exs index ade71ebb4..e6e406b3e 100644 --- a/test/cadet/assessments/assessments_test.exs +++ b/test/cadet/assessments/assessments_test.exs @@ -32,7 +32,9 @@ defmodule Cadet.AssessmentsTest do library: build(:library), question: %{ content: Faker.Pokemon.name(), - solution_template: Faker.Lorem.Shakespeare.as_you_like_it(), + prepend: "", + template: Faker.Lorem.Shakespeare.as_you_like_it(), + postpend: "", solution: Faker.Lorem.Shakespeare.hamlet() } }, diff --git a/test/cadet/assessments/question_test.exs b/test/cadet/assessments/question_test.exs index d157abfbd..3f1e5e7d8 100644 --- a/test/cadet/assessments/question_test.exs +++ b/test/cadet/assessments/question_test.exs @@ -15,7 +15,9 @@ defmodule Cadet.Assessments.QuestionTest do library: build(:library), question: %{ content: Faker.Pokemon.name(), - solution_template: Faker.Lorem.Shakespeare.as_you_like_it(), + prepend: "", + template: Faker.Lorem.Shakespeare.as_you_like_it(), + postpend: "", solution: Faker.Lorem.Shakespeare.hamlet() } } diff --git a/test/cadet/assessments/question_types/programming_question_test.exs b/test/cadet/assessments/question_types/programming_question_test.exs index cf860b8d8..77f187bdc 100644 --- a/test/cadet/assessments/question_types/programming_question_test.exs +++ b/test/cadet/assessments/question_types/programming_question_test.exs @@ -8,14 +8,20 @@ defmodule Cadet.Assessments.QuestionTypes.ProgrammingQuestionTest do assert_changeset( %{ content: "asd", - solution_template: "asd" + template: "asd" }, :valid ) end test "invalid changesets" do - assert_changeset(%{content: "asd"}, :invalid) + assert_changeset( + %{ + content: 1, + template: "asd" + }, + :invalid + ) assert_changeset( %{ diff --git a/test/cadet/assessments/question_types/programming_question_testcases_test.exs b/test/cadet/assessments/question_types/programming_question_testcases_test.exs new file mode 100644 index 000000000..3d599e0eb --- /dev/null +++ b/test/cadet/assessments/question_types/programming_question_testcases_test.exs @@ -0,0 +1,53 @@ +defmodule Cadet.Assessments.QuestionTypes.ProgrammingQuestionTestcaseTest do + alias Cadet.Assessments.QuestionTypes.Testcase + + use Cadet.ChangesetCase, entity: Testcase + + describe "Changesets" do + test "valid changeset" do + assert_changeset( + %{ + score: 1, + answer: "asd", + program: "asd" + }, + :valid + ) + end + + test "invalid changesets" do + assert_changeset( + %{ + score: -1, + answer: "asd", + program: "asd" + }, + :invalid + ) + + assert_changeset( + %{ + score: 1, + answer: "asd" + }, + :invalid + ) + + assert_changeset( + %{ + answer: "asd", + program: "asd" + }, + :invalid + ) + + assert_changeset( + %{ + score: 1, + program: "asd" + }, + :invalid + ) + end + end +end diff --git a/test/cadet/jobs/autograder/lambda_worker_test.exs b/test/cadet/jobs/autograder/lambda_worker_test.exs index 7acfaadfe..5df641fe9 100644 --- a/test/cadet/jobs/autograder/lambda_worker_test.exs +++ b/test/cadet/jobs/autograder/lambda_worker_test.exs @@ -20,9 +20,11 @@ defmodule Cadet.Autograder.LambdaWorkerTest do %{ question: build(:programming_question_content, %{ - autograder: [ - "function ek0chei0y1() {\n return f(0) === 0 ? 1 : 0;\n }\n\n ek0chei0y1();", - "function ek0chei0y1() {\n const test1 = f(7) === 13;\n const test2 = f(10) === 55;\n const test3 = f(12) === 144;\n return test1 && test2 && test3 ? 4 : 0;\n }\n\n ek0chei0y1();" + public: [ + %{"score" => 1, "answer" => "1", "program" => "f(1);"} + ], + private: [ + %{"score" => 1, "answer" => "45", "program" => "f(10);"} ] }) } @@ -56,7 +58,14 @@ defmodule Cadet.Autograder.LambdaWorkerTest do assert_called( Que.add(ResultStoreWorker, %{ answer_id: answer.id, - result: %{errors: [], grade: 5, status: :success} + result: %{ + result: [ + %{"resultType" => "pass", "score" => 1}, + %{"resultType" => "pass", "score" => 1} + ], + grade: 2, + status: :success + } }) ) end @@ -75,16 +84,32 @@ defmodule Cadet.Autograder.LambdaWorkerTest do Que.add(ResultStoreWorker, %{ answer_id: answer.id, result: %{ - errors: [ + result: [ %{ - errors: [%{"errorType" => "syntax", "line" => 1, "location" => "student"}], - grader_program: - "function ek0chei0y1() {\n return f(0) === 0 ? 1 : 0;\n }\n\n ek0chei0y1();" + "resultType" => "error", + "errors" => [ + %{ + "errorType" => "syntax", + "line" => 1, + "location" => "student", + "errorLine" => + "consst f = i => i === 0 ? 0 : i < 3 ? 1 : f(i-1) + f(i-2);", + "errorExplanation" => "SyntaxError: Unexpected token (2:7)" + } + ] }, %{ - errors: [%{"errorType" => "syntax", "line" => 1, "location" => "student"}], - grader_program: - "function ek0chei0y1() {\n const test1 = f(7) === 13;\n const test2 = f(10) === 55;\n const test3 = f(12) === 144;\n return test1 && test2 && test3 ? 4 : 0;\n }\n\n ek0chei0y1();" + "resultType" => "error", + "errors" => [ + %{ + "errorType" => "syntax", + "line" => 1, + "location" => "student", + "errorLine" => + "consst f = i => i === 0 ? 0 : i < 3 ? 1 : f(i-1) + f(i-2);", + "errorExplanation" => "SyntaxError: Unexpected token (2:7)" + } + ] } ], grade: 0, @@ -97,16 +122,34 @@ defmodule Cadet.Autograder.LambdaWorkerTest do end test "lambda errors", %{question: question, answer: answer} do - error_response = %{"errorMessage" => "Some error message"} - - with_mock ExAws, request!: fn _ -> error_response end do - expected_error = inspect(error_response) - - assert_raise RuntimeError, expected_error, fn -> + use_cassette "autograder/errors#2", custom: true do + with_mock Que, add: fn _, _ -> nil end do LambdaWorker.perform(%{ question: Repo.get(Question, question.id), answer: Repo.get(Answer, answer.id) }) + + assert_called( + Que.add(ResultStoreWorker, %{ + answer_id: answer.id, + result: %{ + grade: 0, + status: :failed, + result: [ + %{ + "resultType" => "error", + "errors" => [ + %{ + "errorType" => "systemError", + "errorMessage" => + "2019-05-18T05:26:11.299Z 21606396-02e0-4fd5-a294-963bb7994e75 Task timed out after 10.01 seconds" + } + ] + } + ] + } + }) + ) end end end @@ -130,19 +173,28 @@ defmodule Cadet.Autograder.LambdaWorkerTest do assert log =~ "Task timed out after 1.00 seconds" assert_called( - Que.add(ResultStoreWorker, %{ - answer_id: answer.id, - result: %{ - errors: [ - %{ - "systemError" => - "Autograder runtime error. Please contact a system administrator" - } - ], - grade: 0, - status: :failed + Que.add( + ResultStoreWorker, + %{ + answer_id: answer.id, + result: %{ + grade: 0, + status: :failed, + result: [ + %{ + "resultType" => "error", + "errors" => [ + %{ + "errorType" => "systemError", + "errorMessage" => + "Autograder runtime error. Please contact a system administrator" + } + ] + } + ] + } } - }) + ) ) end end @@ -151,8 +203,10 @@ defmodule Cadet.Autograder.LambdaWorkerTest do describe "#build_request_params" do test "it should build correct params", %{question: question, answer: answer} do expected = %{ - graderPrograms: question.question["autograder"], - studentProgram: answer.answer["code"], + prependProgram: question.question.prepend, + postpendProgram: question.question.postpend, + testcases: question.question.public ++ question.question.private, + studentProgram: answer.answer.code, library: %{ chapter: question.grading_library.chapter, external: %{ @@ -163,7 +217,10 @@ defmodule Cadet.Autograder.LambdaWorkerTest do } } - assert LambdaWorker.build_request_params(%{question: question, answer: answer}) == expected + assert LambdaWorker.build_request_params(%{ + question: Repo.get(Question, question.id), + answer: Repo.get(Answer, answer.id) + }) == expected end end end diff --git a/test/cadet/jobs/autograder/result_store_worker_test.exs b/test/cadet/jobs/autograder/result_store_worker_test.exs index b8a84b517..3c7717f1e 100644 --- a/test/cadet/jobs/autograder/result_store_worker_test.exs +++ b/test/cadet/jobs/autograder/result_store_worker_test.exs @@ -8,19 +8,33 @@ defmodule Cadet.Autograder.ResultStoreWorkerTest do setup do answer = insert(:answer, %{question: insert(:question), submission: insert(:submission)}) - success_no_errors = %{status: :success, grade: 10, errors: []} + success_no_errors = %{status: :success, grade: 10, result: []} success_with_errors = %{ - errors: [ + result: [ %{ - errors: [%{"errorType" => "syntax", "line" => 1, "location" => "student"}], - grader_program: - "function ek0chei0y1() {\n return f(0) === 0 ? 1 : 0;\n }\n\n ek0chei0y1();" + "resultType" => "error", + "errors" => [ + %{ + "errorType" => "syntax", + "line" => 1, + "location" => "student", + "errorLine" => "consst f = i => i === 0 ? 0 : i < 3 ? 1 : f(i-1) + f(i-2);", + "errorExplanation" => "SyntaxError: Unexpected token (2:7)" + } + ] }, %{ - errors: [%{"errorType" => "syntax", "line" => 1, "location" => "student"}], - grader_program: - "function ek0chei0y1() {\n const test1 = f(7) === 13;\n const test2 = f(10) === 55;\n const test3 = f(12) === 144;\n return test1 && test2 && test3 ? 4 : 0;\n }\n\n ek0chei0y1();" + "resultType" => "error", + "errors" => [ + %{ + "errorType" => "syntax", + "line" => 1, + "location" => "student", + "errorLine" => "consst f = i => i === 0 ? 0 : i < 3 ? 1 : f(i-1) + f(i-2);", + "errorExplanation" => "SyntaxError: Unexpected token (2:7)" + } + ] } ], grade: 0, @@ -28,7 +42,7 @@ defmodule Cadet.Autograder.ResultStoreWorkerTest do } failed = %{ - errors: [ + result: [ %{ "systemError" => "Autograder runtime error. Please contact a system administrator" } @@ -63,7 +77,7 @@ defmodule Cadet.Autograder.ResultStoreWorkerTest do |> Repo.get(answer.id) errors_string_keys = - Enum.map(result.errors, fn err -> + Enum.map(result.result, fn err -> Enum.reduce(err, %{}, fn {k, v}, acc -> Map.put(acc, "#{k}", v) end) @@ -83,7 +97,7 @@ defmodule Cadet.Autograder.ResultStoreWorkerTest do end assert answer.autograding_status == result.status - assert answer.autograding_errors == errors_string_keys + assert answer.autograding_results == errors_string_keys end end end diff --git a/test/cadet_web/controllers/assessments_controller_test.exs b/test/cadet_web/controllers/assessments_controller_test.exs index ff54e122f..af9935a16 100644 --- a/test/cadet_web/controllers/assessments_controller_test.exs +++ b/test/cadet_web/controllers/assessments_controller_test.exs @@ -248,7 +248,16 @@ defmodule CadetWeb.AssessmentsControllerTest do "id" => &1.id, "type" => "#{&1.type}", "content" => &1.question.content, - "solutionTemplate" => &1.question.solution_template + "solutionTemplate" => &1.question.template, + "prepend" => &1.question.prepend, + "postpend" => &1.question.postpend, + "testcases" => + Enum.map( + &1.question.public, + fn testcase -> + for {k, v} <- testcase, into: %{}, do: {Atom.to_string(k), v} + end + ) } ) @@ -291,6 +300,8 @@ defmodule CadetWeb.AssessmentsControllerTest do |> Enum.map(&Map.delete(&1, "maxGrade")) |> Enum.map(&Map.delete(&1, "grader")) |> Enum.map(&Map.delete(&1, "gradedAt")) + |> Enum.map(&Map.delete(&1, "autogradingResults")) + |> Enum.map(&Map.delete(&1, "autogradingStatus")) assert expected_questions == resp_questions end diff --git a/test/cadet_web/controllers/grading_controller_test.exs b/test/cadet_web/controllers/grading_controller_test.exs index 027f9322a..86abe4415 100644 --- a/test/cadet_web/controllers/grading_controller_test.exs +++ b/test/cadet_web/controllers/grading_controller_test.exs @@ -227,7 +227,16 @@ defmodule CadetWeb.GradingControllerTest do :programming -> %{ "question" => %{ - "solutionTemplate" => &1.question.question.solution_template, + "prepend" => &1.question.question.prepend, + "postpend" => &1.question.question.postpend, + "testcases" => + Enum.map( + &1.question.question.public, + fn testcase -> + for {k, v} <- testcase, into: %{}, do: {Atom.to_string(k), v} + end + ), + "solutionTemplate" => &1.question.question.template, "type" => "#{&1.question.type}", "id" => &1.question.id, "library" => %{ @@ -333,7 +342,16 @@ defmodule CadetWeb.GradingControllerTest do :programming -> %{ "question" => %{ - "solutionTemplate" => &1.question.question.solution_template, + "prepend" => &1.question.question.prepend, + "postpend" => &1.question.question.postpend, + "testcases" => + Enum.map( + &1.question.question.public, + fn testcase -> + for {k, v} <- testcase, into: %{}, do: {Atom.to_string(k), v} + end + ), + "solutionTemplate" => &1.question.question.template, "type" => "#{&1.question.type}", "id" => &1.question.id, "library" => %{ @@ -621,7 +639,16 @@ defmodule CadetWeb.GradingControllerTest do :programming -> %{ "question" => %{ - "solutionTemplate" => &1.question.question.solution_template, + "prepend" => &1.question.question.prepend, + "postpend" => &1.question.question.postpend, + "testcases" => + Enum.map( + &1.question.question.public, + fn testcase -> + for {k, v} <- testcase, into: %{}, do: {Atom.to_string(k), v} + end + ), + "solutionTemplate" => &1.question.question.template, "type" => "#{&1.question.type}", "id" => &1.question.id, "library" => %{ diff --git a/test/factories/assessments/question_factory.ex b/test/factories/assessments/question_factory.ex index ca8277f27..9ff7c2252 100644 --- a/test/factories/assessments/question_factory.ex +++ b/test/factories/assessments/question_factory.ex @@ -27,13 +27,25 @@ defmodule Cadet.Assessments.QuestionFactory do def programming_question_content_factory do %{ + prepend: Faker.Pokemon.location(), content: Faker.Pokemon.name(), - solution_template: Faker.Lorem.Shakespeare.as_you_like_it(), + postpend: Faker.Pokemon.location(), + template: Faker.Lorem.Shakespeare.as_you_like_it(), solution: Faker.Lorem.Shakespeare.hamlet(), - autograder: - (&Faker.Lorem.Shakespeare.king_richard_iii/0) - |> Stream.repeatedly() - |> Enum.take(Enum.random(0..2)) + public: [ + %{ + score: :rand.uniform(5), + answer: Faker.StarWars.character(), + program: Faker.Lorem.Shakespeare.king_richard_iii() + } + ], + private: [ + %{ + score: :rand.uniform(5), + answer: Faker.StarWars.character(), + program: Faker.Lorem.Shakespeare.king_richard_iii() + } + ] } end diff --git a/test/fixtures/custom_cassettes/autograder/errors#1.json b/test/fixtures/custom_cassettes/autograder/errors#1.json index c5b005e74..639ac9d49 100644 --- a/test/fixtures/custom_cassettes/autograder/errors#1.json +++ b/test/fixtures/custom_cassettes/autograder/errors#1.json @@ -2,7 +2,7 @@ { "request": { "body": - "{\"studentProgram\":\"consst f = i => i === 0 ? 0 : i < 3 ? 1 : f(i-1) + f(i-2);\",\"library\":{\"globals\":[],\"external\":{\"symbols\":[],\"name\":\"NONE\"},\"chapter\":3},\"graderPrograms\":[\"function ek0chei0y1() {\\n return f(0) === 0 ? 1 : 0;\\n }\\n\\n ek0chei0y1();\",\"function ek0chei0y1() {\\n const test1 = f(7) === 13;\\n const test2 = f(10) === 55;\\n const test3 = f(12) === 144;\\n return test1 && test2 && test3 ? 4 : 0;\\n }\\n\\n ek0chei0y1();\"]}", + "{\r\n \"library\": {\r\n \"chapter\": 2,\r\n \"external\": {\r\n \"name\": \"NONE\",\r\n \"symbols\": []\r\n },\r\n \"globals\": []\r\n },\r\n \"prependProgram\": \"\",\r\n \"studentProgram\": \"consst f = i => i === 0 ? 0 : i < 3 ? 1 : f(i-1) + f(i-2);\",\r\n \"postpendProgram\": \"\",\r\n \"testCases\": [\r\n {\r\n \"answer\": \"1\",\r\n \"program\": \"f(1);\",\r\n \"score\": 1\r\n },\r\n {\r\n \"answer\": \"55\",\r\n \"program\": \"f(10);\",\r\n \"score\": 1\r\n }\r\n ]\r\n}", "headers": { "Authorization": "hello_world", "host": "lambda.ap-southeast-1.amazonaws.com", @@ -21,7 +21,7 @@ "response": { "binary": false, "body": - "[{\"resultType\":\"error\",\"errors\":[{\"errorType\":\"syntax\",\"line\":1,\"location\":\"student\"}]},{\"resultType\":\"error\",\"errors\":[{\"errorType\":\"syntax\",\"line\":1,\"location\":\"student\"}]}]", + "{\r\n \"totalScore\": 0,\r\n \"results\": [\r\n {\r\n \"resultType\": \"error\",\r\n \"errors\": [\r\n {\r\n \"errorType\": \"syntax\",\r\n \"line\": 1,\r\n \"location\": \"student\",\r\n \"errorLine\": \"consst f = i => i === 0 ? 0 : i < 3 ? 1 : f(i-1) + f(i-2);\",\r\n \"errorExplanation\": \"SyntaxError: Unexpected token (2:7)\"\r\n }\r\n ]\r\n },\r\n {\r\n \"resultType\": \"error\",\r\n \"errors\": [\r\n {\r\n \"errorType\": \"syntax\",\r\n \"line\": 1,\r\n \"location\": \"student\",\r\n \"errorLine\": \"consst f = i => i === 0 ? 0 : i < 3 ? 1 : f(i-1) + f(i-2);\",\r\n \"errorExplanation\": \"SyntaxError: Unexpected token (2:7)\"\r\n }\r\n ]\r\n }\r\n ]\r\n}", "headers": { "Date": "Thu, 09 Aug 2018 09:53:29 GMT", "Content-Type": "application/json", diff --git a/test/fixtures/custom_cassettes/autograder/errors#2.json b/test/fixtures/custom_cassettes/autograder/errors#2.json index 5d65de1fe..617b9db0d 100644 --- a/test/fixtures/custom_cassettes/autograder/errors#2.json +++ b/test/fixtures/custom_cassettes/autograder/errors#2.json @@ -2,7 +2,7 @@ { "request": { "body": - "{\"studentProgram\":\"const f = i => f(i);\",\"library\":{\"globals\":[],\"external\":{\"symbols\":[],\"name\":\"NONE\"},\"chapter\":7},\"graderPrograms\":[\"function ek0chei0y1() {\\n return f(0) === 0 ? 1 : 0;\\n }\\n\\n ek0chei0y1();\",\"function ek0chei0y1() {\\n const test1 = f(7) === 13;\\n const test2 = f(10) === 55;\\n const test3 = f(12) === 144;\\n return test1 && test2 && test3 ? 4 : 0;\\n }\\n\\n ek0chei0y1();\"]}", + "{\r\n \"library\": {\r\n \"chapter\": 2,\r\n \"external\": {\r\n \"name\": \"NONE\",\r\n \"symbols\": []\r\n },\r\n \"globals\": []\r\n },\r\n \"prependProgram\": \"\",\r\n \"studentProgram\": \"const f = i => f(i);\",\r\n \"postpendProgram\": \"\",\r\n \"testCases\": [\r\n {\r\n \"answer\": \"1\",\r\n \"program\": \"f(1);\",\r\n \"score\": 1\r\n },\r\n {\r\n \"answer\": \"55\",\r\n \"program\": \"f(10);\",\r\n \"score\": 1\r\n }\r\n ]\r\n}", "headers": { "Authorization": "hello_world", "host": "lambda.ap-southeast-1.amazonaws.com", @@ -21,8 +21,8 @@ "response": { "binary": false, "body": - "{\"errorMessage\":\"2018-08-09T09:55:45.692Z 6049db71-33fa-4faa-88c4-38f57b6754f3 Task timed out after 1.00 seconds\"}", - "headers": { + "{\"errorMessage\": \"2019-05-18T05:26:11.299Z 21606396-02e0-4fd5-a294-963bb7994e75 Task timed out after 10.01 seconds\"}", + "headers": { "Date": "Thu, 09 Aug 2018 09:55:45 GMT", "Content-Type": "application/json", "Content-Length": "114", diff --git a/test/fixtures/custom_cassettes/autograder/success#1.json b/test/fixtures/custom_cassettes/autograder/success#1.json index b08c9aa53..68f3f6845 100644 --- a/test/fixtures/custom_cassettes/autograder/success#1.json +++ b/test/fixtures/custom_cassettes/autograder/success#1.json @@ -2,7 +2,7 @@ { "request": { "body": - "{\"studentProgram\":\"const f = i => i === 0 ? 0 : i < 3 ? 1 : f(i-1) + f(i-2);\",\"library\":{\"globals\":[],\"external\":{\"symbols\":[],\"name\":\"NONE\"},\"chapter\":9},\"graderPrograms\":[\"function ek0chei0y1() {\\n return f(0) === 0 ? 1 : 0;\\n }\\n\\n ek0chei0y1();\",\"function ek0chei0y1() {\\n const test1 = f(7) === 13;\\n const test2 = f(10) === 55;\\n const test3 = f(12) === 144;\\n return test1 && test2 && test3 ? 4 : 0;\\n }\\n\\n ek0chei0y1();\"]}", + "{\r\n \"library\": {\r\n \"chapter\": 2,\r\n \"external\": {\r\n \"name\": \"NONE\",\r\n \"symbols\": []\r\n },\r\n \"globals\": []\r\n },\r\n \"prependProgram\": \"\",\r\n \"studentProgram\": \"const f = i => i === 0 ? 0 : i < 3 ? 1 : f(i-1) + f(i-2);\",\r\n \"postpendProgram\": \"\",\r\n \"testCases\": [\r\n {\r\n \"answer\": \"1\",\r\n \"program\": \"f(1);\",\r\n \"score\": 1\r\n },\r\n {\r\n \"answer\": \"55\",\r\n \"program\": \"f(10);\",\r\n \"score\": 1\r\n }\r\n ]\r\n}", "headers": { "Authorization": "AWS4-HMAC-SHA256 Credential=AKIAIAHN5DV7RTRUVP6A/20180809/ap-southeast-1/lambda/aws4_request,SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date,Signature=5bb3b478cc5669bfd3a6188dd7a72590fd71b0f557673dec67b6b3bc81b3dd94", @@ -22,7 +22,7 @@ "response": { "binary": false, "body": - "[{\"resultType\":\"pass\",\"grade\":1},{\"resultType\":\"pass\",\"grade\":4}]", + "{\r\n \"totalScore\": 2,\r\n \"results\": [\r\n {\r\n \"resultType\": \"pass\",\r\n \"score\": 1\r\n },\r\n {\r\n \"resultType\": \"pass\",\r\n \"score\": 1\r\n }\r\n ]\r\n}", "headers": { "Date": "Thu, 09 Aug 2018 09:50:53 GMT", "Content-Type": "application/json", diff --git a/test/support/xml_generator.ex b/test/support/xml_generator.ex index 10bc92f86..aedcd6b24 100644 --- a/test/support/xml_generator.ex +++ b/test/support/xml_generator.ex @@ -108,7 +108,11 @@ defmodule Cadet.Test.XMLGenerator do end :programming -> - template_field = [template(question.question.solution_template)] + prepend_field = [prepend(question.question.prepend)] + + template_field = [template(question.question.template)] + + postpend_field = [postpend(question.question.postpend)] solution_field = if question.question[:solution] do @@ -117,14 +121,27 @@ defmodule Cadet.Test.XMLGenerator do [] end - grader_fields = - if question.question[:autograder] do - Enum.map(question.question[:autograder], &grader/1) - else - [] - end - - [snippet(template_field ++ solution_field ++ grader_fields)] + testcases_fields = [ + testcases( + [ + for testcase <- question.question[:public] do + public(%{score: testcase.score, answer: testcase.answer}, testcase.program) + end + ] ++ + [ + for testcase <- question.question[:private] do + private(%{score: testcase.score, answer: testcase.answer}, testcase.program) + end + ] + ) + ] + + [ + snippet( + prepend_field ++ + template_field ++ postpend_field ++ solution_field ++ testcases_fields + ) + ] end end @@ -227,16 +244,32 @@ defmodule Cadet.Test.XMLGenerator do {"SNIPPET", nil, children} end + defp prepend(content) do + {"PREPEND", nil, content} + end + defp template(content) do {"TEMPLATE", nil, content} end + defp postpend(content) do + {"POSTPEND", nil, content} + end + defp solution(content) do {"SOLUTION", nil, content} end - defp grader(content) do - {"GRADER", nil, content} + defp testcases(children) do + {"TESTCASES", nil, children} + end + + defp public(raw_attrs, content) do + {"PUBLIC", map_permit_keys(raw_attrs, ~w(score answer)a), content} + end + + defp private(raw_attrs, content) do + {"PRIVATE", map_permit_keys(raw_attrs, ~w(score answer)a), content} end defp map_permit_keys(map, keys) when is_map(map) and is_list(keys) do