From 663453769c2f76d3590c51356d91967a0fbcfc1a Mon Sep 17 00:00:00 2001 From: Tyler Wilding Date: Sat, 3 Oct 2020 18:32:01 -0400 Subject: [PATCH] Final polish on the integer tests, time to start converting the rest --- CMakeLists.txt | 2 +- game/CMakeLists.txt | 2 +- test/CMakeLists.txt | 2 +- test/goalc/framework/test_runner.cpp | 18 ++- test/goalc/source_generated/.gitignore | 2 +- test/goalc/test_compiler.cpp | 121 ----------------- test/goalc/test_compiler_integer.cpp | 177 +++++++++++++++++++++++++ test/test_compiler_and_runtime.cpp | 8 -- test/test_main.cpp | 11 ++ 9 files changed, 207 insertions(+), 136 deletions(-) delete mode 100644 test/goalc/test_compiler.cpp create mode 100644 test/goalc/test_compiler_integer.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 70931db1bc..fd0fdbd80e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.16) project(jak) -set(CMAKE_CXX_STANDARD 14) +set(CMAKE_CXX_STANDARD 17) # Set default compile flags for GCC # optimization level can be set here. Note that game/ overwrites this for building game C++ code. diff --git a/game/CMakeLists.txt b/game/CMakeLists.txt index d8aeb06f32..12e7a7747a 100644 --- a/game/CMakeLists.txt +++ b/game/CMakeLists.txt @@ -1,5 +1,5 @@ # We define our own compilation flags here. -set(CMAKE_CXX_STANDARD 14) +set(CMAKE_CXX_STANDARD 17) # Set default compile flags for GCC # optimization level can be set here. Note that game/ overwrites this for building game C++ code. diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 358cc69ff4..983b8fc00e 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -14,7 +14,7 @@ add_executable(goalc-test #test_emitter_integer_math.cpp #test_common_util.cpp #test_compiler_and_runtime.cpp - "goalc/test_compiler.cpp" + "goalc/test_compiler_integer.cpp" "goalc/framework/test_runner.cpp" "goalc/framework/test_runner.h") diff --git a/test/goalc/framework/test_runner.cpp b/test/goalc/framework/test_runner.cpp index 76fd56217b..d090a0271b 100644 --- a/test/goalc/framework/test_runner.cpp +++ b/test/goalc/framework/test_runner.cpp @@ -47,12 +47,24 @@ void CompilerTestRunner::run_test(const std::string& test_file, if (testing::Test::HasFailure()) { std::string testFile = file_util::get_file_path({"test/goalc/source_generated/" + test_file}); - // TODO - put the index and such there incase there are multiple failures - std::string failedFile = file_util::get_file_path({"test/goalc/source_generated/" + test_file + ".failed"}); + // TODO - put the expected and unexpected values as comments in the file as well + std::string failedFile = + file_util::get_file_path({"test/goalc/source_generated/failed/" + test_file}); std::ifstream src(testFile, std::ios::binary); std::ofstream dst(failedFile, std::ios::binary); - dst << src.rdbuf(); + + std::string testOutput = "\n\n;------TEST OUTPUT------\n;-------Expected-------\n"; + + for (auto& x : expected) { + testOutput += fmt::format("; \"{}\"\n", escaped_string(x)); + } + testOutput += "\n;--------Actual--------\n"; + for (auto& x : result) { + testOutput += fmt::format("; \"{}\"\n", escaped_string(x)); + } + + dst << src.rdbuf() << testOutput; } tests.push_back({expected, result, test_file, false}); diff --git a/test/goalc/source_generated/.gitignore b/test/goalc/source_generated/.gitignore index 8579ed2b0d..ffa96252f8 100644 --- a/test/goalc/source_generated/.gitignore +++ b/test/goalc/source_generated/.gitignore @@ -1,2 +1,2 @@ *.gc -*.gc.failed +failed/ \ No newline at end of file diff --git a/test/goalc/test_compiler.cpp b/test/goalc/test_compiler.cpp deleted file mode 100644 index e4af41f9ad..0000000000 --- a/test/goalc/test_compiler.cpp +++ /dev/null @@ -1,121 +0,0 @@ -// https://github.com/google/googletest/blob/master/googletest/docs/advanced.md#value-parameterized-tests - -#include -#include - -#include "gtest/gtest.h" -#include "game/runtime.h" -#include "goalc/listener/Listener.h" -#include "goalc/compiler/Compiler.h" - -#include "third-party/inja.hpp" -#include "third-party/json.hpp" -#include - -#include - -// TODO - put into the framework - -#include -#include -#include -#include -#include -#include - -struct IntegerParam { - s64 val; - bool hex; - - IntegerParam(s64 val, bool hex = false) : val(val), hex(hex) {} - - std::string toLisp() { - // Append hex reader macro '#x' - if (hex) { - return std::string("#x") + std::string(std::to_string(val)); - } - return std::to_string(val); - } - - std::string eval() { - if (hex) { - int64_t hexVal; - std::stringstream ss; - ss << std::hex << std::to_string(val); - ss >> hexVal; - return std::string(std::to_string(hexVal)) + "\n"; - } - if (val == 123) - return std::to_string(val); - return std::to_string(val) + "\n"; - } -}; - -// TODO - make sure i log the input/output if there is a failure -// - maybe i don't have to, the last test may exit and the file would remain? - -class IntegerTests : public testing::TestWithParam {}; - -TEST_P(IntegerTests, IntegerTests) { - // TODO - might be slow if we open / close the thread for each test. - // we might want to persist the compiler/test runner instance long term...shouldn't be that - // difficult, this is C++ right..no rules! pointers pointers pointers. - std::thread runtime_thread(GoalTest::runtime_no_kernel); - Compiler compiler; - GoalTest::CompilerTestRunner runner; - runner.c = &compiler; - - // With separate input and output path - std::string templateDir = file_util::get_file_path({"test/goalc/source_templates/"}); - std::string generatedDir = file_util::get_file_path({"test/goalc/source_generated/"}); - inja::Environment env{templateDir, generatedDir}; - - IntegerParam param = GetParam(); - - nlohmann::json data; - data["integer"] = param.toLisp(); - - env.write("integer-test.template.gc", data, "integer-test.generated.gc"); - - runner.run_test("integer-test.generated.gc", {param.eval()}); - - compiler.shutdown_target(); - runtime_thread.join(); - runner.print_summary(); -} - -// Generates a collection of evenly distributed tests -std::vector genIntegerTests(int numTests, bool includeHex, bool includeNegative) { - std::vector tests; - std::random_device dev; - std::mt19937 rng(dev()); - std::uniform_int_distribution dist6(0, UINT32_MAX); - int testCases = includeNegative ? 2 : 1; - if (includeHex) { - testCases *= 2; - } - for (int i = 0; i < numTests; i++) { - switch (i % testCases) { - case 0: - tests.push_back(IntegerParam(dist6(rng))); - break; - case 1: - tests.push_back(IntegerParam(dist6(rng) * -1)); - break; - case 2: - tests.push_back(IntegerParam(dist6(rng), true)); - tests.push_back(IntegerParam(123)); - break; - case 3: - tests.push_back(IntegerParam(dist6(rng) * -1, true)); - break; - } - } - return tests; -} - -// TODO - don't really need generated tests here, proof of concept -// specific examples for integers is more than enough -INSTANTIATE_TEST_SUITE_P(InstantiationName, - IntegerTests, - testing::ValuesIn(genIntegerTests(10, true, true))); diff --git a/test/goalc/test_compiler_integer.cpp b/test/goalc/test_compiler_integer.cpp new file mode 100644 index 0000000000..a17c8097e5 --- /dev/null +++ b/test/goalc/test_compiler_integer.cpp @@ -0,0 +1,177 @@ +// https://github.com/google/googletest/blob/master/googletest/docs/advanced.md#value-parameterized-tests + +#include +#include + +#include "gtest/gtest.h" +#include "game/runtime.h" +#include "goalc/listener/Listener.h" +#include "goalc/compiler/Compiler.h" + +#include "third-party/inja.hpp" +#include "third-party/json.hpp" +#include + +#include + +#include +#include +#include +#include +#include +#include + +// -------- +// This is a very over-engineered integer test, but it serves as a decent example of how to use the +// test+template framework +// I've heavily annotated it with comments and links to docs to help +// -------- + +// We are using Google Test's paramaterized test feature +// This allows us to define a single generic test, and pass in a whole bunch of values +// See - https://github.com/google/googletest/blob/master/googletest/docs/advanced.md#value-parameterized-tests +struct IntegerParam { + // An index is needed to be explicitly set because I couldn't find a way to pull the test-index number from google's API + // TODO - if you can find a way, please improve! + // But this is needed so we can uniquely save the template files, especially if they error out + // Why? - since you may choose to generate random values, it's nice for them to be stored after the tests complete. Some tests may be complex as well + int index; + // Each integer test has a signed value, and can be represented as hex or an integral + s64 val; + bool hex; + + IntegerParam(s64 val, bool hex = false, int index = 0) : val(val), hex(hex), index(index) {} + + // This is used to generate the value that is passed into the template engine + // and injected into the file of lisp code. + // In most cases this will probably be a string but look into inja's capabilities + // - https://github.com/pantor/inja + std::string toLisp() { + // Append hex reader macro '#x' + if (hex) { + return std::string("#x") + std::string(std::to_string(val)); + } + return std::to_string(val); + } + + // This is used by the test runner code to know what the expected value is + // For a simple example like this, a single eval is all that's required, but for + // more complex tests, this may not be the case. + std::string eval() { + if (hex) { + int64_t hexVal; + std::stringstream ss; + ss << std::hex << std::to_string(val); + ss >> hexVal; + return std::string(std::to_string(hexVal)) + "\n"; + } + return std::to_string(val) + "\n"; + } +}; + +// This is a helper function that is used to generate a bunch of tests +// once again, very over-engineered for just testing engineers, but you might imagine +// a more complex test template that has several conditionals / loops / etc +std::vector genIntegerTests(int numTests, + bool includeHex, + bool includeNegative, + std::vector additionalTests = {}) { + std::vector tests; + std::random_device dev; + std::mt19937 rng(dev()); + std::uniform_int_distribution dist6(0, UINT32_MAX); + int testCases = includeNegative ? 2 : 1; + if (includeHex) { + testCases *= 2; + } + for (int i = 0; i < numTests; i++) { + switch (i % testCases) { + case 0: + tests.push_back(IntegerParam(dist6(rng), false, i)); + break; + case 1: + tests.push_back(IntegerParam(dist6(rng) * -1, false, i)); + break; + case 2: + tests.push_back(IntegerParam(dist6(rng), true, i)); + break; + case 3: + tests.push_back(IntegerParam(dist6(rng) * -1, true, i)); + break; + } + } + + for (int i = 0; i < additionalTests.size(); i++) { + IntegerParam test = additionalTests.at(i); + test.index = i + numTests - 1; + tests.push_back(test); + } + + return tests; +} + +// In the interest of speed, we want to share the same thread/compiler across +// all the tests in this suite, so we have to over-ride this. +class IntegerTests : public testing::TestWithParam { + public: + // Per-test-suite set-up. + // Called before the first test in this test suite. + static void SetUpTestSuite() { + runtime_thread = std::thread((GoalTest::runtime_no_kernel)); + runner.c = &compiler; + } + + // Per-test-suite tear-down. + // Called after the last test in this test suite. + static void TearDownTestSuite() { + compiler.shutdown_target(); + runtime_thread.join(); + } + + // You can define per-test set-up logic as usual. + virtual void SetUp() {} + + // You can define per-test tear-down logic as usual. + virtual void TearDown() {} + + // Common Resources Across all Tests in the Suite + static std::thread runtime_thread; + static Compiler compiler; + static GoalTest::CompilerTestRunner runner; +}; + +// You must initialize the static variables outside of the declaration, or you'll run into unresolved external errors +std::thread IntegerTests::runtime_thread; +Compiler IntegerTests::compiler; +GoalTest::CompilerTestRunner IntegerTests::runner; + + +// Finally, we define our generic test, given our custom class that represents our test inputs +// we can generate the lisp file, and pass along the path to the test runner +// If the test fails, the test runner will save the template file, with the expected/actual results into the `failed/` directory +TEST_P(IntegerTests, IntegerTests) { + // With separate input and output path + std::string templateDir = file_util::get_file_path({"test/goalc/source_templates/"}); + std::string generatedDir = file_util::get_file_path({"test/goalc/source_generated/"}); + inja::Environment env{templateDir, generatedDir}; + + IntegerParam param = GetParam(); + + nlohmann::json data; + data["integer"] = param.toLisp(); + + std::string testFile = "integer-test-" + std::to_string(param.index) + ".generated.gc"; + env.write("integer-test.template.gc", data, testFile); + + runner.run_test(testFile, {param.eval()}); +} + +// ValuesIn, is not the only way to use a parameterized test, but the most applicable for this example +// You can actually get googletest to compute the permutations for you, which may be useful. Consult their docs. +INSTANTIATE_TEST_SUITE_P(InstantiationName, + IntegerTests, + testing::ValuesIn(genIntegerTests(4, + true, + true, + {IntegerParam(-2147483648), + IntegerParam(0), IntegerParam(-0)}))); diff --git a/test/test_compiler_and_runtime.cpp b/test/test_compiler_and_runtime.cpp index 26f35b5970..47d0cf4ea7 100644 --- a/test/test_compiler_and_runtime.cpp +++ b/test/test_compiler_and_runtime.cpp @@ -120,14 +120,6 @@ TEST(CompilerAndRuntime, StartRuntime) { runtime_thread.join(); } -TEST(CompilerAndRuntime, SendProgram) { - std::thread runtime_thread(runtime_no_kernel); - Compiler compiler; - compiler.run_test("goal_src/test/test-return-integer-1.gc"); - compiler.shutdown_target(); - runtime_thread.join(); -} - TEST(CompilerAndRuntime, BuildGameAndTest) { Compiler compiler; diff --git a/test/test_main.cpp b/test/test_main.cpp index 9a17845d82..18d44c8300 100644 --- a/test/test_main.cpp +++ b/test/test_main.cpp @@ -1,6 +1,17 @@ #include "gtest/gtest.h" +#include +#include + int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); + + // Re-init failed folder + std::string failedFolder = file_util::get_file_path({"test/goalc/source_generated/failed/"}); + if (std::filesystem::exists(failedFolder)) { + std::filesystem::remove_all(failedFolder); + } + std::filesystem::create_directory(failedFolder); + return RUN_ALL_TESTS(); } \ No newline at end of file