Merge pull request #67 from xTVaser/generative-tests

Create more general Test Framework and Organize the bulk of current tests

.github/workflows/workflow.yaml

@@ -5,7 +5,7 @@ jobs:
   common:
     # TODO - Start using clang-tidy - requires a bit more setup though - http://clang.llvm.org/docs/HowToSetupToolingForLLVM.html
     name: Linting & Formatting
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
       - name: Checkout Repository
         uses: actions/checkout@v2
@@ -19,7 +19,7 @@ jobs:
           ./third-party/run-clang-format/run-clang-format.py -r common decompiler game goalc test --color always
   build-linux:
     name: (Linux) Build & Test
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     defaults:
       run:
         shell: bash
@@ -50,7 +50,7 @@ jobs:
   build-windows:
     # Very good resource - https://cristianadam.eu/20191222/using-github-actions-with-c-plus-plus-and-cmake/
     name: (Windows) Build & Test
-    runs-on: windows-latest
+    runs-on: windows-2019
     defaults:
       run:
         shell: powershell

CMakeLists.txt

@@ -1,7 +1,6 @@
 # Top Level CMakeLists.txt
 cmake_minimum_required(VERSION 3.16)
 project(jak)
-
 if(NOT CMAKE_BUILD_TYPE)
     set(CMAKE_BUILD_TYPE "Debug")
 endif()
@@ -25,8 +24,7 @@ if (CMAKE_COMPILER_IS_GNUCXX)
         -Wmissing-include-dirs \
         -Woverloaded-virtual \
         -Wredundant-decls \
-        -Wshadow \
-        -Wsign-promo")
+        -Wsign-promo") # TODO - add back -Wshadow once fixed inside inja, lots of compiler warnings
 else ()
     set(CMAKE_CXX_FLAGS "/EHsc")
     set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /STACK:10000000")

goal_src/test/test-add-int-constants.gc

@@ -1 +0,0 @@
-(+ 15 -2)

goal_src/test/test-empty-pair.gc

@@ -1,2 +0,0 @@
-(format #t "~A~%" '())
-0

goal_src/test/test-let-1.gc

@@ -1,4 +0,0 @@
-  (let ((x 1)
-	(y (test-function 1 2 3 4))
-	(z 3))
-    y)

goal_src/test/test-return-integer-2.gc

@@ -1 +0,0 @@
-#x17

goal_src/test/test-return-integer-3.gc

@@ -1 +0,0 @@
--17

goal_src/test/test-return-integer-4.gc

@@ -1 +0,0 @@
--2147483648

goal_src/test/test-return-integer-5.gc

@@ -1 +0,0 @@
--2147483649

goal_src/test/test-return-integer-6.gc

@@ -1 +0,0 @@
-0

goal_src/test/test-return-integer-7.gc

@@ -1 +0,0 @@
--123

test/CMakeLists.txt

@@ -1,3 +1,7 @@
+set(CMAKE_CXX_STANDARD 17)
+
+include("goalc/CMakeLists.txt")
+
 add_executable(goalc-test
         test_main.cpp
         test_test.cpp
@@ -13,10 +17,9 @@ add_executable(goalc-test
         test_emitter_xmm32.cpp
         test_emitter_integer_math.cpp
         test_common_util.cpp
-        test_compiler_and_runtime.cpp
         test_deftype.cpp
-        test_pretty_print.cpp
-        )
+				${GOALC_TEST_FRAMEWORK_SOURCES}
+				${GOALC_TEST_CASES})
 
 enable_testing()
 
@@ -24,7 +27,7 @@ IF (WIN32)
 	set(gtest_force_shared_crt ON CACHE BOOL "" FORCE)
 
 	# TODO - split out these declarations for platform specific includes
-  target_link_libraries(goalc-test cross_sockets listener mman goos common_util runtime compiler type_system gtest)
+  target_link_libraries(goalc-test cross_sockets goos common_util listener runtime compiler type_system gtest mman)
 ELSE()
   target_link_libraries(goalc-test cross_sockets goos common_util listener runtime compiler type_system gtest)
 ENDIF()

test/goalc/CMakeLists.txt

@@ -0,0 +1,25 @@
+# TODO - probably a more cmakey way to do this
+
+set(GOALC_TEST_CASES
+		"goalc/test_arithmetic.cpp"
+		"goalc/test_compiler.cpp"
+    "goalc/test_control_statements.cpp"
+		"goalc/test_collections.cpp"
+		"goalc/test_float.cpp"
+		"goalc/test_functions.cpp"
+		"goalc/test_library.cpp"
+		"goalc/test_logic.cpp"
+		"goalc/test_loop_recur.cpp"
+		"goalc/test_macros.cpp"
+		"goalc/test_methods.cpp"
+		"goalc/test_pointers.cpp"
+		"goalc/test_strings.cpp"
+		"goalc/test_symbols.cpp"
+		"goalc/test_variables.cpp"
+		"goalc/test_with_game.cpp"
+)
+
+set(GOALC_TEST_FRAMEWORK_SOURCES
+		"goalc/framework/test_runner.cpp"
+		"goalc/framework/test_runner.h"
+)

test/goalc/README.md

@@ -0,0 +1,8 @@
+# Some Documentation
+
+TODO!
+
+# TODO
+
+- If it can't make the file successfully, currently the tests just hang
+- How do i share the same fixture (compiler/thread instance), but with different Params.  I don't think this is possible...maybe with templates?

test/goalc/framework/test_runner.cpp

@@ -0,0 +1,117 @@
+
+#include "test_runner.h"
+
+#include <string>
+
+#include "gtest/gtest.h"
+#include "third-party/inja.hpp"
+#include "third-party/json.hpp"
+
+#include "game/runtime.h"
+#include "goalc/listener/Listener.h"
+#include "goalc/compiler/Compiler.h"
+
+#include "common/util/FileUtil.h"
+#include <filesystem>
+
+namespace GoalTest {
+
+std::string escaped_string(const std::string& in) {
+  std::string result;
+  for (auto x : in) {
+    switch (x) {
+      case '\n':
+        result.append("\\n");
+        break;
+      case '\t':
+        result.append("\\t");
+        break;
+      default:
+        result.push_back(x);
+    }
+  }
+  return result;
+}
+
+void CompilerTestRunner::run_static_test(inja::Environment& env,
+                                         std::string& testCategory,
+                                         const std::string& test_file,
+                                         const std::vector<std::string>& expected,
+                                         MatchParam<int> truncate) {
+  env.write(test_file, {}, test_file);
+  run_test(testCategory, test_file, expected, truncate);
+}
+
+void CompilerTestRunner::run_test(const std::string& test_category,
+                                  const std::string& test_file,
+                                  const std::vector<std::string>& expected,
+                                  MatchParam<int> truncate) {
+  fprintf(stderr, "Testing %s\n", test_file.c_str());
+  auto result = c->run_test("test/goalc/source_generated/" + test_category + "/" + test_file);
+  if (!truncate.is_wildcard) {
+    for (auto& x : result) {
+      x = x.substr(0, truncate.value);
+    }
+  }
+
+  bool assertionFailed = false;
+  EXPECT_EQ(result, expected) << (assertionFailed = true);
+
+  if (assertionFailed) {
+    std::string testFile = GoalTest::getGeneratedDir(test_category) + test_file;
+    std::string failedFile = GoalTest::getFailedDir(test_category) + test_file;
+
+    GoalTest::createDirIfAbsent(GoalTest::getFailedDir(test_category));
+
+    std::ifstream src(testFile, std::ios::binary);
+    std::ofstream dst(failedFile, std::ios::binary);
+
+    std::string testOutput = "\n\n;------TEST OUTPUT------\n;-------Expected-------\n";
+
+    for (auto& x : expected) {
+      testOutput += fmt::format("; \"{}\"\n", escaped_string(x));
+    }
+    testOutput += "\n;--------Actual--------\n";
+    for (auto& x : result) {
+      testOutput += fmt::format("; \"{}\"\n", escaped_string(x));
+    }
+
+    dst << src.rdbuf() << testOutput;
+  }
+
+  tests.push_back({expected, result, test_file, false});
+}
+
+void CompilerTestRunner::run_always_pass(const std::string& test_category,
+                                         const std::string& test_file) {
+  c->run_test("test/goalc/source_generated/" + test_category + "/" + test_file);
+  tests.push_back({{}, {}, test_file, true});
+}
+
+void runtime_no_kernel() {
+  constexpr int argc = 4;
+  const char* argv[argc] = {"", "-fakeiso", "-debug", "-nokernel"};
+  exec_runtime(argc, const_cast<char**>(argv));
+}
+
+void runtime_with_kernel() {
+  constexpr int argc = 3;
+  const char* argv[argc] = {"", "-fakeiso", "-debug"};
+  exec_runtime(argc, const_cast<char**>(argv));
+}
+
+void createDirIfAbsent(const std::string& path) {
+  if (!std::filesystem::is_directory(path) || !std::filesystem::exists(path)) {
+    std::filesystem::create_directory(path);
+  }
+}
+std::string getTemplateDir(const std::string& category) {
+  return file_util::get_file_path({"test/goalc/source_templates", category + "/"});
+}
+std::string getGeneratedDir(const std::string& category) {
+  return file_util::get_file_path({"test/goalc/source_generated", category + "/"});
+}
+std::string getFailedDir(const std::string& category) {
+  return file_util::get_file_path({"test/goalc/source_generated/failed", category + "/"});
+}
+}  // namespace GoalTest

test/goalc/framework/test_runner.h

@@ -0,0 +1,49 @@
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "third-party/inja.hpp"
+#include "goalc/compiler/Compiler.h"
+
+namespace GoalTest {
+
+std::string escaped_string(const std::string& in);
+
+struct CompilerTestRunner {
+ public:
+  Compiler* c = nullptr;
+
+  struct Test {
+    std::vector<std::string> expected, actual;
+    std::string test_name;
+    bool auto_pass = false;
+  };
+
+  std::vector<Test> tests;
+
+  void run_static_test(inja::Environment& env,
+                       std::string& testCategory,
+                       const std::string& test_file,
+                       const std::vector<std::string>& expected,
+                       MatchParam<int> truncate = {});
+
+  void run_test(const std::string& test_category,
+                const std::string& test_file,
+                const std::vector<std::string>& expected,
+                MatchParam<int> truncate = {});
+
+  void run_always_pass(const std::string& test_category, const std::string& test_file);
+
+  void print_summary();
+};
+
+void runtime_no_kernel();
+void runtime_with_kernel();
+
+void createDirIfAbsent(const std::string& path);
+std::string getTemplateDir(const std::string& category);
+std::string getGeneratedDir(const std::string& category);
+std::string getFailedDir(const std::string& category);
+
+}  // namespace GoalTest

test/goalc/source_generated/.gitignore

@@ -0,0 +1,2 @@
+*.gc
+failed/

test/goalc/source_templates/arithmetic/add-int-literals.static.gc

@@ -0,0 +1,4 @@
+; TODO this would be an easy function to templatize
+; but I'm not sure how to easily do it in test_arithmetic.cpp without repeating the fixture boilerplate
+
+(+ 15 -2)

test/goalc/source_templates/arithmetic/eval-integer.template.gc

@@ -1,2 +1,2 @@
 ; simply return an integer
-#x123456789
+{{ integer }}

test/goalc/source_templates/collections/car-cdr-get.static.gc

@@ -1,3 +1,5 @@
+(define format _format)
+
 (let ((my-pair (cons 'a 'b)))
   (format #t "~A~A~%" (car my-pair) (cdr my-pair))
   )

test/goalc/source_templates/collections/car-cdr-set.static.gc

@@ -1,3 +1,5 @@
+(define format _format)
+
 (let ((my-pair (cons 'a 'b)))
   (set! (car my-pair) 'c)
   (set! (cdr my-pair) 'd)

test/goalc/source_templates/collections/cons.static.gc

@@ -1,2 +1,4 @@
+(define format _format)
+
 (format #t "~A~%" (cons 'a 'b))
 0

test/goalc/source_templates/collections/empty-pair.static.gc

@@ -0,0 +1,4 @@
+(define format _format)
+
+(format #t "~A~%" '())
+0

test/goalc/source_templates/collections/inline-array-field.static.gc

@@ -1,3 +1,5 @@
+(define format _format)
+
 (let* ((base (the int integer))
        (field (the int (-> integer method-table)))
        (offset (- field base)))

test/goalc/source_templates/collections/list.static.gc

@@ -1 +1,3 @@
+(define format _format)
+
 (format #t "~A~%" (list 'a 'b 'c 'd))

test/goalc/source_templates/collections/nested-car-cdr-set.static.gc

@@ -1,3 +1,5 @@
+(define format _format)
+
 (let ((my-pair (cons (cons 'a 'b) (cons 'c 'd))))
   (set! (car (car my-pair)) 'e)
   (set! (car (cdr my-pair)) 'f)

test/goalc/source_templates/collections/pair-check.static.gc

@@ -1,2 +1,4 @@
+(define format _format)
+
 (format #t "~A~A~%" (pair? '()) (pair? integer))
 0

test/goalc/source_templates/control-statements/conditional-compilation.static.gc

@@ -0,0 +1,12 @@
+;; test the use of #cond to evaluate goos expressions at compile time
+
+(#cond
+ ((> 2 (+ 2 1))
+  1
+  (invalid-code)
+  )
+
+ ((< 2 (+ 1 2))
+  3
+  )
+ )

test/goalc/source_templates/control-statements/return-value-of-if.static.gc

@@ -1,3 +1,5 @@
+(define format _format)
+
 (let ((x (if (> 1 2) "a string!"))
       (y (if (> 2 1) 123)))
   (if x

test/goalc/source_templates/float/float-function.static.gc

@@ -1,3 +1,5 @@
+(define format _format)
+
 (defun float-testing-function ((x float) (y float))
   (* x y (* x x))
   )

test/goalc/source_templates/float/float-in-symbol.static.gc

@@ -1,2 +1,4 @@
+(define format _format)
+
 (define float-symbol 2345.6)
 (format #t "~f~%" float-symbol)

test/goalc/source_templates/float/float-pow.static.gc

@@ -1,3 +1,5 @@
+(define format _format)
+
 (defun pow-test ((base float) (exponent integer))
   (let ((result base))
     (while (> exponent 1)

test/goalc/source_templates/float/float-product.static.gc

@@ -1 +1,3 @@
+(define format _format)
+
 (format #t "~f~%" (* 1.2 25.0 4.0))

test/goalc/source_templates/float/function-return-float-constant.static.gc

@@ -1,3 +1,5 @@
+(define format _format)
+
 (defun return-const-float ()
   3.1415
   )

test/goalc/source_templates/methods/methods.static.gc

@@ -1,3 +1,5 @@
+(define format _format)
+
 (format #t "~A~A~%" (eq? (-> process method-table 2) (method process print))
                     (eq? (-> string method-table 3) (method "test" inspect))
                     )

test/goalc/source_templates/strings/quote-symbol.static.gc

@@ -1,3 +1,5 @@
+(define format _format)
+
 (define my-thing 'apple)
 (set! my-thing 'banana)
 (format #t "~A~%" my-thing)

test/goalc/source_templates/variables/let-star.static.gc

@@ -1,3 +1,6 @@
+(define-extern test-function (function int int int int int))
+(test-function 1 2 3 4)
+
 (define *test-result*
   (let* ((x 1)
 	 (y 2)

test/goalc/source_templates/variables/let.static.gc

@@ -0,0 +1,7 @@
+(define-extern test-function (function int int int int int))
+(test-function 1 2 3 4)
+
+  (let ((x 1)
+	(y (test-function 1 2 3 4))
+	(z 3))
+    y)

test/goalc/source_templates/with_game/defun-return-constant.static.gc

@@ -0,0 +1,10 @@
+(defun return-13 ()
+  13)
+
+(defun return-12 ()
+  12)
+
+(defun return-11 ()
+  11)
+
+(return-12)

test/goalc/source_templates/with_game/defun-return-symbol.static.gc

@@ -0,0 +1,8 @@
+(define my-number 36)
+
+(defun return-my-number ()
+  my-number)
+
+(define my-number 42)
+
+(return-my-number)

test/goalc/source_templates/with_game/test-access-inline-array.gc

@@ -1,3 +1,5 @@
+(define format _format)
+
 (let* ((print-method (method bfloat print))
        (my-float (new 'global 'bfloat))
        )