diff --git a/CMakeLists.txt b/CMakeLists.txt index af6508aeee..28c2792700 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,7 +5,7 @@ project(jak) set(CMAKE_CXX_STANDARD 14) # optimization level can be set here. Note that game/ overwrites this for building game C++ code. -set(CMAKE_CXX_FLAGS "-O3 -ggdb") +set(CMAKE_CXX_FLAGS "-O3 -ggdb /EHsc") # Set default compile flags for GCC if(CMAKE_COMPILER_IS_GNUCXX) diff --git a/goalc/goos/Reader.cpp b/goalc/goos/Reader.cpp index a0357f9103..a87fd8d024 100644 --- a/goalc/goos/Reader.cpp +++ b/goalc/goos/Reader.cpp @@ -66,6 +66,10 @@ void TextStream::seek_past_whitespace_and_comments() { } } +Reader::~Reader() { + printf("destroying reader\n"); +} + Reader::Reader() { // third-party library used for a fancy line in linenoise::SetHistoryMaxLen(400); @@ -76,33 +80,24 @@ Reader::Reader() { add_reader_macro(",", "unquote"); add_reader_macro(",@", "unquote-splicing"); - // setup table of which characters are valid for starting a symbol - for (auto& x : valid_symbols_chars) { - x = false; - } - - for (char x = 'a'; x <= 'z'; x++) { - valid_symbols_chars[(int)x] = true; - } - - for (char x = 'A'; x <= 'Z'; x++) { - valid_symbols_chars[(int)x] = true; - } - - for (char x = '0'; x <= '9'; x++) { - valid_symbols_chars[(int)x] = true; - } - - const char bonus[] = "!$%&*+-/\\.,@^_-;:<>?~=#"; - - for (const char* c = bonus; *c; c++) { - valid_symbols_chars[(int)*c] = true; + uint8_t valid[256] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 0, 1, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + for (int i = 0; i < 256; i++) { + valid_symbols_chars[i] = valid[i]; } // find the source directory auto result = std::getenv("NEXT_DIR"); if (!result) { - throw std::exception( + throw std::runtime_error( "Environment variable NEXT_DIR is not set. Please set this to point to next/"); } @@ -639,6 +634,7 @@ bool Reader::try_token_as_hex(const Token& tok, Object& obj) { * 64-bit signed. Won't accept values between INT64_MAX and UINT64_MAX. */ bool Reader::try_token_as_integer(const Token& tok, Object& obj) { + printf("try token as integer %ld %s\n", tok.text.size(), tok.text.c_str()); if (decimal_start(tok.text[0]) && !str_contains(tok.text, '.')) { // determine if we look like a number or not. If we look like a number, but stoll fails, // it means that the number is too big or too small, and we should error @@ -653,19 +649,29 @@ bool Reader::try_token_as_integer(const Token& tok, Object& obj) { } } + printf("going to try stoll...\n"); uint64_t v = 0; try { std::size_t end = 0; v = std::stoll(tok.text, &end); - if (end != tok.text.size()) + printf("stoll didn't throw, got %ld\n", v); + if (end != tok.text.size()) { + printf("didn't read whole thing\n"); return false; + } + printf("returning object!\n"); obj = Object::make_integer(v); return true; - } catch (std::exception& e) { - throw std::runtime_error("The number cannot be an integer constant"); + } + + catch (std::exception& e) { + printf("stoll threw\n"); + throw std::runtime_error("The number " + tok.text + " cannot be an integer constant"); + } } return false; + } bool Reader::try_token_as_char(const Token& tok, Object& obj) { diff --git a/goalc/goos/Reader.h b/goalc/goos/Reader.h index 1ac7e6a7a1..06710994b4 100644 --- a/goalc/goos/Reader.h +++ b/goalc/goos/Reader.h @@ -68,6 +68,7 @@ struct Token { class Reader { public: Reader(); + ~Reader(); Object read_from_string(const std::string& str); Object read_from_stdin(const std::string& prompt_name); Object read_from_file(const std::string& filename); diff --git a/test/test_reader.cpp b/test/test_reader.cpp index 534da6a60d..69c9b3099f 100644 --- a/test/test_reader.cpp +++ b/test/test_reader.cpp @@ -54,7 +54,6 @@ TEST(GoosReader, Integer) { // too big or too small. EXPECT_ANY_THROW(reader.read_from_string("9223372036854775808")); EXPECT_ANY_THROW(reader.read_from_string("-9223372036854775809")); - printf("got here"); } TEST(GoosReader, Hex) { @@ -116,7 +115,6 @@ TEST(GoosReader, Binary) { EXPECT_TRUE(check_first_symbol(reader.read_from_string("#b.1"), "#b.1")); printf("got here"); } - TEST(GoosReader, Float) { Reader reader; @@ -237,110 +235,110 @@ bool first_char_matches(Object o, char c) { } // namespace -//TEST(GoosReader, List) { -// Reader reader; -// auto r = [&](std::string s) { return reader.read_from_string(s); }; -// EXPECT_TRUE(first_list_matches(r("()"), {})); -// EXPECT_TRUE(first_list_matches(r("(1)"), {r("1")})); -// EXPECT_TRUE(first_list_matches(r(" ( 1 ) "), {r("1")})); -// EXPECT_TRUE(first_list_matches(r("(1 2 3)"), {r("1"), r("2"), r("3")})); -// EXPECT_TRUE(first_list_matches(r(" ( 1 bbbb 3 ) "), {r("1"), r("bbbb"), r("3")})); -// -// EXPECT_TRUE(first_pair_matches(r("(1 . 2)"), Object::make_integer(1), Object::make_integer(2))); -// -// EXPECT_TRUE(print_matches(r(" ( 1 . 2 ) "), "(1 . 2)")); -// EXPECT_TRUE(print_matches(r(" ( 1 1 . 2 ) "), "(1 1 . 2)")); -// EXPECT_TRUE(print_matches(r(" ( 1 . ( 1 . 2 ) ) "), "(1 1 . 2)")); -// EXPECT_TRUE( -// print_matches(r(" ( 1 ( 1 2 ) ( 1 ( 12 3 ) ) . 3 ) "), "(1 (1 2) (1 (12 3)) . 3)")); -// EXPECT_TRUE( -// print_matches(r(" ( 1 ( 1 2 ) ( 1 ( 12 3 ) ) . ( ) ) "), "(1 (1 2) (1 (12 3)))")); -// -// std::vector expected_to_throw = {"(", ")", " (", " )()() ", -// ")(", "(1 2 ))", "(( 1 2)", "(1 . . 2)", -// "(1 . )", "(1 . 2 3)", "( . 2)"}; -// -// for (const auto& x : expected_to_throw) { -// EXPECT_ANY_THROW(r(x)); -// } -// printf("got here"); -//} -// -//TEST(GoosReader, Comments) { -// Reader reader; -// auto r = [&](std::string s) { return reader.read_from_string(s); }; -// EXPECT_TRUE(first_list_matches(r(";;\n(1)\n;;"), {r("1")})); -// EXPECT_TRUE(first_list_matches(r(";;\n(;1\n1;)\n);;\n;"), {r("1")})); -// -// r(";"); -// r(" ;"); -// r("\n;"); -// r(";\n"); -// -// EXPECT_TRUE(first_list_matches( -// r("#|multi line\n com(((((ment |# (1) #| multi line\n comm)))))ent |#"), {r("1")})); -// EXPECT_TRUE(first_list_matches( -// r("#| #| multi l#|ine\n com#|ment |# (1) #| multi line\n commen))))))t |#"), {r("1")})); -// -// std::vector expected_to_throw = {"|#", "#| |# |#"}; -// -// for (const auto& x : expected_to_throw) { -// EXPECT_ANY_THROW(r(x)); -// } -//} -// -//TEST(GoosReader, Char) { -// Reader reader; -// auto r = [&](std::string s) { return reader.read_from_string(s); }; -// -// EXPECT_TRUE(first_char_matches(r("#\\c"), 'c')); -// EXPECT_TRUE(first_char_matches(r("#\\n"), 'n')); -// EXPECT_TRUE(first_char_matches(r("#\\\\n"), '\n')); -// EXPECT_TRUE(first_char_matches(r("#\\\\t"), '\t')); -// EXPECT_TRUE(first_char_matches(r("#\\\\s"), ' ')); -// printf("got here"); -//} +TEST(GoosReader, List) { + Reader reader; + auto r = [&](std::string s) { return reader.read_from_string(s); }; + EXPECT_TRUE(first_list_matches(r("()"), {})); + EXPECT_TRUE(first_list_matches(r("(1)"), {r("1")})); + EXPECT_TRUE(first_list_matches(r(" ( 1 ) "), {r("1")})); + EXPECT_TRUE(first_list_matches(r("(1 2 3)"), {r("1"), r("2"), r("3")})); + EXPECT_TRUE(first_list_matches(r(" ( 1 bbbb 3 ) "), {r("1"), r("bbbb"), r("3")})); -//TEST(GoosReader, Array) { -// Reader reader; -// auto r = [&](std::string s) { return reader.read_from_string(s); }; -// EXPECT_TRUE(print_matches(r(" #( ) "), "#()")); -// EXPECT_TRUE(first_array_matches(r("#()"), {})); -// EXPECT_TRUE(first_array_matches(r("#(1 2)"), {Object::make_integer(1), Object::make_integer(2)})); -// EXPECT_TRUE(first_array_matches(r("#( 1 #| 2 |# 3 )"), -// {Object::make_integer(1), Object::make_integer(3)})); -// EXPECT_TRUE( -// first_array_matches(r("#( 1 #|2|# 3 )"), {Object::make_integer(1), Object::make_integer(3)})); -//} -// -//TEST(GoosReader, Macros) { -// Reader reader; -// auto r = [&](std::string s) { return reader.read_from_string(s); }; -// EXPECT_TRUE(print_matches(r("'x"), "(quote x)")); -// EXPECT_TRUE(print_matches(r("`x"), "(quasiquote x)")); -// EXPECT_TRUE(print_matches(r(",x"), "(unquote x)")); -// EXPECT_TRUE(print_matches(r(",@x"), "(unquote-splicing x)")); -//} -// -//TEST(GoosReader, TopLevel) { -// Reader reader; -// auto r = [&](std::string s) { return reader.read_from_string(s); }; -// EXPECT_EQ(r("x").print(), "(top-level x)"); -//} -// -//TEST(GoosReader, FromFile) { -// Reader reader; -// auto result = -// reader.read_from_file(util::combine_path({"test", "test_reader_file0.gc"})).print(); -// EXPECT_TRUE(result == "(top-level (1 2 3 4))"); -//} -// -//TEST(GoosReader, TextDb) { -// // very specific to this particular test file, but whatever. -// Reader reader; -// auto file_path = util::combine_path({"test", "test_reader_file0.gc"}); -// auto result = reader.read_from_file(file_path).as_pair()->cdr.as_pair()->car; -// std::string expected = "text from " + util::combine_path(reader.get_source_dir(), file_path) + -// ", line: 5\n(1 2 3 4)\n"; -// EXPECT_EQ(expected, reader.db.get_info_for(result)); -//} + EXPECT_TRUE(first_pair_matches(r("(1 . 2)"), Object::make_integer(1), Object::make_integer(2))); + + EXPECT_TRUE(print_matches(r(" ( 1 . 2 ) "), "(1 . 2)")); + EXPECT_TRUE(print_matches(r(" ( 1 1 . 2 ) "), "(1 1 . 2)")); + EXPECT_TRUE(print_matches(r(" ( 1 . ( 1 . 2 ) ) "), "(1 1 . 2)")); + EXPECT_TRUE( + print_matches(r(" ( 1 ( 1 2 ) ( 1 ( 12 3 ) ) . 3 ) "), "(1 (1 2) (1 (12 3)) . 3)")); + EXPECT_TRUE( + print_matches(r(" ( 1 ( 1 2 ) ( 1 ( 12 3 ) ) . ( ) ) "), "(1 (1 2) (1 (12 3)))")); + + std::vector expected_to_throw = {"(", ")", " (", " )()() ", + ")(", "(1 2 ))", "(( 1 2)", "(1 . . 2)", + "(1 . )", "(1 . 2 3)", "( . 2)"}; + + for (const auto& x : expected_to_throw) { + EXPECT_ANY_THROW(r(x)); + } + printf("got here"); +} + +TEST(GoosReader, Comments) { + Reader reader; + auto r = [&](std::string s) { return reader.read_from_string(s); }; + EXPECT_TRUE(first_list_matches(r(";;\n(1)\n;;"), {r("1")})); + EXPECT_TRUE(first_list_matches(r(";;\n(;1\n1;)\n);;\n;"), {r("1")})); + + r(";"); + r(" ;"); + r("\n;"); + r(";\n"); + + EXPECT_TRUE(first_list_matches( + r("#|multi line\n com(((((ment |# (1) #| multi line\n comm)))))ent |#"), {r("1")})); + EXPECT_TRUE(first_list_matches( + r("#| #| multi l#|ine\n com#|ment |# (1) #| multi line\n commen))))))t |#"), {r("1")})); + + std::vector expected_to_throw = {"|#", "#| |# |#"}; + + for (const auto& x : expected_to_throw) { + EXPECT_ANY_THROW(r(x)); + } +} + +TEST(GoosReader, Char) { + Reader reader; + auto r = [&](std::string s) { return reader.read_from_string(s); }; + + EXPECT_TRUE(first_char_matches(r("#\\c"), 'c')); + EXPECT_TRUE(first_char_matches(r("#\\n"), 'n')); + EXPECT_TRUE(first_char_matches(r("#\\\\n"), '\n')); + EXPECT_TRUE(first_char_matches(r("#\\\\t"), '\t')); + EXPECT_TRUE(first_char_matches(r("#\\\\s"), ' ')); + printf("got here"); +} + +TEST(GoosReader, Array) { + Reader reader; + auto r = [&](std::string s) { return reader.read_from_string(s); }; + EXPECT_TRUE(print_matches(r(" #( ) "), "#()")); + EXPECT_TRUE(first_array_matches(r("#()"), {})); + EXPECT_TRUE(first_array_matches(r("#(1 2)"), {Object::make_integer(1), Object::make_integer(2)})); + EXPECT_TRUE(first_array_matches(r("#( 1 #| 2 |# 3 )"), + {Object::make_integer(1), Object::make_integer(3)})); + EXPECT_TRUE( + first_array_matches(r("#( 1 #|2|# 3 )"), {Object::make_integer(1), Object::make_integer(3)})); +} + +TEST(GoosReader, Macros) { + Reader reader; + auto r = [&](std::string s) { return reader.read_from_string(s); }; + EXPECT_TRUE(print_matches(r("'x"), "(quote x)")); + EXPECT_TRUE(print_matches(r("`x"), "(quasiquote x)")); + EXPECT_TRUE(print_matches(r(",x"), "(unquote x)")); + EXPECT_TRUE(print_matches(r(",@x"), "(unquote-splicing x)")); +} + +TEST(GoosReader, TopLevel) { + Reader reader; + auto r = [&](std::string s) { return reader.read_from_string(s); }; + EXPECT_EQ(r("x").print(), "(top-level x)"); +} + +TEST(GoosReader, FromFile) { + Reader reader; + auto result = + reader.read_from_file(util::combine_path({"test", "test_reader_file0.gc"})).print(); + EXPECT_TRUE(result == "(top-level (1 2 3 4))"); +} + +TEST(GoosReader, TextDb) { + // very specific to this particular test file, but whatever. + Reader reader; + auto file_path = util::combine_path({"test", "test_reader_file0.gc"}); + auto result = reader.read_from_file(file_path).as_pair()->cdr.as_pair()->car; + std::string expected = "text from " + util::combine_path(reader.get_source_dir(), file_path) + + ", line: 5\n(1 2 3 4)\n"; + EXPECT_EQ(expected, reader.db.get_info_for(result)); +}