#include #include "Reader.h" #include "TextDb.h" #include "third-party/linenoise.h" Reader::Reader() { linenoise::SetHistoryMaxLen(400); for (auto& x : valid_symbols_chars) { x = false; } for (char x = 'a'; x <= 'z'; x++) { valid_symbols_chars[(int)x] = true; } for (char x = 'A'; x <= 'Z'; x++) { valid_symbols_chars[(int)x] = true; } for (char x = '0'; x <= '9'; x++) { valid_symbols_chars[(int)x] = true; } const char bonus[] = "!$%&*+-/\\.,@^_-;:<>?~=#"; for (const char* c = bonus; *c; c++) { valid_symbols_chars[(int)*c] = true; } } Object Reader::read_from_stdin(const std::string& prompt_name) { // // display prompt // printf("%s> ", prompt_name.c_str()); // // // read text // std::string line; // std::getline(std::cin, line); std::string line; std::string prompt_full = "\033[0m" + prompt_name + "> "; linenoise::Readline(prompt_full.c_str(), line); linenoise::AddHistory(line.c_str()); // todo, decide if we should keep reading or not. // create text fragment and add to the DB auto textFrag = std::make_shared(line); db.insert(textFrag); // perform read auto result = internal_read(textFrag); db.link(result, textFrag, 0); return result; } Object Reader::read_from_string(const std::string& str) { // create text fragment and add to the DB auto textFrag = std::make_shared(str); db.insert(textFrag); // perform read auto result = internal_read(textFrag); db.link(result, textFrag, 0); return result; } Object Reader::read_from_file(const std::string& filename) { auto textFrag = std::make_shared(get_next_dir() + "/" + filename); db.insert(textFrag); auto result = internal_read(textFrag); db.link(result, textFrag, 0); return result; } Object Reader::internal_read(std::shared_ptr text) { // first create stream TextStream ts(text); // clean up first whitespace seek_past_whitespace_and_comments(ts); // read list! auto objs = read_list(ts, false); return PairObject::make_new(SymbolObject::make_new(symbolTable, "top-level"), objs); } void Reader::seek_past_whitespace_and_comments(TextStream& stream) { while (stream.text_remains()) { char c = stream.peek(); switch (c) { case ' ': case '\t': case '\n': // just a whitespace, eat it! stream.read(); break; case ';': // line comment. while (stream.text_remains() && stream.read() != '\n') { } break; case '#': if (stream.text_remains(1) && stream.peek(1) == '|') { assert(stream.read() == '#'); // # assert(stream.read() == '|'); // | bool found_end = false; // find |# while (stream.text_remains() && !found_end) { // find | while (stream.text_remains() && stream.read() != '|') { } if (stream.text_remains() && stream.read() == '#') { found_end = true; } } return; } else { // not a line comment return; } break; default: return; } } } // given a stream starting at first character of the token, // return the token. does not consume any whitespace at the end. Token Reader::get_next_token(TextStream& stream) { assert(stream.text_remains()); Token t; t.source_line = stream.line_count; t.source_offset = stream.seek; t.source_text = stream.text; char first = stream.read(); t.text.push_back(first); // paren/double quote is its own token. if (first == '(' || first == ')' || first == '"' || first == '\'' || first == '`') return t; if (first == ',' && stream.text_remains() && stream.peek() == '@') { t.text.push_back(stream.read()); return t; } else if (first == ',') { return t; } while (stream.text_remains()) { char next = stream.peek(); if (next == ' ' || next == '\n' || next == '\t' || next == ')' || next == ';' /*|| next == '#'*/ || next == '(') { return t; } else { t.text.push_back(stream.read()); } } return t; } const static std::unordered_map reader_macros = { {"'", "quote"}, {"`", "quasiquote"}, {",", "unquote"}, {",@", "unquote-splicing"}}; // call on char after open paren. Object Reader::read_list(TextStream& ts, bool expect_close_paren) { seek_past_whitespace_and_comments(ts); std::vector objects; bool got_close_paren = false; int start_offset = ts.seek; while (ts.text_remains()) { auto tok = get_next_token(ts); // reader macro thing: bool got_reader_macro = false; std::string reader_macro_string; auto kv = reader_macros.find(tok.text); if (kv != reader_macros.end()) { got_reader_macro = true; reader_macro_string = kv->second; tok = get_next_token(ts); } auto insert_object = [&](Object o) { if (got_reader_macro) { objects.push_back( build_list({SymbolObject::make_new(symbolTable, reader_macro_string), o})); } else { objects.push_back(o); } }; if (tok.text.empty()) { // empty list break; } else if (tok.text[0] == '(') { assert(tok.text.length() == 1); insert_object(read_list(ts, true)); seek_past_whitespace_and_comments(ts); continue; } else if (tok.text[0] == ')') { got_close_paren = true; assert(tok.text.length() == 1); break; } else { Object obj; // try as an array // try as char // try as integer if (try_token_as_integer(tok, obj)) { seek_past_whitespace_and_comments(ts); insert_object(obj); continue; } // try as hex if (try_token_as_hex(tok, obj)) { seek_past_whitespace_and_comments(ts); insert_object(obj); continue; } // try as binary if (try_token_as_binary(tok, obj)) { seek_past_whitespace_and_comments(ts); insert_object(obj); continue; } // try as float if (try_token_as_float(tok, obj)) { seek_past_whitespace_and_comments(ts); insert_object(obj); continue; } // try as bool if (try_token_as_boolean(tok, obj)) { seek_past_whitespace_and_comments(ts); insert_object(obj); continue; } // try as string if (tok.text[0] == '"') { // it's a string. assert(tok.text.length() == 1); if (read_string(ts, obj)) { seek_past_whitespace_and_comments(ts); insert_object(obj); } else { display_error_info(ts, "failed to read string, close quote not found"); } continue; } // try as symbol if (try_token_as_symbol(tok, obj)) { insert_object(obj); seek_past_whitespace_and_comments(ts); continue; } display_error_info(ts, "invalid token encountered in reader: " + tok.text); } } if (expect_close_paren && !got_close_paren) { display_error_info(ts, "failed to find close paren"); } if (got_close_paren && !expect_close_paren) { display_error_info(ts, "found an unexpected close paren"); } auto rv = build_list(objects); db.link(rv, ts.text, start_offset); return rv; } bool Reader::try_token_as_symbol(const Token& tok, Object& obj) { // check start character is valid: assert(!tok.text.empty()); char start = tok.text[0]; // if(start == '!' || start == '*' || start == '/' || start == '+' || start == '-' || start == '=' // || start == '<' || start == '>' || start == '?' || (start >= 'A' && start < 'Z') || (start >= // 'a' && start <= 'z')) { if (valid_symbols_chars[(int)start]) { obj = SymbolObject::make_new(symbolTable, tok.text); return true; } else { return false; } } // todo, handle \n in a string. bool Reader::read_string(TextStream& stream, Object& obj) { bool got_close_quote = false; std::string str; while (stream.text_remains()) { char c = stream.read(); if (c == '"') { obj = StringObject::make_new(str); got_close_quote = true; break; } str.push_back(c); } return got_close_quote; } static bool number_start(char c) { return (c >= '0' && c <= '9') || c == '-'; } static bool str_contains(const std::string& str, char c) { for (auto& x : str) if (x == c) return true; return false; } bool Reader::try_token_as_float(const Token& tok, Object& obj) { if (number_start(tok.text[0]) && str_contains(tok.text, '.')) { try { std::size_t end = 0; double v = std::stod(tok.text, &end); if (end != tok.text.size()) return false; obj = Object::make_float(v); return true; } catch (std::exception& e) { return false; } } return false; } bool Reader::try_token_as_binary(const Token& tok, Object& obj) { if (tok.text.size() >= 3 && tok.text[0] == '#' && tok.text[1] == 'b') { uint64_t value = 0; if (tok.text.size() > 64 + 2) return false; for (uint32_t i = 2; i < tok.text.size(); i++) { value <<= 1u; if (tok.text[i] == '1') value++; else if (tok.text[i] != '0') return false; } obj = Object::make_integer((int64_t)value); return true; } return false; } bool Reader::try_token_as_hex(const Token& tok, Object& obj) { if (tok.text.size() >= 3 && tok.text[0] == '#' && tok.text[1] == 'x') { uint64_t v = 0; try { std::size_t end = 0; v = std::stoll(tok.text.substr(2), &end, 16); if (end + 2 != tok.text.size()) return false; obj = Object::make_integer(v); return true; } catch (std::exception& e) { return false; } } return false; } bool Reader::try_token_as_integer(const Token& tok, Object& obj) { if (number_start(tok.text[0]) && !str_contains(tok.text, '.')) { uint64_t v = 0; try { std::size_t end = 0; v = std::stoll(tok.text, &end); if (end != tok.text.size()) return false; obj = Object::make_integer(v); return true; } catch (std::exception& e) { return false; } } return false; } bool Reader::try_token_as_boolean(const Token& tok, Object& obj) { if (tok.text.size() != 2) return false; if (tok.text[0] == '#') { if (tok.text[1] == 'f') { obj = SymbolObject::make_new(symbolTable, "#f"); return true; } else if (tok.text[1] == 't') { obj = SymbolObject::make_new(symbolTable, "#t"); return true; } } return false; } void Reader::display_error_info(TextStream& here, const std::string& err) { printf("Reader error:\n%s\nat %s", err.c_str(), db.get_info_for(here.text, here.seek).c_str()); throw std::runtime_error(err); } std::string Reader::get_next_dir() { auto result = std::getenv("NEXT_DIR"); if (!result) throw std::runtime_error( "Environment variable NEXT_DIR is not set. Please set this to point to next/"); return {result}; }