numerous refactoring

- Split parser core and compiler core. Fix #14
- AST int type to `u32`
- Updated asdl_rs.py and update_asdl.sh fix #6
- Use `ruff_python_ast::SourceLocation` for Python source location. Deleted our own Location.
- Renamed ast::Located to ast::Attributed to distinguish terms for TextSize and SourceLocation
- `ast::<Node>`s for TextSize located ast. `ast::located::<Node>` for Python source located ast.
- And also strictly renaming `located` to refer only python location related interfaces.
- `SourceLocator` to convert locations.
- New `source-code` features of to disable python locations when unnecessary.
- Also including fully merging https://github.com/astral-sh/RustPython/pull/4 closes #9
This commit is contained in:
Jeong YunWon
2023-05-10 02:36:52 +09:00
parent 09a6afdd04
commit a3d9d8cb14
29 changed files with 9737 additions and 12000 deletions

View File

@@ -15,10 +15,10 @@
use crate::{
ast,
lexer::{self, LexResult, LexicalError, LexicalErrorType},
mode::Mode,
python,
text_size::TextSize,
token::Tok,
Mode,
};
use itertools::Itertools;
use std::iter;
@@ -187,7 +187,7 @@ pub fn parse_tokens(
mode: Mode,
source_path: &str,
) -> Result<ast::Mod, ParseError> {
let marker_token = (mode.to_marker(), Default::default());
let marker_token = (Tok::start_marker(mode), Default::default());
let lexer = iter::once(Ok(marker_token))
.chain(lxr)
.filter_ok(|(tok, _)| !matches!(tok, Tok::Comment { .. } | Tok::NonLogicalNewline));
@@ -202,7 +202,7 @@ pub fn parse_tokens(
/// Represents represent errors that occur during parsing and are
/// returned by the `parse_*` functions.
pub type ParseError = rustpython_compiler_core::BaseError<ParseErrorType>;
pub type ParseError = rustpython_parser_core::BaseError<ParseErrorType>;
/// Represents the different types of errors that can occur during parsing.
#[derive(Debug, PartialEq)]