Python
/
ruff
mirror of https://github.com/astral-sh/ruff
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add `rome_formatter` fork as `ruff_formatter` (#2872) 

The Ruff autoformatter is going to be based on an intermediate representation (IR) formatted via [Wadler's algorithm](https://homepages.inf.ed.ac.uk/wadler/papers/prettier/prettier.pdf). This is architecturally similar to [Rome](https://github.com/rome/tools), Prettier, [Skip](https://github.com/skiplang/skip/blob/master/src/tools/printer/printer.sk), and others.

This PR adds a fork of the `rome_formatter` crate from [Rome](https://github.com/rome/tools), renamed here to `ruff_formatter`, which provides generic definitions for a formatter IR as well as a generic IR printer. (We've also pulled in `rome_rowan`, `rome_text_size`, and `rome_text_edit`, though some of these will be removed in future PRs.)

Why fork? `rome_formatter` contains code that's specific to Rome's AST representation (e.g., it relies on a fork of rust-analyzer's `rowan`), and we'll likely want to support different abstractions and formatting capabilities (there are already a few changes coming in future PRs). Once we've dropped `ruff_rowan` and trimmed down `ruff_formatter` to the code we currently need, it's also not a huge surface area to maintain and update.
This commit is contained in:
Charlie Marsh 2023-02-14 19:22:55 -05:00 committed by GitHub
parent ac028cd9f8
commit 3ef1c2e303
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
83 changed files with 27547 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

136
Cargo.lock generated
View File

@ -536,6 +536,12 @@ version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc"
[[package]]
name = "countme"
version = "3.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7704b5fdd17b18ae31c4c1da5a2e0305a2bf17b5249300a9ee9ed7b72114c636"
[[package]]
name = "cpufeatures"
version = "0.2.5"
@ -620,7 +626,7 @@ dependencies = [
"autocfg",
"cfg-if",
"crossbeam-utils",
"memoffset",
"memoffset 0.7.1",
"scopeguard",
]
@ -762,6 +768,12 @@ version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10"
[[package]]
name = "drop_bomb"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9bda8e21c04aca2ae33ffc2fd8c23134f3cac46db123ba97bd9d3f3b8a4a85e1"
[[package]]
name = "dyn-clone"
version = "1.0.10"
@ -798,6 +810,16 @@ dependencies = [
"cfg-if",
]
[[package]]
name = "env_logger"
version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a19187fea3ac7e84da7dacf48de0c45d63c6a76f9490dae389aead16c243fce3"
dependencies = [
"log",
"regex",
]
[[package]]
name = "env_proxy"
version = "0.4.1"
@ -1204,6 +1226,12 @@ dependencies = [
"want",
]
[[package]]
name = "iai"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "71a816c97c42258aa5834d07590b718b4c9a598944cd39a52dc25b351185d678"
[[package]]
name = "iana-time-zone"
version = "0.1.53"
@ -1646,6 +1674,15 @@ version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
[[package]]
name = "memoffset"
version = "0.6.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce"
dependencies = [
"autocfg",
]
[[package]]
name = "memoffset"
version = "0.7.1"
@ -2265,6 +2302,28 @@ dependencies = [
"memchr",
]
[[package]]
name = "quickcheck"
version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "588f6378e4dd99458b60ec275b4477add41ce4fa9f64dcba6f15adccb19b50d6"
dependencies = [
"env_logger",
"log",
"rand",
]
[[package]]
name = "quickcheck_macros"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b22a693222d716a9587786f37ac3f6b4faedb5b80c23914e7303ff5a1d8016e9"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "quote"
version = "1.0.23"
@ -2557,6 +2616,23 @@ dependencies = [
"textwrap",
]
[[package]]
name = "ruff_formatter"
version = "0.0.0"
dependencies = [
"cfg-if",
"countme",
"drop_bomb",
"indexmap",
"insta",
"ruff_rowan",
"rustc-hash",
"schemars",
"serde",
"tracing",
"unicode-width",
]
[[package]]
name = "ruff_macros"
version = "0.0.0"
@ -2577,6 +2653,45 @@ dependencies = [
"rustc-hash",
]
[[package]]
name = "ruff_rowan"
version = "0.0.0"
dependencies = [
"countme",
"hashbrown",
"iai",
"memoffset 0.6.5",
"quickcheck",
"quickcheck_macros",
"ruff_text_edit",
"ruff_text_size",
"rustc-hash",
"schemars",
"serde",
"serde_json",
"tracing",
]
[[package]]
name = "ruff_text_edit"
version = "0.0.0"
dependencies = [
"ruff_text_size",
"schemars",
"serde",
"similar",
]
[[package]]
name = "ruff_text_size"
version = "0.0.0"
dependencies = [
"schemars",
"serde",
"serde_test",
"static_assertions",
]
[[package]]
name = "rust-stemmers"
version = "1.2.0"
@ -2842,6 +2957,15 @@ dependencies = [
"serde",
]
[[package]]
name = "serde_test"
version = "1.0.152"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3611210d2d67e3513204742004d6ac6f589e521861dabb0f649b070eea8bed9e"
dependencies = [
"serde",
]
[[package]]
name = "serde_urlencoded"
version = "0.7.1"
@ -2900,6 +3024,10 @@ name = "similar"
version = "2.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "420acb44afdae038210c99e69aae24109f32f15500aa708e81d46c9f29d55fcf"
dependencies = [
"bstr 0.2.17",
"unicode-segmentation",
]
[[package]]
name = "siphasher"
@ -3518,6 +3646,12 @@ dependencies = [
"tinyvec",
]
[[package]]
name = "unicode-segmentation"
version = "1.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36"
[[package]]
name = "unicode-width"
version = "0.1.10"

106
LICENSE
View File

@ -1062,3 +1062,109 @@ are:
"""
- flake8-django, licensed under the GPL license.
- rust-analyzer/rowan, licensed under the MIT license:
"""
Permission is hereby granted, free of charge, to any
person obtaining a copy of this software and associated
documentation files (the "Software"), to deal in the
Software without restriction, including without
limitation the rights to use, copy, modify, merge,
publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software
is furnished to do so, subject to the following
conditions:
The above copyright notice and this permission notice
shall be included in all copies or substantial portions
of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF
ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED
TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR
IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
"""
- rust-analyzer/text-size, licensed under the MIT license:
"""
Permission is hereby granted, free of charge, to any
person obtaining a copy of this software and associated
documentation files (the "Software"), to deal in the
Software without restriction, including without
limitation the rights to use, copy, modify, merge,
publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software
is furnished to do so, subject to the following
conditions:
The above copyright notice and this permission notice
shall be included in all copies or substantial portions
of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF
ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED
TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR
IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
"""
- rust-analyzer/text-edit, licensed under the MIT license:
"""
Permission is hereby granted, free of charge, to any
person obtaining a copy of this software and associated
documentation files (the "Software"), to deal in the
Software without restriction, including without
limitation the rights to use, copy, modify, merge,
publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software
is furnished to do so, subject to the following
conditions:
The above copyright notice and this permission notice
shall be included in all copies or substantial portions
of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF
ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED
TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR
IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
"""
- rome/tools, licensed under the MIT license:
"""
MIT License
Copyright (c) Rome Tools, Inc. and its affiliates.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""

4
_typos.toml
View File

@ -1,2 +1,6 @@
[files]
extend-exclude = ["snapshots"]
[default.extend-words]
trivias = "trivias"
hel = "hel"

23
crates/ruff_formatter/Cargo.toml Normal file
View File

@ -0,0 +1,23 @@
[package]
name = "ruff_formatter"
version = "0.0.0"
publish = false
edition = "2021"
[dependencies]
cfg-if = { version = "1.0.0" }
countme = { version = "3.0.1" }
drop_bomb = { version = "0.1.5" }
indexmap = { version = "1.9.1" }
ruff_rowan = { path = "../ruff_rowan" }
rustc-hash = { workspace = true }
schemars = { version = "0.8.10", optional = true }
serde = { version = "1.0.136", features = ["derive"], optional = true }
tracing = { version = "0.1.31", default-features = false, features = ["std"] }
unicode-width = { version = "0.1.9" }
[dev-dependencies]
insta = { version = "1.19.0" }
[features]
serde = ["dep:serde", "schemars", "ruff_rowan/serde"]

160
crates/ruff_formatter/src/arguments.rs Normal file
View File

@ -0,0 +1,160 @@
use super::{Buffer, Format, Formatter};
use crate::FormatResult;
use std::ffi::c_void;
use std::marker::PhantomData;
/// Mono-morphed type to format an object. Used by the [crate::format!], [crate::format_args!], and
/// [crate::write!] macros.
///
/// This struct is similar to a dynamic dispatch (using `dyn Format`) because it stores a pointer to the value.
/// However, it doesn't store the pointer to `dyn Format`'s vtable, instead it statically resolves the function
/// pointer of `Format::format` and stores it in `formatter`.
pub struct Argument<'fmt, Context> {
/// The value to format stored as a raw pointer where `lifetime` stores the value's lifetime.
value: *const c_void,
/// Stores the lifetime of the value. To get the most out of our dear borrow checker.
lifetime: PhantomData<&'fmt ()>,
/// The function pointer to `value`'s `Format::format` method
formatter: fn(*const c_void, &mut Formatter<'_, Context>) -> FormatResult<()>,
}
impl<Context> Clone for Argument<'_, Context> {
fn clone(&self) -> Self {
*self
}
}
impl<Context> Copy for Argument<'_, Context> {}
impl<'fmt, Context> Argument<'fmt, Context> {
/// Called by the [ruff_formatter::format_args] macro. Creates a mono-morphed value for formatting
/// an object.
#[doc(hidden)]
#[inline]
pub fn new<F: Format<Context>>(value: &'fmt F) -> Self {
#[inline(always)]
fn formatter<F: Format<Context>, Context>(
ptr: *const c_void,
fmt: &mut Formatter<Context>,
) -> FormatResult<()> {
// SAFETY: Safe because the 'fmt lifetime is captured by the 'lifetime' field.
F::fmt(unsafe { &*(ptr as *const F) }, fmt)
}
Self {
value: value as *const F as *const c_void,
lifetime: PhantomData,
formatter: formatter::<F, Context>,
}
}
/// Formats the value stored by this argument using the given formatter.
#[inline(always)]
pub(super) fn format(&self, f: &mut Formatter<Context>) -> FormatResult<()> {
(self.formatter)(self.value, f)
}
}
/// Sequence of objects that should be formatted in the specified order.
///
/// The [`format_args!`] macro will safely create an instance of this structure.
///
/// You can use the `Arguments<a>` that [`format_args!]` return in `Format` context as seen below.
/// It will call the `format` function for every of it's objects.
///
/// ```rust
/// use ruff_formatter::prelude::*;
/// use ruff_formatter::{format, format_args};
///
/// # fn main() -> FormatResult<()> {
/// let formatted = format!(SimpleFormatContext::default(), [
/// format_args!(text("a"), space(), text("b"))
/// ])?;
///
/// assert_eq!("a b", formatted.print()?.as_code());
/// # Ok(())
/// # }
/// ```
pub struct Arguments<'fmt, Context>(pub &'fmt [Argument<'fmt, Context>]);
impl<'fmt, Context> Arguments<'fmt, Context> {
#[doc(hidden)]
#[inline(always)]
pub fn new(arguments: &'fmt [Argument<'fmt, Context>]) -> Self {
Self(arguments)
}
/// Returns the arguments
#[inline]
pub(super) fn items(&self) -> &'fmt [Argument<'fmt, Context>] {
self.0
}
}
impl<Context> Copy for Arguments<'_, Context> {}
impl<Context> Clone for Arguments<'_, Context> {
fn clone(&self) -> Self {
Self(self.0)
}
}
impl<Context> Format<Context> for Arguments<'_, Context> {
#[inline(always)]
fn fmt(&self, formatter: &mut Formatter<Context>) -> FormatResult<()> {
formatter.write_fmt(*self)
}
}
impl<Context> std::fmt::Debug for Arguments<'_, Context> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str("Arguments[...]")
}
}
impl<'fmt, Context> From<&'fmt Argument<'fmt, Context>> for Arguments<'fmt, Context> {
fn from(argument: &'fmt Argument<'fmt, Context>) -> Self {
Arguments::new(std::slice::from_ref(argument))
}
}
#[cfg(test)]
mod tests {
use crate::format_element::tag::Tag;
use crate::prelude::*;
use crate::{format_args, write, FormatState, VecBuffer};
#[test]
fn test_nesting() {
let mut context = FormatState::new(());
let mut buffer = VecBuffer::new(&mut context);
write!(
&mut buffer,
[
text("function"),
space(),
text("a"),
space(),
group(&format_args!(text("("), text(")")))
]
)
.unwrap();
assert_eq!(
buffer.into_vec(),
vec![
FormatElement::StaticText { text: "function" },
FormatElement::Space,
FormatElement::StaticText { text: "a" },
FormatElement::Space,
// Group
FormatElement::Tag(Tag::StartGroup(tag::Group::new())),
FormatElement::StaticText { text: "(" },
FormatElement::StaticText { text: ")" },
FormatElement::Tag(Tag::EndGroup)
]
);
}
}

723
crates/ruff_formatter/src/buffer.rs Normal file
View File

@ -0,0 +1,723 @@
use super::{write, Arguments, FormatElement};
use crate::format_element::Interned;
use crate::prelude::LineMode;
use crate::{Format, FormatResult, FormatState};
use rustc_hash::FxHashMap;
use std::any::{Any, TypeId};
use std::fmt::Debug;
use std::ops::{Deref, DerefMut};
/// A trait for writing or formatting into [FormatElement]-accepting buffers or streams.
pub trait Buffer {
/// The context used during formatting
type Context;
/// Writes a [crate::FormatElement] into this buffer, returning whether the write succeeded.
///
/// # Errors
/// This function will return an instance of [crate::FormatError] on error.
///
/// # Examples
///
/// ```
/// use ruff_formatter::{Buffer, FormatElement, FormatState, SimpleFormatContext, VecBuffer};
///
/// let mut state = FormatState::new(SimpleFormatContext::default());
/// let mut buffer = VecBuffer::new(&mut state);
///
/// buffer.write_element(FormatElement::StaticText { text: "test"}).unwrap();
///
/// assert_eq!(buffer.into_vec(), vec![FormatElement::StaticText { text: "test" }]);
/// ```
///
fn write_element(&mut self, element: FormatElement) -> FormatResult<()>;
/// Returns a slice containing all elements written into this buffer.
///
/// Prefer using [BufferExtensions::start_recording] over accessing [Buffer::elements] directly.
#[doc(hidden)]
fn elements(&self) -> &[FormatElement];
/// Glue for usage of the [`write!`] macro with implementors of this trait.
///
/// This method should generally not be invoked manually, but rather through the [`write!`] macro itself.
///
/// # Examples
///
/// ```
/// use ruff_formatter::prelude::*;
/// use ruff_formatter::{Buffer, FormatState, SimpleFormatContext, VecBuffer, format_args};
///
/// let mut state = FormatState::new(SimpleFormatContext::default());
/// let mut buffer = VecBuffer::new(&mut state);
///
/// buffer.write_fmt(format_args!(text("Hello World"))).unwrap();
///
/// assert_eq!(buffer.into_vec(), vec![FormatElement::StaticText{ text: "Hello World" }]);
/// ```
fn write_fmt(mut self: &mut Self, arguments: Arguments<Self::Context>) -> FormatResult<()> {
write(&mut self, arguments)
}
/// Returns the formatting state relevant for this formatting session.
fn state(&self) -> &FormatState<Self::Context>;
/// Returns the mutable formatting state relevant for this formatting session.
fn state_mut(&mut self) -> &mut FormatState<Self::Context>;
/// Takes a snapshot of the Buffers state, excluding the formatter state.
fn snapshot(&self) -> BufferSnapshot;
/// Restores the snapshot buffer
///
/// ## Panics
/// If the passed snapshot id is a snapshot of another buffer OR
/// if the snapshot is restored out of order
fn restore_snapshot(&mut self, snapshot: BufferSnapshot);
}
/// Snapshot of a buffer state that can be restored at a later point.
///
/// Used in cases where the formatting of an object fails but a parent formatter knows an alternative
/// strategy on how to format the object that might succeed.
#[derive(Debug)]
pub enum BufferSnapshot {
/// Stores an absolute position of a buffers state, for example, the offset of the last written element.
Position(usize),
/// Generic structure for custom buffers that need to store more complex data. Slightly more
/// expensive because it requires allocating the buffer state on the heap.
Any(Box<dyn Any>),
}
impl BufferSnapshot {
/// Creates a new buffer snapshot that points to the specified position.
pub const fn position(index: usize) -> Self {
Self::Position(index)
}
/// Unwraps the position value.
///
/// # Panics
///
/// If self is not a [`BufferSnapshot::Position`]
pub fn unwrap_position(&self) -> usize {
match self {
BufferSnapshot::Position(index) => *index,
BufferSnapshot::Any(_) => panic!("Tried to unwrap Any snapshot as a position."),
}
}
/// Unwraps the any value.
///
/// # Panics
///
/// If `self` is not a [`BufferSnapshot::Any`].
pub fn unwrap_any<T: 'static>(self) -> T {
match self {
BufferSnapshot::Position(_) => {
panic!("Tried to unwrap Position snapshot as Any snapshot.")
}
BufferSnapshot::Any(value) => match value.downcast::<T>() {
Ok(snapshot) => *snapshot,
Err(err) => {
panic!(
"Tried to unwrap snapshot of type {:?} as {:?}",
err.type_id(),
TypeId::of::<T>()
)
}
},
}
}
}
/// Implements the `[Buffer]` trait for all mutable references of objects implementing [Buffer].
impl<W: Buffer<Context = Context> + ?Sized, Context> Buffer for &mut W {
type Context = Context;
fn write_element(&mut self, element: FormatElement) -> FormatResult<()> {
(**self).write_element(element)
}
fn elements(&self) -> &[FormatElement] {
(**self).elements()
}
fn write_fmt(&mut self, args: Arguments<Context>) -> FormatResult<()> {
(**self).write_fmt(args)
}
fn state(&self) -> &FormatState<Self::Context> {
(**self).state()
}
fn state_mut(&mut self) -> &mut FormatState<Self::Context> {
(**self).state_mut()
}
fn snapshot(&self) -> BufferSnapshot {
(**self).snapshot()
}
fn restore_snapshot(&mut self, snapshot: BufferSnapshot) {
(**self).restore_snapshot(snapshot)
}
}
/// Vector backed [`Buffer`] implementation.
///
/// The buffer writes all elements into the internal elements buffer.
#[derive(Debug)]
pub struct VecBuffer<'a, Context> {
state: &'a mut FormatState<Context>,
elements: Vec<FormatElement>,
}
impl<'a, Context> VecBuffer<'a, Context> {
pub fn new(state: &'a mut FormatState<Context>) -> Self {
Self::new_with_vec(state, Vec::new())
}
pub fn new_with_vec(state: &'a mut FormatState<Context>, elements: Vec<FormatElement>) -> Self {
Self { state, elements }
}
/// Creates a buffer with the specified capacity
pub fn with_capacity(capacity: usize, state: &'a mut FormatState<Context>) -> Self {
Self {
state,
elements: Vec::with_capacity(capacity),
}
}
/// Consumes the buffer and returns the written [`FormatElement]`s as a vector.
pub fn into_vec(self) -> Vec<FormatElement> {
self.elements
}
/// Takes the elements without consuming self
pub fn take_vec(&mut self) -> Vec<FormatElement> {
std::mem::take(&mut self.elements)
}
}
impl<Context> Deref for VecBuffer<'_, Context> {
type Target = [FormatElement];
fn deref(&self) -> &Self::Target {
&self.elements
}
}
impl<Context> DerefMut for VecBuffer<'_, Context> {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.elements
}
}
impl<Context> Buffer for VecBuffer<'_, Context> {
type Context = Context;
fn write_element(&mut self, element: FormatElement) -> FormatResult<()> {
self.elements.push(element);
Ok(())
}
fn elements(&self) -> &[FormatElement] {
self
}
fn state(&self) -> &FormatState<Self::Context> {
self.state
}
fn state_mut(&mut self) -> &mut FormatState<Self::Context> {
self.state
}
fn snapshot(&self) -> BufferSnapshot {
BufferSnapshot::position(self.elements.len())
}
fn restore_snapshot(&mut self, snapshot: BufferSnapshot) {
let position = snapshot.unwrap_position();
assert!(
self.elements.len() >= position,
r#"Outdated snapshot. This buffer contains fewer elements than at the time the snapshot was taken.
Make sure that you take and restore the snapshot in order and that this snapshot belongs to the current buffer."#
);
self.elements.truncate(position);
}
}
/// This struct wraps an existing buffer and emits a preamble text when the first text is written.
///
/// This can be useful if you, for example, want to write some content if what gets written next isn't empty.
///
/// # Examples
///
/// ```
/// use ruff_formatter::{FormatState, Formatted, PreambleBuffer, SimpleFormatContext, VecBuffer, write};
/// use ruff_formatter::prelude::*;
///
/// struct Preamble;
///
/// impl Format<SimpleFormatContext> for Preamble {
/// fn fmt(&self, f: &mut Formatter<SimpleFormatContext>) -> FormatResult<()> {
/// write!(f, [text("# heading"), hard_line_break()])
/// }
/// }
///
/// # fn main() -> FormatResult<()> {
/// let mut state = FormatState::new(SimpleFormatContext::default());
/// let mut buffer = VecBuffer::new(&mut state);
///
/// {
/// let mut with_preamble = PreambleBuffer::new(&mut buffer, Preamble);
///
/// write!(&mut with_preamble, [text("this text will be on a new line")])?;
/// }
///
/// let formatted = Formatted::new(Document::from(buffer.into_vec()), SimpleFormatContext::default());
/// assert_eq!("# heading\nthis text will be on a new line", formatted.print()?.as_code());
///
/// # Ok(())
/// # }
/// ```
///
/// The pre-amble does not get written if no content is written to the buffer.
///
/// ```
/// use ruff_formatter::{FormatState, Formatted, PreambleBuffer, SimpleFormatContext, VecBuffer, write};
/// use ruff_formatter::prelude::*;
///
/// struct Preamble;
///
/// impl Format<SimpleFormatContext> for Preamble {
/// fn fmt(&self, f: &mut Formatter<SimpleFormatContext>) -> FormatResult<()> {
/// write!(f, [text("# heading"), hard_line_break()])
/// }
/// }
///
/// # fn main() -> FormatResult<()> {
/// let mut state = FormatState::new(SimpleFormatContext::default());
/// let mut buffer = VecBuffer::new(&mut state);
/// {
/// let mut with_preamble = PreambleBuffer::new(&mut buffer, Preamble);
/// }
///
/// let formatted = Formatted::new(Document::from(buffer.into_vec()), SimpleFormatContext::default());
/// assert_eq!("", formatted.print()?.as_code());
/// # Ok(())
/// # }
/// ```
pub struct PreambleBuffer<'buf, Preamble, Context> {
/// The wrapped buffer
inner: &'buf mut dyn Buffer<Context = Context>,
/// The pre-amble to write once the first content gets written to this buffer.
preamble: Preamble,
/// Whether some content (including the pre-amble) has been written at this point.
empty: bool,
}
impl<'buf, Preamble, Context> PreambleBuffer<'buf, Preamble, Context> {
pub fn new(inner: &'buf mut dyn Buffer<Context = Context>, preamble: Preamble) -> Self {
Self {
inner,
preamble,
empty: true,
}
}
/// Returns `true` if the preamble has been written, `false` otherwise.
pub fn did_write_preamble(&self) -> bool {
!self.empty
}
}
impl<Preamble, Context> Buffer for PreambleBuffer<'_, Preamble, Context>
where
Preamble: Format<Context>,
{
type Context = Context;
fn write_element(&mut self, element: FormatElement) -> FormatResult<()> {
if self.empty {
write!(self.inner, [&self.preamble])?;
self.empty = false;
}
self.inner.write_element(element)
}
fn elements(&self) -> &[FormatElement] {
self.inner.elements()
}
fn state(&self) -> &FormatState<Self::Context> {
self.inner.state()
}
fn state_mut(&mut self) -> &mut FormatState<Self::Context> {
self.inner.state_mut()
}
fn snapshot(&self) -> BufferSnapshot {
BufferSnapshot::Any(Box::new(PreambleBufferSnapshot {
inner: self.inner.snapshot(),
empty: self.empty,
}))
}
fn restore_snapshot(&mut self, snapshot: BufferSnapshot) {
let snapshot = snapshot.unwrap_any::<PreambleBufferSnapshot>();
self.empty = snapshot.empty;
self.inner.restore_snapshot(snapshot.inner);
}
}
struct PreambleBufferSnapshot {
inner: BufferSnapshot,
empty: bool,
}
/// Buffer that allows you inspecting elements as they get written to the formatter.
pub struct Inspect<'inner, Context, Inspector> {
inner: &'inner mut dyn Buffer<Context = Context>,
inspector: Inspector,
}
impl<'inner, Context, Inspector> Inspect<'inner, Context, Inspector> {
fn new(inner: &'inner mut dyn Buffer<Context = Context>, inspector: Inspector) -> Self {
Self { inner, inspector }
}
}
impl<'inner, Context, Inspector> Buffer for Inspect<'inner, Context, Inspector>
where
Inspector: FnMut(&FormatElement),
{
type Context = Context;
fn write_element(&mut self, element: FormatElement) -> FormatResult<()> {
(self.inspector)(&element);
self.inner.write_element(element)
}
fn elements(&self) -> &[FormatElement] {
self.inner.elements()
}
fn state(&self) -> &FormatState<Self::Context> {
self.inner.state()
}
fn state_mut(&mut self) -> &mut FormatState<Self::Context> {
self.inner.state_mut()
}
fn snapshot(&self) -> BufferSnapshot {
self.inner.snapshot()
}
fn restore_snapshot(&mut self, snapshot: BufferSnapshot) {
self.inner.restore_snapshot(snapshot)
}
}
/// A Buffer that removes any soft line breaks.
///
/// * Removes [`lines`](FormatElement::Line) with the mode [`Soft`](LineMode::Soft).
/// * Replaces [`lines`](FormatElement::Line) with the mode [`Soft`](LineMode::SoftOrSpace) with a [`Space`](FormatElement::Space)
///
/// # Examples
///
/// ```
/// use ruff_formatter::prelude::*;
/// use ruff_formatter::{format, write};
///
/// # fn main() -> FormatResult<()> {
/// use ruff_formatter::{RemoveSoftLinesBuffer, SimpleFormatContext, VecBuffer};
/// use ruff_formatter::prelude::format_with;
/// let formatted = format!(
/// SimpleFormatContext::default(),
/// [format_with(|f| {
/// let mut buffer = RemoveSoftLinesBuffer::new(f);
///
/// write!(
/// buffer,
/// [
/// text("The next soft line or space gets replaced by a space"),
/// soft_line_break_or_space(),
/// text("and the line here"),
/// soft_line_break(),
/// text("is removed entirely.")
/// ]
/// )
/// })]
/// )?;
///
/// assert_eq!(
/// formatted.document().as_ref(),
/// &[
/// FormatElement::StaticText { text: "The next soft line or space gets replaced by a space" },
/// FormatElement::Space,
/// FormatElement::StaticText { text: "and the line here" },
/// FormatElement::StaticText { text: "is removed entirely." }
/// ]
/// );
///
/// # Ok(())
/// # }
/// ```
pub struct RemoveSoftLinesBuffer<'a, Context> {
inner: &'a mut dyn Buffer<Context = Context>,
/// Caches the interned elements after the soft line breaks have been removed.
///
/// The `key` is the [Interned] element as it has been passed to [Self::write_element] or the child of another
/// [Interned] element. The `value` is the matching document of the key where all soft line breaks have been removed.
///
/// It's fine to not snapshot the cache. The worst that can happen is that it holds on interned elements
/// that are now unused. But there's little harm in that and the cache is cleaned when dropping the buffer.
interned_cache: FxHashMap<Interned, Interned>,
}
impl<'a, Context> RemoveSoftLinesBuffer<'a, Context> {
/// Creates a new buffer that removes the soft line breaks before writing them into `buffer`.
pub fn new(inner: &'a mut dyn Buffer<Context = Context>) -> Self {
Self {
inner,
interned_cache: FxHashMap::default(),
}
}
/// Removes the soft line breaks from an interned element.
fn clean_interned(&mut self, interned: &Interned) -> Interned {
clean_interned(interned, &mut self.interned_cache)
}
}
// Extracted to function to avoid monomorphization
fn clean_interned(
interned: &Interned,
interned_cache: &mut FxHashMap<Interned, Interned>,
) -> Interned {
match interned_cache.get(interned) {
Some(cleaned) => cleaned.clone(),
None => {
// Find the first soft line break element or interned element that must be changed
let result = interned
.iter()
.enumerate()
.find_map(|(index, element)| match element {
FormatElement::Line(LineMode::Soft | LineMode::SoftOrSpace) => {
let mut cleaned = Vec::new();
cleaned.extend_from_slice(&interned[..index]);
Some((cleaned, &interned[index..]))
}
FormatElement::Interned(inner) => {
let cleaned_inner = clean_interned(inner, interned_cache);
if &cleaned_inner != inner {
let mut cleaned = Vec::with_capacity(interned.len());
cleaned.extend_from_slice(&interned[..index]);
cleaned.push(FormatElement::Interned(cleaned_inner));
Some((cleaned, &interned[index + 1..]))
} else {
None
}
}
_ => None,
});
let result = match result {
// Copy the whole interned buffer so that becomes possible to change the necessary elements.
Some((mut cleaned, rest)) => {
for element in rest {
let element = match element {
FormatElement::Line(LineMode::Soft) => continue,
FormatElement::Line(LineMode::SoftOrSpace) => FormatElement::Space,
FormatElement::Interned(interned) => {
FormatElement::Interned(clean_interned(interned, interned_cache))
}
element => element.clone(),
};
cleaned.push(element)
}
Interned::new(cleaned)
}
// No change necessary, return existing interned element
None => interned.clone(),
};
interned_cache.insert(interned.clone(), result.clone());
result
}
}
}
impl<Context> Buffer for RemoveSoftLinesBuffer<'_, Context> {
type Context = Context;
fn write_element(&mut self, element: FormatElement) -> FormatResult<()> {
let element = match element {
FormatElement::Line(LineMode::Soft) => return Ok(()),
FormatElement::Line(LineMode::SoftOrSpace) => FormatElement::Space,
FormatElement::Interned(interned) => {
FormatElement::Interned(self.clean_interned(&interned))
}
element => element,
};
self.inner.write_element(element)
}
fn elements(&self) -> &[FormatElement] {
self.inner.elements()
}
fn state(&self) -> &FormatState<Self::Context> {
self.inner.state()
}
fn state_mut(&mut self) -> &mut FormatState<Self::Context> {
self.inner.state_mut()
}
fn snapshot(&self) -> BufferSnapshot {
self.inner.snapshot()
}
fn restore_snapshot(&mut self, snapshot: BufferSnapshot) {
self.inner.restore_snapshot(snapshot)
}
}
pub trait BufferExtensions: Buffer + Sized {
/// Returns a new buffer that calls the passed inspector for every element that gets written to the output
#[must_use]
fn inspect<F>(&mut self, inspector: F) -> Inspect<Self::Context, F>
where
F: FnMut(&FormatElement),
{
Inspect::new(self, inspector)
}
/// Starts a recording that gives you access to all elements that have been written between the start
/// and end of the recording
///
/// #Examples
///
/// ```
/// use std::ops::Deref;
/// use ruff_formatter::prelude::*;
/// use ruff_formatter::{write, format, SimpleFormatContext};
///
/// # fn main() -> FormatResult<()> {
/// let formatted = format!(SimpleFormatContext::default(), [format_with(|f| {
/// let mut recording = f.start_recording();
///
/// write!(recording, [text("A")])?;
/// write!(recording, [text("B")])?;
///
/// write!(recording, [format_with(|f| write!(f, [text("C"), text("D")]))])?;
///
/// let recorded = recording.stop();
/// assert_eq!(
/// recorded.deref(),
/// &[
/// FormatElement::StaticText{ text: "A" },
/// FormatElement::StaticText{ text: "B" },
/// FormatElement::StaticText{ text: "C" },
/// FormatElement::StaticText{ text: "D" }
/// ]
/// );
///
/// Ok(())
/// })])?;
///
/// assert_eq!(formatted.print()?.as_code(), "ABCD");
/// # Ok(())
/// # }
/// ```
#[must_use]
fn start_recording(&mut self) -> Recording<Self> {
Recording::new(self)
}
/// Writes a sequence of elements into this buffer.
fn write_elements<I>(&mut self, elements: I) -> FormatResult<()>
where
I: IntoIterator<Item = FormatElement>,
{
for element in elements.into_iter() {
self.write_element(element)?;
}
Ok(())
}
}
impl<T> BufferExtensions for T where T: Buffer {}
#[derive(Debug)]
pub struct Recording<'buf, Buffer> {
start: usize,
buffer: &'buf mut Buffer,
}
impl<'buf, B> Recording<'buf, B>
where
B: Buffer,
{
fn new(buffer: &'buf mut B) -> Self {
Self {
start: buffer.elements().len(),
buffer,
}
}
#[inline(always)]
pub fn write_fmt(&mut self, arguments: Arguments<B::Context>) -> FormatResult<()> {
self.buffer.write_fmt(arguments)
}
#[inline(always)]
pub fn write_element(&mut self, element: FormatElement) -> FormatResult<()> {
self.buffer.write_element(element)
}
pub fn stop(self) -> Recorded<'buf> {
let buffer: &'buf B = self.buffer;
let elements = buffer.elements();
let recorded = if self.start > elements.len() {
// May happen if buffer was rewinded.
&[]
} else {
&elements[self.start..]
};
Recorded(recorded)
}
}
#[derive(Debug, Copy, Clone)]
pub struct Recorded<'a>(&'a [FormatElement]);
impl Deref for Recorded<'_> {
type Target = [FormatElement];
fn deref(&self) -> &Self::Target {
self.0
}
}

2243
crates/ruff_formatter/src/builders.rs Normal file
View File

File diff suppressed because it is too large Load Diff

1175
crates/ruff_formatter/src/comments.rs Normal file
View File

File diff suppressed because it is too large Load Diff

632
crates/ruff_formatter/src/comments/builder.rs Normal file
View File

@ -0,0 +1,632 @@
use super::{
map::CommentsMap, CommentPlacement, CommentStyle, CommentTextPosition, DecoratedComment,
SourceComment, TransformSourceMap,
};
use crate::source_map::{DeletedRangeEntry, DeletedRanges};
use crate::{TextRange, TextSize};
use ruff_rowan::syntax::SyntaxElementKey;
use ruff_rowan::{
Direction, Language, SyntaxElement, SyntaxKind, SyntaxNode, SyntaxToken, WalkEvent,
};
use rustc_hash::FxHashSet;
/// Extracts all comments from a syntax tree.
pub(super) struct CommentsBuilderVisitor<'a, Style: CommentStyle> {
builder: CommentsBuilder<Style::Language>,
style: &'a Style,
parentheses: SourceParentheses<'a>,
// State
pending_comments: Vec<DecoratedComment<Style::Language>>,
preceding_node: Option<SyntaxNode<Style::Language>>,
following_node_index: Option<usize>,
parents: Vec<SyntaxNode<Style::Language>>,
last_token: Option<SyntaxToken<Style::Language>>,
}
impl<'a, Style> CommentsBuilderVisitor<'a, Style>
where
Style: CommentStyle,
{
pub(super) fn new(style: &'a Style, source_map: Option<&'a TransformSourceMap>) -> Self {
Self {
style,
builder: Default::default(),
parentheses: SourceParentheses::from_source_map(source_map),
pending_comments: Default::default(),
preceding_node: Default::default(),
following_node_index: Default::default(),
parents: Default::default(),
last_token: Default::default(),
}
}
pub(super) fn visit(
mut self,
root: &SyntaxNode<Style::Language>,
) -> (
CommentsMap<SyntaxElementKey, SourceComment<Style::Language>>,
FxHashSet<SyntaxElementKey>,
) {
for event in root.preorder_with_tokens(Direction::Next) {
match event {
WalkEvent::Enter(SyntaxElement::Node(node)) => {
self.visit_node(WalkEvent::Enter(node))
}
WalkEvent::Leave(SyntaxElement::Node(node)) => {
self.visit_node(WalkEvent::Leave(node))
}
WalkEvent::Enter(SyntaxElement::Token(token)) => self.visit_token(token),
WalkEvent::Leave(SyntaxElement::Token(_)) => {
// Handled as part of enter
}
}
}
assert!(
self.parents.is_empty(),
"Expected all enclosing nodes to have been processed but contains {:#?}",
self.parents
);
// Process any comments attached to the last token.
// Important for range formatting where it isn't guaranteed that the
// last token is an EOF token.
if let Some(last_token) = self.last_token.take() {
self.parents.push(root.clone());
let (comments_start, lines_before, position, trailing_end) =
self.visit_trailing_comments(last_token, None);
Self::update_comments(
&mut self.pending_comments[comments_start..],
position,
lines_before,
trailing_end,
);
}
self.flush_comments(None);
self.builder.finish()
}
fn visit_node(&mut self, event: WalkEvent<SyntaxNode<Style::Language>>) {
match event {
WalkEvent::Enter(node) => {
// Lists cannot have comments attached. They either belong to the entire parent or to
// the first child. So we ignore lists all together
if node.kind().is_list() {
return;
}
let is_root = matches!(self.following_node_index, Some(0));
// Associate comments with the most outer node
// Set following here because it is the "following node" of the next token's leading trivia.
if self.following_node_index.is_none() || is_root {
// Flush in case the node doesn't have any tokens.
self.flush_comments(Some(&node));
self.following_node_index = Some(self.parents.len());
}
self.parents.push(node);
}
WalkEvent::Leave(node) => {
if node.kind().is_list() {
return;
}
self.parents.pop().unwrap();
// We're passed this node, flush any pending comments for its children
self.following_node_index = None;
self.flush_comments(None);
// We're passed this node, so it must precede the sibling that comes next.
self.preceding_node = Some(node);
}
}
}
fn visit_token(&mut self, token: SyntaxToken<Style::Language>) {
// Process the trailing trivia of the last token
let (comments_start, mut lines_before, mut position, mut trailing_end) =
if let Some(last_token) = self.last_token.take() {
self.visit_trailing_comments(last_token, Some(&token))
} else {
(
self.pending_comments.len(),
0,
CommentTextPosition::SameLine,
None,
)
};
// Process the leading trivia of the current token. the trailing trivia is handled as part of the next token
for leading in token.leading_trivia().pieces() {
if leading.is_newline() {
lines_before += 1;
// All comments following from here are own line comments
position = CommentTextPosition::OwnLine;
if trailing_end.is_none() {
trailing_end = Some(self.pending_comments.len());
}
} else if leading.is_skipped() {
self.builder.mark_has_skipped(&token);
lines_before = 0;
break;
} else if let Some(comment) = leading.as_comments() {
let kind = Style::get_comment_kind(&comment);
self.queue_comment(DecoratedComment {
enclosing: self.enclosing_node().clone(),
preceding: self.preceding_node.clone(),
following: None,
following_token: Some(token.clone()),
lines_before,
lines_after: 0,
text_position: position,
kind,
comment,
});
lines_before = 0;
}
}
self.last_token = Some(token);
Self::update_comments(
&mut self.pending_comments[comments_start..],
position,
lines_before,
trailing_end,
);
// Set following node to `None` because it now becomes the enclosing node.
if let Some(following_node) = self.following_node() {
self.flush_comments(Some(&following_node.clone()));
self.following_node_index = None;
// The following node is only set after entering a node
// That means, following node is only set for the first token of a node.
// Unset preceding node if this is the first token because the preceding node belongs to the parent.
self.preceding_node = None;
}
}
fn enclosing_node(&self) -> &SyntaxNode<Style::Language> {
let element = match self.following_node_index {
None => self.parents.last(),
Some(index) if index == 0 => Some(&self.parents[0]),
Some(index) => Some(&self.parents[index - 1]),
};
element.expect("Expected enclosing nodes to at least contain the root node.")
}
fn following_node(&self) -> Option<&SyntaxNode<Style::Language>> {
self.following_node_index.map(|index| {
self.parents
.get(index)
.expect("Expected following node index to point to a valid parent node")
})
}
fn queue_comment(&mut self, comment: DecoratedComment<Style::Language>) {
self.pending_comments.push(comment);
}
fn update_comments(
comments: &mut [DecoratedComment<Style::Language>],
position: CommentTextPosition,
lines_before: u32,
trailing_end: Option<usize>,
) {
let trailing_end = trailing_end.unwrap_or(comments.len());
let mut comments = comments.iter_mut().enumerate().peekable();
// Update the lines after of all comments as well as the positioning of end of line comments.
while let Some((index, comment)) = comments.next() {
// Update the position of all trailing comments to be end of line as we've seen a line break since.
if index < trailing_end && position.is_own_line() {
comment.text_position = CommentTextPosition::EndOfLine;
}
comment.lines_after = comments
.peek()
.map_or(lines_before, |(_, next)| next.lines_before);
}
}
fn flush_comments(&mut self, following: Option<&SyntaxNode<Style::Language>>) {
for mut comment in self.pending_comments.drain(..) {
comment.following = following.cloned();
let placement = self.style.place_comment(comment);
self.builder.add_comment(placement);
}
}
fn visit_trailing_comments(
&mut self,
token: SyntaxToken<Style::Language>,
following_token: Option<&SyntaxToken<Style::Language>>,
) -> (usize, u32, CommentTextPosition, Option<usize>) {
let mut comments_start = 0;
// The index of the last trailing comment in `pending_comments`.
let mut trailing_end: Option<usize> = None;
// Number of lines before the next comment, token, or skipped token trivia
let mut lines_before = 0;
// Trailing comments are all `SameLine` comments EXCEPT if any is followed by a line break,
// a leading comment (that always have line breaks), or there's a line break before the token.
let mut position = CommentTextPosition::SameLine;
// Process the trailing trivia of the last token
for piece in token.trailing_trivia().pieces() {
if piece.is_newline() {
lines_before += 1;
// All comments following from here are own line comments
position = CommentTextPosition::OwnLine;
if trailing_end.is_none() {
trailing_end = Some(self.pending_comments.len());
}
} else if let Some(comment) = piece.as_comments() {
self.queue_comment(DecoratedComment {
enclosing: self.enclosing_node().clone(),
preceding: self.preceding_node.clone(),
following: None,
following_token: following_token.cloned(),
lines_before,
lines_after: 0, // Will be initialized after
text_position: position,
kind: Style::get_comment_kind(&comment),
comment,
});
lines_before = 0;
}
if let Some(parens_source_range) = self
.parentheses
.r_paren_source_range(piece.text_range().end())
{
self.flush_before_r_paren_comments(
parens_source_range,
&token,
position,
lines_before,
comments_start,
trailing_end,
);
lines_before = 0;
position = CommentTextPosition::SameLine;
comments_start = 0;
trailing_end = None;
}
}
(comments_start, lines_before, position, trailing_end)
}
/// Processes comments appearing right before a `)` of a parenthesized expressions.
#[cold]
fn flush_before_r_paren_comments(
&mut self,
parens_source_range: TextRange,
last_token: &SyntaxToken<Style::Language>,
position: CommentTextPosition,
lines_before: u32,
start: usize,
trailing_end: Option<usize>,
) {
let enclosing = self.enclosing_node().clone();
let comments = &mut self.pending_comments[start..];
let trailing_end = trailing_end.unwrap_or(comments.len());
let mut comments = comments.iter_mut().enumerate().peekable();
let parenthesized_node = self
.parentheses
.outer_most_parenthesized_node(last_token, parens_source_range);
let preceding = parenthesized_node;
// Using the `enclosing` as default but it's mainly to satisfy Rust. The only case where it is used
// is if someone formats a Parenthesized expression as the root. Something we explicitly disallow
// in ruff_js_formatter
let enclosing = preceding.parent().unwrap_or(enclosing);
// Update the lines after of all comments as well as the positioning of end of line comments.
while let Some((index, comment)) = comments.next() {
// Update the position of all trailing comments to be end of line as we've seen a line break since.
if index < trailing_end && position.is_own_line() {
comment.text_position = CommentTextPosition::EndOfLine;
}
comment.preceding = Some(preceding.clone());
comment.enclosing = enclosing.clone();
comment.lines_after = comments
.peek()
.map_or(lines_before, |(_, next)| next.lines_before);
}
self.flush_comments(None);
}
}
struct CommentsBuilder<L: Language> {
comments: CommentsMap<SyntaxElementKey, SourceComment<L>>,
skipped: FxHashSet<SyntaxElementKey>,
}
impl<L: Language> CommentsBuilder<L> {
fn add_comment(&mut self, placement: CommentPlacement<L>) {
match placement {
CommentPlacement::Leading { node, comment } => {
self.push_leading_comment(&node, comment);
}
CommentPlacement::Trailing { node, comment } => {
self.push_trailing_comment(&node, comment);
}
CommentPlacement::Dangling { node, comment } => {
self.push_dangling_comment(&node, comment)
}
CommentPlacement::Default(mut comment) => {
match comment.text_position {
CommentTextPosition::EndOfLine => {
match (comment.take_preceding_node(), comment.take_following_node()) {
(Some(preceding), Some(_)) => {
// Attach comments with both preceding and following node to the preceding
// because there's a line break separating it from the following node.
// ```javascript
// a; // comment
// b
// ```
self.push_trailing_comment(&preceding, comment);
}
(Some(preceding), None) => {
self.push_trailing_comment(&preceding, comment);
}
(None, Some(following)) => {
self.push_leading_comment(&following, comment);
}
(None, None) => {
self.push_dangling_comment(
&comment.enclosing_node().clone(),
comment,
);
}
}
}
CommentTextPosition::OwnLine => {
match (comment.take_preceding_node(), comment.take_following_node()) {
// Following always wins for a leading comment
// ```javascript
// a;
// // comment
// b
// ```
// attach the comment to the `b` expression statement
(_, Some(following)) => {
self.push_leading_comment(&following, comment);
}
(Some(preceding), None) => {
self.push_trailing_comment(&preceding, comment);
}
(None, None) => {
self.push_dangling_comment(
&comment.enclosing_node().clone(),
comment,
);
}
}
}
CommentTextPosition::SameLine => {
match (comment.take_preceding_node(), comment.take_following_node()) {
(Some(preceding), Some(following)) => {
// Only make it a trailing comment if it directly follows the preceding node but not if it is separated
// by one or more tokens
// ```javascript
// a /* comment */ b; // Comment is a trailing comment
// a, /* comment */ b; // Comment should be a leading comment
// ```
if preceding.text_range().end()
== comment.piece().as_piece().token().text_range().end()
{
self.push_trailing_comment(&preceding, comment);
} else {
self.push_leading_comment(&following, comment);
}
}
(Some(preceding), None) => {
self.push_trailing_comment(&preceding, comment);
}
(None, Some(following)) => {
self.push_leading_comment(&following, comment);
}
(None, None) => {
self.push_dangling_comment(
&comment.enclosing_node().clone(),
comment,
);
}
}
}
}
}
}
}
fn mark_has_skipped(&mut self, token: &SyntaxToken<L>) {
self.skipped.insert(token.key());
}
fn push_leading_comment(&mut self, node: &SyntaxNode<L>, comment: impl Into<SourceComment<L>>) {
self.comments.push_leading(node.key(), comment.into());
}
fn push_dangling_comment(
&mut self,
node: &SyntaxNode<L>,
comment: impl Into<SourceComment<L>>,
) {
self.comments.push_dangling(node.key(), comment.into());
}
fn push_trailing_comment(
&mut self,
node: &SyntaxNode<L>,
comment: impl Into<SourceComment<L>>,
) {
self.comments.push_trailing(node.key(), comment.into());
}
fn finish(
self,
) -> (
CommentsMap<SyntaxElementKey, SourceComment<L>>,
FxHashSet<SyntaxElementKey>,
) {
(self.comments, self.skipped)
}
}
impl<L: Language> Default for CommentsBuilder<L> {
fn default() -> Self {
Self {
comments: CommentsMap::new(),
skipped: FxHashSet::default(),
}
}
}
enum SourceParentheses<'a> {
Empty,
SourceMap {
map: &'a TransformSourceMap,
next: Option<DeletedRangeEntry<'a>>,
tail: DeletedRanges<'a>,
},
}
impl<'a> SourceParentheses<'a> {
fn from_source_map(source_map: Option<&'a TransformSourceMap>) -> Self {
match source_map {
None => Self::Empty,
Some(source_map) => {
let mut deleted = source_map.deleted_ranges();
SourceParentheses::SourceMap {
map: source_map,
next: deleted.next(),
tail: deleted,
}
}
}
}
/// Returns the range of `node` including its parentheses if any. Otherwise returns the range as is
fn parenthesized_range<L: Language>(&self, node: &SyntaxNode<L>) -> TextRange {
match self {
SourceParentheses::Empty => node.text_trimmed_range(),
SourceParentheses::SourceMap { map, .. } => map.trimmed_source_range(node),
}
}
/// Tests if the next offset is at a position where the original source document used to have an `)`.
///
/// Must be called with offsets in increasing order.
///
/// Returns the source range of the `)` if there's any `)` in the deleted range at this offset. Returns `None` otherwise
fn r_paren_source_range(&mut self, offset: TextSize) -> Option<TextRange> {
match self {
SourceParentheses::Empty => None,
SourceParentheses::SourceMap { next, tail, .. } => {
while let Some(range) = next {
#[allow(clippy::comparison_chain)]
if range.transformed == offset {
// A deleted range can contain multiple tokens. See if there's any `)` in the deleted
// range and compute its source range.
return range.text.find(')').map(|r_paren_position| {
let start = range.source + TextSize::from(r_paren_position as u32);
TextRange::at(start, TextSize::from(1))
});
} else if range.transformed > offset {
return None;
} else {
*next = tail.next();
}
}
None
}
}
}
/// Searches the outer most node that still is inside of the parentheses specified by the `parentheses_source_range`.
fn outer_most_parenthesized_node<L: Language>(
&self,
token: &SyntaxToken<L>,
parentheses_source_range: TextRange,
) -> SyntaxNode<L> {
match self {
SourceParentheses::Empty => token.parent().unwrap(),
SourceParentheses::SourceMap { map, .. } => {
debug_assert_eq!(&map.text()[parentheses_source_range], ")");
// How this works: We search the outer most node that, in the source document ends right after the `)`.
// The issue is, it is possible that multiple nodes end right after the `)`
//
// ```javascript
// !(
// a
// /* comment */
// )
// ```
// The issue is, that in the transformed document, the `ReferenceIdentifier`, `IdentifierExpression`, `UnaryExpression`, and `ExpressionStatement`
// all end at the end position of `)`.
// However, not all the nodes start at the same position. That's why this code also tracks the start.
// We first find the closest node that directly ends at the position of the right paren. We then continue
// upwards to find the most outer node that starts at the same position as that node. (In this case,
// `ReferenceIdentifier` -> `IdentifierExpression`.
let mut start_offset = None;
let r_paren_source_end = parentheses_source_range.end();
let ancestors = token.ancestors().take_while(|node| {
let source_range = self.parenthesized_range(node);
if let Some(start) = start_offset {
TextRange::new(start, r_paren_source_end).contains_range(source_range)
}
// Greater than to guarantee that we always return at least one node AND
// handle the case where a node is wrapped in multiple parentheses.
// Take the first node that fully encloses the parentheses
else if source_range.end() >= r_paren_source_end {
start_offset = Some(source_range.start());
true
} else {
source_range.end() < r_paren_source_end
}
});
// SAFETY:
// * The builder starts with a node which guarantees that every token has a parent node.
// * The above `take_while` guarantees to return `true` for the parent of the token.
// Thus, there's always at least one node
ancestors.last().unwrap()
}
}
}
}

836
crates/ruff_formatter/src/comments/map.rs Normal file
View File

@ -0,0 +1,836 @@
use countme::Count;
use rustc_hash::FxHashMap;
use std::fmt::{Debug, Formatter};
use std::iter::FusedIterator;
use std::num::NonZeroU32;
use std::ops::Range;
/// An optimized multi-map implementation for storing leading, dangling, and trailing parts for a key.
///
/// A naive implementation using three multimaps, one to store the leading, dangling, and trailing parts,
/// requires between `keys < allocations < keys * 3` vec allocations.
///
/// This map implementation optimises for the use case where:
/// * Parts belonging to the same key are inserted together. For example, all parts for the key `a` are inserted
/// before inserting any parts for the key `b`.
/// * The parts per key are inserted in the following order: leading, dangling, and then trailing parts.
///
/// Parts inserted in the above mentioned order are stored in a `Vec` shared by all keys to reduce the number
/// of allocations and increased cache locality. The implementation falls back to
/// storing the leading, dangling, and trailing parts of a key in dedicated `Vec`s if the parts
/// aren't inserted in the above described order. However, this comes with a slight performance penalty due to:
/// * Requiring up to three [Vec] allocations, one for the leading, dangling, and trailing parts.
/// * Requires copying already inserted parts for that key (by cloning) into the newly allocated [Vec]s.
/// * Resolving the slices for every part requires an extra level of indirection.
///
/// ## Limitations
///
/// The map supports storing up to `u32::MAX - 1` parts. Inserting the `u32::MAX`nth part panics.
///
/// ## Comments
///
/// Storing the leading, dangling, and trailing comments is an exemplary use case for this map implementation because
/// it is generally desired to keep the comments in the same order as in the source document. This translates to
/// inserting the comments per node and for every node in leading, dangling, trailing order (same order as this map optimises for).
///
/// Running Rome formatter on real world use cases showed that more than 99.99% of comments get inserted in
/// the described order.
///
/// The size limitation isn't a concern for comments because Rome supports source documents with a size up to 4GB (`u32::MAX`)
/// and every comment has at least a size of 2 bytes:
/// * 1 byte for the start sequence, e.g. `#`
/// * 1 byte for the end sequence, e.g. `\n`
///
/// Meaning, the upper bound for comments parts in a document are `u32::MAX / 2`.
pub(super) struct CommentsMap<K, V> {
/// Lookup table to retrieve the entry for a key.
index: FxHashMap<K, Entry>,
/// Flat array storing all the parts that have been inserted in order.
parts: Vec<V>,
/// Vector containing the leading, dangling, and trailing vectors for out of order entries.
///
/// The length of `out_of_order` is a multiple of 3 where:
/// * `index % 3 == 0`: Leading parts
/// * `index % 3 == 1`: Dangling parts
/// * `index % 3 == 2`: Trailing parts
out_of_order: Vec<Vec<V>>,
}
impl<K: std::hash::Hash + Eq, V> CommentsMap<K, V> {
pub fn new() -> Self {
Self {
index: FxHashMap::default(),
parts: Vec::new(),
out_of_order: Vec::new(),
}
}
/// Pushes a leading part for `key`.
pub fn push_leading(&mut self, key: K, part: V)
where
V: Clone,
{
match self.index.get_mut(&key) {
None => {
let start = self.parts.len();
self.parts.push(part);
self.index.insert(
key,
Entry::InOrder(InOrderEntry::leading(start..self.parts.len())),
);
}
// Has only leading comments and no elements have been pushed since
Some(Entry::InOrder(entry))
if entry.trailing_start.is_none() && self.parts.len() == entry.range().end =>
{
self.parts.push(part);
entry.increment_leading_range();
}
Some(Entry::OutOfOrder(entry)) => {
let leading = &mut self.out_of_order[entry.leading_index()];
leading.push(part);
}
Some(entry) => {
let out_of_order =
Self::entry_to_out_of_order(entry, &self.parts, &mut self.out_of_order);
self.out_of_order[out_of_order.leading_index()].push(part);
}
}
}
/// Pushes a dangling part for `key`
pub fn push_dangling(&mut self, key: K, part: V)
where
V: Clone,
{
match self.index.get_mut(&key) {
None => {
let start = self.parts.len();
self.parts.push(part);
self.index.insert(
key,
Entry::InOrder(InOrderEntry::dangling(start..self.parts.len())),
);
}
// Has leading and dangling comments and its comments are at the end of parts
Some(Entry::InOrder(entry))
if entry.trailing_end.is_none() && self.parts.len() == entry.range().end =>
{
self.parts.push(part);
entry.increment_dangling_range();
}
Some(Entry::OutOfOrder(entry)) => {
let dangling = &mut self.out_of_order[entry.dangling_index()];
dangling.push(part);
}
Some(entry) => {
let out_of_order =
Self::entry_to_out_of_order(entry, &self.parts, &mut self.out_of_order);
self.out_of_order[out_of_order.dangling_index()].push(part);
}
}
}
/// Pushes a trailing part for `key`.
pub fn push_trailing(&mut self, key: K, part: V)
where
V: Clone,
{
match self.index.get_mut(&key) {
None => {
let start = self.parts.len();
self.parts.push(part);
self.index.insert(
key,
Entry::InOrder(InOrderEntry::trailing(start..self.parts.len())),
);
}
// Its comments are at the end
Some(Entry::InOrder(entry)) if entry.range().end == self.parts.len() => {
self.parts.push(part);
entry.increment_trailing_range();
}
Some(Entry::OutOfOrder(entry)) => {
let trailing = &mut self.out_of_order[entry.trailing_index()];
trailing.push(part);
}
Some(entry) => {
let out_of_order =
Self::entry_to_out_of_order(entry, &self.parts, &mut self.out_of_order);
self.out_of_order[out_of_order.trailing_index()].push(part);
}
}
}
#[cold]
fn entry_to_out_of_order<'a>(
entry: &'a mut Entry,
parts: &[V],
out_of_order: &mut Vec<Vec<V>>,
) -> &'a mut OutOfOrderEntry
where
V: Clone,
{
match entry {
Entry::InOrder(in_order) => {
let index = out_of_order.len();
out_of_order.push(parts[in_order.leading_range()].to_vec());
out_of_order.push(parts[in_order.dangling_range()].to_vec());
out_of_order.push(parts[in_order.trailing_range()].to_vec());
*entry = Entry::OutOfOrder(OutOfOrderEntry {
leading_index: index,
_count: Count::new(),
});
match entry {
Entry::InOrder(_) => unreachable!(),
Entry::OutOfOrder(out_of_order) => out_of_order,
}
}
Entry::OutOfOrder(entry) => entry,
}
}
/// Retrieves all leading parts of `key`
pub fn leading(&self, key: &K) -> &[V] {
match self.index.get(key) {
None => &[],
Some(Entry::InOrder(in_order)) => &self.parts[in_order.leading_range()],
Some(Entry::OutOfOrder(entry)) => &self.out_of_order[entry.leading_index()],
}
}
/// Retrieves all dangling parts of `key`.
pub fn dangling(&self, key: &K) -> &[V] {
match self.index.get(key) {
None => &[],
Some(Entry::InOrder(in_order)) => &self.parts[in_order.dangling_range()],
Some(Entry::OutOfOrder(entry)) => &self.out_of_order[entry.dangling_index()],
}
}
/// Retrieves all trailing parts of `key`.
pub fn trailing(&self, key: &K) -> &[V] {
match self.index.get(key) {
None => &[],
Some(Entry::InOrder(in_order)) => &self.parts[in_order.trailing_range()],
Some(Entry::OutOfOrder(entry)) => &self.out_of_order[entry.trailing_index()],
}
}
/// Returns `true` if `key` has any leading, dangling, or trailing part.
pub fn has(&self, key: &K) -> bool {
self.index.get(key).is_some()
}
/// Returns an iterator over all leading, dangling, and trailing parts of `key`.
pub fn parts(&self, key: &K) -> PartsIterator<V> {
match self.index.get(key) {
None => PartsIterator::Slice([].iter()),
Some(entry) => PartsIterator::from_entry(entry, self),
}
}
/// Returns an iterator over the parts of all keys.
#[allow(unused)]
pub fn all_parts(&self) -> impl Iterator<Item = &V> {
self.index
.values()
.flat_map(|entry| PartsIterator::from_entry(entry, self))
}
}
impl<K: std::hash::Hash + Eq, V> Default for CommentsMap<K, V> {
fn default() -> Self {
Self::new()
}
}
impl<K, V> std::fmt::Debug for CommentsMap<K, V>
where
K: std::fmt::Debug,
V: std::fmt::Debug,
{
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let mut builder = f.debug_map();
for (key, entry) in &self.index {
builder.entry(&key, &DebugEntry { entry, map: self });
}
builder.finish()
}
}
/// Iterator to iterate over all leading, dangling, and trailing parts of a key.
pub(super) enum PartsIterator<'a, V> {
/// The slice into the [CommentsMap::parts] [Vec] if this is an in-order entry or the trailing parts
/// of an out-of-order entry.
Slice(std::slice::Iter<'a, V>),
/// Iterator over the leading parts of an out-of-order entry. Returns the dangling parts, and then the
/// trailing parts once the leading iterator is fully consumed.
Leading {
leading: std::slice::Iter<'a, V>,
dangling: &'a [V],
trailing: &'a [V],
},
/// Iterator over the dangling parts of an out-of-order entry. Returns the trailing parts
/// once the leading iterator is fully consumed.
Dangling {
dangling: std::slice::Iter<'a, V>,
trailing: &'a [V],
},
}
impl<'a, V> PartsIterator<'a, V> {
fn from_entry<K>(entry: &Entry, map: &'a CommentsMap<K, V>) -> Self {
match entry {
Entry::OutOfOrder(entry) => PartsIterator::Leading {
leading: map.out_of_order[entry.leading_index()].iter(),
dangling: &map.out_of_order[entry.dangling_index()],
trailing: &map.out_of_order[entry.trailing_index()],
},
Entry::InOrder(entry) => PartsIterator::Slice(map.parts[entry.range()].iter()),
}
}
}
impl<'a, V> Iterator for PartsIterator<'a, V> {
type Item = &'a V;
fn next(&mut self) -> Option<Self::Item> {
match self {
PartsIterator::Slice(inner) => inner.next(),
PartsIterator::Leading {
leading,
dangling,
trailing,
} => match leading.next() {
Some(next) => Some(next),
None if !dangling.is_empty() => {
let mut dangling_iterator = dangling.iter();
let next = dangling_iterator.next().unwrap();
*self = PartsIterator::Dangling {
dangling: dangling_iterator,
trailing,
};
Some(next)
}
None => {
let mut trailing_iterator = trailing.iter();
let next = trailing_iterator.next();
*self = PartsIterator::Slice(trailing_iterator);
next
}
},
PartsIterator::Dangling { dangling, trailing } => match dangling.next() {
Some(next) => Some(next),
None => {
let mut trailing_iterator = trailing.iter();
let next = trailing_iterator.next();
*self = PartsIterator::Slice(trailing_iterator);
next
}
},
}
}
fn size_hint(&self) -> (usize, Option<usize>) {
match self {
PartsIterator::Slice(slice) => slice.size_hint(),
PartsIterator::Leading {
leading,
dangling,
trailing,
} => {
let len = leading.len() + dangling.len() + trailing.len();
(len, Some(len))
}
PartsIterator::Dangling { dangling, trailing } => {
let len = dangling.len() + trailing.len();
(len, Some(len))
}
}
}
fn last(self) -> Option<Self::Item>
where
Self: Sized,
{
match self {
PartsIterator::Slice(slice) => slice.last(),
PartsIterator::Leading {
leading,
dangling,
trailing,
} => trailing
.last()
.or_else(|| dangling.last())
.or_else(|| leading.last()),
PartsIterator::Dangling { dangling, trailing } => {
trailing.last().or_else(|| dangling.last())
}
}
}
}
impl<V> ExactSizeIterator for PartsIterator<'_, V> {}
impl<V> FusedIterator for PartsIterator<'_, V> {}
#[derive(Debug)]
enum Entry {
InOrder(InOrderEntry),
OutOfOrder(OutOfOrderEntry),
}
struct DebugEntry<'a, K, V> {
entry: &'a Entry,
map: &'a CommentsMap<K, V>,
}
impl<K, V> Debug for DebugEntry<'_, K, V>
where
K: Debug,
V: Debug,
{
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
let leading = match self.entry {
Entry::OutOfOrder(entry) => self.map.out_of_order[entry.leading_index()].as_slice(),
Entry::InOrder(entry) => &self.map.parts[entry.leading_range()],
};
let dangling = match self.entry {
Entry::OutOfOrder(entry) => self.map.out_of_order[entry.dangling_index()].as_slice(),
Entry::InOrder(entry) => &self.map.parts[entry.dangling_range()],
};
let trailing = match self.entry {
Entry::OutOfOrder(entry) => self.map.out_of_order[entry.trailing_index()].as_slice(),
Entry::InOrder(entry) => &self.map.parts[entry.trailing_range()],
};
let mut list = f.debug_list();
list.entries(leading.iter().map(DebugValue::Leading));
list.entries(dangling.iter().map(DebugValue::Dangling));
list.entries(trailing.iter().map(DebugValue::Trailing));
list.finish()
}
}
enum DebugValue<'a, V> {
Leading(&'a V),
Dangling(&'a V),
Trailing(&'a V),
}
impl<V> Debug for DebugValue<'_, V>
where
V: Debug,
{
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
match self {
DebugValue::Leading(leading) => f.debug_tuple("Leading").field(leading).finish(),
DebugValue::Dangling(dangling) => f.debug_tuple("Dangling").field(dangling).finish(),
DebugValue::Trailing(trailing) => f.debug_tuple("Trailing").field(trailing).finish(),
}
}
}
#[derive(Debug)]
struct InOrderEntry {
/// Index into the [CommentsMap::parts] vector where the leading parts of this entry start
leading_start: PartIndex,
/// Index into the [CommentsMap::parts] vector where the dangling parts (and, thus, the leading parts end) start.
dangling_start: PartIndex,
/// Index into the [CommentsMap::parts] vector where the trailing parts (and, thus, the dangling parts end) of this entry start
trailing_start: Option<PartIndex>,
/// Index into the [CommentsMap::parts] vector where the trailing parts of this entry end
trailing_end: Option<PartIndex>,
_count: Count<InOrderEntry>,
}
impl InOrderEntry {
fn leading(range: Range<usize>) -> Self {
InOrderEntry {
leading_start: PartIndex::from_len(range.start),
dangling_start: PartIndex::from_len(range.end),
trailing_start: None,
trailing_end: None,
_count: Count::new(),
}
}
fn dangling(range: Range<usize>) -> Self {
let start = PartIndex::from_len(range.start);
InOrderEntry {
leading_start: start,
dangling_start: start,
trailing_start: Some(PartIndex::from_len(range.end)),
trailing_end: None,
_count: Count::new(),
}
}
fn trailing(range: Range<usize>) -> Self {
let start = PartIndex::from_len(range.start);
InOrderEntry {
leading_start: start,
dangling_start: start,
trailing_start: Some(start),
trailing_end: Some(PartIndex::from_len(range.end)),
_count: Count::new(),
}
}
fn increment_leading_range(&mut self) {
assert!(
self.trailing_start.is_none(),
"Can't extend the leading range for an in order entry with dangling comments."
);
self.dangling_start.increment();
}
fn increment_dangling_range(&mut self) {
assert!(
self.trailing_end.is_none(),
"Can't extend the dangling range for an in order entry with trailing comments."
);
match &mut self.trailing_start {
Some(start) => start.increment(),
None => self.trailing_start = Some(self.dangling_start.incremented()),
}
}
fn increment_trailing_range(&mut self) {
match (self.trailing_start, &mut self.trailing_end) {
// Already has some trailing comments
(Some(_), Some(end)) => end.increment(),
// Has dangling comments only
(Some(start), None) => self.trailing_end = Some(start.incremented()),
// Has leading comments only
(None, None) => {
self.trailing_start = Some(self.dangling_start);
self.trailing_end = Some(self.dangling_start.incremented())
}
(None, Some(_)) => {
unreachable!()
}
}
}
fn leading_range(&self) -> Range<usize> {
self.leading_start.value()..self.dangling_start.value()
}
fn dangling_range(&self) -> Range<usize> {
match self.trailing_start {
None => self.dangling_start.value()..self.dangling_start.value(),
Some(trailing_start) => self.dangling_start.value()..trailing_start.value(),
}
}
fn trailing_range(&self) -> Range<usize> {
match (self.trailing_start, self.trailing_end) {
(Some(trailing_start), Some(trailing_end)) => {
trailing_start.value()..trailing_end.value()
}
// Only dangling comments
(Some(trailing_start), None) => trailing_start.value()..trailing_start.value(),
(None, Some(_)) => {
panic!("Trailing end shouldn't be set if trailing start is none");
}
(None, None) => self.dangling_start.value()..self.dangling_start.value(),
}
}
fn range(&self) -> Range<usize> {
self.leading_start.value()
..self
.trailing_end
.or(self.trailing_start)
.unwrap_or(self.dangling_start)
.value()
}
}
#[derive(Debug)]
struct OutOfOrderEntry {
/// Index into the [CommentsMap::out_of_order] vector at which offset the leaading vec is stored.
leading_index: usize,
_count: Count<OutOfOrderEntry>,
}
impl OutOfOrderEntry {
const fn leading_index(&self) -> usize {
self.leading_index
}
const fn dangling_index(&self) -> usize {
self.leading_index + 1
}
const fn trailing_index(&self) -> usize {
self.leading_index + 2
}
}
/// Index into the [CommentsMap::parts] vector.
///
/// Stores the index as a [NonZeroU32], starting at 1 instead of 0 so that
/// `size_of::<PartIndex>() == size_of::<Option<PartIndex>>()`.
///
/// This means, that only `u32 - 1` parts can be stored. This should be sufficient for storing comments
/// because: Comments have length of two or more bytes because they consist of a start and end character sequence (`#` + new line, `/*` and `*/`).
/// Thus, a document with length `u32` can have at most `u32::MAX / 2` comment-parts.
#[repr(transparent)]
#[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)]
struct PartIndex(NonZeroU32);
impl PartIndex {
fn from_len(value: usize) -> Self {
Self(NonZeroU32::try_from(value as u32 + 1).unwrap())
}
fn value(&self) -> usize {
(u32::from(self.0) - 1) as usize
}
fn increment(&mut self) {
*self = self.incremented();
}
fn incremented(&self) -> PartIndex {
PartIndex(NonZeroU32::new(self.0.get() + 1).unwrap())
}
}
#[cfg(test)]
mod tests {
use crate::comments::map::CommentsMap;
static EMPTY: [i32; 0] = [];
#[test]
fn leading_dangling_trailing() {
let mut map = CommentsMap::new();
map.push_leading("a", 1);
map.push_dangling("a", 2);
map.push_dangling("a", 3);
map.push_trailing("a", 4);
assert_eq!(map.parts, vec![1, 2, 3, 4]);
assert_eq!(map.leading(&"a"), &[1]);
assert_eq!(map.dangling(&"a"), &[2, 3]);
assert_eq!(map.trailing(&"a"), &[4]);
assert!(map.has(&"a"));
assert_eq!(
map.parts(&"a").copied().collect::<Vec<_>>(),
vec![1, 2, 3, 4]
);
}
#[test]
fn dangling_trailing() {
let mut map = CommentsMap::new();
map.push_dangling("a", 1);
map.push_dangling("a", 2);
map.push_trailing("a", 3);
assert_eq!(map.parts, vec![1, 2, 3]);
assert_eq!(map.leading(&"a"), &EMPTY);
assert_eq!(map.dangling(&"a"), &[1, 2]);
assert_eq!(map.trailing(&"a"), &[3]);
assert!(map.has(&"a"));
assert_eq!(map.parts(&"a").copied().collect::<Vec<_>>(), vec![1, 2, 3]);
}
#[test]
fn trailing() {
let mut map = CommentsMap::new();
map.push_trailing("a", 1);
map.push_trailing("a", 2);
assert_eq!(map.parts, vec![1, 2]);
assert_eq!(map.leading(&"a"), &EMPTY);
assert_eq!(map.dangling(&"a"), &EMPTY);
assert_eq!(map.trailing(&"a"), &[1, 2]);
assert!(map.has(&"a"));
assert_eq!(map.parts(&"a").copied().collect::<Vec<_>>(), vec![1, 2]);
}
#[test]
fn empty() {
let map = CommentsMap::<&str, i32>::default();
assert_eq!(map.parts, Vec::<i32>::new());
assert_eq!(map.leading(&"a"), &EMPTY);
assert_eq!(map.dangling(&"a"), &EMPTY);
assert_eq!(map.trailing(&"a"), &EMPTY);
assert!(!map.has(&"a"));
assert_eq!(
map.parts(&"a").copied().collect::<Vec<_>>(),
Vec::<i32>::new()
);
}
#[test]
fn multiple_keys() {
let mut map = CommentsMap::new();
map.push_leading("a", 1);
map.push_dangling("b", 2);
map.push_trailing("c", 3);
map.push_leading("d", 4);
map.push_dangling("d", 5);
map.push_trailing("d", 6);
assert_eq!(map.parts, &[1, 2, 3, 4, 5, 6]);
assert_eq!(map.leading(&"a"), &[1]);
assert_eq!(map.dangling(&"a"), &EMPTY);
assert_eq!(map.trailing(&"a"), &EMPTY);
assert_eq!(map.parts(&"a").copied().collect::<Vec<_>>(), vec![1]);
assert_eq!(map.leading(&"b"), &EMPTY);
assert_eq!(map.dangling(&"b"), &[2]);
assert_eq!(map.trailing(&"b"), &EMPTY);
assert_eq!(map.parts(&"b").copied().collect::<Vec<_>>(), vec![2]);
assert_eq!(map.leading(&"c"), &EMPTY);
assert_eq!(map.dangling(&"c"), &EMPTY);
assert_eq!(map.trailing(&"c"), &[3]);
assert_eq!(map.parts(&"c").copied().collect::<Vec<_>>(), vec![3]);
assert_eq!(map.leading(&"d"), &[4]);
assert_eq!(map.dangling(&"d"), &[5]);
assert_eq!(map.trailing(&"d"), &[6]);
assert_eq!(map.parts(&"d").copied().collect::<Vec<_>>(), vec![4, 5, 6]);
}
#[test]
fn dangling_leading() {
let mut map = CommentsMap::new();
map.push_dangling("a", 1);
map.push_leading("a", 2);
map.push_dangling("a", 3);
map.push_trailing("a", 4);
assert_eq!(map.leading(&"a"), [2]);
assert_eq!(map.dangling(&"a"), [1, 3]);
assert_eq!(map.trailing(&"a"), [4]);
assert_eq!(
map.parts(&"a").copied().collect::<Vec<_>>(),
vec![2, 1, 3, 4]
);
assert!(map.has(&"a"));
}
#[test]
fn trailing_leading() {
let mut map = CommentsMap::new();
map.push_trailing("a", 1);
map.push_leading("a", 2);
map.push_dangling("a", 3);
map.push_trailing("a", 4);
assert_eq!(map.leading(&"a"), [2]);
assert_eq!(map.dangling(&"a"), [3]);
assert_eq!(map.trailing(&"a"), [1, 4]);
assert_eq!(
map.parts(&"a").copied().collect::<Vec<_>>(),
vec![2, 3, 1, 4]
);
assert!(map.has(&"a"));
}
#[test]
fn trailing_dangling() {
let mut map = CommentsMap::new();
map.push_trailing("a", 1);
map.push_dangling("a", 2);
map.push_trailing("a", 3);
assert_eq!(map.leading(&"a"), &EMPTY);
assert_eq!(map.dangling(&"a"), &[2]);
assert_eq!(map.trailing(&"a"), &[1, 3]);
assert_eq!(map.parts(&"a").copied().collect::<Vec<_>>(), vec![2, 1, 3]);
assert!(map.has(&"a"));
}
#[test]
fn keys_out_of_order() {
let mut map = CommentsMap::new();
map.push_leading("a", 1);
map.push_dangling("b", 2);
map.push_leading("a", 3);
map.push_trailing("c", 4);
map.push_dangling("b", 5);
map.push_leading("d", 6);
map.push_trailing("c", 7);
assert_eq!(map.leading(&"a"), &[1, 3]);
assert_eq!(map.dangling(&"b"), &[2, 5]);
assert_eq!(map.trailing(&"c"), &[4, 7]);
assert!(map.has(&"a"));
assert!(map.has(&"b"));
assert!(map.has(&"c"));
}
}

176
crates/ruff_formatter/src/diagnostics.rs Normal file
View File

@ -0,0 +1,176 @@
use crate::prelude::TagKind;
use ruff_rowan::{SyntaxError, TextRange};
use std::error::Error;
#[derive(Debug, PartialEq, Eq, Copy, Clone)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
/// Series of errors encountered during formatting
pub enum FormatError {
/// In case a node can't be formatted because it either misses a require child element or
/// a child is present that should not (e.g. a trailing comma after a rest element).
SyntaxError,
/// In case range formatting failed because the provided range was larger
/// than the formatted syntax tree
RangeError { input: TextRange, tree: TextRange },
/// In case printing the document failed because it has an invalid structure.
InvalidDocument(InvalidDocumentError),
/// Formatting failed because some content encountered a situation where a layout
/// choice by an enclosing [crate::Format] resulted in a poor layout for a child [crate::Format].
///
/// It's up to an enclosing [crate::Format] to handle the error and pick another layout.
/// This error should not be raised if there's no outer [crate::Format] handling the poor layout error,
/// avoiding that formatting of the whole document fails.
PoorLayout,
}
impl std::fmt::Display for FormatError {
fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
FormatError::SyntaxError => fmt.write_str("syntax error"),
FormatError::RangeError { input, tree } => std::write!(
fmt,
"formatting range {input:?} is larger than syntax tree {tree:?}"
),
FormatError::InvalidDocument(error) => std::write!(fmt, "Invalid document: {error}\n\n This is an internal Rome error. Please report if necessary."),
FormatError::PoorLayout => {
std::write!(fmt, "Poor layout: The formatter wasn't able to pick a good layout for your document. This is an internal Rome error. Please report if necessary.")
}
}
}
}
impl Error for FormatError {}
impl From<SyntaxError> for FormatError {
fn from(error: SyntaxError) -> Self {
FormatError::from(&error)
}
}
impl From<&SyntaxError> for FormatError {
fn from(syntax_error: &SyntaxError) -> Self {
match syntax_error {
SyntaxError::MissingRequiredChild => FormatError::SyntaxError,
}
}
}
impl From<PrintError> for FormatError {
fn from(error: PrintError) -> Self {
FormatError::from(&error)
}
}
impl From<&PrintError> for FormatError {
fn from(error: &PrintError) -> Self {
match error {
PrintError::InvalidDocument(reason) => FormatError::InvalidDocument(*reason),
}
}
}
#[derive(Debug, Copy, Clone, Eq, PartialEq)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
pub enum InvalidDocumentError {
/// Mismatching start/end kinds
///
/// ```plain
/// StartIndent
/// ...
/// EndGroup
/// ```
StartEndTagMismatch {
start_kind: TagKind,
end_kind: TagKind,
},
/// End tag without a corresponding start tag.
///
/// ```plain
/// Text
/// EndGroup
/// ```
StartTagMissing { kind: TagKind },
/// Expected a specific start tag but instead is:
/// * at the end of the document
/// * at another start tag
/// * at an end tag
ExpectedStart {
expected_start: TagKind,
actual: ActualStart,
},
}
#[derive(Debug, Copy, Clone, Eq, PartialEq)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
pub enum ActualStart {
/// The actual element is not a tag.
Content,
/// The actual element was a start tag of another kind.
Start(TagKind),
/// The actual element is an end tag instead of a start tag.
End(TagKind),
/// Reached the end of the document
EndOfDocument,
}
impl std::fmt::Display for InvalidDocumentError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
InvalidDocumentError::StartEndTagMismatch {
start_kind,
end_kind,
} => {
std::write!(
f,
"Expected end tag of kind {start_kind:?} but found {end_kind:?}."
)
}
InvalidDocumentError::StartTagMissing { kind } => {
std::write!(f, "End tag of kind {kind:?} without matching start tag.")
}
InvalidDocumentError::ExpectedStart {
expected_start,
actual,
} => {
match actual {
ActualStart::EndOfDocument => {
std::write!(f, "Expected start tag of kind {expected_start:?} but at the end of document.")
}
ActualStart::Start(start) => {
std::write!(f, "Expected start tag of kind {expected_start:?} but found start tag of kind {start:?}.")
}
ActualStart::End(end) => {
std::write!(f, "Expected start tag of kind {expected_start:?} but found end tag of kind {end:?}.")
}
ActualStart::Content => {
std::write!(f, "Expected start tag of kind {expected_start:?} but found non-tag element.")
}
}
}
}
}
}
#[derive(Debug, Clone, Eq, PartialEq)]
pub enum PrintError {
InvalidDocument(InvalidDocumentError),
}
impl Error for PrintError {}
impl std::fmt::Display for PrintError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
PrintError::InvalidDocument(inner) => {
std::write!(f, "Invalid document: {inner}")
}
}
}
}

395
crates/ruff_formatter/src/format_element.rs Normal file
View File

@ -0,0 +1,395 @@
pub mod document;
pub mod tag;
use crate::format_element::tag::{LabelId, Tag};
use std::borrow::Cow;
use crate::{TagKind, TextSize};
#[cfg(target_pointer_width = "64")]
use ruff_rowan::static_assert;
use ruff_rowan::SyntaxTokenText;
use std::hash::{Hash, Hasher};
use std::ops::Deref;
use std::rc::Rc;
/// Language agnostic IR for formatting source code.
///
/// Use the helper functions like [crate::builders::space], [crate::builders::soft_line_break] etc. defined in this file to create elements.
#[derive(Clone, Eq, PartialEq)]
pub enum FormatElement {
/// A space token, see [crate::builders::space] for documentation.
Space,
/// A new line, see [crate::builders::soft_line_break], [crate::builders::hard_line_break], and [crate::builders::soft_line_break_or_space] for documentation.
Line(LineMode),
/// Forces the parent group to print in expanded mode.
ExpandParent,
/// Token constructed by the formatter from a static string
StaticText { text: &'static str },
/// Token constructed from the input source as a dynamic
/// string with its start position in the input document.
DynamicText {
/// There's no need for the text to be mutable, using `Box<str>` safes 8 bytes over `String`.
text: Box<str>,
/// The start position of the dynamic token in the unformatted source code
source_position: TextSize,
},
/// A token for a text that is taken as is from the source code (input text and formatted representation are identical).
/// Implementing by taking a slice from a `SyntaxToken` to avoid allocating a new string.
SyntaxTokenTextSlice {
/// The start position of the token in the unformatted source code
source_position: TextSize,
/// The token text
slice: SyntaxTokenText,
},
/// Prevents that line suffixes move past this boundary. Forces the printer to print any pending
/// line suffixes, potentially by inserting a hard line break.
LineSuffixBoundary,
/// An interned format element. Useful when the same content must be emitted multiple times to avoid
/// deep cloning the IR when using the `best_fitting!` macro or `if_group_fits_on_line` and `if_group_breaks`.
Interned(Interned),
/// A list of different variants representing the same content. The printer picks the best fitting content.
/// Line breaks inside of a best fitting don't propagate to parent groups.
BestFitting(BestFitting),
/// A [Tag] that marks the start/end of some content to which some special formatting is applied.
Tag(Tag),
}
impl std::fmt::Debug for FormatElement {
fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
match self {
FormatElement::Space => write!(fmt, "Space"),
FormatElement::Line(mode) => fmt.debug_tuple("Line").field(mode).finish(),
FormatElement::ExpandParent => write!(fmt, "ExpandParent"),
FormatElement::StaticText { text } => {
fmt.debug_tuple("StaticText").field(text).finish()
}
FormatElement::DynamicText { text, .. } => {
fmt.debug_tuple("DynamicText").field(text).finish()
}
FormatElement::SyntaxTokenTextSlice { slice, .. } => fmt
.debug_tuple("SyntaxTokenTextSlice")
.field(slice)
.finish(),
FormatElement::LineSuffixBoundary => write!(fmt, "LineSuffixBoundary"),
FormatElement::BestFitting(best_fitting) => {
fmt.debug_tuple("BestFitting").field(&best_fitting).finish()
}
FormatElement::Interned(interned) => {
fmt.debug_list().entries(interned.deref()).finish()
}
FormatElement::Tag(tag) => fmt.debug_tuple("Tag").field(tag).finish(),
}
}
}
#[derive(Debug, Clone, Copy, Eq, PartialEq)]
pub enum LineMode {
/// See [crate::builders::soft_line_break_or_space] for documentation.
SoftOrSpace,
/// See [crate::builders::soft_line_break] for documentation.
Soft,
/// See [crate::builders::hard_line_break] for documentation.
Hard,
/// See [crate::builders::empty_line] for documentation.
Empty,
}
impl LineMode {
pub const fn is_hard(&self) -> bool {
matches!(self, LineMode::Hard)
}
}
#[derive(Debug, Clone, Copy, Eq, PartialEq)]
pub enum PrintMode {
/// Omits any soft line breaks
Flat,
/// Prints soft line breaks as line breaks
Expanded,
}
impl PrintMode {
pub const fn is_flat(&self) -> bool {
matches!(self, PrintMode::Flat)
}
pub const fn is_expanded(&self) -> bool {
matches!(self, PrintMode::Expanded)
}
}
#[derive(Clone)]
pub struct Interned(Rc<[FormatElement]>);
impl Interned {
pub(super) fn new(content: Vec<FormatElement>) -> Self {
Self(content.into())
}
}
impl PartialEq for Interned {
fn eq(&self, other: &Interned) -> bool {
Rc::ptr_eq(&self.0, &other.0)
}
}
impl Eq for Interned {}
impl Hash for Interned {
fn hash<H>(&self, hasher: &mut H)
where
H: Hasher,
{
Rc::as_ptr(&self.0).hash(hasher);
}
}
impl std::fmt::Debug for Interned {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
self.0.fmt(f)
}
}
impl Deref for Interned {
type Target = [FormatElement];
fn deref(&self) -> &Self::Target {
self.0.deref()
}
}
const LINE_SEPARATOR: char = '\u{2028}';
const PARAGRAPH_SEPARATOR: char = '\u{2029}';
pub const LINE_TERMINATORS: [char; 3] = ['\r', LINE_SEPARATOR, PARAGRAPH_SEPARATOR];
/// Replace the line terminators matching the provided list with "\n"
/// since its the only line break type supported by the printer
pub fn normalize_newlines<const N: usize>(text: &str, terminators: [char; N]) -> Cow<str> {
let mut result = String::new();
let mut last_end = 0;
for (start, part) in text.match_indices(terminators) {
result.push_str(&text[last_end..start]);
result.push('\n');
last_end = start + part.len();
// If the current character is \r and the
// next is \n, skip over the entire sequence
if part == "\r" && text[last_end..].starts_with('\n') {
last_end += 1;
}
}
// If the result is empty no line terminators were matched,
// return the entire input text without allocating a new String
if result.is_empty() {
Cow::Borrowed(text)
} else {
result.push_str(&text[last_end..text.len()]);
Cow::Owned(result)
}
}
impl FormatElement {
/// Returns `true` if self is a [FormatElement::Tag]
pub const fn is_tag(&self) -> bool {
matches!(self, FormatElement::Tag(_))
}
/// Returns `true` if self is a [FormatElement::Tag] and [Tag::is_start] is `true`.
pub const fn is_start_tag(&self) -> bool {
match self {
FormatElement::Tag(tag) => tag.is_start(),
_ => false,
}
}
/// Returns `true` if self is a [FormatElement::Tag] and [Tag::is_end] is `true`.
pub const fn is_end_tag(&self) -> bool {
match self {
FormatElement::Tag(tag) => tag.is_end(),
_ => false,
}
}
pub const fn is_text(&self) -> bool {
matches!(
self,
FormatElement::SyntaxTokenTextSlice { .. }
| FormatElement::DynamicText { .. }
| FormatElement::StaticText { .. }
)
}
pub const fn is_space(&self) -> bool {
matches!(self, FormatElement::Space)
}
}
impl FormatElements for FormatElement {
fn will_break(&self) -> bool {
match self {
FormatElement::ExpandParent => true,
FormatElement::Tag(Tag::StartGroup(group)) => !group.mode().is_flat(),
FormatElement::Line(line_mode) => matches!(line_mode, LineMode::Hard | LineMode::Empty),
FormatElement::StaticText { text } => text.contains('\n'),
FormatElement::DynamicText { text, .. } => text.contains('\n'),
FormatElement::SyntaxTokenTextSlice { slice, .. } => slice.contains('\n'),
FormatElement::Interned(interned) => interned.will_break(),
// Traverse into the most flat version because the content is guaranteed to expand when even
// the most flat version contains some content that forces a break.
FormatElement::BestFitting(best_fitting) => best_fitting.most_flat().will_break(),
FormatElement::LineSuffixBoundary | FormatElement::Space | FormatElement::Tag(_) => {
false
}
}
}
fn has_label(&self, label_id: LabelId) -> bool {
match self {
FormatElement::Tag(Tag::StartLabelled(actual)) => *actual == label_id,
FormatElement::Interned(interned) => interned.deref().has_label(label_id),
_ => false,
}
}
fn start_tag(&self, _: TagKind) -> Option<&Tag> {
None
}
fn end_tag(&self, kind: TagKind) -> Option<&Tag> {
match self {
FormatElement::Tag(tag) if tag.kind() == kind && tag.is_end() => Some(tag),
_ => None,
}
}
}
/// Provides the printer with different representations for the same element so that the printer
/// can pick the best fitting variant.
///
/// Best fitting is defined as the variant that takes the most horizontal space but fits on the line.
#[derive(Clone, Eq, PartialEq)]
pub struct BestFitting {
/// The different variants for this element.
/// The first element is the one that takes up the most space horizontally (the most flat),
/// The last element takes up the least space horizontally (but most horizontal space).
variants: Box<[Box<[FormatElement]>]>,
}
impl BestFitting {
/// Creates a new best fitting IR with the given variants. The method itself isn't unsafe
/// but it is to discourage people from using it because the printer will panic if
/// the slice doesn't contain at least the least and most expanded variants.
///
/// You're looking for a way to create a `BestFitting` object, use the `best_fitting![least_expanded, most_expanded]` macro.
///
/// ## Safety
/// The slice must contain at least two variants.
#[doc(hidden)]
pub unsafe fn from_vec_unchecked(variants: Vec<Box<[FormatElement]>>) -> Self {
debug_assert!(
variants.len() >= 2,
"Requires at least the least expanded and most expanded variants"
);
Self {
variants: variants.into_boxed_slice(),
}
}
/// Returns the most expanded variant
pub fn most_expanded(&self) -> &[FormatElement] {
self.variants.last().expect(
"Most contain at least two elements, as guaranteed by the best fitting builder.",
)
}
pub fn variants(&self) -> &[Box<[FormatElement]>] {
&self.variants
}
/// Returns the least expanded variant
pub fn most_flat(&self) -> &[FormatElement] {
self.variants.first().expect(
"Most contain at least two elements, as guaranteed by the best fitting builder.",
)
}
}
impl std::fmt::Debug for BestFitting {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_list().entries(&*self.variants).finish()
}
}
pub trait FormatElements {
/// Returns true if this [FormatElement] is guaranteed to break across multiple lines by the printer.
/// This is the case if this format element recursively contains a:
/// * [crate::builders::empty_line] or [crate::builders::hard_line_break]
/// * A token containing '\n'
///
/// Use this with caution, this is only a heuristic and the printer may print the element over multiple
/// lines if this element is part of a group and the group doesn't fit on a single line.
fn will_break(&self) -> bool;
/// Returns true if the element has the given label.
fn has_label(&self, label: LabelId) -> bool;
/// Returns the start tag of `kind` if:
/// * the last element is an end tag of `kind`.
/// * there's a matching start tag in this document (may not be true if this slice is an interned element and the `start` is in the document storing the interned element).
fn start_tag(&self, kind: TagKind) -> Option<&Tag>;
/// Returns the end tag if:
/// * the last element is an end tag of `kind`
fn end_tag(&self, kind: TagKind) -> Option<&Tag>;
}
#[cfg(test)]
mod tests {
use crate::format_element::{normalize_newlines, LINE_TERMINATORS};
#[test]
fn test_normalize_newlines() {
assert_eq!(normalize_newlines("a\nb", LINE_TERMINATORS), "a\nb");
assert_eq!(normalize_newlines("a\n\n\nb", LINE_TERMINATORS), "a\n\n\nb");
assert_eq!(normalize_newlines("a\rb", LINE_TERMINATORS), "a\nb");
assert_eq!(normalize_newlines("a\r\nb", LINE_TERMINATORS), "a\nb");
assert_eq!(
normalize_newlines("a\r\n\r\n\r\nb", LINE_TERMINATORS),
"a\n\n\nb"
);
assert_eq!(normalize_newlines("a\u{2028}b", LINE_TERMINATORS), "a\nb");
assert_eq!(normalize_newlines("a\u{2029}b", LINE_TERMINATORS), "a\nb");
}
}
#[cfg(target_pointer_width = "64")]
static_assert!(std::mem::size_of::<ruff_rowan::TextRange>() == 8usize);
#[cfg(target_pointer_width = "64")]
static_assert!(std::mem::size_of::<crate::format_element::tag::VerbatimKind>() == 8usize);
#[cfg(not(debug_assertions))]
#[cfg(target_pointer_width = "64")]
static_assert!(std::mem::size_of::<crate::format_element::Tag>() == 16usize);
// Increasing the size of FormatElement has serious consequences on runtime performance and memory footprint.
// Is there a more efficient way to encode the data to avoid increasing its size? Can the information
// be recomputed at a later point in time?
// You reduced the size of a format element? Excellent work!
#[cfg(not(debug_assertions))]
#[cfg(target_pointer_width = "64")]
static_assert!(std::mem::size_of::<crate::FormatElement>() == 24usize);

714
crates/ruff_formatter/src/format_element/document.rs Normal file
View File

@ -0,0 +1,714 @@
use super::tag::Tag;
use crate::format_element::tag::DedentMode;
use crate::prelude::tag::GroupMode;
use crate::prelude::*;
use crate::printer::LineEnding;
use crate::{format, write};
use crate::{
BufferExtensions, Format, FormatContext, FormatElement, FormatOptions, FormatResult, Formatter,
IndentStyle, LineWidth, PrinterOptions, TransformSourceMap,
};
use ruff_rowan::TextSize;
use rustc_hash::FxHashMap;
use std::collections::HashMap;
use std::ops::Deref;
/// A formatted document.
#[derive(Debug, Clone, Eq, PartialEq, Default)]
pub struct Document {
elements: Vec<FormatElement>,
}
impl Document {
/// Sets [`expand`](tag::Group::expand) to [`GroupMode::Propagated`] if the group contains any of:
/// * a group with [`expand`](tag::Group::expand) set to [GroupMode::Propagated] or [GroupMode::Expand].
/// * a non-soft [line break](FormatElement::Line) with mode [LineMode::Hard], [LineMode::Empty], or [LineMode::Literal].
/// * a [FormatElement::ExpandParent]
///
/// [`BestFitting`] elements act as expand boundaries, meaning that the fact that a
/// [`BestFitting`]'s content expands is not propagated past the [`BestFitting`] element.
///
/// [`BestFitting`]: FormatElement::BestFitting
pub(crate) fn propagate_expand(&mut self) {
#[derive(Debug)]
enum Enclosing<'a> {
Group(&'a tag::Group),
BestFitting,
}
fn expand_parent(enclosing: &[Enclosing]) {
if let Some(Enclosing::Group(group)) = enclosing.last() {
group.propagate_expand();
}
}
fn propagate_expands<'a>(
elements: &'a [FormatElement],
enclosing: &mut Vec<Enclosing<'a>>,
checked_interned: &mut FxHashMap<&'a Interned, bool>,
) -> bool {
let mut expands = false;
for element in elements {
let element_expands = match element {
FormatElement::Tag(Tag::StartGroup(group)) => {
enclosing.push(Enclosing::Group(group));
false
}
FormatElement::Tag(Tag::EndGroup) => match enclosing.pop() {
Some(Enclosing::Group(group)) => !group.mode().is_flat(),
_ => false,
},
FormatElement::Interned(interned) => match checked_interned.get(interned) {
Some(interned_expands) => *interned_expands,
None => {
let interned_expands =
propagate_expands(interned, enclosing, checked_interned);
checked_interned.insert(interned, interned_expands);
interned_expands
}
},
FormatElement::BestFitting(best_fitting) => {
enclosing.push(Enclosing::BestFitting);
for variant in best_fitting.variants() {
propagate_expands(variant, enclosing, checked_interned);
}
// Best fitting acts as a boundary
expands = false;
enclosing.pop();
continue;
}
FormatElement::StaticText { text } => text.contains('\n'),
FormatElement::DynamicText { text, .. } => text.contains('\n'),
FormatElement::SyntaxTokenTextSlice { slice, .. } => slice.contains('\n'),
FormatElement::ExpandParent
| FormatElement::Line(LineMode::Hard | LineMode::Empty) => true,
_ => false,
};
if element_expands {
expands = true;
expand_parent(enclosing)
}
}
expands
}
let mut enclosing: Vec<Enclosing> = Vec::new();
let mut interned: FxHashMap<&Interned, bool> = FxHashMap::default();
propagate_expands(self, &mut enclosing, &mut interned);
}
}
impl From<Vec<FormatElement>> for Document {
fn from(elements: Vec<FormatElement>) -> Self {
Self { elements }
}
}
impl Deref for Document {
type Target = [FormatElement];
fn deref(&self) -> &Self::Target {
self.elements.as_slice()
}
}
impl std::fmt::Display for Document {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let formatted = format!(IrFormatContext::default(), [self.elements.as_slice()])
.expect("Formatting not to throw any FormatErrors");
f.write_str(
formatted
.print()
.expect("Expected a valid document")
.as_code(),
)
}
}
#[derive(Clone, Default, Debug)]
struct IrFormatContext {
/// The interned elements that have been printed to this point
printed_interned_elements: HashMap<Interned, usize>,
}
impl FormatContext for IrFormatContext {
type Options = IrFormatOptions;
fn options(&self) -> &Self::Options {
&IrFormatOptions
}
fn source_map(&self) -> Option<&TransformSourceMap> {
None
}
}
#[derive(Debug, Clone, Default)]
struct IrFormatOptions;
impl FormatOptions for IrFormatOptions {
fn indent_style(&self) -> IndentStyle {
IndentStyle::Space(2)
}
fn line_width(&self) -> LineWidth {
LineWidth(80)
}
fn as_print_options(&self) -> PrinterOptions {
PrinterOptions {
tab_width: 2,
print_width: self.line_width().into(),
line_ending: LineEnding::LineFeed,
indent_style: IndentStyle::Space(2),
}
}
}
impl Format<IrFormatContext> for &[FormatElement] {
fn fmt(&self, f: &mut Formatter<IrFormatContext>) -> FormatResult<()> {
use Tag::*;
write!(f, [ContentArrayStart])?;
let mut tag_stack = Vec::new();
let mut first_element = true;
let mut in_text = false;
let mut iter = self.iter().peekable();
while let Some(element) = iter.next() {
if !first_element && !in_text && !element.is_end_tag() {
// Write a separator between every two elements
write!(f, [text(","), soft_line_break_or_space()])?;
}
first_element = false;
match element {
element @ FormatElement::Space
| element @ FormatElement::StaticText { .. }
| element @ FormatElement::DynamicText { .. }
| element @ FormatElement::SyntaxTokenTextSlice { .. } => {
if !in_text {
write!(f, [text("\"")])?;
}
in_text = true;
match element {
FormatElement::Space => {
write!(f, [text(" ")])?;
}
element if element.is_text() => f.write_element(element.clone())?,
_ => unreachable!(),
}
let is_next_text = iter.peek().map_or(false, |e| e.is_text() || e.is_space());
if !is_next_text {
write!(f, [text("\"")])?;
in_text = false;
}
}
FormatElement::Line(mode) => match mode {
LineMode::SoftOrSpace => {
write!(f, [text("soft_line_break_or_space")])?;
}
LineMode::Soft => {
write!(f, [text("soft_line_break")])?;
}
LineMode::Hard => {
write!(f, [text("hard_line_break")])?;
}
LineMode::Empty => {
write!(f, [text("empty_line")])?;
}
},
FormatElement::ExpandParent => {
write!(f, [text("expand_parent")])?;
}
FormatElement::LineSuffixBoundary => {
write!(f, [text("line_suffix_boundary")])?;
}
FormatElement::BestFitting(best_fitting) => {
write!(f, [text("best_fitting([")])?;
f.write_elements([
FormatElement::Tag(StartIndent),
FormatElement::Line(LineMode::Hard),
])?;
for variant in best_fitting.variants() {
write!(f, [variant.deref(), hard_line_break()])?;
}
f.write_elements([
FormatElement::Tag(EndIndent),
FormatElement::Line(LineMode::Hard),
])?;
write!(f, [text("])")])?;
}
FormatElement::Interned(interned) => {
let interned_elements = &mut f.context_mut().printed_interned_elements;
match interned_elements.get(interned).copied() {
None => {
let index = interned_elements.len();
interned_elements.insert(interned.clone(), index);
write!(
f,
[
dynamic_text(
&std::format!("<interned {index}>"),
TextSize::default()
),
space(),
&interned.deref(),
]
)?;
}
Some(reference) => {
write!(
f,
[dynamic_text(
&std::format!("<ref interned *{reference}>"),
TextSize::default()
)]
)?;
}
}
}
FormatElement::Tag(tag) => {
if tag.is_start() {
first_element = true;
tag_stack.push(tag.kind());
}
// Handle documents with mismatching start/end or superfluous end tags
else {
match tag_stack.pop() {
None => {
// Only write the end tag without any indent to ensure the output document is valid.
write!(
f,
[
text("<END_TAG_WITHOUT_START<"),
dynamic_text(
&std::format!("{:?}", tag.kind()),
TextSize::default()
),
text(">>"),
]
)?;
first_element = false;
continue;
}
Some(start_kind) if start_kind != tag.kind() => {
write!(
f,
[
ContentArrayEnd,
text(")"),
soft_line_break_or_space(),
text("ERROR<START_END_TAG_MISMATCH<start: "),
dynamic_text(
&std::format!("{start_kind:?}"),
TextSize::default()
),
text(", end: "),
dynamic_text(
&std::format!("{:?}", tag.kind()),
TextSize::default()
),
text(">>")
]
)?;
first_element = false;
continue;
}
_ => {
// all ok
}
}
}
match tag {
StartIndent => {
write!(f, [text("indent(")])?;
}
StartDedent(mode) => {
let label = match mode {
DedentMode::Level => "dedent",
DedentMode::Root => "dedentRoot",
};
write!(f, [text(label), text("(")])?;
}
StartAlign(tag::Align(count)) => {
write!(
f,
[
text("align("),
dynamic_text(&count.to_string(), TextSize::default()),
text(","),
space(),
]
)?;
}
StartLineSuffix => {
write!(f, [text("line_suffix(")])?;
}
StartVerbatim(_) => {
write!(f, [text("verbatim(")])?;
}
StartGroup(group) => {
write!(f, [text("group(")])?;
if let Some(group_id) = group.id() {
write!(
f,
[
dynamic_text(
&std::format!("\"{group_id:?}\""),
TextSize::default()
),
text(","),
space(),
]
)?;
}
match group.mode() {
GroupMode::Flat => {}
GroupMode::Expand => {
write!(f, [text("expand: true,"), space()])?;
}
GroupMode::Propagated => {
write!(f, [text("expand: propagated,"), space()])?;
}
}
}
StartIndentIfGroupBreaks(id) => {
write!(
f,
[
text("indent_if_group_breaks("),
dynamic_text(&std::format!("\"{id:?}\""), TextSize::default()),
text(","),
space(),
]
)?;
}
StartConditionalContent(condition) => {
match condition.mode {
PrintMode::Flat => {
write!(f, [text("if_group_fits_on_line(")])?;
}
PrintMode::Expanded => {
write!(f, [text("if_group_breaks(")])?;
}
}
if let Some(group_id) = condition.group_id {
write!(
f,
[
dynamic_text(
&std::format!("\"{group_id:?}\""),
TextSize::default()
),
text(","),
space(),
]
)?;
}
}
StartLabelled(label_id) => {
write!(
f,
[
text("label("),
dynamic_text(
&std::format!("\"{label_id:?}\""),
TextSize::default()
),
text(","),
space(),
]
)?;
}
StartFill => {
write!(f, [text("fill(")])?;
}
StartEntry => {
// handled after the match for all start tags
}
EndEntry => write!(f, [ContentArrayEnd])?,
EndFill
| EndLabelled
| EndConditionalContent
| EndIndentIfGroupBreaks
| EndAlign
| EndIndent
| EndGroup
| EndLineSuffix
| EndDedent
| EndVerbatim => {
write!(f, [ContentArrayEnd, text(")")])?;
}
};
if tag.is_start() {
write!(f, [ContentArrayStart])?;
}
}
}
}
while let Some(top) = tag_stack.pop() {
write!(
f,
[
ContentArrayEnd,
text(")"),
soft_line_break_or_space(),
dynamic_text(
&std::format!("<START_WITHOUT_END<{top:?}>>"),
TextSize::default()
),
]
)?;
}
write!(f, [ContentArrayEnd])
}
}
struct ContentArrayStart;
impl Format<IrFormatContext> for ContentArrayStart {
fn fmt(&self, f: &mut Formatter<IrFormatContext>) -> FormatResult<()> {
use Tag::*;
write!(f, [text("[")])?;
f.write_elements([
FormatElement::Tag(StartGroup(tag::Group::new())),
FormatElement::Tag(StartIndent),
FormatElement::Line(LineMode::Soft),
])
}
}
struct ContentArrayEnd;
impl Format<IrFormatContext> for ContentArrayEnd {
fn fmt(&self, f: &mut Formatter<IrFormatContext>) -> FormatResult<()> {
use Tag::*;
f.write_elements([
FormatElement::Tag(EndIndent),
FormatElement::Line(LineMode::Soft),
FormatElement::Tag(EndGroup),
])?;
write!(f, [text("]")])
}
}
impl FormatElements for [FormatElement] {
fn will_break(&self) -> bool {
use Tag::*;
let mut ignore_depth = 0usize;
for element in self {
match element {
// Line suffix
// Ignore if any of its content breaks
FormatElement::Tag(StartLineSuffix) => {
ignore_depth += 1;
}
FormatElement::Tag(EndLineSuffix) => {
ignore_depth -= 1;
}
FormatElement::Interned(interned) if ignore_depth == 0 => {
if interned.will_break() {
return true;
}
}
element if ignore_depth == 0 && element.will_break() => {
return true;
}
_ => continue,
}
}
debug_assert_eq!(ignore_depth, 0, "Unclosed start container");
false
}
fn has_label(&self, expected: LabelId) -> bool {
self.first()
.map_or(false, |element| element.has_label(expected))
}
fn start_tag(&self, kind: TagKind) -> Option<&Tag> {
// Assert that the document ends at a tag with the specified kind;
let _ = self.end_tag(kind)?;
fn traverse_slice<'a>(
slice: &'a [FormatElement],
kind: TagKind,
depth: &mut usize,
) -> Option<&'a Tag> {
for element in slice.iter().rev() {
match element {
FormatElement::Tag(tag) if tag.kind() == kind => {
if tag.is_start() {
if *depth == 0 {
// Invalid document
return None;
} else if *depth == 1 {
return Some(tag);
} else {
*depth -= 1;
}
} else {
*depth += 1;
}
}
FormatElement::Interned(interned) => {
match traverse_slice(interned, kind, depth) {
Some(start) => {
return Some(start);
}
// Reached end or invalid document
None if *depth == 0 => {
return None;
}
_ => {
// continue with other elements
}
}
}
_ => {}
}
}
None
}
let mut depth = 0usize;
traverse_slice(self, kind, &mut depth)
}
fn end_tag(&self, kind: TagKind) -> Option<&Tag> {
self.last().and_then(|element| element.end_tag(kind))
}
}
#[cfg(test)]
mod tests {
use crate::prelude::*;
use crate::SimpleFormatContext;
use crate::{format, format_args, write};
#[test]
fn display_elements() {
let formatted = format!(
SimpleFormatContext::default(),
[format_with(|f| {
write!(
f,
[group(&format_args![
text("("),
soft_block_indent(&format_args![
text("Some longer content"),
space(),
text("That should ultimately break"),
])
])]
)
})]
)
.unwrap();
let document = formatted.into_document();
assert_eq!(
&std::format!("{document}"),
r#"[
group([
"(",
indent([
soft_line_break,
"Some longer content That should ultimately break"
]),
soft_line_break
])
]"#
);
}
#[test]
fn display_invalid_document() {
use Tag::*;
let document = Document::from(vec![
FormatElement::StaticText { text: "[" },
FormatElement::Tag(StartGroup(tag::Group::new())),
FormatElement::Tag(StartIndent),
FormatElement::Line(LineMode::Soft),
FormatElement::StaticText { text: "a" },
// Close group instead of indent
FormatElement::Tag(EndGroup),
FormatElement::Line(LineMode::Soft),
FormatElement::Tag(EndIndent),
FormatElement::StaticText { text: "]" },
// End tag without start
FormatElement::Tag(EndIndent),
// Start tag without an end
FormatElement::Tag(StartIndent),
]);
assert_eq!(
&std::format!("{document}"),
r#"[
"[",
group([
indent([soft_line_break, "a"])
ERROR<START_END_TAG_MISMATCH<start: Indent, end: Group>>,
soft_line_break
])
ERROR<START_END_TAG_MISMATCH<start: Group, end: Indent>>,
"]"<END_TAG_WITHOUT_START<Indent>>,
indent([])
<START_WITHOUT_END<Indent>>
]"#
);
}
}

287
crates/ruff_formatter/src/format_element/tag.rs Normal file
View File

@ -0,0 +1,287 @@
use crate::format_element::PrintMode;
use crate::{GroupId, TextSize};
#[cfg(debug_assertions)]
use std::any::type_name;
use std::any::TypeId;
use std::cell::Cell;
use std::num::NonZeroU8;
/// A Tag marking the start and end of some content to which some special formatting should be applied.
///
/// Tags always come in pairs of a start and an end tag and the styling defined by this tag
/// will be applied to all elements in between the start/end tags.
#[derive(Clone, Debug, Eq, PartialEq)]
pub enum Tag {
/// Indents the content one level deeper, see [crate::builders::indent] for documentation and examples.
StartIndent,
EndIndent,
/// Variant of [TagKind::Indent] that indents content by a number of spaces. For example, `Align(2)`
/// indents any content following a line break by an additional two spaces.
///
/// Nesting (Aligns)[TagKind::Align] has the effect that all except the most inner align are handled as (Indent)[TagKind::Indent].
StartAlign(Align),
EndAlign,
/// Reduces the indention of the specified content either by one level or to the root, depending on the mode.
/// Reverse operation of `Indent` and can be used to *undo* an `Align` for nested content.
StartDedent(DedentMode),
EndDedent,
/// Creates a logical group where its content is either consistently printed:
/// * on a single line: Omitting `LineMode::Soft` line breaks and printing spaces for `LineMode::SoftOrSpace`
/// * on multiple lines: Printing all line breaks
///
/// See [crate::builders::group] for documentation and examples.
StartGroup(Group),
EndGroup,
/// Allows to specify content that gets printed depending on whatever the enclosing group
/// is printed on a single line or multiple lines. See [crate::builders::if_group_breaks] for examples.
StartConditionalContent(Condition),
EndConditionalContent,
/// Optimized version of [Tag::StartConditionalContent] for the case where some content
/// should be indented if the specified group breaks.
StartIndentIfGroupBreaks(GroupId),
EndIndentIfGroupBreaks,
/// Concatenates multiple elements together with a given separator printed in either
/// flat or expanded mode to fill the print width. Expect that the content is a list of alternating
/// [element, separator] See [crate::Formatter::fill].
StartFill,
EndFill,
/// Entry inside of a [Tag::StartFill]
StartEntry,
EndEntry,
/// Delay the printing of its content until the next line break
StartLineSuffix,
EndLineSuffix,
/// A token that tracks tokens/nodes that are printed as verbatim.
StartVerbatim(VerbatimKind),
EndVerbatim,
/// Special semantic element marking the content with a label.
/// This does not directly influence how the content will be printed.
///
/// See [crate::builders::labelled] for documentation.
StartLabelled(LabelId),
EndLabelled,
}
impl Tag {
/// Returns `true` if `self` is any start tag.
pub const fn is_start(&self) -> bool {
matches!(
self,
Tag::StartIndent
| Tag::StartAlign(_)
| Tag::StartDedent(_)
| Tag::StartGroup { .. }
| Tag::StartConditionalContent(_)
| Tag::StartIndentIfGroupBreaks(_)
| Tag::StartFill
| Tag::StartEntry
| Tag::StartLineSuffix
| Tag::StartVerbatim(_)
| Tag::StartLabelled(_)
)
}
/// Returns `true` if `self` is any end tag.
pub const fn is_end(&self) -> bool {
!self.is_start()
}
pub const fn kind(&self) -> TagKind {
use Tag::*;
match self {
StartIndent | EndIndent => TagKind::Indent,
StartAlign(_) | EndAlign => TagKind::Align,
StartDedent(_) | EndDedent => TagKind::Dedent,
StartGroup(_) | EndGroup => TagKind::Group,
StartConditionalContent(_) | EndConditionalContent => TagKind::ConditionalContent,
StartIndentIfGroupBreaks(_) | EndIndentIfGroupBreaks => TagKind::IndentIfGroupBreaks,
StartFill | EndFill => TagKind::Fill,
StartEntry | EndEntry => TagKind::Entry,
StartLineSuffix | EndLineSuffix => TagKind::LineSuffix,
StartVerbatim(_) | EndVerbatim => TagKind::Verbatim,
StartLabelled(_) | EndLabelled => TagKind::Labelled,
}
}
}
/// The kind of a [Tag].
///
/// Each start end tag pair has its own [tag kind](TagKind).
#[derive(Debug, Copy, Clone, Eq, PartialEq)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
pub enum TagKind {
Indent,
Align,
Dedent,
Group,
ConditionalContent,
IndentIfGroupBreaks,
Fill,
Entry,
LineSuffix,
Verbatim,
Labelled,
}
#[derive(Debug, Copy, Default, Clone, Eq, PartialEq)]
pub enum GroupMode {
/// Print group in flat mode.
#[default]
Flat,
/// The group should be printed in expanded mode
Expand,
/// Expand mode has been propagated from an enclosing group to this group.
Propagated,
}
impl GroupMode {
pub const fn is_flat(&self) -> bool {
matches!(self, GroupMode::Flat)
}
}
#[derive(Debug, Clone, Eq, PartialEq, Default)]
pub struct Group {
id: Option<GroupId>,
mode: Cell<GroupMode>,
}
impl Group {
pub fn new() -> Self {
Self {
id: None,
mode: Cell::new(GroupMode::Flat),
}
}
pub fn with_id(mut self, id: Option<GroupId>) -> Self {
self.id = id;
self
}
pub fn with_mode(mut self, mode: GroupMode) -> Self {
self.mode = Cell::new(mode);
self
}
pub fn mode(&self) -> GroupMode {
self.mode.get()
}
pub fn propagate_expand(&self) {
if self.mode.get() == GroupMode::Flat {
self.mode.set(GroupMode::Propagated)
}
}
pub fn id(&self) -> Option<GroupId> {
self.id
}
}
#[derive(Debug, Copy, Clone, Eq, PartialEq)]
pub enum DedentMode {
/// Reduces the indent by a level (if the current indent is > 0)
Level,
/// Reduces the indent to the root
Root,
}
#[derive(Debug, Clone, Eq, PartialEq)]
pub struct Condition {
/// * Flat -> Omitted if the enclosing group is a multiline group, printed for groups fitting on a single line
/// * Multiline -> Omitted if the enclosing group fits on a single line, printed if the group breaks over multiple lines.
pub(crate) mode: PrintMode,
/// The id of the group for which it should check if it breaks or not. The group must appear in the document
/// before the conditional group (but doesn't have to be in the ancestor chain).
pub(crate) group_id: Option<GroupId>,
}
impl Condition {
pub fn new(mode: PrintMode) -> Self {
Self {
mode,
group_id: None,
}
}
pub fn with_group_id(mut self, id: Option<GroupId>) -> Self {
self.group_id = id;
self
}
pub fn mode(&self) -> PrintMode {
self.mode
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct Align(pub(crate) NonZeroU8);
impl Align {
pub fn count(&self) -> NonZeroU8 {
self.0
}
}
#[derive(Eq, PartialEq, Copy, Clone)]
pub struct LabelId {
id: TypeId,
#[cfg(debug_assertions)]
label: &'static str,
}
#[cfg(debug_assertions)]
impl std::fmt::Debug for LabelId {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(self.label)
}
}
#[cfg(not(debug_assertions))]
impl std::fmt::Debug for LabelId {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
std::write!(f, "#{:?}", self.id)
}
}
impl LabelId {
pub fn of<T: ?Sized + 'static>() -> Self {
Self {
id: TypeId::of::<T>(),
#[cfg(debug_assertions)]
label: type_name::<T>(),
}
}
}
#[derive(Clone, Copy, Eq, PartialEq, Debug)]
pub enum VerbatimKind {
Bogus,
Suppressed,
Verbatim {
/// the length of the formatted node
length: TextSize,
},
}
impl VerbatimKind {
pub const fn is_bogus(&self) -> bool {
matches!(self, VerbatimKind::Bogus)
}
}

178
crates/ruff_formatter/src/format_extensions.rs Normal file
View File

@ -0,0 +1,178 @@
use crate::prelude::*;
use std::cell::RefCell;
use std::marker::PhantomData;
use std::ops::Deref;
use crate::Buffer;
/// Utility trait that allows memorizing the output of a [Format].
/// Useful to avoid re-formatting the same object twice.
pub trait MemoizeFormat<Context> {
/// Returns a formattable object that memoizes the result of `Format` by cloning.
/// Mainly useful if the same sub-tree can appear twice in the formatted output because it's
/// used inside of `if_group_breaks` or `if_group_fits_single_line`.
///
/// ```
/// use std::cell::Cell;
/// use ruff_formatter::{format, write};
/// use ruff_formatter::prelude::*;
/// use ruff_rowan::TextSize;
///
/// struct MyFormat {
/// value: Cell<u64>
/// }
///
/// impl MyFormat {
/// pub fn new() -> Self {
/// Self { value: Cell::new(1) }
/// }
/// }
///
/// impl Format<SimpleFormatContext> for MyFormat {
/// fn fmt(&self, f: &mut Formatter<SimpleFormatContext>) -> FormatResult<()> {
/// let value = self.value.get();
/// self.value.set(value + 1);
///
/// write!(f, [dynamic_text(&std::format!("Formatted {value} times."), TextSize::from(0))])
/// }
/// }
///
/// # fn main() -> FormatResult<()> {
/// let normal = MyFormat::new();
///
/// // Calls `format` for everytime the object gets formatted
/// assert_eq!(
/// "Formatted 1 times. Formatted 2 times.",
/// format!(SimpleFormatContext::default(), [normal, space(), normal])?.print()?.as_code()
/// );
///
/// // Memoized memoizes the result and calls `format` only once.
/// let memoized = normal.memoized();
/// assert_eq!(
/// "Formatted 3 times. Formatted 3 times.",
/// format![SimpleFormatContext::default(), [memoized, space(), memoized]]?.print()?.as_code()
/// );
/// # Ok(())
/// # }
/// ```
///
fn memoized(self) -> Memoized<Self, Context>
where
Self: Sized + Format<Context>,
{
Memoized::new(self)
}
}
impl<T, Context> MemoizeFormat<Context> for T where T: Format<Context> {}
/// Memoizes the output of its inner [Format] to avoid re-formatting a potential expensive object.
#[derive(Debug)]
pub struct Memoized<F, Context> {
inner: F,
memory: RefCell<Option<FormatResult<Option<FormatElement>>>>,
options: PhantomData<Context>,
}
impl<F, Context> Memoized<F, Context>
where
F: Format<Context>,
{
fn new(inner: F) -> Self {
Self {
inner,
memory: RefCell::new(None),
options: PhantomData,
}
}
/// Gives access to the memoized content.
///
/// Performs the formatting if the content hasn't been formatted at this point.
///
/// # Example
///
/// Inspect if some memoized content breaks.
///
/// ```rust
/// use std::cell::Cell;
/// use ruff_formatter::{format, write};
/// use ruff_formatter::prelude::*;
/// use ruff_rowan::TextSize;
///
/// #[derive(Default)]
/// struct Counter {
/// value: Cell<u64>
/// }
///
/// impl Format<SimpleFormatContext> for Counter {
/// fn fmt(&self, f: &mut Formatter<SimpleFormatContext>) -> FormatResult<()> {
/// let current = self.value.get();
///
/// write!(f, [
/// text("Count:"),
/// space(),
/// dynamic_text(&std::format!("{current}"), TextSize::default()),
/// hard_line_break()
/// ])?;
///
/// self.value.set(current + 1);
/// Ok(())
/// }
/// }
///
/// # fn main() -> FormatResult<()> {
/// let content = format_with(|f| {
/// let mut counter = Counter::default().memoized();
/// let counter_content = counter.inspect(f)?;
///
/// if counter_content.will_break() {
/// write!(f, [text("Counter:"), block_indent(&counter)])
/// } else {
/// write!(f, [text("Counter:"), counter])
/// }?;
///
/// write!(f, [counter])
/// });
///
///
/// let formatted = format!(SimpleFormatContext::default(), [content])?;
/// assert_eq!("Counter:\n\tCount: 0\nCount: 0\n", formatted.print()?.as_code());
/// # Ok(())
/// # }
///
/// ```
pub fn inspect(&mut self, f: &mut Formatter<Context>) -> FormatResult<&[FormatElement]> {
let result = self
.memory
.get_mut()
.get_or_insert_with(|| f.intern(&self.inner));
match result.as_ref() {
Ok(Some(FormatElement::Interned(interned))) => Ok(interned.deref()),
Ok(Some(other)) => Ok(std::slice::from_ref(other)),
Ok(None) => Ok(&[]),
Err(error) => Err(*error),
}
}
}
impl<F, Context> Format<Context> for Memoized<F, Context>
where
F: Format<Context>,
{
fn fmt(&self, f: &mut Formatter<Context>) -> FormatResult<()> {
let mut memory = self.memory.borrow_mut();
let result = memory.get_or_insert_with(|| f.intern(&self.inner));
match result {
Ok(Some(elements)) => {
f.write_element(elements.clone())?;
Ok(())
}
Ok(None) => Ok(()),
Err(err) => Err(*err),
}
}
}

288
crates/ruff_formatter/src/formatter.rs Normal file
View File

@ -0,0 +1,288 @@
use crate::buffer::BufferSnapshot;
use crate::builders::{FillBuilder, JoinBuilder, JoinNodesBuilder, Line};
use crate::prelude::*;
use crate::{
Arguments, Buffer, Comments, CstFormatContext, FormatContext, FormatState, FormatStateSnapshot,
GroupId, VecBuffer,
};
/// Handles the formatting of a CST and stores the context how the CST should be formatted (user preferences).
/// The formatter is passed to the [Format] implementation of every node in the CST so that they
/// can use it to format their children.
pub struct Formatter<'buf, Context> {
pub(super) buffer: &'buf mut dyn Buffer<Context = Context>,
}
impl<'buf, Context> Formatter<'buf, Context> {
/// Creates a new context that uses the given formatter context
pub fn new(buffer: &'buf mut (dyn Buffer<Context = Context> + 'buf)) -> Self {
Self { buffer }
}
/// Returns the format options
pub fn options(&self) -> &Context::Options
where
Context: FormatContext,
{
self.context().options()
}
/// Returns the Context specifying how to format the current CST
pub fn context(&self) -> &Context {
self.state().context()
}
/// Returns a mutable reference to the context.
pub fn context_mut(&mut self) -> &mut Context {
self.state_mut().context_mut()
}
/// Creates a new group id that is unique to this document. The passed debug name is used in the
/// [std::fmt::Debug] of the document if this is a debug build.
/// The name is unused for production builds and has no meaning on the equality of two group ids.
pub fn group_id(&self, debug_name: &'static str) -> GroupId {
self.state().group_id(debug_name)
}
/// Joins multiple [Format] together without any separator
///
/// ## Examples
///
/// ```rust
/// use ruff_formatter::format;
/// use ruff_formatter::prelude::*;
///
/// # fn main() -> FormatResult<()> {
/// let formatted = format!(SimpleFormatContext::default(), [format_with(|f| {
/// f.join()
/// .entry(&text("a"))
/// .entry(&space())
/// .entry(&text("+"))
/// .entry(&space())
/// .entry(&text("b"))
/// .finish()
/// })])?;
///
/// assert_eq!(
/// "a + b",
/// formatted.print()?.as_code()
/// );
/// # Ok(())
/// # }
/// ```
pub fn join<'a>(&'a mut self) -> JoinBuilder<'a, 'buf, (), Context> {
JoinBuilder::new(self)
}
/// Joins the objects by placing the specified separator between every two items.
///
/// ## Examples
///
/// Joining different tokens by separating them with a comma and a space.
///
/// ```
/// use ruff_formatter::{format, format_args};
/// use ruff_formatter::prelude::*;
///
/// # fn main() -> FormatResult<()> {
/// let formatted = format!(SimpleFormatContext::default(), [format_with(|f| {
/// f.join_with(&format_args!(text(","), space()))
/// .entry(&text("1"))
/// .entry(&text("2"))
/// .entry(&text("3"))
/// .entry(&text("4"))
/// .finish()
/// })])?;
///
/// assert_eq!(
/// "1, 2, 3, 4",
/// formatted.print()?.as_code()
/// );
/// # Ok(())
/// # }
/// ```
pub fn join_with<'a, Joiner>(
&'a mut self,
joiner: Joiner,
) -> JoinBuilder<'a, 'buf, Joiner, Context>
where
Joiner: Format<Context>,
{
JoinBuilder::with_separator(self, joiner)
}
/// Specialized version of [crate::Formatter::join_with] for joining SyntaxNodes separated by a space, soft
/// line break or empty line depending on the input file.
///
/// This functions inspects the input source and separates consecutive elements with either
/// a [crate::builders::soft_line_break_or_space] or [crate::builders::empty_line] depending on how many line breaks were
/// separating the elements in the original file.
pub fn join_nodes_with_soft_line<'a>(
&'a mut self,
) -> JoinNodesBuilder<'a, 'buf, Line, Context> {
JoinNodesBuilder::new(soft_line_break_or_space(), self)
}
/// Specialized version of [crate::Formatter::join_with] for joining SyntaxNodes separated by one or more
/// line breaks depending on the input file.
///
/// This functions inspects the input source and separates consecutive elements with either
/// a [crate::builders::hard_line_break] or [crate::builders::empty_line] depending on how many line breaks were separating the
/// elements in the original file.
pub fn join_nodes_with_hardline<'a>(&'a mut self) -> JoinNodesBuilder<'a, 'buf, Line, Context> {
JoinNodesBuilder::new(hard_line_break(), self)
}
/// Concatenates a list of [crate::Format] objects with spaces and line breaks to fit
/// them on as few lines as possible. Each element introduces a conceptual group. The printer
/// first tries to print the item in flat mode but then prints it in expanded mode if it doesn't fit.
///
/// ## Examples
///
/// ```rust
/// use ruff_formatter::prelude::*;
/// use ruff_formatter::{format, format_args};
///
/// # fn main() -> FormatResult<()> {
/// let formatted = format!(SimpleFormatContext::default(), [format_with(|f| {
/// f.fill()
/// .entry(&soft_line_break_or_space(), &text("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"))
/// .entry(&soft_line_break_or_space(), &text("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"))
/// .entry(&soft_line_break_or_space(), &text("cccccccccccccccccccccccccccccc"))
/// .entry(&soft_line_break_or_space(), &text("dddddddddddddddddddddddddddddd"))
/// .finish()
/// })])?;
///
/// assert_eq!(
/// "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa bbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\ncccccccccccccccccccccccccccccc dddddddddddddddddddddddddddddd",
/// formatted.print()?.as_code()
/// );
/// # Ok(())
/// # }
/// ```
///
/// ```rust
/// use ruff_formatter::prelude::*;
/// use ruff_formatter::{format, format_args};
///
/// # fn main() -> FormatResult<()> {
/// let entries = vec![
/// text("<b>Important: </b>"),
/// text("Please do not commit memory bugs such as segfaults, buffer overflows, etc. otherwise you "),
/// text("<em>will</em>"),
/// text(" be reprimanded")
/// ];
///
/// let formatted = format!(SimpleFormatContext::default(), [format_with(|f| {
/// f.fill().entries(&soft_line_break(), entries.iter()).finish()
/// })])?;
///
/// assert_eq!(
/// &std::format!("<b>Important: </b>\nPlease do not commit memory bugs such as segfaults, buffer overflows, etc. otherwise you \n<em>will</em> be reprimanded"),
/// formatted.print()?.as_code()
/// );
/// # Ok(())
/// # }
/// ```
pub fn fill<'a>(&'a mut self) -> FillBuilder<'a, 'buf, Context> {
FillBuilder::new(self)
}
/// Formats `content` into an interned element without writing it to the formatter's buffer.
pub fn intern(&mut self, content: &dyn Format<Context>) -> FormatResult<Option<FormatElement>> {
let mut buffer = VecBuffer::new(self.state_mut());
crate::write!(&mut buffer, [content])?;
let elements = buffer.into_vec();
Ok(self.intern_vec(elements))
}
pub fn intern_vec(&mut self, mut elements: Vec<FormatElement>) -> Option<FormatElement> {
match elements.len() {
0 => None,
// Doesn't get cheaper than calling clone, use the element directly
// SAFETY: Safe because of the `len == 1` check in the match arm.
1 => Some(elements.pop().unwrap()),
_ => Some(FormatElement::Interned(Interned::new(elements))),
}
}
}
impl<Context> Formatter<'_, Context>
where
Context: FormatContext,
{
/// Take a snapshot of the state of the formatter
#[inline]
pub fn state_snapshot(&self) -> FormatterSnapshot {
FormatterSnapshot {
buffer: self.buffer.snapshot(),
state: self.state().snapshot(),
}
}
#[inline]
/// Restore the state of the formatter to a previous snapshot
pub fn restore_state_snapshot(&mut self, snapshot: FormatterSnapshot) {
self.state_mut().restore_snapshot(snapshot.state);
self.buffer.restore_snapshot(snapshot.buffer);
}
}
impl<Context> Formatter<'_, Context>
where
Context: CstFormatContext,
{
/// Returns the comments from the context.
pub fn comments(&self) -> &Comments<Context::Language> {
self.context().comments()
}
}
impl<Context> Buffer for Formatter<'_, Context> {
type Context = Context;
#[inline(always)]
fn write_element(&mut self, element: FormatElement) -> FormatResult<()> {
self.buffer.write_element(element)
}
fn elements(&self) -> &[FormatElement] {
self.buffer.elements()
}
#[inline(always)]
fn write_fmt(&mut self, arguments: Arguments<Self::Context>) -> FormatResult<()> {
for argument in arguments.items() {
argument.format(self)?;
}
Ok(())
}
fn state(&self) -> &FormatState<Self::Context> {
self.buffer.state()
}
fn state_mut(&mut self) -> &mut FormatState<Self::Context> {
self.buffer.state_mut()
}
fn snapshot(&self) -> BufferSnapshot {
self.buffer.snapshot()
}
fn restore_snapshot(&mut self, snapshot: BufferSnapshot) {
self.buffer.restore_snapshot(snapshot)
}
}
/// Snapshot of the formatter state used to handle backtracking if
/// errors are encountered in the formatting process and the formatter
/// has to fallback to printing raw tokens
///
/// In practice this only saves the set of printed tokens in debug
/// mode and compiled to nothing in release mode
pub struct FormatterSnapshot {
buffer: BufferSnapshot,
state: FormatStateSnapshot,
}

82
crates/ruff_formatter/src/group_id.rs Normal file
View File

@ -0,0 +1,82 @@
use std::num::NonZeroU32;
use std::sync::atomic::{AtomicU32, Ordering};
#[derive(Clone, Copy, Eq, PartialEq, Hash)]
pub struct DebugGroupId {
value: NonZeroU32,
name: &'static str,
}
impl DebugGroupId {
#[allow(unused)]
fn new(value: NonZeroU32, debug_name: &'static str) -> Self {
Self {
value,
name: debug_name,
}
}
}
impl std::fmt::Debug for DebugGroupId {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "#{}-{}", self.name, self.value)
}
}
/// Unique identification for a group.
///
/// See [crate::Formatter::group_id] on how to get a unique id.
#[repr(transparent)]
#[derive(Clone, Copy, Eq, PartialEq, Hash)]
pub struct ReleaseGroupId {
value: NonZeroU32,
}
impl ReleaseGroupId {
/// Creates a new unique group id with the given debug name (only stored in debug builds)
#[allow(unused)]
fn new(value: NonZeroU32, _: &'static str) -> Self {
Self { value }
}
}
impl From<GroupId> for u32 {
fn from(id: GroupId) -> Self {
id.value.get()
}
}
impl std::fmt::Debug for ReleaseGroupId {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "#{}", self.value)
}
}
#[cfg(not(debug_assertions))]
pub type GroupId = ReleaseGroupId;
#[cfg(debug_assertions)]
pub type GroupId = DebugGroupId;
/// Builder to construct unique group ids that are unique if created with the same builder.
pub(super) struct UniqueGroupIdBuilder {
next_id: AtomicU32,
}
impl UniqueGroupIdBuilder {
/// Creates a new unique group id with the given debug name.
pub fn group_id(&self, debug_name: &'static str) -> GroupId {
let id = self.next_id.fetch_add(1, Ordering::Relaxed);
let id = NonZeroU32::new(id).unwrap_or_else(|| panic!("Group ID counter overflowed"));
GroupId::new(id, debug_name)
}
}
impl Default for UniqueGroupIdBuilder {
fn default() -> Self {
UniqueGroupIdBuilder {
// Start with 1 because `GroupId` wraps a `NonZeroU32` to reduce memory usage.
next_id: AtomicU32::new(1),
}
}
}

1436
crates/ruff_formatter/src/lib.rs Normal file
View File

File diff suppressed because it is too large Load Diff

502
crates/ruff_formatter/src/macros.rs Normal file
View File

@ -0,0 +1,502 @@
/// Constructs the parameters for other formatting macros.
///
/// This macro functions by taking a list of objects implementing [crate::Format]. It canonicalize the
/// arguments into a single type.
///
/// This macro produces a value of type [crate::Arguments]. This value can be passed to
/// the macros within [crate]. All other formatting macros ([`format!`](crate::format!),
/// [`write!`](crate::write!)) are proxied through this one. This macro avoids heap allocations.
///
/// You can use the [`Arguments`] value that `format_args!` returns in `Format` contexts
/// as seen below.
///
/// ```rust
/// use ruff_formatter::{SimpleFormatContext, format, format_args};
/// use ruff_formatter::prelude::*;
///
/// # fn main() -> FormatResult<()> {
/// let formatted = format!(SimpleFormatContext::default(), [
/// format_args!(text("Hello World"))
/// ])?;
///
/// assert_eq!("Hello World", formatted.print()?.as_code());
/// # Ok(())
/// # }
/// ```
///
/// [`Format`]: crate::Format
/// [`Arguments`]: crate::Arguments
#[macro_export]
macro_rules! format_args {
($($value:expr),+ $(,)?) => {
$crate::Arguments::new(&[
$(
$crate::Argument::new(&$value)
),+
])
}
}
/// Writes formatted data into a buffer.
///
/// This macro accepts a 'buffer' and a list of format arguments. Each argument will be formatted
/// and the result will be passed to the buffer. The writer may be any value with a `write_fmt` method;
/// generally this comes from an implementation of the [crate::Buffer] trait.
///
/// # Examples
///
/// ```rust
/// use ruff_formatter::prelude::*;
/// use ruff_formatter::{Buffer, FormatState, SimpleFormatContext, VecBuffer, write};
///
/// # fn main() -> FormatResult<()> {
/// let mut state = FormatState::new(SimpleFormatContext::default());
/// let mut buffer = VecBuffer::new(&mut state);
/// write!(&mut buffer, [text("Hello"), space()])?;
/// write!(&mut buffer, [text("World")])?;
///
/// assert_eq!(
/// buffer.into_vec(),
/// vec![
/// FormatElement::StaticText { text: "Hello" },
/// FormatElement::Space,
/// FormatElement::StaticText { text: "World" },
/// ]
/// );
/// # Ok(())
/// # }
/// ```
#[macro_export]
macro_rules! write {
($dst:expr, [$($arg:expr),+ $(,)?]) => {{
let result = $dst.write_fmt($crate::format_args!($($arg),+));
result
}}
}
/// Writes formatted data into the given buffer and prints all written elements for a quick and dirty debugging.
///
/// An example:
///
/// ```rust
/// use ruff_formatter::prelude::*;
/// use ruff_formatter::{FormatState, VecBuffer};
///
/// # fn main() -> FormatResult<()> {
/// let mut state = FormatState::new(SimpleFormatContext::default());
/// let mut buffer = VecBuffer::new(&mut state);
///
/// dbg_write!(buffer, [text("Hello")])?;
/// // ^-- prints: [src/main.rs:7][0] = StaticToken("Hello")
///
/// assert_eq!(buffer.into_vec(), vec![FormatElement::StaticText { text: "Hello" }]);
/// # Ok(())
/// # }
/// ```
///
/// Note that the macro is intended as debugging tool and therefore you should avoid having
/// uses of it in version control for long periods (other than in tests and similar). Format output
/// from production code is better done with `[write!]`
#[macro_export]
macro_rules! dbg_write {
($dst:expr, [$($arg:expr),+ $(,)?]) => {{
use $crate::BufferExtensions;
let mut count = 0;
let mut inspect = $dst.inspect(|element: &FormatElement| {
std::eprintln!(
"[{}:{}][{}] = {element:#?}",
std::file!(), std::line!(), count
);
count += 1;
});
let result = inspect.write_fmt($crate::format_args!($($arg),+));
result
}}
}
/// Creates the Format IR for a value.
///
/// The first argument `format!` receives is the [crate::FormatContext] that specify how elements must be formatted.
/// Additional parameters passed get formatted by using their [crate::Format] implementation.
///
///
/// ## Examples
///
/// ```
/// use ruff_formatter::prelude::*;
/// use ruff_formatter::format;
///
/// let formatted = format!(SimpleFormatContext::default(), [text("("), text("a"), text(")")]).unwrap();
///
/// assert_eq!(
/// formatted.into_document(),
/// Document::from(vec![
/// FormatElement::StaticText { text: "(" },
/// FormatElement::StaticText { text: "a" },
/// FormatElement::StaticText { text: ")" },
/// ])
/// );
/// ```
#[macro_export]
macro_rules! format {
($context:expr, [$($arg:expr),+ $(,)?]) => {{
($crate::format($context, $crate::format_args!($($arg),+)))
}}
}
/// Provides multiple different alternatives and the printer picks the first one that fits.
/// Use this as last resort because it requires that the printer must try all variants in the worst case.
/// The passed variants must be in the following order:
/// * First: The variant that takes up most space horizontally
/// * Last: The variant that takes up the least space horizontally by splitting the content over multiple lines.
///
/// ## Examples
///
/// ```
/// use ruff_formatter::{Formatted, LineWidth, format, format_args, SimpleFormatOptions};
/// use ruff_formatter::prelude::*;
///
/// # fn main() -> FormatResult<()> {
/// let formatted = format!(
/// SimpleFormatContext::default(),
/// [
/// text("aVeryLongIdentifier"),
/// best_fitting!(
/// // Everything fits on a single line
/// format_args!(
/// text("("),
/// group(&format_args![
/// text("["),
/// soft_block_indent(&format_args![
/// text("1,"),
/// soft_line_break_or_space(),
/// text("2,"),
/// soft_line_break_or_space(),
/// text("3"),
/// ]),
/// text("]")
/// ]),
/// text(")")
/// ),
///
/// // Breaks after `[`, but prints all elements on a single line
/// format_args!(
/// text("("),
/// text("["),
/// block_indent(&text("1, 2, 3")),
/// text("]"),
/// text(")"),
/// ),
///
/// // Breaks after `[` and prints each element on a single line
/// format_args!(
/// text("("),
/// block_indent(&format_args![
/// text("["),
/// block_indent(&format_args![
/// text("1,"),
/// hard_line_break(),
/// text("2,"),
/// hard_line_break(),
/// text("3"),
/// ]),
/// text("]"),
/// ]),
/// text(")")
/// )
/// )
/// ]
/// )?;
///
/// let document = formatted.into_document();
///
/// // Takes the first variant if everything fits on a single line
/// assert_eq!(
/// "aVeryLongIdentifier([1, 2, 3])",
/// Formatted::new(document.clone(), SimpleFormatContext::default())
/// .print()?
/// .as_code()
/// );
///
/// // It takes the second if the first variant doesn't fit on a single line. The second variant
/// // has some additional line breaks to make sure inner groups don't break
/// assert_eq!(
/// "aVeryLongIdentifier([\n\t1, 2, 3\n])",
/// Formatted::new(document.clone(), SimpleFormatContext::new(SimpleFormatOptions { line_width: 21.try_into().unwrap(), ..SimpleFormatOptions::default() }))
/// .print()?
/// .as_code()
/// );
///
/// // Prints the last option as last resort
/// assert_eq!(
/// "aVeryLongIdentifier(\n\t[\n\t\t1,\n\t\t2,\n\t\t3\n\t]\n)",
/// Formatted::new(document.clone(), SimpleFormatContext::new(SimpleFormatOptions { line_width: 20.try_into().unwrap(), ..SimpleFormatOptions::default() }))
/// .print()?
/// .as_code()
/// );
/// # Ok(())
/// # }
/// ```
///
/// ### Enclosing group with `should_expand: true`
///
/// ```
/// use ruff_formatter::{Formatted, LineWidth, format, format_args, SimpleFormatOptions};
/// use ruff_formatter::prelude::*;
///
/// # fn main() -> FormatResult<()> {
/// let formatted = format!(
/// SimpleFormatContext::default(),
/// [
/// best_fitting!(
/// // Prints the method call on the line but breaks the array.
/// format_args!(
/// text("expect(a).toMatch("),
/// group(&format_args![
/// text("["),
/// soft_block_indent(&format_args![
/// text("1,"),
/// soft_line_break_or_space(),
/// text("2,"),
/// soft_line_break_or_space(),
/// text("3"),
/// ]),
/// text("]")
/// ]).should_expand(true),
/// text(")")
/// ),
///
/// // Breaks after `(`
/// format_args!(
/// text("expect(a).toMatch("),
/// group(&soft_block_indent(
/// &group(&format_args![
/// text("["),
/// soft_block_indent(&format_args![
/// text("1,"),
/// soft_line_break_or_space(),
/// text("2,"),
/// soft_line_break_or_space(),
/// text("3"),
/// ]),
/// text("]")
/// ]).should_expand(true),
/// )).should_expand(true),
/// text(")")
/// ),
/// )
/// ]
/// )?;
///
/// let document = formatted.into_document();
///
/// assert_eq!(
/// "expect(a).toMatch([\n\t1,\n\t2,\n\t3\n])",
/// Formatted::new(document.clone(), SimpleFormatContext::default())
/// .print()?
/// .as_code()
/// );
///
/// # Ok(())
/// # }
/// ```
///
/// The first variant fits because all its content up to the first line break fit on the line without exceeding
/// the configured print width.
///
/// ## Complexity
/// Be mindful of using this IR element as it has a considerable performance penalty:
/// * There are multiple representation for the same content. This results in increased memory usage
/// and traversal time in the printer.
/// * The worst case complexity is that the printer tires each variant. This can result in quadratic
/// complexity if used in nested structures.
///
/// ## Behavior
/// This IR is similar to Prettier's `conditionalGroup`. The printer measures each variant, except the [`MostExpanded`], in [`Flat`] mode
/// to find the first variant that fits and prints this variant in [`Flat`] mode. If no variant fits, then
/// the printer falls back to printing the [`MostExpanded`] variant in `[`Expanded`] mode.
///
/// The definition of *fits* differs to groups in that the printer only tests if it is possible to print
/// the content up to the first non-soft line break without exceeding the configured print width.
/// This definition differs from groups as that non-soft line breaks make group expand.
///
/// [crate::BestFitting] acts as a "break" boundary, meaning that it is considered to fit
///
///
/// [`Flat`]: crate::format_element::PrintMode::Flat
/// [`Expanded`]: crate::format_element::PrintMode::Expanded
/// [`MostExpanded`]: crate::format_element::BestFitting::most_expanded
#[macro_export]
macro_rules! best_fitting {
($least_expanded:expr, $($tail:expr),+ $(,)?) => {{
unsafe {
$crate::BestFitting::from_arguments_unchecked($crate::format_args!($least_expanded, $($tail),+))
}
}}
}
#[cfg(test)]
mod tests {
use crate::prelude::*;
use crate::{write, FormatState, SimpleFormatOptions, VecBuffer};
struct TestFormat;
impl Format<()> for TestFormat {
fn fmt(&self, f: &mut Formatter<()>) -> FormatResult<()> {
write!(f, [text("test")])
}
}
#[test]
fn test_single_element() {
let mut state = FormatState::new(());
let mut buffer = VecBuffer::new(&mut state);
write![&mut buffer, [TestFormat]].unwrap();
assert_eq!(
buffer.into_vec(),
vec![FormatElement::StaticText { text: "test" }]
);
}
#[test]
fn test_multiple_elements() {
let mut state = FormatState::new(());
let mut buffer = VecBuffer::new(&mut state);
write![
&mut buffer,
[text("a"), space(), text("simple"), space(), TestFormat]
]
.unwrap();
assert_eq!(
buffer.into_vec(),
vec![
FormatElement::StaticText { text: "a" },
FormatElement::Space,
FormatElement::StaticText { text: "simple" },
FormatElement::Space,
FormatElement::StaticText { text: "test" }
]
);
}
#[test]
fn best_fitting_variants_print_as_lists() {
use crate::prelude::*;
use crate::{format, format_args, Formatted};
// The second variant below should be selected when printing at a width of 30
let formatted_best_fitting = format!(
SimpleFormatContext::default(),
[
text("aVeryLongIdentifier"),
soft_line_break_or_space(),
best_fitting![
format_args![text(
"Something that will not fit on a line with 30 character print width."
)],
format_args![group(&format_args![
text("Start"),
soft_line_break(),
group(&soft_block_indent(&format_args![
text("1,"),
soft_line_break_or_space(),
text("2,"),
soft_line_break_or_space(),
text("3"),
])),
soft_line_break_or_space(),
soft_block_indent(&format_args![
text("1,"),
soft_line_break_or_space(),
text("2,"),
soft_line_break_or_space(),
group(&format_args!(
text("A,"),
soft_line_break_or_space(),
text("B")
)),
soft_line_break_or_space(),
text("3")
]),
soft_line_break_or_space(),
text("End")
])
.should_expand(true)],
format_args!(text("Most"), hard_line_break(), text("Expanded"))
]
]
)
.unwrap();
// This matches the IR above except that the `best_fitting` was replaced with
// the contents of its second variant.
let formatted_normal_list = format!(
SimpleFormatContext::default(),
[
text("aVeryLongIdentifier"),
soft_line_break_or_space(),
format_args![
text("Start"),
soft_line_break(),
&group(&soft_block_indent(&format_args![
text("1,"),
soft_line_break_or_space(),
text("2,"),
soft_line_break_or_space(),
text("3"),
])),
soft_line_break_or_space(),
&soft_block_indent(&format_args![
text("1,"),
soft_line_break_or_space(),
text("2,"),
soft_line_break_or_space(),
group(&format_args!(
text("A,"),
soft_line_break_or_space(),
text("B")
)),
soft_line_break_or_space(),
text("3")
]),
soft_line_break_or_space(),
text("End")
],
]
)
.unwrap();
let best_fitting_code = Formatted::new(
formatted_best_fitting.into_document(),
SimpleFormatContext::new(SimpleFormatOptions {
line_width: 30.try_into().unwrap(),
..SimpleFormatOptions::default()
}),
)
.print()
.expect("Document to be valid")
.as_code()
.to_string();
let normal_list_code = Formatted::new(
formatted_normal_list.into_document(),
SimpleFormatContext::new(SimpleFormatOptions {
line_width: 30.try_into().unwrap(),
..SimpleFormatOptions::default()
}),
)
.print()
.expect("Document to be valid")
.as_code()
.to_string();
// The variant that "fits" will print its contents as if it were a normal list
// outside of a BestFitting element.
assert_eq!(best_fitting_code, normal_list_code);
}
}

21
crates/ruff_formatter/src/prelude.rs Normal file
View File

@ -0,0 +1,21 @@
pub use crate::builders::*;
pub use crate::format_element::*;
pub use crate::format_extensions::{MemoizeFormat, Memoized};
pub use crate::formatter::Formatter;
pub use crate::printer::PrinterOptions;
pub use crate::trivia::{
format_dangling_comments, format_leading_comments, format_only_if_breaks, format_removed,
format_replaced, format_trailing_comments, format_trimmed_token,
};
pub use crate::diagnostics::FormatError;
pub use crate::format_element::document::Document;
pub use crate::format_element::tag::{LabelId, Tag, TagKind};
pub use crate::verbatim::{
format_bogus_node, format_or_verbatim, format_suppressed_node, format_verbatim_node,
};
pub use crate::{
best_fitting, dbg_write, format, format_args, write, Buffer as _, BufferExtensions, Format,
Format as _, FormatResult, FormatRule, FormatWithRule as _, SimpleFormatContext,
};

81
crates/ruff_formatter/src/printed_tokens.rs Normal file
View File

@ -0,0 +1,81 @@
use indexmap::IndexSet;
use ruff_rowan::{Direction, Language, SyntaxNode, SyntaxToken, TextSize};
/// Tracks the ranges of the formatted (including replaced or tokens formatted as verbatim) tokens.
///
/// This implementation uses the fact that no two tokens can have an overlapping range to avoid the need for an interval tree.
/// Thus, testing if a token has already been formatted only requires testing if a token starting at the same offset has been formatted.
#[derive(Debug, Clone, Default)]
pub struct PrintedTokens {
/// Key: Start of a token's range
offsets: IndexSet<TextSize>,
disabled: bool,
}
#[derive(Copy, Clone)]
pub struct PrintedTokensSnapshot {
len: usize,
disabled: bool,
}
impl PrintedTokens {
/// Tracks a formatted token
///
/// ## Panics
/// If this token has been formatted before.
pub fn track_token<L: Language>(&mut self, token: &SyntaxToken<L>) {
if self.disabled {
return;
}
let range = token.text_trimmed_range();
if !self.offsets.insert(range.start()) {
panic!("You tried to print the token '{token:?}' twice, and this is not valid.");
}
}
/// Enables or disables the assertion tracking
pub(crate) fn set_disabled(&mut self, disabled: bool) {
self.disabled = disabled;
}
pub(crate) fn is_disabled(&self) -> bool {
self.disabled
}
pub(crate) fn snapshot(&self) -> PrintedTokensSnapshot {
PrintedTokensSnapshot {
len: self.offsets.len(),
disabled: self.disabled,
}
}
pub(crate) fn restore(&mut self, snapshot: PrintedTokensSnapshot) {
let PrintedTokensSnapshot { len, disabled } = snapshot;
self.offsets.truncate(len);
self.disabled = disabled
}
/// Asserts that all tokens of the passed in node have been tracked
///
/// ## Panics
/// If any descendant token of `root` hasn't been tracked
pub fn assert_all_tracked<L: Language>(&self, root: &SyntaxNode<L>) {
let mut offsets = self.offsets.clone();
for token in root.descendants_tokens(Direction::Next) {
if !offsets.remove(&token.text_trimmed_range().start()) {
panic!("token has not been seen by the formatter: {token:#?}.\
\nUse `format_replaced` if you want to replace a token from the formatted output.\
\nUse `format_removed` if you want to remove a token from the formatted output.\n\
parent: {:#?}", token.parent())
}
}
for offset in offsets {
panic!("tracked offset {offset:?} doesn't match any token of {root:#?}. Have you passed a token from another tree?");
}
}
}

235
crates/ruff_formatter/src/printer/call_stack.rs Normal file
View File

@ -0,0 +1,235 @@
use crate::format_element::tag::TagKind;
use crate::format_element::PrintMode;
use crate::printer::stack::{Stack, StackedStack};
use crate::printer::Indention;
use crate::{IndentStyle, InvalidDocumentError, PrintError, PrintResult};
use std::fmt::Debug;
use std::num::NonZeroU8;
#[derive(Copy, Clone, Eq, PartialEq, Debug)]
pub(super) enum StackFrameKind {
Root,
Tag(TagKind),
}
#[derive(Copy, Clone, Eq, PartialEq, Debug)]
pub(super) struct StackFrame {
kind: StackFrameKind,
args: PrintElementArgs,
}
/// Stores arguments passed to `print_element` call, holding the state specific to printing an element.
/// E.g. the `indent` depends on the token the Printer's currently processing. That's why
/// it must be stored outside of the [PrinterState] that stores the state common to all elements.
///
/// The state is passed by value, which is why it's important that it isn't storing any heavy
/// data structures. Such structures should be stored on the [PrinterState] instead.
#[derive(Debug, Copy, Clone, Eq, PartialEq)]
pub(super) struct PrintElementArgs {
indent: Indention,
mode: PrintMode,
}
impl PrintElementArgs {
pub fn new(indent: Indention) -> Self {
Self {
indent,
..Self::default()
}
}
pub(super) fn mode(&self) -> PrintMode {
self.mode
}
pub(super) fn indention(&self) -> Indention {
self.indent
}
pub fn increment_indent_level(mut self, indent_style: IndentStyle) -> Self {
self.indent = self.indent.increment_level(indent_style);
self
}
pub fn decrement_indent(mut self) -> Self {
self.indent = self.indent.decrement();
self
}
pub fn reset_indent(mut self) -> Self {
self.indent = Indention::default();
self
}
pub fn set_indent_align(mut self, count: NonZeroU8) -> Self {
self.indent = self.indent.set_align(count);
self
}
pub fn with_print_mode(mut self, mode: PrintMode) -> Self {
self.mode = mode;
self
}
}
impl Default for PrintElementArgs {
fn default() -> Self {
Self {
indent: Indention::Level(0),
mode: PrintMode::Expanded,
}
}
}
/// Call stack that stores the [PrintElementCallArgs].
///
/// New [PrintElementCallArgs] are pushed onto the stack for every [`start`](Tag::is_start) [`Tag`](FormatElement::Tag)
/// and popped when reaching the corresponding [`end`](Tag::is_end) [`Tag`](FormatElement::Tag).
pub(super) trait CallStack {
type Stack: Stack<StackFrame> + Debug;
fn stack(&self) -> &Self::Stack;
fn stack_mut(&mut self) -> &mut Self::Stack;
/// Pops the call arguments at the top and asserts that they correspond to a start tag of `kind`.
///
/// Returns `Ok` with the arguments if the kind of the top stack frame matches `kind`, otherwise
/// returns `Err`.
fn pop(&mut self, kind: TagKind) -> PrintResult<PrintElementArgs> {
let last = self.stack_mut().pop();
match last {
Some(StackFrame {
kind: StackFrameKind::Tag(actual_kind),
args,
}) if actual_kind == kind => Ok(args),
// Start / End kind don't match
Some(StackFrame {
kind: StackFrameKind::Tag(expected_kind),
..
}) => Err(PrintError::InvalidDocument(Self::invalid_document_error(
kind,
Some(expected_kind),
))),
// Tried to pop the outer most stack frame, which is not valid
Some(
frame @ StackFrame {
kind: StackFrameKind::Root,
..
},
) => {
// Put it back in to guarantee that the stack is never empty
self.stack_mut().push(frame);
Err(PrintError::InvalidDocument(Self::invalid_document_error(
kind, None,
)))
}
// This should be unreachable but having it for completeness. Happens if the stack is empty.
None => Err(PrintError::InvalidDocument(Self::invalid_document_error(
kind, None,
))),
}
}
#[cold]
fn invalid_document_error(
end_kind: TagKind,
start_kind: Option<TagKind>,
) -> InvalidDocumentError {
match start_kind {
None => InvalidDocumentError::StartTagMissing { kind: end_kind },
Some(start_kind) => InvalidDocumentError::StartEndTagMismatch {
start_kind,
end_kind,
},
}
}
/// Returns the [PrintElementArgs] for the current stack frame.
fn top(&self) -> PrintElementArgs {
self.stack()
.top()
.expect("Expected `stack` to never be empty.")
.args
}
/// Returns the [TagKind] of the current stack frame or [None] if this is the root stack frame.
fn top_kind(&self) -> Option<TagKind> {
match self
.stack()
.top()
.expect("Expected `stack` to never be empty.")
.kind
{
StackFrameKind::Root => None,
StackFrameKind::Tag(kind) => Some(kind),
}
}
/// Creates a new stack frame for a [FormatElement::Tag] of `kind` with `args` as the call arguments.
fn push(&mut self, kind: TagKind, args: PrintElementArgs) {
self.stack_mut().push(StackFrame {
kind: StackFrameKind::Tag(kind),
args,
})
}
}
/// Call stack used for printing the [FormatElement]s
#[derive(Debug, Clone)]
pub(super) struct PrintCallStack(Vec<StackFrame>);
impl PrintCallStack {
pub(super) fn new(args: PrintElementArgs) -> Self {
Self(vec![StackFrame {
kind: StackFrameKind::Root,
args,
}])
}
}
impl CallStack for PrintCallStack {
type Stack = Vec<StackFrame>;
fn stack(&self) -> &Self::Stack {
&self.0
}
fn stack_mut(&mut self) -> &mut Self::Stack {
&mut self.0
}
}
/// Call stack used for measuring if some content fits on the line.
///
/// The stack is a view on top of the [PrintCallStack] because the stack frames are still necessary for printing.
#[must_use]
pub(super) struct FitsCallStack<'print> {
stack: StackedStack<'print, StackFrame>,
}
impl<'print> FitsCallStack<'print> {
pub(super) fn new(print: &'print PrintCallStack, saved: Vec<StackFrame>) -> Self {
let stack = StackedStack::with_vec(&print.0, saved);
Self { stack }
}
pub(super) fn finish(self) -> Vec<StackFrame> {
self.stack.into_vec()
}
}
impl<'a> CallStack for FitsCallStack<'a> {
type Stack = StackedStack<'a, StackFrame>;
fn stack(&self) -> &Self::Stack {
&self.stack
}
fn stack_mut(&mut self) -> &mut Self::Stack {
&mut self.stack
}
}

42
crates/ruff_formatter/src/printer/line_suffixes.rs Normal file
View File

@ -0,0 +1,42 @@
use crate::printer::call_stack::PrintElementArgs;
use crate::FormatElement;
/// Stores the queued line suffixes.
#[derive(Debug, Default)]
pub(super) struct LineSuffixes<'a> {
suffixes: Vec<LineSuffixEntry<'a>>,
}
impl<'a> LineSuffixes<'a> {
/// Extends the line suffixes with `elements`, storing their call stack arguments with them.
pub(super) fn extend<I>(&mut self, args: PrintElementArgs, elements: I)
where
I: IntoIterator<Item = &'a FormatElement>,
{
self.suffixes
.extend(elements.into_iter().map(LineSuffixEntry::Suffix));
self.suffixes.push(LineSuffixEntry::Args(args));
}
/// Takes all the pending line suffixes.
pub(super) fn take_pending<'l>(
&'l mut self,
) -> impl Iterator<Item = LineSuffixEntry<'a>> + DoubleEndedIterator + 'l + ExactSizeIterator
{
self.suffixes.drain(..)
}
/// Returns `true` if there are any line suffixes and `false` otherwise.
pub(super) fn has_pending(&self) -> bool {
!self.suffixes.is_empty()
}
}
#[derive(Debug, Copy, Clone)]
pub(super) enum LineSuffixEntry<'a> {
/// A line suffix to print
Suffix(&'a FormatElement),
/// Potentially changed call arguments that should be used to format any following items.
Args(PrintElementArgs),
}

1649
crates/ruff_formatter/src/printer/mod.rs Normal file
View File

File diff suppressed because it is too large Load Diff

115
crates/ruff_formatter/src/printer/printer_options/mod.rs Normal file
View File

@ -0,0 +1,115 @@
use crate::{FormatOptions, IndentStyle, LineWidth};
/// Options that affect how the [crate::Printer] prints the format tokens
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct PrinterOptions {
/// Width of a single tab character (does it equal 2, 4, ... spaces?)
pub tab_width: u8,
/// What's the max width of a line. Defaults to 80
pub print_width: PrintWidth,
/// The type of line ending to apply to the printed input
pub line_ending: LineEnding,
/// Whether the printer should use tabs or spaces to indent code and if spaces, by how many.
pub indent_style: IndentStyle,
}
#[derive(Debug, Copy, Clone, Eq, PartialEq)]
pub struct PrintWidth(u32);
impl PrintWidth {
pub fn new(width: u32) -> Self {
Self(width)
}
}
impl Default for PrintWidth {
fn default() -> Self {
LineWidth::default().into()
}
}
impl From<LineWidth> for PrintWidth {
fn from(width: LineWidth) -> Self {
Self(u16::from(width) as u32)
}
}
impl From<PrintWidth> for usize {
fn from(width: PrintWidth) -> Self {
width.0 as usize
}
}
impl<'a, O> From<&'a O> for PrinterOptions
where
O: FormatOptions,
{
fn from(options: &'a O) -> Self {
PrinterOptions::default()
.with_indent(options.indent_style())
.with_print_width(options.line_width().into())
}
}
impl PrinterOptions {
pub fn with_print_width(mut self, width: PrintWidth) -> Self {
self.print_width = width;
self
}
pub fn with_indent(mut self, style: IndentStyle) -> Self {
self.indent_style = style;
self
}
pub(crate) fn indent_style(&self) -> IndentStyle {
self.indent_style
}
/// Width of an indent in characters.
pub(super) const fn indent_width(&self) -> u8 {
match self.indent_style {
IndentStyle::Tab => self.tab_width,
IndentStyle::Space(count) => count,
}
}
}
#[allow(dead_code)]
#[derive(Clone, Debug, Eq, PartialEq)]
pub enum LineEnding {
/// Line Feed only (\n), common on Linux and macOS as well as inside git repos
LineFeed,
/// Carriage Return + Line Feed characters (\r\n), common on Windows
CarriageReturnLineFeed,
/// Carriage Return character only (\r), used very rarely
CarriageReturn,
}
impl LineEnding {
#[inline]
pub const fn as_str(&self) -> &'static str {
match self {
LineEnding::LineFeed => "\n",
LineEnding::CarriageReturnLineFeed => "\r\n",
LineEnding::CarriageReturn => "\r",
}
}
}
impl Default for PrinterOptions {
fn default() -> Self {
PrinterOptions {
tab_width: 2,
print_width: PrintWidth::default(),
indent_style: Default::default(),
line_ending: LineEnding::LineFeed,
}
}
}

404
crates/ruff_formatter/src/printer/queue.rs Normal file
View File

@ -0,0 +1,404 @@
use crate::format_element::tag::TagKind;
use crate::prelude::Tag;
use crate::printer::stack::{Stack, StackedStack};
use crate::printer::{invalid_end_tag, invalid_start_tag};
use crate::{FormatElement, PrintResult};
use std::fmt::Debug;
use std::iter::FusedIterator;
use std::marker::PhantomData;
/// Queue of [FormatElement]s.
pub(super) trait Queue<'a> {
type Stack: Stack<&'a [FormatElement]>;
fn stack(&self) -> &Self::Stack;
fn stack_mut(&mut self) -> &mut Self::Stack;
fn next_index(&self) -> usize;
fn set_next_index(&mut self, index: usize);
/// Pops the element at the end of the queue.
fn pop(&mut self) -> Option<&'a FormatElement> {
match self.stack().top() {
Some(top_slice) => {
// SAFETY: Safe because queue ensures that slices inside `slices` are never empty.
let next_index = self.next_index();
let element = &top_slice[next_index];
if next_index + 1 == top_slice.len() {
self.stack_mut().pop().unwrap();
self.set_next_index(0);
} else {
self.set_next_index(next_index + 1);
}
Some(element)
}
None => None,
}
}
/// Returns the next element, not traversing into [FormatElement::Interned].
fn top_with_interned(&self) -> Option<&'a FormatElement> {
self.stack()
.top()
.map(|top_slice| &top_slice[self.next_index()])
}
/// Returns the next element, recursively resolving the first element of [FormatElement::Interned].
fn top(&self) -> Option<&'a FormatElement> {
let mut top = self.top_with_interned();
while let Some(FormatElement::Interned(interned)) = top {
top = interned.first()
}
top
}
/// Queues a single element to process before the other elements in this queue.
fn push(&mut self, element: &'a FormatElement) {
self.extend_back(std::slice::from_ref(element))
}
/// Queues a slice of elements to process before the other elements in this queue.
fn extend_back(&mut self, elements: &'a [FormatElement]) {
match elements {
[] => {
// Don't push empty slices
}
slice => {
let next_index = self.next_index();
let stack = self.stack_mut();
if let Some(top) = stack.pop() {
stack.push(&top[next_index..])
}
stack.push(slice);
self.set_next_index(0);
}
}
}
/// Removes top slice.
fn pop_slice(&mut self) -> Option<&'a [FormatElement]> {
self.set_next_index(0);
self.stack_mut().pop()
}
/// Skips all content until it finds the corresponding end tag with the given kind.
fn skip_content(&mut self, kind: TagKind)
where
Self: Sized,
{
let iter = self.iter_content(kind);
for _ in iter {
// consume whole iterator until end
}
}
/// Iterates over all elements until it finds the matching end tag of the specified kind.
fn iter_content<'q>(&'q mut self, kind: TagKind) -> QueueContentIterator<'a, 'q, Self>
where
Self: Sized,
{
QueueContentIterator::new(self, kind)
}
}
/// Queue with the elements to print.
#[derive(Debug, Default, Clone)]
pub(super) struct PrintQueue<'a> {
slices: Vec<&'a [FormatElement]>,
next_index: usize,
}
impl<'a> PrintQueue<'a> {
pub(super) fn new(slice: &'a [FormatElement]) -> Self {
let slices = match slice {
[] => Vec::default(),
slice => vec![slice],
};
Self {
slices,
next_index: 0,
}
}
pub(super) fn is_empty(&self) -> bool {
self.slices.is_empty()
}
}
impl<'a> Queue<'a> for PrintQueue<'a> {
type Stack = Vec<&'a [FormatElement]>;
fn stack(&self) -> &Self::Stack {
&self.slices
}
fn stack_mut(&mut self) -> &mut Self::Stack {
&mut self.slices
}
fn next_index(&self) -> usize {
self.next_index
}
fn set_next_index(&mut self, index: usize) {
self.next_index = index
}
}
/// Queue for measuring if an element fits on the line.
///
/// The queue is a view on top of the [PrintQueue] because no elements should be removed
/// from the [PrintQueue] while measuring.
#[must_use]
#[derive(Debug)]
pub(super) struct FitsQueue<'a, 'print> {
stack: StackedStack<'print, &'a [FormatElement]>,
next_index: usize,
}
impl<'a, 'print> FitsQueue<'a, 'print> {
pub(super) fn new(
print_queue: &'print PrintQueue<'a>,
saved: Vec<&'a [FormatElement]>,
) -> Self {
let stack = StackedStack::with_vec(&print_queue.slices, saved);
Self {
stack,
next_index: print_queue.next_index,
}
}
pub(super) fn finish(self) -> Vec<&'a [FormatElement]> {
self.stack.into_vec()
}
}
impl<'a, 'print> Queue<'a> for FitsQueue<'a, 'print> {
type Stack = StackedStack<'print, &'a [FormatElement]>;
fn stack(&self) -> &Self::Stack {
&self.stack
}
fn stack_mut(&mut self) -> &mut Self::Stack {
&mut self.stack
}
fn next_index(&self) -> usize {
self.next_index
}
fn set_next_index(&mut self, index: usize) {
self.next_index = index;
}
}
/// Iterator that calls [Queue::pop] until it reaches the end of the document.
///
/// The iterator traverses into the content of any [FormatElement::Interned].
pub(super) struct QueueIterator<'a, 'q, Q: Queue<'a>> {
queue: &'q mut Q,
lifetime: PhantomData<&'a ()>,
}
impl<'a, Q> Iterator for QueueIterator<'a, '_, Q>
where
Q: Queue<'a>,
{
type Item = &'a FormatElement;
fn next(&mut self) -> Option<Self::Item> {
self.queue.pop()
}
}
impl<'a, Q> FusedIterator for QueueIterator<'a, '_, Q> where Q: Queue<'a> {}
pub(super) struct QueueContentIterator<'a, 'q, Q: Queue<'a>> {
queue: &'q mut Q,
kind: TagKind,
depth: usize,
lifetime: PhantomData<&'a ()>,
}
impl<'a, 'q, Q> QueueContentIterator<'a, 'q, Q>
where
Q: Queue<'a>,
{
fn new(queue: &'q mut Q, kind: TagKind) -> Self {
Self {
queue,
kind,
depth: 1,
lifetime: PhantomData,
}
}
}
impl<'a, Q> Iterator for QueueContentIterator<'a, '_, Q>
where
Q: Queue<'a>,
{
type Item = &'a FormatElement;
fn next(&mut self) -> Option<Self::Item> {
match self.depth {
0 => None,
_ => {
let mut top = self.queue.pop();
while let Some(FormatElement::Interned(interned)) = top {
self.queue.extend_back(interned);
top = self.queue.pop();
}
match top.expect("Missing end signal.") {
element @ FormatElement::Tag(tag) if tag.kind() == self.kind => {
if tag.is_start() {
self.depth += 1;
} else {
self.depth -= 1;
if self.depth == 0 {
return None;
}
}
Some(element)
}
element => Some(element),
}
}
}
}
}
impl<'a, Q> FusedIterator for QueueContentIterator<'a, '_, Q> where Q: Queue<'a> {}
/// A predicate determining when to end measuring if some content fits on the line.
///
/// Called for every [`element`](FormatElement) in the [FitsQueue] when measuring if a content
/// fits on the line. The measuring of the content ends after the first element [`element`](FormatElement) for which this
/// predicate returns `true` (similar to a take while iterator except that it takes while the predicate returns `false`).
pub(super) trait FitsEndPredicate {
fn is_end(&mut self, element: &FormatElement) -> PrintResult<bool>;
}
/// Filter that includes all elements until it reaches the end of the document.
pub(super) struct AllPredicate;
impl FitsEndPredicate for AllPredicate {
fn is_end(&mut self, _element: &FormatElement) -> PrintResult<bool> {
Ok(false)
}
}
/// Filter that takes all elements between two matching [Tag::StartEntry] and [Tag::EndEntry] tags.
#[derive(Debug)]
pub(super) enum SingleEntryPredicate {
Entry { depth: usize },
Done,
}
impl SingleEntryPredicate {
pub(super) const fn is_done(&self) -> bool {
matches!(self, SingleEntryPredicate::Done)
}
}
impl Default for SingleEntryPredicate {
fn default() -> Self {
SingleEntryPredicate::Entry { depth: 0 }
}
}
impl FitsEndPredicate for SingleEntryPredicate {
fn is_end(&mut self, element: &FormatElement) -> PrintResult<bool> {
let result = match self {
SingleEntryPredicate::Done => true,
SingleEntryPredicate::Entry { depth } => match element {
FormatElement::Tag(Tag::StartEntry) => {
*depth += 1;
false
}
FormatElement::Tag(Tag::EndEntry) => {
if *depth == 0 {
return invalid_end_tag(TagKind::Entry, None);
}
*depth -= 1;
let is_end = *depth == 0;
if is_end {
*self = SingleEntryPredicate::Done;
}
is_end
}
FormatElement::Interned(_) => false,
element if *depth == 0 => {
return invalid_start_tag(TagKind::Entry, Some(element));
}
_ => false,
},
};
Ok(result)
}
}
#[cfg(test)]
mod tests {
use crate::format_element::LineMode;
use crate::prelude::Tag;
use crate::printer::queue::{PrintQueue, Queue};
use crate::FormatElement;
#[test]
fn extend_back_pop_last() {
let mut queue =
PrintQueue::new(&[FormatElement::Tag(Tag::StartEntry), FormatElement::Space]);
assert_eq!(queue.pop(), Some(&FormatElement::Tag(Tag::StartEntry)));
queue.extend_back(&[FormatElement::Line(LineMode::SoftOrSpace)]);
assert_eq!(
queue.pop(),
Some(&FormatElement::Line(LineMode::SoftOrSpace))
);
assert_eq!(queue.pop(), Some(&FormatElement::Space));
assert_eq!(queue.pop(), None);
}
#[test]
fn extend_back_empty_queue() {
let mut queue =
PrintQueue::new(&[FormatElement::Tag(Tag::StartEntry), FormatElement::Space]);
assert_eq!(queue.pop(), Some(&FormatElement::Tag(Tag::StartEntry)));
assert_eq!(queue.pop(), Some(&FormatElement::Space));
queue.extend_back(&[FormatElement::Line(LineMode::SoftOrSpace)]);
assert_eq!(
queue.pop(),
Some(&FormatElement::Line(LineMode::SoftOrSpace))
);
assert_eq!(queue.pop(), None);
}
}

141
crates/ruff_formatter/src/printer/stack.rs Normal file
View File

@ -0,0 +1,141 @@
/// A school book stack. Allows adding, removing, and inspecting elements at the back.
pub(super) trait Stack<T> {
/// Removes the last element if any and returns it
fn pop(&mut self) -> Option<T>;
/// Pushes a new element at the back
fn push(&mut self, value: T);
/// Returns the last element if any
fn top(&self) -> Option<&T>;
/// Returns `true` if the stack is empty
fn is_empty(&self) -> bool;
}
impl<T> Stack<T> for Vec<T> {
fn pop(&mut self) -> Option<T> {
self.pop()
}
fn push(&mut self, value: T) {
self.push(value)
}
fn top(&self) -> Option<&T> {
self.last()
}
fn is_empty(&self) -> bool {
self.is_empty()
}
}
/// A Stack that is stacked on top of another stack. Guarantees that the underlying stack remains unchanged.
#[derive(Debug, Clone)]
pub(super) struct StackedStack<'a, T> {
/// The content of the original stack.
original: &'a [T],
/// Items that have been pushed since the creation of this stack and aren't part of the `original` stack.
stack: Vec<T>,
}
impl<'a, T> StackedStack<'a, T> {
#[cfg(test)]
pub(super) fn new(original: &'a [T]) -> Self {
Self::with_vec(original, Vec::new())
}
/// Creates a new stack that uses `stack` for storing its elements.
pub(super) fn with_vec(original: &'a [T], stack: Vec<T>) -> Self {
Self { original, stack }
}
/// Returns the underlying `stack` vector.
pub(super) fn into_vec(self) -> Vec<T> {
self.stack
}
}
impl<T> Stack<T> for StackedStack<'_, T>
where
T: Copy,
{
fn pop(&mut self) -> Option<T> {
self.stack.pop().or_else(|| match self.original {
[rest @ .., last] => {
self.original = rest;
Some(*last)
}
_ => None,
})
}
fn push(&mut self, value: T) {
self.stack.push(value);
}
fn top(&self) -> Option<&T> {
self.stack.last().or_else(|| self.original.last())
}
fn is_empty(&self) -> bool {
self.original.is_empty() && self.stack.is_empty()
}
}
#[cfg(test)]
mod tests {
use crate::printer::stack::{Stack, StackedStack};
#[test]
fn restore_consumed_stack() {
let original = vec![1, 2, 3];
let mut restorable = StackedStack::new(&original);
restorable.push(4);
assert_eq!(restorable.pop(), Some(4));
assert_eq!(restorable.pop(), Some(3));
assert_eq!(restorable.pop(), Some(2));
assert_eq!(restorable.pop(), Some(1));
assert_eq!(restorable.pop(), None);
assert_eq!(original, vec![1, 2, 3]);
}
#[test]
fn restore_partially_consumed_stack() {
let original = vec![1, 2, 3];
let mut restorable = StackedStack::new(&original);
restorable.push(4);
assert_eq!(restorable.pop(), Some(4));
assert_eq!(restorable.pop(), Some(3));
assert_eq!(restorable.pop(), Some(2));
restorable.push(5);
restorable.push(6);
restorable.push(7);
assert_eq!(original, vec![1, 2, 3]);
}
#[test]
fn restore_stack() {
let original = vec![1, 2, 3];
let mut restorable = StackedStack::new(&original);
restorable.push(4);
restorable.push(5);
restorable.push(6);
restorable.push(7);
assert_eq!(restorable.pop(), Some(7));
assert_eq!(restorable.pop(), Some(6));
assert_eq!(restorable.pop(), Some(5));
assert_eq!(original, vec![1, 2, 3]);
}
}

229
crates/ruff_formatter/src/separated.rs Normal file
View File

@ -0,0 +1,229 @@
use crate::prelude::*;
use crate::{write, CstFormatContext, GroupId};
use ruff_rowan::{AstNode, AstSeparatedElement, SyntaxResult, SyntaxToken};
pub trait FormatSeparatedElementRule<N>
where
N: AstNode,
{
type Context;
type FormatNode<'a>: Format<Self::Context>
where
N: 'a;
type FormatSeparator<'a>: Format<Self::Context>
where
N: 'a;
fn format_node<'a>(&self, node: &'a N) -> Self::FormatNode<'a>;
fn format_separator<'a>(
&self,
separator: &'a SyntaxToken<N::Language>,
) -> Self::FormatSeparator<'a>;
}
/// Formats a single element inside a separated list.
#[derive(Debug, Clone, Eq, PartialEq)]
pub struct FormatSeparatedElement<N, R>
where
N: AstNode,
R: FormatSeparatedElementRule<N>,
{
element: AstSeparatedElement<N::Language, N>,
rule: R,
is_last: bool,
/// The separator to write if the element has no separator yet.
separator: &'static str,
options: FormatSeparatedOptions,
}
impl<N, R> FormatSeparatedElement<N, R>
where
N: AstNode,
R: FormatSeparatedElementRule<N>,
{
/// Returns the node belonging to the element.
pub fn node(&self) -> SyntaxResult<&N> {
self.element.node()
}
}
impl<N, R, C> Format<C> for FormatSeparatedElement<N, R>
where
N: AstNode,
N::Language: 'static,
R: FormatSeparatedElementRule<N, Context = C>,
C: CstFormatContext<Language = N::Language>,
{
fn fmt(&self, f: &mut Formatter<C>) -> FormatResult<()> {
let node = self.element.node()?;
let separator = self.element.trailing_separator()?;
let format_node = self.rule.format_node(node);
if !self.options.nodes_grouped {
format_node.fmt(f)?;
} else {
group(&format_node).fmt(f)?;
}
// Reuse the existing trailing separator or create it if it wasn't in the
// input source. Only print the last trailing token if the outer group breaks
if let Some(separator) = separator {
let format_separator = self.rule.format_separator(separator);
if self.is_last {
match self.options.trailing_separator {
TrailingSeparator::Allowed => {
// Use format_replaced instead of wrapping the result of format_token
// in order to remove only the token itself when the group doesn't break
// but still print its associated trivia unconditionally
format_only_if_breaks(separator, &format_separator)
.with_group_id(self.options.group_id)
.fmt(f)?;
}
TrailingSeparator::Mandatory => {
write!(f, [format_separator])?;
}
TrailingSeparator::Disallowed => {
// A trailing separator was present where it wasn't allowed, opt out of formatting
return Err(FormatError::SyntaxError);
}
TrailingSeparator::Omit => {
write!(f, [format_removed(separator)])?;
}
}
} else {
write!(f, [format_separator])?;
}
} else if self.is_last {
match self.options.trailing_separator {
TrailingSeparator::Allowed => {
write!(
f,
[if_group_breaks(&text(self.separator))
.with_group_id(self.options.group_id)]
)?;
}
TrailingSeparator::Mandatory => {
text(self.separator).fmt(f)?;
}
TrailingSeparator::Omit | TrailingSeparator::Disallowed => { /* no op */ }
}
} else {
unreachable!(
"This is a syntax error, separator must be present between every two elements"
);
};
Ok(())
}
}
/// Iterator for formatting separated elements. Prints the separator between each element and
/// inserts a trailing separator if necessary
pub struct FormatSeparatedIter<I, Node, Rule>
where
Node: AstNode,
{
next: Option<AstSeparatedElement<Node::Language, Node>>,
rule: Rule,
inner: I,
separator: &'static str,
options: FormatSeparatedOptions,
}
impl<I, Node, Rule> FormatSeparatedIter<I, Node, Rule>
where
Node: AstNode,
{
pub fn new(inner: I, separator: &'static str, rule: Rule) -> Self {
Self {
inner,
rule,
separator,
next: None,
options: FormatSeparatedOptions::default(),
}
}
/// Wraps every node inside of a group
pub fn nodes_grouped(mut self) -> Self {
self.options.nodes_grouped = true;
self
}
pub fn with_trailing_separator(mut self, separator: TrailingSeparator) -> Self {
self.options.trailing_separator = separator;
self
}
#[allow(unused)]
pub fn with_group_id(mut self, group_id: Option<GroupId>) -> Self {
self.options.group_id = group_id;
self
}
}
impl<I, Node, Rule> Iterator for FormatSeparatedIter<I, Node, Rule>
where
Node: AstNode,
I: Iterator<Item = AstSeparatedElement<Node::Language, Node>>,
Rule: FormatSeparatedElementRule<Node> + Clone,
{
type Item = FormatSeparatedElement<Node, Rule>;
fn next(&mut self) -> Option<Self::Item> {
let element = self.next.take().or_else(|| self.inner.next())?;
self.next = self.inner.next();
let is_last = self.next.is_none();
Some(FormatSeparatedElement {
element,
rule: self.rule.clone(),
is_last,
separator: self.separator,
options: self.options,
})
}
}
impl<I, Node, Rule> std::iter::FusedIterator for FormatSeparatedIter<I, Node, Rule>
where
Node: AstNode,
I: Iterator<Item = AstSeparatedElement<Node::Language, Node>> + std::iter::FusedIterator,
Rule: FormatSeparatedElementRule<Node> + Clone,
{
}
impl<I, Node, Rule> std::iter::ExactSizeIterator for FormatSeparatedIter<I, Node, Rule>
where
Node: AstNode,
I: Iterator<Item = AstSeparatedElement<Node::Language, Node>> + ExactSizeIterator,
Rule: FormatSeparatedElementRule<Node> + Clone,
{
}
#[derive(Debug, Copy, Clone, Eq, PartialEq, Default)]
pub enum TrailingSeparator {
/// A trailing separator is allowed and preferred
#[default]
Allowed,
/// A trailing separator is not allowed
Disallowed,
/// A trailing separator is mandatory for the syntax to be correct
Mandatory,
/// A trailing separator might be present, but the consumer
/// decides to remove it
Omit,
}
#[derive(Debug, Default, Copy, Clone, Eq, PartialEq)]
pub struct FormatSeparatedOptions {
trailing_separator: TrailingSeparator,
group_id: Option<GroupId>,
nodes_grouped: bool,
}

770
crates/ruff_formatter/src/source_map.rs Normal file
View File

@ -0,0 +1,770 @@
use crate::{Printed, SourceMarker, TextRange};
use ruff_rowan::TextLen;
use ruff_rowan::{Language, SyntaxNode, TextSize};
use rustc_hash::FxHashMap;
use std::cmp::Ordering;
use std::iter::FusedIterator;
/// A source map for mapping positions of a pre-processed tree back to the locations in the source tree.
///
/// This is not a generic purpose source map but instead focused on supporting the case where
/// a language removes or re-orders nodes that would otherwise complicate the formatting logic.
/// A common use case for pre-processing is the removal of all parenthesized nodes.
/// Removing parenthesized nodes simplifies the formatting logic when it has different behaviour
/// depending if a child or parent is of a specific node kind. Performing such a test with parenthesized
/// nodes present in the source code means that the formatting logic has to skip over all parenthesized nodes
/// until it finds the first non-parenthesized node and then test if that node is of the expected kind.
///
/// This source map implementation supports removing tokens or re-structuring nodes
/// without changing the order of the tokens in the tree (requires no source map).
///
/// The following section uses parentheses as a concrete example to explain the functionality of the source map.
/// However, the source map implementation isn't restricted to removing parentheses only, it supports mapping
/// transformed to source position for any use case where a transform deletes text from the source tree.
///
/// ## Position Mapping
///
/// The source map internally tracks all the ranges that have been deleted from the source code sorted by the start of the deleted range.
/// It further stores the absolute count of deleted bytes preceding a range. The deleted range together
/// with the absolute count allows to re-compute the source location for every transformed location
/// and has the benefit that it requires significantly fewer memory
/// than source maps that use a source to destination position marker for every token.
///
/// ## Map Node Ranges
///
/// Only having the deleted ranges to resolve the original text of a node isn't sufficient.
/// Resolving the original text of a node is needed when formatting a node as verbatim, either because
/// formatting the node failed because of a syntax error, or formatting is suppressed with a `rome-ignore format:` comment.
///
/// ```text
/// // Source // Transformed
/// (a+b) + (c + d) a + b + c + d;
/// ```
///
/// Using the above example, the following source ranges should be returned when querying with the transformed ranges:
///
/// * `a` -> `a`: Should not include the leading `(`
/// * `b` -> `b`: Should not include the trailing `)`
/// * `a + b` -> `(a + b)`: Should include the leading `(` and trailing `)`.
/// * `a + b + c + d` -> `(a + b) + (c + d)`: Should include the fist `(` token and the last `)` token because the expression statement
/// fully encloses the `a + b` and `c + d` nodes.
///
/// This is why the source map also tracks the mapped trimmed ranges for every node.
#[derive(Debug, Clone)]
pub struct TransformSourceMap {
source_text: String,
/// The mappings stored in increasing order
deleted_ranges: Vec<DeletedRange>,
/// Key: Start or end position of node for which the trimmed range should be extended
/// Value: The trimmed range.
mapped_node_ranges: FxHashMap<TextSize, TrimmedNodeRangeMapping>,
}
impl TransformSourceMap {
/// Returns the text of the source document as it was before the transformation.
pub fn text(&self) -> &str {
&self.source_text
}
/// Maps a range of the transformed document to a range in the source document.
///
/// Complexity: `O(log(n))`
pub fn source_range(&self, transformed_range: TextRange) -> TextRange {
let range = TextRange::new(
self.source_offset(transformed_range.start(), RangePosition::Start),
self.source_offset(transformed_range.end(), RangePosition::End),
);
debug_assert!(range.end() <= self.source_text.text_len(), "Mapped range {:?} exceeds the length of the source document {:?}. Please check if the passed `transformed_range` is a range of the transformed tree and not of the source tree, and that it belongs to the tree for which the source map was created for.", range, self.source_text.len());
range
}
/// Maps the trimmed range of the transformed node to the trimmed range in the source document.
///
/// Average Complexity: `O(log(n))`
pub fn trimmed_source_range<L: Language>(&self, node: &SyntaxNode<L>) -> TextRange {
self.trimmed_source_range_from_transformed_range(node.text_trimmed_range())
}
fn resolve_trimmed_range(&self, mut source_range: TextRange) -> TextRange {
let start_mapping = self.mapped_node_ranges.get(&source_range.start());
if let Some(mapping) = start_mapping {
// If the queried node fully encloses the original range of the node, then extend the range
if source_range.contains_range(mapping.original_range) {
source_range = TextRange::new(mapping.extended_range.start(), source_range.end());
}
}
let end_mapping = self.mapped_node_ranges.get(&source_range.end());
if let Some(mapping) = end_mapping {
// If the queried node fully encloses the original range of the node, then extend the range
if source_range.contains_range(mapping.original_range) {
source_range = TextRange::new(source_range.start(), mapping.extended_range.end());
}
}
source_range
}
fn trimmed_source_range_from_transformed_range(
&self,
transformed_range: TextRange,
) -> TextRange {
let source_range = self.source_range(transformed_range);
let mut mapped_range = source_range;
loop {
let resolved = self.resolve_trimmed_range(mapped_range);
if resolved == mapped_range {
break resolved;
} else {
mapped_range = resolved;
}
}
}
/// Returns the source text of the trimmed range of `node`.
pub fn trimmed_source_text<L: Language>(&self, node: &SyntaxNode<L>) -> &str {
let range = self.trimmed_source_range(node);
&self.source_text[range]
}
/// Returns an iterator over all deleted ranges in increasing order by their start position.
pub fn deleted_ranges(&self) -> DeletedRanges {
DeletedRanges {
source_text: &self.source_text,
deleted_ranges: self.deleted_ranges.iter(),
}
}
#[cfg(test)]
fn trimmed_source_text_from_transformed_range(&self, range: TextRange) -> &str {
let range = self.trimmed_source_range_from_transformed_range(range);
&self.source_text[range]
}
fn source_offset(&self, transformed_offset: TextSize, position: RangePosition) -> TextSize {
let index = self
.deleted_ranges
.binary_search_by_key(&transformed_offset, |range| range.transformed_start());
let range = match index {
Ok(index) => Some(&self.deleted_ranges[index]),
Err(index) => {
if index == 0 {
None
} else {
self.deleted_ranges.get(index - 1)
}
}
};
self.source_offset_with_range(transformed_offset, position, range)
}
fn source_offset_with_range(
&self,
transformed_offset: TextSize,
position: RangePosition,
deleted_range: Option<&DeletedRange>,
) -> TextSize {
match deleted_range {
Some(range) => {
debug_assert!(
range.transformed_start() <= transformed_offset,
"Transformed start {:?} must be less than or equal to transformed offset {:?}.",
range.transformed_start(),
transformed_offset
);
// Transformed position directly falls onto a position where a deleted range starts or ends (depending on the position)
// For example when querying: `a` in `(a)` or (a + b)`, or `b`
if range.transformed_start() == transformed_offset {
match position {
RangePosition::Start => range.source_end(),
// `a)`, deleted range is right after the token. That's why `source_start` is the offset
// that truncates the `)` and `source_end` includes it
RangePosition::End => range.source_start(),
}
}
// The position falls outside of a position that has a leading/trailing deleted range.
// For example, if you get the position of `+` in `(a + b)`.
// That means, the trimmed and non-trimmed offsets are the same
else {
let transformed_delta = transformed_offset - range.transformed_start();
range.source_start() + range.len() + transformed_delta
}
}
None => transformed_offset,
}
}
/// Maps the source code positions relative to the transformed tree of `printed` to the location
/// in the original, untransformed source code.
///
/// The printer creates a source map that allows mapping positions from the newly formatted document
/// back to the locations of the tree. However, the source positions stored in [crate::FormatElement::DynamicText]
/// and [crate::FormatElement::SyntaxTokenTextSlice] are relative to the transformed tree
/// and not the original tree passed to [crate::format_node].
///
/// This function re-maps the positions from the positions in the transformed tree back to the positions
/// in the original, untransformed tree.
pub fn map_printed(&self, mut printed: Printed) -> Printed {
self.map_markers(&mut printed.sourcemap);
printed
}
/// Maps the printers source map marker to the source positions.
fn map_markers(&self, markers: &mut [SourceMarker]) {
if self.deleted_ranges.is_empty() {
return;
}
let mut previous_marker: Option<SourceMarker> = None;
let mut next_range_index = 0;
for marker in markers {
// It's not guaranteed that markers are sorted by source location (line suffix comments).
// It can, therefore, be necessary to navigate backwards again.
// In this case, do a binary search for the index of the next deleted range (`O(log(n)`).
let out_of_order_marker =
previous_marker.map_or(false, |previous| previous.source > marker.source);
if out_of_order_marker {
let index = self
.deleted_ranges
.binary_search_by_key(&marker.source, |range| range.transformed_start());
match index {
// Direct match
Ok(index) => {
next_range_index = index + 1;
}
Err(index) => next_range_index = index,
}
} else {
// Find the range for this mapping. In most cases this is a no-op or only involves a single step
// because markers are most of the time in increasing source order.
while next_range_index < self.deleted_ranges.len() {
let next_range = &self.deleted_ranges[next_range_index];
if next_range.transformed_start() > marker.source {
break;
}
next_range_index += 1;
}
}
previous_marker = Some(*marker);
let current_range = if next_range_index == 0 {
None
} else {
self.deleted_ranges.get(next_range_index - 1)
};
let source =
self.source_offset_with_range(marker.source, RangePosition::Start, current_range);
marker.source = source;
}
}
}
#[derive(Debug, Copy, Clone)]
struct TrimmedNodeRangeMapping {
/// The original trimmed range of the node.
///
/// ```javascript
/// (a + b)
/// ```
///
/// `1..6` `a + b`
original_range: TextRange,
/// The range to which the trimmed range of the node should be extended
/// ```javascript
/// (a + b)
/// ```
///
/// `0..7` for `a + b` if its range should also include the parenthesized range.
extended_range: TextRange,
}
#[derive(Copy, Clone, Debug)]
enum RangePosition {
Start,
End,
}
/// Stores the information about a range in the source document that isn't present in the transformed document
/// and provides means to map the transformed position back to the source position.
///
/// # Examples
///
/// ```javascript
/// (a + b)
/// ```
///
/// A transform that removes the parentheses from the above expression removes the ranges `0..1` (`(` token)
/// and `6..7` (`)` token) and the source map creates one [DeletedRange] for each:
///
/// ```text
/// DeletedRange {
/// source_range: 0..1,
/// total_length_preceding_deleted_ranges: 0,
/// },
/// DeletedRange {
/// source_range: 6..7,
/// total_length_preceding_deleted_ranges: 1,
/// }
/// ```
///
/// The first range indicates that the range `0..1` for the `(` token has been removed. The second range
/// indicates that the range `6..7` for the `)` token has been removed and it stores that, up to this point,
/// but not including, 1 more byte has been removed.
#[derive(Debug, Copy, Clone, Eq, PartialEq)]
struct DeletedRange {
/// The range in the source document of the bytes that have been omitted from the transformed document.
source_range: TextRange,
/// The accumulated count of all removed bytes up to (but not including) the start of this range.
total_length_preceding_deleted_ranges: TextSize,
}
impl DeletedRange {
fn new(source_range: TextRange, total_length_preceding_deleted_ranges: TextSize) -> Self {
debug_assert!(source_range.start() >= total_length_preceding_deleted_ranges, "The total number of deleted bytes ({:?}) can not exceed the offset from the start in the source document ({:?}). This is a bug in the source map implementation.", total_length_preceding_deleted_ranges, source_range.start());
Self {
source_range,
total_length_preceding_deleted_ranges,
}
}
/// The number of deleted characters starting from [source offset](DeletedRange::source_start).
fn len(&self) -> TextSize {
self.source_range.len()
}
/// The start position in bytes in the source document of the omitted sequence in the transformed document.
fn source_start(&self) -> TextSize {
self.source_range.start()
}
/// The end position in bytes in the source document of the omitted sequence in the transformed document.
fn source_end(&self) -> TextSize {
self.source_range.end()
}
/// Returns the byte position of [DeleteRange::source_start] in the transformed document.
fn transformed_start(&self) -> TextSize {
self.source_range.start() - self.total_length_preceding_deleted_ranges
}
}
/// Builder for creating a source map.
#[derive(Debug, Default)]
pub struct TransformSourceMapBuilder {
/// The original source text of the tree before it was transformed.
source_text: String,
/// The mappings in increasing order by transformed offset.
deleted_ranges: Vec<TextRange>,
/// The keys are a position in the source map where a trimmed node starts or ends.
/// The values are the metadata about a trimmed node range
mapped_node_ranges: FxHashMap<TextSize, TrimmedNodeRangeMapping>,
}
impl TransformSourceMapBuilder {
/// Creates a new builder.
pub fn new() -> Self {
Self {
..Default::default()
}
}
/// Creates a new builder for a document with the given source.
pub fn with_source(source: String) -> Self {
Self {
source_text: source,
..Default::default()
}
}
/// Appends `text` to the source text of the original document.
pub fn push_source_text(&mut self, text: &str) {
self.source_text.push_str(text);
}
/// Adds a new mapping for a deleted character range.
pub fn add_deleted_range(&mut self, source_range: TextRange) {
self.deleted_ranges.push(source_range);
}
/// Adds a mapping to widen a nodes trimmed range.
///
/// The formatter uses the trimmed range when formatting a node in verbatim either because the node
/// failed to format because of a syntax error or because it's formatting is suppressed with a `rome-ignore format:` comment.
///
/// This method adds a mapping to widen a nodes trimmed range to enclose another range instead. This is
/// e.g. useful when removing parentheses around expressions where `(/* comment */ a /* comment */)` because
/// the trimmed range of `a` should now enclose the full range including the `(` and `)` tokens to ensure
/// that the parentheses are retained when printing that node in verbatim style.
pub fn extend_trimmed_node_range(
&mut self,
original_range: TextRange,
extended_range: TextRange,
) {
let mapping = TrimmedNodeRangeMapping {
original_range,
extended_range,
};
self.mapped_node_ranges
.insert(original_range.start(), mapping);
self.mapped_node_ranges
.insert(original_range.end(), mapping);
}
/// Creates a source map that performs single position lookups in `O(log(n))`.
pub fn finish(mut self) -> TransformSourceMap {
let mut merged_mappings = Vec::with_capacity(self.deleted_ranges.len());
if !self.deleted_ranges.is_empty() {
self.deleted_ranges
.sort_by(|a, b| match a.start().cmp(&b.start()) {
Ordering::Equal => a.end().cmp(&b.end()),
ordering => ordering,
});
let mut last_mapping = DeletedRange::new(
// SAFETY: Safe because of the not empty check above
self.deleted_ranges[0],
TextSize::default(),
);
let mut transformed_offset = last_mapping.len();
for range in self.deleted_ranges.drain(1..) {
// Merge adjacent ranges to ensure there's only ever a single mapping starting at the same transformed offset.
if last_mapping.source_range.end() == range.start() {
last_mapping.source_range = last_mapping.source_range.cover(range);
} else {
merged_mappings.push(last_mapping);
last_mapping = DeletedRange::new(range, transformed_offset);
}
transformed_offset += range.len();
}
merged_mappings.push(last_mapping);
}
TransformSourceMap {
source_text: self.source_text,
deleted_ranges: merged_mappings,
mapped_node_ranges: self.mapped_node_ranges,
}
}
}
#[derive(Debug, Copy, Clone, Eq, PartialEq)]
pub struct DeletedRangeEntry<'a> {
/// The start position of the removed range in the source document
pub source: TextSize,
/// The position in the transformed document where the removed range would have been (but is not, because it was removed)
pub transformed: TextSize,
/// The text of the removed range
pub text: &'a str,
}
/// Iterator over all removed ranges in a document.
///
/// Returns the ranges in increased order by their start position.
pub struct DeletedRanges<'a> {
source_text: &'a str,
/// The mappings stored in increasing order
deleted_ranges: std::slice::Iter<'a, DeletedRange>,
}
impl<'a> Iterator for DeletedRanges<'a> {
type Item = DeletedRangeEntry<'a>;
fn next(&mut self) -> Option<Self::Item> {
let next = self.deleted_ranges.next()?;
Some(DeletedRangeEntry {
source: next.source_range.start(),
transformed: next.transformed_start(),
text: &self.source_text[next.source_range],
})
}
fn size_hint(&self) -> (usize, Option<usize>) {
self.deleted_ranges.size_hint()
}
fn last(self) -> Option<Self::Item>
where
Self: Sized,
{
let last = self.deleted_ranges.last()?;
Some(DeletedRangeEntry {
source: last.source_range.start(),
transformed: last.transformed_start(),
text: &self.source_text[last.source_range],
})
}
}
impl DoubleEndedIterator for DeletedRanges<'_> {
fn next_back(&mut self) -> Option<Self::Item> {
let back = self.deleted_ranges.next_back()?;
Some(DeletedRangeEntry {
source: back.source_range.start(),
transformed: back.transformed_start(),
text: &self.source_text[back.source_range],
})
}
}
impl FusedIterator for DeletedRanges<'_> {}
impl ExactSizeIterator for DeletedRanges<'_> {}
#[cfg(test)]
mod tests {
use crate::source_map::DeletedRangeEntry;
use crate::{TextRange, TextSize, TransformSourceMapBuilder};
use ruff_rowan::raw_language::{RawLanguageKind, RawSyntaxTreeBuilder};
#[test]
fn range_mapping() {
let mut cst_builder = RawSyntaxTreeBuilder::new();
cst_builder.start_node(RawLanguageKind::ROOT);
// The shape of the tree doesn't matter for the test case
cst_builder.token(RawLanguageKind::STRING_TOKEN, "(a + (((b + c)) + d)) + e");
cst_builder.finish_node();
let root = cst_builder.finish();
let mut builder = TransformSourceMapBuilder::new();
builder.push_source_text(&root.text().to_string());
// Add mappings for all removed parentheses.
// `(`
builder.add_deleted_range(TextRange::new(TextSize::from(0), TextSize::from(1)));
// `(((`
builder.add_deleted_range(TextRange::new(TextSize::from(5), TextSize::from(6)));
// Ranges can be added out of order
builder.add_deleted_range(TextRange::new(TextSize::from(7), TextSize::from(8)));
builder.add_deleted_range(TextRange::new(TextSize::from(6), TextSize::from(7)));
// `))`
builder.add_deleted_range(TextRange::new(TextSize::from(13), TextSize::from(14)));
builder.add_deleted_range(TextRange::new(TextSize::from(14), TextSize::from(15)));
// `))`
builder.add_deleted_range(TextRange::new(TextSize::from(19), TextSize::from(20)));
builder.add_deleted_range(TextRange::new(TextSize::from(20), TextSize::from(21)));
let source_map = builder.finish();
// The following mapping assume the transformed string to be (including whitespace):
// "a + b + c + d + e";
// `a`
assert_eq!(
source_map.source_range(TextRange::new(TextSize::from(0), TextSize::from(1))),
TextRange::new(TextSize::from(1), TextSize::from(2))
);
// `b`
assert_eq!(
source_map.source_range(TextRange::new(TextSize::from(4), TextSize::from(5))),
TextRange::new(TextSize::from(8), TextSize::from(9))
);
// `c`
assert_eq!(
source_map.source_range(TextRange::new(TextSize::from(8), TextSize::from(9))),
TextRange::new(TextSize::from(12), TextSize::from(13))
);
// `d`
assert_eq!(
source_map.source_range(TextRange::new(TextSize::from(12), TextSize::from(13))),
TextRange::new(TextSize::from(18), TextSize::from(19))
);
// `e`
assert_eq!(
source_map.source_range(TextRange::new(TextSize::from(16), TextSize::from(17))),
TextRange::new(TextSize::from(24), TextSize::from(25))
);
}
#[test]
fn trimmed_range() {
// Build up a tree for `((a))`
// Don't mind the unknown nodes, it doesn't really matter what the nodes are.
let mut cst_builder = RawSyntaxTreeBuilder::new();
cst_builder.start_node(RawLanguageKind::ROOT);
cst_builder.start_node(RawLanguageKind::BOGUS);
cst_builder.token(RawLanguageKind::STRING_TOKEN, "(");
cst_builder.start_node(RawLanguageKind::BOGUS);
cst_builder.token(RawLanguageKind::BOGUS, "(");
cst_builder.start_node(RawLanguageKind::LITERAL_EXPRESSION);
cst_builder.token(RawLanguageKind::STRING_TOKEN, "a");
cst_builder.finish_node();
cst_builder.token(RawLanguageKind::BOGUS, ")");
cst_builder.finish_node();
cst_builder.token(RawLanguageKind::BOGUS, ")");
cst_builder.finish_node();
cst_builder.token(RawLanguageKind::BOGUS, ";");
cst_builder.finish_node();
let root = cst_builder.finish();
assert_eq!(&root.text(), "((a));");
let mut bogus = root
.descendants()
.filter(|node| node.kind() == RawLanguageKind::BOGUS);
// `((a))`
let outer = bogus.next().unwrap();
// `(a)`
let inner = bogus.next().unwrap();
// `a`
let expression = root
.descendants()
.find(|node| node.kind() == RawLanguageKind::LITERAL_EXPRESSION)
.unwrap();
let mut builder = TransformSourceMapBuilder::new();
builder.push_source_text(&root.text().to_string());
// Add mappings for all removed parentheses.
builder.add_deleted_range(TextRange::new(TextSize::from(0), TextSize::from(2)));
builder.add_deleted_range(TextRange::new(TextSize::from(3), TextSize::from(5)));
// Extend `a` to the range of `(a)`
builder
.extend_trimmed_node_range(expression.text_trimmed_range(), inner.text_trimmed_range());
// Extend `(a)` to the range of `((a))`
builder.extend_trimmed_node_range(inner.text_trimmed_range(), outer.text_trimmed_range());
let source_map = builder.finish();
// Query `a`
assert_eq!(
source_map.trimmed_source_text_from_transformed_range(TextRange::new(
TextSize::from(0),
TextSize::from(1)
)),
"((a))"
);
// Query `a;` expression
assert_eq!(
source_map.trimmed_source_text_from_transformed_range(TextRange::new(
TextSize::from(0),
TextSize::from(2)
)),
"((a));"
);
}
#[test]
fn deleted_ranges() {
let mut cst_builder = RawSyntaxTreeBuilder::new();
cst_builder.start_node(RawLanguageKind::ROOT);
// The shape of the tree doesn't matter for the test case
cst_builder.token(RawLanguageKind::STRING_TOKEN, "(a + (((b + c)) + d)) + e");
cst_builder.finish_node();
let root = cst_builder.finish();
let mut builder = TransformSourceMapBuilder::new();
builder.push_source_text(&root.text().to_string());
// Add mappings for all removed parentheses.
// `(`
builder.add_deleted_range(TextRange::new(TextSize::from(0), TextSize::from(1)));
// `(((`
builder.add_deleted_range(TextRange::new(TextSize::from(5), TextSize::from(6)));
// Ranges can be added out of order
builder.add_deleted_range(TextRange::new(TextSize::from(7), TextSize::from(8)));
builder.add_deleted_range(TextRange::new(TextSize::from(6), TextSize::from(7)));
// `))`
builder.add_deleted_range(TextRange::new(TextSize::from(13), TextSize::from(14)));
builder.add_deleted_range(TextRange::new(TextSize::from(14), TextSize::from(15)));
// `))`
builder.add_deleted_range(TextRange::new(TextSize::from(19), TextSize::from(20)));
builder.add_deleted_range(TextRange::new(TextSize::from(20), TextSize::from(21)));
let source_map = builder.finish();
let deleted_ranges = source_map.deleted_ranges().collect::<Vec<_>>();
assert_eq!(
deleted_ranges,
vec![
DeletedRangeEntry {
source: TextSize::from(0),
transformed: TextSize::from(0),
text: "("
},
DeletedRangeEntry {
source: TextSize::from(5),
transformed: TextSize::from(4),
text: "((("
},
DeletedRangeEntry {
source: TextSize::from(13),
transformed: TextSize::from(9),
text: "))"
},
DeletedRangeEntry {
source: TextSize::from(19),
transformed: TextSize::from(13),
text: "))"
},
]
);
assert_eq!(
source_map.deleted_ranges().last(),
Some(DeletedRangeEntry {
source: TextSize::from(19),
transformed: TextSize::from(13),
text: "))"
})
);
}
}

2
crates/ruff_formatter/src/token/mod.rs Normal file
View File

@ -0,0 +1,2 @@
pub mod number;
pub mod string;

296
crates/ruff_formatter/src/token/number.rs Normal file
View File

@ -0,0 +1,296 @@
use crate::token::string::ToAsciiLowercaseCow;
use ruff_rowan::{Language, SyntaxToken};
use std::borrow::Cow;
use std::num::NonZeroUsize;
use crate::prelude::*;
use crate::{CstFormatContext, Format};
pub fn format_number_token<L>(token: &SyntaxToken<L>) -> CleanedNumberLiteralText<L>
where
L: Language,
{
CleanedNumberLiteralText { token }
}
pub struct CleanedNumberLiteralText<'token, L>
where
L: Language,
{
token: &'token SyntaxToken<L>,
}
impl<L, C> Format<C> for CleanedNumberLiteralText<'_, L>
where
L: Language + 'static,
C: CstFormatContext<Language = L>,
{
fn fmt(&self, f: &mut Formatter<C>) -> FormatResult<()> {
format_replaced(
self.token,
&syntax_token_cow_slice(
format_trimmed_number(self.token.text_trimmed()),
self.token,
self.token.text_trimmed_range().start(),
),
)
.fmt(f)
}
}
enum FormatNumberLiteralState {
IntegerPart,
DecimalPart(FormatNumberLiteralDecimalPart),
Exponent(FormatNumberLiteralExponent),
}
struct FormatNumberLiteralDecimalPart {
dot_index: usize,
last_non_zero_index: Option<NonZeroUsize>,
}
struct FormatNumberLiteralExponent {
e_index: usize,
is_negative: bool,
first_digit_index: Option<NonZeroUsize>,
first_non_zero_index: Option<NonZeroUsize>,
}
// Regex-free version of https://github.com/prettier/prettier/blob/ca246afacee8e6d5db508dae01730c9523bbff1d/src/common/util.js#L341-L356
fn format_trimmed_number(text: &str) -> Cow<str> {
use FormatNumberLiteralState::*;
let text = text.to_ascii_lowercase_cow();
let mut copied_or_ignored_chars = 0usize;
let mut iter = text.chars().enumerate();
let mut curr = iter.next();
let mut state = IntegerPart;
// Will be filled only if and when the first place that needs reformatting is detected.
let mut cleaned_text = String::new();
// Look at only the start of the text, ignore any sign, and make sure numbers always start with a digit. Add 0 if missing.
if let Some((_, '+' | '-')) = curr {
curr = iter.next();
}
if let Some((curr_index, '.')) = curr {
cleaned_text.push_str(&text[copied_or_ignored_chars..curr_index]);
copied_or_ignored_chars = curr_index;
cleaned_text.push('0');
}
// Loop over the rest of the text, applying the remaining rules.
loop {
// We use a None pseudo-char at the end of the string to simplify the match cases that follow
let curr_or_none_terminator_char = match curr {
Some((curr_index, curr_char)) => (curr_index, Some(curr_char)),
None => (text.len(), None),
};
// Look for termination of the decimal part or exponent and see if we need to print it differently.
match (&state, curr_or_none_terminator_char) {
(
DecimalPart(FormatNumberLiteralDecimalPart {
dot_index,
last_non_zero_index: None,
}),
(curr_index, Some('e') | None),
) => {
// The decimal part equals zero, ignore it completely.
// Caveat: Prettier still prints a single `.0` unless there was *only* a trailing dot.
if curr_index > dot_index + 1 {
cleaned_text.push_str(&text[copied_or_ignored_chars..=*dot_index]);
cleaned_text.push('0');
} else {
cleaned_text.push_str(&text[copied_or_ignored_chars..*dot_index]);
}
copied_or_ignored_chars = curr_index;
}
(
DecimalPart(FormatNumberLiteralDecimalPart {
last_non_zero_index: Some(last_non_zero_index),
..
}),
(curr_index, Some('e') | None),
) if last_non_zero_index.get() < curr_index - 1 => {
// The decimal part ends with at least one zero, ignore them but copy the part from the dot until the last non-zero.
cleaned_text.push_str(&text[copied_or_ignored_chars..=last_non_zero_index.get()]);
copied_or_ignored_chars = curr_index;
}
(
Exponent(FormatNumberLiteralExponent {
e_index,
first_non_zero_index: None,
..
}),
(curr_index, None),
) => {
// The exponent equals zero, ignore it completely.
cleaned_text.push_str(&text[copied_or_ignored_chars..*e_index]);
copied_or_ignored_chars = curr_index;
}
(
Exponent(FormatNumberLiteralExponent {
e_index,
is_negative,
first_digit_index: Some(first_digit_index),
first_non_zero_index: Some(first_non_zero_index),
}),
(curr_index, None),
) if (first_digit_index.get() > e_index + 1 && !is_negative)
|| (first_non_zero_index.get() > first_digit_index.get()) =>
{
// The exponent begins with a plus or at least one zero, ignore them but copy the part from the first non-zero until the end.
cleaned_text.push_str(&text[copied_or_ignored_chars..=*e_index]);
if *is_negative {
cleaned_text.push('-');
}
cleaned_text.push_str(&text[first_non_zero_index.get()..curr_index]);
copied_or_ignored_chars = curr_index;
}
_ => {}
}
// Update state after the current char
match (&state, curr) {
// Cases entering or remaining in decimal part
(_, Some((curr_index, '.'))) => {
state = DecimalPart(FormatNumberLiteralDecimalPart {
dot_index: curr_index,
last_non_zero_index: None,
});
}
(DecimalPart(decimal_part), Some((curr_index, '1'..='9'))) => {
state = DecimalPart(FormatNumberLiteralDecimalPart {
last_non_zero_index: Some(unsafe {
// We've already entered InDecimalPart, so curr_index must be >0
NonZeroUsize::new_unchecked(curr_index)
}),
..*decimal_part
});
}
// Cases entering or remaining in exponent
(_, Some((curr_index, 'e'))) => {
state = Exponent(FormatNumberLiteralExponent {
e_index: curr_index,
is_negative: false,
first_digit_index: None,
first_non_zero_index: None,
});
}
(Exponent(exponent), Some((_, '-'))) => {
state = Exponent(FormatNumberLiteralExponent {
is_negative: true,
..*exponent
});
}
(
Exponent(
exponent @ FormatNumberLiteralExponent {
first_digit_index: None,
..
},
),
Some((curr_index, curr_char @ '0'..='9')),
) => {
state = Exponent(FormatNumberLiteralExponent {
first_digit_index: Some(unsafe {
// We've already entered InExponent, so curr_index must be >0
NonZeroUsize::new_unchecked(curr_index)
}),
first_non_zero_index: if curr_char != '0' {
Some(unsafe {
// We've already entered InExponent, so curr_index must be >0
NonZeroUsize::new_unchecked(curr_index)
})
} else {
None
},
..*exponent
});
}
(
Exponent(
exponent @ FormatNumberLiteralExponent {
first_non_zero_index: None,
..
},
),
Some((curr_index, '1'..='9')),
) => {
state = Exponent(FormatNumberLiteralExponent {
first_non_zero_index: Some(unsafe { NonZeroUsize::new_unchecked(curr_index) }),
..*exponent
});
}
_ => {}
}
// Repeat or exit
match curr {
None | Some((_, 'x') /* hex bailout */) => break,
Some(_) => curr = iter.next(),
}
}
if cleaned_text.is_empty() {
text
} else {
// Append any unconsidered text
cleaned_text.push_str(&text[copied_or_ignored_chars..]);
Cow::Owned(cleaned_text)
}
}
#[cfg(test)]
mod tests {
use std::borrow::Cow;
use super::format_trimmed_number;
#[test]
fn removes_unnecessary_plus_and_zeros_from_scientific_notation() {
assert_eq!("1e2", format_trimmed_number("1e02"));
assert_eq!("1e2", format_trimmed_number("1e+2"));
}
#[test]
fn removes_unnecessary_scientific_notation() {
assert_eq!("1", format_trimmed_number("1e0"));
assert_eq!("1", format_trimmed_number("1e-0"));
}
#[test]
fn does_not_get_bamboozled_by_hex() {
assert_eq!("0xe0", format_trimmed_number("0xe0"));
assert_eq!("0x10e0", format_trimmed_number("0x10e0"));
}
#[test]
fn makes_sure_numbers_always_start_with_a_digit() {
assert_eq!("0.2", format_trimmed_number(".2"));
}
#[test]
fn removes_extraneous_trailing_decimal_zeroes() {
assert_eq!("0.1", format_trimmed_number("0.10"));
}
#[test]
fn keeps_one_trailing_decimal_zero() {
assert_eq!("0.0", format_trimmed_number("0.00"));
}
#[test]
fn removes_trailing_dot() {
assert_eq!("1", format_trimmed_number("1."));
}
#[test]
fn cleans_all_at_once() {
assert_eq!("0.0", format_trimmed_number(".00e-0"));
}
#[test]
fn keeps_the_input_string_if_no_change_needed() {
assert!(matches!(
format_trimmed_number("0.1e2"),
Cow::Borrowed("0.1e2")
));
}
}

259
crates/ruff_formatter/src/token/string.rs Normal file
View File

@ -0,0 +1,259 @@
use std::borrow::Cow;
pub trait ToAsciiLowercaseCow {
/// Returns the same value as String::to_lowercase. The only difference
/// is that this functions returns ```Cow``` and does not allocate
/// if the string is already in lowercase.
fn to_ascii_lowercase_cow(&self) -> Cow<str>;
}
impl ToAsciiLowercaseCow for str {
fn to_ascii_lowercase_cow(&self) -> Cow<str> {
debug_assert!(self.is_ascii());
let bytes = self.as_bytes();
for idx in 0..bytes.len() {
let chr = bytes[idx];
if chr != chr.to_ascii_lowercase() {
let mut s = bytes.to_vec();
for b in &mut s[idx..] {
b.make_ascii_lowercase();
}
return Cow::Owned(unsafe { String::from_utf8_unchecked(s) });
}
}
Cow::Borrowed(self)
}
}
impl ToAsciiLowercaseCow for String {
#[inline(always)]
fn to_ascii_lowercase_cow(&self) -> Cow<str> {
self.as_str().to_ascii_lowercase_cow()
}
}
/// This signal is used to tell to the next character what it should do
#[derive(Eq, PartialEq)]
pub enum CharSignal {
/// There hasn't been any signal
None,
/// The function decided to keep the previous character
Keep,
/// The function has decided to print the character. Saves the character that was
/// already written
AlreadyPrinted(char),
}
#[derive(Debug, Eq, PartialEq, Clone, Copy)]
pub enum Quote {
Double,
Single,
}
impl Quote {
pub fn as_char(&self) -> char {
match self {
Quote::Double => '"',
Quote::Single => '\'',
}
}
pub fn as_string(&self) -> &str {
match self {
Quote::Double => "\"",
Quote::Single => "'",
}
}
/// Returns the quote, prepended with a backslash (escaped)
pub fn as_escaped(&self) -> &str {
match self {
Quote::Double => "\\\"",
Quote::Single => "\\'",
}
}
pub fn as_bytes(&self) -> u8 {
self.as_char() as u8
}
/// Given the current quote, it returns the other one
pub fn other(&self) -> Self {
match self {
Quote::Double => Quote::Single,
Quote::Single => Quote::Double,
}
}
}
/// This function is responsible of:
///
/// - reducing the number of escapes
/// - normalising the new lines
///
/// # Escaping
///
/// The way it works is the following: we split the content by analyzing all the
/// characters that could keep the escape.
///
/// Each time we retrieve one of this character, we push inside a new string all the content
/// found **before** the current character.
///
/// After that the function checks if the current character should be also be printed or not.
/// These characters (like quotes) can have an escape that might be removed. If that happens,
/// we use [CharSignal] to tell to the next iteration what it should do with that character.
///
/// For example, let's take this example:
/// ```js
/// ("hello! \'")
/// ```
///
/// Here, we want to remove the backslash (\) from the content. So when we encounter `\`,
/// the algorithm checks if after `\` there's a `'`, and if so, then we push inside the final string
/// only `'` and we ignore the backlash. Then we signal the next iteration with [CharSignal::AlreadyPrinted],
/// so when we process the next `'`, we decide to ignore it and reset the signal.
///
/// Another example is the following:
///
/// ```js
/// (" \\' ")
/// ```
///
/// Here, we need to keep all the backslash. We check the first one and we look ahead. We find another
/// `\`, so we keep it the first and we signal the next iteration with [CharSignal::Keep].
/// Then the next iteration comes along. We have the second `\`, we look ahead we find a `'`. Although,
/// as opposed to the previous example, we have a signal that says that we should keep the current
/// character. Then we do so. The third iteration comes along and we find `'`. We still have the
/// [CharSignal::Keep]. We do so and then we set the signal to [CharSignal::None]
///
/// # Newlines
///
/// By default the formatter uses `\n` as a newline. The function replaces
/// `\r\n` with `\n`,
pub fn normalize_string(raw_content: &str, preferred_quote: Quote) -> Cow<str> {
let alternate_quote = preferred_quote.other();
// A string should be manipulated only if its raw content contains backslash or quotes
if !raw_content.contains(['\\', preferred_quote.as_char(), alternate_quote.as_char()]) {
return Cow::Borrowed(raw_content);
}
let mut reduced_string = String::new();
let mut signal = CharSignal::None;
let mut chars = raw_content.char_indices().peekable();
while let Some((_, current_char)) = chars.next() {
let next_character = chars.peek();
if let CharSignal::AlreadyPrinted(char) = signal {
if char == current_char {
continue;
}
}
match current_char {
'\\' => {
let bytes = raw_content.as_bytes();
if let Some((next_index, next_character)) = next_character {
// If we encounter an alternate quote that is escaped, we have to
// remove the escape from it.
// This is done because of how the enclosed strings can change.
// Check `computed_preferred_quote` for more details.
if *next_character as u8 == alternate_quote.as_bytes()
// This check is a safety net for cases where the backslash is at the end
// of the raw content:
// ("\\")
// The second backslash is at the end.
&& *next_index < bytes.len()
{
match signal {
CharSignal::Keep => {
reduced_string.push(current_char);
}
_ => {
reduced_string.push(alternate_quote.as_char());
signal = CharSignal::AlreadyPrinted(alternate_quote.as_char());
}
}
} else if signal == CharSignal::Keep {
reduced_string.push(current_char);
signal = CharSignal::None;
}
// The next character is another backslash, or
// a character that should be kept in the next iteration
else if "^\n\r\"'01234567\\bfnrtuvx\u{2028}\u{2029}".contains(*next_character)
{
signal = CharSignal::Keep;
// fallback, keep the backslash
reduced_string.push(current_char);
} else {
// these, usually characters that can have their
// escape removed: "\a" => "a"
// So we ignore the current slash and we continue
// to the next iteration
continue;
}
} else {
// fallback, keep the backslash
reduced_string.push(current_char);
}
}
'\n' | '\t' => {
if let CharSignal::AlreadyPrinted(the_char) = signal {
if matches!(the_char, '\n' | '\t') {
signal = CharSignal::None
}
} else {
reduced_string.push(current_char);
}
}
// If the current character is \r and the
// next is \n, skip over the entire sequence
'\r' if next_character.map_or(false, |(_, c)| *c == '\n') => {
reduced_string.push('\n');
signal = CharSignal::AlreadyPrinted('\n');
}
_ => {
// If we encounter a preferred quote and it's not escaped, we have to replace it with
// an escaped version.
// This is done because of how the enclosed strings can change.
// Check `computed_preferred_quote` for more details.
if current_char == preferred_quote.as_char() {
let last_char = &reduced_string.chars().last();
if let Some('\\') = last_char {
reduced_string.push(preferred_quote.as_char());
} else {
reduced_string.push_str(preferred_quote.as_escaped());
}
} else if current_char == alternate_quote.as_char() {
match signal {
CharSignal::None | CharSignal::Keep => {
reduced_string.push(alternate_quote.as_char());
}
CharSignal::AlreadyPrinted(_) => (),
}
} else {
reduced_string.push(current_char);
}
signal = CharSignal::None;
}
}
}
// Don't allocate a new string of this is empty
if reduced_string.is_empty() {
Cow::Borrowed(raw_content)
} else {
// don't allocate a new string if the new string is still equals to the input string
if reduced_string == raw_content {
Cow::Borrowed(raw_content)
} else {
Cow::Owned(reduced_string)
}
}
}

597
crates/ruff_formatter/src/trivia.rs Normal file
View File

@ -0,0 +1,597 @@
//! Provides builders for comments and skipped token trivia.
use crate::format_element::tag::VerbatimKind;
use crate::prelude::*;
use crate::{
comments::{CommentKind, CommentStyle},
write, Argument, Arguments, CstFormatContext, FormatRefWithRule, GroupId, SourceComment,
TextRange,
};
use ruff_rowan::{Language, SyntaxNode, SyntaxToken};
#[cfg(debug_assertions)]
use std::cell::Cell;
/// Formats the leading comments of `node`
pub const fn format_leading_comments<L: Language>(
node: &SyntaxNode<L>,
) -> FormatLeadingComments<L> {
FormatLeadingComments::Node(node)
}
/// Formats the leading comments of a node.
#[derive(Debug, Copy, Clone)]
pub enum FormatLeadingComments<'a, L: Language> {
Node(&'a SyntaxNode<L>),
Comments(&'a [SourceComment<L>]),
}
impl<Context> Format<Context> for FormatLeadingComments<'_, Context::Language>
where
Context: CstFormatContext,
{
fn fmt(&self, f: &mut Formatter<Context>) -> FormatResult<()> {
let comments = f.context().comments().clone();
let leading_comments = match self {
FormatLeadingComments::Node(node) => comments.leading_comments(node),
FormatLeadingComments::Comments(comments) => comments,
};
for comment in leading_comments {
let format_comment = FormatRefWithRule::new(comment, Context::CommentRule::default());
write!(f, [format_comment])?;
match comment.kind() {
CommentKind::Block | CommentKind::InlineBlock => {
match comment.lines_after() {
0 => write!(f, [space()])?,
1 => {
if comment.lines_before() == 0 {
write!(f, [soft_line_break_or_space()])?;
} else {
write!(f, [hard_line_break()])?;
}
}
_ => write!(f, [empty_line()])?,
};
}
CommentKind::Line => match comment.lines_after() {
0 | 1 => write!(f, [hard_line_break()])?,
_ => write!(f, [empty_line()])?,
},
}
comment.mark_formatted()
}
Ok(())
}
}
/// Formats the trailing comments of `node`.
pub const fn format_trailing_comments<L: Language>(
node: &SyntaxNode<L>,
) -> FormatTrailingComments<L> {
FormatTrailingComments::Node(node)
}
/// Formats the trailing comments of `node`
#[derive(Debug, Clone, Copy)]
pub enum FormatTrailingComments<'a, L: Language> {
Node(&'a SyntaxNode<L>),
Comments(&'a [SourceComment<L>]),
}
impl<Context> Format<Context> for FormatTrailingComments<'_, Context::Language>
where
Context: CstFormatContext,
{
fn fmt(&self, f: &mut Formatter<Context>) -> FormatResult<()> {
let comments = f.context().comments().clone();
let trailing_comments = match self {
FormatTrailingComments::Node(node) => comments.trailing_comments(node),
FormatTrailingComments::Comments(comments) => comments,
};
let mut total_lines_before = 0;
for comment in trailing_comments {
total_lines_before += comment.lines_before();
let format_comment = FormatRefWithRule::new(comment, Context::CommentRule::default());
// This allows comments at the end of nested structures:
// {
// x: 1,
// y: 2
// // A comment
// }
// Those kinds of comments are almost always leading comments, but
// here it doesn't go "outside" the block and turns it into a
// trailing comment for `2`. We can simulate the above by checking
// if this a comment on its own line; normal trailing comments are
// always at the end of another expression.
if total_lines_before > 0 {
write!(
f,
[
line_suffix(&format_with(|f| {
match comment.lines_before() {
0 | 1 => write!(f, [hard_line_break()])?,
_ => write!(f, [empty_line()])?,
};
write!(f, [format_comment])
})),
expand_parent()
]
)?;
} else {
let content = format_with(|f| write!(f, [space(), format_comment]));
if comment.kind().is_line() {
write!(f, [line_suffix(&content), expand_parent()])?;
} else {
write!(f, [content])?;
}
}
comment.mark_formatted();
}
Ok(())
}
}
/// Formats the dangling comments of `node`.
pub const fn format_dangling_comments<L: Language>(
node: &SyntaxNode<L>,
) -> FormatDanglingComments<L> {
FormatDanglingComments::Node {
node,
indent: DanglingIndentMode::None,
}
}
/// Formats the dangling trivia of `token`.
pub enum FormatDanglingComments<'a, L: Language> {
Node {
node: &'a SyntaxNode<L>,
indent: DanglingIndentMode,
},
Comments {
comments: &'a [SourceComment<L>],
indent: DanglingIndentMode,
},
}
#[derive(Copy, Clone, Debug)]
pub enum DanglingIndentMode {
/// Writes every comment on its own line and indents them with a block indent.
///
/// # Examples
/// ```ignore
/// [
/// /* comment */
/// ]
///
/// [
/// /* comment */
/// /* multiple */
/// ]
/// ```
Block,
/// Writes every comment on its own line and indents them with a soft line indent.
/// Guarantees to write a line break if the last formatted comment is a [line](CommentKind::Line) comment.
///
/// # Examples
///
/// ```ignore
/// [/* comment */]
///
/// [
/// /* comment */
/// /* other */
/// ]
///
/// [
/// // line
/// ]
/// ```
Soft,
/// Writes every comment on its own line.
None,
}
impl<L: Language> FormatDanglingComments<'_, L> {
/// Indents the comments with a [block](DanglingIndentMode::Block) indent.
pub fn with_block_indent(self) -> Self {
self.with_indent_mode(DanglingIndentMode::Block)
}
/// Indents the comments with a [soft block](DanglingIndentMode::Soft) indent.
pub fn with_soft_block_indent(self) -> Self {
self.with_indent_mode(DanglingIndentMode::Soft)
}
fn with_indent_mode(mut self, mode: DanglingIndentMode) -> Self {
match &mut self {
FormatDanglingComments::Node { indent, .. } => *indent = mode,
FormatDanglingComments::Comments { indent, .. } => *indent = mode,
}
self
}
const fn indent(&self) -> DanglingIndentMode {
match self {
FormatDanglingComments::Node { indent, .. } => *indent,
FormatDanglingComments::Comments { indent, .. } => *indent,
}
}
}
impl<Context> Format<Context> for FormatDanglingComments<'_, Context::Language>
where
Context: CstFormatContext,
{
fn fmt(&self, f: &mut Formatter<Context>) -> FormatResult<()> {
let comments = f.context().comments().clone();
let dangling_comments = match self {
FormatDanglingComments::Node { node, .. } => comments.dangling_comments(node),
FormatDanglingComments::Comments { comments, .. } => *comments,
};
if dangling_comments.is_empty() {
return Ok(());
}
let format_dangling_comments = format_with(|f| {
// Write all comments up to the first skipped token trivia or the token
let mut join = f.join_with(hard_line_break());
for comment in dangling_comments {
let format_comment =
FormatRefWithRule::new(comment, Context::CommentRule::default());
join.entry(&format_comment);
comment.mark_formatted();
}
join.finish()?;
if matches!(self.indent(), DanglingIndentMode::Soft)
&& dangling_comments
.last()
.map_or(false, |comment| comment.kind().is_line())
{
write!(f, [hard_line_break()])?;
}
Ok(())
});
match self.indent() {
DanglingIndentMode::Block => {
write!(f, [block_indent(&format_dangling_comments)])
}
DanglingIndentMode::Soft => {
write!(f, [group(&soft_block_indent(&format_dangling_comments))])
}
DanglingIndentMode::None => {
write!(f, [format_dangling_comments])
}
}
}
}
/// Formats a token without its skipped token trivia
///
/// ## Warning
/// It's your responsibility to format any skipped trivia.
pub const fn format_trimmed_token<L: Language>(token: &SyntaxToken<L>) -> FormatTrimmedToken<L> {
FormatTrimmedToken { token }
}
#[derive(Debug, Eq, PartialEq, Copy, Clone)]
pub struct FormatTrimmedToken<'a, L: Language> {
token: &'a SyntaxToken<L>,
}
impl<L: Language + 'static, C> Format<C> for FormatTrimmedToken<'_, L>
where
C: CstFormatContext<Language = L>,
{
fn fmt(&self, f: &mut Formatter<C>) -> FormatResult<()> {
let trimmed_range = self.token.text_trimmed_range();
syntax_token_text_slice(self.token, trimmed_range).fmt(f)
}
}
/// Formats the skipped token trivia of a removed token and marks the token as tracked.
pub const fn format_removed<L>(token: &SyntaxToken<L>) -> FormatRemoved<L>
where
L: Language,
{
FormatRemoved { token }
}
/// Formats the trivia of a token that is present in the source text but should be omitted in the
/// formatted output.
pub struct FormatRemoved<'a, L>
where
L: Language,
{
token: &'a SyntaxToken<L>,
}
impl<C, L> Format<C> for FormatRemoved<'_, L>
where
L: Language + 'static,
C: CstFormatContext<Language = L>,
{
fn fmt(&self, f: &mut Formatter<C>) -> FormatResult<()> {
f.state_mut().track_token(self.token);
write!(f, [format_skipped_token_trivia(self.token)])
}
}
/// Print out a `token` from the original source with a different `content`.
///
/// This will print the skipped token trivia that belong to `token` to `content`;
/// `token` is then marked as consumed by the formatter.
pub fn format_replaced<'a, 'content, L, Context>(
token: &'a SyntaxToken<L>,
content: &'content impl Format<Context>,
) -> FormatReplaced<'a, 'content, L, Context>
where
L: Language,
{
FormatReplaced {
token,
content: Argument::new(content),
}
}
/// Formats a token's skipped token trivia but uses the provided content instead
/// of the token in the formatted output.
#[derive(Copy, Clone)]
pub struct FormatReplaced<'a, 'content, L, C>
where
L: Language,
{
token: &'a SyntaxToken<L>,
content: Argument<'content, C>,
}
impl<L, C> Format<C> for FormatReplaced<'_, '_, L, C>
where
L: Language + 'static,
C: CstFormatContext<Language = L>,
{
fn fmt(&self, f: &mut Formatter<C>) -> FormatResult<()> {
f.state_mut().track_token(self.token);
write!(f, [format_skipped_token_trivia(self.token)])?;
f.write_fmt(Arguments::from(&self.content))
}
}
/// Formats the given token only if the group does break and otherwise retains the token's skipped token trivia.
pub fn format_only_if_breaks<'a, 'content, L, Content, Context>(
token: &'a SyntaxToken<L>,
content: &'content Content,
) -> FormatOnlyIfBreaks<'a, 'content, L, Context>
where
L: Language,
Content: Format<Context>,
{
FormatOnlyIfBreaks {
token,
content: Argument::new(content),
group_id: None,
}
}
/// Formats a token with its skipped token trivia that only gets printed if its enclosing
/// group does break but otherwise gets omitted from the formatted output.
pub struct FormatOnlyIfBreaks<'a, 'content, L, C>
where
L: Language,
{
token: &'a SyntaxToken<L>,
content: Argument<'content, C>,
group_id: Option<GroupId>,
}
impl<'a, 'content, L, C> FormatOnlyIfBreaks<'a, 'content, L, C>
where
L: Language,
{
pub fn with_group_id(mut self, group_id: Option<GroupId>) -> Self {
self.group_id = group_id;
self
}
}
impl<L, C> Format<C> for FormatOnlyIfBreaks<'_, '_, L, C>
where
L: Language + 'static,
C: CstFormatContext<Language = L>,
{
fn fmt(&self, f: &mut Formatter<C>) -> FormatResult<()> {
write!(
f,
[if_group_breaks(&Arguments::from(&self.content)).with_group_id(self.group_id),]
)?;
if f.comments().has_skipped(self.token) {
// Print the trivia otherwise
write!(
f,
[
if_group_fits_on_line(&format_skipped_token_trivia(self.token))
.with_group_id(self.group_id)
]
)?;
}
Ok(())
}
}
/// Formats the skipped token trivia of `token`.
pub const fn format_skipped_token_trivia<L: Language>(
token: &SyntaxToken<L>,
) -> FormatSkippedTokenTrivia<L> {
FormatSkippedTokenTrivia { token }
}
/// Formats the skipped token trivia of `token`.
pub struct FormatSkippedTokenTrivia<'a, L: Language> {
token: &'a SyntaxToken<L>,
}
impl<L: Language> FormatSkippedTokenTrivia<'_, L> {
#[cold]
fn fmt_skipped<Context>(&self, f: &mut Formatter<Context>) -> FormatResult<()>
where
Context: CstFormatContext<Language = L>,
{
// Lines/spaces before the next token/comment
let (mut lines, mut spaces) = match self.token.prev_token() {
Some(token) => {
let mut lines = 0u32;
let mut spaces = 0u32;
for piece in token.trailing_trivia().pieces().rev() {
if piece.is_whitespace() {
spaces += 1;
} else if piece.is_newline() {
spaces = 0;
lines += 1;
} else {
break;
}
}
(lines, spaces)
}
None => (0, 0),
};
// The comments between the last skipped token trivia and the token
let mut dangling_comments = Vec::new();
let mut skipped_range: Option<TextRange> = None;
// Iterate over the remaining pieces to find the full range from the first to the last skipped token trivia.
// Extract the comments between the last skipped token trivia and the token.
for piece in self.token.leading_trivia().pieces() {
if piece.is_whitespace() {
spaces += 1;
continue;
}
if piece.is_newline() {
lines += 1;
spaces = 0;
} else if let Some(comment) = piece.as_comments() {
let source_comment = SourceComment {
kind: Context::Style::get_comment_kind(&comment),
lines_before: lines,
lines_after: 0,
piece: comment,
#[cfg(debug_assertions)]
formatted: Cell::new(true),
};
dangling_comments.push(source_comment);
lines = 0;
spaces = 0;
} else if piece.is_skipped() {
skipped_range = Some(match skipped_range {
Some(range) => range.cover(piece.text_range()),
None => {
if dangling_comments.is_empty() {
match lines {
0 if spaces == 0 => {
// Token had no space to previous token nor any preceding comment. Keep it that way
}
0 => write!(f, [space()])?,
_ => write!(f, [hard_line_break()])?,
};
} else {
match lines {
0 => write!(f, [space()])?,
1 => write!(f, [hard_line_break()])?,
_ => write!(f, [empty_line()])?,
};
}
piece.text_range()
}
});
lines = 0;
spaces = 0;
dangling_comments.clear();
}
}
let skipped_range =
skipped_range.unwrap_or_else(|| TextRange::empty(self.token.text_range().start()));
f.write_element(FormatElement::Tag(Tag::StartVerbatim(
VerbatimKind::Verbatim {
length: skipped_range.len(),
},
)))?;
write!(f, [syntax_token_text_slice(self.token, skipped_range)])?;
f.write_element(FormatElement::Tag(Tag::EndVerbatim))?;
// Write whitespace separator between skipped/last comment and token
if dangling_comments.is_empty() {
match lines {
0 if spaces == 0 => {
// Don't write a space if there was non in the source document
Ok(())
}
0 => write!(f, [space()]),
_ => write!(f, [hard_line_break()]),
}
} else {
match dangling_comments.first().unwrap().lines_before {
0 => write!(f, [space()])?,
1 => write!(f, [hard_line_break()])?,
_ => write!(f, [empty_line()])?,
}
write!(
f,
[FormatDanglingComments::Comments {
comments: &dangling_comments,
indent: DanglingIndentMode::None
}]
)?;
match lines {
0 => write!(f, [space()]),
_ => write!(f, [hard_line_break()]),
}
}
}
}
impl<Context> Format<Context> for FormatSkippedTokenTrivia<'_, Context::Language>
where
Context: CstFormatContext,
{
fn fmt(&self, f: &mut Formatter<Context>) -> FormatResult<()> {
if f.comments().has_skipped(self.token) {
self.fmt_skipped(f)
} else {
Ok(())
}
}
}

205
crates/ruff_formatter/src/verbatim.rs Normal file
View File

@ -0,0 +1,205 @@
use crate::format_element::tag::VerbatimKind;
use crate::prelude::*;
use crate::trivia::{FormatLeadingComments, FormatTrailingComments};
use crate::{write, CstFormatContext, FormatWithRule};
use ruff_rowan::{AstNode, Direction, Language, SyntaxElement, SyntaxNode, TextRange};
/// "Formats" a node according to its original formatting in the source text. Being able to format
/// a node "as is" is useful if a node contains syntax errors. Formatting a node with syntax errors
/// has the risk that Rome misinterprets the structure of the code and formatting it could
/// "mess up" the developers, yet incomplete, work or accidentally introduce new syntax errors.
///
/// You may be inclined to call `node.text` directly. However, using `text` doesn't track the nodes
/// nor its children source mapping information, resulting in incorrect source maps for this subtree.
///
/// These nodes and tokens get tracked as [VerbatimKind::Verbatim], useful to understand
/// if these nodes still need to have their own implementation.
pub fn format_verbatim_node<L: Language>(node: &SyntaxNode<L>) -> FormatVerbatimNode<L> {
FormatVerbatimNode {
node,
kind: VerbatimKind::Verbatim {
length: node.text_range().len(),
},
format_comments: true,
}
}
#[derive(Debug, Clone, Copy, Eq, PartialEq)]
pub struct FormatVerbatimNode<'node, L: Language> {
node: &'node SyntaxNode<L>,
kind: VerbatimKind,
format_comments: bool,
}
impl<Context> Format<Context> for FormatVerbatimNode<'_, Context::Language>
where
Context: CstFormatContext,
{
fn fmt(&self, f: &mut Formatter<Context>) -> FormatResult<()> {
for element in self.node.descendants_with_tokens(Direction::Next) {
match element {
SyntaxElement::Token(token) => f.state_mut().track_token(&token),
SyntaxElement::Node(node) => {
let comments = f.context().comments();
comments.mark_suppression_checked(&node);
for comment in comments.leading_dangling_trailing_comments(&node) {
comment.mark_formatted();
}
}
}
}
// The trimmed range of a node is its range without any of its leading or trailing trivia.
// Except for nodes that used to be parenthesized, the range than covers the source from the
// `(` to the `)` (the trimmed range of the parenthesized expression, not the inner expression)
let trimmed_source_range = f.context().source_map().map_or_else(
|| self.node.text_trimmed_range(),
|source_map| source_map.trimmed_source_range(self.node),
);
f.write_element(FormatElement::Tag(Tag::StartVerbatim(self.kind)))?;
fn source_range<Context>(f: &Formatter<Context>, range: TextRange) -> TextRange
where
Context: CstFormatContext,
{
f.context()
.source_map()
.map_or_else(|| range, |source_map| source_map.source_range(range))
}
// Format all leading comments that are outside of the node's source range.
if self.format_comments {
let comments = f.context().comments().clone();
let leading_comments = comments.leading_comments(self.node);
let outside_trimmed_range = leading_comments.partition_point(|comment| {
comment.piece().text_range().end() <= trimmed_source_range.start()
});
let (outside_trimmed_range, in_trimmed_range) =
leading_comments.split_at(outside_trimmed_range);
write!(f, [FormatLeadingComments::Comments(outside_trimmed_range)])?;
for comment in in_trimmed_range {
comment.mark_formatted();
}
}
// Find the first skipped token trivia, if any, and include it in the verbatim range because
// the comments only format **up to** but not including skipped token trivia.
let start_source = self
.node
.first_leading_trivia()
.into_iter()
.flat_map(|trivia| trivia.pieces())
.filter(|trivia| trivia.is_skipped())
.map(|trivia| source_range(f, trivia.text_range()).start())
.take_while(|start| *start < trimmed_source_range.start())
.next()
.unwrap_or_else(|| trimmed_source_range.start());
let original_source = f.context().source_map().map_or_else(
|| self.node.text_trimmed().to_string(),
|source_map| {
source_map.text()[trimmed_source_range.cover_offset(start_source)].to_string()
},
);
dynamic_text(
&normalize_newlines(&original_source, LINE_TERMINATORS),
self.node.text_trimmed_range().start(),
)
.fmt(f)?;
for comment in f.context().comments().dangling_comments(self.node) {
comment.mark_formatted();
}
// Format all trailing comments that are outside of the trimmed range.
if self.format_comments {
let comments = f.context().comments().clone();
let trailing_comments = comments.trailing_comments(self.node);
let outside_trimmed_range_start = trailing_comments.partition_point(|comment| {
source_range(f, comment.piece().text_range()).end() <= trimmed_source_range.end()
});
let (in_trimmed_range, outside_trimmed_range) =
trailing_comments.split_at(outside_trimmed_range_start);
for comment in in_trimmed_range {
comment.mark_formatted();
}
write!(f, [FormatTrailingComments::Comments(outside_trimmed_range)])?;
}
f.write_element(FormatElement::Tag(Tag::EndVerbatim))
}
}
impl<L: Language> FormatVerbatimNode<'_, L> {
pub fn skip_comments(mut self) -> Self {
self.format_comments = false;
self
}
}
/// Formats bogus nodes. The difference between this method and `format_verbatim` is that this method
/// doesn't track nodes/tokens as [VerbatimKind::Verbatim]. They are just printed as they are.
pub fn format_bogus_node<L: Language>(node: &SyntaxNode<L>) -> FormatVerbatimNode<L> {
FormatVerbatimNode {
node,
kind: VerbatimKind::Bogus,
format_comments: true,
}
}
/// Format a node having formatter suppression comment applied to it
pub fn format_suppressed_node<L: Language>(node: &SyntaxNode<L>) -> FormatVerbatimNode<L> {
FormatVerbatimNode {
node,
kind: VerbatimKind::Suppressed,
format_comments: true,
}
}
/// Formats an object using its [`Format`] implementation but falls back to printing the object as
/// it is in the source document if formatting it returns an [`FormatError::SyntaxError`].
pub const fn format_or_verbatim<F>(inner: F) -> FormatNodeOrVerbatim<F> {
FormatNodeOrVerbatim { inner }
}
/// Formats a node or falls back to verbatim printing if formatting this node fails.
#[derive(Copy, Clone)]
pub struct FormatNodeOrVerbatim<F> {
inner: F,
}
impl<F, Context, Item> Format<Context> for FormatNodeOrVerbatim<F>
where
F: FormatWithRule<Context, Item = Item>,
Item: AstNode,
Context: CstFormatContext<Language = Item::Language>,
{
fn fmt(&self, f: &mut Formatter<Context>) -> FormatResult<()> {
let snapshot = Formatter::state_snapshot(f);
match self.inner.fmt(f) {
Ok(result) => Ok(result),
Err(FormatError::SyntaxError) => {
f.restore_state_snapshot(snapshot);
// Lists that yield errors are formatted as they were suppressed nodes.
// Doing so, the formatter formats the nodes/tokens as is.
format_suppressed_node(self.inner.item().syntax()).fmt(f)
}
Err(err) => Err(err),
}
}
}

29
crates/ruff_rowan/Cargo.toml Normal file
View File

@ -0,0 +1,29 @@
[package]
name = "ruff_rowan"
version = "0.0.0"
publish = false
edition = "2021"
[dependencies]
countme = { version = "3.0.1" }
hashbrown = { version = "0.12.3", features = ["inline-more"], default-features = false }
memoffset = { version = "0.6.5" }
ruff_text_edit = { path = "../ruff_text_edit" }
ruff_text_size = { path = "../ruff_text_size" }
rustc-hash = { workspace = true }
schemars = { version = "0.8.10", optional = true }
serde = { version = "1.0.133", optional = true, default-features = false }
tracing = { version = "0.1.31", default-features = false, features = ["std"] }
[dev-dependencies]
quickcheck = "1.0.3"
quickcheck_macros = "1.0.0"
serde_json = "1.0.79"
iai = "*"
[features]
serde = ["dep:serde", "schemars", "ruff_text_size/serde"]
[[bench]]
name = "mutation"
harness = false

69
crates/ruff_rowan/benches/mutation.rs Normal file
View File

@ -0,0 +1,69 @@
use ruff_rowan::{
raw_language::{LiteralExpression, RawLanguageKind, RawLanguageRoot, RawSyntaxTreeBuilder},
AstNode, AstNodeExt, BatchMutationExt, SyntaxNodeCast,
};
/// ```
/// 0: ROOT@0..1
/// 0: LITERAL_EXPRESSION@0..1
/// 0: STRING_TOKEN@0..1 "a" [] []
/// ```
fn tree_one(a: &str) -> (RawLanguageRoot, String) {
let mut builder = RawSyntaxTreeBuilder::new();
builder
.start_node(RawLanguageKind::ROOT)
.start_node(RawLanguageKind::LITERAL_EXPRESSION)
.token(RawLanguageKind::STRING_TOKEN, a)
.finish_node()
.finish_node();
let root = builder.finish().cast::<RawLanguageRoot>().unwrap();
let s = format!("{:#?}", root.syntax());
(root, s)
}
fn find(root: &RawLanguageRoot, name: &str) -> LiteralExpression {
root.syntax()
.descendants()
.find(|x| x.kind() == RawLanguageKind::LITERAL_EXPRESSION && x.text_trimmed() == name)
.unwrap()
.cast::<LiteralExpression>()
.unwrap()
}
fn clone_detach(root: &RawLanguageRoot, name: &str) -> LiteralExpression {
root.syntax()
.descendants()
.find(|x| x.kind() == RawLanguageKind::LITERAL_EXPRESSION && x.text_trimmed() == name)
.unwrap()
.detach()
.cast::<LiteralExpression>()
.unwrap()
}
fn mutation_replace_node() -> usize {
let (before, _) = tree_one("a");
let (expected, _) = tree_one("b");
let a = find(&before, "a");
let b = clone_detach(&expected, "b");
let root = before.replace_node(a, b).unwrap();
root.syntax().descendants().count()
}
fn mutation_batch() -> usize {
let (before, _) = tree_one("a");
let (expected, _) = tree_one("b");
let a = find(&before, "a");
let b = clone_detach(&expected, "b");
let mut batch = before.begin();
batch.replace_node(a, b);
let root = batch.commit();
root.descendants().count()
}
iai::main!(mutation_replace_node, mutation_batch);

497
crates/ruff_rowan/src/arc.rs Normal file
View File

@ -0,0 +1,497 @@
//! Vendored and stripped down version of triomphe
use std::{
alloc::{self, Layout},
cmp::Ordering,
hash::{Hash, Hasher},
marker::PhantomData,
mem::{self, ManuallyDrop},
ops::Deref,
ptr,
sync::atomic::{
self,
Ordering::{Acquire, Relaxed, Release},
},
};
use memoffset::offset_of;
/// A soft limit on the amount of references that may be made to an `Arc`.
///
/// Going above this limit will abort your program (although not
/// necessarily) at _exactly_ `MAX_REFCOUNT + 1` references.
const MAX_REFCOUNT: usize = (isize::MAX) as usize;
/// The object allocated by an Arc<T>
#[repr(C)]
pub(crate) struct ArcInner<T: ?Sized> {
pub(crate) count: atomic::AtomicUsize,
pub(crate) data: T,
}
unsafe impl<T: ?Sized + Sync + Send> Send for ArcInner<T> {}
unsafe impl<T: ?Sized + Sync + Send> Sync for ArcInner<T> {}
/// An atomically reference counted shared pointer
///
/// See the documentation for [`Arc`] in the standard library. Unlike the
/// standard library `Arc`, this `Arc` does not support weak reference counting.
///
/// [`Arc`]: https://doc.rust-lang.org/stable/std/sync/struct.Arc.html
#[repr(transparent)]
pub(crate) struct Arc<T: ?Sized> {
pub(crate) p: ptr::NonNull<ArcInner<T>>,
pub(crate) phantom: PhantomData<T>,
}
unsafe impl<T: ?Sized + Sync + Send> Send for Arc<T> {}
unsafe impl<T: ?Sized + Sync + Send> Sync for Arc<T> {}
impl<T> Arc<T> {
/// Reconstruct the Arc<T> from a raw pointer obtained from into_raw()
///
/// Note: This raw pointer will be offset in the allocation and must be preceded
/// by the atomic count.
///
/// It is recommended to use OffsetArc for this
#[inline]
pub(crate) unsafe fn from_raw(ptr: *const T) -> Self {
// To find the corresponding pointer to the `ArcInner` we need
// to subtract the offset of the `data` field from the pointer.
let ptr = (ptr as *const u8).sub(offset_of!(ArcInner<T>, data));
Arc {
p: ptr::NonNull::new_unchecked(ptr as *mut ArcInner<T>),
phantom: PhantomData,
}
}
}
impl<T: ?Sized> Arc<T> {
#[inline]
fn inner(&self) -> &ArcInner<T> {
// This unsafety is ok because while this arc is alive we're guaranteed
// that the inner pointer is valid. Furthermore, we know that the
// `ArcInner` structure itself is `Sync` because the inner data is
// `Sync` as well, so we're ok loaning out an immutable pointer to these
// contents.
unsafe { &*self.ptr() }
}
// Non-inlined part of `drop`. Just invokes the destructor.
#[inline(never)]
unsafe fn drop_slow(&mut self) {
let _ = Box::from_raw(self.ptr());
}
/// Test pointer equality between the two Arcs, i.e. they must be the _same_
/// allocation
#[inline]
pub(crate) fn ptr_eq(this: &Self, other: &Self) -> bool {
this.ptr() == other.ptr()
}
pub(crate) fn ptr(&self) -> *mut ArcInner<T> {
self.p.as_ptr()
}
}
impl<T: ?Sized> Clone for Arc<T> {
#[inline]
fn clone(&self) -> Self {
// Using a relaxed ordering is alright here, as knowledge of the
// original reference prevents other threads from erroneously deleting
// the object.
//
// As explained in the [Boost documentation][1], Increasing the
// reference counter can always be done with memory_order_relaxed: New
// references to an object can only be formed from an existing
// reference, and passing an existing reference from one thread to
// another must already provide any required synchronization.
//
// [1]: (www.boost.org/doc/libs/1_55_0/doc/html/atomic/usage_examples.html)
let old_size = self.inner().count.fetch_add(1, Relaxed);
// However we need to guard against massive refcounts in case someone
// is `mem::forget`ing Arcs. If we don't do this the count can overflow
// and users will use-after free. We racily saturate to `isize::MAX` on
// the assumption that there aren't ~2 billion threads incrementing
// the reference count at once. This branch will never be taken in
// any realistic program.
//
// We abort because such a program is incredibly degenerate, and we
// don't care to support it.
if old_size > MAX_REFCOUNT {
std::process::abort();
}
unsafe {
Arc {
p: ptr::NonNull::new_unchecked(self.ptr()),
phantom: PhantomData,
}
}
}
}
impl<T: ?Sized> Deref for Arc<T> {
type Target = T;
#[inline]
fn deref(&self) -> &T {
&self.inner().data
}
}
impl<T: ?Sized> Arc<T> {
/// Provides mutable access to the contents _if_ the `Arc` is uniquely owned.
#[inline]
pub(crate) fn get_mut(this: &mut Self) -> Option<&mut T> {
if this.is_unique() {
unsafe {
// See make_mut() for documentation of the threadsafety here.
Some(&mut (*this.ptr()).data)
}
} else {
None
}
}
/// Whether or not the `Arc` is uniquely owned (is the refcount 1?).
pub(crate) fn is_unique(&self) -> bool {
// See the extensive discussion in [1] for why this needs to be Acquire.
//
// [1] https://github.com/servo/servo/issues/21186
self.inner().count.load(Acquire) == 1
}
}
impl<T: ?Sized> Drop for Arc<T> {
#[inline]
fn drop(&mut self) {
// Because `fetch_sub` is already atomic, we do not need to synchronize
// with other threads unless we are going to delete the object.
if self.inner().count.fetch_sub(1, Release) != 1 {
return;
}
// FIXME(bholley): Use the updated comment when [2] is merged.
//
// This load is needed to prevent reordering of use of the data and
// deletion of the data. Because it is marked `Release`, the decreasing
// of the reference count synchronizes with this `Acquire` load. This
// means that use of the data happens before decreasing the reference
// count, which happens before this load, which happens before the
// deletion of the data.
//
// As explained in the [Boost documentation][1],
//
// > It is important to enforce any possible access to the object in one
// > thread (through an existing reference) to *happen before* deleting
// > the object in a different thread. This is achieved by a "release"
// > operation after dropping a reference (any access to the object
// > through this reference must obviously happened before), and an
// > "acquire" operation before deleting the object.
//
// [1]: (www.boost.org/doc/libs/1_55_0/doc/html/atomic/usage_examples.html)
// [2]: https://github.com/rust-lang/rust/pull/41714
self.inner().count.load(Acquire);
unsafe {
self.drop_slow();
}
}
}
impl<T: ?Sized + PartialEq> PartialEq for Arc<T> {
fn eq(&self, other: &Arc<T>) -> bool {
Self::ptr_eq(self, other) || *(*self) == *(*other)
}
}
impl<T: ?Sized + PartialOrd> PartialOrd for Arc<T> {
fn partial_cmp(&self, other: &Arc<T>) -> Option<Ordering> {
(**self).partial_cmp(&**other)
}
fn lt(&self, other: &Arc<T>) -> bool {
*(*self) < *(*other)
}
fn le(&self, other: &Arc<T>) -> bool {
*(*self) <= *(*other)
}
fn gt(&self, other: &Arc<T>) -> bool {
*(*self) > *(*other)
}
fn ge(&self, other: &Arc<T>) -> bool {
*(*self) >= *(*other)
}
}
impl<T: ?Sized + Ord> Ord for Arc<T> {
fn cmp(&self, other: &Arc<T>) -> Ordering {
(**self).cmp(&**other)
}
}
impl<T: ?Sized + Eq> Eq for Arc<T> {}
impl<T: ?Sized + Hash> Hash for Arc<T> {
fn hash<H: Hasher>(&self, state: &mut H) {
(**self).hash(state)
}
}
#[derive(Debug, Eq, PartialEq, Hash, PartialOrd)]
#[repr(C)]
pub(crate) struct HeaderSlice<H, T: ?Sized> {
pub(crate) header: H,
length: usize,
slice: T,
}
impl<H, T> HeaderSlice<H, [T]> {
pub(crate) fn slice(&self) -> &[T] {
&self.slice
}
/// Returns the number of items
pub(crate) fn len(&self) -> usize {
self.length
}
}
impl<H, T> Deref for HeaderSlice<H, [T; 0]> {
type Target = HeaderSlice<H, [T]>;
fn deref(&self) -> &Self::Target {
unsafe {
let len = self.length;
let fake_slice: *const [T] =
ptr::slice_from_raw_parts(self as *const _ as *const T, len);
&*(fake_slice as *const HeaderSlice<H, [T]>)
}
}
}
/// A "thin" `Arc` containing dynamically sized data
///
/// This is functionally equivalent to `Arc<(H, [T])>`
///
/// When you create an `Arc` containing a dynamically sized type
/// like `HeaderSlice<H, [T]>`, the `Arc` is represented on the stack
/// as a "fat pointer", where the length of the slice is stored
/// alongside the `Arc`'s pointer. In some situations you may wish to
/// have a thin pointer instead, perhaps for FFI compatibility
/// or space efficiency.
///
/// Note that we use `[T; 0]` in order to have the right alignment for `T`.
///
/// `ThinArc` solves this by storing the length in the allocation itself,
/// via `HeaderSlice`.
#[repr(transparent)]
pub(crate) struct ThinArc<H, T> {
ptr: ptr::NonNull<ArcInner<HeaderSlice<H, [T; 0]>>>,
phantom: PhantomData<(H, T)>,
}
unsafe impl<H: Sync + Send, T: Sync + Send> Send for ThinArc<H, T> {}
unsafe impl<H: Sync + Send, T: Sync + Send> Sync for ThinArc<H, T> {}
// Synthesize a fat pointer from a thin pointer.
fn thin_to_thick<H, T>(
thin: *mut ArcInner<HeaderSlice<H, [T; 0]>>,
) -> *mut ArcInner<HeaderSlice<H, [T]>> {
let len = unsafe { (*thin).data.length };
let fake_slice: *mut [T] = ptr::slice_from_raw_parts_mut(thin as *mut T, len);
// Transplants metadata.
fake_slice as *mut ArcInner<HeaderSlice<H, [T]>>
}
impl<H, T> ThinArc<H, T> {
/// Temporarily converts |self| into a bonafide Arc and exposes it to the
/// provided callback. The refcount is not modified.
#[inline]
pub(crate) fn with_arc<F, U>(&self, f: F) -> U
where
F: FnOnce(&Arc<HeaderSlice<H, [T]>>) -> U,
{
// Synthesize transient Arc, which never touches the refcount of the ArcInner.
let transient = unsafe {
ManuallyDrop::new(Arc {
p: ptr::NonNull::new_unchecked(thin_to_thick(self.ptr.as_ptr())),
phantom: PhantomData,
})
};
// Expose the transient Arc to the callback, which may clone it if it wants.
// Forward the result.
f(&transient)
}
/// Creates a `ThinArc` for a HeaderSlice using the given header struct and
/// iterator to generate the slice.
pub(crate) fn from_header_and_iter<I>(header: H, mut items: I) -> Self
where
I: Iterator<Item = T> + ExactSizeIterator,
{
assert_ne!(mem::size_of::<T>(), 0, "Need to think about ZST");
let num_items = items.len();
// Offset of the start of the slice in the allocation.
let inner_to_data_offset = offset_of!(ArcInner<HeaderSlice<H, [T; 0]>>, data);
let data_to_slice_offset = offset_of!(HeaderSlice<H, [T; 0]>, slice);
let slice_offset = inner_to_data_offset + data_to_slice_offset;
// Compute the size of the real payload.
let slice_size = mem::size_of::<T>()
.checked_mul(num_items)
.expect("size overflows");
let usable_size = slice_offset
.checked_add(slice_size)
.expect("size overflows");
// Round up size to alignment.
let align = mem::align_of::<ArcInner<HeaderSlice<H, [T; 0]>>>();
let size = usable_size.wrapping_add(align - 1) & !(align - 1);
assert!(size >= usable_size, "size overflows");
let layout = Layout::from_size_align(size, align).expect("invalid layout");
let ptr: *mut ArcInner<HeaderSlice<H, [T; 0]>>;
unsafe {
let buffer = alloc::alloc(layout);
if buffer.is_null() {
alloc::handle_alloc_error(layout);
}
// // Synthesize the fat pointer. We do this by claiming we have a direct
// // pointer to a [T], and then changing the type of the borrow. The key
// // point here is that the length portion of the fat pointer applies
// // only to the number of elements in the dynamically-sized portion of
// // the type, so the value will be the same whether it points to a [T]
// // or something else with a [T] as its last member.
// let fake_slice: &mut [T] = slice::from_raw_parts_mut(buffer as *mut T, num_items);
// ptr = fake_slice as *mut [T] as *mut ArcInner<HeaderSlice<H, [T]>>;
ptr = buffer as *mut _;
let count = atomic::AtomicUsize::new(1);
// Write the data.
//
// Note that any panics here (i.e. from the iterator) are safe, since
// we'll just leak the uninitialized memory.
ptr::write(ptr::addr_of_mut!((*ptr).count), count);
ptr::write(ptr::addr_of_mut!((*ptr).data.header), header);
ptr::write(ptr::addr_of_mut!((*ptr).data.length), num_items);
if num_items != 0 {
let mut current = ptr::addr_of_mut!((*ptr).data.slice) as *mut T;
debug_assert_eq!(current as usize - buffer as usize, slice_offset);
for _ in 0..num_items {
ptr::write(
current,
items
.next()
.expect("ExactSizeIterator over-reported length"),
);
current = current.offset(1);
}
assert!(
items.next().is_none(),
"ExactSizeIterator under-reported length"
);
// We should have consumed the buffer exactly.
debug_assert_eq!(current as *mut u8, buffer.add(usable_size));
}
assert!(
items.next().is_none(),
"ExactSizeIterator under-reported length"
);
}
ThinArc {
ptr: unsafe { ptr::NonNull::new_unchecked(ptr) },
phantom: PhantomData,
}
}
}
impl<H, T> Deref for ThinArc<H, T> {
type Target = HeaderSlice<H, [T]>;
#[inline]
fn deref(&self) -> &Self::Target {
unsafe { &(*thin_to_thick(self.ptr.as_ptr())).data }
}
}
impl<H, T> Clone for ThinArc<H, T> {
#[inline]
fn clone(&self) -> Self {
ThinArc::with_arc(self, |a| Arc::into_thin(a.clone()))
}
}
impl<H, T> Drop for ThinArc<H, T> {
#[inline]
fn drop(&mut self) {
let _ = Arc::from_thin(ThinArc {
ptr: self.ptr,
phantom: PhantomData,
});
}
}
impl<H, T> Arc<HeaderSlice<H, [T]>> {
/// Converts an `Arc` into a `ThinArc`. This consumes the `Arc`, so the refcount
/// is not modified.
#[inline]
pub(crate) fn into_thin(a: Self) -> ThinArc<H, T> {
assert_eq!(
a.length,
a.slice.len(),
"Length needs to be correct for ThinArc to work"
);
let fat_ptr: *mut ArcInner<HeaderSlice<H, [T]>> = a.ptr();
mem::forget(a);
let thin_ptr = fat_ptr as *mut [usize] as *mut usize;
ThinArc {
ptr: unsafe {
ptr::NonNull::new_unchecked(thin_ptr as *mut ArcInner<HeaderSlice<H, [T; 0]>>)
},
phantom: PhantomData,
}
}
/// Converts a `ThinArc` into an `Arc`. This consumes the `ThinArc`, so the refcount
/// is not modified.
#[inline]
pub(crate) fn from_thin(a: ThinArc<H, T>) -> Self {
let ptr = thin_to_thick(a.ptr.as_ptr());
mem::forget(a);
unsafe {
Arc {
p: ptr::NonNull::new_unchecked(ptr),
phantom: PhantomData,
}
}
}
}
impl<H: PartialEq, T: PartialEq> PartialEq for ThinArc<H, T> {
#[inline]
fn eq(&self, other: &ThinArc<H, T>) -> bool {
**self == **other
}
}
impl<H: Eq, T: Eq> Eq for ThinArc<H, T> {}
impl<H: Hash, T: Hash> Hash for ThinArc<H, T> {
fn hash<HSR: Hasher>(&self, state: &mut HSR) {
(**self).hash(state)
}
}

550
crates/ruff_rowan/src/ast/batch.rs Normal file
View File

@ -0,0 +1,550 @@
use crate::{
chain_trivia_pieces, AstNode, Language, SyntaxElement, SyntaxKind, SyntaxNode, SyntaxSlot,
SyntaxToken,
};
use ruff_text_edit::TextEdit;
use ruff_text_size::TextRange;
use std::{
cmp,
collections::BinaryHeap,
iter::{empty, once},
};
use tracing::debug;
pub trait BatchMutationExt<L>: AstNode<Language = L>
where
L: Language,
{
/// It starts a [BatchMutation]
#[must_use = "This method consumes the node and return the BatchMutation api that returns the new SynytaxNode on commit"]
fn begin(self) -> BatchMutation<L>;
}
impl<L, T> BatchMutationExt<L> for T
where
L: Language,
T: AstNode<Language = L>,
{
#[must_use = "This method consumes the node and return the BatchMutation api that returns the new SynytaxNode on commit"]
fn begin(self) -> BatchMutation<L> {
BatchMutation::new(self.into_syntax())
}
}
/// Stores the changes internally used by the [BatchMutation::commit] algorithm.
/// It needs to be sorted by depth in decreasing order, then by range start and
/// by slot in increasing order.
///
/// This is necesasry so we can aggregate all changes to the same node using "peek".
#[derive(Debug, Clone)]
struct CommitChange<L: Language> {
parent_depth: usize,
parent: Option<SyntaxNode<L>>,
parent_range: Option<(u32, u32)>,
new_node_slot: usize,
new_node: Option<SyntaxElement<L>>,
}
impl<L: Language> CommitChange<L> {
/// Returns the "ordering key" for a change, controlling in what order this
/// change will be applied relatively to other changes. The key consists of
/// a tuple of numeric values representing the depth, parent start and slot
/// of the corresponding change
fn key(&self) -> (usize, cmp::Reverse<u32>, cmp::Reverse<usize>) {
(
self.parent_depth,
cmp::Reverse(self.parent_range.map(|(start, _)| start).unwrap_or(0)),
cmp::Reverse(self.new_node_slot),
)
}
}
impl<L: Language> PartialEq for CommitChange<L> {
fn eq(&self, other: &Self) -> bool {
self.key() == other.key()
}
}
impl<L: Language> Eq for CommitChange<L> {}
/// We order first by depth. Then by the range of the node.
///
/// The first is important to guarantee that all nodes that will be changed
/// in the future are still valid with using SyntaxNode that we have.
///
/// The second is important to guarante that the ".peek()" we do below is sufficient
/// to see the same node in case of two or more nodes having the same depth.
impl<L: Language> PartialOrd for CommitChange<L> {
fn partial_cmp(&self, other: &Self) -> Option<cmp::Ordering> {
Some(self.cmp(other))
}
}
impl<L: Language> Ord for CommitChange<L> {
fn cmp(&self, other: &Self) -> cmp::Ordering {
self.key().cmp(&other.key())
}
}
#[derive(Debug, Clone)]
pub struct BatchMutation<L>
where
L: Language,
{
root: SyntaxNode<L>,
changes: BinaryHeap<CommitChange<L>>,
}
impl<L> BatchMutation<L>
where
L: Language,
{
pub fn new(root: SyntaxNode<L>) -> Self {
Self {
root,
changes: BinaryHeap::new(),
}
}
/// Push a change to replace the "prev_node" with "next_node".
/// Trivia from "prev_node" is automatically copied to "next_node".
///
/// Changes to take effect must be committed.
pub fn replace_node<T>(&mut self, prev_node: T, next_node: T)
where
T: AstNode<Language = L>,
{
self.replace_element(
prev_node.into_syntax().into(),
next_node.into_syntax().into(),
)
}
/// Push a change to replace the "prev_token" with "next_token".
/// Trivia from "prev_token" is automatically copied to "next_token".
///
/// Changes to take effect must be committed.
pub fn replace_token(&mut self, prev_token: SyntaxToken<L>, next_token: SyntaxToken<L>) {
self.replace_element(prev_token.into(), next_token.into())
}
/// Push a change to replace the "prev_element" with "next_element".
/// Trivia from "prev_element" is automatically copied to "next_element".
///
/// Changes to take effect must be committed.
pub fn replace_element(
&mut self,
prev_element: SyntaxElement<L>,
next_element: SyntaxElement<L>,
) {
let (prev_leading_trivia, prev_trailing_trivia) = match &prev_element {
SyntaxElement::Node(node) => (
node.first_token().map(|token| token.leading_trivia()),
node.last_token().map(|token| token.trailing_trivia()),
),
SyntaxElement::Token(token) => {
(Some(token.leading_trivia()), Some(token.trailing_trivia()))
}
};
let next_element = match next_element {
SyntaxElement::Node(mut node) => {
if let Some(token) = node.first_token() {
let new_token = match prev_leading_trivia {
Some(prev_leading_trivia) => {
token.with_leading_trivia_pieces(prev_leading_trivia.pieces())
}
None => token.with_leading_trivia_pieces(empty()),
};
node = node.replace_child(token.into(), new_token.into()).unwrap();
}
if let Some(token) = node.last_token() {
let new_token = match prev_trailing_trivia {
Some(prev_trailing_trivia) => {
token.with_trailing_trivia_pieces(prev_trailing_trivia.pieces())
}
None => token.with_trailing_trivia_pieces(empty()),
};
node = node.replace_child(token.into(), new_token.into()).unwrap();
}
SyntaxElement::Node(node)
}
SyntaxElement::Token(token) => {
let new_token = match prev_leading_trivia {
Some(prev_leading_trivia) => {
token.with_leading_trivia_pieces(prev_leading_trivia.pieces())
}
None => token.with_leading_trivia_pieces(empty()),
};
let new_token = match prev_trailing_trivia {
Some(prev_trailing_trivia) => {
new_token.with_trailing_trivia_pieces(prev_trailing_trivia.pieces())
}
None => new_token.with_trailing_trivia_pieces(empty()),
};
SyntaxElement::Token(new_token)
}
};
self.push_change(prev_element, Some(next_element))
}
/// Push a change to replace the "prev_node" with "next_node".
///
/// Changes to take effect must be committed.
pub fn replace_node_discard_trivia<T>(&mut self, prev_node: T, next_node: T)
where
T: AstNode<Language = L>,
{
self.replace_element_discard_trivia(
prev_node.into_syntax().into(),
next_node.into_syntax().into(),
)
}
/// Push a change to replace the "prev_token" with "next_token".
///
/// Changes to take effect must be committed.
pub fn replace_token_discard_trivia(
&mut self,
prev_token: SyntaxToken<L>,
next_token: SyntaxToken<L>,
) {
self.replace_element_discard_trivia(prev_token.into(), next_token.into())
}
/// Push a change to replace the "prev_token" with "next_token".
///
/// - leading trivia of `prev_token`
/// - leading trivia of `next_token`
/// - trailing trivia of `prev_token`
/// - trailing trivia of `next_token`
pub fn replace_token_transfer_trivia(
&mut self,
prev_token: SyntaxToken<L>,
next_token: SyntaxToken<L>,
) {
let leading_trivia = chain_trivia_pieces(
prev_token.leading_trivia().pieces(),
next_token.leading_trivia().pieces(),
);
let trailing_trivia = chain_trivia_pieces(
prev_token.trailing_trivia().pieces(),
next_token.trailing_trivia().pieces(),
);
let new_token = next_token
.with_leading_trivia_pieces(leading_trivia)
.with_trailing_trivia_pieces(trailing_trivia);
self.replace_token_discard_trivia(prev_token, new_token)
}
/// Push a change to replace the "prev_element" with "next_element".
///
/// Changes to take effect must be committed.
pub fn replace_element_discard_trivia(
&mut self,
prev_element: SyntaxElement<L>,
next_element: SyntaxElement<L>,
) {
self.push_change(prev_element, Some(next_element))
}
/// Push a change to remove the specified token.
///
/// Changes to take effect must be committed.
pub fn remove_token(&mut self, prev_token: SyntaxToken<L>) {
self.remove_element(prev_token.into())
}
/// Push a change to remove the specified node.
///
/// Changes to take effect must be committed.
pub fn remove_node<T>(&mut self, prev_node: T)
where
T: AstNode<Language = L>,
{
self.remove_element(prev_node.into_syntax().into())
}
/// Push a change to remove the specified element.
///
/// Changes to take effect must be committed.
pub fn remove_element(&mut self, prev_element: SyntaxElement<L>) {
self.push_change(prev_element, None)
}
fn push_change(
&mut self,
prev_element: SyntaxElement<L>,
next_element: Option<SyntaxElement<L>>,
) {
let new_node_slot = prev_element.index();
let parent = prev_element.parent();
let parent_range: Option<(u32, u32)> = parent.as_ref().map(|p| {
let range = p.text_range();
(range.start().into(), range.end().into())
});
let parent_depth = parent.as_ref().map(|p| p.ancestors().count()).unwrap_or(0);
debug!("pushing change...");
self.changes.push(CommitChange {
parent_depth,
parent,
parent_range,
new_node_slot,
new_node: next_element,
});
}
/// Returns the range of the document modified by this mutation along with
/// a list of individual text edits to be performed on the source code, or
/// [None] if the mutation is empty
pub fn as_text_edits(&self) -> Option<(TextRange, TextEdit)> {
let mut range = None;
debug!(" changes {:?}", &self.changes);
for change in &self.changes {
let parent = change.parent.as_ref().unwrap_or(&self.root);
let delete = match parent.slots().nth(change.new_node_slot) {
Some(SyntaxSlot::Node(node)) => node.text_range(),
Some(SyntaxSlot::Token(token)) => token.text_range(),
_ => continue,
};
range = match range {
None => Some(delete),
Some(range) => Some(range.cover(delete)),
};
}
let text_range = range?;
let old = self.root.to_string();
let new = self.clone().commit().to_string();
let text_edit = TextEdit::from_unicode_words(&old, &new);
Some((text_range, text_edit))
}
/// The core of the batch mutation algorithm can be summarized as:
/// 1 - Iterate all requested changes;
/// 2 - Insert them into a heap (priority queue) by depth. Deeper changes are done first;
/// 3 - Loop popping requested changes from the heap, taking the deepest change we have for the moment;
/// 4 - Each requested change has a "parent", an "index" and the "new node" (or None);
/// 5 - Clone the current parent's "parent", the "grandparent";
/// 6 - Detach the current "parent" from the tree;
/// 7 - Replace the old node at "index" at the current "parent" with the current "new node";
/// 8 - Insert into the heap the grandparent as the parent and the current "parent" as the "new node";
///
/// This is the simple case. The algorithm also has a more complex case when to changes have a common ancestor,
/// which can actually be one of the changed nodes.
///
/// To address this case at step 3, when we pop a new change to apply it, we actually aggregate all changes to the current
/// parent together. This is done by the heap because we also sort by node and it's range.
///
pub fn commit(self) -> SyntaxNode<L> {
let BatchMutation { root, mut changes } = self;
// Fill the heap with the requested changes
while let Some(item) = changes.pop() {
// If parent is None, we reached the root
if let Some(current_parent) = item.parent {
// This must be done before the detachment below
// because we need nodes that are still valid in the old tree
let grandparent = current_parent.parent();
let grandparent_range = grandparent.as_ref().map(|g| {
let range = g.text_range();
(range.start().into(), range.end().into())
});
let current_parent_slot = current_parent.index();
// Aggregate all modifications to the current parent
// This works because of the Ord we defined in the [CommitChange] struct
let mut modifications = vec![(item.new_node_slot, item.new_node)];
loop {
if let Some(next_change_parent) = changes.peek().and_then(|i| i.parent.as_ref())
{
if *next_change_parent == current_parent {
// SAFETY: We can .pop().unwrap() because we .peek() above
let next_change = changes.pop().expect("changes.pop");
// If we have two modification to the same slot,
// last write wins
if let Some(last) = modifications.last() {
if last.0 == next_change.new_node_slot {
modifications.pop();
}
}
modifications.push((next_change.new_node_slot, next_change.new_node));
continue;
}
}
break;
}
// Now we detach the current parent, make all the modifications
// and push a pending change to its parent.
let mut current_parent = current_parent.detach();
let is_list = current_parent.kind().is_list();
let mut removed_slots = 0;
for (index, replace_with) in modifications {
debug_assert!(index >= removed_slots);
let index = index.checked_sub(removed_slots)
.unwrap_or_else(|| panic!("cannot replace element in slot {index} with {removed_slots} removed slots"));
current_parent = if is_list && replace_with.is_none() {
removed_slots += 1;
current_parent.clone().splice_slots(index..=index, empty())
} else {
current_parent
.clone()
.splice_slots(index..=index, once(replace_with))
};
}
changes.push(CommitChange {
parent_depth: item.parent_depth - 1,
parent: grandparent,
parent_range: grandparent_range,
new_node_slot: current_parent_slot,
new_node: Some(SyntaxElement::Node(current_parent)),
});
} else {
let root = item
.new_node
.expect("new_node")
.into_node()
.expect("expected root to be a node and not a token");
return root;
}
}
root
}
pub fn root(&self) -> &SyntaxNode<L> {
&self.root
}
}
#[cfg(test)]
pub mod tests {
use crate::{
raw_language::{LiteralExpression, RawLanguageKind, RawLanguageRoot, RawSyntaxTreeBuilder},
AstNode, BatchMutationExt, SyntaxNodeCast,
};
/// ```
/// 0: ROOT@0..1
/// 0: LITERAL_EXPRESSION@0..1
/// 0: STRING_TOKEN@0..1 "a" [] []
/// ```
fn tree_one(a: &str) -> (RawLanguageRoot, String) {
let mut builder = RawSyntaxTreeBuilder::new();
builder
.start_node(RawLanguageKind::ROOT)
.start_node(RawLanguageKind::LITERAL_EXPRESSION)
.token(RawLanguageKind::STRING_TOKEN, a)
.finish_node()
.finish_node();
let root = builder.finish().cast::<RawLanguageRoot>().unwrap();
let s = format!("{:#?}", root.syntax());
(root, s)
}
/// ```
/// 0: ROOT@0..1
/// 0: LITERAL_EXPRESSION@0..1
/// 0: STRING_TOKEN@0..1 "a" [] []
/// 1: LITERAL_EXPRESSION@0..1
/// 0: STRING_TOKEN@0..1 "b" [] []
/// ```
fn tree_two(a: &str, b: &str) -> (RawLanguageRoot, String) {
let mut builder = RawSyntaxTreeBuilder::new();
builder
.start_node(RawLanguageKind::ROOT)
.start_node(RawLanguageKind::LITERAL_EXPRESSION)
.token(RawLanguageKind::STRING_TOKEN, a)
.finish_node()
.start_node(RawLanguageKind::LITERAL_EXPRESSION)
.token(RawLanguageKind::STRING_TOKEN, b)
.finish_node()
.finish_node();
let root = builder.finish().cast::<RawLanguageRoot>().unwrap();
let s = format!("{:#?}", root.syntax());
(root, s)
}
fn find(root: &RawLanguageRoot, name: &str) -> LiteralExpression {
root.syntax()
.descendants()
.find(|x| x.kind() == RawLanguageKind::LITERAL_EXPRESSION && x.text_trimmed() == name)
.unwrap()
.cast::<LiteralExpression>()
.unwrap()
}
fn clone_detach(root: &RawLanguageRoot, name: &str) -> LiteralExpression {
root.syntax()
.descendants()
.find(|x| x.kind() == RawLanguageKind::LITERAL_EXPRESSION && x.text_trimmed() == name)
.unwrap()
.detach()
.cast::<LiteralExpression>()
.unwrap()
}
#[test]
pub fn ok_batch_mutation_no_changes() {
let (before, before_debug) = tree_one("a");
let batch = before.begin();
let after = batch.commit();
assert_eq!(before_debug, format!("{:#?}", after));
}
#[test]
pub fn ok_batch_mutation_one_change() {
let (before, _) = tree_one("a");
let (expected, expected_debug) = tree_one("b");
let a = find(&before, "a");
let b = clone_detach(&expected, "b");
let mut batch = before.begin();
batch.replace_node(a, b);
let root = batch.commit();
assert_eq!(expected_debug, format!("{:#?}", root));
}
#[test]
pub fn ok_batch_mutation_multiple_changes_different_branches() {
let (before, _) = tree_two("a", "b");
let (expected, expected_debug) = tree_two("c", "d");
let a = find(&before, "a");
let b = find(&before, "b");
let c = clone_detach(&expected, "c");
let d = clone_detach(&expected, "d");
let mut batch = before.begin();
batch.replace_node(a, c);
batch.replace_node(b, d);
let after = batch.commit();
assert_eq!(expected_debug, format!("{:#?}", after));
}
}

1067
crates/ruff_rowan/src/ast/mod.rs Normal file
View File

File diff suppressed because it is too large Load Diff

228
crates/ruff_rowan/src/ast/mutation.rs Normal file
View File

@ -0,0 +1,228 @@
use std::ops;
use crate::{AstNode, AstNodeList, AstSeparatedList, SyntaxToken};
pub trait AstNodeExt: AstNode {
/// Return a new version of this node with the node `prev_node` replaced with `next_node`
///
/// `prev_node` can be a direct child of this node, or an indirect child through any descendant node
///
/// Returns `None` if `prev_node` is not a descendant of this node
fn replace_node_discard_trivia<N>(self, prev_node: N, next_node: N) -> Option<Self>
where
N: AstNode<Language = Self::Language>,
Self: Sized;
/// Return a new version of this node with the node `prev_node` replaced with `next_node`,
/// transferring the leading and trailing trivia of `prev_node` to `next_node`
///
/// `prev_node` can be a direct child of this node, or an indirect child through any descendant node
///
/// Returns `None` if `prev_node` is not a descendant of this node
fn replace_node<N>(self, prev_node: N, next_node: N) -> Option<Self>
where
N: AstNode<Language = Self::Language>,
Self: Sized;
/// Return a new version of this node with the token `prev_token` replaced with `next_token`
///
/// `prev_token` can be a direct child of this node, or an indirect child through any descendant node
///
/// Returns `None` if `prev_token` is not a descendant of this node
fn replace_token_discard_trivia(
self,
prev_token: SyntaxToken<Self::Language>,
next_token: SyntaxToken<Self::Language>,
) -> Option<Self>
where
Self: Sized;
/// Return a new version of this node with the token `prev_token` replaced with `next_token`,
/// transferring the leading and trailing trivia of `prev_token` to `next_token`
///
/// `prev_token` can be a direct child of this node, or an indirect child through any descendant node
///
/// Returns `None` if `prev_token` is not a descendant of this node
fn replace_token(
self,
prev_token: SyntaxToken<Self::Language>,
next_token: SyntaxToken<Self::Language>,
) -> Option<Self>
where
Self: Sized;
fn detach(self) -> Self;
}
impl<T> AstNodeExt for T
where
T: AstNode,
{
fn replace_node_discard_trivia<N>(self, prev_node: N, next_node: N) -> Option<Self>
where
N: AstNode<Language = Self::Language>,
Self: Sized,
{
Some(Self::unwrap_cast(self.into_syntax().replace_child(
prev_node.into_syntax().into(),
next_node.into_syntax().into(),
)?))
}
fn replace_node<N>(self, prev_node: N, mut next_node: N) -> Option<Self>
where
N: AstNode<Language = Self::Language>,
Self: Sized,
{
// Lookup the first token of `prev_node` and `next_node`, and transfer the leading
// trivia of the former to the later
let prev_first = prev_node.syntax().first_token();
let next_first = next_node.syntax().first_token();
if let (Some(prev_first), Some(next_first)) = (prev_first, next_first) {
let pieces: Vec<_> = prev_first.leading_trivia().pieces().collect();
next_node = next_node.replace_token_discard_trivia(
next_first.clone(),
next_first
.with_leading_trivia(pieces.iter().map(|piece| (piece.kind(), piece.text()))),
)?;
}
// Lookup the last token of `prev_node` and `next_node`, and transfer the trailing
// trivia of the former to the later
let prev_last = prev_node.syntax().last_token();
let next_last = next_node.syntax().last_token();
if let (Some(prev_last), Some(next_last)) = (prev_last, next_last) {
next_node = next_node.replace_token_discard_trivia(
next_last.clone(),
next_last.with_trailing_trivia_pieces(prev_last.trailing_trivia().pieces()),
)?;
}
// Call replace node with the modified `next_node`
self.replace_node_discard_trivia(prev_node, next_node)
}
fn replace_token_discard_trivia(
self,
prev_token: SyntaxToken<Self::Language>,
next_token: SyntaxToken<Self::Language>,
) -> Option<Self>
where
Self: Sized,
{
Some(Self::unwrap_cast(
self.into_syntax()
.replace_child(prev_token.into(), next_token.into())?,
))
}
fn replace_token(
self,
prev_token: SyntaxToken<Self::Language>,
next_token: SyntaxToken<Self::Language>,
) -> Option<Self>
where
Self: Sized,
{
let leading_trivia = prev_token.leading_trivia().pieces();
let trailing_trivia = prev_token.trailing_trivia().pieces();
self.replace_token_discard_trivia(
prev_token,
next_token
.with_leading_trivia_pieces(leading_trivia)
.with_trailing_trivia_pieces(trailing_trivia),
)
}
fn detach(self) -> Self {
Self::unwrap_cast(self.into_syntax().detach())
}
}
pub trait AstNodeListExt: AstNodeList {
/// Replace a range of the children of this list with the content of an iterator
fn splice<R, I>(self, range: R, replace_with: I) -> Self
where
Self: AstNode<Language = <Self as AstNodeList>::Language> + Sized,
R: ops::RangeBounds<usize>,
I: IntoIterator<Item = Self::Node>;
}
impl<T> AstNodeListExt for T
where
T: AstNodeList,
{
fn splice<R, I>(self, range: R, replace_with: I) -> Self
where
Self: AstNode<Language = <Self as AstNodeList>::Language> + Sized,
R: ops::RangeBounds<usize>,
I: IntoIterator<Item = Self::Node>,
{
Self::unwrap_cast(
self.into_syntax_list().into_node().splice_slots(
range,
replace_with
.into_iter()
.map(|node| Some(node.into_syntax().into())),
),
)
}
}
pub trait AstSeparatedListExt: AstSeparatedList {
/// Replace a range of the children of this list with the content of an iterator
///
/// Both the range and iterator work on pairs of node and separator token
fn splice<R, I>(self, range: R, replace_with: I) -> Self
where
Self: AstNode<Language = <Self as AstSeparatedList>::Language> + Sized,
R: ops::RangeBounds<usize>,
I: IntoIterator<
Item = (
Self::Node,
Option<SyntaxToken<<Self as AstSeparatedList>::Language>>,
),
>;
}
impl<T> AstSeparatedListExt for T
where
T: AstSeparatedList,
{
fn splice<R, I>(self, range: R, replace_with: I) -> Self
where
Self: AstNode<Language = <Self as AstSeparatedList>::Language> + Sized,
R: ops::RangeBounds<usize>,
I: IntoIterator<
Item = (
Self::Node,
Option<SyntaxToken<<Self as AstSeparatedList>::Language>>,
),
>,
{
let start_bound = match range.start_bound() {
ops::Bound::Included(index) => ops::Bound::Included(*index * 2),
ops::Bound::Excluded(index) => ops::Bound::Excluded(*index * 2),
ops::Bound::Unbounded => ops::Bound::Unbounded,
};
let end_bound = match range.end_bound() {
ops::Bound::Included(index) => ops::Bound::Included(*index * 2),
ops::Bound::Excluded(index) => ops::Bound::Excluded(*index * 2),
ops::Bound::Unbounded => ops::Bound::Unbounded,
};
Self::unwrap_cast(self.into_syntax_list().into_node().splice_slots(
(start_bound, end_bound),
replace_with.into_iter().flat_map(|(node, separator)| {
[
Some(node.into_syntax().into()),
separator.map(|token| token.into()),
]
}),
))
}
}

30
crates/ruff_rowan/src/cow_mut.rs Normal file
View File

@ -0,0 +1,30 @@
#[derive(Debug)]
pub(crate) enum CowMut<'a, T> {
Owned(T),
Borrowed(&'a mut T),
}
impl<T> std::ops::Deref for CowMut<'_, T> {
type Target = T;
fn deref(&self) -> &T {
match self {
CowMut::Owned(it) => it,
CowMut::Borrowed(it) => it,
}
}
}
impl<T> std::ops::DerefMut for CowMut<'_, T> {
fn deref_mut(&mut self) -> &mut T {
match self {
CowMut::Owned(it) => it,
CowMut::Borrowed(it) => it,
}
}
}
impl<T: Default> Default for CowMut<'_, T> {
fn default() -> Self {
CowMut::Owned(T::default())
}
}

382
crates/ruff_rowan/src/cursor.rs Normal file
View File

@ -0,0 +1,382 @@
//! Implementation of the cursors -- API for convenient access to syntax trees.
//!
//! Functional programmers will recognize that this module implements a zipper
//! for a purely functional (green) tree.
//!
//! A cursor node (`SyntaxNode`) points to a `GreenNode` and a parent
//! `SyntaxNode`. This allows cursor to provide iteration over both ancestors
//! and descendants, as well as a cheep access to absolute offset of the node in
//! file.
//!
// Implementation notes:
//
// The implementation is utterly and horribly unsafe. This whole module is an
// unsafety boundary. It is believed that the API here is, in principle, sound,
// but the implementation might have bugs.
//
// The core type is `NodeData` -- a heap-allocated reference counted object,
// which points to a green node or a green token, and to the parent `NodeData`.
// Publicly-exposed `SyntaxNode` and `SyntaxToken` own a reference to
// `NodeData`.
//
// `NodeData`s are transient, and are created and destroyed during tree
// traversals. In general, only currently referenced nodes and their ancestors
// are alive at any given moment.
//
// More specifically, `NodeData`'s ref count is equal to the number of
// outstanding `SyntaxNode` and `SyntaxToken` plus the number of children with
// non-zero ref counts. For example, if the user has only a single `SyntaxNode`
// pointing somewhere in the middle of the tree, then all `NodeData` on the path
// from that point towards the root have ref count equal to one.
//
// `NodeData` which doesn't have a parent (is a root) owns the corresponding
// green node or token, and is responsible for freeing it. For child `NodeData`
// however since they hold a strong reference to their parent node and thus
// to the root, their corresponding green node is guaranteed to be alive as
// a reference cycle to is know to exist (child `NodeData` -> root `NodeData`
// -> root `GreenNode` -> child `GreenNode`) and they can safely use a "weak
// reference" (raw pointer) to the corresponding green node as an optimization
// to avoid having to track atomic references on the traversal hot path
mod element;
mod node;
mod token;
mod trivia;
use std::{iter, ops};
use std::{ptr, rc::Rc};
use countme::Count;
pub(crate) use trivia::{SyntaxTrivia, SyntaxTriviaPiecesIterator};
use crate::cursor::node::Siblings;
pub(crate) use crate::cursor::token::SyntaxToken;
use crate::green::{self, GreenElement, GreenNodeData, GreenTokenData};
use crate::{
green::{GreenElementRef, RawSyntaxKind},
NodeOrToken, TextRange, TextSize,
};
pub(crate) use element::SyntaxElement;
pub(crate) use node::{
Preorder, PreorderWithTokens, SyntaxElementChildren, SyntaxNode, SyntaxNodeChildren,
SyntaxSlot, SyntaxSlots,
};
#[derive(Debug)]
struct _SyntaxElement;
pub(crate) fn has_live() -> bool {
countme::get::<_SyntaxElement>().live > 0
}
#[derive(Debug)]
struct NodeData {
_c: Count<_SyntaxElement>,
kind: NodeKind,
slot: u32,
/// Absolute offset for immutable nodes, unused for mutable nodes.
offset: TextSize,
}
/// A single NodeData (red node) is either a "root node" (no parent node and
/// holds a strong reference to the root of the green tree) or a "child node"
/// (holds a strong reference to its parent red node and a weak reference to its
/// counterpart green node)
#[derive(Debug)]
enum NodeKind {
Root {
green: GreenElement,
},
Child {
green: WeakGreenElement,
parent: Rc<NodeData>,
},
}
/// Child SyntaxNodes use "unsafe" weak pointers to refer to their green node.
/// Unlike the safe [std::sync::Weak] these are just a raw pointer: the
/// corresponding [ThinArc](crate::arc::ThinArc) doesn't keep a counter of
/// outstanding weak references or defer the release of the underlying memory
/// until the last `Weak` is dropped. On the other hand, a weak reference to a
/// released green node points to deallocated memory and it is undefined
/// behavior to dereference it, but in the context of `NodeData` this is
/// statically known to never happen
#[derive(Debug, Clone)]
enum WeakGreenElement {
Node { ptr: ptr::NonNull<GreenNodeData> },
Token { ptr: ptr::NonNull<GreenTokenData> },
}
impl WeakGreenElement {
fn new(green: GreenElementRef) -> Self {
match green {
NodeOrToken::Node(ptr) => Self::Node {
ptr: ptr::NonNull::from(ptr),
},
NodeOrToken::Token(ptr) => Self::Token {
ptr: ptr::NonNull::from(ptr),
},
}
}
fn as_deref(&self) -> GreenElementRef {
match self {
WeakGreenElement::Node { ptr } => GreenElementRef::Node(unsafe { ptr.as_ref() }),
WeakGreenElement::Token { ptr } => GreenElementRef::Token(unsafe { ptr.as_ref() }),
}
}
fn to_owned(&self) -> GreenElement {
match self {
WeakGreenElement::Node { ptr } => {
GreenElement::Node(unsafe { ptr.as_ref().to_owned() })
}
WeakGreenElement::Token { ptr } => {
GreenElement::Token(unsafe { ptr.as_ref().to_owned() })
}
}
}
}
impl NodeData {
#[inline]
fn new(kind: NodeKind, slot: u32, offset: TextSize) -> Rc<NodeData> {
let res = NodeData {
_c: Count::new(),
kind,
slot,
offset,
};
Rc::new(res)
}
#[inline]
fn key(&self) -> (ptr::NonNull<()>, TextSize) {
let weak = match &self.kind {
NodeKind::Root { green } => WeakGreenElement::new(green.as_deref()),
NodeKind::Child { green, .. } => green.clone(),
};
let ptr = match weak {
WeakGreenElement::Node { ptr } => ptr.cast(),
WeakGreenElement::Token { ptr } => ptr.cast(),
};
(ptr, self.offset())
}
#[inline]
fn parent_node(&self) -> Option<SyntaxNode> {
debug_assert!(matches!(
self.parent()?.green(),
GreenElementRef::Node { .. }
));
match &self.kind {
NodeKind::Child { parent, .. } => Some(SyntaxNode {
ptr: parent.clone(),
}),
NodeKind::Root { .. } => None,
}
}
#[inline]
fn parent(&self) -> Option<&NodeData> {
match &self.kind {
NodeKind::Child { parent, .. } => Some(&**parent),
NodeKind::Root { .. } => None,
}
}
#[inline]
fn green(&self) -> GreenElementRef<'_> {
match &self.kind {
NodeKind::Root { green } => green.as_deref(),
NodeKind::Child { green, .. } => green.as_deref(),
}
}
/// Returns an iterator over the siblings of this node. The iterator is positioned at the current node.
#[inline]
fn green_siblings(&self) -> Option<Siblings> {
match &self.parent()?.green() {
GreenElementRef::Node(ptr) => Some(Siblings::new(ptr, self.slot())),
GreenElementRef::Token(_) => {
debug_assert!(
false,
"A token should never be a parent of a token or node."
);
None
}
}
}
#[inline]
fn slot(&self) -> u32 {
self.slot
}
#[inline]
fn offset(&self) -> TextSize {
self.offset
}
#[inline]
fn text_range(&self) -> TextRange {
let offset = self.offset();
let len = self.green().text_len();
TextRange::at(offset, len)
}
#[inline]
fn kind(&self) -> RawSyntaxKind {
self.green().kind()
}
fn next_sibling(&self) -> Option<SyntaxNode> {
let siblings = self.green_siblings()?;
siblings.following().find_map(|child| {
child.element().into_node().and_then(|green| {
let parent = self.parent_node()?;
let offset = parent.offset() + child.rel_offset();
Some(SyntaxNode::new_child(green, parent, child.slot(), offset))
})
})
}
fn prev_sibling(&self) -> Option<SyntaxNode> {
let siblings = self.green_siblings()?;
siblings.previous().find_map(|child| {
child.element().into_node().and_then(|green| {
let parent = self.parent_node()?;
let offset = parent.offset() + child.rel_offset();
Some(SyntaxNode::new_child(green, parent, child.slot(), offset))
})
})
}
fn next_sibling_or_token(&self) -> Option<SyntaxElement> {
let siblings = self.green_siblings()?;
siblings.following().next().and_then(|child| {
let parent = self.parent_node()?;
let offset = parent.offset() + child.rel_offset();
Some(SyntaxElement::new(
child.element(),
parent,
child.slot(),
offset,
))
})
}
fn prev_sibling_or_token(&self) -> Option<SyntaxElement> {
let siblings = self.green_siblings()?;
siblings.previous().next().and_then(|child| {
let parent = self.parent_node()?;
let offset = parent.offset() + child.rel_offset();
Some(SyntaxElement::new(
child.element(),
parent,
child.slot(),
offset,
))
})
}
fn into_green(self: Rc<Self>) -> GreenElement {
match Rc::try_unwrap(self) {
Ok(data) => match data.kind {
NodeKind::Root { green } => green,
NodeKind::Child { green, .. } => green.to_owned(),
},
Err(ptr) => ptr.green().to_owned(),
}
}
/// Return a clone of this subtree detached from its parent
#[must_use = "syntax elements are immutable, the result of update methods must be propagated to have any effect"]
fn detach(self: Rc<Self>) -> Rc<Self> {
match &self.kind {
NodeKind::Child { green, .. } => Self::new(
NodeKind::Root {
green: green.to_owned(),
},
0,
0.into(),
),
// If this node is already detached, increment the reference count and return a clone
NodeKind::Root { .. } => self.clone(),
}
}
/// Return a clone of this node with the specified range of slots replaced
/// with the elements of the provided iterator
#[must_use = "syntax elements are immutable, the result of update methods must be propagated to have any effect"]
fn splice_slots<R, I>(mut self: Rc<Self>, range: R, replace_with: I) -> Rc<Self>
where
R: ops::RangeBounds<usize>,
I: Iterator<Item = Option<green::GreenElement>>,
{
let green = match self.green() {
NodeOrToken::Node(green) => green.splice_slots(range, replace_with).into(),
NodeOrToken::Token(_) => panic!("called splice_slots on a token node"),
};
// Try to reuse the underlying memory allocation if self is the only
// outstanding reference to this NodeData
match Rc::get_mut(&mut self) {
Some(node) => {
node.kind = NodeKind::Root { green };
node.slot = 0;
node.offset = TextSize::from(0);
self
}
None => Self::new(NodeKind::Root { green }, 0, 0.into()),
}
}
/// Return a new version of this node with the element `prev_elem` replaced with `next_elem`
///
/// `prev_elem` can be a direct child of this node, or an indirect child through any descendant node
///
/// Returns `None` if `prev_elem` is not a descendant of this node
#[must_use = "syntax elements are immutable, the result of update methods must be propagated to have any effect"]
fn replace_child(
mut self: Rc<Self>,
prev_elem: SyntaxElement,
next_elem: SyntaxElement,
) -> Option<Rc<Self>> {
let mut green = next_elem.into_green();
let mut elem = prev_elem;
loop {
let node = elem.parent()?;
let is_self = node.key() == self.key();
let index = elem.index();
let range = index..=index;
let replace_with = iter::once(Some(green));
green = node.green().splice_slots(range, replace_with).into();
elem = node.into();
if is_self {
break;
}
}
// Try to reuse the underlying memory allocation if self is the only
// outstanding reference to this NodeData
let result = match Rc::get_mut(&mut self) {
Some(node) => {
node.kind = NodeKind::Root { green };
node.slot = 0;
node.offset = TextSize::from(0);
self
}
None => Self::new(NodeKind::Root { green }, 0, 0.into()),
};
Some(result)
}
}

129
crates/ruff_rowan/src/cursor/element.rs Normal file
View File

@ -0,0 +1,129 @@
use crate::cursor::{SyntaxNode, SyntaxToken};
use crate::green::{GreenElement, GreenElementRef};
use crate::{NodeOrToken, RawSyntaxKind, TokenAtOffset};
use ruff_text_size::{TextRange, TextSize};
use std::iter;
pub(crate) type SyntaxElement = NodeOrToken<SyntaxNode, SyntaxToken>;
impl SyntaxElement {
pub(super) fn new(
element: GreenElementRef<'_>,
parent: SyntaxNode,
slot: u32,
offset: TextSize,
) -> SyntaxElement {
match element {
NodeOrToken::Node(node) => SyntaxNode::new_child(node, parent, slot, offset).into(),
NodeOrToken::Token(token) => SyntaxToken::new(token, parent, slot, offset).into(),
}
}
#[inline]
pub fn text_range(&self) -> TextRange {
match self {
NodeOrToken::Node(it) => it.text_range(),
NodeOrToken::Token(it) => it.text_range(),
}
}
#[inline]
pub fn index(&self) -> usize {
match self {
NodeOrToken::Node(it) => it.index(),
NodeOrToken::Token(it) => it.index(),
}
}
#[inline]
pub fn kind(&self) -> RawSyntaxKind {
match self {
NodeOrToken::Node(it) => it.kind(),
NodeOrToken::Token(it) => it.kind(),
}
}
#[inline]
pub fn parent(&self) -> Option<SyntaxNode> {
match self {
NodeOrToken::Node(it) => it.parent(),
NodeOrToken::Token(it) => it.parent(),
}
}
#[inline]
pub fn ancestors(&self) -> impl Iterator<Item = SyntaxNode> {
let first = match self {
NodeOrToken::Node(it) => Some(it.clone()),
NodeOrToken::Token(it) => it.parent(),
};
iter::successors(first, SyntaxNode::parent)
}
pub fn first_token(&self) -> Option<SyntaxToken> {
match self {
NodeOrToken::Node(it) => it.first_token(),
NodeOrToken::Token(it) => Some(it.clone()),
}
}
pub fn last_token(&self) -> Option<SyntaxToken> {
match self {
NodeOrToken::Node(it) => it.last_token(),
NodeOrToken::Token(it) => Some(it.clone()),
}
}
pub fn next_sibling_or_token(&self) -> Option<SyntaxElement> {
match self {
NodeOrToken::Node(it) => it.next_sibling_or_token(),
NodeOrToken::Token(it) => it.next_sibling_or_token(),
}
}
pub fn prev_sibling_or_token(&self) -> Option<SyntaxElement> {
match self {
NodeOrToken::Node(it) => it.prev_sibling_or_token(),
NodeOrToken::Token(it) => it.prev_sibling_or_token(),
}
}
pub(super) fn token_at_offset(&self, offset: TextSize) -> TokenAtOffset<SyntaxToken> {
assert!(self.text_range().start() <= offset && offset <= self.text_range().end());
match self {
NodeOrToken::Token(token) => TokenAtOffset::Single(token.clone()),
NodeOrToken::Node(node) => node.token_at_offset(offset),
}
}
#[must_use = "syntax elements are immutable, the result of update methods must be propagated to have any effect"]
pub fn detach(self) -> Self {
match self {
NodeOrToken::Node(it) => Self::Node(it.detach()),
NodeOrToken::Token(it) => Self::Token(it.detach()),
}
}
pub(crate) fn into_green(self) -> GreenElement {
match self {
NodeOrToken::Node(it) => it.ptr.into_green(),
NodeOrToken::Token(it) => it.into_green(),
}
}
}
// region: impls
impl From<SyntaxNode> for SyntaxElement {
#[inline]
fn from(node: SyntaxNode) -> SyntaxElement {
NodeOrToken::Node(node)
}
}
impl From<SyntaxToken> for SyntaxElement {
#[inline]
fn from(token: SyntaxToken) -> SyntaxElement {
NodeOrToken::Token(token)
}
}
// endregion

959
crates/ruff_rowan/src/cursor/node.rs Normal file
View File

@ -0,0 +1,959 @@
use crate::cursor::{NodeData, SyntaxElement, SyntaxToken, SyntaxTrivia};
use crate::green::{Child, Children, GreenElementRef, Slot};
use crate::{
Direction, GreenNode, GreenNodeData, NodeOrToken, RawSyntaxKind, SyntaxNodeText, TokenAtOffset,
WalkEvent,
};
use ruff_text_size::{TextRange, TextSize};
use std::hash::{Hash, Hasher};
use std::iter::FusedIterator;
use std::ops;
use std::ptr::NonNull;
use std::rc::Rc;
use std::{fmt, iter};
use super::{GreenElement, NodeKind, WeakGreenElement};
#[derive(Clone)]
pub(crate) struct SyntaxNode {
pub(super) ptr: Rc<NodeData>,
}
impl SyntaxNode {
pub(crate) fn new_root(green: GreenNode) -> SyntaxNode {
SyntaxNode {
ptr: NodeData::new(
NodeKind::Root {
green: GreenElement::Node(green),
},
0,
0.into(),
),
}
}
pub(super) fn new_child(
green: &GreenNodeData,
parent: SyntaxNode,
slot: u32,
offset: TextSize,
) -> SyntaxNode {
SyntaxNode {
ptr: NodeData::new(
NodeKind::Child {
green: WeakGreenElement::new(GreenElementRef::Node(green)),
parent: parent.ptr,
},
slot,
offset,
),
}
}
pub fn clone_subtree(&self) -> SyntaxNode {
SyntaxNode::new_root(self.green().into())
}
#[inline]
pub(super) fn data(&self) -> &NodeData {
self.ptr.as_ref()
}
#[inline]
pub fn kind(&self) -> RawSyntaxKind {
self.data().kind()
}
#[inline]
pub(super) fn offset(&self) -> TextSize {
self.data().offset()
}
pub(crate) fn element_in_slot(&self, slot_index: u32) -> Option<SyntaxElement> {
let slot = self
.slots()
.nth(slot_index as usize)
.expect("Slot index out of bounds");
slot.map(|element| element)
}
#[inline]
pub(crate) fn slots(&self) -> SyntaxSlots {
SyntaxSlots::new(self.clone())
}
#[inline]
pub fn text_range(&self) -> TextRange {
self.data().text_range()
}
pub fn text_trimmed_range(&self) -> TextRange {
let range = self.text_range();
let mut start = range.start();
let mut end = range.end();
// Remove all trivia from the start of the node
let mut token = self.first_token();
while let Some(t) = token.take() {
let (leading_len, trailing_len, total_len) = t.green().leading_trailing_total_len();
let token_len: u32 = (total_len - leading_len - trailing_len).into();
if token_len == 0 {
start += total_len;
token = t.next_token();
} else {
start += leading_len;
}
}
// Remove all trivia from the end of the node
let mut token = self.last_token();
while let Some(t) = token.take() {
let (leading_len, trailing_len, total_len) = t.green().leading_trailing_total_len();
let token_len: u32 = (total_len - leading_len - trailing_len).into();
if token_len == 0 {
end -= total_len;
token = t.prev_token();
} else {
end -= trailing_len;
}
}
TextRange::new(start, end.max(start))
}
pub fn first_leading_trivia(&self) -> Option<SyntaxTrivia> {
self.first_token().map(|x| x.leading_trivia())
}
pub fn last_trailing_trivia(&self) -> Option<SyntaxTrivia> {
self.last_token().map(|x| x.trailing_trivia())
}
#[inline]
pub fn index(&self) -> usize {
self.data().slot() as usize
}
#[inline]
pub fn text(&self) -> SyntaxNodeText {
SyntaxNodeText::new(self.clone())
}
#[inline]
pub fn text_trimmed(&self) -> SyntaxNodeText {
SyntaxNodeText::with_range(self.clone(), self.text_trimmed_range())
}
#[inline]
pub(crate) fn key(&self) -> (NonNull<()>, TextSize) {
self.data().key()
}
#[inline]
pub(crate) fn green(&self) -> &GreenNodeData {
self.data().green().into_node().unwrap()
}
#[inline]
pub fn parent(&self) -> Option<SyntaxNode> {
self.data().parent_node()
}
#[inline]
pub fn ancestors(&self) -> impl Iterator<Item = SyntaxNode> {
iter::successors(Some(self.clone()), SyntaxNode::parent)
}
#[inline]
pub fn children(&self) -> SyntaxNodeChildren {
SyntaxNodeChildren::new(self.clone())
}
#[inline]
pub fn children_with_tokens(&self) -> SyntaxElementChildren {
SyntaxElementChildren::new(self.clone())
}
#[inline]
pub fn tokens(&self) -> impl Iterator<Item = SyntaxToken> + DoubleEndedIterator + '_ {
self.green().children().filter_map(|child| {
child.element().into_token().map(|token| {
SyntaxToken::new(
token,
self.clone(),
child.slot(),
self.offset() + child.rel_offset(),
)
})
})
}
pub fn first_child(&self) -> Option<SyntaxNode> {
self.green().children().find_map(|child| {
child.element().into_node().map(|green| {
SyntaxNode::new_child(
green,
self.clone(),
child.slot(),
self.offset() + child.rel_offset(),
)
})
})
}
pub fn last_child(&self) -> Option<SyntaxNode> {
self.green().children().rev().find_map(|child| {
child.element().into_node().map(|green| {
SyntaxNode::new_child(
green,
self.clone(),
child.slot(),
self.offset() + child.rel_offset(),
)
})
})
}
pub fn first_child_or_token(&self) -> Option<SyntaxElement> {
self.green().children().next().map(|child| {
SyntaxElement::new(
child.element(),
self.clone(),
child.slot(),
self.offset() + child.rel_offset(),
)
})
}
pub fn last_child_or_token(&self) -> Option<SyntaxElement> {
self.green().children().next_back().map(|child| {
SyntaxElement::new(
child.element(),
self.clone(),
child.slot(),
self.offset() + child.rel_offset(),
)
})
}
pub fn next_sibling(&self) -> Option<SyntaxNode> {
self.data().next_sibling()
}
pub fn prev_sibling(&self) -> Option<SyntaxNode> {
self.data().prev_sibling()
}
pub fn next_sibling_or_token(&self) -> Option<SyntaxElement> {
self.data().next_sibling_or_token()
}
pub fn prev_sibling_or_token(&self) -> Option<SyntaxElement> {
self.data().prev_sibling_or_token()
}
pub fn first_token(&self) -> Option<SyntaxToken> {
self.descendants_with_tokens(Direction::Next)
.find_map(|x| x.into_token())
}
pub fn last_token(&self) -> Option<SyntaxToken> {
PreorderWithTokens::new(self.clone(), Direction::Prev)
.filter_map(|event| match event {
WalkEvent::Enter(it) => Some(it),
WalkEvent::Leave(_) => None,
})
.find_map(|x| x.into_token())
}
#[inline]
pub fn siblings(&self, direction: Direction) -> impl Iterator<Item = SyntaxNode> {
iter::successors(Some(self.clone()), move |node| match direction {
Direction::Next => node.next_sibling(),
Direction::Prev => node.prev_sibling(),
})
}
#[inline]
pub fn siblings_with_tokens(
&self,
direction: Direction,
) -> impl Iterator<Item = SyntaxElement> {
let me: SyntaxElement = self.clone().into();
iter::successors(Some(me), move |el| match direction {
Direction::Next => el.next_sibling_or_token(),
Direction::Prev => el.prev_sibling_or_token(),
})
}
#[inline]
pub fn descendants(&self) -> impl Iterator<Item = SyntaxNode> {
self.preorder().filter_map(|event| match event {
WalkEvent::Enter(node) => Some(node),
WalkEvent::Leave(_) => None,
})
}
#[inline]
pub fn descendants_with_tokens(
&self,
direction: Direction,
) -> impl Iterator<Item = SyntaxElement> {
self.preorder_with_tokens(direction)
.filter_map(|event| match event {
WalkEvent::Enter(it) => Some(it),
WalkEvent::Leave(_) => None,
})
}
#[inline]
pub fn preorder(&self) -> Preorder {
Preorder::new(self.clone())
}
#[inline]
pub fn preorder_with_tokens(&self, direction: Direction) -> PreorderWithTokens {
PreorderWithTokens::new(self.clone(), direction)
}
pub(crate) fn preorder_slots(&self) -> SlotsPreorder {
SlotsPreorder::new(self.clone())
}
pub fn token_at_offset(&self, offset: TextSize) -> TokenAtOffset<SyntaxToken> {
// TODO: this could be faster if we first drill-down to node, and only
// then switch to token search. We should also replace explicit
// recursion with a loop.
let range = self.text_range();
assert!(
range.start() <= offset && offset <= range.end(),
"Bad offset: range {:?} offset {:?}",
range,
offset
);
if range.is_empty() {
return TokenAtOffset::None;
}
let mut children = self.children_with_tokens().filter(|child| {
let child_range = child.text_range();
!child_range.is_empty() && child_range.contains_inclusive(offset)
});
let left = children.next().unwrap();
let right = children.next();
assert!(children.next().is_none());
if let Some(right) = right {
match (left.token_at_offset(offset), right.token_at_offset(offset)) {
(TokenAtOffset::Single(left), TokenAtOffset::Single(right)) => {
TokenAtOffset::Between(left, right)
}
_ => unreachable!(),
}
} else {
left.token_at_offset(offset)
}
}
pub fn covering_element(&self, range: TextRange) -> SyntaxElement {
let mut res: SyntaxElement = self.clone().into();
loop {
assert!(
res.text_range().contains_range(range),
"Bad range: node range {:?}, range {:?}",
res.text_range(),
range,
);
res = match &res {
NodeOrToken::Token(_) => return res,
NodeOrToken::Node(node) => match node.child_or_token_at_range(range) {
Some(it) => it,
None => return res,
},
};
}
}
pub fn child_or_token_at_range(&self, range: TextRange) -> Option<SyntaxElement> {
let rel_range = range - self.offset();
self.green()
.slot_at_range(rel_range)
.and_then(|(index, rel_offset, slot)| {
slot.as_ref().map(|green| {
SyntaxElement::new(
green,
self.clone(),
index as u32,
self.offset() + rel_offset,
)
})
})
}
#[must_use = "syntax elements are immutable, the result of update methods must be propagated to have any effect"]
pub fn detach(self) -> Self {
Self {
ptr: self.ptr.detach(),
}
}
#[must_use = "syntax elements are immutable, the result of update methods must be propagated to have any effect"]
pub fn splice_slots<R, I>(self, range: R, replace_with: I) -> Self
where
R: ops::RangeBounds<usize>,
I: Iterator<Item = Option<SyntaxElement>>,
{
Self {
ptr: self.ptr.splice_slots(
range,
replace_with.into_iter().map(|element| {
element.map(|child| match child.detach() {
NodeOrToken::Node(it) => it.ptr.into_green(),
NodeOrToken::Token(it) => it.into_green(),
})
}),
),
}
}
#[must_use = "syntax elements are immutable, the result of update methods must be propagated to have any effect"]
pub fn replace_child(self, prev_elem: SyntaxElement, next_elem: SyntaxElement) -> Option<Self> {
Some(Self {
ptr: self.ptr.replace_child(prev_elem, next_elem)?,
})
}
}
// Identity semantics for hash & eq
impl PartialEq for SyntaxNode {
#[inline]
fn eq(&self, other: &SyntaxNode) -> bool {
self.data().key() == other.data().key()
}
}
impl Eq for SyntaxNode {}
impl Hash for SyntaxNode {
#[inline]
fn hash<H: Hasher>(&self, state: &mut H) {
self.data().key().hash(state);
}
}
impl fmt::Debug for SyntaxNode {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("SyntaxNode")
.field("kind", &self.kind())
.field("text_range", &self.text_range())
.finish()
}
}
impl fmt::Display for SyntaxNode {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
self.preorder_with_tokens(Direction::Next)
.filter_map(|event| match event {
WalkEvent::Enter(NodeOrToken::Token(token)) => Some(token),
_ => None,
})
.try_for_each(|it| fmt::Display::fmt(&it, f))
}
}
// region: iterators
#[derive(Clone, Debug)]
pub(crate) struct SyntaxNodeChildren {
next: Option<SyntaxNode>,
}
impl SyntaxNodeChildren {
fn new(parent: SyntaxNode) -> SyntaxNodeChildren {
SyntaxNodeChildren {
next: parent.first_child(),
}
}
}
impl Iterator for SyntaxNodeChildren {
type Item = SyntaxNode;
fn next(&mut self) -> Option<SyntaxNode> {
self.next.take().map(|next| {
self.next = next.next_sibling();
next
})
}
}
impl FusedIterator for SyntaxNodeChildren {}
#[derive(Clone, Debug, Default)]
pub(crate) struct SyntaxElementChildren {
next: Option<SyntaxElement>,
}
impl SyntaxElementChildren {
fn new(parent: SyntaxNode) -> SyntaxElementChildren {
SyntaxElementChildren {
next: parent.first_child_or_token(),
}
}
}
impl Iterator for SyntaxElementChildren {
type Item = SyntaxElement;
fn next(&mut self) -> Option<SyntaxElement> {
self.next.take().map(|next| {
self.next = next.next_sibling_or_token();
next
})
}
}
impl FusedIterator for SyntaxElementChildren {}
pub(crate) struct Preorder {
start: SyntaxNode,
next: Option<WalkEvent<SyntaxNode>>,
skip_subtree: bool,
}
impl Preorder {
fn new(start: SyntaxNode) -> Preorder {
let next = Some(WalkEvent::Enter(start.clone()));
Preorder {
start,
next,
skip_subtree: false,
}
}
pub fn skip_subtree(&mut self) {
self.skip_subtree = true;
}
#[cold]
fn do_skip(&mut self) {
self.next = self.next.take().map(|next| match next {
WalkEvent::Enter(first_child) => WalkEvent::Leave(first_child.parent().unwrap()),
WalkEvent::Leave(parent) => WalkEvent::Leave(parent),
})
}
}
impl Iterator for Preorder {
type Item = WalkEvent<SyntaxNode>;
fn next(&mut self) -> Option<WalkEvent<SyntaxNode>> {
if self.skip_subtree {
self.do_skip();
self.skip_subtree = false;
}
let next = self.next.take();
self.next = next.as_ref().and_then(|next| {
Some(match next {
WalkEvent::Enter(node) => match node.first_child() {
Some(child) => WalkEvent::Enter(child),
None => WalkEvent::Leave(node.clone()),
},
WalkEvent::Leave(node) => {
if node == &self.start {
return None;
}
match node.next_sibling() {
Some(sibling) => WalkEvent::Enter(sibling),
None => WalkEvent::Leave(node.parent()?),
}
}
})
});
next
}
}
impl FusedIterator for Preorder {}
pub(crate) struct PreorderWithTokens {
start: SyntaxElement,
next: Option<WalkEvent<SyntaxElement>>,
skip_subtree: bool,
direction: Direction,
}
impl PreorderWithTokens {
fn new(start: SyntaxNode, direction: Direction) -> PreorderWithTokens {
let next = Some(WalkEvent::Enter(start.clone().into()));
PreorderWithTokens {
start: start.into(),
next,
direction,
skip_subtree: false,
}
}
pub fn skip_subtree(&mut self) {
self.skip_subtree = true;
}
#[cold]
fn do_skip(&mut self) {
self.next = self.next.take().map(|next| match next {
WalkEvent::Enter(first_child) => WalkEvent::Leave(first_child.parent().unwrap().into()),
WalkEvent::Leave(parent) => WalkEvent::Leave(parent),
})
}
}
impl Iterator for PreorderWithTokens {
type Item = WalkEvent<SyntaxElement>;
fn next(&mut self) -> Option<WalkEvent<SyntaxElement>> {
if self.skip_subtree {
self.do_skip();
self.skip_subtree = false;
}
let next = self.next.take();
self.next = next.as_ref().and_then(|next| {
Some(match next {
WalkEvent::Enter(el) => match el {
NodeOrToken::Node(node) => {
let next = match self.direction {
Direction::Next => node.first_child_or_token(),
Direction::Prev => node.last_child_or_token(),
};
match next {
Some(child) => WalkEvent::Enter(child),
None => WalkEvent::Leave(node.clone().into()),
}
}
NodeOrToken::Token(token) => WalkEvent::Leave(token.clone().into()),
},
WalkEvent::Leave(el) if el == &self.start => return None,
WalkEvent::Leave(el) => {
let next = match self.direction {
Direction::Next => el.next_sibling_or_token(),
Direction::Prev => el.prev_sibling_or_token(),
};
match next {
Some(sibling) => WalkEvent::Enter(sibling),
None => WalkEvent::Leave(el.parent()?.into()),
}
}
})
});
next
}
}
impl FusedIterator for PreorderWithTokens {}
/// Represents a cursor to a green node slot. A slot either contains an element or is empty
/// if the child isn't present in the source.
#[derive(Debug, Clone)]
pub(crate) enum SyntaxSlot {
Node(SyntaxNode),
Token(SyntaxToken),
Empty { parent: SyntaxNode, index: u32 },
}
impl From<SyntaxElement> for SyntaxSlot {
fn from(element: SyntaxElement) -> Self {
match element {
SyntaxElement::Node(node) => SyntaxSlot::Node(node),
SyntaxElement::Token(token) => SyntaxSlot::Token(token),
}
}
}
impl SyntaxSlot {
#[inline]
pub fn map<F, R>(self, mapper: F) -> Option<R>
where
F: FnOnce(SyntaxElement) -> R,
{
match self {
SyntaxSlot::Node(node) => Some(mapper(SyntaxElement::Node(node))),
SyntaxSlot::Token(token) => Some(mapper(SyntaxElement::Token(token))),
SyntaxSlot::Empty { .. } => None,
}
}
}
/// Iterator over a node's slots
#[derive(Debug, Clone)]
pub(crate) struct SyntaxSlots {
/// Position of the next element to return.
pos: u32,
/// Position of the last returned element from the back.
/// Initially points one element past the last slot.
///
/// [nth_back]: https://doc.rust-lang.org/std/iter/trait.DoubleEndedIterator.html#method.nth_back
back_pos: u32,
parent: SyntaxNode,
}
impl SyntaxSlots {
#[inline]
fn new(parent: SyntaxNode) -> Self {
Self {
pos: 0,
back_pos: parent.green().slice().len() as u32,
parent,
}
}
/// Returns a slice containing the remaining elements to iterate over
/// an empty slice if the iterator reached the end.
#[inline]
fn slice(&self) -> &[Slot] {
if self.pos < self.back_pos {
&self.parent.green().slice()[self.pos as usize..self.back_pos as usize]
} else {
&[]
}
}
fn map_slot(&self, slot: &Slot, slot_index: u32) -> SyntaxSlot {
match slot {
Slot::Empty { .. } => SyntaxSlot::Empty {
parent: self.parent.clone(),
index: slot_index,
},
Slot::Token { rel_offset, token } => SyntaxSlot::Token(SyntaxToken::new(
token,
self.parent.clone(),
slot_index,
self.parent.offset() + rel_offset,
)),
Slot::Node { rel_offset, node } => SyntaxSlot::Node(SyntaxNode::new_child(
node,
self.parent.clone(),
slot_index,
self.parent.offset() + rel_offset,
)),
}
}
}
impl Iterator for SyntaxSlots {
type Item = SyntaxSlot;
#[inline]
fn next(&mut self) -> Option<Self::Item> {
let slot = self.slice().first()?;
let mapped = self.map_slot(slot, self.pos);
self.pos += 1;
Some(mapped)
}
#[inline(always)]
fn size_hint(&self) -> (usize, Option<usize>) {
let len = self.slice().len();
(len, Some(len))
}
#[inline(always)]
fn count(self) -> usize
where
Self: Sized,
{
self.len()
}
#[inline]
fn last(mut self) -> Option<Self::Item>
where
Self: Sized,
{
self.next_back()
}
#[inline]
fn nth(&mut self, n: usize) -> Option<Self::Item> {
self.pos += n as u32;
self.next()
}
}
impl ExactSizeIterator for SyntaxSlots {
#[inline(always)]
fn len(&self) -> usize {
self.slice().len()
}
}
impl FusedIterator for SyntaxSlots {}
impl DoubleEndedIterator for SyntaxSlots {
#[inline]
fn next_back(&mut self) -> Option<Self::Item> {
let slot = self.slice().last()?;
let mapped = self.map_slot(slot, self.back_pos - 1);
self.back_pos -= 1;
Some(mapped)
}
#[inline]
fn nth_back(&mut self, n: usize) -> Option<Self::Item> {
self.back_pos -= n as u32;
self.next_back()
}
}
/// Iterator to visit a node's slots in pre-order.
pub(crate) struct SlotsPreorder {
start: SyntaxNode,
next: Option<WalkEvent<SyntaxSlot>>,
}
impl SlotsPreorder {
fn new(start: SyntaxNode) -> Self {
let next = Some(WalkEvent::Enter(SyntaxSlot::Node(start.clone())));
SlotsPreorder { start, next }
}
}
impl Iterator for SlotsPreorder {
type Item = WalkEvent<SyntaxSlot>;
fn next(&mut self) -> Option<WalkEvent<SyntaxSlot>> {
let next = self.next.take();
self.next = next.as_ref().and_then(|next| {
Some(match next {
WalkEvent::Enter(slot) => match slot {
SyntaxSlot::Empty { .. } | SyntaxSlot::Token(_) => {
WalkEvent::Leave(slot.clone())
}
SyntaxSlot::Node(node) => match node.slots().next() {
None => WalkEvent::Leave(SyntaxSlot::Node(node.clone())),
Some(first_slot) => WalkEvent::Enter(first_slot),
},
},
WalkEvent::Leave(slot) => {
let (parent, slot_index) = match slot {
SyntaxSlot::Empty { parent, index } => (parent.clone(), *index as usize),
SyntaxSlot::Token(token) => (token.parent()?, token.index()),
SyntaxSlot::Node(node) => {
if node == &self.start {
return None;
}
(node.parent()?, node.index())
}
};
let next_slot = parent.slots().nth(slot_index + 1);
match next_slot {
Some(slot) => WalkEvent::Enter(slot),
None => WalkEvent::Leave(SyntaxSlot::Node(parent)),
}
}
})
});
next
}
}
impl FusedIterator for SlotsPreorder {}
#[derive(Debug, Clone)]
pub(crate) struct Siblings<'a> {
parent: &'a GreenNodeData,
start_slot: u32,
}
impl<'a> Siblings<'a> {
pub fn new(parent: &'a GreenNodeData, start_slot: u32) -> Self {
assert!(
(start_slot as usize) < parent.slots().len(),
"Start slot {} out of bounds {}",
start_slot,
parent.slots().len()
);
Self { parent, start_slot }
}
/// Creates an iterator over the siblings following the start node.
/// For example, the following siblings of the if statement's condition are
/// * the consequence
/// * potentially the else clause
pub fn following(&self) -> Children<'a> {
let mut slots = self.parent.slots().enumerate();
// Navigate to the start slot so that calling `next` returns the first following sibling
slots.nth(self.start_slot as usize);
Children::new(slots)
}
/// Creates an iterator over the siblings preceding the start node in reverse order.
/// For example, the preceding siblings of the if statement's condition are:
/// * opening parentheses: (
/// * if keyword: if
pub fn previous(&self) -> impl Iterator<Item = Child<'a>> {
let mut slots = self.parent.slots().enumerate();
// Navigate to the start slot from the back so that calling `next_back` (or rev().next()) returns
// the first slot preceding the start node
slots.nth_back(slots.len() - 1 - self.start_slot as usize);
Children::new(slots).rev()
}
}
// endregion
#[cfg(test)]
mod tests {
use crate::raw_language::{RawLanguageKind, RawSyntaxTreeBuilder};
#[test]
fn slots_iter() {
let mut builder = RawSyntaxTreeBuilder::new();
builder.start_node(RawLanguageKind::EXPRESSION_LIST);
for number in [1, 2, 3, 4] {
builder.start_node(RawLanguageKind::LITERAL_EXPRESSION);
builder.token(RawLanguageKind::NUMBER_TOKEN, &number.to_string());
builder.finish_node();
}
builder.finish_node();
let list = builder.finish();
let mut iter = list.slots();
assert_eq!(iter.size_hint(), (4, Some(4)));
assert_eq!(
iter.next()
.and_then(|slot| slot.into_node())
.map(|node| node.text().to_string())
.as_deref(),
Some("1")
);
assert_eq!(iter.size_hint(), (3, Some(3)));
assert_eq!(
iter.next_back()
.and_then(|slot| slot.into_node())
.map(|node| node.text().to_string())
.as_deref(),
Some("4")
);
assert_eq!(iter.size_hint(), (2, Some(2)));
assert_eq!(
iter.last()
.and_then(|slot| slot.into_node())
.map(|node| node.text().to_string())
.as_deref(),
Some("3")
);
}
}

258
crates/ruff_rowan/src/cursor/token.rs Normal file
View File

@ -0,0 +1,258 @@
use crate::cursor::{NodeData, SyntaxElement, SyntaxNode, SyntaxTrivia};
use crate::green::GreenElementRef;
use crate::{
green, Direction, GreenToken, GreenTokenData, RawSyntaxKind, SyntaxTokenText, WalkEvent,
};
use ruff_text_size::{TextRange, TextSize};
use std::hash::{Hash, Hasher};
use std::ptr::NonNull;
use std::rc::Rc;
use std::{fmt, iter};
use super::{GreenElement, NodeKind, WeakGreenElement};
#[derive(Clone, Debug)]
pub(crate) struct SyntaxToken {
ptr: Rc<NodeData>,
}
impl SyntaxToken {
pub(super) fn new(
green: &GreenTokenData,
parent: SyntaxNode,
index: u32,
offset: TextSize,
) -> SyntaxToken {
SyntaxToken {
ptr: NodeData::new(
NodeKind::Child {
green: WeakGreenElement::new(GreenElementRef::Token(green)),
parent: parent.ptr,
},
index,
offset,
),
}
}
pub(crate) fn new_detached(green: GreenToken) -> SyntaxToken {
SyntaxToken {
ptr: NodeData::new(
NodeKind::Root {
green: GreenElement::Token(green),
},
0,
TextSize::from(0),
),
}
}
#[inline]
pub(crate) fn green(&self) -> &GreenTokenData {
match self.data().green().as_token() {
Some(token) => token,
None => {
panic!(
"corrupted tree: a node thinks it is a token: {:?}",
self.data().green().as_node().unwrap().to_string()
);
}
}
}
pub(crate) fn key(&self) -> (NonNull<()>, TextSize) {
self.data().key()
}
#[inline]
pub(super) fn data(&self) -> &NodeData {
self.ptr.as_ref()
}
#[inline]
pub(super) fn into_green(self) -> green::GreenElement {
self.ptr.into_green()
}
#[inline]
pub fn kind(&self) -> RawSyntaxKind {
self.data().kind()
}
#[inline]
pub fn text_range(&self) -> TextRange {
self.data().text_range()
}
#[inline]
pub fn text_trimmed_range(&self) -> TextRange {
let green_token = self.green();
let leading_len = green_token.leading_trivia().text_len();
let trailing_len = green_token.trailing_trivia().text_len();
let range = self.text_range();
TextRange::new(range.start() + leading_len, range.end() - trailing_len)
}
#[inline]
pub fn index(&self) -> usize {
self.data().slot() as usize
}
#[inline]
pub fn text(&self) -> &str {
self.green().text()
}
#[inline]
pub fn token_text(&self) -> SyntaxTokenText {
SyntaxTokenText::new(self.green().to_owned())
}
#[inline]
pub fn token_text_trimmed(&self) -> SyntaxTokenText {
let green = self.green().to_owned();
let mut range = self.text_trimmed_range();
range -= self.data().offset;
SyntaxTokenText::with_range(green, range)
}
#[inline]
pub fn text_trimmed(&self) -> &str {
self.green().text_trimmed()
}
#[inline]
pub fn parent(&self) -> Option<SyntaxNode> {
self.data().parent_node()
}
#[inline]
pub fn ancestors(&self) -> impl Iterator<Item = SyntaxNode> {
std::iter::successors(self.parent(), SyntaxNode::parent)
}
pub fn next_sibling_or_token(&self) -> Option<SyntaxElement> {
self.data().next_sibling_or_token()
}
pub fn prev_sibling_or_token(&self) -> Option<SyntaxElement> {
self.data().prev_sibling_or_token()
}
#[inline]
pub fn siblings_with_tokens(
&self,
direction: Direction,
) -> impl Iterator<Item = SyntaxElement> {
let next = move |el: &SyntaxElement| match direction {
Direction::Next => el.next_sibling_or_token(),
Direction::Prev => el.prev_sibling_or_token(),
};
let me: SyntaxElement = self.clone().into();
iter::successors(next(&me), next)
}
pub fn next_token(&self) -> Option<SyntaxToken> {
self.next_token_impl(Direction::Next)
}
pub fn prev_token(&self) -> Option<SyntaxToken> {
self.next_token_impl(Direction::Prev)
}
/// Returns the token preceding or following this token depending on the passed `direction`.
fn next_token_impl(&self, direction: Direction) -> Option<SyntaxToken> {
let mut current: WalkEvent<SyntaxElement> =
WalkEvent::Leave(SyntaxElement::Token(self.clone()));
loop {
current = match current {
WalkEvent::Enter(element) => match element {
SyntaxElement::Token(token) => break Some(token),
SyntaxElement::Node(node) => {
let first_child = match direction {
Direction::Next => node.first_child_or_token(),
Direction::Prev => node.last_child_or_token(),
};
match first_child {
// If node is empty, leave parent
None => WalkEvent::Leave(SyntaxElement::Node(node)),
// Otherwise traverse full sub-tree
Some(child) => WalkEvent::Enter(child),
}
}
},
WalkEvent::Leave(element) => {
let mut current_element = element;
loop {
// Only traverse the left (pref) / right (next) siblings of the parent
// to avoid traversing into the same children again.
let sibling = match direction {
Direction::Next => current_element.next_sibling_or_token(),
Direction::Prev => current_element.prev_sibling_or_token(),
};
match sibling {
// Traverse all children of the sibling
Some(sibling) => break WalkEvent::Enter(sibling),
None => {
match current_element.parent() {
Some(node) => {
current_element = SyntaxElement::Node(node);
}
None => {
return None; // Reached root, no token found
}
}
}
}
}
}
}
}
}
#[must_use = "syntax elements are immutable, the result of update methods must be propagated to have any effect"]
pub fn detach(self) -> Self {
Self {
ptr: self.ptr.detach(),
}
}
#[inline]
pub fn leading_trivia(&self) -> SyntaxTrivia {
SyntaxTrivia::leading(self.clone())
}
#[inline]
pub fn trailing_trivia(&self) -> SyntaxTrivia {
SyntaxTrivia::trailing(self.clone())
}
}
// Identity semantics for hash & eq
impl PartialEq for SyntaxToken {
#[inline]
fn eq(&self, other: &SyntaxToken) -> bool {
self.data().key() == other.data().key()
}
}
impl Eq for SyntaxToken {}
impl Hash for SyntaxToken {
#[inline]
fn hash<H: Hasher>(&self, state: &mut H) {
self.data().key().hash(state);
}
}
impl fmt::Display for SyntaxToken {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
fmt::Display::fmt(self.text(), f)
}
}

182
crates/ruff_rowan/src/cursor/trivia.rs Normal file
View File

@ -0,0 +1,182 @@
use crate::cursor::SyntaxToken;
use crate::green::GreenTrivia;
use crate::TriviaPiece;
use ruff_text_size::{TextRange, TextSize};
use std::fmt;
use std::iter::FusedIterator;
#[derive(PartialEq, Eq, Clone, Hash)]
pub(crate) struct SyntaxTrivia {
token: SyntaxToken,
is_leading: bool,
}
impl SyntaxTrivia {
pub(super) fn leading(token: SyntaxToken) -> Self {
Self {
token,
is_leading: true,
}
}
pub(super) fn trailing(token: SyntaxToken) -> Self {
Self {
token,
is_leading: false,
}
}
pub(crate) fn text(&self) -> &str {
let trivia_range = self.text_range();
let relative_range = TextRange::at(
trivia_range.start() - self.token.data().offset,
trivia_range.len(),
);
&self.token.text()[relative_range]
}
pub(crate) fn token(&self) -> &SyntaxToken {
&self.token
}
pub(crate) fn text_range(&self) -> TextRange {
let length = self.green_trivia().text_len();
let token_range = self.token.text_range();
match self.is_leading {
true => TextRange::at(token_range.start(), length),
false => TextRange::at(token_range.end() - length, length),
}
}
/// Get the number of TriviaPiece inside this trivia
pub(crate) fn len(&self) -> usize {
self.green_trivia().len()
}
/// Gets index-th trivia piece when the token associated with this trivia was created.
/// See [SyntaxTriviaPiece].
pub(crate) fn get_piece(&self, index: usize) -> Option<&TriviaPiece> {
self.green_trivia().get_piece(index)
}
fn green_trivia(&self) -> &GreenTrivia {
match self.is_leading {
true => self.token.green().leading_trivia(),
false => self.token.green().trailing_trivia(),
}
}
/// Returns the last trivia piece element
pub(crate) fn last(&self) -> Option<&TriviaPiece> {
self.green_trivia().pieces().last()
}
/// Returns the first trivia piece element
pub(crate) fn first(&self) -> Option<&TriviaPiece> {
self.green_trivia().pieces().first()
}
/// Iterate over all pieces of the trivia. The iterator returns the offset
/// of the trivia as [TextSize] and its data as [Trivia], which contains its length.
/// See [SyntaxTriviaPiece].
pub(crate) fn pieces(&self) -> SyntaxTriviaPiecesIterator {
let range = self.text_range();
SyntaxTriviaPiecesIterator {
raw: self.clone(),
next_index: 0,
next_offset: range.start(),
end_index: self.len(),
end_offset: range.end(),
}
}
}
impl fmt::Debug for SyntaxTrivia {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let mut f = f.debug_struct("SyntaxTrivia");
f.field("text_range", &self.text_range());
f.finish()
}
}
impl fmt::Display for SyntaxTrivia {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
fmt::Display::fmt(self.text(), f)
}
}
#[derive(Clone)]
pub struct SyntaxTriviaPiecesIterator {
pub(crate) raw: SyntaxTrivia,
pub(crate) next_index: usize,
pub(crate) next_offset: TextSize,
pub(crate) end_index: usize,
pub(crate) end_offset: TextSize,
}
impl Iterator for SyntaxTriviaPiecesIterator {
type Item = (TextSize, TriviaPiece);
fn next(&mut self) -> Option<Self::Item> {
let trivia = self.raw.get_piece(self.next_index)?;
let piece = (self.next_offset, *trivia);
self.next_index += 1;
self.next_offset += trivia.text_len();
Some(piece)
}
fn size_hint(&self) -> (usize, Option<usize>) {
let len = self.end_index.saturating_sub(self.next_index);
(len, Some(len))
}
}
impl FusedIterator for SyntaxTriviaPiecesIterator {}
impl DoubleEndedIterator for SyntaxTriviaPiecesIterator {
fn next_back(&mut self) -> Option<Self::Item> {
if self.end_index == self.next_index {
return None;
}
self.end_index -= 1;
let trivia = self.raw.get_piece(self.end_index)?;
self.end_offset -= trivia.text_len();
Some((self.end_offset, *trivia))
}
}
impl ExactSizeIterator for SyntaxTriviaPiecesIterator {}
#[cfg(test)]
mod tests {
use crate::raw_language::{RawLanguage, RawLanguageKind, RawSyntaxTreeBuilder};
use crate::{SyntaxNode, TriviaPiece, TriviaPieceKind};
#[test]
fn trivia_text() {
let mut builder = RawSyntaxTreeBuilder::new();
builder.start_node(RawLanguageKind::ROOT);
builder.token_with_trivia(
RawLanguageKind::WHITESPACE,
"\t let \t\t",
&[TriviaPiece::new(TriviaPieceKind::Whitespace, 2)],
&[TriviaPiece::new(TriviaPieceKind::Whitespace, 3)],
);
builder.finish_node();
let root = builder.finish_green();
let syntax: SyntaxNode<RawLanguage> = SyntaxNode::new_root(root);
let token = syntax.first_token().unwrap();
assert_eq!(token.leading_trivia().text(), "\t ");
assert_eq!(token.trailing_trivia().text(), " \t\t");
}
}

47
crates/ruff_rowan/src/green.rs Normal file
View File

@ -0,0 +1,47 @@
mod element;
mod node;
mod node_cache;
mod token;
mod trivia;
pub(crate) use self::{
element::{GreenElement, GreenElementRef},
node::{Child, Children, GreenNode, GreenNodeData, Slot},
token::{GreenToken, GreenTokenData},
trivia::GreenTrivia,
};
pub use self::node_cache::NodeCache;
pub(crate) use self::node_cache::NodeCacheNodeEntryMut;
/// RawSyntaxKind is a type tag for each token or node.
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct RawSyntaxKind(pub u16);
pub(crate) fn has_live() -> bool {
node::has_live() || token::has_live() || trivia::has_live()
}
#[cfg(test)]
mod tests {
use super::*;
use crate::green::trivia::GreenTrivia;
#[test]
fn assert_send_sync() {
fn f<T: Send + Sync>() {}
f::<GreenNode>();
f::<GreenToken>();
f::<GreenElement>();
}
#[test]
fn test_size_of() {
use std::mem::size_of;
assert_eq!(8, size_of::<GreenNode>());
assert_eq!(8, size_of::<GreenToken>());
assert_eq!(8, size_of::<GreenTrivia>());
assert_eq!(16, size_of::<GreenElement>());
}
}

94
crates/ruff_rowan/src/green/element.rs Normal file
View File

@ -0,0 +1,94 @@
use crate::{
green::{GreenNode, GreenToken, RawSyntaxKind},
GreenNodeData, NodeOrToken, TextSize,
};
use std::borrow::Cow;
use super::GreenTokenData;
pub(crate) type GreenElement = NodeOrToken<GreenNode, GreenToken>;
pub(crate) type GreenElementRef<'a> = NodeOrToken<&'a GreenNodeData, &'a GreenTokenData>;
impl From<GreenNode> for GreenElement {
#[inline]
fn from(node: GreenNode) -> GreenElement {
NodeOrToken::Node(node)
}
}
impl<'a> From<&'a GreenNode> for GreenElementRef<'a> {
#[inline]
fn from(node: &'a GreenNode) -> GreenElementRef<'a> {
NodeOrToken::Node(node)
}
}
impl From<GreenToken> for GreenElement {
#[inline]
fn from(token: GreenToken) -> GreenElement {
NodeOrToken::Token(token)
}
}
impl From<Cow<'_, GreenNodeData>> for GreenElement {
#[inline]
fn from(cow: Cow<'_, GreenNodeData>) -> Self {
NodeOrToken::Node(cow.into_owned())
}
}
impl<'a> From<&'a GreenToken> for GreenElementRef<'a> {
#[inline]
fn from(token: &'a GreenToken) -> GreenElementRef<'a> {
NodeOrToken::Token(token)
}
}
impl GreenElementRef<'_> {
pub fn to_owned(self) -> GreenElement {
match self {
NodeOrToken::Node(it) => NodeOrToken::Node(it.to_owned()),
NodeOrToken::Token(it) => NodeOrToken::Token(it.to_owned()),
}
}
}
impl GreenElement {
/// Returns kind of this element.
#[inline]
pub fn kind(&self) -> RawSyntaxKind {
match self {
NodeOrToken::Node(node) => node.kind(),
NodeOrToken::Token(token) => token.kind(),
}
}
/// Returns the length of the text covered by this element.
#[inline]
pub fn text_len(&self) -> TextSize {
match self {
NodeOrToken::Token(token) => token.text_len(),
NodeOrToken::Node(node) => node.text_len(),
}
}
}
impl GreenElementRef<'_> {
/// Returns kind of this element.
#[inline]
pub fn kind(&self) -> RawSyntaxKind {
match self {
NodeOrToken::Node(it) => it.kind(),
NodeOrToken::Token(it) => it.kind(),
}
}
/// Returns the length of the text covered by this element.
#[inline]
pub fn text_len(self) -> TextSize {
match self {
NodeOrToken::Node(it) => it.text_len(),
NodeOrToken::Token(it) => it.text_len(),
}
}
}

546
crates/ruff_rowan/src/green/node.rs Normal file
View File

@ -0,0 +1,546 @@
use std::convert::TryFrom;
use std::fmt::Formatter;
use std::iter::Enumerate;
use std::{
borrow::{Borrow, Cow},
fmt,
iter::FusedIterator,
mem::{self, ManuallyDrop},
ops, ptr, slice,
};
#[cfg(target_pointer_width = "64")]
use crate::utility_types::static_assert;
use countme::Count;
use crate::{
arc::{Arc, HeaderSlice, ThinArc},
green::{GreenElement, GreenElementRef, RawSyntaxKind},
GreenToken, NodeOrToken, TextRange, TextSize,
};
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub(super) struct GreenNodeHead {
kind: RawSyntaxKind,
text_len: TextSize,
_c: Count<GreenNode>,
}
pub(crate) fn has_live() -> bool {
countme::get::<GreenNode>().live > 0
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub(crate) enum Slot {
Node {
rel_offset: TextSize,
node: GreenNode,
},
Token {
rel_offset: TextSize,
token: GreenToken,
},
/// An empty slot for a child that was missing in the source because:
/// * it's an optional child which is missing for this node
/// * it's a mandatory child but it's missing because of a syntax error
Empty { rel_offset: TextSize },
}
impl std::fmt::Display for Slot {
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
match self {
Slot::Empty { .. } => write!(f, "∅"),
Slot::Node { node, .. } => std::fmt::Display::fmt(node, f),
Slot::Token { token, .. } => std::fmt::Display::fmt(token, f),
}
}
}
#[cfg(target_pointer_width = "64")]
static_assert!(mem::size_of::<Slot>() == mem::size_of::<usize>() * 2);
type Repr = HeaderSlice<GreenNodeHead, [Slot]>;
type ReprThin = HeaderSlice<GreenNodeHead, [Slot; 0]>;
#[repr(transparent)]
pub(crate) struct GreenNodeData {
data: ReprThin,
}
impl PartialEq for GreenNodeData {
fn eq(&self, other: &Self) -> bool {
self.header() == other.header() && self.slice() == other.slice()
}
}
/// Internal node in the immutable tree.
/// It has other nodes and tokens as children.
#[derive(Clone, PartialEq, Eq, Hash)]
#[repr(transparent)]
pub(crate) struct GreenNode {
ptr: ThinArc<GreenNodeHead, Slot>,
}
impl ToOwned for GreenNodeData {
type Owned = GreenNode;
#[inline]
fn to_owned(&self) -> GreenNode {
unsafe {
let green = GreenNode::from_raw(ptr::NonNull::from(self));
let green = ManuallyDrop::new(green);
GreenNode::clone(&green)
}
}
}
impl Borrow<GreenNodeData> for GreenNode {
#[inline]
fn borrow(&self) -> &GreenNodeData {
self
}
}
impl From<Cow<'_, GreenNodeData>> for GreenNode {
#[inline]
fn from(cow: Cow<'_, GreenNodeData>) -> Self {
cow.into_owned()
}
}
impl From<&'_ GreenNodeData> for GreenNode {
#[inline]
fn from(borrow: &'_ GreenNodeData) -> Self {
borrow.to_owned()
}
}
impl fmt::Debug for GreenNodeData {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("GreenNode")
.field("kind", &self.kind())
.field("text_len", &self.text_len())
.field("n_slots", &self.slots().len())
.finish()
}
}
impl fmt::Debug for GreenNode {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let data: &GreenNodeData = self;
fmt::Debug::fmt(data, f)
}
}
impl fmt::Display for GreenNode {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let data: &GreenNodeData = self;
fmt::Display::fmt(data, f)
}
}
impl fmt::Display for GreenNodeData {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
for child in self.slots() {
write!(f, "{}", child)?;
}
Ok(())
}
}
impl GreenNodeData {
#[inline]
fn header(&self) -> &GreenNodeHead {
&self.data.header
}
#[inline]
pub(crate) fn slice(&self) -> &[Slot] {
self.data.slice()
}
/// Kind of this node.
#[inline]
pub fn kind(&self) -> RawSyntaxKind {
self.header().kind
}
/// Returns the length of the text covered by this node.
#[inline]
pub fn text_len(&self) -> TextSize {
self.header().text_len
}
/// Children of this node.
#[inline]
pub fn children(&self) -> Children<'_> {
Children::new(self.slots().enumerate())
}
/// Returns the slots of this node. Every node of a specific kind has the same number of slots
/// to allow using fixed offsets to retrieve a specific child even if some other child is missing.
#[inline]
pub fn slots(&self) -> Slots<'_> {
Slots {
raw: self.slice().iter(),
}
}
pub(crate) fn slot_at_range(
&self,
rel_range: TextRange,
) -> Option<(usize, TextSize, &'_ Slot)> {
let idx = self
.slice()
.binary_search_by(|it| {
let child_range = it.rel_range();
TextRange::ordering(child_range, rel_range)
})
// XXX: this handles empty ranges
.unwrap_or_else(|it| it.saturating_sub(1));
let slot = &self
.slice()
.get(idx)
.filter(|it| it.rel_range().contains_range(rel_range))?;
Some((idx, slot.rel_offset(), slot))
}
#[must_use = "syntax elements are immutable, the result of update methods must be propagated to have any effect"]
pub(crate) fn splice_slots<R, I>(&self, range: R, replace_with: I) -> GreenNode
where
R: ops::RangeBounds<usize>,
I: Iterator<Item = Option<GreenElement>>,
{
let mut slots: Vec<_> = self
.slots()
.map(|slot| match slot {
Slot::Empty { .. } => None,
Slot::Node { node, .. } => Some(NodeOrToken::Node(node.to_owned())),
Slot::Token { token, .. } => Some(NodeOrToken::Token(token.to_owned())),
})
.collect();
slots.splice(range, replace_with);
GreenNode::new(self.kind(), slots)
}
}
impl ops::Deref for GreenNode {
type Target = GreenNodeData;
#[inline]
fn deref(&self) -> &GreenNodeData {
unsafe {
let repr: &Repr = &self.ptr;
let repr: &ReprThin = &*(repr as *const Repr as *const ReprThin);
mem::transmute::<&ReprThin, &GreenNodeData>(repr)
}
}
}
impl GreenNode {
/// Creates new Node.
#[inline]
pub fn new<I>(kind: RawSyntaxKind, slots: I) -> GreenNode
where
I: IntoIterator<Item = Option<GreenElement>>,
I::IntoIter: ExactSizeIterator,
{
let mut text_len: TextSize = 0.into();
let slots = slots.into_iter().map(|el| {
let rel_offset = text_len;
match el {
Some(el) => {
text_len += el.text_len();
match el {
NodeOrToken::Node(node) => Slot::Node { rel_offset, node },
NodeOrToken::Token(token) => Slot::Token { rel_offset, token },
}
}
None => Slot::Empty { rel_offset },
}
});
let data = ThinArc::from_header_and_iter(
GreenNodeHead {
kind,
text_len: 0.into(),
_c: Count::new(),
},
slots,
);
// XXX: fixup `text_len` after construction, because we can't iterate
// `slots` twice.
let data = {
let mut data = Arc::from_thin(data);
Arc::get_mut(&mut data).unwrap().header.text_len = text_len;
Arc::into_thin(data)
};
GreenNode { ptr: data }
}
#[inline]
pub(crate) unsafe fn from_raw(ptr: ptr::NonNull<GreenNodeData>) -> GreenNode {
let arc = Arc::from_raw(&ptr.as_ref().data as *const ReprThin);
let arc = mem::transmute::<Arc<ReprThin>, ThinArc<GreenNodeHead, Slot>>(arc);
GreenNode { ptr: arc }
}
}
impl Slot {
#[inline]
pub(crate) fn as_ref(&self) -> Option<GreenElementRef> {
match self {
Slot::Node { node, .. } => Some(NodeOrToken::Node(node)),
Slot::Token { token, .. } => Some(NodeOrToken::Token(token)),
Slot::Empty { .. } => None,
}
}
#[inline]
pub(crate) fn rel_offset(&self) -> TextSize {
match self {
Slot::Node { rel_offset, .. }
| Slot::Token { rel_offset, .. }
| Slot::Empty { rel_offset } => *rel_offset,
}
}
#[inline]
fn rel_range(&self) -> TextRange {
let text_len = match self.as_ref() {
None => TextSize::from(0),
Some(element) => element.text_len(),
};
TextRange::at(self.rel_offset(), text_len)
}
}
#[derive(Debug, Clone)]
pub(crate) struct Slots<'a> {
pub(crate) raw: slice::Iter<'a, Slot>,
}
// NB: forward everything stable that iter::Slice specializes as of Rust 1.39.0
impl ExactSizeIterator for Slots<'_> {
#[inline(always)]
fn len(&self) -> usize {
self.raw.len()
}
}
impl<'a> Iterator for Slots<'a> {
type Item = &'a Slot;
#[inline]
fn next(&mut self) -> Option<&'a Slot> {
self.raw.next()
}
#[inline]
fn size_hint(&self) -> (usize, Option<usize>) {
self.raw.size_hint()
}
#[inline]
fn count(self) -> usize
where
Self: Sized,
{
self.raw.count()
}
#[inline]
fn last(mut self) -> Option<Self::Item>
where
Self: Sized,
{
self.next_back()
}
#[inline]
fn nth(&mut self, n: usize) -> Option<Self::Item> {
self.raw.nth(n)
}
#[inline]
fn fold<Acc, Fold>(self, init: Acc, mut f: Fold) -> Acc
where
Fold: FnMut(Acc, Self::Item) -> Acc,
{
let mut accum = init;
for x in self {
accum = f(accum, x);
}
accum
}
}
impl<'a> DoubleEndedIterator for Slots<'a> {
#[inline]
fn next_back(&mut self) -> Option<Self::Item> {
self.raw.next_back()
}
#[inline]
fn nth_back(&mut self, n: usize) -> Option<Self::Item> {
self.raw.nth_back(n)
}
#[inline]
fn rfold<Acc, Fold>(mut self, init: Acc, mut f: Fold) -> Acc
where
Fold: FnMut(Acc, Self::Item) -> Acc,
{
let mut accum = init;
while let Some(x) = self.next_back() {
accum = f(accum, x);
}
accum
}
}
impl FusedIterator for Slots<'_> {}
#[derive(Debug, Clone)]
pub(crate) struct Child<'a> {
slot: u32,
rel_offset: TextSize,
element: GreenElementRef<'a>,
}
impl<'a> Child<'a> {
pub fn slot(&self) -> u32 {
self.slot
}
pub fn rel_offset(&self) -> TextSize {
self.rel_offset
}
pub fn element(&self) -> GreenElementRef<'a> {
self.element
}
}
impl<'a> TryFrom<(usize, &'a Slot)> for Child<'a> {
type Error = ();
fn try_from((index, slot): (usize, &'a Slot)) -> Result<Self, Self::Error> {
match slot {
Slot::Empty { .. } => Err(()),
Slot::Node { node, rel_offset } => Ok(Child {
element: NodeOrToken::Node(node),
slot: index as u32,
rel_offset: *rel_offset,
}),
Slot::Token { token, rel_offset } => Ok(Child {
element: NodeOrToken::Token(token),
slot: index as u32,
rel_offset: *rel_offset,
}),
}
}
}
#[derive(Debug, Clone)]
pub(crate) struct Children<'a> {
slots: Enumerate<Slots<'a>>,
}
impl<'a> Children<'a> {
pub fn new(slots: Enumerate<Slots<'a>>) -> Self {
Self { slots }
}
}
impl<'a> Iterator for Children<'a> {
type Item = Child<'a>;
fn next(&mut self) -> Option<Self::Item> {
self.slots.find_map(|it| Child::try_from(it).ok())
}
fn size_hint(&self) -> (usize, Option<usize>) {
self.slots.size_hint()
}
}
impl<'a> DoubleEndedIterator for Children<'a> {
fn next_back(&mut self) -> Option<Self::Item> {
loop {
let next = self.slots.next_back()?;
if let Ok(child) = Child::try_from(next) {
return Some(child);
}
}
}
}
impl FusedIterator for Children<'_> {}
#[cfg(test)]
mod tests {
use crate::raw_language::{RawLanguageKind, RawSyntaxTreeBuilder};
use crate::GreenNode;
fn build_test_list() -> GreenNode {
let mut builder: RawSyntaxTreeBuilder = RawSyntaxTreeBuilder::new();
// list
builder.start_node(RawLanguageKind::SEPARATED_EXPRESSION_LIST);
// element 1
builder.start_node(RawLanguageKind::LITERAL_EXPRESSION);
builder.token(RawLanguageKind::STRING_TOKEN, "a");
builder.finish_node();
// Missing ,
// element 2
builder.start_node(RawLanguageKind::LITERAL_EXPRESSION);
builder.token(RawLanguageKind::STRING_TOKEN, "b");
builder.finish_node();
builder.finish_node();
builder.finish_green()
}
#[test]
fn children() {
let root = build_test_list();
// Test that children skips missing
assert_eq!(root.children().count(), 2);
assert_eq!(
root.children()
.map(|child| child.element.to_string())
.collect::<Vec<_>>(),
vec!["a", "b"]
);
// Slot 2 (index 1) is empty
assert_eq!(
root.children().map(|child| child.slot).collect::<Vec<_>>(),
vec![0, 2]
);
// Same when reverse
assert_eq!(
root.children()
.rev()
.map(|child| child.slot)
.collect::<Vec<_>>(),
vec![2, 0]
);
}
#[test]
fn slots() {
let root = build_test_list();
// Has 3 slots, one is missing
assert_eq!(root.slots().len(), 3);
}
}

358
crates/ruff_rowan/src/green/node_cache.rs Normal file
View File

@ -0,0 +1,358 @@
use hashbrown::hash_map::{RawEntryMut, RawOccupiedEntryMut, RawVacantEntryMut};
use ruff_text_size::TextSize;
use rustc_hash::FxHasher;
use std::hash::{BuildHasherDefault, Hash, Hasher};
use crate::green::Slot;
use crate::syntax::{TriviaPiece, TriviaPieceKind};
use crate::{
green::GreenElementRef, GreenNode, GreenNodeData, GreenToken, GreenTokenData, NodeOrToken,
RawSyntaxKind,
};
use super::element::GreenElement;
use super::trivia::GreenTrivia;
type HashMap<K, V> = hashbrown::HashMap<K, V, BuildHasherDefault<FxHasher>>;
/// A token stored in the `NodeCache`.
/// Does intentionally not implement `Hash` to have compile-time guarantees that the `NodeCache`
/// uses the correct hash.
#[derive(Debug)]
struct CachedToken(GreenToken);
/// A node stored in the `NodeCache`. It stores a pre-computed hash
/// because re-computing the hash requires traversing the whole sub-tree.
/// The hash also differs from the `GreenNode` hash implementation as it
/// only hashes occupied slots and excludes empty slots.
///
/// Does intentionally not implement `Hash` to have compile-time guarantees that the `NodeCache`
/// uses the correct hash.
#[derive(Debug)]
struct CachedNode {
node: GreenNode,
// Store the hash as it's expensive to re-compute
// involves re-computing the hash of the whole sub-tree
hash: u64,
}
/// Interner for GreenTokens and GreenNodes
// XXX: the impl is a bit tricky. As usual when writing interners, we want to
// store all values in one HashSet.
//
// However, hashing trees is fun: hash of the tree is recursively defined. We
// maintain an invariant -- if the tree is interned, then all of its children
// are interned as well.
//
// That means that computing the hash naively is wasteful -- we just *know*
// hashes of children, and we can re-use those.
//
// So here we use *raw* API of hashbrown and provide the hashes manually,
// instead of going via a `Hash` impl. Our manual `Hash` and the
// `#[derive(Hash)]` are actually different! At some point we had a fun bug,
// where we accidentally mixed the two hashes, which made the cache much less
// efficient.
//
// To fix that, we additionally wrap the data in `Cached*` wrappers, to make sure
// we don't accidentally use the wrong hash!
#[derive(Default, Debug)]
pub struct NodeCache {
nodes: HashMap<CachedNode, ()>,
tokens: HashMap<CachedToken, ()>,
trivia: TriviaCache,
}
fn token_hash_of(kind: RawSyntaxKind, text: &str) -> u64 {
let mut h = FxHasher::default();
kind.hash(&mut h);
text.hash(&mut h);
h.finish()
}
fn token_hash(token: &GreenTokenData) -> u64 {
token_hash_of(token.kind(), token.text())
}
fn element_id(elem: GreenElementRef<'_>) -> *const () {
match elem {
NodeOrToken::Node(it) => it as *const GreenNodeData as *const (),
NodeOrToken::Token(it) => it as *const GreenTokenData as *const (),
}
}
impl NodeCache {
/// Hash used for nodes that haven't been cached because it has too many slots or
/// one of its children wasn't cached.
const UNCACHED_NODE_HASH: u64 = 0;
/// Tries to retrieve a node with the given `kind` and `children` from the cache.
///
/// Returns an entry that allows the caller to:
/// * Retrieve the cached node if it is present in the cache
/// * Insert a node if it isn't present in the cache
pub(crate) fn node(
&mut self,
kind: RawSyntaxKind,
children: &[(u64, GreenElement)],
) -> NodeCacheNodeEntryMut {
if children.len() > 3 {
return NodeCacheNodeEntryMut::NoCache(Self::UNCACHED_NODE_HASH);
}
let hash = {
let mut h = FxHasher::default();
kind.hash(&mut h);
for &(hash, _) in children {
if hash == Self::UNCACHED_NODE_HASH {
return NodeCacheNodeEntryMut::NoCache(Self::UNCACHED_NODE_HASH);
}
hash.hash(&mut h);
}
h.finish()
};
// Green nodes are fully immutable, so it's ok to deduplicate them.
// This is the same optimization that Roslyn does
// https://github.com/KirillOsenkov/Bliki/wiki/Roslyn-Immutable-Trees
//
// For example, all `#[inline]` in this file share the same green node!
// For `libsyntax/parse/parser.rs`, measurements show that deduping saves
// 17% of the memory for green nodes!
let entry = self.nodes.raw_entry_mut().from_hash(hash, |no_hash| {
no_hash.node.kind() == kind && {
let lhs = no_hash.node.slots().filter_map(|slot| match slot {
// Ignore empty slots. The queried node only has the present children
Slot::Empty { .. } => None,
Slot::Node { node, .. } => Some(element_id(NodeOrToken::Node(node))),
Slot::Token { token, .. } => Some(element_id(NodeOrToken::Token(token))),
});
let rhs = children
.iter()
.map(|(_, element)| element_id(element.as_deref()));
lhs.eq(rhs)
}
});
match entry {
RawEntryMut::Occupied(entry) => NodeCacheNodeEntryMut::Cached(CachedNodeEntry {
hash,
raw_entry: entry,
}),
RawEntryMut::Vacant(entry) => NodeCacheNodeEntryMut::Vacant(VacantNodeEntry {
raw_entry: entry,
original_kind: kind,
hash,
}),
}
}
pub(crate) fn token(&mut self, kind: RawSyntaxKind, text: &str) -> (u64, GreenToken) {
self.token_with_trivia(kind, text, &[], &[])
}
pub(crate) fn token_with_trivia(
&mut self,
kind: RawSyntaxKind,
text: &str,
leading: &[TriviaPiece],
trailing: &[TriviaPiece],
) -> (u64, GreenToken) {
let hash = token_hash_of(kind, text);
let entry = self.tokens.raw_entry_mut().from_hash(hash, |token| {
token.0.kind() == kind && token.0.text() == text
});
let token = match entry {
RawEntryMut::Occupied(entry) => entry.key().0.clone(),
RawEntryMut::Vacant(entry) => {
let leading = self.trivia.get(leading);
let trailing = self.trivia.get(trailing);
let token = GreenToken::with_trivia(kind, text, leading, trailing);
entry
.insert_with_hasher(hash, CachedToken(token.clone()), (), |t| token_hash(&t.0));
token
}
};
(hash, token)
}
}
pub(crate) enum NodeCacheNodeEntryMut<'a> {
Cached(CachedNodeEntry<'a>),
/// A node that should not be cached
NoCache(u64),
Vacant(VacantNodeEntry<'a>),
}
/// Represents a vacant entry, a node that hasn't been cached yet.
/// The `insert` method allows to place a node inside of the vacant entry. The inserted node
/// may have a different representation (kind or children) than the originally queried node.
/// For example, a node may change its kind to bogus or add empty slots. The only importance is
/// that these changes apply for all nodes that have the same shape as the originally queried node.
pub(crate) struct VacantNodeEntry<'a> {
hash: u64,
original_kind: RawSyntaxKind,
raw_entry: RawVacantEntryMut<'a, CachedNode, (), BuildHasherDefault<FxHasher>>,
}
/// Represents an entry of a cached node.
pub(crate) struct CachedNodeEntry<'a> {
hash: u64,
raw_entry: RawOccupiedEntryMut<'a, CachedNode, (), BuildHasherDefault<FxHasher>>,
}
impl<'a> CachedNodeEntry<'a> {
pub fn node(&self) -> &GreenNode {
&self.raw_entry.key().node
}
pub fn hash(&self) -> u64 {
self.hash
}
}
impl<'a> VacantNodeEntry<'a> {
/// Inserts the `node` into the cache so that future queries for the same kind and children resolve to the passed `node`.
///
/// Returns the hash of the node.
///
/// The cache does not cache the `node` if the kind doesn't match the `kind` of the queried node because
/// cache lookups wouldn't be successful because the hash collision prevention check compares the kinds of the
/// cached and queried node.
pub fn cache(self, node: GreenNode) -> u64 {
if self.original_kind != node.kind() {
// The kind has changed since it has been queried. For example, the node has been converted to an
// unknown node. Never cache these nodes because cache lookups will never match.
NodeCache::UNCACHED_NODE_HASH
} else {
self.raw_entry.insert_with_hasher(
self.hash,
CachedNode {
node,
hash: self.hash,
},
(),
|n| n.hash,
);
self.hash
}
}
}
/// A cached [GreenTrivia].
/// Deliberately doesn't implement `Hash` to make sure all
/// usages go through the custom `FxHasher`.
#[derive(Debug)]
struct CachedTrivia(GreenTrivia);
#[derive(Debug)]
struct TriviaCache {
/// Generic cache for trivia
cache: HashMap<CachedTrivia, ()>,
/// Cached single whitespace trivia.
whitespace: GreenTrivia,
}
impl Default for TriviaCache {
fn default() -> Self {
Self {
cache: Default::default(),
whitespace: GreenTrivia::new([TriviaPiece::whitespace(1)]),
}
}
}
impl TriviaCache {
/// Tries to retrieve a [GreenTrivia] with the given pieces from the cache or creates a new one and caches
/// it for further calls.
fn get(&mut self, pieces: &[TriviaPiece]) -> GreenTrivia {
match pieces {
[] => GreenTrivia::empty(),
[TriviaPiece {
kind: TriviaPieceKind::Whitespace,
length,
}] if *length == TextSize::from(1) => self.whitespace.clone(),
_ => {
let hash = Self::trivia_hash_of(pieces);
let entry = self
.cache
.raw_entry_mut()
.from_hash(hash, |trivia| trivia.0.pieces() == pieces);
match entry {
RawEntryMut::Occupied(entry) => entry.key().0.clone(),
RawEntryMut::Vacant(entry) => {
let trivia = GreenTrivia::new(pieces.iter().copied());
entry.insert_with_hasher(
hash,
CachedTrivia(trivia.clone()),
(),
|cached| Self::trivia_hash_of(cached.0.pieces()),
);
trivia
}
}
}
}
}
fn trivia_hash_of(pieces: &[TriviaPiece]) -> u64 {
let mut h = FxHasher::default();
pieces.len().hash(&mut h);
for piece in pieces {
piece.hash(&mut h);
}
h.finish()
}
}
#[cfg(test)]
mod tests {
use crate::green::node_cache::token_hash;
use crate::green::trivia::GreenTrivia;
use crate::{GreenToken, RawSyntaxKind};
use ruff_text_size::TextSize;
#[test]
fn green_token_hash() {
let kind = RawSyntaxKind(0);
let text = " let ";
let t1 = GreenToken::with_trivia(
kind,
text,
GreenTrivia::whitespace(TextSize::from(1)),
GreenTrivia::whitespace(TextSize::from(1)),
);
let t2 = GreenToken::with_trivia(
kind,
text,
GreenTrivia::whitespace(1),
GreenTrivia::whitespace(1),
);
assert_eq!(token_hash(&t1), token_hash(&t2));
let t3 = GreenToken::new(kind, "let");
assert_ne!(token_hash(&t1), token_hash(&t3));
let t4 = GreenToken::with_trivia(
kind,
"\tlet ",
GreenTrivia::whitespace(1),
GreenTrivia::whitespace(1),
);
assert_ne!(token_hash(&t1), token_hash(&t4));
}
}

235
crates/ruff_rowan/src/green/token.rs Normal file
View File

@ -0,0 +1,235 @@
use std::{
borrow::Borrow,
fmt,
mem::{self, ManuallyDrop},
ops, ptr,
};
use countme::Count;
use crate::green::trivia::GreenTrivia;
use crate::{
arc::{Arc, HeaderSlice, ThinArc},
green::RawSyntaxKind,
TextSize,
};
#[derive(PartialEq, Eq, Hash)]
struct GreenTokenHead {
kind: RawSyntaxKind,
leading: GreenTrivia,
trailing: GreenTrivia,
_c: Count<GreenToken>,
}
pub(crate) fn has_live() -> bool {
countme::get::<GreenToken>().live > 0
}
type Repr = HeaderSlice<GreenTokenHead, [u8]>;
type ReprThin = HeaderSlice<GreenTokenHead, [u8; 0]>;
#[repr(transparent)]
pub(crate) struct GreenTokenData {
data: ReprThin,
}
impl PartialEq for GreenTokenData {
fn eq(&self, other: &Self) -> bool {
self.kind() == other.kind() && self.text() == other.text()
}
}
/// Leaf node in the immutable tree.
#[derive(PartialEq, Eq, Hash, Clone)]
#[repr(transparent)]
pub(crate) struct GreenToken {
ptr: ThinArc<GreenTokenHead, u8>,
}
impl ToOwned for GreenTokenData {
type Owned = GreenToken;
#[inline]
fn to_owned(&self) -> GreenToken {
unsafe {
let green = GreenToken::from_raw(ptr::NonNull::from(self));
let green = ManuallyDrop::new(green);
GreenToken::clone(&green)
}
}
}
impl Borrow<GreenTokenData> for GreenToken {
#[inline]
fn borrow(&self) -> &GreenTokenData {
self
}
}
impl fmt::Debug for GreenTokenData {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("GreenToken")
.field("kind", &self.kind())
.field("text", &self.text())
.field("leading", &self.leading_trivia())
.field("trailing", &self.trailing_trivia())
.finish()
}
}
impl fmt::Debug for GreenToken {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let data: &GreenTokenData = self;
fmt::Debug::fmt(data, f)
}
}
impl fmt::Display for GreenToken {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let data: &GreenTokenData = self;
fmt::Display::fmt(data, f)
}
}
impl fmt::Display for GreenTokenData {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{}", self.text())
}
}
impl GreenTokenData {
/// Kind of this Token.
#[inline]
pub fn kind(&self) -> RawSyntaxKind {
self.data.header.kind
}
/// Whole text of this Token, including all trivia.
#[inline]
pub fn text(&self) -> &str {
unsafe { std::str::from_utf8_unchecked(self.data.slice()) }
}
pub(crate) fn leading_trailing_total_len(&self) -> (TextSize, TextSize, TextSize) {
let leading_len = self.data.header.leading.text_len();
let trailing_len = self.data.header.trailing.text_len();
let total_len = self.data.slice().len() as u32;
(leading_len, trailing_len, total_len.into())
}
/// Text of this Token, excluding all trivia.
#[inline]
pub fn text_trimmed(&self) -> &str {
let (leading_len, trailing_len, total_len) = self.leading_trailing_total_len();
let start: usize = leading_len.into();
let end: usize = (total_len - trailing_len).into();
let text = unsafe { std::str::from_utf8_unchecked(self.data.slice()) };
&text[start..end]
}
/// Returns the length of the text covered by this token.
#[inline]
pub fn text_len(&self) -> TextSize {
TextSize::of(self.text())
}
#[inline]
pub fn leading_trivia(&self) -> &GreenTrivia {
&self.data.header.leading
}
#[inline]
pub fn trailing_trivia(&self) -> &GreenTrivia {
&self.data.header.trailing
}
}
impl GreenToken {
#[inline]
#[cfg(test)]
pub fn new(kind: RawSyntaxKind, text: &str) -> GreenToken {
let leading = GreenTrivia::empty();
let trailing = leading.clone();
Self::with_trivia(kind, text, leading, trailing)
}
#[inline]
pub fn with_trivia(
kind: RawSyntaxKind,
text: &str,
leading: GreenTrivia,
trailing: GreenTrivia,
) -> GreenToken {
let head = GreenTokenHead {
kind,
leading,
trailing,
_c: Count::new(),
};
let ptr = ThinArc::from_header_and_iter(head, text.bytes());
GreenToken { ptr }
}
#[inline]
pub(crate) unsafe fn from_raw(ptr: ptr::NonNull<GreenTokenData>) -> GreenToken {
let arc = Arc::from_raw(&ptr.as_ref().data as *const ReprThin);
let arc = mem::transmute::<Arc<ReprThin>, ThinArc<GreenTokenHead, u8>>(arc);
GreenToken { ptr: arc }
}
}
impl ops::Deref for GreenToken {
type Target = GreenTokenData;
#[inline]
fn deref(&self) -> &GreenTokenData {
unsafe {
let repr: &Repr = &self.ptr;
let repr: &ReprThin = &*(repr as *const Repr as *const ReprThin);
mem::transmute::<&ReprThin, &GreenTokenData>(repr)
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use quickcheck_macros::*;
#[test]
fn green_token_text_and_len() {
let t = GreenToken::with_trivia(
RawSyntaxKind(0),
"\n\t let \t\t",
GreenTrivia::whitespace(3),
GreenTrivia::whitespace(3),
);
assert_eq!("\n\t let \t\t", t.text());
assert_eq!(TextSize::from(9), t.text_len());
assert_eq!("let", t.text_trimmed());
assert_eq!("\n\t let \t\t", format!("{}", t));
}
#[test]
fn empty_text_len() {
assert_eq!(TextSize::from(0), GreenTrivia::empty().text_len());
}
#[quickcheck]
fn whitespace_and_comments_text_len(len: u32) {
let len = TextSize::from(len);
assert_eq!(len, GreenTrivia::whitespace(len).text_len());
assert_eq!(len, GreenTrivia::single_line_comment(len).text_len());
}
#[test]
fn sizes() {
assert_eq!(24, std::mem::size_of::<GreenTokenHead>());
assert_eq!(8, std::mem::size_of::<GreenToken>());
}
}

148
crates/ruff_rowan/src/green/trivia.rs Normal file
View File

@ -0,0 +1,148 @@
use crate::arc::{HeaderSlice, ThinArc};
use crate::TriviaPiece;
use countme::Count;
use ruff_text_size::TextSize;
use std::fmt;
use std::fmt::Formatter;
#[derive(PartialEq, Eq, Hash)]
pub(crate) struct GreenTriviaHead {
_c: Count<GreenTrivia>,
}
pub(crate) fn has_live() -> bool {
countme::get::<GreenTrivia>().live > 0
}
type ReprThin = HeaderSlice<GreenTriviaHead, [TriviaPiece; 0]>;
#[repr(transparent)]
pub(crate) struct GreenTriviaData {
data: ReprThin,
}
impl GreenTriviaData {
#[allow(unused)]
#[inline]
pub fn header(&self) -> &GreenTriviaHead {
&self.data.header
}
#[inline]
pub fn pieces(&self) -> &[TriviaPiece] {
self.data.slice()
}
}
impl PartialEq for GreenTriviaData {
fn eq(&self, other: &Self) -> bool {
self.pieces() == other.pieces()
}
}
impl fmt::Debug for GreenTriviaData {
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
f.debug_list().entries(self.pieces().iter()).finish()
}
}
/// List of trivia. Used to store either the leading or trailing trivia of a token.
/// The identity of a trivia is defined by the kinds and lengths of its items but not by
/// the texts of an individual piece. That means, that `\r` and `\n` can both be represented
/// by the same trivia, a trivia with a single `LINEBREAK` piece with the length 1.
/// This is safe because the text is stored on the token to which the trivia belongs and
/// `a\n` and `a\r` never resolve to the same tokens. Thus, they only share the trivia but are
/// otherwise two different tokens.
#[derive(Eq, PartialEq, Hash, Clone)]
#[repr(transparent)]
pub(crate) struct GreenTrivia {
ptr: Option<ThinArc<GreenTriviaHead, TriviaPiece>>,
}
impl fmt::Debug for GreenTrivia {
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
fmt::Debug::fmt(self.pieces(), f)
}
}
impl GreenTrivia {
/// Creates a new trivia containing the passed in pieces
pub fn new<I>(pieces: I) -> Self
where
I: IntoIterator<Item = TriviaPiece>,
I::IntoIter: ExactSizeIterator,
{
let data =
ThinArc::from_header_and_iter(GreenTriviaHead { _c: Count::new() }, pieces.into_iter());
GreenTrivia { ptr: Some(data) }
}
/// Creates an empty trivia
pub fn empty() -> Self {
GreenTrivia { ptr: None }
}
/// Returns the total length of all pieces
pub fn text_len(&self) -> TextSize {
let mut len = TextSize::default();
for piece in self.pieces() {
len += piece.length
}
len
}
/// Returns the pieces count
pub fn len(&self) -> usize {
match &self.ptr {
None => 0,
Some(ptr) => ptr.len(),
}
}
/// Returns the pieces of the trivia
pub fn pieces(&self) -> &[TriviaPiece] {
match &self.ptr {
None => &[],
Some(ptr) => ptr.slice(),
}
}
/// Returns the piece at the given index.
pub fn get_piece(&self, index: usize) -> Option<&TriviaPiece> {
self.pieces().get(index)
}
}
#[cfg(test)]
mod tests {
use crate::green::trivia::{GreenTrivia, GreenTriviaHead};
use crate::syntax::TriviaPieceKind;
use crate::TriviaPiece;
use ruff_text_size::TextSize;
impl GreenTrivia {
/// Creates a trivia with a single whitespace piece
pub fn whitespace<L: Into<TextSize>>(len: L) -> Self {
Self::single(TriviaPieceKind::Whitespace, len.into())
}
/// Creates a trivia with one single line comment piece
pub fn single_line_comment<L: Into<TextSize>>(len: L) -> Self {
Self::single(TriviaPieceKind::SingleLineComment, len.into())
}
/// Creates a trivia containing a single piece
pub fn single<L: Into<TextSize>>(kind: TriviaPieceKind, len: L) -> Self {
Self::new(std::iter::once(TriviaPiece::new(kind, len)))
}
}
#[test]
fn sizes() {
assert_eq!(0, std::mem::size_of::<GreenTriviaHead>());
assert_eq!(8, std::mem::size_of::<GreenTrivia>());
}
}

63
crates/ruff_rowan/src/lib.rs Normal file
View File

@ -0,0 +1,63 @@
//! A generic library for lossless syntax trees.
//! See `examples/s_expressions.rs` for a tutorial.
#![allow(clippy::pedantic)]
#![forbid(
// missing_debug_implementations,
unconditional_recursion,
future_incompatible,
// missing_docs,
)]
#![deny(unsafe_code)]
#![deny(rustdoc::broken_intra_doc_links)]
#[doc(hidden)]
pub mod macros;
#[allow(unsafe_code)]
pub mod cursor;
#[allow(unsafe_code)]
mod green;
pub mod syntax;
mod syntax_node_text;
mod utility_types;
#[allow(unsafe_code)]
mod arc;
mod ast;
mod cow_mut;
pub mod raw_language;
#[cfg(feature = "serde")]
mod serde_impls;
mod syntax_factory;
mod syntax_token_text;
mod tree_builder;
pub use ruff_text_size::{TextLen, TextRange, TextSize};
pub use crate::{
ast::*,
green::RawSyntaxKind,
syntax::{
chain_trivia_pieces, ChainTriviaPiecesIterator, Language, SendNode, SyntaxElement,
SyntaxElementChildren, SyntaxKind, SyntaxList, SyntaxNode, SyntaxNodeChildren,
SyntaxNodeOptionExt, SyntaxRewriter, SyntaxSlot, SyntaxToken, SyntaxTriviaPiece,
SyntaxTriviaPieceComments, TriviaPiece, TriviaPieceKind, VisitNodeSignal,
},
syntax_factory::*,
syntax_node_text::SyntaxNodeText,
syntax_token_text::SyntaxTokenText,
tree_builder::{Checkpoint, TreeBuilder},
utility_types::{Direction, NodeOrToken, TokenAtOffset, WalkEvent},
};
pub(crate) use crate::green::{GreenNode, GreenNodeData, GreenToken, GreenTokenData};
pub fn check_live() -> Option<String> {
if cursor::has_live() || green::has_live() {
Some(countme::get_all().to_string())
} else {
None
}
}

168
crates/ruff_rowan/src/macros.rs Normal file
View File

@ -0,0 +1,168 @@
use crate::{AstNode, Language};
/// Matches a `SyntaxNode` against an `ast` type.
///
/// # Example:
///
/// ```
/// use ruff_rowan::{match_ast, AstNode};
/// use ruff_rowan::raw_language::{LiteralExpression, RawLanguageRoot, RawLanguageKind, RawSyntaxTreeBuilder};
///
/// let mut builder = RawSyntaxTreeBuilder::new();
/// builder.start_node(RawLanguageKind::ROOT);
/// builder.start_node(RawLanguageKind::LITERAL_EXPRESSION);
/// builder.token(RawLanguageKind::NUMBER_TOKEN, "5");
/// builder.finish_node();
/// builder.finish_node();
///
/// let root = builder.finish();
///
/// let text = match_ast! {
/// match &root {
/// RawLanguageRoot(root) => { format!("root: {}", root.text()) },
/// LiteralExpression(literal) => { format!("literal: {}", literal.text()) },
/// _ => {
/// root.text().to_string()
/// }
/// }
/// };
///
/// assert_eq!(text, "root: 5");
/// ```
#[macro_export]
macro_rules! match_ast {
// Necessary because expressions aren't allowed in front of `{`
(match &$node:ident { $($tt:tt)* }) => { match_ast!(match (&$node) { $($tt)* }) };
(match $node:ident { $($tt:tt)* }) => { match_ast!(match ($node) { $($tt)* }) };
(match ($node:expr) {
$( $( $path:ident )::+ ($it:pat) => $res:expr, )*
_ => $catch_all:expr $(,)?
}) => {{
$( if let Some($it) = $($path::)+cast_ref($node) { $res } else )*
{ $catch_all }
}};
}
/// Declares a custom union AstNode type with an ungram-like syntax
///
/// # Example
///
/// ```ignore
/// declare_node_union! {
/// /// Matches an if statement or a conditional expression
/// pub(crate) JsAnyConditional = JsIfStatement | JsConditionalExpression
/// }
/// ```
#[macro_export]
macro_rules! declare_node_union {
(@merge_kind $head:ident ) => {
$head::KIND_SET
};
(@merge_kind $head:ident $( $rest:ident )* ) => {
$head::KIND_SET.union($crate::declare_node_union!( @merge_kind $( $rest )* ))
};
( $( #[$attr:meta] )* $vis:vis $name:ident = $( $variant:ident )|* ) => {
$( #[$attr] )*
#[allow(clippy::enum_variant_names)]
#[derive(Clone, PartialEq, Eq, Hash)]
$vis enum $name {
$( $variant($variant), )*
}
impl $crate::AstNode for $name {
type Language = <( $( $variant, )* ) as $crate::macros::UnionLanguage>::Language;
const KIND_SET: $crate::SyntaxKindSet<Self::Language> = $crate::declare_node_union!( @merge_kind $( $variant )* );
fn can_cast(kind: <Self::Language as $crate::Language>::Kind) -> bool {
$( $variant::can_cast(kind) )||*
}
fn cast(syntax: $crate::SyntaxNode<Self::Language>) -> Option<Self>
where
Self: Sized,
{
$( if $variant::can_cast(syntax.kind()) {
return Some(Self::$variant($variant::unwrap_cast(syntax)));
} )*
None
}
fn syntax(&self) -> &$crate::SyntaxNode<Self::Language> {
match self {
$( Self::$variant(node) => node.syntax() ),*
}
}
fn into_syntax(self) -> $crate::SyntaxNode<Self::Language> {
match self {
$( Self::$variant(node) => node.into_syntax() ),*
}
}
}
impl std::fmt::Debug for $name {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
$( Self::$variant(it) => std::fmt::Debug::fmt(it, f), )*
}
}
}
impl std::fmt::Display for $name {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
std::fmt::Display::fmt($crate::AstNode::syntax(self), f)
}
}
$( impl From<$variant> for $name {
fn from(node: $variant) -> Self {
Self::$variant(node)
}
} )*
impl From<$name> for $crate::SyntaxNode<<$name as $crate::AstNode>::Language> {
fn from(n: $name) -> $crate::SyntaxNode<<$name as $crate::AstNode>::Language> {
match n {
$( $name::$variant(it) => it.into(), )*
}
}
}
impl From<$name> for $crate::SyntaxElement<<$name as $crate::AstNode>::Language> {
fn from(n: $name) -> $crate::SyntaxElement<<$name as $crate::AstNode>::Language> {
$crate::SyntaxNode::<<$name as $crate::AstNode>::Language>::from(n).into()
}
}
};
}
/// This trait is implemented for tuples of AstNode types of size 1 to 12 if
/// all node types share the same associated language (which is then aliased as
/// the `Language` associated type on [UnionLanguage] itself)
pub trait UnionLanguage {
type Language: Language;
}
macro_rules! impl_union_language {
( $head:ident $( , $rest:ident )* ) => {
impl<$head $( , $rest )*> UnionLanguage for ($head, $( $rest ),*)
where
$head: AstNode $( , $rest: AstNode<Language = <$head as AstNode>::Language> )*
{
type Language = <$head as AstNode>::Language;
}
impl_union_language!( $( $rest ),* );
};
() => {};
}
impl_union_language!(
T00, T01, T02, T03, T04, T05, T06, T07, T08, T09, T10, T11, T12, T13, T14, T15, T16, T17, T18,
T19, T20, T21, T22, T23, T24, T25, T26, T27, T28, T29
);

284
crates/ruff_rowan/src/raw_language.rs Normal file
View File

@ -0,0 +1,284 @@
use crate::raw_language::RawLanguageKind::{COMMA_TOKEN, LITERAL_EXPRESSION, ROOT};
///! Provides a sample language implementation that is useful in API explanation or tests
use crate::{
AstNode, AstSeparatedList, Language, ParsedChildren, RawNodeSlots, RawSyntaxKind,
RawSyntaxNode, SyntaxFactory, SyntaxKind, SyntaxKindSet, SyntaxList, SyntaxNode, TreeBuilder,
};
#[doc(hidden)]
#[derive(Debug, Default, Hash, Copy, Eq, Ord, PartialEq, PartialOrd, Clone)]
pub struct RawLanguage;
impl Language for RawLanguage {
type Kind = RawLanguageKind;
type Root = RawLanguageRoot;
}
#[doc(hidden)]
#[derive(Debug, PartialEq, Eq, Clone, Copy)]
#[repr(u16)]
#[allow(bad_style)]
pub enum RawLanguageKind {
ROOT = 0,
EXPRESSION_LIST = 1,
SEPARATED_EXPRESSION_LIST = 2,
COMMA_TOKEN = 3,
STRING_TOKEN = 4,
NUMBER_TOKEN = 5,
LITERAL_EXPRESSION = 6,
BOGUS = 7,
FOR_KW = 8,
L_PAREN_TOKEN = 9,
SEMICOLON_TOKEN = 10,
R_PAREN_TOKEN = 11,
EQUAL_TOKEN = 12,
LET_TOKEN = 13,
CONDITION = 14,
PLUS_TOKEN = 15,
WHITESPACE = 16,
TOMBSTONE = 17,
EOF = 18,
__LAST,
}
impl SyntaxKind for RawLanguageKind {
const TOMBSTONE: Self = RawLanguageKind::TOMBSTONE;
const EOF: Self = RawLanguageKind::EOF;
fn is_bogus(&self) -> bool {
self == &RawLanguageKind::BOGUS
}
fn to_bogus(&self) -> Self {
RawLanguageKind::BOGUS
}
fn to_raw(&self) -> RawSyntaxKind {
RawSyntaxKind(*self as u16)
}
#[allow(unsafe_code)]
fn from_raw(raw: RawSyntaxKind) -> Self {
assert!(raw.0 < RawLanguageKind::__LAST as u16);
unsafe { std::mem::transmute::<u16, RawLanguageKind>(raw.0) }
}
fn is_root(&self) -> bool {
self == &RawLanguageKind::ROOT
}
fn is_list(&self) -> bool {
matches!(
self,
RawLanguageKind::EXPRESSION_LIST | RawLanguageKind::SEPARATED_EXPRESSION_LIST
)
}
fn to_string(&self) -> Option<&'static str> {
let str = match self {
COMMA_TOKEN => ",",
RawLanguageKind::FOR_KW => "for",
RawLanguageKind::L_PAREN_TOKEN => "(",
RawLanguageKind::SEMICOLON_TOKEN => ";",
RawLanguageKind::R_PAREN_TOKEN => ")",
RawLanguageKind::EQUAL_TOKEN => "=",
RawLanguageKind::LET_TOKEN => "let",
RawLanguageKind::PLUS_TOKEN => "+",
_ => return None,
};
Some(str)
}
}
#[doc(hidden)]
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct RawLanguageRoot {
node: SyntaxNode<RawLanguage>,
}
impl AstNode for RawLanguageRoot {
type Language = RawLanguage;
const KIND_SET: SyntaxKindSet<RawLanguage> =
SyntaxKindSet::from_raw(RawSyntaxKind(ROOT as u16));
fn can_cast(kind: RawLanguageKind) -> bool {
kind == ROOT
}
fn cast(syntax: SyntaxNode<RawLanguage>) -> Option<Self>
where
Self: Sized,
{
if syntax.kind() == ROOT {
Some(RawLanguageRoot { node: syntax })
} else {
None
}
}
fn syntax(&self) -> &SyntaxNode<RawLanguage> {
&self.node
}
fn into_syntax(self) -> SyntaxNode<RawLanguage> {
self.node
}
}
#[doc(hidden)]
#[derive(Clone, Eq, PartialEq, Debug)]
pub struct LiteralExpression {
node: SyntaxNode<RawLanguage>,
}
impl AstNode for LiteralExpression {
type Language = RawLanguage;
const KIND_SET: SyntaxKindSet<RawLanguage> =
SyntaxKindSet::from_raw(RawSyntaxKind(LITERAL_EXPRESSION as u16));
fn can_cast(kind: RawLanguageKind) -> bool {
kind == LITERAL_EXPRESSION
}
fn cast(syntax: SyntaxNode<RawLanguage>) -> Option<Self>
where
Self: Sized,
{
if syntax.kind() == LITERAL_EXPRESSION {
Some(LiteralExpression { node: syntax })
} else {
None
}
}
fn syntax(&self) -> &SyntaxNode<RawLanguage> {
&self.node
}
fn into_syntax(self) -> SyntaxNode<RawLanguage> {
self.node
}
}
#[doc(hidden)]
pub struct SeparatedExpressionList {
syntax_list: SyntaxList<RawLanguage>,
}
impl SeparatedExpressionList {
pub fn new(list: SyntaxList<RawLanguage>) -> Self {
Self { syntax_list: list }
}
}
impl AstSeparatedList for SeparatedExpressionList {
type Language = RawLanguage;
type Node = LiteralExpression;
fn syntax_list(&self) -> &SyntaxList<RawLanguage> {
&self.syntax_list
}
fn into_syntax_list(self) -> SyntaxList<RawLanguage> {
self.syntax_list
}
}
#[doc(hidden)]
#[derive(Debug)]
pub struct RawLanguageSyntaxFactory;
impl SyntaxFactory for RawLanguageSyntaxFactory {
type Kind = RawLanguageKind;
fn make_syntax(
kind: Self::Kind,
children: ParsedChildren<Self::Kind>,
) -> RawSyntaxNode<Self::Kind> {
match kind {
RawLanguageKind::BOGUS | RawLanguageKind::ROOT => {
RawSyntaxNode::new(kind, children.into_iter().map(Some))
}
RawLanguageKind::EXPRESSION_LIST => {
Self::make_node_list_syntax(kind, children, |kind| kind == LITERAL_EXPRESSION)
}
RawLanguageKind::SEPARATED_EXPRESSION_LIST => Self::make_separated_list_syntax(
kind,
children,
|kind| kind == LITERAL_EXPRESSION,
COMMA_TOKEN,
true,
),
RawLanguageKind::LITERAL_EXPRESSION => {
let actual_len = children.len();
if actual_len > 1 {
return RawSyntaxNode::new(kind.to_bogus(), children.into_iter().map(Some));
}
let mut elements = children.into_iter();
let current_element = elements.next();
if let Some(element) = &current_element {
if !matches!(
element.kind(),
RawLanguageKind::STRING_TOKEN | RawLanguageKind::NUMBER_TOKEN
) {
return RawSyntaxNode::new(
kind.to_bogus(),
std::iter::once(current_element),
);
}
} else {
return RawSyntaxNode::new(kind, std::iter::once(None));
}
RawSyntaxNode::new(kind, std::iter::once(current_element))
}
RawLanguageKind::CONDITION => {
let mut elements = (&children).into_iter();
let mut current_element = elements.next();
let mut slots: RawNodeSlots<3> = Default::default();
if let Some(element) = &current_element {
if element.kind() == RawLanguageKind::L_PAREN_TOKEN {
slots.mark_present();
current_element = elements.next();
}
}
slots.next_slot();
if let Some(element) = &current_element {
if element.kind() == RawLanguageKind::LITERAL_EXPRESSION {
slots.mark_present();
current_element = elements.next();
}
}
slots.next_slot();
if let Some(element) = &current_element {
if element.kind() == RawLanguageKind::R_PAREN_TOKEN {
slots.mark_present();
current_element = elements.next();
}
}
slots.next_slot();
if current_element.is_some() {
return RawSyntaxNode::new(kind.to_bogus(), children.into_iter().map(Some));
}
slots.into_node(kind, children)
}
_ => unreachable!("{:?} is not a node kind", kind),
}
}
}
#[doc(hidden)]
pub type RawSyntaxTreeBuilder<'a> = TreeBuilder<'a, RawLanguage, RawLanguageSyntaxFactory>;

89
crates/ruff_rowan/src/serde_impls.rs Normal file
View File

@ -0,0 +1,89 @@
use serde::ser::{Serialize, SerializeMap, SerializeSeq, Serializer};
use std::fmt;
use crate::{
syntax::{Language, SyntaxNode, SyntaxToken},
NodeOrToken,
};
struct SerDisplay<T>(T);
impl<T: fmt::Display> Serialize for SerDisplay<T> {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
serializer.collect_str(&self.0)
}
}
struct DisplayDebug<T>(T);
impl<T: fmt::Debug> fmt::Display for DisplayDebug<T> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
fmt::Debug::fmt(&self.0, f)
}
}
impl<L: Language> Serialize for SyntaxNode<L> {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
let mut state = serializer.serialize_map(Some(3))?;
state.serialize_entry("kind", &SerDisplay(DisplayDebug(self.kind())))?;
state.serialize_entry("text_range", &self.text_range())?;
state.serialize_entry("children", &Children(self))?;
state.end()
}
}
impl<L: Language> Serialize for SyntaxToken<L> {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
let mut state = serializer.serialize_map(Some(3))?;
state.serialize_entry("kind", &SerDisplay(DisplayDebug(self.kind())))?;
state.serialize_entry("text_range", &self.text_range())?;
state.serialize_entry("text", &self.text())?;
// To implement this, SyntaxTrivia will need to expose the kind and the length of each trivia
// state.serialize_entry("leading", &self.leading())?;
// state.serialize_entry("trailing", &self.trailing())?;
state.end()
}
}
struct Children<T>(T);
impl<L: Language> Serialize for Children<&'_ SyntaxNode<L>> {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
let mut state = serializer.serialize_seq(None)?;
self.0
.children_with_tokens()
.try_for_each(|element| match element {
NodeOrToken::Node(it) => state.serialize_element(&it),
NodeOrToken::Token(it) => state.serialize_element(&it),
})?;
state.end()
}
}
#[cfg(test)]
mod test {
use crate::raw_language::{RawLanguage, RawLanguageKind, RawLanguageSyntaxFactory};
#[test]
pub fn serialization() {
let mut builder: crate::TreeBuilder<RawLanguage, RawLanguageSyntaxFactory> =
crate::TreeBuilder::new();
builder.start_node(RawLanguageKind::ROOT);
builder.token(RawLanguageKind::LET_TOKEN, "\n\tlet ");
builder.finish_node();
let root = builder.finish();
assert!(serde_json::to_string(&root).is_ok());
}
}

502
crates/ruff_rowan/src/syntax.rs Normal file
View File

@ -0,0 +1,502 @@
mod element;
mod node;
mod rewriter;
mod token;
mod trivia;
use crate::{AstNode, RawSyntaxKind};
pub use element::{SyntaxElement, SyntaxElementKey};
pub(crate) use node::SyntaxSlots;
pub use node::{
Preorder, PreorderWithTokens, SendNode, SyntaxElementChildren, SyntaxNode, SyntaxNodeChildren,
SyntaxNodeOptionExt, SyntaxSlot,
};
pub use rewriter::{SyntaxRewriter, VisitNodeSignal};
use std::fmt;
use std::fmt::Debug;
pub use token::SyntaxToken;
pub use trivia::{
chain_trivia_pieces, ChainTriviaPiecesIterator, SyntaxTrivia, SyntaxTriviaPiece,
SyntaxTriviaPieceComments, SyntaxTriviaPieceNewline, SyntaxTriviaPieceSkipped,
SyntaxTriviaPieceWhitespace, SyntaxTriviaPiecesIterator, TriviaPiece, TriviaPieceKind,
};
/// Type tag for each node or token of a language
pub trait SyntaxKind: fmt::Debug + PartialEq + Copy {
const TOMBSTONE: Self;
const EOF: Self;
/// Returns `true` if this is a kind of a bogus node.
fn is_bogus(&self) -> bool;
/// Converts this into to the best matching bogus node kind.
fn to_bogus(&self) -> Self;
/// Converts this kind to a raw syntax kind.
fn to_raw(&self) -> RawSyntaxKind;
/// Creates a syntax kind from a raw kind.
fn from_raw(raw: RawSyntaxKind) -> Self;
/// Returns `true` if this kind is for a root node.
fn is_root(&self) -> bool;
/// Returns `true` if this kind is a list node.
fn is_list(&self) -> bool;
/// Returns a string for keywords and punctuation tokens or `None` otherwise.
fn to_string(&self) -> Option<&'static str>;
}
pub trait Language: Sized + Clone + Copy + fmt::Debug + Eq + Ord + std::hash::Hash {
type Kind: SyntaxKind;
type Root: AstNode<Language = Self> + Clone + Eq + fmt::Debug;
}
/// A list of `SyntaxNode`s and/or `SyntaxToken`s
#[derive(Debug, Clone, Eq, PartialEq, Hash)]
pub struct SyntaxList<L: Language> {
list: SyntaxNode<L>,
}
impl<L: Language> SyntaxList<L> {
/// Creates a new list wrapping a List `SyntaxNode`
fn new(node: SyntaxNode<L>) -> Self {
Self { list: node }
}
/// Iterates over the elements in the list.
pub fn iter(&self) -> SyntaxSlots<L> {
self.list.slots()
}
/// Returns the number of items in this list
pub fn len(&self) -> usize {
self.list.slots().len()
}
pub fn is_empty(&self) -> bool {
self.len() == 0
}
pub fn first(&self) -> Option<SyntaxSlot<L>> {
self.list.slots().next()
}
pub fn last(&self) -> Option<SyntaxSlot<L>> {
self.list.slots().last()
}
pub fn node(&self) -> &SyntaxNode<L> {
&self.list
}
pub fn into_node(self) -> SyntaxNode<L> {
self.list
}
}
impl<L: Language> IntoIterator for &SyntaxList<L> {
type Item = SyntaxSlot<L>;
type IntoIter = SyntaxSlots<L>;
fn into_iter(self) -> Self::IntoIter {
self.iter()
}
}
impl<L: Language> IntoIterator for SyntaxList<L> {
type Item = SyntaxSlot<L>;
type IntoIter = SyntaxSlots<L>;
fn into_iter(self) -> Self::IntoIter {
self.iter()
}
}
#[cfg(test)]
mod tests {
use ruff_text_size::TextRange;
use crate::raw_language::{RawLanguageKind, RawSyntaxTreeBuilder};
use crate::syntax::TriviaPiece;
use crate::Direction;
#[test]
fn empty_list() {
let mut builder: RawSyntaxTreeBuilder = RawSyntaxTreeBuilder::new();
builder.start_node(RawLanguageKind::EXPRESSION_LIST);
builder.finish_node();
let list = builder.finish().into_list();
assert!(list.is_empty());
assert_eq!(list.len(), 0);
assert_eq!(list.first(), None);
assert_eq!(list.last(), None);
assert_eq!(list.iter().collect::<Vec<_>>(), Vec::default());
}
#[test]
fn node_list() {
let mut builder = RawSyntaxTreeBuilder::new();
builder.start_node(RawLanguageKind::EXPRESSION_LIST);
builder.start_node(RawLanguageKind::LITERAL_EXPRESSION);
builder.token(RawLanguageKind::NUMBER_TOKEN, "1");
builder.finish_node();
builder.start_node(RawLanguageKind::LITERAL_EXPRESSION);
builder.token(RawLanguageKind::NUMBER_TOKEN, "2");
builder.finish_node();
builder.finish_node();
let node = builder.finish();
let list = node.into_list();
assert!(!list.is_empty());
assert_eq!(list.len(), 2);
let first = list.first().and_then(|e| e.into_node()).unwrap();
assert_eq!(first.kind(), RawLanguageKind::LITERAL_EXPRESSION);
assert_eq!(first.text(), "1");
let last = list.last().and_then(|e| e.into_node()).unwrap();
assert_eq!(last.kind(), RawLanguageKind::LITERAL_EXPRESSION);
assert_eq!(last.text(), "2");
let node_texts: Vec<_> = list
.iter()
.map(|e| e.into_node().map(|n| n.text().to_string()))
.collect();
assert_eq!(
node_texts,
vec![Some(String::from("1")), Some(String::from("2"))]
)
}
#[test]
fn node_or_token_list() {
let mut builder = RawSyntaxTreeBuilder::new();
builder.start_node(RawLanguageKind::SEPARATED_EXPRESSION_LIST);
builder.start_node(RawLanguageKind::LITERAL_EXPRESSION);
builder.token(RawLanguageKind::NUMBER_TOKEN, "1");
builder.finish_node();
builder.token(RawLanguageKind::NUMBER_TOKEN, ",");
builder.start_node(RawLanguageKind::LITERAL_EXPRESSION);
builder.token(RawLanguageKind::NUMBER_TOKEN, "2");
builder.finish_node();
builder.finish_node();
let node = builder.finish();
let list = node.into_list();
assert!(!list.is_empty());
assert_eq!(list.len(), 3);
let first = list.first().and_then(|e| e.into_node()).unwrap();
assert_eq!(first.kind(), RawLanguageKind::LITERAL_EXPRESSION);
assert_eq!(first.text(), "1");
let last = list.last().and_then(|e| e.into_node()).unwrap();
assert_eq!(last.kind(), RawLanguageKind::LITERAL_EXPRESSION);
assert_eq!(last.text(), "2");
let kinds: Vec<_> = list.iter().map(|e| e.kind()).collect();
assert_eq!(
kinds,
vec![
Some(RawLanguageKind::LITERAL_EXPRESSION),
Some(RawLanguageKind::NUMBER_TOKEN),
Some(RawLanguageKind::LITERAL_EXPRESSION)
]
)
}
#[test]
fn siblings() {
let mut builder = RawSyntaxTreeBuilder::new();
// list
builder.start_node(RawLanguageKind::SEPARATED_EXPRESSION_LIST);
// element 1
builder.start_node(RawLanguageKind::LITERAL_EXPRESSION);
builder.token(RawLanguageKind::NUMBER_TOKEN, "a");
builder.finish_node();
// element 2
builder.start_node(RawLanguageKind::LITERAL_EXPRESSION);
builder.token(RawLanguageKind::NUMBER_TOKEN, "b");
builder.finish_node();
// Missing ,
// element 3
builder.start_node(RawLanguageKind::LITERAL_EXPRESSION);
builder.token(RawLanguageKind::NUMBER_TOKEN, "c");
builder.finish_node();
builder.finish_node();
let root = builder.finish();
let first = root.children().next().unwrap();
assert_eq!(first.text().to_string(), "a");
assert_eq!(
first.next_sibling().map(|e| e.text().to_string()),
Some(String::from("b"))
);
let second = root.children().nth(1).unwrap();
assert_eq!(second.text().to_string(), "b");
// Skips the missing element
assert_eq!(
second.next_sibling().map(|e| e.text().to_string()),
Some(String::from("c"))
);
assert_eq!(
second.prev_sibling().map(|e| e.text().to_string()),
Some(String::from("a"))
);
let last = root.children().last().unwrap();
assert_eq!(last.text(), "c");
assert_eq!(last.next_sibling(), None);
assert_eq!(
last.prev_sibling().map(|e| e.text().to_string()),
Some(String::from("b"))
);
assert_eq!(
first
.siblings(Direction::Next)
.map(|s| s.text().to_string())
.collect::<Vec<_>>(),
vec!["a", "b", "c"]
);
assert_eq!(
last.siblings(Direction::Prev)
.map(|s| s.text().to_string())
.collect::<Vec<_>>(),
vec!["c", "b", "a"]
);
}
#[test]
fn siblings_with_tokens() {
let mut builder = RawSyntaxTreeBuilder::new();
builder.start_node(RawLanguageKind::ROOT);
builder.token(RawLanguageKind::FOR_KW, "for");
builder.token(RawLanguageKind::L_PAREN_TOKEN, "(");
builder.token(RawLanguageKind::SEMICOLON_TOKEN, ";");
builder.start_node(RawLanguageKind::LITERAL_EXPRESSION);
builder.token(RawLanguageKind::STRING_TOKEN, "x");
builder.finish_node();
builder.token(RawLanguageKind::SEMICOLON_TOKEN, ";");
builder.token(RawLanguageKind::R_PAREN_TOKEN, ")");
builder.finish_node();
let root = builder.finish();
let first_semicolon = root
.children_with_tokens()
.nth(2)
.and_then(|e| e.into_token())
.unwrap();
assert_eq!(first_semicolon.text(), ";");
assert_eq!(
first_semicolon
.siblings_with_tokens(Direction::Next)
.map(|e| e.to_string())
.collect::<Vec<_>>(),
vec!["x", ";", ")"]
);
assert_eq!(
first_semicolon.next_sibling_or_token(),
first_semicolon.siblings_with_tokens(Direction::Next).next()
);
assert_eq!(
first_semicolon.prev_sibling_or_token(),
first_semicolon.siblings_with_tokens(Direction::Prev).next()
);
}
#[test]
pub fn syntax_text_and_len() {
let mut builder = RawSyntaxTreeBuilder::new();
builder.start_node(RawLanguageKind::ROOT);
builder.token_with_trivia(
RawLanguageKind::LET_TOKEN,
"\n\t let \t\t",
&[TriviaPiece::whitespace(3)],
&[TriviaPiece::whitespace(3)],
);
builder.finish_node();
// // Node texts
let node = builder.finish();
assert_eq!("\n\t let \t\t", node.text());
assert_eq!("let", node.text_trimmed());
assert_eq!("\n\t ", node.first_leading_trivia().unwrap().text());
assert_eq!(" \t\t", node.last_trailing_trivia().unwrap().text());
// Token texts
let token = node.first_token().unwrap();
assert_eq!("\n\t let \t\t", token.text());
assert_eq!("let", token.text_trimmed());
assert_eq!("\n\t ", token.leading_trivia().text());
assert_eq!(" \t\t", token.trailing_trivia().text());
}
#[test]
pub fn syntax_range() {
let mut builder = RawSyntaxTreeBuilder::new();
builder.start_node(RawLanguageKind::ROOT);
builder.token_with_trivia(
RawLanguageKind::LET_TOKEN,
"\n\t let \t\t",
&[TriviaPiece::whitespace(3)],
&[TriviaPiece::whitespace(3)],
);
builder.token_with_trivia(
RawLanguageKind::LET_TOKEN,
"a ",
&[TriviaPiece::whitespace(0)],
&[TriviaPiece::whitespace(1)],
);
builder.token_with_trivia(
RawLanguageKind::EQUAL_TOKEN,
"\n=\n",
&[TriviaPiece::whitespace(1)],
&[TriviaPiece::whitespace(1)],
);
builder.token(RawLanguageKind::NUMBER_TOKEN, "1");
builder.token_with_trivia(
RawLanguageKind::SEMICOLON_TOKEN,
";\t\t",
&[],
&[TriviaPiece::whitespace(2)],
);
builder.finish_node();
let node = builder.finish();
// Node Ranges
assert_eq!(TextRange::new(0.into(), 18.into()), node.text_range());
assert_eq!(
TextRange::new(3.into(), 16.into()),
node.text_trimmed_range()
);
assert_eq!(
TextRange::new(0.into(), 3.into()),
node.first_leading_trivia().unwrap().text_range()
);
assert_eq!(
TextRange::new(16.into(), 18.into()),
node.last_trailing_trivia().unwrap().text_range()
);
// as NodeOrToken
let eq_token = node
.descendants_with_tokens(Direction::Next)
.find(|x| x.kind() == RawLanguageKind::EQUAL_TOKEN)
.unwrap();
assert_eq!(TextRange::new(11.into(), 14.into()), eq_token.text_range());
assert_eq!(
TextRange::new(12.into(), 13.into()),
eq_token.text_trimmed_range()
);
assert_eq!(
TextRange::new(11.into(), 12.into()),
eq_token.leading_trivia().unwrap().text_range()
);
assert_eq!(
TextRange::new(13.into(), 14.into()),
eq_token.trailing_trivia().unwrap().text_range()
);
// as Token
let eq_token = eq_token.as_token().unwrap();
assert_eq!(TextRange::new(11.into(), 14.into()), eq_token.text_range());
assert_eq!(
TextRange::new(12.into(), 13.into()),
eq_token.text_trimmed_range()
);
assert_eq!(
TextRange::new(11.into(), 12.into()),
eq_token.leading_trivia().text_range()
);
assert_eq!(
TextRange::new(13.into(), 14.into()),
eq_token.trailing_trivia().text_range()
);
}
#[test]
pub fn syntax_trivia_pieces() {
use crate::*;
let node = RawSyntaxTreeBuilder::wrap_with_node(RawLanguageKind::ROOT, |builder| {
builder.token_with_trivia(
RawLanguageKind::LET_TOKEN,
"\n\t /**/let \t\t",
&[
TriviaPiece::whitespace(3),
TriviaPiece::single_line_comment(4),
],
&[TriviaPiece::whitespace(3)],
);
});
let pieces: Vec<_> = node.first_leading_trivia().unwrap().pieces().collect();
assert_eq!(2, pieces.len());
assert_eq!("\n\t ", pieces[0].text());
assert_eq!(TextSize::from(3), pieces[0].text_len());
assert_eq!(TextRange::new(0.into(), 3.into()), pieces[0].text_range());
assert!(pieces[0].is_whitespace());
assert_eq!("/**/", pieces[1].text());
assert_eq!(TextSize::from(4), pieces[1].text_len());
assert_eq!(TextRange::new(3.into(), 7.into()), pieces[1].text_range());
assert!(pieces[1].is_comments());
let pieces_rev: Vec<_> = node
.first_leading_trivia()
.unwrap()
.pieces()
.rev()
.collect();
assert_eq!(2, pieces_rev.len());
assert_eq!("/**/", pieces_rev[0].text());
assert_eq!("\n\t ", pieces_rev[1].text());
}
}

137
crates/ruff_rowan/src/syntax/element.rs Normal file
View File

@ -0,0 +1,137 @@
use crate::syntax::SyntaxTrivia;
use crate::{cursor, Language, NodeOrToken, SyntaxNode, SyntaxToken};
use ruff_text_size::{TextRange, TextSize};
use std::iter;
use std::ptr::NonNull;
pub type SyntaxElement<L> = NodeOrToken<SyntaxNode<L>, SyntaxToken<L>>;
impl<L: Language> SyntaxElement<L> {
pub fn key(&self) -> SyntaxElementKey {
match self {
NodeOrToken::Node(it) => it.key(),
NodeOrToken::Token(it) => it.key(),
}
}
pub fn text_range(&self) -> TextRange {
match self {
NodeOrToken::Node(it) => it.text_range(),
NodeOrToken::Token(it) => it.text_range(),
}
}
pub fn text_trimmed_range(&self) -> TextRange {
match self {
NodeOrToken::Node(it) => it.text_trimmed_range(),
NodeOrToken::Token(it) => it.text_trimmed_range(),
}
}
pub fn leading_trivia(&self) -> Option<SyntaxTrivia<L>> {
match self {
NodeOrToken::Node(it) => it.first_leading_trivia(),
NodeOrToken::Token(it) => Some(it.leading_trivia()),
}
}
pub fn trailing_trivia(&self) -> Option<SyntaxTrivia<L>> {
match self {
NodeOrToken::Node(it) => it.last_trailing_trivia(),
NodeOrToken::Token(it) => Some(it.trailing_trivia()),
}
}
pub fn kind(&self) -> L::Kind {
match self {
NodeOrToken::Node(it) => it.kind(),
NodeOrToken::Token(it) => it.kind(),
}
}
pub fn parent(&self) -> Option<SyntaxNode<L>> {
match self {
NodeOrToken::Node(it) => it.parent(),
NodeOrToken::Token(it) => it.parent(),
}
}
pub(crate) fn index(&self) -> usize {
match self {
NodeOrToken::Node(it) => it.index(),
NodeOrToken::Token(it) => it.index(),
}
}
pub fn ancestors(&self) -> impl Iterator<Item = SyntaxNode<L>> {
let first = match self {
NodeOrToken::Node(it) => Some(it.clone()),
NodeOrToken::Token(it) => it.parent(),
};
iter::successors(first, SyntaxNode::parent)
}
pub fn next_sibling_or_token(&self) -> Option<SyntaxElement<L>> {
match self {
NodeOrToken::Node(it) => it.next_sibling_or_token(),
NodeOrToken::Token(it) => it.next_sibling_or_token(),
}
}
pub fn prev_sibling_or_token(&self) -> Option<SyntaxElement<L>> {
match self {
NodeOrToken::Node(it) => it.prev_sibling_or_token(),
NodeOrToken::Token(it) => it.prev_sibling_or_token(),
}
}
#[must_use = "syntax elements are immutable, the result of update methods must be propagated to have any effect"]
pub fn detach(self) -> Self {
match self {
NodeOrToken::Node(it) => Self::Node(it.detach()),
NodeOrToken::Token(it) => Self::Token(it.detach()),
}
}
}
impl<L: Language> From<cursor::SyntaxElement> for SyntaxElement<L> {
fn from(raw: cursor::SyntaxElement) -> SyntaxElement<L> {
match raw {
NodeOrToken::Node(it) => NodeOrToken::Node(it.into()),
NodeOrToken::Token(it) => NodeOrToken::Token(it.into()),
}
}
}
impl<L: Language> From<SyntaxElement<L>> for cursor::SyntaxElement {
fn from(element: SyntaxElement<L>) -> cursor::SyntaxElement {
match element {
NodeOrToken::Node(it) => NodeOrToken::Node(it.into()),
NodeOrToken::Token(it) => NodeOrToken::Token(it.into()),
}
}
}
impl<L: Language> From<SyntaxToken<L>> for SyntaxElement<L> {
fn from(token: SyntaxToken<L>) -> SyntaxElement<L> {
NodeOrToken::Token(token)
}
}
impl<L: Language> From<SyntaxNode<L>> for SyntaxElement<L> {
fn from(node: SyntaxNode<L>) -> SyntaxElement<L> {
NodeOrToken::Node(node)
}
}
#[derive(Copy, Clone, Eq, PartialEq, Hash)]
pub struct SyntaxElementKey {
node_data: NonNull<()>,
offset: TextSize,
}
impl SyntaxElementKey {
pub(crate) fn new(node_data: NonNull<()>, offset: TextSize) -> Self {
Self { node_data, offset }
}
}

817
crates/ruff_rowan/src/syntax/node.rs Normal file
View File

@ -0,0 +1,817 @@
use crate::green::GreenElement;
use crate::syntax::element::{SyntaxElement, SyntaxElementKey};
use crate::syntax::SyntaxTrivia;
use crate::{
cursor, Direction, GreenNode, Language, NodeOrToken, SyntaxKind, SyntaxList, SyntaxNodeText,
SyntaxToken, TokenAtOffset, WalkEvent,
};
use ruff_text_size::{TextRange, TextSize};
#[cfg(feature = "serde")]
use serde::Serialize;
use std::any::TypeId;
use std::fmt::{Debug, Formatter};
use std::iter::FusedIterator;
use std::marker::PhantomData;
use std::{fmt, ops};
#[derive(Clone, PartialEq, Eq, Hash)]
pub struct SyntaxNode<L: Language> {
raw: cursor::SyntaxNode,
_p: PhantomData<L>,
}
impl<L: Language> SyntaxNode<L> {
pub(crate) fn new_root(green: GreenNode) -> SyntaxNode<L> {
SyntaxNode::from(cursor::SyntaxNode::new_root(green))
}
/// Create a new detached (root) node from a syntax kind and an iterator of slots
///
/// In general this function should not be used directly but through the
/// type-checked factory function / builders generated from the grammar of
/// the corresponding language (eg. `ruff_js_factory::make`)
pub fn new_detached<I>(kind: L::Kind, slots: I) -> SyntaxNode<L>
where
I: IntoIterator<Item = Option<SyntaxElement<L>>>,
I::IntoIter: ExactSizeIterator,
{
SyntaxNode::from(cursor::SyntaxNode::new_root(GreenNode::new(
kind.to_raw(),
slots.into_iter().map(|slot| {
slot.map(|element| match element {
NodeOrToken::Node(node) => GreenElement::Node(node.green_node()),
NodeOrToken::Token(token) => GreenElement::Token(token.green_token()),
})
}),
)))
}
fn green_node(&self) -> GreenNode {
self.raw.green().to_owned()
}
pub fn key(&self) -> SyntaxElementKey {
let (node_data, offset) = self.raw.key();
SyntaxElementKey::new(node_data, offset)
}
/// Returns the element stored in the slot with the given index. Returns [None] if the slot is empty.
///
/// ## Panics
/// If the slot index is out of bounds
#[inline]
pub fn element_in_slot(&self, slot: u32) -> Option<SyntaxElement<L>> {
self.raw.element_in_slot(slot).map(SyntaxElement::from)
}
pub fn kind(&self) -> L::Kind {
L::Kind::from_raw(self.raw.kind())
}
/// Returns the text of all descendants tokens combined, including all trivia.
///
/// ```
/// use ruff_rowan::raw_language::{RawLanguage, RawLanguageKind, RawSyntaxTreeBuilder};
/// use ruff_rowan::*;
/// let mut node = RawSyntaxTreeBuilder::wrap_with_node(RawLanguageKind::ROOT, |builder| {
/// builder.token_with_trivia(
/// RawLanguageKind::LET_TOKEN,
/// "\n\t let \t\t",
/// &[TriviaPiece::whitespace(3)],
/// &[TriviaPiece::whitespace(3)],
/// );
/// builder.token(RawLanguageKind::STRING_TOKEN, "a");
/// builder.token_with_trivia(
/// RawLanguageKind::SEMICOLON_TOKEN,
/// "; \t\t",
/// &[TriviaPiece::whitespace(3)],
/// &[TriviaPiece::whitespace(3)],
/// );
/// });
/// assert_eq!("\n\t let \t\ta; \t\t", node.text());
/// ```
pub fn text(&self) -> SyntaxNodeText {
self.raw.text()
}
/// Returns the text of all descendants tokens combined,
/// excluding the first token leading trivia, and the last token trailing trivia.
/// All other trivia is included.
///
/// ```
/// use ruff_rowan::raw_language::{RawLanguage, RawLanguageKind, RawSyntaxTreeBuilder};
/// use ruff_rowan::*;
/// let mut node = RawSyntaxTreeBuilder::wrap_with_node(RawLanguageKind::ROOT, |builder| {
/// builder.token_with_trivia(
/// RawLanguageKind::LET_TOKEN,
/// "\n\t let \t\t",
/// &[TriviaPiece::whitespace(3)],
/// &[TriviaPiece::whitespace(3)],
/// );
/// builder.token(RawLanguageKind::STRING_TOKEN, "a");
/// builder.token_with_trivia(
/// RawLanguageKind::SEMICOLON_TOKEN,
/// "; \t\t",
/// &[],
/// &[TriviaPiece::whitespace(3)],
/// );
/// });
/// assert_eq!("let \t\ta;", node.text_trimmed());
/// ```
pub fn text_trimmed(&self) -> SyntaxNodeText {
self.raw.text_trimmed()
}
/// Returns the range corresponding for the text of all descendants tokens combined, including all trivia.
///
/// ```
/// use ruff_rowan::raw_language::{RawLanguage, RawLanguageKind, RawSyntaxTreeBuilder};
/// use ruff_rowan::*;
/// let mut node = RawSyntaxTreeBuilder::wrap_with_node(RawLanguageKind::ROOT, |builder| {
/// builder.token_with_trivia(
/// RawLanguageKind::LET_TOKEN,
/// "\n\t let \t\t",
/// &[TriviaPiece::whitespace(3)],
/// &[TriviaPiece::whitespace(3)],
/// );
/// builder.token(RawLanguageKind::STRING_TOKEN, "a");
/// builder.token_with_trivia(
/// RawLanguageKind::SEMICOLON_TOKEN,
/// "; \t\t",
/// &[],
/// &[TriviaPiece::whitespace(3)],
/// );
/// });
/// let range = node.text_range();
/// assert_eq!(0u32, u32::from(range.start()));
/// assert_eq!(14u32, u32::from(range.end()));
/// ```
pub fn text_range(&self) -> TextRange {
self.raw.text_range()
}
/// Returns the range corresponding for the text of all descendants tokens combined,
/// excluding the first token leading trivia, and the last token trailing trivia.
/// All other trivia is included.
///
/// ```
/// use ruff_rowan::raw_language::{RawLanguage, RawLanguageKind, RawSyntaxTreeBuilder};
/// use ruff_rowan::*;
/// let mut node = RawSyntaxTreeBuilder::wrap_with_node(RawLanguageKind::ROOT, |builder| {
/// builder.token_with_trivia(
/// RawLanguageKind::LET_TOKEN,
/// "\n\t let \t\t",
/// &[TriviaPiece::whitespace(3)],
/// &[TriviaPiece::whitespace(3)],
/// );
/// builder.token(RawLanguageKind::STRING_TOKEN, "a");
/// builder.token_with_trivia(
/// RawLanguageKind::SEMICOLON_TOKEN,
/// "; \t\t",
/// &[],
/// &[TriviaPiece::whitespace(3)],
/// );
/// });
/// let range = node.text_trimmed_range();
/// assert_eq!(3u32, u32::from(range.start()));
/// assert_eq!(11u32, u32::from(range.end()));
/// ```
pub fn text_trimmed_range(&self) -> TextRange {
self.raw.text_trimmed_range()
}
/// Returns the leading trivia of the [first_token](SyntaxNode::first_token), or [None] if the node does not have any descendant tokens.
///
/// ```
/// use ruff_rowan::raw_language::{RawLanguage, RawLanguageKind, RawSyntaxTreeBuilder};
/// use ruff_rowan::*;
/// let mut node = RawSyntaxTreeBuilder::wrap_with_node(RawLanguageKind::ROOT, |builder| {
/// builder.token_with_trivia(
/// RawLanguageKind::LET_TOKEN,
/// "\n\t let \t\t",
/// &[TriviaPiece::whitespace(3)],
/// &[TriviaPiece::whitespace(3)],
/// );
/// builder.token(RawLanguageKind::STRING_TOKEN, "a");
/// builder.token_with_trivia(
/// RawLanguageKind::SEMICOLON_TOKEN,
/// "; \t\t",
/// &[],
/// &[TriviaPiece::whitespace(3)],
/// );
/// });
/// let trivia = node.first_leading_trivia();
/// assert!(trivia.is_some());
/// assert_eq!("\n\t ", trivia.unwrap().text());
///
/// let mut node = RawSyntaxTreeBuilder::wrap_with_node(RawLanguageKind::ROOT, |builder| {});
/// let trivia = node.first_leading_trivia();
/// assert!(trivia.is_none());
/// ```
pub fn first_leading_trivia(&self) -> Option<SyntaxTrivia<L>> {
self.raw.first_leading_trivia().map(SyntaxTrivia::new)
}
/// Returns the trailing trivia of the [last_token](SyntaxNode::last_token), or [None] if the node does not have any descendant tokens.
///
/// ```
/// use ruff_rowan::raw_language::{RawLanguage, RawLanguageKind, RawSyntaxTreeBuilder};
/// use ruff_rowan::*;
/// let mut node = RawSyntaxTreeBuilder::wrap_with_node(RawLanguageKind::ROOT, |builder| {
/// builder.token_with_trivia(
/// RawLanguageKind::LET_TOKEN,
/// "\n\t let \t\t",
/// &[TriviaPiece::whitespace(3)],
/// &[TriviaPiece::whitespace(3)],
/// );
/// builder.token(RawLanguageKind::STRING_TOKEN, "a");
/// builder.token_with_trivia(
/// RawLanguageKind::SEMICOLON_TOKEN,
/// "; \t\t",
/// &[],
/// &[TriviaPiece::whitespace(3)],
/// );
/// });
/// let trivia = node.last_trailing_trivia();
/// assert!(trivia.is_some());
/// assert_eq!(" \t\t", trivia.unwrap().text());
///
/// let mut node = RawSyntaxTreeBuilder::wrap_with_node(RawLanguageKind::ROOT, |builder| {});
/// let trivia = node.last_trailing_trivia();
/// assert!(trivia.is_none());
/// ```
pub fn last_trailing_trivia(&self) -> Option<SyntaxTrivia<L>> {
self.raw.last_trailing_trivia().map(SyntaxTrivia::new)
}
pub fn parent(&self) -> Option<SyntaxNode<L>> {
self.raw.parent().map(Self::from)
}
/// Returns the grand parent.
pub fn grand_parent(&self) -> Option<SyntaxNode<L>> {
self.parent().and_then(|parent| parent.parent())
}
/// Returns the index of this node inside of its parent
#[inline]
pub fn index(&self) -> usize {
self.raw.index()
}
pub fn ancestors(&self) -> impl Iterator<Item = SyntaxNode<L>> {
self.raw.ancestors().map(SyntaxNode::from)
}
pub fn children(&self) -> SyntaxNodeChildren<L> {
SyntaxNodeChildren {
raw: self.raw.children(),
_p: PhantomData,
}
}
/// Returns an iterator over all the slots of this syntax node.
pub fn slots(&self) -> SyntaxSlots<L> {
SyntaxSlots {
raw: self.raw.slots(),
_p: PhantomData,
}
}
pub fn children_with_tokens(&self) -> SyntaxElementChildren<L> {
SyntaxElementChildren {
raw: self.raw.children_with_tokens(),
_p: PhantomData,
}
}
pub fn tokens(&self) -> impl Iterator<Item = SyntaxToken<L>> + DoubleEndedIterator + '_ {
self.raw.tokens().map(SyntaxToken::from)
}
pub fn first_child(&self) -> Option<SyntaxNode<L>> {
self.raw.first_child().map(Self::from)
}
pub fn last_child(&self) -> Option<SyntaxNode<L>> {
self.raw.last_child().map(Self::from)
}
pub fn first_child_or_token(&self) -> Option<SyntaxElement<L>> {
self.raw.first_child_or_token().map(NodeOrToken::from)
}
pub fn last_child_or_token(&self) -> Option<SyntaxElement<L>> {
self.raw.last_child_or_token().map(NodeOrToken::from)
}
pub fn next_sibling(&self) -> Option<SyntaxNode<L>> {
self.raw.next_sibling().map(Self::from)
}
pub fn prev_sibling(&self) -> Option<SyntaxNode<L>> {
self.raw.prev_sibling().map(Self::from)
}
pub fn next_sibling_or_token(&self) -> Option<SyntaxElement<L>> {
self.raw.next_sibling_or_token().map(NodeOrToken::from)
}
pub fn prev_sibling_or_token(&self) -> Option<SyntaxElement<L>> {
self.raw.prev_sibling_or_token().map(NodeOrToken::from)
}
/// Return the leftmost token in the subtree of this node.
pub fn first_token(&self) -> Option<SyntaxToken<L>> {
self.raw.first_token().map(SyntaxToken::from)
}
/// Return the rightmost token in the subtree of this node.
pub fn last_token(&self) -> Option<SyntaxToken<L>> {
self.raw.last_token().map(SyntaxToken::from)
}
pub fn siblings(&self, direction: Direction) -> impl Iterator<Item = SyntaxNode<L>> {
self.raw.siblings(direction).map(SyntaxNode::from)
}
pub fn siblings_with_tokens(
&self,
direction: Direction,
) -> impl Iterator<Item = SyntaxElement<L>> {
self.raw
.siblings_with_tokens(direction)
.map(SyntaxElement::from)
}
pub fn descendants(&self) -> impl Iterator<Item = SyntaxNode<L>> {
self.raw.descendants().map(SyntaxNode::from)
}
pub fn descendants_tokens(&self, direction: Direction) -> impl Iterator<Item = SyntaxToken<L>> {
self.descendants_with_tokens(direction)
.filter_map(|x| x.as_token().cloned())
}
pub fn descendants_with_tokens(
&self,
direction: Direction,
) -> impl Iterator<Item = SyntaxElement<L>> {
self.raw
.descendants_with_tokens(direction)
.map(NodeOrToken::from)
}
/// Traverse the subtree rooted at the current node (including the current
/// node) in preorder, excluding tokens.
pub fn preorder(&self) -> Preorder<L> {
Preorder {
raw: self.raw.preorder(),
_p: PhantomData,
}
}
/// Traverse the subtree rooted at the current node (including the current
/// node) in preorder, including tokens.
pub fn preorder_with_tokens(&self, direction: Direction) -> PreorderWithTokens<L> {
PreorderWithTokens {
raw: self.raw.preorder_with_tokens(direction),
_p: PhantomData,
}
}
/// Find a token in the subtree corresponding to this node, which covers the offset.
/// Precondition: offset must be within node's range.
pub fn token_at_offset(&self, offset: TextSize) -> TokenAtOffset<SyntaxToken<L>> {
self.raw.token_at_offset(offset).map(SyntaxToken::from)
}
/// Return the deepest node or token in the current subtree that fully
/// contains the range. If the range is empty and is contained in two leaf
/// nodes, either one can be returned. Precondition: range must be contained
/// within the current node
pub fn covering_element(&self, range: TextRange) -> SyntaxElement<L> {
NodeOrToken::from(self.raw.covering_element(range))
}
/// Finds a [`SyntaxElement`] which intersects with a given `range`. If
/// there are several intersecting elements, any one can be returned.
///
/// The method uses binary search internally, so it's complexity is
/// `O(log(N))` where `N = self.children_with_tokens().count()`.
pub fn child_or_token_at_range(&self, range: TextRange) -> Option<SyntaxElement<L>> {
self.raw
.child_or_token_at_range(range)
.map(SyntaxElement::from)
}
/// Returns an independent copy of the subtree rooted at this node.
///
/// The parent of the returned node will be `None`, the start offset will be
/// zero, but, otherwise, it'll be equivalent to the source node.
pub fn clone_subtree(&self) -> SyntaxNode<L> {
SyntaxNode::from(self.raw.clone_subtree())
}
/// Return a new version of this node detached from its parent node
#[must_use = "syntax elements are immutable, the result of update methods must be propagated to have any effect"]
pub fn detach(self) -> Self {
Self {
raw: self.raw.detach(),
_p: PhantomData,
}
}
/// Return a clone of this node with the specified range of slots replaced
/// with the elements of the provided iterator
#[must_use = "syntax elements are immutable, the result of update methods must be propagated to have any effect"]
pub fn splice_slots<R, I>(self, range: R, replace_with: I) -> Self
where
R: ops::RangeBounds<usize>,
I: IntoIterator<Item = Option<SyntaxElement<L>>>,
{
Self {
raw: self.raw.splice_slots(
range,
replace_with
.into_iter()
.map(|element| element.map(cursor::SyntaxElement::from)),
),
_p: PhantomData,
}
}
/// Return a new version of this node with the element `prev_elem` replaced with `next_elem`
///
/// `prev_elem` can be a direct child of this node, or an indirect child through any descendant node
///
/// Returns `None` if `prev_elem` is not a descendant of this node
#[must_use = "syntax elements are immutable, the result of update methods must be propagated to have any effect"]
pub fn replace_child(
self,
prev_elem: SyntaxElement<L>,
next_elem: SyntaxElement<L>,
) -> Option<Self> {
Some(Self {
raw: self.raw.replace_child(prev_elem.into(), next_elem.into())?,
_p: PhantomData,
})
}
pub fn into_list(self) -> SyntaxList<L> {
SyntaxList::new(self)
}
/// Whether the node contains any comments. This function checks
/// **all the descendants** of the current node.
pub fn has_comments_descendants(&self) -> bool {
self.descendants_tokens(Direction::Next)
.any(|tok| tok.has_trailing_comments() || tok.has_leading_comments())
}
/// It checks if the current node has trailing or leading trivia
pub fn has_comments_direct(&self) -> bool {
self.has_trailing_comments() || self.has_leading_comments()
}
/// It checks if the current node has comments at the edges:
/// if first or last tokens contain comments (leading or trailing)
pub fn first_or_last_token_have_comments(&self) -> bool {
self.first_token_has_comments() || self.last_token_has_comments()
}
/// Whether the node contains trailing comments.
pub fn has_trailing_comments(&self) -> bool {
self.last_token()
.map_or(false, |tok| tok.has_trailing_comments())
}
/// Whether the last token of a node has comments (leading or trailing)
pub fn last_token_has_comments(&self) -> bool {
self.last_token().map_or(false, |tok| {
tok.has_trailing_comments() || tok.has_leading_comments()
})
}
/// Whether the first token of a node has comments (leading or trailing)
pub fn first_token_has_comments(&self) -> bool {
self.first_token().map_or(false, |tok| {
tok.has_trailing_comments() || tok.has_leading_comments()
})
}
/// Whether the node contains leading comments.
pub fn has_leading_comments(&self) -> bool {
self.first_token()
.map_or(false, |tok| tok.has_leading_comments())
}
/// Whether the node contains leading newlines.
pub fn has_leading_newline(&self) -> bool {
self.first_token()
.map_or(false, |tok| tok.has_leading_newline())
}
}
impl<L> SyntaxNode<L>
where
L: Language + 'static,
{
/// Create a [Send] + [Sync] handle to this node
///
/// Returns `None` if self is not a root node
pub fn as_send(&self) -> Option<SendNode> {
if self.parent().is_none() {
Some(SendNode {
language: TypeId::of::<L>(),
green: self.green_node(),
})
} else {
None
}
}
}
impl<L: Language> fmt::Debug for SyntaxNode<L> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
if f.alternate() {
let mut level = 0;
for event in self.raw.preorder_slots() {
match event {
WalkEvent::Enter(element) => {
for _ in 0..level {
write!(f, " ")?;
}
match element {
cursor::SyntaxSlot::Node(node) => {
writeln!(f, "{}: {:?}", node.index(), SyntaxNode::<L>::from(node))?
}
cursor::SyntaxSlot::Token(token) => writeln!(
f,
"{}: {:?}",
token.index(),
SyntaxToken::<L>::from(token)
)?,
cursor::SyntaxSlot::Empty { index, .. } => {
writeln!(f, "{}: (empty)", index)?
}
}
level += 1;
}
WalkEvent::Leave(_) => level -= 1,
}
}
assert_eq!(level, 0);
Ok(())
} else {
write!(f, "{:?}@{:?}", self.kind(), self.text_range())
}
}
}
impl<L: Language> fmt::Display for SyntaxNode<L> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
fmt::Display::fmt(&self.raw, f)
}
}
impl<L: Language> From<SyntaxNode<L>> for cursor::SyntaxNode {
fn from(node: SyntaxNode<L>) -> cursor::SyntaxNode {
node.raw
}
}
impl<L: Language> From<cursor::SyntaxNode> for SyntaxNode<L> {
fn from(raw: cursor::SyntaxNode) -> SyntaxNode<L> {
SyntaxNode {
raw,
_p: PhantomData,
}
}
}
/// Language-agnostic representation of the root node of a syntax tree, can be
/// sent or shared between threads
#[derive(Clone)]
pub struct SendNode {
language: TypeId,
green: GreenNode,
}
impl SendNode {
/// Downcast this handle back into a [SyntaxNode]
///
/// Returns `None` if the specified language `L` is not the one this node
/// was created with
pub fn into_node<L>(self) -> Option<SyntaxNode<L>>
where
L: Language + 'static,
{
if TypeId::of::<L>() == self.language {
Some(SyntaxNode::new_root(self.green))
} else {
None
}
}
}
#[derive(Debug, Clone)]
pub struct SyntaxNodeChildren<L: Language> {
raw: cursor::SyntaxNodeChildren,
_p: PhantomData<L>,
}
impl<L: Language> Iterator for SyntaxNodeChildren<L> {
type Item = SyntaxNode<L>;
fn next(&mut self) -> Option<Self::Item> {
self.raw.next().map(SyntaxNode::from)
}
}
#[derive(Clone)]
pub struct SyntaxElementChildren<L: Language> {
raw: cursor::SyntaxElementChildren,
_p: PhantomData<L>,
}
impl<L: Language> Debug for SyntaxElementChildren<L> {
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
f.debug_list().entries(self.clone()).finish()
}
}
impl<L: Language> Default for SyntaxElementChildren<L> {
fn default() -> Self {
SyntaxElementChildren {
raw: cursor::SyntaxElementChildren::default(),
_p: PhantomData,
}
}
}
impl<L: Language> Iterator for SyntaxElementChildren<L> {
type Item = SyntaxElement<L>;
fn next(&mut self) -> Option<Self::Item> {
self.raw.next().map(NodeOrToken::from)
}
}
pub struct Preorder<L: Language> {
raw: cursor::Preorder,
_p: PhantomData<L>,
}
impl<L: Language> Preorder<L> {
pub fn skip_subtree(&mut self) {
self.raw.skip_subtree()
}
}
impl<L: Language> Iterator for Preorder<L> {
type Item = WalkEvent<SyntaxNode<L>>;
fn next(&mut self) -> Option<Self::Item> {
self.raw.next().map(|it| it.map(SyntaxNode::from))
}
}
pub struct PreorderWithTokens<L: Language> {
raw: cursor::PreorderWithTokens,
_p: PhantomData<L>,
}
impl<L: Language> PreorderWithTokens<L> {
pub fn skip_subtree(&mut self) {
self.raw.skip_subtree()
}
}
impl<L: Language> Iterator for PreorderWithTokens<L> {
type Item = WalkEvent<SyntaxElement<L>>;
fn next(&mut self) -> Option<Self::Item> {
self.raw.next().map(|it| it.map(SyntaxElement::from))
}
}
/// Each node has a slot for each of its children regardless if the child is present or not.
/// A child that isn't present either because it's optional or because of a syntax error
/// is stored in an [SyntaxSlot::Empty] to preserve the index of each child.
#[derive(Debug, Clone, Eq, PartialEq)]
#[cfg_attr(feature = "serde", derive(Serialize))]
pub enum SyntaxSlot<L: Language> {
/// Slot that stores a node child
Node(SyntaxNode<L>),
/// Slot that stores a token child
Token(SyntaxToken<L>),
/// Slot that marks that the child in this position isn't present in the source code.
Empty,
}
impl<L: Language> SyntaxSlot<L> {
pub fn into_node(self) -> Option<SyntaxNode<L>> {
match self {
SyntaxSlot::Node(node) => Some(node),
_ => None,
}
}
pub fn into_token(self) -> Option<SyntaxToken<L>> {
match self {
SyntaxSlot::Token(token) => Some(token),
_ => None,
}
}
pub fn into_syntax_element(self) -> Option<SyntaxElement<L>> {
match self {
SyntaxSlot::Node(node) => Some(SyntaxElement::Node(node)),
SyntaxSlot::Token(token) => Some(SyntaxElement::Token(token)),
_ => None,
}
}
pub fn kind(&self) -> Option<L::Kind> {
match self {
SyntaxSlot::Node(node) => Some(node.kind()),
SyntaxSlot::Token(token) => Some(token.kind()),
SyntaxSlot::Empty => None,
}
}
}
impl<L: Language> From<cursor::SyntaxSlot> for SyntaxSlot<L> {
fn from(raw: cursor::SyntaxSlot) -> Self {
match raw {
cursor::SyntaxSlot::Node(node) => SyntaxSlot::Node(node.into()),
cursor::SyntaxSlot::Token(token) => SyntaxSlot::Token(token.into()),
cursor::SyntaxSlot::Empty { .. } => SyntaxSlot::Empty,
}
}
}
/// Iterator over the slots of a node.
#[derive(Debug, Clone)]
pub struct SyntaxSlots<L> {
raw: cursor::SyntaxSlots,
_p: PhantomData<L>,
}
impl<L: Language> Iterator for SyntaxSlots<L> {
type Item = SyntaxSlot<L>;
#[inline]
fn next(&mut self) -> Option<Self::Item> {
self.raw.next().map(SyntaxSlot::from)
}
#[inline(always)]
fn size_hint(&self) -> (usize, Option<usize>) {
self.raw.size_hint()
}
#[inline]
fn last(self) -> Option<Self::Item>
where
Self: Sized,
{
self.raw.last().map(SyntaxSlot::from)
}
#[inline]
fn nth(&mut self, n: usize) -> Option<Self::Item> {
self.raw.nth(n).map(SyntaxSlot::from)
}
}
impl<L: Language> FusedIterator for SyntaxSlots<L> {}
impl<L: Language> ExactSizeIterator for SyntaxSlots<L> {
#[inline(always)]
fn len(&self) -> usize {
self.raw.len()
}
}
impl<L: Language> DoubleEndedIterator for SyntaxSlots<L> {
#[inline]
fn next_back(&mut self) -> Option<Self::Item> {
self.raw.next_back().map(SyntaxSlot::from)
}
#[inline]
fn nth_back(&mut self, n: usize) -> Option<Self::Item> {
self.raw.nth_back(n).map(SyntaxSlot::from)
}
}
/// Trait with extension methods for [Option<SyntaxNode>].
pub trait SyntaxNodeOptionExt<L: Language> {
/// Returns the kind of the node if self is [Some], [None] otherwise.
fn kind(&self) -> Option<L::Kind>;
}
impl<L: Language> SyntaxNodeOptionExt<L> for Option<&SyntaxNode<L>> {
fn kind(&self) -> Option<L::Kind> {
self.map(|node| node.kind())
}
}
impl<L: Language> SyntaxNodeOptionExt<L> for Option<SyntaxNode<L>> {
fn kind(&self) -> Option<L::Kind> {
self.as_ref().kind()
}
}

259
crates/ruff_rowan/src/syntax/rewriter.rs Normal file
View File

@ -0,0 +1,259 @@
//! A module that exports utilities to rewrite a syntax trees
use crate::{Language, SyntaxNode, SyntaxSlot, SyntaxToken};
use std::iter::once;
/// A visitor that re-writes a syntax tree while visiting the nodes.
///
/// The rewriter visits the nodes in pre-order from top-down.
/// Meaning, it first visits the `root`, and then visits the children of the root from left to right,
/// recursively traversing into child nodes and calling [`visit_node`](SyntaxRewriter) for every node.
///
/// Inspired by Roslyn's [`CSharpSyntaxRewriter`](https://docs.microsoft.com/en-us/dotnet/api/microsoft.codeanalysis.csharp.csharpsyntaxrewriter?view=roslyn-dotnet-4.2.0)
///
/// # Unsupported
///
/// The current implementation does not yet support node removal.
///
/// # Examples
///
/// Implementation of a rewritten that replaces all literal expression nodes that contain a number token
/// with a bogus node.
///
/// ```
/// # use std::iter::once;
/// # use ruff_rowan::{AstNode, SyntaxNode, SyntaxRewriter, VisitNodeSignal};
/// # use ruff_rowan::raw_language::{LiteralExpression, RawLanguage, RawLanguageKind, RawSyntaxTreeBuilder};
///
/// struct ReplaceNumberLiteralRewriter;
///
/// impl SyntaxRewriter for ReplaceNumberLiteralRewriter {
/// type Language = RawLanguage;
///
/// fn visit_node(
/// &mut self,
/// node: SyntaxNode<Self::Language>,
/// ) -> VisitNodeSignal<Self::Language> {
/// match node.kind() {
/// RawLanguageKind::LITERAL_EXPRESSION => {
/// let expression = LiteralExpression::unwrap_cast(node);
///
/// let mut token = expression
/// .syntax()
/// .slots()
/// .nth(0)
/// .unwrap()
/// .into_token()
/// .unwrap();
///
/// match token.kind() {
/// RawLanguageKind::NUMBER_TOKEN => {
/// // Use your language's syntax factory instead
/// let bogus_node = SyntaxNode::new_detached(
/// RawLanguageKind::BOGUS,
/// once(Some(token.into())),
/// );
///
/// VisitNodeSignal::Replace(bogus_node)
/// }
/// // Not interested in string literal expressions, continue traversal
/// _ => VisitNodeSignal::Traverse(expression.into_syntax()),
/// }
/// }
/// _ => {
/// // Traverse into the childrens of node
/// VisitNodeSignal::Traverse(node)
/// }
/// }
/// }
/// }
///
/// let mut builder = RawSyntaxTreeBuilder::new();
///
/// builder.start_node(RawLanguageKind::ROOT);
/// builder.start_node(RawLanguageKind::SEPARATED_EXPRESSION_LIST);
///
/// builder.start_node(RawLanguageKind::LITERAL_EXPRESSION);
/// builder.token(RawLanguageKind::NUMBER_TOKEN, "5");
/// builder.finish_node();
///
/// builder.start_node(RawLanguageKind::LITERAL_EXPRESSION);
/// builder.token(RawLanguageKind::STRING_TOKEN, "'abcd'");
/// builder.finish_node();
///
/// builder.finish_node();
/// builder.finish_node();
///
/// let root = builder.finish();
///
/// let transformed = ReplaceNumberLiteralRewriter.transform(root.clone());
///
/// let original_literal_expressions: Vec<_> = root
/// .descendants()
/// .filter(|p| p.kind() == RawLanguageKind::LITERAL_EXPRESSION)
/// .collect();
///
/// assert_ne!(
/// &root, &transformed,
/// "It returns a new root with the updated children"
/// );
///
/// let literal_expressions: Vec<_> = transformed
/// .descendants()
/// .filter(|p| p.kind() == RawLanguageKind::LITERAL_EXPRESSION)
/// .collect();
///
/// // The literal expression containing a string token should be unchanged
/// assert_eq!(&literal_expressions, &original_literal_expressions[1..]);
///
/// let mut bogus: Vec<_> = transformed
/// .descendants()
/// .filter(|p| p.kind() == RawLanguageKind::BOGUS)
/// .collect();
///
/// // It replaced the number literal expression with a bogus node.
/// assert_eq!(bogus.len(), 1);
/// assert_eq!(bogus.pop().unwrap().text(), "5");
/// ```
pub trait SyntaxRewriter {
type Language: Language;
/// Recursively transforms the subtree of `node` by calling [`visit_node`](SyntaxRewriter::visit_node)
/// for every token and [`visit_token`](SyntaxRewriter::visit_token) for every token in the subtree.
///
/// Returns a new syntax tree reflecting the changes by the rewriter if it replaced any node and
/// returns `node` if no changes were made.
fn transform(&mut self, node: SyntaxNode<Self::Language>) -> SyntaxNode<Self::Language>
where
Self: Sized,
{
match self.visit_node(node) {
VisitNodeSignal::Replace(updated) => updated,
VisitNodeSignal::Traverse(node) => traverse(node, self),
}
}
/// Called for every node in the tree. The method should return a signal specifying what should be done with the node
///
/// * [VisitNodeSignal::Traverse]: Recourse into `node` so that [`visit_node`](SyntaxRewriter::visit_node)
/// gets called for all children of `node`. The `node` will only be replaced if any node in its subtree changes.
/// * [VisitNodeSignal::Replace]: Replaces `node` with the node specified in the [`Replace`](VisitNodeSignal::Replace) variant.
/// It's your responsibility to call [`traverse`](SyntaxRewriter::transform) for any child of `node` for which you want the rewritter
/// to recurse into its content.
fn visit_node(&mut self, node: SyntaxNode<Self::Language>) -> VisitNodeSignal<Self::Language> {
VisitNodeSignal::Traverse(node)
}
/// Called for every token in the tree. Returning a new token changes the token in the parent node.
fn visit_token(&mut self, token: SyntaxToken<Self::Language>) -> SyntaxToken<Self::Language> {
token
}
}
#[derive(Debug, Clone)]
pub enum VisitNodeSignal<L: Language> {
/// Signals the [SyntaxRewriter] to replace the current node with the specified node.
Replace(SyntaxNode<L>),
/// Signals the [SyntaxRewriter] to traverse into the children of the specified node.
Traverse(SyntaxNode<L>),
}
fn traverse<R>(mut parent: SyntaxNode<R::Language>, rewriter: &mut R) -> SyntaxNode<R::Language>
where
R: SyntaxRewriter,
{
for slot in parent.slots() {
match slot {
SyntaxSlot::Node(node) => {
let original_key = node.key();
let index = node.index();
let updated = rewriter.transform(node);
if updated.key() != original_key {
parent = parent.splice_slots(index..=index, once(Some(updated.into())));
}
}
SyntaxSlot::Token(token) => {
let original_key = token.key();
let index = token.index();
let updated = rewriter.visit_token(token);
if updated.key() != original_key {
parent = parent.splice_slots(index..=index, once(Some(updated.into())));
}
}
SyntaxSlot::Empty => {
// Nothing to visit
}
}
}
parent
}
#[cfg(test)]
mod tests {
use crate::raw_language::{RawLanguage, RawLanguageKind, RawSyntaxTreeBuilder};
use crate::{SyntaxNode, SyntaxRewriter, SyntaxToken, VisitNodeSignal};
#[test]
pub fn test_visits_each_node() {
let mut builder = RawSyntaxTreeBuilder::new();
builder.start_node(RawLanguageKind::ROOT);
builder.start_node(RawLanguageKind::LITERAL_EXPRESSION);
builder.token(RawLanguageKind::NUMBER_TOKEN, "5");
builder.finish_node();
builder.finish_node();
let root = builder.finish();
let mut recorder = RecordRewritter::default();
let transformed = recorder.transform(root.clone());
assert_eq!(
&root, &transformed,
"It should return the same node if the rewritter doesn't replace a node."
);
let literal_expression = root
.descendants()
.find(|node| node.kind() == RawLanguageKind::LITERAL_EXPRESSION)
.unwrap();
assert_eq!(&recorder.nodes, &[root.clone(), literal_expression]);
let number_literal = root.first_token().unwrap();
assert_eq!(&recorder.tokens, &[number_literal]);
}
/// Visitor that records every `visit_node` and `visit_token` call.
#[derive(Default)]
struct RecordRewritter {
nodes: Vec<SyntaxNode<RawLanguage>>,
tokens: Vec<SyntaxToken<RawLanguage>>,
}
impl SyntaxRewriter for RecordRewritter {
type Language = RawLanguage;
fn visit_node(
&mut self,
node: SyntaxNode<Self::Language>,
) -> VisitNodeSignal<Self::Language> {
self.nodes.push(node.clone());
VisitNodeSignal::Traverse(node)
}
fn visit_token(
&mut self,
token: SyntaxToken<Self::Language>,
) -> SyntaxToken<Self::Language> {
self.tokens.push(token.clone());
token
}
}
}

425
crates/ruff_rowan/src/syntax/token.rs Normal file
View File

@ -0,0 +1,425 @@
use crate::green::{GreenToken, GreenTrivia};
use crate::syntax::element::SyntaxElementKey;
use crate::syntax::SyntaxTrivia;
use crate::syntax_token_text::SyntaxTokenText;
use crate::{
cursor, Direction, Language, NodeOrToken, SyntaxElement, SyntaxKind, SyntaxNode,
SyntaxTriviaPiece, TriviaPiece, TriviaPieceKind,
};
use ruff_text_size::{TextLen, TextRange, TextSize};
use std::fmt;
use std::marker::PhantomData;
#[derive(Clone, PartialEq, Eq, Hash)]
pub struct SyntaxToken<L: Language> {
raw: cursor::SyntaxToken,
_p: PhantomData<L>,
}
impl<L: Language> SyntaxToken<L> {
/// Create a new token detached from any tree
///
/// This is mainly useful for creating a small number of individual tokens
/// when mutating an existing tree, the bulk of the tokens in a given file
/// should be created through the [crate::TreeBuilder] abstraction instead
/// as it will efficiently cache and reuse the created tokens
pub fn new_detached<Leading, Trailing>(
kind: L::Kind,
text: &str,
leading: Leading,
trailing: Trailing,
) -> Self
where
Leading: IntoIterator<Item = TriviaPiece>,
Leading::IntoIter: ExactSizeIterator,
Trailing: IntoIterator<Item = TriviaPiece>,
Trailing::IntoIter: ExactSizeIterator,
{
Self {
raw: cursor::SyntaxToken::new_detached(GreenToken::with_trivia(
kind.to_raw(),
text,
GreenTrivia::new(leading),
GreenTrivia::new(trailing),
)),
_p: PhantomData,
}
}
pub(super) fn green_token(&self) -> GreenToken {
self.raw.green().to_owned()
}
pub fn key(&self) -> SyntaxElementKey {
let (node_data, offset) = self.raw.key();
SyntaxElementKey::new(node_data, offset)
}
pub fn kind(&self) -> L::Kind {
L::Kind::from_raw(self.raw.kind())
}
pub fn text_range(&self) -> TextRange {
self.raw.text_range()
}
pub fn text_trimmed_range(&self) -> TextRange {
self.raw.text_trimmed_range()
}
pub(crate) fn index(&self) -> usize {
self.raw.index()
}
/// Returns the text of the token, including all trivia.
///
/// ```
/// use ruff_rowan::raw_language::{RawLanguage, RawLanguageKind, RawSyntaxTreeBuilder};
/// use ruff_rowan::*;
/// let mut token = RawSyntaxTreeBuilder::wrap_with_node(RawLanguageKind::ROOT, |builder| {
/// builder.token_with_trivia(
/// RawLanguageKind::LET_TOKEN,
/// "\n\t let \t\t",
/// &[TriviaPiece::whitespace(3)],
/// &[TriviaPiece::whitespace(3)],
/// );
/// })
/// .first_token()
/// .unwrap();
/// assert_eq!("\n\t let \t\t", token.text());
/// ```
pub fn text(&self) -> &str {
self.raw.text()
}
/// Returns the text of a token, including all trivia as an owned value.
///
/// ```
/// use ruff_rowan::raw_language::{RawLanguage, RawLanguageKind, RawSyntaxTreeBuilder};
/// use ruff_rowan::*;
/// let mut token = RawSyntaxTreeBuilder::wrap_with_node(RawLanguageKind::ROOT, |builder| {
/// builder.token_with_trivia(
/// RawLanguageKind::LET_TOKEN,
/// "\n\t let \t\t",
/// &[TriviaPiece::whitespace(3)],
/// &[TriviaPiece::whitespace(3)],
/// );
/// })
/// .first_token()
/// .unwrap();
/// assert_eq!("\n\t let \t\t", token.token_text());
/// assert_eq!(
/// format!("{}", "\n\t let \t\t"),
/// format!("{}", token.token_text())
/// );
/// assert_eq!(
/// format!("{:?}", "\n\t let \t\t"),
/// format!("{:?}", token.token_text())
/// );
/// ```
pub fn token_text(&self) -> SyntaxTokenText {
self.raw.token_text()
}
pub fn token_text_trimmed(&self) -> SyntaxTokenText {
self.raw.token_text_trimmed()
}
/// Returns the text of the token, excluding all trivia.
///
/// ```
/// use ruff_rowan::raw_language::{RawLanguage, RawLanguageKind, RawSyntaxTreeBuilder};
/// use ruff_rowan::*;
/// let mut token = RawSyntaxTreeBuilder::wrap_with_node(RawLanguageKind::ROOT, |builder| {
/// builder.token_with_trivia(
/// RawLanguageKind::LET_TOKEN,
/// "\n\t let \t\t",
/// &[TriviaPiece::whitespace(3)],
/// &[TriviaPiece::whitespace(3)],
/// );
/// })
/// .first_token()
/// .unwrap();
/// assert_eq!("let", token.text_trimmed());
/// ```
pub fn text_trimmed(&self) -> &str {
self.raw.text_trimmed()
}
pub fn parent(&self) -> Option<SyntaxNode<L>> {
self.raw.parent().map(SyntaxNode::from)
}
pub fn ancestors(&self) -> impl Iterator<Item = SyntaxNode<L>> {
self.raw.ancestors().map(SyntaxNode::from)
}
pub fn next_sibling_or_token(&self) -> Option<SyntaxElement<L>> {
self.raw.next_sibling_or_token().map(NodeOrToken::from)
}
pub fn prev_sibling_or_token(&self) -> Option<SyntaxElement<L>> {
self.raw.prev_sibling_or_token().map(NodeOrToken::from)
}
pub fn siblings_with_tokens(
&self,
direction: Direction,
) -> impl Iterator<Item = SyntaxElement<L>> {
self.raw
.siblings_with_tokens(direction)
.map(SyntaxElement::from)
}
/// Next token in the tree (i.e, not necessary a sibling).
pub fn next_token(&self) -> Option<SyntaxToken<L>> {
self.raw.next_token().map(SyntaxToken::from)
}
/// Previous token in the tree (i.e, not necessary a sibling).
pub fn prev_token(&self) -> Option<SyntaxToken<L>> {
self.raw.prev_token().map(SyntaxToken::from)
}
/// Return a new version of this token detached from its parent node
#[must_use = "syntax elements are immutable, the result of update methods must be propagated to have any effect"]
pub fn detach(self) -> Self {
Self {
raw: self.raw.detach(),
_p: PhantomData,
}
}
/// Return a new version of this token with its leading trivia replaced with `trivia`
#[must_use = "syntax elements are immutable, the result of update methods must be propagated to have any effect"]
pub fn with_leading_trivia<'a, I>(&self, trivia: I) -> Self
where
I: IntoIterator<Item = (TriviaPieceKind, &'a str)>,
I::IntoIter: ExactSizeIterator,
{
let mut token_text = String::new();
let trivia = trivia.into_iter().map(|(kind, text)| {
token_text.push_str(text);
TriviaPiece::new(kind, TextSize::of(text))
});
let leading = GreenTrivia::new(trivia);
// Copy over token text and trailing trivia
let leading_len = self.raw.green().leading_trivia().text_len();
token_text.push_str(&self.text()[usize::from(leading_len)..]);
Self {
raw: cursor::SyntaxToken::new_detached(GreenToken::with_trivia(
self.kind().to_raw(),
&token_text,
leading,
self.green_token().trailing_trivia().clone(),
)),
_p: PhantomData,
}
}
/// Return a new version of this token with its leading trivia replaced with `trivia`
#[must_use = "syntax elements are immutable, the result of update methods must be propagated to have any effect"]
pub fn with_leading_trivia_pieces<I>(&self, trivia: I) -> Self
where
I: IntoIterator<Item = SyntaxTriviaPiece<L>>,
I::IntoIter: ExactSizeIterator,
{
let mut token_text = String::new();
let trivia = trivia.into_iter().map(|piece| {
token_text.push_str(piece.text());
piece.into_raw_piece()
});
let leading = GreenTrivia::new(trivia);
// Copy over token text and trailing trivia
let leading_len = self.raw.green().leading_trivia().text_len();
token_text.push_str(&self.text()[usize::from(leading_len)..]);
Self {
raw: cursor::SyntaxToken::new_detached(GreenToken::with_trivia(
self.kind().to_raw(),
&token_text,
leading,
self.green_token().trailing_trivia().clone(),
)),
_p: PhantomData,
}
}
/// Return a new version of this token with its trailing trivia replaced with `trivia`
#[must_use = "syntax elements are immutable, the result of update methods must be propagated to have any effect"]
pub fn with_trailing_trivia<'a, I>(&self, trivia: I) -> Self
where
I: IntoIterator<Item = (TriviaPieceKind, &'a str)>,
I::IntoIter: ExactSizeIterator,
{
let mut token_text = String::new();
// copy over leading trivia and token text
let trailing_len = self.green_token().trailing_trivia().text_len();
token_text.push_str(&self.text()[..usize::from(self.text().text_len() - trailing_len)]);
let trivia = trivia.into_iter().map(|(kind, text)| {
token_text.push_str(text);
TriviaPiece::new(kind, TextSize::of(text))
});
let trailing = GreenTrivia::new(trivia);
Self {
raw: cursor::SyntaxToken::new_detached(GreenToken::with_trivia(
self.kind().to_raw(),
&token_text,
self.green_token().leading_trivia().clone(),
trailing,
)),
_p: PhantomData,
}
}
/// Return a new version of this token with its trailing trivia replaced with `trivia`
#[must_use = "syntax elements are immutable, the result of update methods must be propagated to have any effect"]
pub fn with_trailing_trivia_pieces<I>(&self, trivia: I) -> Self
where
I: IntoIterator<Item = SyntaxTriviaPiece<L>>,
I::IntoIter: ExactSizeIterator,
{
let mut token_text = String::new();
// copy over leading trivia and token text
let trailing_len = self.green_token().trailing_trivia().text_len();
token_text.push_str(&self.text()[..usize::from(self.text().text_len() - trailing_len)]);
let trivia = trivia.into_iter().map(|piece| {
token_text.push_str(piece.text());
piece.into_raw_piece()
});
let trailing = GreenTrivia::new(trivia);
Self {
raw: cursor::SyntaxToken::new_detached(GreenToken::with_trivia(
self.kind().to_raw(),
&token_text,
self.green_token().leading_trivia().clone(),
trailing,
)),
_p: PhantomData,
}
}
/// Returns the token's leading trivia.
///
/// Looking backward in the text, a token owns all of its preceding trivia up to and including the first newline character.
///
/// ```
/// use ruff_rowan::raw_language::{RawLanguage, RawLanguageKind, RawSyntaxTreeBuilder};
/// use ruff_rowan::*;
/// let mut token = RawSyntaxTreeBuilder::wrap_with_node(RawLanguageKind::ROOT, |builder| {
/// builder.token_with_trivia(
/// RawLanguageKind::LET_TOKEN,
/// "\n\t let \t\t",
/// &[TriviaPiece::whitespace(3)],
/// &[TriviaPiece::whitespace(3)],
/// );
/// })
/// .first_token()
/// .unwrap();
/// assert_eq!("\n\t ", token.leading_trivia().text());
/// ```
#[inline]
pub fn leading_trivia(&self) -> SyntaxTrivia<L> {
SyntaxTrivia::new(self.raw.leading_trivia())
}
/// Returns the token's trailing trivia.
///
/// A token owns all of its following trivia up to, but not including, the next newline character.
///
/// ```
/// use ruff_rowan::raw_language::{RawLanguage, RawLanguageKind, RawSyntaxTreeBuilder};
/// use ruff_rowan::*;
/// let mut token = RawSyntaxTreeBuilder::wrap_with_node(RawLanguageKind::ROOT, |builder| {
/// builder.token_with_trivia(
/// RawLanguageKind::LET_TOKEN,
/// "\n\t let \t\t",
/// &[TriviaPiece::whitespace(3)],
/// &[TriviaPiece::whitespace(3)],
/// );
/// })
/// .first_token()
/// .unwrap();
/// assert_eq!(" \t\t", token.trailing_trivia().text());
/// ```
#[inline]
pub fn trailing_trivia(&self) -> SyntaxTrivia<L> {
SyntaxTrivia::new(self.raw.trailing_trivia())
}
/// Checks if the current token has trailing comments
pub fn has_trailing_comments(&self) -> bool {
self.trailing_trivia()
.pieces()
.any(|piece| piece.is_comments())
}
/// Checks if the current token has leading comments
pub fn has_leading_comments(&self) -> bool {
self.leading_trivia()
.pieces()
.any(|piece| piece.is_comments())
}
/// Checks if the token has any leading trivia that isn't a whitespace nor a line break
pub fn has_leading_non_whitespace_trivia(&self) -> bool {
self.leading_trivia()
.pieces()
.any(|piece| piece.is_whitespace() || piece.is_newline())
}
/// Checks if the current token has leading newline
pub fn has_leading_newline(&self) -> bool {
self.leading_trivia()
.pieces()
.any(|piece| piece.is_newline())
}
}
impl<L: Language> fmt::Debug for SyntaxToken<L> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(
f,
"{:?}@{:?} {:?} ",
self.kind(),
self.text_range(),
self.text_trimmed()
)?;
self.leading_trivia().fmt(f)?;
write!(f, " ")?;
self.trailing_trivia().fmt(f)
}
}
impl<L: Language> fmt::Display for SyntaxToken<L> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
fmt::Display::fmt(&self.raw, f)
}
}
impl<L: Language> From<SyntaxToken<L>> for cursor::SyntaxToken {
fn from(token: SyntaxToken<L>) -> cursor::SyntaxToken {
token.raw
}
}
impl<L: Language> From<cursor::SyntaxToken> for SyntaxToken<L> {
fn from(raw: cursor::SyntaxToken) -> SyntaxToken<L> {
SyntaxToken {
raw,
_p: PhantomData,
}
}
}

810
crates/ruff_rowan/src/syntax/trivia.rs Normal file
View File

@ -0,0 +1,810 @@
use crate::{cursor, Language, SyntaxToken};
use ruff_text_size::{TextRange, TextSize};
use std::fmt;
use std::fmt::Formatter;
use std::iter::FusedIterator;
use std::marker::PhantomData;
#[derive(Clone, Copy, Eq, PartialEq, Hash, Debug)]
pub enum TriviaPieceKind {
/// A line break (`\n`, `\r`, `\r\n`, ...)
Newline,
/// Any whitespace character
Whitespace,
/// Comment that does not contain any line breaks
SingleLineComment,
/// Comment that contains at least one line break
MultiLineComment,
/// Token that the parser skipped for some reason.
Skipped,
}
impl TriviaPieceKind {
pub const fn is_newline(&self) -> bool {
matches!(self, TriviaPieceKind::Newline)
}
pub const fn is_whitespace(&self) -> bool {
matches!(self, TriviaPieceKind::Whitespace)
}
pub const fn is_single_line_comment(&self) -> bool {
matches!(self, TriviaPieceKind::SingleLineComment)
}
pub const fn is_multiline_comment(&self) -> bool {
matches!(self, TriviaPieceKind::MultiLineComment)
}
pub const fn is_skipped(&self) -> bool {
matches!(self, TriviaPieceKind::Skipped)
}
}
#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)]
pub struct TriviaPiece {
pub(crate) kind: TriviaPieceKind,
pub(crate) length: TextSize,
}
impl TriviaPiece {
/// Creates a new whitespace trivia piece with the given length
pub fn whitespace<L: Into<TextSize>>(len: L) -> Self {
Self::new(TriviaPieceKind::Whitespace, len)
}
/// Creates a new newline trivia piece with the given text length
pub fn newline<L: Into<TextSize>>(len: L) -> Self {
Self::new(TriviaPieceKind::Newline, len)
}
/// Creates a new comment trivia piece that does not contain any line breaks.
/// For example, JavaScript's `//` comments are guaranteed to not spawn multiple lines. However,
/// this can also be a `/* ... */` comment if it doesn't contain any line break characters.
pub fn single_line_comment<L: Into<TextSize>>(len: L) -> Self {
Self::new(TriviaPieceKind::SingleLineComment, len)
}
/// Creates a new comment trivia piece that contains at least one line breaks.
/// For example, a JavaScript `/* ... */` comment that spawns at least two lines (contains at least one line break character).
pub fn multi_line_comment<L: Into<TextSize>>(len: L) -> Self {
Self::new(TriviaPieceKind::MultiLineComment, len)
}
pub fn new<L: Into<TextSize>>(kind: TriviaPieceKind, length: L) -> Self {
Self {
kind,
length: length.into(),
}
}
/// Returns the trivia's length
pub fn text_len(&self) -> TextSize {
self.length
}
/// Returns the trivia's kind
pub fn kind(&self) -> TriviaPieceKind {
self.kind
}
}
#[derive(Debug, Clone)]
pub struct SyntaxTriviaPieceNewline<L: Language>(SyntaxTriviaPiece<L>);
#[derive(Debug, Clone)]
pub struct SyntaxTriviaPieceWhitespace<L: Language>(SyntaxTriviaPiece<L>);
#[derive(Debug, Clone)]
pub struct SyntaxTriviaPieceComments<L: Language>(SyntaxTriviaPiece<L>);
#[derive(Debug, Clone)]
pub struct SyntaxTriviaPieceSkipped<L: Language>(SyntaxTriviaPiece<L>);
impl<L: Language> SyntaxTriviaPieceNewline<L> {
pub fn text(&self) -> &str {
self.0.text()
}
pub fn text_len(&self) -> TextSize {
self.0.text_len()
}
pub fn text_range(&self) -> TextRange {
self.0.text_range()
}
/// Returns a reference to its [SyntaxTriviaPiece]
pub fn as_piece(&self) -> &SyntaxTriviaPiece<L> {
&self.0
}
/// Returns its [SyntaxTriviaPiece]
pub fn into_piece(self) -> SyntaxTriviaPiece<L> {
self.0
}
}
impl<L: Language> SyntaxTriviaPieceWhitespace<L> {
pub fn text(&self) -> &str {
self.0.text()
}
pub fn text_len(&self) -> TextSize {
self.0.text_len()
}
pub fn text_range(&self) -> TextRange {
self.0.text_range()
}
/// Returns a reference to its [SyntaxTriviaPiece]
pub fn as_piece(&self) -> &SyntaxTriviaPiece<L> {
&self.0
}
/// Returns its [SyntaxTriviaPiece]
pub fn into_piece(self) -> SyntaxTriviaPiece<L> {
self.0
}
}
impl<L: Language> SyntaxTriviaPieceComments<L> {
pub fn text(&self) -> &str {
self.0.text()
}
pub fn text_len(&self) -> TextSize {
self.0.text_len()
}
pub fn text_range(&self) -> TextRange {
self.0.text_range()
}
pub fn has_newline(&self) -> bool {
self.0.trivia.kind.is_multiline_comment()
}
/// Returns a reference to its [SyntaxTriviaPiece]
pub fn as_piece(&self) -> &SyntaxTriviaPiece<L> {
&self.0
}
/// Returns its [SyntaxTriviaPiece]
pub fn into_piece(self) -> SyntaxTriviaPiece<L> {
self.0
}
}
impl<L: Language> SyntaxTriviaPieceSkipped<L> {
pub fn text(&self) -> &str {
self.0.text()
}
pub fn text_len(&self) -> TextSize {
self.0.text_len()
}
pub fn text_range(&self) -> TextRange {
self.0.text_range()
}
/// Returns a reference to its [SyntaxTriviaPiece]
pub fn as_piece(&self) -> &SyntaxTriviaPiece<L> {
&self.0
}
/// Returns its [SyntaxTriviaPiece]
pub fn into_piece(self) -> SyntaxTriviaPiece<L> {
self.0
}
}
/// [SyntaxTriviaPiece] gives access to the most granular information about the trivia
/// that was specified by the lexer at the token creation time.
///
/// For example:
///
/// ```no_test
/// builder.token_with_trivia(
/// RawSyntaxKind(1),
/// "\n\t /**/let \t\t",
/// &[TriviaPiece::whitespace(3), TriviaPiece::single_line_comment(4)],
/// &[TriviaPiece::whitespace(3)],
/// );
/// });
/// ```
/// This token has two pieces in the leading trivia, and one piece at the trailing trivia. Each
/// piece is defined by the [TriviaPiece]; its content is irrelevant.
///
#[derive(Clone)]
pub struct SyntaxTriviaPiece<L: Language> {
raw: cursor::SyntaxTrivia,
/// Absolute offset from the beginning of the file
offset: TextSize,
trivia: TriviaPiece,
_p: PhantomData<L>,
}
impl<L: Language> SyntaxTriviaPiece<L> {
pub(crate) fn into_raw_piece(self) -> TriviaPiece {
self.trivia
}
/// Returns the internal kind of this trivia piece
pub fn kind(&self) -> TriviaPieceKind {
self.trivia.kind()
}
/// Returns the associated text just for this trivia piece. This is different from [SyntaxTrivia::text()],
/// which returns the text of the whole trivia.
///
/// ```
/// use ruff_rowan::raw_language::{RawLanguage, RawLanguageKind, RawSyntaxTreeBuilder};
/// use ruff_rowan::*;
/// use std::iter::Iterator;
/// let mut node = RawSyntaxTreeBuilder::wrap_with_node(RawLanguageKind::ROOT, |builder| {
/// builder.token_with_trivia(
/// RawLanguageKind::LET_TOKEN,
/// "\n\t /**/let \t\t",
/// &[
/// TriviaPiece::whitespace(3),
/// TriviaPiece::single_line_comment(4),
/// ],
/// &[TriviaPiece::whitespace(3)],
/// );
/// });
/// let leading: Vec<_> = node.first_leading_trivia().unwrap().pieces().collect();
/// assert_eq!("\n\t ", leading[0].text());
/// assert_eq!("/**/", leading[1].text());
///
/// let trailing: Vec<_> = node.last_trailing_trivia().unwrap().pieces().collect();
/// assert_eq!(" \t\t", trailing[0].text());
/// ```
pub fn text(&self) -> &str {
let token = self.raw.token();
let txt = token.text();
// Compute the offset relative to the token
let start = self.offset - token.text_range().start();
let end = start + self.text_len();
// Don't use self.raw.text(). It iterates over all pieces
&txt[start.into()..end.into()]
}
/// Returns the associated text length just for this trivia piece. This is different from `SyntaxTrivia::len()`,
/// which returns the text length of the whole trivia.
///
/// ```
/// use ruff_rowan::raw_language::{RawLanguage, RawLanguageKind, RawSyntaxTreeBuilder};
/// use ruff_rowan::*;
/// use std::iter::Iterator;
/// let mut node = RawSyntaxTreeBuilder::wrap_with_node(RawLanguageKind::ROOT, |builder| {
/// builder.token_with_trivia(
/// RawLanguageKind::LET_TOKEN,
/// "\n\t /**/let \t\t",
/// &[
/// TriviaPiece::whitespace(3),
/// TriviaPiece::single_line_comment(4),
/// ],
/// &[TriviaPiece::whitespace(3)],
/// );
/// });
/// let pieces: Vec<_> = node.first_leading_trivia().unwrap().pieces().collect();
/// assert_eq!(TextSize::from(3), pieces[0].text_len());
/// ```
pub fn text_len(&self) -> TextSize {
self.trivia.text_len()
}
/// Returns the associated text range just for this trivia piece. This is different from [SyntaxTrivia::text_range()],
/// which returns the text range of the whole trivia.
///
/// ```
/// use ruff_rowan::raw_language::{RawLanguage, RawLanguageKind, RawSyntaxTreeBuilder};
/// use ruff_rowan::*;
/// use std::iter::Iterator;
/// let mut node = RawSyntaxTreeBuilder::wrap_with_node(RawLanguageKind::ROOT, |builder| {
/// builder.token_with_trivia(
/// RawLanguageKind::LET_TOKEN,
/// "\n\t /**/let \t\t",
/// &[
/// TriviaPiece::whitespace(3),
/// TriviaPiece::single_line_comment(4),
/// ],
/// &[TriviaPiece::whitespace(3)],
/// );
/// });
/// let pieces: Vec<_> = node.first_leading_trivia().unwrap().pieces().collect();
/// assert_eq!(TextRange::new(0.into(), 3.into()), pieces[0].text_range());
/// ```
pub fn text_range(&self) -> TextRange {
TextRange::at(self.offset, self.text_len())
}
/// Returns true if this trivia piece is a [SyntaxTriviaPieceNewline].
///
/// ```
/// use ruff_rowan::raw_language::{RawLanguage, RawLanguageKind, RawSyntaxTreeBuilder};
/// use ruff_rowan::*;
/// use std::iter::Iterator;
/// let mut node = RawSyntaxTreeBuilder::wrap_with_node(RawLanguageKind::ROOT, |builder| {
/// builder.token_with_trivia(
/// RawLanguageKind::LET_TOKEN,
/// "\n\t/**/let",
/// &[
/// TriviaPiece::newline(1),
/// TriviaPiece::whitespace(1),
/// TriviaPiece::single_line_comment(4),
/// ],
/// &[],
/// );
/// });
/// let pieces: Vec<_> = node.first_leading_trivia().unwrap().pieces().collect();
/// assert!(pieces[0].is_newline())
/// ```
pub fn is_newline(&self) -> bool {
self.trivia.kind.is_newline()
}
/// Returns true if this trivia piece is a [SyntaxTriviaPieceWhitespace].
///
/// ```
/// use ruff_rowan::raw_language::{RawLanguage, RawLanguageKind, RawSyntaxTreeBuilder};
/// use ruff_rowan::*;
/// use std::iter::Iterator;
/// let mut node = RawSyntaxTreeBuilder::wrap_with_node(RawLanguageKind::ROOT, |builder| {
/// builder.token_with_trivia(
/// RawLanguageKind::LET_TOKEN,
/// "\n\t/**/let",
/// &[
/// TriviaPiece::newline(1),
/// TriviaPiece::whitespace(1),
/// TriviaPiece::single_line_comment(4),
/// ],
/// &[],
/// );
/// });
/// let pieces: Vec<_> = node.first_leading_trivia().unwrap().pieces().collect();
/// assert!(pieces[1].is_whitespace())
/// ```
pub fn is_whitespace(&self) -> bool {
self.trivia.kind.is_whitespace()
}
/// Returns true if this trivia piece is a [SyntaxTriviaPieceComments].
///
/// ```
/// use ruff_rowan::raw_language::{RawLanguage, RawLanguageKind, RawSyntaxTreeBuilder};
/// use ruff_rowan::*;
/// use std::iter::Iterator;
/// let mut node = RawSyntaxTreeBuilder::wrap_with_node(RawLanguageKind::ROOT, |builder| {
/// builder.token_with_trivia(
/// RawLanguageKind::LET_TOKEN,
/// "\n\t/**/let",
/// &[
/// TriviaPiece::newline(1),
/// TriviaPiece::whitespace(1),
/// TriviaPiece::single_line_comment(4),
/// ],
/// &[],
/// );
/// });
/// let pieces: Vec<_> = node.first_leading_trivia().unwrap().pieces().collect();
/// assert!(pieces[2].is_comments())
/// ```
pub const fn is_comments(&self) -> bool {
matches!(
self.trivia.kind,
TriviaPieceKind::SingleLineComment | TriviaPieceKind::MultiLineComment
)
}
/// Returns true if this trivia piece is a [SyntaxTriviaPieceSkipped].
pub fn is_skipped(&self) -> bool {
self.trivia.kind.is_skipped()
}
/// Cast this trivia piece to [SyntaxTriviaPieceNewline].
///
/// ```
/// use ruff_rowan::raw_language::{RawLanguage, RawLanguageKind, RawSyntaxTreeBuilder};
/// use ruff_rowan::*;
/// use std::iter::Iterator;
/// let mut node = RawSyntaxTreeBuilder::wrap_with_node(RawLanguageKind::ROOT, |builder| {
/// builder.token_with_trivia(
/// RawLanguageKind::LET_TOKEN,
/// "\n/**/let \t\t",
/// &[TriviaPiece::newline(1), TriviaPiece::single_line_comment(4)],
/// &[TriviaPiece::newline(3)],
/// );
/// });
/// let pieces: Vec<_> = node.first_leading_trivia().unwrap().pieces().collect();
/// let w = pieces[0].as_newline();
/// assert!(w.is_some());
/// let w = pieces[1].as_newline();
/// assert!(w.is_none());
/// ```
pub fn as_newline(&self) -> Option<SyntaxTriviaPieceNewline<L>> {
match &self.trivia.kind {
TriviaPieceKind::Newline => Some(SyntaxTriviaPieceNewline(self.clone())),
_ => None,
}
}
/// Cast this trivia piece to [SyntaxTriviaPieceWhitespace].
///
/// ```
/// use ruff_rowan::raw_language::{RawLanguage, RawLanguageKind, RawSyntaxTreeBuilder};
/// use ruff_rowan::*;
/// use std::iter::Iterator;
/// let mut node = RawSyntaxTreeBuilder::wrap_with_node(RawLanguageKind::ROOT, |builder| {
/// builder.token_with_trivia(
/// RawLanguageKind::LET_TOKEN,
/// "\t /**/let \t\t",
/// &[
/// TriviaPiece::whitespace(2),
/// TriviaPiece::single_line_comment(4),
/// ],
/// &[TriviaPiece::whitespace(3)],
/// );
/// });
/// let pieces: Vec<_> = node.first_leading_trivia().unwrap().pieces().collect();
/// let w = pieces[0].as_whitespace();
/// assert!(w.is_some());
/// let w = pieces[1].as_whitespace();
/// assert!(w.is_none());
/// ```
pub fn as_whitespace(&self) -> Option<SyntaxTriviaPieceWhitespace<L>> {
match &self.trivia.kind {
TriviaPieceKind::Whitespace => Some(SyntaxTriviaPieceWhitespace(self.clone())),
_ => None,
}
}
/// Cast this trivia piece to [SyntaxTriviaPieceComments].
///
/// ```
/// use ruff_rowan::raw_language::{RawLanguage, RawLanguageKind, RawSyntaxTreeBuilder};
/// use ruff_rowan::*;
/// use std::iter::Iterator;
/// let mut node = RawSyntaxTreeBuilder::wrap_with_node(RawLanguageKind::ROOT, |builder| {
/// builder.token_with_trivia(
/// RawLanguageKind::LET_TOKEN,
/// "\n\t /**/let \t\t",
/// &[
/// TriviaPiece::whitespace(3),
/// TriviaPiece::single_line_comment(4),
/// ],
/// &[TriviaPiece::whitespace(3)],
/// );
/// });
/// let pieces: Vec<_> = node.first_leading_trivia().unwrap().pieces().collect();
/// let w = pieces[0].as_comments();
/// assert!(w.is_none());
/// let w = pieces[1].as_comments();
/// assert!(w.is_some());
/// ```
pub fn as_comments(&self) -> Option<SyntaxTriviaPieceComments<L>> {
match &self.trivia.kind {
TriviaPieceKind::SingleLineComment | TriviaPieceKind::MultiLineComment => {
Some(SyntaxTriviaPieceComments(self.clone()))
}
_ => None,
}
}
/// Casts this piece to a skipped trivia piece.
pub fn as_skipped(&self) -> Option<SyntaxTriviaPieceSkipped<L>> {
match &self.trivia.kind {
TriviaPieceKind::Skipped => Some(SyntaxTriviaPieceSkipped(self.clone())),
_ => None,
}
}
pub fn token(&self) -> SyntaxToken<L> {
SyntaxToken::from(self.raw.token().clone())
}
}
impl<L: Language> fmt::Debug for SyntaxTriviaPiece<L> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self.trivia.kind {
TriviaPieceKind::Newline => write!(f, "Newline(")?,
TriviaPieceKind::Whitespace => write!(f, "Whitespace(")?,
TriviaPieceKind::SingleLineComment | TriviaPieceKind::MultiLineComment => {
write!(f, "Comments(")?
}
TriviaPieceKind::Skipped => write!(f, "Skipped(")?,
}
print_debug_str(self.text(), f)?;
write!(f, ")")
}
}
#[derive(Clone, PartialEq, Eq, Hash)]
pub struct SyntaxTrivia<L: Language> {
raw: cursor::SyntaxTrivia,
_p: PhantomData<L>,
}
#[derive(Clone)]
pub struct SyntaxTriviaPiecesIterator<L: Language> {
iter: cursor::SyntaxTriviaPiecesIterator,
_p: PhantomData<L>,
}
impl<L: Language> Iterator for SyntaxTriviaPiecesIterator<L> {
type Item = SyntaxTriviaPiece<L>;
fn next(&mut self) -> Option<Self::Item> {
let (offset, trivia) = self.iter.next()?;
Some(SyntaxTriviaPiece {
raw: self.iter.raw.clone(),
offset,
trivia,
_p: PhantomData,
})
}
fn size_hint(&self) -> (usize, Option<usize>) {
self.iter.size_hint()
}
}
impl<L: Language> DoubleEndedIterator for SyntaxTriviaPiecesIterator<L> {
fn next_back(&mut self) -> Option<Self::Item> {
let (offset, trivia) = self.iter.next_back()?;
Some(SyntaxTriviaPiece {
raw: self.iter.raw.clone(),
offset,
trivia,
_p: PhantomData,
})
}
}
impl<L: Language> ExactSizeIterator for SyntaxTriviaPiecesIterator<L> {}
impl<L: Language> SyntaxTrivia<L> {
pub(super) fn new(raw: cursor::SyntaxTrivia) -> Self {
Self {
raw,
_p: PhantomData,
}
}
/// Returns all [SyntaxTriviaPiece] of this trivia.
///
/// ```
/// use crate::*;
/// use ruff_rowan::raw_language::{RawLanguage, RawLanguageKind, RawSyntaxTreeBuilder};
/// use ruff_rowan::*;
/// use std::iter::Iterator;
/// let mut node = RawSyntaxTreeBuilder::wrap_with_node(RawLanguageKind::ROOT, |builder| {
/// builder.token_with_trivia(
/// RawLanguageKind::LET_TOKEN,
/// "\n\t /**/let \t\t",
/// &[
/// TriviaPiece::whitespace(3),
/// TriviaPiece::single_line_comment(4),
/// ],
/// &[TriviaPiece::whitespace(3)],
/// );
/// });
/// let pieces: Vec<_> = node.first_leading_trivia().unwrap().pieces().collect();
/// assert_eq!(2, pieces.len());
/// let pieces: Vec<_> = node.last_trailing_trivia().unwrap().pieces().collect();
/// assert_eq!(1, pieces.len());
/// ```
pub fn pieces(&self) -> SyntaxTriviaPiecesIterator<L> {
SyntaxTriviaPiecesIterator {
iter: self.raw.pieces(),
_p: PhantomData,
}
}
pub fn last(&self) -> Option<SyntaxTriviaPiece<L>> {
let piece = self.raw.last()?;
Some(SyntaxTriviaPiece {
raw: self.raw.clone(),
offset: self.raw.text_range().end() - piece.length,
trivia: *piece,
_p: Default::default(),
})
}
pub fn first(&self) -> Option<SyntaxTriviaPiece<L>> {
let piece = self.raw.first()?;
Some(SyntaxTriviaPiece {
raw: self.raw.clone(),
offset: self.raw.text_range().start(),
trivia: *piece,
_p: Default::default(),
})
}
pub fn text(&self) -> &str {
self.raw.text()
}
pub fn text_range(&self) -> TextRange {
self.raw.text_range()
}
pub fn is_empty(&self) -> bool {
self.raw.len() == 0
}
pub fn has_skipped(&self) -> bool {
self.pieces().any(|piece| piece.is_skipped())
}
}
fn print_debug_str<S: AsRef<str>>(text: S, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let text = text.as_ref();
if text.len() < 25 {
write!(f, "{:?}", text)
} else {
for idx in 21..25 {
if text.is_char_boundary(idx) {
let text = format!("{} ...", &text[..idx]);
return write!(f, "{:?}", text);
}
}
write!(f, "")
}
}
impl<L: Language> std::fmt::Debug for SyntaxTrivia<L> {
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
write!(f, "[")?;
let mut first_piece = true;
for piece in self.pieces() {
if !first_piece {
write!(f, ", ")?;
}
first_piece = false;
write!(f, "{:?}", piece)?;
}
write!(f, "]")
}
}
/// It creates an iterator by chaining two trivia pieces. This iterator
/// of trivia can be attached to a token using `*_pieces` APIs.
///
/// ## Examples
///
/// ```
/// use ruff_rowan::raw_language::{RawLanguage, RawLanguageKind, RawSyntaxTreeBuilder};
/// use ruff_rowan::{chain_trivia_pieces, RawSyntaxToken, SyntaxToken, TriviaPiece, TriviaPieceKind};
///
/// let first_token = SyntaxToken::<RawLanguage>::new_detached(
/// RawLanguageKind::LET_TOKEN,
/// "\n\t let \t\t",
/// [TriviaPiece::whitespace(3)],
/// [TriviaPiece::whitespace(3)]
/// );
/// let second_token = SyntaxToken::<RawLanguage>::new_detached(
/// RawLanguageKind::SEMICOLON_TOKEN,
/// "; \t\t",
/// [TriviaPiece::whitespace(1)],
/// [TriviaPiece::whitespace(1)],
/// );
///
/// let leading_trivia = chain_trivia_pieces(
/// first_token.leading_trivia().pieces(),
/// second_token.leading_trivia().pieces()
/// );
///
/// let new_first_token = first_token.with_leading_trivia_pieces(leading_trivia);
///
/// let new_token = format!("{:?}", new_first_token);
/// assert_eq!(new_token, "LET_TOKEN@0..10 \"let\" [Whitespace(\"\\n\\t \"), Whitespace(\";\")] [Whitespace(\" \\t\\t\")]");
///
/// ```
///
pub fn chain_trivia_pieces<L, F, S>(first: F, second: S) -> ChainTriviaPiecesIterator<F, S>
where
L: Language,
F: Iterator<Item = SyntaxTriviaPiece<L>>,
S: Iterator<Item = SyntaxTriviaPiece<L>>,
{
ChainTriviaPiecesIterator::new(first, second)
}
/// Chain iterator that chains two iterators over syntax trivia together.
///
/// This is the same as Rust's [std::iter::Chain] iterator but implements [ExactSizeIterator].
/// Rust doesn't implement [ExactSizeIterator] because adding the sizes of both pieces may overflow.
///
/// Implementing [ExactSizeIterator] in our case is safe because this may only overflow if
/// a source document has more than 2^32 trivia which isn't possible because our source documents are limited to 2^32
/// length.
pub struct ChainTriviaPiecesIterator<F, S> {
first: Option<F>,
second: S,
}
impl<F, S> ChainTriviaPiecesIterator<F, S> {
fn new(first: F, second: S) -> Self {
Self {
first: Some(first),
second,
}
}
}
impl<L, F, S> Iterator for ChainTriviaPiecesIterator<F, S>
where
L: Language,
F: Iterator<Item = SyntaxTriviaPiece<L>>,
S: Iterator<Item = SyntaxTriviaPiece<L>>,
{
type Item = SyntaxTriviaPiece<L>;
fn next(&mut self) -> Option<Self::Item> {
match &mut self.first {
Some(first) => match first.next() {
Some(next) => Some(next),
None => {
self.first.take();
self.second.next()
}
},
None => self.second.next(),
}
}
fn size_hint(&self) -> (usize, Option<usize>) {
match &self.first {
Some(first) => {
let (first_lower, first_upper) = first.size_hint();
let (second_lower, second_upper) = self.second.size_hint();
let lower = first_lower.saturating_add(second_lower);
let upper = match (first_upper, second_upper) {
(Some(first), Some(second)) => first.checked_add(second),
_ => None,
};
(lower, upper)
}
None => self.second.size_hint(),
}
}
}
impl<L, F, S> FusedIterator for ChainTriviaPiecesIterator<F, S>
where
L: Language,
F: Iterator<Item = SyntaxTriviaPiece<L>>,
S: Iterator<Item = SyntaxTriviaPiece<L>>,
{
}
impl<L, F, S> ExactSizeIterator for ChainTriviaPiecesIterator<F, S>
where
L: Language,
F: ExactSizeIterator<Item = SyntaxTriviaPiece<L>>,
S: ExactSizeIterator<Item = SyntaxTriviaPiece<L>>,
{
fn len(&self) -> usize {
match &self.first {
Some(first) => {
let first_len = first.len();
let second_len = self.second.len();
// SAFETY: Should be safe because a program can never contain more than u32 pieces
// because the text ranges are represented as u32 (and each piece must at least contain a single character).
first_len + second_len
}
None => self.second.len(),
}
}
}

275
crates/ruff_rowan/src/syntax_factory.rs Normal file
View File

@ -0,0 +1,275 @@
mod parsed_children;
mod raw_syntax;
use crate::SyntaxKind;
use std::fmt;
use std::iter::{FusedIterator, Peekable};
pub use self::parsed_children::{
ParsedChildren, ParsedChildrenIntoIterator, ParsedChildrenIterator,
};
pub use self::raw_syntax::{
RawSyntaxElement, RawSyntaxElementRef, RawSyntaxNode, RawSyntaxNodeRef, RawSyntaxToken,
RawSyntaxTokenRef,
};
/// Factory for creating syntax nodes of a particular kind.
pub trait SyntaxFactory: fmt::Debug {
/// The syntax kind used by the nodes constructed by this syntax factory.
type Kind: SyntaxKind;
/// Creates a new syntax node of the passed `kind` with the given children.
///
/// The `children` contains the parsed direct children of the node. There may be fewer children
/// in case there's a syntax error and a required child or an optional child isn't present in the source code.
/// The `make_syntax` implementation must then fill in empty slots to match the slots as they're defined in the grammar.
///
/// The implementation is free to change the `kind` of the node but that has the consequence that
/// such a node will not be cached. The reason for not caching these nodes is that the cache lookup is performed
/// before calling `make_syntax`, thus querying the cache with the old kind.
///
/// It's important that the factory function is idempotent, meaning, calling the function
/// multiple times with the same `kind` and `children` returns syntax nodes with the same structure.
/// This is important because the returned nodes may be cached by `kind` and what `children` are present.
fn make_syntax(
kind: Self::Kind,
children: ParsedChildren<Self::Kind>,
) -> RawSyntaxNode<Self::Kind>;
/// Crates a *node list* syntax node. Validates if all elements are valid and changes the node's kind to
/// [SyntaxKind::to_bogus] if that's not the case.
fn make_node_list_syntax<F>(
kind: Self::Kind,
children: ParsedChildren<Self::Kind>,
can_cast: F,
) -> RawSyntaxNode<Self::Kind>
where
F: Fn(Self::Kind) -> bool,
{
let valid = (&children)
.into_iter()
.all(|element| can_cast(element.kind()));
let kind = if valid { kind } else { kind.to_bogus() };
RawSyntaxNode::new(kind, children.into_iter().map(Some))
}
/// Creates a *separated list* syntax node. Validates if the elements are valid, are correctly
/// separated by the specified separator token.
///
/// It changes the kind of the node to [SyntaxKind::to_bogus] if an element isn't a valid list-node
/// nor separator.
///
/// It inserts empty slots for missing elements or missing markers
fn make_separated_list_syntax<F>(
kind: Self::Kind,
children: ParsedChildren<Self::Kind>,
can_cast: F,
separator: Self::Kind,
allow_trailing: bool,
) -> RawSyntaxNode<Self::Kind>
where
F: Fn(Self::Kind) -> bool,
{
let mut next_node = true;
let mut missing_count = 0;
let mut valid = true;
for child in &children {
let kind = child.kind();
if next_node {
if can_cast(kind) {
next_node = false;
} else if kind == separator {
// a missing element
missing_count += 1;
} else {
// an invalid element
valid = false;
break;
}
} else if kind == separator {
next_node = true;
} else if can_cast(kind) {
// a missing separator
missing_count += 1;
} else {
// something unexpected
valid = false;
}
}
if next_node && !allow_trailing && !children.is_empty() {
// a trailing comma in a list that doesn't support trailing commas
missing_count += 1;
}
if !valid {
RawSyntaxNode::new(kind.to_bogus(), children.into_iter().map(Some))
} else if missing_count > 0 {
RawSyntaxNode::new(
kind,
SeparatedListWithMissingNodesOrSeparatorSlotsIterator {
inner: children.into_iter().peekable(),
missing_count,
next_node: true,
separator,
},
)
} else {
RawSyntaxNode::new(kind, children.into_iter().map(Some))
}
}
}
/// Iterator that "fixes up" a separated list by inserting empty slots for any missing
/// separator or element.
struct SeparatedListWithMissingNodesOrSeparatorSlotsIterator<'a, K: SyntaxKind> {
inner: Peekable<ParsedChildrenIntoIterator<'a, K>>,
missing_count: usize,
next_node: bool,
separator: K,
}
impl<'a, K: SyntaxKind> Iterator for SeparatedListWithMissingNodesOrSeparatorSlotsIterator<'a, K> {
type Item = Option<RawSyntaxElement<K>>;
#[cold]
fn next(&mut self) -> Option<Self::Item> {
let peeked = self.inner.peek();
if let Some(peeked) = peeked {
let is_separator = self.separator == peeked.kind();
if self.next_node {
self.next_node = false;
if !is_separator {
Some(self.inner.next())
} else {
self.missing_count -= 1;
Some(None) // Missing separator
}
} else if is_separator {
self.next_node = true;
Some(self.inner.next())
} else {
// Missing node
self.missing_count -= 1;
self.next_node = true;
Some(None)
}
} else if self.missing_count > 0 {
// at a trailing comma in a list that doesn't allow trailing commas.
self.missing_count -= 1;
Some(None)
} else {
None
}
}
fn size_hint(&self) -> (usize, Option<usize>) {
let len = self.len();
(len, Some(len))
}
}
impl<'a, K: SyntaxKind> FusedIterator
for SeparatedListWithMissingNodesOrSeparatorSlotsIterator<'a, K>
{
}
impl<'a, K: SyntaxKind> ExactSizeIterator
for SeparatedListWithMissingNodesOrSeparatorSlotsIterator<'a, K>
{
fn len(&self) -> usize {
self.inner.len() + self.missing_count
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum SlotContent {
Present,
Absent,
}
/// Description of the slots of a node in combination with [ParsedChildren].
/// It stores for each slot if the node is present in [ParsedChildren] or not, allowing
/// to generate a node with the right number of empty slots.
#[derive(Debug)]
pub struct RawNodeSlots<const COUNT: usize> {
slots: [SlotContent; COUNT],
current_slot: usize,
}
impl<const COUNT: usize> Default for RawNodeSlots<COUNT> {
fn default() -> Self {
Self {
slots: [SlotContent::Absent; COUNT],
current_slot: 0,
}
}
}
impl<const COUNT: usize> RawNodeSlots<COUNT> {
/// Progresses to the next slot
pub fn next_slot(&mut self) {
debug_assert!(self.current_slot < COUNT);
self.current_slot += 1;
}
/// Marks that the node for the current slot is *present* in the source code.
pub fn mark_present(&mut self) {
debug_assert!(self.current_slot < COUNT);
self.slots[self.current_slot] = SlotContent::Present;
}
/// Creates a node with the kind `kind`, filling in the nodes from the `children`.
pub fn into_node<K: SyntaxKind>(
self,
kind: K,
children: ParsedChildren<K>,
) -> RawSyntaxNode<K> {
debug_assert!(self.current_slot == COUNT, "Missing slots");
RawSyntaxNode::new(
kind,
RawNodeSlotIterator {
children: children.into_iter(),
slots: self.slots.as_slice().iter(),
},
)
}
}
struct RawNodeSlotIterator<'a, K: SyntaxKind> {
children: ParsedChildrenIntoIterator<'a, K>,
slots: std::slice::Iter<'a, SlotContent>,
}
impl<'a, K: SyntaxKind> Iterator for RawNodeSlotIterator<'a, K> {
type Item = Option<RawSyntaxElement<K>>;
fn next(&mut self) -> Option<Self::Item> {
let slot = self.slots.next()?;
match slot {
SlotContent::Present => {
Some(Some(self.children.next().expect(
"Expected a present node according to the slot description",
)))
}
SlotContent::Absent => Some(None),
}
}
fn size_hint(&self) -> (usize, Option<usize>) {
(self.slots.len(), Some(self.slots.len()))
}
}
impl<'a, K: SyntaxKind> FusedIterator for RawNodeSlotIterator<'a, K> {}
impl<'a, K: SyntaxKind> ExactSizeIterator for RawNodeSlotIterator<'a, K> {}

132
crates/ruff_rowan/src/syntax_factory/parsed_children.rs Normal file
View File

@ -0,0 +1,132 @@
use crate::green::GreenElement;
use crate::syntax_factory::raw_syntax::{RawSyntaxElement, RawSyntaxElementRef};
use crate::SyntaxKind;
use std::iter::FusedIterator;
use std::marker::PhantomData;
/// The parsed children of a node, not accounting for any missing children (required or optional)
#[derive(Debug)]
pub struct ParsedChildren<'a, K> {
/// Reference to an array containing all children of this node or any of its parents
all_children: &'a mut Vec<(u64, GreenElement)>,
/// The index of the first child of this node in the `all_children` array
first_child: usize,
ph: PhantomData<K>,
}
impl<'a, K: SyntaxKind> ParsedChildren<'a, K> {
pub(crate) fn new(all_children: &'a mut Vec<(u64, GreenElement)>, first_child: usize) -> Self {
Self {
all_children,
first_child,
ph: PhantomData,
}
}
pub fn len(&self) -> usize {
(self.first_child..self.all_children.len()).len()
}
pub fn is_empty(&self) -> bool {
self.len() == 0
}
}
impl<'a, K: SyntaxKind> IntoIterator for ParsedChildren<'a, K> {
type Item = RawSyntaxElement<K>;
type IntoIter = ParsedChildrenIntoIterator<'a, K>;
fn into_iter(self) -> Self::IntoIter {
ParsedChildrenIntoIterator {
inner: self.all_children.drain(self.first_child..),
ph: PhantomData,
}
}
}
#[derive(Debug)]
pub struct ParsedChildrenIntoIterator<'a, K> {
inner: std::vec::Drain<'a, (u64, GreenElement)>,
ph: PhantomData<K>,
}
impl<'a, K: SyntaxKind> Iterator for ParsedChildrenIntoIterator<'a, K> {
type Item = RawSyntaxElement<K>;
fn next(&mut self) -> Option<Self::Item> {
self.inner
.next()
.map(|(_, raw)| RawSyntaxElement::from(raw))
}
fn size_hint(&self) -> (usize, Option<usize>) {
let len = self.len();
(len, Some(len))
}
}
impl<'a, K: SyntaxKind> FusedIterator for ParsedChildrenIntoIterator<'a, K> {}
impl<'a, K: SyntaxKind> ExactSizeIterator for ParsedChildrenIntoIterator<'a, K> {
fn len(&self) -> usize {
self.inner.len()
}
}
impl<'a, K: SyntaxKind> DoubleEndedIterator for ParsedChildrenIntoIterator<'a, K> {
fn next_back(&mut self) -> Option<Self::Item> {
self.inner
.next_back()
.map(|(_, raw)| RawSyntaxElement::from(raw))
}
}
impl<'a, K: SyntaxKind> IntoIterator for &'a ParsedChildren<'a, K> {
type Item = RawSyntaxElementRef<'a, K>;
type IntoIter = ParsedChildrenIterator<'a, K>;
fn into_iter(self) -> Self::IntoIter {
ParsedChildrenIterator {
inner: self.all_children[self.first_child..].iter(),
ph: PhantomData,
}
}
}
#[derive(Debug)]
pub struct ParsedChildrenIterator<'a, K> {
inner: std::slice::Iter<'a, (u64, GreenElement)>,
ph: PhantomData<K>,
}
impl<'a, K: SyntaxKind> Iterator for ParsedChildrenIterator<'a, K> {
type Item = RawSyntaxElementRef<'a, K>;
fn next(&mut self) -> Option<Self::Item> {
self.inner
.next()
.map(|(_, raw)| RawSyntaxElementRef::from(raw))
}
fn size_hint(&self) -> (usize, Option<usize>) {
let len = self.len();
(len, Some(len))
}
}
impl<'a, K: SyntaxKind> FusedIterator for ParsedChildrenIterator<'a, K> {}
impl<'a, K: SyntaxKind> ExactSizeIterator for ParsedChildrenIterator<'a, K> {
fn len(&self) -> usize {
self.inner.len()
}
}
impl<'a, K: SyntaxKind> DoubleEndedIterator for ParsedChildrenIterator<'a, K> {
fn next_back(&mut self) -> Option<Self::Item> {
self.inner
.next_back()
.map(|(_, raw)| RawSyntaxElementRef::from(raw))
}
}

191
crates/ruff_rowan/src/syntax_factory/raw_syntax.rs Normal file
View File

@ -0,0 +1,191 @@
use crate::green::GreenElement;
use crate::{GreenNode, GreenToken, NodeOrToken, SyntaxKind};
use std::marker::PhantomData;
/// New-type wrapper around a `GreenNode`.
///
/// Allows third-party crates to access limited information about a `GreenNode` or construct
/// a `GreenNode` in a limited places.
#[derive(Debug)]
pub struct RawSyntaxNode<K: SyntaxKind> {
raw: GreenNode,
ph: PhantomData<K>,
}
impl<K: SyntaxKind> RawSyntaxNode<K> {
/// Creates a new node with the given `kind` and `slots`.
#[inline]
pub fn new<I>(kind: K, slots: I) -> Self
where
I: IntoIterator<Item = Option<RawSyntaxElement<K>>>,
I::IntoIter: ExactSizeIterator,
{
Self {
raw: GreenNode::new(
kind.to_raw(),
slots
.into_iter()
.map(|slot| slot.map(|element| element.into_green())),
),
ph: PhantomData,
}
}
#[inline]
pub fn kind(&self) -> K {
K::from_raw(self.raw.kind())
}
/// Unwraps this raw syntax into it's underlying green node.
#[inline]
pub(crate) fn into_green(self) -> GreenNode {
self.raw
}
}
impl<K: SyntaxKind> From<GreenNode> for RawSyntaxNode<K> {
#[inline]
fn from(node: GreenNode) -> Self {
Self {
raw: node,
ph: PhantomData,
}
}
}
/// New-type wrapper around a `GreenToken`. Allows third-party crates to access limited information
/// on not yet fully constructed nodes.
#[derive(Debug)]
pub struct RawSyntaxToken<K: SyntaxKind> {
raw: GreenToken,
ph: PhantomData<K>,
}
impl<K: SyntaxKind> RawSyntaxToken<K> {
#[inline]
pub fn kind(&self) -> K {
K::from_raw(self.raw.kind())
}
}
impl<K: SyntaxKind> From<GreenToken> for RawSyntaxToken<K> {
fn from(token: GreenToken) -> Self {
Self {
raw: token,
ph: PhantomData,
}
}
}
pub type RawSyntaxElement<K> = NodeOrToken<RawSyntaxNode<K>, RawSyntaxToken<K>>;
impl<K: SyntaxKind> RawSyntaxElement<K> {
#[inline]
pub fn kind(&self) -> K {
match self {
NodeOrToken::Node(node) => node.kind(),
NodeOrToken::Token(token) => token.kind(),
}
}
#[inline]
fn into_green(self) -> GreenElement {
match self {
NodeOrToken::Node(node) => NodeOrToken::Node(node.raw),
NodeOrToken::Token(token) => NodeOrToken::Token(token.raw),
}
}
}
impl<K: SyntaxKind> From<GreenElement> for RawSyntaxElement<K> {
#[inline]
fn from(element: GreenElement) -> Self {
match element {
NodeOrToken::Node(node) => NodeOrToken::Node(RawSyntaxNode::from(node)),
NodeOrToken::Token(token) => NodeOrToken::Token(RawSyntaxToken::from(token)),
}
}
}
/// New-type wrapper to a reference of a `GreenNode`.
#[derive(Debug)]
pub struct RawSyntaxNodeRef<'a, K: SyntaxKind> {
raw: &'a GreenNode,
ph: PhantomData<K>,
}
impl<'a, K: SyntaxKind> RawSyntaxNodeRef<'a, K> {
#[inline]
pub fn kind(&self) -> K {
K::from_raw(self.raw.kind())
}
}
impl<'a, K: SyntaxKind> From<&'a GreenNode> for RawSyntaxNodeRef<'a, K> {
#[inline]
fn from(node: &'a GreenNode) -> Self {
Self {
raw: node,
ph: PhantomData,
}
}
}
/// New-type wrapper to a reference of a `GreenToken`
#[derive(Debug)]
pub struct RawSyntaxTokenRef<'a, K: SyntaxKind> {
raw: &'a GreenToken,
ph: PhantomData<K>,
}
impl<'a, K: SyntaxKind> RawSyntaxTokenRef<'a, K> {
#[inline]
pub fn kind(&self) -> K {
K::from_raw(self.raw.kind())
}
}
impl<'a, K: SyntaxKind> From<&'a GreenToken> for RawSyntaxTokenRef<'a, K> {
#[inline]
fn from(token: &'a GreenToken) -> Self {
Self {
raw: token,
ph: PhantomData,
}
}
}
pub type RawSyntaxElementRef<'a, K> =
NodeOrToken<RawSyntaxNodeRef<'a, K>, RawSyntaxTokenRef<'a, K>>;
impl<'a, K: SyntaxKind> RawSyntaxElementRef<'a, K> {
#[inline]
pub fn kind(&self) -> K {
match self {
NodeOrToken::Node(node) => node.kind(),
NodeOrToken::Token(token) => token.kind(),
}
}
}
impl<'a, K: SyntaxKind> From<NodeOrToken<&'a GreenNode, &'a GreenToken>>
for RawSyntaxElementRef<'a, K>
{
#[inline]
fn from(element: NodeOrToken<&'a GreenNode, &'a GreenToken>) -> Self {
match element {
NodeOrToken::Node(node) => NodeOrToken::Node(RawSyntaxNodeRef::from(node)),
NodeOrToken::Token(token) => NodeOrToken::Token(RawSyntaxTokenRef::from(token)),
}
}
}
impl<'a, K: SyntaxKind> From<&'a GreenElement> for RawSyntaxElementRef<'a, K> {
#[inline]
fn from(element: &'a GreenElement) -> Self {
match element {
NodeOrToken::Node(node) => NodeOrToken::Node(RawSyntaxNodeRef::from(node)),
NodeOrToken::Token(token) => NodeOrToken::Token(RawSyntaxTokenRef::from(token)),
}
}
}

444
crates/ruff_rowan/src/syntax_node_text.rs Normal file
View File

@ -0,0 +1,444 @@
use crate::{
cursor::{SyntaxNode, SyntaxToken},
TextRange, TextSize, TokenAtOffset,
};
use ruff_text_size::TextLen;
use std::fmt;
use std::iter::FusedIterator;
#[derive(Clone)]
pub struct SyntaxNodeText {
node: SyntaxNode,
range: TextRange,
}
impl SyntaxNodeText {
pub(crate) fn new(node: SyntaxNode) -> SyntaxNodeText {
let range = node.text_range();
SyntaxNodeText { node, range }
}
pub(crate) fn with_range(node: SyntaxNode, range: TextRange) -> SyntaxNodeText {
SyntaxNodeText { node, range }
}
pub fn len(&self) -> TextSize {
self.range.len()
}
pub fn is_empty(&self) -> bool {
self.range.is_empty()
}
pub fn contains_char(&self, c: char) -> bool {
self.try_for_each_chunk(|chunk| if chunk.contains(c) { Err(()) } else { Ok(()) })
.is_err()
}
pub fn find_char(&self, c: char) -> Option<TextSize> {
let mut acc: TextSize = 0.into();
let res = self.try_for_each_chunk(|chunk| {
if let Some(pos) = chunk.find(c) {
let pos: TextSize = (pos as u32).into();
return Err(acc + pos);
}
acc += TextSize::of(chunk);
Ok(())
});
found(res)
}
pub fn char_at(&self, offset: TextSize) -> Option<char> {
let mut start: TextSize = 0.into();
let res = self.try_for_each_chunk(|chunk| {
let end = start + TextSize::of(chunk);
if start <= offset && offset < end {
let off: usize = u32::from(offset - start) as usize;
return Err(chunk[off..].chars().next().unwrap());
}
start = end;
Ok(())
});
found(res)
}
pub fn slice<R: private::SyntaxTextRange>(&self, range: R) -> SyntaxNodeText {
let start = range.start().unwrap_or_default();
let end = range.end().unwrap_or_else(|| self.len());
assert!(start <= end);
let len = end - start;
let start = self.range.start() + start;
let end = start + len;
assert!(
start <= end,
"invalid slice, range: {:?}, slice: {:?}",
self.range,
(range.start(), range.end()),
);
let range = TextRange::new(start, end);
assert!(
self.range.contains_range(range),
"invalid slice, range: {:?}, slice: {:?}",
self.range,
range,
);
SyntaxNodeText {
node: self.node.clone(),
range,
}
}
pub fn try_fold_chunks<T, F, E>(&self, init: T, mut f: F) -> Result<T, E>
where
F: FnMut(T, &str) -> Result<T, E>,
{
self.tokens_with_ranges()
.try_fold(init, move |acc, (token, range)| {
f(acc, &token.text()[range])
})
}
pub fn try_for_each_chunk<F: FnMut(&str) -> Result<(), E>, E>(
&self,
mut f: F,
) -> Result<(), E> {
self.try_fold_chunks((), move |(), chunk| f(chunk))
}
pub fn for_each_chunk<F: FnMut(&str)>(&self, mut f: F) {
enum Void {}
match self.try_for_each_chunk(|chunk| {
f(chunk);
Ok::<(), Void>(())
}) {
Ok(()) => (),
Err(void) => match void {},
}
}
fn tokens_with_ranges(&self) -> impl Iterator<Item = (SyntaxToken, TextRange)> + FusedIterator {
SyntaxNodeTokenWithRanges::new(self)
}
pub fn chars(&self) -> impl Iterator<Item = char> + FusedIterator {
SyntaxNodeTextChars::new(self)
}
}
#[derive(Clone)]
struct SyntaxNodeTokenWithRanges {
text_range: TextRange,
next_token: Option<(SyntaxToken, TextRange)>,
}
impl SyntaxNodeTokenWithRanges {
fn new(text: &SyntaxNodeText) -> Self {
let text_range = text.range;
let token = match text.node.token_at_offset(text_range.start()) {
TokenAtOffset::None => None,
TokenAtOffset::Single(token) => Some(token),
TokenAtOffset::Between(_, next) => Some(next),
};
Self {
next_token: token.and_then(|token| Self::with_intersecting_range(token, text_range)),
text_range,
}
}
fn with_intersecting_range(
token: SyntaxToken,
text_range: TextRange,
) -> Option<(SyntaxToken, TextRange)> {
let token_range = token.text_range();
let range = text_range.intersect(token_range)?;
Some((token, range - token_range.start()))
}
}
impl Iterator for SyntaxNodeTokenWithRanges {
type Item = (SyntaxToken, TextRange);
fn next(&mut self) -> Option<Self::Item> {
let (token, range) = self.next_token.take()?;
self.next_token = token
.next_token()
.and_then(|token| Self::with_intersecting_range(token, self.text_range));
Some((token, range))
}
}
impl FusedIterator for SyntaxNodeTokenWithRanges {}
#[derive(Clone)]
struct SyntaxNodeTextChars {
head: Option<(SyntaxToken, TextRange)>,
tail: SyntaxNodeTokenWithRanges,
index: TextSize,
}
impl SyntaxNodeTextChars {
fn new(text: &SyntaxNodeText) -> Self {
let mut chunks = SyntaxNodeTokenWithRanges::new(text);
Self {
head: chunks.next(),
tail: chunks,
index: TextSize::default(),
}
}
}
impl Iterator for SyntaxNodeTextChars {
type Item = char;
fn next(&mut self) -> Option<Self::Item> {
loop {
let (token, range) = self.head.as_ref()?;
if self.index >= range.end() {
self.head = self.tail.next();
self.index = TextSize::default();
continue;
}
let text = token.text();
// SAFETY: Index check above guarantees that there's at least some text left
let next_char = text[TextRange::new(self.index, range.end())]
.chars()
.next()
.unwrap();
self.index += next_char.text_len();
break Some(next_char);
}
}
}
impl FusedIterator for SyntaxNodeTextChars {}
fn found<T>(res: Result<(), T>) -> Option<T> {
match res {
Ok(()) => None,
Err(it) => Some(it),
}
}
impl fmt::Debug for SyntaxNodeText {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
fmt::Debug::fmt(&self.to_string(), f)
}
}
impl fmt::Display for SyntaxNodeText {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
self.try_for_each_chunk(|chunk| fmt::Display::fmt(chunk, f))
}
}
impl From<SyntaxNodeText> for String {
fn from(text: SyntaxNodeText) -> String {
text.to_string()
}
}
impl PartialEq<str> for SyntaxNodeText {
fn eq(&self, mut rhs: &str) -> bool {
self.try_for_each_chunk(|chunk| {
if !rhs.starts_with(chunk) {
return Err(());
}
rhs = &rhs[chunk.len()..];
Ok(())
})
.is_ok()
&& rhs.is_empty()
}
}
impl PartialEq<SyntaxNodeText> for str {
fn eq(&self, rhs: &SyntaxNodeText) -> bool {
rhs == self
}
}
impl PartialEq<&'_ str> for SyntaxNodeText {
fn eq(&self, rhs: &&str) -> bool {
self == *rhs
}
}
impl PartialEq<SyntaxNodeText> for &'_ str {
fn eq(&self, rhs: &SyntaxNodeText) -> bool {
rhs == self
}
}
impl PartialEq for SyntaxNodeText {
fn eq(&self, other: &SyntaxNodeText) -> bool {
if self.range.len() != other.range.len() {
return false;
}
let mut lhs = self.tokens_with_ranges();
let mut rhs = other.tokens_with_ranges();
zip_texts(&mut lhs, &mut rhs).is_none()
&& lhs.all(|it| it.1.is_empty())
&& rhs.all(|it| it.1.is_empty())
}
}
fn zip_texts<I: Iterator<Item = (SyntaxToken, TextRange)>>(xs: &mut I, ys: &mut I) -> Option<()> {
let mut x = xs.next()?;
let mut y = ys.next()?;
loop {
while x.1.is_empty() {
x = xs.next()?;
}
while y.1.is_empty() {
y = ys.next()?;
}
let x_text = &x.0.text()[x.1];
let y_text = &y.0.text()[y.1];
if !(x_text.starts_with(y_text) || y_text.starts_with(x_text)) {
return Some(());
}
let advance = std::cmp::min(x.1.len(), y.1.len());
x.1 = TextRange::new(x.1.start() + advance, x.1.end());
y.1 = TextRange::new(y.1.start() + advance, y.1.end());
}
}
impl Eq for SyntaxNodeText {}
mod private {
use std::ops;
use crate::{TextRange, TextSize};
pub trait SyntaxTextRange {
fn start(&self) -> Option<TextSize>;
fn end(&self) -> Option<TextSize>;
}
impl SyntaxTextRange for TextRange {
fn start(&self) -> Option<TextSize> {
Some(TextRange::start(*self))
}
fn end(&self) -> Option<TextSize> {
Some(TextRange::end(*self))
}
}
impl SyntaxTextRange for ops::Range<TextSize> {
fn start(&self) -> Option<TextSize> {
Some(self.start)
}
fn end(&self) -> Option<TextSize> {
Some(self.end)
}
}
impl SyntaxTextRange for ops::RangeFrom<TextSize> {
fn start(&self) -> Option<TextSize> {
Some(self.start)
}
fn end(&self) -> Option<TextSize> {
None
}
}
impl SyntaxTextRange for ops::RangeTo<TextSize> {
fn start(&self) -> Option<TextSize> {
None
}
fn end(&self) -> Option<TextSize> {
Some(self.end)
}
}
impl SyntaxTextRange for ops::RangeFull {
fn start(&self) -> Option<TextSize> {
None
}
fn end(&self) -> Option<TextSize> {
None
}
}
}
#[cfg(test)]
mod tests {
use crate::raw_language::{RawLanguage, RawLanguageKind, RawSyntaxTreeBuilder};
use crate::SyntaxNode;
fn build_tree(chunks: &[&str]) -> SyntaxNode<RawLanguage> {
let mut builder = RawSyntaxTreeBuilder::new();
builder.start_node(RawLanguageKind::ROOT);
for &chunk in chunks.iter() {
builder.token(RawLanguageKind::STRING_TOKEN, chunk);
}
builder.finish_node();
builder.finish()
}
#[test]
fn test_text_equality() {
fn do_check(t1: &[&str], t2: &[&str]) {
let t1 = build_tree(t1).text();
let t2 = build_tree(t2).text();
let expected = t1.to_string() == t2.to_string();
let actual = t1 == t2;
assert_eq!(
expected, actual,
"`{}` (SyntaxText) `{}` (SyntaxText)",
t1, t2
);
let actual = t1 == *t2.to_string();
assert_eq!(expected, actual, "`{}` (SyntaxText) `{}` (&str)", t1, t2);
}
fn check(t1: &[&str], t2: &[&str]) {
do_check(t1, t2);
do_check(t2, t1)
}
check(&[""], &[""]);
check(&["a"], &[""]);
check(&["a"], &["a"]);
check(&["abc"], &["def"]);
check(&["hello", "world"], &["hello", "world"]);
check(&["hellowo", "rld"], &["hell", "oworld"]);
check(&["hel", "lowo", "rld"], &["helloworld"]);
check(&["{", "abc", "}"], &["{", "123", "}"]);
check(&["{", "abc", "}", "{"], &["{", "123", "}"]);
check(&["{", "abc", "}"], &["{", "123", "}", "{"]);
check(&["{", "abc", "}ab"], &["{", "abc", "}", "ab"]);
}
#[test]
fn test_chars() {
fn check(t1: &[&str], expected: &str) {
let t1 = build_tree(t1).text();
let actual = t1.chars().collect::<String>();
assert_eq!(
expected, &actual,
"`{}` (SyntaxText) `{}` (SyntaxText)",
actual, expected
);
}
check(&[""], "");
check(&["a"], "a");
check(&["hello", "world"], "helloworld");
check(&["hellowo", "rld"], "helloworld");
check(&["hel", "lowo", "rld"], "helloworld");
check(&["{", "abc", "}"], "{abc}");
check(&["{", "abc", "}", "{"], "{abc}{");
check(&["{", "abc", "}ab"], "{abc}ab");
}
}

105
crates/ruff_rowan/src/syntax_token_text.rs Normal file
View File

@ -0,0 +1,105 @@
use crate::GreenToken;
use ruff_text_size::{TextRange, TextSize};
use std::ops::Deref;
use std::{borrow::Borrow, fmt::Formatter};
/// Reference to the text of a SyntaxToken without having to worry about the lifetime of `&str`.
#[derive(Eq, Clone)]
pub struct SyntaxTokenText {
// Using a green token to ensure this type is Send + Sync.
token: GreenToken,
/// Relative range of the "selected" token text.
range: TextRange,
}
impl std::hash::Hash for SyntaxTokenText {
fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
self.text().hash(state);
}
}
impl SyntaxTokenText {
pub(crate) fn new(token: GreenToken) -> SyntaxTokenText {
let range = TextRange::at(TextSize::default(), token.text_len());
Self { token, range }
}
pub(crate) fn with_range(token: GreenToken, range: TextRange) -> SyntaxTokenText {
debug_assert!(range.end() <= token.text_len());
Self { token, range }
}
/// Returns the length of the text
pub fn len(&self) -> TextSize {
self.range.len()
}
/// Returns `true` if the text is empty
pub fn is_empty(&self) -> bool {
self.range.is_empty()
}
/// Returns a subslice of the text.
pub fn slice(mut self, range: TextRange) -> SyntaxTokenText {
assert!(
self.range.contains_range(range),
"Range {range:?} exceeds bounds {:?}",
self.range
);
self.range = range;
self
}
pub fn range(&self) -> TextRange {
self.range
}
pub fn text(&self) -> &str {
&self.token.text()[self.range]
}
}
impl Deref for SyntaxTokenText {
type Target = str;
fn deref(&self) -> &Self::Target {
self.text()
}
}
impl std::fmt::Display for SyntaxTokenText {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.text())
}
}
impl std::fmt::Debug for SyntaxTokenText {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
write!(f, "{:?}", self.text())
}
}
impl PartialEq for SyntaxTokenText {
fn eq(&self, other: &Self) -> bool {
**self == **other
}
}
impl PartialEq<&'_ str> for SyntaxTokenText {
fn eq(&self, rhs: &&'_ str) -> bool {
**self == **rhs
}
}
impl PartialEq<SyntaxTokenText> for &'_ str {
fn eq(&self, other: &SyntaxTokenText) -> bool {
**self == **other
}
}
impl Borrow<str> for SyntaxTokenText {
fn borrow(&self) -> &str {
self.text()
}
}

282
crates/ruff_rowan/src/tree_builder.rs Normal file
View File

@ -0,0 +1,282 @@
use crate::green::NodeCacheNodeEntryMut;
use crate::{
cow_mut::CowMut,
green::{GreenElement, NodeCache},
syntax::TriviaPiece,
GreenNode, Language, NodeOrToken, ParsedChildren, SyntaxFactory, SyntaxKind, SyntaxNode,
};
use std::marker::PhantomData;
/// A checkpoint for maybe wrapping a node. See `GreenNodeBuilder::checkpoint` for details.
#[derive(Clone, Copy, Debug)]
pub struct Checkpoint(usize);
/// A builder for a syntax tree.
#[derive(Debug)]
pub struct TreeBuilder<'cache, L: Language, S: SyntaxFactory<Kind = L::Kind>> {
cache: CowMut<'cache, NodeCache>,
parents: Vec<(L::Kind, usize)>,
children: Vec<(u64, GreenElement)>,
ph: PhantomData<S>,
}
impl<L: Language, S: SyntaxFactory<Kind = L::Kind>> Default for TreeBuilder<'_, L, S> {
fn default() -> Self {
Self {
cache: CowMut::default(),
parents: Vec::default(),
children: Vec::default(),
ph: PhantomData,
}
}
}
impl<L: Language, S: SyntaxFactory<Kind = L::Kind>> TreeBuilder<'_, L, S> {
/// Creates new builder.
pub fn new() -> TreeBuilder<'static, L, S> {
TreeBuilder::default()
}
/// Reusing `NodeCache` between different [TreeBuilder]`s saves memory.
/// It allows to structurally share underlying trees.
pub fn with_cache(cache: &mut NodeCache) -> TreeBuilder<'_, L, S> {
TreeBuilder {
cache: CowMut::Borrowed(cache),
parents: Vec::new(),
children: Vec::new(),
ph: PhantomData,
}
}
/// Method to quickly wrap a tree with a node.
///
/// TreeBuilder::<RawLanguage>::wrap_with_node(RawSyntaxKind(0), |builder| {
/// builder.token(RawSyntaxKind(1), "let");
/// });
pub fn wrap_with_node<F>(kind: L::Kind, build: F) -> SyntaxNode<L>
where
F: Fn(&mut Self),
{
let mut builder = TreeBuilder::<L, S>::new();
builder.start_node(kind);
build(&mut builder);
builder.finish_node();
builder.finish()
}
/// Adds new token to the current branch.
#[inline]
pub fn token(&mut self, kind: L::Kind, text: &str) -> &mut Self {
let (hash, token) = self.cache.token(kind.to_raw(), text);
self.children.push((hash, token.into()));
self
}
/// Adds new token to the current branch.
#[inline]
pub fn token_with_trivia(
&mut self,
kind: L::Kind,
text: &str,
leading: &[TriviaPiece],
trailing: &[TriviaPiece],
) {
let (hash, token) = self
.cache
.token_with_trivia(kind.to_raw(), text, leading, trailing);
self.children.push((hash, token.into()));
}
/// Start new node and make it current.
#[inline]
pub fn start_node(&mut self, kind: L::Kind) -> &mut Self {
let len = self.children.len();
self.parents.push((kind, len));
self
}
/// Finish current branch and restore previous
/// branch as current.
#[inline]
pub fn finish_node(&mut self) -> &mut Self {
let (kind, first_child) = self.parents.pop().unwrap();
let raw_kind = kind.to_raw();
let slots = &self.children[first_child..];
let node_entry = self.cache.node(raw_kind, slots);
let mut build_node = || {
let children = ParsedChildren::new(&mut self.children, first_child);
S::make_syntax(kind, children).into_green()
};
let (hash, node) = match node_entry {
NodeCacheNodeEntryMut::NoCache(hash) => (hash, build_node()),
NodeCacheNodeEntryMut::Vacant(entry) => {
let node = build_node();
let hash = entry.cache(node.clone());
(hash, node)
}
NodeCacheNodeEntryMut::Cached(cached) => {
self.children.truncate(first_child);
(cached.hash(), cached.node().clone())
}
};
self.children.push((hash, node.into()));
self
}
/// Prepare for maybe wrapping the next node.
/// The way wrapping works is that you first of all get a checkpoint,
/// then you place all tokens you want to wrap, and then *maybe* call
/// `start_node_at`.
/// Example:
/// ```rust
/// # use ruff_rowan::raw_language::{RawLanguage, RawLanguageKind, RawSyntaxTreeBuilder};
/// # const PLUS: RawLanguageKind = RawLanguageKind::PLUS_TOKEN;
/// # const OPERATION: RawLanguageKind = RawLanguageKind::ROOT;
/// # struct Parser;
/// # impl Parser {
/// # fn peek(&self) -> Option<RawLanguageKind> { None }
/// # fn parse_expr(&mut self) {}
/// # }
/// # let mut builder = RawSyntaxTreeBuilder::new();
/// # let mut parser = Parser;
/// let checkpoint = builder.checkpoint();
/// parser.parse_expr();
/// if parser.peek() == Some(PLUS) {
/// // 1 + 2 = Add(1, 2)
/// builder.start_node_at(checkpoint, OPERATION);
/// parser.parse_expr();
/// builder.finish_node();
/// }
/// ```
#[inline]
pub fn checkpoint(&self) -> Checkpoint {
Checkpoint(self.children.len())
}
/// Wrap the previous branch marked by `checkpoint` in a new branch and
/// make it current.
#[inline]
pub fn start_node_at(&mut self, checkpoint: Checkpoint, kind: L::Kind) {
let Checkpoint(checkpoint) = checkpoint;
assert!(
checkpoint <= self.children.len(),
"checkpoint no longer valid, was finish_node called early?"
);
if let Some(&(_, first_child)) = self.parents.last() {
assert!(
checkpoint >= first_child,
"checkpoint no longer valid, was an unmatched start_node_at called?"
);
}
self.parents.push((kind, checkpoint));
}
/// Complete tree building. Make sure that
/// `start_node_at` and `finish_node` calls
/// are paired!
#[inline]
#[must_use]
pub fn finish(self) -> SyntaxNode<L> {
SyntaxNode::new_root(self.finish_green())
}
// For tests
#[must_use]
pub(crate) fn finish_green(mut self) -> GreenNode {
assert_eq!(self.children.len(), 1);
match self.children.pop().unwrap().1 {
NodeOrToken::Node(node) => node,
_ => panic!(),
}
}
}
#[cfg(test)]
mod tests {
use crate::green::GreenElementRef;
use crate::raw_language::{RawLanguageKind, RawSyntaxTreeBuilder};
use crate::{GreenNodeData, GreenTokenData, NodeOrToken};
// Builds a "Condition" like structure where the closing ) is missing
fn build_condition_with_missing_closing_parenthesis(builder: &mut RawSyntaxTreeBuilder) {
builder.start_node(RawLanguageKind::CONDITION);
builder.token(RawLanguageKind::L_PAREN_TOKEN, "(");
builder.start_node(RawLanguageKind::LITERAL_EXPRESSION);
builder.token(RawLanguageKind::STRING_TOKEN, "a");
builder.finish_node();
// missing )
builder.finish_node();
}
#[test]
fn caches_identical_nodes_with_empty_slots() {
let mut builder = RawSyntaxTreeBuilder::new();
builder.start_node(RawLanguageKind::ROOT); // Root
build_condition_with_missing_closing_parenthesis(&mut builder);
build_condition_with_missing_closing_parenthesis(&mut builder);
builder.finish_node();
let root = builder.finish_green();
let first = root.children().next().unwrap();
let last = root.children().last().unwrap();
assert_eq!(first.element(), last.element());
assert_same_elements(first.element(), last.element());
}
#[test]
fn doesnt_cache_node_if_empty_slots_differ() {
let mut builder = RawSyntaxTreeBuilder::new();
builder.start_node(RawLanguageKind::ROOT); // Root
build_condition_with_missing_closing_parenthesis(&mut builder); // misses the ')'
// Create a well formed condition
builder.start_node(RawLanguageKind::CONDITION);
builder.token(RawLanguageKind::L_PAREN_TOKEN, "(");
builder.start_node(RawLanguageKind::LITERAL_EXPRESSION);
builder.token(RawLanguageKind::STRING_TOKEN, "a");
builder.finish_node();
// missing )
builder.token(RawLanguageKind::R_PAREN_TOKEN, ")");
builder.finish_node();
// finish root
builder.finish_node();
let root = builder.finish_green();
let first_condition = root.children().next().unwrap();
let last_condition = root.children().last().unwrap();
assert_ne!(first_condition.element(), last_condition.element());
}
fn assert_same_elements(left: GreenElementRef<'_>, right: GreenElementRef<'_>) {
fn element_id(element: GreenElementRef<'_>) -> *const () {
match element {
NodeOrToken::Node(node) => node as *const GreenNodeData as *const (),
NodeOrToken::Token(token) => token as *const GreenTokenData as *const (),
}
}
assert_eq!(element_id(left), element_id(right),);
}
}

157
crates/ruff_rowan/src/utility_types.rs Normal file
View File

@ -0,0 +1,157 @@
use std::{fmt, ops::Deref};
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum NodeOrToken<N, T> {
Node(N),
Token(T),
}
impl<N, T> NodeOrToken<N, T> {
pub fn into_node(self) -> Option<N> {
match self {
NodeOrToken::Node(node) => Some(node),
NodeOrToken::Token(_) => None,
}
}
pub fn into_token(self) -> Option<T> {
match self {
NodeOrToken::Node(_) => None,
NodeOrToken::Token(token) => Some(token),
}
}
pub fn as_node(&self) -> Option<&N> {
match self {
NodeOrToken::Node(node) => Some(node),
NodeOrToken::Token(_) => None,
}
}
pub fn as_token(&self) -> Option<&T> {
match self {
NodeOrToken::Node(_) => None,
NodeOrToken::Token(token) => Some(token),
}
}
}
impl<N: Deref, T: Deref> NodeOrToken<N, T> {
pub(crate) fn as_deref(&self) -> NodeOrToken<&N::Target, &T::Target> {
match self {
NodeOrToken::Node(node) => NodeOrToken::Node(&**node),
NodeOrToken::Token(token) => NodeOrToken::Token(&**token),
}
}
}
impl<N: fmt::Display, T: fmt::Display> fmt::Display for NodeOrToken<N, T> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
NodeOrToken::Node(node) => fmt::Display::fmt(node, f),
NodeOrToken::Token(token) => fmt::Display::fmt(token, f),
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum Direction {
Next,
Prev,
}
/// `WalkEvent` describes tree walking process.
#[derive(Debug, Copy, Clone)]
pub enum WalkEvent<T> {
/// Fired before traversing the node.
Enter(T),
/// Fired after the node is traversed.
Leave(T),
}
impl<T> WalkEvent<T> {
pub fn map<F: FnOnce(T) -> U, U>(self, f: F) -> WalkEvent<U> {
match self {
WalkEvent::Enter(it) => WalkEvent::Enter(f(it)),
WalkEvent::Leave(it) => WalkEvent::Leave(f(it)),
}
}
}
/// There might be zero, one or two leaves at a given offset.
#[derive(Clone, Debug)]
pub enum TokenAtOffset<T> {
/// No leaves at offset -- possible for the empty file.
None,
/// Only a single leaf at offset.
Single(T),
/// Offset is exactly between two leaves.
Between(T, T),
}
impl<T> TokenAtOffset<T> {
pub fn map<F: Fn(T) -> U, U>(self, f: F) -> TokenAtOffset<U> {
match self {
TokenAtOffset::None => TokenAtOffset::None,
TokenAtOffset::Single(it) => TokenAtOffset::Single(f(it)),
TokenAtOffset::Between(l, r) => TokenAtOffset::Between(f(l), f(r)),
}
}
/// Convert to option, preferring the right leaf in case of a tie.
pub fn right_biased(self) -> Option<T> {
match self {
TokenAtOffset::None => None,
TokenAtOffset::Single(node) => Some(node),
TokenAtOffset::Between(_, right) => Some(right),
}
}
/// Convert to option, preferring the left leaf in case of a tie.
pub fn left_biased(self) -> Option<T> {
match self {
TokenAtOffset::None => None,
TokenAtOffset::Single(node) => Some(node),
TokenAtOffset::Between(left, _) => Some(left),
}
}
}
impl<T> Iterator for TokenAtOffset<T> {
type Item = T;
fn next(&mut self) -> Option<T> {
match std::mem::replace(self, TokenAtOffset::None) {
TokenAtOffset::None => None,
TokenAtOffset::Single(node) => {
*self = TokenAtOffset::None;
Some(node)
}
TokenAtOffset::Between(left, right) => {
*self = TokenAtOffset::Single(right);
Some(left)
}
}
}
fn size_hint(&self) -> (usize, Option<usize>) {
match self {
TokenAtOffset::None => (0, Some(0)),
TokenAtOffset::Single(_) => (1, Some(1)),
TokenAtOffset::Between(_, _) => (2, Some(2)),
}
}
}
impl<T> ExactSizeIterator for TokenAtOffset<T> {}
#[cfg(target_pointer_width = "64")]
#[macro_export]
macro_rules! static_assert {
($expr:expr) => {
const _: i32 = 0 / $expr as i32;
};
}
#[cfg(target_pointer_width = "64")]
pub use static_assert;

14
crates/ruff_text_edit/Cargo.toml Normal file
View File

@ -0,0 +1,14 @@
[package]
name = "ruff_text_edit"
version = "0.0.0"
publish = false
edition = "2021"
[dependencies]
ruff_text_size = { path = "../ruff_text_size", features = ["serde"] }
serde = { version = "1.0.136", features = ["derive"] }
schemars = { version = "0.8.10", optional = true }
similar = { version = "2.1.0", features = ["unicode"] }
[features]
schemars = ["dep:schemars", "ruff_text_size/schemars"]

364
crates/ruff_text_edit/src/lib.rs Normal file
View File

@ -0,0 +1,364 @@
//! Representation of a `TextEdit`.
//!
//! This is taken from [rust-analyzer's `text_edit` crate](https://rust-analyzer.github.io/rust-analyzer/text_edit/index.html)
#![warn(
rust_2018_idioms,
unused_lifetimes,
semicolon_in_expressions_from_macros
)]
use std::{cmp::Ordering, num::NonZeroU32};
use ruff_text_size::{TextRange, TextSize};
use serde::{Deserialize, Serialize};
pub use similar::ChangeTag;
use similar::{utils::TextDiffRemapper, TextDiff};
#[derive(Default, Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
pub struct TextEdit {
dictionary: String,
ops: Vec<CompressedOp>,
}
#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
pub enum CompressedOp {
DiffOp(DiffOp),
EqualLines { line_count: NonZeroU32 },
}
#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
pub enum DiffOp {
Equal { range: TextRange },
Insert { range: TextRange },
Delete { range: TextRange },
}
impl DiffOp {
pub fn tag(self) -> ChangeTag {
match self {
DiffOp::Equal { .. } => ChangeTag::Equal,
DiffOp::Insert { .. } => ChangeTag::Insert,
DiffOp::Delete { .. } => ChangeTag::Delete,
}
}
pub fn text(self, diff: &TextEdit) -> &str {
let range = match self {
DiffOp::Equal { range } => range,
DiffOp::Insert { range } => range,
DiffOp::Delete { range } => range,
};
diff.get_text(range)
}
}
#[derive(Debug, Default, Clone)]
pub struct TextEditBuilder {
index: Vec<TextRange>,
edit: TextEdit,
}
impl TextEdit {
/// Convenience method for creating a new [`TextEditBuilder`]
pub fn builder() -> TextEditBuilder {
TextEditBuilder::default()
}
/// Create a diff of `old` to `new`, tokenized by Unicode words
pub fn from_unicode_words(old: &str, new: &str) -> Self {
let mut builder = Self::builder();
let diff = TextDiff::configure()
.newline_terminated(true)
.diff_unicode_words(old, new);
let remapper = TextDiffRemapper::from_text_diff(&diff, old, new);
for (tag, text) in diff.ops().iter().flat_map(|op| remapper.iter_slices(op)) {
match tag {
ChangeTag::Equal => {
builder.equal(text);
}
ChangeTag::Delete => {
builder.delete(text);
}
ChangeTag::Insert => {
builder.insert(text);
}
}
}
builder.finish()
}
/// Returns the number of [`DiffOp`] in this [`TextEdit`]
pub fn len(&self) -> usize {
self.ops.len()
}
/// Return `true` is this [`TextEdit`] doesn't contain any [`DiffOp`]
pub fn is_empty(&self) -> bool {
self.ops.is_empty()
}
/// Returns an [Iterator] over the [`DiffOp`] of this [`TextEdit`]
pub fn iter(&self) -> std::slice::Iter<'_, CompressedOp> {
self.into_iter()
}
/// Return the text value of range interned in this [`TextEdit`] dictionary
pub fn get_text(&self, range: TextRange) -> &str {
&self.dictionary[range]
}
/// Return the content of the "new" revision of the text represented in
/// this [`TextEdit`]. This methods needs to be provided with the "old"
/// revision of the string since [`TextEdit`] doesn't store the content of
/// text sections that are equal between revisions
pub fn new_string(&self, old_string: &str) -> String {
let mut output = String::new();
let mut input_position = TextSize::from(0);
for op in &self.ops {
match op {
CompressedOp::DiffOp(DiffOp::Equal { range }) => {
output.push_str(&self.dictionary[*range]);
input_position += range.len();
}
CompressedOp::DiffOp(DiffOp::Insert { range }) => {
output.push_str(&self.dictionary[*range]);
}
CompressedOp::DiffOp(DiffOp::Delete { range }) => {
input_position += range.len();
}
CompressedOp::EqualLines { line_count } => {
let start = u32::from(input_position) as usize;
let input = &old_string[start..];
let line_break_count = line_count.get() as usize + 1;
for line in input.split_inclusive('\n').take(line_break_count) {
output.push_str(line);
input_position += TextSize::of(line);
}
}
}
}
output
}
}
impl IntoIterator for TextEdit {
type Item = CompressedOp;
type IntoIter = std::vec::IntoIter<CompressedOp>;
fn into_iter(self) -> Self::IntoIter {
self.ops.into_iter()
}
}
impl<'a> IntoIterator for &'a TextEdit {
type Item = &'a CompressedOp;
type IntoIter = std::slice::Iter<'a, CompressedOp>;
fn into_iter(self) -> Self::IntoIter {
self.ops.iter()
}
}
impl TextEditBuilder {
pub fn is_empty(&self) -> bool {
self.edit.ops.is_empty()
}
/// Add a piece of string to the dictionary, returning the corresponding
/// range in the dictionary string
fn intern(&mut self, value: &str) -> TextRange {
let value_bytes = value.as_bytes();
let value_len = TextSize::of(value);
let index = self.index.binary_search_by(|range| {
let entry = self.edit.dictionary[*range].as_bytes();
for (lhs, rhs) in entry.iter().zip(value_bytes) {
match lhs.cmp(rhs) {
Ordering::Equal => continue,
ordering => return ordering,
}
}
match entry.len().cmp(&value_bytes.len()) {
// If all bytes in the shared sub-slice match, the dictionary
// entry is allowed to be longer than the text being inserted
Ordering::Greater => Ordering::Equal,
ordering => ordering,
}
});
match index {
Ok(index) => {
let range = self.index[index];
let len = value_len.min(range.len());
TextRange::at(range.start(), len)
}
Err(index) => {
let start = TextSize::of(&self.edit.dictionary);
self.edit.dictionary.push_str(value);
let range = TextRange::at(start, value_len);
self.index.insert(index, range);
range
}
}
}
pub fn equal(&mut self, text: &str) {
if let Some((start, mid, end)) = compress_equal_op(text) {
let start = self.intern(start);
self.edit
.ops
.push(CompressedOp::DiffOp(DiffOp::Equal { range: start }));
self.edit
.ops
.push(CompressedOp::EqualLines { line_count: mid });
let end = self.intern(end);
self.edit
.ops
.push(CompressedOp::DiffOp(DiffOp::Equal { range: end }));
} else {
let range = self.intern(text);
self.edit
.ops
.push(CompressedOp::DiffOp(DiffOp::Equal { range }));
}
}
pub fn insert(&mut self, text: &str) {
let range = self.intern(text);
self.edit
.ops
.push(CompressedOp::DiffOp(DiffOp::Insert { range }));
}
pub fn delete(&mut self, text: &str) {
let range = self.intern(text);
self.edit
.ops
.push(CompressedOp::DiffOp(DiffOp::Delete { range }));
}
pub fn replace(&mut self, old: &str, new: &str) {
self.delete(old);
self.insert(new);
}
pub fn finish(self) -> TextEdit {
self.edit
}
}
/// Number of lines to keep as [`DiffOp::Equal`] operations around a
/// [`CompressedOp::EqualCompressedLines`] operation. This has the effect of
/// making the compressed diff retain a few line of equal content around
/// changes, which is useful for display as it makes it possible to print a few
/// context lines around changes without having to keep the full original text
/// around.
const COMPRESSED_DIFFS_CONTEXT_LINES: usize = 2;
fn compress_equal_op(text: &str) -> Option<(&str, NonZeroU32, &str)> {
let mut iter = text.split('\n');
let mut leading_len = COMPRESSED_DIFFS_CONTEXT_LINES;
for _ in 0..=COMPRESSED_DIFFS_CONTEXT_LINES {
leading_len += iter.next()?.len();
}
let mut trailing_len = COMPRESSED_DIFFS_CONTEXT_LINES;
for _ in 0..=COMPRESSED_DIFFS_CONTEXT_LINES {
trailing_len += iter.next_back()?.len();
}
let mid_count = iter.count();
let mid_count = u32::try_from(mid_count).ok()?;
let mid_count = NonZeroU32::new(mid_count)?;
let trailing_start = text.len().saturating_sub(trailing_len);
Some((&text[..leading_len], mid_count, &text[trailing_start..]))
}
#[cfg(test)]
mod tests {
use std::num::NonZeroU32;
use crate::{compress_equal_op, TextEdit};
#[test]
fn compress_short() {
let output = compress_equal_op(
"
start 1
start 2
end 1
end 2
",
);
assert_eq!(output, None);
}
#[test]
fn compress_long() {
let output = compress_equal_op(
"
start 1
start 2
mid 1
mid 2
mid 3
end 1
end 2
",
);
assert_eq!(
output,
Some((
"\nstart 1\nstart 2",
NonZeroU32::new(3).unwrap(),
"end 1\nend 2\n"
))
);
}
#[test]
fn new_string_compressed() {
const OLD: &str = "line 1 old
line 2
line 3
line 4
line 5
line 6
line 7 old";
const NEW: &str = "line 1 new
line 2
line 3
line 4
line 5
line 6
line 7 new";
let diff = TextEdit::from_unicode_words(OLD, NEW);
let new_string = diff.new_string(OLD);
assert_eq!(new_string, NEW);
}
}

18
crates/ruff_text_size/Cargo.toml Normal file
View File

@ -0,0 +1,18 @@
[package]
name = "ruff_text_size"
version = "0.0.0"
publish = false
edition = "2021"
[dependencies]
serde = { version = "1.0", optional = true, default_features = false }
schemars = { version = "0.8.10", optional = true }
[dev-dependencies]
serde_test = "1.0"
static_assertions = "1.1"
[[test]]
name = "serde"
path = "tests/serde.rs"
required-features = ["serde"]

34
crates/ruff_text_size/src/lib.rs Normal file
View File

@ -0,0 +1,34 @@
//! Newtypes for working with text sizes/ranges in a more type-safe manner.
//!
//! This library can help with two things:
//! * Reducing storage requirements for offsets and ranges, under the
//! assumption that 32 bits is enough.
//! * Providing standard vocabulary types for applications where text ranges
//! are pervasive.
//!
//! However, you should not use this library simply because you work with
//! strings. In the overwhelming majority of cases, using `usize` and
//! `std::ops::Range<usize>` is better. In particular, if you are publishing a
//! library, using only std types in the interface would make it more
//! interoperable. Similarly, if you are writing something like a lexer, which
//! produces, but does not *store* text ranges, then sticking to `usize` would
//! be better.
//!
//! Minimal Supported Rust Version: latest stable.
#![forbid(unsafe_code)]
#![warn(missing_debug_implementations, missing_docs)]
mod range;
mod size;
mod traits;
#[cfg(feature = "schemars")]
mod schemars_impls;
#[cfg(feature = "serde")]
mod serde_impls;
pub use crate::{range::TextRange, size::TextSize, traits::TextLen};
#[cfg(target_pointer_width = "16")]
compile_error!("text-size assumes usize >= u32 and does not work on 16-bit targets");

537
crates/ruff_text_size/src/range.rs Normal file
View File

@ -0,0 +1,537 @@
use cmp::Ordering;
use {
crate::TextSize,
std::{
cmp, fmt,
ops::{Add, AddAssign, Bound, Index, IndexMut, Range, RangeBounds, Sub, SubAssign},
},
};
/// A range in text, represented as a pair of [`TextSize`][struct@TextSize].
///
/// It is a logic error for `start` to be greater than `end`.
#[derive(Default, Copy, Clone, Eq, PartialEq, Hash)]
pub struct TextRange {
// Invariant: start <= end
start: TextSize,
end: TextSize,
}
impl fmt::Debug for TextRange {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{}..{}", self.start().raw, self.end().raw)
}
}
impl TextRange {
/// Creates a new `TextRange` with the given `start` and `end` (`start..end`).
///
/// # Panics
///
/// Panics if `end < start`.
///
/// # Examples
///
/// ```rust
/// # use ruff_text_size::*;
/// let start = TextSize::from(5);
/// let end = TextSize::from(10);
/// let range = TextRange::new(start, end);
///
/// assert_eq!(range.start(), start);
/// assert_eq!(range.end(), end);
/// assert_eq!(range.len(), end - start);
/// ```
#[inline]
pub fn new(start: TextSize, end: TextSize) -> TextRange {
assert!(start <= end);
TextRange { start, end }
}
/// Create a new `TextRange` with the given `offset` and `len` (`offset..offset + len`).
///
/// # Examples
///
/// ```rust
/// # use ruff_text_size::*;
/// let text = "0123456789";
///
/// let offset = TextSize::from(2);
/// let length = TextSize::from(5);
/// let range = TextRange::at(offset, length);
///
/// assert_eq!(range, TextRange::new(offset, offset + length));
/// assert_eq!(&text[range], "23456")
/// ```
#[inline]
pub fn at(offset: TextSize, len: TextSize) -> TextRange {
TextRange::new(offset, offset + len)
}
/// Create a zero-length range at the specified offset (`offset..offset`).
///
/// # Examples
///
/// ```rust
/// # use ruff_text_size::*;
/// let point: TextSize;
/// # point = TextSize::from(3);
/// let range = TextRange::empty(point);
/// assert!(range.is_empty());
/// assert_eq!(range, TextRange::new(point, point));
/// ```
#[inline]
pub fn empty(offset: TextSize) -> TextRange {
TextRange {
start: offset,
end: offset,
}
}
/// Create a range up to the given end (`..end`).
///
/// # Examples
///
/// ```rust
/// # use ruff_text_size::*;
/// let point: TextSize;
/// # point = TextSize::from(12);
/// let range = TextRange::up_to(point);
///
/// assert_eq!(range.len(), point);
/// assert_eq!(range, TextRange::new(0.into(), point));
/// assert_eq!(range, TextRange::at(0.into(), point));
/// ```
#[inline]
pub fn up_to(end: TextSize) -> TextRange {
TextRange {
start: 0.into(),
end,
}
}
}
/// Identity methods.
impl TextRange {
/// The start point of this range.
#[inline]
pub const fn start(self) -> TextSize {
self.start
}
/// The end point of this range.
#[inline]
pub const fn end(self) -> TextSize {
self.end
}
/// The size of this range.
#[inline]
pub const fn len(self) -> TextSize {
// HACK for const fn: math on primitives only
TextSize {
raw: self.end().raw - self.start().raw,
}
}
/// Check if this range is empty.
#[inline]
pub const fn is_empty(self) -> bool {
// HACK for const fn: math on primitives only
self.start().raw == self.end().raw
}
}
/// Manipulation methods.
impl TextRange {
/// Check if this range contains an offset.
///
/// The end index is considered excluded.
///
/// # Examples
///
/// ```rust
/// # use ruff_text_size::*;
/// let (start, end): (TextSize, TextSize);
/// # start = 10.into(); end = 20.into();
/// let range = TextRange::new(start, end);
/// assert!(range.contains(start));
/// assert!(!range.contains(end));
/// ```
#[inline]
pub fn contains(self, offset: TextSize) -> bool {
self.start() <= offset && offset < self.end()
}
/// Check if this range contains an offset.
///
/// The end index is considered included.
///
/// # Examples
///
/// ```rust
/// # use ruff_text_size::*;
/// let (start, end): (TextSize, TextSize);
/// # start = 10.into(); end = 20.into();
/// let range = TextRange::new(start, end);
/// assert!(range.contains_inclusive(start));
/// assert!(range.contains_inclusive(end));
/// ```
#[inline]
pub fn contains_inclusive(self, offset: TextSize) -> bool {
self.start() <= offset && offset <= self.end()
}
/// Check if this range completely contains another range.
///
/// # Examples
///
/// ```rust
/// # use ruff_text_size::*;
/// let larger = TextRange::new(0.into(), 20.into());
/// let smaller = TextRange::new(5.into(), 15.into());
/// assert!(larger.contains_range(smaller));
/// assert!(!smaller.contains_range(larger));
///
/// // a range always contains itself
/// assert!(larger.contains_range(larger));
/// assert!(smaller.contains_range(smaller));
/// ```
#[inline]
pub fn contains_range(self, other: TextRange) -> bool {
self.start() <= other.start() && other.end() <= self.end()
}
/// The range covered by both ranges, if it exists.
/// If the ranges touch but do not overlap, the output range is empty.
///
/// # Examples
///
/// ```rust
/// # use ruff_text_size::*;
/// assert_eq!(
/// TextRange::intersect(
/// TextRange::new(0.into(), 10.into()),
/// TextRange::new(5.into(), 15.into()),
/// ),
/// Some(TextRange::new(5.into(), 10.into())),
/// );
/// ```
#[inline]
pub fn intersect(self, other: TextRange) -> Option<TextRange> {
let start = cmp::max(self.start(), other.start());
let end = cmp::min(self.end(), other.end());
if end < start {
return None;
}
Some(TextRange::new(start, end))
}
/// Extends the range to cover `other` as well.
///
/// # Examples
///
/// ```rust
/// # use ruff_text_size::*;
/// assert_eq!(
/// TextRange::cover(
/// TextRange::new(0.into(), 5.into()),
/// TextRange::new(15.into(), 20.into()),
/// ),
/// TextRange::new(0.into(), 20.into()),
/// );
/// ```
#[inline]
#[must_use]
pub fn cover(self, other: TextRange) -> TextRange {
let start = cmp::min(self.start(), other.start());
let end = cmp::max(self.end(), other.end());
TextRange::new(start, end)
}
/// Extends the range to cover `other` offsets as well.
///
/// # Examples
///
/// ```rust
/// # use ruff_text_size::*;
/// assert_eq!(
/// TextRange::empty(0.into()).cover_offset(20.into()),
/// TextRange::new(0.into(), 20.into()),
/// )
/// ```
#[inline]
#[must_use]
pub fn cover_offset(self, offset: TextSize) -> TextRange {
self.cover(TextRange::empty(offset))
}
/// Add an offset to this range.
///
/// Note that this is not appropriate for changing where a `TextRange` is
/// within some string; rather, it is for changing the reference anchor
/// that the `TextRange` is measured against.
///
/// The unchecked version (`Add::add`) will _always_ panic on overflow,
/// in contrast to primitive integers, which check in debug mode only.
#[inline]
pub fn checked_add(self, offset: TextSize) -> Option<TextRange> {
Some(TextRange {
start: self.start.checked_add(offset)?,
end: self.end.checked_add(offset)?,
})
}
/// Subtract an offset from this range.
///
/// Note that this is not appropriate for changing where a `TextRange` is
/// within some string; rather, it is for changing the reference anchor
/// that the `TextRange` is measured against.
///
/// The unchecked version (`Sub::sub`) will _always_ panic on overflow,
/// in contrast to primitive integers, which check in debug mode only.
#[inline]
pub fn checked_sub(self, offset: TextSize) -> Option<TextRange> {
Some(TextRange {
start: self.start.checked_sub(offset)?,
end: self.end.checked_sub(offset)?,
})
}
/// Relative order of the two ranges (overlapping ranges are considered
/// equal).
///
///
/// This is useful when, for example, binary searching an array of disjoint
/// ranges.
///
/// # Examples
///
/// ```
/// # use ruff_text_size::*;
/// # use std::cmp::Ordering;
///
/// let a = TextRange::new(0.into(), 3.into());
/// let b = TextRange::new(4.into(), 5.into());
/// assert_eq!(a.ordering(b), Ordering::Less);
///
/// let a = TextRange::new(0.into(), 3.into());
/// let b = TextRange::new(3.into(), 5.into());
/// assert_eq!(a.ordering(b), Ordering::Less);
///
/// let a = TextRange::new(0.into(), 3.into());
/// let b = TextRange::new(2.into(), 5.into());
/// assert_eq!(a.ordering(b), Ordering::Equal);
///
/// let a = TextRange::new(0.into(), 3.into());
/// let b = TextRange::new(2.into(), 2.into());
/// assert_eq!(a.ordering(b), Ordering::Equal);
///
/// let a = TextRange::new(2.into(), 3.into());
/// let b = TextRange::new(2.into(), 2.into());
/// assert_eq!(a.ordering(b), Ordering::Greater);
/// ```
#[inline]
pub fn ordering(self, other: TextRange) -> Ordering {
if self.end() <= other.start() {
Ordering::Less
} else if other.end() <= self.start() {
Ordering::Greater
} else {
Ordering::Equal
}
}
/// Subtracts an offset from the start position.
///
///
/// ## Panics
/// If `start - amount` is less than zero.
///
/// ## Examples
///
/// ```
/// use ruff_text_size::{TextRange, TextSize};
///
/// let range = TextRange::new(TextSize::from(5), TextSize::from(10));
/// assert_eq!(range.sub_start(TextSize::from(2)), TextRange::new(TextSize::from(3), TextSize::from(10)));
/// ```
#[inline]
#[must_use]
pub fn sub_start(&self, amount: TextSize) -> TextRange {
TextRange::new(self.start() - amount, self.end())
}
/// Adds an offset to the start position.
///
/// ## Panics
/// If `start + amount > end`
///
/// ## Examples
///
/// ```
/// use ruff_text_size::{TextRange, TextSize};
///
/// let range = TextRange::new(TextSize::from(5), TextSize::from(10));
/// assert_eq!(range.add_start(TextSize::from(3)), TextRange::new(TextSize::from(8), TextSize::from(10)));
/// ```
#[inline]
#[must_use]
pub fn add_start(&self, amount: TextSize) -> TextRange {
TextRange::new(self.start() + amount, self.end())
}
/// Subtracts an offset from the end position.
///
///
/// ## Panics
/// If `end - amount < 0` or `end - amount < start`
///
/// ## Examples
///
/// ```
/// use ruff_text_size::{TextRange, TextSize};
///
/// let range = TextRange::new(TextSize::from(5), TextSize::from(10));
/// assert_eq!(range.sub_end(TextSize::from(2)), TextRange::new(TextSize::from(5), TextSize::from(8)));
/// ```
#[inline]
#[must_use]
pub fn sub_end(&self, amount: TextSize) -> TextRange {
TextRange::new(self.start(), self.end() - amount)
}
/// Adds an offset to the end position.
///
///
/// ## Panics
/// If `end + amount > u32::MAX`
///
/// ## Examples
///
/// ```
/// use ruff_text_size::{TextRange, TextSize};
///
/// let range = TextRange::new(TextSize::from(5), TextSize::from(10));
/// assert_eq!(range.add_end(TextSize::from(2)), TextRange::new(TextSize::from(5), TextSize::from(12)));
/// ```
#[inline]
#[must_use]
pub fn add_end(&self, amount: TextSize) -> TextRange {
TextRange::new(self.start(), self.end() + amount)
}
}
impl Index<TextRange> for str {
type Output = str;
#[inline]
fn index(&self, index: TextRange) -> &str {
&self[Range::<usize>::from(index)]
}
}
impl Index<TextRange> for String {
type Output = str;
#[inline]
fn index(&self, index: TextRange) -> &str {
&self[Range::<usize>::from(index)]
}
}
impl IndexMut<TextRange> for str {
#[inline]
fn index_mut(&mut self, index: TextRange) -> &mut str {
&mut self[Range::<usize>::from(index)]
}
}
impl IndexMut<TextRange> for String {
#[inline]
fn index_mut(&mut self, index: TextRange) -> &mut str {
&mut self[Range::<usize>::from(index)]
}
}
impl RangeBounds<TextSize> for TextRange {
fn start_bound(&self) -> Bound<&TextSize> {
Bound::Included(&self.start)
}
fn end_bound(&self) -> Bound<&TextSize> {
Bound::Excluded(&self.end)
}
}
impl<T> From<TextRange> for Range<T>
where
T: From<TextSize>,
{
#[inline]
fn from(r: TextRange) -> Self {
r.start().into()..r.end().into()
}
}
macro_rules! ops {
(impl $Op:ident for TextRange by fn $f:ident = $op:tt) => {
impl $Op<&TextSize> for TextRange {
type Output = TextRange;
#[inline]
fn $f(self, other: &TextSize) -> TextRange {
self $op *other
}
}
impl<T> $Op<T> for &TextRange
where
TextRange: $Op<T, Output=TextRange>,
{
type Output = TextRange;
#[inline]
fn $f(self, other: T) -> TextRange {
*self $op other
}
}
};
}
impl Add<TextSize> for TextRange {
type Output = TextRange;
#[inline]
fn add(self, offset: TextSize) -> TextRange {
self.checked_add(offset)
.expect("TextRange +offset overflowed")
}
}
impl Sub<TextSize> for TextRange {
type Output = TextRange;
#[inline]
fn sub(self, offset: TextSize) -> TextRange {
self.checked_sub(offset)
.expect("TextRange -offset overflowed")
}
}
ops!(impl Add for TextRange by fn add = +);
ops!(impl Sub for TextRange by fn sub = -);
impl<A> AddAssign<A> for TextRange
where
TextRange: Add<A, Output = TextRange>,
{
#[inline]
fn add_assign(&mut self, rhs: A) {
*self = *self + rhs;
}
}
impl<S> SubAssign<S> for TextRange
where
TextRange: Sub<S, Output = TextRange>,
{
#[inline]
fn sub_assign(&mut self, rhs: S) {
*self = *self - rhs;
}
}

33
crates/ruff_text_size/src/schemars_impls.rs Normal file
View File

@ -0,0 +1,33 @@
//! This module implements the [`JsonSchema`] trait from the `schemars` crate for
//! [`TextSize`] and [`TextRange`] if the `schemars` feature is enabled. This trait
//! exposes meta-information on how a given type is serialized and deserialized
//! using `serde`, and is currently used to generate autocomplete information
//! for the `rome.json` configuration file and TypeScript types for the node.js
//! bindings to the Workspace API
use crate::{TextRange, TextSize};
use schemars::{gen::SchemaGenerator, schema::Schema, JsonSchema};
impl JsonSchema for TextSize {
fn schema_name() -> String {
String::from("TextSize")
}
fn json_schema(gen: &mut SchemaGenerator) -> Schema {
// TextSize is represented as a raw u32, see serde_impls.rs for the
// actual implementation
<u32>::json_schema(gen)
}
}
impl JsonSchema for TextRange {
fn schema_name() -> String {
String::from("TextRange")
}
fn json_schema(gen: &mut SchemaGenerator) -> Schema {
// TextSize is represented as (TextSize, TextSize), see serde_impls.rs
// for the actual implementation
<(TextSize, TextSize)>::json_schema(gen)
}
}

48
crates/ruff_text_size/src/serde_impls.rs Normal file
View File

@ -0,0 +1,48 @@
use {
crate::{TextRange, TextSize},
serde::{de, Deserialize, Deserializer, Serialize, Serializer},
};
impl Serialize for TextSize {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
self.raw.serialize(serializer)
}
}
impl<'de> Deserialize<'de> for TextSize {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
u32::deserialize(deserializer).map(TextSize::from)
}
}
impl Serialize for TextRange {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
(self.start(), self.end()).serialize(serializer)
}
}
impl<'de> Deserialize<'de> for TextRange {
#[allow(clippy::nonminimal_bool)]
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
let (start, end) = Deserialize::deserialize(deserializer)?;
if !(start <= end) {
return Err(de::Error::custom(format!(
"invalid range: {:?}..{:?}",
start, end
)));
}
Ok(TextRange::new(start, end))
}
}

161
crates/ruff_text_size/src/size.rs Normal file
View File

@ -0,0 +1,161 @@
use {
crate::TextLen,
std::{
convert::TryFrom,
fmt, iter,
num::TryFromIntError,
ops::{Add, AddAssign, Sub, SubAssign},
u32,
},
};
/// A measure of text length. Also, equivalently, an index into text.
///
/// This is a UTF-8 bytes offset stored as `u32`, but
/// most clients should treat it as an opaque measure.
///
/// For cases that need to escape `TextSize` and return to working directly
/// with primitive integers, `TextSize` can be converted losslessly to/from
/// `u32` via [`From`] conversions as well as losslessly be converted [`Into`]
/// `usize`. The `usize -> TextSize` direction can be done via [`TryFrom`].
///
/// These escape hatches are primarily required for unit testing and when
/// converting from UTF-8 size to another coordinate space, such as UTF-16.
#[derive(Clone, Copy, Default, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct TextSize {
pub(crate) raw: u32,
}
impl fmt::Debug for TextSize {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{}", self.raw)
}
}
impl TextSize {
/// The text size of some primitive text-like object.
///
/// Accepts `char`, `&str`, and `&String`.
///
/// # Examples
///
/// ```rust
/// # use ruff_text_size::*;
/// let char_size = TextSize::of('🦀');
/// assert_eq!(char_size, TextSize::from(4));
///
/// let str_size = TextSize::of("rust-analyzer");
/// assert_eq!(str_size, TextSize::from(13));
/// ```
#[inline]
pub fn of<T: TextLen>(text: T) -> TextSize {
text.text_len()
}
}
/// Methods to act like a primitive integer type, where reasonably applicable.
// Last updated for parity with Rust 1.42.0.
impl TextSize {
/// Checked addition. Returns `None` if overflow occurred.
#[inline]
pub fn checked_add(self, rhs: TextSize) -> Option<TextSize> {
self.raw.checked_add(rhs.raw).map(|raw| TextSize { raw })
}
/// Checked subtraction. Returns `None` if overflow occurred.
#[inline]
pub fn checked_sub(self, rhs: TextSize) -> Option<TextSize> {
self.raw.checked_sub(rhs.raw).map(|raw| TextSize { raw })
}
}
impl From<u32> for TextSize {
#[inline]
fn from(raw: u32) -> Self {
TextSize { raw }
}
}
impl From<TextSize> for u32 {
#[inline]
fn from(value: TextSize) -> Self {
value.raw
}
}
impl TryFrom<usize> for TextSize {
type Error = TryFromIntError;
#[inline]
fn try_from(value: usize) -> Result<Self, TryFromIntError> {
Ok(u32::try_from(value)?.into())
}
}
impl From<TextSize> for usize {
#[inline]
fn from(value: TextSize) -> Self {
value.raw as usize
}
}
macro_rules! ops {
(impl $Op:ident for TextSize by fn $f:ident = $op:tt) => {
impl $Op<TextSize> for TextSize {
type Output = TextSize;
#[inline]
fn $f(self, other: TextSize) -> TextSize {
TextSize { raw: self.raw $op other.raw }
}
}
impl $Op<&TextSize> for TextSize {
type Output = TextSize;
#[inline]
fn $f(self, other: &TextSize) -> TextSize {
self $op *other
}
}
impl<T> $Op<T> for &TextSize
where
TextSize: $Op<T, Output=TextSize>,
{
type Output = TextSize;
#[inline]
fn $f(self, other: T) -> TextSize {
*self $op other
}
}
};
}
ops!(impl Add for TextSize by fn add = +);
ops!(impl Sub for TextSize by fn sub = -);
impl<A> AddAssign<A> for TextSize
where
TextSize: Add<A, Output = TextSize>,
{
#[inline]
fn add_assign(&mut self, rhs: A) {
*self = *self + rhs;
}
}
impl<S> SubAssign<S> for TextSize
where
TextSize: Sub<S, Output = TextSize>,
{
#[inline]
fn sub_assign(&mut self, rhs: S) {
*self = *self - rhs;
}
}
impl<A> iter::Sum<A> for TextSize
where
TextSize: Add<A, Output = TextSize>,
{
#[inline]
fn sum<I: Iterator<Item = A>>(iter: I) -> TextSize {
iter.fold(0.into(), Add::add)
}
}

37
crates/ruff_text_size/src/traits.rs Normal file
View File

@ -0,0 +1,37 @@
use {crate::TextSize, std::convert::TryInto};
use priv_in_pub::Sealed;
mod priv_in_pub {
pub trait Sealed {}
}
/// Primitives with a textual length that can be passed to [`TextSize::of`].
pub trait TextLen: Copy + Sealed {
/// The textual length of this primitive.
fn text_len(self) -> TextSize;
}
impl Sealed for &'_ str {}
impl TextLen for &'_ str {
#[inline]
fn text_len(self) -> TextSize {
self.len().try_into().unwrap()
}
}
impl Sealed for &'_ String {}
impl TextLen for &'_ String {
#[inline]
fn text_len(self) -> TextSize {
self.as_str().text_len()
}
}
impl Sealed for char {}
impl TextLen for char {
#[inline]
#[allow(clippy::cast_possible_truncation)]
fn text_len(self) -> TextSize {
(self.len_utf8() as u32).into()
}
}

18
crates/ruff_text_size/tests/auto_traits.rs Normal file
View File

@ -0,0 +1,18 @@
use {
ruff_text_size::{TextRange, TextSize},
static_assertions::assert_impl_all,
std::{
fmt::Debug,
hash::Hash,
marker::{Send, Sync},
panic::{RefUnwindSafe, UnwindSafe},
},
};
// auto traits
assert_impl_all!(TextSize: Send, Sync, Unpin, UnwindSafe, RefUnwindSafe);
assert_impl_all!(TextRange: Send, Sync, Unpin, UnwindSafe, RefUnwindSafe);
// common traits
assert_impl_all!(TextSize: Copy, Debug, Default, Hash, Ord);
assert_impl_all!(TextRange: Copy, Debug, Default, Hash, Eq);

24
crates/ruff_text_size/tests/constructors.rs Normal file
View File

@ -0,0 +1,24 @@
use ruff_text_size::TextSize;
#[derive(Copy, Clone)]
struct BadRope<'a>(&'a [&'a str]);
impl BadRope<'_> {
fn text_len(self) -> TextSize {
self.0.iter().copied().map(TextSize::of).sum()
}
}
#[test]
fn main() {
let x: char = 'c';
let _ = TextSize::of(x);
let x: &str = "hello";
let _ = TextSize::of(x);
let x: &String = &"hello".into();
let _ = TextSize::of(x);
let _ = BadRope(&[""]).text_len();
}

8
crates/ruff_text_size/tests/indexing.rs Normal file
View File

@ -0,0 +1,8 @@
use ruff_text_size::TextRange;
#[test]
fn main() {
let range = TextRange::default();
let _ = &""[range];
let _ = &String::new()[range];
}

79
crates/ruff_text_size/tests/main.rs Normal file
View File

@ -0,0 +1,79 @@
use {
ruff_text_size::{TextRange, TextSize},
std::ops,
};
fn size(x: u32) -> TextSize {
TextSize::from(x)
}
fn range(x: ops::Range<u32>) -> TextRange {
TextRange::new(x.start.into(), x.end.into())
}
#[test]
fn sum() {
let xs: Vec<TextSize> = vec![size(0), size(1), size(2)];
assert_eq!(xs.iter().sum::<TextSize>(), size(3));
assert_eq!(xs.into_iter().sum::<TextSize>(), size(3));
}
#[test]
fn math() {
assert_eq!(size(10) + size(5), size(15));
assert_eq!(size(10) - size(5), size(5));
}
#[test]
fn checked_math() {
assert_eq!(size(1).checked_add(size(1)), Some(size(2)));
assert_eq!(size(1).checked_sub(size(1)), Some(size(0)));
assert_eq!(size(1).checked_sub(size(2)), None);
assert_eq!(size(!0).checked_add(size(1)), None);
}
#[test]
#[rustfmt::skip]
fn contains() {
assert!( range(2..4).contains_range(range(2..3)));
assert!( ! range(2..4).contains_range(range(1..3)));
}
#[test]
fn intersect() {
assert_eq!(range(1..2).intersect(range(2..3)), Some(range(2..2)));
assert_eq!(range(1..5).intersect(range(2..3)), Some(range(2..3)));
assert_eq!(range(1..2).intersect(range(3..4)), None);
}
#[test]
fn cover() {
assert_eq!(range(1..2).cover(range(2..3)), range(1..3));
assert_eq!(range(1..5).cover(range(2..3)), range(1..5));
assert_eq!(range(1..2).cover(range(4..5)), range(1..5));
}
#[test]
fn cover_offset() {
assert_eq!(range(1..3).cover_offset(size(0)), range(0..3));
assert_eq!(range(1..3).cover_offset(size(1)), range(1..3));
assert_eq!(range(1..3).cover_offset(size(2)), range(1..3));
assert_eq!(range(1..3).cover_offset(size(3)), range(1..3));
assert_eq!(range(1..3).cover_offset(size(4)), range(1..4));
}
#[test]
#[rustfmt::skip]
fn contains_point() {
assert!( ! range(1..3).contains(size(0)));
assert!( range(1..3).contains(size(1)));
assert!( range(1..3).contains(size(2)));
assert!( ! range(1..3).contains(size(3)));
assert!( ! range(1..3).contains(size(4)));
assert!( ! range(1..3).contains_inclusive(size(0)));
assert!( range(1..3).contains_inclusive(size(1)));
assert!( range(1..3).contains_inclusive(size(2)));
assert!( range(1..3).contains_inclusive(size(3)));
assert!( ! range(1..3).contains_inclusive(size(4)));
}

83
crates/ruff_text_size/tests/serde.rs Normal file
View File

@ -0,0 +1,83 @@
use {
ruff_text_size::{TextRange, TextSize},
serde_test::{assert_de_tokens_error, assert_tokens, Token},
std::ops,
};
fn size(x: u32) -> TextSize {
TextSize::from(x)
}
fn range(x: ops::Range<u32>) -> TextRange {
TextRange::new(x.start.into(), x.end.into())
}
#[test]
fn size_serialization() {
assert_tokens(&size(00), &[Token::U32(00)]);
assert_tokens(&size(10), &[Token::U32(10)]);
assert_tokens(&size(20), &[Token::U32(20)]);
assert_tokens(&size(30), &[Token::U32(30)]);
}
#[test]
fn range_serialization() {
assert_tokens(
&range(00..10),
&[
Token::Tuple { len: 2 },
Token::U32(00),
Token::U32(10),
Token::TupleEnd,
],
);
assert_tokens(
&range(10..20),
&[
Token::Tuple { len: 2 },
Token::U32(10),
Token::U32(20),
Token::TupleEnd,
],
);
assert_tokens(
&range(20..30),
&[
Token::Tuple { len: 2 },
Token::U32(20),
Token::U32(30),
Token::TupleEnd,
],
);
assert_tokens(
&range(30..40),
&[
Token::Tuple { len: 2 },
Token::U32(30),
Token::U32(40),
Token::TupleEnd,
],
);
}
#[test]
fn invalid_range_deserialization() {
assert_tokens::<TextRange>(
&range(62..92),
&[
Token::Tuple { len: 2 },
Token::U32(62),
Token::U32(92),
Token::TupleEnd,
],
);
assert_de_tokens_error::<TextRange>(
&[
Token::Tuple { len: 2 },
Token::U32(92),
Token::U32(62),
Token::TupleEnd,
],
"invalid range: 92..62",
);
}