Move Token, TokenKind and Tokens to ruff-python-ast (#21760)

crates/ruff_python_ast/src/lib.rs

@@ -29,6 +29,7 @@ pub mod statement_visitor;
 pub mod stmt_if;
 pub mod str;
 pub mod str_prefix;
+pub mod token;
 pub mod traversal;
 pub mod types;
 pub mod visitor;

crates/ruff_python_ast/src/token.rs

@@ -0,0 +1,851 @@
+//! Token kinds for Python source code created by the lexer and consumed by the `ruff_python_parser`.
+//!
+//! This module defines the tokens that the lexer recognizes. The tokens are
+//! loosely based on the token definitions found in the [CPython source].
+//!
+//! [CPython source]: https://github.com/python/cpython/blob/dfc2e065a2e71011017077e549cd2f9bf4944c54/Grammar/Tokens
+
+use std::fmt;
+
+use bitflags::bitflags;
+
+use crate::str::{Quote, TripleQuotes};
+use crate::str_prefix::{
+    AnyStringPrefix, ByteStringPrefix, FStringPrefix, StringLiteralPrefix, TStringPrefix,
+};
+use crate::{AnyStringFlags, BoolOp, Operator, StringFlags, UnaryOp};
+use ruff_text_size::{Ranged, TextRange};
+
+mod tokens;
+
+pub use tokens::{TokenAt, TokenIterWithContext, Tokens};
+
+#[derive(Clone, Copy, PartialEq, Eq)]
+#[cfg_attr(feature = "get-size", derive(get_size2::GetSize))]
+pub struct Token {
+    /// The kind of the token.
+    kind: TokenKind,
+    /// The range of the token.
+    range: TextRange,
+    /// The set of flags describing this token.
+    flags: TokenFlags,
+}
+
+impl Token {
+    pub fn new(kind: TokenKind, range: TextRange, flags: TokenFlags) -> Token {
+        Self { kind, range, flags }
+    }
+
+    /// Returns the token kind.
+    #[inline]
+    pub const fn kind(&self) -> TokenKind {
+        self.kind
+    }
+
+    /// Returns the token as a tuple of (kind, range).
+    #[inline]
+    pub const fn as_tuple(&self) -> (TokenKind, TextRange) {
+        (self.kind, self.range)
+    }
+
+    /// Returns `true` if the current token is a triple-quoted string of any kind.
+    ///
+    /// # Panics
+    ///
+    /// If it isn't a string or any f/t-string tokens.
+    pub fn is_triple_quoted_string(self) -> bool {
+        self.unwrap_string_flags().is_triple_quoted()
+    }
+
+    /// Returns the [`Quote`] style for the current string token of any kind.
+    ///
+    /// # Panics
+    ///
+    /// If it isn't a string or any f/t-string tokens.
+    pub fn string_quote_style(self) -> Quote {
+        self.unwrap_string_flags().quote_style()
+    }
+
+    /// Returns the [`AnyStringFlags`] style for the current string token of any kind.
+    ///
+    /// # Panics
+    ///
+    /// If it isn't a string or any f/t-string tokens.
+    pub fn unwrap_string_flags(self) -> AnyStringFlags {
+        self.string_flags()
+            .unwrap_or_else(|| panic!("token to be a string"))
+    }
+
+    /// Returns true if the current token is a string and it is raw.
+    pub fn string_flags(self) -> Option<AnyStringFlags> {
+        if self.is_any_string() {
+            Some(self.flags.as_any_string_flags())
+        } else {
+            None
+        }
+    }
+
+    /// Returns `true` if this is any kind of string token - including
+    /// tokens in t-strings (which do not have type `str`).
+    const fn is_any_string(self) -> bool {
+        matches!(
+            self.kind,
+            TokenKind::String
+                | TokenKind::FStringStart
+                | TokenKind::FStringMiddle
+                | TokenKind::FStringEnd
+                | TokenKind::TStringStart
+                | TokenKind::TStringMiddle
+                | TokenKind::TStringEnd
+        )
+    }
+}
+
+impl Ranged for Token {
+    fn range(&self) -> TextRange {
+        self.range
+    }
+}
+
+impl fmt::Debug for Token {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{:?} {:?}", self.kind, self.range)?;
+        if !self.flags.is_empty() {
+            f.write_str(" (flags = ")?;
+            let mut first = true;
+            for (name, _) in self.flags.iter_names() {
+                if first {
+                    first = false;
+                } else {
+                    f.write_str(" | ")?;
+                }
+                f.write_str(name)?;
+            }
+            f.write_str(")")?;
+        }
+        Ok(())
+    }
+}
+
+/// A kind of a token.
+#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, PartialOrd, Ord)]
+#[cfg_attr(feature = "get-size", derive(get_size2::GetSize))]
+pub enum TokenKind {
+    /// Token kind for a name, commonly known as an identifier.
+    Name,
+    /// Token kind for an integer.
+    Int,
+    /// Token kind for a floating point number.
+    Float,
+    /// Token kind for a complex number.
+    Complex,
+    /// Token kind for a string.
+    String,
+    /// Token kind for the start of an f-string. This includes the `f`/`F`/`fr` prefix
+    /// and the opening quote(s).
+    FStringStart,
+    /// Token kind that includes the portion of text inside the f-string that's not
+    /// part of the expression part and isn't an opening or closing brace.
+    FStringMiddle,
+    /// Token kind for the end of an f-string. This includes the closing quote.
+    FStringEnd,
+    /// Token kind for the start of a t-string. This includes the `t`/`T`/`tr` prefix
+    /// and the opening quote(s).
+    TStringStart,
+    /// Token kind that includes the portion of text inside the t-string that's not
+    /// part of the interpolation part and isn't an opening or closing brace.
+    TStringMiddle,
+    /// Token kind for the end of a t-string. This includes the closing quote.
+    TStringEnd,
+    /// Token kind for a IPython escape command.
+    IpyEscapeCommand,
+    /// Token kind for a comment. These are filtered out of the token stream prior to parsing.
+    Comment,
+    /// Token kind for a newline.
+    Newline,
+    /// Token kind for a newline that is not a logical line break. These are filtered out of
+    /// the token stream prior to parsing.
+    NonLogicalNewline,
+    /// Token kind for an indent.
+    Indent,
+    /// Token kind for a dedent.
+    Dedent,
+    EndOfFile,
+    /// Token kind for a question mark `?`.
+    Question,
+    /// Token kind for an exclamation mark `!`.
+    Exclamation,
+    /// Token kind for a left parenthesis `(`.
+    Lpar,
+    /// Token kind for a right parenthesis `)`.
+    Rpar,
+    /// Token kind for a left square bracket `[`.
+    Lsqb,
+    /// Token kind for a right square bracket `]`.
+    Rsqb,
+    /// Token kind for a colon `:`.
+    Colon,
+    /// Token kind for a comma `,`.
+    Comma,
+    /// Token kind for a semicolon `;`.
+    Semi,
+    /// Token kind for plus `+`.
+    Plus,
+    /// Token kind for minus `-`.
+    Minus,
+    /// Token kind for star `*`.
+    Star,
+    /// Token kind for slash `/`.
+    Slash,
+    /// Token kind for vertical bar `|`.
+    Vbar,
+    /// Token kind for ampersand `&`.
+    Amper,
+    /// Token kind for less than `<`.
+    Less,
+    /// Token kind for greater than `>`.
+    Greater,
+    /// Token kind for equal `=`.
+    Equal,
+    /// Token kind for dot `.`.
+    Dot,
+    /// Token kind for percent `%`.
+    Percent,
+    /// Token kind for left bracket `{`.
+    Lbrace,
+    /// Token kind for right bracket `}`.
+    Rbrace,
+    /// Token kind for double equal `==`.
+    EqEqual,
+    /// Token kind for not equal `!=`.
+    NotEqual,
+    /// Token kind for less than or equal `<=`.
+    LessEqual,
+    /// Token kind for greater than or equal `>=`.
+    GreaterEqual,
+    /// Token kind for tilde `~`.
+    Tilde,
+    /// Token kind for caret `^`.
+    CircumFlex,
+    /// Token kind for left shift `<<`.
+    LeftShift,
+    /// Token kind for right shift `>>`.
+    RightShift,
+    /// Token kind for double star `**`.
+    DoubleStar,
+    /// Token kind for double star equal `**=`.
+    DoubleStarEqual,
+    /// Token kind for plus equal `+=`.
+    PlusEqual,
+    /// Token kind for minus equal `-=`.
+    MinusEqual,
+    /// Token kind for star equal `*=`.
+    StarEqual,
+    /// Token kind for slash equal `/=`.
+    SlashEqual,
+    /// Token kind for percent equal `%=`.
+    PercentEqual,
+    /// Token kind for ampersand equal `&=`.
+    AmperEqual,
+    /// Token kind for vertical bar equal `|=`.
+    VbarEqual,
+    /// Token kind for caret equal `^=`.
+    CircumflexEqual,
+    /// Token kind for left shift equal `<<=`.
+    LeftShiftEqual,
+    /// Token kind for right shift equal `>>=`.
+    RightShiftEqual,
+    /// Token kind for double slash `//`.
+    DoubleSlash,
+    /// Token kind for double slash equal `//=`.
+    DoubleSlashEqual,
+    /// Token kind for colon equal `:=`.
+    ColonEqual,
+    /// Token kind for at `@`.
+    At,
+    /// Token kind for at equal `@=`.
+    AtEqual,
+    /// Token kind for arrow `->`.
+    Rarrow,
+    /// Token kind for ellipsis `...`.
+    Ellipsis,
+
+    // The keywords should be sorted in alphabetical order. If the boundary tokens for the
+    // "Keywords" and "Soft keywords" group change, update the related methods on `TokenKind`.
+
+    // Keywords
+    And,
+    As,
+    Assert,
+    Async,
+    Await,
+    Break,
+    Class,
+    Continue,
+    Def,
+    Del,
+    Elif,
+    Else,
+    Except,
+    False,
+    Finally,
+    For,
+    From,
+    Global,
+    If,
+    Import,
+    In,
+    Is,
+    Lambda,
+    None,
+    Nonlocal,
+    Not,
+    Or,
+    Pass,
+    Raise,
+    Return,
+    True,
+    Try,
+    While,
+    With,
+    Yield,
+
+    // Soft keywords
+    Case,
+    Match,
+    Type,
+
+    Unknown,
+}
+
+impl TokenKind {
+    /// Returns `true` if this is an end of file token.
+    #[inline]
+    pub const fn is_eof(self) -> bool {
+        matches!(self, TokenKind::EndOfFile)
+    }
+
+    /// Returns `true` if this is either a newline or non-logical newline token.
+    #[inline]
+    pub const fn is_any_newline(self) -> bool {
+        matches!(self, TokenKind::Newline | TokenKind::NonLogicalNewline)
+    }
+
+    /// Returns `true` if the token is a keyword (including soft keywords).
+    ///
+    /// See also [`is_soft_keyword`], [`is_non_soft_keyword`].
+    ///
+    /// [`is_soft_keyword`]: TokenKind::is_soft_keyword
+    /// [`is_non_soft_keyword`]: TokenKind::is_non_soft_keyword
+    #[inline]
+    pub fn is_keyword(self) -> bool {
+        TokenKind::And <= self && self <= TokenKind::Type
+    }
+
+    /// Returns `true` if the token is strictly a soft keyword.
+    ///
+    /// See also [`is_keyword`], [`is_non_soft_keyword`].
+    ///
+    /// [`is_keyword`]: TokenKind::is_keyword
+    /// [`is_non_soft_keyword`]: TokenKind::is_non_soft_keyword
+    #[inline]
+    pub fn is_soft_keyword(self) -> bool {
+        TokenKind::Case <= self && self <= TokenKind::Type
+    }
+
+    /// Returns `true` if the token is strictly a non-soft keyword.
+    ///
+    /// See also [`is_keyword`], [`is_soft_keyword`].
+    ///
+    /// [`is_keyword`]: TokenKind::is_keyword
+    /// [`is_soft_keyword`]: TokenKind::is_soft_keyword
+    #[inline]
+    pub fn is_non_soft_keyword(self) -> bool {
+        TokenKind::And <= self && self <= TokenKind::Yield
+    }
+
+    #[inline]
+    pub const fn is_operator(self) -> bool {
+        matches!(
+            self,
+            TokenKind::Lpar
+                | TokenKind::Rpar
+                | TokenKind::Lsqb
+                | TokenKind::Rsqb
+                | TokenKind::Comma
+                | TokenKind::Semi
+                | TokenKind::Plus
+                | TokenKind::Minus
+                | TokenKind::Star
+                | TokenKind::Slash
+                | TokenKind::Vbar
+                | TokenKind::Amper
+                | TokenKind::Less
+                | TokenKind::Greater
+                | TokenKind::Equal
+                | TokenKind::Dot
+                | TokenKind::Percent
+                | TokenKind::Lbrace
+                | TokenKind::Rbrace
+                | TokenKind::EqEqual
+                | TokenKind::NotEqual
+                | TokenKind::LessEqual
+                | TokenKind::GreaterEqual
+                | TokenKind::Tilde
+                | TokenKind::CircumFlex
+                | TokenKind::LeftShift
+                | TokenKind::RightShift
+                | TokenKind::DoubleStar
+                | TokenKind::PlusEqual
+                | TokenKind::MinusEqual
+                | TokenKind::StarEqual
+                | TokenKind::SlashEqual
+                | TokenKind::PercentEqual
+                | TokenKind::AmperEqual
+                | TokenKind::VbarEqual
+                | TokenKind::CircumflexEqual
+                | TokenKind::LeftShiftEqual
+                | TokenKind::RightShiftEqual
+                | TokenKind::DoubleStarEqual
+                | TokenKind::DoubleSlash
+                | TokenKind::DoubleSlashEqual
+                | TokenKind::At
+                | TokenKind::AtEqual
+                | TokenKind::Rarrow
+                | TokenKind::Ellipsis
+                | TokenKind::ColonEqual
+                | TokenKind::Colon
+                | TokenKind::And
+                | TokenKind::Or
+                | TokenKind::Not
+                | TokenKind::In
+                | TokenKind::Is
+        )
+    }
+
+    /// Returns `true` if this is a singleton token i.e., `True`, `False`, or `None`.
+    #[inline]
+    pub const fn is_singleton(self) -> bool {
+        matches!(self, TokenKind::False | TokenKind::True | TokenKind::None)
+    }
+
+    /// Returns `true` if this is a trivia token i.e., a comment or a non-logical newline.
+    #[inline]
+    pub const fn is_trivia(&self) -> bool {
+        matches!(self, TokenKind::Comment | TokenKind::NonLogicalNewline)
+    }
+
+    /// Returns `true` if this is a comment token.
+    #[inline]
+    pub const fn is_comment(&self) -> bool {
+        matches!(self, TokenKind::Comment)
+    }
+
+    #[inline]
+    pub const fn is_arithmetic(self) -> bool {
+        matches!(
+            self,
+            TokenKind::DoubleStar
+                | TokenKind::Star
+                | TokenKind::Plus
+                | TokenKind::Minus
+                | TokenKind::Slash
+                | TokenKind::DoubleSlash
+                | TokenKind::At
+        )
+    }
+
+    #[inline]
+    pub const fn is_bitwise_or_shift(self) -> bool {
+        matches!(
+            self,
+            TokenKind::LeftShift
+                | TokenKind::LeftShiftEqual
+                | TokenKind::RightShift
+                | TokenKind::RightShiftEqual
+                | TokenKind::Amper
+                | TokenKind::AmperEqual
+                | TokenKind::Vbar
+                | TokenKind::VbarEqual
+                | TokenKind::CircumFlex
+                | TokenKind::CircumflexEqual
+                | TokenKind::Tilde
+        )
+    }
+
+    /// Returns `true` if the current token is a unary arithmetic operator.
+    #[inline]
+    pub const fn is_unary_arithmetic_operator(self) -> bool {
+        matches!(self, TokenKind::Plus | TokenKind::Minus)
+    }
+
+    #[inline]
+    pub const fn is_interpolated_string_end(self) -> bool {
+        matches!(self, TokenKind::FStringEnd | TokenKind::TStringEnd)
+    }
+
+    /// Returns the [`UnaryOp`] that corresponds to this token kind, if it is a unary arithmetic
+    /// operator, otherwise return [None].
+    ///
+    /// Use [`as_unary_operator`] to match against any unary operator.
+    ///
+    /// [`as_unary_operator`]: TokenKind::as_unary_operator
+    #[inline]
+    pub const fn as_unary_arithmetic_operator(self) -> Option<UnaryOp> {
+        Some(match self {
+            TokenKind::Plus => UnaryOp::UAdd,
+            TokenKind::Minus => UnaryOp::USub,
+            _ => return None,
+        })
+    }
+
+    /// Returns the [`UnaryOp`] that corresponds to this token kind, if it is a unary operator,
+    /// otherwise return [None].
+    ///
+    /// Use [`as_unary_arithmetic_operator`] to match against only an arithmetic unary operator.
+    ///
+    /// [`as_unary_arithmetic_operator`]: TokenKind::as_unary_arithmetic_operator
+    #[inline]
+    pub const fn as_unary_operator(self) -> Option<UnaryOp> {
+        Some(match self {
+            TokenKind::Plus => UnaryOp::UAdd,
+            TokenKind::Minus => UnaryOp::USub,
+            TokenKind::Tilde => UnaryOp::Invert,
+            TokenKind::Not => UnaryOp::Not,
+            _ => return None,
+        })
+    }
+
+    /// Returns the [`BoolOp`] that corresponds to this token kind, if it is a boolean operator,
+    /// otherwise return [None].
+    #[inline]
+    pub const fn as_bool_operator(self) -> Option<BoolOp> {
+        Some(match self {
+            TokenKind::And => BoolOp::And,
+            TokenKind::Or => BoolOp::Or,
+            _ => return None,
+        })
+    }
+
+    /// Returns the binary [`Operator`] that corresponds to the current token, if it's a binary
+    /// operator, otherwise return [None].
+    ///
+    /// Use [`as_augmented_assign_operator`] to match against an augmented assignment token.
+    ///
+    /// [`as_augmented_assign_operator`]: TokenKind::as_augmented_assign_operator
+    pub const fn as_binary_operator(self) -> Option<Operator> {
+        Some(match self {
+            TokenKind::Plus => Operator::Add,
+            TokenKind::Minus => Operator::Sub,
+            TokenKind::Star => Operator::Mult,
+            TokenKind::At => Operator::MatMult,
+            TokenKind::DoubleStar => Operator::Pow,
+            TokenKind::Slash => Operator::Div,
+            TokenKind::DoubleSlash => Operator::FloorDiv,
+            TokenKind::Percent => Operator::Mod,
+            TokenKind::Amper => Operator::BitAnd,
+            TokenKind::Vbar => Operator::BitOr,
+            TokenKind::CircumFlex => Operator::BitXor,
+            TokenKind::LeftShift => Operator::LShift,
+            TokenKind::RightShift => Operator::RShift,
+            _ => return None,
+        })
+    }
+
+    /// Returns the [`Operator`] that corresponds to this token kind, if it is
+    /// an augmented assignment operator, or [`None`] otherwise.
+    #[inline]
+    pub const fn as_augmented_assign_operator(self) -> Option<Operator> {
+        Some(match self {
+            TokenKind::PlusEqual => Operator::Add,
+            TokenKind::MinusEqual => Operator::Sub,
+            TokenKind::StarEqual => Operator::Mult,
+            TokenKind::AtEqual => Operator::MatMult,
+            TokenKind::DoubleStarEqual => Operator::Pow,
+            TokenKind::SlashEqual => Operator::Div,
+            TokenKind::DoubleSlashEqual => Operator::FloorDiv,
+            TokenKind::PercentEqual => Operator::Mod,
+            TokenKind::AmperEqual => Operator::BitAnd,
+            TokenKind::VbarEqual => Operator::BitOr,
+            TokenKind::CircumflexEqual => Operator::BitXor,
+            TokenKind::LeftShiftEqual => Operator::LShift,
+            TokenKind::RightShiftEqual => Operator::RShift,
+            _ => return None,
+        })
+    }
+}
+
+impl From<BoolOp> for TokenKind {
+    #[inline]
+    fn from(op: BoolOp) -> Self {
+        match op {
+            BoolOp::And => TokenKind::And,
+            BoolOp::Or => TokenKind::Or,
+        }
+    }
+}
+
+impl From<UnaryOp> for TokenKind {
+    #[inline]
+    fn from(op: UnaryOp) -> Self {
+        match op {
+            UnaryOp::Invert => TokenKind::Tilde,
+            UnaryOp::Not => TokenKind::Not,
+            UnaryOp::UAdd => TokenKind::Plus,
+            UnaryOp::USub => TokenKind::Minus,
+        }
+    }
+}
+
+impl From<Operator> for TokenKind {
+    #[inline]
+    fn from(op: Operator) -> Self {
+        match op {
+            Operator::Add => TokenKind::Plus,
+            Operator::Sub => TokenKind::Minus,
+            Operator::Mult => TokenKind::Star,
+            Operator::MatMult => TokenKind::At,
+            Operator::Div => TokenKind::Slash,
+            Operator::Mod => TokenKind::Percent,
+            Operator::Pow => TokenKind::DoubleStar,
+            Operator::LShift => TokenKind::LeftShift,
+            Operator::RShift => TokenKind::RightShift,
+            Operator::BitOr => TokenKind::Vbar,
+            Operator::BitXor => TokenKind::CircumFlex,
+            Operator::BitAnd => TokenKind::Amper,
+            Operator::FloorDiv => TokenKind::DoubleSlash,
+        }
+    }
+}
+
+impl fmt::Display for TokenKind {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let value = match self {
+            TokenKind::Unknown => "Unknown",
+            TokenKind::Newline => "newline",
+            TokenKind::NonLogicalNewline => "NonLogicalNewline",
+            TokenKind::Indent => "indent",
+            TokenKind::Dedent => "dedent",
+            TokenKind::EndOfFile => "end of file",
+            TokenKind::Name => "name",
+            TokenKind::Int => "int",
+            TokenKind::Float => "float",
+            TokenKind::Complex => "complex",
+            TokenKind::String => "string",
+            TokenKind::FStringStart => "FStringStart",
+            TokenKind::FStringMiddle => "FStringMiddle",
+            TokenKind::FStringEnd => "FStringEnd",
+            TokenKind::TStringStart => "TStringStart",
+            TokenKind::TStringMiddle => "TStringMiddle",
+            TokenKind::TStringEnd => "TStringEnd",
+            TokenKind::IpyEscapeCommand => "IPython escape command",
+            TokenKind::Comment => "comment",
+            TokenKind::Question => "`?`",
+            TokenKind::Exclamation => "`!`",
+            TokenKind::Lpar => "`(`",
+            TokenKind::Rpar => "`)`",
+            TokenKind::Lsqb => "`[`",
+            TokenKind::Rsqb => "`]`",
+            TokenKind::Lbrace => "`{`",
+            TokenKind::Rbrace => "`}`",
+            TokenKind::Equal => "`=`",
+            TokenKind::ColonEqual => "`:=`",
+            TokenKind::Dot => "`.`",
+            TokenKind::Colon => "`:`",
+            TokenKind::Semi => "`;`",
+            TokenKind::Comma => "`,`",
+            TokenKind::Rarrow => "`->`",
+            TokenKind::Plus => "`+`",
+            TokenKind::Minus => "`-`",
+            TokenKind::Star => "`*`",
+            TokenKind::DoubleStar => "`**`",
+            TokenKind::Slash => "`/`",
+            TokenKind::DoubleSlash => "`//`",
+            TokenKind::Percent => "`%`",
+            TokenKind::Vbar => "`|`",
+            TokenKind::Amper => "`&`",
+            TokenKind::CircumFlex => "`^`",
+            TokenKind::LeftShift => "`<<`",
+            TokenKind::RightShift => "`>>`",
+            TokenKind::Tilde => "`~`",
+            TokenKind::At => "`@`",
+            TokenKind::Less => "`<`",
+            TokenKind::Greater => "`>`",
+            TokenKind::EqEqual => "`==`",
+            TokenKind::NotEqual => "`!=`",
+            TokenKind::LessEqual => "`<=`",
+            TokenKind::GreaterEqual => "`>=`",
+            TokenKind::PlusEqual => "`+=`",
+            TokenKind::MinusEqual => "`-=`",
+            TokenKind::StarEqual => "`*=`",
+            TokenKind::DoubleStarEqual => "`**=`",
+            TokenKind::SlashEqual => "`/=`",
+            TokenKind::DoubleSlashEqual => "`//=`",
+            TokenKind::PercentEqual => "`%=`",
+            TokenKind::VbarEqual => "`|=`",
+            TokenKind::AmperEqual => "`&=`",
+            TokenKind::CircumflexEqual => "`^=`",
+            TokenKind::LeftShiftEqual => "`<<=`",
+            TokenKind::RightShiftEqual => "`>>=`",
+            TokenKind::AtEqual => "`@=`",
+            TokenKind::Ellipsis => "`...`",
+            TokenKind::False => "`False`",
+            TokenKind::None => "`None`",
+            TokenKind::True => "`True`",
+            TokenKind::And => "`and`",
+            TokenKind::As => "`as`",
+            TokenKind::Assert => "`assert`",
+            TokenKind::Async => "`async`",
+            TokenKind::Await => "`await`",
+            TokenKind::Break => "`break`",
+            TokenKind::Class => "`class`",
+            TokenKind::Continue => "`continue`",
+            TokenKind::Def => "`def`",
+            TokenKind::Del => "`del`",
+            TokenKind::Elif => "`elif`",
+            TokenKind::Else => "`else`",
+            TokenKind::Except => "`except`",
+            TokenKind::Finally => "`finally`",
+            TokenKind::For => "`for`",
+            TokenKind::From => "`from`",
+            TokenKind::Global => "`global`",
+            TokenKind::If => "`if`",
+            TokenKind::Import => "`import`",
+            TokenKind::In => "`in`",
+            TokenKind::Is => "`is`",
+            TokenKind::Lambda => "`lambda`",
+            TokenKind::Nonlocal => "`nonlocal`",
+            TokenKind::Not => "`not`",
+            TokenKind::Or => "`or`",
+            TokenKind::Pass => "`pass`",
+            TokenKind::Raise => "`raise`",
+            TokenKind::Return => "`return`",
+            TokenKind::Try => "`try`",
+            TokenKind::While => "`while`",
+            TokenKind::Match => "`match`",
+            TokenKind::Type => "`type`",
+            TokenKind::Case => "`case`",
+            TokenKind::With => "`with`",
+            TokenKind::Yield => "`yield`",
+        };
+        f.write_str(value)
+    }
+}
+
+bitflags! {
+    #[derive(Clone, Copy, Debug, PartialEq, Eq)]
+    pub struct TokenFlags: u16 {
+        /// The token is a string with double quotes (`"`).
+        const DOUBLE_QUOTES = 1 << 0;
+        /// The token is a triple-quoted string i.e., it starts and ends with three consecutive
+        /// quote characters (`"""` or `'''`).
+        const TRIPLE_QUOTED_STRING = 1 << 1;
+
+        /// The token is a unicode string i.e., prefixed with `u` or `U`
+        const UNICODE_STRING = 1 << 2;
+        /// The token is a byte string i.e., prefixed with `b` or `B`
+        const BYTE_STRING = 1 << 3;
+        /// The token is an f-string i.e., prefixed with `f` or `F`
+        const F_STRING = 1 << 4;
+        /// The token is a t-string i.e., prefixed with `t` or `T`
+        const T_STRING = 1 << 5;
+        /// The token is a raw string and the prefix character is in lowercase.
+        const RAW_STRING_LOWERCASE = 1 << 6;
+        /// The token is a raw string and the prefix character is in uppercase.
+        const RAW_STRING_UPPERCASE = 1 << 7;
+        /// String without matching closing quote(s)
+        const UNCLOSED_STRING = 1 << 8;
+
+        /// The token is a raw string i.e., prefixed with `r` or `R`
+        const RAW_STRING = Self::RAW_STRING_LOWERCASE.bits() | Self::RAW_STRING_UPPERCASE.bits();
+
+    }
+}
+
+#[cfg(feature = "get-size")]
+impl get_size2::GetSize for TokenFlags {}
+
+impl StringFlags for TokenFlags {
+    fn quote_style(self) -> Quote {
+        if self.intersects(TokenFlags::DOUBLE_QUOTES) {
+            Quote::Double
+        } else {
+            Quote::Single
+        }
+    }
+
+    fn triple_quotes(self) -> TripleQuotes {
+        if self.intersects(TokenFlags::TRIPLE_QUOTED_STRING) {
+            TripleQuotes::Yes
+        } else {
+            TripleQuotes::No
+        }
+    }
+
+    fn prefix(self) -> AnyStringPrefix {
+        if self.intersects(TokenFlags::F_STRING) {
+            if self.intersects(TokenFlags::RAW_STRING_LOWERCASE) {
+                AnyStringPrefix::Format(FStringPrefix::Raw { uppercase_r: false })
+            } else if self.intersects(TokenFlags::RAW_STRING_UPPERCASE) {
+                AnyStringPrefix::Format(FStringPrefix::Raw { uppercase_r: true })
+            } else {
+                AnyStringPrefix::Format(FStringPrefix::Regular)
+            }
+        } else if self.intersects(TokenFlags::T_STRING) {
+            if self.intersects(TokenFlags::RAW_STRING_LOWERCASE) {
+                AnyStringPrefix::Template(TStringPrefix::Raw { uppercase_r: false })
+            } else if self.intersects(TokenFlags::RAW_STRING_UPPERCASE) {
+                AnyStringPrefix::Template(TStringPrefix::Raw { uppercase_r: true })
+            } else {
+                AnyStringPrefix::Template(TStringPrefix::Regular)
+            }
+        } else if self.intersects(TokenFlags::BYTE_STRING) {
+            if self.intersects(TokenFlags::RAW_STRING_LOWERCASE) {
+                AnyStringPrefix::Bytes(ByteStringPrefix::Raw { uppercase_r: false })
+            } else if self.intersects(TokenFlags::RAW_STRING_UPPERCASE) {
+                AnyStringPrefix::Bytes(ByteStringPrefix::Raw { uppercase_r: true })
+            } else {
+                AnyStringPrefix::Bytes(ByteStringPrefix::Regular)
+            }
+        } else if self.intersects(TokenFlags::RAW_STRING_LOWERCASE) {
+            AnyStringPrefix::Regular(StringLiteralPrefix::Raw { uppercase: false })
+        } else if self.intersects(TokenFlags::RAW_STRING_UPPERCASE) {
+            AnyStringPrefix::Regular(StringLiteralPrefix::Raw { uppercase: true })
+        } else if self.intersects(TokenFlags::UNICODE_STRING) {
+            AnyStringPrefix::Regular(StringLiteralPrefix::Unicode)
+        } else {
+            AnyStringPrefix::Regular(StringLiteralPrefix::Empty)
+        }
+    }
+
+    fn is_unclosed(self) -> bool {
+        self.intersects(TokenFlags::UNCLOSED_STRING)
+    }
+}
+
+impl TokenFlags {
+    /// Returns `true` if the token is an f-string.
+    pub const fn is_f_string(self) -> bool {
+        self.intersects(TokenFlags::F_STRING)
+    }
+
+    /// Returns `true` if the token is a t-string.
+    pub const fn is_t_string(self) -> bool {
+        self.intersects(TokenFlags::T_STRING)
+    }
+
+    /// Returns `true` if the token is a t-string.
+    pub const fn is_interpolated_string(self) -> bool {
+        self.intersects(TokenFlags::T_STRING.union(TokenFlags::F_STRING))
+    }
+
+    /// Returns `true` if the token is a triple-quoted t-string.
+    pub fn is_triple_quoted_interpolated_string(self) -> bool {
+        self.intersects(TokenFlags::TRIPLE_QUOTED_STRING) && self.is_interpolated_string()
+    }
+
+    /// Returns `true` if the token is a raw string.
+    pub const fn is_raw_string(self) -> bool {
+        self.intersects(TokenFlags::RAW_STRING)
+    }
+}

crates/ruff_python_ast/src/token/tokens.rs

@@ -0,0 +1,520 @@
+use std::{iter::FusedIterator, ops::Deref};
+
+use super::{Token, TokenKind};
+use ruff_python_trivia::CommentRanges;
+use ruff_text_size::{Ranged as _, TextRange, TextSize};
+
+/// Tokens represents a vector of lexed [`Token`].
+#[derive(Debug, Clone, PartialEq, Eq)]
+#[cfg_attr(feature = "get-size", derive(get_size2::GetSize))]
+pub struct Tokens {
+    raw: Vec<Token>,
+}
+
+impl Tokens {
+    pub fn new(tokens: Vec<Token>) -> Tokens {
+        Tokens { raw: tokens }
+    }
+
+    /// Returns an iterator over all the tokens that provides context.
+    pub fn iter_with_context(&self) -> TokenIterWithContext<'_> {
+        TokenIterWithContext::new(&self.raw)
+    }
+
+    /// Performs a binary search to find the index of the **first** token that starts at the given `offset`.
+    ///
+    /// Unlike `binary_search_by_key`, this method ensures that if multiple tokens start at the same offset,
+    /// it returns the index of the first one. Multiple tokens can start at the same offset in cases where
+    /// zero-length tokens are involved (like `Dedent` or `Newline` at the end of the file).
+    pub fn binary_search_by_start(&self, offset: TextSize) -> Result<usize, usize> {
+        let partition_point = self.partition_point(|token| token.start() < offset);
+
+        let after = &self[partition_point..];
+
+        if after.first().is_some_and(|first| first.start() == offset) {
+            Ok(partition_point)
+        } else {
+            Err(partition_point)
+        }
+    }
+
+    /// Returns a slice of [`Token`] that are within the given `range`.
+    ///
+    /// The start and end offset of the given range should be either:
+    /// 1. Token boundary
+    /// 2. Gap between the tokens
+    ///
+    /// For example, considering the following tokens and their corresponding range:
+    ///
+    /// | Token               | Range     |
+    /// |---------------------|-----------|
+    /// | `Def`               | `0..3`    |
+    /// | `Name`              | `4..7`    |
+    /// | `Lpar`              | `7..8`    |
+    /// | `Rpar`              | `8..9`    |
+    /// | `Colon`             | `9..10`   |
+    /// | `Newline`           | `10..11`  |
+    /// | `Comment`           | `15..24`  |
+    /// | `NonLogicalNewline` | `24..25`  |
+    /// | `Indent`            | `25..29`  |
+    /// | `Pass`              | `29..33`  |
+    ///
+    /// Here, for (1) a token boundary is considered either the start or end offset of any of the
+    /// above tokens. For (2), the gap would be any offset between the `Newline` and `Comment`
+    /// token which are 12, 13, and 14.
+    ///
+    /// Examples:
+    /// 1) `4..10` would give `Name`, `Lpar`, `Rpar`, `Colon`
+    /// 2) `11..25` would give `Comment`, `NonLogicalNewline`
+    /// 3) `12..25` would give same as (2) and offset 12 is in the "gap"
+    /// 4) `9..12` would give `Colon`, `Newline` and offset 12 is in the "gap"
+    /// 5) `18..27` would panic because both the start and end offset is within a token
+    ///
+    /// ## Note
+    ///
+    /// The returned slice can contain the [`TokenKind::Unknown`] token if there was a lexical
+    /// error encountered within the given range.
+    ///
+    /// # Panics
+    ///
+    /// If either the start or end offset of the given range is within a token range.
+    pub fn in_range(&self, range: TextRange) -> &[Token] {
+        let tokens_after_start = self.after(range.start());
+
+        Self::before_impl(tokens_after_start, range.end())
+    }
+
+    /// Searches the token(s) at `offset`.
+    ///
+    /// Returns [`TokenAt::Between`] if `offset` points directly inbetween two tokens
+    /// (the left token ends at `offset` and the right token starts at `offset`).
+    pub fn at_offset(&self, offset: TextSize) -> TokenAt {
+        match self.binary_search_by_start(offset) {
+            // The token at `index` starts exactly at `offset.
+            // ```python
+            // object.attribute
+            //        ^ OFFSET
+            // ```
+            Ok(index) => {
+                let token = self[index];
+                // `token` starts exactly at `offset`. Test if the offset is right between
+                // `token` and the previous token (if there's any)
+                if let Some(previous) = index.checked_sub(1).map(|idx| self[idx]) {
+                    if previous.end() == offset {
+                        return TokenAt::Between(previous, token);
+                    }
+                }
+
+                TokenAt::Single(token)
+            }
+
+            // No token found that starts exactly at the given offset. But it's possible that
+            // the token starting before `offset` fully encloses `offset` (it's end range ends after `offset`).
+            // ```python
+            // object.attribute
+            //   ^ OFFSET
+            // # or
+            // if True:
+            //     print("test")
+            //  ^ OFFSET
+            // ```
+            Err(index) => {
+                if let Some(previous) = index.checked_sub(1).map(|idx| self[idx]) {
+                    if previous.range().contains_inclusive(offset) {
+                        return TokenAt::Single(previous);
+                    }
+                }
+
+                TokenAt::None
+            }
+        }
+    }
+
+    /// Returns a slice of tokens before the given [`TextSize`] offset.
+    ///
+    /// If the given offset is between two tokens, the returned slice will end just before the
+    /// following token. In other words, if the offset is between the end of previous token and
+    /// start of next token, the returned slice will end just before the next token.
+    ///
+    /// # Panics
+    ///
+    /// If the given offset is inside a token range at any point
+    /// other than the start of the range.
+    pub fn before(&self, offset: TextSize) -> &[Token] {
+        Self::before_impl(&self.raw, offset)
+    }
+
+    fn before_impl(tokens: &[Token], offset: TextSize) -> &[Token] {
+        let partition_point = tokens.partition_point(|token| token.start() < offset);
+        let before = &tokens[..partition_point];
+
+        if let Some(last) = before.last() {
+            // If it's equal to the end offset, then it's at a token boundary which is
+            // valid. If it's greater than the end offset, then it's in the gap between
+            // the tokens which is valid as well.
+            assert!(
+                offset >= last.end(),
+                "Offset {:?} is inside a token range {:?}",
+                offset,
+                last.range()
+            );
+        }
+        before
+    }
+
+    /// Returns a slice of tokens after the given [`TextSize`] offset.
+    ///
+    /// If the given offset is between two tokens, the returned slice will start from the following
+    /// token. In other words, if the offset is between the end of previous token and start of next
+    /// token, the returned slice will start from the next token.
+    ///
+    /// # Panics
+    ///
+    /// If the given offset is inside a token range at any point
+    /// other than the start of the range.
+    pub fn after(&self, offset: TextSize) -> &[Token] {
+        let partition_point = self.partition_point(|token| token.end() <= offset);
+        let after = &self[partition_point..];
+
+        if let Some(first) = after.first() {
+            // valid. If it's greater than the end offset, then it's in the gap between
+            // the tokens which is valid as well.
+            assert!(
+                offset <= first.start(),
+                "Offset {:?} is inside a token range {:?}",
+                offset,
+                first.range()
+            );
+        }
+
+        after
+    }
+}
+
+impl<'a> IntoIterator for &'a Tokens {
+    type Item = &'a Token;
+    type IntoIter = std::slice::Iter<'a, Token>;
+
+    fn into_iter(self) -> Self::IntoIter {
+        self.iter()
+    }
+}
+
+impl Deref for Tokens {
+    type Target = [Token];
+
+    fn deref(&self) -> &Self::Target {
+        &self.raw
+    }
+}
+
+/// A token that encloses a given offset or ends exactly at it.
+#[derive(Debug, Clone)]
+pub enum TokenAt {
+    /// There's no token at the given offset
+    None,
+
+    /// There's a single token at the given offset.
+    Single(Token),
+
+    /// The offset falls exactly between two tokens. E.g. `CURSOR` in `call<CURSOR>(arguments)` is
+    /// positioned exactly between the `call` and `(` tokens.
+    Between(Token, Token),
+}
+
+impl Iterator for TokenAt {
+    type Item = Token;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        match *self {
+            TokenAt::None => None,
+            TokenAt::Single(token) => {
+                *self = TokenAt::None;
+                Some(token)
+            }
+            TokenAt::Between(first, second) => {
+                *self = TokenAt::Single(second);
+                Some(first)
+            }
+        }
+    }
+}
+
+impl FusedIterator for TokenAt {}
+
+impl From<&Tokens> for CommentRanges {
+    fn from(tokens: &Tokens) -> Self {
+        let mut ranges = vec![];
+        for token in tokens {
+            if token.kind() == TokenKind::Comment {
+                ranges.push(token.range());
+            }
+        }
+        CommentRanges::new(ranges)
+    }
+}
+
+/// An iterator over the [`Token`]s with context.
+///
+/// This struct is created by the [`iter_with_context`] method on [`Tokens`]. Refer to its
+/// documentation for more details.
+///
+/// [`iter_with_context`]: Tokens::iter_with_context
+#[derive(Debug, Clone)]
+pub struct TokenIterWithContext<'a> {
+    inner: std::slice::Iter<'a, Token>,
+    nesting: u32,
+}
+
+impl<'a> TokenIterWithContext<'a> {
+    fn new(tokens: &'a [Token]) -> TokenIterWithContext<'a> {
+        TokenIterWithContext {
+            inner: tokens.iter(),
+            nesting: 0,
+        }
+    }
+
+    /// Return the nesting level the iterator is currently in.
+    pub const fn nesting(&self) -> u32 {
+        self.nesting
+    }
+
+    /// Returns `true` if the iterator is within a parenthesized context.
+    pub const fn in_parenthesized_context(&self) -> bool {
+        self.nesting > 0
+    }
+
+    /// Returns the next [`Token`] in the iterator without consuming it.
+    pub fn peek(&self) -> Option<&'a Token> {
+        self.clone().next()
+    }
+}
+
+impl<'a> Iterator for TokenIterWithContext<'a> {
+    type Item = &'a Token;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        let token = self.inner.next()?;
+
+        match token.kind() {
+            TokenKind::Lpar | TokenKind::Lbrace | TokenKind::Lsqb => self.nesting += 1,
+            TokenKind::Rpar | TokenKind::Rbrace | TokenKind::Rsqb => {
+                self.nesting = self.nesting.saturating_sub(1);
+            }
+            // This mimics the behavior of re-lexing which reduces the nesting level on the lexer.
+            // We don't need to reduce it by 1 because unlike the lexer we see the final token
+            // after recovering from every unclosed parenthesis.
+            TokenKind::Newline if self.nesting > 0 => {
+                self.nesting = 0;
+            }
+            _ => {}
+        }
+
+        Some(token)
+    }
+}
+
+impl FusedIterator for TokenIterWithContext<'_> {}
+
+#[cfg(test)]
+mod tests {
+    use std::ops::Range;
+
+    use ruff_text_size::TextSize;
+
+    use crate::token::{Token, TokenFlags, TokenKind};
+
+    use super::*;
+
+    /// Test case containing a "gap" between two tokens.
+    ///
+    /// Code: <https://play.ruff.rs/a3658340-6df8-42c5-be80-178744bf1193>
+    const TEST_CASE_WITH_GAP: [(TokenKind, Range<u32>); 10] = [
+        (TokenKind::Def, 0..3),
+        (TokenKind::Name, 4..7),
+        (TokenKind::Lpar, 7..8),
+        (TokenKind::Rpar, 8..9),
+        (TokenKind::Colon, 9..10),
+        (TokenKind::Newline, 10..11),
+        // Gap               ||..||
+        (TokenKind::Comment, 15..24),
+        (TokenKind::NonLogicalNewline, 24..25),
+        (TokenKind::Indent, 25..29),
+        (TokenKind::Pass, 29..33),
+        // No newline at the end to keep the token set full of unique tokens
+    ];
+
+    /// Helper function to create [`Tokens`] from an iterator of (kind, range).
+    fn new_tokens(tokens: impl Iterator<Item = (TokenKind, Range<u32>)>) -> Tokens {
+        Tokens::new(
+            tokens
+                .map(|(kind, range)| {
+                    Token::new(
+                        kind,
+                        TextRange::new(TextSize::new(range.start), TextSize::new(range.end)),
+                        TokenFlags::empty(),
+                    )
+                })
+                .collect(),
+        )
+    }
+
+    #[test]
+    fn tokens_after_offset_at_token_start() {
+        let tokens = new_tokens(TEST_CASE_WITH_GAP.into_iter());
+        let after = tokens.after(TextSize::new(8));
+        assert_eq!(after.len(), 7);
+        assert_eq!(after.first().unwrap().kind(), TokenKind::Rpar);
+    }
+
+    #[test]
+    fn tokens_after_offset_at_token_end() {
+        let tokens = new_tokens(TEST_CASE_WITH_GAP.into_iter());
+        let after = tokens.after(TextSize::new(11));
+        assert_eq!(after.len(), 4);
+        assert_eq!(after.first().unwrap().kind(), TokenKind::Comment);
+    }
+
+    #[test]
+    fn tokens_after_offset_between_tokens() {
+        let tokens = new_tokens(TEST_CASE_WITH_GAP.into_iter());
+        let after = tokens.after(TextSize::new(13));
+        assert_eq!(after.len(), 4);
+        assert_eq!(after.first().unwrap().kind(), TokenKind::Comment);
+    }
+
+    #[test]
+    fn tokens_after_offset_at_last_token_end() {
+        let tokens = new_tokens(TEST_CASE_WITH_GAP.into_iter());
+        let after = tokens.after(TextSize::new(33));
+        assert_eq!(after.len(), 0);
+    }
+
+    #[test]
+    #[should_panic(expected = "Offset 5 is inside a token range 4..7")]
+    fn tokens_after_offset_inside_token() {
+        let tokens = new_tokens(TEST_CASE_WITH_GAP.into_iter());
+        tokens.after(TextSize::new(5));
+    }
+
+    #[test]
+    fn tokens_before_offset_at_first_token_start() {
+        let tokens = new_tokens(TEST_CASE_WITH_GAP.into_iter());
+        let before = tokens.before(TextSize::new(0));
+        assert_eq!(before.len(), 0);
+    }
+
+    #[test]
+    fn tokens_before_offset_after_first_token_gap() {
+        let tokens = new_tokens(TEST_CASE_WITH_GAP.into_iter());
+        let before = tokens.before(TextSize::new(3));
+        assert_eq!(before.len(), 1);
+        assert_eq!(before.last().unwrap().kind(), TokenKind::Def);
+    }
+
+    #[test]
+    fn tokens_before_offset_at_second_token_start() {
+        let tokens = new_tokens(TEST_CASE_WITH_GAP.into_iter());
+        let before = tokens.before(TextSize::new(4));
+        assert_eq!(before.len(), 1);
+        assert_eq!(before.last().unwrap().kind(), TokenKind::Def);
+    }
+
+    #[test]
+    fn tokens_before_offset_at_token_start() {
+        let tokens = new_tokens(TEST_CASE_WITH_GAP.into_iter());
+        let before = tokens.before(TextSize::new(8));
+        assert_eq!(before.len(), 3);
+        assert_eq!(before.last().unwrap().kind(), TokenKind::Lpar);
+    }
+
+    #[test]
+    fn tokens_before_offset_at_token_end() {
+        let tokens = new_tokens(TEST_CASE_WITH_GAP.into_iter());
+        let before = tokens.before(TextSize::new(11));
+        assert_eq!(before.len(), 6);
+        assert_eq!(before.last().unwrap().kind(), TokenKind::Newline);
+    }
+
+    #[test]
+    fn tokens_before_offset_between_tokens() {
+        let tokens = new_tokens(TEST_CASE_WITH_GAP.into_iter());
+        let before = tokens.before(TextSize::new(13));
+        assert_eq!(before.len(), 6);
+        assert_eq!(before.last().unwrap().kind(), TokenKind::Newline);
+    }
+
+    #[test]
+    fn tokens_before_offset_at_last_token_end() {
+        let tokens = new_tokens(TEST_CASE_WITH_GAP.into_iter());
+        let before = tokens.before(TextSize::new(33));
+        assert_eq!(before.len(), 10);
+        assert_eq!(before.last().unwrap().kind(), TokenKind::Pass);
+    }
+
+    #[test]
+    #[should_panic(expected = "Offset 5 is inside a token range 4..7")]
+    fn tokens_before_offset_inside_token() {
+        let tokens = new_tokens(TEST_CASE_WITH_GAP.into_iter());
+        tokens.before(TextSize::new(5));
+    }
+
+    #[test]
+    fn tokens_in_range_at_token_offset() {
+        let tokens = new_tokens(TEST_CASE_WITH_GAP.into_iter());
+        let in_range = tokens.in_range(TextRange::new(4.into(), 10.into()));
+        assert_eq!(in_range.len(), 4);
+        assert_eq!(in_range.first().unwrap().kind(), TokenKind::Name);
+        assert_eq!(in_range.last().unwrap().kind(), TokenKind::Colon);
+    }
+
+    #[test]
+    fn tokens_in_range_start_offset_at_token_end() {
+        let tokens = new_tokens(TEST_CASE_WITH_GAP.into_iter());
+        let in_range = tokens.in_range(TextRange::new(11.into(), 29.into()));
+        assert_eq!(in_range.len(), 3);
+        assert_eq!(in_range.first().unwrap().kind(), TokenKind::Comment);
+        assert_eq!(in_range.last().unwrap().kind(), TokenKind::Indent);
+    }
+
+    #[test]
+    fn tokens_in_range_end_offset_at_token_start() {
+        let tokens = new_tokens(TEST_CASE_WITH_GAP.into_iter());
+        let in_range = tokens.in_range(TextRange::new(8.into(), 15.into()));
+        assert_eq!(in_range.len(), 3);
+        assert_eq!(in_range.first().unwrap().kind(), TokenKind::Rpar);
+        assert_eq!(in_range.last().unwrap().kind(), TokenKind::Newline);
+    }
+
+    #[test]
+    fn tokens_in_range_start_offset_between_tokens() {
+        let tokens = new_tokens(TEST_CASE_WITH_GAP.into_iter());
+        let in_range = tokens.in_range(TextRange::new(13.into(), 29.into()));
+        assert_eq!(in_range.len(), 3);
+        assert_eq!(in_range.first().unwrap().kind(), TokenKind::Comment);
+        assert_eq!(in_range.last().unwrap().kind(), TokenKind::Indent);
+    }
+
+    #[test]
+    fn tokens_in_range_end_offset_between_tokens() {
+        let tokens = new_tokens(TEST_CASE_WITH_GAP.into_iter());
+        let in_range = tokens.in_range(TextRange::new(9.into(), 13.into()));
+        assert_eq!(in_range.len(), 2);
+        assert_eq!(in_range.first().unwrap().kind(), TokenKind::Colon);
+        assert_eq!(in_range.last().unwrap().kind(), TokenKind::Newline);
+    }
+
+    #[test]
+    #[should_panic(expected = "Offset 5 is inside a token range 4..7")]
+    fn tokens_in_range_start_offset_inside_token() {
+        let tokens = new_tokens(TEST_CASE_WITH_GAP.into_iter());
+        tokens.in_range(TextRange::new(5.into(), 10.into()));
+    }
+
+    #[test]
+    #[should_panic(expected = "Offset 6 is inside a token range 4..7")]
+    fn tokens_in_range_end_offset_inside_token() {
+        let tokens = new_tokens(TEST_CASE_WITH_GAP.into_iter());
+        tokens.in_range(TextRange::new(0.into(), 6.into()));
+    }
+}