Skip to content

Token

Source code in pbi_parsers/base/tokens.py 
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
@dataclass
class BaseToken:
    tok_type: Any
    text_slice: TextSlice = field(default_factory=TextSlice)

    def __eq__(self, other: object) -> bool:
        """Checks equality based on token type and text slice."""
        if not isinstance(other, BaseToken):
            return NotImplemented
        return self.tok_type == other.tok_type and self.text_slice == other.text_slice

    def __hash__(self) -> int:
        """Returns a hash based on token type and text slice."""
        return hash((self.tok_type, self.text_slice))

    def __repr__(self) -> str:
        pretty_text = self.text_slice.get_text().replace("\n", "\\n").replace("\r", "\\r")
        return f"Token(type={self.tok_type.name}, text='{pretty_text}')"

    def position(self) -> tuple[int, int]:
        """Returns the start and end positions of the token.

        Returns:
            tuple[int, int]: A tuple containing the start and end positions of the token within the source text.

        """
        return self.text_slice.start, self.text_slice.end

    @property
    def text(self) -> str:
        """Returns the text underlying the token.

        Returns:
            str: The text of the token as a string.

        """
        return self.text_slice.get_text()

text property 

text: str

Returns the text underlying the token.

Returns:

Name Type Description
str str

The text of the token as a string.

__eq__ 

__eq__(other: object) -> bool

Checks equality based on token type and text slice.

Source code in pbi_parsers/base/tokens.py 
35
36
37
38
39
def __eq__(self, other: object) -> bool:
    """Checks equality based on token type and text slice."""
    if not isinstance(other, BaseToken):
        return NotImplemented
    return self.tok_type == other.tok_type and self.text_slice == other.text_slice

__hash__ 

__hash__() -> int

Returns a hash based on token type and text slice.

Source code in pbi_parsers/base/tokens.py 
41
42
43
def __hash__(self) -> int:
    """Returns a hash based on token type and text slice."""
    return hash((self.tok_type, self.text_slice))

position 

position() -> tuple[int, int]

Returns the start and end positions of the token.

Returns:

Type Description
tuple[int, int]

tuple[int, int]: A tuple containing the start and end positions of the token within the source text.
Source code in pbi_parsers/base/tokens.py 
49
50
51
52
53
54
55
56
def position(self) -> tuple[int, int]:
    """Returns the start and end positions of the token.

    Returns:
        tuple[int, int]: A tuple containing the start and end positions of the token within the source text.

    """
    return self.text_slice.start, self.text_slice.end