{-# LANGUAGE CPP, DeriveDataTypeable #-}
module Language.Python.Common.Token (
Token (..),
debugTokenString,
tokenString,
hasLiteral,
TokenClass (..),
classifyToken
) where
import Language.Python.Common.Pretty
import Language.Python.Common.SrcLocation (SrcSpan (..), Span(getSpan))
import Data.Data
data Token
= IndentToken { Token -> SrcSpan
token_span :: !SrcSpan }
| DedentToken { token_span :: !SrcSpan }
| NewlineToken { token_span :: !SrcSpan }
| LineJoinToken { token_span :: !SrcSpan }
| { token_span :: !SrcSpan, Token -> String
token_literal :: !String }
| IdentifierToken { token_span :: !SrcSpan, token_literal :: !String }
| StringToken { token_span :: !SrcSpan, token_literal :: !String }
| ByteStringToken { token_span :: !SrcSpan, token_literal :: !String }
| UnicodeStringToken { token_span :: !SrcSpan, token_literal :: !String }
| IntegerToken { token_span :: !SrcSpan, token_literal :: !String, Token -> Integer
token_integer :: !Integer }
| LongIntegerToken { token_span :: !SrcSpan, token_literal :: !String, token_integer :: !Integer }
| FloatToken { token_span :: !SrcSpan, token_literal :: !String, Token -> Double
token_double :: !Double }
| ImaginaryToken { token_span :: !SrcSpan, token_literal :: !String, token_double :: !Double }
| DefToken { token_span :: !SrcSpan }
| WhileToken { token_span :: !SrcSpan }
| IfToken { token_span :: !SrcSpan }
| TrueToken { token_span :: !SrcSpan }
| FalseToken { token_span :: !SrcSpan }
| ReturnToken { token_span :: !SrcSpan }
| TryToken { token_span :: !SrcSpan }
| ExceptToken { token_span :: !SrcSpan }
| RaiseToken { token_span :: !SrcSpan }
| InToken { token_span :: !SrcSpan }
| IsToken { token_span :: !SrcSpan }
| LambdaToken { token_span :: !SrcSpan }
| ClassToken { token_span :: !SrcSpan }
| FinallyToken { token_span :: !SrcSpan }
| NoneToken { token_span :: !SrcSpan }
| ForToken { token_span :: !SrcSpan }
| FromToken { token_span :: !SrcSpan }
| GlobalToken { token_span :: !SrcSpan }
| WithToken { token_span :: !SrcSpan }
| AsToken { token_span :: !SrcSpan }
| ElifToken { token_span :: !SrcSpan }
| YieldToken { token_span :: !SrcSpan }
| AssertToken { token_span :: !SrcSpan }
| ImportToken { token_span :: !SrcSpan }
| PassToken { token_span :: !SrcSpan }
| BreakToken { token_span :: !SrcSpan }
| ContinueToken { token_span :: !SrcSpan }
| DeleteToken { token_span :: !SrcSpan }
| ElseToken { token_span :: !SrcSpan }
| NotToken { token_span :: !SrcSpan }
| AndToken { token_span :: !SrcSpan }
| OrToken { token_span :: !SrcSpan }
| NonLocalToken { token_span :: !SrcSpan }
| AsyncToken { token_span :: !SrcSpan }
| AwaitToken { token_span :: !SrcSpan }
| PrintToken { token_span :: !SrcSpan }
| ExecToken { token_span :: !SrcSpan }
| AtToken { token_span :: !SrcSpan }
| LeftRoundBracketToken { token_span :: !SrcSpan }
| RightRoundBracketToken { token_span :: !SrcSpan }
| LeftSquareBracketToken { token_span :: !SrcSpan }
| RightSquareBracketToken { token_span :: !SrcSpan }
| LeftBraceToken { token_span :: !SrcSpan }
| RightBraceToken { token_span :: !SrcSpan }
| DotToken { token_span :: !SrcSpan }
| CommaToken { token_span :: !SrcSpan }
| SemiColonToken { token_span :: !SrcSpan }
| ColonToken { token_span :: !SrcSpan }
| EllipsisToken { token_span :: !SrcSpan }
| RightArrowToken { token_span :: !SrcSpan }
| AssignToken { token_span :: !SrcSpan }
| PlusAssignToken { token_span :: !SrcSpan }
| MinusAssignToken { token_span :: !SrcSpan }
| MultAssignToken { token_span :: !SrcSpan }
| DivAssignToken { token_span :: !SrcSpan }
| ModAssignToken { token_span :: !SrcSpan }
| PowAssignToken { token_span :: !SrcSpan }
| BinAndAssignToken { token_span :: !SrcSpan }
| BinOrAssignToken { token_span :: !SrcSpan }
| BinXorAssignToken { token_span :: !SrcSpan }
| LeftShiftAssignToken { token_span :: !SrcSpan }
| RightShiftAssignToken { token_span :: !SrcSpan }
| FloorDivAssignToken { token_span :: !SrcSpan }
| MatrixMultAssignToken { token_span :: !SrcSpan }
| BackQuoteToken { token_span :: !SrcSpan }
| PlusToken { token_span :: !SrcSpan }
| MinusToken { token_span :: !SrcSpan }
| MultToken { token_span :: !SrcSpan }
| DivToken { token_span :: !SrcSpan }
| GreaterThanToken { token_span :: !SrcSpan }
| LessThanToken { token_span :: !SrcSpan }
| EqualityToken { token_span :: !SrcSpan }
| GreaterThanEqualsToken { token_span :: !SrcSpan }
| LessThanEqualsToken { token_span :: !SrcSpan }
| ExponentToken { token_span :: !SrcSpan }
| BinaryOrToken { token_span :: !SrcSpan }
| XorToken { token_span :: !SrcSpan }
| BinaryAndToken { token_span :: !SrcSpan }
| ShiftLeftToken { token_span :: !SrcSpan }
| ShiftRightToken { token_span :: !SrcSpan }
| ModuloToken { token_span :: !SrcSpan }
| FloorDivToken { token_span :: !SrcSpan }
| TildeToken { token_span :: !SrcSpan }
| NotEqualsToken { token_span :: !SrcSpan }
| NotEqualsV2Token { token_span :: !SrcSpan }
| EOFToken { token_span :: !SrcSpan }
deriving (Token -> Token -> Bool
(Token -> Token -> Bool) -> (Token -> Token -> Bool) -> Eq Token
forall a. (a -> a -> Bool) -> (a -> a -> Bool) -> Eq a
/= :: Token -> Token -> Bool
$c/= :: Token -> Token -> Bool
== :: Token -> Token -> Bool
$c== :: Token -> Token -> Bool
Eq,Eq Token
Eq Token
-> (Token -> Token -> Ordering)
-> (Token -> Token -> Bool)
-> (Token -> Token -> Bool)
-> (Token -> Token -> Bool)
-> (Token -> Token -> Bool)
-> (Token -> Token -> Token)
-> (Token -> Token -> Token)
-> Ord Token
Token -> Token -> Bool
Token -> Token -> Ordering
Token -> Token -> Token
forall a.
Eq a
-> (a -> a -> Ordering)
-> (a -> a -> Bool)
-> (a -> a -> Bool)
-> (a -> a -> Bool)
-> (a -> a -> Bool)
-> (a -> a -> a)
-> (a -> a -> a)
-> Ord a
min :: Token -> Token -> Token
$cmin :: Token -> Token -> Token
max :: Token -> Token -> Token
$cmax :: Token -> Token -> Token
>= :: Token -> Token -> Bool
$c>= :: Token -> Token -> Bool
> :: Token -> Token -> Bool
$c> :: Token -> Token -> Bool
<= :: Token -> Token -> Bool
$c<= :: Token -> Token -> Bool
< :: Token -> Token -> Bool
$c< :: Token -> Token -> Bool
compare :: Token -> Token -> Ordering
$ccompare :: Token -> Token -> Ordering
Ord,Int -> Token -> ShowS
[Token] -> ShowS
Token -> String
(Int -> Token -> ShowS)
-> (Token -> String) -> ([Token] -> ShowS) -> Show Token
forall a.
(Int -> a -> ShowS) -> (a -> String) -> ([a] -> ShowS) -> Show a
showList :: [Token] -> ShowS
$cshowList :: [Token] -> ShowS
show :: Token -> String
$cshow :: Token -> String
showsPrec :: Int -> Token -> ShowS
$cshowsPrec :: Int -> Token -> ShowS
Show,Typeable,Typeable Token
Typeable Token
-> (forall (c :: * -> *).
(forall d b. Data d => c (d -> b) -> d -> c b)
-> (forall g. g -> c g) -> Token -> c Token)
-> (forall (c :: * -> *).
(forall b r. Data b => c (b -> r) -> c r)
-> (forall r. r -> c r) -> Constr -> c Token)
-> (Token -> Constr)
-> (Token -> DataType)
-> (forall (t :: * -> *) (c :: * -> *).
Typeable t =>
(forall d. Data d => c (t d)) -> Maybe (c Token))
-> (forall (t :: * -> * -> *) (c :: * -> *).
Typeable t =>
(forall d e. (Data d, Data e) => c (t d e)) -> Maybe (c Token))
-> ((forall b. Data b => b -> b) -> Token -> Token)
-> (forall r r'.
(r -> r' -> r) -> r -> (forall d. Data d => d -> r') -> Token -> r)
-> (forall r r'.
(r' -> r -> r) -> r -> (forall d. Data d => d -> r') -> Token -> r)
-> (forall u. (forall d. Data d => d -> u) -> Token -> [u])
-> (forall u. Int -> (forall d. Data d => d -> u) -> Token -> u)
-> (forall (m :: * -> *).
Monad m =>
(forall d. Data d => d -> m d) -> Token -> m Token)
-> (forall (m :: * -> *).
MonadPlus m =>
(forall d. Data d => d -> m d) -> Token -> m Token)
-> (forall (m :: * -> *).
MonadPlus m =>
(forall d. Data d => d -> m d) -> Token -> m Token)
-> Data Token
Token -> DataType
Token -> Constr
(forall b. Data b => b -> b) -> Token -> Token
forall a.
Typeable a
-> (forall (c :: * -> *).
(forall d b. Data d => c (d -> b) -> d -> c b)
-> (forall g. g -> c g) -> a -> c a)
-> (forall (c :: * -> *).
(forall b r. Data b => c (b -> r) -> c r)
-> (forall r. r -> c r) -> Constr -> c a)
-> (a -> Constr)
-> (a -> DataType)
-> (forall (t :: * -> *) (c :: * -> *).
Typeable t =>
(forall d. Data d => c (t d)) -> Maybe (c a))
-> (forall (t :: * -> * -> *) (c :: * -> *).
Typeable t =>
(forall d e. (Data d, Data e) => c (t d e)) -> Maybe (c a))
-> ((forall b. Data b => b -> b) -> a -> a)
-> (forall r r'.
(r -> r' -> r) -> r -> (forall d. Data d => d -> r') -> a -> r)
-> (forall r r'.
(r' -> r -> r) -> r -> (forall d. Data d => d -> r') -> a -> r)
-> (forall u. (forall d. Data d => d -> u) -> a -> [u])
-> (forall u. Int -> (forall d. Data d => d -> u) -> a -> u)
-> (forall (m :: * -> *).
Monad m =>
(forall d. Data d => d -> m d) -> a -> m a)
-> (forall (m :: * -> *).
MonadPlus m =>
(forall d. Data d => d -> m d) -> a -> m a)
-> (forall (m :: * -> *).
MonadPlus m =>
(forall d. Data d => d -> m d) -> a -> m a)
-> Data a
forall u. Int -> (forall d. Data d => d -> u) -> Token -> u
forall u. (forall d. Data d => d -> u) -> Token -> [u]
forall r r'.
(r -> r' -> r) -> r -> (forall d. Data d => d -> r') -> Token -> r
forall r r'.
(r' -> r -> r) -> r -> (forall d. Data d => d -> r') -> Token -> r
forall (m :: * -> *).
Monad m =>
(forall d. Data d => d -> m d) -> Token -> m Token
forall (m :: * -> *).
MonadPlus m =>
(forall d. Data d => d -> m d) -> Token -> m Token
forall (c :: * -> *).
(forall b r. Data b => c (b -> r) -> c r)
-> (forall r. r -> c r) -> Constr -> c Token
forall (c :: * -> *).
(forall d b. Data d => c (d -> b) -> d -> c b)
-> (forall g. g -> c g) -> Token -> c Token
forall (t :: * -> *) (c :: * -> *).
Typeable t =>
(forall d. Data d => c (t d)) -> Maybe (c Token)
forall (t :: * -> * -> *) (c :: * -> *).
Typeable t =>
(forall d e. (Data d, Data e) => c (t d e)) -> Maybe (c Token)
gmapMo :: forall (m :: * -> *).
MonadPlus m =>
(forall d. Data d => d -> m d) -> Token -> m Token
$cgmapMo :: forall (m :: * -> *).
MonadPlus m =>
(forall d. Data d => d -> m d) -> Token -> m Token
gmapMp :: forall (m :: * -> *).
MonadPlus m =>
(forall d. Data d => d -> m d) -> Token -> m Token
$cgmapMp :: forall (m :: * -> *).
MonadPlus m =>
(forall d. Data d => d -> m d) -> Token -> m Token
gmapM :: forall (m :: * -> *).
Monad m =>
(forall d. Data d => d -> m d) -> Token -> m Token
$cgmapM :: forall (m :: * -> *).
Monad m =>
(forall d. Data d => d -> m d) -> Token -> m Token
gmapQi :: forall u. Int -> (forall d. Data d => d -> u) -> Token -> u
$cgmapQi :: forall u. Int -> (forall d. Data d => d -> u) -> Token -> u
gmapQ :: forall u. (forall d. Data d => d -> u) -> Token -> [u]
$cgmapQ :: forall u. (forall d. Data d => d -> u) -> Token -> [u]
gmapQr :: forall r r'.
(r' -> r -> r) -> r -> (forall d. Data d => d -> r') -> Token -> r
$cgmapQr :: forall r r'.
(r' -> r -> r) -> r -> (forall d. Data d => d -> r') -> Token -> r
gmapQl :: forall r r'.
(r -> r' -> r) -> r -> (forall d. Data d => d -> r') -> Token -> r
$cgmapQl :: forall r r'.
(r -> r' -> r) -> r -> (forall d. Data d => d -> r') -> Token -> r
gmapT :: (forall b. Data b => b -> b) -> Token -> Token
$cgmapT :: (forall b. Data b => b -> b) -> Token -> Token
dataCast2 :: forall (t :: * -> * -> *) (c :: * -> *).
Typeable t =>
(forall d e. (Data d, Data e) => c (t d e)) -> Maybe (c Token)
$cdataCast2 :: forall (t :: * -> * -> *) (c :: * -> *).
Typeable t =>
(forall d e. (Data d, Data e) => c (t d e)) -> Maybe (c Token)
dataCast1 :: forall (t :: * -> *) (c :: * -> *).
Typeable t =>
(forall d. Data d => c (t d)) -> Maybe (c Token)
$cdataCast1 :: forall (t :: * -> *) (c :: * -> *).
Typeable t =>
(forall d. Data d => c (t d)) -> Maybe (c Token)
dataTypeOf :: Token -> DataType
$cdataTypeOf :: Token -> DataType
toConstr :: Token -> Constr
$ctoConstr :: Token -> Constr
gunfold :: forall (c :: * -> *).
(forall b r. Data b => c (b -> r) -> c r)
-> (forall r. r -> c r) -> Constr -> c Token
$cgunfold :: forall (c :: * -> *).
(forall b r. Data b => c (b -> r) -> c r)
-> (forall r. r -> c r) -> Constr -> c Token
gfoldl :: forall (c :: * -> *).
(forall d b. Data d => c (d -> b) -> d -> c b)
-> (forall g. g -> c g) -> Token -> c Token
$cgfoldl :: forall (c :: * -> *).
(forall d b. Data d => c (d -> b) -> d -> c b)
-> (forall g. g -> c g) -> Token -> c Token
Data)
instance Span Token where
getSpan :: Token -> SrcSpan
getSpan = Token -> SrcSpan
token_span
debugTokenString :: Token -> String
debugTokenString :: Token -> String
debugTokenString Token
token =
Doc -> String
render (String -> Doc
text (Constr -> String
forall a. Show a => a -> String
show (Constr -> String) -> Constr -> String
forall a b. (a -> b) -> a -> b
$ Token -> Constr
forall a. Data a => a -> Constr
toConstr Token
token) Doc -> Doc -> Doc
<+> SrcSpan -> Doc
forall a. Pretty a => a -> Doc
pretty (Token -> SrcSpan
token_span Token
token) Doc -> Doc -> Doc
<+>
if Token -> Bool
hasLiteral Token
token then String -> Doc
text (Token -> String
token_literal Token
token) else Doc
empty)
hasLiteral :: Token -> Bool
hasLiteral :: Token -> Bool
hasLiteral Token
token =
case Token
token of
CommentToken {} -> Bool
True
IdentifierToken {} -> Bool
True
StringToken {} -> Bool
True
ByteStringToken {} -> Bool
True
UnicodeStringToken {} -> Bool
True
IntegerToken {} -> Bool
True
LongIntegerToken {} -> Bool
True
FloatToken {} -> Bool
True
ImaginaryToken {} -> Bool
True
Token
other -> Bool
False
data TokenClass
=
| Number
| Identifier
| Punctuation
| Bracket
| Layout
| Keyword
| String
| Operator
| Assignment
deriving (Int -> TokenClass -> ShowS
[TokenClass] -> ShowS
TokenClass -> String
(Int -> TokenClass -> ShowS)
-> (TokenClass -> String)
-> ([TokenClass] -> ShowS)
-> Show TokenClass
forall a.
(Int -> a -> ShowS) -> (a -> String) -> ([a] -> ShowS) -> Show a
showList :: [TokenClass] -> ShowS
$cshowList :: [TokenClass] -> ShowS
show :: TokenClass -> String
$cshow :: TokenClass -> String
showsPrec :: Int -> TokenClass -> ShowS
$cshowsPrec :: Int -> TokenClass -> ShowS
Show, TokenClass -> TokenClass -> Bool
(TokenClass -> TokenClass -> Bool)
-> (TokenClass -> TokenClass -> Bool) -> Eq TokenClass
forall a. (a -> a -> Bool) -> (a -> a -> Bool) -> Eq a
/= :: TokenClass -> TokenClass -> Bool
$c/= :: TokenClass -> TokenClass -> Bool
== :: TokenClass -> TokenClass -> Bool
$c== :: TokenClass -> TokenClass -> Bool
Eq, Eq TokenClass
Eq TokenClass
-> (TokenClass -> TokenClass -> Ordering)
-> (TokenClass -> TokenClass -> Bool)
-> (TokenClass -> TokenClass -> Bool)
-> (TokenClass -> TokenClass -> Bool)
-> (TokenClass -> TokenClass -> Bool)
-> (TokenClass -> TokenClass -> TokenClass)
-> (TokenClass -> TokenClass -> TokenClass)
-> Ord TokenClass
TokenClass -> TokenClass -> Bool
TokenClass -> TokenClass -> Ordering
TokenClass -> TokenClass -> TokenClass
forall a.
Eq a
-> (a -> a -> Ordering)
-> (a -> a -> Bool)
-> (a -> a -> Bool)
-> (a -> a -> Bool)
-> (a -> a -> Bool)
-> (a -> a -> a)
-> (a -> a -> a)
-> Ord a
min :: TokenClass -> TokenClass -> TokenClass
$cmin :: TokenClass -> TokenClass -> TokenClass
max :: TokenClass -> TokenClass -> TokenClass
$cmax :: TokenClass -> TokenClass -> TokenClass
>= :: TokenClass -> TokenClass -> Bool
$c>= :: TokenClass -> TokenClass -> Bool
> :: TokenClass -> TokenClass -> Bool
$c> :: TokenClass -> TokenClass -> Bool
<= :: TokenClass -> TokenClass -> Bool
$c<= :: TokenClass -> TokenClass -> Bool
< :: TokenClass -> TokenClass -> Bool
$c< :: TokenClass -> TokenClass -> Bool
compare :: TokenClass -> TokenClass -> Ordering
$ccompare :: TokenClass -> TokenClass -> Ordering
Ord)
classifyToken :: Token -> TokenClass
classifyToken :: Token -> TokenClass
classifyToken Token
token =
case Token
token of
IndentToken {} -> TokenClass
Layout
DedentToken {} -> TokenClass
Layout
NewlineToken {} -> TokenClass
Layout
CommentToken {} -> TokenClass
Comment
IdentifierToken {} -> TokenClass
Identifier
StringToken {} -> TokenClass
String
ByteStringToken {} -> TokenClass
String
UnicodeStringToken {} -> TokenClass
String
IntegerToken {} -> TokenClass
Number
LongIntegerToken {} -> TokenClass
Number
FloatToken {} -> TokenClass
Number
ImaginaryToken {} -> TokenClass
Number
DefToken {} -> TokenClass
Keyword
WhileToken {} -> TokenClass
Keyword
IfToken {} -> TokenClass
Keyword
TrueToken {} -> TokenClass
Keyword
FalseToken {} -> TokenClass
Keyword
ReturnToken {} -> TokenClass
Keyword
TryToken {} -> TokenClass
Keyword
ExceptToken {} -> TokenClass
Keyword
RaiseToken {} -> TokenClass
Keyword
InToken {} -> TokenClass
Keyword
IsToken {} -> TokenClass
Keyword
LambdaToken {} -> TokenClass
Keyword
ClassToken {} -> TokenClass
Keyword
FinallyToken {} -> TokenClass
Keyword
NoneToken {} -> TokenClass
Keyword
ForToken {} -> TokenClass
Keyword
FromToken {} -> TokenClass
Keyword
GlobalToken {} -> TokenClass
Keyword
WithToken {} -> TokenClass
Keyword
AsToken {} -> TokenClass
Keyword
ElifToken {} -> TokenClass
Keyword
YieldToken {} -> TokenClass
Keyword
AsyncToken {} -> TokenClass
Keyword
AwaitToken {} -> TokenClass
Keyword
AssertToken {} -> TokenClass
Keyword
ImportToken {} -> TokenClass
Keyword
PassToken {} -> TokenClass
Keyword
BreakToken {} -> TokenClass
Keyword
ContinueToken {} -> TokenClass
Keyword
DeleteToken {} -> TokenClass
Keyword
ElseToken {} -> TokenClass
Keyword
NotToken {} -> TokenClass
Keyword
AndToken {} -> TokenClass
Keyword
OrToken {} -> TokenClass
Keyword
NonLocalToken {} -> TokenClass
Keyword
PrintToken {} -> TokenClass
Keyword
ExecToken {} -> TokenClass
Keyword
AtToken {} -> TokenClass
Keyword
LeftRoundBracketToken {} -> TokenClass
Bracket
RightRoundBracketToken {} -> TokenClass
Bracket
LeftSquareBracketToken {} -> TokenClass
Bracket
RightSquareBracketToken {} -> TokenClass
Bracket
LeftBraceToken {} -> TokenClass
Bracket
RightBraceToken {} -> TokenClass
Bracket
DotToken {} -> TokenClass
Operator
CommaToken {} -> TokenClass
Punctuation
SemiColonToken {} -> TokenClass
Punctuation
ColonToken {} -> TokenClass
Punctuation
EllipsisToken {} -> TokenClass
Keyword
RightArrowToken {} -> TokenClass
Punctuation
AssignToken {} -> TokenClass
Assignment
PlusAssignToken {} -> TokenClass
Assignment
MinusAssignToken {} -> TokenClass
Assignment
MultAssignToken {} -> TokenClass
Assignment
DivAssignToken {} -> TokenClass
Assignment
ModAssignToken {} -> TokenClass
Assignment
PowAssignToken {} -> TokenClass
Assignment
BinAndAssignToken {} -> TokenClass
Assignment
BinOrAssignToken {} -> TokenClass
Assignment
BinXorAssignToken {} -> TokenClass
Assignment
LeftShiftAssignToken {} -> TokenClass
Assignment
RightShiftAssignToken {} -> TokenClass
Assignment
FloorDivAssignToken {} -> TokenClass
Assignment
MatrixMultAssignToken {} -> TokenClass
Assignment
BackQuoteToken {} -> TokenClass
Punctuation
PlusToken {} -> TokenClass
Operator
MinusToken {} -> TokenClass
Operator
MultToken {} -> TokenClass
Operator
DivToken {} -> TokenClass
Operator
GreaterThanToken {} -> TokenClass
Operator
LessThanToken {} -> TokenClass
Operator
EqualityToken {} -> TokenClass
Operator
GreaterThanEqualsToken {} -> TokenClass
Operator
LessThanEqualsToken {} -> TokenClass
Operator
ExponentToken {} -> TokenClass
Operator
BinaryOrToken {} -> TokenClass
Operator
XorToken {} -> TokenClass
Operator
BinaryAndToken {} -> TokenClass
Operator
ShiftLeftToken {} -> TokenClass
Operator
ShiftRightToken {} -> TokenClass
Operator
ModuloToken {} -> TokenClass
Operator
FloorDivToken {} -> TokenClass
Operator
TildeToken {} -> TokenClass
Operator
NotEqualsToken {} -> TokenClass
Operator
NotEqualsV2Token {} -> TokenClass
Operator
LineJoinToken {} -> TokenClass
Layout
EOFToken {} -> TokenClass
Layout
tokenString :: Token -> String
tokenString :: Token -> String
tokenString Token
token =
case Token
token of
IndentToken {} -> String
""
DedentToken {} -> String
""
NewlineToken {} -> String
""
CommentToken {} -> Token -> String
token_literal Token
token
IdentifierToken {} -> Token -> String
token_literal Token
token
StringToken {} -> Token -> String
token_literal Token
token
ByteStringToken {} -> Token -> String
token_literal Token
token
UnicodeStringToken {} -> Token -> String
token_literal Token
token
IntegerToken {} -> Token -> String
token_literal Token
token
LongIntegerToken {} -> Token -> String
token_literal Token
token
FloatToken {} -> Token -> String
token_literal Token
token
ImaginaryToken {} -> Token -> String
token_literal Token
token
DefToken {} -> String
"def"
WhileToken {} -> String
"while"
IfToken {} -> String
"if"
TrueToken {} -> String
"True"
FalseToken {} -> String
"False"
ReturnToken {} -> String
"return"
TryToken {} -> String
"try"
ExceptToken {} -> String
"except"
RaiseToken {} -> String
"raise"
InToken {} -> String
"in"
IsToken {} -> String
"is"
LambdaToken {} -> String
"lambda"
ClassToken {} -> String
"class"
FinallyToken {} -> String
"finally"
NoneToken {} -> String
"None"
ForToken {} -> String
"for"
FromToken {} -> String
"from"
GlobalToken {} -> String
"global"
WithToken {} -> String
"with"
AsToken {} -> String
"as"
ElifToken {} -> String
"elif"
YieldToken {} -> String
"yield"
AsyncToken {} -> String
"async"
AwaitToken {} -> String
"await"
AssertToken {} -> String
"assert"
ImportToken {} -> String
"import"
PassToken {} -> String
"pass"
BreakToken {} -> String
"break"
ContinueToken {} -> String
"continue"
DeleteToken {} -> String
"delete"
ElseToken {} -> String
"else"
NotToken {} -> String
"not"
AndToken {} -> String
"and"
OrToken {} -> String
"or"
NonLocalToken {} -> String
"nonlocal"
PrintToken {} -> String
"print"
ExecToken {} -> String
"exec"
AtToken {} -> String
"at"
LeftRoundBracketToken {} -> String
"("
RightRoundBracketToken {} -> String
")"
LeftSquareBracketToken {} -> String
"["
RightSquareBracketToken {} -> String
"]"
LeftBraceToken {} -> String
"{"
RightBraceToken {} -> String
"}"
DotToken {} -> String
"."
CommaToken {} -> String
","
SemiColonToken {} -> String
";"
ColonToken {} -> String
":"
EllipsisToken {} -> String
"..."
RightArrowToken {} -> String
"->"
AssignToken {} -> String
"="
PlusAssignToken {} -> String
"+="
MinusAssignToken {} -> String
"-="
MultAssignToken {} -> String
"*="
DivAssignToken {} -> String
"/="
ModAssignToken {} -> String
"%="
PowAssignToken {} -> String
"**="
BinAndAssignToken {} -> String
"&="
BinOrAssignToken {} -> String
"|="
BinXorAssignToken {} -> String
"^="
LeftShiftAssignToken {} -> String
"<<="
RightShiftAssignToken {} -> String
">>="
FloorDivAssignToken {} -> String
"//="
MatrixMultAssignToken {} -> String
"@="
BackQuoteToken {} -> String
"`"
PlusToken {} -> String
"+"
MinusToken {} -> String
"-"
MultToken {} -> String
"*"
DivToken {} -> String
"/"
GreaterThanToken {} -> String
">"
LessThanToken {} -> String
"<"
EqualityToken {} -> String
"=="
GreaterThanEqualsToken {} -> String
">="
LessThanEqualsToken {} -> String
"<="
ExponentToken {} -> String
"**"
BinaryOrToken {} -> String
"|"
XorToken {} -> String
"^"
BinaryAndToken {} -> String
"&"
ShiftLeftToken {} -> String
"<<"
ShiftRightToken {} -> String
">>"
ModuloToken {} -> String
"%"
FloorDivToken {} -> String
"//"
TildeToken {} -> String
"~"
NotEqualsToken {} -> String
"!="
NotEqualsV2Token {} -> String
"<>"
LineJoinToken {} -> String
"\\"
EOFToken {} -> String
""