֦iBT R d Z dZddlmZ ddlmZmZ ddlZddl Z ddl
m Z ddl Z
ddlZddlZddl ddlmZ ddlZ ej& d ej( Z ej& d
ej( ZddlZej. g d z Z[ G d d
ej0 d
d Zd Zd Zd Z dZ dZ e ede z z e e z Z dZ dZ!dZ"dZ#dZ$ ee!e"e#e$ Z%dZ& edd e e& z Z'd e&z Z( ee'e( Z) ed e)d z Z* ee*e)e% Z+d Z,e jZ d! Z. e e, Z/d"Z0d#Z1d$Z2d%Z3 ee/d&z e/d'z Z4 ee/d(z e/d)z Z5 e e6ejn e8ed*+ Z9 ed,e9 Z: ee+e:e5e Z;e e;z Z< ee/d-z ed.d z e/d/z ed0d z Z= ed1e e4 Z>e ee>e+e:e=e z Z?i Z@ e, D ]" ZAe0e@eAd.z < e1e@eAd0z < e2e@eAd&z < e3e@eAd'z < $ [A eB ZC eB ZD e, D ]B ZEeEd0z eEd.z fD ] ZFeCj eF eEd'z eEd&z fD ] ZFeDj eF D [E[Fd2ZH G d3 d4eI ZJ G d5 d6eI ZK G d7 d8 ZLd9 ZMd: ZNd; ZOd< Zd= ZPd> ZQd? ZRd@ ZSdCdAZTeUdBk( r eR yy)DzKa-Ping Yee zpGvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro, Raymond Hettinger, Trent Nelson, Michael Foord )open)lookupBOM_UTF8N)
TextIOWrapper)*)EXACT_TOKEN_TYPESz&^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)s ^[ \t\f]*(?:[#\r\n]|$))tokenizegenerate_tokensdetect_encoding
untokenize TokenInfoc " e Zd Zd Zed Zy)r
c p d| j t | j fz }d| j | z S )Nz%d (%s)z8TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r))type)r tok_name_replace)selfannotated_types !/usr/lib64/python3.12/tokenize.py__repr__zTokenInfo.__repr__0 s9 "dii$))1D%EEJ
>
23 4 c | j t k( r%| j t v rt | j S | j S N)r OPstringr r s r
exact_typezTokenInfo.exact_type5 s2 99?t{{.? ?$T[[1 199 r N)__name__
__module__ __qualname__r propertyr r r r
r
/ s 4
r r
ztype string start end linec 0 ddj | z dz S )N(|))joinchoicess r groupr* < s C#((7"3 3c 99r c t | dz S )Nr r* r( s r anyr- = s %/C//r c t | dz S )N?r, r( s r mayber0 > s E7Oc 11r z[ \f\t]*z #[^\r\n]*z\\\r?\nz\w+z0[xX](?:_?[0-9a-fA-F])+z0[bB](?:_?[01])+z0[oO](?:_?[0-7])+z (?:0(?:_?0)*|[1-9](?:_?[0-9])*)z[eE][-+]?[0-9](?:_?[0-9])*z)[0-9](?:_?[0-9])*\.(?:[0-9](?:_?[0-9])*)?z\.[0-9](?:_?[0-9])*z[0-9](?:_?[0-9])*z[0-9](?:_?[0-9])*[jJ]z[jJ]c
g d} dh}| D ]q }t j | D ]W }t j |D cg c] }||j f c} D ]" }|j dj | $ Y s |S c c}w )N)brufbrfr )
_itertools permutationsproductupperaddr' )_valid_string_prefixesresultprefixtcr4 s r _all_string_prefixesrC U s >TF(((0A ''!)D!Q1aggi.!)DE
2771:& F 1 ) M *Es B c J t j | t j S r )recompileUNICODE)exprs r _compilerI d s
::dBJJ ''r z[^'\\]*(?:\\.[^'\\]*)*'z[^"\\]*(?:\\.[^"\\]*)*"z%[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''z%[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""z'''z"""z '[^\n'\\]*(?:\\.[^\n'\\]*)*'z "[^\n"\\]*(?:\\.[^\n"\\]*)*"T)reversez\r?\nz'[^\n'\\]*(?:\\.[^\n'\\]*)*'z"[^\n"\\]*(?:\\.[^\n"\\]*)*"z
\\\r?\n|\Z c e Zd Zy)
TokenErrorNr r r r" r r rO rO r rO c e Zd Zy)StopTokenizingNrP r" r r rS rS rQ r rS c 0 e Zd Zd Zd Zd Zd Zd Zd Zy) Untokenizerc X g | _ d| _ d| _ d | _ d| _ d | _ y )N r r8 )tokensprev_rowprev_col prev_type prev_lineencodingr s r __init__zUntokenizer.__init__ s,
r c L |\ }}|| j k s || j k( r@|| j k r1t dj ||| j | j | j | || j z
}|r | j
j
d|z y y )Nz+start ({},{}) precedes previous end ({},{}) )rY rZ
ValueErrorformatadd_backslash_continuationrX append)r startrowcol
col_offsets r add_whitespacezUntokenizer.add_whitespace s S # "63 ;N J $fS#t}}dmm LN
N ''.4==(
KK sZ / 0 r c | |d }|| j z
}|dk( ry | j j d rdnd}| j j d }dj t j t j t | }| j j |d| |z z d| _ y )Nr z
z\
r8 \)
rY r\ endswithrstripr' r9 takewhilestrisspacereversedrX rd rZ )r re rf
row_offsetnewlinelinewss r rc z&Untokenizer.add_backslash_continuation s
Ah4==(
? NN 3 3F ;&~~$$X.
WWZ))#++x~F
G 2"WI ;;<
r c P g }d}|D ] }|dk( r|rd}n|j | |dk( rZt d t j dj |dd d D }|dz d k( s|d d
k7 r|j | nd }|j | d j | S )
NF}{c 3 K | ] }d yw)rW Nr" ).0chars r z.Untokenizer.escape_brackets.. s $#$A #s rl r NTr8 )rd sumr9 ro __eq__r' )r token
charactersconsume_until_next_bracket character
n_backslashess r escape_bracketszUntokenizer.escape_brackets s
%*" IC -16. %%i0C # $","6"6 "26r6*#$ !
!1$)Z^s-B %%i015. i (# $ wwz""r c t | }g }d}|D ] }t | dk( r| j || n|\ }}}} }
|t k( r|| _ @|t
k( r nx|t k( r|j | g|t k( r |j | \ | _
| _ |t t fv rd}n|rF|rD|d } |d t | k\ r+| j j | t | | _ d}ne|t k( r\d|v sd|v rT| j! | } |j" d } | \ }
}| j% d | j% d z }|
||z f} | j' | | j j | | \ | _
| _ |t t fv r | xj dz
c_
d
| _ || _ |
| _ d j- | j S ) NFr Tr rW ry rx z{{z}}r r8 )iterlencompatENCODINGr] ENDMARKERINDENTrd DEDENTpoprY rZ NEWLINENLrX FSTRING_MIDDLEr
splitlinescountri r[ r\ r' )r iterableitindents startlinerA tok_typer re endru indent last_lineend_lineend_col extra_charss r r zUntokenizer.untokenize s
(^ A1v{ Ar"01 -HeUC8# %
9$6!u%V#
/2,
t}gr]* w 8s6{*KK&&v.$'KDM ! ^+%<3%< 0 0 7E 0 0 0 22 6I(+%Hg"+//$"7)//$:O"OK#W{%:;C & KK u %+. (DM4=GR=(
"
!
%DN !DNQ R wwt{{##r c , g }| j j }|d t t fv }d}d}t j
|g| D ]L }|d d \ } }
| t k( r|
| _ | t t fv r|
dz
}
| t k( r
|rd|
z }
d}nd}| t k( r|dz
}n| t k( r|dz }| t k( r|j |
}| t k( r|j | t t fv rd}n,|r|r ||d d}n| t k( r| j# |
}
|
dv r%| j r| j d |
k( r|rd|
z }
| t t fv r3| j$ t t fv r| j j d ||
| | _ O y ) Nr Fr r` TrW r > ry rx )rX rd r r r9 chainr r] NAMENUMBERSTRING
FSTRING_START FSTRING_ENDr r r r r r[ ) r r r r toks_appendr
prevstring
in_fstringtoktoknumtokvals r r zUntokenizer.compat s kk(( !H"
-
##UGX6C !WNFF! &
$ '#
6\F !
"
&a
;&a
v&6!
GR=( wGBK( ! >) --f5 # B68QV` v &-00T^^P[G\5\ ""3' #DNY 7r N) r r r r^ ri rc r r r r" r r rU rU s! 1"#.-$^3$r rU c t }|j | }|j |j |j }|S r )rU r r] encode)r utouts r r r H s?
B
--
!C {{ jj% Jr c | d d j j dd }|dk( s|j d ry|dv s|j d ry| S ) N _-utf-8zutf-8-)zlatin-1
iso-8859-1z iso-latin-1)zlatin-1-z iso-8859-1-z iso-latin-1-r )lowerreplace
startswith)orig_encencs r _get_normal_namer ] sY 2
3B-
'
'S
1C
g~ 1
66
~~AB Or c l j j dd }d} fd}fd} | }|j t r d|dd }d}|s|g fS || }|r||gfS t
j
| s||gfS | }|s||gfS || }|r|||gfS |||gfS # t $ r d Y w xY w)NFr c 2 S # t $ r Y yw xY w)Nr )
StopIterationreadlines r read_or_stopz%detect_encoding..read_or_stop s" : s
c | j d }t j | }|sy t
|j d } t | }r5|dk7 r+
d}t | dj }t | |d z
}|S # t $ r" d}dj | }t | w xY w# t $ r0 d|z }t | dj | }t | w xY w)
Nr z'invalid or missing encoding declarationz {} for {!r}rW zunknown encoding: z unknown encoding for {!r}: {}zencoding problem: utf-8z encoding problem for {!r}: utf-8z-sig)
decodeUnicodeDecodeErrorrb SyntaxError cookie_rematchr r* r LookupError)ru line_stringmsgr r] codec bom_foundfilenames r find_cookiez$detect_encoding..find_cookie s( # ++g.K ,#EKKN3 #8$E 7"#3C "#&& =CCHMC!#&& H= " #;C##**39 c" " # # *X5 c" " 6<>%
^F 6"H% (( UFO ##O s B% % B32B3c t | d } t |j \ }}|j d t ||d }d|_ |S # |j
xY w)Nrbr T)line_bufferingr3 )
_builtin_openr r seekr modeclose)r bufferr] linestexts r r r sd 8T
*F)&//:% A VXdC
s ?A A"c # K t | \ }}t j |t | d }| |dk( rd}t t
|ddd t
|j |d E d { y 7 w)Nr r r )r r r8 T extra_tokens)r r9 r r r
r !_generate_tokens_from_c_tokenizer__next__)r r] consumedrl_gens r r r sr $ )2Hh
hXs(;
.perror s&
!
r c |r|f|z | fz } d|z n|r |d| n d| z t j d y )Nz%s:%d:%d: error: %sz : error: z error: %srW )r exit)r r locationargsr s r errorzmain..error sO ;)WJ6D (4/ 0
h8 9 ;( ) r zpython -m tokenize)progr r/ z filename.pyz'the file to tokenize; defaults to stdin)destnargsmetavarhelpz-ez--exactexact
store_truez(display token names using the exact type)r actionr r zTr z %d,%d-%d,%d:2015rW r z interrupted
zunexpected error: %s)NN)argparseArgumentParser add_argument
parse_argsr r listr r r r stdinr r r re r printr r IndentationErrorr rO r OSErrorKeyboardInterrupt Exception)r r parserr r r5 rX r
token_type token_rangeerrru columnr s @r mainr s
$
$*>
$
?F
Zs - F H igl G I D" ==}}H x.! hqzz 23 /. !H6 ""7F
E Jzz " - -
(EKK%)),CDK 4ellD
E /. 5xx{1Q' f
chhqk8dF^44 5xx{ f
chhqk8dF^44
c8
c
o %+,
sg $E D= B E =EE H&.F H&+G H&
G H&'G44H&
H& H!!H&c d| v ry| S )Nz)unterminated triple-quoted string literalzEOF in multi-line stringr" )r s r _transform_msgr 7 s
3c9) Jr c # l K |t j | | }nt j | || } |D ] }t j | y # t $ rR}t | t k7 r|d t
|j }t ||j |j f d d }~ww xY ww)Nr )r] r ) _tokenize
TokenizerIterr
_maker r r r rO linenooffset)sourcer] r r infoer s r r r A s a
$
$V,
G
$
$Vh\
Z>D//$' ' > 7k ! QUU#qxx 23= >s) 4B4 A B4 B1 A
B,,B11B4__main__)NF)V
__author__ __credits__builtinsr r codecsr r collections functoolsior itertoolsr9 rE r r r r rF ASCIIr r __all__
namedtupler
r* r- r0
WhitespaceCommentIgnoreName Hexnumber Binnumber Octnumber Decnumber IntnumberExponent
PointfloatExpfloat Floatnumber
ImagnumberNumberrC lru_cacherI StringPrefixSingleDoubleSingle3Double3TripleStringmapescapesortedSpecialFunny
PlainTokenTokenContStr PseudoExtras PseudoTokenendpats_prefixset
single_quoted
triple_quotedrA r4 r= tabsizer rO rS rU r r r r r
r r r r r" r r rS s , *
+ #
# BJJ@"(( K 2::0"(( ;
-- 6 6 & &&{4PQ : / 1
c*z12 2U7^ C
& . )Y 9 = (
?)+-28_=
( *J)
+[7-B
C
z; 2
( (
*,-
$ #
2
2 |e#\E%9 : | == ==
? RYY'8$ GH
I
h
65&$
/
==c:&' ==c:&' ( ]GV4 5 vugtLL
#%G #GGcM #GGcM &GGeO &GGeO &
A#gq3w
! %iU
#! $
q
! ! &Y %^$ ^$B* Z$z
_8J=~
> zF r