U
e5dd @ sD d Z dZddlmZ ddlmZmZ ddlZddl m
Z
ddl Z ddl
Z
ddlZddlT ddlmZ e
d e
jZe
d
e
jZddlZejd d d
ddg Z[G dd deddZdd Zdd Zdd ZdZdZ eede ee Z dZ d Z d Z d Z!d Z"ee e e!e"Z#d Z$ed!d"ee$ Z%d#e$ Z&ee%e&Z'ed$e'd% Z(ee(e'e#Z)d&d' Z*d(d) Z+ee* Z,d*Z-d+Z.d,Z/d-Z0ee,d. e,d/ Z1ee,d0 e,d1 Z2ee3e
j4e5ed2d3 Z6ed4e6Z7ee)e7e2e Z8e e8 Z9ee,d5 ed6d e,d7 ed8d Z:ed9e e1Z;eee;e)e7e:e Ze-e=e>d6 < e.e=e>d8 < e/e=e>d. < e0e=e>d/ < q(e? Z@e? ZAe* D ]JZBeBd8 eBd6 fD ]ZCe@DeC qeBd/ eBd. fD ]ZCeADeC qqrd:ZEG d;d< d d>eFZHG d?d@ d@ZIdAd ZJdBdC ZKdDd
ZLdEdF ZdGd ZMdHdI ZNdJd ZOdKdL ZPeQdMkr@eP dS )NzKa-Ping Yee zpGvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro, Raymond Hettinger, Trent Nelson, Michael Foord )open)lookupBOM_UTF8N)
TextIOWrapper)*)EXACT_TOKEN_TYPESz&^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)s ^[ \t\f]*(?:[#\r\n]|$)tokenizegenerate_tokensdetect_encoding
untokenize TokenInfoc @ s e Zd Zdd Zedd ZdS )r c C s$ d| j t| j f }d| j|d S )Nz%d (%s)z8TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r))type)r
tok_name_replace)selfannotated_type r /usr/lib64/python3.8/tokenize.py__repr__. s
zTokenInfo.__repr__c C s( | j tkr | jtkr t| j S | j S d S N)r
OPstringr r r r r
exact_type3 s
zTokenInfo.exact_typeN)__name__
__module__ __qualname__r propertyr r r r r r - s ztype string start end linec G s dd | d S )N(|))joinchoicesr r r group: r$ c G s t | d S )Nr r$ r" r r r any; r% r' c G s t | d S )N?r& r" r r r maybe< r% r) z[ \f\t]*z #[^\r\n]*z\\\r?\nz\w+z0[xX](?:_?[0-9a-fA-F])+z0[bB](?:_?[01])+z0[oO](?:_?[0-7])+z (?:0(?:_?0)*|[1-9](?:_?[0-9])*)z[eE][-+]?[0-9](?:_?[0-9])*z)[0-9](?:_?[0-9])*\.(?:[0-9](?:_?[0-9])*)?z\.[0-9](?:_?[0-9])*z[0-9](?:_?[0-9])*z[0-9](?:_?[0-9])*[jJ]z[jJ]c C s^ ddddddg} dh}| D ]>}t |D ].}t jdd |D D ]}|d| q@q(q|S )
Nbrufbrfr c S s g | ]}|| fqS r )upper).0cr r r
^ s z(_all_string_prefixes..)
_itertools permutationsproductaddr! )_valid_string_prefixesresultprefixtr, r r r _all_string_prefixesS s r= c C s t | t jS r )recompileUNICODE)exprr r r _compileb s rB z[^'\\]*(?:\\.[^'\\]*)*'z[^"\\]*(?:\\.[^"\\]*)*"z%[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''z%[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""z'''z"""z '[^\n'\\]*(?:\\.[^\n'\\]*)*'z "[^\n"\\]*(?:\\.[^\n"\\]*)*"T)reversez\r?\nz'[^\n'\\]*(?:\\.[^\n'\\]*)*'z"[^\n"\\]*(?:\\.[^\n"\\]*)*"z
\\\r?\n|\Z c @ s e Zd ZdS )
TokenErrorNr r r r r r r rG s rG c @ s e Zd ZdS )StopTokenizingNrH r r r r rI s rI c @ s, e Zd Zdd Zdd Zdd Zdd Zd S )
Untokenizerc C s g | _ d| _d| _d | _d S )N r )tokensprev_rowprev_colencodingr r r r __init__ s zUntokenizer.__init__c C s |\}}|| j k s&|| j kr>|| jk r>td||| j | j|| j }|rb| jd| d| _|| j }|r| jd| d S )Nz+start ({},{}) precedes previous end ({},{})\
r )rM rN
ValueErrorformatrL append)r startrowcol
row_offset
col_offsetr r r add_whitespace s
zUntokenizer.add_whitespacec C s6 t |}g }d}|D ]}t|dkr8| || q*|\}}}} }
|tkrV|| _q|tkrd q*|tkrz|| qnl|tkr| | \| _
| _ qnL|t t
fkrd}n:|r|r|d } |d t| kr| j| t| | _ d}| | | j| | \| _
| _ |t t
fkr| j
d7 _
d| _ qd| jS )NF TrK r r0 )iterlencompatENCODINGrO ENDMARKERINDENTrU DEDENTpoprM rN NEWLINENLrL r[ r! ) r iterableitindents startliner< tok_typetokenrV endlineindentr r r r sF
zUntokenizer.untokenizec
C s g }| j j}|d ttfk}d}t|g|D ]}|d d \}} |tkrR| | _q.|tt fkrf| d7 } |t
kr|rzd| } d}nd}|t kr|| q.n>|t kr|
q.n*|ttfkrd}n|r|r||d d}|| q.d S )Nr Fr\ rR Tr] )rL rU rf rg r5 chainra rO NAMENUMBERSTRINGrc rd re )
r rm rh rj toks_appendrk
prevstringtoktoknumtokvalr r r r` s8
zUntokenizer.compatN)r r r rP r[ r r` r r r r rJ s
%rJ c C s* t }|| }|jd k r&||j}|S r )rJ r rO encode)rh utoutr r r r s
c C sH | d d dd}|dks*|dr.dS |dks@|drDdS | S ) N _-utf-8zutf-8-)zlatin-1
iso-8859-1z iso-latin-1)zlatin-1-z iso-8859-1-z iso-latin-1-r )lowerreplace
startswith)orig_encencr r r _get_normal_name s r c s z j jW n tk
r$ d Y nX d d }d}fdd} fdd}| }|trpd |dd }d }|s||g fS ||}|r||gfS t|s||gfS | }|s||gfS ||}|r|||gfS |||gfS )
NFr c s$ z W S t k
r Y dS X d S Nr% )
StopIterationr readliner r read_or_stop? s z%detect_encoding..read_or_stopc s z| d}W n4 tk
rB d}d k r6d|}t|Y nX t|}|sVd S t|d}z t|}W n: t k
r d krd| }n d|}t|Y nX r|dkr؈d krd}n
d}t||d 7 }|S )
Nr z'invalid or missing encoding declarationz {} for {!r}rK zunknown encoding: z unknown encoding for {!r}: {}zencoding problem: utf-8z encoding problem for {!r}: utf-8z-sig)
decodeUnicodeDecodeErrorrT SyntaxError cookie_rematchr r$ r LookupError)ro line_stringmsgr rO codec) bom_foundfilenamer r find_cookieE s8
z$detect_encoding..find_cookieT utf-8-sig)__self__nameAttributeErrorr r blank_rer )r rO defaultr r firstsecondr )r r r r r
' s8
&
c C sX t | d}z2t|j\}}|d t||dd}d|_|W S | Y nX d S )Nrbr T)line_bufferingr+ )
_builtin_openr
r seekr modeclose)r bufferrO linestextr r r r s
r c C s6 t | \}}td}t|t| d|}t|j|S r )r
r5 repeatrq r^ _tokenize__next__)r rO consumedemptyrl_genr r r r s
c c sl d } }}d}d\}}d }dg} |d k rH|dkr6d}t t|dddV d}
d} z| }
| } W n tk
rv d} Y nX |d k r| |} |d 7 }dt| } }
|rp| std
||| }|r|d } }t t|| d | |||f|| V d\}}d }nf|rZ| d d d krZ| d
d dkrZt t || ||t| f|V d}d }qPn|| }|| }qPn|dkr|s| sqd}| |
k r| | dkr|d 7 }n8| | dkr|t
d t
}n| | dkrd}nq| d 7 } q| |
krq| | dkr| | dkr^| | d d}t t ||| f|| t| f| V | t|7 } t t
| | d || f|t| f| V qP|| d kr| | t t| d | |df|| f| V || d k r.|| krtdd|| | f| d d } t td|| f|| f| V qn| s*td|dfd}| |
k rPtt| | }|r|d \}}||f||f| }}} ||krq.| || | | }}||ks|dkr|dkr|dkrt t|||| V q|dkr|dkrt t
|||| V nt t|||| V q|dkr2t t |||| V q|tkrtt| }|| | }|r|d} | || }t t|||| f| V n||f}| |d }| }qPq|tks|d d tks|d d tkrF|d d kr2||f}tt|pt|d pt|d }| |d d }}| }qPnt t|||| V nf| rdt t |||| V nH|d krtd }n8|d kr|d 7 }n|d kr|d 8 }t t |||| V n*t t | | || f|| d f| V | d 7 } q.qP|
r |
d dkr t td|d t|
f|d t|
d fdV | d d D ] }t td|df|dfdV q,t t d|df|dfdV d S )!Nr
0123456789)r0 r r r )r r r0 r% rK zEOF in multi-line stringrQ z\
rR z#
#z
r] z3unindent does not match any outer indentation levelz
zEOF in multi-line statement.z...r\ r
\z([{z)]}) r ra r r r_ rG r rn rt
ERRORTOKENtabsizerstripCOMMENTrg rU rc IndentationErrorrd rB PseudoTokenspanrs rf
triple_quotedendpats
single_quotedget isidentifierrr r rb )r rO lnumparenlev continuednumcharscontstrneedcontcontlinerj last_linero posmaxstrstartendprogendmatchrn column
comment_token pseudomatchrV sposeposrm initialrp r r r r s<
*
"
. r c C s
t | d S r )r r r r r r d s c
s$ dd l } dd d fdd }| jdd}|jdd d
d d |jd
ddddd | }z|jr|j}t|d}tt|j}W 5 Q R X nd}t t
j jd }|D ]>}|j }|j
r|j}d|j|j } td| t| |jf qW n8 tk
r6 }
z0|
jd dd \} } ||
jd || | f W 5 d }
~
X Y n tk
r| }
z(|
jd \} } ||
jd || | f W 5 d }
~
X Y n tk
r }
z||
| W 5 d }
~
X Y nx tk
r }
z ||
W 5 d }
~
X Y nN tk
r td Y n2 tk
r }
z d|
W 5 d }
~
X Y nX d S ) Nr c S s t j| t jd d S )Nr )sysstderrwrite)messager r r perrorp s zmain..perrorc sR |r"|f| | f } d| n"|r8 d|| f n d| t d d S )Nz%s:%d:%d: error: %sz
%s: error: %sz error: %srK )r exit)r r locationargsr r r errort s zmain..errorzpython -m tokenize)progr r( z filename.pyz'the file to tokenize; defaults to stdin)destnargsmetavarhelpz-ez--exactexact
store_truez(display token names using the exact type)r actionr r zz %d,%d-%d,%d:z%-20s%-15s%-15rrK r z interrupted
zunexpected error: %s)NN)argparseArgumentParser add_argument
parse_argsr r listr r r r stdinr
r r rV rn printr r r r rG r OSErrorKeyboardInterrupt Exception)
r r parserr r r- rL rm
token_type token_rangeerrro r r r r mainl sT && r __main__)R
__author__ __credits__builtinsr r codecsr r collectionsior itertoolsr5 r> r rm r r? ASCIIr r __all__
namedtupler r$ r' r)
WhitespaceCommentIgnoreName Hexnumber Binnumber Octnumber Decnumber IntnumberExponent
PointfloatExpfloat Floatnumber
ImagnumberNumberr= rB StringPrefixSingleDoubleSingle3Double3TripleStringmapescapesortedSpecialFunny
PlainTokenTokenContStr PseudoExtrasr r _prefixsetr r r< r, r8 r r rG rI rJ r r r
r r r r r r r r r s
_ ] 8=