3
\8s - @ s d Z dZddlmZ ddlmZmZ ddlZddl m
Z
ddl m Z ddl Z
ddlZddlZddlT ejd ejZejd
ejZddlZejd d d
ddddg Z[eZd ee< ed Zdee< ed Zdee< ed7 Zee e e e e e!e"e#e$e%e&e'e(e)e*e+e,e-e.e/e0e1e2e3e4e5e6e7e8e9e:e;ee?e@eAeBeCeDeEeFd,ZGG dd dejHddZIdd ZJdd ZKd d ZLd ZMd ZNeMeKd eM eLeN ZOd!ZPd"ZQd#ZRd$ZSd%ZTeJeQeReSeTZUd&ZVeJd'd(eLeV ZWd)eV ZXeJeWeXZYeJd*eYd+ ZZeJeZeYeUZ[d,d- Z\d.d/ Z]eJe\ Z^d0Z_d1Z`d2Zad3ZbeJe^d4 e^d5 ZceJe^d6 e^d7 ZdeJd8d9d:d;dd?Zed@ZfeJdAdBdCZgeJeeefegZheJe[ehedePZieOei ZjeJe^dD eJdEd e^dF eJdGd ZkeJdHeNecZleMeJele[ehekeP Zmi Znx@e\ D ]6Zoe_eneodE < e`eneodG < eaeneod4 < ebeneod5 < qW ep Zqep Zrx\e\ D ]RZsx$esdG esdE fD ]Zteqjuet q4W x$esd5 esd4 fD ]Zterjuet qZW q W dIZvG dJdK dKewZxG dLdM dMewZyG dNdO dOZzdPd Z{dQdR Z|dSd
Z}dTdU ZdVd Z~dWdX ZdYdZ Zd[d\ Zed]kre dS )^zKa-Ping Yee zpGvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro, Raymond Hettinger, Trent Nelson, Michael Foord )open)lookupBOM_UTF8N)
TextIOWrapper)chain)*z&^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)s ^[ \t\f]*(?:[#\r\n]|$)COMMENTtokenizedetect_encodingNL
untokenizeENCODING TokenInfo ),()[]:,;+-r /|&<>=.%{}z==z!=z<=z>=~^z<>z**z+=z-=z*=z/=z%=z&=z|=z^=z<<=z>>=z**=z//z//=@z@=c @ s e Zd Zdd Zedd ZdS )r c C s$ d| j t| j f }d| j|d S )Nz%d (%s)z8TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r))type)r( tok_name_replace)selfannotated_type r- /usr/lib64/python3.6/tokenize.py__repr__d s zTokenInfo.__repr__c C s( | j tkr | jtkr t| j S | j S d S )N)r( OPstringEXACT_TOKEN_TYPES)r+ r- r- r.
exact_typei s
zTokenInfo.exact_typeN)__name__
__module__ __qualname__r/ propertyr3 r- r- r- r. r c s ztype string start end linec G s ddj | d S )Nr r r )join)choicesr- r- r. groupp s r: c G s t | d S )Nr )r: )r9 r- r- r. anyq s r; c G s t | d S )N?)r: )r9 r- r- r. mayber s r= z[ \f\t]*z #[^\r\n]*z\\\r?\nz\w+z0[xX](?:_?[0-9a-fA-F])+z0[bB](?:_?[01])+z0[oO](?:_?[0-7])+z (?:0(?:_?0)*|[1-9](?:_?[0-9])*)z[eE][-+]?[0-9](?:_?[0-9])*z)[0-9](?:_?[0-9])*\.(?:[0-9](?:_?[0-9])*)?z\.[0-9](?:_?[0-9])*z[0-9](?:_?[0-9])*z[0-9](?:_?[0-9])*[jJ]z[jJ]c C sn ddddddg} t dg}xN| D ]F}x@tj|D ]2}x,tjdd |D D ]}|jdj| qJW q0W q W |S )
Nbrufbrfr c S s g | ]}||j fqS r- )upper).0cr- r- r.
s z(_all_string_prefixes..)set
_itertools permutationsproductaddr8 )_valid_string_prefixesresultprefixtr@ r- r- r. _all_string_prefixes s
rR c C s t j| t jS )N)recompileUNICODE)exprr- r- r. _compile s rW z[^'\\]*(?:\\.[^'\\]*)*'z[^"\\]*(?:\\.[^"\\]*)*"z%[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''z%[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""z'''z"""z '[^\n'\\]*(?:\\.[^\n'\\]*)*'z "[^\n"\\]*(?:\\.[^\n"\\]*)*"z\*\*=?z>>=?z<<=?z!=z//=?z->z[+\-*/%&@|^=<>]=?r% z[][(){}]z\r?\nz\.\.\.z[:;.,@]z'[^\n'\\]*(?:\\.[^\n'\\]*)*'z"[^\n"\\]*(?:\\.[^\n"\\]*)*"z
\\\r?\n|\Z c @ s e Zd ZdS )
TokenErrorN)r4 r5 r6 r- r- r- r. r[ s r[ c @ s e Zd ZdS )StopTokenizingN)r4 r5 r6 r- r- r- r. r\ s r\ c @ s, e Zd Zdd Zdd Zdd Zdd Zd S )
Untokenizerc C s g | _ d| _d| _d | _d S )Nr r )tokensprev_rowprev_colencoding)r+ r- r- r. __init__ s zUntokenizer.__init__c C s |\}}|| j k s&|| j kr>|| jk r>tdj||| j | j|| j }|rb| jjd| d| _|| j }|r| jjd| d S )Nz+start ({},{}) precedes previous end ({},{})z\
r )r_ r`
ValueErrorformatr^ append)r+ startrowcol
row_offset
col_offsetr- r- r. add_whitespace s
zUntokenizer.add_whitespacec C s4 t |}g }d}x|D ]
}t|dkr8| j|| P |\}}}} }
|tkrV|| _q|tkr`P |tkrv|j| qnl|tkr|j | \| _
| _ qnL|t t
fkrd}n:|r|r|d } |d t| kr| jj| t| | _ d}| j| | jj| | \| _
| _ |t t
fkr| j
d7 _
d| _ qW dj| jS )NFr Tr r rD )iterlencompatr
ra ENDMARKERINDENTrf DEDENTpopr_ r` NEWLINEr r^ rl r8 ) r+ iterableitindents startlinerQ tok_typetokenrg endlineindentr- r- r. r sF
zUntokenizer.untokenizec
C s g }| j j}|d ttfk}d}xt|g|D ]}|d d \}} |tkrR| | _q.|ttt t
fkrj| d7 } |t kr|r~d| } d}nd}|t kr|j| q.n>|t
kr|j q.n*|ttfkrd}n|r|r||d d}|| q.W d S )Nr Fr rc Tr rm )r^ rf ru r r r
ra NAMENUMBERASYNCAWAITSTRINGrr rs rt )
r+ r{ rv rx toks_appendry
prevstringtoktoknumtokvalr- r- r. rp s8
zUntokenizer.compatN)r4 r5 r6 rb rl r rp r- r- r- r. r] s
%r] c C s* t }|j| }|jd k r&|j|j}|S )N)r] r ra encode)rv utoutr- r- r. r = s
c C sH | d d j jdd}|dks*|jdr.dS |d ks@|jd
rDdS | S )N _r zutf-8zutf-8-latin-1
iso-8859-1 iso-latin-1latin-1- iso-8859-1- iso-latin-1-)r r r )r r r )lowerreplace
startswith)orig_encencr- r- r. _get_normal_nameX s
r c s y j jW n tk
r$ d Y nX d d }d}fdd} fdd}| }|jtrpd |dd }d }|s||g fS ||}|r||gfS tj|s||gfS | }|s||gfS ||}|r|||gfS |||gfS )
NFzutf-8c s y S t k
r dS X d S )N )
StopIterationr- )readliner- r. read_or_stop{ s z%detect_encoding..read_or_stopc s y| j d}W n4 tk
rB d}d k r6dj|}t|Y nX tj|}|sVd S t|jd}y t|}W n: t k
r d krd| }n dj|}t|Y nX r|dkr؈d krd}n
dj}t||d 7 }|S )
Nzutf-8z'invalid or missing encoding declarationz {} for {!r}r zunknown encoding: z unknown encoding for {!r}: {}zencoding problem: utf-8z encoding problem for {!r}: utf-8z-sig)
decodeUnicodeDecodeErrorre SyntaxError cookie_rematchr r: r LookupError)r} line_stringmsgr ra codec) bom_foundfilenamer- r. find_cookie s6
z$detect_encoding..find_cookieTr z utf-8-sig)__self__nameAttributeErrorr r blank_rer )r ra defaultr r firstsecondr- )r r r r. r
c s8
&
c C sV t | d}y0t|j\}}|jd t||dd}d|_|S |j Y nX d S )Nrbr T)line_bufferingr? )
_builtin_openr
r seekr modeclose)r bufferra linestextr- r- r. r s
r c C sB ddl m}m} t| \}}t| d}|d}t||||j|S )Nr )r repeatr ) itertoolsr r r
rn _tokenize__next__)r r r ra consumedrl_genemptyr- r- r. r s
c ! c s d } }}d}d!\}}d }dg} d }
d} d} d}
|d k rX|dkrFd}t t|d"d#dV d}d}x
y|}| }W n tk
r d}Y nX |d k r|j|}|d7 }dt| }}|r|std ||j|}|r|jd }}t t||d | |||f|| V d$\}}d }nf|rn|d%d d krn|d&d d
krnt t || ||t|f|V d}d }qdn|| }|| }qdn@|dkr| r|sP d}xf||k r|| dkr|d7 }n6|| dkr|t
d t
}n|| dkrd}nP |d7 }qW ||krP || dkr|| dkr||d j d}|t| }t t |||f||t| f|V t t
||d ||f|t|f|V qdt t
t f|| dk ||d ||f|t|f|V qd|| d' kr | j| t t|d | |df||f|V xv|| d( k r|| kr8tdd|||f| d d) } | rd| | d* krdd} d}
d} t td||f||f|V qW | r|
r| | d+ krd} d}
d} n|std|dfd}x||k rhttj||}|r:|jd\}}||f||f| }} }||kr q||| || } } | |ksZ| dkrp| dkrp| dkrpt t| || |V qd| dkr|
r|
V d }
|dkrt t
| || |V n t t| || |V | r8d}
qd| dkr|
r|
V d }
t t | || |V qd| tkrltt| }|j||}|rN|jd}||| } t t| |||f|V n||f}||d }|}P qd| tks| d d
tks| d d tkr| d, dkr||f}ttj| ptj| d ptj| d
}||d d }}|}P nt t| || |V qd| j r| d-krL| rLt | dkr8t nt | || |V qt t | || |} | dkrv|
rv| }
q| d kr|
r|
j t kr|
j dkrd} | d. } t t |
j |
j!|
j|
j"V d }
|
r|
V d }
| V nX| d krd}nH| d kr|d7 }n| d kr|d8 }|
r&|
V d }
t t#| || |V n*t t || ||f||d f|V |d7 }qW qdW |
r~|
V d }
|r|d/ dkrt td|d t|f|d t|d fdV x0| dd D ] } t td|df|dfdV qW t t$d|df|dfdV d S )0Nr
0123456789rD Fz utf-8-sigzutf-8r r zEOF in multi-line stringr z\
r z\
rc z#
#z
z3unindent does not match any outer indentation levelz
zEOF in multi-line statementr! z...T
asyncawaitdef\z([{z)]})rD r )r r )r r )rD r rm rm rm rm rm rm )r r rm rm )%r r
r r ro r[ r r| r
ERRORTOKENtabsizerstripr r rf rr IndentationErrorrs rW PseudoTokenspanr ru
triple_quotedendpats
single_quotedget isidentifierr r r r( r1 rg r} r0 rq )!r ra lnumparenlev continuednumcharscontstrneedcontcontlinerx stashed async_defasync_def_indent async_def_nl last_liner} posmaxstrstartendprogendmatchr| column
comment_tokennl_pos pseudomatchrg sposeposr{ initialr r~ r- r- r. r sr
*
"
. r c C s
t | d S )N)r )r r- r- r. generate_tokens s r c
s( dd l } dd d fdd }| jdd}|jdd d
d d |jd
ddddd |j }y|jr|j}t|d}tt|j}W d Q R X nd}t t
j jd }xF|D ]>}|j }|j
r|j}d|j|j } td| t| |jf qW W n8 tk
r: }
z2|
jd dd \} } ||
jd || | f W Y d d }
~
X n tk
r }
z*|
jd \} } ||
jd || | f W Y d d }
~
X n tk
r }
z||
| W Y d d }
~
X nx tk
r }
z||
W Y d d }
~
X nN tk
r td Y n2 tk
r" }
z d|
W Y d d }
~
X nX d S ) Nr c S s t | tjd d S )N)file)printsysstderr)messager- r- r. perror s zmain..perrorc sR |r"|f| | f } d| n"|r8 d|| f n d| t jd d S )Nz%s:%d:%d: error: %sz
%s: error: %sz error: %sr )r exit)r r locationargs)r r- r. error s zmain..errorzpython -m tokenize)progr r< z filename.pyz'the file to tokenize; defaults to stdin)destnargsmetavarhelpz-ez--exactexact
store_truez(display token names using the exact type)r actionr r zz %d,%d-%d,%d:z%-20s%-15s%-15rr r z interrupted
zunexpected error: %s)NN)argparseArgumentParser add_argument
parse_argsr r listr r r r stdinr( r r3 rg r| r r) r1 r r r[ r OSErrorKeyboardInterrupt Exception)
r r parserr r rA r^ r{
token_type token_rangeerrr} r r- )r r. main sN
&& r __main__)
__author__ __credits__builtinsr r codecsr r collectionsior r r rJ rS r r{ rT ASCIIr r __all__N_TOKENSr r) r r
LPARRPARLSQBRSQBCOLONCOMMASEMIPLUSMINUSSTARSLASHVBARAMPERLESSGREATEREQUALDOTPERCENTLBRACERBRACEEQEQUALNOTEQUAL LESSEQUAL GREATEREQUALTILDE
CIRCUMFLEX LEFTSHIFT
RIGHTSHIFT
DOUBLESTAR PLUSEQUALMINEQUAL STAREQUAL
SLASHEQUAL PERCENTEQUAL
AMPEREQUAL VBAREQUALCIRCUMFLEXEQUALLEFTSHIFTEQUALRIGHTSHIFTEQUALDOUBLESTAREQUAL DOUBLESLASHDOUBLESLASHEQUALATATEQUALr2
namedtupler r: r; r=
WhitespaceCommentIgnoreName Hexnumber Binnumber Octnumber Decnumber IntnumberExponent
PointfloatExpfloat Floatnumber
ImagnumberNumberrR rW StringPrefixSingleDoubleSingle3Double3TripleStringOperatorBracketSpecialFunny
PlainTokenTokenContStr PseudoExtrasr r _prefixrI r r rQ r@ rM r r
r[ r\ r] r r r
r r r r r4 r- r- r- r. s
_ ] x<