509318396 ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ 3 \NX=@sdZdZddlZddlZddlmZmZddlTddlm Z dd e e Dd d d gZ [ ye Wne k rzeZ YnXd dZddZddZdZdZeedeeeZdZdZdZdZeddZeeeeeZdZeddeeZdeZeeeZed ed!ZeeeeZ d"Z!d#Z"d$Z#d%Z$d&Z%ee%d'e%d(Z&ee%d)e%d*Z'ed+d,d-d.d/d0d1d2d3 Z(d4Z)ed5d6Z*ee(e)e*Z+ee e+e'eZ,ee,Z-ee%d7ed8de%d9ed:dZ.edee&Z/eee/e e+e.eZ0e1e2ej3e-e0e#e$f\Z4Z5Z6Z7ej3e!ej3e"e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7ddddddddd;4Z8iZ9xdD]Z:e:e9e:<qWiZ;xdD]Z:e:e;e:<qWdZGddde=Z?ddZ@e@fdd ZAddZBGdddZCej3dejDZEej3dejDZFddZGddZHdd ZIdd ZJeKdkrddlLZLeMeLjNdkrxeAeOeLjNdjPn eAeLjQjPdS)zKa-Ping Yee z@GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip MontanaroN)BOM_UTF8lookup)*)tokencCsg|]}|ddkr|qS)r_).0xrr./usr/lib64/python3.6/lib2to3/pgen2/tokenize.py %sr tokenizegenerate_tokens untokenizecGsddj|dS)N(|))join)choicesrrr group0srcGs t|dS)Nr)r)rrrr any1srcGs t|dS)N?)r)rrrr maybe2srz[ \f\t]*z #[^\r\n]*z\\\r?\nz [a-zA-Z_]\w*z0[bB]_?[01]+(?:_[01]+)*z(0[xX]_?[\da-fA-F]+(?:_[\da-fA-F]+)*[lL]?z0[oO]?_?[0-7]+(?:_[0-7]+)*[lL]?z[1-9]\d*(?:_\d+)*[lL]?z0[lL]?z[eE][-+]?\d+(?:_\d+)*z\d+(?:_\d+)*\.(?:\d+(?:_\d+)*)?z\.\d+(?:_\d+)*z \d+(?:_\d+)*z\d+(?:_\d+)*[jJ]z[jJ]z[^'\\]*(?:\\.[^'\\]*)*'z[^"\\]*(?:\\.[^"\\]*)*"z%[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''z%[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""z#(?:[uUrRbBfF]|[rR][bB]|[bBuU][rR])?'''"""z'[^\n'\\]*(?:\\.[^\n'\\]*)*'z"[^\n"\\]*(?:\\.[^\n"\\]*)*"z\*\*=?z>>=?z<<=?z<>z!=z//=?z->z[+\-*/%&@|^=<>]=?~z[][(){}]z\r?\nz[:;.,`@]z'[^\n'\\]*(?:\\.[^\n'\\]*)*'z"[^\n"\\]*(?:\\.[^\n"\\]*)*")4rrz'''z"""zr'''zr"""zu'''zu"""zb'''zb"""zf'''zf"""zur'''zur"""zbr'''zbr"""zrb'''zrb"""zR'''zR"""zU'''zU"""zB'''zB"""zF'''zF"""zuR'''zuR"""zUr'''zUr"""zUR'''zUR"""zbR'''zbR"""zBr'''zBr"""zBR'''zBR"""zrB'''zrB"""zRb'''zRb"""zRB'''zRB"""rRuUfFbBr'''r"""R'''R"""u'''u"""U'''U"""b'''b"""B'''B"""f'''f"""F'''F"""ur'''ur"""Ur'''Ur"""uR'''uR"""UR'''UR"""br'''br"""Br'''Br"""bR'''bR"""BR'''BR"""rb'''rb"""Rb'''Rb"""rB'''rB"""RB'''RB"""r'r"R'R"u'u"U'U"b'b"B'B"f'f"F'F"ur'ur"Ur'Ur"uR'uR"UR'UR"br'br"Br'Br"bR'bR"BR'BR"rb'rb"Rb'Rb"rB'rB"RB'RB"c@s eZdZdS) TokenErrorN)__name__ __module__ __qualname__rrrr rwsrwc@s eZdZdS)StopTokenizingN)rxryrzrrrr r{sr{c Cs4|\}}|\}}td||||t|t|fdS)Nz%d,%d-%d,%d: %s %s)printtok_namerepr) typerZxxx_todo_changemeZxxx_todo_changeme1lineZsrowZscolZerowZecolrrr printtokensrc Cs(yt||Wntk r"YnXdS)N) tokenize_loopr{)readline tokeneaterrrr r s cCsxt|D] }||q WdS)N)r)rrZ token_inforrr rsrc@s,eZdZddZddZddZddZd S) UntokenizercCsg|_d|_d|_dS)Nrr)tokensprev_rowprev_col)selfrrr __init__szUntokenizer.__init__cCs*|\}}||j}|r&|jjd|dS)N )rrappend)rstartrowcol col_offsetrrr add_whitespaces zUntokenizer.add_whitespacecCsxv|D]n}t|dkr$|j||P|\}}}}}|j||jj||\|_|_|ttfkr|jd7_d|_qWdj |jS)Nrr) lencompatrrrrrNEWLINENLr)riterablettok_typerrendrrrr rs        zUntokenizer.untokenizec Csd}g}|jj}|\}}|ttfkr,|d7}|ttfkr|t kr|j qBn*|ttfkrd}n|r|r||dd}||qBWdS)NFrTrr) rrNAMENUMBERrrASYNCAWAITINDENTDEDENTpop) rrr startlineindents toks_appendtoknumtokvaltokrrr rs0      zUntokenizer.compatN)rxryrzrrrrrrrr rsrz&^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)s^[ \t\f]*(?:[#\r\n]|$)cCsH|ddjjdd}|dks*|jdr.dS|d ks@|jd rDdS|S)N r-zutf-8zutf-8-latin-1 iso-8859-1 iso-latin-1latin-1- iso-8859-1- iso-latin-1-)rrr)rrr)lowerreplace startswith)orig_encencrrr _get_normal_names rcsdd}d}fdd}fdd}|}|jtrHd|dd}d }|sT|gfS||}|rj||gfStj|s~||gfS|}|s||gfS||}|r|||gfS|||gfS) NFzutf-8c s"yStk rtSXdS)N) StopIterationbytesr)rrr read_or_stop sz%detect_encoding..read_or_stopcsy|jd}Wntk r"dSXtj|}|s6dSt|jd}y t|}Wn tk rptd|YnXr|j dkrtd|d7}|S)Nasciirzunknown encoding: zutf-8zencoding problem: utf-8z-sig) decodeUnicodeDecodeError cookie_rematchrrr LookupError SyntaxErrorname)r line_stringrencodingcodec) bom_foundrr find_cookie&s"   z$detect_encoding..find_cookieTz utf-8-sig)rrblank_rer)rrdefaultrrfirstsecondr)rrr detect_encoding s0         rcCst}|j|S)N)rr)rutrrr rTsc!csd}}}tjdd}}d\}}d}dg} d} d} d} d} xy |}Wntk rdd}YnX|d}dt|}}|rF|std||j|}|r|jd}}t||d||||f||fVd \}}d}nd|r0|d!dd kr0|d"dd kr0t||||t|f|fVd}d}qBn||}||}qBnF|dkrt| rt|s`Pd}xf||kr||d kr|d}n6||d kr|t dt }n||dkrd}nP|d}qfW||krP| r| Vd} ||dkr||dkrh||dj d}|t|}t |||f||t|f|fVt ||d||f|t|f|fVqBt t f||dk||d||f|t|f|fVqB|| d#kr| j |t|d||df||f|fVxt|| d$krJ|| krtdd|||f| dd%} | r.| | d&kr.d} d} d} td||f||f|fVqW| r| r| | d'krd} d} d} n|std|dfd}x||kr8tj||}|r |jd\}}||f||f|}}}|||||}}||ks|dkr|dkrt||||fVq4|dkrft}|dkr8t }n | rBd} | rR| Vd} |||||fVq4|dkr| r| Vd} t ||||fVq4|tkrt|}|j||}|r|jd}|||}| r| Vd} t||||f|fVn||f}||d}|}Pq4|tks@|ddtks@|dd tkr|d(dkr||f}t|pxt|dpxt|d}||dd}}|}Pn | r| Vd} t||||fVq4||kr|d)kr| r|dkrtnt||||fVqt||||f}|dkr| r|} q|dkrx| rx| dtkrx| ddkrxd} | d*} t| d| d| d | dfVd} | r| Vd} |Vnz|dkr| r| Vd} t ||||f|fVd}nF|dkr|d}n|dkr|d}| r| Vd} t||||fVn(t||||f||df|fV|d}qWqBW| rN| Vd} x.| ddD]} td|df|dfdfVq\Wtd|df|dfdfVdS)+Nrr 0123456789rFrzEOF in multi-line stringrz\ rz\ r  z# #z z3unindent does not match any outer indentation levelz zEOF in multi-line statement.T asyncawaitdef\z([{z)]})rr)rrrrrrrr)rrr)stringZ ascii_lettersrrrwrrSTRING ERRORTOKENtabsizerstripCOMMENTrrrIndentationErrorr pseudoprogspanrr triple_quotedendprogs single_quotedrrrOP ENDMARKER)!rlnumparenlev continuedZ namecharsnumcharscontstrneedcontcontlinerstashed async_defasync_def_indent async_def_nlrposmaxstrstartendprogendmatchrcolumn comment_tokennl_pos pseudomatchrsposeposrinitialnewlinerindentrrr risp     *                             __main__)*rrr&r'r(r)r*r+r,r-r.r/r0r1r2r3r4r5r6r7r8r9r:r;r<r=r>r?r@rArBrCrDrErFrGrHrIrJrKrLrM)*rrrNrOrPrQrRrSrTrUrVrWrXrYrZr[r\r]r^r_r`rarbrcrdrerfrgrhrirjrkrlrmrnrorprqrrrsrtru)R __author__ __credits__rrecodecsrrZlib2to3.pgen2.tokenrrdir__all__r NameErrorstrrrr WhitespaceCommentIgnoreName Binnumber Hexnumber Octnumber Decnumber IntnumberExponent PointfloatExpfloat Floatnumber ImagnumberNumberSingleDoubleSingle3Double3Z _litprefixTripleStringOperatorBracketSpecialFunny PlainTokenTokenContStr PseudoExtras PseudoTokenlistmapcompileZ tokenprogrZ single3progZ double3progrrrrr Exceptionrwr{rr rrASCIIrrrrrrrxsysrargvopenrstdinrrrr s              8 Ic