Name	Size	Mode	Actions
2to3.rst.txt	15898	0644	editdlrm
abc.rst.txt	11805	0644	editdlrm
aifc.rst.txt	7503	0644	editdlrm
allos.rst.txt	678	0644	editdlrm
archiving.rst.txt	440	0644	editdlrm
argparse.rst.txt	77480	0644	editdlrm
array.rst.txt	10812	0644	editdlrm
ast.rst.txt	10159	0644	editdlrm
asynchat.rst.txt	8528	0644	editdlrm
asyncio-dev.rst.txt	13398	0644	editdlrm
asyncio-eventloop.rst.txt	33701	0644	editdlrm
asyncio-eventloops.rst.txt	7660	0644	editdlrm
asyncio-protocol.rst.txt	25170	0644	editdlrm
asyncio-queue.rst.txt	4328	0644	editdlrm
asyncio-stream.rst.txt	14363	0644	editdlrm
asyncio-subprocess.rst.txt	15544	0644	editdlrm
asyncio-sync.rst.txt	9078	0644	editdlrm
asyncio-task.rst.txt	25406	0644	editdlrm
asyncio.rst.txt	2272	0644	editdlrm
asyncore.rst.txt	13624	0644	editdlrm
atexit.rst.txt	3669	0644	editdlrm
audioop.rst.txt	10712	0644	editdlrm
base64.rst.txt	10371	0644	editdlrm
bdb.rst.txt	12985	0644	editdlrm
binary.rst.txt	654	0644	editdlrm
binascii.rst.txt	6539	0644	editdlrm
binhex.rst.txt	1709	0644	editdlrm
bisect.rst.txt	5398	0644	editdlrm
builtins.rst.txt	1465	0644	editdlrm
bz2.rst.txt	8825	0644	editdlrm
calendar.rst.txt	10988	0644	editdlrm
cgi.rst.txt	22650	0644	editdlrm
cgitb.rst.txt	2918	0644	editdlrm
chunk.rst.txt	5094	0644	editdlrm
cmath.rst.txt	9326	0644	editdlrm
cmd.rst.txt	13795	0644	editdlrm
code.rst.txt	7817	0644	editdlrm
codecs.rst.txt	75564	0644	editdlrm
codeop.rst.txt	3021	0644	editdlrm
collections.abc.rst.txt	12861	0644	editdlrm
collections.rst.txt	46063	0644	editdlrm
colorsys.rst.txt	1820	0644	editdlrm
compileall.rst.txt	8975	0644	editdlrm
concurrency.rst.txt	673	0644	editdlrm
concurrent.futures.rst.txt	16910	0644	editdlrm
concurrent.rst.txt	171	0644	editdlrm
configparser.rst.txt	48756	0644	editdlrm
constants.rst.txt	3331	0644	editdlrm
contextlib.rst.txt	28175	0644	editdlrm
copy.rst.txt	3378	0644	editdlrm
copyreg.rst.txt	2182	0644	editdlrm
crypt.rst.txt	5122	0644	editdlrm
crypto.rst.txt	411	0644	editdlrm
csv.rst.txt	19902	0644	editdlrm
ctypes.rst.txt	88613	0644	editdlrm
curses.ascii.rst.txt	8920	0644	editdlrm
curses.panel.rst.txt	2764	0644	editdlrm
curses.rst.txt	76397	0644	editdlrm
custominterp.rst.txt	569	0644	editdlrm
datatypes.rst.txt	751	0644	editdlrm
datetime.rst.txt	90524	0644	editdlrm
dbm.rst.txt	14293	0644	editdlrm
debug.rst.txt	471	0644	editdlrm
decimal.rst.txt	74869	0644	editdlrm
development.rst.txt	704	0644	editdlrm
difflib.rst.txt	29861	0644	editdlrm
dis.rst.txt	32567	0644	editdlrm
distribution.rst.txt	452	0644	editdlrm
distutils.rst.txt	1974	0644	editdlrm
doctest.rst.txt	71743	0644	editdlrm
dummy_threading.rst.txt	784	0644	editdlrm
email.charset.rst.txt	9293	0644	editdlrm
email.compat32-message.rst.txt	33493	0644	editdlrm
email.contentmanager.rst.txt	9112	0644	editdlrm
email.encoders.rst.txt	2715	0644	editdlrm
email.errors.rst.txt	4792	0644	editdlrm
email.examples.rst.txt	1915	0644	editdlrm
email.generator.rst.txt	13637	0644	editdlrm
email.header.rst.txt	9188	0644	editdlrm
email.headerregistry.rst.txt	17957	0644	editdlrm
email.iterators.rst.txt	2795	0644	editdlrm
email.message.rst.txt	33008	0644	editdlrm
email.mime.rst.txt	11721	0644	editdlrm
email.parser.rst.txt	14092	0644	editdlrm
email.policy.rst.txt	27022	0644	editdlrm
email.rst.txt	6785	0644	editdlrm
email.util.rst.txt	9214	0644	editdlrm
ensurepip.rst.txt	4987	0644	editdlrm
enum.rst.txt	32751	0644	editdlrm
errno.rst.txt	6811	0644	editdlrm
exceptions.rst.txt	25570	0644	editdlrm
faulthandler.rst.txt	6136	0644	editdlrm
fcntl.rst.txt	7152	0644	editdlrm
filecmp.rst.txt	5623	0644	editdlrm
fileformats.rst.txt	287	0644	editdlrm
fileinput.rst.txt	8051	0644	editdlrm
filesys.rst.txt	961	0644	editdlrm
fnmatch.rst.txt	3103	0644	editdlrm
formatter.rst.txt	13242	0644	editdlrm
fpectl.rst.txt	4180	0644	editdlrm
fractions.rst.txt	6364	0644	editdlrm
frameworks.rst.txt	391	0644	editdlrm
ftplib.rst.txt	17552	0644	editdlrm
functional.rst.txt	365	0644	editdlrm
functions.rst.txt	71621	0644	editdlrm
functools.rst.txt	18503	0644	editdlrm
gc.rst.txt	9694	0644	editdlrm
getopt.rst.txt	6557	0644	editdlrm
getpass.rst.txt	1882	0644	editdlrm
gettext.rst.txt	25588	0644	editdlrm
glob.rst.txt	3508	0644	editdlrm
grp.rst.txt	2417	0644	editdlrm
gzip.rst.txt	8301	0644	editdlrm
hashlib.rst.txt	26523	0644	editdlrm
heapq.rst.txt	13402	0644	editdlrm
hmac.rst.txt	4020	0644	editdlrm
html.entities.rst.txt	1319	0644	editdlrm
html.parser.rst.txt	11275	0644	editdlrm
html.rst.txt	1304	0644	editdlrm
http.client.rst.txt	18870	0644	editdlrm
http.cookiejar.rst.txt	27832	0644	editdlrm
http.cookies.rst.txt	8645	0644	editdlrm
http.rst.txt	7096	0644	editdlrm
http.server.rst.txt	17639	0644	editdlrm
i18n.rst.txt	408	0644	editdlrm
idle.rst.txt	25834	0644	editdlrm
imaplib.rst.txt	20367	0644	editdlrm
imghdr.rst.txt	2938	0644	editdlrm
imp.rst.txt	15548	0644	editdlrm
importlib.rst.txt	53563	0644	editdlrm
index.rst.txt	2258	0644	editdlrm
inspect.rst.txt	53928	0644	editdlrm
internet.rst.txt	992	0644	editdlrm
intro.rst.txt	2645	0644	editdlrm
io.rst.txt	40328	0644	editdlrm
ipaddress.rst.txt	32340	0644	editdlrm
ipc.rst.txt	649	0644	editdlrm
itertools.rst.txt	38459	0644	editdlrm
json.rst.txt	27876	0644	editdlrm
keyword.rst.txt	617	0644	editdlrm
language.rst.txt	522	0644	editdlrm
linecache.rst.txt	2408	0644	editdlrm
locale.rst.txt	25470	0644	editdlrm
logging.config.rst.txt	33886	0644	editdlrm
logging.handlers.rst.txt	42497	0644	editdlrm
logging.rst.txt	60479	0644	editdlrm
lzma.rst.txt	17361	0644	editdlrm
macpath.rst.txt	707	0644	editdlrm
mailbox.rst.txt	63055	0644	editdlrm
mailcap.rst.txt	3691	0644	editdlrm
markup.rst.txt	679	0644	editdlrm
marshal.rst.txt	4855	0644	editdlrm
math.rst.txt	14375	0644	editdlrm
mimetypes.rst.txt	9839	0644	editdlrm
misc.rst.txt	247	0644	editdlrm
mm.rst.txt	431	0644	editdlrm
mmap.rst.txt	11232	0644	editdlrm
modulefinder.rst.txt	3240	0644	editdlrm
modules.rst.txt	355	0644	editdlrm
msilib.rst.txt	18612	0644	editdlrm
msvcrt.rst.txt	4392	0644	editdlrm
multiprocessing.rst.txt	104494	0644	editdlrm
netdata.rst.txt	339	0644	editdlrm
netrc.rst.txt	3007	0644	editdlrm
nis.rst.txt	1993	0644	editdlrm
nntplib.rst.txt	21821	0644	editdlrm
numbers.rst.txt	8089	0644	editdlrm
numeric.rst.txt	696	0644	editdlrm
operator.rst.txt	18989	0644	editdlrm
optparse.rst.txt	76996	0644	editdlrm
os.path.rst.txt	16419	0644	editdlrm
os.rst.txt	137534	0644	editdlrm
ossaudiodev.rst.txt	17845	0644	editdlrm
othergui.rst.txt	2822	0644	editdlrm
parser.rst.txt	15148	0644	editdlrm
pathlib.rst.txt	30045	0644	editdlrm
pdb.rst.txt	19454	0644	editdlrm
persistence.rst.txt	591	0644	editdlrm
pickle.rst.txt	37440	0644	editdlrm
pickletools.rst.txt	3729	0644	editdlrm
pipes.rst.txt	2557	0644	editdlrm
pkgutil.rst.txt	8730	0644	editdlrm
platform.rst.txt	9624	0644	editdlrm
plistlib.rst.txt	7398	0644	editdlrm
poplib.rst.txt	8197	0644	editdlrm
posix.rst.txt	3694	0644	editdlrm
pprint.rst.txt	14274	0644	editdlrm
profile.rst.txt	27969	0644	editdlrm
pty.rst.txt	3123	0644	editdlrm
pwd.rst.txt	2739	0644	editdlrm
pyclbr.rst.txt	3297	0644	editdlrm
pydoc.rst.txt	4688	0644	editdlrm
pyexpat.rst.txt	28620	0644	editdlrm
python.rst.txt	475	0644	editdlrm
py_compile.rst.txt	3851	0644	editdlrm
queue.rst.txt	7268	0644	editdlrm
quopri.rst.txt	2571	0644	editdlrm
random.rst.txt	18503	0644	editdlrm
re.rst.txt	64936	0644	editdlrm
readline.rst.txt	11943	0644	editdlrm
reprlib.rst.txt	5162	0644	editdlrm
resource.rst.txt	12291	0644	editdlrm
rlcompleter.rst.txt	2294	0644	editdlrm
runpy.rst.txt	8285	0644	editdlrm
sched.rst.txt	4842	0644	editdlrm
secrets.rst.txt	5930	0644	editdlrm
select.rst.txt	28175	0644	editdlrm
selectors.rst.txt	8925	0644	editdlrm
shelve.rst.txt	8390	0644	editdlrm
shlex.rst.txt	16441	0644	editdlrm
shutil.rst.txt	26014	0644	editdlrm
signal.rst.txt	17218	0644	editdlrm
site.rst.txt	9628	0644	editdlrm
smtpd.rst.txt	10806	0644	editdlrm
smtplib.rst.txt	23439	0644	editdlrm
sndhdr.rst.txt	1994	0644	editdlrm
socket.rst.txt	68202	0644	editdlrm
socketserver.rst.txt	23712	0644	editdlrm
spwd.rst.txt	2958	0644	editdlrm
sqlite3.rst.txt	39215	0644	editdlrm
ssl.rst.txt	96021	0644	editdlrm
stat.rst.txt	9805	0644	editdlrm
statistics.rst.txt	15301	0644	editdlrm
stdtypes.rst.txt	179240	0644	editdlrm
string.rst.txt	35374	0644	editdlrm
stringprep.rst.txt	4279	0644	editdlrm
struct.rst.txt	19581	0644	editdlrm
subprocess.rst.txt	45793	0644	editdlrm
sunau.rst.txt	7390	0644	editdlrm
superseded.rst.txt	258	0644	editdlrm
symbol.rst.txt	975	0644	editdlrm
symtable.rst.txt	4964	0644	editdlrm
sys.rst.txt	58273	0644	editdlrm
sysconfig.rst.txt	8749	0644	editdlrm
syslog.rst.txt	4302	0644	editdlrm
tabnanny.rst.txt	2007	0644	editdlrm
tarfile.rst.txt	31749	0644	editdlrm
telnetlib.rst.txt	7899	0644	editdlrm
tempfile.rst.txt	13857	0644	editdlrm
termios.rst.txt	3751	0644	editdlrm
test.rst.txt	25764	0644	editdlrm
text.rst.txt	584	0644	editdlrm
textwrap.rst.txt	10519	0644	editdlrm
threading.rst.txt	39257	0644	editdlrm
time.rst.txt	32967	0644	editdlrm
timeit.rst.txt	13000	0644	editdlrm
tk.rst.txt	1639	0644	editdlrm
tkinter.rst.txt	33027	0644	editdlrm
tkinter.scrolledtext.rst.txt	1255	0644	editdlrm
tkinter.tix.rst.txt	22653	0644	editdlrm
tkinter.ttk.rst.txt	58489	0644	editdlrm
token.rst.txt	2607	0644	editdlrm
tokenize.rst.txt	10002	0644	editdlrm
trace.rst.txt	6914	0644	editdlrm
traceback.rst.txt	17854	0644	editdlrm
tracemalloc.rst.txt	22089	0644	editdlrm
tty.rst.txt	1097	0644	editdlrm
turtle.rst.txt	71221	0644	editdlrm
types.rst.txt	9999	0644	editdlrm
typing.rst.txt	34332	0644	editdlrm
undoc.rst.txt	780	0644	editdlrm
unicodedata.rst.txt	5762	0644	editdlrm
unittest.mock-examples.rst.txt	46206	0644	editdlrm
unittest.mock.rst.txt	85991	0644	editdlrm
unittest.rst.txt	93304	0644	editdlrm
unix.rst.txt	446	0644	editdlrm
urllib.error.rst.txt	2199	0644	editdlrm
urllib.parse.rst.txt	27332	0644	editdlrm
urllib.request.rst.txt	59360	0644	editdlrm
urllib.robotparser.rst.txt	2969	0644	editdlrm
urllib.rst.txt	466	0644	editdlrm
uu.rst.txt	2387	0644	editdlrm
uuid.rst.txt	8760	0644	editdlrm
venv.rst.txt	20621	0644	editdlrm
warnings.rst.txt	20152	0644	editdlrm
wave.rst.txt	6870	0644	editdlrm
weakref.rst.txt	21328	0644	editdlrm
webbrowser.rst.txt	9783	0644	editdlrm
windows.rst.txt	272	0644	editdlrm
winreg.rst.txt	24073	0644	editdlrm
winsound.rst.txt	5134	0644	editdlrm
wsgiref.rst.txt	33047	0644	editdlrm
xdrlib.rst.txt	8078	0644	editdlrm
xml.dom.minidom.rst.txt	10198	0644	editdlrm
xml.dom.pulldom.rst.txt	5186	0644	editdlrm
xml.dom.rst.txt	39507	0644	editdlrm
xml.etree.elementtree.rst.txt	43871	0644	editdlrm
xml.rst.txt	6071	0644	editdlrm
xml.sax.handler.rst.txt	15399	0644	editdlrm
xml.sax.reader.rst.txt	12133	0644	editdlrm
xml.sax.rst.txt	7159	0644	editdlrm
xml.sax.utils.rst.txt	3901	0644	editdlrm
xmlrpc.client.rst.txt	23071	0644	editdlrm
xmlrpc.rst.txt	475	0644	editdlrm
xmlrpc.server.rst.txt	14990	0644	editdlrm
zipapp.rst.txt	17392	0644	editdlrm
zipfile.rst.txt	24005	0644	editdlrm
zipimport.rst.txt	5805	0644	editdlrm
zlib.rst.txt	13912	0644	editdlrm
_dummy_thread.rst.txt	762	0644	editdlrm
_thread.rst.txt	6886	0644	editdlrm
__future__.rst.txt	5245	0644	editdlrm
__main__.rst.txt	904	0644	editdlrm

:mod:`tokenize` --- Tokenizer for Python source =============================================== .. module:: tokenize :synopsis: Lexical scanner for Python source code. .. moduleauthor:: Ka Ping Yee .. sectionauthor:: Fred L. Drake, Jr. **Source code:** :source:`Lib/tokenize.py` -------------- The :mod:`tokenize` module provides a lexical scanner for Python source code, implemented in Python. The scanner in this module returns comments as tokens as well, making it useful for implementing "pretty-printers," including colorizers for on-screen displays. To simplify token stream handling, all :ref:`operator ` and :ref:`delimiter ` tokens and :data:`Ellipsis` are returned using the generic :data:`~token.OP` token type. The exact type can be determined by checking the ``exact_type`` property on the :term:`named tuple` returned from :func:`tokenize.tokenize`. Tokenizing Input ---------------- The primary entry point is a :term:`generator`: .. function:: tokenize(readline) The :func:`.tokenize` generator requires one argument, *readline*, which must be a callable object which provides the same interface as the :meth:`io.IOBase.readline` method of file objects. Each call to the function should return one line of input as bytes. The generator produces 5-tuples with these members: the token type; the token string; a 2-tuple ``(srow, scol)`` of ints specifying the row and column where the token begins in the source; a 2-tuple ``(erow, ecol)`` of ints specifying the row and column where the token ends in the source; and the line on which the token was found. The line passed (the last tuple item) is the *logical* line; continuation lines are included. The 5 tuple is returned as a :term:`named tuple` with the field names: ``type string start end line``. The returned :term:`named tuple` has an additional property named ``exact_type`` that contains the exact operator type for :data:`token.OP` tokens. For all other token types ``exact_type`` equals the named tuple ``type`` field. .. versionchanged:: 3.1 Added support for named tuples. .. versionchanged:: 3.3 Added support for ``exact_type``. :func:`.tokenize` determines the source encoding of the file by looking for a UTF-8 BOM or encoding cookie, according to :pep:`263`. All constants from the :mod:`token` module are also exported from :mod:`tokenize`, as are three additional token type values: .. data:: COMMENT Token value used to indicate a comment. .. data:: NL Token value used to indicate a non-terminating newline. The NEWLINE token indicates the end of a logical line of Python code; NL tokens are generated when a logical line of code is continued over multiple physical lines. .. data:: ENCODING Token value that indicates the encoding used to decode the source bytes into text. The first token returned by :func:`.tokenize` will always be an ENCODING token. Another function is provided to reverse the tokenization process. This is useful for creating tools that tokenize a script, modify the token stream, and write back the modified script. .. function:: untokenize(iterable) Converts tokens back into Python source code. The *iterable* must return sequences with at least two elements, the token type and the token string. Any additional sequence elements are ignored. The reconstructed script is returned as a single string. The result is guaranteed to tokenize back to match the input so that the conversion is lossless and round-trips are assured. The guarantee applies only to the token type and token string as the spacing between tokens (column positions) may change. It returns bytes, encoded using the ENCODING token, which is the first token sequence output by :func:`.tokenize`. :func:`.tokenize` needs to detect the encoding of source files it tokenizes. The function it uses to do this is available: .. function:: detect_encoding(readline) The :func:`detect_encoding` function is used to detect the encoding that should be used to decode a Python source file. It requires one argument, readline, in the same way as the :func:`.tokenize` generator. It will call readline a maximum of twice, and return the encoding used (as a string) and a list of any lines (not decoded from bytes) it has read in. It detects the encoding from the presence of a UTF-8 BOM or an encoding cookie as specified in :pep:`263`. If both a BOM and a cookie are present, but disagree, a SyntaxError will be raised. Note that if the BOM is found, ``'utf-8-sig'`` will be returned as an encoding. If no encoding is specified, then the default of ``'utf-8'`` will be returned. Use :func:`.open` to open Python source files: it uses :func:`detect_encoding` to detect the file encoding. .. function:: open(filename) Open a file in read only mode using the encoding detected by :func:`detect_encoding`. .. versionadded:: 3.2 .. exception:: TokenError Raised when either a docstring or expression that may be split over several lines is not completed anywhere in the file, for example:: """Beginning of docstring or:: [1, 2, 3 Note that unclosed single-quoted strings do not cause an error to be raised. They are tokenized as ``ERRORTOKEN``, followed by the tokenization of their contents. .. _tokenize-cli: Command-Line Usage ------------------ .. versionadded:: 3.3 The :mod:`tokenize` module can be executed as a script from the command line. It is as simple as: .. code-block:: sh python -m tokenize [-e] [filename.py] The following options are accepted: .. program:: tokenize .. cmdoption:: -h, --help show this help message and exit .. cmdoption:: -e, --exact display token names using the exact type If :file:`filename.py` is specified its contents are tokenized to stdout. Otherwise, tokenization is performed on stdin. Examples ------------------ Example of a script rewriter that transforms float literals into Decimal objects:: from tokenize import tokenize, untokenize, NUMBER, STRING, NAME, OP from io import BytesIO def decistmt(s): """Substitute Decimals for floats in a string of statements. >>> from decimal import Decimal >>> s = 'print(+21.3e-5*-.1234/81.7)' >>> decistmt(s) "print (+Decimal ('21.3e-5')*-Decimal ('.1234')/Decimal ('81.7'))" The format of the exponent is inherited from the platform C library. Known cases are "e-007" (Windows) and "e-07" (not Windows). Since we're only showing 12 digits, and the 13th isn't close to 5, the rest of the output should be platform-independent. >>> exec(s) #doctest: +ELLIPSIS -3.21716034272e-0...7 Output from calculations with Decimal should be identical across all platforms. >>> exec(decistmt(s)) -3.217160342717258261933904529E-7 """ result = [] g = tokenize(BytesIO(s.encode('utf-8')).readline) # tokenize the string for toknum, tokval, _, _, _ in g: if toknum == NUMBER and '.' in tokval: # replace NUMBER tokens result.extend([ (NAME, 'Decimal'), (OP, '('), (STRING, repr(tokval)), (OP, ')') ]) else: result.append((toknum, tokval)) return untokenize(result).decode('utf-8') Example of tokenizing from the command line. The script:: def say_hello(): print("Hello, World!") say_hello() will be tokenized to the following output where the first column is the range of the line/column coordinates where the token is found, the second column is the name of the token, and the final column is the value of the token (if any) .. code-block:: shell-session $ python -m tokenize hello.py 0,0-0,0: ENCODING 'utf-8' 1,0-1,3: NAME 'def' 1,4-1,13: NAME 'say_hello' 1,13-1,14: OP '(' 1,14-1,15: OP ')' 1,15-1,16: OP ':' 1,16-1,17: NEWLINE '\n' 2,0-2,4: INDENT ' ' 2,4-2,9: NAME 'print' 2,9-2,10: OP '(' 2,10-2,25: STRING '"Hello, World!"' 2,25-2,26: OP ')' 2,26-2,27: NEWLINE '\n' 3,0-3,1: NL '\n' 4,0-4,0: DEDENT '' 4,0-4,9: NAME 'say_hello' 4,9-4,10: OP '(' 4,10-4,11: OP ')' 4,11-4,12: NEWLINE '\n' 5,0-5,0: ENDMARKER '' The exact token type names can be displayed using the ``-e`` option: .. code-block:: shell-session $ python -m tokenize -e hello.py 0,0-0,0: ENCODING 'utf-8' 1,0-1,3: NAME 'def' 1,4-1,13: NAME 'say_hello' 1,13-1,14: LPAR '(' 1,14-1,15: RPAR ')' 1,15-1,16: COLON ':' 1,16-1,17: NEWLINE '\n' 2,0-2,4: INDENT ' ' 2,4-2,9: NAME 'print' 2,9-2,10: LPAR '(' 2,10-2,25: STRING '"Hello, World!"' 2,25-2,26: RPAR ')' 2,26-2,27: NEWLINE '\n' 3,0-3,1: NL '\n' 4,0-4,0: DEDENT '' 4,0-4,9: NAME 'say_hello' 4,9-4,10: LPAR '(' 4,10-4,11: RPAR ')' 4,11-4,12: NEWLINE '\n' 5,0-5,0: ENDMARKER ''