forked from argotorg/fe
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathmod.rs.html
More file actions
106 lines (104 loc) · 7.45 KB
/
Copy pathmod.rs.html
File metadata and controls
106 lines (104 loc) · 7.45 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Source of the Rust file `parser/src/tokenizer/mod.rs`."><meta name="keywords" content="rust, rustlang, rust-lang"><title>mod.rs - source</title><link rel="stylesheet" type="text/css" href="../../../normalize.css"><link rel="stylesheet" type="text/css" href="../../../rustdoc.css" id="mainThemeStyle"><link rel="stylesheet" type="text/css" href="../../../light.css" id="themeStyle"><link rel="stylesheet" type="text/css" href="../../../dark.css" disabled ><link rel="stylesheet" type="text/css" href="../../../ayu.css" disabled ><script src="../../../storage.js"></script><noscript><link rel="stylesheet" href="../../../noscript.css"></noscript><link rel="icon" type="image/svg+xml" href="../../../favicon.svg">
<link rel="alternate icon" type="image/png" href="../../../favicon-16x16.png">
<link rel="alternate icon" type="image/png" href="../../../favicon-32x32.png"><style type="text/css">#crate-search{background-image:url("../../../down-arrow.svg");}</style></head><body class="rustdoc source"><!--[if lte IE 8]><div class="warning">This old browser is unsupported and will most likely display funky things.</div><![endif]--><nav class="sidebar"><div class="sidebar-menu">☰</div><a href='../../../fe_parser/index.html'><div class='logo-container rust-logo'><img src='../../../rust-logo.png' alt='logo'></div></a></nav><div class="theme-picker"><button id="theme-picker" aria-label="Pick another theme!"><img src="../../../brush.svg" width="18" alt="Pick another theme!"></button><div id="theme-choices"></div></div><script src="../../../theme.js"></script><nav class="sub"><form class="search-form"><div class="search-container"><div><select id="crate-search"><option value="All crates">All crates</option></select><input class="search-input" name="search" disabled autocomplete="off" spellcheck="false" placeholder="Click or press ‘S’ to search, ‘?’ for more options…" type="search"></div><span class="help-button">?</span>
<a id="settings-menu" href="../../../settings.html"><img src="../../../wheel.svg" width="18" alt="Change settings"></a></div></form></nav><section id="main" class="content"><pre class="line-numbers"><span id="1"> 1</span>
<span id="2"> 2</span>
<span id="3"> 3</span>
<span id="4"> 4</span>
<span id="5"> 5</span>
<span id="6"> 6</span>
<span id="7"> 7</span>
<span id="8"> 8</span>
<span id="9"> 9</span>
<span id="10">10</span>
<span id="11">11</span>
<span id="12">12</span>
<span id="13">13</span>
<span id="14">14</span>
<span id="15">15</span>
<span id="16">16</span>
<span id="17">17</span>
<span id="18">18</span>
<span id="19">19</span>
<span id="20">20</span>
<span id="21">21</span>
<span id="22">22</span>
<span id="23">23</span>
<span id="24">24</span>
<span id="25">25</span>
<span id="26">26</span>
<span id="27">27</span>
<span id="28">28</span>
<span id="29">29</span>
<span id="30">30</span>
<span id="31">31</span>
<span id="32">32</span>
<span id="33">33</span>
<span id="34">34</span>
<span id="35">35</span>
<span id="36">36</span>
<span id="37">37</span>
<span id="38">38</span>
<span id="39">39</span>
<span id="40">40</span>
<span id="41">41</span>
<span id="42">42</span>
<span id="43">43</span>
<span id="44">44</span>
<span id="45">45</span>
<span id="46">46</span>
<span id="47">47</span>
<span id="48">48</span>
<span id="49">49</span>
<span id="50">50</span>
</pre><div class="example-wrap"><pre class="rust ">
<span class="doccomment">//! A Rust port of [Python's std lib `tokenize`</span>
<span class="doccomment">//! module](https://github.com/python/cpython/blob/2a58b0636d1f620f8a85a2e4c030cc10551936a5/Lib/tokenize.py).</span>
<span class="doccomment">//!</span>
<span class="doccomment">//! ## Usage</span>
<span class="doccomment">//!</span>
<span class="doccomment">//! ```rust</span>
<span class="doccomment">//! use fe_parser::tokenizer::tokenize;</span>
<span class="doccomment">//!</span>
<span class="doccomment">//! let source_string = r#"</span>
<span class="doccomment">//! class Foo:</span>
<span class="doccomment">//! bar = "baz"</span>
<span class="doccomment">//! "#;</span>
<span class="doccomment">//! let token_vector = tokenize(source_string).unwrap();</span>
<span class="doccomment">//! ```</span>
<span class="doccomment">//!</span>
<span class="doccomment">//! ## Differences/similarities with python's `tokenize` module</span>
<span class="doccomment">//!</span>
<span class="doccomment">//! * The [`self::tokenize::tokenize`] function generates all token types</span>
<span class="doccomment">//! produced by python's `tokenize.tokenize` method *except* for an initial</span>
<span class="doccomment">//! `ENCODING` token. Source files are always assumed to be encoded in utf-8</span>
<span class="doccomment">//! so this token is redundant.</span>
<span class="doccomment">//! * The [`self::types::Token`] struct, which we use to represent parsed</span>
<span class="doccomment">//! tokens, includes a single `span` field containing the byte offsets</span>
<span class="doccomment">//! representing the beginning and exclusive end of the token in a source</span>
<span class="doccomment">//! string. This differs from Python's `TokenInfo` instances which use</span>
<span class="doccomment">//! line/column tuples to represent the beginning and ending positions of a</span>
<span class="doccomment">//! token.</span>
<span class="doccomment">//! * The [`self::tokenize::tokenize`] function has no streaming behavior and</span>
<span class="doccomment">//! accepts a reference to an entire source string available in memory.</span>
<span class="doccomment">//! Python's `tokenize.tokenize` function accepts a reference to a function</span>
<span class="doccomment">//! that progressively yields lines of text from a source file.</span>
<span class="doccomment">//!</span>
<span class="doccomment">//! As Python's `tokenize` module's implementation is pretty ugly, so is the</span>
<span class="doccomment">//! implementation of [`self::tokenize::tokenize`]. It may be a candidate for</span>
<span class="doccomment">//! later cleanup. However, it may also be fine as is assuming it doesn't need</span>
<span class="doccomment">//! to be modified often.</span>
<span class="kw">mod</span> <span class="ident">regex</span>;
<span class="kw">pub</span> <span class="kw">mod</span> <span class="ident">tokenize</span>;
<span class="kw">pub</span> <span class="kw">mod</span> <span class="ident">types</span>;
<span class="kw">pub</span> <span class="kw">mod</span> <span class="ident">wasm</span>;
<span class="kw">pub</span> <span class="kw">use</span> <span class="self">self</span>::<span class="ident">tokenize</span>::{
<span class="ident">tokenize</span>,
<span class="ident">TokenizeError</span>,
};
<span class="kw">pub</span> <span class="kw">use</span> <span class="self">self</span>::<span class="ident">types</span>::{
<span class="ident">Token</span>,
<span class="ident">TokenType</span>,
};
</pre></div>
</section><section id="search" class="content hidden"></section><section class="footer"></section><script>window.rootPath = "../../../";window.currentCrate = "fe_parser";</script><script src="../../../main.js"></script><script src="../../../source-script.js"></script><script src="../../../source-files.js"></script><script defer src="../../../search-index.js"></script></body></html>