Skip to content

Commit c7ad41e

Browse files
youknowoneyouknowone
authored
Merge pull request #4119 from youknowone/compiler
Reorganize compiler crates
2 parents a9cacc7 + a52c5e5 commit c7ad41e

97 files changed

Lines changed: 385 additions & 265 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.gitattributes‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
Lib/** linguist-vendored
22
Cargo.lock linguist-generated -merge
3+
*.snap linguist-generated -merge
34
ast/src/ast_gen.rs linguist-generated -merge
45
vm/src/stdlib/ast/gen.rs linguist-generated -merge
5-
parser/python.lalrpop text eol=LF
6+
compiler/parser/python.lalrpop text eol=LF
67
Lib/*.py text working-tree-encoding=UTF-8 eol=LF
78
**/*.rs text working-tree-encoding=UTF-8 eol=LF

‎.github/workflows/ci.yaml‎

Lines changed: 19 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -112,8 +112,8 @@ jobs:
112112
- name: Cache generated parser
113113
uses: actions/cache@v2
114114
with:
115-
path: parser/python.rs
116-
key: lalrpop-${{ hashFiles('parser/python.lalrpop') }}
115+
path: compiler/parser/python.rs
116+
key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
117117
- uses: dtolnay/rust-toolchain@stable
118118
- name: Set up the Windows environment
119119
shell: bash
@@ -156,8 +156,8 @@ jobs:
156156
- name: Cache generated parser
157157
uses: actions/cache@v2
158158
with:
159-
path: parser/python.rs
160-
key: lalrpop-${{ hashFiles('parser/python.lalrpop') }}
159+
path: compiler/parser/python.rs
160+
key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
161161

162162
- uses: dtolnay/rust-toolchain@stable
163163
with:
@@ -226,8 +226,8 @@ jobs:
226226
- name: Cache generated parser
227227
uses: actions/cache@v2
228228
with:
229-
path: parser/python.rs
230-
key: lalrpop-${{ hashFiles('parser/python.lalrpop') }}
229+
path: compiler/parser/python.rs
230+
key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
231231

232232
- uses: dtolnay/rust-toolchain@stable
233233
- uses: actions/setup-python@v2
@@ -288,17 +288,17 @@ jobs:
288288
- name: Cache generated parser
289289
uses: actions/cache@v2
290290
with:
291-
path: parser/python.rs
292-
key: lalrpop-${{ hashFiles('parser/python.lalrpop') }}
291+
path: compiler/parser/python.rs
292+
key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
293293
- name: Check if cached generated parser exists
294294
id: generated_parser
295295
uses: andstor/file-existence-action@v1
296296
with:
297-
files: "parser/python.rs"
297+
files: "compiler/parser/python.rs"
298298
- if: runner.os == 'Windows'
299299
name: Force python.lalrpop to be lf # actions@checkout ignore .gitattributes
300300
run: |
301-
set file parser/python.lalrpop; ((Get-Content $file) -join "`n") + "`n" | Set-Content -NoNewline $file
301+
set file compiler/parser/python.lalrpop; ((Get-Content $file) -join "`n") + "`n" | Set-Content -NoNewline $file
302302
- name: Install lalrpop
303303
if: steps.generated_parser.outputs.files_exists == 'false'
304304
uses: baptiste0928/cargo-install@v1
@@ -307,7 +307,7 @@ jobs:
307307
version: "0.19.8"
308308
- name: Run lalrpop
309309
if: steps.generated_parser.outputs.files_exists == 'false'
310-
run: lalrpop parser/python.lalrpop
310+
run: lalrpop compiler/parser/python.lalrpop
311311

312312
lint:
313313
name: Check Rust code with rustfmt and clippy
@@ -318,8 +318,8 @@ jobs:
318318
- name: Cache generated parser
319319
uses: actions/cache@v2
320320
with:
321-
path: parser/python.rs
322-
key: lalrpop-${{ hashFiles('parser/python.lalrpop') }}
321+
path: compiler/parser/python.rs
322+
key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
323323
- uses: dtolnay/rust-toolchain@stable
324324
with:
325325
components: rustfmt, clippy
@@ -351,8 +351,8 @@ jobs:
351351
- name: Cache generated parser
352352
uses: actions/cache@v2
353353
with:
354-
path: parser/python.rs
355-
key: lalrpop-${{ hashFiles('parser/python.lalrpop') }}
354+
path: compiler/parser/python.rs
355+
key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
356356
- uses: dtolnay/rust-toolchain@master
357357
with:
358358
toolchain: nightly
@@ -372,8 +372,8 @@ jobs:
372372
- name: Cache generated parser
373373
uses: actions/cache@v2
374374
with:
375-
path: parser/python.rs
376-
key: lalrpop-${{ hashFiles('parser/python.lalrpop') }}
375+
path: compiler/parser/python.rs
376+
key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
377377
- uses: dtolnay/rust-toolchain@stable
378378
- uses: Swatinem/rust-cache@v1
379379
- name: install wasm-pack
@@ -420,8 +420,8 @@ jobs:
420420
- name: Cache generated parser
421421
uses: actions/cache@v2
422422
with:
423-
path: parser/python.rs
424-
key: lalrpop-${{ hashFiles('parser/python.lalrpop') }}
423+
path: compiler/parser/python.rs
424+
key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
425425
- uses: dtolnay/rust-toolchain@stable
426426
with:
427427
target: wasm32-wasi

‎.gitignore‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,4 +20,4 @@ flamescope.json
2020
extra_tests/snippets/resources
2121
extra_tests/not_impl.py
2222

23-
parser/python.rs
23+
compiler/parser/python.rs

‎Cargo.lock‎

Lines changed: 18 additions & 19 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,8 @@ include = ["LICENSE", "Cargo.toml", "src/**/*.rs"]
1414
[workspace]
1515
resolver = "2"
1616
members = [
17-
".", "ast", "bytecode", "common", "compiler", "compiler/porcelain",
18-
"derive", "jit", "parser", "vm", "vm/pylib-crate", "stdlib", "wasm/lib",
17+
"compiler", "compiler/ast", "compiler/bytecode", "compiler/codegen", "compiler/parser",
18+
".", "common", "derive", "jit", "vm", "vm/pylib-crate", "stdlib", "wasm/lib",
1919
]
2020

2121
[features]
@@ -36,10 +36,10 @@ ssl-vendor = ["rustpython-stdlib/ssl-vendor"]
3636
log = "0.4.16"
3737
env_logger = { version = "0.9.0", default-features = false, features = ["atty", "termcolor"] }
3838
clap = "2.34"
39-
rustpython-compiler = { path = "compiler/porcelain", version = "0.1.1" }
40-
rustpython-parser = { path = "parser", version = "0.1.1" }
41-
rustpython-vm = { path = "vm", version = "0.1.1", default-features = false, features = ["compile-parse"] }
42-
rustpython-stdlib = {path = "stdlib", optional = true, default-features = false, features = ["compile-parse"]}
39+
rustpython-compiler = { path = "compiler", version = "0.1.1" }
40+
rustpython-parser = { path = "compiler/parser", version = "0.1.1" }
41+
rustpython-vm = { path = "vm", version = "0.1.1", default-features = false, features = ["compiler"] }
42+
rustpython-stdlib = {path = "stdlib", optional = true, default-features = false}
4343
dirs = { package = "dirs-next", version = "2.0.0" }
4444
num-traits = "0.2.14"
4545
cfg-if = "1.0.0"

‎DEVELOPMENT.md‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -115,10 +115,10 @@ exists a raw html viewer which is currently broken, and we welcome a PR to fix i
115115
Understanding a new codebase takes time. Here's a brief view of the
116116
repository's structure:
117117

118-
- `bytecode/src`: python bytecode representation in rust structures
119118
- `compiler/src`: python compilation to bytecode
119+
- `bytecode/src`: python bytecode representation in rust structures
120+
- `parser/src`: python lexing, parsing and ast
120121
- `derive/src`: Rust language extensions and macros specific to rustpython
121-
- `parser/src`: python lexing, parsing and ast
122122
- `Lib`: Carefully selected / copied files from CPython sourcecode. This is
123123
the python side of the standard library.
124124
- `test`: CPython test suite
@@ -137,7 +137,7 @@ implementation is found in the `src` directory (specifically, `src/lib.rs`).
137137

138138
The top-level `rustpython` binary depends on several lower-level crates including:
139139

140-
- `rustpython-parser` (implementation in `parser/src`)
140+
- `rustpython-parser` (implementation in `compiler/parser/src`)
141141
- `rustpython-compiler` (implementation in `compiler/src`)
142142
- `rustpython-vm` (implementation in `vm/src`)
143143

@@ -155,15 +155,15 @@ enable a line of code to go through a series of steps:
155155
This crate contains the lexer and parser to convert a line of code to
156156
an Abstract Syntax Tree (AST):
157157

158-
- Lexer: `parser/src/lexer.rs` converts Python source code into tokens
159-
- Parser: `parser/src/parser.rs` takes the tokens generated by the lexer and parses
158+
- Lexer: `compiler/parser/src/lexer.rs` converts Python source code into tokens
159+
- Parser: `compiler/parser/src/parser.rs` takes the tokens generated by the lexer and parses
160160
the tokens into an AST (Abstract Syntax Tree) where the nodes of the syntax
161161
tree are Rust structs and enums.
162162
- The Parser relies on `LALRPOP`, a Rust parser generator framework. The
163-
LALRPOP definition of Python's grammar is in `parser/src/python.lalrpop`.
163+
LALRPOP definition of Python's grammar is in `compiler/parser/src/python.lalrpop`.
164164
- More information on parsers and a tutorial can be found in the
165165
[LALRPOP book](https://lalrpop.github.io/lalrpop/).
166-
- AST: `ast/` implements in Rust the Python types and expressions
166+
- AST: `compiler/ast/` implements in Rust the Python types and expressions
167167
represented by the AST nodes.
168168

169169
### rustpython-compiler

‎compiler/Cargo.toml‎

Lines changed: 6 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,12 @@
11
[package]
2-
name = "rustpython-compiler-core"
2+
name = "rustpython-compiler"
33
version = "0.1.2"
4-
description = "Compiler for python code into bytecode for the rustpython VM."
4+
description = "A usability wrapper around rustpython-parser and rustpython-compiler-core"
55
authors = ["RustPython Team"]
6-
repository = "https://github.com/RustPython/RustPython"
7-
license = "MIT"
86
edition = "2021"
97

108
[dependencies]
11-
indexmap = "1.8.1"
12-
itertools = "0.10.3"
13-
rustpython-bytecode = { path = "../bytecode", version = "0.1.1" }
14-
rustpython-ast = { path = "../ast", features = ["unparse"] }
15-
num-complex = { version = "0.4.0", features = ["serde"] }
16-
num-traits = "0.2.14"
17-
log = "0.4.16"
18-
ahash = "0.7.6"
19-
20-
[dev-dependencies]
21-
rustpython-parser = { path = "../parser" }
22-
insta = "1.14.0"
9+
thiserror = "1.0"
10+
rustpython-codegen = { path = "codegen" }
11+
rustpython-parser = { path = "parser" }
12+
rustpython-bytecode = { path = "bytecode" }

ast/Cargo.toml renamed to compiler/ast/Cargo.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,5 +12,5 @@ unparse = ["rustpython-common"]
1212

1313
[dependencies]
1414
num-bigint = "0.4.3"
15-
rustpython-common = { path = "../common", optional = true }
15+
rustpython-common = { path = "../../common", optional = true }
1616
rustpython-bytecode = { path = "../bytecode"}

ast/Python.asdl renamed to compiler/ast/Python.asdl

File renamed without changes.

ast/asdl.py renamed to compiler/ast/asdl.py

File renamed without changes.

0 commit comments

Comments
 (0)