youknowone · Aug 21, 2022 · Aug 21, 2022 · Aug 21, 2022 · Aug 21, 2022 · Aug 21, 2022
diff --git a/.gitattributes b/.gitattributes
@@ -1,7 +1,8 @@
 Lib/**      linguist-vendored
 Cargo.lock  linguist-generated -merge
+*.snap      linguist-generated -merge
 ast/src/ast_gen.rs          linguist-generated -merge
 vm/src/stdlib/ast/gen.rs    linguist-generated -merge
-parser/python.lalrpop       text eol=LF
+compiler/parser/python.lalrpop       text eol=LF
 Lib/*.py    text working-tree-encoding=UTF-8 eol=LF
 **/*.rs     text working-tree-encoding=UTF-8 eol=LF
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -112,8 +112,8 @@ jobs:
       - name: Cache generated parser
         uses: actions/cache@v2
         with:
-          path: parser/python.rs
-          key: lalrpop-${{ hashFiles('parser/python.lalrpop') }}
+          path: compiler/parser/python.rs
+          key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
       - uses: dtolnay/rust-toolchain@stable
       - name: Set up the Windows environment
         shell: bash
@@ -156,8 +156,8 @@ jobs:
       - name: Cache generated parser
         uses: actions/cache@v2
         with:
-          path: parser/python.rs
-          key: lalrpop-${{ hashFiles('parser/python.lalrpop') }}
+          path: compiler/parser/python.rs
+          key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
 
       - uses: dtolnay/rust-toolchain@stable
         with:
@@ -226,8 +226,8 @@ jobs:
       - name: Cache generated parser
         uses: actions/cache@v2
         with:
-          path: parser/python.rs
-          key: lalrpop-${{ hashFiles('parser/python.lalrpop') }}
+          path: compiler/parser/python.rs
+          key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
 
       - uses: dtolnay/rust-toolchain@stable
       - uses: actions/setup-python@v2
@@ -288,17 +288,17 @@ jobs:
       - name: Cache generated parser
         uses: actions/cache@v2
         with:
-          path: parser/python.rs
-          key: lalrpop-${{ hashFiles('parser/python.lalrpop') }}
+          path: compiler/parser/python.rs
+          key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
       - name: Check if cached generated parser exists
         id: generated_parser
         uses: andstor/file-existence-action@v1
         with:
-          files: "parser/python.rs"
+          files: "compiler/parser/python.rs"
       - if: runner.os == 'Windows'
         name: Force python.lalrpop to be lf  # actions@checkout ignore .gitattributes
         run: |
-          set file parser/python.lalrpop; ((Get-Content $file) -join "`n") + "`n" | Set-Content -NoNewline $file
+          set file compiler/parser/python.lalrpop; ((Get-Content $file) -join "`n") + "`n" | Set-Content -NoNewline $file
       - name: Install lalrpop
         if: steps.generated_parser.outputs.files_exists == 'false'
         uses: baptiste0928/cargo-install@v1
@@ -307,7 +307,7 @@ jobs:
           version: "0.19.8"
       - name: Run lalrpop
         if: steps.generated_parser.outputs.files_exists == 'false'
-        run: lalrpop parser/python.lalrpop
+        run: lalrpop compiler/parser/python.lalrpop
 
   lint:
     name: Check Rust code with rustfmt and clippy
@@ -318,8 +318,8 @@ jobs:
       - name: Cache generated parser
         uses: actions/cache@v2
         with:
-          path: parser/python.rs
-          key: lalrpop-${{ hashFiles('parser/python.lalrpop') }}
+          path: compiler/parser/python.rs
+          key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
       - uses: dtolnay/rust-toolchain@stable
         with:
             components: rustfmt, clippy
@@ -351,8 +351,8 @@ jobs:
       - name: Cache generated parser
         uses: actions/cache@v2
         with:
-          path: parser/python.rs
-          key: lalrpop-${{ hashFiles('parser/python.lalrpop') }}
+          path: compiler/parser/python.rs
+          key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
       - uses: dtolnay/rust-toolchain@master
         with:
             toolchain: nightly
@@ -372,8 +372,8 @@ jobs:
       - name: Cache generated parser
         uses: actions/cache@v2
         with:
-          path: parser/python.rs
-          key: lalrpop-${{ hashFiles('parser/python.lalrpop') }}
+          path: compiler/parser/python.rs
+          key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
       - uses: dtolnay/rust-toolchain@stable
       - uses: Swatinem/rust-cache@v1
       - name: install wasm-pack
@@ -420,8 +420,8 @@ jobs:
       - name: Cache generated parser
         uses: actions/cache@v2
         with:
-          path: parser/python.rs
-          key: lalrpop-${{ hashFiles('parser/python.lalrpop') }}
+          path: compiler/parser/python.rs
+          key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
       - uses: dtolnay/rust-toolchain@stable
         with:
           target: wasm32-wasi

diff --git a/.gitignore b/.gitignore
@@ -20,4 +20,4 @@ flamescope.json
 extra_tests/snippets/resources
 extra_tests/not_impl.py
 
-parser/python.rs
+compiler/parser/python.rs
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -14,8 +14,8 @@ include = ["LICENSE", "Cargo.toml", "src/**/*.rs"]
 [workspace]
 resolver = "2"
 members = [
-    ".", "ast", "bytecode", "common", "compiler", "compiler/porcelain",
-    "derive", "jit", "parser", "vm", "vm/pylib-crate", "stdlib", "wasm/lib",
+    "compiler", "compiler/ast", "compiler/bytecode", "compiler/codegen", "compiler/parser",
+    ".", "common", "derive", "jit", "vm", "vm/pylib-crate", "stdlib", "wasm/lib",
 ]
 
 [features]
@@ -36,10 +36,10 @@ ssl-vendor = ["rustpython-stdlib/ssl-vendor"]
 log = "0.4.16"
 env_logger = { version = "0.9.0", default-features = false, features = ["atty", "termcolor"] }
 clap = "2.34"
-rustpython-compiler = { path = "compiler/porcelain", version = "0.1.1" }
-rustpython-parser = { path = "parser", version = "0.1.1" }
-rustpython-vm = { path = "vm", version = "0.1.1", default-features = false, features = ["compile-parse"] }
-rustpython-stdlib = {path = "stdlib", optional = true, default-features = false, features = ["compile-parse"]}
+rustpython-compiler = { path = "compiler", version = "0.1.1" }
+rustpython-parser = { path = "compiler/parser", version = "0.1.1" }
+rustpython-vm = { path = "vm", version = "0.1.1", default-features = false, features = ["compiler"] }
+rustpython-stdlib = {path = "stdlib", optional = true, default-features = false}
 dirs = { package = "dirs-next", version = "2.0.0" }
 num-traits = "0.2.14"
 cfg-if = "1.0.0"

diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md
@@ -115,10 +115,10 @@ exists a raw html viewer which is currently broken, and we welcome a PR to fix i
 Understanding a new codebase takes time. Here's a brief view of the
 repository's structure:
 
-- `bytecode/src`: python bytecode representation in rust structures
 - `compiler/src`: python compilation to bytecode
+  - `bytecode/src`: python bytecode representation in rust structures
+  - `parser/src`: python lexing, parsing and ast
 - `derive/src`: Rust language extensions and macros specific to rustpython
-- `parser/src`: python lexing, parsing and ast
 - `Lib`: Carefully selected / copied files from CPython sourcecode. This is
    the python side of the standard library.
   - `test`: CPython test suite
@@ -137,7 +137,7 @@ implementation is found in the `src` directory (specifically, `src/lib.rs`).
 
 The top-level `rustpython` binary depends on several lower-level crates including:
 
-- `rustpython-parser` (implementation in `parser/src`)
+- `rustpython-parser` (implementation in `compiler/parser/src`)
 - `rustpython-compiler` (implementation in `compiler/src`)
 - `rustpython-vm` (implementation in `vm/src`)
 
@@ -155,15 +155,15 @@ enable a line of code to go through a series of steps:
 This crate contains the lexer and parser to convert a line of code to
 an Abstract Syntax Tree (AST):
 
-- Lexer: `parser/src/lexer.rs` converts Python source code into tokens
-- Parser: `parser/src/parser.rs` takes the tokens generated by the lexer and parses
+- Lexer: `compiler/parser/src/lexer.rs` converts Python source code into tokens
+- Parser: `compiler/parser/src/parser.rs` takes the tokens generated by the lexer and parses
   the tokens into an AST (Abstract Syntax Tree) where the nodes of the syntax
   tree are Rust structs and enums.
   - The Parser relies on `LALRPOP`, a Rust parser generator framework. The
-    LALRPOP definition of Python's grammar is in `parser/src/python.lalrpop`.
+    LALRPOP definition of Python's grammar is in `compiler/parser/src/python.lalrpop`.
   - More information on parsers and a tutorial can be found in the 
     [LALRPOP book](https://lalrpop.github.io/lalrpop/).
-- AST: `ast/` implements in Rust the Python types and expressions
+- AST: `compiler/ast/` implements in Rust the Python types and expressions
   represented by the AST nodes.
 
 ### rustpython-compiler

diff --git a/compiler/Cargo.toml b/compiler/Cargo.toml
@@ -1,22 +1,12 @@
 [package]
-name = "rustpython-compiler-core"
+name = "rustpython-compiler"
 version = "0.1.2"
-description = "Compiler for python code into bytecode for the rustpython VM."
+description = "A usability wrapper around rustpython-parser and rustpython-compiler-core"
 authors = ["RustPython Team"]
-repository = "https://github.com/RustPython/RustPython"
-license = "MIT"
 edition = "2021"
 
 [dependencies]
-indexmap = "1.8.1"
-itertools = "0.10.3"
-rustpython-bytecode = { path = "../bytecode", version = "0.1.1" }
-rustpython-ast = { path = "../ast", features = ["unparse"] }
-num-complex = { version = "0.4.0", features = ["serde"] }
-num-traits = "0.2.14"
-log = "0.4.16"
-ahash = "0.7.6"
-
-[dev-dependencies]
-rustpython-parser = { path = "../parser" }
-insta = "1.14.0"
+thiserror = "1.0"
+rustpython-codegen = { path = "codegen" }
+rustpython-parser = { path = "parser" }
+rustpython-bytecode = { path = "bytecode" }
diff --git a/ast/Cargo.toml → compiler/ast/Cargo.toml b/ast/Cargo.toml → compiler/ast/Cargo.toml
@@ -12,5 +12,5 @@ unparse = ["rustpython-common"]
 
 [dependencies]
 num-bigint = "0.4.3"
-rustpython-common = { path = "../common", optional = true }
+rustpython-common = { path = "../../common", optional = true }
 rustpython-bytecode = { path = "../bytecode"}
diff --git a/ast/Python.asdl → compiler/ast/Python.asdl b/ast/Python.asdl → compiler/ast/Python.asdl
diff --git a/ast/asdl.py → compiler/ast/asdl.py b/ast/asdl.py → compiler/ast/asdl.py
diff --git a/ast/asdl_rs.py → compiler/ast/asdl_rs.py b/ast/asdl_rs.py → compiler/ast/asdl_rs.py
diff --git a/ast/src/ast_gen.rs → compiler/ast/src/ast_gen.rs b/ast/src/ast_gen.rs → compiler/ast/src/ast_gen.rs
diff --git a/ast/src/constant.rs → compiler/ast/src/constant.rs b/ast/src/constant.rs → compiler/ast/src/constant.rs
diff --git a/ast/src/fold_helpers.rs → compiler/ast/src/fold_helpers.rs b/ast/src/fold_helpers.rs → compiler/ast/src/fold_helpers.rs
diff --git a/ast/src/impls.rs → compiler/ast/src/impls.rs b/ast/src/impls.rs → compiler/ast/src/impls.rs
diff --git a/ast/src/lib.rs → compiler/ast/src/lib.rs b/ast/src/lib.rs → compiler/ast/src/lib.rs
diff --git a/ast/src/location.rs → compiler/ast/src/location.rs b/ast/src/location.rs → compiler/ast/src/location.rs
diff --git a/ast/src/unparse.rs → compiler/ast/src/unparse.rs b/ast/src/unparse.rs → compiler/ast/src/unparse.rs
diff --git a/bytecode/Cargo.toml → compiler/bytecode/Cargo.toml b/bytecode/Cargo.toml → compiler/bytecode/Cargo.toml
diff --git a/bytecode/src/lib.rs → compiler/bytecode/src/lib.rs b/bytecode/src/lib.rs → compiler/bytecode/src/lib.rs
diff --git a/compiler/codegen/Cargo.toml b/compiler/codegen/Cargo.toml
@@ -0,0 +1,22 @@
+[package]
+name = "rustpython-codegen"
+version = "0.1.2"
+description = "Compiler for python code into bytecode for the rustpython VM."
+authors = ["RustPython Team"]
+repository = "https://github.com/RustPython/RustPython"
+license = "MIT"
+edition = "2021"
+
+[dependencies]
+indexmap = "1.8.1"
+itertools = "0.10.3"
+rustpython-bytecode = { path = "../bytecode", version = "0.1.1" }
+rustpython-ast = { path = "../ast", features = ["unparse"] }
+num-complex = { version = "0.4.0", features = ["serde"] }
+num-traits = "0.2.14"
+log = "0.4.16"
+ahash = "0.7.6"
+
+[dev-dependencies]
+rustpython-parser = { path = "../parser" }
+insta = "1.14.0"
diff --git a/compiler/src/compile.rs → compiler/codegen/src/compile.rs b/compiler/src/compile.rs → compiler/codegen/src/compile.rs
@@ -5,13 +5,11 @@
 //!   <https://github.com/python/cpython/blob/main/Python/compile.c>
 //!   <https://github.com/micropython/micropython/blob/master/py/compile.c>
 
-use crate::ir::{self, CodeInfo};
-pub use crate::mode::Mode;
-use crate::symboltable::{make_symbol_table, make_symbol_table_expr, SymbolScope, SymbolTable};
-use crate::IndexSet;
 use crate::{
     error::{CompileError, CompileErrorType},
-    symboltable,
+    ir,
+    symboltable::{self, make_symbol_table, make_symbol_table_expr, SymbolScope, SymbolTable},
+    IndexSet,
 };
 use itertools::Itertools;
 use num_complex::Complex64;
@@ -20,6 +18,8 @@ use rustpython_ast as ast;
 use rustpython_bytecode::{self as bytecode, CodeObject, ConstantData, Instruction};
 use std::borrow::Cow;
 
+pub use crate::mode::Mode;
+
 type CompileResult<T> = Result<T, CompileError>;
 
 #[derive(PartialEq, Eq, Clone, Copy)]
@@ -61,7 +61,7 @@ fn is_forbidden_name(name: &str) -> bool {
 
 /// Main structure holding the state of compilation.
 struct Compiler {
-    code_stack: Vec<CodeInfo>,
+    code_stack: Vec<ir::CodeInfo>,
     symbol_table_stack: Vec<SymbolTable>,
     source_path: String,
     current_source_location: ast::Location,
@@ -199,7 +199,7 @@ pub fn compile_expression(
 
 impl Compiler {
     fn new(opts: CompileOpts, source_path: String, code_name: String) -> Self {
-        let module_code = CodeInfo {
+        let module_code = ir::CodeInfo {
             flags: bytecode::CodeFlags::NEW_LOCALS,
             posonlyarg_count: 0,
             arg_count: 0,
@@ -278,7 +278,7 @@ impl Compiler {
 
         self.symbol_table_stack.push(table);
 
-        let info = CodeInfo {
+        let info = ir::CodeInfo {
             flags,
             posonlyarg_count,
             arg_count,
@@ -318,7 +318,7 @@ impl Compiler {
     fn _name_inner(
         &mut self,
         name: &str,
-        cache: impl FnOnce(&mut CodeInfo) -> &mut IndexSet<String>,
+        cache: impl FnOnce(&mut ir::CodeInfo) -> &mut IndexSet<String>,
     ) -> bytecode::NameIdx {
         let name = self.mangle(name);
         let cache = cache(self.current_codeinfo());
@@ -2583,7 +2583,7 @@ impl Compiler {
         self.emit(Instruction::LoadConst { idx })
     }
 
-    fn current_codeinfo(&mut self) -> &mut CodeInfo {
+    fn current_codeinfo(&mut self) -> &mut ir::CodeInfo {
         self.code_stack.last_mut().expect("no code on stack")
     }
 

diff --git a/compiler/src/error.rs → compiler/codegen/src/error.rs b/compiler/src/error.rs → compiler/codegen/src/error.rs
diff --git a/compiler/src/ir.rs → compiler/codegen/src/ir.rs b/compiler/src/ir.rs → compiler/codegen/src/ir.rs