Merge pull request RustPython#2397 from RustPython/coolreader18/fasthash

youknowone · web-flow · commit eb6edabfd3f8 · 2021-01-11T01:37:12.000+09:00
Use ahash instead of siphash in a few places
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/bytecode/Cargo.toml b/bytecode/Cargo.toml
@@ -11,7 +11,7 @@ license = "MIT"
 [dependencies]
 bincode =  "1.1"
 bitflags = "1.1"
-lz4_flex = "0.5"
+lz4_flex = "0.7"
 num-bigint = { version = "0.3", features = ["serde"] }
 num-complex = { version = "0.3", features = ["serde"] }
 serde = { version = "1.0", features = ["derive"] }
diff --git a/compiler/Cargo.toml b/compiler/Cargo.toml
@@ -15,7 +15,7 @@ rustpython-ast = { path = "../ast" }
 num-complex = { version = "0.3", features = ["serde"] }
 num-traits = "0.2"
 log = "0.4"
-arrayvec = "0.5"
+ahash = "0.6"
 
 [dev-dependencies]
 rustpython-parser = { path = "../parser" }
diff --git a/compiler/src/compile.rs b/compiler/src/compile.rs
@@ -9,7 +9,7 @@ use crate::error::{CompileError, CompileErrorType};
 use crate::ir::{self, CodeInfo};
 pub use crate::mode::Mode;
 use crate::symboltable::{make_symbol_table, statements_to_symbol_table, SymbolScope, SymbolTable};
-use indexmap::IndexSet;
+use crate::IndexSet;
 use itertools::Itertools;
 use num_complex::Complex64;
 use num_traits::ToPrimitive;
@@ -140,10 +140,10 @@ impl Compiler {
             blocks: vec![ir::Block::default()],
             current_block: bytecode::Label(0),
             constants: Vec::new(),
-            name_cache: IndexSet::new(),
-            varname_cache: IndexSet::new(),
-            cellvar_cache: IndexSet::new(),
-            freevar_cache: IndexSet::new(),
+            name_cache: IndexSet::default(),
+            varname_cache: IndexSet::default(),
+            cellvar_cache: IndexSet::default(),
+            freevar_cache: IndexSet::default(),
         };
         Compiler {
             code_stack: vec![module_code],
@@ -217,8 +217,8 @@ impl Compiler {
             blocks: vec![ir::Block::default()],
             current_block: bytecode::Label(0),
             constants: Vec::new(),
-            name_cache: IndexSet::new(),
-            varname_cache: IndexSet::new(),
+            name_cache: IndexSet::default(),
+            varname_cache: IndexSet::default(),
             cellvar_cache,
             freevar_cache,
         };
diff --git a/compiler/src/ir.rs b/compiler/src/ir.rs
@@ -1,4 +1,4 @@
-use indexmap::IndexSet;
+use crate::IndexSet;
 use rustpython_bytecode::{CodeFlags, CodeObject, ConstantData, Instruction, Label, Location};
 
 pub type BlockIdx = Label;
diff --git a/compiler/src/lib.rs b/compiler/src/lib.rs
@@ -5,6 +5,9 @@
 #[macro_use]
 extern crate log;
 
+type IndexMap<K, V> = indexmap::IndexMap<K, V, ahash::RandomState>;
+type IndexSet<T> = indexmap::IndexSet<T, ahash::RandomState>;
+
 pub mod compile;
 pub mod error;
 pub mod ir;
diff --git a/compiler/src/symboltable.rs b/compiler/src/symboltable.rs
@@ -8,7 +8,7 @@ Inspirational file: https://github.com/python/cpython/blob/master/Python/symtabl
 */
 
 use crate::error::{CompileError, CompileErrorType};
-use indexmap::map::IndexMap;
+use crate::IndexMap;
 use rustpython_ast::{self as ast, Location};
 use std::fmt;
 
@@ -58,7 +58,7 @@ impl SymbolTable {
             typ,
             line_number,
             is_nested,
-            symbols: IndexMap::new(),
+            symbols: IndexMap::default(),
             sub_tables: vec![],
         }
     }
diff --git a/parser/Cargo.toml b/parser/Cargo.toml
@@ -20,3 +20,6 @@ num-traits = "0.2"
 unic-emoji-char = "0.9"
 unic-ucd-ident  = "0.9"
 unicode_names2 = "0.4"
+phf = { version = "0.8", features = ["macros"] }
+hashbrown = "0.9"
+ahash = "0.6"
diff --git a/parser/src/function.rs b/parser/src/function.rs
@@ -1,3 +1,4 @@
+use ahash::RandomState;
 use std::collections::HashSet;
 
 use crate::ast;
@@ -45,7 +46,7 @@ pub fn parse_args(func_args: Vec<FunctionArgument>) -> Result<ast::ArgumentList,
     let mut args = vec![];
     let mut keywords = vec![];
 
-    let mut keyword_names = HashSet::with_capacity(func_args.len());
+    let mut keyword_names = HashSet::with_capacity_and_hasher(func_args.len(), RandomState::new());
     for (name, value) in func_args {
         match name {
             Some(n) => {
diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs
@@ -10,7 +10,6 @@ use num_traits::identities::Zero;
 use num_traits::Num;
 use std::char;
 use std::cmp::Ordering;
-use std::collections::HashMap;
 use std::str::FromStr;
 use unic_emoji_char::is_emoji_presentation;
 use unic_ucd_ident::{is_xid_continue, is_xid_start};
@@ -66,52 +65,48 @@ pub struct Lexer<T: Iterator<Item = char>> {
     chr1: Option<char>,
     chr2: Option<char>,
     location: Location,
-    keywords: HashMap<String, Tok>,
 }
 
-pub fn get_keywords() -> HashMap<String, Tok> {
-    let mut keywords: HashMap<String, Tok> = HashMap::new();
-
+pub static KEYWORDS: phf::Map<&'static str, Tok> = phf::phf_map! {
     // Alphabetical keywords:
-    keywords.insert(String::from("..."), Tok::Ellipsis);
-    keywords.insert(String::from("False"), Tok::False);
-    keywords.insert(String::from("None"), Tok::None);
-    keywords.insert(String::from("True"), Tok::True);
-
-    keywords.insert(String::from("and"), Tok::And);
-    keywords.insert(String::from("as"), Tok::As);
-    keywords.insert(String::from("assert"), Tok::Assert);
-    keywords.insert(String::from("async"), Tok::Async);
-    keywords.insert(String::from("await"), Tok::Await);
-    keywords.insert(String::from("break"), Tok::Break);
-    keywords.insert(String::from("class"), Tok::Class);
-    keywords.insert(String::from("continue"), Tok::Continue);
-    keywords.insert(String::from("def"), Tok::Def);
-    keywords.insert(String::from("del"), Tok::Del);
-    keywords.insert(String::from("elif"), Tok::Elif);
-    keywords.insert(String::from("else"), Tok::Else);
-    keywords.insert(String::from("except"), Tok::Except);
-    keywords.insert(String::from("finally"), Tok::Finally);
-    keywords.insert(String::from("for"), Tok::For);
-    keywords.insert(String::from("from"), Tok::From);
-    keywords.insert(String::from("global"), Tok::Global);
-    keywords.insert(String::from("if"), Tok::If);
-    keywords.insert(String::from("import"), Tok::Import);
-    keywords.insert(String::from("in"), Tok::In);
-    keywords.insert(String::from("is"), Tok::Is);
-    keywords.insert(String::from("lambda"), Tok::Lambda);
-    keywords.insert(String::from("nonlocal"), Tok::Nonlocal);
-    keywords.insert(String::from("not"), Tok::Not);
-    keywords.insert(String::from("or"), Tok::Or);
-    keywords.insert(String::from("pass"), Tok::Pass);
-    keywords.insert(String::from("raise"), Tok::Raise);
-    keywords.insert(String::from("return"), Tok::Return);
-    keywords.insert(String::from("try"), Tok::Try);
-    keywords.insert(String::from("while"), Tok::While);
-    keywords.insert(String::from("with"), Tok::With);
-    keywords.insert(String::from("yield"), Tok::Yield);
-    keywords
-}
+    "..." => Tok::Ellipsis,
+    "False" => Tok::False,
+    "None" => Tok::None,
+    "True" => Tok::True,
+
+    "and" => Tok::And,
+    "as" => Tok::As,
+    "assert" => Tok::Assert,
+    "async" => Tok::Async,
+    "await" => Tok::Await,
+    "break" => Tok::Break,
+    "class" => Tok::Class,
+    "continue" => Tok::Continue,
+    "def" => Tok::Def,
+    "del" => Tok::Del,
+    "elif" => Tok::Elif,
+    "else" => Tok::Else,
+    "except" => Tok::Except,
+    "finally" => Tok::Finally,
+    "for" => Tok::For,
+    "from" => Tok::From,
+    "global" => Tok::Global,
+    "if" => Tok::If,
+    "import" => Tok::Import,
+    "in" => Tok::In,
+    "is" => Tok::Is,
+    "lambda" => Tok::Lambda,
+    "nonlocal" => Tok::Nonlocal,
+    "not" => Tok::Not,
+    "or" => Tok::Or,
+    "pass" => Tok::Pass,
+    "raise" => Tok::Raise,
+    "return" => Tok::Return,
+    "try" => Tok::Try,
+    "while" => Tok::While,
+    "with" => Tok::With,
+    "yield" => Tok::Yield,
+};
 
 pub type Spanned = (Location, Tok, Location);
 pub type LexResult = Result<Spanned, LexicalError>;
@@ -193,7 +188,6 @@ where
             location: Location::new(0, 0),
             chr1: None,
             chr2: None,
-            keywords: get_keywords(),
         };
         lxr.next_char();
         lxr.next_char();
@@ -245,8 +239,8 @@ where
         }
         let end_pos = self.get_pos();
 
-        if self.keywords.contains_key(&name) {
-            Ok((start_pos, self.keywords[&name].clone(), end_pos))
+        if let Some(tok) = KEYWORDS.get(name.as_str()) {
+            Ok((start_pos, tok.clone(), end_pos))
         } else {
             Ok((start_pos, Tok::Name { name }, end_pos))
         }
diff --git a/vm/Cargo.toml b/vm/Cargo.toml
@@ -57,8 +57,8 @@ itertools = "0.9"
 hex = "0.4.0"
 hexf-parse = "0.1.0"
 indexmap = "1.0.2"
+ahash = "0.6"
 crc = "^1.0.0"
-maplit = "1.0"
 bitflags = "1.2.1"
 libc = "0.2"
 nix = "0.18"
diff --git a/vm/src/builtins/dict.rs b/vm/src/builtins/dict.rs
@@ -450,7 +450,7 @@ impl PyDictRef {
 
     /// Take a python dictionary and convert it to attributes.
     pub fn to_attributes(self) -> PyAttributes {
-        let mut attrs = PyAttributes::new();
+        let mut attrs = PyAttributes::default();
         for (key, value) in self {
             let key = pystr::clone_value(&key);
             attrs.insert(key, value);
diff --git a/vm/src/builtins/int.rs b/vm/src/builtins/int.rs
@@ -908,7 +908,7 @@ pub(crate) fn try_int(obj: &PyObjectRef, vm: &VirtualMachine) -> PyResult<BigInt
                 vm.to_repr(obj)?,
             ))),
         }
-    };
+    }
 
     // test for strings and bytes
     if let Some(s) = obj.downcast_ref::<PyStr>() {
diff --git a/vm/src/builtins/pytype.rs b/vm/src/builtins/pytype.rs
@@ -2,7 +2,7 @@ use crate::common::lock::{
     PyMappedRwLockReadGuard, PyRwLock, PyRwLockReadGuard, PyRwLockUpgradableReadGuard,
     PyRwLockWriteGuard,
 };
-use std::collections::{HashMap, HashSet};
+use std::collections::HashSet;
 use std::fmt;
 
 use super::classmethod::PyClassMethod;
@@ -133,7 +133,7 @@ impl PyType {
 
     pub fn get_attributes(&self) -> PyAttributes {
         // Gather all members here:
-        let mut attributes = PyAttributes::new();
+        let mut attributes = PyAttributes::default();
 
         for bc in self.iter_mro().rev() {
             for (name, value) in bc.attributes.read().iter() {
@@ -791,7 +791,7 @@ pub fn new(
     name: &str,
     base: PyTypeRef,
     bases: Vec<PyTypeRef>,
-    attrs: HashMap<String, PyObjectRef>,
+    attrs: PyAttributes,
     mut slots: PyTypeSlots,
 ) -> Result<PyTypeRef, String> {
     // Check for duplicates in bases.
@@ -914,8 +914,7 @@ fn best_base(bases: &[PyTypeRef], vm: &VirtualMachine) -> PyResult<PyTypeRef> {
 
 #[cfg(test)]
 mod tests {
-    use super::{linearise_mro, new};
-    use super::{HashMap, IdProtocol, PyContext, PyTypeRef};
+    use super::*;
 
     fn map_ids(obj: Result<Vec<PyTypeRef>, String>) -> Result<Vec<usize>, String> {
         Ok(obj?.into_iter().map(|x| x.get_id()).collect())
@@ -932,7 +931,7 @@ mod tests {
             "A",
             object.clone(),
             vec![object.clone()],
-            HashMap::new(),
+            PyAttributes::default(),
             Default::default(),
         )
         .unwrap();
@@ -941,7 +940,7 @@ mod tests {
             "B",
             object.clone(),
             vec![object.clone()],
-            HashMap::new(),
+            PyAttributes::default(),
             Default::default(),
         )
         .unwrap();
diff --git a/vm/src/frozen.rs b/vm/src/frozen.rs
@@ -19,8 +19,10 @@ pub fn map_frozen<'a>(
         })
 }
 
-pub fn get_module_inits(vm: &VirtualMachine) -> HashMap<String, code::FrozenModule> {
-    let mut modules = HashMap::new();
+pub fn get_module_inits(
+    vm: &VirtualMachine,
+) -> HashMap<String, code::FrozenModule, ahash::RandomState> {
+    let mut modules = HashMap::default();
 
     macro_rules! ext_modules {
         ($($t:tt)*) => {
diff --git a/vm/src/function.rs b/vm/src/function.rs
@@ -10,7 +10,6 @@ use crate::vm::VirtualMachine;
 use indexmap::IndexMap;
 use itertools::Itertools;
 use result_like::impl_option_like;
-use std::collections::HashMap;
 use std::marker::PhantomData;
 use std::ops::RangeInclusive;
 
@@ -325,9 +324,9 @@ impl<T> KwArgs<T> {
         self.0.remove(name)
     }
 }
-impl<T> From<HashMap<String, T>> for KwArgs<T> {
-    fn from(kwargs: HashMap<String, T>) -> Self {
-        KwArgs(kwargs.into_iter().collect())
+impl<T> std::iter::FromIterator<(String, T)> for KwArgs<T> {
+    fn from_iter<I: IntoIterator<Item = (String, T)>>(iter: I) -> Self {
+        KwArgs(iter.into_iter().collect())
     }
 }
 impl<T> Default for KwArgs<T> {
diff --git a/vm/src/lib.rs b/vm/src/lib.rs
@@ -21,8 +21,6 @@ extern crate flamer;
 extern crate bitflags;
 #[macro_use]
 extern crate log;
-#[macro_use]
-extern crate maplit;
 // extern crate env_logger;
 
 #[macro_use]
diff --git a/vm/src/macros.rs b/vm/src/macros.rs
@@ -256,3 +256,21 @@ cfg_if::cfg_if! {
         }
     }
 }
+
+/// A modified version of the hashmap! macro from the maplit crate
+macro_rules! hashmap {
+    (@single $($x:tt)*) => (());
+    (@count $($rest:expr),*) => (<[()]>::len(&[$(hashmap!(@single $rest)),*]));
+
+    (hasher=$hasher:expr, $($key:expr => $value:expr,)+) => { hashmap!(hasher=$hasher, $($key => $value),+) };
+    (hasher=$hasher:expr, $($key:expr => $value:expr),*) => {
+        {
+            let _cap = hashmap!(@count $($key),*);
+            let mut _map = ::std::collections::HashMap::with_capacity_and_hasher(_cap, $hasher);
+            $(
+                let _ = _map.insert($key, $value);
+            )*
+            _map
+        }
+    };
+}
diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs
@@ -64,7 +64,7 @@ pub type PyResult<T = PyObjectRef> = Result<T, PyBaseExceptionRef>; // A valid v
 /// For attributes we do not use a dict, but a hashmap. This is probably
 /// faster, unordered, and only supports strings as keys.
 /// TODO: class attributes should maintain insertion order (use IndexMap here)
-pub type PyAttributes = HashMap<String, PyObjectRef>;
+pub type PyAttributes = HashMap<String, PyObjectRef, ahash::RandomState>;
 
 // TODO: remove this impl
 impl fmt::Display for PyObjectRef {
diff --git a/vm/src/pyobjectrc.rs b/vm/src/pyobjectrc.rs
diff --git a/vm/src/stdlib/keyword.rs b/vm/src/stdlib/keyword.rs
diff --git a/vm/src/stdlib/mod.rs b/vm/src/stdlib/mod.rs
diff --git a/vm/src/stdlib/thread.rs b/vm/src/stdlib/thread.rs
diff --git a/vm/src/types.rs b/vm/src/types.rs
diff --git a/vm/src/vm.rs b/vm/src/vm.rs

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-use indexmap::IndexSet;`
	`1`	`+use crate::IndexSet;`
`2`	`2`	`use rustpython_bytecode::{CodeFlags, CodeObject, ConstantData, Instruction, Label, Location};`
`3`	`3`
`4`	`4`	`pub type BlockIdx = Label;`