Skip to content

Commit 6a8a4a2

Browse files
authored
feat: Unified CLI bin cache with env var override support (#863)
1 parent a877580 commit 6a8a4a2

File tree

25 files changed

+365
-675
lines changed

25 files changed

+365
-675
lines changed

Cargo.lock

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

package.json

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,8 @@
2323
},
2424
"scripts": {
2525
"build": "cd wasm-bindgen && cargo build -p wasm-bindgen-cli --bin wasm-bindgen && cd .. && cargo build -p worker-build",
26-
"test": "cd test && NO_MINIFY=1 WASM_BINDGEN_PATH=../wasm-bindgen/target/debug/wasm-bindgen ../target/debug/worker-build --dev && NODE_OPTIONS='--experimental-vm-modules' npx vitest run",
27-
"test-http": "cd test && NO_MINIFY=1 WASM_BINDGEN_PATH=../wasm-bindgen/target/debug/wasm-bindgen ../target/debug/worker-build --release --features http && NODE_OPTIONS='--experimental-vm-modules' npx vitest run",
26+
"test": "cd test && NO_MINIFY=1 WASM_BINDGEN_BIN=../wasm-bindgen/target/debug/wasm-bindgen ../target/debug/worker-build --dev && NODE_OPTIONS='--experimental-vm-modules' npx vitest run",
27+
"test-http": "cd test && NO_MINIFY=1 WASM_BINDGEN_BIN=../wasm-bindgen/target/debug/wasm-bindgen ../target/debug/worker-build --release --features http && NODE_OPTIONS='--experimental-vm-modules' npx vitest run",
2828
"test-mem": "cd test && npx wrangler dev --enable-containers=false",
2929
"lint": "cargo clippy --features d1,queue --all-targets --workspace -- -D warnings",
3030
"lint:fix": "cargo fmt && cargo clippy --features d1,queue --all-targets --workspace --fix -- -D warnings"

test/wrangler.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ bucket_name = 'delete-bucket'
6969
preview_bucket_name = 'delete-bucket'
7070

7171
[build]
72-
command = "WASM_BINDGEN_PATH=../wasm-bindgen/target/debug/wasm-bindgen ../target/debug/worker-build --release"
72+
command = "WASM_BINDGEN_BIN=../wasm-bindgen/target/debug/wasm-bindgen ../target/debug/worker-build --release"
7373

7474
[[migrations]]
7575
tag = "v1"

worker-build/Cargo.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ console = "0.16.0"
1717
dirs-next = "2.0"
1818
flate2 = "1.1"
1919
glob = "0.3.1"
20+
heck = "0.5.0"
2021
log = "0.4.17"
2122
parking_lot = "0.12.1"
2223
path-clean = "1.0.1"

worker-build/README.md

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,4 +17,20 @@ main = "./shim.mjs"
1717
[[build.upload.rules]]
1818
globs = ["**/*.wasm"]
1919
type = "CompiledWasm"
20+
```
21+
22+
## Environment Variables
23+
24+
You can override the default binary lookup/download behavior by setting these environment variables:
25+
26+
- **`WASM_BINDGEN_BIN`**: Path to a custom `wasm-bindgen` binary. When set, worker-build will use this binary instead of downloading or looking for a globally installed version.
27+
28+
- **`WASM_OPT_BIN`**: Path to a custom `wasm-opt` binary. When set, worker-build will use this binary instead of downloading one.
29+
30+
### Example
31+
32+
```bash
33+
export WASM_BINDGEN_BIN=/path/to/custom/wasm-bindgen
34+
export WASM_OPT_BIN=/path/to/custom/wasm-opt
35+
worker-build --release
2036
```

worker-build/src/binary.rs

Lines changed: 291 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,291 @@
1+
use crate::emoji::{CONFIG, DOWN_ARROW};
2+
use crate::wasm_pack::PBAR;
3+
use anyhow::{bail, Context, Result};
4+
use flate2::read::GzDecoder;
5+
use heck::ToShoutySnakeCase;
6+
use std::env;
7+
use std::{
8+
fs::{create_dir_all, read_dir, OpenOptions},
9+
path::{Path, PathBuf},
10+
};
11+
12+
pub trait GetBinary: BinaryDep {
13+
/// Get the given binary path for a binary dependency
14+
fn get_binary(&self, bin_name: Option<&str>) -> Result<PathBuf> {
15+
let full_name = self.full_name();
16+
let name = self.name();
17+
let target = self.target();
18+
let version = self.version();
19+
20+
// 1. First check {BIN_NAME}_BIN env override
21+
let check_env_var = bin_name.unwrap_or(name).to_shouty_snake_case() + "_BIN";
22+
if let Ok(custom_bin) = env::var(&check_env_var) {
23+
match which::which(&custom_bin) {
24+
Ok(resolved) => {
25+
PBAR.info(&format!(
26+
"{CONFIG}Using custom {full_name} from {check_env_var}: {}",
27+
resolved.display()
28+
));
29+
return Ok(resolved);
30+
}
31+
Err(_) => {
32+
PBAR.warn(&format!("{check_env_var}={custom_bin} not found, falling back to internal {full_name} implementation"));
33+
}
34+
}
35+
}
36+
37+
// 2. Then check the cache path
38+
let cache_path = cache_path(name, version, target)?;
39+
let bin_path = cache_path.join(self.bin_path(bin_name)?);
40+
if bin_path.exists() {
41+
return Ok(bin_path);
42+
}
43+
44+
// 3. Finally perform a download, clearing cache for this name and target first
45+
let url = self.download_url();
46+
let _ = remove_all_versions(name, target);
47+
PBAR.info(&format!("{DOWN_ARROW}Downloading {full_name}@{version}..."));
48+
download(&url, &cache_path)?;
49+
if !bin_path.exists() {
50+
bail!(
51+
"Unable to locate binary {} in {full_name}",
52+
bin_path.to_string_lossy()
53+
);
54+
}
55+
Ok(bin_path)
56+
}
57+
}
58+
59+
pub trait BinaryDep: Sized {
60+
/// Returns the name of the binary
61+
fn name(&self) -> &'static str;
62+
63+
/// Returns the full name of the binary
64+
fn full_name(&self) -> &'static str;
65+
66+
/// Returns the target of the binary
67+
fn target(&self) -> &'static str;
68+
69+
/// Returns the latest current version of the binary
70+
fn version(&self) -> &'static str;
71+
72+
/// Returns the URL for the binary to be downloaded
73+
/// as well as the path string within the archive to use
74+
fn download_url(&self) -> String;
75+
76+
/// Get the relative path of the given binary in the package
77+
/// If None, returns the default binary
78+
fn bin_path(&self, name: Option<&str>) -> Result<String>;
79+
}
80+
81+
impl<T: BinaryDep> GetBinary for T {}
82+
83+
const MAYBE_EXE: &str = if cfg!(windows) { ".exe" } else { "" };
84+
85+
/// For clearing the cache, remove all files for the given binary and target
86+
fn remove_all_versions(name: &str, target: &str) -> Result<usize> {
87+
let prefix_name = format!("{name}-{target}-");
88+
let dir = dirs_next::cache_dir()
89+
.unwrap_or_else(std::env::temp_dir)
90+
.join("worker-build");
91+
92+
let mut deleted_count = 0;
93+
for entry in read_dir(dir)? {
94+
let entry = entry?;
95+
let file_name = entry.file_name();
96+
97+
if let Some(name_str) = file_name.to_str() {
98+
if name_str.starts_with(&prefix_name) {
99+
let path = entry.path();
100+
if path.is_dir() {
101+
std::fs::remove_dir_all(&path)?;
102+
} else {
103+
std::fs::remove_file(&path)?;
104+
}
105+
deleted_count += 1;
106+
}
107+
}
108+
}
109+
110+
Ok(deleted_count)
111+
}
112+
113+
/// Cache path for this binary instance
114+
fn cache_path(name: &str, version: &str, target: &str) -> Result<PathBuf> {
115+
let path_name = format!("{name}-{target}-{version}{MAYBE_EXE}");
116+
let path = dirs_next::cache_dir()
117+
.unwrap_or_else(std::env::temp_dir)
118+
.join("worker-build")
119+
.join(&path_name);
120+
if !path.exists() {
121+
create_dir_all(&path)?;
122+
}
123+
Ok(path)
124+
}
125+
126+
#[cfg(target_family = "unix")]
127+
fn fix_permissions(options: &mut OpenOptions) -> &mut OpenOptions {
128+
use std::os::unix::fs::OpenOptionsExt;
129+
options.mode(0o755)
130+
}
131+
132+
#[cfg(target_family = "windows")]
133+
fn fix_permissions(options: &mut OpenOptions) -> &mut OpenOptions {
134+
options
135+
}
136+
137+
/// Download this binary instance into its cache path
138+
fn download(url: &str, bin_dir: &Path) -> Result<()> {
139+
let mut res = ureq::get(url)
140+
.call()
141+
.with_context(|| format!("Failed to fetch URL {url}"))?;
142+
let body = res.body_mut().as_reader();
143+
let deflater = GzDecoder::new(body);
144+
let mut archive = tar::Archive::new(deflater);
145+
146+
for entry in archive.entries()? {
147+
let mut entry = entry?;
148+
let path_stripped = entry.path()?.components().skip(1).collect::<PathBuf>();
149+
let bin_path = bin_dir.join(path_stripped);
150+
151+
if entry.header().entry_type().is_dir() {
152+
std::fs::create_dir_all(&bin_path)?;
153+
} else {
154+
if let Some(parent) = bin_path.parent() {
155+
std::fs::create_dir_all(parent)?;
156+
}
157+
158+
let mut options = std::fs::OpenOptions::new();
159+
let options = fix_permissions(&mut options);
160+
let mut file = options.create(true).write(true).open(&bin_path)?;
161+
std::io::copy(&mut entry, &mut file)?;
162+
}
163+
}
164+
165+
Ok(())
166+
}
167+
168+
pub struct Esbuild;
169+
170+
impl BinaryDep for Esbuild {
171+
fn full_name(&self) -> &'static str {
172+
"Esbuild"
173+
}
174+
fn name(&self) -> &'static str {
175+
"esbuild"
176+
}
177+
fn version(&self) -> &'static str {
178+
"0.25.11"
179+
}
180+
fn target(&self) -> &'static str {
181+
match (std::env::consts::OS, std::env::consts::ARCH) {
182+
("android", "arm") => "android-arm",
183+
("android", "aarch64") => "android-arm64",
184+
("android", "x86_64") => "android-x64",
185+
("macos", "aarch64") => "darwin-arm64",
186+
("macos", "x86_64") => "darwin-x64",
187+
("freebsd", "aarch64") => "freebsd-arm64",
188+
("freebsd", "x86_64") => "freebsd-x64",
189+
("linux", "arm") => "linux-arm",
190+
("linux", "aarch64") => "linux-arm64",
191+
("linux", "x86") => "linux-ia32",
192+
("linux", "powerpc64") => "linux-ppc64",
193+
("linux", "s390x") => "linux-s390x",
194+
("linux", "x86_64") => "linux-x64",
195+
("netbsd", "aarch64") => "netbsd-arm64",
196+
("netbsd", "x86_64") => "netbsd-x64",
197+
("openbsd", "aarch64") => "openbsd-arm64",
198+
("openbsd", "x86_64") => "openbsd-x64",
199+
("solaris", "x86_64") => "sunos-x64",
200+
("windows", "aarch64") => "win32-arm64",
201+
("windows", "x86") => "win32-ia32",
202+
("windows", "x86_64") => "win32-x64",
203+
_ => panic!("Platform unsupported by esbuild."),
204+
}
205+
}
206+
fn download_url(&self) -> String {
207+
let version = self.version();
208+
let target = self.target();
209+
format!("https://registry.npmjs.org/@esbuild/{target}/-/{target}-{version}.tgz")
210+
}
211+
fn bin_path(&self, name: Option<&str>) -> Result<String> {
212+
Ok(match name {
213+
None | Some("esbuild") => format!("bin/esbuild{MAYBE_EXE}"),
214+
Some(name) => bail!("Unknown binary {name} in {}", self.full_name()),
215+
})
216+
}
217+
}
218+
219+
pub struct WasmOpt;
220+
221+
impl BinaryDep for WasmOpt {
222+
fn full_name(&self) -> &'static str {
223+
"Wasm Opt"
224+
}
225+
fn name(&self) -> &'static str {
226+
"wasm-opt"
227+
}
228+
fn version(&self) -> &'static str {
229+
"124"
230+
}
231+
fn target(&self) -> &'static str {
232+
match (std::env::consts::OS, std::env::consts::ARCH) {
233+
("macos", "aarch64") => "arm64-macos",
234+
("macos", "x86_64") => "x86_64-macos",
235+
("linux" | "freebsd" | "netbsd" | "openbsd" | "android", "aarch64") => "aarch64-linux",
236+
("linux" | "freebsd" | "netbsd" | "openbsd", "x86_64") => "x86_64-linux",
237+
("windows", "aarch64") => "arm64-windows",
238+
("windows", "x86_64") => "x86_64-windows",
239+
_ => panic!("Platform unsupported for {}", self.full_name()),
240+
}
241+
}
242+
fn download_url(&self) -> String {
243+
let version = self.version();
244+
let target = self.target();
245+
format!("https://github.com/WebAssembly/binaryen/releases/download/version_{version}/binaryen-version_{version}-{target}.tar.gz")
246+
}
247+
fn bin_path(&self, name: Option<&str>) -> Result<String> {
248+
Ok(match name {
249+
None | Some("wasm-opt") => format!("bin/wasm-opt{MAYBE_EXE}"),
250+
Some(name) => bail!("Unknown binary {name} in {}", self.full_name()),
251+
})
252+
}
253+
}
254+
255+
pub struct WasmBindgen;
256+
257+
impl BinaryDep for WasmBindgen {
258+
fn full_name(&self) -> &'static str {
259+
"Wasm Bindgen"
260+
}
261+
fn name(&self) -> &'static str {
262+
"wasm-bindgen"
263+
}
264+
fn version(&self) -> &'static str {
265+
"0.2.105"
266+
}
267+
fn target(&self) -> &'static str {
268+
match (std::env::consts::OS, std::env::consts::ARCH) {
269+
("macos", "aarch64") => "aarch64-apple-darwin",
270+
("macos", "x86_64") => "x86_64-apple-darwin",
271+
("linux" | "freebsd" | "netbsd" | "openbsd" | "android", "aarch64") => {
272+
"aarch64-unknown-linux-musl"
273+
}
274+
("linux" | "freebsd" | "netbsd" | "openbsd", "x86_64") => "x86_64-unknown-linux-musl",
275+
("windows", "x86_64" | "aarch64") => "x86_64-pc-windows-msvc",
276+
_ => panic!("Platform unsupported for {}", self.full_name()),
277+
}
278+
}
279+
fn download_url(&self) -> String {
280+
let version = self.version();
281+
let target = self.target();
282+
format!("https://github.com/wasm-bindgen/wasm-bindgen/releases/download/{version}/wasm-bindgen-{version}-{target}.tar.gz")
283+
}
284+
fn bin_path(&self, name: Option<&str>) -> Result<String> {
285+
Ok(match name {
286+
None | Some("wasm-bindgen") => format!("wasm-bindgen{MAYBE_EXE}"),
287+
Some("wasm-bindgen-test-runner") => format!("wasm-bindgen-test-runner{MAYBE_EXE}"),
288+
Some(name) => bail!("Unknown binary {name} in {}", self.full_name()),
289+
})
290+
}
291+
}

worker-build/src/wasm_pack/emoji.rs renamed to worker-build/src/emoji.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,5 +17,6 @@ pub static TARGET: Emoji = Emoji("🎯 ", "");
1717
pub static CYCLONE: Emoji = Emoji("🌀 ", "");
1818
pub static DOWN_ARROW: Emoji = Emoji("⬇️ ", "");
1919
pub static SPARKLE: Emoji = Emoji("✨ ", ":-)");
20+
pub static CONFIG: Emoji = Emoji("⚙️. ", "");
2021
pub static PACKAGE: Emoji = Emoji("📦 ", ":-)");
2122
pub static WARN: Emoji = Emoji("⚠️ ", ":-)");

0 commit comments

Comments
 (0)