foster/denoland-deno
1
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2025-01-21 04:52:26 -05:00
Code Issues Packages 1 Wiki Activity

Merge branch 'main' of github.com:denoland/deno into sqlite_node_cppgc

Browse source
This commit is contained in:
Divy Srivastava 2024-12-31 22:36:10 +05:30
commit 8b030f3247
273 changed files with 17921 additions and 6821 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.github/workflows/ci.generate.ts vendored
View file

@ -5,7 +5,7 @@ import { stringify } from "jsr:@std/yaml@^0.221/stringify";
// Bump this number when you want to purge the cache.
// Note: the tools/release/01_bump_crate_versions.ts script will update this version
// automatically via regex, so ensure that this line maintains this format.
const cacheVersion = 30;
const cacheVersion = 32;
const ubuntuX86Runner = "ubuntu-24.04";
const ubuntuX86XlRunner = "ubuntu-24.04-xl";

8
.github/workflows/ci.yml vendored
View file

@ -184,8 +184,8 @@ jobs:
~/.cargo/registry/index
~/.cargo/registry/cache
~/.cargo/git/db
key: '30-cargo-home-${{ matrix.os }}-${{ matrix.arch }}-${{ hashFiles(''Cargo.lock'') }}'
restore-keys: '30-cargo-home-${{ matrix.os }}-${{ matrix.arch }}-'
key: '32-cargo-home-${{ matrix.os }}-${{ matrix.arch }}-${{ hashFiles(''Cargo.lock'') }}'
restore-keys: '32-cargo-home-${{ matrix.os }}-${{ matrix.arch }}-'
if: '!(matrix.skip)'
- uses: dsherret/rust-toolchain-file@v1
if: '!(matrix.skip)'
@ -379,7 +379,7 @@ jobs:
!./target/*/*.zip
!./target/*/*.tar.gz
key: never_saved
restore-keys: '30-cargo-target-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.profile }}-${{ matrix.job }}-'
restore-keys: '32-cargo-target-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.profile }}-${{ matrix.job }}-'
- name: Apply and update mtime cache
if: '!(matrix.skip) && (!startsWith(github.ref, ''refs/tags/''))'
uses: ./.github/mtime_cache
@ -689,7 +689,7 @@ jobs:
!./target/*/gn_root
!./target/*/*.zip
!./target/*/*.tar.gz
key: '30-cargo-target-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.profile }}-${{ matrix.job }}-${{ github.sha }}'
key: '32-cargo-target-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.profile }}-${{ matrix.job }}-${{ github.sha }}'
publish-canary:
name: publish canary
runs-on: ubuntu-24.04

350
Cargo.lock generated
View file

@ -39,9 +39,9 @@ dependencies = [
[[package]]
name = "aead-gcm-stream"
version = "0.3.0"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4947a169074c7e038fa43051d1c4e073f4488b0e4b0a30658f1e1a1b06449ce8"
checksum = "e70c8dec860340effb00f6945c49c0daaa6dac963602750db862eabb74bf7886"
dependencies = [
"aead",
"aes",
@ -380,7 +380,7 @@ dependencies = [
"rustversion",
"serde",
"sync_wrapper",
"tower",
"tower 0.4.13",
"tower-layer",
"tower-service",
]
@ -596,9 +596,9 @@ dependencies = [
[[package]]
name = "boxed_error"
version = "0.2.2"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "69aae56aaf59d1994b902ed5c0c79024012bdc2426741def75a635999a030e7e"
checksum = "17d4f95e880cfd28c4ca5a006cf7f6af52b4bcb7b5866f573b2faa126fb7affb"
dependencies = [
"quote",
"syn 2.0.87",
@ -670,13 +670,35 @@ checksum = "1bf2a5fb3207c12b5d208ebc145f967fea5cac41a021c37417ccc31ba40f39ee"
[[package]]
name = "capacity_builder"
version = "0.1.1"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ab97838e07d98bdddf1e7e191ffe3c718cba7477c0b6607fdfb96ecd696202df"
checksum = "58ec49028cb308564429cd8fac4ef21290067a0afe8f5955330a8d487d0d790c"
dependencies = [
"itoa",
]
[[package]]
name = "capacity_builder"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f2d24a6dcf0cd402a21b65d35340f3a49ff3475dc5fdac91d22d2733e6641c6"
dependencies = [
"capacity_builder_macros",
"ecow",
"hipstr",
"itoa",
]
[[package]]
name = "capacity_builder_macros"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3b4a6cae9efc04cc6cbb8faf338d2c497c165c83e74509cf4dbedea948bbf6e5"
dependencies = [
"quote",
"syn 2.0.87",
]
[[package]]
name = "caseless"
version = "0.2.1"
@ -728,6 +750,12 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e"
[[package]]
name = "cfg_aliases"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
[[package]]
name = "chrono"
version = "0.4.37"
@ -839,6 +867,7 @@ dependencies = [
"regex",
"reqwest",
"serde",
"sys_traits",
"test_server",
"tokio",
"url",
@ -1193,9 +1222,9 @@ checksum = "7e962a19be5cfc3f3bf6dd8f61eb50107f356ad6270fbb3ed41476571db78be5"
[[package]]
name = "data-url"
version = "0.3.0"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41b319d1b62ffbd002e057f36bebd1f42b9f97927c9577461d855f3513c4289f"
checksum = "5c297a1c74b71ae29df00c3e22dd9534821d60eb9af5a0192823fa2acea70c2a"
[[package]]
name = "debug-ignore"
@ -1221,8 +1250,10 @@ dependencies = [
"async-trait",
"base64 0.21.7",
"bincode",
"boxed_error",
"bytes",
"cache_control",
"capacity_builder 0.5.0",
"chrono",
"clap",
"clap_complete",
@ -1237,13 +1268,14 @@ dependencies = [
"deno_config",
"deno_core",
"deno_doc",
"deno_error",
"deno_graph",
"deno_lint",
"deno_lockfile",
"deno_npm",
"deno_npm_cache",
"deno_package_json",
"deno_path_util",
"deno_path_util 0.3.0",
"deno_resolver",
"deno_runtime",
"deno_semver",
@ -1309,6 +1341,7 @@ dependencies = [
"spki",
"sqlformat",
"strsim",
"sys_traits",
"tar",
"tempfile",
"test_server",
@ -1388,7 +1421,7 @@ dependencies = [
[[package]]
name = "deno_bench_util"
version = "0.176.0"
version = "0.178.0"
dependencies = [
"bencher",
"deno_core",
@ -1397,7 +1430,7 @@ dependencies = [
[[package]]
name = "deno_broadcast_channel"
version = "0.176.0"
version = "0.178.0"
dependencies = [
"async-trait",
"deno_core",
@ -1408,7 +1441,7 @@ dependencies = [
[[package]]
name = "deno_cache"
version = "0.114.0"
version = "0.116.0"
dependencies = [
"async-trait",
"deno_core",
@ -1421,13 +1454,21 @@ dependencies = [
[[package]]
name = "deno_cache_dir"
version = "0.14.0"
version = "0.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cca43605c8cbce6c6787e0daf227864487c07c2b31d438c0bf43d1b38da94b7f"
checksum = "e73ed17f285731a23df9779ca1e0e721de866db6776ed919ebd9235e0a107c4c"
dependencies = [
"async-trait",
"base32",
"base64 0.21.7",
"boxed_error",
"cache_control",
"chrono",
"data-url",
"deno_error",
"deno_media_type",
"deno_path_util",
"deno_path_util 0.3.0",
"http 1.1.0",
"indexmap 2.3.0",
"log",
"once_cell",
@ -1435,13 +1476,14 @@ dependencies = [
"serde",
"serde_json",
"sha2",
"sys_traits",
"thiserror 1.0.64",
"url",
]
[[package]]
name = "deno_canvas"
version = "0.51.0"
version = "0.53.0"
dependencies = [
"deno_core",
"deno_webgpu",
@ -1452,13 +1494,13 @@ dependencies = [
[[package]]
name = "deno_config"
version = "0.39.3"
version = "0.42.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ce717af3fe6788dae63965d58d5637fd62be8fe4f345f189137ffc06c51837d2"
checksum = "b45aaf31e58ca915d5c0746bf8e2d07b94635154ad9e5afe5ff265cae6187b19"
dependencies = [
"anyhow",
"deno_package_json",
"deno_path_util",
"deno_path_util 0.3.0",
"deno_semver",
"glob",
"ignore",
@ -1470,22 +1512,23 @@ dependencies = [
"phf",
"serde",
"serde_json",
"sys_traits",
"thiserror 1.0.64",
"url",
]
[[package]]
name = "deno_console"
version = "0.182.0"
version = "0.184.0"
dependencies = [
"deno_core",
]
[[package]]
name = "deno_core"
version = "0.326.0"
version = "0.327.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed157162dc5320a2b46ffeeaec24788339df0f2437cfaea78a8d82696715ad7f"
checksum = "eaf8dff204b9c2415deb47b9f30d4d38b0925d0d88f1f9074e8e76f59e6d7ded"
dependencies = [
"anyhow",
"az",
@ -1493,7 +1536,7 @@ dependencies = [
"bit-set",
"bit-vec",
"bytes",
"capacity_builder",
"capacity_builder 0.1.3",
"cooked-waker",
"deno_core_icudata",
"deno_ops",
@ -1525,7 +1568,7 @@ checksum = "fe4dccb6147bb3f3ba0c7a48e993bfeb999d2c2e47a81badee80e2b370c8d695"
[[package]]
name = "deno_cron"
version = "0.62.0"
version = "0.64.0"
dependencies = [
"anyhow",
"async-trait",
@ -1538,7 +1581,7 @@ dependencies = [
[[package]]
name = "deno_crypto"
version = "0.196.0"
version = "0.198.0"
dependencies = [
"aes",
"aes-gcm",
@ -1584,7 +1627,7 @@ dependencies = [
"comrak",
"deno_ast",
"deno_graph",
"deno_path_util",
"deno_path_util 0.2.2",
"handlebars",
"html-escape",
"import_map",
@ -1612,6 +1655,7 @@ dependencies = [
"libc",
"serde",
"serde_json",
"url",
]
[[package]]
@ -1627,18 +1671,19 @@ dependencies = [
[[package]]
name = "deno_fetch"
version = "0.206.0"
version = "0.208.0"
dependencies = [
"base64 0.21.7",
"bytes",
"data-url",
"deno_core",
"deno_path_util",
"deno_path_util 0.3.0",
"deno_permissions",
"deno_tls",
"dyn-clone",
"error_reporter",
"fast-socks5",
"h2 0.4.4",
"hickory-resolver",
"http 1.1.0",
"http-body-util",
@ -1655,14 +1700,14 @@ dependencies = [
"tokio-rustls",
"tokio-socks",
"tokio-util",
"tower",
"tower 0.5.2",
"tower-http",
"tower-service",
]
[[package]]
name = "deno_ffi"
version = "0.169.0"
version = "0.171.0"
dependencies = [
"deno_core",
"deno_permissions",
@ -1682,14 +1727,14 @@ dependencies = [
[[package]]
name = "deno_fs"
version = "0.92.0"
version = "0.94.0"
dependencies = [
"async-trait",
"base32",
"boxed_error",
"deno_core",
"deno_io",
"deno_path_util",
"deno_path_util 0.3.0",
"deno_permissions",
"filetime",
"junction",
@ -1705,14 +1750,16 @@ dependencies = [
[[package]]
name = "deno_graph"
version = "0.86.3"
version = "0.86.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc78ed0b4bbcb4197300f0d6e7d1edc2d2c5019cdb9dedba7ff229158441885b"
checksum = "83af194ca492ea7b624d21055f933676d3f3d27586de93be31c8f1babcc73510"
dependencies = [
"anyhow",
"async-trait",
"capacity_builder 0.5.0",
"data-url",
"deno_ast",
"deno_path_util 0.3.0",
"deno_semver",
"deno_unsync",
"encoding_rs",
@ -1727,6 +1774,7 @@ dependencies = [
"serde",
"serde_json",
"sha2",
"sys_traits",
"thiserror 2.0.3",
"twox-hash",
"url",
@ -1735,7 +1783,7 @@ dependencies = [
[[package]]
name = "deno_http"
version = "0.180.0"
version = "0.182.0"
dependencies = [
"async-compression",
"async-trait",
@ -1774,7 +1822,7 @@ dependencies = [
[[package]]
name = "deno_io"
version = "0.92.0"
version = "0.94.0"
dependencies = [
"async-trait",
"deno_core",
@ -1795,7 +1843,7 @@ dependencies = [
[[package]]
name = "deno_kv"
version = "0.90.0"
version = "0.92.0"
dependencies = [
"anyhow",
"async-trait",
@ -1805,7 +1853,7 @@ dependencies = [
"chrono",
"deno_core",
"deno_fetch",
"deno_path_util",
"deno_path_util 0.3.0",
"deno_permissions",
"deno_tls",
"denokv_proto",
@ -1845,9 +1893,9 @@ dependencies = [
[[package]]
name = "deno_lockfile"
version = "0.23.2"
version = "0.24.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "559c19feb00af0c34f0bd4a20e56e12463fafd5c5069d6005f3ce33008027eea"
checksum = "632e835a53ed667d62fdd766c5780fe8361c831d3e3fbf1a760a0b7896657587"
dependencies = [
"deno_semver",
"serde",
@ -1857,9 +1905,9 @@ dependencies = [
[[package]]
name = "deno_media_type"
version = "0.2.1"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7fcf552fbdedbe81c89705349d7d2485c7051382b000dfddbdbf7fc25931cf83"
checksum = "eaa135b8a9febc9a51c16258e294e268a1276750780d69e46edb31cced2826e4"
dependencies = [
"data-url",
"serde",
@ -1868,7 +1916,7 @@ dependencies = [
[[package]]
name = "deno_napi"
version = "0.113.0"
version = "0.115.0"
dependencies = [
"deno_core",
"deno_permissions",
@ -1896,7 +1944,7 @@ dependencies = [
[[package]]
name = "deno_net"
version = "0.174.0"
version = "0.176.0"
dependencies = [
"deno_core",
"deno_permissions",
@ -1904,6 +1952,7 @@ dependencies = [
"hickory-proto",
"hickory-resolver",
"pin-project",
"quinn",
"rustls-tokio-stream",
"serde",
"socket2",
@ -1913,7 +1962,7 @@ dependencies = [
[[package]]
name = "deno_node"
version = "0.119.0"
version = "0.122.0"
dependencies = [
"aead-gcm-stream",
"aes",
@ -1933,7 +1982,7 @@ dependencies = [
"deno_media_type",
"deno_net",
"deno_package_json",
"deno_path_util",
"deno_path_util 0.3.0",
"deno_permissions",
"deno_whoami",
"der",
@ -1993,6 +2042,7 @@ dependencies = [
"sm3",
"spki",
"stable_deref_trait",
"sys_traits",
"thiserror 2.0.3",
"tokio",
"tokio-eld",
@ -2007,12 +2057,13 @@ dependencies = [
[[package]]
name = "deno_npm"
version = "0.26.0"
version = "0.27.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2f125a5dba7839c46394a0a9c835da9fe60f5f412587ab4956a76492a1cc6a8"
checksum = "5f818ad5dc4c206b50b5cfa6f10b4b94b127e15c8342c152768eba40c225ca23"
dependencies = [
"anyhow",
"async-trait",
"capacity_builder 0.5.0",
"deno_error",
"deno_lockfile",
"deno_semver",
"futures",
@ -2026,7 +2077,7 @@ dependencies = [
[[package]]
name = "deno_npm_cache"
version = "0.2.0"
version = "0.3.0"
dependencies = [
"anyhow",
"async-trait",
@ -2034,7 +2085,9 @@ dependencies = [
"boxed_error",
"deno_cache_dir",
"deno_core",
"deno_error",
"deno_npm",
"deno_path_util 0.3.0",
"deno_semver",
"deno_unsync",
"faster-hex",
@ -2047,6 +2100,7 @@ dependencies = [
"rand",
"ring",
"serde_json",
"sys_traits",
"tar",
"tempfile",
"thiserror 2.0.3",
@ -2055,9 +2109,9 @@ dependencies = [
[[package]]
name = "deno_ops"
version = "0.202.0"
version = "0.203.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4dd8ac1af251e292388e516dd339b9a3b982a6d1e7f8644c08e34671ca39003c"
checksum = "b146ca74cac431843486ade58e2accc16c11315fb2c6934590a52a73c56b7ec3"
dependencies = [
"proc-macro-rules",
"proc-macro2",
@ -2071,38 +2125,54 @@ dependencies = [
[[package]]
name = "deno_package_json"
version = "0.2.1"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "80b0a3d81c592624a1ae15332a04b4dc2b7c163ef1dfc7c60171f736d1babdf5"
checksum = "e1d3c0f699ba2040669204ce24ab73720499fc290af843e4ce0fc8a9b3d67735"
dependencies = [
"boxed_error",
"deno_error",
"deno_path_util",
"deno_path_util 0.3.0",
"deno_semver",
"indexmap 2.3.0",
"serde",
"serde_json",
"sys_traits",
"thiserror 2.0.3",
"url",
]
[[package]]
name = "deno_path_util"
version = "0.2.1"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ff25f6e08e7a0214bbacdd6f7195c7f1ebcd850c87a624e4ff06326b68b42d99"
checksum = "b02c7d341e1b2cf089daff0f4fb2b4be8f3b5511b1d96040b3f7ed63a66c737b"
dependencies = [
"deno_error",
"percent-encoding",
"thiserror 1.0.64",
"thiserror 2.0.3",
"url",
]
[[package]]
name = "deno_path_util"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "420e8211aaba7fde83ccaa9a5dad855c3b940ed988d70c95159acd600a70dc87"
dependencies = [
"deno_error",
"percent-encoding",
"sys_traits",
"thiserror 2.0.3",
"url",
]
[[package]]
name = "deno_permissions"
version = "0.42.0"
version = "0.43.0"
dependencies = [
"capacity_builder",
"capacity_builder 0.5.0",
"deno_core",
"deno_path_util",
"deno_path_util 0.3.0",
"deno_terminal 0.2.0",
"fqdn",
"libc",
@ -2117,7 +2187,7 @@ dependencies = [
[[package]]
name = "deno_resolver"
version = "0.14.0"
version = "0.15.0"
dependencies = [
"anyhow",
"base32",
@ -2126,9 +2196,10 @@ dependencies = [
"deno_config",
"deno_media_type",
"deno_package_json",
"deno_path_util",
"deno_path_util 0.3.0",
"deno_semver",
"node_resolver",
"sys_traits",
"test_server",
"thiserror 2.0.3",
"url",
@ -2136,7 +2207,7 @@ dependencies = [
[[package]]
name = "deno_runtime"
version = "0.191.0"
version = "0.192.0"
dependencies = [
"color-print",
"deno_ast",
@ -2156,7 +2227,7 @@ dependencies = [
"deno_napi",
"deno_net",
"deno_node",
"deno_path_util",
"deno_path_util 0.3.0",
"deno_permissions",
"deno_telemetry",
"deno_terminal 0.2.0",
@ -2191,6 +2262,7 @@ dependencies = [
"serde",
"signal-hook",
"signal-hook-registry",
"sys_traits",
"tempfile",
"test_server",
"thiserror 2.0.3",
@ -2205,11 +2277,14 @@ dependencies = [
[[package]]
name = "deno_semver"
version = "0.6.1"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7d1259270d66a5e6d29bb75c9289656541874f79ae9ff6c9f1c790846d5c07ba"
checksum = "4775271f9b5602482698f76d24ea9ed8ba27af7f587a7e9a876916300c542435"
dependencies = [
"capacity_builder 0.5.0",
"deno_error",
"ecow",
"hipstr",
"monch",
"once_cell",
"serde",
@ -2237,7 +2312,7 @@ dependencies = [
[[package]]
name = "deno_telemetry"
version = "0.4.0"
version = "0.6.0"
dependencies = [
"async-trait",
"deno_core",
@ -2278,7 +2353,7 @@ dependencies = [
[[package]]
name = "deno_tls"
version = "0.169.0"
version = "0.171.0"
dependencies = [
"deno_core",
"deno_native_certs",
@ -2311,7 +2386,7 @@ dependencies = [
"serde_json",
"tokio",
"tokio-util",
"tower",
"tower 0.4.13",
"tracing",
]
@ -2328,7 +2403,7 @@ dependencies = [
[[package]]
name = "deno_url"
version = "0.182.0"
version = "0.184.0"
dependencies = [
"deno_bench_util",
"deno_console",
@ -2340,7 +2415,7 @@ dependencies = [
[[package]]
name = "deno_web"
version = "0.213.0"
version = "0.215.0"
dependencies = [
"async-trait",
"base64-simd 0.8.0",
@ -2362,7 +2437,7 @@ dependencies = [
[[package]]
name = "deno_webgpu"
version = "0.149.0"
version = "0.151.0"
dependencies = [
"deno_core",
"raw-window-handle",
@ -2375,7 +2450,7 @@ dependencies = [
[[package]]
name = "deno_webidl"
version = "0.182.0"
version = "0.184.0"
dependencies = [
"deno_bench_util",
"deno_core",
@ -2383,7 +2458,7 @@ dependencies = [
[[package]]
name = "deno_websocket"
version = "0.187.0"
version = "0.189.0"
dependencies = [
"bytes",
"deno_core",
@ -2405,7 +2480,7 @@ dependencies = [
[[package]]
name = "deno_webstorage"
version = "0.177.0"
version = "0.179.0"
dependencies = [
"deno_core",
"deno_web",
@ -2876,6 +2951,15 @@ dependencies = [
"spki",
]
[[package]]
name = "ecow"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e42fc0a93992b20c58b99e59d61eaf1635a25bfbe49e4275c34ba0aee98119ba"
dependencies = [
"serde",
]
[[package]]
name = "ed25519"
version = "2.2.3"
@ -3812,6 +3896,17 @@ dependencies = [
"tracing",
]
[[package]]
name = "hipstr"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "97971ffc85d4c98de12e2608e992a43f5294ebb625fdb045b27c731b64c4c6d6"
dependencies = [
"serde",
"serde_bytes",
"sptr",
]
[[package]]
name = "hkdf"
version = "0.12.4"
@ -4012,9 +4107,9 @@ dependencies = [
[[package]]
name = "hyper-timeout"
version = "0.5.1"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3203a961e5c83b6f5498933e78b6b263e208c197b63e9c6c53cc82ffd3f63793"
checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0"
dependencies = [
"hyper 1.4.1",
"hyper-util",
@ -4541,9 +4636,9 @@ dependencies = [
[[package]]
name = "libc"
version = "0.2.153"
version = "0.2.168"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd"
checksum = "5aaeb2981e0606ca11d79718f8bb01164f1d6ed75080182d3abf017e6d244b6d"
[[package]]
name = "libffi"
@ -4919,7 +5014,7 @@ dependencies = [
[[package]]
name = "napi_sym"
version = "0.112.0"
version = "0.114.0"
dependencies = [
"quote",
"serde",
@ -4974,20 +5069,21 @@ dependencies = [
[[package]]
name = "node_resolver"
version = "0.21.0"
version = "0.22.0"
dependencies = [
"anyhow",
"async-trait",
"boxed_error",
"deno_media_type",
"deno_package_json",
"deno_path_util",
"deno_path_util 0.3.0",
"futures",
"lazy-regex",
"once_cell",
"path-clean",
"regex",
"serde_json",
"sys_traits",
"thiserror 2.0.3",
"tokio",
"url",
@ -5895,49 +5991,54 @@ dependencies = [
[[package]]
name = "quinn"
version = "0.11.2"
version = "0.11.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e4ceeeeabace7857413798eb1ffa1e9c905a9946a57d81fb69b4b71c4d8eb3ad"
checksum = "62e96808277ec6f97351a2380e6c25114bc9e67037775464979f3037c92d05ef"
dependencies = [
"bytes",
"pin-project-lite",
"quinn-proto",
"quinn-udp",
"rustc-hash 1.1.0",
"rustc-hash 2.0.0",
"rustls",
"thiserror 1.0.64",
"socket2",
"thiserror 2.0.3",
"tokio",
"tracing",
]
[[package]]
name = "quinn-proto"
version = "0.11.8"
version = "0.11.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fadfaed2cd7f389d0161bb73eeb07b7b78f8691047a6f3e73caaeae55310a4a6"
checksum = "a2fe5ef3495d7d2e377ff17b1a8ce2ee2ec2a18cde8b6ad6619d65d0701c135d"
dependencies = [
"bytes",
"getrandom",
"rand",
"ring",
"rustc-hash 2.0.0",
"rustls",
"rustls-pki-types",
"slab",
"thiserror 1.0.64",
"thiserror 2.0.3",
"tinyvec",
"tracing",
"web-time",
]
[[package]]
name = "quinn-udp"
version = "0.5.2"
version = "0.5.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9096629c45860fc7fb143e125eb826b5e721e10be3263160c7d60ca832cf8c46"
checksum = "52cd4b1eff68bf27940dd39811292c49e007f4d0b4c357358dc9b0197be6b527"
dependencies = [
"cfg_aliases 0.2.1",
"libc",
"once_cell",
"socket2",
"tracing",
"windows-sys 0.52.0",
"windows-sys 0.59.0",
]
[[package]]
@ -6414,6 +6515,9 @@ name = "rustls-pki-types"
version = "1.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc0a2ce646f8655401bb81e7927b812614bd5d91dbc968696be50603510fcaf0"
dependencies = [
"web-time",
]
[[package]]
name = "rustls-tokio-stream"
@ -6697,9 +6801,9 @@ dependencies = [
[[package]]
name = "serde_v8"
version = "0.235.0"
version = "0.236.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d07afd8b67b4a442ecc2823038473ac0e9e5682de93c213323b60661afdd7eb4"
checksum = "e23b3abce64010612f88f4ff689a959736f99eb3dc0dbf1c7903434b8bd8cda5"
dependencies = [
"num-bigint",
"serde",
@ -6975,6 +7079,12 @@ dependencies = [
"der",
]
[[package]]
name = "sptr"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3b9b39299b249ad65f3b7e96443bad61c02ca5cd3589f46cb6d610a0fd6c0d6a"
[[package]]
name = "sqlformat"
version = "0.3.2"
@ -7570,6 +7680,19 @@ dependencies = [
"syn 2.0.87",
]
[[package]]
name = "sys_traits"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6683465f4e1d8fd75069cbc36c646258c05b7d8d6676bcb5d71968b99b7d5ae2"
dependencies = [
"filetime",
"getrandom",
"libc",
"parking_lot",
"windows-sys 0.59.0",
]
[[package]]
name = "tagptr"
version = "0.2.0"
@ -7965,7 +8088,7 @@ dependencies = [
"socket2",
"tokio",
"tokio-stream",
"tower",
"tower 0.4.13",
"tower-layer",
"tower-service",
"tracing",
@ -7991,6 +8114,21 @@ dependencies = [
"tracing",
]
[[package]]
name = "tower"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9"
dependencies = [
"futures-core",
"futures-util",
"pin-project-lite",
"sync_wrapper",
"tokio",
"tower-layer",
"tower-service",
]
[[package]]
name = "tower-http"
version = "0.6.1"
@ -8019,9 +8157,9 @@ checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e"
[[package]]
name = "tower-service"
version = "0.3.2"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52"
checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3"
[[package]]
name = "tracing"
@ -8495,6 +8633,16 @@ dependencies = [
"wasm-bindgen",
]
[[package]]
name = "web-time"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb"
dependencies = [
"js-sys",
"wasm-bindgen",
]
[[package]]
name = "webpki-root-certs"
version = "0.26.6"
@ -8522,7 +8670,7 @@ dependencies = [
"arrayvec",
"bit-vec",
"bitflags 2.6.0",
"cfg_aliases",
"cfg_aliases 0.1.1",
"codespan-reporting",
"document-features",
"indexmap 2.3.0",
@ -8554,7 +8702,7 @@ dependencies = [
"bit-set",
"bitflags 2.6.0",
"block",
"cfg_aliases",
"cfg_aliases 0.1.1",
"core-graphics-types",
"d3d12",
"glow",

88
Cargo.toml
View file

@ -48,19 +48,19 @@ repository = "https://github.com/denoland/deno"
[workspace.dependencies]
deno_ast = { version = "=0.44.0", features = ["transpiling"] }
deno_core = { version = "0.326.0" }
deno_core = { version = "0.327.0" }
deno_bench_util = { version = "0.176.0", path = "./bench_util" }
deno_config = { version = "=0.39.3", features = ["workspace", "sync"] }
deno_lockfile = "=0.23.2"
deno_bench_util = { version = "0.178.0", path = "./bench_util" }
deno_config = { version = "=0.42.0", features = ["workspace", "sync"] }
deno_lockfile = "=0.24.0"
deno_media_type = { version = "0.2.0", features = ["module_specifier"] }
deno_npm = "=0.26.0"
deno_path_util = "=0.2.1"
deno_permissions = { version = "0.42.0", path = "./runtime/permissions" }
deno_runtime = { version = "0.191.0", path = "./runtime" }
deno_semver = "=0.6.1"
deno_npm = "=0.27.0"
deno_path_util = "=0.3.0"
deno_permissions = { version = "0.43.0", path = "./runtime/permissions" }
deno_runtime = { version = "0.192.0", path = "./runtime" }
deno_semver = "=0.7.1"
deno_terminal = "0.2.0"
napi_sym = { version = "0.112.0", path = "./ext/napi/sym" }
napi_sym = { version = "0.114.0", path = "./ext/napi/sym" }
test_util = { package = "test_server", path = "./tests/util/server" }
denokv_proto = "0.8.4"
@ -69,34 +69,34 @@ denokv_remote = "0.8.4"
denokv_sqlite = { default-features = false, version = "0.8.4" }
# exts
deno_broadcast_channel = { version = "0.176.0", path = "./ext/broadcast_channel" }
deno_cache = { version = "0.114.0", path = "./ext/cache" }
deno_canvas = { version = "0.51.0", path = "./ext/canvas" }
deno_console = { version = "0.182.0", path = "./ext/console" }
deno_cron = { version = "0.62.0", path = "./ext/cron" }
deno_crypto = { version = "0.196.0", path = "./ext/crypto" }
deno_fetch = { version = "0.206.0", path = "./ext/fetch" }
deno_ffi = { version = "0.169.0", path = "./ext/ffi" }
deno_fs = { version = "0.92.0", path = "./ext/fs" }
deno_http = { version = "0.180.0", path = "./ext/http" }
deno_io = { version = "0.92.0", path = "./ext/io" }
deno_kv = { version = "0.90.0", path = "./ext/kv" }
deno_napi = { version = "0.113.0", path = "./ext/napi" }
deno_net = { version = "0.174.0", path = "./ext/net" }
deno_node = { version = "0.119.0", path = "./ext/node" }
deno_telemetry = { version = "0.4.0", path = "./ext/telemetry" }
deno_tls = { version = "0.169.0", path = "./ext/tls" }
deno_url = { version = "0.182.0", path = "./ext/url" }
deno_web = { version = "0.213.0", path = "./ext/web" }
deno_webgpu = { version = "0.149.0", path = "./ext/webgpu" }
deno_webidl = { version = "0.182.0", path = "./ext/webidl" }
deno_websocket = { version = "0.187.0", path = "./ext/websocket" }
deno_webstorage = { version = "0.177.0", path = "./ext/webstorage" }
deno_broadcast_channel = { version = "0.178.0", path = "./ext/broadcast_channel" }
deno_cache = { version = "0.116.0", path = "./ext/cache" }
deno_canvas = { version = "0.53.0", path = "./ext/canvas" }
deno_console = { version = "0.184.0", path = "./ext/console" }
deno_cron = { version = "0.64.0", path = "./ext/cron" }
deno_crypto = { version = "0.198.0", path = "./ext/crypto" }
deno_fetch = { version = "0.208.0", path = "./ext/fetch" }
deno_ffi = { version = "0.171.0", path = "./ext/ffi" }
deno_fs = { version = "0.94.0", path = "./ext/fs" }
deno_http = { version = "0.182.0", path = "./ext/http" }
deno_io = { version = "0.94.0", path = "./ext/io" }
deno_kv = { version = "0.92.0", path = "./ext/kv" }
deno_napi = { version = "0.115.0", path = "./ext/napi" }
deno_net = { version = "0.176.0", path = "./ext/net" }
deno_node = { version = "0.122.0", path = "./ext/node" }
deno_telemetry = { version = "0.6.0", path = "./ext/telemetry" }
deno_tls = { version = "0.171.0", path = "./ext/tls" }
deno_url = { version = "0.184.0", path = "./ext/url" }
deno_web = { version = "0.215.0", path = "./ext/web" }
deno_webgpu = { version = "0.151.0", path = "./ext/webgpu" }
deno_webidl = { version = "0.184.0", path = "./ext/webidl" }
deno_websocket = { version = "0.189.0", path = "./ext/websocket" }
deno_webstorage = { version = "0.179.0", path = "./ext/webstorage" }
# resolvers
deno_npm_cache = { version = "0.2.0", path = "./resolvers/npm_cache" }
deno_resolver = { version = "0.14.0", path = "./resolvers/deno" }
node_resolver = { version = "0.21.0", path = "./resolvers/node" }
deno_npm_cache = { version = "0.3.0", path = "./resolvers/npm_cache" }
deno_resolver = { version = "0.15.0", path = "./resolvers/deno" }
node_resolver = { version = "0.22.0", path = "./resolvers/node" }
aes = "=0.8.3"
anyhow = "1.0.57"
@ -104,11 +104,11 @@ async-trait = "0.1.73"
base32 = "=0.5.1"
base64 = "0.21.7"
bencher = "0.1"
boxed_error = "0.2.2"
boxed_error = "0.2.3"
brotli = "6.0.0"
bytes = "1.4.0"
cache_control = "=0.2.0"
capacity_builder = "0.1.0"
capacity_builder = "0.5.0"
cbc = { version = "=0.1.2", features = ["alloc"] }
# Note: Do not use the "clock" feature of chrono, as it links us to CoreFoundation on macOS.
# Instead use util::time::utc_now()
@ -117,9 +117,10 @@ color-print = "0.3.5"
console_static_text = "=0.8.1"
dashmap = "5.5.3"
data-encoding = "2.3.3"
data-url = "=0.3.0"
deno_cache_dir = "=0.14.0"
deno_package_json = { version = "0.2.1", default-features = false }
data-url = "=0.3.1"
deno_cache_dir = "=0.16.0"
deno_error = "=0.5.2"
deno_package_json = { version = "0.4.0", default-features = false }
deno_unsync = "0.4.2"
dlopen2 = "0.6.1"
ecb = "=0.1.2"
@ -148,7 +149,7 @@ indexmap = { version = "2", features = ["serde"] }
ipnet = "2.3"
jsonc-parser = { version = "=0.26.2", features = ["serde"] }
lazy-regex = "3"
libc = "0.2.126"
libc = "0.2.168"
libz-sys = { version = "1.1.20", default-features = false }
log = { version = "0.4.20", features = ["kv"] }
lsp-types = "=0.97.0" # used by tower-lsp and "proposed" feature is unstable in patch releases
@ -192,6 +193,7 @@ slab = "0.4"
smallvec = "1.8"
socket2 = { version = "0.5.3", features = ["all"] }
spki = "0.7.2"
sys_traits = "=0.1.4"
tar = "=0.4.40"
tempfile = "3.4.0"
termcolor = "1.1.3"
@ -201,7 +203,7 @@ tokio-metrics = { version = "0.3.0", features = ["rt"] }
tokio-rustls = { version = "0.26.0", default-features = false, features = ["ring", "tls12"] }
tokio-socks = "0.5.1"
tokio-util = "0.7.4"
tower = { version = "0.4.13", default-features = false, features = ["util"] }
tower = { version = "0.5.2", default-features = false, features = ["retry", "util"] }
tower-http = { version = "0.6.1", features = ["decompression-br", "decompression-gzip"] }
tower-lsp = { package = "deno_tower_lsp", version = "0.1.0", features = ["proposed"] }
tower-service = "0.3.2"

2
bench_util/Cargo.toml
View file

@ -2,7 +2,7 @@
[package]
name = "deno_bench_util"
version = "0.176.0"
version = "0.178.0"
authors.workspace = true
edition.workspace = true
license.workspace = true

7
cli/Cargo.toml
View file

@ -73,7 +73,8 @@ deno_cache_dir.workspace = true
deno_config.workspace = true
deno_core = { workspace = true, features = ["include_js_files_for_snapshotting"] }
deno_doc = { version = "=0.161.3", features = ["rust", "comrak"] }
deno_graph = { version = "=0.86.3" }
deno_error.workspace = true
deno_graph = { version = "=0.86.6" }
deno_lint = { version = "=0.68.2", features = ["docs"] }
deno_lockfile.workspace = true
deno_npm.workspace = true
@ -93,8 +94,10 @@ anstream = "0.6.14"
async-trait.workspace = true
base64.workspace = true
bincode = "=1.3.3"
boxed_error.workspace = true
bytes.workspace = true
cache_control.workspace = true
capacity_builder.workspace = true
chrono = { workspace = true, features = ["now"] }
clap = { version = "=4.5.16", features = ["env", "string", "wrap_help", "error-context"] }
clap_complete = "=4.5.24"
@ -155,6 +158,7 @@ shell-escape = "=0.1.5"
spki = { version = "0.7", features = ["pem"] }
sqlformat = "=0.3.2"
strsim = "0.11.1"
sys_traits = { workspace = true, features = ["getrandom", "filetime", "libc", "real", "strip_unc", "winapi"] }
tar.workspace = true
tempfile.workspace = true
text-size = "=1.1.0"
@ -183,6 +187,7 @@ nix.workspace = true
[dev-dependencies]
deno_bench_util.workspace = true
pretty_assertions.workspace = true
sys_traits = { workspace = true, features = ["memory"] }
test_util.workspace = true
[package.metadata.winres]

56
cli/args/deno_json.rs
View file

@ -8,62 +8,6 @@ use deno_semver::jsr::JsrDepPackageReq;
use deno_semver::jsr::JsrPackageReqReference;
use deno_semver::npm::NpmPackageReqReference;
#[cfg(test)] // happens to only be used by the tests at the moment
pub struct DenoConfigFsAdapter<'a>(
pub &'a dyn deno_runtime::deno_fs::FileSystem,
);
#[cfg(test)]
impl<'a> deno_config::fs::DenoConfigFs for DenoConfigFsAdapter<'a> {
fn read_to_string_lossy(
&self,
path: &std::path::Path,
) -> Result<std::borrow::Cow<'static, str>, std::io::Error> {
self
.0
.read_text_file_lossy_sync(path, None)
.map_err(|err| err.into_io_error())
}
fn stat_sync(
&self,
path: &std::path::Path,
) -> Result<deno_config::fs::FsMetadata, std::io::Error> {
self
.0
.stat_sync(path)
.map(|stat| deno_config::fs::FsMetadata {
is_file: stat.is_file,
is_directory: stat.is_directory,
is_symlink: stat.is_symlink,
})
.map_err(|err| err.into_io_error())
}
fn read_dir(
&self,
path: &std::path::Path,
) -> Result<Vec<deno_config::fs::FsDirEntry>, std::io::Error> {
self
.0
.read_dir_sync(path)
.map_err(|err| err.into_io_error())
.map(|entries| {
entries
.into_iter()
.map(|e| deno_config::fs::FsDirEntry {
path: path.join(e.name),
metadata: deno_config::fs::FsMetadata {
is_file: e.is_file,
is_directory: e.is_directory,
is_symlink: e.is_symlink,
},
})
.collect()
})
}
}
pub fn import_map_deps(
import_map: &serde_json::Value,
) -> HashSet<JsrDepPackageReq> {

2
cli/args/flags.rs
View file

@ -1006,6 +1006,8 @@ impl Flags {
OtelConfig {
tracing_enabled: !disabled
&& otel_var("OTEL_DENO_TRACING").unwrap_or(default),
metrics_enabled: !disabled
&& otel_var("OTEL_DENO_METRICS").unwrap_or(default),
console: match std::env::var("OTEL_DENO_CONSOLE").as_deref() {
Ok(_) if disabled => OtelConsoleConfig::Ignore,
Ok("ignore") => OtelConsoleConfig::Ignore,

12
cli/args/import_map.rs
View file

@ -4,21 +4,21 @@ use deno_core::error::AnyError;
use deno_core::serde_json;
use deno_core::url::Url;
use crate::file_fetcher::FileFetcher;
use crate::file_fetcher::CliFileFetcher;
use crate::file_fetcher::TextDecodedFile;
pub async fn resolve_import_map_value_from_specifier(
specifier: &Url,
file_fetcher: &FileFetcher,
file_fetcher: &CliFileFetcher,
) -> Result<serde_json::Value, AnyError> {
if specifier.scheme() == "data" {
let data_url_text =
deno_graph::source::RawDataUrl::parse(specifier)?.decode()?;
Ok(serde_json::from_str(&data_url_text)?)
} else {
let file = file_fetcher
.fetch_bypass_permissions(specifier)
.await?
.into_text_decoded()?;
let file = TextDecodedFile::decode(
file_fetcher.fetch_bypass_permissions(specifier).await?,
)?;
Ok(serde_json::from_str(&file.source)?)
}
}

23
cli/args/lockfile.rs
View file

@ -12,12 +12,13 @@ use deno_core::parking_lot::MutexGuard;
use deno_core::serde_json;
use deno_lockfile::WorkspaceMemberConfig;
use deno_package_json::PackageJsonDepValue;
use deno_path_util::fs::atomic_write_file_with_retries;
use deno_runtime::deno_node::PackageJson;
use deno_semver::jsr::JsrDepPackageReq;
use crate::args::deno_json::import_map_deps;
use crate::cache;
use crate::util::fs::atomic_write_file_with_retries;
use crate::sys::CliSys;
use crate::Flags;
use crate::args::DenoSubcommand;
@ -35,6 +36,7 @@ pub struct CliLockfileReadFromPathOptions {
#[derive(Debug)]
pub struct CliLockfile {
sys: CliSys,
lockfile: Mutex<Lockfile>,
pub filename: PathBuf,
frozen: bool,
@ -91,8 +93,9 @@ impl CliLockfile {
// do an atomic write to reduce the chance of multiple deno
// processes corrupting the file
atomic_write_file_with_retries(
&self.sys,
&lockfile.filename,
bytes,
&bytes,
cache::CACHE_PERM,
)
.context("Failed writing lockfile.")?;
@ -101,6 +104,7 @@ impl CliLockfile {
}
pub fn discover(
sys: &CliSys,
flags: &Flags,
workspace: &Workspace,
maybe_external_import_map: Option<&serde_json::Value>,
@ -163,11 +167,14 @@ impl CliLockfile {
.unwrap_or(false)
});
let lockfile = Self::read_from_path(CliLockfileReadFromPathOptions {
file_path,
frozen,
skip_write: flags.internal.lockfile_skip_write,
})?;
let lockfile = Self::read_from_path(
sys,
CliLockfileReadFromPathOptions {
file_path,
frozen,
skip_write: flags.internal.lockfile_skip_write,
},
)?;
// initialize the lockfile with the workspace's configuration
let root_url = workspace.root_dir();
@ -223,6 +230,7 @@ impl CliLockfile {
}
pub fn read_from_path(
sys: &CliSys,
opts: CliLockfileReadFromPathOptions,
) -> Result<CliLockfile, AnyError> {
let lockfile = match std::fs::read_to_string(&opts.file_path) {
@ -241,6 +249,7 @@ impl CliLockfile {
}
};
Ok(CliLockfile {
sys: sys.clone(),
filename: lockfile.filename.clone(),
lockfile: Mutex::new(lockfile),
frozen: opts.frozen,

114
cli/args/mod.rs
View file

@ -9,6 +9,7 @@ mod package_json;
use deno_ast::MediaType;
use deno_ast::SourceMapOption;
use deno_cache_dir::file_fetcher::CacheSetting;
use deno_config::deno_json::NodeModulesDirMode;
use deno_config::workspace::CreateResolverOptions;
use deno_config::workspace::FolderConfigs;
@ -23,13 +24,14 @@ use deno_config::workspace::WorkspaceLintConfig;
use deno_config::workspace::WorkspaceResolver;
use deno_core::resolve_url_or_path;
use deno_graph::GraphKind;
use deno_lint::linter::LintConfig as DenoLintConfig;
use deno_npm::npm_rc::NpmRc;
use deno_npm::npm_rc::ResolvedNpmRc;
use deno_npm::resolution::ValidSerializedNpmResolutionSnapshot;
use deno_npm::NpmSystemInfo;
use deno_npm_cache::NpmCacheSetting;
use deno_path_util::normalize_path;
use deno_semver::npm::NpmPackageReqReference;
use deno_semver::StackString;
use deno_telemetry::OtelConfig;
use deno_telemetry::OtelRuntimeConfig;
use import_map::resolve_import_map_value_from_specifier;
@ -81,11 +83,12 @@ use std::num::NonZeroUsize;
use std::path::Path;
use std::path::PathBuf;
use std::sync::Arc;
use sys_traits::EnvHomeDir;
use thiserror::Error;
use crate::cache;
use crate::cache::DenoDirProvider;
use crate::file_fetcher::FileFetcher;
use crate::file_fetcher::CliFileFetcher;
use crate::sys::CliSys;
use crate::util::fs::canonicalize_path_maybe_not_exists;
use crate::version;
@ -217,52 +220,6 @@ pub fn ts_config_to_transpile_and_emit_options(
))
}
/// Indicates how cached source files should be handled.
#[derive(Debug, Clone, Eq, PartialEq)]
pub enum CacheSetting {
/// Only the cached files should be used. Any files not in the cache will
/// error. This is the equivalent of `--cached-only` in the CLI.
Only,
/// No cached source files should be used, and all files should be reloaded.
/// This is the equivalent of `--reload` in the CLI.
ReloadAll,
/// Only some cached resources should be used. This is the equivalent of
/// `--reload=jsr:@std/http/file-server` or
/// `--reload=jsr:@std/http/file-server,jsr:@std/assert/assert-equals`.
ReloadSome(Vec<String>),
/// The usability of a cached value is determined by analyzing the cached
/// headers and other metadata associated with a cached response, reloading
/// any cached "non-fresh" cached responses.
RespectHeaders,
/// The cached source files should be used for local modules. This is the
/// default behavior of the CLI.
Use,
}
impl CacheSetting {
pub fn as_npm_cache_setting(&self) -> NpmCacheSetting {
match self {
CacheSetting::Only => NpmCacheSetting::Only,
CacheSetting::ReloadAll => NpmCacheSetting::ReloadAll,
CacheSetting::ReloadSome(values) => {
if values.iter().any(|v| v == "npm:") {
NpmCacheSetting::ReloadAll
} else {
NpmCacheSetting::ReloadSome {
npm_package_names: values
.iter()
.filter_map(|v| v.strip_prefix("npm:"))
.map(|n| n.to_string())
.collect(),
}
}
}
CacheSetting::RespectHeaders => unreachable!(), // not supported
CacheSetting::Use => NpmCacheSetting::Use,
}
}
}
pub struct WorkspaceBenchOptions {
pub filter: Option<String>,
pub json: bool,
@ -616,7 +573,7 @@ fn discover_npmrc(
// TODO(bartlomieju): update to read both files - one in the project root and one and
// home dir and then merge them.
// 3. Try `.npmrc` in the user's home directory
if let Some(home_dir) = cache::home_dir() {
if let Some(home_dir) = crate::sys::CliSys::default().env_home_dir() {
match try_to_read_npmrc(&home_dir) {
Ok(Some((source, path))) => {
return try_to_parse_npmrc(source, &path).map(|r| (r, Some(path)));
@ -815,7 +772,9 @@ pub struct CliOptions {
}
impl CliOptions {
#[allow(clippy::too_many_arguments)]
pub fn new(
sys: &CliSys,
flags: Arc<Flags>,
initial_cwd: PathBuf,
maybe_lockfile: Option<Arc<CliLockfile>>,
@ -840,8 +799,10 @@ impl CliOptions {
}
let maybe_lockfile = maybe_lockfile.filter(|_| !force_global_cache);
let deno_dir_provider =
Arc::new(DenoDirProvider::new(flags.internal.cache_path.clone()));
let deno_dir_provider = Arc::new(DenoDirProvider::new(
sys.clone(),
flags.internal.cache_path.clone(),
));
let maybe_node_modules_folder = resolve_node_modules_folder(
&initial_cwd,
&flags,
@ -866,7 +827,7 @@ impl CliOptions {
})
}
pub fn from_flags(flags: Arc<Flags>) -> Result<Self, AnyError> {
pub fn from_flags(sys: &CliSys, flags: Arc<Flags>) -> Result<Self, AnyError> {
let initial_cwd =
std::env::current_dir().with_context(|| "Failed getting cwd.")?;
let maybe_vendor_override = flags.vendor.map(|v| match v {
@ -889,7 +850,6 @@ impl CliOptions {
log::debug!("package.json auto-discovery is disabled");
}
WorkspaceDiscoverOptions {
fs: Default::default(), // use real fs
deno_json_cache: None,
pkg_json_cache: Some(&node_resolver::PackageJsonThreadLocalCache),
workspace_cache: None,
@ -911,6 +871,7 @@ impl CliOptions {
ConfigFlag::Discover => {
if let Some(start_paths) = flags.config_path_args(&initial_cwd) {
WorkspaceDirectory::discover(
sys,
WorkspaceDiscoverStart::Paths(&start_paths),
&resolve_workspace_discover_options(),
)?
@ -921,6 +882,7 @@ impl CliOptions {
ConfigFlag::Path(path) => {
let config_path = normalize_path(initial_cwd.join(path));
WorkspaceDirectory::discover(
sys,
WorkspaceDiscoverStart::ConfigFile(&config_path),
&resolve_workspace_discover_options(),
)?
@ -959,6 +921,7 @@ impl CliOptions {
};
let maybe_lock_file = CliLockfile::discover(
sys,
&flags,
&start_dir.workspace,
external_import_map.as_ref().map(|(_, v)| v),
@ -967,6 +930,7 @@ impl CliOptions {
log::debug!("Finished config loading.");
Self::new(
sys,
flags,
initial_cwd,
maybe_lock_file.map(Arc::new),
@ -1037,24 +1001,24 @@ impl CliOptions {
// https://nodejs.org/api/process.html
match target.as_str() {
"aarch64-apple-darwin" => NpmSystemInfo {
os: "darwin".to_string(),
cpu: "arm64".to_string(),
os: "darwin".into(),
cpu: "arm64".into(),
},
"aarch64-unknown-linux-gnu" => NpmSystemInfo {
os: "linux".to_string(),
cpu: "arm64".to_string(),
os: "linux".into(),
cpu: "arm64".into(),
},
"x86_64-apple-darwin" => NpmSystemInfo {
os: "darwin".to_string(),
cpu: "x64".to_string(),
os: "darwin".into(),
cpu: "x64".into(),
},
"x86_64-unknown-linux-gnu" => NpmSystemInfo {
os: "linux".to_string(),
cpu: "x64".to_string(),
os: "linux".into(),
cpu: "x64".into(),
},
"x86_64-pc-windows-msvc" => NpmSystemInfo {
os: "win32".to_string(),
cpu: "x64".to_string(),
os: "win32".into(),
cpu: "x64".into(),
},
value => {
log::warn!(
@ -1091,7 +1055,7 @@ impl CliOptions {
pub async fn create_workspace_resolver(
&self,
file_fetcher: &FileFetcher,
file_fetcher: &CliFileFetcher,
pkg_json_dep_resolution: PackageJsonDepResolution,
) -> Result<WorkspaceResolver, AnyError> {
let overrode_no_import_map: bool = self
@ -1397,9 +1361,7 @@ impl CliOptions {
Ok(result)
}
pub fn resolve_deno_lint_config(
&self,
) -> Result<deno_lint::linter::LintConfig, AnyError> {
pub fn resolve_deno_lint_config(&self) -> Result<DenoLintConfig, AnyError> {
let ts_config_result =
self.resolve_ts_config_for_emit(TsConfigType::Emit)?;
@ -1408,11 +1370,11 @@ impl CliOptions {
ts_config_result.ts_config,
)?;
Ok(deno_lint::linter::LintConfig {
Ok(DenoLintConfig {
default_jsx_factory: (!transpile_options.jsx_automatic)
.then(|| transpile_options.jsx_factory.clone()),
.then_some(transpile_options.jsx_factory),
default_jsx_fragment_factory: (!transpile_options.jsx_automatic)
.then(|| transpile_options.jsx_fragment_factory.clone()),
.then_some(transpile_options.jsx_fragment_factory),
})
}
@ -1993,15 +1955,17 @@ pub fn has_flag_env_var(name: &str) -> bool {
pub fn npm_pkg_req_ref_to_binary_command(
req_ref: &NpmPackageReqReference,
) -> String {
let binary_name = req_ref.sub_path().unwrap_or(req_ref.req().name.as_str());
binary_name.to_string()
req_ref
.sub_path()
.map(|s| s.to_string())
.unwrap_or_else(|| req_ref.req().name.to_string())
}
pub fn config_to_deno_graph_workspace_member(
config: &ConfigFile,
) -> Result<deno_graph::WorkspaceMember, AnyError> {
let name = match &config.json.name {
Some(name) => name.clone(),
let name: StackString = match &config.json.name {
Some(name) => name.as_str().into(),
None => bail!("Missing 'name' field in config file."),
};
let version = match &config.json.version {

31
cli/args/package_json.rs
View file

@ -11,19 +11,20 @@ use deno_package_json::PackageJsonDepValueParseError;
use deno_package_json::PackageJsonDepWorkspaceReq;
use deno_semver::npm::NpmPackageReqReference;
use deno_semver::package::PackageReq;
use deno_semver::StackString;
use deno_semver::VersionReq;
use thiserror::Error;
#[derive(Debug)]
pub struct InstallNpmRemotePkg {
pub alias: Option<String>,
pub alias: Option<StackString>,
pub base_dir: PathBuf,
pub req: PackageReq,
}
#[derive(Debug)]
pub struct InstallNpmWorkspacePkg {
pub alias: Option<String>,
pub alias: Option<StackString>,
pub target_dir: PathBuf,
}
@ -31,7 +32,7 @@ pub struct InstallNpmWorkspacePkg {
#[error("Failed to install '{}'\n at {}", alias, location)]
pub struct PackageJsonDepValueParseWithLocationError {
pub location: Url,
pub alias: String,
pub alias: StackString,
#[source]
pub source: PackageJsonDepValueParseError,
}
@ -100,10 +101,8 @@ impl NpmInstallDepsProvider {
let mut pkg_pkgs = Vec::with_capacity(
deps.dependencies.len() + deps.dev_dependencies.len(),
);
for (alias, dep) in deps
.dependencies
.into_iter()
.chain(deps.dev_dependencies.into_iter())
for (alias, dep) in
deps.dependencies.iter().chain(deps.dev_dependencies.iter())
{
let dep = match dep {
Ok(dep) => dep,
@ -111,8 +110,8 @@ impl NpmInstallDepsProvider {
pkg_json_dep_errors.push(
PackageJsonDepValueParseWithLocationError {
location: pkg_json.specifier(),
alias,
source: err,
alias: alias.clone(),
source: err.clone(),
},
);
continue;
@ -121,28 +120,28 @@ impl NpmInstallDepsProvider {
match dep {
PackageJsonDepValue::Req(pkg_req) => {
let workspace_pkg = workspace_npm_pkgs.iter().find(|pkg| {
pkg.matches_req(&pkg_req)
pkg.matches_req(pkg_req)
// do not resolve to the current package
&& pkg.pkg_json.path != pkg_json.path
});
if let Some(pkg) = workspace_pkg {
workspace_pkgs.push(InstallNpmWorkspacePkg {
alias: Some(alias),
alias: Some(alias.clone()),
target_dir: pkg.pkg_json.dir_path().to_path_buf(),
});
} else {
pkg_pkgs.push(InstallNpmRemotePkg {
alias: Some(alias),
alias: Some(alias.clone()),
base_dir: pkg_json.dir_path().to_path_buf(),
req: pkg_req,
req: pkg_req.clone(),
});
}
}
PackageJsonDepValue::Workspace(workspace_version_req) => {
let version_req = match workspace_version_req {
PackageJsonDepWorkspaceReq::VersionReq(version_req) => {
version_req
version_req.clone()
}
PackageJsonDepWorkspaceReq::Tilde
| PackageJsonDepWorkspaceReq::Caret => {
@ -150,10 +149,10 @@ impl NpmInstallDepsProvider {
}
};
if let Some(pkg) = workspace_npm_pkgs.iter().find(|pkg| {
pkg.matches_name_and_version_req(&alias, &version_req)
pkg.matches_name_and_version_req(alias, &version_req)
}) {
workspace_pkgs.push(InstallNpmWorkspacePkg {
alias: Some(alias),
alias: Some(alias.clone()),
target_dir: pkg.pkg_json.dir_path().to_path_buf(),
});
}

369
cli/auth_tokens.rs
View file

@ -1,369 +0,0 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use base64::prelude::BASE64_STANDARD;
use base64::Engine;
use deno_core::ModuleSpecifier;
use log::debug;
use log::error;
use std::borrow::Cow;
use std::fmt;
use std::net::IpAddr;
use std::net::Ipv4Addr;
use std::net::Ipv6Addr;
use std::net::SocketAddr;
use std::str::FromStr;
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum AuthTokenData {
Bearer(String),
Basic { username: String, password: String },
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct AuthToken {
host: AuthDomain,
token: AuthTokenData,
}
impl fmt::Display for AuthToken {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match &self.token {
AuthTokenData::Bearer(token) => write!(f, "Bearer {token}"),
AuthTokenData::Basic { username, password } => {
let credentials = format!("{username}:{password}");
write!(f, "Basic {}", BASE64_STANDARD.encode(credentials))
}
}
}
}
/// A structure which contains bearer tokens that can be used when sending
/// requests to websites, intended to authorize access to private resources
/// such as remote modules.
#[derive(Debug, Clone)]
pub struct AuthTokens(Vec<AuthToken>);
/// An authorization domain, either an exact or suffix match.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum AuthDomain {
Ip(IpAddr),
IpPort(SocketAddr),
/// Suffix match, no dot. May include a port.
Suffix(Cow<'static, str>),
}
impl<T: ToString> From<T> for AuthDomain {
fn from(value: T) -> Self {
let s = value.to_string().to_lowercase();
if let Ok(ip) = SocketAddr::from_str(&s) {
return AuthDomain::IpPort(ip);
};
if s.starts_with('[') && s.ends_with(']') {
if let Ok(ip) = Ipv6Addr::from_str(&s[1..s.len() - 1]) {
return AuthDomain::Ip(ip.into());
}
} else if let Ok(ip) = Ipv4Addr::from_str(&s) {
return AuthDomain::Ip(ip.into());
}
if let Some(s) = s.strip_prefix('.') {
AuthDomain::Suffix(Cow::Owned(s.to_owned()))
} else {
AuthDomain::Suffix(Cow::Owned(s))
}
}
}
impl AuthDomain {
pub fn matches(&self, specifier: &ModuleSpecifier) -> bool {
let Some(host) = specifier.host_str() else {
return false;
};
match *self {
Self::Ip(ip) => {
let AuthDomain::Ip(parsed) = AuthDomain::from(host) else {
return false;
};
ip == parsed && specifier.port().is_none()
}
Self::IpPort(ip) => {
let AuthDomain::Ip(parsed) = AuthDomain::from(host) else {
return false;
};
ip.ip() == parsed && specifier.port() == Some(ip.port())
}
Self::Suffix(ref suffix) => {
let hostname = if let Some(port) = specifier.port() {
Cow::Owned(format!("{}:{}", host, port))
} else {
Cow::Borrowed(host)
};
if suffix.len() == hostname.len() {
return suffix == &hostname;
}
// If it's a suffix match, ensure a dot
if hostname.ends_with(suffix.as_ref())
&& hostname.ends_with(&format!(".{suffix}"))
{
return true;
}
false
}
}
}
}
impl AuthTokens {
/// Create a new set of tokens based on the provided string. It is intended
/// that the string be the value of an environment variable and the string is
/// parsed for token values. The string is expected to be a semi-colon
/// separated string, where each value is `{token}@{hostname}`.
pub fn new(maybe_tokens_str: Option<String>) -> Self {
let mut tokens = Vec::new();
if let Some(tokens_str) = maybe_tokens_str {
for token_str in tokens_str.trim().split(';') {
if token_str.contains('@') {
let mut iter = token_str.rsplitn(2, '@');
let host = AuthDomain::from(iter.next().unwrap());
let token = iter.next().unwrap();
if token.contains(':') {
let mut iter = token.rsplitn(2, ':');
let password = iter.next().unwrap().to_owned();
let username = iter.next().unwrap().to_owned();
tokens.push(AuthToken {
host,
token: AuthTokenData::Basic { username, password },
});
} else {
tokens.push(AuthToken {
host,
token: AuthTokenData::Bearer(token.to_string()),
});
}
} else {
error!("Badly formed auth token discarded.");
}
}
debug!("Parsed {} auth token(s).", tokens.len());
}
Self(tokens)
}
/// Attempt to match the provided specifier to the tokens in the set. The
/// matching occurs from the right of the hostname plus port, irrespective of
/// scheme. For example `https://www.deno.land:8080/` would match a token
/// with a host value of `deno.land:8080` but not match `www.deno.land`. The
/// matching is case insensitive.
pub fn get(&self, specifier: &ModuleSpecifier) -> Option<AuthToken> {
self.0.iter().find_map(|t| {
if t.host.matches(specifier) {
Some(t.clone())
} else {
None
}
})
}
}
#[cfg(test)]
mod tests {
use super::*;
use deno_core::resolve_url;
#[test]
fn test_auth_token() {
let auth_tokens = AuthTokens::new(Some("abc123@deno.land".to_string()));
let fixture = resolve_url("https://deno.land/x/mod.ts").unwrap();
assert_eq!(
auth_tokens.get(&fixture).unwrap().to_string(),
"Bearer abc123"
);
let fixture = resolve_url("https://www.deno.land/x/mod.ts").unwrap();
assert_eq!(
auth_tokens.get(&fixture).unwrap().to_string(),
"Bearer abc123".to_string()
);
let fixture = resolve_url("http://127.0.0.1:8080/x/mod.ts").unwrap();
assert_eq!(auth_tokens.get(&fixture), None);
let fixture =
resolve_url("https://deno.land.example.com/x/mod.ts").unwrap();
assert_eq!(auth_tokens.get(&fixture), None);
let fixture = resolve_url("https://deno.land:8080/x/mod.ts").unwrap();
assert_eq!(auth_tokens.get(&fixture), None);
}
#[test]
fn test_auth_tokens_multiple() {
let auth_tokens =
AuthTokens::new(Some("abc123@deno.land;def456@example.com".to_string()));
let fixture = resolve_url("https://deno.land/x/mod.ts").unwrap();
assert_eq!(
auth_tokens.get(&fixture).unwrap().to_string(),
"Bearer abc123".to_string()
);
let fixture = resolve_url("http://example.com/a/file.ts").unwrap();
assert_eq!(
auth_tokens.get(&fixture).unwrap().to_string(),
"Bearer def456".to_string()
);
}
#[test]
fn test_auth_tokens_space() {
let auth_tokens = AuthTokens::new(Some(
" abc123@deno.land;def456@example.com\t".to_string(),
));
let fixture = resolve_url("https://deno.land/x/mod.ts").unwrap();
assert_eq!(
auth_tokens.get(&fixture).unwrap().to_string(),
"Bearer abc123".to_string()
);
let fixture = resolve_url("http://example.com/a/file.ts").unwrap();
assert_eq!(
auth_tokens.get(&fixture).unwrap().to_string(),
"Bearer def456".to_string()
);
}
#[test]
fn test_auth_tokens_newline() {
let auth_tokens = AuthTokens::new(Some(
"\nabc123@deno.land;def456@example.com\n".to_string(),
));
let fixture = resolve_url("https://deno.land/x/mod.ts").unwrap();
assert_eq!(
auth_tokens.get(&fixture).unwrap().to_string(),
"Bearer abc123".to_string()
);
let fixture = resolve_url("http://example.com/a/file.ts").unwrap();
assert_eq!(
auth_tokens.get(&fixture).unwrap().to_string(),
"Bearer def456".to_string()
);
}
#[test]
fn test_auth_tokens_port() {
let auth_tokens =
AuthTokens::new(Some("abc123@deno.land:8080".to_string()));
let fixture = resolve_url("https://deno.land/x/mod.ts").unwrap();
assert_eq!(auth_tokens.get(&fixture), None);
let fixture = resolve_url("http://deno.land:8080/x/mod.ts").unwrap();
assert_eq!(
auth_tokens.get(&fixture).unwrap().to_string(),
"Bearer abc123".to_string()
);
}
#[test]
fn test_auth_tokens_contain_at() {
let auth_tokens = AuthTokens::new(Some("abc@123@deno.land".to_string()));
let fixture = resolve_url("https://deno.land/x/mod.ts").unwrap();
assert_eq!(
auth_tokens.get(&fixture).unwrap().to_string(),
"Bearer abc@123".to_string()
);
}
#[test]
fn test_auth_token_basic() {
let auth_tokens = AuthTokens::new(Some("abc:123@deno.land".to_string()));
let fixture = resolve_url("https://deno.land/x/mod.ts").unwrap();
assert_eq!(
auth_tokens.get(&fixture).unwrap().to_string(),
"Basic YWJjOjEyMw=="
);
let fixture = resolve_url("https://www.deno.land/x/mod.ts").unwrap();
assert_eq!(
auth_tokens.get(&fixture).unwrap().to_string(),
"Basic YWJjOjEyMw==".to_string()
);
let fixture = resolve_url("http://127.0.0.1:8080/x/mod.ts").unwrap();
assert_eq!(auth_tokens.get(&fixture), None);
let fixture =
resolve_url("https://deno.land.example.com/x/mod.ts").unwrap();
assert_eq!(auth_tokens.get(&fixture), None);
let fixture = resolve_url("https://deno.land:8080/x/mod.ts").unwrap();
assert_eq!(auth_tokens.get(&fixture), None);
}
#[test]
fn test_parse_ip() {
let ip = AuthDomain::from("[2001:db8:a::123]");
assert_eq!("Ip(2001:db8:a::123)", format!("{ip:?}"));
let ip = AuthDomain::from("[2001:db8:a::123]:8080");
assert_eq!("IpPort([2001:db8:a::123]:8080)", format!("{ip:?}"));
let ip = AuthDomain::from("1.1.1.1");
assert_eq!("Ip(1.1.1.1)", format!("{ip:?}"));
}
#[test]
fn test_case_insensitive() {
let domain = AuthDomain::from("EXAMPLE.com");
assert!(
domain.matches(&ModuleSpecifier::parse("http://example.com").unwrap())
);
assert!(
domain.matches(&ModuleSpecifier::parse("http://example.COM").unwrap())
);
}
#[test]
fn test_matches() {
let candidates = [
"example.com",
"www.example.com",
"1.1.1.1",
"[2001:db8:a::123]",
// These will never match
"example.com.evil.com",
"1.1.1.1.evil.com",
"notexample.com",
"www.notexample.com",
];
let domains = [
("example.com", vec!["example.com", "www.example.com"]),
(".example.com", vec!["example.com", "www.example.com"]),
("www.example.com", vec!["www.example.com"]),
("1.1.1.1", vec!["1.1.1.1"]),
("[2001:db8:a::123]", vec!["[2001:db8:a::123]"]),
];
let url = |c: &str| ModuleSpecifier::parse(&format!("http://{c}")).unwrap();
let url_port =
|c: &str| ModuleSpecifier::parse(&format!("http://{c}:8080")).unwrap();
// Generate each candidate with and without a port
let candidates = candidates
.into_iter()
.flat_map(|c| [url(c), url_port(c)])
.collect::<Vec<_>>();
for (domain, expected_domain) in domains {
// Test without a port -- all candidates return without a port
let auth_domain = AuthDomain::from(domain);
let actual = candidates
.iter()
.filter(|c| auth_domain.matches(c))
.cloned()
.collect::<Vec<_>>();
let expected = expected_domain.iter().map(|u| url(u)).collect::<Vec<_>>();
assert_eq!(actual, expected);
// Test with a port, all candidates return with a port
let auth_domain = AuthDomain::from(&format!("{domain}:8080"));
let actual = candidates
.iter()
.filter(|c| auth_domain.matches(c))
.cloned()
.collect::<Vec<_>>();
let expected = expected_domain
.iter()
.map(|u| url_port(u))
.collect::<Vec<_>>();
assert_eq!(actual, expected);
}
}
}

166
cli/cache/deno_dir.rs vendored
View file

@ -1,7 +1,10 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use deno_cache_dir::DenoDirResolutionError;
use once_cell::sync::OnceCell;
use crate::sys::CliSys;
use super::DiskCache;
use std::env;
@ -10,24 +13,37 @@ use std::path::PathBuf;
/// Lazily creates the deno dir which might be useful in scenarios
/// where functionality wants to continue if the DENO_DIR can't be created.
pub struct DenoDirProvider {
sys: CliSys,
maybe_custom_root: Option<PathBuf>,
deno_dir: OnceCell<std::io::Result<DenoDir>>,
deno_dir: OnceCell<Result<DenoDir, DenoDirResolutionError>>,
}
impl DenoDirProvider {
pub fn new(maybe_custom_root: Option<PathBuf>) -> Self {
pub fn new(sys: CliSys, maybe_custom_root: Option<PathBuf>) -> Self {
Self {
sys,
maybe_custom_root,
deno_dir: Default::default(),
}
}
pub fn get_or_create(&self) -> Result<&DenoDir, std::io::Error> {
pub fn get_or_create(&self) -> Result<&DenoDir, DenoDirResolutionError> {
self
.deno_dir
.get_or_init(|| DenoDir::new(self.maybe_custom_root.clone()))
.get_or_init(|| {
DenoDir::new(self.sys.clone(), self.maybe_custom_root.clone())
})
.as_ref()
.map_err(|err| std::io::Error::new(err.kind(), err.to_string()))
.map_err(|err| match err {
DenoDirResolutionError::NoCacheOrHomeDir => {
DenoDirResolutionError::NoCacheOrHomeDir
}
DenoDirResolutionError::FailedCwd { source } => {
DenoDirResolutionError::FailedCwd {
source: std::io::Error::new(source.kind(), source.to_string()),
}
}
})
}
}
@ -42,33 +58,20 @@ pub struct DenoDir {
}
impl DenoDir {
pub fn new(maybe_custom_root: Option<PathBuf>) -> std::io::Result<Self> {
let maybe_custom_root =
maybe_custom_root.or_else(|| env::var("DENO_DIR").map(String::into).ok());
let root: PathBuf = if let Some(root) = maybe_custom_root {
root
} else if let Some(cache_dir) = dirs::cache_dir() {
// We use the OS cache dir because all files deno writes are cache files
// Once that changes we need to start using different roots if DENO_DIR
// is not set, and keep a single one if it is.
cache_dir.join("deno")
} else if let Some(home_dir) = dirs::home_dir() {
// fallback path
home_dir.join(".deno")
} else {
panic!("Could not set the Deno root directory")
};
let root = if root.is_absolute() {
root
} else {
std::env::current_dir()?.join(root)
};
pub fn new(
sys: CliSys,
maybe_custom_root: Option<PathBuf>,
) -> Result<Self, deno_cache_dir::DenoDirResolutionError> {
let root = deno_cache_dir::resolve_deno_dir(
&sys_traits::impls::RealSys,
maybe_custom_root,
)?;
assert!(root.is_absolute());
let gen_path = root.join("gen");
let deno_dir = Self {
root,
gen_cache: DiskCache::new(&gen_path),
gen_cache: DiskCache::new(sys, &gen_path),
};
Ok(deno_dir)
@ -166,112 +169,3 @@ impl DenoDir {
self.root.join("dl")
}
}
/// To avoid the poorly managed dirs crate
#[cfg(not(windows))]
pub mod dirs {
use std::path::PathBuf;
pub fn cache_dir() -> Option<PathBuf> {
if cfg!(target_os = "macos") {
home_dir().map(|h| h.join("Library/Caches"))
} else {
std::env::var_os("XDG_CACHE_HOME")
.map(PathBuf::from)
.or_else(|| home_dir().map(|h| h.join(".cache")))
}
}
pub fn home_dir() -> Option<PathBuf> {
std::env::var_os("HOME")
.and_then(|h| if h.is_empty() { None } else { Some(h) })
.or_else(|| {
// TODO(bartlomieju):
#[allow(clippy::undocumented_unsafe_blocks)]
unsafe {
fallback()
}
})
.map(PathBuf::from)
}
// This piece of code is taken from the deprecated home_dir() function in Rust's standard library: https://github.com/rust-lang/rust/blob/master/src/libstd/sys/unix/os.rs#L579
// The same code is used by the dirs crate
unsafe fn fallback() -> Option<std::ffi::OsString> {
let amt = match libc::sysconf(libc::_SC_GETPW_R_SIZE_MAX) {
n if n < 0 => 512_usize,
n => n as usize,
};
let mut buf = Vec::with_capacity(amt);
let mut passwd: libc::passwd = std::mem::zeroed();
let mut result = std::ptr::null_mut();
match libc::getpwuid_r(
libc::getuid(),
&mut passwd,
buf.as_mut_ptr(),
buf.capacity(),
&mut result,
) {
0 if !result.is_null() => {
let ptr = passwd.pw_dir as *const _;
let bytes = std::ffi::CStr::from_ptr(ptr).to_bytes().to_vec();
Some(std::os::unix::ffi::OsStringExt::from_vec(bytes))
}
_ => None,
}
}
}
/// To avoid the poorly managed dirs crate
// Copied from
// https://github.com/dirs-dev/dirs-sys-rs/blob/ec7cee0b3e8685573d847f0a0f60aae3d9e07fa2/src/lib.rs#L140-L164
// MIT license. Copyright (c) 2018-2019 dirs-rs contributors
#[cfg(windows)]
pub mod dirs {
use std::ffi::OsString;
use std::os::windows::ffi::OsStringExt;
use std::path::PathBuf;
use winapi::shared::winerror;
use winapi::um::combaseapi;
use winapi::um::knownfolders;
use winapi::um::shlobj;
use winapi::um::shtypes;
use winapi::um::winbase;
use winapi::um::winnt;
fn known_folder(folder_id: shtypes::REFKNOWNFOLDERID) -> Option<PathBuf> {
// SAFETY: winapi calls
unsafe {
let mut path_ptr: winnt::PWSTR = std::ptr::null_mut();
let result = shlobj::SHGetKnownFolderPath(
folder_id,
0,
std::ptr::null_mut(),
&mut path_ptr,
);
if result == winerror::S_OK {
let len = winbase::lstrlenW(path_ptr) as usize;
let path = std::slice::from_raw_parts(path_ptr, len);
let ostr: OsString = OsStringExt::from_wide(path);
combaseapi::CoTaskMemFree(path_ptr as *mut winapi::ctypes::c_void);
Some(PathBuf::from(ostr))
} else {
None
}
}
}
pub fn cache_dir() -> Option<PathBuf> {
known_folder(&knownfolders::FOLDERID_LocalAppData)
}
pub fn home_dir() -> Option<PathBuf> {
if let Some(userprofile) = std::env::var_os("USERPROFILE") {
if !userprofile.is_empty() {
return Some(PathBuf::from(userprofile));
}
}
known_folder(&knownfolders::FOLDERID_Profile)
}
}

18
cli/cache/disk_cache.rs vendored
View file

@ -1,11 +1,13 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use crate::sys::CliSys;
use super::CACHE_PERM;
use crate::util::fs::atomic_write_file_with_retries;
use deno_cache_dir::url_to_filename;
use deno_core::url::Host;
use deno_core::url::Url;
use deno_path_util::fs::atomic_write_file_with_retries;
use std::ffi::OsStr;
use std::fs;
use std::path::Component;
@ -16,14 +18,16 @@ use std::str;
#[derive(Debug, Clone)]
pub struct DiskCache {
sys: CliSys,
pub location: PathBuf,
}
impl DiskCache {
/// `location` must be an absolute path.
pub fn new(location: &Path) -> Self {
pub fn new(sys: CliSys, location: &Path) -> Self {
assert!(location.is_absolute());
Self {
sys,
location: location.to_owned(),
}
}
@ -120,7 +124,7 @@ impl DiskCache {
pub fn set(&self, filename: &Path, data: &[u8]) -> std::io::Result<()> {
let path = self.location.join(filename);
atomic_write_file_with_retries(&path, data, CACHE_PERM)
atomic_write_file_with_retries(&self.sys, &path, data, CACHE_PERM)
}
}
@ -133,7 +137,7 @@ mod tests {
fn test_set_get_cache_file() {
let temp_dir = TempDir::new();
let sub_dir = temp_dir.path().join("sub_dir");
let cache = DiskCache::new(&sub_dir.to_path_buf());
let cache = DiskCache::new(CliSys::default(), &sub_dir.to_path_buf());
let path = PathBuf::from("foo/bar.txt");
cache.set(&path, b"hello").unwrap();
assert_eq!(cache.get(&path).unwrap(), b"hello");
@ -147,7 +151,7 @@ mod tests {
PathBuf::from("/deno_dir/")
};
let cache = DiskCache::new(&cache_location);
let cache = DiskCache::new(CliSys::default(), &cache_location);
let mut test_cases = vec![
(
@ -203,7 +207,7 @@ mod tests {
} else {
"/foo"
};
let cache = DiskCache::new(&PathBuf::from(p));
let cache = DiskCache::new(CliSys::default(), &PathBuf::from(p));
let mut test_cases = vec![
(
@ -251,7 +255,7 @@ mod tests {
PathBuf::from("/deno_dir/")
};
let cache = DiskCache::new(&cache_location);
let cache = DiskCache::new(CliSys::default(), &cache_location);
let mut test_cases = vec!["unknown://localhost/test.ts"];

5
cli/cache/emit.rs vendored
View file

@ -159,12 +159,15 @@ impl EmitFileSerializer {
mod test {
use test_util::TempDir;
use crate::sys::CliSys;
use super::*;
#[test]
pub fn emit_cache_general_use() {
let temp_dir = TempDir::new();
let disk_cache = DiskCache::new(temp_dir.path().as_path());
let disk_cache =
DiskCache::new(CliSys::default(), temp_dir.path().as_path());
let cache = EmitCache {
disk_cache: disk_cache.clone(),
file_serializer: EmitFileSerializer {

231
cli/cache/mod.rs vendored
View file

@ -1,18 +1,17 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use crate::args::jsr_url;
use crate::args::CacheSetting;
use crate::errors::get_error_class_name;
use crate::file_fetcher::CliFetchNoFollowErrorKind;
use crate::file_fetcher::CliFileFetcher;
use crate::file_fetcher::FetchNoFollowOptions;
use crate::file_fetcher::FetchOptions;
use crate::file_fetcher::FetchPermissionsOptionRef;
use crate::file_fetcher::FileFetcher;
use crate::file_fetcher::FileOrRedirect;
use crate::util::fs::atomic_write_file_with_retries;
use crate::util::fs::atomic_write_file_with_retries_and_fs;
use crate::util::fs::AtomicWriteFileFsAdapter;
use crate::sys::CliSys;
use deno_ast::MediaType;
use deno_cache_dir::file_fetcher::CacheSetting;
use deno_cache_dir::file_fetcher::FetchNoFollowErrorKind;
use deno_cache_dir::file_fetcher::FileOrRedirect;
use deno_core::error::AnyError;
use deno_core::futures;
use deno_core::futures::FutureExt;
use deno_core::ModuleSpecifier;
@ -20,15 +19,11 @@ use deno_graph::source::CacheInfo;
use deno_graph::source::LoadFuture;
use deno_graph::source::LoadResponse;
use deno_graph::source::Loader;
use deno_runtime::deno_fs;
use deno_runtime::deno_permissions::PermissionsContainer;
use node_resolver::InNpmPackageChecker;
use std::borrow::Cow;
use std::collections::HashMap;
use std::path::Path;
use std::path::PathBuf;
use std::sync::Arc;
use std::time::SystemTime;
mod cache_db;
mod caches;
@ -49,7 +44,6 @@ pub use caches::Caches;
pub use check::TypeCheckCache;
pub use code_cache::CodeCache;
pub use common::FastInsecureHasher;
pub use deno_dir::dirs::home_dir;
pub use deno_dir::DenoDir;
pub use deno_dir::DenoDirProvider;
pub use disk_cache::DiskCache;
@ -62,121 +56,11 @@ pub use parsed_source::LazyGraphSourceParser;
pub use parsed_source::ParsedSourceCache;
/// Permissions used to save a file in the disk caches.
pub const CACHE_PERM: u32 = 0o644;
pub use deno_cache_dir::CACHE_PERM;
#[derive(Debug, Clone)]
pub struct RealDenoCacheEnv;
impl deno_cache_dir::DenoCacheEnv for RealDenoCacheEnv {
fn read_file_bytes(
&self,
path: &Path,
) -> std::io::Result<Cow<'static, [u8]>> {
std::fs::read(path).map(Cow::Owned)
}
fn atomic_write_file(
&self,
path: &Path,
bytes: &[u8],
) -> std::io::Result<()> {
atomic_write_file_with_retries(path, bytes, CACHE_PERM)
}
fn canonicalize_path(&self, path: &Path) -> std::io::Result<PathBuf> {
crate::util::fs::canonicalize_path(path)
}
fn create_dir_all(&self, path: &Path) -> std::io::Result<()> {
std::fs::create_dir_all(path)
}
fn modified(&self, path: &Path) -> std::io::Result<Option<SystemTime>> {
match std::fs::metadata(path) {
Ok(metadata) => Ok(Some(
metadata.modified().unwrap_or_else(|_| SystemTime::now()),
)),
Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(None),
Err(err) => Err(err),
}
}
fn is_file(&self, path: &Path) -> bool {
path.is_file()
}
fn time_now(&self) -> SystemTime {
SystemTime::now()
}
}
#[derive(Debug, Clone)]
pub struct DenoCacheEnvFsAdapter<'a>(
pub &'a dyn deno_runtime::deno_fs::FileSystem,
);
impl<'a> deno_cache_dir::DenoCacheEnv for DenoCacheEnvFsAdapter<'a> {
fn read_file_bytes(
&self,
path: &Path,
) -> std::io::Result<Cow<'static, [u8]>> {
self
.0
.read_file_sync(path, None)
.map_err(|err| err.into_io_error())
}
fn atomic_write_file(
&self,
path: &Path,
bytes: &[u8],
) -> std::io::Result<()> {
atomic_write_file_with_retries_and_fs(
&AtomicWriteFileFsAdapter {
fs: self.0,
write_mode: CACHE_PERM,
},
path,
bytes,
)
}
fn canonicalize_path(&self, path: &Path) -> std::io::Result<PathBuf> {
self.0.realpath_sync(path).map_err(|e| e.into_io_error())
}
fn create_dir_all(&self, path: &Path) -> std::io::Result<()> {
self
.0
.mkdir_sync(path, true, None)
.map_err(|e| e.into_io_error())
}
fn modified(&self, path: &Path) -> std::io::Result<Option<SystemTime>> {
self
.0
.stat_sync(path)
.map(|stat| {
stat
.mtime
.map(|ts| SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(ts))
})
.map_err(|e| e.into_io_error())
}
fn is_file(&self, path: &Path) -> bool {
self.0.is_file_sync(path)
}
fn time_now(&self) -> SystemTime {
SystemTime::now()
}
}
pub type GlobalHttpCache = deno_cache_dir::GlobalHttpCache<RealDenoCacheEnv>;
pub type LocalHttpCache = deno_cache_dir::LocalHttpCache<RealDenoCacheEnv>;
pub type LocalLspHttpCache =
deno_cache_dir::LocalLspHttpCache<RealDenoCacheEnv>;
pub type GlobalHttpCache = deno_cache_dir::GlobalHttpCache<CliSys>;
pub type LocalHttpCache = deno_cache_dir::LocalHttpCache<CliSys>;
pub type LocalLspHttpCache = deno_cache_dir::LocalLspHttpCache<CliSys>;
pub use deno_cache_dir::HttpCache;
pub struct FetchCacherOptions {
@ -190,31 +74,31 @@ pub struct FetchCacherOptions {
/// a concise interface to the DENO_DIR when building module graphs.
pub struct FetchCacher {
pub file_header_overrides: HashMap<ModuleSpecifier, HashMap<String, String>>,
file_fetcher: Arc<FileFetcher>,
fs: Arc<dyn deno_fs::FileSystem>,
file_fetcher: Arc<CliFileFetcher>,
global_http_cache: Arc<GlobalHttpCache>,
in_npm_pkg_checker: Arc<dyn InNpmPackageChecker>,
module_info_cache: Arc<ModuleInfoCache>,
permissions: PermissionsContainer,
sys: CliSys,
is_deno_publish: bool,
cache_info_enabled: bool,
}
impl FetchCacher {
pub fn new(
file_fetcher: Arc<FileFetcher>,
fs: Arc<dyn deno_fs::FileSystem>,
file_fetcher: Arc<CliFileFetcher>,
global_http_cache: Arc<GlobalHttpCache>,
in_npm_pkg_checker: Arc<dyn InNpmPackageChecker>,
module_info_cache: Arc<ModuleInfoCache>,
sys: CliSys,
options: FetchCacherOptions,
) -> Self {
Self {
file_fetcher,
fs,
global_http_cache,
in_npm_pkg_checker,
module_info_cache,
sys,
file_header_overrides: options.file_header_overrides,
permissions: options.permissions,
is_deno_publish: options.is_deno_publish,
@ -276,9 +160,8 @@ impl Loader for FetchCacher {
// symlinked to `/my-project-2/node_modules`), so first we checked if the path
// is in a node_modules dir to avoid needlessly canonicalizing, then now compare
// against the canonicalized specifier.
let specifier = crate::node::resolve_specifier_into_node_modules(
specifier,
self.fs.as_ref(),
let specifier = node_resolver::resolve_specifier_into_node_modules(
&self.sys, specifier,
);
if self.in_npm_pkg_checker.in_npm_package(&specifier) {
return Box::pin(futures::future::ready(Ok(Some(
@ -320,18 +203,18 @@ impl Loader for FetchCacher {
LoaderCacheSetting::Only => Some(CacheSetting::Only),
};
file_fetcher
.fetch_no_follow_with_options(FetchNoFollowOptions {
fetch_options: FetchOptions {
specifier: &specifier,
permissions: if is_statically_analyzable {
FetchPermissionsOptionRef::StaticContainer(&permissions)
} else {
FetchPermissionsOptionRef::DynamicContainer(&permissions)
},
maybe_auth: None,
maybe_accept: None,
maybe_cache_setting: maybe_cache_setting.as_ref(),
},
.fetch_no_follow(
&specifier,
FetchPermissionsOptionRef::Restricted(&permissions,
if is_statically_analyzable {
deno_runtime::deno_permissions::CheckSpecifierKind::Static
} else {
deno_runtime::deno_permissions::CheckSpecifierKind::Dynamic
}),
FetchNoFollowOptions {
maybe_auth: None,
maybe_accept: None,
maybe_cache_setting: maybe_cache_setting.as_ref(),
maybe_checksum: options.maybe_checksum.as_ref(),
})
.await
@ -348,7 +231,7 @@ impl Loader for FetchCacher {
(None, None) => None,
};
Ok(Some(LoadResponse::Module {
specifier: file.specifier,
specifier: file.url,
maybe_headers,
content: file.source,
}))
@ -361,18 +244,46 @@ impl Loader for FetchCacher {
}
})
.unwrap_or_else(|err| {
if let Some(io_err) = err.downcast_ref::<std::io::Error>() {
if io_err.kind() == std::io::ErrorKind::NotFound {
return Ok(None);
} else {
return Err(err);
}
}
let error_class_name = get_error_class_name(&err);
match error_class_name {
"NotFound" => Ok(None),
"NotCached" if options.cache_setting == LoaderCacheSetting::Only => Ok(None),
_ => Err(err),
let err = err.into_kind();
match err {
CliFetchNoFollowErrorKind::FetchNoFollow(err) => {
let err = err.into_kind();
match err {
FetchNoFollowErrorKind::NotFound(_) => Ok(None),
FetchNoFollowErrorKind::UrlToFilePath { .. } |
FetchNoFollowErrorKind::ReadingBlobUrl { .. } |
FetchNoFollowErrorKind::ReadingFile { .. } |
FetchNoFollowErrorKind::FetchingRemote { .. } |
FetchNoFollowErrorKind::ClientError { .. } |
FetchNoFollowErrorKind::NoRemote { .. } |
FetchNoFollowErrorKind::DataUrlDecode { .. } |
FetchNoFollowErrorKind::RedirectResolution { .. } |
FetchNoFollowErrorKind::CacheRead { .. } |
FetchNoFollowErrorKind::CacheSave { .. } |
FetchNoFollowErrorKind::UnsupportedScheme { .. } |
FetchNoFollowErrorKind::RedirectHeaderParse { .. } |
FetchNoFollowErrorKind::InvalidHeader { .. } => Err(AnyError::from(err)),
FetchNoFollowErrorKind::NotCached { .. } => {
if options.cache_setting == LoaderCacheSetting::Only {
Ok(None)
} else {
Err(AnyError::from(err))
}
},
FetchNoFollowErrorKind::ChecksumIntegrity(err) => {
// convert to the equivalent deno_graph error so that it
// enhances it if this is passed to deno_graph
Err(
deno_graph::source::ChecksumIntegrityError {
actual: err.actual,
expected: err.expected,
}
.into(),
)
}
}
},
CliFetchNoFollowErrorKind::PermissionCheck(permission_check_error) => Err(AnyError::from(permission_check_error)),
}
})
}

1
cli/clippy.toml
View file

@ -4,6 +4,7 @@ disallowed-methods = [
]
disallowed-types = [
{ path = "reqwest::Client", reason = "use crate::http_util::HttpClient instead" },
{ path = "sys_traits::impls::RealSys", reason = "use crate::sys::CliSys instead" },
]
ignore-interior-mutability = [
"lsp_types::Uri",

35
cli/emit.rs
View file

@ -5,6 +5,7 @@ use crate::cache::FastInsecureHasher;
use crate::cache::ParsedSourceCache;
use crate::resolver::CjsTracker;
use deno_ast::EmittedSourceText;
use deno_ast::ModuleKind;
use deno_ast::SourceMapOption;
use deno_ast::SourceRange;
@ -132,6 +133,7 @@ impl Emitter {
&transpile_and_emit_options.0,
&transpile_and_emit_options.1,
)
.map(|r| r.text)
}
})
.await
@ -166,7 +168,8 @@ impl Emitter {
source.clone(),
&self.transpile_and_emit_options.0,
&self.transpile_and_emit_options.1,
)?;
)?
.text;
helper.post_emit_parsed_source(
specifier,
&transpiled_source,
@ -177,6 +180,31 @@ impl Emitter {
}
}
pub fn emit_parsed_source_for_deno_compile(
&self,
specifier: &ModuleSpecifier,
media_type: MediaType,
module_kind: deno_ast::ModuleKind,
source: &Arc<str>,
) -> Result<(String, String), AnyError> {
let mut emit_options = self.transpile_and_emit_options.1.clone();
emit_options.inline_sources = false;
emit_options.source_map = SourceMapOption::Separate;
// strip off the path to have more deterministic builds as we don't care
// about the source name because we manually provide the source map to v8
emit_options.source_map_base = Some(deno_path_util::url_parent(specifier));
let source = EmitParsedSourceHelper::transpile(
&self.parsed_source_cache,
specifier,
media_type,
module_kind,
source.clone(),
&self.transpile_and_emit_options.0,
&emit_options,
)?;
Ok((source.text, source.source_map.unwrap()))
}
/// Expects a file URL, panics otherwise.
pub async fn load_and_emit_for_hmr(
&self,
@ -282,7 +310,7 @@ impl<'a> EmitParsedSourceHelper<'a> {
source: Arc<str>,
transpile_options: &deno_ast::TranspileOptions,
emit_options: &deno_ast::EmitOptions,
) -> Result<String, AnyError> {
) -> Result<EmittedSourceText, AnyError> {
// nothing else needs the parsed source at this point, so remove from
// the cache in order to not transpile owned
let parsed_source = parsed_source_cache
@ -302,8 +330,7 @@ impl<'a> EmitParsedSourceHelper<'a> {
source
}
};
debug_assert!(transpiled_source.source_map.is_none());
Ok(transpiled_source.text)
Ok(transpiled_source)
}
pub fn post_emit_parsed_source(

97
cli/factory.rs
View file

@ -11,7 +11,6 @@ use crate::args::StorageKeyResolver;
use crate::args::TsConfigType;
use crate::cache::Caches;
use crate::cache::CodeCache;
use crate::cache::DenoCacheEnvFsAdapter;
use crate::cache::DenoDir;
use crate::cache::DenoDirProvider;
use crate::cache::EmitCache;
@ -22,7 +21,7 @@ use crate::cache::ModuleInfoCache;
use crate::cache::NodeAnalysisCache;
use crate::cache::ParsedSourceCache;
use crate::emit::Emitter;
use crate::file_fetcher::FileFetcher;
use crate::file_fetcher::CliFileFetcher;
use crate::graph_container::MainModuleGraphContainer;
use crate::graph_util::FileWatcherReporter;
use crate::graph_util::ModuleGraphBuilder;
@ -32,6 +31,8 @@ use crate::module_loader::CliModuleLoaderFactory;
use crate::module_loader::ModuleLoadPreparer;
use crate::node::CliCjsCodeAnalyzer;
use crate::node::CliNodeCodeTranslator;
use crate::node::CliNodeResolver;
use crate::node::CliPackageJsonResolver;
use crate::npm::create_cli_npm_resolver;
use crate::npm::create_in_npm_pkg_checker;
use crate::npm::CliByonmNpmResolverCreateOptions;
@ -43,14 +44,14 @@ use crate::npm::CliNpmResolverManagedSnapshotOption;
use crate::npm::CreateInNpmPkgCheckerOptions;
use crate::resolver::CjsTracker;
use crate::resolver::CliDenoResolver;
use crate::resolver::CliDenoResolverFs;
use crate::resolver::CliNpmReqResolver;
use crate::resolver::CliResolver;
use crate::resolver::CliResolverOptions;
use crate::resolver::CliSloppyImportsResolver;
use crate::resolver::NpmModuleLoader;
use crate::resolver::SloppyImportsCachedFs;
use crate::standalone::DenoCompileBinaryWriter;
use crate::standalone::binary::DenoCompileBinaryWriter;
use crate::sys::CliSys;
use crate::tools::check::TypeChecker;
use crate::tools::coverage::CoverageCollector;
use crate::tools::lint::LintRuleProvider;
@ -76,9 +77,8 @@ use deno_resolver::npm::NpmReqResolverOptions;
use deno_resolver::DenoResolverOptions;
use deno_resolver::NodeAndNpmReqResolver;
use deno_runtime::deno_fs;
use deno_runtime::deno_node::DenoFsNodeResolverEnv;
use deno_runtime::deno_node::NodeResolver;
use deno_runtime::deno_node::PackageJsonResolver;
use deno_runtime::deno_fs::RealFs;
use deno_runtime::deno_node::RealIsBuiltInNodeModuleChecker;
use deno_runtime::deno_permissions::Permissions;
use deno_runtime::deno_permissions::PermissionsContainer;
use deno_runtime::deno_tls::rustls::RootCertStore;
@ -185,7 +185,7 @@ struct CliFactoryServices {
emit_cache: Deferred<Arc<EmitCache>>,
emitter: Deferred<Arc<Emitter>>,
feature_checker: Deferred<Arc<FeatureChecker>>,
file_fetcher: Deferred<Arc<FileFetcher>>,
file_fetcher: Deferred<Arc<CliFileFetcher>>,
fs: Deferred<Arc<dyn deno_fs::FileSystem>>,
global_http_cache: Deferred<Arc<GlobalHttpCache>>,
http_cache: Deferred<Arc<dyn HttpCache>>,
@ -199,13 +199,14 @@ struct CliFactoryServices {
module_info_cache: Deferred<Arc<ModuleInfoCache>>,
module_load_preparer: Deferred<Arc<ModuleLoadPreparer>>,
node_code_translator: Deferred<Arc<CliNodeCodeTranslator>>,
node_resolver: Deferred<Arc<NodeResolver>>,
node_resolver: Deferred<Arc<CliNodeResolver>>,
npm_cache_dir: Deferred<Arc<NpmCacheDir>>,
npm_req_resolver: Deferred<Arc<CliNpmReqResolver>>,
npm_resolver: Deferred<Arc<dyn CliNpmResolver>>,
parsed_source_cache: Deferred<Arc<ParsedSourceCache>>,
permission_desc_parser: Deferred<Arc<RuntimePermissionDescriptorParser>>,
pkg_json_resolver: Deferred<Arc<PackageJsonResolver>>,
permission_desc_parser:
Deferred<Arc<RuntimePermissionDescriptorParser<CliSys>>>,
pkg_json_resolver: Deferred<Arc<CliPackageJsonResolver>>,
resolver: Deferred<Arc<CliResolver>>,
root_cert_store_provider: Deferred<Arc<dyn RootCertStoreProvider>>,
root_permissions_container: Deferred<PermissionsContainer>,
@ -255,7 +256,7 @@ impl CliFactory {
pub fn cli_options(&self) -> Result<&Arc<CliOptions>, AnyError> {
self.services.cli_options.get_or_try_init(|| {
CliOptions::from_flags(self.flags.clone()).map(Arc::new)
CliOptions::from_flags(&self.sys(), self.flags.clone()).map(Arc::new)
})
}
@ -318,8 +319,8 @@ impl CliFactory {
pub fn global_http_cache(&self) -> Result<&Arc<GlobalHttpCache>, AnyError> {
self.services.global_http_cache.get_or_try_init(|| {
Ok(Arc::new(GlobalHttpCache::new(
self.sys(),
self.deno_dir()?.remote_folder_path(),
crate::cache::RealDenoCacheEnv,
)))
})
}
@ -350,22 +351,28 @@ impl CliFactory {
})
}
pub fn file_fetcher(&self) -> Result<&Arc<FileFetcher>, AnyError> {
pub fn file_fetcher(&self) -> Result<&Arc<CliFileFetcher>, AnyError> {
self.services.file_fetcher.get_or_try_init(|| {
let cli_options = self.cli_options()?;
Ok(Arc::new(FileFetcher::new(
Ok(Arc::new(CliFileFetcher::new(
self.http_cache()?.clone(),
cli_options.cache_setting(),
!cli_options.no_remote(),
self.http_client_provider().clone(),
self.sys(),
self.blob_store().clone(),
Some(self.text_only_progress_bar().clone()),
!cli_options.no_remote(),
cli_options.cache_setting(),
log::Level::Info,
)))
})
}
pub fn fs(&self) -> &Arc<dyn deno_fs::FileSystem> {
self.services.fs.get_or_init(|| Arc::new(deno_fs::RealFs))
self.services.fs.get_or_init(|| Arc::new(RealFs))
}
pub fn sys(&self) -> CliSys {
CliSys::default() // very cheap to make
}
pub fn in_npm_pkg_checker(
@ -391,11 +398,10 @@ impl CliFactory {
pub fn npm_cache_dir(&self) -> Result<&Arc<NpmCacheDir>, AnyError> {
self.services.npm_cache_dir.get_or_try_init(|| {
let fs = self.fs();
let global_path = self.deno_dir()?.npm_folder_path();
let cli_options = self.cli_options()?;
Ok(Arc::new(NpmCacheDir::new(
&DenoCacheEnvFsAdapter(fs.as_ref()),
&self.sys(),
global_path,
cli_options.npmrc().get_all_known_registries_urls(),
)))
@ -410,12 +416,11 @@ impl CliFactory {
.npm_resolver
.get_or_try_init_async(
async {
let fs = self.fs();
let cli_options = self.cli_options()?;
create_cli_npm_resolver(if cli_options.use_byonm() {
CliNpmResolverCreateOptions::Byonm(
CliByonmNpmResolverCreateOptions {
fs: CliDenoResolverFs(fs.clone()),
sys: self.sys(),
pkg_json_resolver: self.pkg_json_resolver().clone(),
root_node_modules_dir: Some(
match cli_options.node_modules_dir_path() {
@ -433,6 +438,13 @@ impl CliFactory {
} else {
CliNpmResolverCreateOptions::Managed(
CliManagedNpmResolverCreateOptions {
http_client_provider: self.http_client_provider().clone(),
npm_install_deps_provider: Arc::new(
NpmInstallDepsProvider::from_workspace(
cli_options.workspace(),
),
),
sys: self.sys(),
snapshot: match cli_options.resolve_npm_resolution_snapshot()? {
Some(snapshot) => {
CliNpmResolverManagedSnapshotOption::Specified(Some(
@ -451,19 +463,12 @@ impl CliFactory {
},
},
maybe_lockfile: cli_options.maybe_lockfile().cloned(),
fs: fs.clone(),
http_client_provider: self.http_client_provider().clone(),
npm_cache_dir: self.npm_cache_dir()?.clone(),
cache_setting: cli_options.cache_setting(),
text_only_progress_bar: self.text_only_progress_bar().clone(),
maybe_node_modules_path: cli_options
.node_modules_dir_path()
.cloned(),
npm_install_deps_provider: Arc::new(
NpmInstallDepsProvider::from_workspace(
cli_options.workspace(),
),
),
npm_system_info: cli_options.npm_system_info(),
npmrc: cli_options.npmrc().clone(),
lifecycle_scripts: cli_options.lifecycle_scripts_config(),
@ -486,7 +491,7 @@ impl CliFactory {
.get_or_try_init(|| {
Ok(self.cli_options()?.unstable_sloppy_imports().then(|| {
Arc::new(CliSloppyImportsResolver::new(SloppyImportsCachedFs::new(
self.fs().clone(),
self.sys(),
)))
}))
})
@ -647,21 +652,22 @@ impl CliFactory {
))
}
pub async fn node_resolver(&self) -> Result<&Arc<NodeResolver>, AnyError> {
pub async fn node_resolver(&self) -> Result<&Arc<CliNodeResolver>, AnyError> {
self
.services
.node_resolver
.get_or_try_init_async(
async {
Ok(Arc::new(NodeResolver::new(
DenoFsNodeResolverEnv::new(self.fs().clone()),
Ok(Arc::new(CliNodeResolver::new(
self.in_npm_pkg_checker()?.clone(),
RealIsBuiltInNodeModuleChecker,
self
.npm_resolver()
.await?
.clone()
.into_npm_pkg_folder_resolver(),
self.pkg_json_resolver().clone(),
self.sys(),
)))
}
.boxed_local(),
@ -689,7 +695,6 @@ impl CliFactory {
Ok(Arc::new(NodeCodeTranslator::new(
cjs_esm_analyzer,
DenoFsNodeResolverEnv::new(self.fs().clone()),
self.in_npm_pkg_checker()?.clone(),
node_resolver,
self
@ -698,6 +703,7 @@ impl CliFactory {
.clone()
.into_npm_pkg_folder_resolver(),
self.pkg_json_resolver().clone(),
self.sys(),
)))
})
.await
@ -713,7 +719,7 @@ impl CliFactory {
let npm_resolver = self.npm_resolver().await?;
Ok(Arc::new(CliNpmReqResolver::new(NpmReqResolverOptions {
byonm_resolver: (npm_resolver.clone()).into_maybe_byonm(),
fs: CliDenoResolverFs(self.fs().clone()),
sys: self.sys(),
in_npm_pkg_checker: self.in_npm_pkg_checker()?.clone(),
node_resolver: self.node_resolver().await?.clone(),
npm_req_resolver: npm_resolver.clone().into_npm_req_resolver(),
@ -722,12 +728,11 @@ impl CliFactory {
.await
}
pub fn pkg_json_resolver(&self) -> &Arc<PackageJsonResolver> {
self.services.pkg_json_resolver.get_or_init(|| {
Arc::new(PackageJsonResolver::new(DenoFsNodeResolverEnv::new(
self.fs().clone(),
)))
})
pub fn pkg_json_resolver(&self) -> &Arc<CliPackageJsonResolver> {
self
.services
.pkg_json_resolver
.get_or_init(|| Arc::new(CliPackageJsonResolver::new(self.sys())))
}
pub async fn type_checker(&self) -> Result<&Arc<TypeChecker>, AnyError> {
@ -764,7 +769,6 @@ impl CliFactory {
self.cjs_tracker()?.clone(),
cli_options.clone(),
self.file_fetcher()?.clone(),
self.fs().clone(),
self.global_http_cache()?.clone(),
self.in_npm_pkg_checker()?.clone(),
cli_options.maybe_lockfile().cloned(),
@ -774,6 +778,7 @@ impl CliFactory {
self.parsed_source_cache().clone(),
self.resolver().await?.clone(),
self.root_permissions_container()?.clone(),
self.sys(),
)))
})
.await
@ -863,10 +868,9 @@ impl CliFactory {
pub fn permission_desc_parser(
&self,
) -> Result<&Arc<RuntimePermissionDescriptorParser>, AnyError> {
) -> Result<&Arc<RuntimePermissionDescriptorParser<CliSys>>, AnyError> {
self.services.permission_desc_parser.get_or_try_init(|| {
let fs = self.fs().clone();
Ok(Arc::new(RuntimePermissionDescriptorParser::new(fs)))
Ok(Arc::new(RuntimePermissionDescriptorParser::new(self.sys())))
})
}
@ -959,7 +963,6 @@ impl CliFactory {
None
},
self.emitter()?.clone(),
fs.clone(),
in_npm_pkg_checker.clone(),
self.main_module_graph_container().await?.clone(),
self.module_load_preparer().await?.clone(),
@ -974,6 +977,7 @@ impl CliFactory {
),
self.parsed_source_cache().clone(),
self.resolver().await?.clone(),
self.sys(),
)),
node_resolver.clone(),
npm_resolver.clone(),
@ -981,6 +985,7 @@ impl CliFactory {
self.root_cert_store_provider().clone(),
self.root_permissions_container()?.clone(),
StorageKeyResolver::from_options(cli_options),
self.sys(),
cli_options.sub_command().clone(),
self.create_cli_main_worker_options()?,
self.cli_options()?.otel_config(),

1430
cli/file_fetcher.rs
View file

File diff suppressed because it is too large Load diff

171
cli/graph_util.rs
View file

@ -1,5 +1,42 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use std::collections::HashSet;
use std::error::Error;
use std::ops::Deref;
use std::path::PathBuf;
use std::sync::Arc;
use deno_config::deno_json::JsxImportSourceConfig;
use deno_config::workspace::JsrPackageConfig;
use deno_core::anyhow::bail;
use deno_core::error::custom_error;
use deno_core::error::AnyError;
use deno_core::parking_lot::Mutex;
use deno_core::ModuleSpecifier;
use deno_graph::source::Loader;
use deno_graph::source::LoaderChecksum;
use deno_graph::source::ResolutionKind;
use deno_graph::source::ResolveError;
use deno_graph::FillFromLockfileOptions;
use deno_graph::GraphKind;
use deno_graph::JsrLoadError;
use deno_graph::ModuleError;
use deno_graph::ModuleGraph;
use deno_graph::ModuleGraphError;
use deno_graph::ModuleLoadError;
use deno_graph::ResolutionError;
use deno_graph::SpecifierError;
use deno_graph::WorkspaceFastCheckOption;
use deno_path_util::url_to_file_path;
use deno_resolver::sloppy_imports::SloppyImportsResolutionKind;
use deno_runtime::deno_node;
use deno_runtime::deno_permissions::PermissionsContainer;
use deno_semver::jsr::JsrDepPackageReq;
use deno_semver::package::PackageNv;
use deno_semver::SmallStackString;
use import_map::ImportMapError;
use node_resolver::InNpmPackageChecker;
use crate::args::config_to_deno_graph_workspace_member;
use crate::args::jsr_url;
use crate::args::CliLockfile;
@ -13,52 +50,17 @@ use crate::cache::ModuleInfoCache;
use crate::cache::ParsedSourceCache;
use crate::colors;
use crate::errors::get_error_class_name;
use crate::file_fetcher::FileFetcher;
use crate::file_fetcher::CliFileFetcher;
use crate::npm::CliNpmResolver;
use crate::resolver::CjsTracker;
use crate::resolver::CliResolver;
use crate::resolver::CliSloppyImportsResolver;
use crate::resolver::SloppyImportsCachedFs;
use crate::sys::CliSys;
use crate::tools::check;
use crate::tools::check::TypeChecker;
use crate::util::file_watcher::WatcherCommunicator;
use crate::util::fs::canonicalize_path;
use deno_config::deno_json::JsxImportSourceConfig;
use deno_config::workspace::JsrPackageConfig;
use deno_core::anyhow::bail;
use deno_graph::source::LoaderChecksum;
use deno_graph::source::ResolutionKind;
use deno_graph::FillFromLockfileOptions;
use deno_graph::JsrLoadError;
use deno_graph::ModuleLoadError;
use deno_graph::WorkspaceFastCheckOption;
use deno_core::error::custom_error;
use deno_core::error::AnyError;
use deno_core::parking_lot::Mutex;
use deno_core::ModuleSpecifier;
use deno_graph::source::Loader;
use deno_graph::source::ResolveError;
use deno_graph::GraphKind;
use deno_graph::ModuleError;
use deno_graph::ModuleGraph;
use deno_graph::ModuleGraphError;
use deno_graph::ResolutionError;
use deno_graph::SpecifierError;
use deno_path_util::url_to_file_path;
use deno_resolver::sloppy_imports::SloppyImportsResolutionKind;
use deno_runtime::deno_fs::FileSystem;
use deno_runtime::deno_node;
use deno_runtime::deno_permissions::PermissionsContainer;
use deno_semver::jsr::JsrDepPackageReq;
use deno_semver::package::PackageNv;
use import_map::ImportMapError;
use node_resolver::InNpmPackageChecker;
use std::collections::HashSet;
use std::error::Error;
use std::ops::Deref;
use std::path::PathBuf;
use std::sync::Arc;
#[derive(Clone)]
pub struct GraphValidOptions {
@ -79,7 +81,7 @@ pub struct GraphValidOptions {
/// for the CLI.
pub fn graph_valid(
graph: &ModuleGraph,
fs: &Arc<dyn FileSystem>,
sys: &CliSys,
roots: &[ModuleSpecifier],
options: GraphValidOptions,
) -> Result<(), AnyError> {
@ -89,7 +91,7 @@ pub fn graph_valid(
let mut errors = graph_walk_errors(
graph,
fs,
sys,
roots,
GraphWalkErrorsOptions {
check_js: options.check_js,
@ -139,7 +141,7 @@ pub struct GraphWalkErrorsOptions {
/// and enhances them with CLI information.
pub fn graph_walk_errors<'a>(
graph: &'a ModuleGraph,
fs: &'a Arc<dyn FileSystem>,
sys: &'a CliSys,
roots: &'a [ModuleSpecifier],
options: GraphWalkErrorsOptions,
) -> impl Iterator<Item = AnyError> + 'a {
@ -174,7 +176,7 @@ pub fn graph_walk_errors<'a>(
}
ModuleGraphError::ModuleError(error) => {
enhanced_integrity_error_message(error)
.or_else(|| enhanced_sloppy_imports_error_message(fs, error))
.or_else(|| enhanced_sloppy_imports_error_message(sys, error))
.unwrap_or_else(|| format_deno_graph_error(error))
}
};
@ -431,8 +433,7 @@ pub struct ModuleGraphBuilder {
caches: Arc<cache::Caches>,
cjs_tracker: Arc<CjsTracker>,
cli_options: Arc<CliOptions>,
file_fetcher: Arc<FileFetcher>,
fs: Arc<dyn FileSystem>,
file_fetcher: Arc<CliFileFetcher>,
global_http_cache: Arc<GlobalHttpCache>,
in_npm_pkg_checker: Arc<dyn InNpmPackageChecker>,
lockfile: Option<Arc<CliLockfile>>,
@ -442,6 +443,7 @@ pub struct ModuleGraphBuilder {
parsed_source_cache: Arc<ParsedSourceCache>,
resolver: Arc<CliResolver>,
root_permissions_container: PermissionsContainer,
sys: CliSys,
}
impl ModuleGraphBuilder {
@ -450,8 +452,7 @@ impl ModuleGraphBuilder {
caches: Arc<cache::Caches>,
cjs_tracker: Arc<CjsTracker>,
cli_options: Arc<CliOptions>,
file_fetcher: Arc<FileFetcher>,
fs: Arc<dyn FileSystem>,
file_fetcher: Arc<CliFileFetcher>,
global_http_cache: Arc<GlobalHttpCache>,
in_npm_pkg_checker: Arc<dyn InNpmPackageChecker>,
lockfile: Option<Arc<CliLockfile>>,
@ -461,13 +462,13 @@ impl ModuleGraphBuilder {
parsed_source_cache: Arc<ParsedSourceCache>,
resolver: Arc<CliResolver>,
root_permissions_container: PermissionsContainer,
sys: CliSys,
) -> Self {
Self {
caches,
cjs_tracker,
cli_options,
file_fetcher,
fs,
global_http_cache,
in_npm_pkg_checker,
lockfile,
@ -477,6 +478,7 @@ impl ModuleGraphBuilder {
parsed_source_cache,
resolver,
root_permissions_container,
sys,
}
}
@ -592,7 +594,7 @@ impl ModuleGraphBuilder {
is_dynamic: options.is_dynamic,
passthrough_jsr_specifiers: false,
executor: Default::default(),
file_system: &DenoGraphFsAdapter(self.fs.as_ref()),
file_system: &self.sys,
jsr_url_provider: &CliJsrUrlProvider,
npm_resolver: Some(&graph_npm_resolver),
module_analyzer: &analyzer,
@ -680,7 +682,7 @@ impl ModuleGraphBuilder {
for (from, to) in graph.packages.mappings() {
lockfile.insert_package_specifier(
JsrDepPackageReq::jsr(from.clone()),
to.version.to_string(),
to.version.to_custom_string::<SmallStackString>(),
);
}
}
@ -746,10 +748,10 @@ impl ModuleGraphBuilder {
) -> cache::FetchCacher {
cache::FetchCacher::new(
self.file_fetcher.clone(),
self.fs.clone(),
self.global_http_cache.clone(),
self.in_npm_pkg_checker.clone(),
self.module_info_cache.clone(),
self.sys.clone(),
cache::FetchCacherOptions {
file_header_overrides: self.cli_options.resolve_file_header_overrides(),
permissions,
@ -778,7 +780,7 @@ impl ModuleGraphBuilder {
) -> Result<(), AnyError> {
graph_valid(
graph,
&self.fs,
&self.sys,
roots,
GraphValidOptions {
kind: if self.cli_options.type_check_mode().is_true() {
@ -834,13 +836,13 @@ pub fn enhanced_resolution_error_message(error: &ResolutionError) -> String {
}
fn enhanced_sloppy_imports_error_message(
fs: &Arc<dyn FileSystem>,
sys: &CliSys,
error: &ModuleError,
) -> Option<String> {
match error {
ModuleError::LoadingErr(specifier, _, ModuleLoadError::Loader(_)) // ex. "Is a directory" error
| ModuleError::Missing(specifier, _) => {
let additional_message = CliSloppyImportsResolver::new(SloppyImportsCachedFs::new(fs.clone()))
let additional_message = CliSloppyImportsResolver::new(SloppyImportsCachedFs::new(sys.clone()))
.resolve(specifier, SloppyImportsResolutionKind::Execution)?
.as_suggestion_message();
Some(format!(
@ -1081,71 +1083,6 @@ impl deno_graph::source::Reporter for FileWatcherReporter {
}
}
pub struct DenoGraphFsAdapter<'a>(
pub &'a dyn deno_runtime::deno_fs::FileSystem,
);
impl<'a> deno_graph::source::FileSystem for DenoGraphFsAdapter<'a> {
fn read_dir(
&self,
dir_url: &deno_graph::ModuleSpecifier,
) -> Vec<deno_graph::source::DirEntry> {
use deno_core::anyhow;
use deno_graph::source::DirEntry;
use deno_graph::source::DirEntryKind;
let dir_path = match dir_url.to_file_path() {
Ok(path) => path,
// ignore, treat as non-analyzable
Err(()) => return vec![],
};
let entries = match self.0.read_dir_sync(&dir_path) {
Ok(dir) => dir,
Err(err)
if matches!(
err.kind(),
std::io::ErrorKind::PermissionDenied | std::io::ErrorKind::NotFound
) =>
{
return vec![];
}
Err(err) => {
return vec![DirEntry {
kind: DirEntryKind::Error(
anyhow::Error::from(err)
.context("Failed to read directory.".to_string()),
),
url: dir_url.clone(),
}];
}
};
let mut dir_entries = Vec::with_capacity(entries.len());
for entry in entries {
let entry_path = dir_path.join(&entry.name);
dir_entries.push(if entry.is_directory {
DirEntry {
kind: DirEntryKind::Dir,
url: ModuleSpecifier::from_directory_path(&entry_path).unwrap(),
}
} else if entry.is_file {
DirEntry {
kind: DirEntryKind::File,
url: ModuleSpecifier::from_file_path(&entry_path).unwrap(),
}
} else if entry.is_symlink {
DirEntry {
kind: DirEntryKind::Symlink,
url: ModuleSpecifier::from_file_path(&entry_path).unwrap(),
}
} else {
continue;
});
}
dir_entries
}
}
pub fn format_range_with_colors(referrer: &deno_graph::Range) -> String {
format!(
"{}:{}:{}",

907
cli/http_util.rs
View file

File diff suppressed because it is too large Load diff

1122
cli/js/40_lint.js Normal file
View file

File diff suppressed because it is too large Load diff

1014
cli/js/40_lint_selector.js Normal file
View file

File diff suppressed because it is too large Load diff

132
cli/js/40_lint_types.d.ts vendored Normal file
View file

@ -0,0 +1,132 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
export interface NodeFacade {
type: string;
range: [number, number];
[key: string]: unknown;
}
export interface AstContext {
buf: Uint8Array;
strTable: Map<number, string>;
strTableOffset: number;
rootOffset: number;
nodes: Map<number, NodeFacade>;
strByType: number[];
strByProp: number[];
typeByStr: Map<string, number>;
propByStr: Map<string, number>;
matcher: MatchContext;
}
// TODO(@marvinhagemeister) Remove once we land "official" types
export interface RuleContext {
id: string;
}
// TODO(@marvinhagemeister) Remove once we land "official" types
export interface LintRule {
create(ctx: RuleContext): Record<string, (node: unknown) => void>;
destroy?(ctx: RuleContext): void;
}
// TODO(@marvinhagemeister) Remove once we land "official" types
export interface LintPlugin {
name: string;
rules: Record<string, LintRule>;
}
export interface LintState {
plugins: LintPlugin[];
installedPlugins: Set<string>;
}
export type VisitorFn = (node: unknown) => void;
export interface CompiledVisitor {
matcher: (ctx: MatchContext, offset: number) => boolean;
info: { enter: VisitorFn; exit: VisitorFn };
}
export interface AttrExists {
type: 3;
prop: number[];
}
export interface AttrBin {
type: 4;
prop: number[];
op: number;
// deno-lint-ignore no-explicit-any
value: any;
}
export type AttrSelector = AttrExists | AttrBin;
export interface ElemSelector {
type: 1;
wildcard: boolean;
elem: number;
}
export interface PseudoNthChild {
type: 5;
op: string | null;
step: number;
stepOffset: number;
of: Selector | null;
repeat: boolean;
}
export interface PseudoHas {
type: 6;
selectors: Selector[];
}
export interface PseudoNot {
type: 7;
selectors: Selector[];
}
export interface PseudoFirstChild {
type: 8;
}
export interface PseudoLastChild {
type: 9;
}
export interface Relation {
type: 2;
op: number;
}
export type Selector = Array<
| ElemSelector
| Relation
| AttrExists
| AttrBin
| PseudoNthChild
| PseudoNot
| PseudoHas
| PseudoFirstChild
| PseudoLastChild
>;
export interface SelectorParseCtx {
root: Selector;
current: Selector;
}
export interface MatchContext {
getFirstChild(id: number): number;
getLastChild(id: number): number;
getSiblings(id: number): number[];
getParent(id: number): number;
getType(id: number): number;
hasAttrPath(id: number, propIds: number[], idx: number): boolean;
getAttrPathValue(id: number, propIds: number[], idx: number): unknown;
}
export type NextFn = (ctx: MatchContext, id: number) => boolean;
export type MatcherFn = (ctx: MatchContext, id: number) => boolean;
export type TransformFn = (value: string) => number;
export {};

6
cli/jsr.rs
View file

@ -1,7 +1,7 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use crate::args::jsr_url;
use crate::file_fetcher::FileFetcher;
use crate::file_fetcher::CliFileFetcher;
use dashmap::DashMap;
use deno_core::serde_json;
use deno_graph::packages::JsrPackageInfo;
@ -19,11 +19,11 @@ pub struct JsrFetchResolver {
/// It can be large and we don't want to store it.
info_by_nv: DashMap<PackageNv, Option<Arc<JsrPackageVersionInfo>>>,
info_by_name: DashMap<String, Option<Arc<JsrPackageInfo>>>,
file_fetcher: Arc<FileFetcher>,
file_fetcher: Arc<CliFileFetcher>,
}
impl JsrFetchResolver {
pub fn new(file_fetcher: Arc<FileFetcher>) -> Self {
pub fn new(file_fetcher: Arc<CliFileFetcher>) -> Self {
Self {
nv_by_req: Default::default(),
info_by_nv: Default::default(),

37
cli/lsp/analysis.rs
View file

@ -36,6 +36,8 @@ use deno_semver::package::PackageNv;
use deno_semver::package::PackageNvReference;
use deno_semver::package::PackageReq;
use deno_semver::package::PackageReqReference;
use deno_semver::SmallStackString;
use deno_semver::StackString;
use deno_semver::Version;
use import_map::ImportMap;
use node_resolver::NodeResolutionKind;
@ -270,13 +272,24 @@ impl<'a> TsResponseImportMapper<'a> {
}
}
if specifier.scheme() == "node" {
return Some(specifier.to_string());
}
if let Some(jsr_path) = specifier.as_str().strip_prefix(jsr_url().as_str())
{
let mut segments = jsr_path.split('/');
let name = if jsr_path.starts_with('@') {
format!("{}/{}", segments.next()?, segments.next()?)
let scope = segments.next()?;
let name = segments.next()?;
capacity_builder::StringBuilder::<StackString>::build(|builder| {
builder.append(scope);
builder.append("/");
builder.append(name);
})
.unwrap()
} else {
segments.next()?.to_string()
StackString::from(segments.next()?)
};
let version = Version::parse_standard(segments.next()?).ok()?;
let nv = PackageNv { name, version };
@ -286,7 +299,9 @@ impl<'a> TsResponseImportMapper<'a> {
&path,
Some(&self.file_referrer),
)?;
let sub_path = (export != ".").then_some(export);
let sub_path = (export != ".")
.then_some(export)
.map(SmallStackString::from_string);
let mut req = None;
req = req.or_else(|| {
let import_map = self.maybe_import_map?;
@ -599,18 +614,24 @@ fn try_reverse_map_package_json_exports(
/// For a set of tsc changes, can them for any that contain something that looks
/// like an import and rewrite the import specifier to include the extension
pub fn fix_ts_import_changes(
referrer: &ModuleSpecifier,
resolution_mode: ResolutionMode,
changes: &[tsc::FileTextChanges],
language_server: &language_server::Inner,
) -> Result<Vec<tsc::FileTextChanges>, AnyError> {
let import_mapper = language_server.get_ts_response_import_mapper(referrer);
let mut r = Vec::new();
for change in changes {
let Ok(referrer) = ModuleSpecifier::parse(&change.file_name) else {
continue;
};
let referrer_doc = language_server.get_asset_or_document(&referrer).ok();
let resolution_mode = referrer_doc
.as_ref()
.map(|d| d.resolution_mode())
.unwrap_or(ResolutionMode::Import);
let import_mapper =
language_server.get_ts_response_import_mapper(&referrer);
let mut text_changes = Vec::new();
for text_change in &change.text_changes {
let lines = text_change.new_text.split('\n');
let new_lines: Vec<String> = lines
.map(|line| {
// This assumes that there's only one import per line.
@ -618,7 +639,7 @@ pub fn fix_ts_import_changes(
let specifier =
captures.iter().skip(1).find_map(|s| s).unwrap().as_str();
if let Some(new_specifier) = import_mapper
.check_unresolved_specifier(specifier, referrer, resolution_mode)
.check_unresolved_specifier(specifier, &referrer, resolution_mode)
{
line.replace(specifier, &new_specifier)
} else {

10
cli/lsp/cache.rs
View file

@ -7,6 +7,7 @@ use crate::cache::LocalLspHttpCache;
use crate::lsp::config::Config;
use crate::lsp::logging::lsp_log;
use crate::lsp::logging::lsp_warn;
use crate::sys::CliSys;
use deno_core::url::Url;
use deno_core::ModuleSpecifier;
@ -91,12 +92,11 @@ impl LspCache {
})
.ok()
});
let deno_dir = DenoDir::new(global_cache_path)
let sys = CliSys::default();
let deno_dir = DenoDir::new(sys.clone(), global_cache_path)
.expect("should be infallible with absolute custom root");
let global = Arc::new(GlobalHttpCache::new(
deno_dir.remote_folder_path(),
crate::cache::RealDenoCacheEnv,
));
let global =
Arc::new(GlobalHttpCache::new(sys, deno_dir.remote_folder_path()));
Self {
deno_dir,
global,

140
cli/lsp/config.rs
View file

@ -9,8 +9,6 @@ use deno_config::deno_json::LintConfig;
use deno_config::deno_json::NodeModulesDirMode;
use deno_config::deno_json::TestConfig;
use deno_config::deno_json::TsConfig;
use deno_config::fs::DenoConfigFs;
use deno_config::fs::RealDenoConfigFs;
use deno_config::glob::FilePatterns;
use deno_config::glob::PathOrPatternSet;
use deno_config::workspace::CreateResolverOptions;
@ -41,7 +39,6 @@ use deno_path_util::url_to_file_path;
use deno_runtime::deno_node::PackageJson;
use indexmap::IndexSet;
use lsp_types::ClientCapabilities;
use std::borrow::Cow;
use std::collections::BTreeMap;
use std::collections::BTreeSet;
use std::collections::HashMap;
@ -63,10 +60,11 @@ use crate::args::ConfigFile;
use crate::args::LintFlags;
use crate::args::LintOptions;
use crate::cache::FastInsecureHasher;
use crate::file_fetcher::FileFetcher;
use crate::file_fetcher::CliFileFetcher;
use crate::lsp::logging::lsp_warn;
use crate::resolver::CliSloppyImportsResolver;
use crate::resolver::SloppyImportsCachedFs;
use crate::sys::CliSys;
use crate::tools::lint::CliLinter;
use crate::tools::lint::CliLinterOptions;
use crate::tools::lint::LintRuleProvider;
@ -459,6 +457,19 @@ impl Default for LanguagePreferences {
}
}
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)]
#[serde(rename_all = "camelCase")]
pub struct SuggestionActionsSettings {
#[serde(default = "is_true")]
pub enabled: bool,
}
impl Default for SuggestionActionsSettings {
fn default() -> Self {
SuggestionActionsSettings { enabled: true }
}
}
#[derive(Debug, Default, Clone, Deserialize, Serialize, PartialEq, Eq)]
#[serde(rename_all = "camelCase")]
pub struct UpdateImportsOnFileMoveOptions {
@ -490,6 +501,8 @@ pub struct LanguageWorkspaceSettings {
#[serde(default)]
pub suggest: CompletionSettings,
#[serde(default)]
pub suggestion_actions: SuggestionActionsSettings,
#[serde(default)]
pub update_imports_on_file_move: UpdateImportsOnFileMoveOptions,
}
@ -1203,9 +1216,8 @@ impl ConfigData {
specified_config: Option<&Path>,
scope: &ModuleSpecifier,
settings: &Settings,
file_fetcher: &Arc<FileFetcher>,
file_fetcher: &Arc<CliFileFetcher>,
// sync requirement is because the lsp requires sync
cached_deno_config_fs: &(dyn DenoConfigFs + Sync),
deno_json_cache: &(dyn DenoJsonCache + Sync),
pkg_json_cache: &(dyn PackageJsonCache + Sync),
workspace_cache: &(dyn WorkspaceCache + Sync),
@ -1215,6 +1227,7 @@ impl ConfigData {
Ok(scope_dir_path) => {
let paths = [scope_dir_path];
WorkspaceDirectory::discover(
&CliSys::default(),
match specified_config {
Some(config_path) => {
deno_config::workspace::WorkspaceDiscoverStart::ConfigFile(
@ -1226,7 +1239,6 @@ impl ConfigData {
}
},
&WorkspaceDiscoverOptions {
fs: cached_deno_config_fs,
additional_config_file_names: &[],
deno_json_cache: Some(deno_json_cache),
pkg_json_cache: Some(pkg_json_cache),
@ -1298,7 +1310,7 @@ impl ConfigData {
member_dir: Arc<WorkspaceDirectory>,
scope: Arc<ModuleSpecifier>,
settings: &Settings,
file_fetcher: Option<&Arc<FileFetcher>>,
file_fetcher: Option<&Arc<CliFileFetcher>>,
) -> Self {
let (settings, workspace_folder) = settings.get_for_specifier(&scope);
let mut watched_files = HashMap::with_capacity(10);
@ -1603,9 +1615,7 @@ impl ConfigData {
|| unstable.contains("sloppy-imports");
let sloppy_imports_resolver = unstable_sloppy_imports.then(|| {
Arc::new(CliSloppyImportsResolver::new(
SloppyImportsCachedFs::new_without_stat_cache(Arc::new(
deno_runtime::deno_fs::RealFs,
)),
SloppyImportsCachedFs::new_without_stat_cache(CliSys::default()),
))
});
let resolver = Arc::new(resolver);
@ -1819,13 +1829,12 @@ impl ConfigTree {
&mut self,
settings: &Settings,
workspace_files: &IndexSet<ModuleSpecifier>,
file_fetcher: &Arc<FileFetcher>,
file_fetcher: &Arc<CliFileFetcher>,
) {
lsp_log!("Refreshing configuration tree...");
// since we're resolving a workspace multiple times in different
// folders, we want to cache all the lookups and config files across
// ConfigData::load calls
let cached_fs = CachedDenoConfigFs::default();
let deno_json_cache = DenoJsonMemCache::default();
let pkg_json_cache = PackageJsonMemCache::default();
let workspace_cache = WorkspaceMemCache::default();
@ -1850,7 +1859,6 @@ impl ConfigTree {
folder_uri,
settings,
file_fetcher,
&cached_fs,
&deno_json_cache,
&pkg_json_cache,
&workspace_cache,
@ -1881,7 +1889,6 @@ impl ConfigTree {
&scope,
settings,
file_fetcher,
&cached_fs,
&deno_json_cache,
&pkg_json_cache,
&workspace_cache,
@ -1898,7 +1905,6 @@ impl ConfigTree {
member_scope,
settings,
file_fetcher,
&cached_fs,
&deno_json_cache,
&pkg_json_cache,
&workspace_cache,
@ -1913,21 +1919,24 @@ impl ConfigTree {
#[cfg(test)]
pub async fn inject_config_file(&mut self, config_file: ConfigFile) {
use sys_traits::FsCreateDirAll;
use sys_traits::FsWrite;
let scope = config_file.specifier.join(".").unwrap();
let json_text = serde_json::to_string(&config_file.json).unwrap();
let test_fs = deno_runtime::deno_fs::InMemoryFs::default();
let memory_sys = sys_traits::impls::InMemorySys::default();
let config_path = url_to_file_path(&config_file.specifier).unwrap();
test_fs.setup_text_files(vec![(
config_path.to_string_lossy().to_string(),
json_text,
)]);
memory_sys
.fs_create_dir_all(config_path.parent().unwrap())
.unwrap();
memory_sys.fs_write(&config_path, json_text).unwrap();
let workspace_dir = Arc::new(
WorkspaceDirectory::discover(
&memory_sys,
deno_config::workspace::WorkspaceDiscoverStart::ConfigFile(
&config_path,
),
&deno_config::workspace::WorkspaceDiscoverOptions {
fs: &crate::args::deno_json::DenoConfigFsAdapter(&test_fs),
..Default::default()
},
)
@ -2000,11 +2009,14 @@ fn resolve_lockfile_from_path(
lockfile_path: PathBuf,
frozen: bool,
) -> Option<CliLockfile> {
match CliLockfile::read_from_path(CliLockfileReadFromPathOptions {
file_path: lockfile_path,
frozen,
skip_write: false,
}) {
match CliLockfile::read_from_path(
&CliSys::default(),
CliLockfileReadFromPathOptions {
file_path: lockfile_path,
frozen,
skip_write: false,
},
) {
Ok(value) => {
if value.filename.exists() {
if let Ok(specifier) = ModuleSpecifier::from_file_path(&value.filename)
@ -2061,78 +2073,6 @@ impl deno_config::workspace::WorkspaceCache for WorkspaceMemCache {
}
}
#[derive(Default)]
struct CachedFsItems<T: Clone> {
items: HashMap<PathBuf, Result<T, std::io::Error>>,
}
impl<T: Clone> CachedFsItems<T> {
pub fn get(
&mut self,
path: &Path,
action: impl FnOnce(&Path) -> Result<T, std::io::Error>,
) -> Result<T, std::io::Error> {
let value = if let Some(value) = self.items.get(path) {
value
} else {
let value = action(path);
// just in case this gets really large for some reason
if self.items.len() == 16_384 {
return value;
}
self.items.insert(path.to_owned(), value);
self.items.get(path).unwrap()
};
value
.as_ref()
.map(|v| (*v).clone())
.map_err(|e| std::io::Error::new(e.kind(), e.to_string()))
}
}
#[derive(Default)]
struct InnerData {
stat_calls: CachedFsItems<deno_config::fs::FsMetadata>,
read_to_string_calls: CachedFsItems<Cow<'static, str>>,
}
#[derive(Default)]
struct CachedDenoConfigFs(Mutex<InnerData>);
impl DenoConfigFs for CachedDenoConfigFs {
fn stat_sync(
&self,
path: &Path,
) -> Result<deno_config::fs::FsMetadata, std::io::Error> {
self
.0
.lock()
.stat_calls
.get(path, |path| RealDenoConfigFs.stat_sync(path))
}
fn read_to_string_lossy(
&self,
path: &Path,
) -> Result<Cow<'static, str>, std::io::Error> {
self
.0
.lock()
.read_to_string_calls
.get(path, |path| RealDenoConfigFs.read_to_string_lossy(path))
}
fn read_dir(
&self,
path: &Path,
) -> Result<Vec<deno_config::fs::FsDirEntry>, std::io::Error> {
// no need to cache these because the workspace cache will ensure
// we only do read_dir calls once (read_dirs are only used for
// npm workspace resolution)
RealDenoConfigFs.read_dir(path)
}
}
#[cfg(test)]
mod tests {
use deno_config::deno_json::ConfigParseOptions;
@ -2292,6 +2232,7 @@ mod tests {
enabled: true,
},
},
suggestion_actions: SuggestionActionsSettings { enabled: true },
update_imports_on_file_move: UpdateImportsOnFileMoveOptions {
enabled: UpdateImportsOnFileMoveEnabled::Prompt
}
@ -2338,6 +2279,7 @@ mod tests {
enabled: true,
},
},
suggestion_actions: SuggestionActionsSettings { enabled: true },
update_imports_on_file_move: UpdateImportsOnFileMoveOptions {
enabled: UpdateImportsOnFileMoveEnabled::Prompt
}

24
cli/lsp/diagnostics.rs
View file

@ -21,9 +21,11 @@ use crate::graph_util::enhanced_resolution_error_message;
use crate::lsp::lsp_custom::DiagnosticBatchNotificationParams;
use crate::resolver::CliSloppyImportsResolver;
use crate::resolver::SloppyImportsCachedFs;
use crate::sys::CliSys;
use crate::tools::lint::CliLinter;
use crate::tools::lint::CliLinterOptions;
use crate::tools::lint::LintRuleProvider;
use crate::tsc::DiagnosticCategory;
use crate::util::path::to_percent_decoded_str;
use deno_ast::MediaType;
@ -44,9 +46,9 @@ use deno_graph::source::ResolveError;
use deno_graph::Resolution;
use deno_graph::ResolutionError;
use deno_graph::SpecifierError;
use deno_lint::linter::LintConfig as DenoLintConfig;
use deno_resolver::sloppy_imports::SloppyImportsResolution;
use deno_resolver::sloppy_imports::SloppyImportsResolutionKind;
use deno_runtime::deno_fs;
use deno_runtime::deno_node;
use deno_runtime::tokio_util::create_basic_runtime;
use deno_semver::jsr::JsrPackageReqReference;
@ -833,7 +835,7 @@ fn generate_lint_diagnostics(
lint_rule_provider.resolve_lint_rules(Default::default(), None)
},
fix: false,
deno_lint_config: deno_lint::linter::LintConfig {
deno_lint_config: DenoLintConfig {
default_jsx_factory: None,
default_jsx_fragment_factory: None,
},
@ -906,8 +908,22 @@ async fn generate_ts_diagnostics(
} else {
Default::default()
};
for (specifier_str, ts_json_diagnostics) in ts_diagnostics_map {
for (specifier_str, mut ts_json_diagnostics) in ts_diagnostics_map {
let specifier = resolve_url(&specifier_str)?;
let suggestion_actions_settings = snapshot
.config
.language_settings_for_specifier(&specifier)
.map(|s| s.suggestion_actions.clone())
.unwrap_or_default();
if !suggestion_actions_settings.enabled {
ts_json_diagnostics.retain(|d| {
d.category != DiagnosticCategory::Suggestion
// Still show deprecated and unused diagnostics.
// https://github.com/microsoft/vscode/blob/ce50bd4876af457f64d83cfd956bc916535285f4/extensions/typescript-language-features/src/languageFeatures/diagnostics.ts#L113-L114
|| d.reports_deprecated == Some(true)
|| d.reports_unnecessary == Some(true)
});
}
let version = snapshot
.documents
.get(&specifier)
@ -1265,7 +1281,7 @@ impl DenoDiagnostic {
Self::NotInstalledNpm(pkg_req, specifier) => (lsp::DiagnosticSeverity::ERROR, format!("npm package \"{pkg_req}\" is not installed or doesn't exist."), Some(json!({ "specifier": specifier }))),
Self::NoLocal(specifier) => {
let maybe_sloppy_resolution = CliSloppyImportsResolver::new(
SloppyImportsCachedFs::new(Arc::new(deno_fs::RealFs))
SloppyImportsCachedFs::new(CliSys::default())
).resolve(specifier, SloppyImportsResolutionKind::Execution);
let data = maybe_sloppy_resolution.as_ref().map(|res| {
json!({

7
cli/lsp/documents.rs
View file

@ -251,6 +251,13 @@ impl AssetOrDocument {
pub fn document_lsp_version(&self) -> Option<i32> {
self.document().and_then(|d| d.maybe_lsp_version())
}
pub fn resolution_mode(&self) -> ResolutionMode {
match self {
AssetOrDocument::Asset(_) => ResolutionMode::Import,
AssetOrDocument::Document(d) => d.resolution_mode(),
}
}
}
type ModuleResult = Result<deno_graph::JsModule, deno_graph::ModuleGraphError>;

36
cli/lsp/jsr.rs
View file

@ -2,7 +2,8 @@
use crate::args::jsr_api_url;
use crate::args::jsr_url;
use crate::file_fetcher::FileFetcher;
use crate::file_fetcher::CliFileFetcher;
use crate::file_fetcher::TextDecodedFile;
use crate::jsr::partial_jsr_package_version_info_from_slice;
use crate::jsr::JsrFetchResolver;
use dashmap::DashMap;
@ -17,6 +18,7 @@ use deno_graph::ModuleSpecifier;
use deno_semver::jsr::JsrPackageReqReference;
use deno_semver::package::PackageNv;
use deno_semver::package::PackageReq;
use deno_semver::StackString;
use deno_semver::Version;
use serde::Deserialize;
use std::collections::HashMap;
@ -32,8 +34,8 @@ pub struct JsrCacheResolver {
/// The `module_graph` fields of the version infos should be forcibly absent.
/// It can be large and we don't want to store it.
info_by_nv: DashMap<PackageNv, Option<Arc<JsrPackageVersionInfo>>>,
info_by_name: DashMap<String, Option<Arc<JsrPackageInfo>>>,
workspace_scope_by_name: HashMap<String, ModuleSpecifier>,
info_by_name: DashMap<StackString, Option<Arc<JsrPackageInfo>>>,
workspace_scope_by_name: HashMap<StackString, ModuleSpecifier>,
cache: Arc<dyn HttpCache>,
}
@ -58,7 +60,7 @@ impl JsrCacheResolver {
continue;
};
let nv = PackageNv {
name: jsr_pkg_config.name.clone(),
name: jsr_pkg_config.name.as_str().into(),
version: version.clone(),
};
info_by_name.insert(
@ -124,8 +126,8 @@ impl JsrCacheResolver {
return nv.value().clone();
}
let maybe_get_nv = || {
let name = req.name.clone();
let package_info = self.package_info(&name)?;
let name = &req.name;
let package_info = self.package_info(name)?;
// Find the first matching version of the package which is cached.
let mut versions = package_info.versions.keys().collect::<Vec<_>>();
versions.sort();
@ -143,7 +145,10 @@ impl JsrCacheResolver {
self.package_version_info(&nv).is_some()
})
.cloned()?;
Some(PackageNv { name, version })
Some(PackageNv {
name: name.clone(),
version,
})
};
let nv = maybe_get_nv();
self.nv_by_req.insert(req.clone(), nv.clone());
@ -215,7 +220,10 @@ impl JsrCacheResolver {
None
}
pub fn package_info(&self, name: &str) -> Option<Arc<JsrPackageInfo>> {
pub fn package_info(
&self,
name: &StackString,
) -> Option<Arc<JsrPackageInfo>> {
if let Some(info) = self.info_by_name.get(name) {
return info.value().clone();
}
@ -225,7 +233,7 @@ impl JsrCacheResolver {
serde_json::from_slice::<JsrPackageInfo>(&meta_bytes).ok()
};
let info = read_cached_package_info().map(Arc::new);
self.info_by_name.insert(name.to_string(), info.clone());
self.info_by_name.insert(name.clone(), info.clone());
info
}
@ -267,7 +275,7 @@ fn read_cached_url(
#[derive(Debug)]
pub struct CliJsrSearchApi {
file_fetcher: Arc<FileFetcher>,
file_fetcher: Arc<CliFileFetcher>,
resolver: JsrFetchResolver,
search_cache: DashMap<String, Arc<Vec<String>>>,
versions_cache: DashMap<String, Arc<Vec<Version>>>,
@ -275,7 +283,7 @@ pub struct CliJsrSearchApi {
}
impl CliJsrSearchApi {
pub fn new(file_fetcher: Arc<FileFetcher>) -> Self {
pub fn new(file_fetcher: Arc<CliFileFetcher>) -> Self {
let resolver = JsrFetchResolver::new(file_fetcher.clone());
Self {
file_fetcher,
@ -309,10 +317,8 @@ impl PackageSearchApi for CliJsrSearchApi {
let file_fetcher = self.file_fetcher.clone();
// spawn due to the lsp's `Send` requirement
let file = deno_core::unsync::spawn(async move {
file_fetcher
.fetch_bypass_permissions(&search_url)
.await?
.into_text_decoded()
let file = file_fetcher.fetch_bypass_permissions(&search_url).await?;
TextDecodedFile::decode(file)
})
.await??;
let names = Arc::new(parse_jsr_search_response(&file.source)?);

65
cli/lsp/language_server.rs
View file

@ -1,6 +1,7 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use deno_ast::MediaType;
use deno_cache_dir::file_fetcher::CacheSetting;
use deno_config::workspace::WorkspaceDirectory;
use deno_config::workspace::WorkspaceDiscoverOptions;
use deno_core::anyhow::anyhow;
@ -95,19 +96,19 @@ use crate::args::create_default_npmrc;
use crate::args::get_root_cert_store;
use crate::args::has_flag_env_var;
use crate::args::CaData;
use crate::args::CacheSetting;
use crate::args::CliOptions;
use crate::args::Flags;
use crate::args::InternalFlags;
use crate::args::UnstableFmtOptions;
use crate::factory::CliFactory;
use crate::file_fetcher::FileFetcher;
use crate::file_fetcher::CliFileFetcher;
use crate::graph_util;
use crate::http_util::HttpClientProvider;
use crate::lsp::config::ConfigWatchedFileType;
use crate::lsp::logging::init_log_file;
use crate::lsp::tsc::file_text_changes_to_workspace_edit;
use crate::lsp::urls::LspUrlKind;
use crate::sys::CliSys;
use crate::tools::fmt::format_file;
use crate::tools::fmt::format_parsed_source;
use crate::tools::upgrade::check_for_upgrades_for_lsp;
@ -279,7 +280,7 @@ impl LanguageServer {
.await?;
graph_util::graph_valid(
&graph,
factory.fs(),
&CliSys::default(),
&roots,
graph_util::GraphValidOptions {
kind: GraphKind::All,
@ -958,15 +959,16 @@ impl Inner {
}
async fn refresh_config_tree(&mut self) {
let mut file_fetcher = FileFetcher::new(
let file_fetcher = CliFileFetcher::new(
self.cache.global().clone(),
CacheSetting::RespectHeaders,
true,
self.http_client_provider.clone(),
CliSys::default(),
Default::default(),
None,
true,
CacheSetting::RespectHeaders,
super::logging::lsp_log_level(),
);
file_fetcher.set_download_log_level(super::logging::lsp_log_level());
let file_fetcher = Arc::new(file_fetcher);
self
.config
@ -1855,20 +1857,12 @@ impl Inner {
}
let changes = if code_action_data.fix_id == "fixMissingImport" {
fix_ts_import_changes(
&code_action_data.specifier,
maybe_asset_or_doc
.as_ref()
.and_then(|d| d.document())
.map(|d| d.resolution_mode())
.unwrap_or(ResolutionMode::Import),
&combined_code_actions.changes,
self,
)
.map_err(|err| {
error!("Unable to remap changes: {:#}", err);
LspError::internal_error()
})?
fix_ts_import_changes(&combined_code_actions.changes, self).map_err(
|err| {
error!("Unable to remap changes: {:#}", err);
LspError::internal_error()
},
)?
} else {
combined_code_actions.changes
};
@ -1912,20 +1906,16 @@ impl Inner {
asset_or_doc.scope().cloned(),
)
.await?;
if kind_suffix == ".rewrite.function.returnType" {
refactor_edit_info.edits = fix_ts_import_changes(
&action_data.specifier,
asset_or_doc
.document()
.map(|d| d.resolution_mode())
.unwrap_or(ResolutionMode::Import),
&refactor_edit_info.edits,
self,
)
.map_err(|err| {
error!("Unable to remap changes: {:#}", err);
LspError::internal_error()
})?
if kind_suffix == ".rewrite.function.returnType"
|| kind_suffix == ".move.newFile"
{
refactor_edit_info.edits =
fix_ts_import_changes(&refactor_edit_info.edits, self).map_err(
|err| {
error!("Unable to remap changes: {:#}", err);
LspError::internal_error()
},
)?
}
code_action.edit = refactor_edit_info.to_workspace_edit(self)?;
code_action
@ -3624,11 +3614,11 @@ impl Inner {
let workspace = match config_data {
Some(d) => d.member_dir.clone(),
None => Arc::new(WorkspaceDirectory::discover(
&CliSys::default(),
deno_config::workspace::WorkspaceDiscoverStart::Paths(&[
initial_cwd.clone()
]),
&WorkspaceDiscoverOptions {
fs: Default::default(), // use real fs,
deno_json_cache: None,
pkg_json_cache: None,
workspace_cache: None,
@ -3645,6 +3635,7 @@ impl Inner {
)?),
};
let cli_options = CliOptions::new(
&CliSys::default(),
Arc::new(Flags {
internal: InternalFlags {
cache_path: Some(self.cache.deno_dir().root.clone()),
@ -3793,7 +3784,7 @@ impl Inner {
for (name, command) in scripts {
result.push(TaskDefinition {
name: name.clone(),
command: command.clone(),
command: Some(command.clone()),
source_uri: url_to_uri(&package_json.specifier())
.map_err(|_| LspError::internal_error())?,
});

2
cli/lsp/lsp_custom.rs
View file

@ -14,7 +14,7 @@ pub const LATEST_DIAGNOSTIC_BATCH_INDEX: &str =
#[serde(rename_all = "camelCase")]
pub struct TaskDefinition {
pub name: String,
pub command: String,
pub command: Option<String>,
pub source_uri: lsp::Uri,
}

13
cli/lsp/npm.rs
View file

@ -11,21 +11,22 @@ use serde::Deserialize;
use std::sync::Arc;
use crate::args::npm_registry_url;
use crate::file_fetcher::FileFetcher;
use crate::file_fetcher::CliFileFetcher;
use crate::file_fetcher::TextDecodedFile;
use crate::npm::NpmFetchResolver;
use super::search::PackageSearchApi;
#[derive(Debug)]
pub struct CliNpmSearchApi {
file_fetcher: Arc<FileFetcher>,
file_fetcher: Arc<CliFileFetcher>,
resolver: NpmFetchResolver,
search_cache: DashMap<String, Arc<Vec<String>>>,
versions_cache: DashMap<String, Arc<Vec<Version>>>,
}
impl CliNpmSearchApi {
pub fn new(file_fetcher: Arc<FileFetcher>) -> Self {
pub fn new(file_fetcher: Arc<CliFileFetcher>) -> Self {
let resolver = NpmFetchResolver::new(
file_fetcher.clone(),
Arc::new(NpmRc::default().as_resolved(npm_registry_url()).unwrap()),
@ -57,10 +58,8 @@ impl PackageSearchApi for CliNpmSearchApi {
.append_pair("text", &format!("{} boost-exact:false", query));
let file_fetcher = self.file_fetcher.clone();
let file = deno_core::unsync::spawn(async move {
file_fetcher
.fetch_bypass_permissions(&search_url)
.await?
.into_text_decoded()
let file = file_fetcher.fetch_bypass_permissions(&search_url).await?;
TextDecodedFile::decode(file)
})
.await??;
let names = Arc::new(parse_npm_search_response(&file.source)?);

69
cli/lsp/registries.rs
View file

@ -12,14 +12,16 @@ use super::path_to_regex::StringOrNumber;
use super::path_to_regex::StringOrVec;
use super::path_to_regex::Token;
use crate::args::CacheSetting;
use crate::cache::GlobalHttpCache;
use crate::cache::HttpCache;
use crate::file_fetcher::CliFileFetcher;
use crate::file_fetcher::FetchOptions;
use crate::file_fetcher::FetchPermissionsOptionRef;
use crate::file_fetcher::FileFetcher;
use crate::file_fetcher::TextDecodedFile;
use crate::http_util::HttpClientProvider;
use crate::sys::CliSys;
use deno_cache_dir::file_fetcher::CacheSetting;
use deno_core::anyhow::anyhow;
use deno_core::error::AnyError;
use deno_core::serde::Deserialize;
@ -418,7 +420,7 @@ enum VariableItems {
pub struct ModuleRegistry {
origins: HashMap<String, Vec<RegistryConfiguration>>,
pub location: PathBuf,
pub file_fetcher: Arc<FileFetcher>,
pub file_fetcher: Arc<CliFileFetcher>,
http_cache: Arc<GlobalHttpCache>,
}
@ -428,19 +430,18 @@ impl ModuleRegistry {
http_client_provider: Arc<HttpClientProvider>,
) -> Self {
// the http cache should always be the global one for registry completions
let http_cache = Arc::new(GlobalHttpCache::new(
location.clone(),
crate::cache::RealDenoCacheEnv,
));
let mut file_fetcher = FileFetcher::new(
let http_cache =
Arc::new(GlobalHttpCache::new(CliSys::default(), location.clone()));
let file_fetcher = CliFileFetcher::new(
http_cache.clone(),
CacheSetting::RespectHeaders,
true,
http_client_provider,
CliSys::default(),
Default::default(),
None,
true,
CacheSetting::RespectHeaders,
super::logging::lsp_log_level(),
);
file_fetcher.set_download_log_level(super::logging::lsp_log_level());
Self {
origins: HashMap::new(),
@ -479,13 +480,15 @@ impl ModuleRegistry {
let specifier = specifier.clone();
async move {
file_fetcher
.fetch_with_options(FetchOptions {
specifier: &specifier,
permissions: FetchPermissionsOptionRef::AllowAll,
maybe_auth: None,
maybe_accept: Some("application/vnd.deno.reg.v2+json, application/vnd.deno.reg.v1+json;q=0.9, application/json;q=0.8"),
maybe_cache_setting: None,
})
.fetch_with_options(
&specifier,
FetchPermissionsOptionRef::AllowAll,
FetchOptions {
maybe_auth: None,
maybe_accept: Some("application/vnd.deno.reg.v2+json, application/vnd.deno.reg.v1+json;q=0.9, application/json;q=0.8"),
maybe_cache_setting: None,
}
)
.await
}
}).await?;
@ -500,7 +503,7 @@ impl ModuleRegistry {
);
self.http_cache.set(specifier, headers_map, &[])?;
}
let file = fetch_result?.into_text_decoded()?;
let file = TextDecodedFile::decode(fetch_result?)?;
let config: RegistryConfigurationJson = serde_json::from_str(&file.source)?;
validate_config(&config)?;
Ok(config.registries)
@ -584,12 +587,11 @@ impl ModuleRegistry {
// spawn due to the lsp's `Send` requirement
let file = deno_core::unsync::spawn({
async move {
file_fetcher
let file = file_fetcher
.fetch_bypass_permissions(&endpoint)
.await
.ok()?
.into_text_decoded()
.ok()
.ok()?;
TextDecodedFile::decode(file).ok()
}
})
.await
@ -983,12 +985,11 @@ impl ModuleRegistry {
let file_fetcher = self.file_fetcher.clone();
// spawn due to the lsp's `Send` requirement
let file = deno_core::unsync::spawn(async move {
file_fetcher
let file = file_fetcher
.fetch_bypass_permissions(&specifier)
.await
.ok()?
.into_text_decoded()
.ok()
.ok()?;
TextDecodedFile::decode(file).ok()
})
.await
.ok()??;
@ -1049,7 +1050,7 @@ impl ModuleRegistry {
let file_fetcher = self.file_fetcher.clone();
let specifier = specifier.clone();
async move {
file_fetcher
let file = file_fetcher
.fetch_bypass_permissions(&specifier)
.await
.map_err(|err| {
@ -1058,9 +1059,8 @@ impl ModuleRegistry {
specifier, err
);
})
.ok()?
.into_text_decoded()
.ok()
.ok()?;
TextDecodedFile::decode(file).ok()
}
})
.await
@ -1095,7 +1095,7 @@ impl ModuleRegistry {
let file_fetcher = self.file_fetcher.clone();
let specifier = specifier.clone();
async move {
file_fetcher
let file = file_fetcher
.fetch_bypass_permissions(&specifier)
.await
.map_err(|err| {
@ -1104,9 +1104,8 @@ impl ModuleRegistry {
specifier, err
);
})
.ok()?
.into_text_decoded()
.ok()
.ok()?;
TextDecodedFile::decode(file).ok()
}
})
.await

59
cli/lsp/resolver.rs
View file

@ -2,6 +2,7 @@
use dashmap::DashMap;
use deno_ast::MediaType;
use deno_cache_dir::file_fetcher::CacheSetting;
use deno_cache_dir::npm::NpmCacheDir;
use deno_cache_dir::HttpCache;
use deno_config::deno_json::JsxImportSourceConfig;
@ -18,9 +19,7 @@ use deno_resolver::cjs::IsCjsResolutionMode;
use deno_resolver::npm::NpmReqResolverOptions;
use deno_resolver::DenoResolverOptions;
use deno_resolver::NodeAndNpmReqResolver;
use deno_runtime::deno_fs;
use deno_runtime::deno_node::NodeResolver;
use deno_runtime::deno_node::PackageJsonResolver;
use deno_runtime::deno_node::RealIsBuiltInNodeModuleChecker;
use deno_semver::jsr::JsrPackageReqReference;
use deno_semver::npm::NpmPackageReqReference;
use deno_semver::package::PackageNv;
@ -39,10 +38,8 @@ use std::sync::Arc;
use super::cache::LspCache;
use super::jsr::JsrCacheResolver;
use crate::args::create_default_npmrc;
use crate::args::CacheSetting;
use crate::args::CliLockfile;
use crate::args::NpmInstallDepsProvider;
use crate::cache::DenoCacheEnvFsAdapter;
use crate::factory::Deferred;
use crate::graph_util::to_node_resolution_kind;
use crate::graph_util::to_node_resolution_mode;
@ -51,6 +48,8 @@ use crate::http_util::HttpClientProvider;
use crate::lsp::config::Config;
use crate::lsp::config::ConfigData;
use crate::lsp::logging::lsp_warn;
use crate::node::CliNodeResolver;
use crate::node::CliPackageJsonResolver;
use crate::npm::create_cli_npm_resolver_for_lsp;
use crate::npm::CliByonmNpmResolverCreateOptions;
use crate::npm::CliManagedInNpmPkgCheckerCreateOptions;
@ -61,12 +60,12 @@ use crate::npm::CliNpmResolverManagedSnapshotOption;
use crate::npm::CreateInNpmPkgCheckerOptions;
use crate::npm::ManagedCliNpmResolver;
use crate::resolver::CliDenoResolver;
use crate::resolver::CliDenoResolverFs;
use crate::resolver::CliNpmReqResolver;
use crate::resolver::CliResolver;
use crate::resolver::CliResolverOptions;
use crate::resolver::IsCjsResolver;
use crate::resolver::WorkerCliNpmGraphResolver;
use crate::sys::CliSys;
use crate::tsc::into_specifier_and_media_type;
use crate::util::progress_bar::ProgressBar;
use crate::util::progress_bar::ProgressBarStyle;
@ -78,9 +77,9 @@ struct LspScopeResolver {
is_cjs_resolver: Arc<IsCjsResolver>,
jsr_resolver: Option<Arc<JsrCacheResolver>>,
npm_resolver: Option<Arc<dyn CliNpmResolver>>,
node_resolver: Option<Arc<NodeResolver>>,
node_resolver: Option<Arc<CliNodeResolver>>,
npm_pkg_req_resolver: Option<Arc<CliNpmReqResolver>>,
pkg_json_resolver: Arc<PackageJsonResolver>,
pkg_json_resolver: Arc<CliPackageJsonResolver>,
redirect_resolver: Option<Arc<RedirectResolver>>,
graph_imports: Arc<IndexMap<ModuleSpecifier, GraphImport>>,
dep_info: Arc<Mutex<Arc<ScopeDepInfo>>>,
@ -384,7 +383,7 @@ impl LspResolver {
pub fn pkg_json_resolver(
&self,
referrer: &ModuleSpecifier,
) -> &Arc<PackageJsonResolver> {
) -> &Arc<CliPackageJsonResolver> {
let resolver = self.get_scope_resolver(Some(referrer));
&resolver.pkg_json_resolver
}
@ -592,28 +591,26 @@ struct ResolverFactoryServices {
cli_resolver: Deferred<Arc<CliResolver>>,
in_npm_pkg_checker: Deferred<Arc<dyn InNpmPackageChecker>>,
is_cjs_resolver: Deferred<Arc<IsCjsResolver>>,
node_resolver: Deferred<Option<Arc<NodeResolver>>>,
node_resolver: Deferred<Option<Arc<CliNodeResolver>>>,
npm_pkg_req_resolver: Deferred<Option<Arc<CliNpmReqResolver>>>,
npm_resolver: Option<Arc<dyn CliNpmResolver>>,
}
struct ResolverFactory<'a> {
config_data: Option<&'a Arc<ConfigData>>,
fs: Arc<dyn deno_fs::FileSystem>,
pkg_json_resolver: Arc<PackageJsonResolver>,
pkg_json_resolver: Arc<CliPackageJsonResolver>,
sys: CliSys,
services: ResolverFactoryServices,
}
impl<'a> ResolverFactory<'a> {
pub fn new(config_data: Option<&'a Arc<ConfigData>>) -> Self {
let fs = Arc::new(deno_fs::RealFs);
let pkg_json_resolver = Arc::new(PackageJsonResolver::new(
deno_runtime::deno_node::DenoFsNodeResolverEnv::new(fs.clone()),
));
let sys = CliSys::default();
let pkg_json_resolver = Arc::new(CliPackageJsonResolver::new(sys.clone()));
Self {
config_data,
fs,
pkg_json_resolver,
sys,
services: Default::default(),
}
}
@ -624,9 +621,10 @@ impl<'a> ResolverFactory<'a> {
cache: &LspCache,
) {
let enable_byonm = self.config_data.map(|d| d.byonm).unwrap_or(false);
let sys = CliSys::default();
let options = if enable_byonm {
CliNpmResolverCreateOptions::Byonm(CliByonmNpmResolverCreateOptions {
fs: CliDenoResolverFs(Arc::new(deno_fs::RealFs)),
sys,
pkg_json_resolver: self.pkg_json_resolver.clone(),
root_node_modules_dir: self.config_data.and_then(|config_data| {
config_data.node_modules_dir.clone().or_else(|| {
@ -642,12 +640,14 @@ impl<'a> ResolverFactory<'a> {
.and_then(|d| d.npmrc.clone())
.unwrap_or_else(create_default_npmrc);
let npm_cache_dir = Arc::new(NpmCacheDir::new(
&DenoCacheEnvFsAdapter(self.fs.as_ref()),
&sys,
cache.deno_dir().npm_folder_path(),
npmrc.get_all_known_registries_urls(),
));
CliNpmResolverCreateOptions::Managed(CliManagedNpmResolverCreateOptions {
http_client_provider: http_client_provider.clone(),
// only used for top level install, so we can ignore this
npm_install_deps_provider: Arc::new(NpmInstallDepsProvider::empty()),
snapshot: match self.config_data.and_then(|d| d.lockfile.as_ref()) {
Some(lockfile) => {
CliNpmResolverManagedSnapshotOption::ResolveFromLockfile(
@ -656,10 +656,7 @@ impl<'a> ResolverFactory<'a> {
}
None => CliNpmResolverManagedSnapshotOption::Specified(None),
},
// Don't provide the lockfile. We don't want these resolvers
// updating it. Only the cache request should update the lockfile.
maybe_lockfile: None,
fs: Arc::new(deno_fs::RealFs),
sys: CliSys::default(),
npm_cache_dir,
// Use an "only" cache setting in order to make the
// user do an explicit "cache" command and prevent
@ -667,11 +664,12 @@ impl<'a> ResolverFactory<'a> {
// the user is typing.
cache_setting: CacheSetting::Only,
text_only_progress_bar: ProgressBar::new(ProgressBarStyle::TextOnly),
// Don't provide the lockfile. We don't want these resolvers
// updating it. Only the cache request should update the lockfile.
maybe_lockfile: None,
maybe_node_modules_path: self
.config_data
.and_then(|d| d.node_modules_dir.clone()),
// only used for top level install, so we can ignore this
npm_install_deps_provider: Arc::new(NpmInstallDepsProvider::empty()),
npmrc,
npm_system_info: NpmSystemInfo::default(),
lifecycle_scripts: Default::default(),
@ -731,7 +729,7 @@ impl<'a> ResolverFactory<'a> {
})
}
pub fn pkg_json_resolver(&self) -> &Arc<PackageJsonResolver> {
pub fn pkg_json_resolver(&self) -> &Arc<CliPackageJsonResolver> {
&self.pkg_json_resolver
}
@ -772,17 +770,18 @@ impl<'a> ResolverFactory<'a> {
})
}
pub fn node_resolver(&self) -> Option<&Arc<NodeResolver>> {
pub fn node_resolver(&self) -> Option<&Arc<CliNodeResolver>> {
self
.services
.node_resolver
.get_or_init(|| {
let npm_resolver = self.services.npm_resolver.as_ref()?;
Some(Arc::new(NodeResolver::new(
deno_runtime::deno_node::DenoFsNodeResolverEnv::new(self.fs.clone()),
Some(Arc::new(CliNodeResolver::new(
self.in_npm_pkg_checker().clone(),
RealIsBuiltInNodeModuleChecker,
npm_resolver.clone().into_npm_pkg_folder_resolver(),
self.pkg_json_resolver.clone(),
self.sys.clone(),
)))
})
.as_ref()
@ -797,10 +796,10 @@ impl<'a> ResolverFactory<'a> {
let npm_resolver = self.npm_resolver()?;
Some(Arc::new(CliNpmReqResolver::new(NpmReqResolverOptions {
byonm_resolver: (npm_resolver.clone()).into_maybe_byonm(),
fs: CliDenoResolverFs(self.fs.clone()),
in_npm_pkg_checker: self.in_npm_pkg_checker().clone(),
node_resolver: node_resolver.clone(),
npm_req_resolver: npm_resolver.clone().into_npm_req_resolver(),
sys: self.sys.clone(),
})))
})
.as_ref()

4
cli/lsp/search.rs
View file

@ -67,7 +67,9 @@ pub mod tests {
&self,
nv: &PackageNv,
) -> Result<Arc<Vec<String>>, AnyError> {
let Some(exports_by_version) = self.package_versions.get(&nv.name) else {
let Some(exports_by_version) =
self.package_versions.get(nv.name.as_str())
else {
return Err(anyhow!("Package not found."));
};
let Some(exports) = exports_by_version.get(&nv.version) else {

58
cli/lsp/tsc.rs
View file

@ -64,6 +64,7 @@ use deno_core::OpState;
use deno_core::PollEventLoopOptions;
use deno_core::RuntimeOptions;
use deno_path_util::url_to_file_path;
use deno_runtime::deno_node::SUPPORTED_BUILTIN_NODE_MODULES;
use deno_runtime::inspector_server::InspectorServer;
use deno_runtime::tokio_util::create_basic_runtime;
use indexmap::IndexMap;
@ -3411,10 +3412,18 @@ fn parse_code_actions(
additional_text_edits.extend(change.text_changes.iter().map(|tc| {
let mut text_edit = tc.as_text_edit(asset_or_doc.line_index());
if let Some(specifier_rewrite) = &data.specifier_rewrite {
text_edit.new_text = text_edit.new_text.replace(
&specifier_rewrite.old_specifier,
&specifier_rewrite.new_specifier,
);
let specifier_index = text_edit
.new_text
.char_indices()
.find_map(|(b, c)| (c == '\'' || c == '"').then_some(b));
if let Some(i) = specifier_index {
let mut specifier_part = text_edit.new_text.split_off(i);
specifier_part = specifier_part.replace(
&specifier_rewrite.old_specifier,
&specifier_rewrite.new_specifier,
);
text_edit.new_text.push_str(&specifier_part);
}
if let Some(deno_types_specifier) =
&specifier_rewrite.new_deno_types_specifier
{
@ -3587,10 +3596,17 @@ impl CompletionEntryDetails {
&mut insert_replace_edit.new_text
}
};
*new_text = new_text.replace(
&specifier_rewrite.old_specifier,
&specifier_rewrite.new_specifier,
);
let specifier_index = new_text
.char_indices()
.find_map(|(b, c)| (c == '\'' || c == '"').then_some(b));
if let Some(i) = specifier_index {
let mut specifier_part = new_text.split_off(i);
specifier_part = specifier_part.replace(
&specifier_rewrite.old_specifier,
&specifier_rewrite.new_specifier,
);
new_text.push_str(&specifier_part);
}
if let Some(deno_types_specifier) =
&specifier_rewrite.new_deno_types_specifier
{
@ -3729,7 +3745,7 @@ pub struct CompletionItemData {
#[serde(rename_all = "camelCase")]
struct CompletionEntryDataAutoImport {
module_specifier: String,
file_name: String,
file_name: Option<String>,
}
#[derive(Debug)]
@ -3786,9 +3802,20 @@ impl CompletionEntry {
else {
return;
};
if let Ok(normalized) = specifier_map.normalize(&raw.file_name) {
self.auto_import_data =
Some(CompletionNormalizedAutoImportData { raw, normalized });
if let Some(file_name) = &raw.file_name {
if let Ok(normalized) = specifier_map.normalize(file_name) {
self.auto_import_data =
Some(CompletionNormalizedAutoImportData { raw, normalized });
}
} else if SUPPORTED_BUILTIN_NODE_MODULES
.contains(&raw.module_specifier.as_str())
{
if let Ok(normalized) =
resolve_url(&format!("node:{}", &raw.module_specifier))
{
self.auto_import_data =
Some(CompletionNormalizedAutoImportData { raw, normalized });
}
}
}
@ -5516,7 +5543,6 @@ impl TscRequest {
mod tests {
use super::*;
use crate::cache::HttpCache;
use crate::http_util::HeadersMap;
use crate::lsp::cache::LspCache;
use crate::lsp::config::Config;
use crate::lsp::config::WorkspaceSettings;
@ -5746,6 +5772,7 @@ mod tests {
"sourceLine": " import { A } from \".\";",
"category": 2,
"code": 6133,
"reportsUnnecessary": true,
}]
})
);
@ -5828,6 +5855,7 @@ mod tests {
"sourceLine": " import {",
"category": 2,
"code": 6192,
"reportsUnnecessary": true,
}, {
"start": {
"line": 8,
@ -5951,7 +5979,7 @@ mod tests {
.global()
.set(
&specifier_dep,
HeadersMap::default(),
Default::default(),
b"export const b = \"b\";\n",
)
.unwrap();
@ -5990,7 +6018,7 @@ mod tests {
.global()
.set(
&specifier_dep,
HeadersMap::default(),
Default::default(),
b"export const b = \"b\";\n\nexport const a = \"b\";\n",
)
.unwrap();

2
cli/main.rs
View file

@ -1,7 +1,6 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
mod args;
mod auth_tokens;
mod cache;
mod cdp;
mod emit;
@ -21,6 +20,7 @@ mod ops;
mod resolver;
mod shared;
mod standalone;
mod sys;
mod task_runner;
mod tools;
mod tsc;

8
cli/mainrt.rs
View file

@ -8,7 +8,6 @@
mod standalone;
mod args;
mod auth_tokens;
mod cache;
mod emit;
mod errors;
@ -19,6 +18,7 @@ mod node;
mod npm;
mod resolver;
mod shared;
mod sys;
mod task_runner;
mod util;
mod version;
@ -32,11 +32,13 @@ use deno_runtime::tokio_util::create_and_run_current_thread_with_maybe_metrics;
pub use deno_runtime::UNSTABLE_GRANULAR_FLAGS;
use deno_terminal::colors;
use indexmap::IndexMap;
use standalone::DenoCompileFileSystem;
use std::borrow::Cow;
use std::collections::HashMap;
use std::env;
use std::env::current_exe;
use std::sync::Arc;
use crate::args::Flags;
@ -93,7 +95,9 @@ fn main() {
Some(data.metadata.otel_config.clone()),
);
load_env_vars(&data.metadata.env_vars_from_env_file);
let exit_code = standalone::run(data).await?;
let fs = DenoCompileFileSystem::new(data.vfs.clone());
let sys = crate::sys::CliSys::DenoCompile(fs.clone());
let exit_code = standalone::run(Arc::new(fs), sys, data).await?;
deno_runtime::exit(exit_code);
}
Ok(None) => Ok(()),

89
cli/module_loader.rs
View file

@ -11,36 +11,8 @@ use std::sync::atomic::AtomicU16;
use std::sync::atomic::Ordering;
use std::sync::Arc;
use crate::args::jsr_url;
use crate::args::CliLockfile;
use crate::args::CliOptions;
use crate::args::DenoSubcommand;
use crate::args::TsTypeLib;
use crate::cache::CodeCache;
use crate::cache::FastInsecureHasher;
use crate::cache::ParsedSourceCache;
use crate::emit::Emitter;
use crate::graph_container::MainModuleGraphContainer;
use crate::graph_container::ModuleGraphContainer;
use crate::graph_container::ModuleGraphUpdatePermit;
use crate::graph_util::CreateGraphOptions;
use crate::graph_util::ModuleGraphBuilder;
use crate::node;
use crate::node::CliNodeCodeTranslator;
use crate::npm::CliNpmResolver;
use crate::resolver::CjsTracker;
use crate::resolver::CliNpmReqResolver;
use crate::resolver::CliResolver;
use crate::resolver::ModuleCodeStringSource;
use crate::resolver::NotSupportedKindInNpmError;
use crate::resolver::NpmModuleLoader;
use crate::tools::check;
use crate::tools::check::TypeChecker;
use crate::util::progress_bar::ProgressBar;
use crate::util::text_encoding::code_without_source_map;
use crate::util::text_encoding::source_map_from_code;
use crate::worker::CreateModuleLoaderResult;
use crate::worker::ModuleLoaderFactory;
use crate::node::CliNodeResolver;
use crate::sys::CliSys;
use deno_ast::MediaType;
use deno_ast::ModuleKind;
use deno_core::anyhow::anyhow;
@ -69,16 +41,45 @@ use deno_graph::ModuleGraph;
use deno_graph::Resolution;
use deno_graph::WasmModule;
use deno_runtime::code_cache;
use deno_runtime::deno_fs::FileSystem;
use deno_runtime::deno_node::create_host_defined_options;
use deno_runtime::deno_node::NodeRequireLoader;
use deno_runtime::deno_node::NodeResolver;
use deno_runtime::deno_permissions::PermissionsContainer;
use deno_semver::npm::NpmPackageReqReference;
use node_resolver::errors::ClosestPkgJsonError;
use node_resolver::InNpmPackageChecker;
use node_resolver::NodeResolutionKind;
use node_resolver::ResolutionMode;
use sys_traits::FsRead;
use crate::args::jsr_url;
use crate::args::CliLockfile;
use crate::args::CliOptions;
use crate::args::DenoSubcommand;
use crate::args::TsTypeLib;
use crate::cache::CodeCache;
use crate::cache::FastInsecureHasher;
use crate::cache::ParsedSourceCache;
use crate::emit::Emitter;
use crate::graph_container::MainModuleGraphContainer;
use crate::graph_container::ModuleGraphContainer;
use crate::graph_container::ModuleGraphUpdatePermit;
use crate::graph_util::CreateGraphOptions;
use crate::graph_util::ModuleGraphBuilder;
use crate::node::CliNodeCodeTranslator;
use crate::npm::CliNpmResolver;
use crate::resolver::CjsTracker;
use crate::resolver::CliNpmReqResolver;
use crate::resolver::CliResolver;
use crate::resolver::ModuleCodeStringSource;
use crate::resolver::NotSupportedKindInNpmError;
use crate::resolver::NpmModuleLoader;
use crate::tools::check;
use crate::tools::check::TypeChecker;
use crate::util::progress_bar::ProgressBar;
use crate::util::text_encoding::code_without_source_map;
use crate::util::text_encoding::source_map_from_code;
use crate::worker::CreateModuleLoaderResult;
use crate::worker::ModuleLoaderFactory;
pub struct ModuleLoadPreparer {
options: Arc<CliOptions>,
@ -215,17 +216,17 @@ struct SharedCliModuleLoaderState {
cjs_tracker: Arc<CjsTracker>,
code_cache: Option<Arc<CodeCache>>,
emitter: Arc<Emitter>,
fs: Arc<dyn FileSystem>,
in_npm_pkg_checker: Arc<dyn InNpmPackageChecker>,
main_module_graph_container: Arc<MainModuleGraphContainer>,
module_load_preparer: Arc<ModuleLoadPreparer>,
node_code_translator: Arc<CliNodeCodeTranslator>,
node_resolver: Arc<NodeResolver>,
node_resolver: Arc<CliNodeResolver>,
npm_req_resolver: Arc<CliNpmReqResolver>,
npm_resolver: Arc<dyn CliNpmResolver>,
npm_module_loader: NpmModuleLoader,
parsed_source_cache: Arc<ParsedSourceCache>,
resolver: Arc<CliResolver>,
sys: CliSys,
in_flight_loads_tracker: InFlightModuleLoadsTracker,
}
@ -275,17 +276,17 @@ impl CliModuleLoaderFactory {
cjs_tracker: Arc<CjsTracker>,
code_cache: Option<Arc<CodeCache>>,
emitter: Arc<Emitter>,
fs: Arc<dyn FileSystem>,
in_npm_pkg_checker: Arc<dyn InNpmPackageChecker>,
main_module_graph_container: Arc<MainModuleGraphContainer>,
module_load_preparer: Arc<ModuleLoadPreparer>,
node_code_translator: Arc<CliNodeCodeTranslator>,
node_resolver: Arc<NodeResolver>,
node_resolver: Arc<CliNodeResolver>,
npm_req_resolver: Arc<CliNpmReqResolver>,
npm_resolver: Arc<dyn CliNpmResolver>,
npm_module_loader: NpmModuleLoader,
parsed_source_cache: Arc<ParsedSourceCache>,
resolver: Arc<CliResolver>,
sys: CliSys,
) -> Self {
Self {
shared: Arc::new(SharedCliModuleLoaderState {
@ -301,7 +302,6 @@ impl CliModuleLoaderFactory {
cjs_tracker,
code_cache,
emitter,
fs,
in_npm_pkg_checker,
main_module_graph_container,
module_load_preparer,
@ -312,6 +312,7 @@ impl CliModuleLoaderFactory {
npm_module_loader,
parsed_source_cache,
resolver,
sys,
in_flight_loads_tracker: InFlightModuleLoadsTracker {
loads_number: Arc::new(AtomicU16::new(0)),
cleanup_task_timeout: 10_000,
@ -344,7 +345,7 @@ impl CliModuleLoaderFactory {
let node_require_loader = Rc::new(CliNodeRequireLoader {
cjs_tracker: self.shared.cjs_tracker.clone(),
emitter: self.shared.emitter.clone(),
fs: self.shared.fs.clone(),
sys: self.shared.sys.clone(),
graph_container,
in_npm_pkg_checker: self.shared.in_npm_pkg_checker.clone(),
npm_resolver: self.shared.npm_resolver.clone(),
@ -593,9 +594,9 @@ impl<TGraphContainer: ModuleGraphContainer>
Some(Module::Json(module)) => module.specifier.clone(),
Some(Module::Wasm(module)) => module.specifier.clone(),
Some(Module::External(module)) => {
node::resolve_specifier_into_node_modules(
node_resolver::resolve_specifier_into_node_modules(
&self.shared.sys,
&module.specifier,
self.shared.fs.as_ref(),
)
}
None => specifier.into_owned(),
@ -996,7 +997,7 @@ impl<TGraphContainer: ModuleGraphContainer> ModuleLoader
std::future::ready(()).boxed_local()
}
fn get_source_map(&self, file_name: &str) -> Option<Vec<u8>> {
fn get_source_map(&self, file_name: &str) -> Option<Cow<[u8]>> {
let specifier = resolve_url(file_name).ok()?;
match specifier.scheme() {
// we should only be looking for emits for schemes that denote external
@ -1008,7 +1009,7 @@ impl<TGraphContainer: ModuleGraphContainer> ModuleLoader
.0
.load_prepared_module_for_source_map_sync(&specifier)
.ok()??;
source_map_from_code(source.code.as_bytes())
source_map_from_code(source.code.as_bytes()).map(Cow::Owned)
}
fn get_source_mapped_source_line(
@ -1091,7 +1092,7 @@ impl ModuleGraphUpdatePermit for WorkerModuleGraphUpdatePermit {
struct CliNodeRequireLoader<TGraphContainer: ModuleGraphContainer> {
cjs_tracker: Arc<CjsTracker>,
emitter: Arc<Emitter>,
fs: Arc<dyn FileSystem>,
sys: CliSys,
graph_container: TGraphContainer,
in_npm_pkg_checker: Arc<dyn InNpmPackageChecker>,
npm_resolver: Arc<dyn CliNpmResolver>,
@ -1120,7 +1121,7 @@ impl<TGraphContainer: ModuleGraphContainer> NodeRequireLoader
) -> Result<Cow<'static, str>, AnyError> {
// todo(dsherret): use the preloaded module from the graph if available?
let media_type = MediaType::from_path(path);
let text = self.fs.read_text_file_lossy_sync(path, None)?;
let text = self.sys.fs_read_to_string_lossy(path)?;
if media_type.is_emittable() {
let specifier = deno_path_util::url_from_file_path(path)?;
if self.in_npm_pkg_checker.in_npm_package(&specifier) {

27
cli/node.rs
View file

@ -8,7 +8,7 @@ use deno_ast::ModuleSpecifier;
use deno_core::error::AnyError;
use deno_graph::ParsedSourceStore;
use deno_runtime::deno_fs;
use deno_runtime::deno_node::DenoFsNodeResolverEnv;
use deno_runtime::deno_node::RealIsBuiltInNodeModuleChecker;
use node_resolver::analyze::CjsAnalysis as ExtNodeCjsAnalysis;
use node_resolver::analyze::CjsAnalysisExports;
use node_resolver::analyze::CjsCodeAnalyzer;
@ -20,24 +20,15 @@ use crate::cache::CacheDBHash;
use crate::cache::NodeAnalysisCache;
use crate::cache::ParsedSourceCache;
use crate::resolver::CjsTracker;
use crate::sys::CliSys;
pub type CliNodeCodeTranslator =
NodeCodeTranslator<CliCjsCodeAnalyzer, DenoFsNodeResolverEnv>;
/// Resolves a specifier that is pointing into a node_modules folder.
///
/// Note: This should be called whenever getting the specifier from
/// a Module::External(module) reference because that module might
/// not be fully resolved at the time deno_graph is analyzing it
/// because the node_modules folder might not exist at that time.
pub fn resolve_specifier_into_node_modules(
specifier: &ModuleSpecifier,
fs: &dyn deno_fs::FileSystem,
) -> ModuleSpecifier {
node_resolver::resolve_specifier_into_node_modules(specifier, &|path| {
fs.realpath_sync(path).map_err(|err| err.into_io_error())
})
}
pub type CliNodeCodeTranslator = NodeCodeTranslator<
CliCjsCodeAnalyzer,
RealIsBuiltInNodeModuleChecker,
CliSys,
>;
pub type CliNodeResolver = deno_runtime::deno_node::NodeResolver<CliSys>;
pub type CliPackageJsonResolver = node_resolver::PackageJsonResolver<CliSys>;
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub enum CliCjsAnalysis {

8
cli/npm/byonm.rs
View file

@ -4,27 +4,25 @@ use std::borrow::Cow;
use std::path::Path;
use std::sync::Arc;
use crate::sys::CliSys;
use deno_core::error::AnyError;
use deno_core::serde_json;
use deno_resolver::npm::ByonmNpmResolver;
use deno_resolver::npm::ByonmNpmResolverCreateOptions;
use deno_resolver::npm::CliNpmReqResolver;
use deno_runtime::deno_node::DenoFsNodeResolverEnv;
use deno_runtime::deno_node::NodePermissions;
use deno_runtime::ops::process::NpmProcessStateProvider;
use node_resolver::NpmPackageFolderResolver;
use crate::args::NpmProcessState;
use crate::args::NpmProcessStateKind;
use crate::resolver::CliDenoResolverFs;
use super::CliNpmResolver;
use super::InnerCliNpmResolverRef;
pub type CliByonmNpmResolverCreateOptions =
ByonmNpmResolverCreateOptions<CliDenoResolverFs, DenoFsNodeResolverEnv>;
pub type CliByonmNpmResolver =
ByonmNpmResolver<CliDenoResolverFs, DenoFsNodeResolverEnv>;
ByonmNpmResolverCreateOptions<CliSys>;
pub type CliByonmNpmResolver = ByonmNpmResolver<CliSys>;
// todo(dsherret): the services hanging off `CliNpmResolver` doesn't seem ideal. We should probably decouple.
#[derive(Debug)]

111
cli/npm/managed/mod.rs
View file

@ -5,6 +5,7 @@ use std::path::Path;
use std::path::PathBuf;
use std::sync::Arc;
use crate::sys::CliSys;
use deno_ast::ModuleSpecifier;
use deno_cache_dir::npm::NpmCacheDir;
use deno_core::anyhow::Context;
@ -20,9 +21,10 @@ use deno_npm::resolution::ValidSerializedNpmResolutionSnapshot;
use deno_npm::NpmPackageId;
use deno_npm::NpmResolutionPackage;
use deno_npm::NpmSystemInfo;
use deno_npm_cache::NpmCacheSetting;
use deno_path_util::fs::canonicalize_path_maybe_not_exists;
use deno_resolver::npm::CliNpmReqResolver;
use deno_runtime::colors;
use deno_runtime::deno_fs::FileSystem;
use deno_runtime::deno_node::NodePermissions;
use deno_runtime::ops::process::NpmProcessStateProvider;
use deno_semver::package::PackageNv;
@ -40,7 +42,6 @@ use crate::args::NpmProcessState;
use crate::args::NpmProcessStateKind;
use crate::args::PackageJsonDepValueParseWithLocationError;
use crate::cache::FastInsecureHasher;
use crate::util::fs::canonicalize_path_maybe_not_exists_with_fs;
use crate::util::progress_bar::ProgressBar;
use crate::util::sync::AtomicFlag;
@ -49,7 +50,7 @@ use self::resolvers::create_npm_fs_resolver;
use self::resolvers::NpmPackageFsResolver;
use super::CliNpmCache;
use super::CliNpmCacheEnv;
use super::CliNpmCacheHttpClient;
use super::CliNpmRegistryInfoProvider;
use super::CliNpmResolver;
use super::CliNpmTarballCache;
@ -67,10 +68,10 @@ pub enum CliNpmResolverManagedSnapshotOption {
pub struct CliManagedNpmResolverCreateOptions {
pub snapshot: CliNpmResolverManagedSnapshotOption,
pub maybe_lockfile: Option<Arc<CliLockfile>>,
pub fs: Arc<dyn deno_runtime::deno_fs::FileSystem>,
pub http_client_provider: Arc<crate::http_util::HttpClientProvider>,
pub npm_cache_dir: Arc<NpmCacheDir>,
pub cache_setting: crate::args::CacheSetting,
pub sys: CliSys,
pub cache_setting: deno_cache_dir::file_fetcher::CacheSetting,
pub text_only_progress_bar: crate::util::progress_bar::ProgressBar,
pub maybe_node_modules_path: Option<PathBuf>,
pub npm_system_info: NpmSystemInfo,
@ -82,9 +83,12 @@ pub struct CliManagedNpmResolverCreateOptions {
pub async fn create_managed_npm_resolver_for_lsp(
options: CliManagedNpmResolverCreateOptions,
) -> Arc<dyn CliNpmResolver> {
let cache_env = create_cache_env(&options);
let npm_cache = create_cache(cache_env.clone(), &options);
let npm_api = create_api(npm_cache.clone(), cache_env.clone(), &options);
let npm_cache = create_cache(&options);
let http_client = Arc::new(CliNpmCacheHttpClient::new(
options.http_client_provider.clone(),
options.text_only_progress_bar.clone(),
));
let npm_api = create_api(npm_cache.clone(), http_client.clone(), &options);
// spawn due to the lsp's `Send` requirement
deno_core::unsync::spawn(async move {
let snapshot = match resolve_snapshot(&npm_api, options.snapshot).await {
@ -95,14 +99,14 @@ pub async fn create_managed_npm_resolver_for_lsp(
}
};
create_inner(
cache_env,
options.fs,
options.maybe_lockfile,
npm_api,
http_client,
npm_cache,
options.npmrc,
options.npm_install_deps_provider,
npm_api,
options.sys,
options.text_only_progress_bar,
options.maybe_lockfile,
options.npmrc,
options.maybe_node_modules_path,
options.npm_system_info,
snapshot,
@ -116,19 +120,22 @@ pub async fn create_managed_npm_resolver_for_lsp(
pub async fn create_managed_npm_resolver(
options: CliManagedNpmResolverCreateOptions,
) -> Result<Arc<dyn CliNpmResolver>, AnyError> {
let npm_cache_env = create_cache_env(&options);
let npm_cache = create_cache(npm_cache_env.clone(), &options);
let api = create_api(npm_cache.clone(), npm_cache_env.clone(), &options);
let npm_cache = create_cache(&options);
let http_client = Arc::new(CliNpmCacheHttpClient::new(
options.http_client_provider.clone(),
options.text_only_progress_bar.clone(),
));
let api = create_api(npm_cache.clone(), http_client.clone(), &options);
let snapshot = resolve_snapshot(&api, options.snapshot).await?;
Ok(create_inner(
npm_cache_env,
options.fs,
options.maybe_lockfile,
api,
http_client,
npm_cache,
options.npmrc,
options.npm_install_deps_provider,
api,
options.sys,
options.text_only_progress_bar,
options.maybe_lockfile,
options.npmrc,
options.maybe_node_modules_path,
options.npm_system_info,
snapshot,
@ -138,14 +145,14 @@ pub async fn create_managed_npm_resolver(
#[allow(clippy::too_many_arguments)]
fn create_inner(
env: Arc<CliNpmCacheEnv>,
fs: Arc<dyn deno_runtime::deno_fs::FileSystem>,
maybe_lockfile: Option<Arc<CliLockfile>>,
registry_info_provider: Arc<CliNpmRegistryInfoProvider>,
http_client: Arc<CliNpmCacheHttpClient>,
npm_cache: Arc<CliNpmCache>,
npm_rc: Arc<ResolvedNpmRc>,
npm_install_deps_provider: Arc<NpmInstallDepsProvider>,
registry_info_provider: Arc<CliNpmRegistryInfoProvider>,
sys: CliSys,
text_only_progress_bar: crate::util::progress_bar::ProgressBar,
maybe_lockfile: Option<Arc<CliLockfile>>,
npm_rc: Arc<ResolvedNpmRc>,
node_modules_dir_path: Option<PathBuf>,
npm_system_info: NpmSystemInfo,
snapshot: Option<ValidSerializedNpmResolutionSnapshot>,
@ -158,28 +165,29 @@ fn create_inner(
));
let tarball_cache = Arc::new(CliNpmTarballCache::new(
npm_cache.clone(),
env,
http_client,
sys.clone(),
npm_rc.clone(),
));
let fs_resolver = create_npm_fs_resolver(
fs.clone(),
npm_cache.clone(),
&npm_install_deps_provider,
&text_only_progress_bar,
resolution.clone(),
sys.clone(),
tarball_cache.clone(),
node_modules_dir_path,
npm_system_info.clone(),
lifecycle_scripts.clone(),
);
Arc::new(ManagedCliNpmResolver::new(
fs,
fs_resolver,
maybe_lockfile,
registry_info_provider,
npm_cache,
npm_install_deps_provider,
resolution,
sys,
tarball_cache,
text_only_progress_bar,
npm_system_info,
@ -187,36 +195,25 @@ fn create_inner(
))
}
fn create_cache_env(
options: &CliManagedNpmResolverCreateOptions,
) -> Arc<CliNpmCacheEnv> {
Arc::new(CliNpmCacheEnv::new(
options.fs.clone(),
options.http_client_provider.clone(),
options.text_only_progress_bar.clone(),
))
}
fn create_cache(
env: Arc<CliNpmCacheEnv>,
options: &CliManagedNpmResolverCreateOptions,
) -> Arc<CliNpmCache> {
Arc::new(CliNpmCache::new(
options.npm_cache_dir.clone(),
options.cache_setting.as_npm_cache_setting(),
env,
options.sys.clone(),
NpmCacheSetting::from_cache_setting(&options.cache_setting),
options.npmrc.clone(),
))
}
fn create_api(
cache: Arc<CliNpmCache>,
env: Arc<CliNpmCacheEnv>,
http_client: Arc<CliNpmCacheHttpClient>,
options: &CliManagedNpmResolverCreateOptions,
) -> Arc<CliNpmRegistryInfoProvider> {
Arc::new(CliNpmRegistryInfoProvider::new(
cache,
env,
http_client,
options.npmrc.clone(),
))
}
@ -305,12 +302,12 @@ pub enum PackageCaching<'a> {
/// An npm resolver where the resolution is managed by Deno rather than
/// the user bringing their own node_modules (BYONM) on the file system.
pub struct ManagedCliNpmResolver {
fs: Arc<dyn FileSystem>,
fs_resolver: Arc<dyn NpmPackageFsResolver>,
maybe_lockfile: Option<Arc<CliLockfile>>,
registry_info_provider: Arc<CliNpmRegistryInfoProvider>,
npm_cache: Arc<CliNpmCache>,
npm_install_deps_provider: Arc<NpmInstallDepsProvider>,
sys: CliSys,
resolution: Arc<NpmResolution>,
tarball_cache: Arc<CliNpmTarballCache>,
text_only_progress_bar: ProgressBar,
@ -330,20 +327,19 @@ impl std::fmt::Debug for ManagedCliNpmResolver {
impl ManagedCliNpmResolver {
#[allow(clippy::too_many_arguments)]
pub fn new(
fs: Arc<dyn FileSystem>,
fs_resolver: Arc<dyn NpmPackageFsResolver>,
maybe_lockfile: Option<Arc<CliLockfile>>,
registry_info_provider: Arc<CliNpmRegistryInfoProvider>,
npm_cache: Arc<CliNpmCache>,
npm_install_deps_provider: Arc<NpmInstallDepsProvider>,
resolution: Arc<NpmResolution>,
sys: CliSys,
tarball_cache: Arc<CliNpmTarballCache>,
text_only_progress_bar: ProgressBar,
npm_system_info: NpmSystemInfo,
lifecycle_scripts: LifecycleScriptsConfig,
) -> Self {
Self {
fs,
fs_resolver,
maybe_lockfile,
registry_info_provider,
@ -351,6 +347,7 @@ impl ManagedCliNpmResolver {
npm_install_deps_provider,
text_only_progress_bar,
resolution,
sys,
tarball_cache,
npm_system_info,
top_level_install_flag: Default::default(),
@ -363,8 +360,7 @@ impl ManagedCliNpmResolver {
pkg_id: &NpmPackageId,
) -> Result<PathBuf, AnyError> {
let path = self.fs_resolver.package_folder(pkg_id)?;
let path =
canonicalize_path_maybe_not_exists_with_fs(&path, self.fs.as_ref())?;
let path = canonicalize_path_maybe_not_exists(&self.sys, &path)?;
log::debug!(
"Resolved package folder of {} to {}",
pkg_id.as_serialized(),
@ -559,11 +555,11 @@ impl ManagedCliNpmResolver {
&self,
) -> Result<(), Box<PackageJsonDepValueParseWithLocationError>> {
for err in self.npm_install_deps_provider.pkg_json_dep_errors() {
match &err.source {
deno_package_json::PackageJsonDepValueParseError::VersionReq(_) => {
match err.source.as_kind() {
deno_package_json::PackageJsonDepValueParseErrorKind::VersionReq(_) => {
return Err(Box::new(err.clone()));
}
deno_package_json::PackageJsonDepValueParseError::Unsupported {
deno_package_json::PackageJsonDepValueParseErrorKind::Unsupported {
..
} => {
// only warn for this one
@ -666,12 +662,13 @@ impl NpmPackageFolderResolver for ManagedCliNpmResolver {
.fs_resolver
.resolve_package_folder_from_package(name, referrer)?;
let path =
canonicalize_path_maybe_not_exists_with_fs(&path, self.fs.as_ref())
.map_err(|err| PackageFolderResolveIoError {
canonicalize_path_maybe_not_exists(&self.sys, &path).map_err(|err| {
PackageFolderResolveIoError {
package_name: name.to_string(),
referrer: referrer.clone(),
source: err,
})?;
}
})?;
log::debug!("Resolved {} from {} to {}", name, referrer, path.display());
Ok(path)
}
@ -727,13 +724,12 @@ impl CliNpmResolver for ManagedCliNpmResolver {
));
Arc::new(ManagedCliNpmResolver::new(
self.fs.clone(),
create_npm_fs_resolver(
self.fs.clone(),
self.npm_cache.clone(),
&self.npm_install_deps_provider,
&self.text_only_progress_bar,
npm_resolution.clone(),
self.sys.clone(),
self.tarball_cache.clone(),
self.root_node_modules_path().map(ToOwned::to_owned),
self.npm_system_info.clone(),
@ -744,6 +740,7 @@ impl CliNpmResolver for ManagedCliNpmResolver {
self.npm_cache.clone(),
self.npm_install_deps_provider.clone(),
npm_resolution,
self.sys.clone(),
self.tarball_cache.clone(),
self.text_only_progress_bar.clone(),
self.npm_system_info.clone(),

10
cli/npm/managed/resolution.rs
View file

@ -4,6 +4,7 @@ use std::collections::HashMap;
use std::collections::HashSet;
use std::sync::Arc;
use capacity_builder::StringBuilder;
use deno_core::error::AnyError;
use deno_lockfile::NpmPackageDependencyLockfileInfo;
use deno_lockfile::NpmPackageLockfileInfo;
@ -24,6 +25,7 @@ use deno_npm::NpmSystemInfo;
use deno_semver::jsr::JsrDepPackageReq;
use deno_semver::package::PackageNv;
use deno_semver::package::PackageReq;
use deno_semver::SmallStackString;
use deno_semver::VersionReq;
use crate::args::CliLockfile;
@ -336,7 +338,13 @@ fn populate_lockfile_from_snapshot(
let id = &snapshot.resolve_package_from_deno_module(nv).unwrap().id;
lockfile.insert_package_specifier(
JsrDepPackageReq::npm(package_req.clone()),
format!("{}{}", id.nv.version, id.peer_deps_serialized()),
{
StringBuilder::<SmallStackString>::build(|builder| {
builder.append(&id.nv.version);
builder.append(&id.peer_dependencies);
})
.unwrap()
},
);
}
for package in snapshot.all_packages_for_every_system() {

13
cli/npm/managed/resolvers/common.rs
View file

@ -11,7 +11,6 @@ use std::path::PathBuf;
use std::sync::Arc;
use std::sync::Mutex;
use super::super::PackageCaching;
use async_trait::async_trait;
use deno_ast::ModuleSpecifier;
use deno_core::anyhow::Context;
@ -21,11 +20,13 @@ use deno_core::futures::StreamExt;
use deno_npm::NpmPackageCacheFolderId;
use deno_npm::NpmPackageId;
use deno_npm::NpmResolutionPackage;
use deno_runtime::deno_fs::FileSystem;
use deno_runtime::deno_node::NodePermissions;
use node_resolver::errors::PackageFolderResolveError;
use sys_traits::FsCanonicalize;
use super::super::PackageCaching;
use crate::npm::CliNpmTarballCache;
use crate::sys::CliSys;
/// Part of the resolution that interacts with the file system.
#[async_trait(?Send)]
@ -73,15 +74,15 @@ pub trait NpmPackageFsResolver: Send + Sync {
#[derive(Debug)]
pub struct RegistryReadPermissionChecker {
fs: Arc<dyn FileSystem>,
sys: CliSys,
cache: Mutex<HashMap<PathBuf, PathBuf>>,
registry_path: PathBuf,
}
impl RegistryReadPermissionChecker {
pub fn new(fs: Arc<dyn FileSystem>, registry_path: PathBuf) -> Self {
pub fn new(sys: CliSys, registry_path: PathBuf) -> Self {
Self {
fs,
sys,
registry_path,
cache: Default::default(),
}
@ -108,7 +109,7 @@ impl RegistryReadPermissionChecker {
|path: &Path| -> Result<Option<PathBuf>, AnyError> {
match cache.get(path) {
Some(canon) => Ok(Some(canon.clone())),
None => match self.fs.realpath_sync(path) {
None => match self.sys.fs_canonicalize(path) {
Ok(canon) => {
cache.insert(path.to_path_buf(), canon.clone());
Ok(Some(canon))

4
cli/npm/managed/resolvers/common/bin_entries.rs
View file

@ -28,8 +28,10 @@ fn default_bin_name(package: &NpmResolutionPackage) -> &str {
.id
.nv
.name
.as_str()
.rsplit_once('/')
.map_or(package.id.nv.name.as_str(), |(_, name)| name)
.map(|(_, name)| name)
.unwrap_or(package.id.nv.name.as_str())
}
pub fn warn_missing_entrypoint(

6
cli/npm/managed/resolvers/global.rs
View file

@ -11,6 +11,7 @@ use crate::colors;
use crate::npm::managed::PackageCaching;
use crate::npm::CliNpmCache;
use crate::npm::CliNpmTarballCache;
use crate::sys::CliSys;
use async_trait::async_trait;
use deno_ast::ModuleSpecifier;
use deno_core::error::AnyError;
@ -18,7 +19,6 @@ use deno_npm::NpmPackageCacheFolderId;
use deno_npm::NpmPackageId;
use deno_npm::NpmResolutionPackage;
use deno_npm::NpmSystemInfo;
use deno_runtime::deno_fs::FileSystem;
use deno_runtime::deno_node::NodePermissions;
use node_resolver::errors::PackageFolderResolveError;
use node_resolver::errors::PackageNotFoundError;
@ -47,15 +47,15 @@ pub struct GlobalNpmPackageResolver {
impl GlobalNpmPackageResolver {
pub fn new(
cache: Arc<CliNpmCache>,
fs: Arc<dyn FileSystem>,
tarball_cache: Arc<CliNpmTarballCache>,
resolution: Arc<NpmResolution>,
sys: CliSys,
system_info: NpmSystemInfo,
lifecycle_scripts: LifecycleScriptsConfig,
) -> Self {
Self {
registry_read_permission_checker: RegistryReadPermissionChecker::new(
fs,
sys,
cache.root_dir_path().to_path_buf(),
),
cache,

53
cli/npm/managed/resolvers/local.rs
View file

@ -15,11 +15,6 @@ use std::path::PathBuf;
use std::rc::Rc;
use std::sync::Arc;
use crate::args::LifecycleScriptsConfig;
use crate::colors;
use crate::npm::managed::PackageCaching;
use crate::npm::CliNpmCache;
use crate::npm::CliNpmTarballCache;
use async_trait::async_trait;
use deno_ast::ModuleSpecifier;
use deno_cache_dir::npm::mixed_case_package_name_decode;
@ -34,21 +29,28 @@ use deno_npm::NpmPackageCacheFolderId;
use deno_npm::NpmPackageId;
use deno_npm::NpmResolutionPackage;
use deno_npm::NpmSystemInfo;
use deno_path_util::fs::atomic_write_file_with_retries;
use deno_path_util::fs::canonicalize_path_maybe_not_exists;
use deno_resolver::npm::normalize_pkg_name_for_node_modules_deno_folder;
use deno_runtime::deno_fs;
use deno_runtime::deno_node::NodePermissions;
use deno_semver::package::PackageNv;
use deno_semver::StackString;
use node_resolver::errors::PackageFolderResolveError;
use node_resolver::errors::PackageFolderResolveIoError;
use node_resolver::errors::PackageNotFoundError;
use node_resolver::errors::ReferrerNotFoundError;
use serde::Deserialize;
use serde::Serialize;
use sys_traits::FsMetadata;
use crate::args::LifecycleScriptsConfig;
use crate::args::NpmInstallDepsProvider;
use crate::cache::CACHE_PERM;
use crate::util::fs::atomic_write_file_with_retries;
use crate::util::fs::canonicalize_path_maybe_not_exists_with_fs;
use crate::colors;
use crate::npm::managed::PackageCaching;
use crate::npm::CliNpmCache;
use crate::npm::CliNpmTarballCache;
use crate::sys::CliSys;
use crate::util::fs::clone_dir_recursive;
use crate::util::fs::symlink_dir;
use crate::util::fs::LaxSingleProcessFsFlag;
@ -65,10 +67,10 @@ use super::common::RegistryReadPermissionChecker;
#[derive(Debug)]
pub struct LocalNpmPackageResolver {
cache: Arc<CliNpmCache>,
fs: Arc<dyn deno_fs::FileSystem>,
npm_install_deps_provider: Arc<NpmInstallDepsProvider>,
progress_bar: ProgressBar,
resolution: Arc<NpmResolution>,
sys: CliSys,
tarball_cache: Arc<CliNpmTarballCache>,
root_node_modules_path: PathBuf,
root_node_modules_url: Url,
@ -81,10 +83,10 @@ impl LocalNpmPackageResolver {
#[allow(clippy::too_many_arguments)]
pub fn new(
cache: Arc<CliNpmCache>,
fs: Arc<dyn deno_fs::FileSystem>,
npm_install_deps_provider: Arc<NpmInstallDepsProvider>,
progress_bar: ProgressBar,
resolution: Arc<NpmResolution>,
sys: CliSys,
tarball_cache: Arc<CliNpmTarballCache>,
node_modules_folder: PathBuf,
system_info: NpmSystemInfo,
@ -92,15 +94,15 @@ impl LocalNpmPackageResolver {
) -> Self {
Self {
cache,
fs: fs.clone(),
npm_install_deps_provider,
progress_bar,
resolution,
tarball_cache,
registry_read_permission_checker: RegistryReadPermissionChecker::new(
fs,
sys.clone(),
node_modules_folder.clone(),
),
sys,
root_node_modules_url: Url::from_directory_path(&node_modules_folder)
.unwrap(),
root_node_modules_path: node_modules_folder,
@ -139,8 +141,7 @@ impl LocalNpmPackageResolver {
};
// Canonicalize the path so it's not pointing to the symlinked directory
// in `node_modules` directory of the referrer.
canonicalize_path_maybe_not_exists_with_fs(&path, self.fs.as_ref())
.map(Some)
canonicalize_path_maybe_not_exists(&self.sys, &path).map(Some)
}
fn resolve_package_folder_from_specifier(
@ -209,7 +210,7 @@ impl NpmPackageFsResolver for LocalNpmPackageResolver {
};
let sub_dir = join_package_name(&node_modules_folder, name);
if self.fs.is_dir_sync(&sub_dir) {
if self.sys.fs_is_dir_no_err(&sub_dir) {
return Ok(sub_dir);
}
@ -355,8 +356,10 @@ async fn sync_resolution_with_fs(
let package_partitions =
snapshot.all_system_packages_partitioned(system_info);
let mut cache_futures = FuturesUnordered::new();
let mut newest_packages_by_name: HashMap<&String, &NpmResolutionPackage> =
HashMap::with_capacity(package_partitions.packages.len());
let mut newest_packages_by_name: HashMap<
&StackString,
&NpmResolutionPackage,
> = HashMap::with_capacity(package_partitions.packages.len());
let bin_entries = Rc::new(RefCell::new(bin_entries::BinEntries::new()));
let mut lifecycle_scripts =
super::common::lifecycle_scripts::LifecycleScripts::new(
@ -536,7 +539,7 @@ async fn sync_resolution_with_fs(
}
}
let mut found_names: HashMap<&String, &PackageNv> = HashMap::new();
let mut found_names: HashMap<&StackString, &PackageNv> = HashMap::new();
// set of node_modules in workspace packages that we've already ensured exist
let mut existing_child_node_modules_dirs: HashSet<PathBuf> = HashSet::new();
@ -922,7 +925,13 @@ impl SetupCache {
}
bincode::serialize(&self.current).ok().and_then(|data| {
atomic_write_file_with_retries(&self.file_path, data, CACHE_PERM).ok()
atomic_write_file_with_retries(
&CliSys::default(),
&self.file_path,
&data,
CACHE_PERM,
)
.ok()
});
true
}
@ -1012,10 +1021,10 @@ fn get_package_folder_id_from_folder_name(
) -> Option<NpmPackageCacheFolderId> {
let folder_name = folder_name.replace('+', "/");
let (name, ending) = folder_name.rsplit_once('@')?;
let name = if let Some(encoded_name) = name.strip_prefix('_') {
mixed_case_package_name_decode(encoded_name)?
let name: StackString = if let Some(encoded_name) = name.strip_prefix('_') {
StackString::from_string(mixed_case_package_name_decode(encoded_name)?)
} else {
name.to_string()
name.into()
};
let (raw_version, copy_index) = match ending.split_once('_') {
Some((raw_version, copy_index)) => {

8
cli/npm/managed/resolvers/mod.rs
View file

@ -7,8 +7,8 @@ mod local;
use std::path::PathBuf;
use std::sync::Arc;
use crate::sys::CliSys;
use deno_npm::NpmSystemInfo;
use deno_runtime::deno_fs::FileSystem;
use crate::args::LifecycleScriptsConfig;
use crate::args::NpmInstallDepsProvider;
@ -25,11 +25,11 @@ use super::resolution::NpmResolution;
#[allow(clippy::too_many_arguments)]
pub fn create_npm_fs_resolver(
fs: Arc<dyn FileSystem>,
npm_cache: Arc<CliNpmCache>,
npm_install_deps_provider: &Arc<NpmInstallDepsProvider>,
progress_bar: &ProgressBar,
resolution: Arc<NpmResolution>,
sys: CliSys,
tarball_cache: Arc<CliNpmTarballCache>,
maybe_node_modules_path: Option<PathBuf>,
system_info: NpmSystemInfo,
@ -38,10 +38,10 @@ pub fn create_npm_fs_resolver(
match maybe_node_modules_path {
Some(node_modules_folder) => Arc::new(LocalNpmPackageResolver::new(
npm_cache,
fs,
npm_install_deps_provider.clone(),
progress_bar.clone(),
resolution,
sys,
tarball_cache,
node_modules_folder,
system_info,
@ -49,9 +49,9 @@ pub fn create_npm_fs_resolver(
)),
None => Arc::new(GlobalNpmPackageResolver::new(
npm_cache,
fs,
tarball_cache,
resolution,
sys,
system_info,
lifecycle_scripts,
)),

61
cli/npm/mod.rs
View file

@ -7,6 +7,7 @@ use std::borrow::Cow;
use std::path::Path;
use std::sync::Arc;
use crate::sys::CliSys;
use dashmap::DashMap;
use deno_core::error::AnyError;
use deno_core::serde_json;
@ -17,7 +18,6 @@ use deno_resolver::npm::ByonmInNpmPackageChecker;
use deno_resolver::npm::ByonmNpmResolver;
use deno_resolver::npm::CliNpmReqResolver;
use deno_resolver::npm::ResolvePkgFolderFromDenoReqError;
use deno_runtime::deno_fs::FileSystem;
use deno_runtime::deno_node::NodePermissions;
use deno_runtime::ops::process::NpmProcessStateProvider;
use deno_semver::package::PackageNv;
@ -28,11 +28,8 @@ use managed::create_managed_in_npm_pkg_checker;
use node_resolver::InNpmPackageChecker;
use node_resolver::NpmPackageFolderResolver;
use crate::file_fetcher::FileFetcher;
use crate::file_fetcher::CliFileFetcher;
use crate::http_util::HttpClientProvider;
use crate::util::fs::atomic_write_file_with_retries_and_fs;
use crate::util::fs::hard_link_dir_recursive;
use crate::util::fs::AtomicWriteFileFsAdapter;
use crate::util::progress_bar::ProgressBar;
pub use self::byonm::CliByonmNpmResolver;
@ -43,26 +40,24 @@ pub use self::managed::CliNpmResolverManagedSnapshotOption;
pub use self::managed::ManagedCliNpmResolver;
pub use self::managed::PackageCaching;
pub type CliNpmTarballCache = deno_npm_cache::TarballCache<CliNpmCacheEnv>;
pub type CliNpmCache = deno_npm_cache::NpmCache<CliNpmCacheEnv>;
pub type CliNpmTarballCache =
deno_npm_cache::TarballCache<CliNpmCacheHttpClient, CliSys>;
pub type CliNpmCache = deno_npm_cache::NpmCache<CliSys>;
pub type CliNpmRegistryInfoProvider =
deno_npm_cache::RegistryInfoProvider<CliNpmCacheEnv>;
deno_npm_cache::RegistryInfoProvider<CliNpmCacheHttpClient, CliSys>;
#[derive(Debug)]
pub struct CliNpmCacheEnv {
fs: Arc<dyn FileSystem>,
pub struct CliNpmCacheHttpClient {
http_client_provider: Arc<HttpClientProvider>,
progress_bar: ProgressBar,
}
impl CliNpmCacheEnv {
impl CliNpmCacheHttpClient {
pub fn new(
fs: Arc<dyn FileSystem>,
http_client_provider: Arc<HttpClientProvider>,
progress_bar: ProgressBar,
) -> Self {
Self {
fs,
http_client_provider,
progress_bar,
}
@ -70,35 +65,7 @@ impl CliNpmCacheEnv {
}
#[async_trait::async_trait(?Send)]
impl deno_npm_cache::NpmCacheEnv for CliNpmCacheEnv {
fn exists(&self, path: &Path) -> bool {
self.fs.exists_sync(path)
}
fn hard_link_dir_recursive(
&self,
from: &Path,
to: &Path,
) -> Result<(), AnyError> {
// todo(dsherret): use self.fs here instead
hard_link_dir_recursive(from, to)
}
fn atomic_write_file_with_retries(
&self,
file_path: &Path,
data: &[u8],
) -> std::io::Result<()> {
atomic_write_file_with_retries_and_fs(
&AtomicWriteFileFsAdapter {
fs: self.fs.as_ref(),
write_mode: crate::cache::CACHE_PERM,
},
file_path,
data,
)
}
impl deno_npm_cache::NpmCacheHttpClient for CliNpmCacheHttpClient {
async fn download_with_retries_on_any_tokio_runtime(
&self,
url: Url,
@ -115,14 +82,14 @@ impl deno_npm_cache::NpmCacheEnv for CliNpmCacheEnv {
.download_with_progress_and_retries(url, maybe_auth_header, &guard)
.await
.map_err(|err| {
use crate::http_util::DownloadError::*;
let status_code = match &err {
use crate::http_util::DownloadErrorKind::*;
let status_code = match err.as_kind() {
Fetch { .. }
| UrlParse { .. }
| HttpParse { .. }
| Json { .. }
| ToStr { .. }
| NoRedirectHeader { .. }
| RedirectHeaderParse { .. }
| TooManyRedirects => None,
BadResponse(bad_response_error) => {
Some(bad_response_error.status_code)
@ -232,13 +199,13 @@ pub trait CliNpmResolver: NpmPackageFolderResolver + CliNpmReqResolver {
pub struct NpmFetchResolver {
nv_by_req: DashMap<PackageReq, Option<PackageNv>>,
info_by_name: DashMap<String, Option<Arc<NpmPackageInfo>>>,
file_fetcher: Arc<FileFetcher>,
file_fetcher: Arc<CliFileFetcher>,
npmrc: Arc<ResolvedNpmRc>,
}
impl NpmFetchResolver {
pub fn new(
file_fetcher: Arc<FileFetcher>,
file_fetcher: Arc<CliFileFetcher>,
npmrc: Arc<ResolvedNpmRc>,
) -> Self {
Self {

34
cli/ops/lint.rs Normal file
View file

@ -0,0 +1,34 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use deno_ast::MediaType;
use deno_ast::ModuleSpecifier;
use deno_core::error::generic_error;
use deno_core::error::AnyError;
use deno_core::op2;
use crate::tools::lint;
deno_core::extension!(deno_lint, ops = [op_lint_create_serialized_ast,],);
#[op2]
#[buffer]
fn op_lint_create_serialized_ast(
#[string] file_name: &str,
#[string] source: String,
) -> Result<Vec<u8>, AnyError> {
let file_text = deno_ast::strip_bom(source);
let path = std::env::current_dir()?.join(file_name);
let specifier = ModuleSpecifier::from_file_path(&path).map_err(|_| {
generic_error(format!("Failed to parse path as URL: {}", path.display()))
})?;
let media_type = MediaType::from_specifier(&specifier);
let parsed_source = deno_ast::parse_program(deno_ast::ParseParams {
specifier,
text: file_text.into(),
media_type,
capture_tokens: false,
scope_analysis: false,
maybe_syntax: None,
})?;
Ok(lint::serialize_ast_to_buffer(&parsed_source))
}

1
cli/ops/mod.rs
View file

@ -2,4 +2,5 @@
pub mod bench;
pub mod jupyter;
pub mod lint;
pub mod testing;

92
cli/resolver.rs
View file

@ -1,5 +1,11 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use std::borrow::Cow;
use std::path::Path;
use std::path::PathBuf;
use std::sync::Arc;
use crate::sys::CliSys;
use async_trait::async_trait;
use dashmap::DashMap;
use dashmap::DashSet;
@ -20,16 +26,13 @@ use deno_npm::resolution::NpmResolutionError;
use deno_resolver::sloppy_imports::SloppyImportsResolver;
use deno_runtime::colors;
use deno_runtime::deno_fs;
use deno_runtime::deno_fs::FileSystem;
use deno_runtime::deno_node::is_builtin_node_module;
use deno_runtime::deno_node::DenoFsNodeResolverEnv;
use deno_runtime::deno_node::RealIsBuiltInNodeModuleChecker;
use deno_semver::package::PackageReq;
use node_resolver::NodeResolutionKind;
use node_resolver::ResolutionMode;
use std::borrow::Cow;
use std::path::Path;
use std::path::PathBuf;
use std::sync::Arc;
use sys_traits::FsMetadata;
use sys_traits::FsMetadataValue;
use thiserror::Error;
use crate::args::NpmCachingStrategy;
@ -40,18 +43,17 @@ use crate::npm::InnerCliNpmResolverRef;
use crate::util::sync::AtomicFlag;
use crate::util::text_encoding::from_utf8_lossy_cow;
pub type CjsTracker = deno_resolver::cjs::CjsTracker<DenoFsNodeResolverEnv>;
pub type IsCjsResolver =
deno_resolver::cjs::IsCjsResolver<DenoFsNodeResolverEnv>;
pub type CjsTracker = deno_resolver::cjs::CjsTracker<CliSys>;
pub type IsCjsResolver = deno_resolver::cjs::IsCjsResolver<CliSys>;
pub type CliSloppyImportsResolver =
SloppyImportsResolver<SloppyImportsCachedFs>;
pub type CliDenoResolver = deno_resolver::DenoResolver<
CliDenoResolverFs,
DenoFsNodeResolverEnv,
RealIsBuiltInNodeModuleChecker,
SloppyImportsCachedFs,
CliSys,
>;
pub type CliNpmReqResolver =
deno_resolver::npm::NpmReqResolver<CliDenoResolverFs, DenoFsNodeResolverEnv>;
deno_resolver::npm::NpmReqResolver<RealIsBuiltInNodeModuleChecker, CliSys>;
pub struct ModuleCodeStringSource {
pub code: ModuleSourceCode,
@ -59,53 +61,6 @@ pub struct ModuleCodeStringSource {
pub media_type: MediaType,
}
#[derive(Debug, Clone)]
pub struct CliDenoResolverFs(pub Arc<dyn FileSystem>);
impl deno_resolver::fs::DenoResolverFs for CliDenoResolverFs {
fn read_to_string_lossy(
&self,
path: &Path,
) -> std::io::Result<Cow<'static, str>> {
self
.0
.read_text_file_lossy_sync(path, None)
.map_err(|e| e.into_io_error())
}
fn realpath_sync(&self, path: &Path) -> std::io::Result<PathBuf> {
self.0.realpath_sync(path).map_err(|e| e.into_io_error())
}
fn exists_sync(&self, path: &Path) -> bool {
self.0.exists_sync(path)
}
fn is_dir_sync(&self, path: &Path) -> bool {
self.0.is_dir_sync(path)
}
fn read_dir_sync(
&self,
dir_path: &Path,
) -> std::io::Result<Vec<deno_resolver::fs::DirEntry>> {
self
.0
.read_dir_sync(dir_path)
.map(|entries| {
entries
.into_iter()
.map(|e| deno_resolver::fs::DirEntry {
name: e.name,
is_file: e.is_file,
is_directory: e.is_directory,
})
.collect::<Vec<_>>()
})
.map_err(|err| err.into_io_error())
}
}
#[derive(Debug, Error)]
#[error("{media_type} files are not supported in npm packages: {specifier}")]
pub struct NotSupportedKindInNpmError {
@ -440,7 +395,7 @@ impl<'a> deno_graph::source::NpmResolver for WorkerCliNpmGraphResolver<'a> {
#[derive(Debug)]
pub struct SloppyImportsCachedFs {
fs: Arc<dyn deno_fs::FileSystem>,
sys: CliSys,
cache: Option<
DashMap<
PathBuf,
@ -450,15 +405,18 @@ pub struct SloppyImportsCachedFs {
}
impl SloppyImportsCachedFs {
pub fn new(fs: Arc<dyn FileSystem>) -> Self {
pub fn new(sys: CliSys) -> Self {
Self {
fs,
sys,
cache: Some(Default::default()),
}
}
pub fn new_without_stat_cache(fs: Arc<dyn FileSystem>) -> Self {
Self { fs, cache: None }
pub fn new_without_stat_cache(fs: CliSys) -> Self {
Self {
sys: fs,
cache: None,
}
}
}
@ -475,10 +433,10 @@ impl deno_resolver::sloppy_imports::SloppyImportResolverFs
}
}
let entry = self.fs.stat_sync(path).ok().and_then(|stat| {
if stat.is_file {
let entry = self.sys.fs_metadata(path).ok().and_then(|stat| {
if stat.file_type().is_file() {
Some(deno_resolver::sloppy_imports::SloppyImportsFsEntry::File)
} else if stat.is_directory {
} else if stat.file_type().is_dir() {
Some(deno_resolver::sloppy_imports::SloppyImportsFsEntry::Dir)
} else {
None

1
cli/schemas/config-file.v1.json
View file

@ -446,7 +446,6 @@
},
"command": {
"type": "string",
"required": true,
"description": "The task to execute"
},
"dependencies": {

204
cli/standalone/binary.rs
View file

@ -37,7 +37,6 @@ use deno_core::futures::AsyncReadExt;
use deno_core::futures::AsyncSeekExt;
use deno_core::serde_json;
use deno_core::url::Url;
use deno_graph::source::RealFileSystem;
use deno_graph::ModuleGraph;
use deno_npm::resolution::SerializedNpmResolutionSnapshot;
use deno_npm::resolution::SerializedNpmResolutionSnapshotPackage;
@ -71,7 +70,7 @@ use crate::args::UnstableConfig;
use crate::cache::DenoDir;
use crate::cache::FastInsecureHasher;
use crate::emit::Emitter;
use crate::file_fetcher::FileFetcher;
use crate::file_fetcher::CliFileFetcher;
use crate::http_util::HttpClientProvider;
use crate::npm::CliNpmResolver;
use crate::npm::InnerCliNpmResolverRef;
@ -91,6 +90,7 @@ use super::serialization::DenoCompileModuleData;
use super::serialization::DeserializedDataSection;
use super::serialization::RemoteModulesStore;
use super::serialization::RemoteModulesStoreBuilder;
use super::serialization::SourceMapStore;
use super::virtual_fs::output_vfs;
use super::virtual_fs::BuiltVfs;
use super::virtual_fs::FileBackedVfs;
@ -98,6 +98,7 @@ use super::virtual_fs::VfsBuilder;
use super::virtual_fs::VfsFileSubDataKind;
use super::virtual_fs::VfsRoot;
use super::virtual_fs::VirtualDirectory;
use super::virtual_fs::VirtualDirectoryEntries;
use super::virtual_fs::WindowsSystemRootablePath;
pub static DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME: &str =
@ -203,18 +204,25 @@ pub struct Metadata {
pub otel_config: OtelConfig,
}
#[allow(clippy::too_many_arguments)]
fn write_binary_bytes(
mut file_writer: File,
original_bin: Vec<u8>,
metadata: &Metadata,
npm_snapshot: Option<SerializedNpmResolutionSnapshot>,
remote_modules: &RemoteModulesStoreBuilder,
source_map_store: &SourceMapStore,
vfs: &BuiltVfs,
compile_flags: &CompileFlags,
) -> Result<(), AnyError> {
let data_section_bytes =
serialize_binary_data_section(metadata, npm_snapshot, remote_modules, vfs)
.context("Serializing binary data section.")?;
let data_section_bytes = serialize_binary_data_section(
metadata,
npm_snapshot,
remote_modules,
source_map_store,
vfs,
)
.context("Serializing binary data section.")?;
let target = compile_flags.resolve_target();
if target.contains("linux") {
@ -251,11 +259,11 @@ pub fn is_standalone_binary(exe_path: &Path) -> bool {
}
pub struct StandaloneData {
pub fs: Arc<dyn deno_fs::FileSystem>,
pub metadata: Metadata,
pub modules: StandaloneModules,
pub npm_snapshot: Option<ValidSerializedNpmResolutionSnapshot>,
pub root_path: PathBuf,
pub source_maps: SourceMapStore,
pub vfs: Arc<FileBackedVfs>,
}
@ -283,13 +291,12 @@ impl StandaloneModules {
pub fn read<'a>(
&'a self,
specifier: &'a ModuleSpecifier,
kind: VfsFileSubDataKind,
) -> Result<Option<DenoCompileModuleData<'a>>, AnyError> {
if specifier.scheme() == "file" {
let path = deno_path_util::url_to_file_path(specifier)?;
let bytes = match self.vfs.file_entry(&path) {
Ok(entry) => self
.vfs
.read_file_all(entry, VfsFileSubDataKind::ModuleGraph)?,
Ok(entry) => self.vfs.read_file_all(entry, kind)?,
Err(err) if err.kind() == ErrorKind::NotFound => {
match RealFs.read_file_sync(&path, None) {
Ok(bytes) => bytes,
@ -307,7 +314,18 @@ impl StandaloneModules {
data: bytes,
}))
} else {
self.remote_modules.read(specifier)
self.remote_modules.read(specifier).map(|maybe_entry| {
maybe_entry.map(|entry| DenoCompileModuleData {
media_type: entry.media_type,
specifier: entry.specifier,
data: match kind {
VfsFileSubDataKind::Raw => entry.data,
VfsFileSubDataKind::ModuleGraph => {
entry.transpiled_data.unwrap_or(entry.data)
}
},
})
})
}
}
}
@ -328,7 +346,8 @@ pub fn extract_standalone(
mut metadata,
npm_snapshot,
remote_modules,
mut vfs_dir,
source_maps,
vfs_root_entries,
vfs_files_data,
} = match deserialize_binary_data_section(data)? {
Some(data_section) => data_section,
@ -351,20 +370,18 @@ pub fn extract_standalone(
metadata.argv.push(arg.into_string().unwrap());
}
let vfs = {
// align the name of the directory with the root dir
vfs_dir.name = root_path.file_name().unwrap().to_string_lossy().to_string();
let fs_root = VfsRoot {
dir: vfs_dir,
dir: VirtualDirectory {
// align the name of the directory with the root dir
name: root_path.file_name().unwrap().to_string_lossy().to_string(),
entries: vfs_root_entries,
},
root_path: root_path.clone(),
start_file_offset: 0,
};
Arc::new(FileBackedVfs::new(Cow::Borrowed(vfs_files_data), fs_root))
};
let fs: Arc<dyn deno_fs::FileSystem> =
Arc::new(DenoCompileFileSystem::new(vfs.clone()));
Ok(Some(StandaloneData {
fs,
metadata,
modules: StandaloneModules {
remote_modules,
@ -372,6 +389,7 @@ pub fn extract_standalone(
},
npm_snapshot,
root_path,
source_maps,
vfs,
}))
}
@ -390,7 +408,7 @@ pub struct DenoCompileBinaryWriter<'a> {
cli_options: &'a CliOptions,
deno_dir: &'a DenoDir,
emitter: &'a Emitter,
file_fetcher: &'a FileFetcher,
file_fetcher: &'a CliFileFetcher,
http_client_provider: &'a HttpClientProvider,
npm_resolver: &'a dyn CliNpmResolver,
workspace_resolver: &'a WorkspaceResolver,
@ -404,7 +422,7 @@ impl<'a> DenoCompileBinaryWriter<'a> {
cli_options: &'a CliOptions,
deno_dir: &'a DenoDir,
emitter: &'a Emitter,
file_fetcher: &'a FileFetcher,
file_fetcher: &'a CliFileFetcher,
http_client_provider: &'a HttpClientProvider,
npm_resolver: &'a dyn CliNpmResolver,
workspace_resolver: &'a WorkspaceResolver,
@ -451,7 +469,7 @@ impl<'a> DenoCompileBinaryWriter<'a> {
)
}
}
self.write_standalone_binary(options, original_binary).await
self.write_standalone_binary(options, original_binary)
}
async fn get_base_binary(
@ -554,7 +572,7 @@ impl<'a> DenoCompileBinaryWriter<'a> {
/// This functions creates a standalone deno binary by appending a bundle
/// and magic trailer to the currently executing binary.
#[allow(clippy::too_many_arguments)]
async fn write_standalone_binary(
fn write_standalone_binary(
&self,
options: WriteBinOptions<'_>,
original_bin: Vec<u8>,
@ -598,71 +616,81 @@ impl<'a> DenoCompileBinaryWriter<'a> {
.with_context(|| format!("Including {}", path.display()))?;
}
let mut remote_modules_store = RemoteModulesStoreBuilder::default();
let mut code_cache_key_hasher = if self.cli_options.code_cache_enabled() {
Some(FastInsecureHasher::new_deno_versioned())
} else {
None
};
let mut source_maps = Vec::with_capacity(graph.specifiers_count());
// todo(dsherret): transpile in parallel
for module in graph.modules() {
if module.specifier().scheme() == "data" {
continue; // don't store data urls as an entry as they're in the code
}
if let Some(hasher) = &mut code_cache_key_hasher {
if let Some(source) = module.source() {
hasher.write(module.specifier().as_str().as_bytes());
hasher.write(source.as_bytes());
}
}
let (maybe_source, media_type) = match module {
let (maybe_original_source, maybe_transpiled, media_type) = match module {
deno_graph::Module::Js(m) => {
let source = if m.media_type.is_emittable() {
let original_bytes = m.source.as_bytes().to_vec();
let maybe_transpiled = if m.media_type.is_emittable() {
let is_cjs = self.cjs_tracker.is_cjs_with_known_is_script(
&m.specifier,
m.media_type,
m.is_script,
)?;
let module_kind = ModuleKind::from_is_cjs(is_cjs);
let source = self
.emitter
.emit_parsed_source(
let (source, source_map) =
self.emitter.emit_parsed_source_for_deno_compile(
&m.specifier,
m.media_type,
module_kind,
&m.source,
)
.await?;
source.into_bytes()
)?;
if source != m.source.as_ref() {
source_maps.push((&m.specifier, source_map));
Some(source.into_bytes())
} else {
None
}
} else {
m.source.as_bytes().to_vec()
None
};
(Some(source), m.media_type)
(Some(original_bytes), maybe_transpiled, m.media_type)
}
deno_graph::Module::Json(m) => {
(Some(m.source.as_bytes().to_vec()), m.media_type)
(Some(m.source.as_bytes().to_vec()), None, m.media_type)
}
deno_graph::Module::Wasm(m) => {
(Some(m.source.to_vec()), MediaType::Wasm)
(Some(m.source.to_vec()), None, MediaType::Wasm)
}
deno_graph::Module::Npm(_)
| deno_graph::Module::Node(_)
| deno_graph::Module::External(_) => (None, MediaType::Unknown),
| deno_graph::Module::External(_) => (None, None, MediaType::Unknown),
};
if module.specifier().scheme() == "file" {
let file_path = deno_path_util::url_to_file_path(module.specifier())?;
vfs
.add_file_with_data(
&file_path,
match maybe_source {
Some(source) => source,
None => RealFs.read_file_sync(&file_path, None)?.into_owned(),
},
VfsFileSubDataKind::ModuleGraph,
)
.with_context(|| {
format!("Failed adding '{}'", file_path.display())
})?;
} else if let Some(source) = maybe_source {
remote_modules_store.add(module.specifier(), media_type, source);
if let Some(original_source) = maybe_original_source {
if module.specifier().scheme() == "file" {
let file_path = deno_path_util::url_to_file_path(module.specifier())?;
vfs
.add_file_with_data(
&file_path,
original_source,
VfsFileSubDataKind::Raw,
)
.with_context(|| {
format!("Failed adding '{}'", file_path.display())
})?;
if let Some(transpiled_source) = maybe_transpiled {
vfs
.add_file_with_data(
&file_path,
transpiled_source,
VfsFileSubDataKind::ModuleGraph,
)
.with_context(|| {
format!("Failed adding '{}'", file_path.display())
})?;
}
} else {
remote_modules_store.add(
module.specifier(),
media_type,
original_source,
maybe_transpiled,
);
}
}
}
remote_modules_store.add_redirects(&graph.redirects);
@ -695,6 +723,28 @@ impl<'a> DenoCompileBinaryWriter<'a> {
None => StandaloneRelativeFileBaseUrl::WindowsSystemRoot,
};
let code_cache_key = if self.cli_options.code_cache_enabled() {
let mut hasher = FastInsecureHasher::new_deno_versioned();
for module in graph.modules() {
if let Some(source) = module.source() {
hasher
.write(root_dir_url.specifier_key(module.specifier()).as_bytes());
hasher.write(source.as_bytes());
}
}
Some(hasher.finish())
} else {
None
};
let mut source_map_store = SourceMapStore::with_capacity(source_maps.len());
for (specifier, source_map) in source_maps {
source_map_store.add(
Cow::Owned(root_dir_url.specifier_key(specifier).into_owned()),
Cow::Owned(source_map.into_bytes()),
);
}
let node_modules = match self.npm_resolver.as_inner() {
InnerCliNpmResolverRef::Managed(_) => {
npm_snapshot.as_ref().map(|_| NodeModules::Managed {
@ -742,7 +792,7 @@ impl<'a> DenoCompileBinaryWriter<'a> {
let metadata = Metadata {
argv: compile_flags.args.clone(),
seed: self.cli_options.seed(),
code_cache_key: code_cache_key_hasher.map(|h| h.finish()),
code_cache_key,
location: self.cli_options.location_flag().clone(),
permissions: self.cli_options.permission_flags().clone(),
v8_flags: self.cli_options.v8_flags().clone(),
@ -809,6 +859,7 @@ impl<'a> DenoCompileBinaryWriter<'a> {
&metadata,
npm_snapshot.map(|s| s.into_serialized()),
&remote_modules_store,
&source_map_store,
&vfs,
compile_flags,
)
@ -903,10 +954,10 @@ impl<'a> DenoCompileBinaryWriter<'a> {
root_dir.name = DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME.to_string();
let mut new_entries = Vec::with_capacity(root_dir.entries.len());
let mut localhost_entries = IndexMap::new();
for entry in std::mem::take(&mut root_dir.entries) {
for entry in root_dir.entries.take_inner() {
match entry {
VfsEntry::Dir(dir) => {
for entry in dir.entries {
VfsEntry::Dir(mut dir) => {
for entry in dir.entries.take_inner() {
log::debug!("Flattening {} into node_modules", entry.name());
if let Some(existing) =
localhost_entries.insert(entry.name().to_string(), entry)
@ -925,11 +976,11 @@ impl<'a> DenoCompileBinaryWriter<'a> {
}
new_entries.push(VfsEntry::Dir(VirtualDirectory {
name: "localhost".to_string(),
entries: localhost_entries.into_iter().map(|(_, v)| v).collect(),
entries: VirtualDirectoryEntries::new(
localhost_entries.into_iter().map(|(_, v)| v).collect(),
),
}));
// needs to be sorted by name
new_entries.sort_by(|a, b| a.name().cmp(b.name()));
root_dir.entries = new_entries;
root_dir.entries = VirtualDirectoryEntries::new(new_entries);
// it's better to not expose the user's cache directory, so take it out
// of there
@ -937,10 +988,7 @@ impl<'a> DenoCompileBinaryWriter<'a> {
let parent_dir = vfs.get_dir_mut(parent).unwrap();
let index = parent_dir
.entries
.iter()
.position(|entry| {
entry.name() == DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME
})
.binary_search(DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME)
.unwrap();
let npm_global_cache_dir_entry = parent_dir.entries.remove(index);
@ -950,11 +998,7 @@ impl<'a> DenoCompileBinaryWriter<'a> {
Cow::Borrowed(DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME);
for ancestor in parent.ancestors() {
let dir = vfs.get_dir_mut(ancestor).unwrap();
if let Some(index) = dir
.entries
.iter()
.position(|entry| entry.name() == last_name)
{
if let Ok(index) = dir.entries.binary_search(&last_name) {
dir.entries.remove(index);
}
last_name = Cow::Owned(dir.name.clone());
@ -965,7 +1009,7 @@ impl<'a> DenoCompileBinaryWriter<'a> {
// now build the vfs and add the global cache dir entry there
let mut built_vfs = vfs.build();
built_vfs.root.insert_entry(npm_global_cache_dir_entry);
built_vfs.entries.insert(npm_global_cache_dir_entry);
built_vfs
}
InnerCliNpmResolverRef::Byonm(_) => vfs.build(),

5
cli/standalone/code_cache.rs
View file

@ -15,11 +15,11 @@ use deno_core::anyhow::bail;
use deno_core::error::AnyError;
use deno_core::parking_lot::Mutex;
use deno_core::unsync::sync::AtomicFlag;
use deno_path_util::get_atomic_path;
use deno_runtime::code_cache::CodeCache;
use deno_runtime::code_cache::CodeCacheType;
use crate::cache::FastInsecureHasher;
use crate::util::path::get_atomic_file_path;
use crate::worker::CliCodeCache;
enum CodeCacheStrategy {
@ -189,7 +189,8 @@ impl FirstRunCodeCacheStrategy {
cache_data: &HashMap<CodeCacheKey, DenoCompileCodeCacheEntry>,
) {
let count = cache_data.len();
let temp_file = get_atomic_file_path(&self.file_path);
let temp_file =
get_atomic_path(&sys_traits::impls::RealSys, &self.file_path);
match serialize(&temp_file, self.cache_key, cache_data) {
Ok(()) => {
if let Err(err) = std::fs::rename(&temp_file, &self.file_path) {

449
cli/standalone/file_system.rs
View file

@ -1,9 +1,13 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use std::borrow::Cow;
use std::io::ErrorKind;
use std::path::Path;
use std::path::PathBuf;
use std::rc::Rc;
use std::sync::Arc;
use std::time::Duration;
use std::time::SystemTime;
use deno_runtime::deno_fs::AccessCheckCb;
use deno_runtime::deno_fs::FileSystem;
@ -15,8 +19,16 @@ use deno_runtime::deno_io::fs::File;
use deno_runtime::deno_io::fs::FsError;
use deno_runtime::deno_io::fs::FsResult;
use deno_runtime::deno_io::fs::FsStat;
use sys_traits::boxed::BoxedFsDirEntry;
use sys_traits::boxed::BoxedFsMetadataValue;
use sys_traits::boxed::FsMetadataBoxed;
use sys_traits::boxed::FsReadDirBoxed;
use sys_traits::FsMetadata;
use super::virtual_fs::FileBackedVfs;
use super::virtual_fs::FileBackedVfsDirEntry;
use super::virtual_fs::FileBackedVfsFile;
use super::virtual_fs::FileBackedVfsMetadata;
use super::virtual_fs::VfsFileSubDataKind;
#[derive(Debug, Clone)]
@ -82,7 +94,7 @@ impl FileSystem for DenoCompileFileSystem {
access_check: Option<AccessCheckCb>,
) -> FsResult<Rc<dyn File>> {
if self.0.is_path_within(path) {
Ok(self.0.open_file(path)?)
Ok(Rc::new(self.0.open_file(path)?))
} else {
RealFs.open_sync(path, options, access_check)
}
@ -94,7 +106,7 @@ impl FileSystem for DenoCompileFileSystem {
access_check: Option<AccessCheckCb<'a>>,
) -> FsResult<Rc<dyn File>> {
if self.0.is_path_within(&path) {
Ok(self.0.open_file(&path)?)
Ok(Rc::new(self.0.open_file(&path)?))
} else {
RealFs.open_async(path, options, access_check).await
}
@ -214,14 +226,14 @@ impl FileSystem for DenoCompileFileSystem {
fn stat_sync(&self, path: &Path) -> FsResult<FsStat> {
if self.0.is_path_within(path) {
Ok(self.0.stat(path)?)
Ok(self.0.stat(path)?.as_fs_stat())
} else {
RealFs.stat_sync(path)
}
}
async fn stat_async(&self, path: PathBuf) -> FsResult<FsStat> {
if self.0.is_path_within(&path) {
Ok(self.0.stat(&path)?)
Ok(self.0.stat(&path)?.as_fs_stat())
} else {
RealFs.stat_async(path).await
}
@ -229,14 +241,14 @@ impl FileSystem for DenoCompileFileSystem {
fn lstat_sync(&self, path: &Path) -> FsResult<FsStat> {
if self.0.is_path_within(path) {
Ok(self.0.lstat(path)?)
Ok(self.0.lstat(path)?.as_fs_stat())
} else {
RealFs.lstat_sync(path)
}
}
async fn lstat_async(&self, path: PathBuf) -> FsResult<FsStat> {
if self.0.is_path_within(&path) {
Ok(self.0.lstat(&path)?)
Ok(self.0.lstat(&path)?.as_fs_stat())
} else {
RealFs.lstat_async(path).await
}
@ -397,3 +409,428 @@ impl FileSystem for DenoCompileFileSystem {
.await
}
}
impl sys_traits::BaseFsHardLink for DenoCompileFileSystem {
#[inline]
fn base_fs_hard_link(&self, src: &Path, dst: &Path) -> std::io::Result<()> {
self.link_sync(src, dst).map_err(|err| err.into_io_error())
}
}
impl sys_traits::BaseFsRead for DenoCompileFileSystem {
#[inline]
fn base_fs_read(&self, path: &Path) -> std::io::Result<Cow<'static, [u8]>> {
self
.read_file_sync(path, None)
.map_err(|err| err.into_io_error())
}
}
impl sys_traits::FsMetadataValue for FileBackedVfsMetadata {
fn file_type(&self) -> sys_traits::FileType {
self.file_type
}
fn len(&self) -> u64 {
self.len
}
fn accessed(&self) -> std::io::Result<SystemTime> {
Err(not_supported("accessed time"))
}
fn created(&self) -> std::io::Result<SystemTime> {
Err(not_supported("created time"))
}
fn changed(&self) -> std::io::Result<SystemTime> {
Err(not_supported("changed time"))
}
fn modified(&self) -> std::io::Result<SystemTime> {
Err(not_supported("modified time"))
}
fn dev(&self) -> std::io::Result<u64> {
Ok(0)
}
fn ino(&self) -> std::io::Result<u64> {
Ok(0)
}
fn mode(&self) -> std::io::Result<u32> {
Ok(0)
}
fn nlink(&self) -> std::io::Result<u64> {
Ok(0)
}
fn uid(&self) -> std::io::Result<u32> {
Ok(0)
}
fn gid(&self) -> std::io::Result<u32> {
Ok(0)
}
fn rdev(&self) -> std::io::Result<u64> {
Ok(0)
}
fn blksize(&self) -> std::io::Result<u64> {
Ok(0)
}
fn blocks(&self) -> std::io::Result<u64> {
Ok(0)
}
fn is_block_device(&self) -> std::io::Result<bool> {
Ok(false)
}
fn is_char_device(&self) -> std::io::Result<bool> {
Ok(false)
}
fn is_fifo(&self) -> std::io::Result<bool> {
Ok(false)
}
fn is_socket(&self) -> std::io::Result<bool> {
Ok(false)
}
fn file_attributes(&self) -> std::io::Result<u32> {
Ok(0)
}
}
fn not_supported(name: &str) -> std::io::Error {
std::io::Error::new(
ErrorKind::Unsupported,
format!(
"{} is not supported for an embedded deno compile file",
name
),
)
}
impl sys_traits::FsDirEntry for FileBackedVfsDirEntry {
type Metadata = BoxedFsMetadataValue;
fn file_name(&self) -> Cow<std::ffi::OsStr> {
Cow::Borrowed(self.metadata.name.as_ref())
}
fn file_type(&self) -> std::io::Result<sys_traits::FileType> {
Ok(self.metadata.file_type)
}
fn metadata(&self) -> std::io::Result<Self::Metadata> {
Ok(BoxedFsMetadataValue(Box::new(self.metadata.clone())))
}
fn path(&self) -> Cow<Path> {
Cow::Owned(self.parent_path.join(&self.metadata.name))
}
}
impl sys_traits::BaseFsReadDir for DenoCompileFileSystem {
type ReadDirEntry = BoxedFsDirEntry;
fn base_fs_read_dir(
&self,
path: &Path,
) -> std::io::Result<
Box<dyn Iterator<Item = std::io::Result<Self::ReadDirEntry>> + '_>,
> {
if self.0.is_path_within(path) {
let entries = self.0.read_dir_with_metadata(path)?;
Ok(Box::new(
entries.map(|entry| Ok(BoxedFsDirEntry::new(entry))),
))
} else {
#[allow(clippy::disallowed_types)] // ok because we're implementing the fs
sys_traits::impls::RealSys.fs_read_dir_boxed(path)
}
}
}
impl sys_traits::BaseFsCanonicalize for DenoCompileFileSystem {
#[inline]
fn base_fs_canonicalize(&self, path: &Path) -> std::io::Result<PathBuf> {
self.realpath_sync(path).map_err(|err| err.into_io_error())
}
}
impl sys_traits::BaseFsMetadata for DenoCompileFileSystem {
type Metadata = BoxedFsMetadataValue;
#[inline]
fn base_fs_metadata(&self, path: &Path) -> std::io::Result<Self::Metadata> {
if self.0.is_path_within(path) {
Ok(BoxedFsMetadataValue::new(self.0.stat(path)?))
} else {
#[allow(clippy::disallowed_types)] // ok because we're implementing the fs
sys_traits::impls::RealSys.fs_metadata_boxed(path)
}
}
#[inline]
fn base_fs_symlink_metadata(
&self,
path: &Path,
) -> std::io::Result<Self::Metadata> {
if self.0.is_path_within(path) {
Ok(BoxedFsMetadataValue::new(self.0.lstat(path)?))
} else {
#[allow(clippy::disallowed_types)] // ok because we're implementing the fs
sys_traits::impls::RealSys.fs_symlink_metadata_boxed(path)
}
}
}
impl sys_traits::BaseFsCreateDir for DenoCompileFileSystem {
#[inline]
fn base_fs_create_dir(
&self,
path: &Path,
options: &sys_traits::CreateDirOptions,
) -> std::io::Result<()> {
self
.mkdir_sync(path, options.recursive, options.mode)
.map_err(|err| err.into_io_error())
}
}
impl sys_traits::BaseFsRemoveFile for DenoCompileFileSystem {
#[inline]
fn base_fs_remove_file(&self, path: &Path) -> std::io::Result<()> {
self
.remove_sync(path, false)
.map_err(|err| err.into_io_error())
}
}
impl sys_traits::BaseFsRename for DenoCompileFileSystem {
#[inline]
fn base_fs_rename(&self, from: &Path, to: &Path) -> std::io::Result<()> {
self
.rename_sync(from, to)
.map_err(|err| err.into_io_error())
}
}
pub enum FsFileAdapter {
Real(sys_traits::impls::RealFsFile),
Vfs(FileBackedVfsFile),
}
impl sys_traits::FsFile for FsFileAdapter {}
impl sys_traits::FsFileAsRaw for FsFileAdapter {
#[cfg(windows)]
fn fs_file_as_raw_handle(&self) -> Option<std::os::windows::io::RawHandle> {
match self {
Self::Real(file) => file.fs_file_as_raw_handle(),
Self::Vfs(_) => None,
}
}
#[cfg(unix)]
fn fs_file_as_raw_fd(&self) -> Option<std::os::fd::RawFd> {
match self {
Self::Real(file) => file.fs_file_as_raw_fd(),
Self::Vfs(_) => None,
}
}
}
impl sys_traits::FsFileSyncData for FsFileAdapter {
fn fs_file_sync_data(&mut self) -> std::io::Result<()> {
match self {
Self::Real(file) => file.fs_file_sync_data(),
Self::Vfs(_) => Ok(()),
}
}
}
impl sys_traits::FsFileSyncAll for FsFileAdapter {
fn fs_file_sync_all(&mut self) -> std::io::Result<()> {
match self {
Self::Real(file) => file.fs_file_sync_all(),
Self::Vfs(_) => Ok(()),
}
}
}
impl sys_traits::FsFileSetPermissions for FsFileAdapter {
#[inline]
fn fs_file_set_permissions(&mut self, mode: u32) -> std::io::Result<()> {
match self {
Self::Real(file) => file.fs_file_set_permissions(mode),
Self::Vfs(_) => Ok(()),
}
}
}
impl std::io::Read for FsFileAdapter {
#[inline]
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
match self {
Self::Real(file) => file.read(buf),
Self::Vfs(file) => file.read_to_buf(buf),
}
}
}
impl std::io::Seek for FsFileAdapter {
fn seek(&mut self, pos: std::io::SeekFrom) -> std::io::Result<u64> {
match self {
Self::Real(file) => file.seek(pos),
Self::Vfs(file) => file.seek(pos),
}
}
}
impl std::io::Write for FsFileAdapter {
#[inline]
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
match self {
Self::Real(file) => file.write(buf),
Self::Vfs(_) => Err(not_supported("writing files")),
}
}
#[inline]
fn flush(&mut self) -> std::io::Result<()> {
match self {
Self::Real(file) => file.flush(),
Self::Vfs(_) => Err(not_supported("writing files")),
}
}
}
impl sys_traits::FsFileSetLen for FsFileAdapter {
#[inline]
fn fs_file_set_len(&mut self, len: u64) -> std::io::Result<()> {
match self {
Self::Real(file) => file.fs_file_set_len(len),
Self::Vfs(_) => Err(not_supported("setting file length")),
}
}
}
impl sys_traits::FsFileSetTimes for FsFileAdapter {
fn fs_file_set_times(
&mut self,
times: sys_traits::FsFileTimes,
) -> std::io::Result<()> {
match self {
Self::Real(file) => file.fs_file_set_times(times),
Self::Vfs(_) => Err(not_supported("setting file times")),
}
}
}
impl sys_traits::FsFileLock for FsFileAdapter {
fn fs_file_lock(
&mut self,
mode: sys_traits::FsFileLockMode,
) -> std::io::Result<()> {
match self {
Self::Real(file) => file.fs_file_lock(mode),
Self::Vfs(_) => Err(not_supported("locking files")),
}
}
fn fs_file_try_lock(
&mut self,
mode: sys_traits::FsFileLockMode,
) -> std::io::Result<()> {
match self {
Self::Real(file) => file.fs_file_try_lock(mode),
Self::Vfs(_) => Err(not_supported("locking files")),
}
}
fn fs_file_unlock(&mut self) -> std::io::Result<()> {
match self {
Self::Real(file) => file.fs_file_unlock(),
Self::Vfs(_) => Err(not_supported("unlocking files")),
}
}
}
impl sys_traits::FsFileIsTerminal for FsFileAdapter {
#[inline]
fn fs_file_is_terminal(&self) -> bool {
match self {
Self::Real(file) => file.fs_file_is_terminal(),
Self::Vfs(_) => false,
}
}
}
impl sys_traits::BaseFsOpen for DenoCompileFileSystem {
type File = FsFileAdapter;
fn base_fs_open(
&self,
path: &Path,
options: &sys_traits::OpenOptions,
) -> std::io::Result<Self::File> {
if self.0.is_path_within(path) {
Ok(FsFileAdapter::Vfs(self.0.open_file(path)?))
} else {
#[allow(clippy::disallowed_types)] // ok because we're implementing the fs
Ok(FsFileAdapter::Real(
sys_traits::impls::RealSys.base_fs_open(path, options)?,
))
}
}
}
impl sys_traits::SystemRandom for DenoCompileFileSystem {
#[inline]
fn sys_random(&self, buf: &mut [u8]) -> std::io::Result<()> {
#[allow(clippy::disallowed_types)] // ok because we're implementing the fs
sys_traits::impls::RealSys.sys_random(buf)
}
}
impl sys_traits::SystemTimeNow for DenoCompileFileSystem {
#[inline]
fn sys_time_now(&self) -> SystemTime {
#[allow(clippy::disallowed_types)] // ok because we're implementing the fs
sys_traits::impls::RealSys.sys_time_now()
}
}
impl sys_traits::ThreadSleep for DenoCompileFileSystem {
#[inline]
fn thread_sleep(&self, dur: Duration) {
#[allow(clippy::disallowed_types)] // ok because we're implementing the fs
sys_traits::impls::RealSys.thread_sleep(dur)
}
}
impl sys_traits::EnvCurrentDir for DenoCompileFileSystem {
fn env_current_dir(&self) -> std::io::Result<PathBuf> {
#[allow(clippy::disallowed_types)] // ok because we're implementing the fs
sys_traits::impls::RealSys.env_current_dir()
}
}
impl sys_traits::BaseEnvVar for DenoCompileFileSystem {
fn base_env_var_os(
&self,
key: &std::ffi::OsStr,
) -> Option<std::ffi::OsString> {
#[allow(clippy::disallowed_types)] // ok because we're implementing the fs
sys_traits::impls::RealSys.base_env_var_os(key)
}
}

117
cli/standalone/mod.rs
View file

@ -9,6 +9,7 @@ use binary::StandaloneData;
use binary::StandaloneModules;
use code_cache::DenoCompileCodeCache;
use deno_ast::MediaType;
use deno_cache_dir::file_fetcher::CacheSetting;
use deno_cache_dir::npm::NpmCacheDir;
use deno_config::workspace::MappedResolution;
use deno_config::workspace::MappedResolutionError;
@ -35,10 +36,11 @@ use deno_package_json::PackageJsonDepValue;
use deno_resolver::cjs::IsCjsResolutionMode;
use deno_resolver::npm::NpmReqResolverOptions;
use deno_runtime::deno_fs;
use deno_runtime::deno_fs::FileSystem;
use deno_runtime::deno_node::create_host_defined_options;
use deno_runtime::deno_node::NodeRequireLoader;
use deno_runtime::deno_node::NodeResolver;
use deno_runtime::deno_node::PackageJsonResolver;
use deno_runtime::deno_node::RealIsBuiltInNodeModuleChecker;
use deno_runtime::deno_permissions::Permissions;
use deno_runtime::deno_permissions::PermissionsContainer;
use deno_runtime::deno_tls::rustls::RootCertStore;
@ -54,6 +56,7 @@ use node_resolver::errors::ClosestPkgJsonError;
use node_resolver::NodeResolutionKind;
use node_resolver::ResolutionMode;
use serialization::DenoCompileModuleSource;
use serialization::SourceMapStore;
use std::borrow::Cow;
use std::rc::Rc;
use std::sync::Arc;
@ -64,18 +67,17 @@ use crate::args::create_default_npmrc;
use crate::args::get_root_cert_store;
use crate::args::npm_pkg_req_ref_to_binary_command;
use crate::args::CaData;
use crate::args::CacheSetting;
use crate::args::NpmInstallDepsProvider;
use crate::args::StorageKeyResolver;
use crate::cache::Caches;
use crate::cache::DenoCacheEnvFsAdapter;
use crate::cache::DenoDirProvider;
use crate::cache::FastInsecureHasher;
use crate::cache::NodeAnalysisCache;
use crate::cache::RealDenoCacheEnv;
use crate::http_util::HttpClientProvider;
use crate::node::CliCjsCodeAnalyzer;
use crate::node::CliNodeCodeTranslator;
use crate::node::CliNodeResolver;
use crate::node::CliPackageJsonResolver;
use crate::npm::create_cli_npm_resolver;
use crate::npm::create_in_npm_pkg_checker;
use crate::npm::CliByonmNpmResolverCreateOptions;
@ -86,9 +88,9 @@ use crate::npm::CliNpmResolverCreateOptions;
use crate::npm::CliNpmResolverManagedSnapshotOption;
use crate::npm::CreateInNpmPkgCheckerOptions;
use crate::resolver::CjsTracker;
use crate::resolver::CliDenoResolverFs;
use crate::resolver::CliNpmReqResolver;
use crate::resolver::NpmModuleLoader;
use crate::sys::CliSys;
use crate::util::progress_bar::ProgressBar;
use crate::util::progress_bar::ProgressBarStyle;
use crate::util::text_encoding::from_utf8_lossy_cow;
@ -105,12 +107,12 @@ mod file_system;
mod serialization;
mod virtual_fs;
pub use self::file_system::DenoCompileFileSystem;
pub use binary::extract_standalone;
pub use binary::is_standalone_binary;
pub use binary::DenoCompileBinaryWriter;
use self::binary::Metadata;
use self::file_system::DenoCompileFileSystem;
struct SharedModuleLoaderState {
cjs_tracker: Arc<CjsTracker>,
@ -118,10 +120,11 @@ struct SharedModuleLoaderState {
fs: Arc<dyn deno_fs::FileSystem>,
modules: StandaloneModules,
node_code_translator: Arc<CliNodeCodeTranslator>,
node_resolver: Arc<NodeResolver>,
node_resolver: Arc<CliNodeResolver>,
npm_module_loader: Arc<NpmModuleLoader>,
npm_req_resolver: Arc<CliNpmReqResolver>,
npm_resolver: Arc<dyn CliNpmResolver>,
source_maps: SourceMapStore,
vfs: Arc<FileBackedVfs>,
workspace_resolver: WorkspaceResolver,
}
@ -396,7 +399,11 @@ impl ModuleLoader for EmbeddedModuleLoader {
);
}
match self.shared.modules.read(original_specifier) {
match self
.shared
.modules
.read(original_specifier, VfsFileSubDataKind::ModuleGraph)
{
Ok(Some(module)) => {
let media_type = module.media_type;
let (module_specifier, module_type, module_source) =
@ -495,6 +502,45 @@ impl ModuleLoader for EmbeddedModuleLoader {
}
std::future::ready(()).boxed_local()
}
fn get_source_map(&self, file_name: &str) -> Option<Cow<[u8]>> {
if file_name.starts_with("file:///") {
let url =
deno_path_util::url_from_directory_path(self.shared.vfs.root()).ok()?;
let file_url = ModuleSpecifier::parse(file_name).ok()?;
let relative_path = url.make_relative(&file_url)?;
self.shared.source_maps.get(&relative_path)
} else {
self.shared.source_maps.get(file_name)
}
.map(Cow::Borrowed)
}
fn get_source_mapped_source_line(
&self,
file_name: &str,
line_number: usize,
) -> Option<String> {
let specifier = ModuleSpecifier::parse(file_name).ok()?;
let data = self
.shared
.modules
.read(&specifier, VfsFileSubDataKind::Raw)
.ok()??;
let source = String::from_utf8_lossy(&data.data);
// Do NOT use .lines(): it skips the terminating empty line.
// (due to internally using_terminator() instead of .split())
let lines: Vec<&str> = source.split('\n').collect();
if line_number >= lines.len() {
Some(format!(
"{} Couldn't format source line: Line {} is out of bounds (source may have changed at runtime)",
crate::colors::yellow("Warning"), line_number + 1,
))
} else {
Some(lines[line_number].to_string())
}
}
}
impl NodeRequireLoader for EmbeddedModuleLoader {
@ -583,16 +629,20 @@ impl RootCertStoreProvider for StandaloneRootCertStoreProvider {
}
}
pub async fn run(data: StandaloneData) -> Result<i32, AnyError> {
pub async fn run(
fs: Arc<dyn FileSystem>,
sys: CliSys,
data: StandaloneData,
) -> Result<i32, AnyError> {
let StandaloneData {
fs,
metadata,
modules,
npm_snapshot,
root_path,
source_maps,
vfs,
} = data;
let deno_dir_provider = Arc::new(DenoDirProvider::new(None));
let deno_dir_provider = Arc::new(DenoDirProvider::new(sys.clone(), None));
let root_cert_store_provider = Arc::new(StandaloneRootCertStoreProvider {
ca_stores: metadata.ca_stores,
ca_data: metadata.ca_data.map(CaData::Bytes),
@ -610,9 +660,7 @@ pub async fn run(data: StandaloneData) -> Result<i32, AnyError> {
let main_module = root_dir_url.join(&metadata.entrypoint_key).unwrap();
let npm_global_cache_dir = root_path.join(".deno_compile_node_modules");
let cache_setting = CacheSetting::Only;
let pkg_json_resolver = Arc::new(PackageJsonResolver::new(
deno_runtime::deno_node::DenoFsNodeResolverEnv::new(fs.clone()),
));
let pkg_json_resolver = Arc::new(CliPackageJsonResolver::new(sys.clone()));
let (in_npm_pkg_checker, npm_resolver) = match metadata.node_modules {
Some(binary::NodeModules::Managed { node_modules_dir }) => {
// create an npmrc that uses the fake npm_registry_url to resolve packages
@ -625,7 +673,7 @@ pub async fn run(data: StandaloneData) -> Result<i32, AnyError> {
registry_configs: Default::default(),
});
let npm_cache_dir = Arc::new(NpmCacheDir::new(
&DenoCacheEnvFsAdapter(fs.as_ref()),
&sys,
npm_global_cache_dir,
npmrc.get_all_known_registries_urls(),
));
@ -646,17 +694,17 @@ pub async fn run(data: StandaloneData) -> Result<i32, AnyError> {
snapshot,
)),
maybe_lockfile: None,
fs: fs.clone(),
http_client_provider: http_client_provider.clone(),
npm_cache_dir,
cache_setting,
text_only_progress_bar: progress_bar,
maybe_node_modules_path,
npm_system_info: Default::default(),
npm_install_deps_provider: Arc::new(
// this is only used for installing packages, which isn't necessary with deno compile
NpmInstallDepsProvider::empty(),
),
sys: sys.clone(),
text_only_progress_bar: progress_bar,
cache_setting,
maybe_node_modules_path,
npm_system_info: Default::default(),
npmrc,
lifecycle_scripts: Default::default(),
},
@ -673,7 +721,7 @@ pub async fn run(data: StandaloneData) -> Result<i32, AnyError> {
create_in_npm_pkg_checker(CreateInNpmPkgCheckerOptions::Byonm);
let npm_resolver = create_cli_npm_resolver(
CliNpmResolverCreateOptions::Byonm(CliByonmNpmResolverCreateOptions {
fs: CliDenoResolverFs(fs.clone()),
sys: sys.clone(),
pkg_json_resolver: pkg_json_resolver.clone(),
root_node_modules_dir,
}),
@ -686,7 +734,7 @@ pub async fn run(data: StandaloneData) -> Result<i32, AnyError> {
// so no need to create actual `.npmrc` configuration.
let npmrc = create_default_npmrc();
let npm_cache_dir = Arc::new(NpmCacheDir::new(
&DenoCacheEnvFsAdapter(fs.as_ref()),
&sys,
npm_global_cache_dir,
npmrc.get_all_known_registries_urls(),
));
@ -701,18 +749,18 @@ pub async fn run(data: StandaloneData) -> Result<i32, AnyError> {
create_cli_npm_resolver(CliNpmResolverCreateOptions::Managed(
CliManagedNpmResolverCreateOptions {
snapshot: CliNpmResolverManagedSnapshotOption::Specified(None),
maybe_lockfile: None,
fs: fs.clone(),
http_client_provider: http_client_provider.clone(),
npm_cache_dir,
cache_setting,
text_only_progress_bar: progress_bar,
maybe_node_modules_path: None,
npm_system_info: Default::default(),
npm_install_deps_provider: Arc::new(
// this is only used for installing packages, which isn't necessary with deno compile
NpmInstallDepsProvider::empty(),
),
sys: sys.clone(),
cache_setting,
text_only_progress_bar: progress_bar,
npm_cache_dir,
maybe_lockfile: None,
maybe_node_modules_path: None,
npm_system_info: Default::default(),
npmrc: create_default_npmrc(),
lifecycle_scripts: Default::default(),
},
@ -724,10 +772,11 @@ pub async fn run(data: StandaloneData) -> Result<i32, AnyError> {
let has_node_modules_dir = npm_resolver.root_node_modules_path().is_some();
let node_resolver = Arc::new(NodeResolver::new(
deno_runtime::deno_node::DenoFsNodeResolverEnv::new(fs.clone()),
in_npm_pkg_checker.clone(),
RealIsBuiltInNodeModuleChecker,
npm_resolver.clone().into_npm_pkg_folder_resolver(),
pkg_json_resolver.clone(),
sys.clone(),
));
let cjs_tracker = Arc::new(CjsTracker::new(
in_npm_pkg_checker.clone(),
@ -745,7 +794,7 @@ pub async fn run(data: StandaloneData) -> Result<i32, AnyError> {
let npm_req_resolver =
Arc::new(CliNpmReqResolver::new(NpmReqResolverOptions {
byonm_resolver: (npm_resolver.clone()).into_maybe_byonm(),
fs: CliDenoResolverFs(fs.clone()),
sys: sys.clone(),
in_npm_pkg_checker: in_npm_pkg_checker.clone(),
node_resolver: node_resolver.clone(),
npm_req_resolver: npm_resolver.clone().into_npm_req_resolver(),
@ -758,11 +807,11 @@ pub async fn run(data: StandaloneData) -> Result<i32, AnyError> {
);
let node_code_translator = Arc::new(NodeCodeTranslator::new(
cjs_esm_code_analyzer,
deno_runtime::deno_node::DenoFsNodeResolverEnv::new(fs.clone()),
in_npm_pkg_checker,
node_resolver.clone(),
npm_resolver.clone().into_npm_pkg_folder_resolver(),
pkg_json_resolver.clone(),
sys.clone(),
));
let workspace_resolver = {
let import_map = match metadata.workspace_resolver.import_map {
@ -841,6 +890,7 @@ pub async fn run(data: StandaloneData) -> Result<i32, AnyError> {
)),
npm_resolver: npm_resolver.clone(),
npm_req_resolver,
source_maps,
vfs,
workspace_resolver,
}),
@ -864,7 +914,7 @@ pub async fn run(data: StandaloneData) -> Result<i32, AnyError> {
}
let desc_parser =
Arc::new(RuntimePermissionDescriptorParser::new(fs.clone()));
Arc::new(RuntimePermissionDescriptorParser::new(sys.clone()));
let permissions =
Permissions::from_options(desc_parser.as_ref(), &permissions)?;
PermissionsContainer::new(desc_parser, permissions)
@ -894,6 +944,7 @@ pub async fn run(data: StandaloneData) -> Result<i32, AnyError> {
root_cert_store_provider,
permissions,
StorageKeyResolver::empty(),
sys,
crate::args::DenoSubcommand::Run(Default::default()),
CliMainWorkerOptions {
argv: metadata.argv,

315
cli/standalone/serialization.rs
View file

@ -1,10 +1,13 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use std::borrow::Cow;
use std::cell::Cell;
use std::collections::BTreeMap;
use std::collections::HashMap;
use std::io::Write;
use capacity_builder::BytesAppendable;
use deno_ast::swc::common::source_map;
use deno_ast::MediaType;
use deno_core::anyhow::bail;
use deno_core::anyhow::Context;
@ -19,12 +22,15 @@ use deno_npm::resolution::SerializedNpmResolutionSnapshotPackage;
use deno_npm::resolution::ValidSerializedNpmResolutionSnapshot;
use deno_npm::NpmPackageId;
use deno_semver::package::PackageReq;
use deno_semver::StackString;
use indexmap::IndexMap;
use crate::standalone::virtual_fs::VirtualDirectory;
use super::binary::Metadata;
use super::virtual_fs::BuiltVfs;
use super::virtual_fs::VfsBuilder;
use super::virtual_fs::VirtualDirectoryEntries;
const MAGIC_BYTES: &[u8; 8] = b"d3n0l4nd";
@ -32,60 +38,64 @@ const MAGIC_BYTES: &[u8; 8] = b"d3n0l4nd";
/// * d3n0l4nd
/// * <metadata_len><metadata>
/// * <npm_snapshot_len><npm_snapshot>
/// * <remote_modules_len><remote_modules>
/// * <remote_modules>
/// * <vfs_headers_len><vfs_headers>
/// * <vfs_file_data_len><vfs_file_data>
/// * <source_map_data>
/// * d3n0l4nd
pub fn serialize_binary_data_section(
metadata: &Metadata,
npm_snapshot: Option<SerializedNpmResolutionSnapshot>,
remote_modules: &RemoteModulesStoreBuilder,
source_map_store: &SourceMapStore,
vfs: &BuiltVfs,
) -> Result<Vec<u8>, AnyError> {
fn write_bytes_with_len(bytes: &mut Vec<u8>, data: &[u8]) {
bytes.extend_from_slice(&(data.len() as u64).to_le_bytes());
bytes.extend_from_slice(data);
}
let metadata = serde_json::to_string(metadata)?;
let npm_snapshot =
npm_snapshot.map(serialize_npm_snapshot).unwrap_or_default();
let serialized_vfs = serde_json::to_string(&vfs.entries)?;
let mut bytes = Vec::new();
bytes.extend_from_slice(MAGIC_BYTES);
// 1. Metadata
{
let metadata = serde_json::to_string(metadata)?;
write_bytes_with_len(&mut bytes, metadata.as_bytes());
}
// 2. Npm snapshot
{
let npm_snapshot =
npm_snapshot.map(serialize_npm_snapshot).unwrap_or_default();
write_bytes_with_len(&mut bytes, &npm_snapshot);
}
// 3. Remote modules
{
let update_index = bytes.len();
bytes.extend_from_slice(&(0_u64).to_le_bytes());
let start_index = bytes.len();
remote_modules.write(&mut bytes)?;
let length = bytes.len() - start_index;
let length_bytes = (length as u64).to_le_bytes();
bytes[update_index..update_index + length_bytes.len()]
.copy_from_slice(&length_bytes);
}
// 4. VFS
{
let serialized_vfs = serde_json::to_string(&vfs.root)?;
write_bytes_with_len(&mut bytes, serialized_vfs.as_bytes());
let vfs_bytes_len = vfs.files.iter().map(|f| f.len() as u64).sum::<u64>();
bytes.extend_from_slice(&vfs_bytes_len.to_le_bytes());
for file in &vfs.files {
bytes.extend_from_slice(file);
let bytes = capacity_builder::BytesBuilder::build(|builder| {
builder.append(MAGIC_BYTES);
// 1. Metadata
{
builder.append_le(metadata.len() as u64);
builder.append(&metadata);
}
// 2. Npm snapshot
{
builder.append_le(npm_snapshot.len() as u64);
builder.append(&npm_snapshot);
}
// 3. Remote modules
{
remote_modules.write(builder);
}
// 4. VFS
{
builder.append_le(serialized_vfs.len() as u64);
builder.append(&serialized_vfs);
let vfs_bytes_len = vfs.files.iter().map(|f| f.len() as u64).sum::<u64>();
builder.append_le(vfs_bytes_len);
for file in &vfs.files {
builder.append(file);
}
}
// 5. Source maps
{
builder.append_le(source_map_store.data.len() as u32);
for (specifier, source_map) in &source_map_store.data {
builder.append_le(specifier.len() as u32);
builder.append(specifier);
builder.append_le(source_map.len() as u32);
builder.append(source_map.as_ref());
}
}
}
// write the magic bytes at the end so we can use it
// to make sure we've deserialized correctly
bytes.extend_from_slice(MAGIC_BYTES);
// write the magic bytes at the end so we can use it
// to make sure we've deserialized correctly
builder.append(MAGIC_BYTES);
})?;
Ok(bytes)
}
@ -94,19 +104,14 @@ pub struct DeserializedDataSection {
pub metadata: Metadata,
pub npm_snapshot: Option<ValidSerializedNpmResolutionSnapshot>,
pub remote_modules: RemoteModulesStore,
pub vfs_dir: VirtualDirectory,
pub source_maps: SourceMapStore,
pub vfs_root_entries: VirtualDirectoryEntries,
pub vfs_files_data: &'static [u8],
}
pub fn deserialize_binary_data_section(
data: &'static [u8],
) -> Result<Option<DeserializedDataSection>, AnyError> {
fn read_bytes_with_len(input: &[u8]) -> Result<(&[u8], &[u8]), AnyError> {
let (input, len) = read_u64(input)?;
let (input, data) = read_bytes(input, len as usize)?;
Ok((input, data))
}
fn read_magic_bytes(input: &[u8]) -> Result<(&[u8], bool), AnyError> {
if input.len() < MAGIC_BYTES.len() {
bail!("Unexpected end of data. Could not find magic bytes.");
@ -118,34 +123,51 @@ pub fn deserialize_binary_data_section(
Ok((input, true))
}
#[allow(clippy::type_complexity)]
fn read_source_map_entry(
input: &[u8],
) -> Result<(&[u8], (Cow<str>, &[u8])), AnyError> {
let (input, specifier) = read_string_lossy(input)?;
let (input, source_map) = read_bytes_with_u32_len(input)?;
Ok((input, (specifier, source_map)))
}
let (input, found) = read_magic_bytes(data)?;
if !found {
return Ok(None);
}
// 1. Metadata
let (input, data) = read_bytes_with_len(input).context("reading metadata")?;
let (input, data) =
read_bytes_with_u64_len(input).context("reading metadata")?;
let metadata: Metadata =
serde_json::from_slice(data).context("deserializing metadata")?;
// 2. Npm snapshot
let (input, data) =
read_bytes_with_len(input).context("reading npm snapshot")?;
read_bytes_with_u64_len(input).context("reading npm snapshot")?;
let npm_snapshot = if data.is_empty() {
None
} else {
Some(deserialize_npm_snapshot(data).context("deserializing npm snapshot")?)
};
// 3. Remote modules
let (input, data) =
read_bytes_with_len(input).context("reading remote modules data")?;
let remote_modules =
RemoteModulesStore::build(data).context("deserializing remote modules")?;
let (input, remote_modules) =
RemoteModulesStore::build(input).context("deserializing remote modules")?;
// 4. VFS
let (input, data) = read_bytes_with_len(input).context("vfs")?;
let vfs_dir: VirtualDirectory =
let (input, data) = read_bytes_with_u64_len(input).context("vfs")?;
let vfs_root_entries: VirtualDirectoryEntries =
serde_json::from_slice(data).context("deserializing vfs data")?;
let (input, vfs_files_data) =
read_bytes_with_len(input).context("reading vfs files data")?;
read_bytes_with_u64_len(input).context("reading vfs files data")?;
// 5. Source maps
let (mut input, source_map_data_len) = read_u32_as_usize(input)?;
let mut source_maps = SourceMapStore::with_capacity(source_map_data_len);
for _ in 0..source_map_data_len {
let (current_input, (specifier, source_map)) =
read_source_map_entry(input)?;
input = current_input;
source_maps.add(specifier, Cow::Borrowed(source_map));
}
// finally ensure we read the magic bytes at the end
let (_input, found) = read_magic_bytes(input)?;
@ -157,7 +179,8 @@ pub fn deserialize_binary_data_section(
metadata,
npm_snapshot,
remote_modules,
vfs_dir,
source_maps,
vfs_root_entries,
vfs_files_data,
}))
}
@ -165,19 +188,31 @@ pub fn deserialize_binary_data_section(
#[derive(Default)]
pub struct RemoteModulesStoreBuilder {
specifiers: Vec<(String, u64)>,
data: Vec<(MediaType, Vec<u8>)>,
data: Vec<(MediaType, Vec<u8>, Option<Vec<u8>>)>,
data_byte_len: u64,
redirects: Vec<(String, String)>,
redirects_len: u64,
}
impl RemoteModulesStoreBuilder {
pub fn add(&mut self, specifier: &Url, media_type: MediaType, data: Vec<u8>) {
pub fn add(
&mut self,
specifier: &Url,
media_type: MediaType,
data: Vec<u8>,
maybe_transpiled: Option<Vec<u8>>,
) {
log::debug!("Adding '{}' ({})", specifier, media_type);
let specifier = specifier.to_string();
self.specifiers.push((specifier, self.data_byte_len));
self.data_byte_len += 1 + 8 + data.len() as u64; // media type (1 byte), data length (8 bytes), data
self.data.push((media_type, data));
let maybe_transpiled_len = match &maybe_transpiled {
// data length (4 bytes), data
Some(data) => 4 + data.len() as u64,
None => 0,
};
// media type (1 byte), data length (4 bytes), data, has transpiled (1 byte), transpiled length
self.data_byte_len += 1 + 4 + data.len() as u64 + 1 + maybe_transpiled_len;
self.data.push((media_type, data, maybe_transpiled));
}
pub fn add_redirects(&mut self, redirects: &BTreeMap<Url, Url>) {
@ -191,26 +226,50 @@ impl RemoteModulesStoreBuilder {
}
}
fn write(&self, writer: &mut dyn Write) -> Result<(), AnyError> {
writer.write_all(&(self.specifiers.len() as u32).to_le_bytes())?;
writer.write_all(&(self.redirects.len() as u32).to_le_bytes())?;
fn write<'a, TBytes: capacity_builder::BytesType>(
&'a self,
builder: &mut capacity_builder::BytesBuilder<'a, TBytes>,
) {
builder.append_le(self.specifiers.len() as u32);
builder.append_le(self.redirects.len() as u32);
for (specifier, offset) in &self.specifiers {
writer.write_all(&(specifier.len() as u32).to_le_bytes())?;
writer.write_all(specifier.as_bytes())?;
writer.write_all(&offset.to_le_bytes())?;
builder.append_le(specifier.len() as u32);
builder.append(specifier);
builder.append_le(*offset);
}
for (from, to) in &self.redirects {
writer.write_all(&(from.len() as u32).to_le_bytes())?;
writer.write_all(from.as_bytes())?;
writer.write_all(&(to.len() as u32).to_le_bytes())?;
writer.write_all(to.as_bytes())?;
builder.append_le(from.len() as u32);
builder.append(from);
builder.append_le(to.len() as u32);
builder.append(to);
}
for (media_type, data) in &self.data {
writer.write_all(&[serialize_media_type(*media_type)])?;
writer.write_all(&(data.len() as u64).to_le_bytes())?;
writer.write_all(data)?;
builder.append_le(
self
.data
.iter()
.map(|(_, data, maybe_transpiled)| {
1 + 4
+ (data.len() as u64)
+ 1
+ match maybe_transpiled {
Some(transpiled) => 4 + (transpiled.len() as u64),
None => 0,
}
})
.sum::<u64>(),
);
for (media_type, data, maybe_transpiled) in &self.data {
builder.append(serialize_media_type(*media_type));
builder.append_le(data.len() as u32);
builder.append(data);
if let Some(transpiled) = maybe_transpiled {
builder.append(1);
builder.append_le(transpiled.len() as u32);
builder.append(transpiled);
} else {
builder.append(0);
}
}
Ok(())
}
}
@ -238,6 +297,30 @@ impl DenoCompileModuleSource {
}
}
pub struct SourceMapStore {
data: IndexMap<Cow<'static, str>, Cow<'static, [u8]>>,
}
impl SourceMapStore {
pub fn with_capacity(capacity: usize) -> Self {
Self {
data: IndexMap::with_capacity(capacity),
}
}
pub fn add(
&mut self,
specifier: Cow<'static, str>,
source_map: Cow<'static, [u8]>,
) {
self.data.insert(specifier, source_map);
}
pub fn get(&self, specifier: &str) -> Option<&[u8]> {
self.data.get(specifier).map(|v| v.as_ref())
}
}
pub struct DenoCompileModuleData<'a> {
pub specifier: &'a Url,
pub media_type: MediaType,
@ -284,6 +367,13 @@ impl<'a> DenoCompileModuleData<'a> {
}
}
pub struct RemoteModuleEntry<'a> {
pub specifier: &'a Url,
pub media_type: MediaType,
pub data: Cow<'static, [u8]>,
pub transpiled_data: Option<Cow<'static, [u8]>>,
}
enum RemoteModulesStoreSpecifierValue {
Data(usize),
Redirect(Url),
@ -295,7 +385,7 @@ pub struct RemoteModulesStore {
}
impl RemoteModulesStore {
fn build(data: &'static [u8]) -> Result<Self, AnyError> {
fn build(input: &'static [u8]) -> Result<(&'static [u8], Self), AnyError> {
fn read_specifier(input: &[u8]) -> Result<(&[u8], (Url, u64)), AnyError> {
let (input, specifier) = read_string_lossy(input)?;
let specifier = Url::parse(&specifier)?;
@ -338,12 +428,16 @@ impl RemoteModulesStore {
Ok((input, specifiers))
}
let (files_data, specifiers) = read_headers(data)?;
let (input, specifiers) = read_headers(input)?;
let (input, files_data) = read_bytes_with_u64_len(input)?;
Ok(Self {
specifiers,
files_data,
})
Ok((
input,
Self {
specifiers,
files_data,
},
))
}
pub fn resolve_specifier<'a>(
@ -374,7 +468,7 @@ impl RemoteModulesStore {
pub fn read<'a>(
&'a self,
original_specifier: &'a Url,
) -> Result<Option<DenoCompileModuleData<'a>>, AnyError> {
) -> Result<Option<RemoteModuleEntry<'a>>, AnyError> {
let mut count = 0;
let mut specifier = original_specifier;
loop {
@ -390,12 +484,25 @@ impl RemoteModulesStore {
let input = &self.files_data[*offset..];
let (input, media_type_byte) = read_bytes(input, 1)?;
let media_type = deserialize_media_type(media_type_byte[0])?;
let (input, len) = read_u64(input)?;
let (_input, data) = read_bytes(input, len as usize)?;
return Ok(Some(DenoCompileModuleData {
let (input, data) = read_bytes_with_u32_len(input)?;
check_has_len(input, 1)?;
let (input, has_transpiled) = (&input[1..], input[0]);
let (_, transpiled_data) = match has_transpiled {
0 => (input, None),
1 => {
let (input, data) = read_bytes_with_u32_len(input)?;
(input, Some(data))
}
value => bail!(
"Invalid transpiled data flag: {}. Compiled data is corrupt.",
value
),
};
return Ok(Some(RemoteModuleEntry {
specifier,
media_type,
data: Cow::Borrowed(data),
transpiled_data: transpiled_data.map(Cow::Borrowed),
}));
}
None => {
@ -479,12 +586,13 @@ fn deserialize_npm_snapshot(
#[allow(clippy::needless_lifetimes)] // clippy bug
fn parse_package_dep<'a>(
id_to_npm_id: &'a impl Fn(usize) -> Result<NpmPackageId, AnyError>,
) -> impl Fn(&[u8]) -> Result<(&[u8], (String, NpmPackageId)), AnyError> + 'a
) -> impl Fn(&[u8]) -> Result<(&[u8], (StackString, NpmPackageId)), AnyError> + 'a
{
|input| {
let (input, req) = read_string_lossy(input)?;
let (input, id) = read_u32_as_usize(input)?;
Ok((input, (req.into_owned(), id_to_npm_id(id)?)))
let req = StackString::from_cow(req);
Ok((input, (req, id_to_npm_id(id)?)))
}
}
@ -634,17 +742,34 @@ fn parse_vec_n_times_with_index<TResult>(
Ok((input, results))
}
fn read_bytes_with_u64_len(input: &[u8]) -> Result<(&[u8], &[u8]), AnyError> {
let (input, len) = read_u64(input)?;
let (input, data) = read_bytes(input, len as usize)?;
Ok((input, data))
}
fn read_bytes_with_u32_len(input: &[u8]) -> Result<(&[u8], &[u8]), AnyError> {
let (input, len) = read_u32_as_usize(input)?;
let (input, data) = read_bytes(input, len)?;
Ok((input, data))
}
fn read_bytes(input: &[u8], len: usize) -> Result<(&[u8], &[u8]), AnyError> {
if input.len() < len {
bail!("Unexpected end of data.",);
}
check_has_len(input, len)?;
let (len_bytes, input) = input.split_at(len);
Ok((input, len_bytes))
}
#[inline(always)]
fn check_has_len(input: &[u8], len: usize) -> Result<(), AnyError> {
if input.len() < len {
bail!("Unexpected end of data.");
}
Ok(())
}
fn read_string_lossy(input: &[u8]) -> Result<(&[u8], Cow<str>), AnyError> {
let (input, str_len) = read_u32_as_usize(input)?;
let (input, data_bytes) = read_bytes(input, str_len)?;
let (input, data_bytes) = read_bytes_with_u32_len(input)?;
Ok((input, String::from_utf8_lossy(data_bytes)))
}

423
cli/standalone/virtual_fs.rs
View file

@ -51,8 +51,16 @@ pub enum WindowsSystemRootablePath {
impl WindowsSystemRootablePath {
pub fn join(&self, name_component: &str) -> PathBuf {
// this method doesn't handle multiple components
debug_assert!(!name_component.contains('\\'));
debug_assert!(!name_component.contains('/'));
debug_assert!(
!name_component.contains('\\'),
"Invalid component: {}",
name_component
);
debug_assert!(
!name_component.contains('/'),
"Invalid component: {}",
name_component
);
match self {
WindowsSystemRootablePath::WindowSystemRoot => {
@ -67,7 +75,7 @@ impl WindowsSystemRootablePath {
#[derive(Debug)]
pub struct BuiltVfs {
pub root_path: WindowsSystemRootablePath,
pub root: VirtualDirectory,
pub entries: VirtualDirectoryEntries,
pub files: Vec<Vec<u8>>,
}
@ -95,7 +103,7 @@ impl VfsBuilder {
Self {
executable_root: VirtualDirectory {
name: "/".to_string(),
entries: Vec::new(),
entries: Default::default(),
},
files: Vec::new(),
current_offset: 0,
@ -208,23 +216,20 @@ impl VfsBuilder {
continue;
}
let name = component.as_os_str().to_string_lossy();
let index = match current_dir
.entries
.binary_search_by(|e| e.name().cmp(&name))
{
let index = match current_dir.entries.binary_search(&name) {
Ok(index) => index,
Err(insert_index) => {
current_dir.entries.insert(
current_dir.entries.0.insert(
insert_index,
VfsEntry::Dir(VirtualDirectory {
name: name.to_string(),
entries: Vec::new(),
entries: Default::default(),
}),
);
insert_index
}
};
match &mut current_dir.entries[index] {
match &mut current_dir.entries.0[index] {
VfsEntry::Dir(dir) => {
current_dir = dir;
}
@ -248,14 +253,8 @@ impl VfsBuilder {
continue;
}
let name = component.as_os_str().to_string_lossy();
let index = match current_dir
.entries
.binary_search_by(|e| e.name().cmp(&name))
{
Ok(index) => index,
Err(_) => return None,
};
match &mut current_dir.entries[index] {
let entry = current_dir.entries.get_mut_by_name(&name)?;
match entry {
VfsEntry::Dir(dir) => {
current_dir = dir;
}
@ -320,9 +319,9 @@ impl VfsBuilder {
offset,
len: data.len() as u64,
};
match dir.entries.binary_search_by(|e| e.name().cmp(&name)) {
match dir.entries.binary_search(&name) {
Ok(index) => {
let entry = &mut dir.entries[index];
let entry = &mut dir.entries.0[index];
match entry {
VfsEntry::File(virtual_file) => match sub_data_kind {
VfsFileSubDataKind::Raw => {
@ -336,7 +335,7 @@ impl VfsBuilder {
}
}
Err(insert_index) => {
dir.entries.insert(
dir.entries.0.insert(
insert_index,
VfsEntry::File(VirtualFile {
name: name.to_string(),
@ -384,10 +383,10 @@ impl VfsBuilder {
let target = normalize_path(path.parent().unwrap().join(&target));
let dir = self.add_dir_raw(path.parent().unwrap());
let name = path.file_name().unwrap().to_string_lossy();
match dir.entries.binary_search_by(|e| e.name().cmp(&name)) {
match dir.entries.binary_search(&name) {
Ok(_) => {} // previously inserted
Err(insert_index) => {
dir.entries.insert(
dir.entries.0.insert(
insert_index,
VfsEntry::Symlink(VirtualSymlink {
name: name.to_string(),
@ -426,7 +425,7 @@ impl VfsBuilder {
dir: &mut VirtualDirectory,
parts: &[String],
) {
for entry in &mut dir.entries {
for entry in &mut dir.entries.0 {
match entry {
VfsEntry::Dir(dir) => {
strip_prefix_from_symlinks(dir, parts);
@ -454,13 +453,13 @@ impl VfsBuilder {
if self.min_root_dir.as_ref() == Some(&current_path) {
break;
}
match &current_dir.entries[0] {
match &current_dir.entries.0[0] {
VfsEntry::Dir(dir) => {
if dir.name == DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME {
// special directory we want to maintain
break;
}
match current_dir.entries.remove(0) {
match current_dir.entries.0.remove(0) {
VfsEntry::Dir(dir) => {
current_path =
WindowsSystemRootablePath::Path(current_path.join(&dir.name));
@ -480,7 +479,7 @@ impl VfsBuilder {
}
BuiltVfs {
root_path: current_path,
root: current_dir,
entries: current_dir.entries,
files: self.files,
}
}
@ -506,7 +505,7 @@ pub fn output_vfs(vfs: &BuiltVfs, executable_name: &str) {
return; // no need to compute if won't output
}
if vfs.root.entries.is_empty() {
if vfs.entries.is_empty() {
return; // nothing to output
}
@ -696,7 +695,7 @@ fn vfs_as_display_tree(
fn dir_size(dir: &VirtualDirectory, seen_offsets: &mut HashSet<u64>) -> Size {
let mut size = Size::default();
for entry in &dir.entries {
for entry in dir.entries.iter() {
match entry {
VfsEntry::Dir(virtual_directory) => {
size = size + dir_size(virtual_directory, seen_offsets);
@ -760,15 +759,10 @@ fn vfs_as_display_tree(
fn include_all_entries<'a>(
dir_path: &WindowsSystemRootablePath,
vfs_dir: &'a VirtualDirectory,
entries: &'a VirtualDirectoryEntries,
seen_offsets: &mut HashSet<u64>,
) -> Vec<DirEntryOutput<'a>> {
if vfs_dir.name == DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME {
return show_global_node_modules_dir(vfs_dir, seen_offsets);
}
vfs_dir
.entries
entries
.iter()
.map(|entry| DirEntryOutput {
name: Cow::Borrowed(entry.name()),
@ -826,10 +820,12 @@ fn vfs_as_display_tree(
} else {
EntryOutput::Subset(children)
}
} else if vfs_dir.name == DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME {
EntryOutput::Subset(show_global_node_modules_dir(vfs_dir, seen_offsets))
} else {
EntryOutput::Subset(include_all_entries(
&WindowsSystemRootablePath::Path(dir),
vfs_dir,
&vfs_dir.entries,
seen_offsets,
))
}
@ -839,7 +835,7 @@ fn vfs_as_display_tree(
// user might not have context about what's being shown
let mut seen_offsets = HashSet::with_capacity(vfs.files.len());
let mut child_entries =
include_all_entries(&vfs.root_path, &vfs.root, &mut seen_offsets);
include_all_entries(&vfs.root_path, &vfs.entries, &mut seen_offsets);
for child_entry in &mut child_entries {
child_entry.collapse_leaf_nodes();
}
@ -859,78 +855,28 @@ enum VfsEntryRef<'a> {
Symlink(&'a VirtualSymlink),
}
impl<'a> VfsEntryRef<'a> {
pub fn as_fs_stat(&self) -> FsStat {
impl VfsEntryRef<'_> {
pub fn as_metadata(&self) -> FileBackedVfsMetadata {
FileBackedVfsMetadata {
file_type: match self {
Self::Dir(_) => sys_traits::FileType::Dir,
Self::File(_) => sys_traits::FileType::File,
Self::Symlink(_) => sys_traits::FileType::Symlink,
},
name: self.name().to_string(),
len: match self {
Self::Dir(_) => 0,
Self::File(file) => file.offset.len,
Self::Symlink(_) => 0,
},
}
}
pub fn name(&self) -> &str {
match self {
VfsEntryRef::Dir(_) => FsStat {
is_directory: true,
is_file: false,
is_symlink: false,
atime: None,
birthtime: None,
mtime: None,
ctime: None,
blksize: 0,
size: 0,
dev: 0,
ino: 0,
mode: 0,
nlink: 0,
uid: 0,
gid: 0,
rdev: 0,
blocks: 0,
is_block_device: false,
is_char_device: false,
is_fifo: false,
is_socket: false,
},
VfsEntryRef::File(file) => FsStat {
is_directory: false,
is_file: true,
is_symlink: false,
atime: None,
birthtime: None,
mtime: None,
ctime: None,
blksize: 0,
size: file.offset.len,
dev: 0,
ino: 0,
mode: 0,
nlink: 0,
uid: 0,
gid: 0,
rdev: 0,
blocks: 0,
is_block_device: false,
is_char_device: false,
is_fifo: false,
is_socket: false,
},
VfsEntryRef::Symlink(_) => FsStat {
is_directory: false,
is_file: false,
is_symlink: true,
atime: None,
birthtime: None,
mtime: None,
ctime: None,
blksize: 0,
size: 0,
dev: 0,
ino: 0,
mode: 0,
nlink: 0,
uid: 0,
gid: 0,
rdev: 0,
blocks: 0,
is_block_device: false,
is_char_device: false,
is_fifo: false,
is_socket: false,
},
Self::Dir(dir) => &dir.name,
Self::File(file) => &file.name,
Self::Symlink(symlink) => &symlink.name,
}
}
}
@ -946,9 +892,9 @@ pub enum VfsEntry {
impl VfsEntry {
pub fn name(&self) -> &str {
match self {
VfsEntry::Dir(dir) => &dir.name,
VfsEntry::File(file) => &file.name,
VfsEntry::Symlink(symlink) => &symlink.name,
Self::Dir(dir) => &dir.name,
Self::File(file) => &file.name,
Self::Symlink(symlink) => &symlink.name,
}
}
@ -961,27 +907,70 @@ impl VfsEntry {
}
}
#[derive(Debug, Default, Serialize, Deserialize)]
pub struct VirtualDirectoryEntries(Vec<VfsEntry>);
impl VirtualDirectoryEntries {
pub fn new(mut entries: Vec<VfsEntry>) -> Self {
// needs to be sorted by name
entries.sort_by(|a, b| a.name().cmp(b.name()));
Self(entries)
}
pub fn take_inner(&mut self) -> Vec<VfsEntry> {
std::mem::take(&mut self.0)
}
pub fn is_empty(&self) -> bool {
self.0.is_empty()
}
pub fn len(&self) -> usize {
self.0.len()
}
pub fn get_by_name(&self, name: &str) -> Option<&VfsEntry> {
self.binary_search(name).ok().map(|index| &self.0[index])
}
pub fn get_mut_by_name(&mut self, name: &str) -> Option<&mut VfsEntry> {
self
.binary_search(name)
.ok()
.map(|index| &mut self.0[index])
}
pub fn binary_search(&self, name: &str) -> Result<usize, usize> {
self.0.binary_search_by(|e| e.name().cmp(name))
}
pub fn insert(&mut self, entry: VfsEntry) {
match self.binary_search(entry.name()) {
Ok(index) => {
self.0[index] = entry;
}
Err(insert_index) => {
self.0.insert(insert_index, entry);
}
}
}
pub fn remove(&mut self, index: usize) -> VfsEntry {
self.0.remove(index)
}
pub fn iter(&self) -> std::slice::Iter<'_, VfsEntry> {
self.0.iter()
}
}
#[derive(Debug, Serialize, Deserialize)]
pub struct VirtualDirectory {
#[serde(rename = "n")]
pub name: String,
// should be sorted by name
#[serde(rename = "e")]
pub entries: Vec<VfsEntry>,
}
impl VirtualDirectory {
pub fn insert_entry(&mut self, entry: VfsEntry) {
let name = entry.name();
match self.entries.binary_search_by(|e| e.name().cmp(name)) {
Ok(index) => {
self.entries[index] = entry;
}
Err(insert_index) => {
self.entries.insert(insert_index, entry);
}
}
}
pub entries: VirtualDirectoryEntries,
}
#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
@ -1136,34 +1125,27 @@ impl VfsRoot {
}
};
let component = component.to_string_lossy();
match current_dir
current_entry = current_dir
.entries
.binary_search_by(|e| e.name().cmp(&component))
{
Ok(index) => {
current_entry = current_dir.entries[index].as_ref();
}
Err(_) => {
return Err(std::io::Error::new(
std::io::ErrorKind::NotFound,
"path not found",
));
}
}
.get_by_name(&component)
.ok_or_else(|| {
std::io::Error::new(std::io::ErrorKind::NotFound, "path not found")
})?
.as_ref();
}
Ok((final_path, current_entry))
}
}
struct FileBackedVfsFile {
pub struct FileBackedVfsFile {
file: VirtualFile,
pos: RefCell<u64>,
vfs: Arc<FileBackedVfs>,
}
impl FileBackedVfsFile {
fn seek(&self, pos: SeekFrom) -> FsResult<u64> {
pub fn seek(&self, pos: SeekFrom) -> std::io::Result<u64> {
match pos {
SeekFrom::Start(pos) => {
*self.pos.borrow_mut() = pos;
@ -1172,10 +1154,10 @@ impl FileBackedVfsFile {
SeekFrom::End(offset) => {
if offset < 0 && -offset as u64 > self.file.offset.len {
let msg = "An attempt was made to move the file pointer before the beginning of the file.";
Err(
std::io::Error::new(std::io::ErrorKind::PermissionDenied, msg)
.into(),
)
Err(std::io::Error::new(
std::io::ErrorKind::PermissionDenied,
msg,
))
} else {
let mut current_pos = self.pos.borrow_mut();
*current_pos = if offset >= 0 {
@ -1191,7 +1173,7 @@ impl FileBackedVfsFile {
if offset >= 0 {
*current_pos += offset as u64;
} else if -offset as u64 > *current_pos {
return Err(std::io::Error::new(std::io::ErrorKind::PermissionDenied, "An attempt was made to move the file pointer before the beginning of the file.").into());
return Err(std::io::Error::new(std::io::ErrorKind::PermissionDenied, "An attempt was made to move the file pointer before the beginning of the file."));
} else {
*current_pos -= -offset as u64;
}
@ -1200,7 +1182,7 @@ impl FileBackedVfsFile {
}
}
fn read_to_buf(&self, buf: &mut [u8]) -> FsResult<usize> {
pub fn read_to_buf(&self, buf: &mut [u8]) -> std::io::Result<usize> {
let read_pos = {
let mut pos = self.pos.borrow_mut();
let read_pos = *pos;
@ -1208,10 +1190,7 @@ impl FileBackedVfsFile {
*pos = std::cmp::min(self.file.offset.len, *pos + buf.len() as u64);
read_pos
};
self
.vfs
.read_file(&self.file, read_pos, buf)
.map_err(|err| err.into())
self.vfs.read_file(&self.file, read_pos, buf)
}
fn read_to_end(&self) -> FsResult<Cow<'static, [u8]>> {
@ -1246,7 +1225,7 @@ impl FileBackedVfsFile {
#[async_trait::async_trait(?Send)]
impl deno_io::fs::File for FileBackedVfsFile {
fn read_sync(self: Rc<Self>, buf: &mut [u8]) -> FsResult<usize> {
self.read_to_buf(buf)
self.read_to_buf(buf).map_err(Into::into)
}
async fn read_byob(
self: Rc<Self>,
@ -1290,10 +1269,10 @@ impl deno_io::fs::File for FileBackedVfsFile {
}
fn seek_sync(self: Rc<Self>, pos: SeekFrom) -> FsResult<u64> {
self.seek(pos)
self.seek(pos).map_err(|err| err.into())
}
async fn seek_async(self: Rc<Self>, pos: SeekFrom) -> FsResult<u64> {
self.seek(pos)
self.seek(pos).map_err(|err| err.into())
}
fn datasync_sync(self: Rc<Self>) -> FsResult<()> {
@ -1369,6 +1348,47 @@ impl deno_io::fs::File for FileBackedVfsFile {
}
}
#[derive(Debug, Clone)]
pub struct FileBackedVfsDirEntry {
pub parent_path: PathBuf,
pub metadata: FileBackedVfsMetadata,
}
#[derive(Debug, Clone)]
pub struct FileBackedVfsMetadata {
pub name: String,
pub file_type: sys_traits::FileType,
pub len: u64,
}
impl FileBackedVfsMetadata {
pub fn as_fs_stat(&self) -> FsStat {
FsStat {
is_directory: self.file_type == sys_traits::FileType::Dir,
is_file: self.file_type == sys_traits::FileType::File,
is_symlink: self.file_type == sys_traits::FileType::Symlink,
atime: None,
birthtime: None,
mtime: None,
ctime: None,
blksize: 0,
size: self.len,
dev: 0,
ino: 0,
mode: 0,
nlink: 0,
uid: 0,
gid: 0,
rdev: 0,
blocks: 0,
is_block_device: false,
is_char_device: false,
is_fifo: false,
is_socket: false,
}
}
}
#[derive(Debug)]
pub struct FileBackedVfs {
vfs_data: Cow<'static, [u8]>,
@ -1394,13 +1414,13 @@ impl FileBackedVfs {
pub fn open_file(
self: &Arc<Self>,
path: &Path,
) -> std::io::Result<Rc<dyn deno_io::fs::File>> {
) -> std::io::Result<FileBackedVfsFile> {
let file = self.file_entry(path)?;
Ok(Rc::new(FileBackedVfsFile {
Ok(FileBackedVfsFile {
file: file.clone(),
vfs: self.clone(),
pos: Default::default(),
}))
})
}
pub fn read_dir(&self, path: &Path) -> std::io::Result<Vec<FsDirEntry>> {
@ -1419,6 +1439,18 @@ impl FileBackedVfs {
)
}
pub fn read_dir_with_metadata<'a>(
&'a self,
path: &Path,
) -> std::io::Result<impl Iterator<Item = FileBackedVfsDirEntry> + 'a> {
let dir = self.dir_entry(path)?;
let path = path.to_path_buf();
Ok(dir.entries.iter().map(move |entry| FileBackedVfsDirEntry {
parent_path: path.to_path_buf(),
metadata: entry.as_ref().as_metadata(),
}))
}
pub fn read_link(&self, path: &Path) -> std::io::Result<PathBuf> {
let (_, entry) = self.fs_root.find_entry_no_follow(path)?;
match entry {
@ -1432,14 +1464,14 @@ impl FileBackedVfs {
}
}
pub fn lstat(&self, path: &Path) -> std::io::Result<FsStat> {
pub fn lstat(&self, path: &Path) -> std::io::Result<FileBackedVfsMetadata> {
let (_, entry) = self.fs_root.find_entry_no_follow(path)?;
Ok(entry.as_fs_stat())
Ok(entry.as_metadata())
}
pub fn stat(&self, path: &Path) -> std::io::Result<FsStat> {
pub fn stat(&self, path: &Path) -> std::io::Result<FileBackedVfsMetadata> {
let (_, entry) = self.fs_root.find_entry(path)?;
Ok(entry.as_fs_stat())
Ok(entry.as_metadata())
}
pub fn canonicalize(&self, path: &Path) -> std::io::Result<PathBuf> {
@ -1532,6 +1564,7 @@ impl FileBackedVfs {
#[cfg(test)]
mod test {
use console_static_text::ansi::strip_ansi_codes;
use deno_io::fs::File;
use std::io::Write;
use test_util::assert_contains;
use test_util::TempDir;
@ -1617,25 +1650,31 @@ mod test {
);
// metadata
assert!(
assert_eq!(
virtual_fs
.lstat(&dest_path.join("sub_dir").join("e.txt"))
.unwrap()
.is_symlink
.file_type,
sys_traits::FileType::Symlink,
);
assert!(
assert_eq!(
virtual_fs
.stat(&dest_path.join("sub_dir").join("e.txt"))
.unwrap()
.is_file
.file_type,
sys_traits::FileType::File,
);
assert!(
assert_eq!(
virtual_fs
.stat(&dest_path.join("sub_dir"))
.unwrap()
.is_directory,
.file_type,
sys_traits::FileType::Dir,
);
assert_eq!(
virtual_fs.stat(&dest_path.join("e.txt")).unwrap().file_type,
sys_traits::FileType::File
);
assert!(virtual_fs.stat(&dest_path.join("e.txt")).unwrap().is_file,);
}
#[test]
@ -1672,11 +1711,12 @@ mod test {
read_file(&virtual_fs, &dest_path.join("sub_dir_link").join("c.txt")),
"c",
);
assert!(
assert_eq!(
virtual_fs
.lstat(&dest_path.join("sub_dir_link"))
.unwrap()
.is_symlink
.file_type,
sys_traits::FileType::Symlink,
);
assert_eq!(
@ -1706,7 +1746,10 @@ mod test {
FileBackedVfs::new(
Cow::Owned(data),
VfsRoot {
dir: vfs.root,
dir: VirtualDirectory {
name: "".to_string(),
entries: vfs.entries,
},
root_path: dest_path.to_path_buf(),
start_file_offset: 0,
},
@ -1745,37 +1788,35 @@ mod test {
let (dest_path, virtual_fs) = into_virtual_fs(builder, &temp_dir);
let virtual_fs = Arc::new(virtual_fs);
let file = virtual_fs.open_file(&dest_path.join("a.txt")).unwrap();
file.clone().seek_sync(SeekFrom::Current(2)).unwrap();
file.seek(SeekFrom::Current(2)).unwrap();
let mut buf = vec![0; 2];
file.clone().read_sync(&mut buf).unwrap();
file.read_to_buf(&mut buf).unwrap();
assert_eq!(buf, b"23");
file.clone().read_sync(&mut buf).unwrap();
file.read_to_buf(&mut buf).unwrap();
assert_eq!(buf, b"45");
file.clone().seek_sync(SeekFrom::Current(-4)).unwrap();
file.clone().read_sync(&mut buf).unwrap();
file.seek(SeekFrom::Current(-4)).unwrap();
file.read_to_buf(&mut buf).unwrap();
assert_eq!(buf, b"23");
file.clone().seek_sync(SeekFrom::Start(2)).unwrap();
file.clone().read_sync(&mut buf).unwrap();
file.seek(SeekFrom::Start(2)).unwrap();
file.read_to_buf(&mut buf).unwrap();
assert_eq!(buf, b"23");
file.clone().seek_sync(SeekFrom::End(2)).unwrap();
file.clone().read_sync(&mut buf).unwrap();
file.seek(SeekFrom::End(2)).unwrap();
file.read_to_buf(&mut buf).unwrap();
assert_eq!(buf, b"89");
file.clone().seek_sync(SeekFrom::Current(-8)).unwrap();
file.clone().read_sync(&mut buf).unwrap();
file.seek(SeekFrom::Current(-8)).unwrap();
file.read_to_buf(&mut buf).unwrap();
assert_eq!(buf, b"23");
assert_eq!(
file
.clone()
.seek_sync(SeekFrom::Current(-5))
.err()
.unwrap()
.into_io_error()
.seek(SeekFrom::Current(-5))
.unwrap_err()
.to_string(),
"An attempt was made to move the file pointer before the beginning of the file."
);
// go beyond the file length, then back
file.clone().seek_sync(SeekFrom::Current(40)).unwrap();
file.clone().seek_sync(SeekFrom::Current(-38)).unwrap();
file.seek(SeekFrom::Current(40)).unwrap();
file.seek(SeekFrom::Current(-38)).unwrap();
let file = Rc::new(file);
let read_buf = file.clone().read(2).await.unwrap();
assert_eq!(read_buf.to_vec(), b"67");
file.clone().seek_sync(SeekFrom::Current(-2)).unwrap();

218
cli/sys.rs Normal file
View file

@ -0,0 +1,218 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
// todo(dsherret): this should instead use conditional compilation and directly
// surface the underlying implementation.
//
// The problem atm is that there's no way to have conditional compilation for
// denort or the deno binary. We should extract out denort to a separate binary.
use std::borrow::Cow;
use sys_traits::boxed::BoxedFsDirEntry;
use sys_traits::boxed::BoxedFsFile;
use sys_traits::boxed::BoxedFsMetadataValue;
use sys_traits::boxed::FsMetadataBoxed;
use sys_traits::boxed::FsOpenBoxed;
use sys_traits::boxed::FsReadDirBoxed;
use sys_traits::CreateDirOptions;
use crate::standalone::DenoCompileFileSystem;
#[derive(Debug, Clone)]
pub enum CliSys {
#[allow(dead_code)] // will be dead code for denort
#[allow(clippy::disallowed_types)] // ok because sys impl
Real(sys_traits::impls::RealSys),
#[allow(dead_code)] // will be dead code for deno
DenoCompile(DenoCompileFileSystem),
}
impl Default for CliSys {
fn default() -> Self {
Self::Real(sys_traits::impls::RealSys)
}
}
impl deno_runtime::deno_node::ExtNodeSys for CliSys {}
impl sys_traits::BaseFsHardLink for CliSys {
fn base_fs_hard_link(
&self,
src: &std::path::Path,
dst: &std::path::Path,
) -> std::io::Result<()> {
match self {
Self::Real(sys) => sys.base_fs_hard_link(src, dst),
Self::DenoCompile(sys) => sys.base_fs_hard_link(src, dst),
}
}
}
impl sys_traits::BaseFsRead for CliSys {
fn base_fs_read(
&self,
p: &std::path::Path,
) -> std::io::Result<Cow<'static, [u8]>> {
match self {
Self::Real(sys) => sys.base_fs_read(p),
Self::DenoCompile(sys) => sys.base_fs_read(p),
}
}
}
impl sys_traits::BaseFsReadDir for CliSys {
type ReadDirEntry = BoxedFsDirEntry;
fn base_fs_read_dir(
&self,
p: &std::path::Path,
) -> std::io::Result<
Box<dyn Iterator<Item = std::io::Result<Self::ReadDirEntry>> + '_>,
> {
match self {
Self::Real(sys) => sys.fs_read_dir_boxed(p),
Self::DenoCompile(sys) => sys.fs_read_dir_boxed(p),
}
}
}
impl sys_traits::BaseFsCanonicalize for CliSys {
fn base_fs_canonicalize(
&self,
p: &std::path::Path,
) -> std::io::Result<std::path::PathBuf> {
match self {
Self::Real(sys) => sys.base_fs_canonicalize(p),
Self::DenoCompile(sys) => sys.base_fs_canonicalize(p),
}
}
}
impl sys_traits::BaseFsMetadata for CliSys {
type Metadata = BoxedFsMetadataValue;
fn base_fs_metadata(
&self,
path: &std::path::Path,
) -> std::io::Result<Self::Metadata> {
match self {
Self::Real(sys) => sys.fs_metadata_boxed(path),
Self::DenoCompile(sys) => sys.fs_metadata_boxed(path),
}
}
fn base_fs_symlink_metadata(
&self,
path: &std::path::Path,
) -> std::io::Result<Self::Metadata> {
match self {
Self::Real(sys) => sys.fs_symlink_metadata_boxed(path),
Self::DenoCompile(sys) => sys.fs_symlink_metadata_boxed(path),
}
}
}
impl sys_traits::BaseFsCreateDir for CliSys {
fn base_fs_create_dir(
&self,
p: &std::path::Path,
options: &CreateDirOptions,
) -> std::io::Result<()> {
match self {
Self::Real(sys) => sys.base_fs_create_dir(p, options),
Self::DenoCompile(sys) => sys.base_fs_create_dir(p, options),
}
}
}
impl sys_traits::BaseFsOpen for CliSys {
type File = BoxedFsFile;
fn base_fs_open(
&self,
path: &std::path::Path,
options: &sys_traits::OpenOptions,
) -> std::io::Result<Self::File> {
match self {
Self::Real(sys) => sys.fs_open_boxed(path, options),
Self::DenoCompile(sys) => sys.fs_open_boxed(path, options),
}
}
}
impl sys_traits::BaseFsRemoveFile for CliSys {
fn base_fs_remove_file(&self, p: &std::path::Path) -> std::io::Result<()> {
match self {
Self::Real(sys) => sys.base_fs_remove_file(p),
Self::DenoCompile(sys) => sys.base_fs_remove_file(p),
}
}
}
impl sys_traits::BaseFsRename for CliSys {
fn base_fs_rename(
&self,
old: &std::path::Path,
new: &std::path::Path,
) -> std::io::Result<()> {
match self {
Self::Real(sys) => sys.base_fs_rename(old, new),
Self::DenoCompile(sys) => sys.base_fs_rename(old, new),
}
}
}
impl sys_traits::SystemRandom for CliSys {
fn sys_random(&self, buf: &mut [u8]) -> std::io::Result<()> {
match self {
Self::Real(sys) => sys.sys_random(buf),
Self::DenoCompile(sys) => sys.sys_random(buf),
}
}
}
impl sys_traits::SystemTimeNow for CliSys {
fn sys_time_now(&self) -> std::time::SystemTime {
match self {
Self::Real(sys) => sys.sys_time_now(),
Self::DenoCompile(sys) => sys.sys_time_now(),
}
}
}
impl sys_traits::ThreadSleep for CliSys {
fn thread_sleep(&self, dur: std::time::Duration) {
match self {
Self::Real(sys) => sys.thread_sleep(dur),
Self::DenoCompile(sys) => sys.thread_sleep(dur),
}
}
}
impl sys_traits::EnvCurrentDir for CliSys {
fn env_current_dir(&self) -> std::io::Result<std::path::PathBuf> {
match self {
Self::Real(sys) => sys.env_current_dir(),
Self::DenoCompile(sys) => sys.env_current_dir(),
}
}
}
impl sys_traits::BaseEnvVar for CliSys {
fn base_env_var_os(
&self,
key: &std::ffi::OsStr,
) -> Option<std::ffi::OsString> {
match self {
Self::Real(sys) => sys.base_env_var_os(key),
Self::DenoCompile(sys) => sys.base_env_var_os(key),
}
}
}
impl sys_traits::EnvHomeDir for CliSys {
fn env_home_dir(&self) -> Option<std::path::PathBuf> {
#[allow(clippy::disallowed_types)] // ok because sys impl
sys_traits::impls::RealSys.env_home_dir()
}
}

6
cli/task_runner.rs
View file

@ -10,7 +10,6 @@ use deno_core::anyhow::Context;
use deno_core::error::AnyError;
use deno_core::futures;
use deno_core::futures::future::LocalBoxFuture;
use deno_runtime::deno_node::NodeResolver;
use deno_semver::package::PackageNv;
use deno_task_shell::ExecutableCommand;
use deno_task_shell::ExecuteResult;
@ -25,6 +24,7 @@ use tokio::task::JoinHandle;
use tokio::task::LocalSet;
use tokio_util::sync::CancellationToken;
use crate::node::CliNodeResolver;
use crate::npm::CliNpmResolver;
use crate::npm::InnerCliNpmResolverRef;
use crate::npm::ManagedCliNpmResolver;
@ -415,7 +415,7 @@ impl ShellCommand for NodeModulesFileRunCommand {
pub fn resolve_custom_commands(
npm_resolver: &dyn CliNpmResolver,
node_resolver: &NodeResolver,
node_resolver: &CliNodeResolver,
) -> Result<HashMap<String, Rc<dyn ShellCommand>>, AnyError> {
let mut commands = match npm_resolver.as_inner() {
InnerCliNpmResolverRef::Byonm(npm_resolver) => {
@ -522,7 +522,7 @@ fn resolve_execution_path_from_npx_shim(
fn resolve_managed_npm_commands(
npm_resolver: &ManagedCliNpmResolver,
node_resolver: &NodeResolver,
node_resolver: &CliNodeResolver,
) -> Result<HashMap<String, Rc<dyn ShellCommand>>, AnyError> {
let mut result = HashMap::new();
let snapshot = npm_resolver.snapshot();

3
cli/tools/bench/mod.rs
View file

@ -7,6 +7,7 @@ use crate::display::write_json_to_stdout;
use crate::factory::CliFactory;
use crate::graph_util::has_graph_root_local_dependent_changed;
use crate::ops;
use crate::sys::CliSys;
use crate::tools::test::format_test_error;
use crate::tools::test::TestFilter;
use crate::util::file_watcher;
@ -265,7 +266,7 @@ async fn bench_specifier_inner(
async fn bench_specifiers(
worker_factory: Arc<CliMainWorkerFactory>,
permissions: &Permissions,
permissions_desc_parser: &Arc<RuntimePermissionDescriptorParser>,
permissions_desc_parser: &Arc<RuntimePermissionDescriptorParser<CliSys>>,
specifiers: Vec<ModuleSpecifier>,
options: BenchSpecifierOptions,
) -> Result<(), AnyError> {

8
cli/tools/check.rs
View file

@ -9,7 +9,6 @@ use deno_ast::ModuleSpecifier;
use deno_core::error::AnyError;
use deno_graph::Module;
use deno_graph::ModuleGraph;
use deno_runtime::deno_node::NodeResolver;
use deno_terminal::colors;
use once_cell::sync::Lazy;
use regex::Regex;
@ -29,6 +28,7 @@ use crate::cache::TypeCheckCache;
use crate::factory::CliFactory;
use crate::graph_util::BuildFastCheckGraphOptions;
use crate::graph_util::ModuleGraphBuilder;
use crate::node::CliNodeResolver;
use crate::npm::CliNpmResolver;
use crate::tsc;
use crate::tsc::Diagnostics;
@ -64,7 +64,7 @@ pub async fn check(
let file = file_fetcher.fetch(&s, root_permissions).await?;
let snippet_files = extract::extract_snippet_files(file)?;
for snippet_file in snippet_files {
specifiers_for_typecheck.push(snippet_file.specifier.clone());
specifiers_for_typecheck.push(snippet_file.url.clone());
file_fetcher.insert_memory_files(snippet_file);
}
}
@ -103,7 +103,7 @@ pub struct TypeChecker {
cjs_tracker: Arc<TypeCheckingCjsTracker>,
cli_options: Arc<CliOptions>,
module_graph_builder: Arc<ModuleGraphBuilder>,
node_resolver: Arc<NodeResolver>,
node_resolver: Arc<CliNodeResolver>,
npm_resolver: Arc<dyn CliNpmResolver>,
}
@ -113,7 +113,7 @@ impl TypeChecker {
cjs_tracker: Arc<TypeCheckingCjsTracker>,
cli_options: Arc<CliOptions>,
module_graph_builder: Arc<ModuleGraphBuilder>,
node_resolver: Arc<NodeResolver>,
node_resolver: Arc<CliNodeResolver>,
npm_resolver: Arc<dyn CliNpmResolver>,
) -> Self {
Self {

3
cli/tools/clean.rs
View file

@ -7,6 +7,7 @@ use std::path::Path;
use crate::cache::DenoDir;
use crate::colors;
use crate::display;
use crate::sys::CliSys;
use crate::util::progress_bar::ProgressBar;
use crate::util::progress_bar::ProgressBarStyle;
use crate::util::progress_bar::ProgressMessagePrompt;
@ -28,7 +29,7 @@ impl CleanState {
}
pub fn clean() -> Result<(), AnyError> {
let deno_dir = DenoDir::new(None)?;
let deno_dir = DenoDir::new(CliSys::default(), None)?;
if deno_dir.root.exists() {
let no_of_files = walkdir::WalkDir::new(&deno_dir.root).into_iter().count();
let progress_bar = ProgressBar::new(ProgressBarStyle::ProgressBars);

2
cli/tools/compile.rs
View file

@ -5,8 +5,8 @@ use crate::args::CompileFlags;
use crate::args::Flags;
use crate::factory::CliFactory;
use crate::http_util::HttpClientProvider;
use crate::standalone::binary::is_standalone_binary;
use crate::standalone::binary::WriteBinOptions;
use crate::standalone::is_standalone_binary;
use deno_ast::MediaType;
use deno_ast::ModuleSpecifier;
use deno_core::anyhow::bail;

35
cli/tools/coverage/mod.rs
View file

@ -6,6 +6,8 @@ use crate::args::FileFlags;
use crate::args::Flags;
use crate::cdp;
use crate::factory::CliFactory;
use crate::file_fetcher::TextDecodedFile;
use crate::sys::CliSys;
use crate::tools::fmt::format_json;
use crate::tools::test::is_supported_test_path;
use crate::util::text_encoding::source_map_from_code;
@ -197,7 +199,7 @@ pub struct CoverageReport {
fn generate_coverage_report(
script_coverage: &cdp::ScriptCoverage,
script_source: String,
maybe_source_map: &Option<Vec<u8>>,
maybe_source_map: Option<&[u8]>,
output: &Option<PathBuf>,
) -> CoverageReport {
let maybe_source_map = maybe_source_map
@ -427,7 +429,7 @@ fn collect_coverages(
.ignore_git_folder()
.ignore_node_modules()
.set_vendor_folder(cli_options.vendor_dir_path().map(ToOwned::to_owned))
.collect_file_patterns(&deno_config::fs::RealDenoConfigFs, file_patterns)?;
.collect_file_patterns(&CliSys::default(), file_patterns)?;
let coverage_patterns = FilePatterns {
base: initial_cwd.to_path_buf(),
@ -559,6 +561,12 @@ pub fn cover_files(
},
None => None,
};
let get_message = |specifier: &ModuleSpecifier| -> String {
format!(
"Failed to fetch \"{}\" from cache. Before generating coverage report, run `deno test --coverage` to ensure consistent state.",
specifier,
)
};
for script_coverage in script_coverages {
let module_specifier = deno_core::resolve_url_or_path(
@ -566,21 +574,14 @@ pub fn cover_files(
cli_options.initial_cwd(),
)?;
let maybe_file = if module_specifier.scheme() == "file" {
file_fetcher.get_source(&module_specifier)
} else {
file_fetcher
.fetch_cached(&module_specifier, 10)
.with_context(|| {
format!("Failed to fetch \"{module_specifier}\" from cache.")
})?
let maybe_file_result = file_fetcher
.get_cached_source_or_local(&module_specifier)
.map_err(AnyError::from);
let file = match maybe_file_result {
Ok(Some(file)) => TextDecodedFile::decode(file)?,
Ok(None) => return Err(anyhow!("{}", get_message(&module_specifier))),
Err(err) => return Err(err).context(get_message(&module_specifier)),
};
let file = maybe_file.ok_or_else(|| {
anyhow!("Failed to fetch \"{}\" from cache.
Before generating coverage report, run `deno test --coverage` to ensure consistent state.",
module_specifier
)
})?.into_text_decoded()?;
let original_source = file.source.clone();
// Check if file was transpiled
@ -625,7 +626,7 @@ pub fn cover_files(
let coverage_report = generate_coverage_report(
&script_coverage,
runtime_code.as_str().to_owned(),
&source_map,
source_map.as_deref(),
&out_mode,
);

9
cli/tools/doc.rs
View file

@ -10,6 +10,7 @@ use crate::factory::CliFactory;
use crate::graph_util::graph_exit_integrity_errors;
use crate::graph_util::graph_walk_errors;
use crate::graph_util::GraphWalkErrorsOptions;
use crate::sys::CliSys;
use crate::tsc::get_types_declaration_file_text;
use crate::util::fs::collect_specifiers;
use deno_ast::diagnostics::Diagnostic;
@ -114,7 +115,7 @@ pub async fn doc(
}
DocSourceFileFlag::Paths(ref source_files) => {
let module_graph_creator = factory.module_graph_creator().await?;
let fs = factory.fs();
let sys = CliSys::default();
let module_specifiers = collect_specifiers(
FilePatterns {
@ -141,7 +142,7 @@ pub async fn doc(
graph_exit_integrity_errors(&graph);
let errors = graph_walk_errors(
&graph,
fs,
&sys,
&module_specifiers,
GraphWalkErrorsOptions {
check_js: false,
@ -343,14 +344,14 @@ impl deno_doc::html::HrefResolver for DocResolver {
let name = &res.req().name;
Some((
format!("https://www.npmjs.com/package/{name}"),
name.to_owned(),
name.to_string(),
))
}
"jsr" => {
let res =
deno_semver::jsr::JsrPackageReqReference::from_str(module).ok()?;
let name = &res.req().name;
Some((format!("https://jsr.io/{name}"), name.to_owned()))
Some((format!("https://jsr.io/{name}"), name.to_string()))
}
_ => None,
}

5
cli/tools/fmt.rs
View file

@ -17,6 +17,7 @@ use crate::args::UnstableFmtOptions;
use crate::cache::Caches;
use crate::colors;
use crate::factory::CliFactory;
use crate::sys::CliSys;
use crate::util::diff::diff;
use crate::util::file_watcher;
use crate::util::fs::canonicalize_path;
@ -57,7 +58,7 @@ pub async fn format(
fmt_flags: FmtFlags,
) -> Result<(), AnyError> {
if fmt_flags.is_stdin() {
let cli_options = CliOptions::from_flags(flags)?;
let cli_options = CliOptions::from_flags(&CliSys::default(), flags)?;
let start_dir = &cli_options.start_dir;
let fmt_config = start_dir
.to_fmt_config(FilePatterns::new_with_base(start_dir.dir_path()))?;
@ -230,7 +231,7 @@ fn collect_fmt_files(
.ignore_node_modules()
.use_gitignore()
.set_vendor_folder(cli_options.vendor_dir_path().map(ToOwned::to_owned))
.collect_file_patterns(&deno_config::fs::RealDenoConfigFs, files)
.collect_file_patterns(&CliSys::default(), files)
}
/// Formats markdown (using <https://github.com/dprint/dprint-plugin-markdown>) and its code blocks

19
cli/tools/info.rs
View file

@ -278,8 +278,10 @@ fn add_npm_packages_to_json(
});
if let Some(pkg) = maybe_package {
if let Some(module) = module.as_object_mut() {
module
.insert("npmPackage".to_string(), pkg.id.as_serialized().into());
module.insert(
"npmPackage".to_string(),
pkg.id.as_serialized().into_string().into(),
);
}
}
}
@ -296,7 +298,7 @@ fn add_npm_packages_to_json(
{
dep.insert(
"npmPackage".to_string(),
pkg.id.as_serialized().into(),
pkg.id.as_serialized().into_string().into(),
);
}
}
@ -324,19 +326,19 @@ fn add_npm_packages_to_json(
let mut json_packages = serde_json::Map::with_capacity(sorted_packages.len());
for pkg in sorted_packages {
let mut kv = serde_json::Map::new();
kv.insert("name".to_string(), pkg.id.nv.name.clone().into());
kv.insert("name".to_string(), pkg.id.nv.name.to_string().into());
kv.insert("version".to_string(), pkg.id.nv.version.to_string().into());
let mut deps = pkg.dependencies.values().collect::<Vec<_>>();
deps.sort();
let deps = deps
.into_iter()
.map(|id| serde_json::Value::String(id.as_serialized()))
.map(|id| serde_json::Value::String(id.as_serialized().into_string()))
.collect::<Vec<_>>();
kv.insert("dependencies".to_string(), deps.into());
let registry_url = npmrc.get_registry_url(&pkg.id.nv.name);
kv.insert("registryUrl".to_string(), registry_url.to_string().into());
json_packages.insert(pkg.id.as_serialized(), kv.into());
json_packages.insert(pkg.id.as_serialized().into_string(), kv.into());
}
json.insert("npmPackages".to_string(), json_packages.into());
@ -549,7 +551,7 @@ impl<'a> GraphDisplayContext<'a> {
None => Specifier(module.specifier().clone()),
};
let was_seen = !self.seen.insert(match &package_or_specifier {
Package(package) => package.id.as_serialized(),
Package(package) => package.id.as_serialized().into_string(),
Specifier(specifier) => specifier.to_string(),
});
let header_text = if was_seen {
@ -631,7 +633,8 @@ impl<'a> GraphDisplayContext<'a> {
));
if let Some(package) = self.npm_info.packages.get(dep_id) {
if !package.dependencies.is_empty() {
let was_seen = !self.seen.insert(package.id.as_serialized());
let was_seen =
!self.seen.insert(package.id.as_serialized().into_string());
if was_seen {
child.text = format!("{} {}", child.text, colors::gray("*"));
} else {

17
cli/tools/installer.rs
View file

@ -3,7 +3,6 @@
use crate::args::resolve_no_prompt;
use crate::args::AddFlags;
use crate::args::CaData;
use crate::args::CacheSetting;
use crate::args::ConfigFlag;
use crate::args::Flags;
use crate::args::InstallFlags;
@ -13,13 +12,14 @@ use crate::args::TypeCheckMode;
use crate::args::UninstallFlags;
use crate::args::UninstallKind;
use crate::factory::CliFactory;
use crate::file_fetcher::FileFetcher;
use crate::file_fetcher::CliFileFetcher;
use crate::graph_container::ModuleGraphContainer;
use crate::http_util::HttpClientProvider;
use crate::jsr::JsrFetchResolver;
use crate::npm::NpmFetchResolver;
use crate::util::fs::canonicalize_path_maybe_not_exists;
use deno_cache_dir::file_fetcher::CacheSetting;
use deno_core::anyhow::bail;
use deno_core::anyhow::Context;
use deno_core::error::generic_error;
@ -161,11 +161,11 @@ pub async fn infer_name_from_url(
let npm_ref = npm_ref.into_inner();
if let Some(sub_path) = npm_ref.sub_path {
if !sub_path.contains('/') {
return Some(sub_path);
return Some(sub_path.to_string());
}
}
if !npm_ref.req.name.contains('/') {
return Some(npm_ref.req.name);
return Some(npm_ref.req.name.into_string());
}
return None;
}
@ -361,18 +361,19 @@ async fn install_global(
let cli_options = factory.cli_options()?;
let http_client = factory.http_client_provider();
let deps_http_cache = factory.global_http_cache()?;
let mut deps_file_fetcher = FileFetcher::new(
let deps_file_fetcher = CliFileFetcher::new(
deps_http_cache.clone(),
CacheSetting::ReloadAll,
true,
http_client.clone(),
factory.sys(),
Default::default(),
None,
true,
CacheSetting::ReloadAll,
log::Level::Trace,
);
let npmrc = factory.cli_options().unwrap().npmrc();
deps_file_fetcher.set_download_log_level(log::Level::Trace);
let deps_file_fetcher = Arc::new(deps_file_fetcher);
let jsr_resolver = Arc::new(JsrFetchResolver::new(deps_file_fetcher.clone()));
let npm_resolver = Arc::new(NpmFetchResolver::new(

536
cli/tools/lint/ast_buffer/buffer.rs Normal file
View file

@ -0,0 +1,536 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use std::fmt::Display;
use deno_ast::swc::common::Span;
use deno_ast::swc::common::DUMMY_SP;
use indexmap::IndexMap;
/// Each property has this flag to mark what kind of value it holds-
/// Plain objects and arrays are not supported yet, but could be easily
/// added if needed.
#[derive(Debug, PartialEq)]
pub enum PropFlags {
Ref,
RefArr,
String,
Bool,
Null,
Undefined,
}
impl From<PropFlags> for u8 {
fn from(m: PropFlags) -> u8 {
m as u8
}
}
impl TryFrom<u8> for PropFlags {
type Error = &'static str;
fn try_from(value: u8) -> Result<Self, Self::Error> {
match value {
0 => Ok(PropFlags::Ref),
1 => Ok(PropFlags::RefArr),
2 => Ok(PropFlags::String),
3 => Ok(PropFlags::Bool),
4 => Ok(PropFlags::Null),
5 => Ok(PropFlags::Undefined),
_ => Err("Unknown Prop flag"),
}
}
}
const MASK_U32_1: u32 = 0b11111111_00000000_00000000_00000000;
const MASK_U32_2: u32 = 0b00000000_11111111_00000000_00000000;
const MASK_U32_3: u32 = 0b00000000_00000000_11111111_00000000;
const MASK_U32_4: u32 = 0b00000000_00000000_00000000_11111111;
// TODO: There is probably a native Rust function to do this.
pub fn append_u32(result: &mut Vec<u8>, value: u32) {
let v1: u8 = ((value & MASK_U32_1) >> 24) as u8;
let v2: u8 = ((value & MASK_U32_2) >> 16) as u8;
let v3: u8 = ((value & MASK_U32_3) >> 8) as u8;
let v4: u8 = (value & MASK_U32_4) as u8;
result.push(v1);
result.push(v2);
result.push(v3);
result.push(v4);
}
pub fn append_usize(result: &mut Vec<u8>, value: usize) {
let raw = u32::try_from(value).unwrap();
append_u32(result, raw);
}
pub fn write_usize(result: &mut [u8], value: usize, idx: usize) {
let raw = u32::try_from(value).unwrap();
let v1: u8 = ((raw & MASK_U32_1) >> 24) as u8;
let v2: u8 = ((raw & MASK_U32_2) >> 16) as u8;
let v3: u8 = ((raw & MASK_U32_3) >> 8) as u8;
let v4: u8 = (raw & MASK_U32_4) as u8;
result[idx] = v1;
result[idx + 1] = v2;
result[idx + 2] = v3;
result[idx + 3] = v4;
}
#[derive(Debug)]
pub struct StringTable {
id: usize,
table: IndexMap<String, usize>,
}
impl StringTable {
pub fn new() -> Self {
Self {
id: 0,
table: IndexMap::new(),
}
}
pub fn insert(&mut self, s: &str) -> usize {
if let Some(id) = self.table.get(s) {
return *id;
}
let id = self.id;
self.id += 1;
self.table.insert(s.to_string(), id);
id
}
pub fn serialize(&mut self) -> Vec<u8> {
let mut result: Vec<u8> = vec![];
append_u32(&mut result, self.table.len() as u32);
// Assume that it's sorted by id
for (s, _id) in &self.table {
let bytes = s.as_bytes();
append_u32(&mut result, bytes.len() as u32);
result.append(&mut bytes.to_vec());
}
result
}
}
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct NodeRef(pub usize);
/// Represents an offset to a node whose schema hasn't been committed yet
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct PendingNodeRef(pub NodeRef);
#[derive(Debug)]
pub struct BoolPos(pub usize);
#[derive(Debug)]
pub struct FieldPos(pub usize);
#[derive(Debug)]
pub struct FieldArrPos(pub usize);
#[derive(Debug)]
pub struct StrPos(pub usize);
#[derive(Debug)]
pub struct UndefPos(pub usize);
#[derive(Debug)]
pub struct NullPos(pub usize);
#[derive(Debug)]
pub enum NodePos {
Bool(BoolPos),
#[allow(dead_code)]
Field(FieldPos),
#[allow(dead_code)]
FieldArr(FieldArrPos),
Str(StrPos),
Undef(UndefPos),
#[allow(dead_code)]
Null(NullPos),
}
pub trait AstBufSerializer<K, P>
where
K: Into<u8> + Display,
P: Into<u8> + Display,
{
fn header(&mut self, kind: K, parent: NodeRef, span: &Span)
-> PendingNodeRef;
fn ref_field(&mut self, prop: P) -> FieldPos;
fn ref_vec_field(&mut self, prop: P, len: usize) -> FieldArrPos;
fn str_field(&mut self, prop: P) -> StrPos;
fn bool_field(&mut self, prop: P) -> BoolPos;
fn undefined_field(&mut self, prop: P) -> UndefPos;
#[allow(dead_code)]
fn null_field(&mut self, prop: P) -> NullPos;
fn commit_schema(&mut self, offset: PendingNodeRef) -> NodeRef;
fn write_ref(&mut self, pos: FieldPos, value: NodeRef);
fn write_maybe_ref(&mut self, pos: FieldPos, value: Option<NodeRef>);
fn write_refs(&mut self, pos: FieldArrPos, value: Vec<NodeRef>);
fn write_str(&mut self, pos: StrPos, value: &str);
fn write_bool(&mut self, pos: BoolPos, value: bool);
fn serialize(&mut self) -> Vec<u8>;
}
#[derive(Debug)]
pub struct SerializeCtx {
buf: Vec<u8>,
start_buf: NodeRef,
str_table: StringTable,
kind_map: Vec<usize>,
prop_map: Vec<usize>,
field_count: u8,
}
/// This is the internal context used to allocate and fill the buffer. The point
/// is to be able to write absolute offsets directly in place.
///
/// The typical workflow is to reserve all necessary space for the currrent
/// node with placeholders for the offsets of the child nodes. Once child
/// nodes have been traversed, we know their offsets and can replace the
/// placeholder values with the actual ones.
impl SerializeCtx {
pub fn new(kind_len: u8, prop_len: u8) -> Self {
let kind_size = kind_len as usize;
let prop_size = prop_len as usize;
let mut ctx = Self {
start_buf: NodeRef(0),
buf: vec![],
str_table: StringTable::new(),
kind_map: vec![0; kind_size],
prop_map: vec![0; prop_size],
field_count: 0,
};
let empty_str = ctx.str_table.insert("");
// Placeholder node is always 0
ctx.append_node(0, NodeRef(0), &DUMMY_SP, 0);
ctx.kind_map[0] = empty_str;
ctx.start_buf = NodeRef(ctx.buf.len());
// Insert default props that are always present
let type_str = ctx.str_table.insert("type");
let parent_str = ctx.str_table.insert("parent");
let range_str = ctx.str_table.insert("range");
let length_str = ctx.str_table.insert("length");
// These values are expected to be in this order on the JS side
ctx.prop_map[0] = empty_str;
ctx.prop_map[1] = type_str;
ctx.prop_map[2] = parent_str;
ctx.prop_map[3] = range_str;
ctx.prop_map[4] = length_str;
ctx
}
/// Allocate a node's header
fn field_header<P>(&mut self, prop: P, prop_flags: PropFlags) -> usize
where
P: Into<u8> + Display + Clone,
{
self.field_count += 1;
let offset = self.buf.len();
let n: u8 = prop.clone().into();
self.buf.push(n);
if let Some(v) = self.prop_map.get::<usize>(n.into()) {
if *v == 0 {
let id = self.str_table.insert(&format!("{prop}"));
self.prop_map[n as usize] = id;
}
}
let flags: u8 = prop_flags.into();
self.buf.push(flags);
offset
}
/// Allocate a property pointing to another node.
fn field<P>(&mut self, prop: P, prop_flags: PropFlags) -> usize
where
P: Into<u8> + Display + Clone,
{
let offset = self.field_header(prop, prop_flags);
append_usize(&mut self.buf, 0);
offset
}
fn append_node(
&mut self,
kind: u8,
parent: NodeRef,
span: &Span,
prop_count: usize,
) -> PendingNodeRef {
let offset = self.buf.len();
// Node type fits in a u8
self.buf.push(kind);
// Offset to the parent node. Will be 0 if none exists
append_usize(&mut self.buf, parent.0);
// Span, the start and end location of this node
append_u32(&mut self.buf, span.lo.0);
append_u32(&mut self.buf, span.hi.0);
// No node has more than <10 properties
debug_assert!(prop_count < 10);
self.buf.push(prop_count as u8);
PendingNodeRef(NodeRef(offset))
}
pub fn commit_schema(&mut self, node_ref: PendingNodeRef) -> NodeRef {
let mut offset = node_ref.0 .0;
// type + parentId + span lo + span hi
offset += 1 + 4 + 4 + 4;
self.buf[offset] = self.field_count;
self.field_count = 0;
node_ref.0
}
/// Allocate the node header. It's always the same for every node.
/// <type u8>
/// <parent offset u32>
/// <span lo u32>
/// <span high u32>
/// <property count u8> (There is no node with more than 10 properties)
pub fn header<N>(
&mut self,
kind: N,
parent: NodeRef,
span: &Span,
) -> PendingNodeRef
where
N: Into<u8> + Display + Clone,
{
let n: u8 = kind.clone().into();
if let Some(v) = self.kind_map.get::<usize>(n.into()) {
if *v == 0 {
let id = self.str_table.insert(&format!("{kind}"));
self.kind_map[n as usize] = id;
}
}
// Prop count will be filled with the actual value when the
// schema is committed.
self.append_node(n, parent, span, 0)
}
/// Allocate a reference property that will hold the offset of
/// another node.
pub fn ref_field<P>(&mut self, prop: P) -> usize
where
P: Into<u8> + Display + Clone,
{
self.field(prop, PropFlags::Ref)
}
/// Allocate a property that is a vec of node offsets pointing to other
/// nodes.
pub fn ref_vec_field<P>(&mut self, prop: P, len: usize) -> usize
where
P: Into<u8> + Display + Clone,
{
let offset = self.field(prop, PropFlags::RefArr);
for _ in 0..len {
append_u32(&mut self.buf, 0);
}
offset
}
// Allocate a property representing a string. Strings are deduplicated
// in the message and the property will only contain the string id.
pub fn str_field<P>(&mut self, prop: P) -> usize
where
P: Into<u8> + Display + Clone,
{
self.field(prop, PropFlags::String)
}
/// Allocate a bool field
pub fn bool_field<P>(&mut self, prop: P) -> usize
where
P: Into<u8> + Display + Clone,
{
let offset = self.field_header(prop, PropFlags::Bool);
self.buf.push(0);
offset
}
/// Allocate an undefined field
pub fn undefined_field<P>(&mut self, prop: P) -> usize
where
P: Into<u8> + Display + Clone,
{
self.field_header(prop, PropFlags::Undefined)
}
/// Allocate an undefined field
#[allow(dead_code)]
pub fn null_field<P>(&mut self, prop: P) -> usize
where
P: Into<u8> + Display + Clone,
{
self.field_header(prop, PropFlags::Null)
}
/// Replace the placeholder of a reference field with the actual offset
/// to the node we want to point to.
pub fn write_ref(&mut self, field_offset: usize, value: NodeRef) {
#[cfg(debug_assertions)]
{
let value_kind = self.buf[field_offset + 1];
if PropFlags::try_from(value_kind).unwrap() != PropFlags::Ref {
panic!("Trying to write a ref into a non-ref field")
}
}
write_usize(&mut self.buf, value.0, field_offset + 2);
}
/// Helper for writing optional node offsets
pub fn write_maybe_ref(
&mut self,
field_offset: usize,
value: Option<NodeRef>,
) {
#[cfg(debug_assertions)]
{
let value_kind = self.buf[field_offset + 1];
if PropFlags::try_from(value_kind).unwrap() != PropFlags::Ref {
panic!("Trying to write a ref into a non-ref field")
}
}
let ref_value = if let Some(v) = value { v } else { NodeRef(0) };
write_usize(&mut self.buf, ref_value.0, field_offset + 2);
}
/// Write a vec of node offsets into the property. The necessary space
/// has been reserved earlier.
pub fn write_refs(&mut self, field_offset: usize, value: Vec<NodeRef>) {
#[cfg(debug_assertions)]
{
let value_kind = self.buf[field_offset + 1];
if PropFlags::try_from(value_kind).unwrap() != PropFlags::RefArr {
panic!("Trying to write a ref into a non-ref array field")
}
}
let mut offset = field_offset + 2;
write_usize(&mut self.buf, value.len(), offset);
offset += 4;
for item in value {
write_usize(&mut self.buf, item.0, offset);
offset += 4;
}
}
/// Store the string in our string table and save the id of the string
/// in the current field.
pub fn write_str(&mut self, field_offset: usize, value: &str) {
#[cfg(debug_assertions)]
{
let value_kind = self.buf[field_offset + 1];
if PropFlags::try_from(value_kind).unwrap() != PropFlags::String {
panic!("Trying to write a ref into a non-string field")
}
}
let id = self.str_table.insert(value);
write_usize(&mut self.buf, id, field_offset + 2);
}
/// Write a bool to a field.
pub fn write_bool(&mut self, field_offset: usize, value: bool) {
#[cfg(debug_assertions)]
{
let value_kind = self.buf[field_offset + 1];
if PropFlags::try_from(value_kind).unwrap() != PropFlags::Bool {
panic!("Trying to write a ref into a non-bool field")
}
}
self.buf[field_offset + 2] = if value { 1 } else { 0 };
}
/// Serialize all information we have into a buffer that can be sent to JS.
/// It has the following structure:
///
/// <...ast>
/// <string table>
/// <node kind map> <- node kind id maps to string id
/// <node prop map> <- node property id maps to string id
/// <offset kind map>
/// <offset prop map>
/// <offset str table>
pub fn serialize(&mut self) -> Vec<u8> {
let mut buf: Vec<u8> = vec![];
// The buffer starts with the serialized AST first, because that
// contains absolute offsets. By butting this at the start of the
// message we don't have to waste time updating any offsets.
buf.append(&mut self.buf);
// Next follows the string table. We'll keep track of the offset
// in the message of where the string table begins
let offset_str_table = buf.len();
// Serialize string table
buf.append(&mut self.str_table.serialize());
// Next, serialize the mappings of kind -> string of encountered
// nodes in the AST. We use this additional lookup table to compress
// the message so that we can save space by using a u8 . All nodes of
// JS, TS and JSX together are <200
let offset_kind_map = buf.len();
// Write the total number of entries in the kind -> str mapping table
// TODO: make this a u8
append_usize(&mut buf, self.kind_map.len());
for v in &self.kind_map {
append_usize(&mut buf, *v);
}
// Store offset to prop -> string map. It's the same as with node kind
// as the total number of properties is <120 which allows us to store it
// as u8.
let offset_prop_map = buf.len();
// Write the total number of entries in the kind -> str mapping table
append_usize(&mut buf, self.prop_map.len());
for v in &self.prop_map {
append_usize(&mut buf, *v);
}
// Putting offsets of relevant parts of the buffer at the end. This
// allows us to hop to the relevant part by merely looking at the last
// for values in the message. Each value represents an offset into the
// buffer.
append_usize(&mut buf, offset_kind_map);
append_usize(&mut buf, offset_prop_map);
append_usize(&mut buf, offset_str_table);
append_usize(&mut buf, self.start_buf.0);
buf
}
}

13
cli/tools/lint/ast_buffer/mod.rs Normal file
View file

@ -0,0 +1,13 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use deno_ast::ParsedSource;
use swc::serialize_swc_to_buffer;
mod buffer;
mod swc;
mod ts_estree;
pub fn serialize_ast_to_buffer(parsed_source: &ParsedSource) -> Vec<u8> {
// TODO: We could support multiple languages here
serialize_swc_to_buffer(parsed_source)
}

3134
cli/tools/lint/ast_buffer/swc.rs Normal file
View file

File diff suppressed because it is too large Load diff

520
cli/tools/lint/ast_buffer/ts_estree.rs Normal file
View file

@ -0,0 +1,520 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use std::fmt;
use std::fmt::Debug;
use std::fmt::Display;
use deno_ast::swc::common::Span;
use super::buffer::AstBufSerializer;
use super::buffer::BoolPos;
use super::buffer::FieldArrPos;
use super::buffer::FieldPos;
use super::buffer::NodeRef;
use super::buffer::NullPos;
use super::buffer::PendingNodeRef;
use super::buffer::SerializeCtx;
use super::buffer::StrPos;
use super::buffer::UndefPos;
#[derive(Debug, Clone, PartialEq)]
pub enum AstNode {
// First node must always be the empty/invalid node
Invalid,
// Typically the
Program,
// Module declarations
ExportAllDeclaration,
ExportDefaultDeclaration,
ExportNamedDeclaration,
ImportDeclaration,
TsExportAssignment,
TsImportEquals,
TsNamespaceExport,
// Decls
ClassDeclaration,
FunctionDeclaration,
TSEnumDeclaration,
TSInterface,
TsModule,
TsTypeAlias,
Using,
VariableDeclaration,
// Statements
BlockStatement,
BreakStatement,
ContinueStatement,
DebuggerStatement,
DoWhileStatement,
EmptyStatement,
ExpressionStatement,
ForInStatement,
ForOfStatement,
ForStatement,
IfStatement,
LabeledStatement,
ReturnStatement,
SwitchCase,
SwitchStatement,
ThrowStatement,
TryStatement,
WhileStatement,
WithStatement,
// Expressions
ArrayExpression,
ArrowFunctionExpression,
AssignmentExpression,
AwaitExpression,
BinaryExpression,
CallExpression,
ChainExpression,
ClassExpression,
ConditionalExpression,
FunctionExpression,
Identifier,
ImportExpression,
LogicalExpression,
MemberExpression,
MetaProp,
NewExpression,
ObjectExpression,
PrivateIdentifier,
SequenceExpression,
Super,
TaggedTemplateExpression,
TemplateLiteral,
ThisExpression,
TSAsExpression,
TsConstAssertion,
TsInstantiation,
TSNonNullExpression,
TSSatisfiesExpression,
TSTypeAssertion,
UnaryExpression,
UpdateExpression,
YieldExpression,
// TODO: TSEsTree uses a single literal node
// Literals
StringLiteral,
Bool,
Null,
NumericLiteral,
BigIntLiteral,
RegExpLiteral,
EmptyExpr,
SpreadElement,
Property,
VariableDeclarator,
CatchClause,
RestElement,
ExportSpecifier,
TemplateElement,
MethodDefinition,
ClassBody,
// Patterns
ArrayPattern,
AssignmentPattern,
ObjectPattern,
// JSX
JSXAttribute,
JSXClosingElement,
JSXClosingFragment,
JSXElement,
JSXEmptyExpression,
JSXExpressionContainer,
JSXFragment,
JSXIdentifier,
JSXMemberExpression,
JSXNamespacedName,
JSXOpeningElement,
JSXOpeningFragment,
JSXSpreadAttribute,
JSXSpreadChild,
JSXText,
TSTypeAnnotation,
TSTypeParameterDeclaration,
TSTypeParameter,
TSTypeParameterInstantiation,
TSEnumMember,
TSInterfaceBody,
TSInterfaceHeritage,
TSTypeReference,
TSThisType,
TSLiteralType,
TSInferType,
TSConditionalType,
TSUnionType,
TSIntersectionType,
TSMappedType,
TSTypeQuery,
TSTupleType,
TSNamedTupleMember,
TSFunctionType,
TsCallSignatureDeclaration,
TSPropertySignature,
TSMethodSignature,
TSIndexSignature,
TSIndexedAccessType,
TSTypeOperator,
TSTypePredicate,
TSImportType,
TSRestType,
TSArrayType,
TSClassImplements,
TSAnyKeyword,
TSBigIntKeyword,
TSBooleanKeyword,
TSIntrinsicKeyword,
TSNeverKeyword,
TSNullKeyword,
TSNumberKeyword,
TSObjectKeyword,
TSStringKeyword,
TSSymbolKeyword,
TSUndefinedKeyword,
TSUnknownKeyword,
TSVoidKeyword,
TSEnumBody, // Last value is used for max value
}
impl Display for AstNode {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
Debug::fmt(self, f)
}
}
impl From<AstNode> for u8 {
fn from(m: AstNode) -> u8 {
m as u8
}
}
#[derive(Debug, Clone)]
pub enum AstProp {
// Base, these must be in sync with JS in the same order.
Invalid,
Type,
Parent,
Range,
Length, // Not used in AST, but can be used in attr selectors
// Starting from here the order doesn't matter.
// Following are all possible AST node properties.
Abstract,
Accessibility,
Alternate,
Argument,
Arguments,
Asserts,
Async,
Attributes,
Await,
Block,
Body,
Callee,
Cases,
Children,
CheckType,
ClosingElement,
ClosingFragment,
Computed,
Consequent,
Const,
Constraint,
Cooked,
Declaration,
Declarations,
Declare,
Default,
Definite,
Delegate,
Discriminant,
Elements,
ElementType,
ElementTypes,
ExprName,
Expression,
Expressions,
Exported,
Extends,
ExtendsType,
FalseType,
Finalizer,
Flags,
Generator,
Handler,
Id,
In,
IndexType,
Init,
Initializer,
Implements,
Key,
Kind,
Label,
Left,
Literal,
Local,
Members,
Meta,
Method,
Name,
Namespace,
NameType,
Object,
ObjectType,
OpeningElement,
OpeningFragment,
Operator,
Optional,
Out,
Param,
ParameterName,
Params,
Pattern,
Prefix,
Properties,
Property,
Qualifier,
Quasi,
Quasis,
Raw,
Readonly,
ReturnType,
Right,
SelfClosing,
Shorthand,
Source,
SourceType,
Specifiers,
Static,
SuperClass,
SuperTypeArguments,
Tag,
Tail,
Test,
TrueType,
TypeAnnotation,
TypeArguments,
TypeName,
TypeParameter,
TypeParameters,
Types,
Update,
Value, // Last value is used for max value
}
// TODO: Feels like there should be an easier way to iterater over an
// enum in Rust and lowercase the first letter.
impl Display for AstProp {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
let s = match self {
AstProp::Invalid => "__invalid__", // unused
AstProp::Parent => "parent",
AstProp::Range => "range",
AstProp::Type => "type",
AstProp::Length => "length",
AstProp::Abstract => "abstract",
AstProp::Accessibility => "accessibility",
AstProp::Alternate => "alternate",
AstProp::Argument => "argument",
AstProp::Arguments => "arguments",
AstProp::Asserts => "asserts",
AstProp::Async => "async",
AstProp::Attributes => "attributes",
AstProp::Await => "await",
AstProp::Block => "block",
AstProp::Body => "body",
AstProp::Callee => "callee",
AstProp::Cases => "cases",
AstProp::Children => "children",
AstProp::CheckType => "checkType",
AstProp::ClosingElement => "closingElement",
AstProp::ClosingFragment => "closingFragment",
AstProp::Computed => "computed",
AstProp::Consequent => "consequent",
AstProp::Const => "const",
AstProp::Constraint => "constraint",
AstProp::Cooked => "cooked",
AstProp::Declaration => "declaration",
AstProp::Declarations => "declarations",
AstProp::Declare => "declare",
AstProp::Default => "default",
AstProp::Definite => "definite",
AstProp::Delegate => "delegate",
AstProp::Discriminant => "discriminant",
AstProp::Elements => "elements",
AstProp::ElementType => "elementType",
AstProp::ElementTypes => "elementTypes",
AstProp::ExprName => "exprName",
AstProp::Expression => "expression",
AstProp::Expressions => "expressions",
AstProp::Exported => "exported",
AstProp::Extends => "extends",
AstProp::ExtendsType => "extendsType",
AstProp::FalseType => "falseType",
AstProp::Finalizer => "finalizer",
AstProp::Flags => "flags",
AstProp::Generator => "generator",
AstProp::Handler => "handler",
AstProp::Id => "id",
AstProp::In => "in",
AstProp::IndexType => "indexType",
AstProp::Init => "init",
AstProp::Initializer => "initializer",
AstProp::Implements => "implements",
AstProp::Key => "key",
AstProp::Kind => "kind",
AstProp::Label => "label",
AstProp::Left => "left",
AstProp::Literal => "literal",
AstProp::Local => "local",
AstProp::Members => "members",
AstProp::Meta => "meta",
AstProp::Method => "method",
AstProp::Name => "name",
AstProp::Namespace => "namespace",
AstProp::NameType => "nameType",
AstProp::Object => "object",
AstProp::ObjectType => "objectType",
AstProp::OpeningElement => "openingElement",
AstProp::OpeningFragment => "openingFragment",
AstProp::Operator => "operator",
AstProp::Optional => "optional",
AstProp::Out => "out",
AstProp::Param => "param",
AstProp::ParameterName => "parameterName",
AstProp::Params => "params",
AstProp::Pattern => "pattern",
AstProp::Prefix => "prefix",
AstProp::Properties => "properties",
AstProp::Property => "property",
AstProp::Qualifier => "qualifier",
AstProp::Quasi => "quasi",
AstProp::Quasis => "quasis",
AstProp::Raw => "raw",
AstProp::Readonly => "readonly",
AstProp::ReturnType => "returnType",
AstProp::Right => "right",
AstProp::SelfClosing => "selfClosing",
AstProp::Shorthand => "shorthand",
AstProp::Source => "source",
AstProp::SourceType => "sourceType",
AstProp::Specifiers => "specifiers",
AstProp::Static => "static",
AstProp::SuperClass => "superClass",
AstProp::SuperTypeArguments => "superTypeArguments",
AstProp::Tag => "tag",
AstProp::Tail => "tail",
AstProp::Test => "test",
AstProp::TrueType => "trueType",
AstProp::TypeAnnotation => "typeAnnotation",
AstProp::TypeArguments => "typeArguments",
AstProp::TypeName => "typeName",
AstProp::TypeParameter => "typeParameter",
AstProp::TypeParameters => "typeParameters",
AstProp::Types => "types",
AstProp::Update => "update",
AstProp::Value => "value",
};
write!(f, "{}", s)
}
}
impl From<AstProp> for u8 {
fn from(m: AstProp) -> u8 {
m as u8
}
}
pub struct TsEsTreeBuilder {
ctx: SerializeCtx,
}
// TODO: Add a builder API to make it easier to convert from different source
// ast formats.
impl TsEsTreeBuilder {
pub fn new() -> Self {
// Max values
// TODO: Maybe there is a rust macro to grab the last enum value?
let kind_max_count: u8 = u8::from(AstNode::TSEnumBody) + 1;
let prop_max_count: u8 = u8::from(AstProp::Value) + 1;
Self {
ctx: SerializeCtx::new(kind_max_count, prop_max_count),
}
}
}
impl AstBufSerializer<AstNode, AstProp> for TsEsTreeBuilder {
fn header(
&mut self,
kind: AstNode,
parent: NodeRef,
span: &Span,
) -> PendingNodeRef {
self.ctx.header(kind, parent, span)
}
fn commit_schema(&mut self, offset: PendingNodeRef) -> NodeRef {
self.ctx.commit_schema(offset)
}
fn ref_field(&mut self, prop: AstProp) -> FieldPos {
FieldPos(self.ctx.ref_field(prop))
}
fn ref_vec_field(&mut self, prop: AstProp, len: usize) -> FieldArrPos {
FieldArrPos(self.ctx.ref_vec_field(prop, len))
}
fn str_field(&mut self, prop: AstProp) -> StrPos {
StrPos(self.ctx.str_field(prop))
}
fn bool_field(&mut self, prop: AstProp) -> BoolPos {
BoolPos(self.ctx.bool_field(prop))
}
fn undefined_field(&mut self, prop: AstProp) -> UndefPos {
UndefPos(self.ctx.undefined_field(prop))
}
fn null_field(&mut self, prop: AstProp) -> NullPos {
NullPos(self.ctx.null_field(prop))
}
fn write_ref(&mut self, pos: FieldPos, value: NodeRef) {
self.ctx.write_ref(pos.0, value);
}
fn write_maybe_ref(&mut self, pos: FieldPos, value: Option<NodeRef>) {
self.ctx.write_maybe_ref(pos.0, value);
}
fn write_refs(&mut self, pos: FieldArrPos, value: Vec<NodeRef>) {
self.ctx.write_refs(pos.0, value);
}
fn write_str(&mut self, pos: StrPos, value: &str) {
self.ctx.write_str(pos.0, value);
}
fn write_bool(&mut self, pos: BoolPos, value: bool) {
self.ctx.write_bool(pos.0, value);
}
fn serialize(&mut self) -> Vec<u8> {
self.ctx.serialize()
}
}

6
cli/tools/lint/linter.rs
View file

@ -15,8 +15,9 @@ use deno_lint::linter::LintConfig as DenoLintConfig;
use deno_lint::linter::LintFileOptions;
use deno_lint::linter::Linter as DenoLintLinter;
use deno_lint::linter::LinterOptions;
use deno_path_util::fs::atomic_write_file_with_retries;
use crate::util::fs::atomic_write_file_with_retries;
use crate::sys::CliSys;
use crate::util::fs::specifier_from_file_path;
use super::rules::FileOrPackageLintRule;
@ -176,8 +177,9 @@ impl CliLinter {
if fix_iterations > 0 {
// everything looks good and the file still parses, so write it out
atomic_write_file_with_retries(
&CliSys::default(),
file_path,
source.text().as_ref(),
source.text().as_bytes(),
crate::cache::CACHE_PERM,
)
.context("Failed writing fix to file.")?;

489
cli/tools/lint/mod.rs
View file

@ -20,7 +20,7 @@ use deno_core::unsync::future::LocalFutureExt;
use deno_core::unsync::future::SharedLocal;
use deno_graph::ModuleGraph;
use deno_lint::diagnostic::LintDiagnostic;
use deno_lint::linter::LintConfig;
use deno_lint::linter::LintConfig as DenoLintConfig;
use log::debug;
use reporters::create_reporter;
use reporters::LintReporter;
@ -29,7 +29,6 @@ use std::collections::HashSet;
use std::fs;
use std::io::stdin;
use std::io::Read;
use std::path::Path;
use std::path::PathBuf;
use std::rc::Rc;
use std::sync::Arc;
@ -44,17 +43,22 @@ use crate::cache::IncrementalCache;
use crate::colors;
use crate::factory::CliFactory;
use crate::graph_util::ModuleGraphCreator;
use crate::sys::CliSys;
use crate::tools::fmt::run_parallelized;
use crate::util::display;
use crate::util::file_watcher;
use crate::util::file_watcher::WatcherCommunicator;
use crate::util::fs::canonicalize_path;
use crate::util::path::is_script_ext;
use crate::util::sync::AtomicFlag;
mod ast_buffer;
mod linter;
mod reporters;
mod rules;
// TODO(bartlomieju): remove once we wire plugins through the CLI linter
pub use ast_buffer::serialize_ast_to_buffer;
pub use linter::CliLinter;
pub use linter::CliLinterOptions;
pub use rules::collect_no_slow_type_diagnostics;
@ -69,136 +73,139 @@ pub async fn lint(
flags: Arc<Flags>,
lint_flags: LintFlags,
) -> Result<(), AnyError> {
if let Some(watch_flags) = &lint_flags.watch {
if lint_flags.watch.is_some() {
if lint_flags.is_stdin() {
return Err(generic_error(
"Lint watch on standard input is not supported.",
));
}
file_watcher::watch_func(
flags,
file_watcher::PrintConfig::new("Lint", !watch_flags.no_clear_screen),
move |flags, watcher_communicator, changed_paths| {
let lint_flags = lint_flags.clone();
watcher_communicator.show_path_changed(changed_paths.clone());
Ok(async move {
let factory = CliFactory::from_flags(flags);
let cli_options = factory.cli_options()?;
let lint_config = cli_options.resolve_deno_lint_config()?;
let mut paths_with_options_batches =
resolve_paths_with_options_batches(cli_options, &lint_flags)?;
for paths_with_options in &mut paths_with_options_batches {
_ = watcher_communicator
.watch_paths(paths_with_options.paths.clone());
let files = std::mem::take(&mut paths_with_options.paths);
paths_with_options.paths = if let Some(paths) = &changed_paths {
// lint all files on any changed (https://github.com/denoland/deno/issues/12446)
files
.iter()
.any(|path| {
canonicalize_path(path)
.map(|p| paths.contains(&p))
.unwrap_or(false)
})
.then_some(files)
.unwrap_or_else(|| [].to_vec())
} else {
files
};
}
return lint_with_watch(flags, lint_flags).await;
}
let mut linter = WorkspaceLinter::new(
factory.caches()?.clone(),
factory.lint_rule_provider().await?,
factory.module_graph_creator().await?.clone(),
cli_options.start_dir.clone(),
&cli_options.resolve_workspace_lint_options(&lint_flags)?,
);
for paths_with_options in paths_with_options_batches {
linter
.lint_files(
cli_options,
paths_with_options.options,
lint_config.clone(),
paths_with_options.dir,
paths_with_options.paths,
)
.await?;
}
linter.finish();
Ok(())
})
},
)
.await?;
let factory = CliFactory::from_flags(flags);
let cli_options = factory.cli_options()?;
let lint_rule_provider = factory.lint_rule_provider().await?;
let is_stdin = lint_flags.is_stdin();
let deno_lint_config = cli_options.resolve_deno_lint_config()?;
let workspace_lint_options =
cli_options.resolve_workspace_lint_options(&lint_flags)?;
let success = if is_stdin {
lint_stdin(
cli_options,
lint_rule_provider,
workspace_lint_options,
lint_flags,
deno_lint_config,
)?
} else {
let factory = CliFactory::from_flags(flags);
let cli_options = factory.cli_options()?;
let is_stdin = lint_flags.is_stdin();
let deno_lint_config = cli_options.resolve_deno_lint_config()?;
let workspace_lint_options =
cli_options.resolve_workspace_lint_options(&lint_flags)?;
let success = if is_stdin {
let start_dir = &cli_options.start_dir;
let reporter_lock = Arc::new(Mutex::new(create_reporter(
workspace_lint_options.reporter_kind,
)));
let lint_config = start_dir
.to_lint_config(FilePatterns::new_with_base(start_dir.dir_path()))?;
let lint_options = LintOptions::resolve(lint_config, &lint_flags);
let lint_rules = factory
.lint_rule_provider()
.await?
.resolve_lint_rules_err_empty(
lint_options.rules,
start_dir.maybe_deno_json().map(|c| c.as_ref()),
)?;
let mut file_path = cli_options.initial_cwd().join(STDIN_FILE_NAME);
if let Some(ext) = cli_options.ext_flag() {
file_path.set_extension(ext);
}
let r = lint_stdin(&file_path, lint_rules, deno_lint_config);
let success = handle_lint_result(
&file_path.to_string_lossy(),
r,
reporter_lock.clone(),
);
reporter_lock.lock().close(1);
success
} else {
let mut linter = WorkspaceLinter::new(
factory.caches()?.clone(),
factory.lint_rule_provider().await?,
factory.module_graph_creator().await?.clone(),
cli_options.start_dir.clone(),
&workspace_lint_options,
);
let paths_with_options_batches =
resolve_paths_with_options_batches(cli_options, &lint_flags)?;
for paths_with_options in paths_with_options_batches {
linter
.lint_files(
cli_options,
paths_with_options.options,
deno_lint_config.clone(),
paths_with_options.dir,
paths_with_options.paths,
)
.await?;
}
linter.finish()
};
if !success {
deno_runtime::exit(1);
let mut linter = WorkspaceLinter::new(
factory.caches()?.clone(),
lint_rule_provider,
factory.module_graph_creator().await?.clone(),
cli_options.start_dir.clone(),
&workspace_lint_options,
);
let paths_with_options_batches =
resolve_paths_with_options_batches(cli_options, &lint_flags)?;
for paths_with_options in paths_with_options_batches {
linter
.lint_files(
cli_options,
paths_with_options.options,
deno_lint_config.clone(),
paths_with_options.dir,
paths_with_options.paths,
)
.await?;
}
linter.finish()
};
if !success {
deno_runtime::exit(1);
}
Ok(())
}
async fn lint_with_watch_inner(
flags: Arc<Flags>,
lint_flags: LintFlags,
watcher_communicator: Arc<WatcherCommunicator>,
changed_paths: Option<Vec<PathBuf>>,
) -> Result<(), AnyError> {
let factory = CliFactory::from_flags(flags);
let cli_options = factory.cli_options()?;
let lint_config = cli_options.resolve_deno_lint_config()?;
let mut paths_with_options_batches =
resolve_paths_with_options_batches(cli_options, &lint_flags)?;
for paths_with_options in &mut paths_with_options_batches {
_ = watcher_communicator.watch_paths(paths_with_options.paths.clone());
let files = std::mem::take(&mut paths_with_options.paths);
paths_with_options.paths = if let Some(paths) = &changed_paths {
// lint all files on any changed (https://github.com/denoland/deno/issues/12446)
files
.iter()
.any(|path| {
canonicalize_path(path)
.map(|p| paths.contains(&p))
.unwrap_or(false)
})
.then_some(files)
.unwrap_or_else(|| [].to_vec())
} else {
files
};
}
let mut linter = WorkspaceLinter::new(
factory.caches()?.clone(),
factory.lint_rule_provider().await?,
factory.module_graph_creator().await?.clone(),
cli_options.start_dir.clone(),
&cli_options.resolve_workspace_lint_options(&lint_flags)?,
);
for paths_with_options in paths_with_options_batches {
linter
.lint_files(
cli_options,
paths_with_options.options,
lint_config.clone(),
paths_with_options.dir,
paths_with_options.paths,
)
.await?;
}
linter.finish();
Ok(())
}
async fn lint_with_watch(
flags: Arc<Flags>,
lint_flags: LintFlags,
) -> Result<(), AnyError> {
let watch_flags = lint_flags.watch.as_ref().unwrap();
file_watcher::watch_func(
flags,
file_watcher::PrintConfig::new("Lint", !watch_flags.no_clear_screen),
move |flags, watcher_communicator, changed_paths| {
let lint_flags = lint_flags.clone();
watcher_communicator.show_path_changed(changed_paths.clone());
Ok(lint_with_watch_inner(
flags,
lint_flags,
watcher_communicator,
changed_paths,
))
},
)
.await
}
struct PathsWithOptions {
dir: WorkspaceDirectory,
paths: Vec<PathBuf>,
@ -269,7 +276,7 @@ impl WorkspaceLinter {
&mut self,
cli_options: &Arc<CliOptions>,
lint_options: LintOptions,
lint_config: LintConfig,
lint_config: DenoLintConfig,
member_dir: WorkspaceDirectory,
paths: Vec<PathBuf>,
) -> Result<(), AnyError> {
@ -294,112 +301,63 @@ impl WorkspaceLinter {
deno_lint_config: lint_config,
}));
let has_error = self.has_error.clone();
let reporter_lock = self.reporter_lock.clone();
let mut futures = Vec::with_capacity(2);
if linter.has_package_rules() {
if self.workspace_module_graph.is_none() {
let module_graph_creator = self.module_graph_creator.clone();
let packages = self.workspace_dir.jsr_packages_for_publish();
self.workspace_module_graph = Some(
async move {
module_graph_creator
.create_and_validate_publish_graph(&packages, true)
.await
.map(Rc::new)
.map_err(Rc::new)
}
.boxed_local()
.shared_local(),
);
}
let workspace_module_graph_future =
self.workspace_module_graph.as_ref().unwrap().clone();
let publish_config = member_dir.maybe_package_config();
if let Some(publish_config) = publish_config {
let has_error = self.has_error.clone();
let reporter_lock = self.reporter_lock.clone();
let linter = linter.clone();
let path_urls = paths
.iter()
.filter_map(|p| ModuleSpecifier::from_file_path(p).ok())
.collect::<HashSet<_>>();
futures.push(
async move {
let graph = workspace_module_graph_future
.await
.map_err(|err| anyhow!("{:#}", err))?;
let export_urls =
publish_config.config_file.resolve_export_value_urls()?;
if !export_urls.iter().any(|url| path_urls.contains(url)) {
return Ok(()); // entrypoint is not specified, so skip
}
let diagnostics = linter.lint_package(&graph, &export_urls);
if !diagnostics.is_empty() {
has_error.raise();
let mut reporter = reporter_lock.lock();
for diagnostic in &diagnostics {
reporter.visit_diagnostic(diagnostic);
}
}
Ok(())
}
.boxed_local(),
);
if let Some(fut) = self.run_package_rules(&linter, &member_dir, &paths) {
futures.push(fut);
}
}
futures.push({
let has_error = self.has_error.clone();
let reporter_lock = self.reporter_lock.clone();
let maybe_incremental_cache = maybe_incremental_cache.clone();
let linter = linter.clone();
let cli_options = cli_options.clone();
async move {
run_parallelized(paths, {
move |file_path| {
let file_text =
deno_ast::strip_bom(fs::read_to_string(&file_path)?);
let maybe_incremental_cache_ = maybe_incremental_cache.clone();
let linter = linter.clone();
let cli_options = cli_options.clone();
let fut = async move {
let operation = move |file_path: PathBuf| {
let file_text = deno_ast::strip_bom(fs::read_to_string(&file_path)?);
// don't bother rechecking this file if it didn't have any diagnostics before
if let Some(incremental_cache) = &maybe_incremental_cache {
if incremental_cache.is_file_same(&file_path, &file_text) {
return Ok(());
}
}
let r = linter.lint_file(
&file_path,
file_text,
cli_options.ext_flag().as_deref(),
);
if let Ok((file_source, file_diagnostics)) = &r {
if let Some(incremental_cache) = &maybe_incremental_cache {
if file_diagnostics.is_empty() {
// update the incremental cache if there were no diagnostics
incremental_cache.update_file(
&file_path,
// ensure the returned text is used here as it may have been modified via --fix
file_source.text(),
)
}
}
}
let success = handle_lint_result(
&file_path.to_string_lossy(),
r,
reporter_lock.clone(),
);
if !success {
has_error.raise();
}
Ok(())
// don't bother rechecking this file if it didn't have any diagnostics before
if let Some(incremental_cache) = &maybe_incremental_cache_ {
if incremental_cache.is_file_same(&file_path, &file_text) {
return Ok(());
}
})
.await
}
.boxed_local()
});
}
let r = linter.lint_file(
&file_path,
file_text,
cli_options.ext_flag().as_deref(),
);
if let Ok((file_source, file_diagnostics)) = &r {
if let Some(incremental_cache) = &maybe_incremental_cache_ {
if file_diagnostics.is_empty() {
// update the incremental cache if there were no diagnostics
incremental_cache.update_file(
&file_path,
// ensure the returned text is used here as it may have been modified via --fix
file_source.text(),
)
}
}
}
let success = handle_lint_result(
&file_path.to_string_lossy(),
r,
reporter_lock.clone(),
);
if !success {
has_error.raise();
}
Ok(())
};
run_parallelized(paths, operation).await
}
.boxed_local();
futures.push(fut);
if lint_options.fix {
// run sequentially when using `--fix` to lower the chances of weird
@ -419,6 +377,63 @@ impl WorkspaceLinter {
Ok(())
}
fn run_package_rules(
&mut self,
linter: &Arc<CliLinter>,
member_dir: &WorkspaceDirectory,
paths: &[PathBuf],
) -> Option<LocalBoxFuture<Result<(), AnyError>>> {
if self.workspace_module_graph.is_none() {
let module_graph_creator = self.module_graph_creator.clone();
let packages = self.workspace_dir.jsr_packages_for_publish();
self.workspace_module_graph = Some(
async move {
module_graph_creator
.create_and_validate_publish_graph(&packages, true)
.await
.map(Rc::new)
.map_err(Rc::new)
}
.boxed_local()
.shared_local(),
);
}
let workspace_module_graph_future =
self.workspace_module_graph.as_ref().unwrap().clone();
let maybe_publish_config = member_dir.maybe_package_config();
let publish_config = maybe_publish_config?;
let has_error = self.has_error.clone();
let reporter_lock = self.reporter_lock.clone();
let linter = linter.clone();
let path_urls = paths
.iter()
.filter_map(|p| ModuleSpecifier::from_file_path(p).ok())
.collect::<HashSet<_>>();
let fut = async move {
let graph = workspace_module_graph_future
.await
.map_err(|err| anyhow!("{:#}", err))?;
let export_urls =
publish_config.config_file.resolve_export_value_urls()?;
if !export_urls.iter().any(|url| path_urls.contains(url)) {
return Ok(()); // entrypoint is not specified, so skip
}
let diagnostics = linter.lint_package(&graph, &export_urls);
if !diagnostics.is_empty() {
has_error.raise();
let mut reporter = reporter_lock.lock();
for diagnostic in &diagnostics {
reporter.visit_diagnostic(diagnostic);
}
}
Ok(())
}
.boxed_local();
Some(fut)
}
pub fn finish(self) -> bool {
debug!("Found {} files", self.file_count);
self.reporter_lock.lock().close(self.file_count);
@ -438,7 +453,7 @@ fn collect_lint_files(
.ignore_node_modules()
.use_gitignore()
.set_vendor_folder(cli_options.vendor_dir_path().map(ToOwned::to_owned))
.collect_file_patterns(&deno_config::fs::RealDenoConfigFs, files)
.collect_file_patterns(&CliSys::default(), files)
}
#[allow(clippy::print_stdout)]
@ -494,10 +509,27 @@ pub fn print_rules_list(json: bool, maybe_rules_tags: Option<Vec<String>>) {
/// Treats input as TypeScript.
/// Compatible with `--json` flag.
fn lint_stdin(
file_path: &Path,
configured_rules: ConfiguredRules,
deno_lint_config: LintConfig,
) -> Result<(ParsedSource, Vec<LintDiagnostic>), AnyError> {
cli_options: &Arc<CliOptions>,
lint_rule_provider: LintRuleProvider,
workspace_lint_options: WorkspaceLintOptions,
lint_flags: LintFlags,
deno_lint_config: DenoLintConfig,
) -> Result<bool, AnyError> {
let start_dir = &cli_options.start_dir;
let reporter_lock = Arc::new(Mutex::new(create_reporter(
workspace_lint_options.reporter_kind,
)));
let lint_config = start_dir
.to_lint_config(FilePatterns::new_with_base(start_dir.dir_path()))?;
let lint_options = LintOptions::resolve(lint_config, &lint_flags);
let configured_rules = lint_rule_provider.resolve_lint_rules_err_empty(
lint_options.rules,
start_dir.maybe_deno_json().map(|c| c.as_ref()),
)?;
let mut file_path = cli_options.initial_cwd().join(STDIN_FILE_NAME);
if let Some(ext) = cli_options.ext_flag() {
file_path.set_extension(ext);
}
let mut source_code = String::new();
if stdin().read_to_string(&mut source_code).is_err() {
return Err(generic_error("Failed to read from stdin"));
@ -509,9 +541,14 @@ fn lint_stdin(
deno_lint_config,
});
linter
.lint_file(file_path, deno_ast::strip_bom(source_code), None)
.map_err(AnyError::from)
let r = linter
.lint_file(&file_path, deno_ast::strip_bom(source_code), None)
.map_err(AnyError::from);
let success =
handle_lint_result(&file_path.to_string_lossy(), r, reporter_lock.clone());
reporter_lock.lock().close(1);
Ok(success)
}
fn handle_lint_result(

5
cli/tools/registry/mod.rs
View file

@ -26,6 +26,7 @@ use deno_core::serde_json;
use deno_core::serde_json::json;
use deno_core::serde_json::Value;
use deno_core::url::Url;
use deno_runtime::deno_fetch;
use deno_terminal::colors;
use http_body_util::BodyExt;
use serde::Deserialize;
@ -911,9 +912,7 @@ async fn publish_package(
package.config
);
let body = http_body_util::Full::new(package.tarball.bytes.clone())
.map_err(|never| match never {})
.boxed();
let body = deno_fetch::ReqBody::full(package.tarball.bytes.clone());
let response = http_client
.post(url.parse()?, body)?
.header(

7
cli/tools/registry/paths.rs
View file

@ -6,6 +6,7 @@ use std::collections::HashSet;
use std::path::Path;
use std::path::PathBuf;
use crate::sys::CliSys;
use deno_ast::MediaType;
use deno_ast::ModuleSpecifier;
use deno_config::glob::FileCollector;
@ -323,11 +324,11 @@ fn collect_paths(
file_patterns: FilePatterns,
) -> Result<Vec<PathBuf>, AnyError> {
FileCollector::new(|e| {
if !e.metadata.is_file {
if !e.metadata.file_type().is_file() {
if let Ok(specifier) = ModuleSpecifier::from_file_path(e.path) {
diagnostics_collector.push(PublishDiagnostic::UnsupportedFileType {
specifier,
kind: if e.metadata.is_symlink {
kind: if e.metadata.file_type().is_symlink() {
"symlink".to_string()
} else {
"Unknown".to_string()
@ -345,5 +346,5 @@ fn collect_paths(
.ignore_node_modules()
.set_vendor_folder(cli_options.vendor_dir_path().map(ToOwned::to_owned))
.use_gitignore()
.collect_file_patterns(&deno_config::fs::RealDenoConfigFs, file_patterns)
.collect_file_patterns(&CliSys::default(), file_patterns)
}

46
cli/tools/registry/pm.rs
View file

@ -4,6 +4,7 @@ use std::path::Path;
use std::path::PathBuf;
use std::sync::Arc;
use deno_cache_dir::file_fetcher::CacheSetting;
use deno_core::anyhow::bail;
use deno_core::anyhow::Context;
use deno_core::error::AnyError;
@ -14,6 +15,7 @@ use deno_semver::jsr::JsrPackageReqReference;
use deno_semver::npm::NpmPackageReqReference;
use deno_semver::package::PackageNv;
use deno_semver::package::PackageReq;
use deno_semver::StackString;
use deno_semver::Version;
use deno_semver::VersionReq;
use deps::KeyPath;
@ -23,12 +25,11 @@ use jsonc_parser::cst::CstRootNode;
use jsonc_parser::json;
use crate::args::AddFlags;
use crate::args::CacheSetting;
use crate::args::CliOptions;
use crate::args::Flags;
use crate::args::RemoveFlags;
use crate::factory::CliFactory;
use crate::file_fetcher::FileFetcher;
use crate::file_fetcher::CliFileFetcher;
use crate::jsr::JsrFetchResolver;
use crate::npm::NpmFetchResolver;
@ -283,7 +284,7 @@ fn package_json_dependency_entry(
(npm_package.into(), selected.version_req)
} else {
(
selected.import_name,
selected.import_name.into_string(),
format!("npm:{}@{}", npm_package, selected.version_req),
)
}
@ -292,7 +293,7 @@ fn package_json_dependency_entry(
let scope_replaced = jsr_package.replace('/', "__");
let version_req =
format!("npm:@jsr/{scope_replaced}@{}", selected.version_req);
(selected.import_name, version_req)
(selected.import_name.into_string(), version_req)
} else {
(selected.package_name, selected.version_req)
}
@ -411,18 +412,19 @@ pub async fn add(
let http_client = cli_factory.http_client_provider();
let deps_http_cache = cli_factory.global_http_cache()?;
let mut deps_file_fetcher = FileFetcher::new(
let deps_file_fetcher = CliFileFetcher::new(
deps_http_cache.clone(),
CacheSetting::ReloadAll,
true,
http_client.clone(),
cli_factory.sys(),
Default::default(),
None,
true,
CacheSetting::ReloadAll,
log::Level::Trace,
);
let npmrc = cli_factory.cli_options().unwrap().npmrc();
deps_file_fetcher.set_download_log_level(log::Level::Trace);
let deps_file_fetcher = Arc::new(deps_file_fetcher);
let jsr_resolver = Arc::new(JsrFetchResolver::new(deps_file_fetcher.clone()));
let npm_resolver =
@ -549,10 +551,10 @@ pub async fn add(
}
struct SelectedPackage {
import_name: String,
import_name: StackString,
package_name: String,
version_req: String,
selected_version: String,
selected_version: StackString,
}
enum NotFoundHelp {
@ -683,7 +685,7 @@ async fn find_package_and_select_version_for_req(
import_name: add_package_req.alias,
package_name: prefixed_name,
version_req: format!("{}{}", range_symbol, &nv.version),
selected_version: nv.version.to_string(),
selected_version: nv.version.to_custom_string::<StackString>(),
}))
}
@ -705,7 +707,7 @@ enum AddRmPackageReqValue {
#[derive(Debug, PartialEq, Eq)]
pub struct AddRmPackageReq {
alias: String,
alias: StackString,
value: AddRmPackageReqValue,
}
@ -753,7 +755,11 @@ impl AddRmPackageReq {
return Ok(Err(PackageReq::from_str(entry_text)?));
}
(maybe_prefix.unwrap(), Some(alias.to_string()), entry_text)
(
maybe_prefix.unwrap(),
Some(StackString::from(alias)),
entry_text,
)
}
None => return Ok(Err(PackageReq::from_str(entry_text)?)),
},
@ -765,7 +771,7 @@ impl AddRmPackageReq {
JsrPackageReqReference::from_str(&format!("jsr:{}", entry_text))?;
let package_req = req_ref.into_inner().req;
Ok(Ok(AddRmPackageReq {
alias: maybe_alias.unwrap_or_else(|| package_req.name.to_string()),
alias: maybe_alias.unwrap_or_else(|| package_req.name.clone()),
value: AddRmPackageReqValue::Jsr(package_req),
}))
}
@ -785,7 +791,7 @@ impl AddRmPackageReq {
);
}
Ok(Ok(AddRmPackageReq {
alias: maybe_alias.unwrap_or_else(|| package_req.name.to_string()),
alias: maybe_alias.unwrap_or_else(|| package_req.name.clone()),
value: AddRmPackageReqValue::Npm(package_req),
}))
}
@ -878,14 +884,14 @@ mod test {
assert_eq!(
AddRmPackageReq::parse("jsr:foo").unwrap().unwrap(),
AddRmPackageReq {
alias: "foo".to_string(),
alias: "foo".into(),
value: AddRmPackageReqValue::Jsr(PackageReq::from_str("foo").unwrap())
}
);
assert_eq!(
AddRmPackageReq::parse("alias@jsr:foo").unwrap().unwrap(),
AddRmPackageReq {
alias: "alias".to_string(),
alias: "alias".into(),
value: AddRmPackageReqValue::Jsr(PackageReq::from_str("foo").unwrap())
}
);
@ -894,7 +900,7 @@ mod test {
.unwrap()
.unwrap(),
AddRmPackageReq {
alias: "@alias/pkg".to_string(),
alias: "@alias/pkg".into(),
value: AddRmPackageReqValue::Npm(
PackageReq::from_str("foo@latest").unwrap()
)
@ -905,7 +911,7 @@ mod test {
.unwrap()
.unwrap(),
AddRmPackageReq {
alias: "@alias/pkg".to_string(),
alias: "@alias/pkg".into(),
value: AddRmPackageReqValue::Jsr(PackageReq::from_str("foo").unwrap())
}
);
@ -914,7 +920,7 @@ mod test {
.unwrap()
.unwrap(),
AddRmPackageReq {
alias: "alias".to_string(),
alias: "alias".into(),
value: AddRmPackageReqValue::Jsr(
PackageReq::from_str("foo@^1.5.0").unwrap()
)

112
cli/tools/registry/pm/deps.rs
View file

@ -3,7 +3,6 @@
use std::borrow::Cow;
use std::collections::HashMap;
use std::path::PathBuf;
use std::sync::atomic::AtomicBool;
use std::sync::Arc;
use deno_ast::ModuleSpecifier;
@ -28,6 +27,8 @@ use deno_semver::npm::NpmPackageReqReference;
use deno_semver::package::PackageNv;
use deno_semver::package::PackageReq;
use deno_semver::package::PackageReqReference;
use deno_semver::StackString;
use deno_semver::Version;
use deno_semver::VersionReq;
use import_map::ImportMap;
use import_map::ImportMapWithDiagnostics;
@ -42,6 +43,7 @@ use crate::jsr::JsrFetchResolver;
use crate::module_loader::ModuleLoadPreparer;
use crate::npm::CliNpmResolver;
use crate::npm::NpmFetchResolver;
use crate::util::sync::AtomicFlag;
use super::ConfigUpdater;
@ -138,13 +140,7 @@ pub enum KeyPart {
Scopes,
Dependencies,
DevDependencies,
String(String),
}
impl From<String> for KeyPart {
fn from(value: String) -> Self {
KeyPart::String(value)
}
String(StackString),
}
impl From<PackageJsonDepKind> for KeyPart {
@ -163,7 +159,7 @@ impl KeyPart {
KeyPart::Scopes => "scopes",
KeyPart::Dependencies => "dependencies",
KeyPart::DevDependencies => "devDependencies",
KeyPart::String(s) => s,
KeyPart::String(s) => s.as_str(),
}
}
}
@ -216,12 +212,12 @@ fn import_map_entries(
.chain(import_map.scopes().flat_map(|scope| {
let path = KeyPath::from_parts([
KeyPart::Scopes,
scope.raw_key.to_string().into(),
KeyPart::String(scope.raw_key.into()),
]);
scope.imports.entries().map(move |entry| {
let mut full_path = path.clone();
full_path.push(KeyPart::String(entry.raw_key.to_string()));
full_path.push(KeyPart::String(entry.raw_key.into()));
(full_path, entry)
})
}))
@ -337,7 +333,7 @@ fn add_deps_from_package_json(
package_json: &PackageJsonRc,
mut filter: impl DepFilter,
package_dep_kind: PackageJsonDepKind,
package_json_deps: PackageJsonDepsMap,
package_json_deps: &PackageJsonDepsMap,
deps: &mut Vec<Dep>,
) {
for (k, v) in package_json_deps {
@ -352,7 +348,7 @@ fn add_deps_from_package_json(
deno_package_json::PackageJsonDepValue::Req(req) => {
let alias = k.as_str();
let alias = (alias != req.name).then(|| alias.to_string());
if !filter.should_include(alias.as_deref(), &req, DepKind::Npm) {
if !filter.should_include(alias.as_deref(), req, DepKind::Npm) {
continue;
}
let id = DepId(deps.len());
@ -361,9 +357,12 @@ fn add_deps_from_package_json(
kind: DepKind::Npm,
location: DepLocation::PackageJson(
package_json.clone(),
KeyPath::from_parts([package_dep_kind.into(), k.into()]),
KeyPath::from_parts([
package_dep_kind.into(),
KeyPart::String(k.clone()),
]),
),
req,
req: req.clone(),
alias,
})
}
@ -376,14 +375,14 @@ fn add_deps_from_package_json(
package_json,
filter,
PackageJsonDepKind::Normal,
package_json_deps.dependencies,
&package_json_deps.dependencies,
deps,
);
iterate(
package_json,
filter,
PackageJsonDepKind::Dev,
package_json_deps.dev_dependencies,
&package_json_deps.dev_dependencies,
deps,
);
}
@ -447,7 +446,7 @@ pub struct DepManager {
pending_changes: Vec<Change>,
dependencies_resolved: AtomicBool,
dependencies_resolved: AtomicFlag,
module_load_preparer: Arc<ModuleLoadPreparer>,
// TODO(nathanwhit): probably shouldn't be pub
pub(crate) jsr_fetch_resolver: Arc<JsrFetchResolver>,
@ -489,7 +488,7 @@ impl DepManager {
resolved_versions: Vec::new(),
latest_versions: Vec::new(),
jsr_fetch_resolver,
dependencies_resolved: AtomicBool::new(false),
dependencies_resolved: AtomicFlag::lowered(),
module_load_preparer,
npm_fetch_resolver,
npm_resolver,
@ -530,10 +529,7 @@ impl DepManager {
}
async fn run_dependency_resolution(&self) -> Result<(), AnyError> {
if self
.dependencies_resolved
.load(std::sync::atomic::Ordering::Relaxed)
{
if self.dependencies_resolved.is_raised() {
return Ok(());
}
@ -556,9 +552,7 @@ impl DepManager {
}
DepKind::Jsr => graph.packages.mappings().contains_key(&dep.req),
}) {
self
.dependencies_resolved
.store(true, std::sync::atomic::Ordering::Relaxed);
self.dependencies_resolved.raise();
graph_permit.commit();
return Ok(());
}
@ -613,6 +607,7 @@ impl DepManager {
)
.await?;
self.dependencies_resolved.raise();
graph_permit.commit();
Ok(())
@ -655,10 +650,6 @@ impl DepManager {
if self.latest_versions.len() == self.deps.len() {
return Ok(self.latest_versions.clone());
}
let latest_tag_req = deno_semver::VersionReq::from_raw_text_and_inner(
"latest".into(),
deno_semver::RangeSetOrTag::Tag("latest".into()),
);
let mut latest_versions = Vec::with_capacity(self.deps.len());
let npm_sema = Semaphore::new(32);
@ -670,14 +661,25 @@ impl DepManager {
DepKind::Npm => futs.push_back(
async {
let semver_req = &dep.req;
let latest_req = PackageReq {
name: dep.req.name.clone(),
version_req: latest_tag_req.clone(),
};
let _permit = npm_sema.acquire().await;
let semver_compatible =
self.npm_fetch_resolver.req_to_nv(semver_req).await;
let latest = self.npm_fetch_resolver.req_to_nv(&latest_req).await;
let info =
self.npm_fetch_resolver.package_info(&semver_req.name).await;
let latest = info
.and_then(|info| {
let latest_tag = info.dist_tags.get("latest")?;
let lower_bound = &semver_compatible.as_ref()?.version;
if latest_tag > lower_bound {
Some(latest_tag.clone())
} else {
latest_version(Some(latest_tag), info.versions.keys())
}
})
.map(|version| PackageNv {
name: semver_req.name.clone(),
version,
});
PackageLatestVersion {
latest,
semver_compatible,
@ -688,14 +690,29 @@ impl DepManager {
DepKind::Jsr => futs.push_back(
async {
let semver_req = &dep.req;
let latest_req = PackageReq {
name: dep.req.name.clone(),
version_req: deno_semver::WILDCARD_VERSION_REQ.clone(),
};
let _permit = jsr_sema.acquire().await;
let semver_compatible =
self.jsr_fetch_resolver.req_to_nv(semver_req).await;
let latest = self.jsr_fetch_resolver.req_to_nv(&latest_req).await;
let info =
self.jsr_fetch_resolver.package_info(&semver_req.name).await;
let latest = info
.and_then(|info| {
let lower_bound = &semver_compatible.as_ref()?.version;
latest_version(
Some(lower_bound),
info.versions.iter().filter_map(|(version, version_info)| {
if !version_info.yanked {
Some(version)
} else {
None
}
}),
)
})
.map(|version| PackageNv {
name: semver_req.name.clone(),
version,
});
PackageLatestVersion {
latest,
semver_compatible,
@ -893,3 +910,18 @@ fn parse_req_reference(
DepKind::Jsr => JsrPackageReqReference::from_str(input)?.into_inner(),
})
}
fn latest_version<'a>(
start: Option<&Version>,
versions: impl IntoIterator<Item = &'a Version>,
) -> Option<Version> {
let mut best = start;
for version in versions {
match best {
Some(best_version) if version > best_version => best = Some(version),
None => best = Some(version),
_ => {}
}
}
best.cloned()
}

16
cli/tools/registry/pm/outdated.rs
View file

@ -3,19 +3,20 @@
use std::collections::HashSet;
use std::sync::Arc;
use deno_cache_dir::file_fetcher::CacheSetting;
use deno_core::anyhow::bail;
use deno_core::error::AnyError;
use deno_semver::package::PackageNv;
use deno_semver::package::PackageReq;
use deno_semver::StackString;
use deno_semver::VersionReq;
use deno_terminal::colors;
use crate::args::CacheSetting;
use crate::args::CliOptions;
use crate::args::Flags;
use crate::args::OutdatedFlags;
use crate::factory::CliFactory;
use crate::file_fetcher::FileFetcher;
use crate::file_fetcher::CliFileFetcher;
use crate::jsr::JsrFetchResolver;
use crate::npm::NpmFetchResolver;
use crate::tools::registry::pm::deps::DepKind;
@ -31,7 +32,7 @@ struct OutdatedPackage {
latest: String,
semver_compatible: String,
current: String,
name: String,
name: StackString,
}
#[allow(clippy::print_stdout)]
@ -181,15 +182,16 @@ pub async fn outdated(
let workspace = cli_options.workspace();
let http_client = factory.http_client_provider();
let deps_http_cache = factory.global_http_cache()?;
let mut file_fetcher = FileFetcher::new(
let file_fetcher = CliFileFetcher::new(
deps_http_cache.clone(),
CacheSetting::RespectHeaders,
true,
http_client.clone(),
factory.sys(),
Default::default(),
None,
true,
CacheSetting::RespectHeaders,
log::Level::Trace,
);
file_fetcher.set_download_log_level(log::Level::Trace);
let file_fetcher = Arc::new(file_fetcher);
let npm_fetch_resolver = Arc::new(NpmFetchResolver::new(
file_fetcher.clone(),

9
cli/tools/registry/unfurl.rs
View file

@ -658,12 +658,12 @@ mod tests {
use crate::resolver::SloppyImportsCachedFs;
use super::*;
use crate::sys::CliSys;
use deno_ast::MediaType;
use deno_ast::ModuleSpecifier;
use deno_config::workspace::ResolverWorkspaceJsrPackage;
use deno_core::serde_json::json;
use deno_core::url::Url;
use deno_runtime::deno_fs::RealFs;
use deno_runtime::deno_node::PackageJson;
use deno_semver::Version;
use import_map::ImportMapWithDiagnostics;
@ -722,10 +722,9 @@ mod tests {
vec![Arc::new(package_json)],
deno_config::workspace::PackageJsonDepResolution::Enabled,
);
let fs = Arc::new(RealFs);
let unfurler = SpecifierUnfurler::new(
Some(Arc::new(CliSloppyImportsResolver::new(
SloppyImportsCachedFs::new(fs),
SloppyImportsCachedFs::new(CliSys::default()),
))),
Arc::new(workspace_resolver),
true,
@ -863,10 +862,10 @@ const warn2 = await import(`${expr}`);
],
deno_config::workspace::PackageJsonDepResolution::Enabled,
);
let fs = Arc::new(RealFs);
let sys = CliSys::default();
let unfurler = SpecifierUnfurler::new(
Some(Arc::new(CliSloppyImportsResolver::new(
SloppyImportsCachedFs::new(fs),
SloppyImportsCachedFs::new(sys),
))),
Arc::new(workspace_resolver),
true,

7
cli/tools/repl/mod.rs
View file

@ -11,7 +11,8 @@ use crate::args::ReplFlags;
use crate::cdp;
use crate::colors;
use crate::factory::CliFactory;
use crate::file_fetcher::FileFetcher;
use crate::file_fetcher::CliFileFetcher;
use crate::file_fetcher::TextDecodedFile;
use deno_core::error::AnyError;
use deno_core::futures::StreamExt;
use deno_core::serde_json;
@ -143,7 +144,7 @@ async fn read_line_and_poll(
async fn read_eval_file(
cli_options: &CliOptions,
file_fetcher: &FileFetcher,
file_fetcher: &CliFileFetcher,
eval_file: &str,
) -> Result<Arc<str>, AnyError> {
let specifier =
@ -151,7 +152,7 @@ async fn read_eval_file(
let file = file_fetcher.fetch_bypass_permissions(&specifier).await?;
Ok(file.into_text_decoded()?.source)
Ok(TextDecodedFile::decode(file)?.source)
}
#[allow(clippy::print_stdout)]

6
cli/tools/run/mod.rs
View file

@ -3,6 +3,7 @@
use std::io::Read;
use std::sync::Arc;
use deno_cache_dir::file_fetcher::File;
use deno_config::deno_json::NodeModulesDirMode;
use deno_core::error::AnyError;
use deno_runtime::WorkerExecutionMode;
@ -11,7 +12,6 @@ use crate::args::EvalFlags;
use crate::args::Flags;
use crate::args::WatchFlagsWithPaths;
use crate::factory::CliFactory;
use crate::file_fetcher::File;
use crate::util;
use crate::util::file_watcher::WatcherRestartMode;
@ -97,7 +97,7 @@ pub async fn run_from_stdin(flags: Arc<Flags>) -> Result<i32, AnyError> {
// Save a fake file into file fetcher cache
// to allow module access by TS compiler
file_fetcher.insert_memory_files(File {
specifier: main_module.clone(),
url: main_module.clone(),
maybe_headers: None,
source: source.into(),
});
@ -184,7 +184,7 @@ pub async fn eval_command(
// Save a fake file into file fetcher cache
// to allow module access by TS compiler.
file_fetcher.insert_memory_files(File {
specifier: main_module.clone(),
url: main_module.clone(),
maybe_headers: None,
source: source_code.into_bytes().into(),
});

82
cli/tools/task.rs
View file

@ -25,7 +25,6 @@ use deno_core::futures::FutureExt;
use deno_core::futures::StreamExt;
use deno_core::url::Url;
use deno_path_util::normalize_path;
use deno_runtime::deno_node::NodeResolver;
use deno_task_shell::KillSignal;
use deno_task_shell::ShellCommand;
use indexmap::IndexMap;
@ -36,6 +35,7 @@ use crate::args::Flags;
use crate::args::TaskFlags;
use crate::colors;
use crate::factory::CliFactory;
use crate::node::CliNodeResolver;
use crate::npm::CliNpmResolver;
use crate::task_runner;
use crate::task_runner::run_future_forwarding_signals;
@ -94,7 +94,7 @@ pub async fn execute_script(
return Ok(0);
};
let task_regex = arg_to_regex(task_name)?;
let task_name_filter = arg_to_task_name_filter(task_name)?;
let mut packages_task_info: Vec<PackageTaskInfo> = vec![];
for folder in workspace.config_folders() {
@ -137,12 +137,20 @@ pub async fn execute_script(
// Match tasks in deno.json
for name in tasks_config.task_names() {
if task_regex.is_match(name) && !visited.contains(name) {
let matches_filter = match &task_name_filter {
TaskNameFilter::Exact(n) => *n == name,
TaskNameFilter::Regex(re) => re.is_match(name),
};
if matches_filter && !visited.contains(name) {
matched.insert(name.to_string());
visit_task(&tasks_config, &mut visited, name);
}
}
if matched.is_empty() {
continue;
}
packages_task_info.push(PackageTaskInfo {
matched_tasks: matched
.iter()
@ -223,7 +231,7 @@ pub async fn execute_script(
&Url::from_directory_path(cli_options.initial_cwd()).unwrap(),
"",
&TaskDefinition {
command: task_flags.task.as_ref().unwrap().to_string(),
command: Some(task_flags.task.as_ref().unwrap().to_string()),
dependencies: vec![],
description: None,
},
@ -259,7 +267,7 @@ struct RunSingleOptions<'a> {
struct TaskRunner<'a> {
task_flags: &'a TaskFlags,
npm_resolver: &'a dyn CliNpmResolver,
node_resolver: &'a NodeResolver,
node_resolver: &'a CliNodeResolver,
env_vars: HashMap<String, String>,
cli_options: &'a CliOptions,
concurrency: usize,
@ -440,6 +448,16 @@ impl<'a> TaskRunner<'a> {
kill_signal: KillSignal,
argv: &'a [String],
) -> Result<i32, deno_core::anyhow::Error> {
let Some(command) = &definition.command else {
log::info!(
"{} {} {}",
colors::green("Task"),
colors::cyan(task_name),
colors::gray("(no command)")
);
return Ok(0);
};
if let Some(npm_resolver) = self.npm_resolver.as_managed() {
npm_resolver.ensure_top_level_package_json_install().await?;
npm_resolver
@ -461,7 +479,7 @@ impl<'a> TaskRunner<'a> {
self
.run_single(RunSingleOptions {
task_name,
script: &definition.command,
script: command,
cwd: &cwd,
custom_commands,
kill_signal,
@ -829,7 +847,7 @@ fn print_available_tasks(
is_deno: false,
name: name.to_string(),
task: deno_config::deno_json::TaskDefinition {
command: script.to_string(),
command: Some(script.to_string()),
dependencies: vec![],
description: None,
},
@ -865,11 +883,13 @@ fn print_available_tasks(
)?;
}
}
writeln!(
writer,
" {}",
strip_ansi_codes_and_escape_control_chars(&desc.task.command)
)?;
if let Some(command) = &desc.task.command {
writeln!(
writer,
" {}",
strip_ansi_codes_and_escape_control_chars(command)
)?;
};
if !desc.task.dependencies.is_empty() {
let dependencies = desc
.task
@ -902,3 +922,41 @@ fn strip_ansi_codes_and_escape_control_chars(s: &str) -> String {
})
.collect()
}
fn arg_to_task_name_filter(input: &str) -> Result<TaskNameFilter, AnyError> {
if !input.contains("*") {
return Ok(TaskNameFilter::Exact(input));
}
let mut regex_str = regex::escape(input);
regex_str = regex_str.replace("\\*", ".*");
let re = Regex::new(&regex_str)?;
Ok(TaskNameFilter::Regex(re))
}
#[derive(Debug)]
enum TaskNameFilter<'s> {
Exact(&'s str),
Regex(regex::Regex),
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_arg_to_task_name_filter() {
assert!(matches!(
arg_to_task_name_filter("test").unwrap(),
TaskNameFilter::Exact("test")
));
assert!(matches!(
arg_to_task_name_filter("test-").unwrap(),
TaskNameFilter::Exact("test-")
));
assert!(matches!(
arg_to_task_name_filter("test*").unwrap(),
TaskNameFilter::Regex(_)
));
}
}

18
cli/tools/test/mod.rs
View file

@ -7,10 +7,10 @@ use crate::args::TestReporterConfig;
use crate::colors;
use crate::display;
use crate::factory::CliFactory;
use crate::file_fetcher::File;
use crate::file_fetcher::FileFetcher;
use crate::file_fetcher::CliFileFetcher;
use crate::graph_util::has_graph_root_local_dependent_changed;
use crate::ops;
use crate::sys::CliSys;
use crate::util::extract::extract_doc_tests;
use crate::util::file_watcher;
use crate::util::fs::collect_specifiers;
@ -21,6 +21,7 @@ use crate::worker::CliMainWorkerFactory;
use crate::worker::CoverageCollector;
use deno_ast::MediaType;
use deno_cache_dir::file_fetcher::File;
use deno_config::glob::FilePatterns;
use deno_config::glob::WalkEntry;
use deno_core::anyhow;
@ -616,7 +617,10 @@ async fn configure_main_worker(
WorkerExecutionMode::Test,
specifier.clone(),
permissions_container,
vec![ops::testing::deno_test::init_ops(worker_sender.sender)],
vec![
ops::testing::deno_test::init_ops(worker_sender.sender),
ops::lint::deno_lint::init_ops(),
],
Stdio {
stdin: StdioPipe::inherit(),
stdout: StdioPipe::file(worker_sender.stdout),
@ -1191,7 +1195,7 @@ static HAS_TEST_RUN_SIGINT_HANDLER: AtomicBool = AtomicBool::new(false);
async fn test_specifiers(
worker_factory: Arc<CliMainWorkerFactory>,
permissions: &Permissions,
permission_desc_parser: &Arc<RuntimePermissionDescriptorParser>,
permission_desc_parser: &Arc<RuntimePermissionDescriptorParser<CliSys>>,
specifiers: Vec<ModuleSpecifier>,
options: TestSpecifiersOptions,
) -> Result<(), AnyError> {
@ -1514,7 +1518,7 @@ fn collect_specifiers_with_test_mode(
/// as well.
async fn fetch_specifiers_with_test_mode(
cli_options: &CliOptions,
file_fetcher: &FileFetcher,
file_fetcher: &CliFileFetcher,
member_patterns: impl Iterator<Item = FilePatterns>,
doc: &bool,
) -> Result<Vec<(ModuleSpecifier, TestMode)>, AnyError> {
@ -1822,7 +1826,7 @@ pub async fn run_tests_with_watch(
/// Extracts doc tests from files specified by the given specifiers.
async fn get_doc_tests(
specifiers_with_mode: &[(Url, TestMode)],
file_fetcher: &FileFetcher,
file_fetcher: &CliFileFetcher,
) -> Result<Vec<File>, AnyError> {
let specifiers_needing_extraction = specifiers_with_mode
.iter()
@ -1847,7 +1851,7 @@ fn get_target_specifiers(
specifiers_with_mode
.into_iter()
.filter_map(|(s, mode)| mode.needs_test_run().then_some(s))
.chain(doc_tests.iter().map(|d| d.specifier.clone()))
.chain(doc_tests.iter().map(|d| d.url.clone()))
.collect()
}

5
cli/tools/upgrade.rs
View file

@ -21,6 +21,7 @@ use deno_core::anyhow::Context;
use deno_core::error::AnyError;
use deno_core::unsync::spawn;
use deno_core::url::Url;
use deno_semver::SmallStackString;
use deno_semver::Version;
use once_cell::sync::Lazy;
use std::borrow::Cow;
@ -255,7 +256,7 @@ async fn print_release_notes(
let is_deno_2_rc = new_semver.major == 2
&& new_semver.minor == 0
&& new_semver.patch == 0
&& new_semver.pre.first() == Some(&"rc".to_string());
&& new_semver.pre.first().map(|s| s.as_str()) == Some("rc");
if is_deno_2_rc || is_switching_from_deno1_to_deno2 {
log::info!(
@ -674,7 +675,7 @@ impl RequestedVersion {
);
};
if semver.pre.contains(&"rc".to_string()) {
if semver.pre.contains(&SmallStackString::from_static("rc")) {
(ReleaseChannel::Rc, passed_version)
} else {
(ReleaseChannel::Stable, passed_version)

1
cli/tsc/00_typescript.js vendored
View file

@ -136063,6 +136063,7 @@ var unprefixedNodeCoreModuleList = [
"https",
"http2",
"inspector",
"inspector/promises",
"module",
"net",
"os",

7
cli/tsc/99_main_compiler.js
View file

@ -41,6 +41,13 @@ delete Object.prototype.__proto__;
"listen",
"listenDatagram",
"openKv",
"connectQuic",
"listenQuic",
"QuicBidirectionalStream",
"QuicConn",
"QuicListener",
"QuicReceiveStream",
"QuicSendStream",
]);
const unstableMsgSuggestion =
"If not, try changing the 'lib' compiler option to include 'deno.unstable' " +

6
cli/tsc/diagnostics.rs
View file

@ -133,6 +133,12 @@ pub struct Diagnostic {
pub file_name: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub related_information: Option<Vec<Diagnostic>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub reports_deprecated: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub reports_unnecessary: Option<bool>,
#[serde(flatten)]
pub other: deno_core::serde_json::Map<String, deno_core::serde_json::Value>,
}
impl Diagnostic {

19
cli/tsc/mod.rs
View file

@ -4,9 +4,10 @@ use crate::args::TsConfig;
use crate::args::TypeCheckMode;
use crate::cache::FastInsecureHasher;
use crate::cache::ModuleInfoCache;
use crate::node;
use crate::node::CliNodeResolver;
use crate::npm::CliNpmResolver;
use crate::resolver::CjsTracker;
use crate::sys::CliSys;
use crate::util::checksum;
use crate::util::path::mapped_specifier_for_tsc;
use crate::worker::create_isolate_create_params;
@ -35,12 +36,11 @@ use deno_graph::Module;
use deno_graph::ModuleGraph;
use deno_graph::ResolutionResolved;
use deno_resolver::npm::ResolvePkgFolderFromDenoReqError;
use deno_runtime::deno_fs;
use deno_runtime::deno_node::NodeResolver;
use deno_semver::npm::NpmPackageReqReference;
use node_resolver::errors::NodeJsErrorCode;
use node_resolver::errors::NodeJsErrorCoded;
use node_resolver::errors::PackageSubpathResolveError;
use node_resolver::resolve_specifier_into_node_modules;
use node_resolver::NodeResolutionKind;
use node_resolver::ResolutionMode;
use once_cell::sync::Lazy;
@ -380,7 +380,7 @@ impl TypeCheckingCjsTracker {
#[derive(Debug)]
pub struct RequestNpmState {
pub cjs_tracker: Arc<TypeCheckingCjsTracker>,
pub node_resolver: Arc<NodeResolver>,
pub node_resolver: Arc<CliNodeResolver>,
pub npm_resolver: Arc<dyn CliNpmResolver>,
}
@ -660,9 +660,9 @@ fn op_load_inner(
None
} else {
// means it's Deno code importing an npm module
let specifier = node::resolve_specifier_into_node_modules(
let specifier = resolve_specifier_into_node_modules(
&CliSys::default(),
&module.specifier,
&deno_fs::RealFs,
);
Some(Cow::Owned(load_from_node_modules(
&specifier,
@ -924,9 +924,9 @@ fn resolve_graph_specifier_types(
Some(Module::External(module)) => {
// we currently only use "External" for when the module is in an npm package
Ok(state.maybe_npm.as_ref().map(|_| {
let specifier = node::resolve_specifier_into_node_modules(
let specifier = resolve_specifier_into_node_modules(
&CliSys::default(),
&module.specifier,
&deno_fs::RealFs,
);
into_specifier_and_media_type(Some(specifier))
}))
@ -1444,6 +1444,9 @@ mod tests {
source_line: None,
file_name: None,
related_information: None,
reports_deprecated: None,
reports_unnecessary: None,
other: Default::default(),
}]),
stats: Stats(vec![("a".to_string(), 12)])
})

19
cli/util/extract.rs
View file

@ -13,6 +13,7 @@ use deno_ast::swc::visit::VisitMut;
use deno_ast::swc::visit::VisitWith as _;
use deno_ast::MediaType;
use deno_ast::SourceRangedForSpanned as _;
use deno_cache_dir::file_fetcher::File;
use deno_core::error::AnyError;
use deno_core::ModuleSpecifier;
use regex::Regex;
@ -20,7 +21,7 @@ use std::collections::BTreeSet;
use std::fmt::Write as _;
use std::sync::Arc;
use crate::file_fetcher::File;
use crate::file_fetcher::TextDecodedFile;
use crate::util::path::mapped_specifier_for_tsc;
/// Extracts doc tests from a given file, transforms them into pseudo test
@ -52,7 +53,7 @@ fn extract_inner(
file: File,
wrap_kind: WrapKind,
) -> Result<Vec<File>, AnyError> {
let file = file.into_text_decoded()?;
let file = TextDecodedFile::decode(file)?;
let exports = match deno_ast::parse_program(deno_ast::ParseParams {
specifier: file.specifier.clone(),
@ -230,7 +231,7 @@ fn extract_files_from_regex_blocks(
.unwrap_or(file_specifier);
Some(File {
specifier: file_specifier,
url: file_specifier,
maybe_headers: None,
source: file_source.into_bytes().into(),
})
@ -558,7 +559,7 @@ fn generate_pseudo_file(
exports: &ExportCollector,
wrap_kind: WrapKind,
) -> Result<File, AnyError> {
let file = file.into_text_decoded()?;
let file = TextDecodedFile::decode(file)?;
let parsed = deno_ast::parse_program(deno_ast::ParseParams {
specifier: file.specifier.clone(),
@ -594,7 +595,7 @@ fn generate_pseudo_file(
log::debug!("{}:\n{}", file.specifier, source);
Ok(File {
specifier: file.specifier,
url: file.specifier,
maybe_headers: None,
source: source.into_bytes().into(),
})
@ -1199,14 +1200,14 @@ Deno.test("file:///main.ts$3-7.ts", async ()=>{
for test in tests {
let file = File {
specifier: ModuleSpecifier::parse(test.input.specifier).unwrap(),
url: ModuleSpecifier::parse(test.input.specifier).unwrap(),
maybe_headers: None,
source: test.input.source.as_bytes().into(),
};
let got_decoded = extract_doc_tests(file)
.unwrap()
.into_iter()
.map(|f| f.into_text_decoded().unwrap())
.map(|f| TextDecodedFile::decode(f).unwrap())
.collect::<Vec<_>>();
let expected = test
.expected
@ -1435,14 +1436,14 @@ add('1', '2');
for test in tests {
let file = File {
specifier: ModuleSpecifier::parse(test.input.specifier).unwrap(),
url: ModuleSpecifier::parse(test.input.specifier).unwrap(),
maybe_headers: None,
source: test.input.source.as_bytes().into(),
};
let got_decoded = extract_snippet_files(file)
.unwrap()
.into_iter()
.map(|f| f.into_text_decoded().unwrap())
.map(|f| TextDecodedFile::decode(f).unwrap())
.collect::<Vec<_>>();
let expected = test
.expected

316
cli/util/fs.rs
View file

@ -1,9 +1,7 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use std::fs::OpenOptions;
use std::io::Error;
use std::io::ErrorKind;
use std::io::Write;
use std::path::Path;
use std::path::PathBuf;
use std::sync::Arc;
@ -19,185 +17,12 @@ use deno_core::anyhow::Context;
use deno_core::error::AnyError;
use deno_core::unsync::spawn_blocking;
use deno_core::ModuleSpecifier;
use deno_runtime::deno_fs::FileSystem;
use crate::util::path::get_atomic_file_path;
use crate::sys::CliSys;
use crate::util::progress_bar::ProgressBar;
use crate::util::progress_bar::ProgressBarStyle;
use crate::util::progress_bar::ProgressMessagePrompt;
/// Writes the file to the file system at a temporary path, then
/// renames it to the destination in a single sys call in order
/// to never leave the file system in a corrupted state.
///
/// This also handles creating the directory if a NotFound error
/// occurs.
pub fn atomic_write_file_with_retries<T: AsRef<[u8]>>(
file_path: &Path,
data: T,
mode: u32,
) -> std::io::Result<()> {
struct RealAtomicWriteFileFs {
mode: u32,
}
impl AtomicWriteFileFs for RealAtomicWriteFileFs {
fn write_file(&self, path: &Path, bytes: &[u8]) -> std::io::Result<()> {
write_file(path, bytes, self.mode)
}
fn rename_file(&self, from: &Path, to: &Path) -> std::io::Result<()> {
std::fs::rename(from, to)
}
fn remove_file(&self, path: &Path) -> std::io::Result<()> {
std::fs::remove_file(path)
}
fn create_dir_all(&self, dir_path: &Path) -> std::io::Result<()> {
std::fs::create_dir_all(dir_path)
}
fn path_exists(&self, path: &Path) -> bool {
path.exists()
}
}
atomic_write_file_with_retries_and_fs(
&RealAtomicWriteFileFs { mode },
file_path,
data.as_ref(),
)
}
pub trait AtomicWriteFileFs {
fn write_file(&self, path: &Path, bytes: &[u8]) -> std::io::Result<()>;
fn rename_file(&self, from: &Path, to: &Path) -> std::io::Result<()>;
fn remove_file(&self, path: &Path) -> std::io::Result<()>;
fn create_dir_all(&self, dir_path: &Path) -> std::io::Result<()>;
fn path_exists(&self, path: &Path) -> bool;
}
pub struct AtomicWriteFileFsAdapter<'a> {
pub fs: &'a dyn FileSystem,
pub write_mode: u32,
}
impl<'a> AtomicWriteFileFs for AtomicWriteFileFsAdapter<'a> {
fn write_file(&self, path: &Path, bytes: &[u8]) -> std::io::Result<()> {
self
.fs
.write_file_sync(
path,
deno_runtime::deno_fs::OpenOptions::write(
true,
false,
false,
Some(self.write_mode),
),
None,
bytes,
)
.map_err(|e| e.into_io_error())
}
fn rename_file(&self, from: &Path, to: &Path) -> std::io::Result<()> {
self.fs.rename_sync(from, to).map_err(|e| e.into_io_error())
}
fn remove_file(&self, path: &Path) -> std::io::Result<()> {
self
.fs
.remove_sync(path, false)
.map_err(|e| e.into_io_error())
}
fn create_dir_all(&self, dir_path: &Path) -> std::io::Result<()> {
self
.fs
.mkdir_sync(dir_path, /* recursive */ true, None)
.map_err(|e| e.into_io_error())
}
fn path_exists(&self, path: &Path) -> bool {
self.fs.exists_sync(path)
}
}
pub fn atomic_write_file_with_retries_and_fs<T: AsRef<[u8]>>(
fs: &impl AtomicWriteFileFs,
file_path: &Path,
data: T,
) -> std::io::Result<()> {
let mut count = 0;
loop {
match atomic_write_file(fs, file_path, data.as_ref()) {
Ok(()) => return Ok(()),
Err(err) => {
if count >= 5 {
// too many retries, return the error
return Err(err);
}
count += 1;
let sleep_ms = std::cmp::min(50, 10 * count);
std::thread::sleep(std::time::Duration::from_millis(sleep_ms));
}
}
}
}
/// Writes the file to the file system at a temporary path, then
/// renames it to the destination in a single sys call in order
/// to never leave the file system in a corrupted state.
///
/// This also handles creating the directory if a NotFound error
/// occurs.
fn atomic_write_file(
fs: &impl AtomicWriteFileFs,
file_path: &Path,
data: &[u8],
) -> std::io::Result<()> {
fn atomic_write_file_raw(
fs: &impl AtomicWriteFileFs,
temp_file_path: &Path,
file_path: &Path,
data: &[u8],
) -> std::io::Result<()> {
fs.write_file(temp_file_path, data)?;
fs.rename_file(temp_file_path, file_path)
.inspect_err(|_err| {
// clean up the created temp file on error
let _ = fs.remove_file(temp_file_path);
})
}
let temp_file_path = get_atomic_file_path(file_path);
if let Err(write_err) =
atomic_write_file_raw(fs, &temp_file_path, file_path, data)
{
if write_err.kind() == ErrorKind::NotFound {
let parent_dir_path = file_path.parent().unwrap();
match fs.create_dir_all(parent_dir_path) {
Ok(()) => {
return atomic_write_file_raw(fs, &temp_file_path, file_path, data)
.map_err(|err| add_file_context_to_err(file_path, err));
}
Err(create_err) => {
if !fs.path_exists(parent_dir_path) {
return Err(Error::new(
create_err.kind(),
format!(
"{:#} (for '{}')\nCheck the permission of the directory.",
create_err,
parent_dir_path.display()
),
));
}
}
}
}
return Err(add_file_context_to_err(file_path, write_err));
}
Ok(())
}
/// Creates a std::fs::File handling if the parent does not exist.
pub fn create_file(file_path: &Path) -> std::io::Result<std::fs::File> {
match std::fs::File::create(file_path) {
@ -236,45 +61,6 @@ fn add_file_context_to_err(file_path: &Path, err: Error) -> Error {
)
}
pub fn write_file<T: AsRef<[u8]>>(
filename: &Path,
data: T,
mode: u32,
) -> std::io::Result<()> {
write_file_2(filename, data, true, mode, true, false)
}
pub fn write_file_2<T: AsRef<[u8]>>(
filename: &Path,
data: T,
update_mode: bool,
mode: u32,
is_create: bool,
is_append: bool,
) -> std::io::Result<()> {
let mut file = OpenOptions::new()
.read(false)
.write(true)
.append(is_append)
.truncate(!is_append)
.create(is_create)
.open(filename)?;
if update_mode {
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
let mode = mode & 0o777;
let permissions = PermissionsExt::from_mode(mode);
file.set_permissions(permissions)?;
}
#[cfg(not(unix))]
let _ = mode;
}
file.write_all(data.as_ref())
}
/// Similar to `std::fs::canonicalize()` but strips UNC prefixes on Windows.
pub fn canonicalize_path(path: &Path) -> Result<PathBuf, Error> {
Ok(deno_path_util::strip_unc_prefix(path.canonicalize()?))
@ -289,16 +75,10 @@ pub fn canonicalize_path(path: &Path) -> Result<PathBuf, Error> {
pub fn canonicalize_path_maybe_not_exists(
path: &Path,
) -> Result<PathBuf, Error> {
deno_path_util::canonicalize_path_maybe_not_exists(path, &canonicalize_path)
}
pub fn canonicalize_path_maybe_not_exists_with_fs(
path: &Path,
fs: &dyn FileSystem,
) -> Result<PathBuf, Error> {
deno_path_util::canonicalize_path_maybe_not_exists(path, &|path| {
fs.realpath_sync(path).map_err(|err| err.into_io_error())
})
deno_path_util::fs::canonicalize_path_maybe_not_exists(
&CliSys::default(),
path,
)
}
/// Collects module specifiers that satisfy the given predicate as a file path, by recursively walking `include`.
@ -346,7 +126,7 @@ pub fn collect_specifiers(
.ignore_git_folder()
.ignore_node_modules()
.set_vendor_folder(vendor_folder)
.collect_file_patterns(&deno_config::fs::RealDenoConfigFs, files)?;
.collect_file_patterns(&CliSys::default(), files)?;
let mut collected_files_as_urls = collected_files
.iter()
.map(|f| specifier_from_file_path(f).unwrap())
@ -418,7 +198,11 @@ mod clone_dir_imp {
from: &std::path::Path,
to: &std::path::Path,
) -> Result<(), deno_core::error::AnyError> {
if let Err(e) = super::hard_link_dir_recursive(from, to) {
use crate::sys::CliSys;
if let Err(e) =
deno_npm_cache::hard_link_dir_recursive(&CliSys::default(), from, to)
{
log::debug!("Failed to hard link dir {:?} to {:?}: {}", from, to, e);
super::copy_dir_recursive(from, to)?;
}
@ -465,84 +249,6 @@ pub fn copy_dir_recursive(from: &Path, to: &Path) -> Result<(), AnyError> {
Ok(())
}
/// Hardlinks the files in one directory to another directory.
///
/// Note: Does not handle symlinks.
pub fn hard_link_dir_recursive(from: &Path, to: &Path) -> Result<(), AnyError> {
std::fs::create_dir_all(to)
.with_context(|| format!("Creating {}", to.display()))?;
let read_dir = std::fs::read_dir(from)
.with_context(|| format!("Reading {}", from.display()))?;
for entry in read_dir {
let entry = entry?;
let file_type = entry.file_type()?;
let new_from = from.join(entry.file_name());
let new_to = to.join(entry.file_name());
if file_type.is_dir() {
hard_link_dir_recursive(&new_from, &new_to).with_context(|| {
format!("Dir {} to {}", new_from.display(), new_to.display())
})?;
} else if file_type.is_file() {
// note: chance for race conditions here between attempting to create,
// then removing, then attempting to create. There doesn't seem to be
// a way to hard link with overwriting in Rust, but maybe there is some
// way with platform specific code. The workaround here is to handle
// scenarios where something else might create or remove files.
if let Err(err) = std::fs::hard_link(&new_from, &new_to) {
if err.kind() == ErrorKind::AlreadyExists {
if let Err(err) = std::fs::remove_file(&new_to) {
if err.kind() == ErrorKind::NotFound {
// Assume another process/thread created this hard link to the file we are wanting
// to remove then sleep a little bit to let the other process/thread move ahead
// faster to reduce contention.
std::thread::sleep(Duration::from_millis(10));
} else {
return Err(err).with_context(|| {
format!(
"Removing file to hard link {} to {}",
new_from.display(),
new_to.display()
)
});
}
}
// Always attempt to recreate the hardlink. In contention scenarios, the other process
// might have been killed or exited after removing the file, but before creating the hardlink
if let Err(err) = std::fs::hard_link(&new_from, &new_to) {
// Assume another process/thread created this hard link to the file we are wanting
// to now create then sleep a little bit to let the other process/thread move ahead
// faster to reduce contention.
if err.kind() == ErrorKind::AlreadyExists {
std::thread::sleep(Duration::from_millis(10));
} else {
return Err(err).with_context(|| {
format!(
"Hard linking {} to {}",
new_from.display(),
new_to.display()
)
});
}
}
} else {
return Err(err).with_context(|| {
format!(
"Hard linking {} to {}",
new_from.display(),
new_to.display()
)
});
}
}
}
}
Ok(())
}
pub fn symlink_dir(oldpath: &Path, newpath: &Path) -> Result<(), Error> {
let err_mapper = |err: Error, kind: Option<ErrorKind>| {
Error::new(

14
cli/util/path.rs
View file

@ -1,7 +1,6 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use std::borrow::Cow;
use std::fmt::Write;
use std::path::Path;
use std::path::PathBuf;
@ -52,19 +51,6 @@ pub fn get_extension(file_path: &Path) -> Option<String> {
.map(|e| e.to_lowercase());
}
pub fn get_atomic_file_path(file_path: &Path) -> PathBuf {
let rand = gen_rand_path_component();
let extension = format!("{rand}.tmp");
file_path.with_extension(extension)
}
fn gen_rand_path_component() -> String {
(0..4).fold(String::with_capacity(8), |mut output, _| {
write!(&mut output, "{:02x}", rand::random::<u8>()).unwrap();
output
})
}
/// TypeScript figures out the type of file based on the extension, but we take
/// other factors into account like the file headers. The hack here is to map the
/// specifier passed to TypeScript to a new specifier with the file extension.

12
cli/util/text_encoding.rs
View file

@ -140,23 +140,23 @@ mod tests {
#[test]
fn test_source_map_from_code() {
let to_string =
|bytes: Vec<u8>| -> String { String::from_utf8(bytes).unwrap() };
|bytes: Vec<u8>| -> String { String::from_utf8(bytes.to_vec()).unwrap() };
assert_eq!(
source_map_from_code(
b"test\n//# sourceMappingURL=data:application/json;base64,dGVzdGluZ3Rlc3Rpbmc=",
b"test\n//# sourceMappingURL=data:application/json;base64,dGVzdGluZ3Rlc3Rpbmc="
).map(to_string),
Some("testingtesting".to_string())
);
assert_eq!(
source_map_from_code(
b"test\n//# sourceMappingURL=data:application/json;base64,dGVzdGluZ3Rlc3Rpbmc=\n \n",
b"test\n//# sourceMappingURL=data:application/json;base64,dGVzdGluZ3Rlc3Rpbmc=\n \n"
).map(to_string),
Some("testingtesting".to_string())
);
assert_eq!(
source_map_from_code(
b"test\n//# sourceMappingURL=data:application/json;base64,dGVzdGluZ3Rlc3Rpbmc=\n test\n",
),
b"test\n//# sourceMappingURL=data:application/json;base64,dGVzdGluZ3Rlc3Rpbmc=\n test\n"
).map(to_string),
None
);
assert_eq!(
@ -164,7 +164,7 @@ mod tests {
b"\"use strict\";
throw new Error(\"Hello world!\");
//# sourceMappingURL=data:application/json;base64,{",
//# sourceMappingURL=data:application/json;base64,{"
),
None
);

33
cli/worker.rs
View file

@ -23,8 +23,6 @@ use deno_runtime::deno_fs;
use deno_runtime::deno_node::NodeExtInitServices;
use deno_runtime::deno_node::NodeRequireLoader;
use deno_runtime::deno_node::NodeRequireLoaderRc;
use deno_runtime::deno_node::NodeResolver;
use deno_runtime::deno_node::PackageJsonResolver;
use deno_runtime::deno_permissions::PermissionsContainer;
use deno_runtime::deno_tls::RootCertStoreProvider;
use deno_runtime::deno_web::BlobStore;
@ -53,7 +51,10 @@ use crate::args::DenoSubcommand;
use crate::args::NpmCachingStrategy;
use crate::args::StorageKeyResolver;
use crate::errors;
use crate::node::CliNodeResolver;
use crate::node::CliPackageJsonResolver;
use crate::npm::CliNpmResolver;
use crate::sys::CliSys;
use crate::util::checksum;
use crate::util::file_watcher::WatcherCommunicator;
use crate::util::file_watcher::WatcherRestartMode;
@ -145,13 +146,14 @@ struct SharedWorkerState {
maybe_inspector_server: Option<Arc<InspectorServer>>,
maybe_lockfile: Option<Arc<CliLockfile>>,
module_loader_factory: Box<dyn ModuleLoaderFactory>,
node_resolver: Arc<NodeResolver>,
node_resolver: Arc<CliNodeResolver>,
npm_resolver: Arc<dyn CliNpmResolver>,
pkg_json_resolver: Arc<PackageJsonResolver>,
pkg_json_resolver: Arc<CliPackageJsonResolver>,
root_cert_store_provider: Arc<dyn RootCertStoreProvider>,
root_permissions: PermissionsContainer,
shared_array_buffer_store: SharedArrayBufferStore,
storage_key_resolver: StorageKeyResolver,
sys: CliSys,
options: CliMainWorkerOptions,
subcommand: DenoSubcommand,
otel_config: OtelConfig,
@ -162,12 +164,13 @@ impl SharedWorkerState {
pub fn create_node_init_services(
&self,
node_require_loader: NodeRequireLoaderRc,
) -> NodeExtInitServices {
) -> NodeExtInitServices<CliSys> {
NodeExtInitServices {
node_require_loader,
node_resolver: self.node_resolver.clone(),
npm_resolver: self.npm_resolver.clone().into_npm_pkg_folder_resolver(),
pkg_json_resolver: self.pkg_json_resolver.clone(),
sys: self.sys.clone(),
}
}
@ -418,12 +421,13 @@ impl CliMainWorkerFactory {
maybe_inspector_server: Option<Arc<InspectorServer>>,
maybe_lockfile: Option<Arc<CliLockfile>>,
module_loader_factory: Box<dyn ModuleLoaderFactory>,
node_resolver: Arc<NodeResolver>,
node_resolver: Arc<CliNodeResolver>,
npm_resolver: Arc<dyn CliNpmResolver>,
pkg_json_resolver: Arc<PackageJsonResolver>,
pkg_json_resolver: Arc<CliPackageJsonResolver>,
root_cert_store_provider: Arc<dyn RootCertStoreProvider>,
root_permissions: PermissionsContainer,
storage_key_resolver: StorageKeyResolver,
sys: CliSys,
subcommand: DenoSubcommand,
options: CliMainWorkerOptions,
otel_config: OtelConfig,
@ -448,6 +452,7 @@ impl CliMainWorkerFactory {
root_permissions,
shared_array_buffer_store: Default::default(),
storage_key_resolver,
sys,
options,
subcommand,
otel_config,
@ -612,6 +617,7 @@ impl CliMainWorkerFactory {
serve_port: shared.options.serve_port,
serve_host: shared.options.serve_host.clone(),
otel_config: shared.otel_config.clone(),
close_on_idle: true,
},
extensions: custom_extensions,
startup_snapshot: crate::js::deno_isolate_init(),
@ -655,7 +661,10 @@ impl CliMainWorkerFactory {
"40_test_common.js",
"40_test.js",
"40_bench.js",
"40_jupyter.js"
"40_jupyter.js",
// TODO(bartlomieju): probably shouldn't include these files here?
"40_lint_selector.js",
"40_lint.js"
);
}
@ -812,6 +821,7 @@ fn create_web_worker_callback(
serve_port: shared.options.serve_port,
serve_host: shared.options.serve_host.clone(),
otel_config: shared.otel_config.clone(),
close_on_idle: args.close_on_idle,
},
extensions: vec![],
startup_snapshot: crate::js::deno_isolate_init(),
@ -864,14 +874,15 @@ mod tests {
let main_module =
resolve_path("./hello.js", &std::env::current_dir().unwrap()).unwrap();
let fs = Arc::new(RealFs);
let permission_desc_parser =
Arc::new(RuntimePermissionDescriptorParser::new(fs.clone()));
let permission_desc_parser = Arc::new(
RuntimePermissionDescriptorParser::new(crate::sys::CliSys::default()),
);
let options = WorkerOptions {
startup_snapshot: crate::js::deno_isolate_init(),
..Default::default()
};
MainWorker::bootstrap_from_options(
MainWorker::bootstrap_from_options::<CliSys>(
main_module,
WorkerServiceOptions {
module_loader: Rc::new(FsModuleLoader),

Some files were not shown because too many files have changed in this diff Show more