foster/denoland-deno
1
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2025-01-21 04:52:26 -05:00
Code Issues Packages 1 Wiki Activity

Merge remote-tracking branch 'upstream/main' into check-workspace-member-compiler-options

Browse source
This commit is contained in:
Nayeem Rahman 2024-12-30 18:12:34 +00:00
commit 03cb9732a7
194 changed files with 14681 additions and 3283 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.github/workflows/ci.generate.ts vendored
View file

@ -5,7 +5,7 @@ import { stringify } from "jsr:@std/yaml@^0.221/stringify";
// Bump this number when you want to purge the cache.
// Note: the tools/release/01_bump_crate_versions.ts script will update this version
// automatically via regex, so ensure that this line maintains this format.
const cacheVersion = 31;
const cacheVersion = 32;
const ubuntuX86Runner = "ubuntu-24.04";
const ubuntuX86XlRunner = "ubuntu-24.04-xl";

8
.github/workflows/ci.yml vendored
View file

@ -184,8 +184,8 @@ jobs:
~/.cargo/registry/index
~/.cargo/registry/cache
~/.cargo/git/db
key: '31-cargo-home-${{ matrix.os }}-${{ matrix.arch }}-${{ hashFiles(''Cargo.lock'') }}'
restore-keys: '31-cargo-home-${{ matrix.os }}-${{ matrix.arch }}-'
key: '32-cargo-home-${{ matrix.os }}-${{ matrix.arch }}-${{ hashFiles(''Cargo.lock'') }}'
restore-keys: '32-cargo-home-${{ matrix.os }}-${{ matrix.arch }}-'
if: '!(matrix.skip)'
- uses: dsherret/rust-toolchain-file@v1
if: '!(matrix.skip)'
@ -379,7 +379,7 @@ jobs:
!./target/*/*.zip
!./target/*/*.tar.gz
key: never_saved
restore-keys: '31-cargo-target-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.profile }}-${{ matrix.job }}-'
restore-keys: '32-cargo-target-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.profile }}-${{ matrix.job }}-'
- name: Apply and update mtime cache
if: '!(matrix.skip) && (!startsWith(github.ref, ''refs/tags/''))'
uses: ./.github/mtime_cache
@ -689,7 +689,7 @@ jobs:
!./target/*/gn_root
!./target/*/*.zip
!./target/*/*.tar.gz
key: '31-cargo-target-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.profile }}-${{ matrix.job }}-${{ github.sha }}'
key: '32-cargo-target-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.profile }}-${{ matrix.job }}-${{ github.sha }}'
publish-canary:
name: publish canary
runs-on: ubuntu-24.04

273
Cargo.lock generated
View file

@ -39,9 +39,9 @@ dependencies = [
[[package]]
name = "aead-gcm-stream"
version = "0.3.0"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4947a169074c7e038fa43051d1c4e073f4488b0e4b0a30658f1e1a1b06449ce8"
checksum = "e70c8dec860340effb00f6945c49c0daaa6dac963602750db862eabb74bf7886"
dependencies = [
"aead",
"aes",
@ -677,6 +677,28 @@ dependencies = [
"itoa",
]
[[package]]
name = "capacity_builder"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f2d24a6dcf0cd402a21b65d35340f3a49ff3475dc5fdac91d22d2733e6641c6"
dependencies = [
"capacity_builder_macros",
"ecow",
"hipstr",
"itoa",
]
[[package]]
name = "capacity_builder_macros"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3b4a6cae9efc04cc6cbb8faf338d2c497c165c83e74509cf4dbedea948bbf6e5"
dependencies = [
"quote",
"syn 2.0.87",
]
[[package]]
name = "caseless"
version = "0.2.1"
@ -728,6 +750,12 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e"
[[package]]
name = "cfg_aliases"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
[[package]]
name = "chrono"
version = "0.4.37"
@ -839,6 +867,7 @@ dependencies = [
"regex",
"reqwest",
"serde",
"sys_traits",
"test_server",
"tokio",
"url",
@ -1224,7 +1253,7 @@ dependencies = [
"boxed_error",
"bytes",
"cache_control",
"capacity_builder",
"capacity_builder 0.5.0",
"chrono",
"clap",
"clap_complete",
@ -1246,7 +1275,7 @@ dependencies = [
"deno_npm",
"deno_npm_cache",
"deno_package_json",
"deno_path_util",
"deno_path_util 0.3.0",
"deno_resolver",
"deno_runtime",
"deno_semver",
@ -1312,6 +1341,7 @@ dependencies = [
"spki",
"sqlformat",
"strsim",
"sys_traits",
"tar",
"tempfile",
"test_server",
@ -1391,7 +1421,7 @@ dependencies = [
[[package]]
name = "deno_bench_util"
version = "0.177.0"
version = "0.178.0"
dependencies = [
"bencher",
"deno_core",
@ -1400,7 +1430,7 @@ dependencies = [
[[package]]
name = "deno_broadcast_channel"
version = "0.177.0"
version = "0.178.0"
dependencies = [
"async-trait",
"deno_core",
@ -1411,7 +1441,7 @@ dependencies = [
[[package]]
name = "deno_cache"
version = "0.115.0"
version = "0.116.0"
dependencies = [
"async-trait",
"deno_core",
@ -1424,9 +1454,9 @@ dependencies = [
[[package]]
name = "deno_cache_dir"
version = "0.15.0"
version = "0.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "54df1c5177ace01d92b872584ab9af8290681bb150fd9b423c37a494ad5ddbdc"
checksum = "e73ed17f285731a23df9779ca1e0e721de866db6776ed919ebd9235e0a107c4c"
dependencies = [
"async-trait",
"base32",
@ -1437,7 +1467,7 @@ dependencies = [
"data-url",
"deno_error",
"deno_media_type",
"deno_path_util",
"deno_path_util 0.3.0",
"http 1.1.0",
"indexmap 2.3.0",
"log",
@ -1446,13 +1476,14 @@ dependencies = [
"serde",
"serde_json",
"sha2",
"sys_traits",
"thiserror 1.0.64",
"url",
]
[[package]]
name = "deno_canvas"
version = "0.52.0"
version = "0.53.0"
dependencies = [
"deno_core",
"deno_webgpu",
@ -1463,12 +1494,12 @@ dependencies = [
[[package]]
name = "deno_config"
version = "0.39.3"
source = "git+https://github.com/denoland/deno_config.git?rev=0d588fb1831bf4d33d3279a1a75db6138d81a75b#0d588fb1831bf4d33d3279a1a75db6138d81a75b"
version = "0.42.0"
source = "git+https://github.com/denoland/deno_config.git?rev=4cbb63704442a7834dc6bed2e7e310a0d46ade09#4cbb63704442a7834dc6bed2e7e310a0d46ade09"
dependencies = [
"anyhow",
"deno_package_json",
"deno_path_util",
"deno_path_util 0.3.0",
"deno_semver",
"glob",
"ignore",
@ -1480,22 +1511,23 @@ dependencies = [
"phf",
"serde",
"serde_json",
"sys_traits",
"thiserror 1.0.64",
"url",
]
[[package]]
name = "deno_console"
version = "0.183.0"
version = "0.184.0"
dependencies = [
"deno_core",
]
[[package]]
name = "deno_core"
version = "0.326.0"
version = "0.327.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed157162dc5320a2b46ffeeaec24788339df0f2437cfaea78a8d82696715ad7f"
checksum = "eaf8dff204b9c2415deb47b9f30d4d38b0925d0d88f1f9074e8e76f59e6d7ded"
dependencies = [
"anyhow",
"az",
@ -1503,7 +1535,7 @@ dependencies = [
"bit-set",
"bit-vec",
"bytes",
"capacity_builder",
"capacity_builder 0.1.3",
"cooked-waker",
"deno_core_icudata",
"deno_ops",
@ -1535,7 +1567,7 @@ checksum = "fe4dccb6147bb3f3ba0c7a48e993bfeb999d2c2e47a81badee80e2b370c8d695"
[[package]]
name = "deno_cron"
version = "0.63.0"
version = "0.64.0"
dependencies = [
"anyhow",
"async-trait",
@ -1548,7 +1580,7 @@ dependencies = [
[[package]]
name = "deno_crypto"
version = "0.197.0"
version = "0.198.0"
dependencies = [
"aes",
"aes-gcm",
@ -1594,7 +1626,7 @@ dependencies = [
"comrak",
"deno_ast",
"deno_graph",
"deno_path_util",
"deno_path_util 0.2.2",
"handlebars",
"html-escape",
"import_map",
@ -1638,13 +1670,13 @@ dependencies = [
[[package]]
name = "deno_fetch"
version = "0.207.0"
version = "0.208.0"
dependencies = [
"base64 0.21.7",
"bytes",
"data-url",
"deno_core",
"deno_path_util",
"deno_path_util 0.3.0",
"deno_permissions",
"deno_tls",
"dyn-clone",
@ -1674,7 +1706,7 @@ dependencies = [
[[package]]
name = "deno_ffi"
version = "0.170.0"
version = "0.171.0"
dependencies = [
"deno_core",
"deno_permissions",
@ -1694,22 +1726,24 @@ dependencies = [
[[package]]
name = "deno_fs"
version = "0.93.0"
version = "0.94.0"
dependencies = [
"async-trait",
"base32",
"boxed_error",
"deno_core",
"deno_io",
"deno_path_util",
"deno_path_util 0.3.0",
"deno_permissions",
"filetime",
"getrandom",
"junction",
"libc",
"nix",
"rand",
"rayon",
"serde",
"sys_traits",
"thiserror 2.0.3",
"winapi",
"windows-sys 0.59.0",
@ -1717,14 +1751,16 @@ dependencies = [
[[package]]
name = "deno_graph"
version = "0.86.3"
version = "0.86.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc78ed0b4bbcb4197300f0d6e7d1edc2d2c5019cdb9dedba7ff229158441885b"
checksum = "83af194ca492ea7b624d21055f933676d3f3d27586de93be31c8f1babcc73510"
dependencies = [
"anyhow",
"async-trait",
"capacity_builder 0.5.0",
"data-url",
"deno_ast",
"deno_path_util 0.3.0",
"deno_semver",
"deno_unsync",
"encoding_rs",
@ -1739,6 +1775,7 @@ dependencies = [
"serde",
"serde_json",
"sha2",
"sys_traits",
"thiserror 2.0.3",
"twox-hash",
"url",
@ -1747,7 +1784,7 @@ dependencies = [
[[package]]
name = "deno_http"
version = "0.181.0"
version = "0.182.0"
dependencies = [
"async-compression",
"async-trait",
@ -1786,7 +1823,7 @@ dependencies = [
[[package]]
name = "deno_io"
version = "0.93.0"
version = "0.94.0"
dependencies = [
"async-trait",
"deno_core",
@ -1807,7 +1844,7 @@ dependencies = [
[[package]]
name = "deno_kv"
version = "0.91.0"
version = "0.92.0"
dependencies = [
"anyhow",
"async-trait",
@ -1817,7 +1854,7 @@ dependencies = [
"chrono",
"deno_core",
"deno_fetch",
"deno_path_util",
"deno_path_util 0.3.0",
"deno_permissions",
"deno_tls",
"denokv_proto",
@ -1857,9 +1894,9 @@ dependencies = [
[[package]]
name = "deno_lockfile"
version = "0.23.2"
version = "0.24.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "559c19feb00af0c34f0bd4a20e56e12463fafd5c5069d6005f3ce33008027eea"
checksum = "632e835a53ed667d62fdd766c5780fe8361c831d3e3fbf1a760a0b7896657587"
dependencies = [
"deno_semver",
"serde",
@ -1880,7 +1917,7 @@ dependencies = [
[[package]]
name = "deno_napi"
version = "0.114.0"
version = "0.115.0"
dependencies = [
"deno_core",
"deno_permissions",
@ -1908,7 +1945,7 @@ dependencies = [
[[package]]
name = "deno_net"
version = "0.175.0"
version = "0.176.0"
dependencies = [
"deno_core",
"deno_permissions",
@ -1916,6 +1953,7 @@ dependencies = [
"hickory-proto",
"hickory-resolver",
"pin-project",
"quinn",
"rustls-tokio-stream",
"serde",
"socket2",
@ -1925,7 +1963,7 @@ dependencies = [
[[package]]
name = "deno_node"
version = "0.121.0"
version = "0.122.0"
dependencies = [
"aead-gcm-stream",
"aes",
@ -1945,7 +1983,7 @@ dependencies = [
"deno_media_type",
"deno_net",
"deno_package_json",
"deno_path_util",
"deno_path_util 0.3.0",
"deno_permissions",
"deno_whoami",
"der",
@ -2017,12 +2055,13 @@ dependencies = [
[[package]]
name = "deno_npm"
version = "0.26.0"
version = "0.27.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2f125a5dba7839c46394a0a9c835da9fe60f5f412587ab4956a76492a1cc6a8"
checksum = "5f818ad5dc4c206b50b5cfa6f10b4b94b127e15c8342c152768eba40c225ca23"
dependencies = [
"anyhow",
"async-trait",
"capacity_builder 0.5.0",
"deno_error",
"deno_lockfile",
"deno_semver",
"futures",
@ -2044,7 +2083,9 @@ dependencies = [
"boxed_error",
"deno_cache_dir",
"deno_core",
"deno_error",
"deno_npm",
"deno_path_util 0.3.0",
"deno_semver",
"deno_unsync",
"faster-hex",
@ -2057,6 +2098,7 @@ dependencies = [
"rand",
"ring",
"serde_json",
"sys_traits",
"tar",
"tempfile",
"thiserror 2.0.3",
@ -2065,9 +2107,9 @@ dependencies = [
[[package]]
name = "deno_ops"
version = "0.202.0"
version = "0.203.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4dd8ac1af251e292388e516dd339b9a3b982a6d1e7f8644c08e34671ca39003c"
checksum = "b146ca74cac431843486ade58e2accc16c11315fb2c6934590a52a73c56b7ec3"
dependencies = [
"proc-macro-rules",
"proc-macro2",
@ -2081,16 +2123,18 @@ dependencies = [
[[package]]
name = "deno_package_json"
version = "0.2.1"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "80b0a3d81c592624a1ae15332a04b4dc2b7c163ef1dfc7c60171f736d1babdf5"
checksum = "e1d3c0f699ba2040669204ce24ab73720499fc290af843e4ce0fc8a9b3d67735"
dependencies = [
"boxed_error",
"deno_error",
"deno_path_util",
"deno_path_util 0.3.0",
"deno_semver",
"indexmap 2.3.0",
"serde",
"serde_json",
"sys_traits",
"thiserror 2.0.3",
"url",
]
@ -2107,13 +2151,26 @@ dependencies = [
"url",
]
[[package]]
name = "deno_path_util"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "420e8211aaba7fde83ccaa9a5dad855c3b940ed988d70c95159acd600a70dc87"
dependencies = [
"deno_error",
"percent-encoding",
"sys_traits",
"thiserror 2.0.3",
"url",
]
[[package]]
name = "deno_permissions"
version = "0.43.0"
dependencies = [
"capacity_builder",
"capacity_builder 0.5.0",
"deno_core",
"deno_path_util",
"deno_path_util 0.3.0",
"deno_terminal 0.2.0",
"fqdn",
"libc",
@ -2137,9 +2194,10 @@ dependencies = [
"deno_config",
"deno_media_type",
"deno_package_json",
"deno_path_util",
"deno_path_util 0.3.0",
"deno_semver",
"node_resolver",
"sys_traits",
"test_server",
"thiserror 2.0.3",
"url",
@ -2167,7 +2225,7 @@ dependencies = [
"deno_napi",
"deno_net",
"deno_node",
"deno_path_util",
"deno_path_util 0.3.0",
"deno_permissions",
"deno_telemetry",
"deno_terminal 0.2.0",
@ -2216,11 +2274,14 @@ dependencies = [
[[package]]
name = "deno_semver"
version = "0.6.1"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7d1259270d66a5e6d29bb75c9289656541874f79ae9ff6c9f1c790846d5c07ba"
checksum = "4775271f9b5602482698f76d24ea9ed8ba27af7f587a7e9a876916300c542435"
dependencies = [
"capacity_builder 0.5.0",
"deno_error",
"ecow",
"hipstr",
"monch",
"once_cell",
"serde",
@ -2248,7 +2309,7 @@ dependencies = [
[[package]]
name = "deno_telemetry"
version = "0.5.0"
version = "0.6.0"
dependencies = [
"async-trait",
"deno_core",
@ -2289,7 +2350,7 @@ dependencies = [
[[package]]
name = "deno_tls"
version = "0.170.0"
version = "0.171.0"
dependencies = [
"deno_core",
"deno_native_certs",
@ -2339,7 +2400,7 @@ dependencies = [
[[package]]
name = "deno_url"
version = "0.183.0"
version = "0.184.0"
dependencies = [
"deno_bench_util",
"deno_console",
@ -2351,7 +2412,7 @@ dependencies = [
[[package]]
name = "deno_web"
version = "0.214.0"
version = "0.215.0"
dependencies = [
"async-trait",
"base64-simd 0.8.0",
@ -2373,7 +2434,7 @@ dependencies = [
[[package]]
name = "deno_webgpu"
version = "0.150.0"
version = "0.151.0"
dependencies = [
"deno_core",
"raw-window-handle",
@ -2386,7 +2447,7 @@ dependencies = [
[[package]]
name = "deno_webidl"
version = "0.183.0"
version = "0.184.0"
dependencies = [
"deno_bench_util",
"deno_core",
@ -2394,7 +2455,7 @@ dependencies = [
[[package]]
name = "deno_websocket"
version = "0.188.0"
version = "0.189.0"
dependencies = [
"bytes",
"deno_core",
@ -2416,7 +2477,7 @@ dependencies = [
[[package]]
name = "deno_webstorage"
version = "0.178.0"
version = "0.179.0"
dependencies = [
"deno_core",
"deno_web",
@ -2887,6 +2948,15 @@ dependencies = [
"spki",
]
[[package]]
name = "ecow"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e42fc0a93992b20c58b99e59d61eaf1635a25bfbe49e4275c34ba0aee98119ba"
dependencies = [
"serde",
]
[[package]]
name = "ed25519"
version = "2.2.3"
@ -3823,6 +3893,17 @@ dependencies = [
"tracing",
]
[[package]]
name = "hipstr"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "97971ffc85d4c98de12e2608e992a43f5294ebb625fdb045b27c731b64c4c6d6"
dependencies = [
"serde",
"serde_bytes",
"sptr",
]
[[package]]
name = "hkdf"
version = "0.12.4"
@ -4023,9 +4104,9 @@ dependencies = [
[[package]]
name = "hyper-timeout"
version = "0.5.1"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3203a961e5c83b6f5498933e78b6b263e208c197b63e9c6c53cc82ffd3f63793"
checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0"
dependencies = [
"hyper 1.4.1",
"hyper-util",
@ -4930,7 +5011,7 @@ dependencies = [
[[package]]
name = "napi_sym"
version = "0.113.0"
version = "0.114.0"
dependencies = [
"quote",
"serde",
@ -4992,13 +5073,14 @@ dependencies = [
"boxed_error",
"deno_media_type",
"deno_package_json",
"deno_path_util",
"deno_path_util 0.3.0",
"futures",
"lazy-regex",
"once_cell",
"path-clean",
"regex",
"serde_json",
"sys_traits",
"thiserror 2.0.3",
"tokio",
"url",
@ -5906,49 +5988,54 @@ dependencies = [
[[package]]
name = "quinn"
version = "0.11.2"
version = "0.11.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e4ceeeeabace7857413798eb1ffa1e9c905a9946a57d81fb69b4b71c4d8eb3ad"
checksum = "62e96808277ec6f97351a2380e6c25114bc9e67037775464979f3037c92d05ef"
dependencies = [
"bytes",
"pin-project-lite",
"quinn-proto",
"quinn-udp",
"rustc-hash 1.1.0",
"rustc-hash 2.0.0",
"rustls",
"thiserror 1.0.64",
"socket2",
"thiserror 2.0.3",
"tokio",
"tracing",
]
[[package]]
name = "quinn-proto"
version = "0.11.8"
version = "0.11.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fadfaed2cd7f389d0161bb73eeb07b7b78f8691047a6f3e73caaeae55310a4a6"
checksum = "a2fe5ef3495d7d2e377ff17b1a8ce2ee2ec2a18cde8b6ad6619d65d0701c135d"
dependencies = [
"bytes",
"getrandom",
"rand",
"ring",
"rustc-hash 2.0.0",
"rustls",
"rustls-pki-types",
"slab",
"thiserror 1.0.64",
"thiserror 2.0.3",
"tinyvec",
"tracing",
"web-time",
]
[[package]]
name = "quinn-udp"
version = "0.5.2"
version = "0.5.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9096629c45860fc7fb143e125eb826b5e721e10be3263160c7d60ca832cf8c46"
checksum = "52cd4b1eff68bf27940dd39811292c49e007f4d0b4c357358dc9b0197be6b527"
dependencies = [
"cfg_aliases 0.2.1",
"libc",
"once_cell",
"socket2",
"tracing",
"windows-sys 0.52.0",
"windows-sys 0.59.0",
]
[[package]]
@ -6425,6 +6512,9 @@ name = "rustls-pki-types"
version = "1.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc0a2ce646f8655401bb81e7927b812614bd5d91dbc968696be50603510fcaf0"
dependencies = [
"web-time",
]
[[package]]
name = "rustls-tokio-stream"
@ -6708,9 +6798,9 @@ dependencies = [
[[package]]
name = "serde_v8"
version = "0.235.0"
version = "0.236.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d07afd8b67b4a442ecc2823038473ac0e9e5682de93c213323b60661afdd7eb4"
checksum = "e23b3abce64010612f88f4ff689a959736f99eb3dc0dbf1c7903434b8bd8cda5"
dependencies = [
"num-bigint",
"serde",
@ -6986,6 +7076,12 @@ dependencies = [
"der",
]
[[package]]
name = "sptr"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3b9b39299b249ad65f3b7e96443bad61c02ca5cd3589f46cb6d610a0fd6c0d6a"
[[package]]
name = "sqlformat"
version = "0.3.2"
@ -7581,6 +7677,17 @@ dependencies = [
"syn 2.0.87",
]
[[package]]
name = "sys_traits"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a5a12729b699487bb50163466e87be7197871d83d04cc6815d430cf7c893bbd7"
dependencies = [
"getrandom",
"libc",
"windows-sys 0.59.0",
]
[[package]]
name = "tagptr"
version = "0.2.0"
@ -8521,6 +8628,16 @@ dependencies = [
"wasm-bindgen",
]
[[package]]
name = "web-time"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb"
dependencies = [
"js-sys",
"wasm-bindgen",
]
[[package]]
name = "webpki-root-certs"
version = "0.26.6"
@ -8548,7 +8665,7 @@ dependencies = [
"arrayvec",
"bit-vec",
"bitflags 2.6.0",
"cfg_aliases",
"cfg_aliases 0.1.1",
"codespan-reporting",
"document-features",
"indexmap 2.3.0",
@ -8580,7 +8697,7 @@ dependencies = [
"bit-set",
"bitflags 2.6.0",
"block",
"cfg_aliases",
"cfg_aliases 0.1.1",
"core-graphics-types",
"d3d12",
"glow",

69
Cargo.toml
View file

@ -48,20 +48,20 @@ repository = "https://github.com/denoland/deno"
[workspace.dependencies]
deno_ast = { version = "=0.44.0", features = ["transpiling"] }
deno_core = { version = "0.326.0" }
deno_core = { version = "0.327.0" }
deno_bench_util = { version = "0.177.0", path = "./bench_util" }
deno_bench_util = { version = "0.178.0", path = "./bench_util" }
# TODO(nayeemrmn): Use proper version when https://github.com/denoland/deno_config/pull/143 lands!
deno_config = { git = "https://github.com/denoland/deno_config.git", rev = "0d588fb1831bf4d33d3279a1a75db6138d81a75b", features = ["workspace", "sync"] }
deno_lockfile = "=0.23.2"
deno_config = { git = "https://github.com/denoland/deno_config.git", rev = "4cbb63704442a7834dc6bed2e7e310a0d46ade09", features = ["workspace", "sync"] }
deno_lockfile = "=0.24.0"
deno_media_type = { version = "0.2.0", features = ["module_specifier"] }
deno_npm = "=0.26.0"
deno_path_util = "=0.2.2"
deno_npm = "=0.27.0"
deno_path_util = "=0.3.0"
deno_permissions = { version = "0.43.0", path = "./runtime/permissions" }
deno_runtime = { version = "0.192.0", path = "./runtime" }
deno_semver = "=0.6.1"
deno_semver = "=0.7.1"
deno_terminal = "0.2.0"
napi_sym = { version = "0.113.0", path = "./ext/napi/sym" }
napi_sym = { version = "0.114.0", path = "./ext/napi/sym" }
test_util = { package = "test_server", path = "./tests/util/server" }
denokv_proto = "0.8.4"
@ -70,29 +70,29 @@ denokv_remote = "0.8.4"
denokv_sqlite = { default-features = false, version = "0.8.4" }
# exts
deno_broadcast_channel = { version = "0.177.0", path = "./ext/broadcast_channel" }
deno_cache = { version = "0.115.0", path = "./ext/cache" }
deno_canvas = { version = "0.52.0", path = "./ext/canvas" }
deno_console = { version = "0.183.0", path = "./ext/console" }
deno_cron = { version = "0.63.0", path = "./ext/cron" }
deno_crypto = { version = "0.197.0", path = "./ext/crypto" }
deno_fetch = { version = "0.207.0", path = "./ext/fetch" }
deno_ffi = { version = "0.170.0", path = "./ext/ffi" }
deno_fs = { version = "0.93.0", path = "./ext/fs" }
deno_http = { version = "0.181.0", path = "./ext/http" }
deno_io = { version = "0.93.0", path = "./ext/io" }
deno_kv = { version = "0.91.0", path = "./ext/kv" }
deno_napi = { version = "0.114.0", path = "./ext/napi" }
deno_net = { version = "0.175.0", path = "./ext/net" }
deno_node = { version = "0.121.0", path = "./ext/node" }
deno_telemetry = { version = "0.5.0", path = "./ext/telemetry" }
deno_tls = { version = "0.170.0", path = "./ext/tls" }
deno_url = { version = "0.183.0", path = "./ext/url" }
deno_web = { version = "0.214.0", path = "./ext/web" }
deno_webgpu = { version = "0.150.0", path = "./ext/webgpu" }
deno_webidl = { version = "0.183.0", path = "./ext/webidl" }
deno_websocket = { version = "0.188.0", path = "./ext/websocket" }
deno_webstorage = { version = "0.178.0", path = "./ext/webstorage" }
deno_broadcast_channel = { version = "0.178.0", path = "./ext/broadcast_channel" }
deno_cache = { version = "0.116.0", path = "./ext/cache" }
deno_canvas = { version = "0.53.0", path = "./ext/canvas" }
deno_console = { version = "0.184.0", path = "./ext/console" }
deno_cron = { version = "0.64.0", path = "./ext/cron" }
deno_crypto = { version = "0.198.0", path = "./ext/crypto" }
deno_fetch = { version = "0.208.0", path = "./ext/fetch" }
deno_ffi = { version = "0.171.0", path = "./ext/ffi" }
deno_fs = { version = "0.94.0", path = "./ext/fs" }
deno_http = { version = "0.182.0", path = "./ext/http" }
deno_io = { version = "0.94.0", path = "./ext/io" }
deno_kv = { version = "0.92.0", path = "./ext/kv" }
deno_napi = { version = "0.115.0", path = "./ext/napi" }
deno_net = { version = "0.176.0", path = "./ext/net" }
deno_node = { version = "0.122.0", path = "./ext/node" }
deno_telemetry = { version = "0.6.0", path = "./ext/telemetry" }
deno_tls = { version = "0.171.0", path = "./ext/tls" }
deno_url = { version = "0.184.0", path = "./ext/url" }
deno_web = { version = "0.215.0", path = "./ext/web" }
deno_webgpu = { version = "0.151.0", path = "./ext/webgpu" }
deno_webidl = { version = "0.184.0", path = "./ext/webidl" }
deno_websocket = { version = "0.189.0", path = "./ext/websocket" }
deno_webstorage = { version = "0.179.0", path = "./ext/webstorage" }
# resolvers
deno_npm_cache = { version = "0.3.0", path = "./resolvers/npm_cache" }
@ -109,7 +109,7 @@ boxed_error = "0.2.3"
brotli = "6.0.0"
bytes = "1.4.0"
cache_control = "=0.2.0"
capacity_builder = "0.1.3"
capacity_builder = "0.5.0"
cbc = { version = "=0.1.2", features = ["alloc"] }
# Note: Do not use the "clock" feature of chrono, as it links us to CoreFoundation on macOS.
# Instead use util::time::utc_now()
@ -119,9 +119,9 @@ console_static_text = "=0.8.1"
dashmap = "5.5.3"
data-encoding = "2.3.3"
data-url = "=0.3.1"
deno_cache_dir = "=0.15.0"
deno_cache_dir = "=0.16.0"
deno_error = "=0.5.2"
deno_package_json = { version = "0.2.1", default-features = false }
deno_package_json = { version = "0.4.0", default-features = false }
deno_unsync = "0.4.2"
dlopen2 = "0.6.1"
ecb = "=0.1.2"
@ -194,6 +194,7 @@ slab = "0.4"
smallvec = "1.8"
socket2 = { version = "0.5.3", features = ["all"] }
spki = "0.7.2"
sys_traits = "=0.1.1"
tar = "=0.4.40"
tempfile = "3.4.0"
termcolor = "1.1.3"

2
bench_util/Cargo.toml
View file

@ -2,7 +2,7 @@
[package]
name = "deno_bench_util"
version = "0.177.0"
version = "0.178.0"
authors.workspace = true
edition.workspace = true
license.workspace = true

3
cli/Cargo.toml
View file

@ -74,7 +74,7 @@ deno_config.workspace = true
deno_core = { workspace = true, features = ["include_js_files_for_snapshotting"] }
deno_doc = { version = "=0.161.3", features = ["rust", "comrak"] }
deno_error.workspace = true
deno_graph = { version = "=0.86.3" }
deno_graph = { version = "=0.86.6" }
deno_lint = { version = "=0.68.2", features = ["docs"] }
deno_lockfile.workspace = true
deno_npm.workspace = true
@ -158,6 +158,7 @@ shell-escape = "=0.1.5"
spki = { version = "0.7", features = ["pem"] }
sqlformat = "=0.3.2"
strsim = "0.11.1"
sys_traits = { workspace = true, features = ["libc", "real", "winapi"] }
tar.workspace = true
tempfile.workspace = true
text-size = "=1.1.0"

56
cli/args/deno_json.rs
View file

@ -8,62 +8,6 @@ use deno_semver::jsr::JsrDepPackageReq;
use deno_semver::jsr::JsrPackageReqReference;
use deno_semver::npm::NpmPackageReqReference;
#[cfg(test)] // happens to only be used by the tests at the moment
pub struct DenoConfigFsAdapter<'a>(
pub &'a dyn deno_runtime::deno_fs::FileSystem,
);
#[cfg(test)]
impl<'a> deno_config::fs::DenoConfigFs for DenoConfigFsAdapter<'a> {
fn read_to_string_lossy(
&self,
path: &std::path::Path,
) -> Result<std::borrow::Cow<'static, str>, std::io::Error> {
self
.0
.read_text_file_lossy_sync(path, None)
.map_err(|err| err.into_io_error())
}
fn stat_sync(
&self,
path: &std::path::Path,
) -> Result<deno_config::fs::FsMetadata, std::io::Error> {
self
.0
.stat_sync(path)
.map(|stat| deno_config::fs::FsMetadata {
is_file: stat.is_file,
is_directory: stat.is_directory,
is_symlink: stat.is_symlink,
})
.map_err(|err| err.into_io_error())
}
fn read_dir(
&self,
path: &std::path::Path,
) -> Result<Vec<deno_config::fs::FsDirEntry>, std::io::Error> {
self
.0
.read_dir_sync(path)
.map_err(|err| err.into_io_error())
.map(|entries| {
entries
.into_iter()
.map(|e| deno_config::fs::FsDirEntry {
path: path.join(e.name),
metadata: deno_config::fs::FsMetadata {
is_file: e.is_file,
is_directory: e.is_directory,
is_symlink: e.is_symlink,
},
})
.collect()
})
}
}
pub fn import_map_deps(
import_map: &serde_json::Value,
) -> HashSet<JsrDepPackageReq> {

2
cli/args/flags.rs
View file

@ -1006,6 +1006,8 @@ impl Flags {
OtelConfig {
tracing_enabled: !disabled
&& otel_var("OTEL_DENO_TRACING").unwrap_or(default),
metrics_enabled: !disabled
&& otel_var("OTEL_DENO_METRICS").unwrap_or(default),
console: match std::env::var("OTEL_DENO_CONSOLE").as_deref() {
Ok(_) if disabled => OtelConsoleConfig::Ignore,
Ok("ignore") => OtelConsoleConfig::Ignore,

6
cli/args/lockfile.rs
View file

@ -12,12 +12,13 @@ use deno_core::parking_lot::MutexGuard;
use deno_core::serde_json;
use deno_lockfile::WorkspaceMemberConfig;
use deno_package_json::PackageJsonDepValue;
use deno_path_util::fs::atomic_write_file_with_retries;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use deno_runtime::deno_node::PackageJson;
use deno_semver::jsr::JsrDepPackageReq;
use crate::args::deno_json::import_map_deps;
use crate::cache;
use crate::util::fs::atomic_write_file_with_retries;
use crate::Flags;
use crate::args::DenoSubcommand;
@ -91,8 +92,9 @@ impl CliLockfile {
// do an atomic write to reduce the chance of multiple deno
// processes corrupting the file
atomic_write_file_with_retries(
&FsSysTraitsAdapter::new_real(),
&lockfile.filename,
bytes,
&bytes,
cache::CACHE_PERM,
)
.context("Failed writing lockfile.")?;

43
cli/args/mod.rs
View file

@ -30,7 +30,9 @@ use deno_npm::npm_rc::ResolvedNpmRc;
use deno_npm::resolution::ValidSerializedNpmResolutionSnapshot;
use deno_npm::NpmSystemInfo;
use deno_path_util::normalize_path;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use deno_semver::npm::NpmPackageReqReference;
use deno_semver::StackString;
use deno_telemetry::OtelConfig;
use deno_telemetry::OtelRuntimeConfig;
use import_map::resolve_import_map_value_from_specifier;
@ -82,9 +84,9 @@ use std::net::SocketAddr;
use std::path::Path;
use std::path::PathBuf;
use std::sync::Arc;
use sys_traits::EnvHomeDir;
use thiserror::Error;
use crate::cache;
use crate::cache::DenoDirProvider;
use crate::file_fetcher::CliFileFetcher;
use crate::util::fs::canonicalize_path_maybe_not_exists;
@ -520,7 +522,7 @@ fn discover_npmrc(
// TODO(bartlomieju): update to read both files - one in the project root and one and
// home dir and then merge them.
// 3. Try `.npmrc` in the user's home directory
if let Some(home_dir) = cache::home_dir() {
if let Some(home_dir) = sys_traits::impls::RealSys.env_home_dir() {
match try_to_read_npmrc(&home_dir) {
Ok(Some((source, path))) => {
return try_to_parse_npmrc(source, &path).map(|r| (r, Some(path)));
@ -821,7 +823,6 @@ impl CliOptions {
log::debug!("package.json auto-discovery is disabled");
}
WorkspaceDiscoverOptions {
fs: Default::default(), // use real fs
deno_json_cache: None,
pkg_json_cache: Some(&node_resolver::PackageJsonThreadLocalCache),
workspace_cache: None,
@ -843,6 +844,7 @@ impl CliOptions {
ConfigFlag::Discover => {
if let Some(start_paths) = flags.config_path_args(&initial_cwd) {
WorkspaceDirectory::discover(
&FsSysTraitsAdapter::new_real(),
WorkspaceDiscoverStart::Paths(&start_paths),
&resolve_workspace_discover_options(),
)?
@ -853,6 +855,7 @@ impl CliOptions {
ConfigFlag::Path(path) => {
let config_path = normalize_path(initial_cwd.join(path));
WorkspaceDirectory::discover(
&FsSysTraitsAdapter::new_real(),
WorkspaceDiscoverStart::ConfigFile(&config_path),
&resolve_workspace_discover_options(),
)?
@ -984,24 +987,24 @@ impl CliOptions {
// https://nodejs.org/api/process.html
match target.as_str() {
"aarch64-apple-darwin" => NpmSystemInfo {
os: "darwin".to_string(),
cpu: "arm64".to_string(),
os: "darwin".into(),
cpu: "arm64".into(),
},
"aarch64-unknown-linux-gnu" => NpmSystemInfo {
os: "linux".to_string(),
cpu: "arm64".to_string(),
os: "linux".into(),
cpu: "arm64".into(),
},
"x86_64-apple-darwin" => NpmSystemInfo {
os: "darwin".to_string(),
cpu: "x64".to_string(),
os: "darwin".into(),
cpu: "x64".into(),
},
"x86_64-unknown-linux-gnu" => NpmSystemInfo {
os: "linux".to_string(),
cpu: "x64".to_string(),
os: "linux".into(),
cpu: "x64".into(),
},
"x86_64-pc-windows-msvc" => NpmSystemInfo {
os: "win32".to_string(),
cpu: "x64".to_string(),
os: "win32".into(),
cpu: "x64".into(),
},
value => {
log::warn!(
@ -1396,9 +1399,9 @@ impl CliOptions {
Ok(DenoLintConfig {
default_jsx_factory: (!transpile_options.jsx_automatic)
.then(|| transpile_options.jsx_factory.clone()),
.then_some(transpile_options.jsx_factory),
default_jsx_fragment_factory: (!transpile_options.jsx_automatic)
.then(|| transpile_options.jsx_fragment_factory.clone()),
.then_some(transpile_options.jsx_fragment_factory),
})
}
@ -1981,15 +1984,17 @@ pub fn has_flag_env_var(name: &str) -> bool {
pub fn npm_pkg_req_ref_to_binary_command(
req_ref: &NpmPackageReqReference,
) -> String {
let binary_name = req_ref.sub_path().unwrap_or(req_ref.req().name.as_str());
binary_name.to_string()
req_ref
.sub_path()
.map(|s| s.to_string())
.unwrap_or_else(|| req_ref.req().name.to_string())
}
pub fn config_to_deno_graph_workspace_member(
config: &ConfigFile,
) -> Result<deno_graph::WorkspaceMember, AnyError> {
let name = match &config.json.name {
Some(name) => name.clone(),
let name: StackString = match &config.json.name {
Some(name) => name.as_str().into(),
None => bail!("Missing 'name' field in config file."),
};
let version = match &config.json.version {

31
cli/args/package_json.rs
View file

@ -11,19 +11,20 @@ use deno_package_json::PackageJsonDepValueParseError;
use deno_package_json::PackageJsonDepWorkspaceReq;
use deno_semver::npm::NpmPackageReqReference;
use deno_semver::package::PackageReq;
use deno_semver::StackString;
use deno_semver::VersionReq;
use thiserror::Error;
#[derive(Debug)]
pub struct InstallNpmRemotePkg {
pub alias: Option<String>,
pub alias: Option<StackString>,
pub base_dir: PathBuf,
pub req: PackageReq,
}
#[derive(Debug)]
pub struct InstallNpmWorkspacePkg {
pub alias: Option<String>,
pub alias: Option<StackString>,
pub target_dir: PathBuf,
}
@ -31,7 +32,7 @@ pub struct InstallNpmWorkspacePkg {
#[error("Failed to install '{}'\n at {}", alias, location)]
pub struct PackageJsonDepValueParseWithLocationError {
pub location: Url,
pub alias: String,
pub alias: StackString,
#[source]
pub source: PackageJsonDepValueParseError,
}
@ -100,10 +101,8 @@ impl NpmInstallDepsProvider {
let mut pkg_pkgs = Vec::with_capacity(
deps.dependencies.len() + deps.dev_dependencies.len(),
);
for (alias, dep) in deps
.dependencies
.into_iter()
.chain(deps.dev_dependencies.into_iter())
for (alias, dep) in
deps.dependencies.iter().chain(deps.dev_dependencies.iter())
{
let dep = match dep {
Ok(dep) => dep,
@ -111,8 +110,8 @@ impl NpmInstallDepsProvider {
pkg_json_dep_errors.push(
PackageJsonDepValueParseWithLocationError {
location: pkg_json.specifier(),
alias,
source: err,
alias: alias.clone(),
source: err.clone(),
},
);
continue;
@ -121,28 +120,28 @@ impl NpmInstallDepsProvider {
match dep {
PackageJsonDepValue::Req(pkg_req) => {
let workspace_pkg = workspace_npm_pkgs.iter().find(|pkg| {
pkg.matches_req(&pkg_req)
pkg.matches_req(pkg_req)
// do not resolve to the current package
&& pkg.pkg_json.path != pkg_json.path
});
if let Some(pkg) = workspace_pkg {
workspace_pkgs.push(InstallNpmWorkspacePkg {
alias: Some(alias),
alias: Some(alias.clone()),
target_dir: pkg.pkg_json.dir_path().to_path_buf(),
});
} else {
pkg_pkgs.push(InstallNpmRemotePkg {
alias: Some(alias),
alias: Some(alias.clone()),
base_dir: pkg_json.dir_path().to_path_buf(),
req: pkg_req,
req: pkg_req.clone(),
});
}
}
PackageJsonDepValue::Workspace(workspace_version_req) => {
let version_req = match workspace_version_req {
PackageJsonDepWorkspaceReq::VersionReq(version_req) => {
version_req
version_req.clone()
}
PackageJsonDepWorkspaceReq::Tilde
| PackageJsonDepWorkspaceReq::Caret => {
@ -150,10 +149,10 @@ impl NpmInstallDepsProvider {
}
};
if let Some(pkg) = workspace_npm_pkgs.iter().find(|pkg| {
pkg.matches_name_and_version_req(&alias, &version_req)
pkg.matches_name_and_version_req(alias, &version_req)
}) {
workspace_pkgs.push(InstallNpmWorkspacePkg {
alias: Some(alias),
alias: Some(alias.clone()),
target_dir: pkg.pkg_json.dir_path().to_path_buf(),
});
}

153
cli/cache/deno_dir.rs vendored
View file

@ -1,5 +1,6 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use deno_cache_dir::DenoDirResolutionError;
use once_cell::sync::OnceCell;
use super::DiskCache;
@ -11,7 +12,7 @@ use std::path::PathBuf;
/// where functionality wants to continue if the DENO_DIR can't be created.
pub struct DenoDirProvider {
maybe_custom_root: Option<PathBuf>,
deno_dir: OnceCell<std::io::Result<DenoDir>>,
deno_dir: OnceCell<Result<DenoDir, DenoDirResolutionError>>,
}
impl DenoDirProvider {
@ -22,12 +23,21 @@ impl DenoDirProvider {
}
}
pub fn get_or_create(&self) -> Result<&DenoDir, std::io::Error> {
pub fn get_or_create(&self) -> Result<&DenoDir, DenoDirResolutionError> {
self
.deno_dir
.get_or_init(|| DenoDir::new(self.maybe_custom_root.clone()))
.as_ref()
.map_err(|err| std::io::Error::new(err.kind(), err.to_string()))
.map_err(|err| match err {
DenoDirResolutionError::NoCacheOrHomeDir => {
DenoDirResolutionError::NoCacheOrHomeDir
}
DenoDirResolutionError::FailedCwd { source } => {
DenoDirResolutionError::FailedCwd {
source: std::io::Error::new(source.kind(), source.to_string()),
}
}
})
}
}
@ -42,27 +52,13 @@ pub struct DenoDir {
}
impl DenoDir {
pub fn new(maybe_custom_root: Option<PathBuf>) -> std::io::Result<Self> {
let maybe_custom_root =
maybe_custom_root.or_else(|| env::var("DENO_DIR").map(String::into).ok());
let root: PathBuf = if let Some(root) = maybe_custom_root {
root
} else if let Some(cache_dir) = dirs::cache_dir() {
// We use the OS cache dir because all files deno writes are cache files
// Once that changes we need to start using different roots if DENO_DIR
// is not set, and keep a single one if it is.
cache_dir.join("deno")
} else if let Some(home_dir) = dirs::home_dir() {
// fallback path
home_dir.join(".deno")
} else {
panic!("Could not set the Deno root directory")
};
let root = if root.is_absolute() {
root
} else {
std::env::current_dir()?.join(root)
};
pub fn new(
maybe_custom_root: Option<PathBuf>,
) -> Result<Self, deno_cache_dir::DenoDirResolutionError> {
let root = deno_cache_dir::resolve_deno_dir(
&sys_traits::impls::RealSys,
maybe_custom_root,
)?;
assert!(root.is_absolute());
let gen_path = root.join("gen");
@ -166,112 +162,3 @@ impl DenoDir {
self.root.join("dl")
}
}
/// To avoid the poorly managed dirs crate
#[cfg(not(windows))]
pub mod dirs {
use std::path::PathBuf;
pub fn cache_dir() -> Option<PathBuf> {
if cfg!(target_os = "macos") {
home_dir().map(|h| h.join("Library/Caches"))
} else {
std::env::var_os("XDG_CACHE_HOME")
.map(PathBuf::from)
.or_else(|| home_dir().map(|h| h.join(".cache")))
}
}
pub fn home_dir() -> Option<PathBuf> {
std::env::var_os("HOME")
.and_then(|h| if h.is_empty() { None } else { Some(h) })
.or_else(|| {
// TODO(bartlomieju):
#[allow(clippy::undocumented_unsafe_blocks)]
unsafe {
fallback()
}
})
.map(PathBuf::from)
}
// This piece of code is taken from the deprecated home_dir() function in Rust's standard library: https://github.com/rust-lang/rust/blob/master/src/libstd/sys/unix/os.rs#L579
// The same code is used by the dirs crate
unsafe fn fallback() -> Option<std::ffi::OsString> {
let amt = match libc::sysconf(libc::_SC_GETPW_R_SIZE_MAX) {
n if n < 0 => 512_usize,
n => n as usize,
};
let mut buf = Vec::with_capacity(amt);
let mut passwd: libc::passwd = std::mem::zeroed();
let mut result = std::ptr::null_mut();
match libc::getpwuid_r(
libc::getuid(),
&mut passwd,
buf.as_mut_ptr(),
buf.capacity(),
&mut result,
) {
0 if !result.is_null() => {
let ptr = passwd.pw_dir as *const _;
let bytes = std::ffi::CStr::from_ptr(ptr).to_bytes().to_vec();
Some(std::os::unix::ffi::OsStringExt::from_vec(bytes))
}
_ => None,
}
}
}
/// To avoid the poorly managed dirs crate
// Copied from
// https://github.com/dirs-dev/dirs-sys-rs/blob/ec7cee0b3e8685573d847f0a0f60aae3d9e07fa2/src/lib.rs#L140-L164
// MIT license. Copyright (c) 2018-2019 dirs-rs contributors
#[cfg(windows)]
pub mod dirs {
use std::ffi::OsString;
use std::os::windows::ffi::OsStringExt;
use std::path::PathBuf;
use winapi::shared::winerror;
use winapi::um::combaseapi;
use winapi::um::knownfolders;
use winapi::um::shlobj;
use winapi::um::shtypes;
use winapi::um::winbase;
use winapi::um::winnt;
fn known_folder(folder_id: shtypes::REFKNOWNFOLDERID) -> Option<PathBuf> {
// SAFETY: winapi calls
unsafe {
let mut path_ptr: winnt::PWSTR = std::ptr::null_mut();
let result = shlobj::SHGetKnownFolderPath(
folder_id,
0,
std::ptr::null_mut(),
&mut path_ptr,
);
if result == winerror::S_OK {
let len = winbase::lstrlenW(path_ptr) as usize;
let path = std::slice::from_raw_parts(path_ptr, len);
let ostr: OsString = OsStringExt::from_wide(path);
combaseapi::CoTaskMemFree(path_ptr as *mut winapi::ctypes::c_void);
Some(PathBuf::from(ostr))
} else {
None
}
}
}
pub fn cache_dir() -> Option<PathBuf> {
known_folder(&knownfolders::FOLDERID_LocalAppData)
}
pub fn home_dir() -> Option<PathBuf> {
if let Some(userprofile) = std::env::var_os("USERPROFILE") {
if !userprofile.is_empty() {
return Some(PathBuf::from(userprofile));
}
}
known_folder(&knownfolders::FOLDERID_Profile)
}
}

10
cli/cache/disk_cache.rs vendored
View file

@ -1,11 +1,12 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use super::CACHE_PERM;
use crate::util::fs::atomic_write_file_with_retries;
use deno_cache_dir::url_to_filename;
use deno_core::url::Host;
use deno_core::url::Url;
use deno_path_util::fs::atomic_write_file_with_retries;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use std::ffi::OsStr;
use std::fs;
use std::path::Component;
@ -120,7 +121,12 @@ impl DiskCache {
pub fn set(&self, filename: &Path, data: &[u8]) -> std::io::Result<()> {
let path = self.location.join(filename);
atomic_write_file_with_retries(&path, data, CACHE_PERM)
atomic_write_file_with_retries(
&FsSysTraitsAdapter::new_real(),
&path,
data,
CACHE_PERM,
)
}
}

137
cli/cache/mod.rs vendored
View file

@ -5,9 +5,6 @@ use crate::file_fetcher::CliFetchNoFollowErrorKind;
use crate::file_fetcher::CliFileFetcher;
use crate::file_fetcher::FetchNoFollowOptions;
use crate::file_fetcher::FetchPermissionsOptionRef;
use crate::util::fs::atomic_write_file_with_retries;
use crate::util::fs::atomic_write_file_with_retries_and_fs;
use crate::util::fs::AtomicWriteFileFsAdapter;
use deno_ast::MediaType;
use deno_cache_dir::file_fetcher::CacheSetting;
@ -21,15 +18,12 @@ use deno_graph::source::CacheInfo;
use deno_graph::source::LoadFuture;
use deno_graph::source::LoadResponse;
use deno_graph::source::Loader;
use deno_runtime::deno_fs;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use deno_runtime::deno_permissions::PermissionsContainer;
use node_resolver::InNpmPackageChecker;
use std::borrow::Cow;
use std::collections::HashMap;
use std::path::Path;
use std::path::PathBuf;
use std::sync::Arc;
use std::time::SystemTime;
mod cache_db;
mod caches;
@ -50,7 +44,6 @@ pub use caches::Caches;
pub use check::TypeCheckCache;
pub use code_cache::CodeCache;
pub use common::FastInsecureHasher;
pub use deno_dir::dirs::home_dir;
pub use deno_dir::DenoDir;
pub use deno_dir::DenoDirProvider;
pub use disk_cache::DiskCache;
@ -63,121 +56,12 @@ pub use parsed_source::LazyGraphSourceParser;
pub use parsed_source::ParsedSourceCache;
/// Permissions used to save a file in the disk caches.
pub const CACHE_PERM: u32 = 0o644;
pub use deno_cache_dir::CACHE_PERM;
#[derive(Debug, Clone)]
pub struct RealDenoCacheEnv;
impl deno_cache_dir::DenoCacheEnv for RealDenoCacheEnv {
fn read_file_bytes(
&self,
path: &Path,
) -> std::io::Result<Cow<'static, [u8]>> {
std::fs::read(path).map(Cow::Owned)
}
fn atomic_write_file(
&self,
path: &Path,
bytes: &[u8],
) -> std::io::Result<()> {
atomic_write_file_with_retries(path, bytes, CACHE_PERM)
}
fn canonicalize_path(&self, path: &Path) -> std::io::Result<PathBuf> {
crate::util::fs::canonicalize_path(path)
}
fn create_dir_all(&self, path: &Path) -> std::io::Result<()> {
std::fs::create_dir_all(path)
}
fn modified(&self, path: &Path) -> std::io::Result<Option<SystemTime>> {
match std::fs::metadata(path) {
Ok(metadata) => Ok(Some(
metadata.modified().unwrap_or_else(|_| SystemTime::now()),
)),
Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(None),
Err(err) => Err(err),
}
}
fn is_file(&self, path: &Path) -> bool {
path.is_file()
}
fn time_now(&self) -> SystemTime {
SystemTime::now()
}
}
#[derive(Debug, Clone)]
pub struct DenoCacheEnvFsAdapter<'a>(
pub &'a dyn deno_runtime::deno_fs::FileSystem,
);
impl<'a> deno_cache_dir::DenoCacheEnv for DenoCacheEnvFsAdapter<'a> {
fn read_file_bytes(
&self,
path: &Path,
) -> std::io::Result<Cow<'static, [u8]>> {
self
.0
.read_file_sync(path, None)
.map_err(|err| err.into_io_error())
}
fn atomic_write_file(
&self,
path: &Path,
bytes: &[u8],
) -> std::io::Result<()> {
atomic_write_file_with_retries_and_fs(
&AtomicWriteFileFsAdapter {
fs: self.0,
write_mode: CACHE_PERM,
},
path,
bytes,
)
}
fn canonicalize_path(&self, path: &Path) -> std::io::Result<PathBuf> {
self.0.realpath_sync(path).map_err(|e| e.into_io_error())
}
fn create_dir_all(&self, path: &Path) -> std::io::Result<()> {
self
.0
.mkdir_sync(path, true, None)
.map_err(|e| e.into_io_error())
}
fn modified(&self, path: &Path) -> std::io::Result<Option<SystemTime>> {
self
.0
.stat_sync(path)
.map(|stat| {
stat
.mtime
.map(|ts| SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(ts))
})
.map_err(|e| e.into_io_error())
}
fn is_file(&self, path: &Path) -> bool {
self.0.is_file_sync(path)
}
fn time_now(&self) -> SystemTime {
SystemTime::now()
}
}
pub type GlobalHttpCache = deno_cache_dir::GlobalHttpCache<RealDenoCacheEnv>;
pub type LocalHttpCache = deno_cache_dir::LocalHttpCache<RealDenoCacheEnv>;
pub type GlobalHttpCache = deno_cache_dir::GlobalHttpCache<FsSysTraitsAdapter>;
pub type LocalHttpCache = deno_cache_dir::LocalHttpCache<FsSysTraitsAdapter>;
pub type LocalLspHttpCache =
deno_cache_dir::LocalLspHttpCache<RealDenoCacheEnv>;
deno_cache_dir::LocalLspHttpCache<FsSysTraitsAdapter>;
pub use deno_cache_dir::HttpCache;
pub struct FetchCacherOptions {
@ -192,11 +76,11 @@ pub struct FetchCacherOptions {
pub struct FetchCacher {
pub file_header_overrides: HashMap<ModuleSpecifier, HashMap<String, String>>,
file_fetcher: Arc<CliFileFetcher>,
fs: Arc<dyn deno_fs::FileSystem>,
global_http_cache: Arc<GlobalHttpCache>,
in_npm_pkg_checker: Arc<dyn InNpmPackageChecker>,
module_info_cache: Arc<ModuleInfoCache>,
permissions: PermissionsContainer,
sys: FsSysTraitsAdapter,
is_deno_publish: bool,
cache_info_enabled: bool,
}
@ -204,18 +88,18 @@ pub struct FetchCacher {
impl FetchCacher {
pub fn new(
file_fetcher: Arc<CliFileFetcher>,
fs: Arc<dyn deno_fs::FileSystem>,
global_http_cache: Arc<GlobalHttpCache>,
in_npm_pkg_checker: Arc<dyn InNpmPackageChecker>,
module_info_cache: Arc<ModuleInfoCache>,
sys: FsSysTraitsAdapter,
options: FetchCacherOptions,
) -> Self {
Self {
file_fetcher,
fs,
global_http_cache,
in_npm_pkg_checker,
module_info_cache,
sys,
file_header_overrides: options.file_header_overrides,
permissions: options.permissions,
is_deno_publish: options.is_deno_publish,
@ -277,9 +161,8 @@ impl Loader for FetchCacher {
// symlinked to `/my-project-2/node_modules`), so first we checked if the path
// is in a node_modules dir to avoid needlessly canonicalizing, then now compare
// against the canonicalized specifier.
let specifier = crate::node::resolve_specifier_into_node_modules(
specifier,
self.fs.as_ref(),
let specifier = node_resolver::resolve_specifier_into_node_modules(
&self.sys, specifier,
);
if self.in_npm_pkg_checker.in_npm_package(&specifier) {
return Box::pin(futures::future::ready(Ok(Some(

35
cli/emit.rs
View file

@ -5,6 +5,7 @@ use crate::cache::FastInsecureHasher;
use crate::cache::ParsedSourceCache;
use crate::resolver::CjsTracker;
use deno_ast::EmittedSourceText;
use deno_ast::ModuleKind;
use deno_ast::SourceMapOption;
use deno_ast::SourceRange;
@ -132,6 +133,7 @@ impl Emitter {
&transpile_and_emit_options.0,
&transpile_and_emit_options.1,
)
.map(|r| r.text)
}
})
.await
@ -166,7 +168,8 @@ impl Emitter {
source.clone(),
&self.transpile_and_emit_options.0,
&self.transpile_and_emit_options.1,
)?;
)?
.text;
helper.post_emit_parsed_source(
specifier,
&transpiled_source,
@ -177,6 +180,31 @@ impl Emitter {
}
}
pub fn emit_parsed_source_for_deno_compile(
&self,
specifier: &ModuleSpecifier,
media_type: MediaType,
module_kind: deno_ast::ModuleKind,
source: &Arc<str>,
) -> Result<(String, String), AnyError> {
let mut emit_options = self.transpile_and_emit_options.1.clone();
emit_options.inline_sources = false;
emit_options.source_map = SourceMapOption::Separate;
// strip off the path to have more deterministic builds as we don't care
// about the source name because we manually provide the source map to v8
emit_options.source_map_base = Some(deno_path_util::url_parent(specifier));
let source = EmitParsedSourceHelper::transpile(
&self.parsed_source_cache,
specifier,
media_type,
module_kind,
source.clone(),
&self.transpile_and_emit_options.0,
&emit_options,
)?;
Ok((source.text, source.source_map.unwrap()))
}
/// Expects a file URL, panics otherwise.
pub async fn load_and_emit_for_hmr(
&self,
@ -282,7 +310,7 @@ impl<'a> EmitParsedSourceHelper<'a> {
source: Arc<str>,
transpile_options: &deno_ast::TranspileOptions,
emit_options: &deno_ast::EmitOptions,
) -> Result<String, AnyError> {
) -> Result<EmittedSourceText, AnyError> {
// nothing else needs the parsed source at this point, so remove from
// the cache in order to not transpile owned
let parsed_source = parsed_source_cache
@ -302,8 +330,7 @@ impl<'a> EmitParsedSourceHelper<'a> {
source
}
};
debug_assert!(transpiled_source.source_map.is_none());
Ok(transpiled_source.text)
Ok(transpiled_source)
}
pub fn post_emit_parsed_source(

40
cli/factory.rs
View file

@ -12,7 +12,6 @@ use crate::args::StorageKeyResolver;
use crate::args::TsConfigType;
use crate::cache::Caches;
use crate::cache::CodeCache;
use crate::cache::DenoCacheEnvFsAdapter;
use crate::cache::DenoDir;
use crate::cache::DenoDirProvider;
use crate::cache::EmitCache;
@ -44,7 +43,6 @@ use crate::npm::CliNpmResolverManagedSnapshotOption;
use crate::npm::CreateInNpmPkgCheckerOptions;
use crate::resolver::CjsTracker;
use crate::resolver::CliDenoResolver;
use crate::resolver::CliDenoResolverFs;
use crate::resolver::CliNpmReqResolver;
use crate::resolver::CliResolver;
use crate::resolver::CliResolverOptions;
@ -88,9 +86,10 @@ use deno_resolver::npm::NpmReqResolverOptions;
use deno_resolver::DenoResolverOptions;
use deno_resolver::NodeAndNpmReqResolver;
use deno_runtime::deno_fs;
use deno_runtime::deno_node::DenoFsNodeResolverEnv;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use deno_runtime::deno_node::NodeResolver;
use deno_runtime::deno_node::PackageJsonResolver;
use deno_runtime::deno_node::RealIsBuiltInNodeModuleChecker;
use deno_runtime::deno_permissions::Permissions;
use deno_runtime::deno_permissions::PermissionsContainer;
use deno_runtime::deno_permissions::PermissionsOptions;
@ -331,8 +330,8 @@ impl CliFactory {
pub fn global_http_cache(&self) -> Result<&Arc<GlobalHttpCache>, AnyError> {
self.services.global_http_cache.get_or_try_init(|| {
Ok(Arc::new(GlobalHttpCache::new(
FsSysTraitsAdapter(self.fs().clone()),
self.deno_dir()?.remote_folder_path(),
crate::cache::RealDenoCacheEnv,
)))
})
}
@ -409,7 +408,7 @@ impl CliFactory {
let global_path = self.deno_dir()?.npm_folder_path();
let cli_options = self.cli_options()?;
Ok(Arc::new(NpmCacheDir::new(
&DenoCacheEnvFsAdapter(fs.as_ref()),
&FsSysTraitsAdapter(fs.clone()),
global_path,
cli_options.npmrc().get_all_known_registries_urls(),
)))
@ -429,7 +428,7 @@ impl CliFactory {
create_cli_npm_resolver(if cli_options.use_byonm() {
CliNpmResolverCreateOptions::Byonm(
CliByonmNpmResolverCreateOptions {
fs: CliDenoResolverFs(fs.clone()),
sys: FsSysTraitsAdapter(fs.clone()),
pkg_json_resolver: self.pkg_json_resolver().clone(),
root_node_modules_dir: Some(
match cli_options.node_modules_dir_path() {
@ -447,6 +446,13 @@ impl CliFactory {
} else {
CliNpmResolverCreateOptions::Managed(
CliManagedNpmResolverCreateOptions {
http_client_provider: self.http_client_provider().clone(),
npm_install_deps_provider: Arc::new(
NpmInstallDepsProvider::from_workspace(
cli_options.workspace(),
),
),
sys: FsSysTraitsAdapter(self.fs().clone()),
snapshot: match cli_options.resolve_npm_resolution_snapshot()? {
Some(snapshot) => {
CliNpmResolverManagedSnapshotOption::Specified(Some(
@ -465,19 +471,12 @@ impl CliFactory {
},
},
maybe_lockfile: cli_options.maybe_lockfile().cloned(),
fs: fs.clone(),
http_client_provider: self.http_client_provider().clone(),
npm_cache_dir: self.npm_cache_dir()?.clone(),
cache_setting: cli_options.cache_setting(),
text_only_progress_bar: self.text_only_progress_bar().clone(),
maybe_node_modules_path: cli_options
.node_modules_dir_path()
.cloned(),
npm_install_deps_provider: Arc::new(
NpmInstallDepsProvider::from_workspace(
cli_options.workspace(),
),
),
npm_system_info: cli_options.npm_system_info(),
npmrc: cli_options.npmrc().clone(),
lifecycle_scripts: cli_options.lifecycle_scripts_config(),
@ -500,7 +499,7 @@ impl CliFactory {
.get_or_try_init(|| {
Ok(self.cli_options()?.unstable_sloppy_imports().then(|| {
Arc::new(CliSloppyImportsResolver::new(SloppyImportsCachedFs::new(
self.fs().clone(),
FsSysTraitsAdapter(self.fs().clone()),
)))
}))
})
@ -668,14 +667,15 @@ impl CliFactory {
.get_or_try_init_async(
async {
Ok(Arc::new(NodeResolver::new(
DenoFsNodeResolverEnv::new(self.fs().clone()),
self.in_npm_pkg_checker()?.clone(),
RealIsBuiltInNodeModuleChecker,
self
.npm_resolver()
.await?
.clone()
.into_npm_pkg_folder_resolver(),
self.pkg_json_resolver().clone(),
FsSysTraitsAdapter(self.fs().clone()),
)))
}
.boxed_local(),
@ -703,7 +703,6 @@ impl CliFactory {
Ok(Arc::new(NodeCodeTranslator::new(
cjs_esm_analyzer,
DenoFsNodeResolverEnv::new(self.fs().clone()),
self.in_npm_pkg_checker()?.clone(),
node_resolver,
self
@ -712,6 +711,7 @@ impl CliFactory {
.clone()
.into_npm_pkg_folder_resolver(),
self.pkg_json_resolver().clone(),
FsSysTraitsAdapter(self.fs().clone()),
)))
})
.await
@ -727,7 +727,7 @@ impl CliFactory {
let npm_resolver = self.npm_resolver().await?;
Ok(Arc::new(CliNpmReqResolver::new(NpmReqResolverOptions {
byonm_resolver: (npm_resolver.clone()).into_maybe_byonm(),
fs: CliDenoResolverFs(self.fs().clone()),
sys: FsSysTraitsAdapter(self.fs().clone()),
in_npm_pkg_checker: self.in_npm_pkg_checker()?.clone(),
node_resolver: self.node_resolver().await?.clone(),
npm_req_resolver: npm_resolver.clone().into_npm_req_resolver(),
@ -738,7 +738,7 @@ impl CliFactory {
pub fn pkg_json_resolver(&self) -> &Arc<PackageJsonResolver> {
self.services.pkg_json_resolver.get_or_init(|| {
Arc::new(PackageJsonResolver::new(DenoFsNodeResolverEnv::new(
Arc::new(PackageJsonResolver::new(FsSysTraitsAdapter(
self.fs().clone(),
)))
})
@ -778,7 +778,6 @@ impl CliFactory {
self.cjs_tracker()?.clone(),
cli_options.clone(),
self.file_fetcher()?.clone(),
self.fs().clone(),
self.global_http_cache()?.clone(),
self.in_npm_pkg_checker()?.clone(),
cli_options.maybe_lockfile().cloned(),
@ -788,6 +787,7 @@ impl CliFactory {
self.parsed_source_cache().clone(),
self.resolver().await?.clone(),
self.root_permissions_container()?.clone(),
FsSysTraitsAdapter(self.fs().clone()),
)))
})
.await
@ -973,7 +973,6 @@ impl CliFactory {
None
},
self.emitter()?.clone(),
fs.clone(),
in_npm_pkg_checker.clone(),
self.main_module_graph_container().await?.clone(),
self.module_load_preparer().await?.clone(),
@ -988,6 +987,7 @@ impl CliFactory {
),
self.parsed_source_cache().clone(),
self.resolver().await?.clone(),
FsSysTraitsAdapter(self.fs().clone()),
)),
node_resolver.clone(),
npm_resolver.clone(),

55
cli/file_fetcher.rs
View file

@ -1,11 +1,8 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use crate::cache::HttpCache;
use crate::cache::RealDenoCacheEnv;
use crate::colors;
use crate::http_util::get_response_body_with_progress;
use crate::http_util::HttpClientProvider;
use crate::util::progress_bar::ProgressBar;
use std::borrow::Cow;
use std::collections::HashMap;
use std::sync::Arc;
use boxed_error::Boxed;
use deno_ast::MediaType;
@ -27,7 +24,7 @@ use deno_core::url::Url;
use deno_core::ModuleSpecifier;
use deno_error::JsError;
use deno_graph::source::LoaderChecksum;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use deno_runtime::deno_permissions::CheckSpecifierKind;
use deno_runtime::deno_permissions::PermissionCheckError;
use deno_runtime::deno_permissions::PermissionsContainer;
@ -35,12 +32,14 @@ use deno_runtime::deno_web::BlobStore;
use http::header;
use http::HeaderMap;
use http::StatusCode;
use std::borrow::Cow;
use std::collections::HashMap;
use std::env;
use std::sync::Arc;
use thiserror::Error;
use crate::cache::HttpCache;
use crate::colors;
use crate::http_util::get_response_body_with_progress;
use crate::http_util::HttpClientProvider;
use crate::util::progress_bar::ProgressBar;
#[derive(Debug, Clone, Eq, PartialEq)]
pub struct TextDecodedFile {
pub media_type: MediaType,
@ -268,7 +267,7 @@ pub struct FetchNoFollowOptions<'a> {
type DenoCacheDirFileFetcher = deno_cache_dir::file_fetcher::FileFetcher<
BlobStoreAdapter,
RealDenoCacheEnv,
FsSysTraitsAdapter,
HttpClientAdapter,
>;
@ -290,9 +289,11 @@ impl CliFileFetcher {
download_log_level: log::Level,
) -> Self {
let memory_files = Arc::new(MemoryFiles::default());
let sys = FsSysTraitsAdapter::new_real();
let auth_tokens = AuthTokens::new_from_sys(&sys);
let file_fetcher = DenoCacheDirFileFetcher::new(
BlobStoreAdapter(blob_store),
RealDenoCacheEnv,
sys,
http_cache,
HttpClientAdapter {
http_client_provider: http_client_provider.clone(),
@ -303,7 +304,7 @@ impl CliFileFetcher {
FileFetcherOptions {
allow_remote,
cache_setting,
auth_tokens: AuthTokens::new(env::var("DENO_AUTH_TOKENS").ok()),
auth_tokens,
},
);
Self {
@ -497,7 +498,6 @@ fn validate_scheme(specifier: &Url) -> Result<(), UnsupportedSchemeError> {
#[cfg(test)]
mod tests {
use crate::cache::GlobalHttpCache;
use crate::cache::RealDenoCacheEnv;
use crate::http_util::HttpClientProvider;
use super::*;
@ -538,7 +538,10 @@ mod tests {
let temp_dir = maybe_temp_dir.unwrap_or_default();
let location = temp_dir.path().join("remote").to_path_buf();
let blob_store: Arc<BlobStore> = Default::default();
let cache = Arc::new(GlobalHttpCache::new(location, RealDenoCacheEnv));
let cache = Arc::new(GlobalHttpCache::new(
FsSysTraitsAdapter::new_real(),
location,
));
let file_fetcher = CliFileFetcher::new(
cache.clone(),
Arc::new(HttpClientProvider::new(None, None)),
@ -752,8 +755,8 @@ mod tests {
let location = temp_dir.path().join("remote").to_path_buf();
let file_fetcher = CliFileFetcher::new(
Arc::new(GlobalHttpCache::new(
FsSysTraitsAdapter::new_real(),
location,
crate::cache::RealDenoCacheEnv,
)),
Arc::new(HttpClientProvider::new(None, None)),
Default::default(),
@ -781,8 +784,8 @@ mod tests {
resolve_url("http://localhost:4545/subdir/mismatch_ext.ts").unwrap();
let http_cache = Arc::new(GlobalHttpCache::new(
FsSysTraitsAdapter::new_real(),
location.clone(),
crate::cache::RealDenoCacheEnv,
));
let file_modified_01 = {
let file_fetcher = CliFileFetcher::new(
@ -808,8 +811,8 @@ mod tests {
let file_modified_02 = {
let file_fetcher = CliFileFetcher::new(
Arc::new(GlobalHttpCache::new(
FsSysTraitsAdapter::new_real(),
location,
crate::cache::RealDenoCacheEnv,
)),
Arc::new(HttpClientProvider::new(None, None)),
Default::default(),
@ -938,8 +941,8 @@ mod tests {
let redirected_specifier =
resolve_url("http://localhost:4546/subdir/mismatch_ext.ts").unwrap();
let http_cache = Arc::new(GlobalHttpCache::new(
FsSysTraitsAdapter::new_real(),
location.clone(),
crate::cache::RealDenoCacheEnv,
));
let metadata_file_modified_01 = {
@ -1073,8 +1076,8 @@ mod tests {
let location = temp_dir.path().join("remote").to_path_buf();
let file_fetcher = CliFileFetcher::new(
Arc::new(GlobalHttpCache::new(
FsSysTraitsAdapter::new_real(),
location,
crate::cache::RealDenoCacheEnv,
)),
Arc::new(HttpClientProvider::new(None, None)),
Default::default(),
@ -1110,7 +1113,10 @@ mod tests {
let temp_dir = TempDir::new();
let location = temp_dir.path().join("remote").to_path_buf();
let file_fetcher_01 = CliFileFetcher::new(
Arc::new(GlobalHttpCache::new(location.clone(), RealDenoCacheEnv)),
Arc::new(GlobalHttpCache::new(
FsSysTraitsAdapter::new_real(),
location.clone(),
)),
Arc::new(HttpClientProvider::new(None, None)),
Default::default(),
None,
@ -1119,7 +1125,10 @@ mod tests {
log::Level::Info,
);
let file_fetcher_02 = CliFileFetcher::new(
Arc::new(GlobalHttpCache::new(location, RealDenoCacheEnv)),
Arc::new(GlobalHttpCache::new(
FsSysTraitsAdapter::new_real(),
location,
)),
Arc::new(HttpClientProvider::new(None, None)),
Default::default(),
None,

161
cli/graph_util.rs
View file

@ -1,5 +1,41 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use std::error::Error;
use std::ops::Deref;
use std::path::PathBuf;
use std::sync::Arc;
use deno_config::deno_json::JsxImportSourceConfig;
use deno_config::workspace::JsrPackageConfig;
use deno_core::anyhow::bail;
use deno_core::error::custom_error;
use deno_core::error::AnyError;
use deno_core::parking_lot::Mutex;
use deno_core::ModuleSpecifier;
use deno_graph::source::Loader;
use deno_graph::source::LoaderChecksum;
use deno_graph::source::ResolutionKind;
use deno_graph::source::ResolveError;
use deno_graph::FillFromLockfileOptions;
use deno_graph::GraphKind;
use deno_graph::JsrLoadError;
use deno_graph::ModuleError;
use deno_graph::ModuleGraph;
use deno_graph::ModuleGraphError;
use deno_graph::ModuleLoadError;
use deno_graph::ResolutionError;
use deno_graph::SpecifierError;
use deno_graph::WorkspaceFastCheckOption;
use deno_resolver::sloppy_imports::SloppyImportsResolutionKind;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use deno_runtime::deno_node;
use deno_runtime::deno_permissions::PermissionsContainer;
use deno_semver::jsr::JsrDepPackageReq;
use deno_semver::package::PackageNv;
use deno_semver::SmallStackString;
use import_map::ImportMapError;
use node_resolver::InNpmPackageChecker;
use crate::args::config_to_deno_graph_workspace_member;
use crate::args::jsr_url;
use crate::args::CliLockfile;
@ -22,40 +58,6 @@ use crate::resolver::SloppyImportsCachedFs;
use crate::tools::check;
use crate::tools::check::TypeChecker;
use crate::util::file_watcher::WatcherCommunicator;
use deno_config::deno_json::JsxImportSourceConfig;
use deno_config::workspace::JsrPackageConfig;
use deno_core::anyhow::bail;
use deno_graph::source::LoaderChecksum;
use deno_graph::source::ResolutionKind;
use deno_graph::FillFromLockfileOptions;
use deno_graph::JsrLoadError;
use deno_graph::ModuleLoadError;
use deno_graph::WorkspaceFastCheckOption;
use deno_core::error::custom_error;
use deno_core::error::AnyError;
use deno_core::parking_lot::Mutex;
use deno_core::ModuleSpecifier;
use deno_graph::source::Loader;
use deno_graph::source::ResolveError;
use deno_graph::GraphKind;
use deno_graph::ModuleError;
use deno_graph::ModuleGraph;
use deno_graph::ModuleGraphError;
use deno_graph::ResolutionError;
use deno_graph::SpecifierError;
use deno_resolver::sloppy_imports::SloppyImportsResolutionKind;
use deno_runtime::deno_fs::FileSystem;
use deno_runtime::deno_node;
use deno_runtime::deno_permissions::PermissionsContainer;
use deno_semver::jsr::JsrDepPackageReq;
use deno_semver::package::PackageNv;
use import_map::ImportMapError;
use node_resolver::InNpmPackageChecker;
use std::error::Error;
use std::ops::Deref;
use std::path::PathBuf;
use std::sync::Arc;
#[derive(Clone)]
pub struct GraphValidOptions {
@ -76,7 +78,7 @@ pub struct GraphValidOptions {
/// for the CLI.
pub fn graph_valid(
graph: &ModuleGraph,
fs: &Arc<dyn FileSystem>,
sys: &FsSysTraitsAdapter,
roots: &[ModuleSpecifier],
options: GraphValidOptions,
) -> Result<(), AnyError> {
@ -86,7 +88,7 @@ pub fn graph_valid(
let mut errors = graph_walk_errors(
graph,
fs,
sys,
roots,
GraphWalkErrorsOptions {
check_js: options.check_js,
@ -136,7 +138,7 @@ pub struct GraphWalkErrorsOptions {
/// and enhances them with CLI information.
pub fn graph_walk_errors<'a>(
graph: &'a ModuleGraph,
fs: &'a Arc<dyn FileSystem>,
sys: &'a FsSysTraitsAdapter,
roots: &'a [ModuleSpecifier],
options: GraphWalkErrorsOptions,
) -> impl Iterator<Item = AnyError> + 'a {
@ -171,7 +173,7 @@ pub fn graph_walk_errors<'a>(
}
ModuleGraphError::ModuleError(error) => {
enhanced_integrity_error_message(error)
.or_else(|| enhanced_sloppy_imports_error_message(fs, error))
.or_else(|| enhanced_sloppy_imports_error_message(sys, error))
.unwrap_or_else(|| format_deno_graph_error(error))
}
};
@ -430,7 +432,6 @@ pub struct ModuleGraphBuilder {
cjs_tracker: Arc<CjsTracker>,
cli_options: Arc<CliOptions>,
file_fetcher: Arc<CliFileFetcher>,
fs: Arc<dyn FileSystem>,
global_http_cache: Arc<GlobalHttpCache>,
in_npm_pkg_checker: Arc<dyn InNpmPackageChecker>,
lockfile: Option<Arc<CliLockfile>>,
@ -440,6 +441,7 @@ pub struct ModuleGraphBuilder {
parsed_source_cache: Arc<ParsedSourceCache>,
resolver: Arc<CliResolver>,
root_permissions_container: PermissionsContainer,
sys: FsSysTraitsAdapter,
}
impl ModuleGraphBuilder {
@ -449,7 +451,6 @@ impl ModuleGraphBuilder {
cjs_tracker: Arc<CjsTracker>,
cli_options: Arc<CliOptions>,
file_fetcher: Arc<CliFileFetcher>,
fs: Arc<dyn FileSystem>,
global_http_cache: Arc<GlobalHttpCache>,
in_npm_pkg_checker: Arc<dyn InNpmPackageChecker>,
lockfile: Option<Arc<CliLockfile>>,
@ -459,13 +460,13 @@ impl ModuleGraphBuilder {
parsed_source_cache: Arc<ParsedSourceCache>,
resolver: Arc<CliResolver>,
root_permissions_container: PermissionsContainer,
sys: FsSysTraitsAdapter,
) -> Self {
Self {
caches,
cjs_tracker,
cli_options,
file_fetcher,
fs,
global_http_cache,
in_npm_pkg_checker,
lockfile,
@ -475,6 +476,7 @@ impl ModuleGraphBuilder {
parsed_source_cache,
resolver,
root_permissions_container,
sys,
}
}
@ -590,7 +592,7 @@ impl ModuleGraphBuilder {
is_dynamic: options.is_dynamic,
passthrough_jsr_specifiers: false,
executor: Default::default(),
file_system: &DenoGraphFsAdapter(self.fs.as_ref()),
file_system: &self.sys,
jsr_url_provider: &CliJsrUrlProvider,
npm_resolver: Some(&graph_npm_resolver),
module_analyzer: &analyzer,
@ -678,7 +680,7 @@ impl ModuleGraphBuilder {
for (from, to) in graph.packages.mappings() {
lockfile.insert_package_specifier(
JsrDepPackageReq::jsr(from.clone()),
to.version.to_string(),
to.version.to_custom_string::<SmallStackString>(),
);
}
}
@ -744,10 +746,10 @@ impl ModuleGraphBuilder {
) -> cache::FetchCacher {
cache::FetchCacher::new(
self.file_fetcher.clone(),
self.fs.clone(),
self.global_http_cache.clone(),
self.in_npm_pkg_checker.clone(),
self.module_info_cache.clone(),
self.sys.clone(),
cache::FetchCacherOptions {
file_header_overrides: self.cli_options.resolve_file_header_overrides(),
permissions,
@ -776,7 +778,7 @@ impl ModuleGraphBuilder {
) -> Result<(), AnyError> {
graph_valid(
graph,
&self.fs,
&self.sys,
roots,
GraphValidOptions {
kind: if self.cli_options.type_check_mode().is_true() {
@ -832,13 +834,13 @@ pub fn enhanced_resolution_error_message(error: &ResolutionError) -> String {
}
fn enhanced_sloppy_imports_error_message(
fs: &Arc<dyn FileSystem>,
sys: &FsSysTraitsAdapter,
error: &ModuleError,
) -> Option<String> {
match error {
ModuleError::LoadingErr(specifier, _, ModuleLoadError::Loader(_)) // ex. "Is a directory" error
| ModuleError::Missing(specifier, _) => {
let additional_message = CliSloppyImportsResolver::new(SloppyImportsCachedFs::new(fs.clone()))
let additional_message = CliSloppyImportsResolver::new(SloppyImportsCachedFs::new(sys.clone()))
.resolve(specifier, SloppyImportsResolutionKind::Execution)?
.as_suggestion_message();
Some(format!(
@ -1048,71 +1050,6 @@ impl deno_graph::source::Reporter for FileWatcherReporter {
}
}
pub struct DenoGraphFsAdapter<'a>(
pub &'a dyn deno_runtime::deno_fs::FileSystem,
);
impl<'a> deno_graph::source::FileSystem for DenoGraphFsAdapter<'a> {
fn read_dir(
&self,
dir_url: &deno_graph::ModuleSpecifier,
) -> Vec<deno_graph::source::DirEntry> {
use deno_core::anyhow;
use deno_graph::source::DirEntry;
use deno_graph::source::DirEntryKind;
let dir_path = match dir_url.to_file_path() {
Ok(path) => path,
// ignore, treat as non-analyzable
Err(()) => return vec![],
};
let entries = match self.0.read_dir_sync(&dir_path) {
Ok(dir) => dir,
Err(err)
if matches!(
err.kind(),
std::io::ErrorKind::PermissionDenied | std::io::ErrorKind::NotFound
) =>
{
return vec![];
}
Err(err) => {
return vec![DirEntry {
kind: DirEntryKind::Error(
anyhow::Error::from(err)
.context("Failed to read directory.".to_string()),
),
url: dir_url.clone(),
}];
}
};
let mut dir_entries = Vec::with_capacity(entries.len());
for entry in entries {
let entry_path = dir_path.join(&entry.name);
dir_entries.push(if entry.is_directory {
DirEntry {
kind: DirEntryKind::Dir,
url: ModuleSpecifier::from_directory_path(&entry_path).unwrap(),
}
} else if entry.is_file {
DirEntry {
kind: DirEntryKind::File,
url: ModuleSpecifier::from_file_path(&entry_path).unwrap(),
}
} else if entry.is_symlink {
DirEntry {
kind: DirEntryKind::Symlink,
url: ModuleSpecifier::from_file_path(&entry_path).unwrap(),
}
} else {
continue;
});
}
dir_entries
}
}
pub fn format_range_with_colors(referrer: &deno_graph::Range) -> String {
format!(
"{}:{}:{}",

1122
cli/js/40_lint.js Normal file
View file

File diff suppressed because it is too large Load diff

1014
cli/js/40_lint_selector.js Normal file
View file

File diff suppressed because it is too large Load diff

132
cli/js/40_lint_types.d.ts vendored Normal file
View file

@ -0,0 +1,132 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
export interface NodeFacade {
type: string;
range: [number, number];
[key: string]: unknown;
}
export interface AstContext {
buf: Uint8Array;
strTable: Map<number, string>;
strTableOffset: number;
rootOffset: number;
nodes: Map<number, NodeFacade>;
strByType: number[];
strByProp: number[];
typeByStr: Map<string, number>;
propByStr: Map<string, number>;
matcher: MatchContext;
}
// TODO(@marvinhagemeister) Remove once we land "official" types
export interface RuleContext {
id: string;
}
// TODO(@marvinhagemeister) Remove once we land "official" types
export interface LintRule {
create(ctx: RuleContext): Record<string, (node: unknown) => void>;
destroy?(ctx: RuleContext): void;
}
// TODO(@marvinhagemeister) Remove once we land "official" types
export interface LintPlugin {
name: string;
rules: Record<string, LintRule>;
}
export interface LintState {
plugins: LintPlugin[];
installedPlugins: Set<string>;
}
export type VisitorFn = (node: unknown) => void;
export interface CompiledVisitor {
matcher: (ctx: MatchContext, offset: number) => boolean;
info: { enter: VisitorFn; exit: VisitorFn };
}
export interface AttrExists {
type: 3;
prop: number[];
}
export interface AttrBin {
type: 4;
prop: number[];
op: number;
// deno-lint-ignore no-explicit-any
value: any;
}
export type AttrSelector = AttrExists | AttrBin;
export interface ElemSelector {
type: 1;
wildcard: boolean;
elem: number;
}
export interface PseudoNthChild {
type: 5;
op: string | null;
step: number;
stepOffset: number;
of: Selector | null;
repeat: boolean;
}
export interface PseudoHas {
type: 6;
selectors: Selector[];
}
export interface PseudoNot {
type: 7;
selectors: Selector[];
}
export interface PseudoFirstChild {
type: 8;
}
export interface PseudoLastChild {
type: 9;
}
export interface Relation {
type: 2;
op: number;
}
export type Selector = Array<
| ElemSelector
| Relation
| AttrExists
| AttrBin
| PseudoNthChild
| PseudoNot
| PseudoHas
| PseudoFirstChild
| PseudoLastChild
>;
export interface SelectorParseCtx {
root: Selector;
current: Selector;
}
export interface MatchContext {
getFirstChild(id: number): number;
getLastChild(id: number): number;
getSiblings(id: number): number[];
getParent(id: number): number;
getType(id: number): number;
hasAttrPath(id: number, propIds: number[], idx: number): boolean;
getAttrPathValue(id: number, propIds: number[], idx: number): unknown;
}
export type NextFn = (ctx: MatchContext, id: number) => boolean;
export type MatcherFn = (ctx: MatchContext, id: number) => boolean;
export type TransformFn = (value: string) => number;
export {};

33
cli/lsp/analysis.rs
View file

@ -36,6 +36,8 @@ use deno_semver::package::PackageNv;
use deno_semver::package::PackageNvReference;
use deno_semver::package::PackageReq;
use deno_semver::package::PackageReqReference;
use deno_semver::SmallStackString;
use deno_semver::StackString;
use deno_semver::Version;
use import_map::ImportMap;
use node_resolver::NodeResolutionKind;
@ -278,9 +280,16 @@ impl<'a> TsResponseImportMapper<'a> {
{
let mut segments = jsr_path.split('/');
let name = if jsr_path.starts_with('@') {
format!("{}/{}", segments.next()?, segments.next()?)
let scope = segments.next()?;
let name = segments.next()?;
capacity_builder::StringBuilder::<StackString>::build(|builder| {
builder.append(scope);
builder.append("/");
builder.append(name);
})
.unwrap()
} else {
segments.next()?.to_string()
StackString::from(segments.next()?)
};
let version = Version::parse_standard(segments.next()?).ok()?;
let nv = PackageNv { name, version };
@ -290,7 +299,9 @@ impl<'a> TsResponseImportMapper<'a> {
&path,
Some(&self.file_referrer),
)?;
let sub_path = (export != ".").then_some(export);
let sub_path = (export != ".")
.then_some(export)
.map(SmallStackString::from_string);
let mut req = None;
req = req.or_else(|| {
let import_map = self.maybe_import_map?;
@ -603,18 +614,24 @@ fn try_reverse_map_package_json_exports(
/// For a set of tsc changes, can them for any that contain something that looks
/// like an import and rewrite the import specifier to include the extension
pub fn fix_ts_import_changes(
referrer: &ModuleSpecifier,
resolution_mode: ResolutionMode,
changes: &[tsc::FileTextChanges],
language_server: &language_server::Inner,
) -> Result<Vec<tsc::FileTextChanges>, AnyError> {
let import_mapper = language_server.get_ts_response_import_mapper(referrer);
let mut r = Vec::new();
for change in changes {
let Ok(referrer) = ModuleSpecifier::parse(&change.file_name) else {
continue;
};
let referrer_doc = language_server.get_asset_or_document(&referrer).ok();
let resolution_mode = referrer_doc
.as_ref()
.map(|d| d.resolution_mode())
.unwrap_or(ResolutionMode::Import);
let import_mapper =
language_server.get_ts_response_import_mapper(&referrer);
let mut text_changes = Vec::new();
for text_change in &change.text_changes {
let lines = text_change.new_text.split('\n');
let new_lines: Vec<String> = lines
.map(|line| {
// This assumes that there's only one import per line.
@ -622,7 +639,7 @@ pub fn fix_ts_import_changes(
let specifier =
captures.iter().skip(1).find_map(|s| s).unwrap().as_str();
if let Some(new_specifier) = import_mapper
.check_unresolved_specifier(specifier, referrer, resolution_mode)
.check_unresolved_specifier(specifier, &referrer, resolution_mode)
{
line.replace(specifier, &new_specifier)
} else {

3
cli/lsp/cache.rs
View file

@ -11,6 +11,7 @@ use crate::lsp::logging::lsp_warn;
use deno_core::url::Url;
use deno_core::ModuleSpecifier;
use deno_path_util::url_to_file_path;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use std::collections::BTreeMap;
use std::fs;
use std::path::Path;
@ -94,8 +95,8 @@ impl LspCache {
let deno_dir = DenoDir::new(global_cache_path)
.expect("should be infallible with absolute custom root");
let global = Arc::new(GlobalHttpCache::new(
FsSysTraitsAdapter::new_real(),
deno_dir.remote_folder_path(),
crate::cache::RealDenoCacheEnv,
));
Self {
deno_dir,

93
cli/lsp/config.rs
View file

@ -9,8 +9,6 @@ use deno_config::deno_json::LintConfig;
use deno_config::deno_json::NodeModulesDirMode;
use deno_config::deno_json::TestConfig;
use deno_config::deno_json::TsConfig;
use deno_config::fs::DenoConfigFs;
use deno_config::fs::RealDenoConfigFs;
use deno_config::glob::FilePatterns;
use deno_config::glob::PathOrPatternSet;
use deno_config::workspace::CreateResolverOptions;
@ -38,10 +36,10 @@ use deno_lint::linter::LintConfig as DenoLintConfig;
use deno_npm::npm_rc::ResolvedNpmRc;
use deno_package_json::PackageJsonCache;
use deno_path_util::url_to_file_path;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use deno_runtime::deno_node::PackageJson;
use indexmap::IndexSet;
use lsp_types::ClientCapabilities;
use std::borrow::Cow;
use std::collections::BTreeMap;
use std::collections::BTreeSet;
use std::collections::HashMap;
@ -1220,7 +1218,6 @@ impl ConfigData {
settings: &Settings,
file_fetcher: &Arc<CliFileFetcher>,
// sync requirement is because the lsp requires sync
cached_deno_config_fs: &(dyn DenoConfigFs + Sync),
deno_json_cache: &(dyn DenoJsonCache + Sync),
pkg_json_cache: &(dyn PackageJsonCache + Sync),
workspace_cache: &(dyn WorkspaceCache + Sync),
@ -1230,6 +1227,7 @@ impl ConfigData {
Ok(scope_dir_path) => {
let paths = [scope_dir_path];
WorkspaceDirectory::discover(
&FsSysTraitsAdapter::new_real(),
match specified_config {
Some(config_path) => {
deno_config::workspace::WorkspaceDiscoverStart::ConfigFile(
@ -1241,7 +1239,6 @@ impl ConfigData {
}
},
&WorkspaceDiscoverOptions {
fs: cached_deno_config_fs,
additional_config_file_names: &[],
deno_json_cache: Some(deno_json_cache),
pkg_json_cache: Some(pkg_json_cache),
@ -1620,9 +1617,9 @@ impl ConfigData {
|| unstable.contains("sloppy-imports");
let sloppy_imports_resolver = unstable_sloppy_imports.then(|| {
Arc::new(CliSloppyImportsResolver::new(
SloppyImportsCachedFs::new_without_stat_cache(Arc::new(
deno_runtime::deno_fs::RealFs,
)),
SloppyImportsCachedFs::new_without_stat_cache(
FsSysTraitsAdapter::new_real(),
),
))
});
let resolver = Arc::new(resolver);
@ -1841,7 +1838,6 @@ impl ConfigTree {
// since we're resolving a workspace multiple times in different
// folders, we want to cache all the lookups and config files across
// ConfigData::load calls
let cached_fs = CachedDenoConfigFs::default();
let deno_json_cache = DenoJsonMemCache::default();
let pkg_json_cache = PackageJsonMemCache::default();
let workspace_cache = WorkspaceMemCache::default();
@ -1866,7 +1862,6 @@ impl ConfigTree {
folder_uri,
settings,
file_fetcher,
&cached_fs,
&deno_json_cache,
&pkg_json_cache,
&workspace_cache,
@ -1897,7 +1892,6 @@ impl ConfigTree {
&scope,
settings,
file_fetcher,
&cached_fs,
&deno_json_cache,
&pkg_json_cache,
&workspace_cache,
@ -1914,7 +1908,6 @@ impl ConfigTree {
member_scope,
settings,
file_fetcher,
&cached_fs,
&deno_json_cache,
&pkg_json_cache,
&workspace_cache,
@ -1931,7 +1924,7 @@ impl ConfigTree {
pub async fn inject_config_file(&mut self, config_file: ConfigFile) {
let scope = config_file.specifier.join(".").unwrap();
let json_text = serde_json::to_string(&config_file.json).unwrap();
let test_fs = deno_runtime::deno_fs::InMemoryFs::default();
let test_fs = Arc::new(deno_runtime::deno_fs::InMemoryFs::default());
let config_path = url_to_file_path(&config_file.specifier).unwrap();
test_fs.setup_text_files(vec![(
config_path.to_string_lossy().to_string(),
@ -1939,11 +1932,11 @@ impl ConfigTree {
)]);
let workspace_dir = Arc::new(
WorkspaceDirectory::discover(
&FsSysTraitsAdapter(test_fs.clone()),
deno_config::workspace::WorkspaceDiscoverStart::ConfigFile(
&config_path,
),
&deno_config::workspace::WorkspaceDiscoverOptions {
fs: &crate::args::deno_json::DenoConfigFsAdapter(&test_fs),
..Default::default()
},
)
@ -2077,78 +2070,6 @@ impl deno_config::workspace::WorkspaceCache for WorkspaceMemCache {
}
}
#[derive(Default)]
struct CachedFsItems<T: Clone> {
items: HashMap<PathBuf, Result<T, std::io::Error>>,
}
impl<T: Clone> CachedFsItems<T> {
pub fn get(
&mut self,
path: &Path,
action: impl FnOnce(&Path) -> Result<T, std::io::Error>,
) -> Result<T, std::io::Error> {
let value = if let Some(value) = self.items.get(path) {
value
} else {
let value = action(path);
// just in case this gets really large for some reason
if self.items.len() == 16_384 {
return value;
}
self.items.insert(path.to_owned(), value);
self.items.get(path).unwrap()
};
value
.as_ref()
.map(|v| (*v).clone())
.map_err(|e| std::io::Error::new(e.kind(), e.to_string()))
}
}
#[derive(Default)]
struct InnerData {
stat_calls: CachedFsItems<deno_config::fs::FsMetadata>,
read_to_string_calls: CachedFsItems<Cow<'static, str>>,
}
#[derive(Default)]
struct CachedDenoConfigFs(Mutex<InnerData>);
impl DenoConfigFs for CachedDenoConfigFs {
fn stat_sync(
&self,
path: &Path,
) -> Result<deno_config::fs::FsMetadata, std::io::Error> {
self
.0
.lock()
.stat_calls
.get(path, |path| RealDenoConfigFs.stat_sync(path))
}
fn read_to_string_lossy(
&self,
path: &Path,
) -> Result<Cow<'static, str>, std::io::Error> {
self
.0
.lock()
.read_to_string_calls
.get(path, |path| RealDenoConfigFs.read_to_string_lossy(path))
}
fn read_dir(
&self,
path: &Path,
) -> Result<Vec<deno_config::fs::FsDirEntry>, std::io::Error> {
// no need to cache these because the workspace cache will ensure
// we only do read_dir calls once (read_dirs are only used for
// npm workspace resolution)
RealDenoConfigFs.read_dir(path)
}
}
#[cfg(test)]
mod tests {
use deno_config::deno_json::ConfigParseOptions;

4
cli/lsp/diagnostics.rs
View file

@ -48,7 +48,7 @@ use deno_graph::SpecifierError;
use deno_lint::linter::LintConfig as DenoLintConfig;
use deno_resolver::sloppy_imports::SloppyImportsResolution;
use deno_resolver::sloppy_imports::SloppyImportsResolutionKind;
use deno_runtime::deno_fs;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use deno_runtime::deno_node;
use deno_runtime::tokio_util::create_basic_runtime;
use deno_semver::jsr::JsrPackageReqReference;
@ -1281,7 +1281,7 @@ impl DenoDiagnostic {
Self::NotInstalledNpm(pkg_req, specifier) => (lsp::DiagnosticSeverity::ERROR, format!("npm package \"{pkg_req}\" is not installed or doesn't exist."), Some(json!({ "specifier": specifier }))),
Self::NoLocal(specifier) => {
let maybe_sloppy_resolution = CliSloppyImportsResolver::new(
SloppyImportsCachedFs::new(Arc::new(deno_fs::RealFs))
SloppyImportsCachedFs::new(FsSysTraitsAdapter::new_real())
).resolve(specifier, SloppyImportsResolutionKind::Execution);
let data = maybe_sloppy_resolution.as_ref().map(|res| {
json!({

7
cli/lsp/documents.rs
View file

@ -251,6 +251,13 @@ impl AssetOrDocument {
pub fn document_lsp_version(&self) -> Option<i32> {
self.document().and_then(|d| d.maybe_lsp_version())
}
pub fn resolution_mode(&self) -> ResolutionMode {
match self {
AssetOrDocument::Asset(_) => ResolutionMode::Import,
AssetOrDocument::Document(d) => d.resolution_mode(),
}
}
}
type ModuleResult = Result<deno_graph::JsModule, deno_graph::ModuleGraphError>;

23
cli/lsp/jsr.rs
View file

@ -18,6 +18,7 @@ use deno_graph::ModuleSpecifier;
use deno_semver::jsr::JsrPackageReqReference;
use deno_semver::package::PackageNv;
use deno_semver::package::PackageReq;
use deno_semver::StackString;
use deno_semver::Version;
use serde::Deserialize;
use std::collections::HashMap;
@ -33,8 +34,8 @@ pub struct JsrCacheResolver {
/// The `module_graph` fields of the version infos should be forcibly absent.
/// It can be large and we don't want to store it.
info_by_nv: DashMap<PackageNv, Option<Arc<JsrPackageVersionInfo>>>,
info_by_name: DashMap<String, Option<Arc<JsrPackageInfo>>>,
workspace_scope_by_name: HashMap<String, ModuleSpecifier>,
info_by_name: DashMap<StackString, Option<Arc<JsrPackageInfo>>>,
workspace_scope_by_name: HashMap<StackString, ModuleSpecifier>,
cache: Arc<dyn HttpCache>,
}
@ -59,7 +60,7 @@ impl JsrCacheResolver {
continue;
};
let nv = PackageNv {
name: jsr_pkg_config.name.clone(),
name: jsr_pkg_config.name.as_str().into(),
version: version.clone(),
};
info_by_name.insert(
@ -125,8 +126,8 @@ impl JsrCacheResolver {
return nv.value().clone();
}
let maybe_get_nv = || {
let name = req.name.clone();
let package_info = self.package_info(&name)?;
let name = &req.name;
let package_info = self.package_info(name)?;
// Find the first matching version of the package which is cached.
let mut versions = package_info.versions.keys().collect::<Vec<_>>();
versions.sort();
@ -144,7 +145,10 @@ impl JsrCacheResolver {
self.package_version_info(&nv).is_some()
})
.cloned()?;
Some(PackageNv { name, version })
Some(PackageNv {
name: name.clone(),
version,
})
};
let nv = maybe_get_nv();
self.nv_by_req.insert(req.clone(), nv.clone());
@ -216,7 +220,10 @@ impl JsrCacheResolver {
None
}
pub fn package_info(&self, name: &str) -> Option<Arc<JsrPackageInfo>> {
pub fn package_info(
&self,
name: &StackString,
) -> Option<Arc<JsrPackageInfo>> {
if let Some(info) = self.info_by_name.get(name) {
return info.value().clone();
}
@ -226,7 +233,7 @@ impl JsrCacheResolver {
serde_json::from_slice::<JsrPackageInfo>(&meta_bytes).ok()
};
let info = read_cached_package_info().map(Arc::new);
self.info_by_name.insert(name.to_string(), info.clone());
self.info_by_name.insert(name.clone(), info.clone());
info
}

51
cli/lsp/language_server.rs
View file

@ -17,6 +17,7 @@ use deno_core::ModuleSpecifier;
use deno_graph::GraphKind;
use deno_graph::Resolution;
use deno_path_util::url_to_file_path;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use deno_runtime::deno_tls::rustls::RootCertStore;
use deno_runtime::deno_tls::RootCertStoreProvider;
use deno_semver::jsr::JsrPackageReqReference;
@ -279,7 +280,7 @@ impl LanguageServer {
.await?;
graph_util::graph_valid(
&graph,
factory.fs(),
&FsSysTraitsAdapter(factory.fs().clone()),
&roots,
graph_util::GraphValidOptions {
kind: GraphKind::All,
@ -1855,20 +1856,12 @@ impl Inner {
}
let changes = if code_action_data.fix_id == "fixMissingImport" {
fix_ts_import_changes(
&code_action_data.specifier,
maybe_asset_or_doc
.as_ref()
.and_then(|d| d.document())
.map(|d| d.resolution_mode())
.unwrap_or(ResolutionMode::Import),
&combined_code_actions.changes,
self,
)
.map_err(|err| {
error!("Unable to remap changes: {:#}", err);
LspError::internal_error()
})?
fix_ts_import_changes(&combined_code_actions.changes, self).map_err(
|err| {
error!("Unable to remap changes: {:#}", err);
LspError::internal_error()
},
)?
} else {
combined_code_actions.changes
};
@ -1912,20 +1905,16 @@ impl Inner {
asset_or_doc.scope().cloned(),
)
.await?;
if kind_suffix == ".rewrite.function.returnType" {
refactor_edit_info.edits = fix_ts_import_changes(
&action_data.specifier,
asset_or_doc
.document()
.map(|d| d.resolution_mode())
.unwrap_or(ResolutionMode::Import),
&refactor_edit_info.edits,
self,
)
.map_err(|err| {
error!("Unable to remap changes: {:#}", err);
LspError::internal_error()
})?
if kind_suffix == ".rewrite.function.returnType"
|| kind_suffix == ".move.newFile"
{
refactor_edit_info.edits =
fix_ts_import_changes(&refactor_edit_info.edits, self).map_err(
|err| {
error!("Unable to remap changes: {:#}", err);
LspError::internal_error()
},
)?
}
code_action.edit = refactor_edit_info.to_workspace_edit(self)?;
code_action
@ -3624,11 +3613,11 @@ impl Inner {
let workspace = match config_data {
Some(d) => d.member_dir.clone(),
None => Arc::new(WorkspaceDirectory::discover(
&FsSysTraitsAdapter::new_real(),
deno_config::workspace::WorkspaceDiscoverStart::Paths(&[
initial_cwd.clone()
]),
&WorkspaceDiscoverOptions {
fs: Default::default(), // use real fs,
deno_json_cache: None,
pkg_json_cache: None,
workspace_cache: None,
@ -3794,7 +3783,7 @@ impl Inner {
for (name, command) in scripts {
result.push(TaskDefinition {
name: name.clone(),
command: command.clone(),
command: Some(command.clone()),
source_uri: url_to_uri(&package_json.specifier())
.map_err(|_| LspError::internal_error())?,
});

2
cli/lsp/lsp_custom.rs
View file

@ -14,7 +14,7 @@ pub const LATEST_DIAGNOSTIC_BATCH_INDEX: &str =
#[serde(rename_all = "camelCase")]
pub struct TaskDefinition {
pub name: String,
pub command: String,
pub command: Option<String>,
pub source_uri: lsp::Uri,
}

3
cli/lsp/registries.rs
View file

@ -32,6 +32,7 @@ use deno_core::url::Position;
use deno_core::url::Url;
use deno_core::ModuleSpecifier;
use deno_graph::Dependency;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use log::error;
use once_cell::sync::Lazy;
use std::borrow::Cow;
@ -430,8 +431,8 @@ impl ModuleRegistry {
) -> Self {
// the http cache should always be the global one for registry completions
let http_cache = Arc::new(GlobalHttpCache::new(
FsSysTraitsAdapter::new_real(),
location.clone(),
crate::cache::RealDenoCacheEnv,
));
let file_fetcher = CliFileFetcher::new(
http_cache.clone(),

37
cli/lsp/resolver.rs
View file

@ -19,9 +19,10 @@ use deno_resolver::cjs::IsCjsResolutionMode;
use deno_resolver::npm::NpmReqResolverOptions;
use deno_resolver::DenoResolverOptions;
use deno_resolver::NodeAndNpmReqResolver;
use deno_runtime::deno_fs;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use deno_runtime::deno_node::NodeResolver;
use deno_runtime::deno_node::PackageJsonResolver;
use deno_runtime::deno_node::RealIsBuiltInNodeModuleChecker;
use deno_semver::jsr::JsrPackageReqReference;
use deno_semver::npm::NpmPackageReqReference;
use deno_semver::package::PackageNv;
@ -42,7 +43,6 @@ use super::jsr::JsrCacheResolver;
use crate::args::create_default_npmrc;
use crate::args::CliLockfile;
use crate::args::NpmInstallDepsProvider;
use crate::cache::DenoCacheEnvFsAdapter;
use crate::factory::Deferred;
use crate::graph_util::to_node_resolution_kind;
use crate::graph_util::to_node_resolution_mode;
@ -61,7 +61,6 @@ use crate::npm::CliNpmResolverManagedSnapshotOption;
use crate::npm::CreateInNpmPkgCheckerOptions;
use crate::npm::ManagedCliNpmResolver;
use crate::resolver::CliDenoResolver;
use crate::resolver::CliDenoResolverFs;
use crate::resolver::CliNpmReqResolver;
use crate::resolver::CliResolver;
use crate::resolver::CliResolverOptions;
@ -599,21 +598,19 @@ struct ResolverFactoryServices {
struct ResolverFactory<'a> {
config_data: Option<&'a Arc<ConfigData>>,
fs: Arc<dyn deno_fs::FileSystem>,
pkg_json_resolver: Arc<PackageJsonResolver>,
sys: FsSysTraitsAdapter,
services: ResolverFactoryServices,
}
impl<'a> ResolverFactory<'a> {
pub fn new(config_data: Option<&'a Arc<ConfigData>>) -> Self {
let fs = Arc::new(deno_fs::RealFs);
let pkg_json_resolver = Arc::new(PackageJsonResolver::new(
deno_runtime::deno_node::DenoFsNodeResolverEnv::new(fs.clone()),
));
let sys = FsSysTraitsAdapter::new_real();
let pkg_json_resolver = Arc::new(PackageJsonResolver::new(sys.clone()));
Self {
config_data,
fs,
pkg_json_resolver,
sys,
services: Default::default(),
}
}
@ -624,9 +621,10 @@ impl<'a> ResolverFactory<'a> {
cache: &LspCache,
) {
let enable_byonm = self.config_data.map(|d| d.byonm).unwrap_or(false);
let sys = FsSysTraitsAdapter::new_real();
let options = if enable_byonm {
CliNpmResolverCreateOptions::Byonm(CliByonmNpmResolverCreateOptions {
fs: CliDenoResolverFs(Arc::new(deno_fs::RealFs)),
sys,
pkg_json_resolver: self.pkg_json_resolver.clone(),
root_node_modules_dir: self.config_data.and_then(|config_data| {
config_data.node_modules_dir.clone().or_else(|| {
@ -642,12 +640,14 @@ impl<'a> ResolverFactory<'a> {
.and_then(|d| d.npmrc.clone())
.unwrap_or_else(create_default_npmrc);
let npm_cache_dir = Arc::new(NpmCacheDir::new(
&DenoCacheEnvFsAdapter(self.fs.as_ref()),
&sys,
cache.deno_dir().npm_folder_path(),
npmrc.get_all_known_registries_urls(),
));
CliNpmResolverCreateOptions::Managed(CliManagedNpmResolverCreateOptions {
http_client_provider: http_client_provider.clone(),
// only used for top level install, so we can ignore this
npm_install_deps_provider: Arc::new(NpmInstallDepsProvider::empty()),
snapshot: match self.config_data.and_then(|d| d.lockfile.as_ref()) {
Some(lockfile) => {
CliNpmResolverManagedSnapshotOption::ResolveFromLockfile(
@ -656,10 +656,7 @@ impl<'a> ResolverFactory<'a> {
}
None => CliNpmResolverManagedSnapshotOption::Specified(None),
},
// Don't provide the lockfile. We don't want these resolvers
// updating it. Only the cache request should update the lockfile.
maybe_lockfile: None,
fs: Arc::new(deno_fs::RealFs),
sys: FsSysTraitsAdapter::new_real(),
npm_cache_dir,
// Use an "only" cache setting in order to make the
// user do an explicit "cache" command and prevent
@ -667,11 +664,12 @@ impl<'a> ResolverFactory<'a> {
// the user is typing.
cache_setting: CacheSetting::Only,
text_only_progress_bar: ProgressBar::new(ProgressBarStyle::TextOnly),
// Don't provide the lockfile. We don't want these resolvers
// updating it. Only the cache request should update the lockfile.
maybe_lockfile: None,
maybe_node_modules_path: self
.config_data
.and_then(|d| d.node_modules_dir.clone()),
// only used for top level install, so we can ignore this
npm_install_deps_provider: Arc::new(NpmInstallDepsProvider::empty()),
npmrc,
npm_system_info: NpmSystemInfo::default(),
lifecycle_scripts: Default::default(),
@ -779,10 +777,11 @@ impl<'a> ResolverFactory<'a> {
.get_or_init(|| {
let npm_resolver = self.services.npm_resolver.as_ref()?;
Some(Arc::new(NodeResolver::new(
deno_runtime::deno_node::DenoFsNodeResolverEnv::new(self.fs.clone()),
self.in_npm_pkg_checker().clone(),
RealIsBuiltInNodeModuleChecker,
npm_resolver.clone().into_npm_pkg_folder_resolver(),
self.pkg_json_resolver.clone(),
self.sys.clone(),
)))
})
.as_ref()
@ -797,10 +796,10 @@ impl<'a> ResolverFactory<'a> {
let npm_resolver = self.npm_resolver()?;
Some(Arc::new(CliNpmReqResolver::new(NpmReqResolverOptions {
byonm_resolver: (npm_resolver.clone()).into_maybe_byonm(),
fs: CliDenoResolverFs(self.fs.clone()),
in_npm_pkg_checker: self.in_npm_pkg_checker().clone(),
node_resolver: node_resolver.clone(),
npm_req_resolver: npm_resolver.clone().into_npm_req_resolver(),
sys: self.sys.clone(),
})))
})
.as_ref()

4
cli/lsp/search.rs
View file

@ -67,7 +67,9 @@ pub mod tests {
&self,
nv: &PackageNv,
) -> Result<Arc<Vec<String>>, AnyError> {
let Some(exports_by_version) = self.package_versions.get(&nv.name) else {
let Some(exports_by_version) =
self.package_versions.get(nv.name.as_str())
else {
return Err(anyhow!("Package not found."));
};
let Some(exports) = exports_by_version.get(&nv.version) else {

85
cli/module_loader.rs
View file

@ -11,37 +11,6 @@ use std::sync::atomic::AtomicU16;
use std::sync::atomic::Ordering;
use std::sync::Arc;
use crate::args::jsr_url;
use crate::args::CliLockfile;
use crate::args::CliOptions;
use crate::args::DenoSubcommand;
use crate::args::TsTypeLib;
use crate::cache::CodeCache;
use crate::cache::FastInsecureHasher;
use crate::cache::ParsedSourceCache;
use crate::emit::Emitter;
use crate::graph_container::MainModuleGraphContainer;
use crate::graph_container::ModuleGraphContainer;
use crate::graph_container::ModuleGraphUpdatePermit;
use crate::graph_util::CreateGraphOptions;
use crate::graph_util::ModuleGraphBuilder;
use crate::node;
use crate::node::CliNodeCodeTranslator;
use crate::npm::CliNpmResolver;
use crate::resolver::CjsTracker;
use crate::resolver::CliNpmReqResolver;
use crate::resolver::CliResolver;
use crate::resolver::ModuleCodeStringSource;
use crate::resolver::NotSupportedKindInNpmError;
use crate::resolver::NpmModuleLoader;
use crate::tools::check;
use crate::tools::check::MaybeDiagnostics;
use crate::tools::check::TypeChecker;
use crate::util::progress_bar::ProgressBar;
use crate::util::text_encoding::code_without_source_map;
use crate::util::text_encoding::source_map_from_code;
use crate::worker::CreateModuleLoaderResult;
use crate::worker::ModuleLoaderFactory;
use deno_ast::MediaType;
use deno_ast::ModuleKind;
use deno_core::anyhow::anyhow;
@ -70,7 +39,7 @@ use deno_graph::ModuleGraph;
use deno_graph::Resolution;
use deno_graph::WasmModule;
use deno_runtime::code_cache;
use deno_runtime::deno_fs::FileSystem;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use deno_runtime::deno_node::create_host_defined_options;
use deno_runtime::deno_node::NodeRequireLoader;
use deno_runtime::deno_node::NodeResolver;
@ -80,6 +49,38 @@ use node_resolver::errors::ClosestPkgJsonError;
use node_resolver::InNpmPackageChecker;
use node_resolver::NodeResolutionKind;
use node_resolver::ResolutionMode;
use sys_traits::FsRead;
use crate::args::jsr_url;
use crate::args::CliLockfile;
use crate::args::CliOptions;
use crate::args::DenoSubcommand;
use crate::args::TsTypeLib;
use crate::cache::CodeCache;
use crate::cache::FastInsecureHasher;
use crate::cache::ParsedSourceCache;
use crate::emit::Emitter;
use crate::graph_container::MainModuleGraphContainer;
use crate::graph_container::ModuleGraphContainer;
use crate::graph_container::ModuleGraphUpdatePermit;
use crate::graph_util::CreateGraphOptions;
use crate::graph_util::ModuleGraphBuilder;
use crate::node::CliNodeCodeTranslator;
use crate::npm::CliNpmResolver;
use crate::resolver::CjsTracker;
use crate::resolver::CliNpmReqResolver;
use crate::resolver::CliResolver;
use crate::resolver::ModuleCodeStringSource;
use crate::resolver::NotSupportedKindInNpmError;
use crate::resolver::NpmModuleLoader;
use crate::tools::check;
use crate::tools::check::MaybeDiagnostics;
use crate::tools::check::TypeChecker;
use crate::util::progress_bar::ProgressBar;
use crate::util::text_encoding::code_without_source_map;
use crate::util::text_encoding::source_map_from_code;
use crate::worker::CreateModuleLoaderResult;
use crate::worker::ModuleLoaderFactory;
pub struct ModuleLoadPreparer {
options: Arc<CliOptions>,
@ -216,7 +217,6 @@ struct SharedCliModuleLoaderState {
cjs_tracker: Arc<CjsTracker>,
code_cache: Option<Arc<CodeCache>>,
emitter: Arc<Emitter>,
fs: Arc<dyn FileSystem>,
in_npm_pkg_checker: Arc<dyn InNpmPackageChecker>,
main_module_graph_container: Arc<MainModuleGraphContainer>,
module_load_preparer: Arc<ModuleLoadPreparer>,
@ -227,6 +227,7 @@ struct SharedCliModuleLoaderState {
npm_module_loader: NpmModuleLoader,
parsed_source_cache: Arc<ParsedSourceCache>,
resolver: Arc<CliResolver>,
sys: FsSysTraitsAdapter,
in_flight_loads_tracker: InFlightModuleLoadsTracker,
}
@ -276,7 +277,6 @@ impl CliModuleLoaderFactory {
cjs_tracker: Arc<CjsTracker>,
code_cache: Option<Arc<CodeCache>>,
emitter: Arc<Emitter>,
fs: Arc<dyn FileSystem>,
in_npm_pkg_checker: Arc<dyn InNpmPackageChecker>,
main_module_graph_container: Arc<MainModuleGraphContainer>,
module_load_preparer: Arc<ModuleLoadPreparer>,
@ -287,6 +287,7 @@ impl CliModuleLoaderFactory {
npm_module_loader: NpmModuleLoader,
parsed_source_cache: Arc<ParsedSourceCache>,
resolver: Arc<CliResolver>,
sys: FsSysTraitsAdapter,
) -> Self {
Self {
shared: Arc::new(SharedCliModuleLoaderState {
@ -302,7 +303,6 @@ impl CliModuleLoaderFactory {
cjs_tracker,
code_cache,
emitter,
fs,
in_npm_pkg_checker,
main_module_graph_container,
module_load_preparer,
@ -313,6 +313,7 @@ impl CliModuleLoaderFactory {
npm_module_loader,
parsed_source_cache,
resolver,
sys,
in_flight_loads_tracker: InFlightModuleLoadsTracker {
loads_number: Arc::new(AtomicU16::new(0)),
cleanup_task_timeout: 10_000,
@ -345,7 +346,7 @@ impl CliModuleLoaderFactory {
let node_require_loader = Rc::new(CliNodeRequireLoader {
cjs_tracker: self.shared.cjs_tracker.clone(),
emitter: self.shared.emitter.clone(),
fs: self.shared.fs.clone(),
sys: self.shared.sys.clone(),
graph_container,
in_npm_pkg_checker: self.shared.in_npm_pkg_checker.clone(),
npm_resolver: self.shared.npm_resolver.clone(),
@ -594,9 +595,9 @@ impl<TGraphContainer: ModuleGraphContainer>
Some(Module::Json(module)) => module.specifier.clone(),
Some(Module::Wasm(module)) => module.specifier.clone(),
Some(Module::External(module)) => {
node::resolve_specifier_into_node_modules(
node_resolver::resolve_specifier_into_node_modules(
&self.shared.sys,
&module.specifier,
self.shared.fs.as_ref(),
)
}
None => specifier.into_owned(),
@ -997,7 +998,7 @@ impl<TGraphContainer: ModuleGraphContainer> ModuleLoader
std::future::ready(()).boxed_local()
}
fn get_source_map(&self, file_name: &str) -> Option<Vec<u8>> {
fn get_source_map(&self, file_name: &str) -> Option<Cow<[u8]>> {
let specifier = resolve_url(file_name).ok()?;
match specifier.scheme() {
// we should only be looking for emits for schemes that denote external
@ -1009,7 +1010,7 @@ impl<TGraphContainer: ModuleGraphContainer> ModuleLoader
.0
.load_prepared_module_for_source_map_sync(&specifier)
.ok()??;
source_map_from_code(source.code.as_bytes())
source_map_from_code(source.code.as_bytes()).map(Cow::Owned)
}
fn get_source_mapped_source_line(
@ -1092,7 +1093,7 @@ impl ModuleGraphUpdatePermit for WorkerModuleGraphUpdatePermit {
struct CliNodeRequireLoader<TGraphContainer: ModuleGraphContainer> {
cjs_tracker: Arc<CjsTracker>,
emitter: Arc<Emitter>,
fs: Arc<dyn FileSystem>,
sys: FsSysTraitsAdapter,
graph_container: TGraphContainer,
in_npm_pkg_checker: Arc<dyn InNpmPackageChecker>,
npm_resolver: Arc<dyn CliNpmResolver>,
@ -1121,7 +1122,7 @@ impl<TGraphContainer: ModuleGraphContainer> NodeRequireLoader
) -> Result<Cow<'static, str>, AnyError> {
// todo(dsherret): use the preloaded module from the graph if available?
let media_type = MediaType::from_path(path);
let text = self.fs.read_text_file_lossy_sync(path, None)?;
let text = self.sys.fs_read_to_string_lossy(path)?;
if media_type.is_emittable() {
let specifier = deno_path_util::url_from_file_path(path)?;
if self.in_npm_pkg_checker.in_npm_package(&specifier) {

25
cli/node.rs
View file

@ -8,7 +8,8 @@ use deno_ast::ModuleSpecifier;
use deno_core::error::AnyError;
use deno_graph::ParsedSourceStore;
use deno_runtime::deno_fs;
use deno_runtime::deno_node::DenoFsNodeResolverEnv;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use deno_runtime::deno_node::RealIsBuiltInNodeModuleChecker;
use node_resolver::analyze::CjsAnalysis as ExtNodeCjsAnalysis;
use node_resolver::analyze::CjsAnalysisExports;
use node_resolver::analyze::CjsCodeAnalyzer;
@ -21,23 +22,11 @@ use crate::cache::NodeAnalysisCache;
use crate::cache::ParsedSourceCache;
use crate::resolver::CjsTracker;
pub type CliNodeCodeTranslator =
NodeCodeTranslator<CliCjsCodeAnalyzer, DenoFsNodeResolverEnv>;
/// Resolves a specifier that is pointing into a node_modules folder.
///
/// Note: This should be called whenever getting the specifier from
/// a Module::External(module) reference because that module might
/// not be fully resolved at the time deno_graph is analyzing it
/// because the node_modules folder might not exist at that time.
pub fn resolve_specifier_into_node_modules(
specifier: &ModuleSpecifier,
fs: &dyn deno_fs::FileSystem,
) -> ModuleSpecifier {
node_resolver::resolve_specifier_into_node_modules(specifier, &|path| {
fs.realpath_sync(path).map_err(|err| err.into_io_error())
})
}
pub type CliNodeCodeTranslator = NodeCodeTranslator<
CliCjsCodeAnalyzer,
RealIsBuiltInNodeModuleChecker,
FsSysTraitsAdapter,
>;
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub enum CliCjsAnalysis {

8
cli/npm/byonm.rs
View file

@ -9,22 +9,20 @@ use deno_core::serde_json;
use deno_resolver::npm::ByonmNpmResolver;
use deno_resolver::npm::ByonmNpmResolverCreateOptions;
use deno_resolver::npm::CliNpmReqResolver;
use deno_runtime::deno_node::DenoFsNodeResolverEnv;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use deno_runtime::deno_node::NodePermissions;
use deno_runtime::ops::process::NpmProcessStateProvider;
use node_resolver::NpmPackageFolderResolver;
use crate::args::NpmProcessState;
use crate::args::NpmProcessStateKind;
use crate::resolver::CliDenoResolverFs;
use super::CliNpmResolver;
use super::InnerCliNpmResolverRef;
pub type CliByonmNpmResolverCreateOptions =
ByonmNpmResolverCreateOptions<CliDenoResolverFs, DenoFsNodeResolverEnv>;
pub type CliByonmNpmResolver =
ByonmNpmResolver<CliDenoResolverFs, DenoFsNodeResolverEnv>;
ByonmNpmResolverCreateOptions<FsSysTraitsAdapter>;
pub type CliByonmNpmResolver = ByonmNpmResolver<FsSysTraitsAdapter>;
// todo(dsherret): the services hanging off `CliNpmResolver` doesn't seem ideal. We should probably decouple.
#[derive(Debug)]

106
cli/npm/managed/mod.rs
View file

@ -21,9 +21,10 @@ use deno_npm::NpmPackageId;
use deno_npm::NpmResolutionPackage;
use deno_npm::NpmSystemInfo;
use deno_npm_cache::NpmCacheSetting;
use deno_path_util::fs::canonicalize_path_maybe_not_exists;
use deno_resolver::npm::CliNpmReqResolver;
use deno_runtime::colors;
use deno_runtime::deno_fs::FileSystem;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use deno_runtime::deno_node::NodePermissions;
use deno_runtime::ops::process::NpmProcessStateProvider;
use deno_semver::package::PackageNv;
@ -41,7 +42,6 @@ use crate::args::NpmProcessState;
use crate::args::NpmProcessStateKind;
use crate::args::PackageJsonDepValueParseWithLocationError;
use crate::cache::FastInsecureHasher;
use crate::util::fs::canonicalize_path_maybe_not_exists_with_fs;
use crate::util::progress_bar::ProgressBar;
use crate::util::sync::AtomicFlag;
@ -50,7 +50,7 @@ use self::resolvers::create_npm_fs_resolver;
use self::resolvers::NpmPackageFsResolver;
use super::CliNpmCache;
use super::CliNpmCacheEnv;
use super::CliNpmCacheHttpClient;
use super::CliNpmRegistryInfoProvider;
use super::CliNpmResolver;
use super::CliNpmTarballCache;
@ -68,9 +68,9 @@ pub enum CliNpmResolverManagedSnapshotOption {
pub struct CliManagedNpmResolverCreateOptions {
pub snapshot: CliNpmResolverManagedSnapshotOption,
pub maybe_lockfile: Option<Arc<CliLockfile>>,
pub fs: Arc<dyn deno_runtime::deno_fs::FileSystem>,
pub http_client_provider: Arc<crate::http_util::HttpClientProvider>,
pub npm_cache_dir: Arc<NpmCacheDir>,
pub sys: FsSysTraitsAdapter,
pub cache_setting: deno_cache_dir::file_fetcher::CacheSetting,
pub text_only_progress_bar: crate::util::progress_bar::ProgressBar,
pub maybe_node_modules_path: Option<PathBuf>,
@ -83,9 +83,12 @@ pub struct CliManagedNpmResolverCreateOptions {
pub async fn create_managed_npm_resolver_for_lsp(
options: CliManagedNpmResolverCreateOptions,
) -> Arc<dyn CliNpmResolver> {
let cache_env = create_cache_env(&options);
let npm_cache = create_cache(cache_env.clone(), &options);
let npm_api = create_api(npm_cache.clone(), cache_env.clone(), &options);
let npm_cache = create_cache(&options);
let http_client = Arc::new(CliNpmCacheHttpClient::new(
options.http_client_provider.clone(),
options.text_only_progress_bar.clone(),
));
let npm_api = create_api(npm_cache.clone(), http_client.clone(), &options);
// spawn due to the lsp's `Send` requirement
deno_core::unsync::spawn(async move {
let snapshot = match resolve_snapshot(&npm_api, options.snapshot).await {
@ -96,14 +99,14 @@ pub async fn create_managed_npm_resolver_for_lsp(
}
};
create_inner(
cache_env,
options.fs,
options.maybe_lockfile,
npm_api,
http_client,
npm_cache,
options.npmrc,
options.npm_install_deps_provider,
npm_api,
options.sys,
options.text_only_progress_bar,
options.maybe_lockfile,
options.npmrc,
options.maybe_node_modules_path,
options.npm_system_info,
snapshot,
@ -117,19 +120,22 @@ pub async fn create_managed_npm_resolver_for_lsp(
pub async fn create_managed_npm_resolver(
options: CliManagedNpmResolverCreateOptions,
) -> Result<Arc<dyn CliNpmResolver>, AnyError> {
let npm_cache_env = create_cache_env(&options);
let npm_cache = create_cache(npm_cache_env.clone(), &options);
let api = create_api(npm_cache.clone(), npm_cache_env.clone(), &options);
let npm_cache = create_cache(&options);
let http_client = Arc::new(CliNpmCacheHttpClient::new(
options.http_client_provider.clone(),
options.text_only_progress_bar.clone(),
));
let api = create_api(npm_cache.clone(), http_client.clone(), &options);
let snapshot = resolve_snapshot(&api, options.snapshot).await?;
Ok(create_inner(
npm_cache_env,
options.fs,
options.maybe_lockfile,
api,
http_client,
npm_cache,
options.npmrc,
options.npm_install_deps_provider,
api,
options.sys,
options.text_only_progress_bar,
options.maybe_lockfile,
options.npmrc,
options.maybe_node_modules_path,
options.npm_system_info,
snapshot,
@ -139,14 +145,14 @@ pub async fn create_managed_npm_resolver(
#[allow(clippy::too_many_arguments)]
fn create_inner(
env: Arc<CliNpmCacheEnv>,
fs: Arc<dyn deno_runtime::deno_fs::FileSystem>,
maybe_lockfile: Option<Arc<CliLockfile>>,
registry_info_provider: Arc<CliNpmRegistryInfoProvider>,
http_client: Arc<CliNpmCacheHttpClient>,
npm_cache: Arc<CliNpmCache>,
npm_rc: Arc<ResolvedNpmRc>,
npm_install_deps_provider: Arc<NpmInstallDepsProvider>,
registry_info_provider: Arc<CliNpmRegistryInfoProvider>,
sys: FsSysTraitsAdapter,
text_only_progress_bar: crate::util::progress_bar::ProgressBar,
maybe_lockfile: Option<Arc<CliLockfile>>,
npm_rc: Arc<ResolvedNpmRc>,
node_modules_dir_path: Option<PathBuf>,
npm_system_info: NpmSystemInfo,
snapshot: Option<ValidSerializedNpmResolutionSnapshot>,
@ -159,28 +165,29 @@ fn create_inner(
));
let tarball_cache = Arc::new(CliNpmTarballCache::new(
npm_cache.clone(),
env,
http_client,
sys.clone(),
npm_rc.clone(),
));
let fs_resolver = create_npm_fs_resolver(
fs.clone(),
npm_cache.clone(),
&npm_install_deps_provider,
&text_only_progress_bar,
resolution.clone(),
sys.clone(),
tarball_cache.clone(),
node_modules_dir_path,
npm_system_info.clone(),
lifecycle_scripts.clone(),
);
Arc::new(ManagedCliNpmResolver::new(
fs,
fs_resolver,
maybe_lockfile,
registry_info_provider,
npm_cache,
npm_install_deps_provider,
resolution,
sys,
tarball_cache,
text_only_progress_bar,
npm_system_info,
@ -188,36 +195,25 @@ fn create_inner(
))
}
fn create_cache_env(
options: &CliManagedNpmResolverCreateOptions,
) -> Arc<CliNpmCacheEnv> {
Arc::new(CliNpmCacheEnv::new(
options.fs.clone(),
options.http_client_provider.clone(),
options.text_only_progress_bar.clone(),
))
}
fn create_cache(
env: Arc<CliNpmCacheEnv>,
options: &CliManagedNpmResolverCreateOptions,
) -> Arc<CliNpmCache> {
Arc::new(CliNpmCache::new(
options.npm_cache_dir.clone(),
options.sys.clone(),
NpmCacheSetting::from_cache_setting(&options.cache_setting),
env,
options.npmrc.clone(),
))
}
fn create_api(
cache: Arc<CliNpmCache>,
env: Arc<CliNpmCacheEnv>,
http_client: Arc<CliNpmCacheHttpClient>,
options: &CliManagedNpmResolverCreateOptions,
) -> Arc<CliNpmRegistryInfoProvider> {
Arc::new(CliNpmRegistryInfoProvider::new(
cache,
env,
http_client,
options.npmrc.clone(),
))
}
@ -306,12 +302,12 @@ pub enum PackageCaching<'a> {
/// An npm resolver where the resolution is managed by Deno rather than
/// the user bringing their own node_modules (BYONM) on the file system.
pub struct ManagedCliNpmResolver {
fs: Arc<dyn FileSystem>,
fs_resolver: Arc<dyn NpmPackageFsResolver>,
maybe_lockfile: Option<Arc<CliLockfile>>,
registry_info_provider: Arc<CliNpmRegistryInfoProvider>,
npm_cache: Arc<CliNpmCache>,
npm_install_deps_provider: Arc<NpmInstallDepsProvider>,
sys: FsSysTraitsAdapter,
resolution: Arc<NpmResolution>,
tarball_cache: Arc<CliNpmTarballCache>,
text_only_progress_bar: ProgressBar,
@ -331,20 +327,19 @@ impl std::fmt::Debug for ManagedCliNpmResolver {
impl ManagedCliNpmResolver {
#[allow(clippy::too_many_arguments)]
pub fn new(
fs: Arc<dyn FileSystem>,
fs_resolver: Arc<dyn NpmPackageFsResolver>,
maybe_lockfile: Option<Arc<CliLockfile>>,
registry_info_provider: Arc<CliNpmRegistryInfoProvider>,
npm_cache: Arc<CliNpmCache>,
npm_install_deps_provider: Arc<NpmInstallDepsProvider>,
resolution: Arc<NpmResolution>,
sys: FsSysTraitsAdapter,
tarball_cache: Arc<CliNpmTarballCache>,
text_only_progress_bar: ProgressBar,
npm_system_info: NpmSystemInfo,
lifecycle_scripts: LifecycleScriptsConfig,
) -> Self {
Self {
fs,
fs_resolver,
maybe_lockfile,
registry_info_provider,
@ -352,6 +347,7 @@ impl ManagedCliNpmResolver {
npm_install_deps_provider,
text_only_progress_bar,
resolution,
sys,
tarball_cache,
npm_system_info,
top_level_install_flag: Default::default(),
@ -364,8 +360,7 @@ impl ManagedCliNpmResolver {
pkg_id: &NpmPackageId,
) -> Result<PathBuf, AnyError> {
let path = self.fs_resolver.package_folder(pkg_id)?;
let path =
canonicalize_path_maybe_not_exists_with_fs(&path, self.fs.as_ref())?;
let path = canonicalize_path_maybe_not_exists(&self.sys, &path)?;
log::debug!(
"Resolved package folder of {} to {}",
pkg_id.as_serialized(),
@ -560,11 +555,11 @@ impl ManagedCliNpmResolver {
&self,
) -> Result<(), Box<PackageJsonDepValueParseWithLocationError>> {
for err in self.npm_install_deps_provider.pkg_json_dep_errors() {
match &err.source {
deno_package_json::PackageJsonDepValueParseError::VersionReq(_) => {
match err.source.as_kind() {
deno_package_json::PackageJsonDepValueParseErrorKind::VersionReq(_) => {
return Err(Box::new(err.clone()));
}
deno_package_json::PackageJsonDepValueParseError::Unsupported {
deno_package_json::PackageJsonDepValueParseErrorKind::Unsupported {
..
} => {
// only warn for this one
@ -667,12 +662,13 @@ impl NpmPackageFolderResolver for ManagedCliNpmResolver {
.fs_resolver
.resolve_package_folder_from_package(name, referrer)?;
let path =
canonicalize_path_maybe_not_exists_with_fs(&path, self.fs.as_ref())
.map_err(|err| PackageFolderResolveIoError {
canonicalize_path_maybe_not_exists(&self.sys, &path).map_err(|err| {
PackageFolderResolveIoError {
package_name: name.to_string(),
referrer: referrer.clone(),
source: err,
})?;
}
})?;
log::debug!("Resolved {} from {} to {}", name, referrer, path.display());
Ok(path)
}
@ -728,13 +724,12 @@ impl CliNpmResolver for ManagedCliNpmResolver {
));
Arc::new(ManagedCliNpmResolver::new(
self.fs.clone(),
create_npm_fs_resolver(
self.fs.clone(),
self.npm_cache.clone(),
&self.npm_install_deps_provider,
&self.text_only_progress_bar,
npm_resolution.clone(),
self.sys.clone(),
self.tarball_cache.clone(),
self.root_node_modules_path().map(ToOwned::to_owned),
self.npm_system_info.clone(),
@ -745,6 +740,7 @@ impl CliNpmResolver for ManagedCliNpmResolver {
self.npm_cache.clone(),
self.npm_install_deps_provider.clone(),
npm_resolution,
self.sys.clone(),
self.tarball_cache.clone(),
self.text_only_progress_bar.clone(),
self.npm_system_info.clone(),

10
cli/npm/managed/resolution.rs
View file

@ -4,6 +4,7 @@ use std::collections::HashMap;
use std::collections::HashSet;
use std::sync::Arc;
use capacity_builder::StringBuilder;
use deno_core::error::AnyError;
use deno_lockfile::NpmPackageDependencyLockfileInfo;
use deno_lockfile::NpmPackageLockfileInfo;
@ -24,6 +25,7 @@ use deno_npm::NpmSystemInfo;
use deno_semver::jsr::JsrDepPackageReq;
use deno_semver::package::PackageNv;
use deno_semver::package::PackageReq;
use deno_semver::SmallStackString;
use deno_semver::VersionReq;
use crate::args::CliLockfile;
@ -336,7 +338,13 @@ fn populate_lockfile_from_snapshot(
let id = &snapshot.resolve_package_from_deno_module(nv).unwrap().id;
lockfile.insert_package_specifier(
JsrDepPackageReq::npm(package_req.clone()),
format!("{}{}", id.nv.version, id.peer_deps_serialized()),
{
StringBuilder::<SmallStackString>::build(|builder| {
builder.append(&id.nv.version);
builder.append(&id.peer_dependencies);
})
.unwrap()
},
);
}
for package in snapshot.all_packages_for_every_system() {

13
cli/npm/managed/resolvers/common.rs
View file

@ -11,7 +11,6 @@ use std::path::PathBuf;
use std::sync::Arc;
use std::sync::Mutex;
use super::super::PackageCaching;
use async_trait::async_trait;
use deno_ast::ModuleSpecifier;
use deno_core::anyhow::Context;
@ -21,10 +20,12 @@ use deno_core::futures::StreamExt;
use deno_npm::NpmPackageCacheFolderId;
use deno_npm::NpmPackageId;
use deno_npm::NpmResolutionPackage;
use deno_runtime::deno_fs::FileSystem;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use deno_runtime::deno_node::NodePermissions;
use node_resolver::errors::PackageFolderResolveError;
use sys_traits::FsCanonicalize;
use super::super::PackageCaching;
use crate::npm::CliNpmTarballCache;
/// Part of the resolution that interacts with the file system.
@ -73,15 +74,15 @@ pub trait NpmPackageFsResolver: Send + Sync {
#[derive(Debug)]
pub struct RegistryReadPermissionChecker {
fs: Arc<dyn FileSystem>,
sys: FsSysTraitsAdapter,
cache: Mutex<HashMap<PathBuf, PathBuf>>,
registry_path: PathBuf,
}
impl RegistryReadPermissionChecker {
pub fn new(fs: Arc<dyn FileSystem>, registry_path: PathBuf) -> Self {
pub fn new(fs: FsSysTraitsAdapter, registry_path: PathBuf) -> Self {
Self {
fs,
sys: fs,
registry_path,
cache: Default::default(),
}
@ -108,7 +109,7 @@ impl RegistryReadPermissionChecker {
|path: &Path| -> Result<Option<PathBuf>, AnyError> {
match cache.get(path) {
Some(canon) => Ok(Some(canon.clone())),
None => match self.fs.realpath_sync(path) {
None => match self.sys.fs_canonicalize(path) {
Ok(canon) => {
cache.insert(path.to_path_buf(), canon.clone());
Ok(Some(canon))

4
cli/npm/managed/resolvers/common/bin_entries.rs
View file

@ -28,8 +28,10 @@ fn default_bin_name(package: &NpmResolutionPackage) -> &str {
.id
.nv
.name
.as_str()
.rsplit_once('/')
.map_or(package.id.nv.name.as_str(), |(_, name)| name)
.map(|(_, name)| name)
.unwrap_or(package.id.nv.name.as_str())
}
pub fn warn_missing_entrypoint(

6
cli/npm/managed/resolvers/global.rs
View file

@ -18,7 +18,7 @@ use deno_npm::NpmPackageCacheFolderId;
use deno_npm::NpmPackageId;
use deno_npm::NpmResolutionPackage;
use deno_npm::NpmSystemInfo;
use deno_runtime::deno_fs::FileSystem;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use deno_runtime::deno_node::NodePermissions;
use node_resolver::errors::PackageFolderResolveError;
use node_resolver::errors::PackageNotFoundError;
@ -47,15 +47,15 @@ pub struct GlobalNpmPackageResolver {
impl GlobalNpmPackageResolver {
pub fn new(
cache: Arc<CliNpmCache>,
fs: Arc<dyn FileSystem>,
tarball_cache: Arc<CliNpmTarballCache>,
resolution: Arc<NpmResolution>,
sys: FsSysTraitsAdapter,
system_info: NpmSystemInfo,
lifecycle_scripts: LifecycleScriptsConfig,
) -> Self {
Self {
registry_read_permission_checker: RegistryReadPermissionChecker::new(
fs,
sys,
cache.root_dir_path().to_path_buf(),
),
cache,

53
cli/npm/managed/resolvers/local.rs
View file

@ -15,11 +15,6 @@ use std::path::PathBuf;
use std::rc::Rc;
use std::sync::Arc;
use crate::args::LifecycleScriptsConfig;
use crate::colors;
use crate::npm::managed::PackageCaching;
use crate::npm::CliNpmCache;
use crate::npm::CliNpmTarballCache;
use async_trait::async_trait;
use deno_ast::ModuleSpecifier;
use deno_cache_dir::npm::mixed_case_package_name_decode;
@ -34,21 +29,28 @@ use deno_npm::NpmPackageCacheFolderId;
use deno_npm::NpmPackageId;
use deno_npm::NpmResolutionPackage;
use deno_npm::NpmSystemInfo;
use deno_path_util::fs::atomic_write_file_with_retries;
use deno_path_util::fs::canonicalize_path_maybe_not_exists;
use deno_resolver::npm::normalize_pkg_name_for_node_modules_deno_folder;
use deno_runtime::deno_fs;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use deno_runtime::deno_node::NodePermissions;
use deno_semver::package::PackageNv;
use deno_semver::StackString;
use node_resolver::errors::PackageFolderResolveError;
use node_resolver::errors::PackageFolderResolveIoError;
use node_resolver::errors::PackageNotFoundError;
use node_resolver::errors::ReferrerNotFoundError;
use serde::Deserialize;
use serde::Serialize;
use sys_traits::FsMetadata;
use crate::args::LifecycleScriptsConfig;
use crate::args::NpmInstallDepsProvider;
use crate::cache::CACHE_PERM;
use crate::util::fs::atomic_write_file_with_retries;
use crate::util::fs::canonicalize_path_maybe_not_exists_with_fs;
use crate::colors;
use crate::npm::managed::PackageCaching;
use crate::npm::CliNpmCache;
use crate::npm::CliNpmTarballCache;
use crate::util::fs::clone_dir_recursive;
use crate::util::fs::symlink_dir;
use crate::util::fs::LaxSingleProcessFsFlag;
@ -65,10 +67,10 @@ use super::common::RegistryReadPermissionChecker;
#[derive(Debug)]
pub struct LocalNpmPackageResolver {
cache: Arc<CliNpmCache>,
fs: Arc<dyn deno_fs::FileSystem>,
npm_install_deps_provider: Arc<NpmInstallDepsProvider>,
progress_bar: ProgressBar,
resolution: Arc<NpmResolution>,
sys: FsSysTraitsAdapter,
tarball_cache: Arc<CliNpmTarballCache>,
root_node_modules_path: PathBuf,
root_node_modules_url: Url,
@ -81,10 +83,10 @@ impl LocalNpmPackageResolver {
#[allow(clippy::too_many_arguments)]
pub fn new(
cache: Arc<CliNpmCache>,
fs: Arc<dyn deno_fs::FileSystem>,
npm_install_deps_provider: Arc<NpmInstallDepsProvider>,
progress_bar: ProgressBar,
resolution: Arc<NpmResolution>,
sys: FsSysTraitsAdapter,
tarball_cache: Arc<CliNpmTarballCache>,
node_modules_folder: PathBuf,
system_info: NpmSystemInfo,
@ -92,15 +94,15 @@ impl LocalNpmPackageResolver {
) -> Self {
Self {
cache,
fs: fs.clone(),
npm_install_deps_provider,
progress_bar,
resolution,
tarball_cache,
registry_read_permission_checker: RegistryReadPermissionChecker::new(
fs,
sys.clone(),
node_modules_folder.clone(),
),
sys,
root_node_modules_url: Url::from_directory_path(&node_modules_folder)
.unwrap(),
root_node_modules_path: node_modules_folder,
@ -139,8 +141,7 @@ impl LocalNpmPackageResolver {
};
// Canonicalize the path so it's not pointing to the symlinked directory
// in `node_modules` directory of the referrer.
canonicalize_path_maybe_not_exists_with_fs(&path, self.fs.as_ref())
.map(Some)
canonicalize_path_maybe_not_exists(&self.sys, &path).map(Some)
}
fn resolve_package_folder_from_specifier(
@ -209,7 +210,7 @@ impl NpmPackageFsResolver for LocalNpmPackageResolver {
};
let sub_dir = join_package_name(&node_modules_folder, name);
if self.fs.is_dir_sync(&sub_dir) {
if self.sys.fs_is_dir_no_err(&sub_dir) {
return Ok(sub_dir);
}
@ -355,8 +356,10 @@ async fn sync_resolution_with_fs(
let package_partitions =
snapshot.all_system_packages_partitioned(system_info);
let mut cache_futures = FuturesUnordered::new();
let mut newest_packages_by_name: HashMap<&String, &NpmResolutionPackage> =
HashMap::with_capacity(package_partitions.packages.len());
let mut newest_packages_by_name: HashMap<
&StackString,
&NpmResolutionPackage,
> = HashMap::with_capacity(package_partitions.packages.len());
let bin_entries = Rc::new(RefCell::new(bin_entries::BinEntries::new()));
let mut lifecycle_scripts =
super::common::lifecycle_scripts::LifecycleScripts::new(
@ -536,7 +539,7 @@ async fn sync_resolution_with_fs(
}
}
let mut found_names: HashMap<&String, &PackageNv> = HashMap::new();
let mut found_names: HashMap<&StackString, &PackageNv> = HashMap::new();
// set of node_modules in workspace packages that we've already ensured exist
let mut existing_child_node_modules_dirs: HashSet<PathBuf> = HashSet::new();
@ -922,7 +925,13 @@ impl SetupCache {
}
bincode::serialize(&self.current).ok().and_then(|data| {
atomic_write_file_with_retries(&self.file_path, data, CACHE_PERM).ok()
atomic_write_file_with_retries(
&FsSysTraitsAdapter::new_real(),
&self.file_path,
&data,
CACHE_PERM,
)
.ok()
});
true
}
@ -1012,10 +1021,10 @@ fn get_package_folder_id_from_folder_name(
) -> Option<NpmPackageCacheFolderId> {
let folder_name = folder_name.replace('+', "/");
let (name, ending) = folder_name.rsplit_once('@')?;
let name = if let Some(encoded_name) = name.strip_prefix('_') {
mixed_case_package_name_decode(encoded_name)?
let name: StackString = if let Some(encoded_name) = name.strip_prefix('_') {
StackString::from_string(mixed_case_package_name_decode(encoded_name)?)
} else {
name.to_string()
name.into()
};
let (raw_version, copy_index) = match ending.split_once('_') {
Some((raw_version, copy_index)) => {

8
cli/npm/managed/resolvers/mod.rs
View file

@ -8,7 +8,7 @@ use std::path::PathBuf;
use std::sync::Arc;
use deno_npm::NpmSystemInfo;
use deno_runtime::deno_fs::FileSystem;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use crate::args::LifecycleScriptsConfig;
use crate::args::NpmInstallDepsProvider;
@ -25,11 +25,11 @@ use super::resolution::NpmResolution;
#[allow(clippy::too_many_arguments)]
pub fn create_npm_fs_resolver(
fs: Arc<dyn FileSystem>,
npm_cache: Arc<CliNpmCache>,
npm_install_deps_provider: &Arc<NpmInstallDepsProvider>,
progress_bar: &ProgressBar,
resolution: Arc<NpmResolution>,
sys: FsSysTraitsAdapter,
tarball_cache: Arc<CliNpmTarballCache>,
maybe_node_modules_path: Option<PathBuf>,
system_info: NpmSystemInfo,
@ -38,10 +38,10 @@ pub fn create_npm_fs_resolver(
match maybe_node_modules_path {
Some(node_modules_folder) => Arc::new(LocalNpmPackageResolver::new(
npm_cache,
fs,
npm_install_deps_provider.clone(),
progress_bar.clone(),
resolution,
sys,
tarball_cache,
node_modules_folder,
system_info,
@ -49,9 +49,9 @@ pub fn create_npm_fs_resolver(
)),
None => Arc::new(GlobalNpmPackageResolver::new(
npm_cache,
fs,
tarball_cache,
resolution,
sys,
system_info,
lifecycle_scripts,
)),

53
cli/npm/mod.rs
View file

@ -17,7 +17,7 @@ use deno_resolver::npm::ByonmInNpmPackageChecker;
use deno_resolver::npm::ByonmNpmResolver;
use deno_resolver::npm::CliNpmReqResolver;
use deno_resolver::npm::ResolvePkgFolderFromDenoReqError;
use deno_runtime::deno_fs::FileSystem;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use deno_runtime::deno_node::NodePermissions;
use deno_runtime::ops::process::NpmProcessStateProvider;
use deno_semver::package::PackageNv;
@ -30,9 +30,6 @@ use node_resolver::NpmPackageFolderResolver;
use crate::file_fetcher::CliFileFetcher;
use crate::http_util::HttpClientProvider;
use crate::util::fs::atomic_write_file_with_retries_and_fs;
use crate::util::fs::hard_link_dir_recursive;
use crate::util::fs::AtomicWriteFileFsAdapter;
use crate::util::progress_bar::ProgressBar;
pub use self::byonm::CliByonmNpmResolver;
@ -43,26 +40,26 @@ pub use self::managed::CliNpmResolverManagedSnapshotOption;
pub use self::managed::ManagedCliNpmResolver;
pub use self::managed::PackageCaching;
pub type CliNpmTarballCache = deno_npm_cache::TarballCache<CliNpmCacheEnv>;
pub type CliNpmCache = deno_npm_cache::NpmCache<CliNpmCacheEnv>;
pub type CliNpmRegistryInfoProvider =
deno_npm_cache::RegistryInfoProvider<CliNpmCacheEnv>;
pub type CliNpmTarballCache =
deno_npm_cache::TarballCache<CliNpmCacheHttpClient, FsSysTraitsAdapter>;
pub type CliNpmCache = deno_npm_cache::NpmCache<FsSysTraitsAdapter>;
pub type CliNpmRegistryInfoProvider = deno_npm_cache::RegistryInfoProvider<
CliNpmCacheHttpClient,
FsSysTraitsAdapter,
>;
#[derive(Debug)]
pub struct CliNpmCacheEnv {
fs: Arc<dyn FileSystem>,
pub struct CliNpmCacheHttpClient {
http_client_provider: Arc<HttpClientProvider>,
progress_bar: ProgressBar,
}
impl CliNpmCacheEnv {
impl CliNpmCacheHttpClient {
pub fn new(
fs: Arc<dyn FileSystem>,
http_client_provider: Arc<HttpClientProvider>,
progress_bar: ProgressBar,
) -> Self {
Self {
fs,
http_client_provider,
progress_bar,
}
@ -70,35 +67,7 @@ impl CliNpmCacheEnv {
}
#[async_trait::async_trait(?Send)]
impl deno_npm_cache::NpmCacheEnv for CliNpmCacheEnv {
fn exists(&self, path: &Path) -> bool {
self.fs.exists_sync(path)
}
fn hard_link_dir_recursive(
&self,
from: &Path,
to: &Path,
) -> Result<(), AnyError> {
// todo(dsherret): use self.fs here instead
hard_link_dir_recursive(from, to)
}
fn atomic_write_file_with_retries(
&self,
file_path: &Path,
data: &[u8],
) -> std::io::Result<()> {
atomic_write_file_with_retries_and_fs(
&AtomicWriteFileFsAdapter {
fs: self.fs.as_ref(),
write_mode: crate::cache::CACHE_PERM,
},
file_path,
data,
)
}
impl deno_npm_cache::NpmCacheHttpClient for CliNpmCacheHttpClient {
async fn download_with_retries_on_any_tokio_runtime(
&self,
url: Url,

34
cli/ops/lint.rs Normal file
View file

@ -0,0 +1,34 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use deno_ast::MediaType;
use deno_ast::ModuleSpecifier;
use deno_core::error::generic_error;
use deno_core::error::AnyError;
use deno_core::op2;
use crate::tools::lint;
deno_core::extension!(deno_lint, ops = [op_lint_create_serialized_ast,],);
#[op2]
#[buffer]
fn op_lint_create_serialized_ast(
#[string] file_name: &str,
#[string] source: String,
) -> Result<Vec<u8>, AnyError> {
let file_text = deno_ast::strip_bom(source);
let path = std::env::current_dir()?.join(file_name);
let specifier = ModuleSpecifier::from_file_path(&path).map_err(|_| {
generic_error(format!("Failed to parse path as URL: {}", path.display()))
})?;
let media_type = MediaType::from_specifier(&specifier);
let parsed_source = deno_ast::parse_program(deno_ast::ParseParams {
specifier,
text: file_text.into(),
media_type,
capture_tokens: false,
scope_analysis: false,
maybe_syntax: None,
})?;
Ok(lint::serialize_ast_to_buffer(&parsed_source))
}

1
cli/ops/mod.rs
View file

@ -2,4 +2,5 @@
pub mod bench;
pub mod jupyter;
pub mod lint;
pub mod testing;

96
cli/resolver.rs
View file

@ -1,5 +1,10 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use std::borrow::Cow;
use std::path::Path;
use std::path::PathBuf;
use std::sync::Arc;
use async_trait::async_trait;
use dashmap::DashMap;
use dashmap::DashSet;
@ -20,16 +25,14 @@ use deno_npm::resolution::NpmResolutionError;
use deno_resolver::sloppy_imports::SloppyImportsResolver;
use deno_runtime::colors;
use deno_runtime::deno_fs;
use deno_runtime::deno_fs::FileSystem;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use deno_runtime::deno_node::is_builtin_node_module;
use deno_runtime::deno_node::DenoFsNodeResolverEnv;
use deno_runtime::deno_node::RealIsBuiltInNodeModuleChecker;
use deno_semver::package::PackageReq;
use node_resolver::NodeResolutionKind;
use node_resolver::ResolutionMode;
use std::borrow::Cow;
use std::path::Path;
use std::path::PathBuf;
use std::sync::Arc;
use sys_traits::FsMetadata;
use sys_traits::FsMetadataValue;
use thiserror::Error;
use crate::args::NpmCachingStrategy;
@ -40,18 +43,19 @@ use crate::npm::InnerCliNpmResolverRef;
use crate::util::sync::AtomicFlag;
use crate::util::text_encoding::from_utf8_lossy_cow;
pub type CjsTracker = deno_resolver::cjs::CjsTracker<DenoFsNodeResolverEnv>;
pub type IsCjsResolver =
deno_resolver::cjs::IsCjsResolver<DenoFsNodeResolverEnv>;
pub type CjsTracker = deno_resolver::cjs::CjsTracker<FsSysTraitsAdapter>;
pub type IsCjsResolver = deno_resolver::cjs::IsCjsResolver<FsSysTraitsAdapter>;
pub type CliSloppyImportsResolver =
SloppyImportsResolver<SloppyImportsCachedFs>;
pub type CliDenoResolver = deno_resolver::DenoResolver<
CliDenoResolverFs,
DenoFsNodeResolverEnv,
RealIsBuiltInNodeModuleChecker,
SloppyImportsCachedFs,
FsSysTraitsAdapter,
>;
pub type CliNpmReqResolver = deno_resolver::npm::NpmReqResolver<
RealIsBuiltInNodeModuleChecker,
FsSysTraitsAdapter,
>;
pub type CliNpmReqResolver =
deno_resolver::npm::NpmReqResolver<CliDenoResolverFs, DenoFsNodeResolverEnv>;
pub struct ModuleCodeStringSource {
pub code: ModuleSourceCode,
@ -59,53 +63,6 @@ pub struct ModuleCodeStringSource {
pub media_type: MediaType,
}
#[derive(Debug, Clone)]
pub struct CliDenoResolverFs(pub Arc<dyn FileSystem>);
impl deno_resolver::fs::DenoResolverFs for CliDenoResolverFs {
fn read_to_string_lossy(
&self,
path: &Path,
) -> std::io::Result<Cow<'static, str>> {
self
.0
.read_text_file_lossy_sync(path, None)
.map_err(|e| e.into_io_error())
}
fn realpath_sync(&self, path: &Path) -> std::io::Result<PathBuf> {
self.0.realpath_sync(path).map_err(|e| e.into_io_error())
}
fn exists_sync(&self, path: &Path) -> bool {
self.0.exists_sync(path)
}
fn is_dir_sync(&self, path: &Path) -> bool {
self.0.is_dir_sync(path)
}
fn read_dir_sync(
&self,
dir_path: &Path,
) -> std::io::Result<Vec<deno_resolver::fs::DirEntry>> {
self
.0
.read_dir_sync(dir_path)
.map(|entries| {
entries
.into_iter()
.map(|e| deno_resolver::fs::DirEntry {
name: e.name,
is_file: e.is_file,
is_directory: e.is_directory,
})
.collect::<Vec<_>>()
})
.map_err(|err| err.into_io_error())
}
}
#[derive(Debug, Error)]
#[error("{media_type} files are not supported in npm packages: {specifier}")]
pub struct NotSupportedKindInNpmError {
@ -440,7 +397,7 @@ impl<'a> deno_graph::source::NpmResolver for WorkerCliNpmGraphResolver<'a> {
#[derive(Debug)]
pub struct SloppyImportsCachedFs {
fs: Arc<dyn deno_fs::FileSystem>,
sys: FsSysTraitsAdapter,
cache: Option<
DashMap<
PathBuf,
@ -450,15 +407,18 @@ pub struct SloppyImportsCachedFs {
}
impl SloppyImportsCachedFs {
pub fn new(fs: Arc<dyn FileSystem>) -> Self {
pub fn new(sys: FsSysTraitsAdapter) -> Self {
Self {
fs,
sys,
cache: Some(Default::default()),
}
}
pub fn new_without_stat_cache(fs: Arc<dyn FileSystem>) -> Self {
Self { fs, cache: None }
pub fn new_without_stat_cache(fs: FsSysTraitsAdapter) -> Self {
Self {
sys: fs,
cache: None,
}
}
}
@ -475,10 +435,10 @@ impl deno_resolver::sloppy_imports::SloppyImportResolverFs
}
}
let entry = self.fs.stat_sync(path).ok().and_then(|stat| {
if stat.is_file {
let entry = self.sys.fs_metadata(path).ok().and_then(|stat| {
if stat.file_type().is_file() {
Some(deno_resolver::sloppy_imports::SloppyImportsFsEntry::File)
} else if stat.is_directory {
} else if stat.file_type().is_dir() {
Some(deno_resolver::sloppy_imports::SloppyImportsFsEntry::Dir)
} else {
None

1
cli/schemas/config-file.v1.json
View file

@ -446,7 +446,6 @@
},
"command": {
"type": "string",
"required": true,
"description": "The task to execute"
},
"dependencies": {

194
cli/standalone/binary.rs
View file

@ -37,7 +37,6 @@ use deno_core::futures::AsyncReadExt;
use deno_core::futures::AsyncSeekExt;
use deno_core::serde_json;
use deno_core::url::Url;
use deno_graph::source::RealFileSystem;
use deno_graph::ModuleGraph;
use deno_npm::resolution::SerializedNpmResolutionSnapshot;
use deno_npm::resolution::SerializedNpmResolutionSnapshotPackage;
@ -91,6 +90,7 @@ use super::serialization::DenoCompileModuleData;
use super::serialization::DeserializedDataSection;
use super::serialization::RemoteModulesStore;
use super::serialization::RemoteModulesStoreBuilder;
use super::serialization::SourceMapStore;
use super::virtual_fs::output_vfs;
use super::virtual_fs::BuiltVfs;
use super::virtual_fs::FileBackedVfs;
@ -98,6 +98,7 @@ use super::virtual_fs::VfsBuilder;
use super::virtual_fs::VfsFileSubDataKind;
use super::virtual_fs::VfsRoot;
use super::virtual_fs::VirtualDirectory;
use super::virtual_fs::VirtualDirectoryEntries;
use super::virtual_fs::WindowsSystemRootablePath;
pub static DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME: &str =
@ -203,18 +204,25 @@ pub struct Metadata {
pub otel_config: OtelConfig,
}
#[allow(clippy::too_many_arguments)]
fn write_binary_bytes(
mut file_writer: File,
original_bin: Vec<u8>,
metadata: &Metadata,
npm_snapshot: Option<SerializedNpmResolutionSnapshot>,
remote_modules: &RemoteModulesStoreBuilder,
source_map_store: &SourceMapStore,
vfs: &BuiltVfs,
compile_flags: &CompileFlags,
) -> Result<(), AnyError> {
let data_section_bytes =
serialize_binary_data_section(metadata, npm_snapshot, remote_modules, vfs)
.context("Serializing binary data section.")?;
let data_section_bytes = serialize_binary_data_section(
metadata,
npm_snapshot,
remote_modules,
source_map_store,
vfs,
)
.context("Serializing binary data section.")?;
let target = compile_flags.resolve_target();
if target.contains("linux") {
@ -256,6 +264,7 @@ pub struct StandaloneData {
pub modules: StandaloneModules,
pub npm_snapshot: Option<ValidSerializedNpmResolutionSnapshot>,
pub root_path: PathBuf,
pub source_maps: SourceMapStore,
pub vfs: Arc<FileBackedVfs>,
}
@ -283,13 +292,12 @@ impl StandaloneModules {
pub fn read<'a>(
&'a self,
specifier: &'a ModuleSpecifier,
kind: VfsFileSubDataKind,
) -> Result<Option<DenoCompileModuleData<'a>>, AnyError> {
if specifier.scheme() == "file" {
let path = deno_path_util::url_to_file_path(specifier)?;
let bytes = match self.vfs.file_entry(&path) {
Ok(entry) => self
.vfs
.read_file_all(entry, VfsFileSubDataKind::ModuleGraph)?,
Ok(entry) => self.vfs.read_file_all(entry, kind)?,
Err(err) if err.kind() == ErrorKind::NotFound => {
match RealFs.read_file_sync(&path, None) {
Ok(bytes) => bytes,
@ -307,7 +315,18 @@ impl StandaloneModules {
data: bytes,
}))
} else {
self.remote_modules.read(specifier)
self.remote_modules.read(specifier).map(|maybe_entry| {
maybe_entry.map(|entry| DenoCompileModuleData {
media_type: entry.media_type,
specifier: entry.specifier,
data: match kind {
VfsFileSubDataKind::Raw => entry.data,
VfsFileSubDataKind::ModuleGraph => {
entry.transpiled_data.unwrap_or(entry.data)
}
},
})
})
}
}
}
@ -328,7 +347,8 @@ pub fn extract_standalone(
mut metadata,
npm_snapshot,
remote_modules,
mut vfs_dir,
source_maps,
vfs_root_entries,
vfs_files_data,
} = match deserialize_binary_data_section(data)? {
Some(data_section) => data_section,
@ -351,11 +371,12 @@ pub fn extract_standalone(
metadata.argv.push(arg.into_string().unwrap());
}
let vfs = {
// align the name of the directory with the root dir
vfs_dir.name = root_path.file_name().unwrap().to_string_lossy().to_string();
let fs_root = VfsRoot {
dir: vfs_dir,
dir: VirtualDirectory {
// align the name of the directory with the root dir
name: root_path.file_name().unwrap().to_string_lossy().to_string(),
entries: vfs_root_entries,
},
root_path: root_path.clone(),
start_file_offset: 0,
};
@ -372,6 +393,7 @@ pub fn extract_standalone(
},
npm_snapshot,
root_path,
source_maps,
vfs,
}))
}
@ -451,7 +473,7 @@ impl<'a> DenoCompileBinaryWriter<'a> {
)
}
}
self.write_standalone_binary(options, original_binary).await
self.write_standalone_binary(options, original_binary)
}
async fn get_base_binary(
@ -554,7 +576,7 @@ impl<'a> DenoCompileBinaryWriter<'a> {
/// This functions creates a standalone deno binary by appending a bundle
/// and magic trailer to the currently executing binary.
#[allow(clippy::too_many_arguments)]
async fn write_standalone_binary(
fn write_standalone_binary(
&self,
options: WriteBinOptions<'_>,
original_bin: Vec<u8>,
@ -598,71 +620,81 @@ impl<'a> DenoCompileBinaryWriter<'a> {
.with_context(|| format!("Including {}", path.display()))?;
}
let mut remote_modules_store = RemoteModulesStoreBuilder::default();
let mut code_cache_key_hasher = if self.cli_options.code_cache_enabled() {
Some(FastInsecureHasher::new_deno_versioned())
} else {
None
};
let mut source_maps = Vec::with_capacity(graph.specifiers_count());
// todo(dsherret): transpile in parallel
for module in graph.modules() {
if module.specifier().scheme() == "data" {
continue; // don't store data urls as an entry as they're in the code
}
if let Some(hasher) = &mut code_cache_key_hasher {
if let Some(source) = module.source() {
hasher.write(module.specifier().as_str().as_bytes());
hasher.write(source.as_bytes());
}
}
let (maybe_source, media_type) = match module {
let (maybe_original_source, maybe_transpiled, media_type) = match module {
deno_graph::Module::Js(m) => {
let source = if m.media_type.is_emittable() {
let original_bytes = m.source.as_bytes().to_vec();
let maybe_transpiled = if m.media_type.is_emittable() {
let is_cjs = self.cjs_tracker.is_cjs_with_known_is_script(
&m.specifier,
m.media_type,
m.is_script,
)?;
let module_kind = ModuleKind::from_is_cjs(is_cjs);
let source = self
.emitter
.emit_parsed_source(
let (source, source_map) =
self.emitter.emit_parsed_source_for_deno_compile(
&m.specifier,
m.media_type,
module_kind,
&m.source,
)
.await?;
source.into_bytes()
)?;
if source != m.source.as_ref() {
source_maps.push((&m.specifier, source_map));
Some(source.into_bytes())
} else {
None
}
} else {
m.source.as_bytes().to_vec()
None
};
(Some(source), m.media_type)
(Some(original_bytes), maybe_transpiled, m.media_type)
}
deno_graph::Module::Json(m) => {
(Some(m.source.as_bytes().to_vec()), m.media_type)
(Some(m.source.as_bytes().to_vec()), None, m.media_type)
}
deno_graph::Module::Wasm(m) => {
(Some(m.source.to_vec()), MediaType::Wasm)
(Some(m.source.to_vec()), None, MediaType::Wasm)
}
deno_graph::Module::Npm(_)
| deno_graph::Module::Node(_)
| deno_graph::Module::External(_) => (None, MediaType::Unknown),
| deno_graph::Module::External(_) => (None, None, MediaType::Unknown),
};
if module.specifier().scheme() == "file" {
let file_path = deno_path_util::url_to_file_path(module.specifier())?;
vfs
.add_file_with_data(
&file_path,
match maybe_source {
Some(source) => source,
None => RealFs.read_file_sync(&file_path, None)?.into_owned(),
},
VfsFileSubDataKind::ModuleGraph,
)
.with_context(|| {
format!("Failed adding '{}'", file_path.display())
})?;
} else if let Some(source) = maybe_source {
remote_modules_store.add(module.specifier(), media_type, source);
if let Some(original_source) = maybe_original_source {
if module.specifier().scheme() == "file" {
let file_path = deno_path_util::url_to_file_path(module.specifier())?;
vfs
.add_file_with_data(
&file_path,
original_source,
VfsFileSubDataKind::Raw,
)
.with_context(|| {
format!("Failed adding '{}'", file_path.display())
})?;
if let Some(transpiled_source) = maybe_transpiled {
vfs
.add_file_with_data(
&file_path,
transpiled_source,
VfsFileSubDataKind::ModuleGraph,
)
.with_context(|| {
format!("Failed adding '{}'", file_path.display())
})?;
}
} else {
remote_modules_store.add(
module.specifier(),
media_type,
original_source,
maybe_transpiled,
);
}
}
}
remote_modules_store.add_redirects(&graph.redirects);
@ -695,6 +727,28 @@ impl<'a> DenoCompileBinaryWriter<'a> {
None => StandaloneRelativeFileBaseUrl::WindowsSystemRoot,
};
let code_cache_key = if self.cli_options.code_cache_enabled() {
let mut hasher = FastInsecureHasher::new_deno_versioned();
for module in graph.modules() {
if let Some(source) = module.source() {
hasher
.write(root_dir_url.specifier_key(module.specifier()).as_bytes());
hasher.write(source.as_bytes());
}
}
Some(hasher.finish())
} else {
None
};
let mut source_map_store = SourceMapStore::with_capacity(source_maps.len());
for (specifier, source_map) in source_maps {
source_map_store.add(
Cow::Owned(root_dir_url.specifier_key(specifier).into_owned()),
Cow::Owned(source_map.into_bytes()),
);
}
let node_modules = match self.npm_resolver.as_inner() {
InnerCliNpmResolverRef::Managed(_) => {
npm_snapshot.as_ref().map(|_| NodeModules::Managed {
@ -742,7 +796,7 @@ impl<'a> DenoCompileBinaryWriter<'a> {
let metadata = Metadata {
argv: compile_flags.args.clone(),
seed: self.cli_options.seed(),
code_cache_key: code_cache_key_hasher.map(|h| h.finish()),
code_cache_key,
location: self.cli_options.location_flag().clone(),
permissions: self.cli_options.permission_flags().clone(),
v8_flags: self.cli_options.v8_flags().clone(),
@ -809,6 +863,7 @@ impl<'a> DenoCompileBinaryWriter<'a> {
&metadata,
npm_snapshot.map(|s| s.into_serialized()),
&remote_modules_store,
&source_map_store,
&vfs,
compile_flags,
)
@ -903,10 +958,10 @@ impl<'a> DenoCompileBinaryWriter<'a> {
root_dir.name = DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME.to_string();
let mut new_entries = Vec::with_capacity(root_dir.entries.len());
let mut localhost_entries = IndexMap::new();
for entry in std::mem::take(&mut root_dir.entries) {
for entry in root_dir.entries.take_inner() {
match entry {
VfsEntry::Dir(dir) => {
for entry in dir.entries {
VfsEntry::Dir(mut dir) => {
for entry in dir.entries.take_inner() {
log::debug!("Flattening {} into node_modules", entry.name());
if let Some(existing) =
localhost_entries.insert(entry.name().to_string(), entry)
@ -925,11 +980,11 @@ impl<'a> DenoCompileBinaryWriter<'a> {
}
new_entries.push(VfsEntry::Dir(VirtualDirectory {
name: "localhost".to_string(),
entries: localhost_entries.into_iter().map(|(_, v)| v).collect(),
entries: VirtualDirectoryEntries::new(
localhost_entries.into_iter().map(|(_, v)| v).collect(),
),
}));
// needs to be sorted by name
new_entries.sort_by(|a, b| a.name().cmp(b.name()));
root_dir.entries = new_entries;
root_dir.entries = VirtualDirectoryEntries::new(new_entries);
// it's better to not expose the user's cache directory, so take it out
// of there
@ -937,10 +992,7 @@ impl<'a> DenoCompileBinaryWriter<'a> {
let parent_dir = vfs.get_dir_mut(parent).unwrap();
let index = parent_dir
.entries
.iter()
.position(|entry| {
entry.name() == DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME
})
.binary_search(DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME)
.unwrap();
let npm_global_cache_dir_entry = parent_dir.entries.remove(index);
@ -950,11 +1002,7 @@ impl<'a> DenoCompileBinaryWriter<'a> {
Cow::Borrowed(DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME);
for ancestor in parent.ancestors() {
let dir = vfs.get_dir_mut(ancestor).unwrap();
if let Some(index) = dir
.entries
.iter()
.position(|entry| entry.name() == last_name)
{
if let Ok(index) = dir.entries.binary_search(&last_name) {
dir.entries.remove(index);
}
last_name = Cow::Owned(dir.name.clone());
@ -965,7 +1013,7 @@ impl<'a> DenoCompileBinaryWriter<'a> {
// now build the vfs and add the global cache dir entry there
let mut built_vfs = vfs.build();
built_vfs.root.insert_entry(npm_global_cache_dir_entry);
built_vfs.entries.insert(npm_global_cache_dir_entry);
built_vfs
}
InnerCliNpmResolverRef::Byonm(_) => vfs.build(),

6
cli/standalone/code_cache.rs
View file

@ -15,11 +15,12 @@ use deno_core::anyhow::bail;
use deno_core::error::AnyError;
use deno_core::parking_lot::Mutex;
use deno_core::unsync::sync::AtomicFlag;
use deno_path_util::get_atomic_path;
use deno_runtime::code_cache::CodeCache;
use deno_runtime::code_cache::CodeCacheType;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use crate::cache::FastInsecureHasher;
use crate::util::path::get_atomic_file_path;
use crate::worker::CliCodeCache;
enum CodeCacheStrategy {
@ -189,7 +190,8 @@ impl FirstRunCodeCacheStrategy {
cache_data: &HashMap<CodeCacheKey, DenoCompileCodeCacheEntry>,
) {
let count = cache_data.len();
let temp_file = get_atomic_file_path(&self.file_path);
let temp_file =
get_atomic_path(&FsSysTraitsAdapter::new_real(), &self.file_path);
match serialize(&temp_file, self.cache_key, cache_data) {
Ok(()) => {
if let Err(err) = std::fs::rename(&temp_file, &self.file_path) {

1
cli/standalone/file_system.rs
View file

@ -9,6 +9,7 @@ use deno_runtime::deno_fs::AccessCheckCb;
use deno_runtime::deno_fs::FileSystem;
use deno_runtime::deno_fs::FsDirEntry;
use deno_runtime::deno_fs::FsFileType;
use deno_runtime::deno_fs::FsStatSlim;
use deno_runtime::deno_fs::OpenOptions;
use deno_runtime::deno_fs::RealFs;
use deno_runtime::deno_io::fs::File;

96
cli/standalone/mod.rs
View file

@ -36,10 +36,12 @@ use deno_package_json::PackageJsonDepValue;
use deno_resolver::cjs::IsCjsResolutionMode;
use deno_resolver::npm::NpmReqResolverOptions;
use deno_runtime::deno_fs;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use deno_runtime::deno_node::create_host_defined_options;
use deno_runtime::deno_node::NodeRequireLoader;
use deno_runtime::deno_node::NodeResolver;
use deno_runtime::deno_node::PackageJsonResolver;
use deno_runtime::deno_node::RealIsBuiltInNodeModuleChecker;
use deno_runtime::deno_permissions::Permissions;
use deno_runtime::deno_permissions::PermissionsContainer;
use deno_runtime::deno_tls::rustls::RootCertStore;
@ -55,6 +57,7 @@ use node_resolver::errors::ClosestPkgJsonError;
use node_resolver::NodeResolutionKind;
use node_resolver::ResolutionMode;
use serialization::DenoCompileModuleSource;
use serialization::SourceMapStore;
use std::borrow::Cow;
use std::rc::Rc;
use std::sync::Arc;
@ -68,11 +71,9 @@ use crate::args::CaData;
use crate::args::NpmInstallDepsProvider;
use crate::args::StorageKeyResolver;
use crate::cache::Caches;
use crate::cache::DenoCacheEnvFsAdapter;
use crate::cache::DenoDirProvider;
use crate::cache::FastInsecureHasher;
use crate::cache::NodeAnalysisCache;
use crate::cache::RealDenoCacheEnv;
use crate::http_util::HttpClientProvider;
use crate::node::CliCjsCodeAnalyzer;
use crate::node::CliNodeCodeTranslator;
@ -86,7 +87,6 @@ use crate::npm::CliNpmResolverCreateOptions;
use crate::npm::CliNpmResolverManagedSnapshotOption;
use crate::npm::CreateInNpmPkgCheckerOptions;
use crate::resolver::CjsTracker;
use crate::resolver::CliDenoResolverFs;
use crate::resolver::CliNpmReqResolver;
use crate::resolver::NpmModuleLoader;
use crate::util::progress_bar::ProgressBar;
@ -122,6 +122,7 @@ struct SharedModuleLoaderState {
npm_module_loader: Arc<NpmModuleLoader>,
npm_req_resolver: Arc<CliNpmReqResolver>,
npm_resolver: Arc<dyn CliNpmResolver>,
source_maps: SourceMapStore,
vfs: Arc<FileBackedVfs>,
workspace_resolver: WorkspaceResolver,
}
@ -396,7 +397,11 @@ impl ModuleLoader for EmbeddedModuleLoader {
);
}
match self.shared.modules.read(original_specifier) {
match self
.shared
.modules
.read(original_specifier, VfsFileSubDataKind::ModuleGraph)
{
Ok(Some(module)) => {
let media_type = module.media_type;
let (module_specifier, module_type, module_source) =
@ -495,6 +500,45 @@ impl ModuleLoader for EmbeddedModuleLoader {
}
std::future::ready(()).boxed_local()
}
fn get_source_map(&self, file_name: &str) -> Option<Cow<[u8]>> {
if file_name.starts_with("file:///") {
let url =
deno_path_util::url_from_directory_path(self.shared.vfs.root()).ok()?;
let file_url = ModuleSpecifier::parse(file_name).ok()?;
let relative_path = url.make_relative(&file_url)?;
self.shared.source_maps.get(&relative_path)
} else {
self.shared.source_maps.get(file_name)
}
.map(Cow::Borrowed)
}
fn get_source_mapped_source_line(
&self,
file_name: &str,
line_number: usize,
) -> Option<String> {
let specifier = ModuleSpecifier::parse(file_name).ok()?;
let data = self
.shared
.modules
.read(&specifier, VfsFileSubDataKind::Raw)
.ok()??;
let source = String::from_utf8_lossy(&data.data);
// Do NOT use .lines(): it skips the terminating empty line.
// (due to internally using_terminator() instead of .split())
let lines: Vec<&str> = source.split('\n').collect();
if line_number >= lines.len() {
Some(format!(
"{} Couldn't format source line: Line {} is out of bounds (source may have changed at runtime)",
crate::colors::yellow("Warning"), line_number + 1,
))
} else {
Some(lines[line_number].to_string())
}
}
}
impl NodeRequireLoader for EmbeddedModuleLoader {
@ -590,6 +634,7 @@ pub async fn run(data: StandaloneData) -> Result<i32, AnyError> {
modules,
npm_snapshot,
root_path,
source_maps,
vfs,
} = data;
let deno_dir_provider = Arc::new(DenoDirProvider::new(None));
@ -610,9 +655,8 @@ pub async fn run(data: StandaloneData) -> Result<i32, AnyError> {
let main_module = root_dir_url.join(&metadata.entrypoint_key).unwrap();
let npm_global_cache_dir = root_path.join(".deno_compile_node_modules");
let cache_setting = CacheSetting::Only;
let pkg_json_resolver = Arc::new(PackageJsonResolver::new(
deno_runtime::deno_node::DenoFsNodeResolverEnv::new(fs.clone()),
));
let sys = FsSysTraitsAdapter(fs.clone());
let pkg_json_resolver = Arc::new(PackageJsonResolver::new(sys.clone()));
let (in_npm_pkg_checker, npm_resolver) = match metadata.node_modules {
Some(binary::NodeModules::Managed { node_modules_dir }) => {
// create an npmrc that uses the fake npm_registry_url to resolve packages
@ -625,7 +669,7 @@ pub async fn run(data: StandaloneData) -> Result<i32, AnyError> {
registry_configs: Default::default(),
});
let npm_cache_dir = Arc::new(NpmCacheDir::new(
&DenoCacheEnvFsAdapter(fs.as_ref()),
&sys,
npm_global_cache_dir,
npmrc.get_all_known_registries_urls(),
));
@ -646,17 +690,17 @@ pub async fn run(data: StandaloneData) -> Result<i32, AnyError> {
snapshot,
)),
maybe_lockfile: None,
fs: fs.clone(),
http_client_provider: http_client_provider.clone(),
npm_cache_dir,
cache_setting,
text_only_progress_bar: progress_bar,
maybe_node_modules_path,
npm_system_info: Default::default(),
npm_install_deps_provider: Arc::new(
// this is only used for installing packages, which isn't necessary with deno compile
NpmInstallDepsProvider::empty(),
),
sys: sys.clone(),
text_only_progress_bar: progress_bar,
cache_setting,
maybe_node_modules_path,
npm_system_info: Default::default(),
npmrc,
lifecycle_scripts: Default::default(),
},
@ -673,7 +717,7 @@ pub async fn run(data: StandaloneData) -> Result<i32, AnyError> {
create_in_npm_pkg_checker(CreateInNpmPkgCheckerOptions::Byonm);
let npm_resolver = create_cli_npm_resolver(
CliNpmResolverCreateOptions::Byonm(CliByonmNpmResolverCreateOptions {
fs: CliDenoResolverFs(fs.clone()),
sys: sys.clone(),
pkg_json_resolver: pkg_json_resolver.clone(),
root_node_modules_dir,
}),
@ -686,7 +730,7 @@ pub async fn run(data: StandaloneData) -> Result<i32, AnyError> {
// so no need to create actual `.npmrc` configuration.
let npmrc = create_default_npmrc();
let npm_cache_dir = Arc::new(NpmCacheDir::new(
&DenoCacheEnvFsAdapter(fs.as_ref()),
&sys,
npm_global_cache_dir,
npmrc.get_all_known_registries_urls(),
));
@ -701,18 +745,18 @@ pub async fn run(data: StandaloneData) -> Result<i32, AnyError> {
create_cli_npm_resolver(CliNpmResolverCreateOptions::Managed(
CliManagedNpmResolverCreateOptions {
snapshot: CliNpmResolverManagedSnapshotOption::Specified(None),
maybe_lockfile: None,
fs: fs.clone(),
http_client_provider: http_client_provider.clone(),
npm_cache_dir,
cache_setting,
text_only_progress_bar: progress_bar,
maybe_node_modules_path: None,
npm_system_info: Default::default(),
npm_install_deps_provider: Arc::new(
// this is only used for installing packages, which isn't necessary with deno compile
NpmInstallDepsProvider::empty(),
),
sys: sys.clone(),
cache_setting,
text_only_progress_bar: progress_bar,
npm_cache_dir,
maybe_lockfile: None,
maybe_node_modules_path: None,
npm_system_info: Default::default(),
npmrc: create_default_npmrc(),
lifecycle_scripts: Default::default(),
},
@ -724,10 +768,11 @@ pub async fn run(data: StandaloneData) -> Result<i32, AnyError> {
let has_node_modules_dir = npm_resolver.root_node_modules_path().is_some();
let node_resolver = Arc::new(NodeResolver::new(
deno_runtime::deno_node::DenoFsNodeResolverEnv::new(fs.clone()),
in_npm_pkg_checker.clone(),
RealIsBuiltInNodeModuleChecker,
npm_resolver.clone().into_npm_pkg_folder_resolver(),
pkg_json_resolver.clone(),
sys.clone(),
));
let cjs_tracker = Arc::new(CjsTracker::new(
in_npm_pkg_checker.clone(),
@ -745,7 +790,7 @@ pub async fn run(data: StandaloneData) -> Result<i32, AnyError> {
let npm_req_resolver =
Arc::new(CliNpmReqResolver::new(NpmReqResolverOptions {
byonm_resolver: (npm_resolver.clone()).into_maybe_byonm(),
fs: CliDenoResolverFs(fs.clone()),
sys: sys.clone(),
in_npm_pkg_checker: in_npm_pkg_checker.clone(),
node_resolver: node_resolver.clone(),
npm_req_resolver: npm_resolver.clone().into_npm_req_resolver(),
@ -758,11 +803,11 @@ pub async fn run(data: StandaloneData) -> Result<i32, AnyError> {
);
let node_code_translator = Arc::new(NodeCodeTranslator::new(
cjs_esm_code_analyzer,
deno_runtime::deno_node::DenoFsNodeResolverEnv::new(fs.clone()),
in_npm_pkg_checker,
node_resolver.clone(),
npm_resolver.clone().into_npm_pkg_folder_resolver(),
pkg_json_resolver.clone(),
sys,
));
let workspace_resolver = {
let import_map = match metadata.workspace_resolver.import_map {
@ -841,6 +886,7 @@ pub async fn run(data: StandaloneData) -> Result<i32, AnyError> {
)),
npm_resolver: npm_resolver.clone(),
npm_req_resolver,
source_maps,
vfs,
workspace_resolver,
}),

225
cli/standalone/serialization.rs
View file

@ -6,6 +6,8 @@ use std::collections::BTreeMap;
use std::collections::HashMap;
use std::io::Write;
use capacity_builder::BytesAppendable;
use deno_ast::swc::common::source_map;
use deno_ast::MediaType;
use deno_core::anyhow::bail;
use deno_core::anyhow::Context;
@ -20,12 +22,15 @@ use deno_npm::resolution::SerializedNpmResolutionSnapshotPackage;
use deno_npm::resolution::ValidSerializedNpmResolutionSnapshot;
use deno_npm::NpmPackageId;
use deno_semver::package::PackageReq;
use deno_semver::StackString;
use indexmap::IndexMap;
use crate::standalone::virtual_fs::VirtualDirectory;
use super::binary::Metadata;
use super::virtual_fs::BuiltVfs;
use super::virtual_fs::VfsBuilder;
use super::virtual_fs::VirtualDirectoryEntries;
const MAGIC_BYTES: &[u8; 8] = b"d3n0l4nd";
@ -33,21 +38,22 @@ const MAGIC_BYTES: &[u8; 8] = b"d3n0l4nd";
/// * d3n0l4nd
/// * <metadata_len><metadata>
/// * <npm_snapshot_len><npm_snapshot>
/// * <remote_modules_len><remote_modules>
/// * <remote_modules>
/// * <vfs_headers_len><vfs_headers>
/// * <vfs_file_data_len><vfs_file_data>
/// * <source_map_data>
/// * d3n0l4nd
pub fn serialize_binary_data_section(
metadata: &Metadata,
npm_snapshot: Option<SerializedNpmResolutionSnapshot>,
remote_modules: &RemoteModulesStoreBuilder,
source_map_store: &SourceMapStore,
vfs: &BuiltVfs,
) -> Result<Vec<u8>, AnyError> {
let metadata = serde_json::to_string(metadata)?;
let npm_snapshot =
npm_snapshot.map(serialize_npm_snapshot).unwrap_or_default();
let remote_modules_len = Cell::new(0_u64);
let serialized_vfs = serde_json::to_string(&vfs.root)?;
let serialized_vfs = serde_json::to_string(&vfs.entries)?;
let bytes = capacity_builder::BytesBuilder::build(|builder| {
builder.append(MAGIC_BYTES);
@ -63,10 +69,7 @@ pub fn serialize_binary_data_section(
}
// 3. Remote modules
{
builder.append_le(remote_modules_len.get()); // this will be properly initialized on the second pass
let start_index = builder.len();
remote_modules.write(builder);
remote_modules_len.set((builder.len() - start_index) as u64);
}
// 4. VFS
{
@ -78,6 +81,16 @@ pub fn serialize_binary_data_section(
builder.append(file);
}
}
// 5. Source maps
{
builder.append_le(source_map_store.data.len() as u32);
for (specifier, source_map) in &source_map_store.data {
builder.append_le(specifier.len() as u32);
builder.append(specifier);
builder.append_le(source_map.len() as u32);
builder.append(source_map.as_ref());
}
}
// write the magic bytes at the end so we can use it
// to make sure we've deserialized correctly
@ -91,19 +104,14 @@ pub struct DeserializedDataSection {
pub metadata: Metadata,
pub npm_snapshot: Option<ValidSerializedNpmResolutionSnapshot>,
pub remote_modules: RemoteModulesStore,
pub vfs_dir: VirtualDirectory,
pub source_maps: SourceMapStore,
pub vfs_root_entries: VirtualDirectoryEntries,
pub vfs_files_data: &'static [u8],
}
pub fn deserialize_binary_data_section(
data: &'static [u8],
) -> Result<Option<DeserializedDataSection>, AnyError> {
fn read_bytes_with_len(input: &[u8]) -> Result<(&[u8], &[u8]), AnyError> {
let (input, len) = read_u64(input)?;
let (input, data) = read_bytes(input, len as usize)?;
Ok((input, data))
}
fn read_magic_bytes(input: &[u8]) -> Result<(&[u8], bool), AnyError> {
if input.len() < MAGIC_BYTES.len() {
bail!("Unexpected end of data. Could not find magic bytes.");
@ -115,34 +123,51 @@ pub fn deserialize_binary_data_section(
Ok((input, true))
}
#[allow(clippy::type_complexity)]
fn read_source_map_entry(
input: &[u8],
) -> Result<(&[u8], (Cow<str>, &[u8])), AnyError> {
let (input, specifier) = read_string_lossy(input)?;
let (input, source_map) = read_bytes_with_u32_len(input)?;
Ok((input, (specifier, source_map)))
}
let (input, found) = read_magic_bytes(data)?;
if !found {
return Ok(None);
}
// 1. Metadata
let (input, data) = read_bytes_with_len(input).context("reading metadata")?;
let (input, data) =
read_bytes_with_u64_len(input).context("reading metadata")?;
let metadata: Metadata =
serde_json::from_slice(data).context("deserializing metadata")?;
// 2. Npm snapshot
let (input, data) =
read_bytes_with_len(input).context("reading npm snapshot")?;
read_bytes_with_u64_len(input).context("reading npm snapshot")?;
let npm_snapshot = if data.is_empty() {
None
} else {
Some(deserialize_npm_snapshot(data).context("deserializing npm snapshot")?)
};
// 3. Remote modules
let (input, data) =
read_bytes_with_len(input).context("reading remote modules data")?;
let remote_modules =
RemoteModulesStore::build(data).context("deserializing remote modules")?;
let (input, remote_modules) =
RemoteModulesStore::build(input).context("deserializing remote modules")?;
// 4. VFS
let (input, data) = read_bytes_with_len(input).context("vfs")?;
let vfs_dir: VirtualDirectory =
let (input, data) = read_bytes_with_u64_len(input).context("vfs")?;
let vfs_root_entries: VirtualDirectoryEntries =
serde_json::from_slice(data).context("deserializing vfs data")?;
let (input, vfs_files_data) =
read_bytes_with_len(input).context("reading vfs files data")?;
read_bytes_with_u64_len(input).context("reading vfs files data")?;
// 5. Source maps
let (mut input, source_map_data_len) = read_u32_as_usize(input)?;
let mut source_maps = SourceMapStore::with_capacity(source_map_data_len);
for _ in 0..source_map_data_len {
let (current_input, (specifier, source_map)) =
read_source_map_entry(input)?;
input = current_input;
source_maps.add(specifier, Cow::Borrowed(source_map));
}
// finally ensure we read the magic bytes at the end
let (_input, found) = read_magic_bytes(input)?;
@ -154,7 +179,8 @@ pub fn deserialize_binary_data_section(
metadata,
npm_snapshot,
remote_modules,
vfs_dir,
source_maps,
vfs_root_entries,
vfs_files_data,
}))
}
@ -162,19 +188,31 @@ pub fn deserialize_binary_data_section(
#[derive(Default)]
pub struct RemoteModulesStoreBuilder {
specifiers: Vec<(String, u64)>,
data: Vec<(MediaType, Vec<u8>)>,
data: Vec<(MediaType, Vec<u8>, Option<Vec<u8>>)>,
data_byte_len: u64,
redirects: Vec<(String, String)>,
redirects_len: u64,
}
impl RemoteModulesStoreBuilder {
pub fn add(&mut self, specifier: &Url, media_type: MediaType, data: Vec<u8>) {
pub fn add(
&mut self,
specifier: &Url,
media_type: MediaType,
data: Vec<u8>,
maybe_transpiled: Option<Vec<u8>>,
) {
log::debug!("Adding '{}' ({})", specifier, media_type);
let specifier = specifier.to_string();
self.specifiers.push((specifier, self.data_byte_len));
self.data_byte_len += 1 + 8 + data.len() as u64; // media type (1 byte), data length (8 bytes), data
self.data.push((media_type, data));
let maybe_transpiled_len = match &maybe_transpiled {
// data length (4 bytes), data
Some(data) => 4 + data.len() as u64,
None => 0,
};
// media type (1 byte), data length (4 bytes), data, has transpiled (1 byte), transpiled length
self.data_byte_len += 1 + 4 + data.len() as u64 + 1 + maybe_transpiled_len;
self.data.push((media_type, data, maybe_transpiled));
}
pub fn add_redirects(&mut self, redirects: &BTreeMap<Url, Url>) {
@ -188,12 +226,15 @@ impl RemoteModulesStoreBuilder {
}
}
fn write<'a>(&'a self, builder: &mut capacity_builder::BytesBuilder<'a>) {
fn write<'a, TBytes: capacity_builder::BytesType>(
&'a self,
builder: &mut capacity_builder::BytesBuilder<'a, TBytes>,
) {
builder.append_le(self.specifiers.len() as u32);
builder.append_le(self.redirects.len() as u32);
for (specifier, offset) in &self.specifiers {
builder.append_le(specifier.len() as u32);
builder.append(specifier.as_bytes());
builder.append(specifier);
builder.append_le(*offset);
}
for (from, to) in &self.redirects {
@ -202,10 +243,32 @@ impl RemoteModulesStoreBuilder {
builder.append_le(to.len() as u32);
builder.append(to);
}
for (media_type, data) in &self.data {
builder.append_le(
self
.data
.iter()
.map(|(_, data, maybe_transpiled)| {
1 + 4
+ (data.len() as u64)
+ 1
+ match maybe_transpiled {
Some(transpiled) => 4 + (transpiled.len() as u64),
None => 0,
}
})
.sum::<u64>(),
);
for (media_type, data, maybe_transpiled) in &self.data {
builder.append(serialize_media_type(*media_type));
builder.append_le(data.len() as u64);
builder.append_le(data.len() as u32);
builder.append(data);
if let Some(transpiled) = maybe_transpiled {
builder.append(1);
builder.append_le(transpiled.len() as u32);
builder.append(transpiled);
} else {
builder.append(0);
}
}
}
}
@ -234,6 +297,30 @@ impl DenoCompileModuleSource {
}
}
pub struct SourceMapStore {
data: IndexMap<Cow<'static, str>, Cow<'static, [u8]>>,
}
impl SourceMapStore {
pub fn with_capacity(capacity: usize) -> Self {
Self {
data: IndexMap::with_capacity(capacity),
}
}
pub fn add(
&mut self,
specifier: Cow<'static, str>,
source_map: Cow<'static, [u8]>,
) {
self.data.insert(specifier, source_map);
}
pub fn get(&self, specifier: &str) -> Option<&[u8]> {
self.data.get(specifier).map(|v| v.as_ref())
}
}
pub struct DenoCompileModuleData<'a> {
pub specifier: &'a Url,
pub media_type: MediaType,
@ -280,6 +367,13 @@ impl<'a> DenoCompileModuleData<'a> {
}
}
pub struct RemoteModuleEntry<'a> {
pub specifier: &'a Url,
pub media_type: MediaType,
pub data: Cow<'static, [u8]>,
pub transpiled_data: Option<Cow<'static, [u8]>>,
}
enum RemoteModulesStoreSpecifierValue {
Data(usize),
Redirect(Url),
@ -291,7 +385,7 @@ pub struct RemoteModulesStore {
}
impl RemoteModulesStore {
fn build(data: &'static [u8]) -> Result<Self, AnyError> {
fn build(input: &'static [u8]) -> Result<(&'static [u8], Self), AnyError> {
fn read_specifier(input: &[u8]) -> Result<(&[u8], (Url, u64)), AnyError> {
let (input, specifier) = read_string_lossy(input)?;
let specifier = Url::parse(&specifier)?;
@ -334,12 +428,16 @@ impl RemoteModulesStore {
Ok((input, specifiers))
}
let (files_data, specifiers) = read_headers(data)?;
let (input, specifiers) = read_headers(input)?;
let (input, files_data) = read_bytes_with_u64_len(input)?;
Ok(Self {
specifiers,
files_data,
})
Ok((
input,
Self {
specifiers,
files_data,
},
))
}
pub fn resolve_specifier<'a>(
@ -370,7 +468,7 @@ impl RemoteModulesStore {
pub fn read<'a>(
&'a self,
original_specifier: &'a Url,
) -> Result<Option<DenoCompileModuleData<'a>>, AnyError> {
) -> Result<Option<RemoteModuleEntry<'a>>, AnyError> {
let mut count = 0;
let mut specifier = original_specifier;
loop {
@ -386,12 +484,25 @@ impl RemoteModulesStore {
let input = &self.files_data[*offset..];
let (input, media_type_byte) = read_bytes(input, 1)?;
let media_type = deserialize_media_type(media_type_byte[0])?;
let (input, len) = read_u64(input)?;
let (_input, data) = read_bytes(input, len as usize)?;
return Ok(Some(DenoCompileModuleData {
let (input, data) = read_bytes_with_u32_len(input)?;
check_has_len(input, 1)?;
let (input, has_transpiled) = (&input[1..], input[0]);
let (_, transpiled_data) = match has_transpiled {
0 => (input, None),
1 => {
let (input, data) = read_bytes_with_u32_len(input)?;
(input, Some(data))
}
value => bail!(
"Invalid transpiled data flag: {}. Compiled data is corrupt.",
value
),
};
return Ok(Some(RemoteModuleEntry {
specifier,
media_type,
data: Cow::Borrowed(data),
transpiled_data: transpiled_data.map(Cow::Borrowed),
}));
}
None => {
@ -475,12 +586,13 @@ fn deserialize_npm_snapshot(
#[allow(clippy::needless_lifetimes)] // clippy bug
fn parse_package_dep<'a>(
id_to_npm_id: &'a impl Fn(usize) -> Result<NpmPackageId, AnyError>,
) -> impl Fn(&[u8]) -> Result<(&[u8], (String, NpmPackageId)), AnyError> + 'a
) -> impl Fn(&[u8]) -> Result<(&[u8], (StackString, NpmPackageId)), AnyError> + 'a
{
|input| {
let (input, req) = read_string_lossy(input)?;
let (input, id) = read_u32_as_usize(input)?;
Ok((input, (req.into_owned(), id_to_npm_id(id)?)))
let req = StackString::from_cow(req);
Ok((input, (req, id_to_npm_id(id)?)))
}
}
@ -630,17 +742,34 @@ fn parse_vec_n_times_with_index<TResult>(
Ok((input, results))
}
fn read_bytes_with_u64_len(input: &[u8]) -> Result<(&[u8], &[u8]), AnyError> {
let (input, len) = read_u64(input)?;
let (input, data) = read_bytes(input, len as usize)?;
Ok((input, data))
}
fn read_bytes_with_u32_len(input: &[u8]) -> Result<(&[u8], &[u8]), AnyError> {
let (input, len) = read_u32_as_usize(input)?;
let (input, data) = read_bytes(input, len)?;
Ok((input, data))
}
fn read_bytes(input: &[u8], len: usize) -> Result<(&[u8], &[u8]), AnyError> {
if input.len() < len {
bail!("Unexpected end of data.",);
}
check_has_len(input, len)?;
let (len_bytes, input) = input.split_at(len);
Ok((input, len_bytes))
}
#[inline(always)]
fn check_has_len(input: &[u8], len: usize) -> Result<(), AnyError> {
if input.len() < len {
bail!("Unexpected end of data.");
}
Ok(())
}
fn read_string_lossy(input: &[u8]) -> Result<(&[u8], Cow<str>), AnyError> {
let (input, str_len) = read_u32_as_usize(input)?;
let (input, data_bytes) = read_bytes(input, str_len)?;
let (input, data_bytes) = read_bytes_with_u32_len(input)?;
Ok((input, String::from_utf8_lossy(data_bytes)))
}

159
cli/standalone/virtual_fs.rs
View file

@ -67,7 +67,7 @@ impl WindowsSystemRootablePath {
#[derive(Debug)]
pub struct BuiltVfs {
pub root_path: WindowsSystemRootablePath,
pub root: VirtualDirectory,
pub entries: VirtualDirectoryEntries,
pub files: Vec<Vec<u8>>,
}
@ -95,7 +95,7 @@ impl VfsBuilder {
Self {
executable_root: VirtualDirectory {
name: "/".to_string(),
entries: Vec::new(),
entries: Default::default(),
},
files: Vec::new(),
current_offset: 0,
@ -208,23 +208,20 @@ impl VfsBuilder {
continue;
}
let name = component.as_os_str().to_string_lossy();
let index = match current_dir
.entries
.binary_search_by(|e| e.name().cmp(&name))
{
let index = match current_dir.entries.binary_search(&name) {
Ok(index) => index,
Err(insert_index) => {
current_dir.entries.insert(
current_dir.entries.0.insert(
insert_index,
VfsEntry::Dir(VirtualDirectory {
name: name.to_string(),
entries: Vec::new(),
entries: Default::default(),
}),
);
insert_index
}
};
match &mut current_dir.entries[index] {
match &mut current_dir.entries.0[index] {
VfsEntry::Dir(dir) => {
current_dir = dir;
}
@ -248,14 +245,8 @@ impl VfsBuilder {
continue;
}
let name = component.as_os_str().to_string_lossy();
let index = match current_dir
.entries
.binary_search_by(|e| e.name().cmp(&name))
{
Ok(index) => index,
Err(_) => return None,
};
match &mut current_dir.entries[index] {
let entry = current_dir.entries.get_mut_by_name(&name)?;
match entry {
VfsEntry::Dir(dir) => {
current_dir = dir;
}
@ -320,9 +311,9 @@ impl VfsBuilder {
offset,
len: data.len() as u64,
};
match dir.entries.binary_search_by(|e| e.name().cmp(&name)) {
match dir.entries.binary_search(&name) {
Ok(index) => {
let entry = &mut dir.entries[index];
let entry = &mut dir.entries.0[index];
match entry {
VfsEntry::File(virtual_file) => match sub_data_kind {
VfsFileSubDataKind::Raw => {
@ -336,7 +327,7 @@ impl VfsBuilder {
}
}
Err(insert_index) => {
dir.entries.insert(
dir.entries.0.insert(
insert_index,
VfsEntry::File(VirtualFile {
name: name.to_string(),
@ -384,10 +375,10 @@ impl VfsBuilder {
let target = normalize_path(path.parent().unwrap().join(&target));
let dir = self.add_dir_raw(path.parent().unwrap());
let name = path.file_name().unwrap().to_string_lossy();
match dir.entries.binary_search_by(|e| e.name().cmp(&name)) {
match dir.entries.binary_search(&name) {
Ok(_) => {} // previously inserted
Err(insert_index) => {
dir.entries.insert(
dir.entries.0.insert(
insert_index,
VfsEntry::Symlink(VirtualSymlink {
name: name.to_string(),
@ -426,7 +417,7 @@ impl VfsBuilder {
dir: &mut VirtualDirectory,
parts: &[String],
) {
for entry in &mut dir.entries {
for entry in &mut dir.entries.0 {
match entry {
VfsEntry::Dir(dir) => {
strip_prefix_from_symlinks(dir, parts);
@ -454,13 +445,13 @@ impl VfsBuilder {
if self.min_root_dir.as_ref() == Some(&current_path) {
break;
}
match &current_dir.entries[0] {
match &current_dir.entries.0[0] {
VfsEntry::Dir(dir) => {
if dir.name == DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME {
// special directory we want to maintain
break;
}
match current_dir.entries.remove(0) {
match current_dir.entries.0.remove(0) {
VfsEntry::Dir(dir) => {
current_path =
WindowsSystemRootablePath::Path(current_path.join(&dir.name));
@ -480,7 +471,7 @@ impl VfsBuilder {
}
BuiltVfs {
root_path: current_path,
root: current_dir,
entries: current_dir.entries,
files: self.files,
}
}
@ -506,7 +497,7 @@ pub fn output_vfs(vfs: &BuiltVfs, executable_name: &str) {
return; // no need to compute if won't output
}
if vfs.root.entries.is_empty() {
if vfs.entries.is_empty() {
return; // nothing to output
}
@ -696,7 +687,7 @@ fn vfs_as_display_tree(
fn dir_size(dir: &VirtualDirectory, seen_offsets: &mut HashSet<u64>) -> Size {
let mut size = Size::default();
for entry in &dir.entries {
for entry in dir.entries.iter() {
match entry {
VfsEntry::Dir(virtual_directory) => {
size = size + dir_size(virtual_directory, seen_offsets);
@ -760,15 +751,10 @@ fn vfs_as_display_tree(
fn include_all_entries<'a>(
dir_path: &WindowsSystemRootablePath,
vfs_dir: &'a VirtualDirectory,
entries: &'a VirtualDirectoryEntries,
seen_offsets: &mut HashSet<u64>,
) -> Vec<DirEntryOutput<'a>> {
if vfs_dir.name == DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME {
return show_global_node_modules_dir(vfs_dir, seen_offsets);
}
vfs_dir
.entries
entries
.iter()
.map(|entry| DirEntryOutput {
name: Cow::Borrowed(entry.name()),
@ -826,10 +812,12 @@ fn vfs_as_display_tree(
} else {
EntryOutput::Subset(children)
}
} else if vfs_dir.name == DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME {
EntryOutput::Subset(show_global_node_modules_dir(vfs_dir, seen_offsets))
} else {
EntryOutput::Subset(include_all_entries(
&WindowsSystemRootablePath::Path(dir),
vfs_dir,
&vfs_dir.entries,
seen_offsets,
))
}
@ -839,7 +827,7 @@ fn vfs_as_display_tree(
// user might not have context about what's being shown
let mut seen_offsets = HashSet::with_capacity(vfs.files.len());
let mut child_entries =
include_all_entries(&vfs.root_path, &vfs.root, &mut seen_offsets);
include_all_entries(&vfs.root_path, &vfs.entries, &mut seen_offsets);
for child_entry in &mut child_entries {
child_entry.collapse_leaf_nodes();
}
@ -961,27 +949,70 @@ impl VfsEntry {
}
}
#[derive(Debug, Default, Serialize, Deserialize)]
pub struct VirtualDirectoryEntries(Vec<VfsEntry>);
impl VirtualDirectoryEntries {
pub fn new(mut entries: Vec<VfsEntry>) -> Self {
// needs to be sorted by name
entries.sort_by(|a, b| a.name().cmp(b.name()));
Self(entries)
}
pub fn take_inner(&mut self) -> Vec<VfsEntry> {
std::mem::take(&mut self.0)
}
pub fn is_empty(&self) -> bool {
self.0.is_empty()
}
pub fn len(&self) -> usize {
self.0.len()
}
pub fn get_by_name(&self, name: &str) -> Option<&VfsEntry> {
self.binary_search(name).ok().map(|index| &self.0[index])
}
pub fn get_mut_by_name(&mut self, name: &str) -> Option<&mut VfsEntry> {
self
.binary_search(name)
.ok()
.map(|index| &mut self.0[index])
}
pub fn binary_search(&self, name: &str) -> Result<usize, usize> {
self.0.binary_search_by(|e| e.name().cmp(name))
}
pub fn insert(&mut self, entry: VfsEntry) {
match self.binary_search(entry.name()) {
Ok(index) => {
self.0[index] = entry;
}
Err(insert_index) => {
self.0.insert(insert_index, entry);
}
}
}
pub fn remove(&mut self, index: usize) -> VfsEntry {
self.0.remove(index)
}
pub fn iter(&self) -> std::slice::Iter<'_, VfsEntry> {
self.0.iter()
}
}
#[derive(Debug, Serialize, Deserialize)]
pub struct VirtualDirectory {
#[serde(rename = "n")]
pub name: String,
// should be sorted by name
#[serde(rename = "e")]
pub entries: Vec<VfsEntry>,
}
impl VirtualDirectory {
pub fn insert_entry(&mut self, entry: VfsEntry) {
let name = entry.name();
match self.entries.binary_search_by(|e| e.name().cmp(name)) {
Ok(index) => {
self.entries[index] = entry;
}
Err(insert_index) => {
self.entries.insert(insert_index, entry);
}
}
}
pub entries: VirtualDirectoryEntries,
}
#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
@ -1136,20 +1167,13 @@ impl VfsRoot {
}
};
let component = component.to_string_lossy();
match current_dir
current_entry = current_dir
.entries
.binary_search_by(|e| e.name().cmp(&component))
{
Ok(index) => {
current_entry = current_dir.entries[index].as_ref();
}
Err(_) => {
return Err(std::io::Error::new(
std::io::ErrorKind::NotFound,
"path not found",
));
}
}
.get_by_name(&component)
.ok_or_else(|| {
std::io::Error::new(std::io::ErrorKind::NotFound, "path not found")
})?
.as_ref();
}
Ok((final_path, current_entry))
@ -1706,7 +1730,10 @@ mod test {
FileBackedVfs::new(
Cow::Owned(data),
VfsRoot {
dir: vfs.root,
dir: VirtualDirectory {
name: "".to_string(),
entries: vfs.entries,
},
root_path: dest_path.to_path_buf(),
start_file_offset: 0,
},

7
cli/tools/coverage/mod.rs
View file

@ -26,6 +26,7 @@ use deno_core::serde_json;
use deno_core::sourcemap::SourceMap;
use deno_core::url::Url;
use deno_core::LocalInspectorSession;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use node_resolver::InNpmPackageChecker;
use regex::Regex;
use std::fs;
@ -198,7 +199,7 @@ pub struct CoverageReport {
fn generate_coverage_report(
script_coverage: &cdp::ScriptCoverage,
script_source: String,
maybe_source_map: &Option<Vec<u8>>,
maybe_source_map: Option<&[u8]>,
output: &Option<PathBuf>,
) -> CoverageReport {
let maybe_source_map = maybe_source_map
@ -428,7 +429,7 @@ fn collect_coverages(
.ignore_git_folder()
.ignore_node_modules()
.set_vendor_folder(cli_options.vendor_dir_path().map(ToOwned::to_owned))
.collect_file_patterns(&deno_config::fs::RealDenoConfigFs, file_patterns)?;
.collect_file_patterns(&FsSysTraitsAdapter::new_real(), file_patterns)?;
let coverage_patterns = FilePatterns {
base: initial_cwd.to_path_buf(),
@ -625,7 +626,7 @@ pub fn cover_files(
let coverage_report = generate_coverage_report(
&script_coverage,
runtime_code.as_str().to_owned(),
&source_map,
source_map.as_deref(),
&out_mode,
);

9
cli/tools/doc.rs
View file

@ -28,6 +28,7 @@ use deno_graph::EsParser;
use deno_graph::GraphKind;
use deno_graph::ModuleAnalyzer;
use deno_graph::ModuleSpecifier;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use doc::html::ShortPath;
use doc::DocDiagnostic;
use indexmap::IndexMap;
@ -114,7 +115,7 @@ pub async fn doc(
}
DocSourceFileFlag::Paths(ref source_files) => {
let module_graph_creator = factory.module_graph_creator().await?;
let fs = factory.fs();
let fs = FsSysTraitsAdapter(factory.fs().clone());
let module_specifiers = collect_specifiers(
FilePatterns {
@ -141,7 +142,7 @@ pub async fn doc(
graph_exit_integrity_errors(&graph);
let errors = graph_walk_errors(
&graph,
fs,
&fs,
&module_specifiers,
GraphWalkErrorsOptions {
check_js: false,
@ -343,14 +344,14 @@ impl deno_doc::html::HrefResolver for DocResolver {
let name = &res.req().name;
Some((
format!("https://www.npmjs.com/package/{name}"),
name.to_owned(),
name.to_string(),
))
}
"jsr" => {
let res =
deno_semver::jsr::JsrPackageReqReference::from_str(module).ok()?;
let name = &res.req().name;
Some((format!("https://jsr.io/{name}"), name.to_owned()))
Some((format!("https://jsr.io/{name}"), name.to_string()))
}
_ => None,
}

3
cli/tools/fmt.rs
View file

@ -34,6 +34,7 @@ use deno_core::futures;
use deno_core::parking_lot::Mutex;
use deno_core::unsync::spawn_blocking;
use deno_core::url::Url;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use log::debug;
use log::info;
use log::warn;
@ -230,7 +231,7 @@ fn collect_fmt_files(
.ignore_node_modules()
.use_gitignore()
.set_vendor_folder(cli_options.vendor_dir_path().map(ToOwned::to_owned))
.collect_file_patterns(&deno_config::fs::RealDenoConfigFs, files)
.collect_file_patterns(&FsSysTraitsAdapter::new_real(), files)
}
/// Formats markdown (using <https://github.com/dprint/dprint-plugin-markdown>) and its code blocks

19
cli/tools/info.rs
View file

@ -278,8 +278,10 @@ fn add_npm_packages_to_json(
});
if let Some(pkg) = maybe_package {
if let Some(module) = module.as_object_mut() {
module
.insert("npmPackage".to_string(), pkg.id.as_serialized().into());
module.insert(
"npmPackage".to_string(),
pkg.id.as_serialized().into_string().into(),
);
}
}
}
@ -296,7 +298,7 @@ fn add_npm_packages_to_json(
{
dep.insert(
"npmPackage".to_string(),
pkg.id.as_serialized().into(),
pkg.id.as_serialized().into_string().into(),
);
}
}
@ -324,19 +326,19 @@ fn add_npm_packages_to_json(
let mut json_packages = serde_json::Map::with_capacity(sorted_packages.len());
for pkg in sorted_packages {
let mut kv = serde_json::Map::new();
kv.insert("name".to_string(), pkg.id.nv.name.clone().into());
kv.insert("name".to_string(), pkg.id.nv.name.to_string().into());
kv.insert("version".to_string(), pkg.id.nv.version.to_string().into());
let mut deps = pkg.dependencies.values().collect::<Vec<_>>();
deps.sort();
let deps = deps
.into_iter()
.map(|id| serde_json::Value::String(id.as_serialized()))
.map(|id| serde_json::Value::String(id.as_serialized().into_string()))
.collect::<Vec<_>>();
kv.insert("dependencies".to_string(), deps.into());
let registry_url = npmrc.get_registry_url(&pkg.id.nv.name);
kv.insert("registryUrl".to_string(), registry_url.to_string().into());
json_packages.insert(pkg.id.as_serialized(), kv.into());
json_packages.insert(pkg.id.as_serialized().into_string(), kv.into());
}
json.insert("npmPackages".to_string(), json_packages.into());
@ -549,7 +551,7 @@ impl<'a> GraphDisplayContext<'a> {
None => Specifier(module.specifier().clone()),
};
let was_seen = !self.seen.insert(match &package_or_specifier {
Package(package) => package.id.as_serialized(),
Package(package) => package.id.as_serialized().into_string(),
Specifier(specifier) => specifier.to_string(),
});
let header_text = if was_seen {
@ -631,7 +633,8 @@ impl<'a> GraphDisplayContext<'a> {
));
if let Some(package) = self.npm_info.packages.get(dep_id) {
if !package.dependencies.is_empty() {
let was_seen = !self.seen.insert(package.id.as_serialized());
let was_seen =
!self.seen.insert(package.id.as_serialized().into_string());
if was_seen {
child.text = format!("{} {}", child.text, colors::gray("*"));
} else {

4
cli/tools/installer.rs
View file

@ -161,11 +161,11 @@ pub async fn infer_name_from_url(
let npm_ref = npm_ref.into_inner();
if let Some(sub_path) = npm_ref.sub_path {
if !sub_path.contains('/') {
return Some(sub_path);
return Some(sub_path.to_string());
}
}
if !npm_ref.req.name.contains('/') {
return Some(npm_ref.req.name);
return Some(npm_ref.req.name.into_string());
}
return None;
}

519
cli/tools/lint/ast_buffer/buffer.rs Normal file
View file

@ -0,0 +1,519 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use std::fmt::Display;
use deno_ast::swc::common::Span;
use deno_ast::swc::common::DUMMY_SP;
use indexmap::IndexMap;
/// Each property has this flag to mark what kind of value it holds-
/// Plain objects and arrays are not supported yet, but could be easily
/// added if needed.
#[derive(Debug, PartialEq)]
pub enum PropFlags {
Ref,
RefArr,
String,
Bool,
Null,
Undefined,
}
impl From<PropFlags> for u8 {
fn from(m: PropFlags) -> u8 {
m as u8
}
}
impl TryFrom<u8> for PropFlags {
type Error = &'static str;
fn try_from(value: u8) -> Result<Self, Self::Error> {
match value {
0 => Ok(PropFlags::Ref),
1 => Ok(PropFlags::RefArr),
2 => Ok(PropFlags::String),
3 => Ok(PropFlags::Bool),
4 => Ok(PropFlags::Null),
5 => Ok(PropFlags::Undefined),
_ => Err("Unknown Prop flag"),
}
}
}
const MASK_U32_1: u32 = 0b11111111_00000000_00000000_00000000;
const MASK_U32_2: u32 = 0b00000000_11111111_00000000_00000000;
const MASK_U32_3: u32 = 0b00000000_00000000_11111111_00000000;
const MASK_U32_4: u32 = 0b00000000_00000000_00000000_11111111;
// TODO: There is probably a native Rust function to do this.
pub fn append_u32(result: &mut Vec<u8>, value: u32) {
let v1: u8 = ((value & MASK_U32_1) >> 24) as u8;
let v2: u8 = ((value & MASK_U32_2) >> 16) as u8;
let v3: u8 = ((value & MASK_U32_3) >> 8) as u8;
let v4: u8 = (value & MASK_U32_4) as u8;
result.push(v1);
result.push(v2);
result.push(v3);
result.push(v4);
}
pub fn append_usize(result: &mut Vec<u8>, value: usize) {
let raw = u32::try_from(value).unwrap();
append_u32(result, raw);
}
pub fn write_usize(result: &mut [u8], value: usize, idx: usize) {
let raw = u32::try_from(value).unwrap();
let v1: u8 = ((raw & MASK_U32_1) >> 24) as u8;
let v2: u8 = ((raw & MASK_U32_2) >> 16) as u8;
let v3: u8 = ((raw & MASK_U32_3) >> 8) as u8;
let v4: u8 = (raw & MASK_U32_4) as u8;
result[idx] = v1;
result[idx + 1] = v2;
result[idx + 2] = v3;
result[idx + 3] = v4;
}
#[derive(Debug)]
pub struct StringTable {
id: usize,
table: IndexMap<String, usize>,
}
impl StringTable {
pub fn new() -> Self {
Self {
id: 0,
table: IndexMap::new(),
}
}
pub fn insert(&mut self, s: &str) -> usize {
if let Some(id) = self.table.get(s) {
return *id;
}
let id = self.id;
self.id += 1;
self.table.insert(s.to_string(), id);
id
}
pub fn serialize(&mut self) -> Vec<u8> {
let mut result: Vec<u8> = vec![];
append_u32(&mut result, self.table.len() as u32);
// Assume that it's sorted by id
for (s, _id) in &self.table {
let bytes = s.as_bytes();
append_u32(&mut result, bytes.len() as u32);
result.append(&mut bytes.to_vec());
}
result
}
}
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct NodeRef(pub usize);
#[derive(Debug)]
pub struct BoolPos(pub usize);
#[derive(Debug)]
pub struct FieldPos(pub usize);
#[derive(Debug)]
pub struct FieldArrPos(pub usize);
#[derive(Debug)]
pub struct StrPos(pub usize);
#[derive(Debug)]
pub struct UndefPos(pub usize);
#[derive(Debug)]
pub struct NullPos(pub usize);
#[derive(Debug)]
pub enum NodePos {
Bool(BoolPos),
#[allow(dead_code)]
Field(FieldPos),
#[allow(dead_code)]
FieldArr(FieldArrPos),
Str(StrPos),
Undef(UndefPos),
#[allow(dead_code)]
Null(NullPos),
}
pub trait AstBufSerializer<K, P>
where
K: Into<u8> + Display,
P: Into<u8> + Display,
{
fn header(
&mut self,
kind: K,
parent: NodeRef,
span: &Span,
prop_count: usize,
) -> NodeRef;
fn ref_field(&mut self, prop: P) -> FieldPos;
fn ref_vec_field(&mut self, prop: P, len: usize) -> FieldArrPos;
fn str_field(&mut self, prop: P) -> StrPos;
fn bool_field(&mut self, prop: P) -> BoolPos;
fn undefined_field(&mut self, prop: P) -> UndefPos;
#[allow(dead_code)]
fn null_field(&mut self, prop: P) -> NullPos;
fn write_ref(&mut self, pos: FieldPos, value: NodeRef);
fn write_maybe_ref(&mut self, pos: FieldPos, value: Option<NodeRef>);
fn write_refs(&mut self, pos: FieldArrPos, value: Vec<NodeRef>);
fn write_str(&mut self, pos: StrPos, value: &str);
fn write_bool(&mut self, pos: BoolPos, value: bool);
fn serialize(&mut self) -> Vec<u8>;
}
#[derive(Debug)]
pub struct SerializeCtx {
buf: Vec<u8>,
start_buf: NodeRef,
str_table: StringTable,
kind_map: Vec<usize>,
prop_map: Vec<usize>,
}
/// This is the internal context used to allocate and fill the buffer. The point
/// is to be able to write absolute offsets directly in place.
///
/// The typical workflow is to reserve all necessary space for the currrent
/// node with placeholders for the offsets of the child nodes. Once child
/// nodes have been traversed, we know their offsets and can replace the
/// placeholder values with the actual ones.
impl SerializeCtx {
pub fn new(kind_len: u8, prop_len: u8) -> Self {
let kind_size = kind_len as usize;
let prop_size = prop_len as usize;
let mut ctx = Self {
start_buf: NodeRef(0),
buf: vec![],
str_table: StringTable::new(),
kind_map: vec![0; kind_size + 1],
prop_map: vec![0; prop_size + 1],
};
let empty_str = ctx.str_table.insert("");
// Placeholder node is always 0
ctx.append_node(0, NodeRef(0), &DUMMY_SP, 0);
ctx.kind_map[0] = empty_str;
ctx.start_buf = NodeRef(ctx.buf.len());
// Insert default props that are always present
let type_str = ctx.str_table.insert("type");
let parent_str = ctx.str_table.insert("parent");
let range_str = ctx.str_table.insert("range");
let length_str = ctx.str_table.insert("length");
// These values are expected to be in this order on the JS side
ctx.prop_map[0] = empty_str;
ctx.prop_map[1] = type_str;
ctx.prop_map[2] = parent_str;
ctx.prop_map[3] = range_str;
ctx.prop_map[4] = length_str;
ctx
}
/// Allocate a node's header
fn field_header<P>(&mut self, prop: P, prop_flags: PropFlags) -> usize
where
P: Into<u8> + Display + Clone,
{
let offset = self.buf.len();
let n: u8 = prop.clone().into();
self.buf.push(n);
if let Some(v) = self.prop_map.get::<usize>(n.into()) {
if *v == 0 {
let id = self.str_table.insert(&format!("{prop}"));
self.prop_map[n as usize] = id;
}
}
let flags: u8 = prop_flags.into();
self.buf.push(flags);
offset
}
/// Allocate a property pointing to another node.
fn field<P>(&mut self, prop: P, prop_flags: PropFlags) -> usize
where
P: Into<u8> + Display + Clone,
{
let offset = self.field_header(prop, prop_flags);
append_usize(&mut self.buf, 0);
offset
}
fn append_node(
&mut self,
kind: u8,
parent: NodeRef,
span: &Span,
prop_count: usize,
) -> NodeRef {
let offset = self.buf.len();
// Node type fits in a u8
self.buf.push(kind);
// Offset to the parent node. Will be 0 if none exists
append_usize(&mut self.buf, parent.0);
// Span, the start and end location of this node
append_u32(&mut self.buf, span.lo.0);
append_u32(&mut self.buf, span.hi.0);
// No node has more than <10 properties
debug_assert!(prop_count < 10);
self.buf.push(prop_count as u8);
NodeRef(offset)
}
/// Allocate the node header. It's always the same for every node.
/// <type u8>
/// <parent offset u32>
/// <span lo u32>
/// <span high u32>
/// <property count u8> (There is no node with more than 10 properties)
pub fn header<N>(
&mut self,
kind: N,
parent: NodeRef,
span: &Span,
prop_count: usize,
) -> NodeRef
where
N: Into<u8> + Display + Clone,
{
let n: u8 = kind.clone().into();
if let Some(v) = self.kind_map.get::<usize>(n.into()) {
if *v == 0 {
let id = self.str_table.insert(&format!("{kind}"));
self.kind_map[n as usize] = id;
}
}
self.append_node(n, parent, span, prop_count)
}
/// Allocate a reference property that will hold the offset of
/// another node.
pub fn ref_field<P>(&mut self, prop: P) -> usize
where
P: Into<u8> + Display + Clone,
{
self.field(prop, PropFlags::Ref)
}
/// Allocate a property that is a vec of node offsets pointing to other
/// nodes.
pub fn ref_vec_field<P>(&mut self, prop: P, len: usize) -> usize
where
P: Into<u8> + Display + Clone,
{
let offset = self.field(prop, PropFlags::RefArr);
for _ in 0..len {
append_u32(&mut self.buf, 0);
}
offset
}
// Allocate a property representing a string. Strings are deduplicated
// in the message and the property will only contain the string id.
pub fn str_field<P>(&mut self, prop: P) -> usize
where
P: Into<u8> + Display + Clone,
{
self.field(prop, PropFlags::String)
}
/// Allocate a bool field
pub fn bool_field<P>(&mut self, prop: P) -> usize
where
P: Into<u8> + Display + Clone,
{
let offset = self.field_header(prop, PropFlags::Bool);
self.buf.push(0);
offset
}
/// Allocate an undefined field
pub fn undefined_field<P>(&mut self, prop: P) -> usize
where
P: Into<u8> + Display + Clone,
{
self.field_header(prop, PropFlags::Undefined)
}
/// Allocate an undefined field
#[allow(dead_code)]
pub fn null_field<P>(&mut self, prop: P) -> usize
where
P: Into<u8> + Display + Clone,
{
self.field_header(prop, PropFlags::Null)
}
/// Replace the placeholder of a reference field with the actual offset
/// to the node we want to point to.
pub fn write_ref(&mut self, field_offset: usize, value: NodeRef) {
#[cfg(debug_assertions)]
{
let value_kind = self.buf[field_offset + 1];
if PropFlags::try_from(value_kind).unwrap() != PropFlags::Ref {
panic!("Trying to write a ref into a non-ref field")
}
}
write_usize(&mut self.buf, value.0, field_offset + 2);
}
/// Helper for writing optional node offsets
pub fn write_maybe_ref(
&mut self,
field_offset: usize,
value: Option<NodeRef>,
) {
#[cfg(debug_assertions)]
{
let value_kind = self.buf[field_offset + 1];
if PropFlags::try_from(value_kind).unwrap() != PropFlags::Ref {
panic!("Trying to write a ref into a non-ref field")
}
}
let ref_value = if let Some(v) = value { v } else { NodeRef(0) };
write_usize(&mut self.buf, ref_value.0, field_offset + 2);
}
/// Write a vec of node offsets into the property. The necessary space
/// has been reserved earlier.
pub fn write_refs(&mut self, field_offset: usize, value: Vec<NodeRef>) {
#[cfg(debug_assertions)]
{
let value_kind = self.buf[field_offset + 1];
if PropFlags::try_from(value_kind).unwrap() != PropFlags::RefArr {
panic!("Trying to write a ref into a non-ref array field")
}
}
let mut offset = field_offset + 2;
write_usize(&mut self.buf, value.len(), offset);
offset += 4;
for item in value {
write_usize(&mut self.buf, item.0, offset);
offset += 4;
}
}
/// Store the string in our string table and save the id of the string
/// in the current field.
pub fn write_str(&mut self, field_offset: usize, value: &str) {
#[cfg(debug_assertions)]
{
let value_kind = self.buf[field_offset + 1];
if PropFlags::try_from(value_kind).unwrap() != PropFlags::String {
panic!("Trying to write a ref into a non-string field")
}
}
let id = self.str_table.insert(value);
write_usize(&mut self.buf, id, field_offset + 2);
}
/// Write a bool to a field.
pub fn write_bool(&mut self, field_offset: usize, value: bool) {
#[cfg(debug_assertions)]
{
let value_kind = self.buf[field_offset + 1];
if PropFlags::try_from(value_kind).unwrap() != PropFlags::Bool {
panic!("Trying to write a ref into a non-bool field")
}
}
self.buf[field_offset + 2] = if value { 1 } else { 0 };
}
/// Serialize all information we have into a buffer that can be sent to JS.
/// It has the following structure:
///
/// <...ast>
/// <string table>
/// <node kind map> <- node kind id maps to string id
/// <node prop map> <- node property id maps to string id
/// <offset kind map>
/// <offset prop map>
/// <offset str table>
pub fn serialize(&mut self) -> Vec<u8> {
let mut buf: Vec<u8> = vec![];
// The buffer starts with the serialized AST first, because that
// contains absolute offsets. By butting this at the start of the
// message we don't have to waste time updating any offsets.
buf.append(&mut self.buf);
// Next follows the string table. We'll keep track of the offset
// in the message of where the string table begins
let offset_str_table = buf.len();
// Serialize string table
buf.append(&mut self.str_table.serialize());
// Next, serialize the mappings of kind -> string of encountered
// nodes in the AST. We use this additional lookup table to compress
// the message so that we can save space by using a u8 . All nodes of
// JS, TS and JSX together are <200
let offset_kind_map = buf.len();
// Write the total number of entries in the kind -> str mapping table
// TODO: make this a u8
append_usize(&mut buf, self.kind_map.len());
for v in &self.kind_map {
append_usize(&mut buf, *v);
}
// Store offset to prop -> string map. It's the same as with node kind
// as the total number of properties is <120 which allows us to store it
// as u8.
let offset_prop_map = buf.len();
// Write the total number of entries in the kind -> str mapping table
append_usize(&mut buf, self.prop_map.len());
for v in &self.prop_map {
append_usize(&mut buf, *v);
}
// Putting offsets of relevant parts of the buffer at the end. This
// allows us to hop to the relevant part by merely looking at the last
// for values in the message. Each value represents an offset into the
// buffer.
append_usize(&mut buf, offset_kind_map);
append_usize(&mut buf, offset_prop_map);
append_usize(&mut buf, offset_str_table);
append_usize(&mut buf, self.start_buf.0);
buf
}
}

13
cli/tools/lint/ast_buffer/mod.rs Normal file
View file

@ -0,0 +1,13 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use deno_ast::ParsedSource;
use swc::serialize_swc_to_buffer;
mod buffer;
mod swc;
mod ts_estree;
pub fn serialize_ast_to_buffer(parsed_source: &ParsedSource) -> Vec<u8> {
// TODO: We could support multiple languages here
serialize_swc_to_buffer(parsed_source)
}

3018
cli/tools/lint/ast_buffer/swc.rs Normal file
View file

File diff suppressed because it is too large Load diff

516
cli/tools/lint/ast_buffer/ts_estree.rs Normal file
View file

@ -0,0 +1,516 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use std::fmt;
use std::fmt::Debug;
use std::fmt::Display;
use deno_ast::swc::common::Span;
use super::buffer::AstBufSerializer;
use super::buffer::BoolPos;
use super::buffer::FieldArrPos;
use super::buffer::FieldPos;
use super::buffer::NodeRef;
use super::buffer::NullPos;
use super::buffer::SerializeCtx;
use super::buffer::StrPos;
use super::buffer::UndefPos;
#[derive(Debug, Clone, PartialEq)]
pub enum AstNode {
// First node must always be the empty/invalid node
Invalid,
// Typically the
Program,
// Module declarations
ExportAllDeclaration,
ExportDefaultDeclaration,
ExportNamedDeclaration,
ImportDeclaration,
TsExportAssignment,
TsImportEquals,
TsNamespaceExport,
// Decls
ClassDeclaration,
FunctionDeclaration,
TSEnumDeclaration,
TSInterface,
TsModule,
TsTypeAlias,
Using,
VariableDeclaration,
// Statements
BlockStatement,
BreakStatement,
ContinueStatement,
DebuggerStatement,
DoWhileStatement,
EmptyStatement,
ExpressionStatement,
ForInStatement,
ForOfStatement,
ForStatement,
IfStatement,
LabeledStatement,
ReturnStatement,
SwitchCase,
SwitchStatement,
ThrowStatement,
TryStatement,
WhileStatement,
WithStatement,
// Expressions
ArrayExpression,
ArrowFunctionExpression,
AssignmentExpression,
AwaitExpression,
BinaryExpression,
CallExpression,
ChainExpression,
ClassExpression,
ConditionalExpression,
FunctionExpression,
Identifier,
ImportExpression,
LogicalExpression,
MemberExpression,
MetaProp,
NewExpression,
ObjectExpression,
PrivateIdentifier,
SequenceExpression,
Super,
TaggedTemplateExpression,
TemplateLiteral,
ThisExpression,
TSAsExpression,
TsConstAssertion,
TsInstantiation,
TSNonNullExpression,
TSSatisfiesExpression,
TSTypeAssertion,
UnaryExpression,
UpdateExpression,
YieldExpression,
// TODO: TSEsTree uses a single literal node
// Literals
StringLiteral,
Bool,
Null,
NumericLiteral,
BigIntLiteral,
RegExpLiteral,
EmptyExpr,
SpreadElement,
Property,
VariableDeclarator,
CatchClause,
RestElement,
ExportSpecifier,
TemplateElement,
MethodDefinition,
ClassBody,
// Patterns
ArrayPattern,
AssignmentPattern,
ObjectPattern,
// JSX
JSXAttribute,
JSXClosingElement,
JSXClosingFragment,
JSXElement,
JSXEmptyExpression,
JSXExpressionContainer,
JSXFragment,
JSXIdentifier,
JSXMemberExpression,
JSXNamespacedName,
JSXOpeningElement,
JSXOpeningFragment,
JSXSpreadAttribute,
JSXSpreadChild,
JSXText,
TSTypeAnnotation,
TSTypeParameterDeclaration,
TSTypeParameter,
TSTypeParameterInstantiation,
TSEnumMember,
TSInterfaceBody,
TSInterfaceHeritage,
TSTypeReference,
TSThisType,
TSLiteralType,
TSInferType,
TSConditionalType,
TSUnionType,
TSIntersectionType,
TSMappedType,
TSTypeQuery,
TSTupleType,
TSNamedTupleMember,
TSFunctionType,
TsCallSignatureDeclaration,
TSPropertySignature,
TSMethodSignature,
TSIndexSignature,
TSIndexedAccessType,
TSTypeOperator,
TSTypePredicate,
TSImportType,
TSRestType,
TSArrayType,
TSClassImplements,
TSAnyKeyword,
TSBigIntKeyword,
TSBooleanKeyword,
TSIntrinsicKeyword,
TSNeverKeyword,
TSNullKeyword,
TSNumberKeyword,
TSObjectKeyword,
TSStringKeyword,
TSSymbolKeyword,
TSUndefinedKeyword,
TSUnknownKeyword,
TSVoidKeyword,
TSEnumBody, // Last value is used for max value
}
impl Display for AstNode {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
Debug::fmt(self, f)
}
}
impl From<AstNode> for u8 {
fn from(m: AstNode) -> u8 {
m as u8
}
}
#[derive(Debug, Clone)]
pub enum AstProp {
// Base, these must be in sync with JS in the same order.
Invalid,
Type,
Parent,
Range,
Length, // Not used in AST, but can be used in attr selectors
// Starting from here the order doesn't matter.
// Following are all possible AST node properties.
Abstract,
Accessibility,
Alternate,
Argument,
Arguments,
Asserts,
Async,
Attributes,
Await,
Block,
Body,
Callee,
Cases,
Children,
CheckType,
ClosingElement,
ClosingFragment,
Computed,
Consequent,
Const,
Constraint,
Cooked,
Declaration,
Declarations,
Declare,
Default,
Definite,
Delegate,
Discriminant,
Elements,
ElementType,
ElementTypes,
ExprName,
Expression,
Expressions,
Exported,
Extends,
ExtendsType,
FalseType,
Finalizer,
Flags,
Generator,
Handler,
Id,
In,
IndexType,
Init,
Initializer,
Implements,
Key,
Kind,
Label,
Left,
Literal,
Local,
Members,
Meta,
Method,
Name,
Namespace,
NameType,
Object,
ObjectType,
OpeningElement,
OpeningFragment,
Operator,
Optional,
Out,
Param,
ParameterName,
Params,
Pattern,
Prefix,
Properties,
Property,
Qualifier,
Quasi,
Quasis,
Raw,
Readonly,
ReturnType,
Right,
SelfClosing,
Shorthand,
Source,
SourceType,
Specifiers,
Static,
SuperClass,
SuperTypeArguments,
Tag,
Tail,
Test,
TrueType,
TypeAnnotation,
TypeArguments,
TypeName,
TypeParameter,
TypeParameters,
Types,
Update,
Value, // Last value is used for max value
}
// TODO: Feels like there should be an easier way to iterater over an
// enum in Rust and lowercase the first letter.
impl Display for AstProp {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
let s = match self {
AstProp::Invalid => "__invalid__", // unused
AstProp::Parent => "parent",
AstProp::Range => "range",
AstProp::Type => "type",
AstProp::Length => "length",
AstProp::Abstract => "abstract",
AstProp::Accessibility => "accessibility",
AstProp::Alternate => "alternate",
AstProp::Argument => "argument",
AstProp::Arguments => "arguments",
AstProp::Asserts => "asserts",
AstProp::Async => "async",
AstProp::Attributes => "attributes",
AstProp::Await => "await",
AstProp::Block => "block",
AstProp::Body => "body",
AstProp::Callee => "callee",
AstProp::Cases => "cases",
AstProp::Children => "children",
AstProp::CheckType => "checkType",
AstProp::ClosingElement => "closingElement",
AstProp::ClosingFragment => "closingFragment",
AstProp::Computed => "computed",
AstProp::Consequent => "consequent",
AstProp::Const => "const",
AstProp::Constraint => "constraint",
AstProp::Cooked => "cooked",
AstProp::Declaration => "declaration",
AstProp::Declarations => "declarations",
AstProp::Declare => "declare",
AstProp::Default => "default",
AstProp::Definite => "definite",
AstProp::Delegate => "delegate",
AstProp::Discriminant => "discriminant",
AstProp::Elements => "elements",
AstProp::ElementType => "elementType",
AstProp::ElementTypes => "elementTypes",
AstProp::ExprName => "exprName",
AstProp::Expression => "expression",
AstProp::Expressions => "expressions",
AstProp::Exported => "exported",
AstProp::Extends => "extends",
AstProp::ExtendsType => "extendsType",
AstProp::FalseType => "falseType",
AstProp::Finalizer => "finalizer",
AstProp::Flags => "flags",
AstProp::Generator => "generator",
AstProp::Handler => "handler",
AstProp::Id => "id",
AstProp::In => "in",
AstProp::IndexType => "indexType",
AstProp::Init => "init",
AstProp::Initializer => "initializer",
AstProp::Implements => "implements",
AstProp::Key => "key",
AstProp::Kind => "kind",
AstProp::Label => "label",
AstProp::Left => "left",
AstProp::Literal => "literal",
AstProp::Local => "local",
AstProp::Members => "members",
AstProp::Meta => "meta",
AstProp::Method => "method",
AstProp::Name => "name",
AstProp::Namespace => "namespace",
AstProp::NameType => "nameType",
AstProp::Object => "object",
AstProp::ObjectType => "objectType",
AstProp::OpeningElement => "openingElement",
AstProp::OpeningFragment => "openingFragment",
AstProp::Operator => "operator",
AstProp::Optional => "optional",
AstProp::Out => "out",
AstProp::Param => "param",
AstProp::ParameterName => "parameterName",
AstProp::Params => "params",
AstProp::Pattern => "pattern",
AstProp::Prefix => "prefix",
AstProp::Properties => "properties",
AstProp::Property => "property",
AstProp::Qualifier => "qualifier",
AstProp::Quasi => "quasi",
AstProp::Quasis => "quasis",
AstProp::Raw => "raw",
AstProp::Readonly => "readonly",
AstProp::ReturnType => "returnType",
AstProp::Right => "right",
AstProp::SelfClosing => "selfClosing",
AstProp::Shorthand => "shorthand",
AstProp::Source => "source",
AstProp::SourceType => "sourceType",
AstProp::Specifiers => "specifiers",
AstProp::Static => "static",
AstProp::SuperClass => "superClass",
AstProp::SuperTypeArguments => "superTypeArguments",
AstProp::Tag => "tag",
AstProp::Tail => "tail",
AstProp::Test => "test",
AstProp::TrueType => "trueType",
AstProp::TypeAnnotation => "typeAnnotation",
AstProp::TypeArguments => "typeArguments",
AstProp::TypeName => "typeName",
AstProp::TypeParameter => "typeParameter",
AstProp::TypeParameters => "typeParameters",
AstProp::Types => "types",
AstProp::Update => "update",
AstProp::Value => "value",
};
write!(f, "{}", s)
}
}
impl From<AstProp> for u8 {
fn from(m: AstProp) -> u8 {
m as u8
}
}
pub struct TsEsTreeBuilder {
ctx: SerializeCtx,
}
// TODO: Add a builder API to make it easier to convert from different source
// ast formats.
impl TsEsTreeBuilder {
pub fn new() -> Self {
// Max values
// TODO: Maybe there is a rust macro to grab the last enum value?
let kind_count: u8 = AstNode::TSEnumBody.into();
let prop_count: u8 = AstProp::Value.into();
Self {
ctx: SerializeCtx::new(kind_count, prop_count),
}
}
}
impl AstBufSerializer<AstNode, AstProp> for TsEsTreeBuilder {
fn header(
&mut self,
kind: AstNode,
parent: NodeRef,
span: &Span,
prop_count: usize,
) -> NodeRef {
self.ctx.header(kind, parent, span, prop_count)
}
fn ref_field(&mut self, prop: AstProp) -> FieldPos {
FieldPos(self.ctx.ref_field(prop))
}
fn ref_vec_field(&mut self, prop: AstProp, len: usize) -> FieldArrPos {
FieldArrPos(self.ctx.ref_vec_field(prop, len))
}
fn str_field(&mut self, prop: AstProp) -> StrPos {
StrPos(self.ctx.str_field(prop))
}
fn bool_field(&mut self, prop: AstProp) -> BoolPos {
BoolPos(self.ctx.bool_field(prop))
}
fn undefined_field(&mut self, prop: AstProp) -> UndefPos {
UndefPos(self.ctx.undefined_field(prop))
}
fn null_field(&mut self, prop: AstProp) -> NullPos {
NullPos(self.ctx.null_field(prop))
}
fn write_ref(&mut self, pos: FieldPos, value: NodeRef) {
self.ctx.write_ref(pos.0, value);
}
fn write_maybe_ref(&mut self, pos: FieldPos, value: Option<NodeRef>) {
self.ctx.write_maybe_ref(pos.0, value);
}
fn write_refs(&mut self, pos: FieldArrPos, value: Vec<NodeRef>) {
self.ctx.write_refs(pos.0, value);
}
fn write_str(&mut self, pos: StrPos, value: &str) {
self.ctx.write_str(pos.0, value);
}
fn write_bool(&mut self, pos: BoolPos, value: bool) {
self.ctx.write_bool(pos.0, value);
}
fn serialize(&mut self) -> Vec<u8> {
self.ctx.serialize()
}
}

6
cli/tools/lint/linter.rs
View file

@ -15,8 +15,9 @@ use deno_lint::linter::LintConfig as DenoLintConfig;
use deno_lint::linter::LintFileOptions;
use deno_lint::linter::Linter as DenoLintLinter;
use deno_lint::linter::LinterOptions;
use deno_path_util::fs::atomic_write_file_with_retries;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use crate::util::fs::atomic_write_file_with_retries;
use crate::util::fs::specifier_from_file_path;
use super::rules::FileOrPackageLintRule;
@ -176,8 +177,9 @@ impl CliLinter {
if fix_iterations > 0 {
// everything looks good and the file still parses, so write it out
atomic_write_file_with_retries(
&FsSysTraitsAdapter::new_real(),
file_path,
source.text().as_ref(),
source.text().as_bytes(),
crate::cache::CACHE_PERM,
)
.context("Failed writing fix to file.")?;

6
cli/tools/lint/mod.rs
View file

@ -21,6 +21,7 @@ use deno_core::unsync::future::SharedLocal;
use deno_graph::ModuleGraph;
use deno_lint::diagnostic::LintDiagnostic;
use deno_lint::linter::LintConfig as DenoLintConfig;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use log::debug;
use reporters::create_reporter;
use reporters::LintReporter;
@ -51,10 +52,13 @@ use crate::util::fs::canonicalize_path;
use crate::util::path::is_script_ext;
use crate::util::sync::AtomicFlag;
mod ast_buffer;
mod linter;
mod reporters;
mod rules;
// TODO(bartlomieju): remove once we wire plugins through the CLI linter
pub use ast_buffer::serialize_ast_to_buffer;
pub use linter::CliLinter;
pub use linter::CliLinterOptions;
pub use rules::collect_no_slow_type_diagnostics;
@ -449,7 +453,7 @@ fn collect_lint_files(
.ignore_node_modules()
.use_gitignore()
.set_vendor_folder(cli_options.vendor_dir_path().map(ToOwned::to_owned))
.collect_file_patterns(&deno_config::fs::RealDenoConfigFs, files)
.collect_file_patterns(&FsSysTraitsAdapter::new_real(), files)
}
#[allow(clippy::print_stdout)]

7
cli/tools/registry/paths.rs
View file

@ -11,6 +11,7 @@ use deno_ast::ModuleSpecifier;
use deno_config::glob::FileCollector;
use deno_config::glob::FilePatterns;
use deno_core::error::AnyError;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use thiserror::Error;
use crate::args::CliOptions;
@ -323,11 +324,11 @@ fn collect_paths(
file_patterns: FilePatterns,
) -> Result<Vec<PathBuf>, AnyError> {
FileCollector::new(|e| {
if !e.metadata.is_file {
if !e.metadata.file_type().is_file() {
if let Ok(specifier) = ModuleSpecifier::from_file_path(e.path) {
diagnostics_collector.push(PublishDiagnostic::UnsupportedFileType {
specifier,
kind: if e.metadata.is_symlink {
kind: if e.metadata.file_type().is_symlink() {
"symlink".to_string()
} else {
"Unknown".to_string()
@ -345,5 +346,5 @@ fn collect_paths(
.ignore_node_modules()
.set_vendor_folder(cli_options.vendor_dir_path().map(ToOwned::to_owned))
.use_gitignore()
.collect_file_patterns(&deno_config::fs::RealDenoConfigFs, file_patterns)
.collect_file_patterns(&FsSysTraitsAdapter::new_real(), file_patterns)
}

33
cli/tools/registry/pm.rs
View file

@ -15,6 +15,7 @@ use deno_semver::jsr::JsrPackageReqReference;
use deno_semver::npm::NpmPackageReqReference;
use deno_semver::package::PackageNv;
use deno_semver::package::PackageReq;
use deno_semver::StackString;
use deno_semver::Version;
use deno_semver::VersionReq;
use deps::KeyPath;
@ -283,7 +284,7 @@ fn package_json_dependency_entry(
(npm_package.into(), selected.version_req)
} else {
(
selected.import_name,
selected.import_name.into_string(),
format!("npm:{}@{}", npm_package, selected.version_req),
)
}
@ -292,7 +293,7 @@ fn package_json_dependency_entry(
let scope_replaced = jsr_package.replace('/', "__");
let version_req =
format!("npm:@jsr/{scope_replaced}@{}", selected.version_req);
(selected.import_name, version_req)
(selected.import_name.into_string(), version_req)
} else {
(selected.package_name, selected.version_req)
}
@ -549,10 +550,10 @@ pub async fn add(
}
struct SelectedPackage {
import_name: String,
import_name: StackString,
package_name: String,
version_req: String,
selected_version: String,
selected_version: StackString,
}
enum NotFoundHelp {
@ -683,7 +684,7 @@ async fn find_package_and_select_version_for_req(
import_name: add_package_req.alias,
package_name: prefixed_name,
version_req: format!("{}{}", range_symbol, &nv.version),
selected_version: nv.version.to_string(),
selected_version: nv.version.to_custom_string::<StackString>(),
}))
}
@ -705,7 +706,7 @@ enum AddRmPackageReqValue {
#[derive(Debug, PartialEq, Eq)]
pub struct AddRmPackageReq {
alias: String,
alias: StackString,
value: AddRmPackageReqValue,
}
@ -753,7 +754,11 @@ impl AddRmPackageReq {
return Ok(Err(PackageReq::from_str(entry_text)?));
}
(maybe_prefix.unwrap(), Some(alias.to_string()), entry_text)
(
maybe_prefix.unwrap(),
Some(StackString::from(alias)),
entry_text,
)
}
None => return Ok(Err(PackageReq::from_str(entry_text)?)),
},
@ -765,7 +770,7 @@ impl AddRmPackageReq {
JsrPackageReqReference::from_str(&format!("jsr:{}", entry_text))?;
let package_req = req_ref.into_inner().req;
Ok(Ok(AddRmPackageReq {
alias: maybe_alias.unwrap_or_else(|| package_req.name.to_string()),
alias: maybe_alias.unwrap_or_else(|| package_req.name.clone()),
value: AddRmPackageReqValue::Jsr(package_req),
}))
}
@ -785,7 +790,7 @@ impl AddRmPackageReq {
);
}
Ok(Ok(AddRmPackageReq {
alias: maybe_alias.unwrap_or_else(|| package_req.name.to_string()),
alias: maybe_alias.unwrap_or_else(|| package_req.name.clone()),
value: AddRmPackageReqValue::Npm(package_req),
}))
}
@ -878,14 +883,14 @@ mod test {
assert_eq!(
AddRmPackageReq::parse("jsr:foo").unwrap().unwrap(),
AddRmPackageReq {
alias: "foo".to_string(),
alias: "foo".into(),
value: AddRmPackageReqValue::Jsr(PackageReq::from_str("foo").unwrap())
}
);
assert_eq!(
AddRmPackageReq::parse("alias@jsr:foo").unwrap().unwrap(),
AddRmPackageReq {
alias: "alias".to_string(),
alias: "alias".into(),
value: AddRmPackageReqValue::Jsr(PackageReq::from_str("foo").unwrap())
}
);
@ -894,7 +899,7 @@ mod test {
.unwrap()
.unwrap(),
AddRmPackageReq {
alias: "@alias/pkg".to_string(),
alias: "@alias/pkg".into(),
value: AddRmPackageReqValue::Npm(
PackageReq::from_str("foo@latest").unwrap()
)
@ -905,7 +910,7 @@ mod test {
.unwrap()
.unwrap(),
AddRmPackageReq {
alias: "@alias/pkg".to_string(),
alias: "@alias/pkg".into(),
value: AddRmPackageReqValue::Jsr(PackageReq::from_str("foo").unwrap())
}
);
@ -914,7 +919,7 @@ mod test {
.unwrap()
.unwrap(),
AddRmPackageReq {
alias: "alias".to_string(),
alias: "alias".into(),
value: AddRmPackageReqValue::Jsr(
PackageReq::from_str("foo@^1.5.0").unwrap()
)

30
cli/tools/registry/pm/deps.rs
View file

@ -27,6 +27,7 @@ use deno_semver::npm::NpmPackageReqReference;
use deno_semver::package::PackageNv;
use deno_semver::package::PackageReq;
use deno_semver::package::PackageReqReference;
use deno_semver::StackString;
use deno_semver::Version;
use deno_semver::VersionReq;
use import_map::ImportMap;
@ -139,13 +140,7 @@ pub enum KeyPart {
Scopes,
Dependencies,
DevDependencies,
String(String),
}
impl From<String> for KeyPart {
fn from(value: String) -> Self {
KeyPart::String(value)
}
String(StackString),
}
impl From<PackageJsonDepKind> for KeyPart {
@ -164,7 +159,7 @@ impl KeyPart {
KeyPart::Scopes => "scopes",
KeyPart::Dependencies => "dependencies",
KeyPart::DevDependencies => "devDependencies",
KeyPart::String(s) => s,
KeyPart::String(s) => s.as_str(),
}
}
}
@ -217,12 +212,12 @@ fn import_map_entries(
.chain(import_map.scopes().flat_map(|scope| {
let path = KeyPath::from_parts([
KeyPart::Scopes,
scope.raw_key.to_string().into(),
KeyPart::String(scope.raw_key.into()),
]);
scope.imports.entries().map(move |entry| {
let mut full_path = path.clone();
full_path.push(KeyPart::String(entry.raw_key.to_string()));
full_path.push(KeyPart::String(entry.raw_key.into()));
(full_path, entry)
})
}))
@ -338,7 +333,7 @@ fn add_deps_from_package_json(
package_json: &PackageJsonRc,
mut filter: impl DepFilter,
package_dep_kind: PackageJsonDepKind,
package_json_deps: PackageJsonDepsMap,
package_json_deps: &PackageJsonDepsMap,
deps: &mut Vec<Dep>,
) {
for (k, v) in package_json_deps {
@ -353,7 +348,7 @@ fn add_deps_from_package_json(
deno_package_json::PackageJsonDepValue::Req(req) => {
let alias = k.as_str();
let alias = (alias != req.name).then(|| alias.to_string());
if !filter.should_include(alias.as_deref(), &req, DepKind::Npm) {
if !filter.should_include(alias.as_deref(), req, DepKind::Npm) {
continue;
}
let id = DepId(deps.len());
@ -362,9 +357,12 @@ fn add_deps_from_package_json(
kind: DepKind::Npm,
location: DepLocation::PackageJson(
package_json.clone(),
KeyPath::from_parts([package_dep_kind.into(), k.into()]),
KeyPath::from_parts([
package_dep_kind.into(),
KeyPart::String(k.clone()),
]),
),
req,
req: req.clone(),
alias,
})
}
@ -377,14 +375,14 @@ fn add_deps_from_package_json(
package_json,
filter,
PackageJsonDepKind::Normal,
package_json_deps.dependencies,
&package_json_deps.dependencies,
deps,
);
iterate(
package_json,
filter,
PackageJsonDepKind::Dev,
package_json_deps.dev_dependencies,
&package_json_deps.dev_dependencies,
deps,
);
}

3
cli/tools/registry/pm/outdated.rs
View file

@ -8,6 +8,7 @@ use deno_core::anyhow::bail;
use deno_core::error::AnyError;
use deno_semver::package::PackageNv;
use deno_semver::package::PackageReq;
use deno_semver::StackString;
use deno_semver::VersionReq;
use deno_terminal::colors;
@ -31,7 +32,7 @@ struct OutdatedPackage {
latest: String,
semver_compatible: String,
current: String,
name: String,
name: StackString,
}
#[allow(clippy::print_stdout)]

6
cli/tools/registry/unfurl.rs
View file

@ -663,6 +663,7 @@ mod tests {
use deno_config::workspace::ResolverWorkspaceJsrPackage;
use deno_core::serde_json::json;
use deno_core::url::Url;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use deno_runtime::deno_fs::RealFs;
use deno_runtime::deno_node::PackageJson;
use deno_semver::Version;
@ -722,10 +723,9 @@ mod tests {
vec![Arc::new(package_json)],
deno_config::workspace::PackageJsonDepResolution::Enabled,
);
let fs = Arc::new(RealFs);
let unfurler = SpecifierUnfurler::new(
Some(Arc::new(CliSloppyImportsResolver::new(
SloppyImportsCachedFs::new(fs),
SloppyImportsCachedFs::new(FsSysTraitsAdapter::new_real()),
))),
Arc::new(workspace_resolver),
true,
@ -863,7 +863,7 @@ const warn2 = await import(`${expr}`);
],
deno_config::workspace::PackageJsonDepResolution::Enabled,
);
let fs = Arc::new(RealFs);
let fs = FsSysTraitsAdapter(Arc::new(RealFs));
let unfurler = SpecifierUnfurler::new(
Some(Arc::new(CliSloppyImportsResolver::new(
SloppyImportsCachedFs::new(fs),

28
cli/tools/task.rs
View file

@ -231,7 +231,7 @@ pub async fn execute_script(
&Url::from_directory_path(cli_options.initial_cwd()).unwrap(),
"",
&TaskDefinition {
command: task_flags.task.as_ref().unwrap().to_string(),
command: Some(task_flags.task.as_ref().unwrap().to_string()),
dependencies: vec![],
description: None,
},
@ -448,6 +448,16 @@ impl<'a> TaskRunner<'a> {
kill_signal: KillSignal,
argv: &'a [String],
) -> Result<i32, deno_core::anyhow::Error> {
let Some(command) = &definition.command else {
log::info!(
"{} {} {}",
colors::green("Task"),
colors::cyan(task_name),
colors::gray("(no command)")
);
return Ok(0);
};
if let Some(npm_resolver) = self.npm_resolver.as_managed() {
npm_resolver.ensure_top_level_package_json_install().await?;
npm_resolver
@ -469,7 +479,7 @@ impl<'a> TaskRunner<'a> {
self
.run_single(RunSingleOptions {
task_name,
script: &definition.command,
script: command,
cwd: &cwd,
custom_commands,
kill_signal,
@ -837,7 +847,7 @@ fn print_available_tasks(
is_deno: false,
name: name.to_string(),
task: deno_config::deno_json::TaskDefinition {
command: script.to_string(),
command: Some(script.to_string()),
dependencies: vec![],
description: None,
},
@ -873,11 +883,13 @@ fn print_available_tasks(
)?;
}
}
writeln!(
writer,
" {}",
strip_ansi_codes_and_escape_control_chars(&desc.task.command)
)?;
if let Some(command) = &desc.task.command {
writeln!(
writer,
" {}",
strip_ansi_codes_and_escape_control_chars(command)
)?;
};
if !desc.task.dependencies.is_empty() {
let dependencies = desc
.task

5
cli/tools/test/mod.rs
View file

@ -589,7 +589,10 @@ async fn configure_main_worker(
WorkerExecutionMode::Test,
specifier.clone(),
permissions_container,
vec![ops::testing::deno_test::init_ops(worker_sender.sender)],
vec![
ops::testing::deno_test::init_ops(worker_sender.sender),
ops::lint::deno_lint::init_ops(),
],
Stdio {
stdin: StdioPipe::inherit(),
stdout: StdioPipe::file(worker_sender.stdout),

5
cli/tools/upgrade.rs
View file

@ -21,6 +21,7 @@ use deno_core::anyhow::Context;
use deno_core::error::AnyError;
use deno_core::unsync::spawn;
use deno_core::url::Url;
use deno_semver::SmallStackString;
use deno_semver::Version;
use once_cell::sync::Lazy;
use std::borrow::Cow;
@ -255,7 +256,7 @@ async fn print_release_notes(
let is_deno_2_rc = new_semver.major == 2
&& new_semver.minor == 0
&& new_semver.patch == 0
&& new_semver.pre.first() == Some(&"rc".to_string());
&& new_semver.pre.first().map(|s| s.as_str()) == Some("rc");
if is_deno_2_rc || is_switching_from_deno1_to_deno2 {
log::info!(
@ -674,7 +675,7 @@ impl RequestedVersion {
);
};
if semver.pre.contains(&"rc".to_string()) {
if semver.pre.contains(&SmallStackString::from_static("rc")) {
(ReleaseChannel::Rc, passed_version)
} else {
(ReleaseChannel::Stable, passed_version)

7
cli/tsc/99_main_compiler.js
View file

@ -41,6 +41,13 @@ delete Object.prototype.__proto__;
"listen",
"listenDatagram",
"openKv",
"connectQuic",
"listenQuic",
"QuicBidirectionalStream",
"QuicConn",
"QuicListener",
"QuicReceiveStream",
"QuicSendStream",
]);
const unstableMsgSuggestion =
"If not, try changing the 'lib' compiler option to include 'deno.unstable' " +

12
cli/tsc/mod.rs
View file

@ -4,7 +4,6 @@ use crate::args::TsConfig;
use crate::args::TypeCheckMode;
use crate::cache::FastInsecureHasher;
use crate::cache::ModuleInfoCache;
use crate::node;
use crate::npm::CliNpmResolver;
use crate::resolver::CjsTracker;
use crate::util::checksum;
@ -35,12 +34,13 @@ use deno_graph::Module;
use deno_graph::ModuleGraph;
use deno_graph::ResolutionResolved;
use deno_resolver::npm::ResolvePkgFolderFromDenoReqError;
use deno_runtime::deno_fs;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use deno_runtime::deno_node::NodeResolver;
use deno_semver::npm::NpmPackageReqReference;
use node_resolver::errors::NodeJsErrorCode;
use node_resolver::errors::NodeJsErrorCoded;
use node_resolver::errors::PackageSubpathResolveError;
use node_resolver::resolve_specifier_into_node_modules;
use node_resolver::NodeResolutionKind;
use node_resolver::ResolutionMode;
use once_cell::sync::Lazy;
@ -660,9 +660,9 @@ fn op_load_inner(
None
} else {
// means it's Deno code importing an npm module
let specifier = node::resolve_specifier_into_node_modules(
let specifier = resolve_specifier_into_node_modules(
&FsSysTraitsAdapter::new_real(),
&module.specifier,
&deno_fs::RealFs,
);
Some(Cow::Owned(load_from_node_modules(
&specifier,
@ -924,9 +924,9 @@ fn resolve_graph_specifier_types(
Some(Module::External(module)) => {
// we currently only use "External" for when the module is in an npm package
Ok(state.maybe_npm.as_ref().map(|_| {
let specifier = node::resolve_specifier_into_node_modules(
let specifier = resolve_specifier_into_node_modules(
&FsSysTraitsAdapter::new_real(),
&module.specifier,
&deno_fs::RealFs,
);
into_specifier_and_media_type(Some(specifier))
}))

318
cli/util/fs.rs
View file

@ -1,9 +1,7 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use std::fs::OpenOptions;
use std::io::Error;
use std::io::ErrorKind;
use std::io::Write;
use std::path::Path;
use std::path::PathBuf;
use std::sync::Arc;
@ -19,185 +17,12 @@ use deno_core::anyhow::Context;
use deno_core::error::AnyError;
use deno_core::unsync::spawn_blocking;
use deno_core::ModuleSpecifier;
use deno_runtime::deno_fs::FileSystem;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use crate::util::path::get_atomic_file_path;
use crate::util::progress_bar::ProgressBar;
use crate::util::progress_bar::ProgressBarStyle;
use crate::util::progress_bar::ProgressMessagePrompt;
/// Writes the file to the file system at a temporary path, then
/// renames it to the destination in a single sys call in order
/// to never leave the file system in a corrupted state.
///
/// This also handles creating the directory if a NotFound error
/// occurs.
pub fn atomic_write_file_with_retries<T: AsRef<[u8]>>(
file_path: &Path,
data: T,
mode: u32,
) -> std::io::Result<()> {
struct RealAtomicWriteFileFs {
mode: u32,
}
impl AtomicWriteFileFs for RealAtomicWriteFileFs {
fn write_file(&self, path: &Path, bytes: &[u8]) -> std::io::Result<()> {
write_file(path, bytes, self.mode)
}
fn rename_file(&self, from: &Path, to: &Path) -> std::io::Result<()> {
std::fs::rename(from, to)
}
fn remove_file(&self, path: &Path) -> std::io::Result<()> {
std::fs::remove_file(path)
}
fn create_dir_all(&self, dir_path: &Path) -> std::io::Result<()> {
std::fs::create_dir_all(dir_path)
}
fn path_exists(&self, path: &Path) -> bool {
path.exists()
}
}
atomic_write_file_with_retries_and_fs(
&RealAtomicWriteFileFs { mode },
file_path,
data.as_ref(),
)
}
pub trait AtomicWriteFileFs {
fn write_file(&self, path: &Path, bytes: &[u8]) -> std::io::Result<()>;
fn rename_file(&self, from: &Path, to: &Path) -> std::io::Result<()>;
fn remove_file(&self, path: &Path) -> std::io::Result<()>;
fn create_dir_all(&self, dir_path: &Path) -> std::io::Result<()>;
fn path_exists(&self, path: &Path) -> bool;
}
pub struct AtomicWriteFileFsAdapter<'a> {
pub fs: &'a dyn FileSystem,
pub write_mode: u32,
}
impl<'a> AtomicWriteFileFs for AtomicWriteFileFsAdapter<'a> {
fn write_file(&self, path: &Path, bytes: &[u8]) -> std::io::Result<()> {
self
.fs
.write_file_sync(
path,
deno_runtime::deno_fs::OpenOptions::write(
true,
false,
false,
Some(self.write_mode),
),
None,
bytes,
)
.map_err(|e| e.into_io_error())
}
fn rename_file(&self, from: &Path, to: &Path) -> std::io::Result<()> {
self.fs.rename_sync(from, to).map_err(|e| e.into_io_error())
}
fn remove_file(&self, path: &Path) -> std::io::Result<()> {
self
.fs
.remove_sync(path, false)
.map_err(|e| e.into_io_error())
}
fn create_dir_all(&self, dir_path: &Path) -> std::io::Result<()> {
self
.fs
.mkdir_sync(dir_path, /* recursive */ true, None)
.map_err(|e| e.into_io_error())
}
fn path_exists(&self, path: &Path) -> bool {
self.fs.exists_sync(path)
}
}
pub fn atomic_write_file_with_retries_and_fs<T: AsRef<[u8]>>(
fs: &impl AtomicWriteFileFs,
file_path: &Path,
data: T,
) -> std::io::Result<()> {
let mut count = 0;
loop {
match atomic_write_file(fs, file_path, data.as_ref()) {
Ok(()) => return Ok(()),
Err(err) => {
if count >= 5 {
// too many retries, return the error
return Err(err);
}
count += 1;
let sleep_ms = std::cmp::min(50, 10 * count);
std::thread::sleep(std::time::Duration::from_millis(sleep_ms));
}
}
}
}
/// Writes the file to the file system at a temporary path, then
/// renames it to the destination in a single sys call in order
/// to never leave the file system in a corrupted state.
///
/// This also handles creating the directory if a NotFound error
/// occurs.
fn atomic_write_file(
fs: &impl AtomicWriteFileFs,
file_path: &Path,
data: &[u8],
) -> std::io::Result<()> {
fn atomic_write_file_raw(
fs: &impl AtomicWriteFileFs,
temp_file_path: &Path,
file_path: &Path,
data: &[u8],
) -> std::io::Result<()> {
fs.write_file(temp_file_path, data)?;
fs.rename_file(temp_file_path, file_path)
.inspect_err(|_err| {
// clean up the created temp file on error
let _ = fs.remove_file(temp_file_path);
})
}
let temp_file_path = get_atomic_file_path(file_path);
if let Err(write_err) =
atomic_write_file_raw(fs, &temp_file_path, file_path, data)
{
if write_err.kind() == ErrorKind::NotFound {
let parent_dir_path = file_path.parent().unwrap();
match fs.create_dir_all(parent_dir_path) {
Ok(()) => {
return atomic_write_file_raw(fs, &temp_file_path, file_path, data)
.map_err(|err| add_file_context_to_err(file_path, err));
}
Err(create_err) => {
if !fs.path_exists(parent_dir_path) {
return Err(Error::new(
create_err.kind(),
format!(
"{:#} (for '{}')\nCheck the permission of the directory.",
create_err,
parent_dir_path.display()
),
));
}
}
}
}
return Err(add_file_context_to_err(file_path, write_err));
}
Ok(())
}
/// Creates a std::fs::File handling if the parent does not exist.
pub fn create_file(file_path: &Path) -> std::io::Result<std::fs::File> {
match std::fs::File::create(file_path) {
@ -236,45 +61,6 @@ fn add_file_context_to_err(file_path: &Path, err: Error) -> Error {
)
}
pub fn write_file<T: AsRef<[u8]>>(
filename: &Path,
data: T,
mode: u32,
) -> std::io::Result<()> {
write_file_2(filename, data, true, mode, true, false)
}
pub fn write_file_2<T: AsRef<[u8]>>(
filename: &Path,
data: T,
update_mode: bool,
mode: u32,
is_create: bool,
is_append: bool,
) -> std::io::Result<()> {
let mut file = OpenOptions::new()
.read(false)
.write(true)
.append(is_append)
.truncate(!is_append)
.create(is_create)
.open(filename)?;
if update_mode {
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
let mode = mode & 0o777;
let permissions = PermissionsExt::from_mode(mode);
file.set_permissions(permissions)?;
}
#[cfg(not(unix))]
let _ = mode;
}
file.write_all(data.as_ref())
}
/// Similar to `std::fs::canonicalize()` but strips UNC prefixes on Windows.
pub fn canonicalize_path(path: &Path) -> Result<PathBuf, Error> {
Ok(deno_path_util::strip_unc_prefix(path.canonicalize()?))
@ -289,16 +75,10 @@ pub fn canonicalize_path(path: &Path) -> Result<PathBuf, Error> {
pub fn canonicalize_path_maybe_not_exists(
path: &Path,
) -> Result<PathBuf, Error> {
deno_path_util::canonicalize_path_maybe_not_exists(path, &canonicalize_path)
}
pub fn canonicalize_path_maybe_not_exists_with_fs(
path: &Path,
fs: &dyn FileSystem,
) -> Result<PathBuf, Error> {
deno_path_util::canonicalize_path_maybe_not_exists(path, &|path| {
fs.realpath_sync(path).map_err(|err| err.into_io_error())
})
deno_path_util::fs::canonicalize_path_maybe_not_exists(
&FsSysTraitsAdapter::new_real(),
path,
)
}
/// Collects module specifiers that satisfy the given predicate as a file path, by recursively walking `include`.
@ -346,7 +126,7 @@ pub fn collect_specifiers(
.ignore_git_folder()
.ignore_node_modules()
.set_vendor_folder(vendor_folder)
.collect_file_patterns(&deno_config::fs::RealDenoConfigFs, files)?;
.collect_file_patterns(&FsSysTraitsAdapter::new_real(), files)?;
let mut collected_files_as_urls = collected_files
.iter()
.map(|f| specifier_from_file_path(f).unwrap())
@ -418,7 +198,13 @@ mod clone_dir_imp {
from: &std::path::Path,
to: &std::path::Path,
) -> Result<(), deno_core::error::AnyError> {
if let Err(e) = super::hard_link_dir_recursive(from, to) {
use deno_runtime::deno_fs::FsSysTraitsAdapter;
if let Err(e) = deno_npm_cache::hard_link_dir_recursive(
&FsSysTraitsAdapter::new_real(),
from,
to,
) {
log::debug!("Failed to hard link dir {:?} to {:?}: {}", from, to, e);
super::copy_dir_recursive(from, to)?;
}
@ -465,84 +251,6 @@ pub fn copy_dir_recursive(from: &Path, to: &Path) -> Result<(), AnyError> {
Ok(())
}
/// Hardlinks the files in one directory to another directory.
///
/// Note: Does not handle symlinks.
pub fn hard_link_dir_recursive(from: &Path, to: &Path) -> Result<(), AnyError> {
std::fs::create_dir_all(to)
.with_context(|| format!("Creating {}", to.display()))?;
let read_dir = std::fs::read_dir(from)
.with_context(|| format!("Reading {}", from.display()))?;
for entry in read_dir {
let entry = entry?;
let file_type = entry.file_type()?;
let new_from = from.join(entry.file_name());
let new_to = to.join(entry.file_name());
if file_type.is_dir() {
hard_link_dir_recursive(&new_from, &new_to).with_context(|| {
format!("Dir {} to {}", new_from.display(), new_to.display())
})?;
} else if file_type.is_file() {
// note: chance for race conditions here between attempting to create,
// then removing, then attempting to create. There doesn't seem to be
// a way to hard link with overwriting in Rust, but maybe there is some
// way with platform specific code. The workaround here is to handle
// scenarios where something else might create or remove files.
if let Err(err) = std::fs::hard_link(&new_from, &new_to) {
if err.kind() == ErrorKind::AlreadyExists {
if let Err(err) = std::fs::remove_file(&new_to) {
if err.kind() == ErrorKind::NotFound {
// Assume another process/thread created this hard link to the file we are wanting
// to remove then sleep a little bit to let the other process/thread move ahead
// faster to reduce contention.
std::thread::sleep(Duration::from_millis(10));
} else {
return Err(err).with_context(|| {
format!(
"Removing file to hard link {} to {}",
new_from.display(),
new_to.display()
)
});
}
}
// Always attempt to recreate the hardlink. In contention scenarios, the other process
// might have been killed or exited after removing the file, but before creating the hardlink
if let Err(err) = std::fs::hard_link(&new_from, &new_to) {
// Assume another process/thread created this hard link to the file we are wanting
// to now create then sleep a little bit to let the other process/thread move ahead
// faster to reduce contention.
if err.kind() == ErrorKind::AlreadyExists {
std::thread::sleep(Duration::from_millis(10));
} else {
return Err(err).with_context(|| {
format!(
"Hard linking {} to {}",
new_from.display(),
new_to.display()
)
});
}
}
} else {
return Err(err).with_context(|| {
format!(
"Hard linking {} to {}",
new_from.display(),
new_to.display()
)
});
}
}
}
}
Ok(())
}
pub fn symlink_dir(oldpath: &Path, newpath: &Path) -> Result<(), Error> {
let err_mapper = |err: Error, kind: Option<ErrorKind>| {
Error::new(

14
cli/util/path.rs
View file

@ -1,7 +1,6 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use std::borrow::Cow;
use std::fmt::Write;
use std::path::Path;
use std::path::PathBuf;
@ -52,19 +51,6 @@ pub fn get_extension(file_path: &Path) -> Option<String> {
.map(|e| e.to_lowercase());
}
pub fn get_atomic_file_path(file_path: &Path) -> PathBuf {
let rand = gen_rand_path_component();
let extension = format!("{rand}.tmp");
file_path.with_extension(extension)
}
fn gen_rand_path_component() -> String {
(0..4).fold(String::with_capacity(8), |mut output, _| {
write!(&mut output, "{:02x}", rand::random::<u8>()).unwrap();
output
})
}
/// TypeScript figures out the type of file based on the extension, but we take
/// other factors into account like the file headers. The hack here is to map the
/// specifier passed to TypeScript to a new specifier with the file extension.

12
cli/util/text_encoding.rs
View file

@ -140,23 +140,23 @@ mod tests {
#[test]
fn test_source_map_from_code() {
let to_string =
|bytes: Vec<u8>| -> String { String::from_utf8(bytes).unwrap() };
|bytes: Vec<u8>| -> String { String::from_utf8(bytes.to_vec()).unwrap() };
assert_eq!(
source_map_from_code(
b"test\n//# sourceMappingURL=data:application/json;base64,dGVzdGluZ3Rlc3Rpbmc=",
b"test\n//# sourceMappingURL=data:application/json;base64,dGVzdGluZ3Rlc3Rpbmc="
).map(to_string),
Some("testingtesting".to_string())
);
assert_eq!(
source_map_from_code(
b"test\n//# sourceMappingURL=data:application/json;base64,dGVzdGluZ3Rlc3Rpbmc=\n \n",
b"test\n//# sourceMappingURL=data:application/json;base64,dGVzdGluZ3Rlc3Rpbmc=\n \n"
).map(to_string),
Some("testingtesting".to_string())
);
assert_eq!(
source_map_from_code(
b"test\n//# sourceMappingURL=data:application/json;base64,dGVzdGluZ3Rlc3Rpbmc=\n test\n",
),
b"test\n//# sourceMappingURL=data:application/json;base64,dGVzdGluZ3Rlc3Rpbmc=\n test\n"
).map(to_string),
None
);
assert_eq!(
@ -164,7 +164,7 @@ mod tests {
b"\"use strict\";
throw new Error(\"Hello world!\");
//# sourceMappingURL=data:application/json;base64,{",
//# sourceMappingURL=data:application/json;base64,{"
),
None
);

7
cli/worker.rs
View file

@ -612,6 +612,7 @@ impl CliMainWorkerFactory {
serve_port: shared.options.serve_port,
serve_host: shared.options.serve_host.clone(),
otel_config: shared.otel_config.clone(),
close_on_idle: true,
},
extensions: custom_extensions,
startup_snapshot: crate::js::deno_isolate_init(),
@ -655,7 +656,10 @@ impl CliMainWorkerFactory {
"40_test_common.js",
"40_test.js",
"40_bench.js",
"40_jupyter.js"
"40_jupyter.js",
// TODO(bartlomieju): probably shouldn't include these files here?
"40_lint_selector.js",
"40_lint.js"
);
}
@ -812,6 +816,7 @@ fn create_web_worker_callback(
serve_port: shared.options.serve_port,
serve_host: shared.options.serve_host.clone(),
otel_config: shared.otel_config.clone(),
close_on_idle: args.close_on_idle,
},
extensions: vec![],
startup_snapshot: crate::js::deno_isolate_init(),

2
ext/broadcast_channel/Cargo.toml
View file

@ -2,7 +2,7 @@
[package]
name = "deno_broadcast_channel"
version = "0.177.0"
version = "0.178.0"
authors.workspace = true
edition.workspace = true
license.workspace = true

2
ext/cache/Cargo.toml vendored
View file

@ -2,7 +2,7 @@
[package]
name = "deno_cache"
version = "0.115.0"
version = "0.116.0"
authors.workspace = true
edition.workspace = true
license.workspace = true

2
ext/canvas/Cargo.toml
View file

@ -2,7 +2,7 @@
[package]
name = "deno_canvas"
version = "0.52.0"
version = "0.53.0"
authors.workspace = true
edition.workspace = true
license.workspace = true

2
ext/console/Cargo.toml
View file

@ -2,7 +2,7 @@
[package]
name = "deno_console"
version = "0.183.0"
version = "0.184.0"
authors.workspace = true
edition.workspace = true
license.workspace = true

2
ext/cron/Cargo.toml
View file

@ -2,7 +2,7 @@
[package]
name = "deno_cron"
version = "0.63.0"
version = "0.64.0"
authors.workspace = true
edition.workspace = true
license.workspace = true

2
ext/crypto/Cargo.toml
View file

@ -2,7 +2,7 @@
[package]
name = "deno_crypto"
version = "0.197.0"
version = "0.198.0"
authors.workspace = true
edition.workspace = true
license.workspace = true

15
ext/fetch/22_body.js
View file

@ -13,6 +13,7 @@
import { core, primordials } from "ext:core/mod.js";
const {
BadResourcePrototype,
isAnyArrayBuffer,
isArrayBuffer,
isStringObject,
@ -26,6 +27,7 @@ const {
JSONParse,
ObjectDefineProperties,
ObjectPrototypeIsPrototypeOf,
PromisePrototypeCatch,
TypedArrayPrototypeGetBuffer,
TypedArrayPrototypeGetByteLength,
TypedArrayPrototypeGetByteOffset,
@ -160,7 +162,18 @@ class InnerBody {
)
) {
readableStreamThrowIfErrored(this.stream);
return readableStreamCollectIntoUint8Array(this.stream);
return PromisePrototypeCatch(
readableStreamCollectIntoUint8Array(this.stream),
(e) => {
if (ObjectPrototypeIsPrototypeOf(BadResourcePrototype, e)) {
// TODO(kt3k): We probably like to pass e as `cause` if BadResource supports it.
throw new e.constructor(
"Cannot read body as underlying resource unavailable",
);
}
throw e;
},
);
} else {
this.streamOrStatic.consumed = true;
return this.streamOrStatic.body;

2
ext/fetch/Cargo.toml
View file

@ -2,7 +2,7 @@
[package]
name = "deno_fetch"
version = "0.207.0"
version = "0.208.0"
authors.workspace = true
edition.workspace = true
license.workspace = true

2
ext/ffi/Cargo.toml
View file

@ -2,7 +2,7 @@
[package]
name = "deno_ffi"
version = "0.170.0"
version = "0.171.0"
authors.workspace = true
edition.workspace = true
license.workspace = true

7
ext/fs/30_fs.js
View file

@ -77,6 +77,7 @@ const {
Error,
Function,
MathTrunc,
Number,
ObjectEntries,
ObjectDefineProperty,
ObjectPrototypeIsPrototypeOf,
@ -373,12 +374,12 @@ function parseFileInfo(response) {
isDirectory: response.isDirectory,
isSymlink: response.isSymlink,
size: response.size,
mtime: response.mtimeSet === true ? new Date(response.mtime) : null,
atime: response.atimeSet === true ? new Date(response.atime) : null,
mtime: response.mtimeSet === true ? new Date(Number(response.mtime)) : null,
atime: response.atimeSet === true ? new Date(Number(response.atime)) : null,
birthtime: response.birthtimeSet === true
? new Date(response.birthtime)
: null,
ctime: response.ctimeSet === true ? new Date(response.ctime) : null,
ctime: response.ctimeSet === true ? new Date(Number(response.ctime)) : null,
dev: response.dev,
mode: response.mode,
ino: unix ? response.ino : null,

4
ext/fs/Cargo.toml
View file

@ -2,7 +2,7 @@
[package]
name = "deno_fs"
version = "0.93.0"
version = "0.94.0"
authors.workspace = true
edition.workspace = true
license.workspace = true
@ -25,10 +25,12 @@ deno_io.workspace = true
deno_path_util.workspace = true
deno_permissions.workspace = true
filetime.workspace = true
getrandom = "0.2"
libc.workspace = true
rand.workspace = true
rayon = "1.8.0"
serde.workspace = true
sys_traits.workspace = true
thiserror.workspace = true
[target.'cfg(unix)'.dependencies]

342
ext/fs/interface.rs
View file

@ -5,6 +5,8 @@ use std::borrow::Cow;
use std::path::Path;
use std::path::PathBuf;
use std::rc::Rc;
use std::time::Duration;
use std::time::SystemTime;
use serde::Deserialize;
use serde::Serialize;
@ -12,6 +14,8 @@ use serde::Serialize;
use deno_io::fs::File;
use deno_io::fs::FsResult;
use deno_io::fs::FsStat;
use sys_traits::FsFile;
use sys_traits::FsFileSetPermissions;
use crate::sync::MaybeSend;
use crate::sync::MaybeSync;
@ -71,7 +75,7 @@ pub enum FsFileType {
}
/// WARNING: This is part of the public JS Deno API.
#[derive(Serialize)]
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct FsDirEntry {
pub name: String,
@ -100,6 +104,56 @@ impl<T> AccessCheckFn for T where
{
}
#[derive(Debug)]
pub struct FsStatSlim {
file_type: sys_traits::FileType,
modified: Result<SystemTime, std::io::Error>,
}
impl FsStatSlim {
pub fn from_std(metadata: &std::fs::Metadata) -> Self {
Self {
file_type: metadata.file_type().into(),
modified: metadata.modified(),
}
}
pub fn from_deno_fs_stat(data: &FsStat) -> Self {
FsStatSlim {
file_type: if data.is_file {
sys_traits::FileType::File
} else if data.is_directory {
sys_traits::FileType::Dir
} else if data.is_symlink {
sys_traits::FileType::Symlink
} else {
sys_traits::FileType::Unknown
},
modified: data
.mtime
.map(|ms| SystemTime::UNIX_EPOCH + Duration::from_millis(ms))
.ok_or_else(|| {
std::io::Error::new(std::io::ErrorKind::InvalidData, "No mtime")
}),
}
}
}
impl sys_traits::FsMetadataValue for FsStatSlim {
#[inline]
fn file_type(&self) -> sys_traits::FileType {
self.file_type
}
fn modified(&self) -> Result<SystemTime, std::io::Error> {
self
.modified
.as_ref()
.copied()
.map_err(|err| std::io::Error::new(err.kind(), err.to_string()))
}
}
pub type AccessCheckCb<'a> = &'a mut (dyn AccessCheckFn + 'a);
#[async_trait::async_trait(?Send)]
@ -361,3 +415,289 @@ fn string_from_utf8_lossy(buf: Vec<u8>) -> String {
Cow::Borrowed(_) => unsafe { String::from_utf8_unchecked(buf) },
}
}
// todo(dsherret): this is temporary. Instead of using the `FileSystem` trait implementation
// in the CLI, the CLI should instead create it's own file system using `sys_traits` traits
// then that can implement the `FileSystem` trait. Then this `FileSystem` trait can stay here
// for use only for `ext/fs` and not the entire CLI.
#[derive(Debug, Clone)]
pub struct FsSysTraitsAdapter(pub FileSystemRc);
impl FsSysTraitsAdapter {
pub fn new_real() -> Self {
Self(crate::sync::new_rc(crate::RealFs))
}
}
impl sys_traits::BaseFsHardLink for FsSysTraitsAdapter {
#[inline]
fn base_fs_hard_link(&self, src: &Path, dst: &Path) -> std::io::Result<()> {
self
.0
.link_sync(src, dst)
.map_err(|err| err.into_io_error())
}
}
impl sys_traits::BaseFsRead for FsSysTraitsAdapter {
#[inline]
fn base_fs_read(&self, path: &Path) -> std::io::Result<Cow<'static, [u8]>> {
self
.0
.read_file_sync(path, None)
.map_err(|err| err.into_io_error())
}
}
#[derive(Debug)]
pub struct FsSysTraitsAdapterReadDirEntry {
path: PathBuf,
entry: FsDirEntry,
}
impl sys_traits::FsDirEntry for FsSysTraitsAdapterReadDirEntry {
type Metadata = FsStatSlim;
fn file_name(&self) -> Cow<std::ffi::OsStr> {
Cow::Borrowed(self.entry.name.as_ref())
}
fn file_type(&self) -> std::io::Result<sys_traits::FileType> {
if self.entry.is_file {
Ok(sys_traits::FileType::File)
} else if self.entry.is_directory {
Ok(sys_traits::FileType::Dir)
} else if self.entry.is_symlink {
Ok(sys_traits::FileType::Symlink)
} else {
Ok(sys_traits::FileType::Unknown)
}
}
fn metadata(&self) -> std::io::Result<Self::Metadata> {
Ok(FsStatSlim {
file_type: self.file_type().unwrap(),
modified: Err(std::io::Error::new(
std::io::ErrorKind::Other,
"not supported",
)),
})
}
fn path(&self) -> Cow<Path> {
Cow::Borrowed(&self.path)
}
}
impl sys_traits::BaseFsReadDir for FsSysTraitsAdapter {
type ReadDirEntry = FsSysTraitsAdapterReadDirEntry;
fn base_fs_read_dir(
&self,
path: &Path,
) -> std::io::Result<
Box<dyn Iterator<Item = std::io::Result<Self::ReadDirEntry>>>,
> {
// todo(dsherret): needs to actually be iterable and not allocate a vector
let entries = self
.0
.read_dir_sync(path)
.map_err(|err| err.into_io_error())?;
let parent_dir = path.to_path_buf();
Ok(Box::new(entries.into_iter().map(move |entry| {
Ok(FsSysTraitsAdapterReadDirEntry {
path: parent_dir.join(&entry.name),
entry,
})
})))
}
}
impl sys_traits::BaseFsCanonicalize for FsSysTraitsAdapter {
#[inline]
fn base_fs_canonicalize(&self, path: &Path) -> std::io::Result<PathBuf> {
self
.0
.realpath_sync(path)
.map_err(|err| err.into_io_error())
}
}
impl sys_traits::BaseFsMetadata for FsSysTraitsAdapter {
type Metadata = FsStatSlim;
#[inline]
fn base_fs_metadata(&self, path: &Path) -> std::io::Result<Self::Metadata> {
self
.0
.stat_sync(path)
.map(|data| FsStatSlim::from_deno_fs_stat(&data))
.map_err(|err| err.into_io_error())
}
#[inline]
fn base_fs_symlink_metadata(
&self,
path: &Path,
) -> std::io::Result<Self::Metadata> {
self
.0
.lstat_sync(path)
.map(|data| FsStatSlim::from_deno_fs_stat(&data))
.map_err(|err| err.into_io_error())
}
}
impl sys_traits::BaseFsCreateDir for FsSysTraitsAdapter {
#[inline]
fn base_fs_create_dir(
&self,
path: &Path,
options: &sys_traits::CreateDirOptions,
) -> std::io::Result<()> {
self
.0
.mkdir_sync(path, options.recursive, options.mode)
.map_err(|err| err.into_io_error())
}
}
impl sys_traits::BaseFsRemoveFile for FsSysTraitsAdapter {
#[inline]
fn base_fs_remove_file(&self, path: &Path) -> std::io::Result<()> {
self
.0
.remove_sync(path, false)
.map_err(|err| err.into_io_error())
}
}
impl sys_traits::BaseFsRename for FsSysTraitsAdapter {
#[inline]
fn base_fs_rename(&self, from: &Path, to: &Path) -> std::io::Result<()> {
self
.0
.rename_sync(from, to)
.map_err(|err| err.into_io_error())
}
}
pub struct FsFileAdapter(pub Rc<dyn File>);
impl FsFile for FsFileAdapter {}
impl FsFileSetPermissions for FsFileAdapter {
#[inline]
fn fs_file_set_permissions(&mut self, mode: u32) -> std::io::Result<()> {
if cfg!(windows) {
Ok(()) // ignore
} else {
self
.0
.clone()
.chmod_sync(mode)
.map_err(|err| err.into_io_error())
}
}
}
impl std::io::Read for FsFileAdapter {
#[inline]
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
self
.0
.clone()
.read_sync(buf)
.map_err(|err| err.into_io_error())
}
}
impl std::io::Seek for FsFileAdapter {
fn seek(&mut self, pos: std::io::SeekFrom) -> std::io::Result<u64> {
self
.0
.clone()
.seek_sync(pos)
.map_err(|err| err.into_io_error())
}
}
impl std::io::Write for FsFileAdapter {
#[inline]
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
self
.0
.clone()
.write_sync(buf)
.map_err(|err| err.into_io_error())
}
#[inline]
fn flush(&mut self) -> std::io::Result<()> {
self
.0
.clone()
.sync_sync()
.map_err(|err| err.into_io_error())
}
}
impl sys_traits::BaseFsOpen for FsSysTraitsAdapter {
type File = FsFileAdapter;
fn base_fs_open(
&self,
path: &Path,
options: &sys_traits::OpenOptions,
) -> std::io::Result<Self::File> {
self
.0
.open_sync(
path,
OpenOptions {
read: options.read,
write: options.write,
create: options.create,
truncate: options.truncate,
append: options.append,
create_new: options.create_new,
mode: options.mode,
},
None,
)
.map(FsFileAdapter)
.map_err(|err| err.into_io_error())
}
}
impl sys_traits::SystemRandom for FsSysTraitsAdapter {
#[inline]
fn sys_random(&self, buf: &mut [u8]) -> std::io::Result<()> {
getrandom::getrandom(buf).map_err(|err| {
std::io::Error::new(std::io::ErrorKind::Other, err.to_string())
})
}
}
impl sys_traits::SystemTimeNow for FsSysTraitsAdapter {
#[inline]
fn sys_time_now(&self) -> SystemTime {
SystemTime::now()
}
}
impl sys_traits::ThreadSleep for FsSysTraitsAdapter {
#[inline]
fn thread_sleep(&self, dur: Duration) {
std::thread::sleep(dur);
}
}
impl sys_traits::BaseEnvVar for FsSysTraitsAdapter {
fn base_env_var_os(
&self,
key: &std::ffi::OsStr,
) -> Option<std::ffi::OsString> {
std::env::var_os(key)
}
}

2
ext/fs/lib.rs
View file

@ -13,6 +13,8 @@ pub use crate::interface::FileSystem;
pub use crate::interface::FileSystemRc;
pub use crate::interface::FsDirEntry;
pub use crate::interface::FsFileType;
pub use crate::interface::FsStatSlim;
pub use crate::interface::FsSysTraitsAdapter;
pub use crate::interface::OpenOptions;
pub use crate::ops::FsOpsError;
pub use crate::ops::FsOpsErrorKind;

34
ext/fs/std_fs.rs
View file

@ -723,30 +723,34 @@ fn cp(from: &Path, to: &Path) -> FsResult<()> {
}
}
match (fs::metadata(to), fs::symlink_metadata(to)) {
(Ok(m), _) if m.is_dir() => cp_(
source_meta,
from,
&to.join(from.file_name().ok_or_else(|| {
io::Error::new(
io::ErrorKind::InvalidInput,
"the source path is not a valid file",
)
})?),
)?,
(_, Ok(m)) if is_identical(&source_meta, &m) => {
if let Ok(m) = fs::metadata(to) {
if m.is_dir() {
return cp_(
source_meta,
from,
&to.join(from.file_name().ok_or_else(|| {
io::Error::new(
io::ErrorKind::InvalidInput,
"the source path is not a valid file",
)
})?),
);
}
}
if let Ok(m) = fs::symlink_metadata(to) {
if is_identical(&source_meta, &m) {
return Err(
io::Error::new(
io::ErrorKind::InvalidInput,
"the source and destination are the same file",
)
.into(),
)
);
}
_ => cp_(source_meta, from, to)?,
}
Ok(())
cp_(source_meta, from, to)
}
#[cfg(not(windows))]

6
ext/fs/sync.rs
View file

@ -21,3 +21,9 @@ mod inner {
pub trait MaybeSend {}
impl<T> MaybeSend for T where T: ?Sized {}
}
#[allow(clippy::disallowed_types)]
#[inline]
pub fn new_rc<T>(value: T) -> MaybeArc<T> {
MaybeArc::new(value)
}

2
ext/http/Cargo.toml
View file

@ -2,7 +2,7 @@
[package]
name = "deno_http"
version = "0.181.0"
version = "0.182.0"
authors.workspace = true
edition.workspace = true
license.workspace = true

2
ext/io/Cargo.toml
View file

@ -2,7 +2,7 @@
[package]
name = "deno_io"
version = "0.93.0"
version = "0.94.0"
authors.workspace = true
edition.workspace = true
license.workspace = true

2
ext/kv/Cargo.toml
View file

@ -2,7 +2,7 @@
[package]
name = "deno_kv"
version = "0.91.0"
version = "0.92.0"
authors.workspace = true
edition.workspace = true
license.workspace = true

2
ext/napi/Cargo.toml
View file

@ -2,7 +2,7 @@
[package]
name = "deno_napi"
version = "0.114.0"
version = "0.115.0"
authors.workspace = true
edition.workspace = true
license.workspace = true

2
ext/napi/sym/Cargo.toml
View file

@ -2,7 +2,7 @@
[package]
name = "napi_sym"
version = "0.113.0"
version = "0.114.0"
authors.workspace = true
edition.workspace = true
license.workspace = true

367
ext/net/03_quic.js Normal file
View file

@ -0,0 +1,367 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
import { core, primordials } from "ext:core/mod.js";
import {
op_quic_accept,
op_quic_accept_bi,
op_quic_accept_incoming,
op_quic_accept_uni,
op_quic_close_connection,
op_quic_close_endpoint,
op_quic_connect,
op_quic_connection_closed,
op_quic_connection_get_protocol,
op_quic_connection_get_remote_addr,
op_quic_endpoint_get_addr,
op_quic_get_send_stream_priority,
op_quic_incoming_accept,
op_quic_incoming_ignore,
op_quic_incoming_local_ip,
op_quic_incoming_refuse,
op_quic_incoming_remote_addr,
op_quic_incoming_remote_addr_validated,
op_quic_listen,
op_quic_max_datagram_size,
op_quic_open_bi,
op_quic_open_uni,
op_quic_read_datagram,
op_quic_send_datagram,
op_quic_set_send_stream_priority,
} from "ext:core/ops";
import {
getWritableStreamResourceBacking,
ReadableStream,
readableStreamForRid,
WritableStream,
writableStreamForRid,
} from "ext:deno_web/06_streams.js";
import { loadTlsKeyPair } from "ext:deno_net/02_tls.js";
const {
BadResourcePrototype,
} = core;
const {
Uint8Array,
TypedArrayPrototypeSubarray,
SymbolAsyncIterator,
SafePromisePrototypeFinally,
ObjectPrototypeIsPrototypeOf,
} = primordials;
class QuicSendStream extends WritableStream {
get sendOrder() {
return op_quic_get_send_stream_priority(
getWritableStreamResourceBacking(this).rid,
);
}
set sendOrder(p) {
op_quic_set_send_stream_priority(
getWritableStreamResourceBacking(this).rid,
p,
);
}
}
class QuicReceiveStream extends ReadableStream {}
function readableStream(rid, closed) {
// stream can be indirectly closed by closing connection.
SafePromisePrototypeFinally(closed, () => {
core.tryClose(rid);
});
return readableStreamForRid(rid, true, QuicReceiveStream);
}
function writableStream(rid, closed) {
// stream can be indirectly closed by closing connection.
SafePromisePrototypeFinally(closed, () => {
core.tryClose(rid);
});
return writableStreamForRid(rid, true, QuicSendStream);
}
class QuicBidirectionalStream {
#readable;
#writable;
constructor(txRid, rxRid, closed) {
this.#readable = readableStream(rxRid, closed);
this.#writable = writableStream(txRid, closed);
}
get readable() {
return this.#readable;
}
get writable() {
return this.#writable;
}
}
async function* bidiStream(conn, closed) {
try {
while (true) {
const r = await op_quic_accept_bi(conn);
yield new QuicBidirectionalStream(r[0], r[1], closed);
}
} catch (error) {
if (ObjectPrototypeIsPrototypeOf(BadResourcePrototype, error)) {
return;
}
throw error;
}
}
async function* uniStream(conn, closed) {
try {
while (true) {
const uniRid = await op_quic_accept_uni(conn);
yield readableStream(uniRid, closed);
}
} catch (error) {
if (ObjectPrototypeIsPrototypeOf(BadResourcePrototype, error)) {
return;
}
throw error;
}
}
class QuicConn {
#resource;
#bidiStream = null;
#uniStream = null;
#closed;
constructor(resource) {
this.#resource = resource;
this.#closed = op_quic_connection_closed(this.#resource);
core.unrefOpPromise(this.#closed);
}
get protocol() {
return op_quic_connection_get_protocol(this.#resource);
}
get remoteAddr() {
return op_quic_connection_get_remote_addr(this.#resource);
}
async createBidirectionalStream(
{ sendOrder, waitUntilAvailable } = { __proto__: null },
) {
const { 0: txRid, 1: rxRid } = await op_quic_open_bi(
this.#resource,
waitUntilAvailable ?? false,
);
if (sendOrder !== null && sendOrder !== undefined) {
op_quic_set_send_stream_priority(txRid, sendOrder);
}
return new QuicBidirectionalStream(txRid, rxRid, this.#closed);
}
async createUnidirectionalStream(
{ sendOrder, waitUntilAvailable } = { __proto__: null },
) {
const rid = await op_quic_open_uni(
this.#resource,
waitUntilAvailable ?? false,
);
if (sendOrder !== null && sendOrder !== undefined) {
op_quic_set_send_stream_priority(rid, sendOrder);
}
return writableStream(rid, this.#closed);
}
get incomingBidirectionalStreams() {
if (this.#bidiStream === null) {
this.#bidiStream = ReadableStream.from(
bidiStream(this.#resource, this.#closed),
);
}
return this.#bidiStream;
}
get incomingUnidirectionalStreams() {
if (this.#uniStream === null) {
this.#uniStream = ReadableStream.from(
uniStream(this.#resource, this.#closed),
);
}
return this.#uniStream;
}
get maxDatagramSize() {
return op_quic_max_datagram_size(this.#resource);
}
async readDatagram(p) {
const view = p || new Uint8Array(this.maxDatagramSize);
const nread = await op_quic_read_datagram(this.#resource, view);
return TypedArrayPrototypeSubarray(view, 0, nread);
}
async sendDatagram(data) {
await op_quic_send_datagram(this.#resource, data);
}
get closed() {
core.refOpPromise(this.#closed);
return this.#closed;
}
close({ closeCode, reason }) {
op_quic_close_connection(this.#resource, closeCode, reason);
}
}
class QuicIncoming {
#incoming;
constructor(incoming) {
this.#incoming = incoming;
}
get localIp() {
return op_quic_incoming_local_ip(this.#incoming);
}
get remoteAddr() {
return op_quic_incoming_remote_addr(this.#incoming);
}
get remoteAddressValidated() {
return op_quic_incoming_remote_addr_validated(this.#incoming);
}
async accept() {
const conn = await op_quic_incoming_accept(this.#incoming);
return new QuicConn(conn);
}
refuse() {
op_quic_incoming_refuse(this.#incoming);
}
ignore() {
op_quic_incoming_ignore(this.#incoming);
}
}
class QuicListener {
#endpoint;
constructor(endpoint) {
this.#endpoint = endpoint;
}
get addr() {
return op_quic_endpoint_get_addr(this.#endpoint);
}
async accept() {
const conn = await op_quic_accept(this.#endpoint);
return new QuicConn(conn);
}
async incoming() {
const incoming = await op_quic_accept_incoming(this.#endpoint);
return new QuicIncoming(incoming);
}
async next() {
let conn;
try {
conn = await this.accept();
} catch (error) {
if (ObjectPrototypeIsPrototypeOf(BadResourcePrototype, error)) {
return { value: undefined, done: true };
}
throw error;
}
return { value: conn, done: false };
}
[SymbolAsyncIterator]() {
return this;
}
close({ closeCode, reason }) {
op_quic_close_endpoint(this.#endpoint, closeCode, reason);
}
}
async function listenQuic(
{
hostname,
port,
cert,
key,
alpnProtocols,
keepAliveInterval,
maxIdleTimeout,
maxConcurrentBidirectionalStreams,
maxConcurrentUnidirectionalStreams,
},
) {
hostname = hostname || "0.0.0.0";
const keyPair = loadTlsKeyPair("Deno.listenQuic", { cert, key });
const endpoint = await op_quic_listen(
{ hostname, port },
{ alpnProtocols },
{
keepAliveInterval,
maxIdleTimeout,
maxConcurrentBidirectionalStreams,
maxConcurrentUnidirectionalStreams,
},
keyPair,
);
return new QuicListener(endpoint);
}
async function connectQuic(
{
hostname,
port,
serverName,
caCerts,
cert,
key,
alpnProtocols,
keepAliveInterval,
maxIdleTimeout,
maxConcurrentBidirectionalStreams,
maxConcurrentUnidirectionalStreams,
congestionControl,
},
) {
const keyPair = loadTlsKeyPair("Deno.connectQuic", { cert, key });
const conn = await op_quic_connect(
{ hostname, port },
{
caCerts,
alpnProtocols,
serverName,
},
{
keepAliveInterval,
maxIdleTimeout,
maxConcurrentBidirectionalStreams,
maxConcurrentUnidirectionalStreams,
congestionControl,
},
keyPair,
);
return new QuicConn(conn);
}
export {
connectQuic,
listenQuic,
QuicBidirectionalStream,
QuicConn,
QuicIncoming,
QuicListener,
QuicReceiveStream,
QuicSendStream,
};

3
ext/net/Cargo.toml
View file

@ -2,7 +2,7 @@
[package]
name = "deno_net"
version = "0.175.0"
version = "0.176.0"
authors.workspace = true
edition.workspace = true
license.workspace = true
@ -20,6 +20,7 @@ deno_tls.workspace = true
hickory-proto = "0.25.0-alpha.4"
hickory-resolver.workspace = true
pin-project.workspace = true
quinn = { version = "0.11.6", default-features = false, features = ["runtime-tokio", "rustls", "ring"] }
rustls-tokio-stream.workspace = true
serde.workspace = true
socket2.workspace = true

Some files were not shown because too many files have changed in this diff Show more