diff --git a/.github/workflows/cargo_publish.yml b/.github/workflows/cargo_publish.yml index 3af97f4662..eb72e3739f 100644 --- a/.github/workflows/cargo_publish.yml +++ b/.github/workflows/cargo_publish.yml @@ -35,7 +35,7 @@ jobs: - name: Install deno uses: denoland/setup-deno@v2 with: - deno-version: v1.x + deno-version: v2.x - name: Publish env: diff --git a/.github/workflows/ci.generate.ts b/.github/workflows/ci.generate.ts index cb83842c2f..5c2de96006 100755 --- a/.github/workflows/ci.generate.ts +++ b/.github/workflows/ci.generate.ts @@ -5,7 +5,7 @@ import { stringify } from "jsr:@std/yaml@^0.221/stringify"; // Bump this number when you want to purge the cache. // Note: the tools/release/01_bump_crate_versions.ts script will update this version // automatically via regex, so ensure that this line maintains this format. -const cacheVersion = 29; +const cacheVersion = 30; const ubuntuX86Runner = "ubuntu-24.04"; const ubuntuX86XlRunner = "ubuntu-24.04-xl"; @@ -59,6 +59,15 @@ const Runners = { const prCacheKeyPrefix = `${cacheVersion}-cargo-target-\${{ matrix.os }}-\${{ matrix.arch }}-\${{ matrix.profile }}-\${{ matrix.job }}-`; +const prCacheKey = `${prCacheKeyPrefix}\${{ github.sha }}`; +const prCachePath = [ + // this must match for save and restore (https://github.com/actions/cache/issues/1444) + "./target", + "!./target/*/gn_out", + "!./target/*/gn_root", + "!./target/*/*.zip", + "!./target/*/*.tar.gz", +].join("\n"); // Note that you may need to add more version to the `apt-get remove` line below if you change this const llvmVersion = 19; @@ -196,7 +205,7 @@ const installNodeStep = { const installDenoStep = { name: "Install Deno", uses: "denoland/setup-deno@v2", - with: { "deno-version": "v1.x" }, + with: { "deno-version": "v2.x" }, }; const authenticateWithGoogleCloud = { @@ -612,7 +621,7 @@ const ci = { `${cacheVersion}-cargo-home-\${{ matrix.os }}-\${{ matrix.arch }}-\${{ hashFiles('Cargo.lock') }}`, // We will try to restore from the closest cargo-home we can find "restore-keys": - `${cacheVersion}-cargo-home-\${{ matrix.os }}-\${{ matrix.arch }}`, + `${cacheVersion}-cargo-home-\${{ matrix.os }}-\${{ matrix.arch }}-`, }, }, { @@ -622,13 +631,7 @@ const ci = { if: "github.ref != 'refs/heads/main' && !startsWith(github.ref, 'refs/tags/')", with: { - path: [ - "./target", - "!./target/*/gn_out", - "!./target/*/gn_root", - "!./target/*/*.zip", - "!./target/*/*.tar.gz", - ].join("\n"), + path: prCachePath, key: "never_saved", "restore-keys": prCacheKeyPrefix, }, @@ -1080,14 +1083,8 @@ const ci = { if: "(matrix.job == 'test' || matrix.job == 'lint') && github.ref == 'refs/heads/main'", with: { - path: [ - "./target", - "!./target/*/gn_out", - "!./target/*/*.zip", - "!./target/*/*.sha256sum", - "!./target/*/*.tar.gz", - ].join("\n"), - key: prCacheKeyPrefix + "${{ github.sha }}", + path: prCachePath, + key: prCacheKey, }, }, ]), diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6dc71ffa2d..44eb15cb95 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -180,7 +180,7 @@ jobs: name: Install Deno uses: denoland/setup-deno@v2 with: - deno-version: v1.x + deno-version: v2.x - name: Install Python uses: actions/setup-python@v5 with: @@ -361,8 +361,8 @@ jobs: path: |- ~/.cargo/registry/index ~/.cargo/registry/cache - key: '29-cargo-home-${{ matrix.os }}-${{ matrix.arch }}-${{ hashFiles(''Cargo.lock'') }}' - restore-keys: '29-cargo-home-${{ matrix.os }}-${{ matrix.arch }}' + key: '30-cargo-home-${{ matrix.os }}-${{ matrix.arch }}-${{ hashFiles(''Cargo.lock'') }}' + restore-keys: '30-cargo-home-${{ matrix.os }}-${{ matrix.arch }}-' if: '!(matrix.skip)' - name: Restore cache build output (PR) uses: actions/cache/restore@v4 @@ -375,7 +375,7 @@ jobs: !./target/*/*.zip !./target/*/*.tar.gz key: never_saved - restore-keys: '29-cargo-target-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.profile }}-${{ matrix.job }}-' + restore-keys: '30-cargo-target-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.profile }}-${{ matrix.job }}-' - name: Apply and update mtime cache if: '!(matrix.skip) && (!startsWith(github.ref, ''refs/tags/''))' uses: ./.github/mtime_cache @@ -682,10 +682,10 @@ jobs: path: |- ./target !./target/*/gn_out + !./target/*/gn_root !./target/*/*.zip - !./target/*/*.sha256sum !./target/*/*.tar.gz - key: '29-cargo-target-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.profile }}-${{ matrix.job }}-${{ github.sha }}' + key: '30-cargo-target-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.profile }}-${{ matrix.job }}-${{ github.sha }}' publish-canary: name: publish canary runs-on: ubuntu-24.04 diff --git a/.github/workflows/npm_publish.yml b/.github/workflows/npm_publish.yml new file mode 100644 index 0000000000..5e58005926 --- /dev/null +++ b/.github/workflows/npm_publish.yml @@ -0,0 +1,45 @@ +name: npm_publish + +on: + workflow_dispatch: + inputs: + version: + description: 'Version' + type: string + release: + types: [published] + +permissions: + id-token: write + +jobs: + build: + name: npm publish + runs-on: ubuntu-latest + timeout-minutes: 30 + + steps: + - name: Configure git + run: | + git config --global core.symlinks true + git config --global fetch.parallel 32 + + - name: Clone repository + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Install Deno + uses: denoland/setup-deno@v2 + with: + deno-version: v2.x + - name: Install Node + uses: actions/setup-node@v4 + with: + node-version: '22.x' + registry-url: 'https://registry.npmjs.org' + + - name: Publish + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + run: ./tools/release/npm/build.ts ${{ github.event.inputs.version }} --publish diff --git a/.github/workflows/promote_to_release.yml b/.github/workflows/promote_to_release.yml index 79fefa6d6c..4079118d92 100644 --- a/.github/workflows/promote_to_release.yml +++ b/.github/workflows/promote_to_release.yml @@ -42,7 +42,7 @@ jobs: - name: Install deno uses: denoland/setup-deno@v2 with: - deno-version: v1.x + deno-version: v2.x - name: Install rust-codesign run: |- diff --git a/.github/workflows/start_release.yml b/.github/workflows/start_release.yml index 40a44bb61a..35446c1adb 100644 --- a/.github/workflows/start_release.yml +++ b/.github/workflows/start_release.yml @@ -36,7 +36,7 @@ jobs: - name: Install deno uses: denoland/setup-deno@v2 with: - deno-version: v1.x + deno-version: v2.x - name: Create Gist URL env: diff --git a/.github/workflows/version_bump.yml b/.github/workflows/version_bump.yml index 9038fe0d22..306a8642ad 100644 --- a/.github/workflows/version_bump.yml +++ b/.github/workflows/version_bump.yml @@ -41,7 +41,7 @@ jobs: - name: Install deno uses: denoland/setup-deno@v2 with: - deno-version: v1.x + deno-version: v2.x - name: Run version bump run: | diff --git a/Cargo.lock b/Cargo.lock index a61065a12d..7b7fbc0ba0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -658,9 +658,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.6.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" +checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b" [[package]] name = "cache_control" @@ -1215,7 +1215,7 @@ dependencies = [ [[package]] name = "deno" -version = "2.1.3" +version = "2.1.4" dependencies = [ "anstream", "async-trait", @@ -1388,7 +1388,7 @@ dependencies = [ [[package]] name = "deno_bench_util" -version = "0.175.0" +version = "0.176.0" dependencies = [ "bencher", "deno_core", @@ -1397,7 +1397,7 @@ dependencies = [ [[package]] name = "deno_broadcast_channel" -version = "0.175.0" +version = "0.176.0" dependencies = [ "async-trait", "deno_core", @@ -1408,7 +1408,7 @@ dependencies = [ [[package]] name = "deno_cache" -version = "0.113.0" +version = "0.114.0" dependencies = [ "async-trait", "deno_core", @@ -1441,7 +1441,7 @@ dependencies = [ [[package]] name = "deno_canvas" -version = "0.50.0" +version = "0.51.0" dependencies = [ "deno_core", "deno_webgpu", @@ -1476,7 +1476,7 @@ dependencies = [ [[package]] name = "deno_console" -version = "0.181.0" +version = "0.182.0" dependencies = [ "deno_core", ] @@ -1524,7 +1524,7 @@ checksum = "fe4dccb6147bb3f3ba0c7a48e993bfeb999d2c2e47a81badee80e2b370c8d695" [[package]] name = "deno_cron" -version = "0.61.0" +version = "0.62.0" dependencies = [ "anyhow", "async-trait", @@ -1537,7 +1537,7 @@ dependencies = [ [[package]] name = "deno_crypto" -version = "0.195.0" +version = "0.196.0" dependencies = [ "aes", "aes-gcm", @@ -1626,7 +1626,7 @@ dependencies = [ [[package]] name = "deno_fetch" -version = "0.205.0" +version = "0.206.0" dependencies = [ "base64 0.21.7", "bytes", @@ -1661,7 +1661,7 @@ dependencies = [ [[package]] name = "deno_ffi" -version = "0.168.0" +version = "0.169.0" dependencies = [ "deno_core", "deno_permissions", @@ -1681,7 +1681,7 @@ dependencies = [ [[package]] name = "deno_fs" -version = "0.91.0" +version = "0.92.0" dependencies = [ "async-trait", "base32", @@ -1734,7 +1734,7 @@ dependencies = [ [[package]] name = "deno_http" -version = "0.179.0" +version = "0.180.0" dependencies = [ "async-compression", "async-trait", @@ -1773,7 +1773,7 @@ dependencies = [ [[package]] name = "deno_io" -version = "0.91.0" +version = "0.92.0" dependencies = [ "async-trait", "deno_core", @@ -1794,7 +1794,7 @@ dependencies = [ [[package]] name = "deno_kv" -version = "0.89.0" +version = "0.90.0" dependencies = [ "anyhow", "async-trait", @@ -1867,7 +1867,7 @@ dependencies = [ [[package]] name = "deno_napi" -version = "0.112.0" +version = "0.113.0" dependencies = [ "deno_core", "deno_permissions", @@ -1895,7 +1895,7 @@ dependencies = [ [[package]] name = "deno_net" -version = "0.173.0" +version = "0.174.0" dependencies = [ "deno_core", "deno_permissions", @@ -1912,7 +1912,7 @@ dependencies = [ [[package]] name = "deno_node" -version = "0.118.0" +version = "0.119.0" dependencies = [ "aead-gcm-stream", "aes", @@ -2025,7 +2025,7 @@ dependencies = [ [[package]] name = "deno_npm_cache" -version = "0.1.0" +version = "0.2.0" dependencies = [ "anyhow", "async-trait", @@ -2096,8 +2096,9 @@ dependencies = [ [[package]] name = "deno_permissions" -version = "0.41.0" +version = "0.42.0" dependencies = [ + "capacity_builder", "deno_core", "deno_path_util", "deno_terminal 0.2.0", @@ -2114,7 +2115,7 @@ dependencies = [ [[package]] name = "deno_resolver" -version = "0.13.0" +version = "0.14.0" dependencies = [ "anyhow", "base32", @@ -2133,7 +2134,7 @@ dependencies = [ [[package]] name = "deno_runtime" -version = "0.190.0" +version = "0.191.0" dependencies = [ "color-print", "deno_ast", @@ -2202,9 +2203,9 @@ dependencies = [ [[package]] name = "deno_semver" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4756be7351289726087408984db18b9eb5e0186907673f39f858d119d0162071" +checksum = "7d1259270d66a5e6d29bb75c9289656541874f79ae9ff6c9f1c790846d5c07ba" dependencies = [ "deno_error", "monch", @@ -2234,7 +2235,7 @@ dependencies = [ [[package]] name = "deno_telemetry" -version = "0.3.0" +version = "0.4.0" dependencies = [ "async-trait", "deno_core", @@ -2275,7 +2276,7 @@ dependencies = [ [[package]] name = "deno_tls" -version = "0.168.0" +version = "0.169.0" dependencies = [ "deno_core", "deno_native_certs", @@ -2325,7 +2326,7 @@ dependencies = [ [[package]] name = "deno_url" -version = "0.181.0" +version = "0.182.0" dependencies = [ "deno_bench_util", "deno_console", @@ -2337,7 +2338,7 @@ dependencies = [ [[package]] name = "deno_web" -version = "0.212.0" +version = "0.213.0" dependencies = [ "async-trait", "base64-simd 0.8.0", @@ -2359,7 +2360,7 @@ dependencies = [ [[package]] name = "deno_webgpu" -version = "0.148.0" +version = "0.149.0" dependencies = [ "deno_core", "raw-window-handle", @@ -2372,7 +2373,7 @@ dependencies = [ [[package]] name = "deno_webidl" -version = "0.181.0" +version = "0.182.0" dependencies = [ "deno_bench_util", "deno_core", @@ -2380,7 +2381,7 @@ dependencies = [ [[package]] name = "deno_websocket" -version = "0.186.0" +version = "0.187.0" dependencies = [ "bytes", "deno_core", @@ -2402,7 +2403,7 @@ dependencies = [ [[package]] name = "deno_webstorage" -version = "0.176.0" +version = "0.177.0" dependencies = [ "deno_core", "deno_web", @@ -4022,9 +4023,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.7" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cde7055719c54e36e95e8719f95883f22072a48ede39db7fc17a4e1d5281e9b9" +checksum = "df2dcfbe0677734ab2f3ffa7fa7bfd4706bfdc1ef393f2ee30184aed67e631b4" dependencies = [ "bytes", "futures-channel", @@ -4035,7 +4036,6 @@ dependencies = [ "pin-project-lite", "socket2", "tokio", - "tower", "tower-service", "tracing", ] @@ -4917,7 +4917,7 @@ dependencies = [ [[package]] name = "napi_sym" -version = "0.111.0" +version = "0.112.0" dependencies = [ "quote", "serde", @@ -4972,7 +4972,7 @@ dependencies = [ [[package]] name = "node_resolver" -version = "0.20.0" +version = "0.21.0" dependencies = [ "anyhow", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index 62b05c6791..ae81662d72 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -50,17 +50,17 @@ repository = "https://github.com/denoland/deno" deno_ast = { version = "=0.44.0", features = ["transpiling"] } deno_core = { version = "0.325.0" } -deno_bench_util = { version = "0.175.0", path = "./bench_util" } +deno_bench_util = { version = "0.176.0", path = "./bench_util" } deno_config = { version = "=0.39.3", features = ["workspace", "sync"] } deno_lockfile = "=0.23.2" deno_media_type = { version = "0.2.0", features = ["module_specifier"] } deno_npm = "=0.26.0" deno_path_util = "=0.2.1" -deno_permissions = { version = "0.41.0", path = "./runtime/permissions" } -deno_runtime = { version = "0.190.0", path = "./runtime" } -deno_semver = "=0.6.0" +deno_permissions = { version = "0.42.0", path = "./runtime/permissions" } +deno_runtime = { version = "0.191.0", path = "./runtime" } +deno_semver = "=0.6.1" deno_terminal = "0.2.0" -napi_sym = { version = "0.111.0", path = "./ext/napi/sym" } +napi_sym = { version = "0.112.0", path = "./ext/napi/sym" } test_util = { package = "test_server", path = "./tests/util/server" } denokv_proto = "0.8.4" @@ -69,34 +69,34 @@ denokv_remote = "0.8.4" denokv_sqlite = { default-features = false, version = "0.8.4" } # exts -deno_broadcast_channel = { version = "0.175.0", path = "./ext/broadcast_channel" } -deno_cache = { version = "0.113.0", path = "./ext/cache" } -deno_canvas = { version = "0.50.0", path = "./ext/canvas" } -deno_console = { version = "0.181.0", path = "./ext/console" } -deno_cron = { version = "0.61.0", path = "./ext/cron" } -deno_crypto = { version = "0.195.0", path = "./ext/crypto" } -deno_fetch = { version = "0.205.0", path = "./ext/fetch" } -deno_ffi = { version = "0.168.0", path = "./ext/ffi" } -deno_fs = { version = "0.91.0", path = "./ext/fs" } -deno_http = { version = "0.179.0", path = "./ext/http" } -deno_io = { version = "0.91.0", path = "./ext/io" } -deno_kv = { version = "0.89.0", path = "./ext/kv" } -deno_napi = { version = "0.112.0", path = "./ext/napi" } -deno_net = { version = "0.173.0", path = "./ext/net" } -deno_node = { version = "0.118.0", path = "./ext/node" } -deno_telemetry = { version = "0.3.0", path = "./ext/telemetry" } -deno_tls = { version = "0.168.0", path = "./ext/tls" } -deno_url = { version = "0.181.0", path = "./ext/url" } -deno_web = { version = "0.212.0", path = "./ext/web" } -deno_webgpu = { version = "0.148.0", path = "./ext/webgpu" } -deno_webidl = { version = "0.181.0", path = "./ext/webidl" } -deno_websocket = { version = "0.186.0", path = "./ext/websocket" } -deno_webstorage = { version = "0.176.0", path = "./ext/webstorage" } +deno_broadcast_channel = { version = "0.176.0", path = "./ext/broadcast_channel" } +deno_cache = { version = "0.114.0", path = "./ext/cache" } +deno_canvas = { version = "0.51.0", path = "./ext/canvas" } +deno_console = { version = "0.182.0", path = "./ext/console" } +deno_cron = { version = "0.62.0", path = "./ext/cron" } +deno_crypto = { version = "0.196.0", path = "./ext/crypto" } +deno_fetch = { version = "0.206.0", path = "./ext/fetch" } +deno_ffi = { version = "0.169.0", path = "./ext/ffi" } +deno_fs = { version = "0.92.0", path = "./ext/fs" } +deno_http = { version = "0.180.0", path = "./ext/http" } +deno_io = { version = "0.92.0", path = "./ext/io" } +deno_kv = { version = "0.90.0", path = "./ext/kv" } +deno_napi = { version = "0.113.0", path = "./ext/napi" } +deno_net = { version = "0.174.0", path = "./ext/net" } +deno_node = { version = "0.119.0", path = "./ext/node" } +deno_telemetry = { version = "0.4.0", path = "./ext/telemetry" } +deno_tls = { version = "0.169.0", path = "./ext/tls" } +deno_url = { version = "0.182.0", path = "./ext/url" } +deno_web = { version = "0.213.0", path = "./ext/web" } +deno_webgpu = { version = "0.149.0", path = "./ext/webgpu" } +deno_webidl = { version = "0.182.0", path = "./ext/webidl" } +deno_websocket = { version = "0.187.0", path = "./ext/websocket" } +deno_webstorage = { version = "0.177.0", path = "./ext/webstorage" } # resolvers -deno_npm_cache = { version = "0.1.0", path = "./resolvers/npm_cache" } -deno_resolver = { version = "0.13.0", path = "./resolvers/deno" } -node_resolver = { version = "0.20.0", path = "./resolvers/node" } +deno_npm_cache = { version = "0.2.0", path = "./resolvers/npm_cache" } +deno_resolver = { version = "0.14.0", path = "./resolvers/deno" } +node_resolver = { version = "0.21.0", path = "./resolvers/node" } aes = "=0.8.3" anyhow = "1.0.57" @@ -108,6 +108,7 @@ boxed_error = "0.2.2" brotli = "6.0.0" bytes = "1.4.0" cache_control = "=0.2.0" +capacity_builder = "0.1.0" cbc = { version = "=0.1.2", features = ["alloc"] } # Note: Do not use the "clock" feature of chrono, as it links us to CoreFoundation on macOS. # Instead use util::time::utc_now() @@ -141,7 +142,7 @@ http_v02 = { package = "http", version = "0.2.9" } httparse = "1.8.0" hyper = { version = "1.4.1", features = ["full"] } hyper-rustls = { version = "0.27.2", default-features = false, features = ["http1", "http2", "tls12", "ring"] } -hyper-util = { version = "=0.1.7", features = ["tokio", "client", "client-legacy", "server", "server-auto"] } +hyper-util = { version = "0.1.10", features = ["tokio", "client", "client-legacy", "server", "server-auto"] } hyper_v014 = { package = "hyper", version = "0.14.26", features = ["runtime", "http1"] } indexmap = { version = "2", features = ["serde"] } ipnet = "2.3" diff --git a/Releases.md b/Releases.md index e395934ac1..aaae202a37 100644 --- a/Releases.md +++ b/Releases.md @@ -6,6 +6,26 @@ https://github.com/denoland/deno/releases We also have one-line install commands at: https://github.com/denoland/deno_install +### 2.1.4 / 2024.12.11 + +- feat(unstable): support caching npm dependencies only as they're needed + (#27300) +- fix(compile): correct read length for transpiled typescript files (#27301) +- fix(ext/node): accept file descriptor in fs.readFile(Sync) (#27252) +- fix(ext/node): handle Float16Array in node:v8 module (#27285) +- fix(lint): do not error providing --allow-import (#27321) +- fix(node): update list of builtin node modules, add missing export to + _http_common (#27294) +- fix(outdated): error when there are no config files (#27306) +- fix(outdated): respect --quiet flag for hints (#27317) +- fix(outdated): show a suggestion for updating (#27304) +- fix(task): do not always kill child on ctrl+c on windows (#27269) +- fix(unstable): don't unwrap optional state in otel (#27292) +- fix: do not error when subpath has an @ symbol (#27290) +- fix: do not panic when fetching invalid file url on Windows (#27259) +- fix: replace the @deno-types with @ts-types (#27310) +- perf(compile): improve FileBackedVfsFile (#27299) + ### 2.1.3 / 2024.12.05 - feat(unstable): add metrics to otel (#27143) diff --git a/bench_util/Cargo.toml b/bench_util/Cargo.toml index 65c51c24d5..9833996fd3 100644 --- a/bench_util/Cargo.toml +++ b/bench_util/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_bench_util" -version = "0.175.0" +version = "0.176.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 74ac159840..b71167e509 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno" -version = "2.1.3" +version = "2.1.4" authors.workspace = true default-run = "deno" edition.workspace = true diff --git a/cli/args/deno_json.rs b/cli/args/deno_json.rs index c2ba31fd36..8853107eef 100644 --- a/cli/args/deno_json.rs +++ b/cli/args/deno_json.rs @@ -64,6 +64,15 @@ impl<'a> deno_config::fs::DenoConfigFs for DenoConfigFsAdapter<'a> { } } +pub fn import_map_deps( + import_map: &serde_json::Value, +) -> HashSet { + let values = imports_values(import_map.get("imports")) + .into_iter() + .chain(scope_values(import_map.get("scopes"))); + values_to_set(values) +} + pub fn deno_json_deps( config: &deno_config::deno_json::ConfigFile, ) -> HashSet { diff --git a/cli/args/flags.rs b/cli/args/flags.rs index cdeaa1b335..418edcf34b 100644 --- a/cli/args/flags.rs +++ b/cli/args/flags.rs @@ -37,6 +37,7 @@ use deno_path_util::url_to_file_path; use deno_runtime::deno_permissions::PermissionsOptions; use deno_runtime::deno_permissions::SysDescriptor; use deno_telemetry::OtelConfig; +use deno_telemetry::OtelConsoleConfig; use log::debug; use log::Level; use serde::Deserialize; @@ -245,7 +246,7 @@ pub struct InstallFlagsGlobal { } #[derive(Clone, Debug, Eq, PartialEq)] -pub enum InstallKind { +pub enum InstallFlags { Local(InstallFlagsLocal), Global(InstallFlagsGlobal), } @@ -257,11 +258,6 @@ pub enum InstallFlagsLocal { Entrypoints(Vec), } -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct InstallFlags { - pub kind: InstallKind, -} - #[derive(Clone, Debug, Eq, PartialEq)] pub struct JSONReferenceFlags { pub json: deno_core::serde_json::Value, @@ -600,6 +596,7 @@ pub struct UnstableConfig { pub bare_node_builtins: bool, pub detect_cjs: bool, pub sloppy_imports: bool, + pub npm_lazy_caching: bool, pub features: Vec, // --unstabe-kv --unstable-cron } @@ -990,21 +987,41 @@ impl Flags { args } - pub fn otel_config(&self) -> Option { - if self + pub fn otel_config(&self) -> OtelConfig { + let has_unstable_flag = self .unstable_config .features - .contains(&String::from("otel")) - { - Some(OtelConfig { - runtime_name: Cow::Borrowed("deno"), - runtime_version: Cow::Borrowed(crate::version::DENO_VERSION_INFO.deno), - deterministic: std::env::var("DENO_UNSTABLE_OTEL_DETERMINISTIC") - .is_ok(), - ..Default::default() - }) - } else { - None + .contains(&String::from("otel")); + + let otel_var = |name| match std::env::var(name) { + Ok(s) if s.to_lowercase() == "true" => Some(true), + Ok(s) if s.to_lowercase() == "false" => Some(false), + _ => None, + }; + + let disabled = + !has_unstable_flag || otel_var("OTEL_SDK_DISABLED").unwrap_or(false); + let default = !disabled && otel_var("OTEL_DENO").unwrap_or(false); + + OtelConfig { + tracing_enabled: !disabled + && otel_var("OTEL_DENO_TRACING").unwrap_or(default), + console: match std::env::var("OTEL_DENO_CONSOLE").as_deref() { + Ok(_) if disabled => OtelConsoleConfig::Ignore, + Ok("ignore") => OtelConsoleConfig::Ignore, + Ok("capture") => OtelConsoleConfig::Capture, + Ok("replace") => OtelConsoleConfig::Replace, + _ => { + if default { + OtelConsoleConfig::Capture + } else { + OtelConsoleConfig::Ignore + } + } + }, + deterministic: std::env::var("DENO_UNSTABLE_OTEL_DETERMINISTIC") + .as_deref() + == Ok("1"), } } @@ -2904,6 +2921,7 @@ To ignore linting on an entire file, you can add an ignore comment at the top of .arg(watch_arg(false)) .arg(watch_exclude_arg()) .arg(no_clear_screen_arg()) + .arg(allow_import_arg()) }) } @@ -4406,6 +4424,16 @@ impl CommandExt for Command { }) .help_heading(UNSTABLE_HEADING) .display_order(next_display_order()) + ).arg( + Arg::new("unstable-npm-lazy-caching") + .long("unstable-npm-lazy-caching") + .help("Enable unstable lazy caching of npm dependencies, downloading them only as needed (disabled: all npm packages in package.json are installed on startup; enabled: only npm packages that are actually referenced in an import are installed") + .env("DENO_UNSTABLE_NPM_LAZY_CACHING") + .value_parser(FalseyValueParser::new()) + .action(ArgAction::SetTrue) + .hide(true) + .help_heading(UNSTABLE_HEADING) + .display_order(next_display_order()), ); for granular_flag in crate::UNSTABLE_GRANULAR_FLAGS.iter() { @@ -4919,15 +4947,14 @@ fn install_parse( let module_url = cmd_values.next().unwrap(); let args = cmd_values.collect(); - flags.subcommand = DenoSubcommand::Install(InstallFlags { - kind: InstallKind::Global(InstallFlagsGlobal { + flags.subcommand = + DenoSubcommand::Install(InstallFlags::Global(InstallFlagsGlobal { name, module_url, args, root, force, - }), - }); + })); return Ok(()); } @@ -4936,22 +4963,19 @@ fn install_parse( allow_scripts_arg_parse(flags, matches)?; if matches.get_flag("entrypoint") { let entrypoints = matches.remove_many::("cmd").unwrap_or_default(); - flags.subcommand = DenoSubcommand::Install(InstallFlags { - kind: InstallKind::Local(InstallFlagsLocal::Entrypoints( - entrypoints.collect(), - )), - }); + flags.subcommand = DenoSubcommand::Install(InstallFlags::Local( + InstallFlagsLocal::Entrypoints(entrypoints.collect()), + )); } else if let Some(add_files) = matches .remove_many("cmd") .map(|packages| add_parse_inner(matches, Some(packages))) { - flags.subcommand = DenoSubcommand::Install(InstallFlags { - kind: InstallKind::Local(InstallFlagsLocal::Add(add_files)), - }) + flags.subcommand = DenoSubcommand::Install(InstallFlags::Local( + InstallFlagsLocal::Add(add_files), + )) } else { - flags.subcommand = DenoSubcommand::Install(InstallFlags { - kind: InstallKind::Local(InstallFlagsLocal::TopLevel), - }); + flags.subcommand = + DenoSubcommand::Install(InstallFlags::Local(InstallFlagsLocal::TopLevel)); } Ok(()) } @@ -5083,6 +5107,7 @@ fn lint_parse( unstable_args_parse(flags, matches, UnstableArgsConfig::ResolutionOnly); ext_arg_parse(flags, matches); config_args_parse(flags, matches); + allow_import_parse(flags, matches); let files = match matches.remove_many::("files") { Some(f) => f.collect(), @@ -5996,6 +6021,8 @@ fn unstable_args_parse( flags.unstable_config.detect_cjs = matches.get_flag("unstable-detect-cjs"); flags.unstable_config.sloppy_imports = matches.get_flag("unstable-sloppy-imports"); + flags.unstable_config.npm_lazy_caching = + matches.get_flag("unstable-npm-lazy-caching"); if matches!(cfg, UnstableArgsConfig::ResolutionAndRuntime) { for granular_flag in crate::UNSTABLE_GRANULAR_FLAGS { @@ -7141,6 +7168,7 @@ mod tests { let r = flags_from_vec(svec![ "deno", "lint", + "--allow-import", "--watch", "script_1.ts", "script_2.ts" @@ -7162,6 +7190,10 @@ mod tests { compact: false, watch: Some(Default::default()), }), + permissions: PermissionFlags { + allow_import: Some(vec![]), + ..Default::default() + }, ..Flags::default() } ); @@ -8599,15 +8631,15 @@ mod tests { assert_eq!( r.unwrap(), Flags { - subcommand: DenoSubcommand::Install(InstallFlags { - kind: InstallKind::Global(InstallFlagsGlobal { + subcommand: DenoSubcommand::Install(InstallFlags::Global( + InstallFlagsGlobal { name: None, module_url: "jsr:@std/http/file-server".to_string(), args: vec![], root: None, force: false, - }), - }), + } + ),), ..Flags::default() } ); @@ -8621,15 +8653,15 @@ mod tests { assert_eq!( r.unwrap(), Flags { - subcommand: DenoSubcommand::Install(InstallFlags { - kind: InstallKind::Global(InstallFlagsGlobal { + subcommand: DenoSubcommand::Install(InstallFlags::Global( + InstallFlagsGlobal { name: None, module_url: "jsr:@std/http/file-server".to_string(), args: vec![], root: None, force: false, - }), - }), + } + ),), ..Flags::default() } ); @@ -8642,15 +8674,15 @@ mod tests { assert_eq!( r.unwrap(), Flags { - subcommand: DenoSubcommand::Install(InstallFlags { - kind: InstallKind::Global(InstallFlagsGlobal { + subcommand: DenoSubcommand::Install(InstallFlags::Global( + InstallFlagsGlobal { name: Some("file_server".to_string()), module_url: "jsr:@std/http/file-server".to_string(), args: svec!["foo", "bar"], root: Some("/foo".to_string()), force: true, - }), - }), + } + ),), import_map_path: Some("import_map.json".to_string()), no_remote: true, config_flag: ConfigFlag::Path("tsconfig.json".to_owned()), @@ -11204,9 +11236,9 @@ mod tests { ..Flags::default() }, "install" => Flags { - subcommand: DenoSubcommand::Install(InstallFlags { - kind: InstallKind::Local(InstallFlagsLocal::Add(flags)), - }), + subcommand: DenoSubcommand::Install(InstallFlags::Local( + InstallFlagsLocal::Add(flags), + )), ..Flags::default() }, _ => unreachable!(), diff --git a/cli/args/lockfile.rs b/cli/args/lockfile.rs index 6c1a2ca0ef..1075f93a6f 100644 --- a/cli/args/lockfile.rs +++ b/cli/args/lockfile.rs @@ -9,18 +9,19 @@ use deno_core::anyhow::Context; use deno_core::error::AnyError; use deno_core::parking_lot::Mutex; use deno_core::parking_lot::MutexGuard; +use deno_core::serde_json; use deno_lockfile::WorkspaceMemberConfig; use deno_package_json::PackageJsonDepValue; use deno_runtime::deno_node::PackageJson; use deno_semver::jsr::JsrDepPackageReq; +use crate::args::deno_json::import_map_deps; use crate::cache; use crate::util::fs::atomic_write_file_with_retries; use crate::Flags; use crate::args::DenoSubcommand; use crate::args::InstallFlags; -use crate::args::InstallKind; use deno_lockfile::Lockfile; @@ -102,6 +103,7 @@ impl CliLockfile { pub fn discover( flags: &Flags, workspace: &Workspace, + maybe_external_import_map: Option<&serde_json::Value>, ) -> Result, AnyError> { fn pkg_json_deps( maybe_pkg_json: Option<&PackageJson>, @@ -136,10 +138,8 @@ impl CliLockfile { if flags.no_lock || matches!( flags.subcommand, - DenoSubcommand::Install(InstallFlags { - kind: InstallKind::Global(..), - .. - }) | DenoSubcommand::Uninstall(_) + DenoSubcommand::Install(InstallFlags::Global(..)) + | DenoSubcommand::Uninstall(_) ) { return Ok(None); @@ -174,7 +174,11 @@ impl CliLockfile { let config = deno_lockfile::WorkspaceConfig { root: WorkspaceMemberConfig { package_json_deps: pkg_json_deps(root_folder.pkg_json.as_deref()), - dependencies: deno_json_deps(root_folder.deno_json.as_deref()), + dependencies: if let Some(map) = maybe_external_import_map { + import_map_deps(map) + } else { + deno_json_deps(root_folder.deno_json.as_deref()) + }, }, members: workspace .config_folders() diff --git a/cli/args/mod.rs b/cli/args/mod.rs index 0b049cf409..450aa11652 100644 --- a/cli/args/mod.rs +++ b/cli/args/mod.rs @@ -31,6 +31,7 @@ use deno_npm_cache::NpmCacheSetting; use deno_path_util::normalize_path; use deno_semver::npm::NpmPackageReqReference; use deno_telemetry::OtelConfig; +use deno_telemetry::OtelRuntimeConfig; use import_map::resolve_import_map_value_from_specifier; pub use deno_config::deno_json::BenchConfig; @@ -807,6 +808,7 @@ pub struct CliOptions { maybe_node_modules_folder: Option, npmrc: Arc, maybe_lockfile: Option>, + maybe_external_import_map: Option<(PathBuf, serde_json::Value)>, overrides: CliOptionOverrides, pub start_dir: Arc, pub deno_dir_provider: Arc, @@ -820,6 +822,7 @@ impl CliOptions { npmrc: Arc, start_dir: Arc, force_global_cache: bool, + maybe_external_import_map: Option<(PathBuf, serde_json::Value)>, ) -> Result { if let Some(insecure_allowlist) = flags.unsafely_ignore_certificate_errors.as_ref() @@ -857,6 +860,7 @@ impl CliOptions { maybe_node_modules_folder, overrides: Default::default(), main_module_cell: std::sync::OnceLock::new(), + maybe_external_import_map, start_dir, deno_dir_provider, }) @@ -932,7 +936,33 @@ impl CliOptions { let (npmrc, _) = discover_npmrc_from_workspace(&start_dir.workspace)?; - let maybe_lock_file = CliLockfile::discover(&flags, &start_dir.workspace)?; + fn load_external_import_map( + deno_json: &ConfigFile, + ) -> Result, AnyError> { + if !deno_json.is_an_import_map() { + if let Some(path) = deno_json.to_import_map_path()? { + let contents = std::fs::read_to_string(&path).with_context(|| { + format!("Unable to read import map at '{}'", path.display()) + })?; + let map = serde_json::from_str(&contents)?; + return Ok(Some((path, map))); + } + } + Ok(None) + } + + let external_import_map = + if let Some(deno_json) = start_dir.workspace.root_deno_json() { + load_external_import_map(deno_json)? + } else { + None + }; + + let maybe_lock_file = CliLockfile::discover( + &flags, + &start_dir.workspace, + external_import_map.as_ref().map(|(_, v)| v), + )?; log::debug!("Finished config loading."); @@ -943,6 +973,7 @@ impl CliOptions { npmrc, Arc::new(start_dir), false, + external_import_map, ) } @@ -970,9 +1001,7 @@ impl CliOptions { match self.sub_command() { DenoSubcommand::Cache(_) => GraphKind::All, DenoSubcommand::Check(_) => GraphKind::TypesOnly, - DenoSubcommand::Install(InstallFlags { - kind: InstallKind::Local(_), - }) => GraphKind::All, + DenoSubcommand::Install(InstallFlags::Local(_)) => GraphKind::All, _ => self.type_check_mode().as_graph_kind(), } } @@ -1065,7 +1094,7 @@ impl CliOptions { file_fetcher: &FileFetcher, pkg_json_dep_resolution: PackageJsonDepResolution, ) -> Result { - let overrode_no_import_map = self + let overrode_no_import_map: bool = self .overrides .import_map_specifier .as_ref() @@ -1093,7 +1122,19 @@ impl CliOptions { value, }) } - None => None, + None => { + if let Some((path, import_map)) = + self.maybe_external_import_map.as_ref() + { + let path_url = deno_path_util::url_from_file_path(path)?; + Some(deno_config::workspace::SpecifiedImportMap { + base_url: path_url, + value: import_map.clone(), + }) + } else { + None + } + } } }; Ok(self.workspace().create_resolver( @@ -1132,7 +1173,7 @@ impl CliOptions { } } - pub fn otel_config(&self) -> Option { + pub fn otel_config(&self) -> OtelConfig { self.flags.otel_config() } @@ -1549,11 +1590,11 @@ impl CliOptions { DenoSubcommand::Check(check_flags) => { Some(files_to_urls(&check_flags.files)) } - DenoSubcommand::Install(InstallFlags { - kind: InstallKind::Global(flags), - }) => Url::parse(&flags.module_url) - .ok() - .map(|url| vec![Cow::Owned(url)]), + DenoSubcommand::Install(InstallFlags::Global(flags)) => { + Url::parse(&flags.module_url) + .ok() + .map(|url| vec![Cow::Owned(url)]) + } DenoSubcommand::Doc(DocFlags { source_files: DocSourceFileFlag::Paths(paths), .. @@ -1689,6 +1730,7 @@ impl CliOptions { "detect-cjs", "fmt-component", "fmt-sql", + "lazy-npm-caching", ]) .collect(); @@ -1767,6 +1809,19 @@ impl CliOptions { ), } } + + pub fn unstable_npm_lazy_caching(&self) -> bool { + self.flags.unstable_config.npm_lazy_caching + || self.workspace().has_unstable("npm-lazy-caching") + } + + pub fn default_npm_caching_strategy(&self) -> NpmCachingStrategy { + if self.flags.unstable_config.npm_lazy_caching { + NpmCachingStrategy::Lazy + } else { + NpmCachingStrategy::Eager + } + } } /// Resolves the path to use for a local node_modules folder. @@ -1981,6 +2036,20 @@ fn load_env_variables_from_env_file(filename: Option<&Vec>) { } } +#[derive(Debug, Clone, Copy)] +pub enum NpmCachingStrategy { + Eager, + Lazy, + Manual, +} + +pub(crate) fn otel_runtime_config() -> OtelRuntimeConfig { + OtelRuntimeConfig { + runtime_name: Cow::Borrowed("deno"), + runtime_version: Cow::Borrowed(crate::version::DENO_VERSION_INFO.deno), + } +} + #[cfg(test)] mod test { use pretty_assertions::assert_eq; diff --git a/cli/factory.rs b/cli/factory.rs index 6937b750f9..f08bf7e4b1 100644 --- a/cli/factory.rs +++ b/cli/factory.rs @@ -984,6 +984,7 @@ impl CliFactory { cli_options.sub_command().clone(), self.create_cli_main_worker_options()?, self.cli_options()?.otel_config(), + self.cli_options()?.default_npm_caching_strategy(), )) } diff --git a/cli/graph_util.rs b/cli/graph_util.rs index 22117990d2..b655dda0f6 100644 --- a/cli/graph_util.rs +++ b/cli/graph_util.rs @@ -4,6 +4,7 @@ use crate::args::config_to_deno_graph_workspace_member; use crate::args::jsr_url; use crate::args::CliLockfile; use crate::args::CliOptions; +pub use crate::args::NpmCachingStrategy; use crate::args::DENO_DISABLE_PEDANTIC_NODE_WARNINGS; use crate::cache; use crate::cache::FetchCacher; @@ -218,6 +219,7 @@ pub struct CreateGraphOptions<'a> { pub is_dynamic: bool, /// Specify `None` to use the default CLI loader. pub loader: Option<&'a mut dyn Loader>, + pub npm_caching: NpmCachingStrategy, } pub struct ModuleGraphCreator { @@ -246,10 +248,11 @@ impl ModuleGraphCreator { &self, graph_kind: GraphKind, roots: Vec, + npm_caching: NpmCachingStrategy, ) -> Result { let mut cache = self.module_graph_builder.create_graph_loader(); self - .create_graph_with_loader(graph_kind, roots, &mut cache) + .create_graph_with_loader(graph_kind, roots, &mut cache, npm_caching) .await } @@ -258,6 +261,7 @@ impl ModuleGraphCreator { graph_kind: GraphKind, roots: Vec, loader: &mut dyn Loader, + npm_caching: NpmCachingStrategy, ) -> Result { self .create_graph_with_options(CreateGraphOptions { @@ -265,6 +269,7 @@ impl ModuleGraphCreator { graph_kind, roots, loader: Some(loader), + npm_caching, }) .await } @@ -317,6 +322,7 @@ impl ModuleGraphCreator { graph_kind: deno_graph::GraphKind::All, roots, loader: Some(&mut publish_loader), + npm_caching: self.options.default_npm_caching_strategy(), }) .await?; self.graph_valid(&graph)?; @@ -376,6 +382,7 @@ impl ModuleGraphCreator { graph_kind, roots, loader: None, + npm_caching: self.options.default_npm_caching_strategy(), }) .await?; @@ -565,7 +572,8 @@ impl ModuleGraphBuilder { }; let cli_resolver = &self.resolver; let graph_resolver = self.create_graph_resolver()?; - let graph_npm_resolver = cli_resolver.create_graph_npm_resolver(); + let graph_npm_resolver = + cli_resolver.create_graph_npm_resolver(options.npm_caching); let maybe_file_watcher_reporter = self .maybe_file_watcher_reporter .as_ref() @@ -592,6 +600,7 @@ impl ModuleGraphBuilder { resolver: Some(&graph_resolver), locker: locker.as_mut().map(|l| l as _), }, + options.npm_caching, ) .await } @@ -602,6 +611,7 @@ impl ModuleGraphBuilder { roots: Vec, loader: &'a mut dyn deno_graph::source::Loader, options: deno_graph::BuildOptions<'a>, + npm_caching: NpmCachingStrategy, ) -> Result<(), AnyError> { // ensure an "npm install" is done if the user has explicitly // opted into using a node_modules directory @@ -612,7 +622,13 @@ impl ModuleGraphBuilder { .unwrap_or(false) { if let Some(npm_resolver) = self.npm_resolver.as_managed() { - npm_resolver.ensure_top_level_package_json_install().await?; + let already_done = + npm_resolver.ensure_top_level_package_json_install().await?; + if !already_done && matches!(npm_caching, NpmCachingStrategy::Eager) { + npm_resolver + .cache_packages(crate::npm::PackageCaching::All) + .await?; + } } } @@ -701,7 +717,9 @@ impl ModuleGraphBuilder { let parser = self.parsed_source_cache.as_capturing_parser(); let cli_resolver = &self.resolver; let graph_resolver = self.create_graph_resolver()?; - let graph_npm_resolver = cli_resolver.create_graph_npm_resolver(); + let graph_npm_resolver = cli_resolver.create_graph_npm_resolver( + self.cli_options.default_npm_caching_strategy(), + ); graph.build_fast_check_type_graph( deno_graph::BuildFastCheckTypeGraphOptions { diff --git a/cli/lsp/analysis.rs b/cli/lsp/analysis.rs index 853708221f..c128372dcd 100644 --- a/cli/lsp/analysis.rs +++ b/cli/lsp/analysis.rs @@ -1387,7 +1387,7 @@ impl CodeActionCollection { character: import_start.column_index as u32, }; let new_text = format!( - "{}// @deno-types=\"{}\"\n", + "{}// @ts-types=\"{}\"\n", if position.character == 0 { "" } else { "\n" }, &types_specifier_text ); @@ -1400,7 +1400,7 @@ impl CodeActionCollection { }; Some(lsp::CodeAction { title: format!( - "Add @deno-types directive for \"{}\"", + "Add @ts-types directive for \"{}\"", &types_specifier_text ), kind: Some(lsp::CodeActionKind::QUICKFIX), diff --git a/cli/lsp/completions.rs b/cli/lsp/completions.rs index 95e5113620..31f0b066ed 100644 --- a/cli/lsp/completions.rs +++ b/cli/lsp/completions.rs @@ -743,13 +743,16 @@ fn get_node_completions( } let items = SUPPORTED_BUILTIN_NODE_MODULES .iter() - .map(|name| { + .filter_map(|name| { + if name.starts_with('_') { + return None; + } let specifier = format!("node:{}", name); let text_edit = Some(lsp::CompletionTextEdit::Edit(lsp::TextEdit { range: *range, new_text: specifier.clone(), })); - lsp::CompletionItem { + Some(lsp::CompletionItem { label: specifier, kind: Some(lsp::CompletionItemKind::FILE), detail: Some("(node)".to_string()), @@ -758,7 +761,7 @@ fn get_node_completions( IMPORT_COMMIT_CHARS.iter().map(|&c| c.into()).collect(), ), ..Default::default() - } + }) }) .collect(); Some(CompletionList { diff --git a/cli/lsp/diagnostics.rs b/cli/lsp/diagnostics.rs index ac4d8c01e4..01fc3bf69e 100644 --- a/cli/lsp/diagnostics.rs +++ b/cli/lsp/diagnostics.rs @@ -1355,7 +1355,7 @@ fn diagnose_resolution( } // don't bother warning about sloppy import redirects from .js to .d.ts // because explaining how to fix this via a diagnostic involves using - // @deno-types and that's a bit complicated to explain + // @ts-types and that's a bit complicated to explain let is_sloppy_import_dts_redirect = doc_specifier.scheme() == "file" && doc.media_type().is_declaration() && !MediaType::from_specifier(specifier).is_declaration(); @@ -1523,7 +1523,7 @@ fn diagnose_dependency( .iter() .map(|i| documents::to_lsp_range(&i.specifier_range)) .collect(); - // TODO(nayeemrmn): This is a crude way of detecting `@deno-types` which has + // TODO(nayeemrmn): This is a crude way of detecting `@ts-types` which has // a different specifier and therefore needs a separate call to // `diagnose_resolution()`. It would be much cleaner if that were modelled as // a separate dependency: https://github.com/denoland/deno_graph/issues/247. @@ -1540,7 +1540,7 @@ fn diagnose_dependency( snapshot, dependency_key, if dependency.maybe_code.is_none() - // If not @deno-types, diagnose the types if the code errored because + // If not @ts-types, diagnose the types if the code errored because // it's likely resolving into the node_modules folder, which might be // erroring correctly due to resolution only being for bundlers. Let this // fail at runtime if necessary, but don't bother erroring in the editor @@ -1951,7 +1951,7 @@ let c: number = "a"; &[( "a.ts", r#" - // @deno-types="bad.d.ts" + // @ts-types="bad.d.ts" import "bad.js"; import "bad.js"; "#, @@ -2005,11 +2005,11 @@ let c: number = "a"; "range": { "start": { "line": 1, - "character": 23 + "character": 21 }, "end": { "line": 1, - "character": 33 + "character": 31 } }, "severity": 1, diff --git a/cli/lsp/documents.rs b/cli/lsp/documents.rs index b9ec8ffc46..bdb64c9da3 100644 --- a/cli/lsp/documents.rs +++ b/cli/lsp/documents.rs @@ -65,6 +65,12 @@ pub enum LanguageId { Html, Css, Yaml, + Sql, + Svelte, + Vue, + Astro, + Vento, + Nunjucks, Unknown, } @@ -81,6 +87,12 @@ impl LanguageId { LanguageId::Html => Some("html"), LanguageId::Css => Some("css"), LanguageId::Yaml => Some("yaml"), + LanguageId::Sql => Some("sql"), + LanguageId::Svelte => Some("svelte"), + LanguageId::Vue => Some("vue"), + LanguageId::Astro => Some("astro"), + LanguageId::Vento => Some("vto"), + LanguageId::Nunjucks => Some("njk"), LanguageId::Unknown => None, } } @@ -96,6 +108,12 @@ impl LanguageId { LanguageId::Html => Some("text/html"), LanguageId::Css => Some("text/css"), LanguageId::Yaml => Some("application/yaml"), + LanguageId::Sql => None, + LanguageId::Svelte => None, + LanguageId::Vue => None, + LanguageId::Astro => None, + LanguageId::Vento => None, + LanguageId::Nunjucks => None, LanguageId::Unknown => None, } } @@ -123,6 +141,12 @@ impl FromStr for LanguageId { "html" => Ok(Self::Html), "css" => Ok(Self::Css), "yaml" => Ok(Self::Yaml), + "sql" => Ok(Self::Sql), + "svelte" => Ok(Self::Svelte), + "vue" => Ok(Self::Vue), + "astro" => Ok(Self::Astro), + "vento" => Ok(Self::Vento), + "nunjucks" => Ok(Self::Nunjucks), _ => Ok(Self::Unknown), } } diff --git a/cli/lsp/language_server.rs b/cli/lsp/language_server.rs index 0caaa94107..839d28469e 100644 --- a/cli/lsp/language_server.rs +++ b/cli/lsp/language_server.rs @@ -270,7 +270,12 @@ impl LanguageServer { open_docs: &open_docs, }; let graph = module_graph_creator - .create_graph_with_loader(GraphKind::All, roots.clone(), &mut loader) + .create_graph_with_loader( + GraphKind::All, + roots.clone(), + &mut loader, + graph_util::NpmCachingStrategy::Eager, + ) .await?; graph_util::graph_valid( &graph, @@ -3671,6 +3676,7 @@ impl Inner { .unwrap_or_else(create_default_npmrc), workspace, force_global_cache, + None, )?; let open_docs = self.documents.documents(DocumentsFilter::OpenDiagnosable); diff --git a/cli/lsp/resolver.rs b/cli/lsp/resolver.rs index 363ad43700..28c7b04fc9 100644 --- a/cli/lsp/resolver.rs +++ b/cli/lsp/resolver.rs @@ -133,7 +133,8 @@ impl LspScopeResolver { cache.for_specifier(config_data.map(|d| d.scope.as_ref())), config_data.and_then(|d| d.lockfile.clone()), ))); - let npm_graph_resolver = cli_resolver.create_graph_npm_resolver(); + let npm_graph_resolver = cli_resolver + .create_graph_npm_resolver(crate::graph_util::NpmCachingStrategy::Eager); let maybe_jsx_import_source_config = config_data.and_then(|d| d.maybe_jsx_import_source_config()); let graph_imports = config_data @@ -343,7 +344,9 @@ impl LspResolver { file_referrer: Option<&ModuleSpecifier>, ) -> WorkerCliNpmGraphResolver { let resolver = self.get_scope_resolver(file_referrer); - resolver.resolver.create_graph_npm_resolver() + resolver + .resolver + .create_graph_npm_resolver(crate::graph_util::NpmCachingStrategy::Eager) } pub fn as_is_cjs_resolver( diff --git a/cli/lsp/tsc.rs b/cli/lsp/tsc.rs index 360e0946b2..957c3a6859 100644 --- a/cli/lsp/tsc.rs +++ b/cli/lsp/tsc.rs @@ -3419,7 +3419,7 @@ fn parse_code_actions( &specifier_rewrite.new_deno_types_specifier { text_edit.new_text = format!( - "// @deno-types=\"{}\"\n{}", + "// @ts-types=\"{}\"\n{}", deno_types_specifier, &text_edit.new_text ); } @@ -3594,10 +3594,8 @@ impl CompletionEntryDetails { if let Some(deno_types_specifier) = &specifier_rewrite.new_deno_types_specifier { - *new_text = format!( - "// @deno-types=\"{}\"\n{}", - deno_types_specifier, new_text - ); + *new_text = + format!("// @ts-types=\"{}\"\n{}", deno_types_specifier, new_text); } } } diff --git a/cli/main.rs b/cli/main.rs index d47f1e363c..0594739fd8 100644 --- a/cli/main.rs +++ b/cli/main.rs @@ -437,20 +437,18 @@ fn resolve_flags_and_init( if err.kind() == clap::error::ErrorKind::DisplayVersion => { // Ignore results to avoid BrokenPipe errors. - util::logger::init(None); + util::logger::init(None, None); let _ = err.print(); deno_runtime::exit(0); } Err(err) => { - util::logger::init(None); + util::logger::init(None, None); exit_for_error(AnyError::from(err)) } }; - if let Some(otel_config) = flags.otel_config() { - deno_telemetry::init(otel_config)?; - } - util::logger::init(flags.log_level); + deno_telemetry::init(crate::args::otel_runtime_config())?; + util::logger::init(flags.log_level, Some(flags.otel_config())); // TODO(bartlomieju): remove in Deno v2.5 and hard error then. if flags.unstable_config.legacy_flag_enabled { diff --git a/cli/mainrt.rs b/cli/mainrt.rs index 7ad3b3744b..18142bd0e7 100644 --- a/cli/mainrt.rs +++ b/cli/mainrt.rs @@ -87,17 +87,18 @@ fn main() { let future = async move { match standalone { Ok(Some(data)) => { - if let Some(otel_config) = data.metadata.otel_config.clone() { - deno_telemetry::init(otel_config)?; - } - util::logger::init(data.metadata.log_level); + deno_telemetry::init(crate::args::otel_runtime_config())?; + util::logger::init( + data.metadata.log_level, + Some(data.metadata.otel_config.clone()), + ); load_env_vars(&data.metadata.env_vars_from_env_file); let exit_code = standalone::run(data).await?; deno_runtime::exit(exit_code); } Ok(None) => Ok(()), Err(err) => { - util::logger::init(None); + util::logger::init(None, None); Err(err) } } diff --git a/cli/module_loader.rs b/cli/module_loader.rs index c5f80d68e0..5e4ff875dc 100644 --- a/cli/module_loader.rs +++ b/cli/module_loader.rs @@ -156,6 +156,7 @@ impl ModuleLoadPreparer { graph_kind: graph.graph_kind(), roots: roots.to_vec(), loader: Some(&mut cache), + npm_caching: self.options.default_npm_caching_strategy(), }, ) .await?; diff --git a/cli/npm/managed/mod.rs b/cli/npm/managed/mod.rs index 5ed25f8272..2c6e6d318a 100644 --- a/cli/npm/managed/mod.rs +++ b/cli/npm/managed/mod.rs @@ -296,6 +296,12 @@ pub fn create_managed_in_npm_pkg_checker( Arc::new(ManagedInNpmPackageChecker { root_dir }) } +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PackageCaching<'a> { + Only(Cow<'a, [PackageReq]>), + All, +} + /// An npm resolver where the resolution is managed by Deno rather than /// the user bringing their own node_modules (BYONM) on the file system. pub struct ManagedCliNpmResolver { @@ -420,19 +426,44 @@ impl ManagedCliNpmResolver { /// Adds package requirements to the resolver and ensures everything is setup. /// This includes setting up the `node_modules` directory, if applicable. - pub async fn add_package_reqs( + pub async fn add_and_cache_package_reqs( &self, packages: &[PackageReq], ) -> Result<(), AnyError> { self - .add_package_reqs_raw(packages) + .add_package_reqs_raw( + packages, + Some(PackageCaching::Only(packages.into())), + ) .await .dependencies_result } - pub async fn add_package_reqs_raw( + pub async fn add_package_reqs_no_cache( &self, packages: &[PackageReq], + ) -> Result<(), AnyError> { + self + .add_package_reqs_raw(packages, None) + .await + .dependencies_result + } + + pub async fn add_package_reqs( + &self, + packages: &[PackageReq], + caching: PackageCaching<'_>, + ) -> Result<(), AnyError> { + self + .add_package_reqs_raw(packages, Some(caching)) + .await + .dependencies_result + } + + pub async fn add_package_reqs_raw<'a>( + &self, + packages: &[PackageReq], + caching: Option>, ) -> AddPkgReqsResult { if packages.is_empty() { return AddPkgReqsResult { @@ -449,7 +480,9 @@ impl ManagedCliNpmResolver { } } if result.dependencies_result.is_ok() { - result.dependencies_result = self.cache_packages().await; + if let Some(caching) = caching { + result.dependencies_result = self.cache_packages(caching).await; + } } result @@ -491,16 +524,20 @@ impl ManagedCliNpmResolver { pub async fn inject_synthetic_types_node_package( &self, ) -> Result<(), AnyError> { + let reqs = &[PackageReq::from_str("@types/node").unwrap()]; // add and ensure this isn't added to the lockfile self - .add_package_reqs(&[PackageReq::from_str("@types/node").unwrap()]) + .add_package_reqs(reqs, PackageCaching::Only(reqs.into())) .await?; Ok(()) } - pub async fn cache_packages(&self) -> Result<(), AnyError> { - self.fs_resolver.cache_packages().await + pub async fn cache_packages( + &self, + caching: PackageCaching<'_>, + ) -> Result<(), AnyError> { + self.fs_resolver.cache_packages(caching).await } pub fn resolve_pkg_folder_from_deno_module( @@ -545,18 +582,18 @@ impl ManagedCliNpmResolver { /// Ensures that the top level `package.json` dependencies are installed. /// This may set up the `node_modules` directory. /// - /// Returns `true` if any changes (such as caching packages) were made. - /// If this returns `false`, `node_modules` has _not_ been set up. + /// Returns `true` if the top level packages are already installed. A + /// return value of `false` means that new packages were added to the NPM resolution. pub async fn ensure_top_level_package_json_install( &self, ) -> Result { if !self.top_level_install_flag.raise() { - return Ok(false); // already did this + return Ok(true); // already did this } let pkg_json_remote_pkgs = self.npm_install_deps_provider.remote_pkgs(); if pkg_json_remote_pkgs.is_empty() { - return Ok(false); + return Ok(true); } // check if something needs resolving before bothering to load all @@ -570,14 +607,16 @@ impl ManagedCliNpmResolver { log::debug!( "All package.json deps resolvable. Skipping top level install." ); - return Ok(false); // everything is already resolvable + return Ok(true); // everything is already resolvable } let pkg_reqs = pkg_json_remote_pkgs .iter() .map(|pkg| pkg.req.clone()) .collect::>(); - self.add_package_reqs(&pkg_reqs).await.map(|_| true) + self.add_package_reqs_no_cache(&pkg_reqs).await?; + + Ok(false) } pub async fn cache_package_info( diff --git a/cli/npm/managed/resolution.rs b/cli/npm/managed/resolution.rs index 66cc6a7428..73c5c31caf 100644 --- a/cli/npm/managed/resolution.rs +++ b/cli/npm/managed/resolution.rs @@ -255,6 +255,10 @@ impl NpmResolution { .read() .as_valid_serialized_for_system(system_info) } + + pub fn subset(&self, package_reqs: &[PackageReq]) -> NpmResolutionSnapshot { + self.snapshot.read().subset(package_reqs) + } } async fn add_package_reqs_to_snapshot( diff --git a/cli/npm/managed/resolvers/common.rs b/cli/npm/managed/resolvers/common.rs index 332756daa4..68e95fb39a 100644 --- a/cli/npm/managed/resolvers/common.rs +++ b/cli/npm/managed/resolvers/common.rs @@ -11,6 +11,7 @@ use std::path::PathBuf; use std::sync::Arc; use std::sync::Mutex; +use super::super::PackageCaching; use async_trait::async_trait; use deno_ast::ModuleSpecifier; use deno_core::anyhow::Context; @@ -57,7 +58,10 @@ pub trait NpmPackageFsResolver: Send + Sync { specifier: &ModuleSpecifier, ) -> Result, AnyError>; - async fn cache_packages(&self) -> Result<(), AnyError>; + async fn cache_packages<'a>( + &self, + caching: PackageCaching<'a>, + ) -> Result<(), AnyError>; #[must_use = "the resolved return value to mitigate time-of-check to time-of-use issues"] fn ensure_read_permission<'a>( diff --git a/cli/npm/managed/resolvers/global.rs b/cli/npm/managed/resolvers/global.rs index 2b48c3d2fc..4e79941af6 100644 --- a/cli/npm/managed/resolvers/global.rs +++ b/cli/npm/managed/resolvers/global.rs @@ -8,6 +8,7 @@ use std::path::PathBuf; use std::sync::Arc; use crate::colors; +use crate::npm::managed::PackageCaching; use crate::npm::CliNpmCache; use crate::npm::CliNpmTarballCache; use async_trait::async_trait; @@ -150,10 +151,19 @@ impl NpmPackageFsResolver for GlobalNpmPackageResolver { ) } - async fn cache_packages(&self) -> Result<(), AnyError> { - let package_partitions = self - .resolution - .all_system_packages_partitioned(&self.system_info); + async fn cache_packages<'a>( + &self, + caching: PackageCaching<'a>, + ) -> Result<(), AnyError> { + let package_partitions = match caching { + PackageCaching::All => self + .resolution + .all_system_packages_partitioned(&self.system_info), + PackageCaching::Only(reqs) => self + .resolution + .subset(&reqs) + .all_system_packages_partitioned(&self.system_info), + }; cache_packages(&package_partitions.packages, &self.tarball_cache).await?; // create the copy package folders diff --git a/cli/npm/managed/resolvers/local.rs b/cli/npm/managed/resolvers/local.rs index d383a5413f..1e83717f15 100644 --- a/cli/npm/managed/resolvers/local.rs +++ b/cli/npm/managed/resolvers/local.rs @@ -17,6 +17,7 @@ use std::sync::Arc; use crate::args::LifecycleScriptsConfig; use crate::colors; +use crate::npm::managed::PackageCaching; use crate::npm::CliNpmCache; use crate::npm::CliNpmTarballCache; use async_trait::async_trait; @@ -253,9 +254,16 @@ impl NpmPackageFsResolver for LocalNpmPackageResolver { )) } - async fn cache_packages(&self) -> Result<(), AnyError> { + async fn cache_packages<'a>( + &self, + caching: PackageCaching<'a>, + ) -> Result<(), AnyError> { + let snapshot = match caching { + PackageCaching::All => self.resolution.snapshot(), + PackageCaching::Only(reqs) => self.resolution.subset(&reqs), + }; sync_resolution_with_fs( - &self.resolution.snapshot(), + &snapshot, &self.cache, &self.npm_install_deps_provider, &self.progress_bar, diff --git a/cli/npm/mod.rs b/cli/npm/mod.rs index 48d90d7dd0..b39e0a340d 100644 --- a/cli/npm/mod.rs +++ b/cli/npm/mod.rs @@ -41,6 +41,7 @@ pub use self::managed::CliManagedInNpmPkgCheckerCreateOptions; pub use self::managed::CliManagedNpmResolverCreateOptions; pub use self::managed::CliNpmResolverManagedSnapshotOption; pub use self::managed::ManagedCliNpmResolver; +pub use self::managed::PackageCaching; pub type CliNpmTarballCache = deno_npm_cache::TarballCache; pub type CliNpmCache = deno_npm_cache::NpmCache; diff --git a/cli/resolver.rs b/cli/resolver.rs index 15ca4aa2b6..f5c3f68f36 100644 --- a/cli/resolver.rs +++ b/cli/resolver.rs @@ -32,6 +32,7 @@ use std::path::PathBuf; use std::sync::Arc; use thiserror::Error; +use crate::args::NpmCachingStrategy; use crate::args::DENO_DISABLE_PEDANTIC_NODE_WARNINGS; use crate::node::CliNodeCodeTranslator; use crate::npm::CliNpmResolver; @@ -240,11 +241,15 @@ impl CliResolver { // todo(dsherret): move this off CliResolver as CliResolver is acting // like a factory by doing this (it's beyond its responsibility) - pub fn create_graph_npm_resolver(&self) -> WorkerCliNpmGraphResolver { + pub fn create_graph_npm_resolver( + &self, + npm_caching: NpmCachingStrategy, + ) -> WorkerCliNpmGraphResolver { WorkerCliNpmGraphResolver { npm_resolver: self.npm_resolver.as_ref(), found_package_json_dep_flag: &self.found_package_json_dep_flag, bare_node_builtins_enabled: self.bare_node_builtins_enabled, + npm_caching, } } @@ -304,6 +309,7 @@ pub struct WorkerCliNpmGraphResolver<'a> { npm_resolver: Option<&'a Arc>, found_package_json_dep_flag: &'a AtomicFlag, bare_node_builtins_enabled: bool, + npm_caching: NpmCachingStrategy, } #[async_trait(?Send)] @@ -373,7 +379,20 @@ impl<'a> deno_graph::source::NpmResolver for WorkerCliNpmGraphResolver<'a> { Ok(()) }; - let result = npm_resolver.add_package_reqs_raw(package_reqs).await; + let result = npm_resolver + .add_package_reqs_raw( + package_reqs, + match self.npm_caching { + NpmCachingStrategy::Eager => { + Some(crate::npm::PackageCaching::All) + } + NpmCachingStrategy::Lazy => { + Some(crate::npm::PackageCaching::Only(package_reqs.into())) + } + NpmCachingStrategy::Manual => None, + }, + ) + .await; NpmResolvePkgReqsResult { results: result diff --git a/cli/schemas/config-file.v1.json b/cli/schemas/config-file.v1.json index a64cb2ff65..1e3abb2c0d 100644 --- a/cli/schemas/config-file.v1.json +++ b/cli/schemas/config-file.v1.json @@ -291,7 +291,7 @@ "type": "array", "description": "List of tag names that will be run. Empty list disables all tags and will only use rules from `include`.", "items": { - "$ref": "https://raw.githubusercontent.com/denoland/deno_lint/main/schemas/tags.v1.json" + "$ref": "lint-tags.v1.json" }, "minItems": 0, "uniqueItems": true @@ -300,7 +300,7 @@ "type": "array", "description": "List of rule names that will be excluded from configured tag sets. If the same rule is in `include` it will be run.", "items": { - "$ref": "https://raw.githubusercontent.com/denoland/deno_lint/main/schemas/rules.v1.json" + "$ref": "lint-rules.v1.json" }, "minItems": 0, "uniqueItems": true @@ -309,7 +309,7 @@ "type": "array", "description": "List of rule names that will be run. Even if the same rule is in `exclude` it will be run.", "items": { - "$ref": "https://raw.githubusercontent.com/denoland/deno_lint/main/schemas/rules.v1.json" + "$ref": "lint-rules.v1.json" }, "minItems": 0, "uniqueItems": true diff --git a/cli/schemas/lint-rules.v1.json b/cli/schemas/lint-rules.v1.json new file mode 100644 index 0000000000..71d1784958 --- /dev/null +++ b/cli/schemas/lint-rules.v1.json @@ -0,0 +1,112 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "enum": [ + "adjacent-overload-signatures", + "ban-ts-comment", + "ban-types", + "ban-unknown-rule-code", + "ban-untagged-ignore", + "ban-untagged-todo", + "ban-unused-ignore", + "camelcase", + "constructor-super", + "default-param-last", + "eqeqeq", + "explicit-function-return-type", + "explicit-module-boundary-types", + "for-direction", + "fresh-handler-export", + "fresh-server-event-handlers", + "getter-return", + "guard-for-in", + "no-array-constructor", + "no-async-promise-executor", + "no-await-in-loop", + "no-await-in-sync-fn", + "no-boolean-literal-for-arguments", + "no-case-declarations", + "no-class-assign", + "no-compare-neg-zero", + "no-cond-assign", + "no-console", + "no-const-assign", + "no-constant-condition", + "no-control-regex", + "no-debugger", + "no-delete-var", + "no-deprecated-deno-api", + "no-dupe-args", + "no-dupe-class-members", + "no-dupe-else-if", + "no-dupe-keys", + "no-duplicate-case", + "no-empty", + "no-empty-character-class", + "no-empty-enum", + "no-empty-interface", + "no-empty-pattern", + "no-eval", + "no-ex-assign", + "no-explicit-any", + "no-external-import", + "no-extra-boolean-cast", + "no-extra-non-null-assertion", + "no-fallthrough", + "no-func-assign", + "no-global-assign", + "no-implicit-declare-namespace-export", + "no-import-assertions", + "no-import-assign", + "no-inferrable-types", + "no-inner-declarations", + "no-invalid-regexp", + "no-invalid-triple-slash-reference", + "no-irregular-whitespace", + "no-misused-new", + "no-namespace", + "no-new-symbol", + "no-node-globals", + "no-non-null-asserted-optional-chain", + "no-non-null-assertion", + "no-obj-calls", + "no-octal", + "no-process-globals", + "no-prototype-builtins", + "no-redeclare", + "no-regex-spaces", + "no-self-assign", + "no-self-compare", + "no-setter-return", + "no-shadow-restricted-names", + "no-sloppy-imports", + "no-slow-types", + "no-sparse-arrays", + "no-sync-fn-in-async-fn", + "no-this-alias", + "no-this-before-super", + "no-throw-literal", + "no-top-level-await", + "no-undef", + "no-unreachable", + "no-unsafe-finally", + "no-unsafe-negation", + "no-unused-labels", + "no-unused-vars", + "no-var", + "no-window", + "no-window-prefix", + "no-with", + "prefer-as-const", + "prefer-ascii", + "prefer-const", + "prefer-namespace-keyword", + "prefer-primordials", + "require-await", + "require-yield", + "single-var-declarator", + "triple-slash-reference", + "use-isnan", + "valid-typeof", + "verbatim-module-syntax" + ] +} diff --git a/cli/schemas/lint-tags.v1.json b/cli/schemas/lint-tags.v1.json new file mode 100644 index 0000000000..4b4f0e48db --- /dev/null +++ b/cli/schemas/lint-tags.v1.json @@ -0,0 +1,4 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "enum": ["fresh", "jsr", "jsx", "react", "recommended"] +} diff --git a/cli/standalone/binary.rs b/cli/standalone/binary.rs index 632f27da6f..85a22cf837 100644 --- a/cli/standalone/binary.rs +++ b/cli/standalone/binary.rs @@ -44,6 +44,9 @@ use deno_npm::resolution::SerializedNpmResolutionSnapshotPackage; use deno_npm::resolution::ValidSerializedNpmResolutionSnapshot; use deno_npm::NpmPackageId; use deno_npm::NpmSystemInfo; +use deno_path_util::url_from_directory_path; +use deno_path_util::url_from_file_path; +use deno_path_util::url_to_file_path; use deno_runtime::deno_fs; use deno_runtime::deno_fs::FileSystem; use deno_runtime::deno_fs::RealFs; @@ -76,6 +79,7 @@ use crate::resolver::CjsTracker; use crate::shared::ReleaseChannel; use crate::standalone::virtual_fs::VfsEntry; use crate::util::archive; +use crate::util::fs::canonicalize_path; use crate::util::fs::canonicalize_path_maybe_not_exists; use crate::util::progress_bar::ProgressBar; use crate::util::progress_bar::ProgressBarStyle; @@ -87,31 +91,29 @@ use super::serialization::DenoCompileModuleData; use super::serialization::DeserializedDataSection; use super::serialization::RemoteModulesStore; use super::serialization::RemoteModulesStoreBuilder; +use super::virtual_fs::output_vfs; +use super::virtual_fs::BuiltVfs; use super::virtual_fs::FileBackedVfs; use super::virtual_fs::VfsBuilder; use super::virtual_fs::VfsFileSubDataKind; use super::virtual_fs::VfsRoot; use super::virtual_fs::VirtualDirectory; +use super::virtual_fs::WindowsSystemRootablePath; + +pub static DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME: &str = + ".deno_compile_node_modules"; /// A URL that can be designated as the base for relative URLs. /// /// After creation, this URL may be used to get the key for a /// module in the binary. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub struct StandaloneRelativeFileBaseUrl<'a>(&'a Url); - -impl<'a> From<&'a Url> for StandaloneRelativeFileBaseUrl<'a> { - fn from(url: &'a Url) -> Self { - Self(url) - } +pub enum StandaloneRelativeFileBaseUrl<'a> { + WindowsSystemRoot, + Path(&'a Url), } impl<'a> StandaloneRelativeFileBaseUrl<'a> { - pub fn new(url: &'a Url) -> Self { - debug_assert_eq!(url.scheme(), "file"); - Self(url) - } - /// Gets the module map key of the provided specifier. /// /// * Descendant file specifiers will be made relative to the base. @@ -121,22 +123,29 @@ impl<'a> StandaloneRelativeFileBaseUrl<'a> { if target.scheme() != "file" { return Cow::Borrowed(target.as_str()); } + let base = match self { + Self::Path(base) => base, + Self::WindowsSystemRoot => return Cow::Borrowed(target.path()), + }; - match self.0.make_relative(target) { + match base.make_relative(target) { Some(relative) => { - if relative.starts_with("../") { - Cow::Borrowed(target.as_str()) - } else { - Cow::Owned(relative) - } + // This is not a great scenario to have because it means that the + // specifier is outside the vfs and could cause the binary to act + // strangely. If you encounter this, the fix is to add more paths + // to the vfs builder by calling `add_possible_min_root_dir`. + debug_assert!( + !relative.starts_with("../"), + "{} -> {} ({})", + base.as_str(), + target.as_str(), + relative, + ); + Cow::Owned(relative) } None => Cow::Borrowed(target.as_str()), } } - - pub fn inner(&self) -> &Url { - self.0 - } } #[derive(Deserialize, Serialize)] @@ -191,7 +200,7 @@ pub struct Metadata { pub entrypoint_key: String, pub node_modules: Option, pub unstable_config: UnstableConfig, - pub otel_config: Option, // None means disabled. + pub otel_config: OtelConfig, } fn write_binary_bytes( @@ -200,7 +209,7 @@ fn write_binary_bytes( metadata: &Metadata, npm_snapshot: Option, remote_modules: &RemoteModulesStoreBuilder, - vfs: VfsBuilder, + vfs: &BuiltVfs, compile_flags: &CompileFlags, ) -> Result<(), AnyError> { let data_section_bytes = @@ -367,6 +376,15 @@ pub fn extract_standalone( })) } +pub struct WriteBinOptions<'a> { + pub writer: File, + pub display_output_filename: &'a str, + pub graph: &'a ModuleGraph, + pub entrypoint: &'a ModuleSpecifier, + pub include_files: &'a [ModuleSpecifier], + pub compile_flags: &'a CompileFlags, +} + pub struct DenoCompileBinaryWriter<'a> { cjs_tracker: &'a CjsTracker, cli_options: &'a CliOptions, @@ -407,18 +425,14 @@ impl<'a> DenoCompileBinaryWriter<'a> { pub async fn write_bin( &self, - writer: File, - graph: &ModuleGraph, - root_dir_url: StandaloneRelativeFileBaseUrl<'_>, - entrypoint: &ModuleSpecifier, - include_files: &[ModuleSpecifier], - compile_flags: &CompileFlags, + options: WriteBinOptions<'_>, ) -> Result<(), AnyError> { // Select base binary based on target - let mut original_binary = self.get_base_binary(compile_flags).await?; + let mut original_binary = + self.get_base_binary(options.compile_flags).await?; - if compile_flags.no_terminal { - let target = compile_flags.resolve_target(); + if options.compile_flags.no_terminal { + let target = options.compile_flags.resolve_target(); if !target.contains("windows") { bail!( "The `--no-terminal` flag is only available when targeting Windows (current: {})", @@ -428,8 +442,8 @@ impl<'a> DenoCompileBinaryWriter<'a> { set_windows_binary_to_gui(&mut original_binary) .context("Setting windows binary to GUI.")?; } - if compile_flags.icon.is_some() { - let target = compile_flags.resolve_target(); + if options.compile_flags.icon.is_some() { + let target = options.compile_flags.resolve_target(); if !target.contains("windows") { bail!( "The `--icon` flag is only available when targeting Windows (current: {})", @@ -437,17 +451,7 @@ impl<'a> DenoCompileBinaryWriter<'a> { ) } } - self - .write_standalone_binary( - writer, - original_binary, - graph, - root_dir_url, - entrypoint, - include_files, - compile_flags, - ) - .await + self.write_standalone_binary(options, original_binary).await } async fn get_base_binary( @@ -552,14 +556,17 @@ impl<'a> DenoCompileBinaryWriter<'a> { #[allow(clippy::too_many_arguments)] async fn write_standalone_binary( &self, - writer: File, + options: WriteBinOptions<'_>, original_bin: Vec, - graph: &ModuleGraph, - root_dir_url: StandaloneRelativeFileBaseUrl<'_>, - entrypoint: &ModuleSpecifier, - include_files: &[ModuleSpecifier], - compile_flags: &CompileFlags, ) -> Result<(), AnyError> { + let WriteBinOptions { + writer, + display_output_filename, + graph, + entrypoint, + include_files, + compile_flags, + } = options; let ca_data = match self.cli_options.ca_data() { Some(CaData::File(ca_file)) => Some( std::fs::read(ca_file).with_context(|| format!("Reading {ca_file}"))?, @@ -567,74 +574,28 @@ impl<'a> DenoCompileBinaryWriter<'a> { Some(CaData::Bytes(bytes)) => Some(bytes.clone()), None => None, }; - let root_path = root_dir_url.inner().to_file_path().unwrap(); - let (maybe_npm_vfs, node_modules, npm_snapshot) = - match self.npm_resolver.as_inner() { - InnerCliNpmResolverRef::Managed(managed) => { - let snapshot = - managed.serialized_valid_snapshot_for_system(&self.npm_system_info); - if !snapshot.as_serialized().packages.is_empty() { - let npm_vfs_builder = self - .build_npm_vfs(&root_path) - .context("Building npm vfs.")?; - ( - Some(npm_vfs_builder), - Some(NodeModules::Managed { - node_modules_dir: self - .npm_resolver - .root_node_modules_path() - .map(|path| { - root_dir_url - .specifier_key( - &ModuleSpecifier::from_directory_path(path).unwrap(), - ) - .into_owned() - }), - }), - Some(snapshot), - ) - } else { - (None, None, None) - } + let mut vfs = VfsBuilder::new(); + let npm_snapshot = match self.npm_resolver.as_inner() { + InnerCliNpmResolverRef::Managed(managed) => { + let snapshot = + managed.serialized_valid_snapshot_for_system(&self.npm_system_info); + if !snapshot.as_serialized().packages.is_empty() { + self.fill_npm_vfs(&mut vfs).context("Building npm vfs.")?; + Some(snapshot) + } else { + None } - InnerCliNpmResolverRef::Byonm(resolver) => { - let npm_vfs_builder = self.build_npm_vfs(&root_path)?; - ( - Some(npm_vfs_builder), - Some(NodeModules::Byonm { - root_node_modules_dir: resolver.root_node_modules_path().map( - |node_modules_dir| { - root_dir_url - .specifier_key( - &ModuleSpecifier::from_directory_path(node_modules_dir) - .unwrap(), - ) - .into_owned() - }, - ), - }), - None, - ) - } - }; - let mut vfs = if let Some(npm_vfs) = maybe_npm_vfs { - npm_vfs - } else { - VfsBuilder::new(root_path.clone())? + } + InnerCliNpmResolverRef::Byonm(_) => { + self.fill_npm_vfs(&mut vfs)?; + None + } }; for include_file in include_files { let path = deno_path_util::url_to_file_path(include_file)?; - if path.is_dir() { - // TODO(#26941): we should analyze if any of these are - // modules in order to include their dependencies - vfs - .add_dir_recursive(&path) - .with_context(|| format!("Including {}", path.display()))?; - } else { - vfs - .add_file_at_path(&path) - .with_context(|| format!("Including {}", path.display()))?; - } + vfs + .add_file_at_path(&path) + .with_context(|| format!("Including {}", path.display()))?; } let mut remote_modules_store = RemoteModulesStoreBuilder::default(); let mut code_cache_key_hasher = if self.cli_options.code_cache_enabled() { @@ -706,6 +667,62 @@ impl<'a> DenoCompileBinaryWriter<'a> { } remote_modules_store.add_redirects(&graph.redirects); + if let Some(import_map) = self.workspace_resolver.maybe_import_map() { + if let Ok(file_path) = url_to_file_path(import_map.base_url()) { + if let Some(import_map_parent_dir) = file_path.parent() { + // tell the vfs about the import map's parent directory in case it + // falls outside what the root of where the VFS will be based + vfs.add_possible_min_root_dir(import_map_parent_dir); + } + } + } + if let Some(node_modules_dir) = self.npm_resolver.root_node_modules_path() { + // ensure the vfs doesn't go below the node_modules directory's parent + if let Some(parent) = node_modules_dir.parent() { + vfs.add_possible_min_root_dir(parent); + } + } + + let vfs = self.build_vfs_consolidating_global_npm_cache(vfs); + let root_dir_url = match &vfs.root_path { + WindowsSystemRootablePath::Path(dir) => { + Some(url_from_directory_path(dir)?) + } + WindowsSystemRootablePath::WindowSystemRoot => None, + }; + let root_dir_url = match &root_dir_url { + Some(url) => StandaloneRelativeFileBaseUrl::Path(url), + None => StandaloneRelativeFileBaseUrl::WindowsSystemRoot, + }; + + let node_modules = match self.npm_resolver.as_inner() { + InnerCliNpmResolverRef::Managed(_) => { + npm_snapshot.as_ref().map(|_| NodeModules::Managed { + node_modules_dir: self.npm_resolver.root_node_modules_path().map( + |path| { + root_dir_url + .specifier_key( + &ModuleSpecifier::from_directory_path(path).unwrap(), + ) + .into_owned() + }, + ), + }) + } + InnerCliNpmResolverRef::Byonm(resolver) => Some(NodeModules::Byonm { + root_node_modules_dir: resolver.root_node_modules_path().map( + |node_modules_dir| { + root_dir_url + .specifier_key( + &ModuleSpecifier::from_directory_path(node_modules_dir) + .unwrap(), + ) + .into_owned() + }, + ), + }), + }; + let env_vars_from_env_file = match self.cli_options.env_file_name() { Some(env_filenames) => { let mut aggregated_env_vars = IndexMap::new(); @@ -720,6 +737,8 @@ impl<'a> DenoCompileBinaryWriter<'a> { None => Default::default(), }; + output_vfs(&vfs, display_output_filename); + let metadata = Metadata { argv: compile_flags.args.clone(), seed: self.cli_options.seed(), @@ -779,6 +798,7 @@ impl<'a> DenoCompileBinaryWriter<'a> { detect_cjs: self.cli_options.unstable_detect_cjs(), sloppy_imports: self.cli_options.unstable_sloppy_imports(), features: self.cli_options.unstable_features(), + npm_lazy_caching: self.cli_options.unstable_npm_lazy_caching(), }, otel_config: self.cli_options.otel_config(), }; @@ -789,13 +809,13 @@ impl<'a> DenoCompileBinaryWriter<'a> { &metadata, npm_snapshot.map(|s| s.into_serialized()), &remote_modules_store, - vfs, + &vfs, compile_flags, ) .context("Writing binary bytes") } - fn build_npm_vfs(&self, root_path: &Path) -> Result { + fn fill_npm_vfs(&self, builder: &mut VfsBuilder) -> Result<(), AnyError> { fn maybe_warn_different_system(system_info: &NpmSystemInfo) { if system_info != &NpmSystemInfo::default() { log::warn!("{} The node_modules directory may be incompatible with the target system.", crate::colors::yellow("Warning")); @@ -806,15 +826,10 @@ impl<'a> DenoCompileBinaryWriter<'a> { InnerCliNpmResolverRef::Managed(npm_resolver) => { if let Some(node_modules_path) = npm_resolver.root_node_modules_path() { maybe_warn_different_system(&self.npm_system_info); - let mut builder = VfsBuilder::new(root_path.to_path_buf())?; builder.add_dir_recursive(node_modules_path)?; - Ok(builder) + Ok(()) } else { - // DO NOT include the user's registry url as it may contain credentials, - // but also don't make this dependent on the registry url - let global_cache_root_path = npm_resolver.global_cache_root_path(); - let mut builder = - VfsBuilder::new(global_cache_root_path.to_path_buf())?; + // we'll flatten to remove any custom registries later let mut packages = npm_resolver.all_system_packages(&self.npm_system_info); packages.sort_by(|a, b| a.id.cmp(&b.id)); // determinism @@ -823,55 +838,11 @@ impl<'a> DenoCompileBinaryWriter<'a> { npm_resolver.resolve_pkg_folder_from_pkg_id(&package.id)?; builder.add_dir_recursive(&folder)?; } - - // Flatten all the registries folders into a single ".deno_compile_node_modules/localhost" folder - // that will be used by denort when loading the npm cache. This avoids us exposing - // the user's private registry information and means we don't have to bother - // serializing all the different registry config into the binary. - builder.with_root_dir(|root_dir| { - root_dir.name = ".deno_compile_node_modules".to_string(); - let mut new_entries = Vec::with_capacity(root_dir.entries.len()); - let mut localhost_entries = IndexMap::new(); - for entry in std::mem::take(&mut root_dir.entries) { - match entry { - VfsEntry::Dir(dir) => { - for entry in dir.entries { - log::debug!( - "Flattening {} into node_modules", - entry.name() - ); - if let Some(existing) = - localhost_entries.insert(entry.name().to_string(), entry) - { - panic!( - "Unhandled scenario where a duplicate entry was found: {:?}", - existing - ); - } - } - } - VfsEntry::File(_) | VfsEntry::Symlink(_) => { - new_entries.push(entry); - } - } - } - new_entries.push(VfsEntry::Dir(VirtualDirectory { - name: "localhost".to_string(), - entries: localhost_entries.into_iter().map(|(_, v)| v).collect(), - })); - // needs to be sorted by name - new_entries.sort_by(|a, b| a.name().cmp(b.name())); - root_dir.entries = new_entries; - }); - - builder.set_new_root_path(root_path.to_path_buf())?; - - Ok(builder) + Ok(()) } } InnerCliNpmResolverRef::Byonm(_) => { maybe_warn_different_system(&self.npm_system_info); - let mut builder = VfsBuilder::new(root_path.to_path_buf())?; for pkg_json in self.cli_options.workspace().package_jsons() { builder.add_file_at_path(&pkg_json.path)?; } @@ -904,10 +875,102 @@ impl<'a> DenoCompileBinaryWriter<'a> { } } } - Ok(builder) + Ok(()) } } } + + fn build_vfs_consolidating_global_npm_cache( + &self, + mut vfs: VfsBuilder, + ) -> BuiltVfs { + match self.npm_resolver.as_inner() { + InnerCliNpmResolverRef::Managed(npm_resolver) => { + if npm_resolver.root_node_modules_path().is_some() { + return vfs.build(); + } + + let global_cache_root_path = npm_resolver.global_cache_root_path(); + + // Flatten all the registries folders into a single ".deno_compile_node_modules/localhost" folder + // that will be used by denort when loading the npm cache. This avoids us exposing + // the user's private registry information and means we don't have to bother + // serializing all the different registry config into the binary. + let Some(root_dir) = vfs.get_dir_mut(global_cache_root_path) else { + return vfs.build(); + }; + + root_dir.name = DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME.to_string(); + let mut new_entries = Vec::with_capacity(root_dir.entries.len()); + let mut localhost_entries = IndexMap::new(); + for entry in std::mem::take(&mut root_dir.entries) { + match entry { + VfsEntry::Dir(dir) => { + for entry in dir.entries { + log::debug!("Flattening {} into node_modules", entry.name()); + if let Some(existing) = + localhost_entries.insert(entry.name().to_string(), entry) + { + panic!( + "Unhandled scenario where a duplicate entry was found: {:?}", + existing + ); + } + } + } + VfsEntry::File(_) | VfsEntry::Symlink(_) => { + new_entries.push(entry); + } + } + } + new_entries.push(VfsEntry::Dir(VirtualDirectory { + name: "localhost".to_string(), + entries: localhost_entries.into_iter().map(|(_, v)| v).collect(), + })); + // needs to be sorted by name + new_entries.sort_by(|a, b| a.name().cmp(b.name())); + root_dir.entries = new_entries; + + // it's better to not expose the user's cache directory, so take it out + // of there + let parent = global_cache_root_path.parent().unwrap(); + let parent_dir = vfs.get_dir_mut(parent).unwrap(); + let index = parent_dir + .entries + .iter() + .position(|entry| { + entry.name() == DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME + }) + .unwrap(); + let npm_global_cache_dir_entry = parent_dir.entries.remove(index); + + // go up from the ancestors removing empty directories... + // this is not as optimized as it could be + let mut last_name = + Cow::Borrowed(DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME); + for ancestor in parent.ancestors() { + let dir = vfs.get_dir_mut(ancestor).unwrap(); + if let Some(index) = dir + .entries + .iter() + .position(|entry| entry.name() == last_name) + { + dir.entries.remove(index); + } + last_name = Cow::Owned(dir.name.clone()); + if !dir.entries.is_empty() { + break; + } + } + + // now build the vfs and add the global cache dir entry there + let mut built_vfs = vfs.build(); + built_vfs.root.insert_entry(npm_global_cache_dir_entry); + built_vfs + } + InnerCliNpmResolverRef::Byonm(_) => vfs.build(), + } + } } fn get_denort_path(deno_exe: PathBuf) -> Option { diff --git a/cli/standalone/mod.rs b/cli/standalone/mod.rs index 53efab2964..22e0b6d115 100644 --- a/cli/standalone/mod.rs +++ b/cli/standalone/mod.rs @@ -924,6 +924,7 @@ pub async fn run(data: StandaloneData) -> Result { serve_host: None, }, metadata.otel_config, + crate::args::NpmCachingStrategy::Lazy, ); // Initialize v8 once from the main thread. diff --git a/cli/standalone/serialization.rs b/cli/standalone/serialization.rs index a5eb649bfd..6062e21019 100644 --- a/cli/standalone/serialization.rs +++ b/cli/standalone/serialization.rs @@ -23,6 +23,7 @@ use deno_semver::package::PackageReq; use crate::standalone::virtual_fs::VirtualDirectory; use super::binary::Metadata; +use super::virtual_fs::BuiltVfs; use super::virtual_fs::VfsBuilder; const MAGIC_BYTES: &[u8; 8] = b"d3n0l4nd"; @@ -39,7 +40,7 @@ pub fn serialize_binary_data_section( metadata: &Metadata, npm_snapshot: Option, remote_modules: &RemoteModulesStoreBuilder, - vfs: VfsBuilder, + vfs: &BuiltVfs, ) -> Result, AnyError> { fn write_bytes_with_len(bytes: &mut Vec, data: &[u8]) { bytes.extend_from_slice(&(data.len() as u64).to_le_bytes()); @@ -73,12 +74,11 @@ pub fn serialize_binary_data_section( } // 4. VFS { - let (vfs, vfs_files) = vfs.into_dir_and_files(); - let vfs = serde_json::to_string(&vfs)?; - write_bytes_with_len(&mut bytes, vfs.as_bytes()); - let vfs_bytes_len = vfs_files.iter().map(|f| f.len() as u64).sum::(); + let serialized_vfs = serde_json::to_string(&vfs.root)?; + write_bytes_with_len(&mut bytes, serialized_vfs.as_bytes()); + let vfs_bytes_len = vfs.files.iter().map(|f| f.len() as u64).sum::(); bytes.extend_from_slice(&vfs_bytes_len.to_le_bytes()); - for file in &vfs_files { + for file in &vfs.files { bytes.extend_from_slice(file); } } diff --git a/cli/standalone/virtual_fs.rs b/cli/standalone/virtual_fs.rs index 66fc835534..8ddd179c7a 100644 --- a/cli/standalone/virtual_fs.rs +++ b/cli/standalone/virtual_fs.rs @@ -1,6 +1,7 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. use std::borrow::Cow; +use std::cell::RefCell; use std::collections::HashMap; use std::collections::HashSet; use std::fs::File; @@ -14,24 +15,61 @@ use std::rc::Rc; use std::sync::Arc; use deno_core::anyhow::anyhow; +use deno_core::anyhow::bail; use deno_core::anyhow::Context; use deno_core::error::AnyError; use deno_core::parking_lot::Mutex; use deno_core::BufMutView; use deno_core::BufView; use deno_core::ResourceHandleFd; +use deno_path_util::normalize_path; +use deno_path_util::strip_unc_prefix; use deno_runtime::deno_fs::FsDirEntry; use deno_runtime::deno_io; use deno_runtime::deno_io::fs::FsError; use deno_runtime::deno_io::fs::FsResult; use deno_runtime::deno_io::fs::FsStat; +use indexmap::IndexSet; use serde::Deserialize; use serde::Serialize; use thiserror::Error; use crate::util; +use crate::util::display::DisplayTreeNode; use crate::util::fs::canonicalize_path; +use super::binary::DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME; + +#[derive(Debug, PartialEq, Eq)] +pub enum WindowsSystemRootablePath { + /// The root of the system above any drive letters. + WindowSystemRoot, + Path(PathBuf), +} + +impl WindowsSystemRootablePath { + pub fn join(&self, name_component: &str) -> PathBuf { + // this method doesn't handle multiple components + debug_assert!(!name_component.contains('\\')); + debug_assert!(!name_component.contains('/')); + + match self { + WindowsSystemRootablePath::WindowSystemRoot => { + // windows drive letter + PathBuf::from(&format!("{}\\", name_component)) + } + WindowsSystemRootablePath::Path(path) => path.join(name_component), + } + } +} + +#[derive(Debug)] +pub struct BuiltVfs { + pub root_path: WindowsSystemRootablePath, + pub root: VirtualDirectory, + pub files: Vec>, +} + #[derive(Debug, Copy, Clone)] pub enum VfsFileSubDataKind { /// Raw bytes of the file. @@ -41,83 +79,84 @@ pub enum VfsFileSubDataKind { ModuleGraph, } -#[derive(Error, Debug)] -#[error( - "Failed to strip prefix '{}' from '{}'", root_path.display(), target.display() -)] -pub struct StripRootError { - root_path: PathBuf, - target: PathBuf, -} - +#[derive(Debug)] pub struct VfsBuilder { - root_path: PathBuf, - root_dir: VirtualDirectory, + executable_root: VirtualDirectory, files: Vec>, current_offset: u64, file_offsets: HashMap, + /// The minimum root directory that should be included in the VFS. + min_root_dir: Option, } impl VfsBuilder { - pub fn new(root_path: PathBuf) -> Result { - let root_path = canonicalize_path(&root_path) - .with_context(|| format!("Canonicalizing {}", root_path.display()))?; - log::debug!("Building vfs with root '{}'", root_path.display()); - Ok(Self { - root_dir: VirtualDirectory { - name: root_path - .file_stem() - .map(|s| s.to_string_lossy().into_owned()) - .unwrap_or("root".to_string()), + pub fn new() -> Self { + Self { + executable_root: VirtualDirectory { + name: "/".to_string(), entries: Vec::new(), }, - root_path, files: Vec::new(), current_offset: 0, file_offsets: Default::default(), - }) + min_root_dir: Default::default(), + } } - pub fn set_new_root_path( - &mut self, - root_path: PathBuf, - ) -> Result<(), AnyError> { - let root_path = canonicalize_path(&root_path)?; - self.root_path = root_path; - self.root_dir = VirtualDirectory { - name: self - .root_path - .file_stem() - .map(|s| s.to_string_lossy().into_owned()) - .unwrap_or("root".to_string()), - entries: vec![VfsEntry::Dir(VirtualDirectory { - name: std::mem::take(&mut self.root_dir.name), - entries: std::mem::take(&mut self.root_dir.entries), - })], - }; - Ok(()) - } + /// Add a directory that might be the minimum root directory + /// of the VFS. + /// + /// For example, say the user has a deno.json and specifies an + /// import map in a parent directory. The import map won't be + /// included in the VFS, but its base will meaning we need to + /// tell the VFS builder to include the base of the import map + /// by calling this method. + pub fn add_possible_min_root_dir(&mut self, path: &Path) { + self.add_dir_raw(path); - pub fn with_root_dir( - &mut self, - with_root: impl FnOnce(&mut VirtualDirectory) -> R, - ) -> R { - with_root(&mut self.root_dir) + match &self.min_root_dir { + Some(WindowsSystemRootablePath::WindowSystemRoot) => { + // already the root dir + } + Some(WindowsSystemRootablePath::Path(current_path)) => { + let mut common_components = Vec::new(); + for (a, b) in current_path.components().zip(path.components()) { + if a != b { + break; + } + common_components.push(a); + } + if common_components.is_empty() { + if cfg!(windows) { + self.min_root_dir = + Some(WindowsSystemRootablePath::WindowSystemRoot); + } else { + self.min_root_dir = + Some(WindowsSystemRootablePath::Path(PathBuf::from("/"))); + } + } else { + self.min_root_dir = Some(WindowsSystemRootablePath::Path( + common_components.iter().collect(), + )); + } + } + None => { + self.min_root_dir = + Some(WindowsSystemRootablePath::Path(path.to_path_buf())); + } + } } pub fn add_dir_recursive(&mut self, path: &Path) -> Result<(), AnyError> { - let target_path = canonicalize_path(path)?; - if path != target_path { - self.add_symlink(path, &target_path)?; - } - self.add_dir_recursive_internal(&target_path) + let target_path = self.resolve_target_path(path)?; + self.add_dir_recursive_not_symlink(&target_path) } - fn add_dir_recursive_internal( + fn add_dir_recursive_not_symlink( &mut self, path: &Path, ) -> Result<(), AnyError> { - self.add_dir(path)?; + self.add_dir_raw(path); let read_dir = std::fs::read_dir(path) .with_context(|| format!("Reading {}", path.display()))?; @@ -130,49 +169,26 @@ impl VfsBuilder { let path = entry.path(); if file_type.is_dir() { - self.add_dir_recursive_internal(&path)?; + self.add_dir_recursive_not_symlink(&path)?; } else if file_type.is_file() { self.add_file_at_path_not_symlink(&path)?; } else if file_type.is_symlink() { - match util::fs::canonicalize_path(&path) { - Ok(target) => { - if let Err(StripRootError { .. }) = self.add_symlink(&path, &target) - { - if target.is_file() { - // this may change behavior, so warn the user about it - log::warn!( - "{} Symlink target is outside '{}'. Inlining symlink at '{}' to '{}' as file.", - crate::colors::yellow("Warning"), - self.root_path.display(), - path.display(), - target.display(), - ); - // inline the symlink and make the target file - let file_bytes = std::fs::read(&target) - .with_context(|| format!("Reading {}", path.display()))?; - self.add_file_with_data_inner( - &path, - file_bytes, - VfsFileSubDataKind::Raw, - )?; - } else { - log::warn!( - "{} Symlink target is outside '{}'. Excluding symlink at '{}' with target '{}'.", - crate::colors::yellow("Warning"), - self.root_path.display(), - path.display(), - target.display(), - ); - } + match self.add_symlink(&path) { + Ok(target) => match target { + SymlinkTarget::File(target) => { + self.add_file_at_path_not_symlink(&target)? } - } + SymlinkTarget::Dir(target) => { + self.add_dir_recursive_not_symlink(&target)?; + } + }, Err(err) => { log::warn!( - "{} Failed resolving symlink. Ignoring.\n Path: {}\n Message: {:#}", - crate::colors::yellow("Warning"), - path.display(), - err - ); + "{} Failed resolving symlink. Ignoring.\n Path: {}\n Message: {:#}", + crate::colors::yellow("Warning"), + path.display(), + err + ); } } } @@ -181,15 +197,15 @@ impl VfsBuilder { Ok(()) } - fn add_dir( - &mut self, - path: &Path, - ) -> Result<&mut VirtualDirectory, StripRootError> { + fn add_dir_raw(&mut self, path: &Path) -> &mut VirtualDirectory { log::debug!("Ensuring directory '{}'", path.display()); - let path = self.path_relative_root(path)?; - let mut current_dir = &mut self.root_dir; + debug_assert!(path.is_absolute()); + let mut current_dir = &mut self.executable_root; for component in path.components() { + if matches!(component, std::path::Component::RootDir) { + continue; + } let name = component.as_os_str().to_string_lossy(); let index = match current_dir .entries @@ -215,15 +231,44 @@ impl VfsBuilder { }; } - Ok(current_dir) + current_dir + } + + pub fn get_system_root_dir_mut(&mut self) -> &mut VirtualDirectory { + &mut self.executable_root + } + + pub fn get_dir_mut(&mut self, path: &Path) -> Option<&mut VirtualDirectory> { + debug_assert!(path.is_absolute()); + let mut current_dir = &mut self.executable_root; + + for component in path.components() { + if matches!(component, std::path::Component::RootDir) { + continue; + } + let name = component.as_os_str().to_string_lossy(); + let index = match current_dir + .entries + .binary_search_by(|e| e.name().cmp(&name)) + { + Ok(index) => index, + Err(_) => return None, + }; + match &mut current_dir.entries[index] { + VfsEntry::Dir(dir) => { + current_dir = dir; + } + _ => unreachable!(), + }; + } + + Some(current_dir) } pub fn add_file_at_path(&mut self, path: &Path) -> Result<(), AnyError> { - let target_path = canonicalize_path(path)?; - if target_path != path { - self.add_symlink(path, &target_path)?; - } - self.add_file_at_path_not_symlink(&target_path) + let file_bytes = std::fs::read(path) + .with_context(|| format!("Reading {}", path.display()))?; + self.add_file_with_data(path, file_bytes, VfsFileSubDataKind::Raw) } fn add_file_at_path_not_symlink( @@ -241,11 +286,15 @@ impl VfsBuilder { data: Vec, sub_data_kind: VfsFileSubDataKind, ) -> Result<(), AnyError> { - let target_path = canonicalize_path(path)?; - if target_path != path { - self.add_symlink(path, &target_path)?; + let metadata = std::fs::symlink_metadata(path).with_context(|| { + format!("Resolving target path for '{}'", path.display()) + })?; + if metadata.is_symlink() { + let target = self.add_symlink(path)?.into_path_buf(); + self.add_file_with_data_inner(&target, data, sub_data_kind) + } else { + self.add_file_with_data_inner(path, data, sub_data_kind) } - self.add_file_with_data_inner(&target_path, data, sub_data_kind) } fn add_file_with_data_inner( @@ -264,19 +313,22 @@ impl VfsBuilder { self.current_offset }; - let dir = self.add_dir(path.parent().unwrap())?; + let dir = self.add_dir_raw(path.parent().unwrap()); let name = path.file_name().unwrap().to_string_lossy(); - let data_len = data.len(); + let offset_and_len = OffsetWithLength { + offset, + len: data.len() as u64, + }; match dir.entries.binary_search_by(|e| e.name().cmp(&name)) { Ok(index) => { let entry = &mut dir.entries[index]; match entry { VfsEntry::File(virtual_file) => match sub_data_kind { VfsFileSubDataKind::Raw => { - virtual_file.offset = offset; + virtual_file.offset = offset_and_len; } VfsFileSubDataKind::ModuleGraph => { - virtual_file.module_graph_offset = offset; + virtual_file.module_graph_offset = offset_and_len; } }, VfsEntry::Dir(_) | VfsEntry::Symlink(_) => unreachable!(), @@ -287,9 +339,8 @@ impl VfsBuilder { insert_index, VfsEntry::File(VirtualFile { name: name.to_string(), - offset, - module_graph_offset: offset, - len: data.len() as u64, + offset: offset_and_len, + module_graph_offset: offset_and_len, }), ); } @@ -298,66 +349,343 @@ impl VfsBuilder { // new file, update the list of files if self.current_offset == offset { self.files.push(data); - self.current_offset += data_len as u64; + self.current_offset += offset_and_len.len; } Ok(()) } - fn add_symlink( + fn resolve_target_path(&mut self, path: &Path) -> Result { + let metadata = std::fs::symlink_metadata(path).with_context(|| { + format!("Resolving target path for '{}'", path.display()) + })?; + if metadata.is_symlink() { + Ok(self.add_symlink(path)?.into_path_buf()) + } else { + Ok(path.to_path_buf()) + } + } + + fn add_symlink(&mut self, path: &Path) -> Result { + self.add_symlink_inner(path, &mut IndexSet::new()) + } + + fn add_symlink_inner( &mut self, path: &Path, - target: &Path, - ) -> Result<(), StripRootError> { - log::debug!( - "Adding symlink '{}' to '{}'", - path.display(), - target.display() + visited: &mut IndexSet, + ) -> Result { + log::debug!("Adding symlink '{}'", path.display()); + let target = strip_unc_prefix( + std::fs::read_link(path) + .with_context(|| format!("Reading symlink '{}'", path.display()))?, ); - let relative_target = self.path_relative_root(target)?; - let relative_path = match self.path_relative_root(path) { - Ok(path) => path, - Err(StripRootError { .. }) => { - // ignore if the original path is outside the root directory - return Ok(()); - } - }; - if relative_target == relative_path { - // it's the same, ignore - return Ok(()); - } - let dir = self.add_dir(path.parent().unwrap())?; + let target = normalize_path(path.parent().unwrap().join(&target)); + let dir = self.add_dir_raw(path.parent().unwrap()); let name = path.file_name().unwrap().to_string_lossy(); match dir.entries.binary_search_by(|e| e.name().cmp(&name)) { - Ok(_) => Ok(()), // previously inserted + Ok(_) => {} // previously inserted Err(insert_index) => { dir.entries.insert( insert_index, VfsEntry::Symlink(VirtualSymlink { name: name.to_string(), - dest_parts: relative_target - .components() - .map(|c| c.as_os_str().to_string_lossy().to_string()) - .collect::>(), + dest_parts: VirtualSymlinkParts::from_path(&target), }), ); - Ok(()) + } + } + let target_metadata = + std::fs::symlink_metadata(&target).with_context(|| { + format!("Reading symlink target '{}'", target.display()) + })?; + if target_metadata.is_symlink() { + if !visited.insert(target.clone()) { + // todo: probably don't error in this scenario + bail!( + "Circular symlink detected: {} -> {}", + visited + .iter() + .map(|p| p.display().to_string()) + .collect::>() + .join(" -> "), + target.display() + ); + } + self.add_symlink_inner(&target, visited) + } else if target_metadata.is_dir() { + Ok(SymlinkTarget::Dir(target)) + } else { + Ok(SymlinkTarget::File(target)) + } + } + + pub fn build(self) -> BuiltVfs { + fn strip_prefix_from_symlinks( + dir: &mut VirtualDirectory, + parts: &[String], + ) { + for entry in &mut dir.entries { + match entry { + VfsEntry::Dir(dir) => { + strip_prefix_from_symlinks(dir, parts); + } + VfsEntry::File(_) => {} + VfsEntry::Symlink(symlink) => { + let old_parts = std::mem::take(&mut symlink.dest_parts.0); + symlink.dest_parts.0 = + old_parts.into_iter().skip(parts.len()).collect(); + } + } + } + } + + let mut current_dir = self.executable_root; + let mut current_path = if cfg!(windows) { + WindowsSystemRootablePath::WindowSystemRoot + } else { + WindowsSystemRootablePath::Path(PathBuf::from("/")) + }; + loop { + if current_dir.entries.len() != 1 { + break; + } + if self.min_root_dir.as_ref() == Some(¤t_path) { + break; + } + match ¤t_dir.entries[0] { + VfsEntry::Dir(dir) => { + if dir.name == DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME { + // special directory we want to maintain + break; + } + match current_dir.entries.remove(0) { + VfsEntry::Dir(dir) => { + current_path = + WindowsSystemRootablePath::Path(current_path.join(&dir.name)); + current_dir = dir; + } + _ => unreachable!(), + }; + } + VfsEntry::File(_) | VfsEntry::Symlink(_) => break, + } + } + if let WindowsSystemRootablePath::Path(path) = ¤t_path { + strip_prefix_from_symlinks( + &mut current_dir, + &VirtualSymlinkParts::from_path(path).0, + ); + } + BuiltVfs { + root_path: current_path, + root: current_dir, + files: self.files, + } + } +} + +#[derive(Debug)] +enum SymlinkTarget { + File(PathBuf), + Dir(PathBuf), +} + +impl SymlinkTarget { + pub fn into_path_buf(self) -> PathBuf { + match self { + Self::File(path) => path, + Self::Dir(path) => path, + } + } +} + +pub fn output_vfs(vfs: &BuiltVfs, executable_name: &str) { + if !log::log_enabled!(log::Level::Info) { + return; // no need to compute if won't output + } + + if vfs.root.entries.is_empty() { + return; // nothing to output + } + + let mut text = String::new(); + let display_tree = vfs_as_display_tree(vfs, executable_name); + display_tree.print(&mut text).unwrap(); // unwrap ok because it's writing to a string + log::info!( + "\n{}\n", + deno_terminal::colors::bold("Embedded File System") + ); + log::info!("{}\n", text.trim()); +} + +fn vfs_as_display_tree( + vfs: &BuiltVfs, + executable_name: &str, +) -> DisplayTreeNode { + enum EntryOutput<'a> { + All, + Subset(Vec>), + File, + Symlink(&'a [String]), + } + + impl<'a> EntryOutput<'a> { + pub fn as_display_tree(&self, name: String) -> DisplayTreeNode { + let mut children = match self { + EntryOutput::Subset(vec) => vec + .iter() + .map(|e| e.output.as_display_tree(e.name.to_string())) + .collect(), + EntryOutput::All | EntryOutput::File | EntryOutput::Symlink(_) => { + vec![] + } + }; + // we only want to collapse leafs so that nodes of the + // same depth have the same indentation + let collapse_single_child = + children.len() == 1 && children[0].children.is_empty(); + DisplayTreeNode { + text: match self { + EntryOutput::All => format!("{}/*", name), + EntryOutput::Subset(_) => { + if collapse_single_child { + format!("{}/{}", name, children[0].text) + } else { + name + } + } + EntryOutput::File => name, + EntryOutput::Symlink(parts) => { + format!("{} --> {}", name, parts.join("/")) + } + }, + children: if collapse_single_child { + children.remove(0).children + } else { + children + }, } } } - pub fn into_dir_and_files(self) -> (VirtualDirectory, Vec>) { - (self.root_dir, self.files) + pub struct DirEntryOutput<'a> { + name: &'a str, + output: EntryOutput<'a>, } - fn path_relative_root(&self, path: &Path) -> Result { - match path.strip_prefix(&self.root_path) { - Ok(p) => Ok(p.to_path_buf()), - Err(_) => Err(StripRootError { - root_path: self.root_path.clone(), - target: path.to_path_buf(), - }), + fn show_global_node_modules_dir( + vfs_dir: &VirtualDirectory, + ) -> Vec { + fn show_subset_deep( + vfs_dir: &VirtualDirectory, + depth: usize, + ) -> EntryOutput { + if depth == 0 { + EntryOutput::All + } else { + EntryOutput::Subset(show_subset(vfs_dir, depth)) + } } + + fn show_subset( + vfs_dir: &VirtualDirectory, + depth: usize, + ) -> Vec { + vfs_dir + .entries + .iter() + .map(|entry| DirEntryOutput { + name: entry.name(), + output: match entry { + VfsEntry::Dir(virtual_directory) => { + show_subset_deep(virtual_directory, depth - 1) + } + VfsEntry::File(_) => EntryOutput::File, + VfsEntry::Symlink(virtual_symlink) => { + EntryOutput::Symlink(&virtual_symlink.dest_parts.0) + } + }, + }) + .collect() + } + + // in this scenario, we want to show + // .deno_compile_node_modules/localhost///* + show_subset(vfs_dir, 3) + } + + fn include_all_entries<'a>( + dir_path: &WindowsSystemRootablePath, + vfs_dir: &'a VirtualDirectory, + ) -> Vec> { + if vfs_dir.name == DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME { + return show_global_node_modules_dir(vfs_dir); + } + + vfs_dir + .entries + .iter() + .map(|entry| DirEntryOutput { + name: entry.name(), + output: analyze_entry(dir_path.join(entry.name()), entry), + }) + .collect() + } + + fn analyze_entry(path: PathBuf, entry: &VfsEntry) -> EntryOutput { + match entry { + VfsEntry::Dir(virtual_directory) => analyze_dir(path, virtual_directory), + VfsEntry::File(_) => EntryOutput::File, + VfsEntry::Symlink(virtual_symlink) => { + EntryOutput::Symlink(&virtual_symlink.dest_parts.0) + } + } + } + + fn analyze_dir(dir: PathBuf, vfs_dir: &VirtualDirectory) -> EntryOutput { + if vfs_dir.name == DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME { + return EntryOutput::Subset(show_global_node_modules_dir(vfs_dir)); + } + + let real_entry_count = std::fs::read_dir(&dir) + .ok() + .map(|entries| entries.flat_map(|e| e.ok()).count()) + .unwrap_or(0); + if real_entry_count == vfs_dir.entries.len() { + let children = vfs_dir + .entries + .iter() + .map(|entry| DirEntryOutput { + name: entry.name(), + output: analyze_entry(dir.join(entry.name()), entry), + }) + .collect::>(); + if children + .iter() + .all(|c| !matches!(c.output, EntryOutput::Subset(_))) + { + EntryOutput::All + } else { + EntryOutput::Subset(children) + } + } else { + EntryOutput::Subset(include_all_entries( + &WindowsSystemRootablePath::Path(dir), + vfs_dir, + )) + } + } + + // always include all the entries for the root directory, otherwise the + // user might not have context about what's being shown + let child_entries = include_all_entries(&vfs.root_path, &vfs.root); + DisplayTreeNode { + text: deno_terminal::colors::italic(executable_name).to_string(), + children: child_entries + .iter() + .map(|entry| entry.output.as_display_tree(entry.name.to_string())) + .collect(), } } @@ -403,7 +731,7 @@ impl<'a> VfsEntryRef<'a> { mtime: None, ctime: None, blksize: 0, - size: file.len, + size: file.offset.len, dev: 0, ino: 0, mode: 0, @@ -472,34 +800,77 @@ impl VfsEntry { #[derive(Debug, Serialize, Deserialize)] pub struct VirtualDirectory { + #[serde(rename = "n")] pub name: String, // should be sorted by name + #[serde(rename = "e")] pub entries: Vec, } +impl VirtualDirectory { + pub fn insert_entry(&mut self, entry: VfsEntry) { + let name = entry.name(); + match self.entries.binary_search_by(|e| e.name().cmp(name)) { + Ok(index) => { + self.entries[index] = entry; + } + Err(insert_index) => { + self.entries.insert(insert_index, entry); + } + } + } +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +pub struct OffsetWithLength { + #[serde(rename = "o")] + pub offset: u64, + #[serde(rename = "l")] + pub len: u64, +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct VirtualFile { + #[serde(rename = "n")] pub name: String, - pub offset: u64, + #[serde(rename = "o")] + pub offset: OffsetWithLength, /// Offset file to use for module loading when it differs from the /// raw file. Often this will be the same offset as above for data /// such as JavaScript files, but for TypeScript files the `offset` /// will be the original raw bytes when included as an asset and this /// offset will be to the transpiled JavaScript source. - pub module_graph_offset: u64, - pub len: u64, + #[serde(rename = "m")] + pub module_graph_offset: OffsetWithLength, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct VirtualSymlinkParts(Vec); + +impl VirtualSymlinkParts { + pub fn from_path(path: &Path) -> Self { + Self( + path + .components() + .filter(|c| !matches!(c, std::path::Component::RootDir)) + .map(|c| c.as_os_str().to_string_lossy().to_string()) + .collect(), + ) + } } #[derive(Debug, Serialize, Deserialize)] pub struct VirtualSymlink { + #[serde(rename = "n")] pub name: String, - pub dest_parts: Vec, + #[serde(rename = "p")] + pub dest_parts: VirtualSymlinkParts, } impl VirtualSymlink { pub fn resolve_dest_from_root(&self, root: &Path) -> PathBuf { let mut dest = root.to_path_buf(); - for part in &self.dest_parts { + for part in &self.dest_parts.0 { dest.push(part); } dest @@ -571,10 +942,10 @@ impl VfsRoot { let mut final_path = self.root_path.clone(); let mut current_entry = VfsEntryRef::Dir(&self.dir); for component in relative_path.components() { - let component = component.as_os_str().to_string_lossy(); + let component = component.as_os_str(); let current_dir = match current_entry { VfsEntryRef::Dir(dir) => { - final_path.push(component.as_ref()); + final_path.push(component); dir } VfsEntryRef::Symlink(symlink) => { @@ -583,7 +954,7 @@ impl VfsRoot { final_path = resolved_path; // overwrite with the new resolved path match entry { VfsEntryRef::Dir(dir) => { - final_path.push(component.as_ref()); + final_path.push(component); dir } _ => { @@ -601,6 +972,7 @@ impl VfsRoot { )); } }; + let component = component.to_string_lossy(); match current_dir .entries .binary_search_by(|e| e.name().cmp(&component)) @@ -621,10 +993,9 @@ impl VfsRoot { } } -#[derive(Clone)] struct FileBackedVfsFile { file: VirtualFile, - pos: Arc>, + pos: RefCell, vfs: Arc, } @@ -632,28 +1003,28 @@ impl FileBackedVfsFile { fn seek(&self, pos: SeekFrom) -> FsResult { match pos { SeekFrom::Start(pos) => { - *self.pos.lock() = pos; + *self.pos.borrow_mut() = pos; Ok(pos) } SeekFrom::End(offset) => { - if offset < 0 && -offset as u64 > self.file.len { + if offset < 0 && -offset as u64 > self.file.offset.len { let msg = "An attempt was made to move the file pointer before the beginning of the file."; Err( std::io::Error::new(std::io::ErrorKind::PermissionDenied, msg) .into(), ) } else { - let mut current_pos = self.pos.lock(); + let mut current_pos = self.pos.borrow_mut(); *current_pos = if offset >= 0 { - self.file.len - (offset as u64) + self.file.offset.len - (offset as u64) } else { - self.file.len + (-offset as u64) + self.file.offset.len + (-offset as u64) }; Ok(*current_pos) } } SeekFrom::Current(offset) => { - let mut current_pos = self.pos.lock(); + let mut current_pos = self.pos.borrow_mut(); if offset >= 0 { *current_pos += offset as u64; } else if -offset as u64 > *current_pos { @@ -668,10 +1039,10 @@ impl FileBackedVfsFile { fn read_to_buf(&self, buf: &mut [u8]) -> FsResult { let read_pos = { - let mut pos = self.pos.lock(); + let mut pos = self.pos.borrow_mut(); let read_pos = *pos; // advance the position due to the read - *pos = std::cmp::min(self.file.len, *pos + buf.len() as u64); + *pos = std::cmp::min(self.file.offset.len, *pos + buf.len() as u64); read_pos }; self @@ -682,16 +1053,16 @@ impl FileBackedVfsFile { fn read_to_end(&self) -> FsResult> { let read_pos = { - let mut pos = self.pos.lock(); + let mut pos = self.pos.borrow_mut(); let read_pos = *pos; // todo(dsherret): should this always set it to the end of the file? - if *pos < self.file.len { + if *pos < self.file.offset.len { // advance the position due to the read - *pos = self.file.len; + *pos = self.file.offset.len; } read_pos }; - if read_pos > self.file.len { + if read_pos > self.file.offset.len { return Ok(Cow::Borrowed(&[])); } if read_pos == 0 { @@ -701,7 +1072,7 @@ impl FileBackedVfsFile { .read_file_all(&self.file, VfsFileSubDataKind::Raw)?, ) } else { - let size = (self.file.len - read_pos) as usize; + let size = (self.file.offset.len - read_pos) as usize; let mut buf = vec![0; size]; self.vfs.read_file(&self.file, read_pos, &mut buf)?; Ok(Cow::Owned(buf)) @@ -718,12 +1089,9 @@ impl deno_io::fs::File for FileBackedVfsFile { self: Rc, mut buf: BufMutView, ) -> FsResult<(usize, BufMutView)> { - let inner = (*self).clone(); - tokio::task::spawn(async move { - let nread = inner.read_to_buf(&mut buf)?; - Ok((nread, buf)) - }) - .await? + // this is fast, no need to spawn a task + let nread = self.read_to_buf(&mut buf)?; + Ok((nread, buf)) } fn write_sync(self: Rc, _buf: &[u8]) -> FsResult { @@ -747,8 +1115,8 @@ impl deno_io::fs::File for FileBackedVfsFile { self.read_to_end() } async fn read_all_async(self: Rc) -> FsResult> { - let inner = (*self).clone(); - tokio::task::spawn_blocking(move || inner.read_to_end()).await? + // this is fast, no need to spawn a task + self.read_to_end() } fn chmod_sync(self: Rc, _pathmode: u32) -> FsResult<()> { @@ -921,7 +1289,11 @@ impl FileBackedVfs { file: &VirtualFile, sub_data_kind: VfsFileSubDataKind, ) -> std::io::Result> { - let read_range = self.get_read_range(file, sub_data_kind, 0, file.len)?; + let read_len = match sub_data_kind { + VfsFileSubDataKind::Raw => file.offset.len, + VfsFileSubDataKind::ModuleGraph => file.module_graph_offset.len, + }; + let read_range = self.get_read_range(file, sub_data_kind, 0, read_len)?; match &self.vfs_data { Cow::Borrowed(data) => Ok(Cow::Borrowed(&data[read_range])), Cow::Owned(data) => Ok(Cow::Owned(data[read_range].to_vec())), @@ -952,19 +1324,20 @@ impl FileBackedVfs { pos: u64, len: u64, ) -> std::io::Result> { - if pos > file.len { + let file_offset_and_len = match sub_data_kind { + VfsFileSubDataKind::Raw => file.offset, + VfsFileSubDataKind::ModuleGraph => file.module_graph_offset, + }; + if pos > file_offset_and_len.len { return Err(std::io::Error::new( std::io::ErrorKind::UnexpectedEof, "unexpected EOF", )); } - let offset = match sub_data_kind { - VfsFileSubDataKind::Raw => file.offset, - VfsFileSubDataKind::ModuleGraph => file.module_graph_offset, - }; - let file_offset = self.fs_root.start_file_offset + offset; + let file_offset = + self.fs_root.start_file_offset + file_offset_and_len.offset; let start = file_offset + pos; - let end = file_offset + std::cmp::min(pos + len, file.len); + let end = file_offset + std::cmp::min(pos + len, file_offset_and_len.len); Ok(start as usize..end as usize) } @@ -995,7 +1368,9 @@ impl FileBackedVfs { #[cfg(test)] mod test { + use console_static_text::ansi::strip_ansi_codes; use std::io::Write; + use test_util::assert_contains; use test_util::TempDir; use super::*; @@ -1019,8 +1394,11 @@ mod test { // will canonicalize the root path let src_path = temp_dir.path().canonicalize().join("src"); src_path.create_dir_all(); + src_path.join("sub_dir").create_dir_all(); + src_path.join("e.txt").write("e"); + src_path.symlink_file("e.txt", "sub_dir/e.txt"); let src_path = src_path.to_path_buf(); - let mut builder = VfsBuilder::new(src_path.clone()).unwrap(); + let mut builder = VfsBuilder::new(); builder .add_file_with_data_inner( &src_path.join("a.txt"), @@ -1050,18 +1428,9 @@ mod test { VfsFileSubDataKind::Raw, ) .unwrap(); + builder.add_file_at_path(&src_path.join("e.txt")).unwrap(); builder - .add_file_with_data_inner( - &src_path.join("e.txt"), - "e".into(), - VfsFileSubDataKind::Raw, - ) - .unwrap(); - builder - .add_symlink( - &src_path.join("sub_dir").join("e.txt"), - &src_path.join("e.txt"), - ) + .add_symlink(&src_path.join("sub_dir").join("e.txt")) .unwrap(); // get the virtual fs @@ -1122,7 +1491,7 @@ mod test { // build and create the virtual fs let src_path = temp_dir_path.join("src").to_path_buf(); - let mut builder = VfsBuilder::new(src_path.clone()).unwrap(); + let mut builder = VfsBuilder::new(); builder.add_dir_recursive(&src_path).unwrap(); let (dest_path, virtual_fs) = into_virtual_fs(builder, &temp_dir); @@ -1160,10 +1529,10 @@ mod test { temp_dir: &TempDir, ) -> (PathBuf, FileBackedVfs) { let virtual_fs_file = temp_dir.path().join("virtual_fs"); - let (root_dir, files) = builder.into_dir_and_files(); + let vfs = builder.build(); { let mut file = std::fs::File::create(&virtual_fs_file).unwrap(); - for file_data in &files { + for file_data in &vfs.files { file.write_all(file_data).unwrap(); } } @@ -1174,7 +1543,7 @@ mod test { FileBackedVfs::new( Cow::Owned(data), VfsRoot { - dir: root_dir, + dir: vfs.root, root_path: dest_path.to_path_buf(), start_file_offset: 0, }, @@ -1187,41 +1556,22 @@ mod test { let temp_dir = TempDir::new(); let src_path = temp_dir.path().canonicalize().join("src"); src_path.create_dir_all(); + src_path.symlink_file("a.txt", "b.txt"); + src_path.symlink_file("b.txt", "c.txt"); + src_path.symlink_file("c.txt", "a.txt"); let src_path = src_path.to_path_buf(); - let mut builder = VfsBuilder::new(src_path.clone()).unwrap(); - builder - .add_symlink(&src_path.join("a.txt"), &src_path.join("b.txt")) - .unwrap(); - builder - .add_symlink(&src_path.join("b.txt"), &src_path.join("c.txt")) - .unwrap(); - builder - .add_symlink(&src_path.join("c.txt"), &src_path.join("a.txt")) - .unwrap(); - let (dest_path, virtual_fs) = into_virtual_fs(builder, &temp_dir); - assert_eq!( - virtual_fs - .file_entry(&dest_path.join("a.txt")) - .err() - .unwrap() - .to_string(), - "circular symlinks", - ); - assert_eq!( - virtual_fs.read_link(&dest_path.join("a.txt")).unwrap(), - dest_path.join("b.txt") - ); - assert_eq!( - virtual_fs.read_link(&dest_path.join("b.txt")).unwrap(), - dest_path.join("c.txt") - ); + let mut builder = VfsBuilder::new(); + let err = builder + .add_symlink(src_path.join("a.txt").as_path()) + .unwrap_err(); + assert_contains!(err.to_string(), "Circular symlink detected",); } #[tokio::test] async fn test_open_file() { let temp_dir = TempDir::new(); let temp_path = temp_dir.path().canonicalize(); - let mut builder = VfsBuilder::new(temp_path.to_path_buf()).unwrap(); + let mut builder = VfsBuilder::new(); builder .add_file_with_data_inner( temp_path.join("a.txt").as_path(), @@ -1281,4 +1631,47 @@ mod test { .unwrap(); assert_eq!(all_buf.to_vec(), b"123456789"); } + + #[test] + fn test_vfs_as_display_tree() { + let temp_dir = TempDir::new(); + temp_dir.write("root.txt", ""); + temp_dir.create_dir_all("a"); + temp_dir.write("a/a.txt", ""); + temp_dir.write("a/b.txt", ""); + temp_dir.create_dir_all("b"); + temp_dir.write("b/a.txt", ""); + temp_dir.write("b/b.txt", ""); + temp_dir.create_dir_all("c"); + temp_dir.write("c/a.txt", "contents"); + temp_dir.symlink_file("c/a.txt", "c/b.txt"); + assert_eq!(temp_dir.read_to_string("c/b.txt"), "contents"); // ensure the symlink works + let mut vfs_builder = VfsBuilder::new(); + // full dir + vfs_builder + .add_dir_recursive(temp_dir.path().join("a").as_path()) + .unwrap(); + // part of the dir + vfs_builder + .add_file_at_path(temp_dir.path().join("b/a.txt").as_path()) + .unwrap(); + // symlink + vfs_builder + .add_dir_recursive(temp_dir.path().join("c").as_path()) + .unwrap(); + temp_dir.write("c/c.txt", ""); // write an extra file so it shows the whole directory + let node = vfs_as_display_tree(&vfs_builder.build(), "executable"); + let mut text = String::new(); + node.print(&mut text).unwrap(); + assert_eq!( + strip_ansi_codes(&text), + r#"executable +├── a/* +├── b/a.txt +└─┬ c + ├── a.txt + └── b.txt --> c/a.txt +"# + ); + } } diff --git a/cli/task_runner.rs b/cli/task_runner.rs index aabdaf5777..d6589a1832 100644 --- a/cli/task_runner.rs +++ b/cli/task_runner.rs @@ -585,7 +585,13 @@ pub async fn run_future_forwarding_signals( async fn listen_ctrl_c(kill_signal: KillSignal) { while let Ok(()) = tokio::signal::ctrl_c().await { - kill_signal.send(deno_task_shell::SignalKind::SIGINT) + // On windows, ctrl+c is sent to the process group, so the signal would + // have already been sent to the child process. We still want to listen + // for ctrl+c here to keep the process alive when receiving it, but no + // need to forward the signal because it's already been sent. + if !cfg!(windows) { + kill_signal.send(deno_task_shell::SignalKind::SIGINT) + } } } diff --git a/cli/tools/bench/mod.rs b/cli/tools/bench/mod.rs index 1d49fa061d..5983590531 100644 --- a/cli/tools/bench/mod.rs +++ b/cli/tools/bench/mod.rs @@ -538,7 +538,11 @@ pub async fn run_benchmarks_with_watch( )?; let graph = module_graph_creator - .create_graph(graph_kind, collected_bench_modules.clone()) + .create_graph( + graph_kind, + collected_bench_modules.clone(), + crate::graph_util::NpmCachingStrategy::Eager, + ) .await?; module_graph_creator.graph_valid(&graph)?; let bench_modules = &graph.roots; diff --git a/cli/tools/compile.rs b/cli/tools/compile.rs index 4fa9963683..7a463a7b09 100644 --- a/cli/tools/compile.rs +++ b/cli/tools/compile.rs @@ -5,7 +5,7 @@ use crate::args::CompileFlags; use crate::args::Flags; use crate::factory::CliFactory; use crate::http_util::HttpClientProvider; -use crate::standalone::binary::StandaloneRelativeFileBaseUrl; +use crate::standalone::binary::WriteBinOptions; use crate::standalone::is_standalone_binary; use deno_ast::MediaType; use deno_ast::ModuleSpecifier; @@ -15,8 +15,12 @@ use deno_core::error::generic_error; use deno_core::error::AnyError; use deno_core::resolve_url_or_path; use deno_graph::GraphKind; +use deno_path_util::url_from_file_path; +use deno_path_util::url_to_file_path; use deno_terminal::colors; use rand::Rng; +use std::collections::HashSet; +use std::collections::VecDeque; use std::path::Path; use std::path::PathBuf; use std::sync::Arc; @@ -69,7 +73,11 @@ pub async fn compile( // create a module graph with types information in it. We don't want to // store that in the binary so create a code only module graph from scratch. module_graph_creator - .create_graph(GraphKind::CodeOnly, module_roots) + .create_graph( + GraphKind::CodeOnly, + module_roots, + crate::graph_util::NpmCachingStrategy::Eager, + ) .await? } else { graph @@ -78,20 +86,6 @@ pub async fn compile( let ts_config_for_emit = cli_options .resolve_ts_config_for_emit(deno_config::deno_json::TsConfigType::Emit)?; check_warn_tsconfig(&ts_config_for_emit); - let root_dir_url = resolve_root_dir_from_specifiers( - cli_options.workspace().root_dir(), - graph - .specifiers() - .map(|(s, _)| s) - .chain( - cli_options - .node_modules_dir_path() - .and_then(|p| ModuleSpecifier::from_directory_path(p).ok()) - .iter(), - ) - .chain(include_files.iter()), - ); - log::debug!("Binary root dir: {}", root_dir_url); log::info!( "{} {} to {}", colors::green("Compile"), @@ -116,14 +110,17 @@ pub async fn compile( })?; let write_result = binary_writer - .write_bin( - file, - &graph, - StandaloneRelativeFileBaseUrl::from(&root_dir_url), + .write_bin(WriteBinOptions { + writer: file, + display_output_filename: &output_path + .file_name() + .unwrap() + .to_string_lossy(), + graph: &graph, entrypoint, - &include_files, - &compile_flags, - ) + include_files: &include_files, + compile_flags: &compile_flags, + }) .await .with_context(|| { format!( @@ -242,15 +239,58 @@ fn get_module_roots_and_include_files( } } - let mut module_roots = Vec::with_capacity(compile_flags.include.len() + 1); - let mut include_files = Vec::with_capacity(compile_flags.include.len()); + fn analyze_path( + url: &ModuleSpecifier, + module_roots: &mut Vec, + include_files: &mut Vec, + searched_paths: &mut HashSet, + ) -> Result<(), AnyError> { + let Ok(path) = url_to_file_path(url) else { + return Ok(()); + }; + let mut pending = VecDeque::from([path]); + while let Some(path) = pending.pop_front() { + if !searched_paths.insert(path.clone()) { + continue; + } + if !path.is_dir() { + let url = url_from_file_path(&path)?; + include_files.push(url.clone()); + if is_module_graph_module(&url) { + module_roots.push(url); + } + continue; + } + for entry in std::fs::read_dir(&path).with_context(|| { + format!("Failed reading directory '{}'", path.display()) + })? { + let entry = entry.with_context(|| { + format!("Failed reading entry in directory '{}'", path.display()) + })?; + pending.push_back(entry.path()); + } + } + Ok(()) + } + + let mut searched_paths = HashSet::new(); + let mut module_roots = Vec::new(); + let mut include_files = Vec::new(); module_roots.push(entrypoint.clone()); for side_module in &compile_flags.include { let url = resolve_url_or_path(side_module, initial_cwd)?; if is_module_graph_module(&url) { - module_roots.push(url); + module_roots.push(url.clone()); + if url.scheme() == "file" { + include_files.push(url); + } } else { - include_files.push(url); + analyze_path( + &url, + &mut module_roots, + &mut include_files, + &mut searched_paths, + )?; } } Ok((module_roots, include_files)) @@ -316,68 +356,6 @@ fn get_os_specific_filepath( } } -fn resolve_root_dir_from_specifiers<'a>( - starting_dir: &ModuleSpecifier, - specifiers: impl Iterator, -) -> ModuleSpecifier { - fn select_common_root<'a>(a: &'a str, b: &'a str) -> &'a str { - let min_length = a.len().min(b.len()); - - let mut last_slash = 0; - for i in 0..min_length { - if a.as_bytes()[i] == b.as_bytes()[i] && a.as_bytes()[i] == b'/' { - last_slash = i; - } else if a.as_bytes()[i] != b.as_bytes()[i] { - break; - } - } - - // Return the common root path up to the last common slash. - // This returns a slice of the original string 'a', up to and including the last matching '/'. - let common = &a[..=last_slash]; - if cfg!(windows) && common == "file:///" { - a - } else { - common - } - } - - fn is_file_system_root(url: &str) -> bool { - let Some(path) = url.strip_prefix("file:///") else { - return false; - }; - if cfg!(windows) { - let Some((_drive, path)) = path.split_once('/') else { - return true; - }; - path.is_empty() - } else { - path.is_empty() - } - } - - let mut found_dir = starting_dir.as_str(); - if !is_file_system_root(found_dir) { - for specifier in specifiers { - if specifier.scheme() == "file" { - found_dir = select_common_root(found_dir, specifier.as_str()); - } - } - } - let found_dir = if is_file_system_root(found_dir) { - found_dir - } else { - // include the parent dir name because it helps create some context - found_dir - .strip_suffix('/') - .unwrap_or(found_dir) - .rfind('/') - .map(|i| &found_dir[..i + 1]) - .unwrap_or(found_dir) - }; - ModuleSpecifier::parse(found_dir).unwrap() -} - #[cfg(test)] mod test { pub use super::*; @@ -454,38 +432,4 @@ mod test { run_test("C:\\my-exe.0.1.2", Some("windows"), "C:\\my-exe.0.1.2.exe"); run_test("my-exe-0.1.2", Some("linux"), "my-exe-0.1.2"); } - - #[test] - fn test_resolve_root_dir_from_specifiers() { - fn resolve(start: &str, specifiers: &[&str]) -> String { - let specifiers = specifiers - .iter() - .map(|s| ModuleSpecifier::parse(s).unwrap()) - .collect::>(); - resolve_root_dir_from_specifiers( - &ModuleSpecifier::parse(start).unwrap(), - specifiers.iter(), - ) - .to_string() - } - - assert_eq!(resolve("file:///a/b/c", &["file:///a/b/c/d"]), "file:///a/"); - assert_eq!( - resolve("file:///a/b/c/", &["file:///a/b/c/d"]), - "file:///a/b/" - ); - assert_eq!( - resolve("file:///a/b/c/", &["file:///a/b/c/d", "file:///a/b/c/e"]), - "file:///a/b/" - ); - assert_eq!(resolve("file:///", &["file:///a/b/c/d"]), "file:///"); - if cfg!(windows) { - assert_eq!(resolve("file:///c:/", &["file:///c:/test"]), "file:///c:/"); - // this will ignore the other one because it's on a separate drive - assert_eq!( - resolve("file:///c:/a/b/c/", &["file:///v:/a/b/c/d"]), - "file:///c:/a/b/" - ); - } - } } diff --git a/cli/tools/doc.rs b/cli/tools/doc.rs index 9a24e458ac..647a36dc48 100644 --- a/cli/tools/doc.rs +++ b/cli/tools/doc.rs @@ -131,7 +131,11 @@ pub async fn doc( |_| true, )?; let graph = module_graph_creator - .create_graph(GraphKind::TypesOnly, module_specifiers.clone()) + .create_graph( + GraphKind::TypesOnly, + module_specifiers.clone(), + crate::graph_util::NpmCachingStrategy::Eager, + ) .await?; graph_exit_integrity_errors(&graph); diff --git a/cli/tools/info.rs b/cli/tools/info.rs index f0cd37772d..7a35f597c3 100644 --- a/cli/tools/info.rs +++ b/cli/tools/info.rs @@ -2,7 +2,6 @@ use std::collections::HashMap; use std::collections::HashSet; -use std::fmt; use std::fmt::Write; use std::sync::Arc; @@ -35,6 +34,7 @@ use crate::graph_util::graph_exit_integrity_errors; use crate::npm::CliNpmResolver; use crate::npm::ManagedCliNpmResolver; use crate::util::checksum; +use crate::util::display::DisplayTreeNode; const JSON_SCHEMA_VERSION: u8 = 1; @@ -123,7 +123,12 @@ pub async fn info( let mut loader = module_graph_builder.create_graph_loader(); loader.enable_loading_cache_info(); // for displaying the cache information let graph = module_graph_creator - .create_graph_with_loader(GraphKind::All, vec![specifier], &mut loader) + .create_graph_with_loader( + GraphKind::All, + vec![specifier], + &mut loader, + crate::graph_util::NpmCachingStrategy::Eager, + ) .await?; // write out the lockfile if there is one @@ -337,76 +342,6 @@ fn add_npm_packages_to_json( json.insert("npmPackages".to_string(), json_packages.into()); } -struct TreeNode { - text: String, - children: Vec, -} - -impl TreeNode { - pub fn from_text(text: String) -> Self { - Self { - text, - children: Default::default(), - } - } -} - -fn print_tree_node( - tree_node: &TreeNode, - writer: &mut TWrite, -) -> fmt::Result { - fn print_children( - writer: &mut TWrite, - prefix: &str, - children: &[TreeNode], - ) -> fmt::Result { - const SIBLING_CONNECTOR: char = '├'; - const LAST_SIBLING_CONNECTOR: char = '└'; - const CHILD_DEPS_CONNECTOR: char = '┬'; - const CHILD_NO_DEPS_CONNECTOR: char = '─'; - const VERTICAL_CONNECTOR: char = '│'; - const EMPTY_CONNECTOR: char = ' '; - - let child_len = children.len(); - for (index, child) in children.iter().enumerate() { - let is_last = index + 1 == child_len; - let sibling_connector = if is_last { - LAST_SIBLING_CONNECTOR - } else { - SIBLING_CONNECTOR - }; - let child_connector = if child.children.is_empty() { - CHILD_NO_DEPS_CONNECTOR - } else { - CHILD_DEPS_CONNECTOR - }; - writeln!( - writer, - "{} {}", - colors::gray(format!("{prefix}{sibling_connector}─{child_connector}")), - child.text - )?; - let child_prefix = format!( - "{}{}{}", - prefix, - if is_last { - EMPTY_CONNECTOR - } else { - VERTICAL_CONNECTOR - }, - EMPTY_CONNECTOR - ); - print_children(writer, &child_prefix, &child.children)?; - } - - Ok(()) - } - - writeln!(writer, "{}", tree_node.text)?; - print_children(writer, "", &tree_node.children)?; - Ok(()) -} - /// Precached information about npm packages that are used in deno info. #[derive(Default)] struct NpmInfo { @@ -563,7 +498,7 @@ impl<'a> GraphDisplayContext<'a> { )?; writeln!(writer)?; let root_node = self.build_module_info(root, false); - print_tree_node(&root_node, writer)?; + root_node.print(writer)?; Ok(()) } Err(err) => { @@ -579,7 +514,7 @@ impl<'a> GraphDisplayContext<'a> { } } - fn build_dep_info(&mut self, dep: &Dependency) -> Vec { + fn build_dep_info(&mut self, dep: &Dependency) -> Vec { let mut children = Vec::with_capacity(2); if !dep.maybe_code.is_none() { if let Some(child) = self.build_resolved_info(&dep.maybe_code, false) { @@ -594,7 +529,11 @@ impl<'a> GraphDisplayContext<'a> { children } - fn build_module_info(&mut self, module: &Module, type_dep: bool) -> TreeNode { + fn build_module_info( + &mut self, + module: &Module, + type_dep: bool, + ) -> DisplayTreeNode { enum PackageOrSpecifier { Package(Box), Specifier(ModuleSpecifier), @@ -640,7 +579,7 @@ impl<'a> GraphDisplayContext<'a> { format!("{} {}", header_text, maybe_size_to_text(maybe_size)) }; - let mut tree_node = TreeNode::from_text(header_text); + let mut tree_node = DisplayTreeNode::from_text(header_text); if !was_seen { match &package_or_specifier { @@ -678,14 +617,14 @@ impl<'a> GraphDisplayContext<'a> { fn build_npm_deps( &mut self, package: &NpmResolutionPackage, - ) -> Vec { + ) -> Vec { let mut deps = package.dependencies.values().collect::>(); deps.sort(); let mut children = Vec::with_capacity(deps.len()); for dep_id in deps.into_iter() { let maybe_size = self.npm_info.package_sizes.get(dep_id).cloned(); let size_str = maybe_size_to_text(maybe_size); - let mut child = TreeNode::from_text(format!( + let mut child = DisplayTreeNode::from_text(format!( "npm:/{} {}", dep_id.as_serialized(), size_str @@ -710,7 +649,7 @@ impl<'a> GraphDisplayContext<'a> { &mut self, err: &ModuleError, specifier: &ModuleSpecifier, - ) -> TreeNode { + ) -> DisplayTreeNode { self.seen.insert(specifier.to_string()); match err { ModuleError::InvalidTypeAssertion { .. } => { @@ -753,8 +692,8 @@ impl<'a> GraphDisplayContext<'a> { &self, specifier: &ModuleSpecifier, error_msg: &str, - ) -> TreeNode { - TreeNode::from_text(format!( + ) -> DisplayTreeNode { + DisplayTreeNode::from_text(format!( "{} {}", colors::red(specifier), colors::red_bold(error_msg) @@ -765,7 +704,7 @@ impl<'a> GraphDisplayContext<'a> { &mut self, resolution: &Resolution, type_dep: bool, - ) -> Option { + ) -> Option { match resolution { Resolution::Ok(resolved) => { let specifier = &resolved.specifier; @@ -773,14 +712,14 @@ impl<'a> GraphDisplayContext<'a> { Some(match self.graph.try_get(resolved_specifier) { Ok(Some(module)) => self.build_module_info(module, type_dep), Err(err) => self.build_error_info(err, resolved_specifier), - Ok(None) => TreeNode::from_text(format!( + Ok(None) => DisplayTreeNode::from_text(format!( "{} {}", colors::red(specifier), colors::red_bold("(missing)") )), }) } - Resolution::Err(err) => Some(TreeNode::from_text(format!( + Resolution::Err(err) => Some(DisplayTreeNode::from_text(format!( "{} {}", colors::italic(err.to_string()), colors::red_bold("(resolve error)") diff --git a/cli/tools/installer.rs b/cli/tools/installer.rs index df5981e6e7..d7c484beba 100644 --- a/cli/tools/installer.rs +++ b/cli/tools/installer.rs @@ -9,7 +9,6 @@ use crate::args::Flags; use crate::args::InstallFlags; use crate::args::InstallFlagsGlobal; use crate::args::InstallFlagsLocal; -use crate::args::InstallKind; use crate::args::TypeCheckMode; use crate::args::UninstallFlags; use crate::args::UninstallKind; @@ -339,11 +338,11 @@ pub async fn install_command( flags: Arc, install_flags: InstallFlags, ) -> Result<(), AnyError> { - match install_flags.kind { - InstallKind::Global(global_flags) => { + match install_flags { + InstallFlags::Global(global_flags) => { install_global(flags, global_flags).await } - InstallKind::Local(local_flags) => { + InstallFlags::Local(local_flags) => { if let InstallFlagsLocal::Add(add_flags) = &local_flags { check_if_installs_a_single_package_globally(Some(add_flags))?; } diff --git a/cli/tools/lint/mod.rs b/cli/tools/lint/mod.rs index fcefb45874..596359bdc0 100644 --- a/cli/tools/lint/mod.rs +++ b/cli/tools/lint/mod.rs @@ -556,3 +556,68 @@ struct LintError { file_path: String, message: String, } + +#[cfg(test)] +mod tests { + use super::*; + use pretty_assertions::assert_eq; + use serde::Deserialize; + use test_util as util; + + #[derive(Serialize, Deserialize)] + struct RulesSchema { + #[serde(rename = "$schema")] + schema: String, + + #[serde(rename = "enum")] + rules: Vec, + } + + fn get_all_rules() -> Vec { + let rule_provider = LintRuleProvider::new(None, None); + let configured_rules = + rule_provider.resolve_lint_rules(Default::default(), None); + let mut all_rules = configured_rules + .all_rule_codes + .into_iter() + .map(|s| s.to_string()) + .collect::>(); + all_rules.sort(); + + all_rules + } + + // TODO(bartlomieju): do the same for tags, once https://github.com/denoland/deno/pull/27162 lands + #[test] + fn all_lint_rules_are_listed_in_schema_file() { + let all_rules = get_all_rules(); + + let rules_schema_path = + util::root_path().join("cli/schemas/lint-rules.v1.json"); + let rules_schema_file = + std::fs::read_to_string(&rules_schema_path).unwrap(); + + let schema: RulesSchema = serde_json::from_str(&rules_schema_file).unwrap(); + + const UPDATE_ENV_VAR_NAME: &str = "UPDATE_EXPECTED"; + + if std::env::var(UPDATE_ENV_VAR_NAME).ok().is_none() { + assert_eq!( + schema.rules, all_rules, + "Lint rules schema file not up to date. Run again with {}=1 to update the expected output", + UPDATE_ENV_VAR_NAME + ); + return; + } + + std::fs::write( + &rules_schema_path, + serde_json::to_string_pretty(&RulesSchema { + schema: schema.schema, + rules: all_rules, + }) + .unwrap(), + ) + .unwrap(); + } +} diff --git a/cli/tools/registry/pm.rs b/cli/tools/registry/pm.rs index 5718cd3ec1..6f89ec7aae 100644 --- a/cli/tools/registry/pm.rs +++ b/cli/tools/registry/pm.rs @@ -432,9 +432,8 @@ pub async fn add( let mut package_reqs = Vec::with_capacity(add_flags.packages.len()); for entry_text in add_flags.packages.iter() { - let req = AddRmPackageReq::parse(entry_text).with_context(|| { - format!("Failed to parse package required: {}", entry_text) - })?; + let req = AddRmPackageReq::parse(entry_text) + .with_context(|| format!("Failed to parse package: {}", entry_text))?; match req { Ok(add_req) => package_reqs.push(add_req), @@ -805,9 +804,8 @@ pub async fn remove( let mut removed_packages = vec![]; for package in &remove_flags.packages { - let req = AddRmPackageReq::parse(package).with_context(|| { - format!("Failed to parse package required: {}", package) - })?; + let req = AddRmPackageReq::parse(package) + .with_context(|| format!("Failed to parse package: {}", package))?; let mut parsed_pkg_name = None; for config in configs.iter_mut().flatten() { match &req { diff --git a/cli/tools/registry/pm/cache_deps.rs b/cli/tools/registry/pm/cache_deps.rs index dbec8a3b3f..814c76cb27 100644 --- a/cli/tools/registry/pm/cache_deps.rs +++ b/cli/tools/registry/pm/cache_deps.rs @@ -6,6 +6,7 @@ use std::sync::Arc; use crate::factory::CliFactory; use crate::graph_container::ModuleGraphContainer; use crate::graph_container::ModuleGraphUpdatePermit; +use crate::graph_util::CreateGraphOptions; use deno_core::error::AnyError; use deno_core::futures::stream::FuturesUnordered; use deno_core::futures::StreamExt; @@ -18,18 +19,16 @@ pub async fn cache_top_level_deps( ) -> Result<(), AnyError> { let npm_resolver = factory.npm_resolver().await?; let cli_options = factory.cli_options()?; - let root_permissions = factory.root_permissions_container()?; if let Some(npm_resolver) = npm_resolver.as_managed() { - if !npm_resolver.ensure_top_level_package_json_install().await? { - if let Some(lockfile) = cli_options.maybe_lockfile() { - lockfile.error_if_changed()?; - } - - npm_resolver.cache_packages().await?; + npm_resolver.ensure_top_level_package_json_install().await?; + if let Some(lockfile) = cli_options.maybe_lockfile() { + lockfile.error_if_changed()?; } } // cache as many entries in the import map as we can let resolver = factory.workspace_resolver().await?; + + let mut maybe_graph_error = Ok(()); if let Some(import_map) = resolver.maybe_import_map() { let jsr_resolver = if let Some(resolver) = jsr_resolver { resolver @@ -122,19 +121,29 @@ pub async fn cache_top_level_deps( } drop(info_futures); - factory - .module_load_preparer() - .await? - .prepare_module_load( + let graph_builder = factory.module_graph_builder().await?; + graph_builder + .build_graph_with_npm_resolution( graph, - &roots, - false, - deno_config::deno_json::TsTypeLib::DenoWorker, - root_permissions.clone(), - None, + CreateGraphOptions { + loader: None, + graph_kind: graph.graph_kind(), + is_dynamic: false, + roots: roots.clone(), + npm_caching: crate::graph_util::NpmCachingStrategy::Manual, + }, ) .await?; + maybe_graph_error = graph_builder.graph_roots_valid(graph, &roots); + } + + if let Some(npm_resolver) = npm_resolver.as_managed() { + npm_resolver + .cache_packages(crate::npm::PackageCaching::All) + .await?; } + maybe_graph_error?; + Ok(()) } diff --git a/cli/tools/registry/pm/deps.rs b/cli/tools/registry/pm/deps.rs index d82e9954cd..e4c38276f7 100644 --- a/cli/tools/registry/pm/deps.rs +++ b/cli/tools/registry/pm/deps.rs @@ -2,6 +2,7 @@ use std::borrow::Cow; use std::collections::HashMap; +use std::path::PathBuf; use std::sync::atomic::AtomicBool; use std::sync::Arc; @@ -11,6 +12,7 @@ use deno_config::deno_json::ConfigFileRc; use deno_config::workspace::Workspace; use deno_config::workspace::WorkspaceDirectory; use deno_core::anyhow::bail; +use deno_core::anyhow::Context; use deno_core::error::AnyError; use deno_core::futures::future::try_join; use deno_core::futures::stream::FuturesOrdered; @@ -43,10 +45,10 @@ use crate::npm::NpmFetchResolver; use super::ConfigUpdater; -#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq)] pub enum ImportMapKind { Inline, - Outline, + Outline(PathBuf), } #[derive(Clone)] @@ -62,9 +64,12 @@ impl DepLocation { pub fn file_path(&self) -> Cow { match self { - DepLocation::DenoJson(arc, _, _) => { - Cow::Owned(arc.specifier.to_file_path().unwrap()) - } + DepLocation::DenoJson(arc, _, kind) => match kind { + ImportMapKind::Inline => { + Cow::Owned(arc.specifier.to_file_path().unwrap()) + } + ImportMapKind::Outline(path) => Cow::Borrowed(path.as_path()), + }, DepLocation::PackageJson(arc, _) => Cow::Borrowed(arc.path.as_ref()), } } @@ -238,22 +243,30 @@ fn to_import_map_value_from_imports( fn deno_json_import_map( deno_json: &ConfigFile, ) -> Result, AnyError> { - let (value, kind) = - if deno_json.json.imports.is_some() || deno_json.json.scopes.is_some() { - ( - to_import_map_value_from_imports(deno_json), - ImportMapKind::Inline, - ) - } else { - match deno_json.to_import_map_path()? { - Some(path) => { - let text = std::fs::read_to_string(&path)?; - let value = serde_json::from_str(&text)?; - (value, ImportMapKind::Outline) - } - None => return Ok(None), + let (value, kind) = if deno_json.json.imports.is_some() + || deno_json.json.scopes.is_some() + { + ( + to_import_map_value_from_imports(deno_json), + ImportMapKind::Inline, + ) + } else { + match deno_json.to_import_map_path()? { + Some(path) => { + let err_context = || { + format!( + "loading import map at '{}' (from \"importMap\" field in '{}')", + path.display(), + deno_json.specifier + ) + }; + let text = std::fs::read_to_string(&path).with_context(err_context)?; + let value = serde_json::from_str(&text).with_context(err_context)?; + (value, ImportMapKind::Outline(path)) } - }; + None => return Ok(None), + } + }; import_map::parse_from_value(deno_json.specifier.clone(), value) .map_err(Into::into) @@ -303,7 +316,7 @@ fn add_deps_from_deno_json( location: DepLocation::DenoJson( deno_json.clone(), key_path, - import_map_kind, + import_map_kind.clone(), ), kind, req, @@ -747,11 +760,7 @@ impl DepManager { let dep = &mut self.deps[dep_id.0]; dep.req.version_req = version_req.clone(); match &dep.location { - DepLocation::DenoJson(arc, key_path, import_map_kind) => { - if matches!(import_map_kind, ImportMapKind::Outline) { - // not supported - continue; - } + DepLocation::DenoJson(arc, key_path, _) => { let updater = get_or_create_updater(&mut config_updaters, &dep.location)?; diff --git a/cli/tools/registry/pm/outdated.rs b/cli/tools/registry/pm/outdated.rs index 2a29014267..aef65a5de0 100644 --- a/cli/tools/registry/pm/outdated.rs +++ b/cli/tools/registry/pm/outdated.rs @@ -3,6 +3,7 @@ use std::collections::HashSet; use std::sync::Arc; +use deno_core::anyhow::bail; use deno_core::error::AnyError; use deno_semver::package::PackageNv; use deno_semver::package::PackageReq; @@ -100,6 +101,23 @@ fn print_outdated_table(packages: &[OutdatedPackage]) { println!("└{package_fill}┴{current_fill}┴{update_fill}┴{latest_fill}┘",); } +fn print_suggestion(compatible: bool) { + log::info!(""); + let (cmd, txt) = if compatible { + ("", "compatible") + } else { + (" --latest", "available") + }; + log::info!( + "{}", + color_print::cformat!( + "Run deno outdated --update{} to update to the latest {} versions,\nor deno outdated --help for more information.", + cmd, + txt, + ) + ); +} + fn print_outdated( deps: &mut DepManager, compatible: bool, @@ -148,6 +166,7 @@ fn print_outdated( if !outdated.is_empty() { outdated.sort(); print_outdated_table(&outdated); + print_suggestion(compatible); } Ok(()) @@ -179,6 +198,15 @@ pub async fn outdated( let jsr_fetch_resolver = Arc::new(JsrFetchResolver::new(file_fetcher.clone())); + if !cli_options.start_dir.has_deno_json() + && !cli_options.start_dir.has_pkg_json() + { + bail!( + "No deno.json or package.json in \"{}\".", + cli_options.initial_cwd().display(), + ); + } + let args = dep_manager_args( &factory, cli_options, diff --git a/cli/tools/repl/session.rs b/cli/tools/repl/session.rs index 26e1eeac2f..02594f1519 100644 --- a/cli/tools/repl/session.rs +++ b/cli/tools/repl/session.rs @@ -727,7 +727,9 @@ impl ReplSession { let has_node_specifier = resolved_imports.iter().any(|url| url.scheme() == "node"); if !npm_imports.is_empty() || has_node_specifier { - npm_resolver.add_package_reqs(&npm_imports).await?; + npm_resolver + .add_and_cache_package_reqs(&npm_imports) + .await?; // prevent messages in the repl about @types/node not being cached if has_node_specifier { diff --git a/cli/tools/run/mod.rs b/cli/tools/run/mod.rs index 8fab544eca..d3f7b093d4 100644 --- a/cli/tools/run/mod.rs +++ b/cli/tools/run/mod.rs @@ -198,13 +198,23 @@ pub async fn eval_command( } pub async fn maybe_npm_install(factory: &CliFactory) -> Result<(), AnyError> { + let cli_options = factory.cli_options()?; // ensure an "npm install" is done if the user has explicitly // opted into using a managed node_modules directory - if factory.cli_options()?.node_modules_dir()? - == Some(NodeModulesDirMode::Auto) - { + if cli_options.node_modules_dir()? == Some(NodeModulesDirMode::Auto) { if let Some(npm_resolver) = factory.npm_resolver().await?.as_managed() { - npm_resolver.ensure_top_level_package_json_install().await?; + let already_done = + npm_resolver.ensure_top_level_package_json_install().await?; + if !already_done + && matches!( + cli_options.default_npm_caching_strategy(), + crate::graph_util::NpmCachingStrategy::Eager + ) + { + npm_resolver + .cache_packages(crate::npm::PackageCaching::All) + .await?; + } } } Ok(()) diff --git a/cli/tools/task.rs b/cli/tools/task.rs index fc1410aa0e..25d1d66710 100644 --- a/cli/tools/task.rs +++ b/cli/tools/task.rs @@ -440,6 +440,13 @@ impl<'a> TaskRunner<'a> { kill_signal: KillSignal, argv: &'a [String], ) -> Result { + if let Some(npm_resolver) = self.npm_resolver.as_managed() { + npm_resolver.ensure_top_level_package_json_install().await?; + npm_resolver + .cache_packages(crate::npm::PackageCaching::All) + .await?; + } + let cwd = match &self.task_flags.cwd { Some(path) => canonicalize_path(&PathBuf::from(path)) .context("failed canonicalizing --cwd")?, @@ -450,6 +457,7 @@ impl<'a> TaskRunner<'a> { self.npm_resolver, self.node_resolver, )?; + self .run_single(RunSingleOptions { task_name, @@ -473,6 +481,9 @@ impl<'a> TaskRunner<'a> { // ensure the npm packages are installed if using a managed resolver if let Some(npm_resolver) = self.npm_resolver.as_managed() { npm_resolver.ensure_top_level_package_json_install().await?; + npm_resolver + .cache_packages(crate::npm::PackageCaching::All) + .await?; } let cwd = match &self.task_flags.cwd { @@ -492,6 +503,7 @@ impl<'a> TaskRunner<'a> { self.npm_resolver, self.node_resolver, )?; + for task_name in &task_names { if let Some(script) = scripts.get(task_name) { let exit_code = self diff --git a/cli/tools/test/mod.rs b/cli/tools/test/mod.rs index 6357ebcae2..2e46bdd4da 100644 --- a/cli/tools/test/mod.rs +++ b/cli/tools/test/mod.rs @@ -1716,7 +1716,11 @@ pub async fn run_tests_with_watch( &cli_options.permissions_options(), )?; let graph = module_graph_creator - .create_graph(graph_kind, test_modules) + .create_graph( + graph_kind, + test_modules, + crate::graph_util::NpmCachingStrategy::Eager, + ) .await?; module_graph_creator.graph_valid(&graph)?; let test_modules = &graph.roots; diff --git a/cli/util/display.rs b/cli/util/display.rs index d18b045d8d..8795d3db68 100644 --- a/cli/util/display.rs +++ b/cli/util/display.rs @@ -2,6 +2,7 @@ use deno_core::error::AnyError; use deno_core::serde_json; +use deno_runtime::colors; use std::io::Write; /// A function that converts a float to a string the represents a human @@ -85,6 +86,78 @@ where Ok(()) } +pub struct DisplayTreeNode { + pub text: String, + pub children: Vec, +} + +impl DisplayTreeNode { + pub fn from_text(text: String) -> Self { + Self { + text, + children: Default::default(), + } + } + + pub fn print( + &self, + writer: &mut TWrite, + ) -> std::fmt::Result { + fn print_children( + writer: &mut TWrite, + prefix: &str, + children: &[DisplayTreeNode], + ) -> std::fmt::Result { + const SIBLING_CONNECTOR: char = '├'; + const LAST_SIBLING_CONNECTOR: char = '└'; + const CHILD_DEPS_CONNECTOR: char = '┬'; + const CHILD_NO_DEPS_CONNECTOR: char = '─'; + const VERTICAL_CONNECTOR: char = '│'; + const EMPTY_CONNECTOR: char = ' '; + + let child_len = children.len(); + for (index, child) in children.iter().enumerate() { + let is_last = index + 1 == child_len; + let sibling_connector = if is_last { + LAST_SIBLING_CONNECTOR + } else { + SIBLING_CONNECTOR + }; + let child_connector = if child.children.is_empty() { + CHILD_NO_DEPS_CONNECTOR + } else { + CHILD_DEPS_CONNECTOR + }; + writeln!( + writer, + "{} {}", + colors::gray(format!( + "{prefix}{sibling_connector}─{child_connector}" + )), + child.text + )?; + let child_prefix = format!( + "{}{}{}", + prefix, + if is_last { + EMPTY_CONNECTOR + } else { + VERTICAL_CONNECTOR + }, + EMPTY_CONNECTOR + ); + print_children(writer, &child_prefix, &child.children)?; + } + + Ok(()) + } + + writeln!(writer, "{}", self.text)?; + print_children(writer, "", &self.children)?; + Ok(()) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/cli/util/logger.rs b/cli/util/logger.rs index 2b8987c3e7..783f8a5f68 100644 --- a/cli/util/logger.rs +++ b/cli/util/logger.rs @@ -1,24 +1,34 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +use super::draw_thread::DrawThread; +use deno_telemetry::OtelConfig; +use deno_telemetry::OtelConsoleConfig; use std::io::Write; -use super::draw_thread::DrawThread; - -struct CliLogger(env_logger::Logger); +struct CliLogger { + otel_console_config: OtelConsoleConfig, + logger: env_logger::Logger, +} impl CliLogger { - pub fn new(logger: env_logger::Logger) -> Self { - Self(logger) + pub fn new( + logger: env_logger::Logger, + otel_console_config: OtelConsoleConfig, + ) -> Self { + Self { + logger, + otel_console_config, + } } pub fn filter(&self) -> log::LevelFilter { - self.0.filter() + self.logger.filter() } } impl log::Log for CliLogger { fn enabled(&self, metadata: &log::Metadata) -> bool { - self.0.enabled(metadata) + self.logger.enabled(metadata) } fn log(&self, record: &log::Record) { @@ -28,18 +38,30 @@ impl log::Log for CliLogger { // could potentially block other threads that access the draw // thread's state DrawThread::hide(); - self.0.log(record); - deno_telemetry::handle_log(record); + + match self.otel_console_config { + OtelConsoleConfig::Ignore => { + self.logger.log(record); + } + OtelConsoleConfig::Capture => { + self.logger.log(record); + deno_telemetry::handle_log(record); + } + OtelConsoleConfig::Replace => { + deno_telemetry::handle_log(record); + } + } + DrawThread::show(); } } fn flush(&self) { - self.0.flush(); + self.logger.flush(); } } -pub fn init(maybe_level: Option) { +pub fn init(maybe_level: Option, otel_config: Option) { let log_level = maybe_level.unwrap_or(log::Level::Info); let logger = env_logger::Builder::from_env( env_logger::Env::new() @@ -93,7 +115,12 @@ pub fn init(maybe_level: Option) { }) .build(); - let cli_logger = CliLogger::new(logger); + let cli_logger = CliLogger::new( + logger, + otel_config + .map(|c| c.console) + .unwrap_or(OtelConsoleConfig::Ignore), + ); let max_level = cli_logger.filter(); let r = log::set_boxed_logger(Box::new(cli_logger)); if r.is_ok() { diff --git a/cli/util/path.rs b/cli/util/path.rs index df66b83766..de72843406 100644 --- a/cli/util/path.rs +++ b/cli/util/path.rs @@ -1,6 +1,7 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. use std::borrow::Cow; +use std::fmt::Write; use std::path::Path; use std::path::PathBuf; @@ -58,8 +59,8 @@ pub fn get_atomic_file_path(file_path: &Path) -> PathBuf { } fn gen_rand_path_component() -> String { - (0..4).fold(String::new(), |mut output, _| { - output.push_str(&format!("{:02x}", rand::random::())); + (0..4).fold(String::with_capacity(8), |mut output, _| { + write!(&mut output, "{:02x}", rand::random::()).unwrap(); output }) } diff --git a/cli/util/progress_bar/renderer.rs b/cli/util/progress_bar/renderer.rs index 6b08dada12..db3d37140f 100644 --- a/cli/util/progress_bar/renderer.rs +++ b/cli/util/progress_bar/renderer.rs @@ -1,5 +1,6 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +use std::fmt::Write; use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering; use std::time::Duration; @@ -81,12 +82,14 @@ impl ProgressBarRenderer for BarProgressBarRenderer { let elapsed_text = get_elapsed_text(data.duration); let mut text = String::new(); if !display_entry.message.is_empty() { - text.push_str(&format!( - "{} {}{}\n", + writeln!( + &mut text, + "{} {}{}", colors::green("Download"), display_entry.message, bytes_text, - )); + ) + .unwrap(); } text.push_str(&elapsed_text); let max_width = (data.terminal_width as i32 - 5).clamp(10, 75) as usize; diff --git a/cli/worker.rs b/cli/worker.rs index 161d8bcc21..0bbc27b29f 100644 --- a/cli/worker.rs +++ b/cli/worker.rs @@ -50,6 +50,7 @@ use tokio::select; use crate::args::CliLockfile; use crate::args::DenoSubcommand; +use crate::args::NpmCachingStrategy; use crate::args::StorageKeyResolver; use crate::errors; use crate::npm::CliNpmResolver; @@ -153,7 +154,8 @@ struct SharedWorkerState { storage_key_resolver: StorageKeyResolver, options: CliMainWorkerOptions, subcommand: DenoSubcommand, - otel_config: Option, // `None` means OpenTelemetry is disabled. + otel_config: OtelConfig, + default_npm_caching_strategy: NpmCachingStrategy, } impl SharedWorkerState { @@ -424,7 +426,8 @@ impl CliMainWorkerFactory { storage_key_resolver: StorageKeyResolver, subcommand: DenoSubcommand, options: CliMainWorkerOptions, - otel_config: Option, + otel_config: OtelConfig, + default_npm_caching_strategy: NpmCachingStrategy, ) -> Self { Self { shared: Arc::new(SharedWorkerState { @@ -448,6 +451,7 @@ impl CliMainWorkerFactory { options, subcommand, otel_config, + default_npm_caching_strategy, }), } } @@ -487,8 +491,19 @@ impl CliMainWorkerFactory { NpmPackageReqReference::from_specifier(&main_module) { if let Some(npm_resolver) = shared.npm_resolver.as_managed() { + let reqs = &[package_ref.req().clone()]; npm_resolver - .add_package_reqs(&[package_ref.req().clone()]) + .add_package_reqs( + reqs, + if matches!( + shared.default_npm_caching_strategy, + NpmCachingStrategy::Lazy + ) { + crate::npm::PackageCaching::Only(reqs.into()) + } else { + crate::npm::PackageCaching::All + }, + ) .await?; } diff --git a/ext/broadcast_channel/Cargo.toml b/ext/broadcast_channel/Cargo.toml index fccec9a66a..5b238aad25 100644 --- a/ext/broadcast_channel/Cargo.toml +++ b/ext/broadcast_channel/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_broadcast_channel" -version = "0.175.0" +version = "0.176.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/cache/Cargo.toml b/ext/cache/Cargo.toml index bf067c8e32..d03779d364 100644 --- a/ext/cache/Cargo.toml +++ b/ext/cache/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_cache" -version = "0.113.0" +version = "0.114.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/canvas/Cargo.toml b/ext/canvas/Cargo.toml index 5ec468ec18..c851b7724a 100644 --- a/ext/canvas/Cargo.toml +++ b/ext/canvas/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_canvas" -version = "0.50.0" +version = "0.51.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/console/Cargo.toml b/ext/console/Cargo.toml index 7e827efc61..4a26917933 100644 --- a/ext/console/Cargo.toml +++ b/ext/console/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_console" -version = "0.181.0" +version = "0.182.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/cron/Cargo.toml b/ext/cron/Cargo.toml index 3762ace1a5..d8f2d949f7 100644 --- a/ext/cron/Cargo.toml +++ b/ext/cron/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_cron" -version = "0.61.0" +version = "0.62.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/crypto/Cargo.toml b/ext/crypto/Cargo.toml index f13e0a3c66..63656bf642 100644 --- a/ext/crypto/Cargo.toml +++ b/ext/crypto/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_crypto" -version = "0.195.0" +version = "0.196.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/fetch/Cargo.toml b/ext/fetch/Cargo.toml index 959fac574a..716d268a04 100644 --- a/ext/fetch/Cargo.toml +++ b/ext/fetch/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_fetch" -version = "0.205.0" +version = "0.206.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/fetch/lib.rs b/ext/fetch/lib.rs index a3f5d03e64..919c6d3044 100644 --- a/ext/fetch/lib.rs +++ b/ext/fetch/lib.rs @@ -206,9 +206,6 @@ pub enum FetchError { RequestBuilderHook(deno_core::error::AnyError), #[error(transparent)] Io(#[from] std::io::Error), - // Only used for node upgrade - #[error(transparent)] - Hyper(#[from] hyper::Error), } pub type CancelableResponseFuture = diff --git a/ext/ffi/Cargo.toml b/ext/ffi/Cargo.toml index 23c71d0030..d54249329d 100644 --- a/ext/ffi/Cargo.toml +++ b/ext/ffi/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_ffi" -version = "0.168.0" +version = "0.169.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/fs/Cargo.toml b/ext/fs/Cargo.toml index f9e3359d07..d11520ad8c 100644 --- a/ext/fs/Cargo.toml +++ b/ext/fs/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_fs" -version = "0.91.0" +version = "0.92.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/http/Cargo.toml b/ext/http/Cargo.toml index 4a398db859..27a91ca61b 100644 --- a/ext/http/Cargo.toml +++ b/ext/http/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_http" -version = "0.179.0" +version = "0.180.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/io/Cargo.toml b/ext/io/Cargo.toml index f5d9e47245..1b73bad348 100644 --- a/ext/io/Cargo.toml +++ b/ext/io/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_io" -version = "0.91.0" +version = "0.92.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/kv/Cargo.toml b/ext/kv/Cargo.toml index c0e030e3c8..c69a962fa3 100644 --- a/ext/kv/Cargo.toml +++ b/ext/kv/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_kv" -version = "0.89.0" +version = "0.90.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/napi/Cargo.toml b/ext/napi/Cargo.toml index 5bd86d31b4..783b4b7cff 100644 --- a/ext/napi/Cargo.toml +++ b/ext/napi/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_napi" -version = "0.112.0" +version = "0.113.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/napi/sym/Cargo.toml b/ext/napi/sym/Cargo.toml index 478443e78f..a3dd56e2bf 100644 --- a/ext/napi/sym/Cargo.toml +++ b/ext/napi/sym/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "napi_sym" -version = "0.111.0" +version = "0.112.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/net/Cargo.toml b/ext/net/Cargo.toml index efe46a79e7..f5aa32c8ce 100644 --- a/ext/net/Cargo.toml +++ b/ext/net/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_net" -version = "0.173.0" +version = "0.174.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/node/Cargo.toml b/ext/node/Cargo.toml index f010e5784f..22180bf952 100644 --- a/ext/node/Cargo.toml +++ b/ext/node/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_node" -version = "0.118.0" +version = "0.119.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/node/lib.rs b/ext/node/lib.rs index aaf61797bb..d1090f6576 100644 --- a/ext/node/lib.rs +++ b/ext/node/lib.rs @@ -364,9 +364,9 @@ deno_core::extension!(deno_node, ops::zlib::brotli::op_create_brotli_decompress, ops::zlib::brotli::op_brotli_decompress_stream, ops::zlib::brotli::op_brotli_decompress_stream_end, - ops::http::op_node_http_request

, ops::http::op_node_http_fetch_response_upgrade, - ops::http::op_node_http_fetch_send, + ops::http::op_node_http_request_with_conn

, + ops::http::op_node_http_await_response, ops::http2::op_http2_connect, ops::http2::op_http2_poll_client_connection, ops::http2::op_http2_client_request, diff --git a/ext/node/ops/http.rs b/ext/node/ops/http.rs index f4adb94060..eb28e68aee 100644 --- a/ext/node/ops/http.rs +++ b/ext/node/ops/http.rs @@ -2,18 +2,20 @@ use std::borrow::Cow; use std::cell::RefCell; +use std::fmt::Debug; use std::pin::Pin; use std::rc::Rc; use std::task::Context; use std::task::Poll; use bytes::Bytes; +use deno_core::error::bad_resource; +use deno_core::error::type_error; use deno_core::futures::stream::Peekable; use deno_core::futures::Future; use deno_core::futures::FutureExt; use deno_core::futures::Stream; use deno_core::futures::StreamExt; -use deno_core::futures::TryFutureExt; use deno_core::op2; use deno_core::serde::Serialize; use deno_core::unsync::spawn; @@ -25,17 +27,17 @@ use deno_core::ByteString; use deno_core::CancelFuture; use deno_core::CancelHandle; use deno_core::CancelTryFuture; +use deno_core::Canceled; use deno_core::OpState; use deno_core::RcRef; use deno_core::Resource; use deno_core::ResourceId; -use deno_fetch::get_or_create_client_from_state; use deno_fetch::FetchCancelHandle; -use deno_fetch::FetchError; -use deno_fetch::FetchRequestResource; use deno_fetch::FetchReturn; -use deno_fetch::HttpClientResource; use deno_fetch::ResBody; +use deno_net::io::TcpStreamResource; +use deno_net::ops_tls::TlsStreamResource; +use deno_permissions::PermissionCheckError; use http::header::HeaderMap; use http::header::HeaderName; use http::header::HeaderValue; @@ -44,41 +46,140 @@ use http::header::CONTENT_LENGTH; use http::Method; use http_body_util::BodyExt; use hyper::body::Frame; +use hyper::body::Incoming; use hyper_util::rt::TokioIo; use std::cmp::min; use tokio::io::AsyncReadExt; use tokio::io::AsyncWriteExt; -#[op2(stack_trace)] +#[derive(Default, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct NodeHttpResponse { + pub status: u16, + pub status_text: String, + pub headers: Vec<(ByteString, ByteString)>, + pub url: String, + pub response_rid: ResourceId, + pub content_length: Option, + pub remote_addr_ip: Option, + pub remote_addr_port: Option, + pub error: Option, +} + +type CancelableResponseResult = + Result, hyper::Error>, Canceled>; + +pub struct NodeHttpClientResponse { + response: Pin>>, + url: String, +} + +impl Debug for NodeHttpClientResponse { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("NodeHttpClientResponse") + .field("url", &self.url) + .finish() + } +} + +impl deno_core::Resource for NodeHttpClientResponse { + fn name(&self) -> Cow { + "nodeHttpClientResponse".into() + } +} + +#[derive(Debug, thiserror::Error)] +pub enum ConnError { + #[error(transparent)] + Resource(deno_core::error::AnyError), + #[error(transparent)] + Permission(#[from] PermissionCheckError), + #[error("Invalid URL {0}")] + InvalidUrl(Url), + #[error(transparent)] + InvalidHeaderName(#[from] http::header::InvalidHeaderName), + #[error(transparent)] + InvalidHeaderValue(#[from] http::header::InvalidHeaderValue), + #[error(transparent)] + Url(#[from] url::ParseError), + #[error(transparent)] + Method(#[from] http::method::InvalidMethod), + #[error(transparent)] + Io(#[from] std::io::Error), + #[error("TLS stream is currently in use")] + TlsStreamBusy, + #[error("TCP stream is currently in use")] + TcpStreamBusy, + #[error(transparent)] + ReuniteTcp(#[from] tokio::net::tcp::ReuniteError), + #[error(transparent)] + Canceled(#[from] deno_core::Canceled), + #[error(transparent)] + Hyper(#[from] hyper::Error), +} + +#[op2(async, stack_trace)] #[serde] -pub fn op_node_http_request

( - state: &mut OpState, +pub async fn op_node_http_request_with_conn

( + state: Rc>, #[serde] method: ByteString, #[string] url: String, #[serde] headers: Vec<(ByteString, ByteString)>, - #[smi] client_rid: Option, #[smi] body: Option, -) -> Result + #[smi] conn_rid: ResourceId, + encrypted: bool, +) -> Result where P: crate::NodePermissions + 'static, { - let client = if let Some(rid) = client_rid { - let r = state + let (_handle, mut sender) = if encrypted { + let resource_rc = state + .borrow_mut() .resource_table - .get::(rid) - .map_err(FetchError::Resource)?; - r.client.clone() + .take::(conn_rid) + .map_err(ConnError::Resource)?; + let resource = + Rc::try_unwrap(resource_rc).map_err(|_e| ConnError::TlsStreamBusy)?; + let (read_half, write_half) = resource.into_inner(); + let tcp_stream = read_half.unsplit(write_half); + let io = TokioIo::new(tcp_stream); + let (sender, conn) = hyper::client::conn::http1::handshake(io).await?; + ( + tokio::task::spawn(async move { conn.with_upgrades().await }), + sender, + ) } else { - get_or_create_client_from_state(state)? + let resource_rc = state + .borrow_mut() + .resource_table + .take::(conn_rid) + .map_err(ConnError::Resource)?; + let resource = + Rc::try_unwrap(resource_rc).map_err(|_| ConnError::TcpStreamBusy)?; + let (read_half, write_half) = resource.into_inner(); + let tcp_stream = read_half.reunite(write_half)?; + let io = TokioIo::new(tcp_stream); + let (sender, conn) = hyper::client::conn::http1::handshake(io).await?; + + // Spawn a task to poll the connection, driving the HTTP state + ( + tokio::task::spawn(async move { + conn.with_upgrades().await?; + Ok::<_, _>(()) + }), + sender, + ) }; + // Create the request. let method = Method::from_bytes(&method)?; - let mut url = Url::parse(&url)?; - let maybe_authority = deno_fetch::extract_authority(&mut url); + let mut url_parsed = Url::parse(&url)?; + let maybe_authority = deno_fetch::extract_authority(&mut url_parsed); { - let permissions = state.borrow_mut::

(); - permissions.check_net_url(&url, "ClientRequest")?; + let mut state_ = state.borrow_mut(); + let permissions = state_.borrow_mut::

(); + permissions.check_net_url(&url_parsed, "ClientRequest")?; } let mut header_map = HeaderMap::new(); @@ -93,9 +194,10 @@ where ( BodyExt::boxed(NodeHttpResourceToBodyAdapter::new( state + .borrow_mut() .resource_table .take_any(body) - .map_err(FetchError::Resource)?, + .map_err(ConnError::Resource)?, )), None, ) @@ -117,10 +219,13 @@ where let mut request = http::Request::new(body); *request.method_mut() = method.clone(); - *request.uri_mut() = url - .as_str() + let path = url_parsed.path(); + let query = url_parsed.query(); + *request.uri_mut() = query + .map(|q| format!("{}?{}", path, q)) + .unwrap_or_else(|| path.to_string()) .parse() - .map_err(|_| FetchError::InvalidUrl(url.clone()))?; + .map_err(|_| ConnError::InvalidUrl(url_parsed.clone()))?; *request.headers_mut() = header_map; if let Some((username, password)) = maybe_authority { @@ -136,86 +241,44 @@ where let cancel_handle = CancelHandle::new_rc(); let cancel_handle_ = cancel_handle.clone(); - let fut = async move { - client - .send(request) - .map_err(Into::into) - .or_cancel(cancel_handle_) - .await - }; + let fut = + async move { sender.send_request(request).or_cancel(cancel_handle_).await }; - let request_rid = state.resource_table.add(FetchRequestResource { - future: Box::pin(fut), - url, - }); + let rid = state + .borrow_mut() + .resource_table + .add(NodeHttpClientResponse { + response: Box::pin(fut), + url: url.clone(), + }); - let cancel_handle_rid = - state.resource_table.add(FetchCancelHandle(cancel_handle)); + let cancel_handle_rid = state + .borrow_mut() + .resource_table + .add(FetchCancelHandle(cancel_handle)); Ok(FetchReturn { - request_rid, + request_rid: rid, cancel_handle_rid: Some(cancel_handle_rid), }) } -#[derive(Default, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct NodeHttpFetchResponse { - pub status: u16, - pub status_text: String, - pub headers: Vec<(ByteString, ByteString)>, - pub url: String, - pub response_rid: ResourceId, - pub content_length: Option, - pub remote_addr_ip: Option, - pub remote_addr_port: Option, - pub error: Option, -} - #[op2(async)] #[serde] -pub async fn op_node_http_fetch_send( +pub async fn op_node_http_await_response( state: Rc>, #[smi] rid: ResourceId, -) -> Result { - let request = state +) -> Result { + let resource = state .borrow_mut() .resource_table - .take::(rid) - .map_err(FetchError::Resource)?; - - let request = Rc::try_unwrap(request) - .ok() - .expect("multiple op_node_http_fetch_send ongoing"); - - let res = match request.future.await { - Ok(Ok(res)) => res, - Ok(Err(err)) => { - // We're going to try and rescue the error cause from a stream and return it from this fetch. - // If any error in the chain is a hyper body error, return that as a special result we can use to - // reconstruct an error chain (eg: `new TypeError(..., { cause: new Error(...) })`). - // TODO(mmastrac): it would be a lot easier if we just passed a v8::Global through here instead - - if let FetchError::ClientSend(err_src) = &err { - if let Some(client_err) = std::error::Error::source(&err_src.source) { - if let Some(err_src) = client_err.downcast_ref::() { - if let Some(err_src) = std::error::Error::source(err_src) { - return Ok(NodeHttpFetchResponse { - error: Some(err_src.to_string()), - ..Default::default() - }); - } - } - } - } - - return Err(err); - } - Err(_) => return Err(FetchError::RequestCanceled), - }; + .take::(rid) + .map_err(ConnError::Resource)?; + let resource = Rc::try_unwrap(resource) + .map_err(|_| ConnError::Resource(bad_resource("NodeHttpClientResponse")))?; + let res = resource.response.await??; let status = res.status(); - let url = request.url.into(); let mut res_headers = Vec::new(); for (key, val) in res.headers().iter() { res_headers.push((key.as_str().into(), val.as_bytes().into())); @@ -232,16 +295,22 @@ pub async fn op_node_http_fetch_send( (None, None) }; + let (parts, body) = res.into_parts(); + let body = body.map_err(deno_core::anyhow::Error::from); + let body = body.boxed(); + + let res = http::Response::from_parts(parts, body); + let response_rid = state .borrow_mut() .resource_table - .add(NodeHttpFetchResponseResource::new(res, content_length)); + .add(NodeHttpResponseResource::new(res, content_length)); - Ok(NodeHttpFetchResponse { + Ok(NodeHttpResponse { status: status.as_u16(), status_text: status.canonical_reason().unwrap_or("").to_string(), headers: res_headers, - url, + url: resource.url, response_rid, content_length, remote_addr_ip, @@ -255,12 +324,12 @@ pub async fn op_node_http_fetch_send( pub async fn op_node_http_fetch_response_upgrade( state: Rc>, #[smi] rid: ResourceId, -) -> Result { +) -> Result { let raw_response = state .borrow_mut() .resource_table - .take::(rid) - .map_err(FetchError::Resource)?; + .take::(rid) + .map_err(ConnError::Resource)?; let raw_response = Rc::try_unwrap(raw_response) .expect("Someone is holding onto NodeHttpFetchResponseResource"); @@ -283,7 +352,7 @@ pub async fn op_node_http_fetch_response_upgrade( } read_tx.write_all(&buf[..read]).await?; } - Ok::<_, FetchError>(()) + Ok::<_, ConnError>(()) }); spawn(async move { let mut buf = [0; 1024]; @@ -294,7 +363,7 @@ pub async fn op_node_http_fetch_response_upgrade( } upgraded_tx.write_all(&buf[..read]).await?; } - Ok::<_, FetchError>(()) + Ok::<_, ConnError>(()) }); } @@ -379,13 +448,13 @@ impl Default for NodeHttpFetchResponseReader { } #[derive(Debug)] -pub struct NodeHttpFetchResponseResource { +pub struct NodeHttpResponseResource { pub response_reader: AsyncRefCell, pub cancel: CancelHandle, pub size: Option, } -impl NodeHttpFetchResponseResource { +impl NodeHttpResponseResource { pub fn new(response: http::Response, size: Option) -> Self { Self { response_reader: AsyncRefCell::new(NodeHttpFetchResponseReader::Start( @@ -400,14 +469,14 @@ impl NodeHttpFetchResponseResource { let reader = self.response_reader.into_inner(); match reader { NodeHttpFetchResponseReader::Start(resp) => { - Ok(hyper::upgrade::on(resp).await?) + hyper::upgrade::on(resp).await } _ => unreachable!(), } } } -impl Resource for NodeHttpFetchResponseResource { +impl Resource for NodeHttpResponseResource { fn name(&self) -> Cow { "fetchResponse".into() } @@ -454,9 +523,7 @@ impl Resource for NodeHttpFetchResponseResource { // safely call `await` on it without creating a race condition. Some(_) => match reader.as_mut().next().await.unwrap() { Ok(chunk) => assert!(chunk.is_empty()), - Err(err) => { - break Err(deno_core::error::type_error(err.to_string())) - } + Err(err) => break Err(type_error(err.to_string())), }, None => break Ok(BufView::empty()), } @@ -464,7 +531,7 @@ impl Resource for NodeHttpFetchResponseResource { }; let cancel_handle = RcRef::map(self, |r| &r.cancel); - fut.try_or_cancel(cancel_handle).await.map_err(Into::into) + fut.try_or_cancel(cancel_handle).await }) } @@ -514,8 +581,9 @@ impl Stream for NodeHttpResourceToBodyAdapter { Poll::Ready(res) => match res { Ok(buf) if buf.is_empty() => Poll::Ready(None), Ok(buf) => { + let bytes: Bytes = buf.to_vec().into(); this.1 = Some(this.0.clone().read(64 * 1024)); - Poll::Ready(Some(Ok(buf.to_vec().into()))) + Poll::Ready(Some(Ok(bytes))) } Err(err) => Poll::Ready(Some(Err(err))), }, diff --git a/ext/node/polyfill.rs b/ext/node/polyfill.rs index 7a5f32c45e..fc95d708eb 100644 --- a/ext/node/polyfill.rs +++ b/ext/node/polyfill.rs @@ -25,6 +25,17 @@ macro_rules! generate_builtin_node_module_lists { // NOTE(bartlomieju): keep this list in sync with `ext/node/polyfills/01_require.js` generate_builtin_node_module_lists! { + "_http_agent", + "_http_common", + "_http_outgoing", + "_http_server", + "_stream_duplex", + "_stream_passthrough", + "_stream_readable", + "_stream_transform", + "_stream_writable", + "_tls_common", + "_tls_wrap", "assert", "assert/strict", "async_hooks", diff --git a/ext/node/polyfills/_http_common.ts b/ext/node/polyfills/_http_common.ts index 8fb7758a55..86143b4dea 100644 --- a/ext/node/polyfills/_http_common.ts +++ b/ext/node/polyfills/_http_common.ts @@ -7,6 +7,7 @@ const { SafeRegExp, Symbol, } = primordials; +import { HTTPParser } from "ext:deno_node/internal_binding/http_parser.ts"; export const CRLF = "\r\n"; export const kIncomingMessage = Symbol("IncomingMessage"); @@ -79,6 +80,8 @@ export { checkIsHttpToken as _checkIsHttpToken, }; +export { HTTPParser }; + export default { _checkInvalidHeaderChar: checkInvalidHeaderChar, _checkIsHttpToken: checkIsHttpToken, @@ -87,4 +90,5 @@ export default { continueExpression, kIncomingMessage, methods, + HTTPParser, }; diff --git a/ext/node/polyfills/_http_outgoing.ts b/ext/node/polyfills/_http_outgoing.ts index 4da6b73e87..5a9a8ad7e6 100644 --- a/ext/node/polyfills/_http_outgoing.ts +++ b/ext/node/polyfills/_http_outgoing.ts @@ -491,19 +491,53 @@ Object.defineProperties( return ret; }, + /** Right after socket is ready, we need to writeHeader() to setup the request and + * client. This is invoked by onSocket(). */ + _flushHeaders() { + if (!this._headerSent) { + this._headerSent = true; + this._writeHeader(); + } + }, + // deno-lint-ignore no-explicit-any _send(data: any, encoding?: string | null, callback?: () => void) { - if (!this._headerSent && this._header !== null) { - this._writeHeader(); - this._headerSent = true; + // if socket is ready, write the data after headers are written. + // if socket is not ready, buffer data in outputbuffer. + if ( + this.socket && !this.socket.connecting && this.outputData.length === 0 + ) { + if (!this._headerSent) { + this._writeHeader(); + this._headerSent = true; + } + + return this._writeRaw(data, encoding, callback); + } else { + this.outputData.push({ data, encoding, callback }); } - return this._writeRaw(data, encoding, callback); + return false; }, _writeHeader() { throw new ERR_METHOD_NOT_IMPLEMENTED("_writeHeader()"); }, + _flushBuffer() { + const outputLength = this.outputData.length; + if (outputLength <= 0 || !this.socket || !this._bodyWriter) { + return undefined; + } + + const { data, encoding, callback } = this.outputData.shift(); + const ret = this._writeRaw(data, encoding, callback); + if (this.outputData.length > 0) { + this.once("drain", this._flushBuffer); + } + + return ret; + }, + _writeRaw( // deno-lint-ignore no-explicit-any data: any, @@ -517,11 +551,15 @@ Object.defineProperties( data = new Uint8Array(data.buffer, data.byteOffset, data.byteLength); } if (data.buffer.byteLength > 0) { - this._bodyWriter.write(data).then(() => { - callback?.(); - this.emit("drain"); - }).catch((e) => { - this._requestSendError = e; + this._bodyWriter.ready.then(() => { + if (this._bodyWriter.desiredSize > 0) { + this._bodyWriter.write(data).then(() => { + callback?.(); + this.emit("drain"); + }).catch((e) => { + this._requestSendError = e; + }); + } }); } return false; @@ -658,7 +696,6 @@ Object.defineProperties( const { header } = state; this._header = header + "\r\n"; - this._headerSent = false; // Wait until the first body chunk, or close(), is sent to flush, // UNLESS we're sending Expect: 100-continue. diff --git a/ext/node/polyfills/_tls_wrap.ts b/ext/node/polyfills/_tls_wrap.ts index 4c7424a328..9e5def9f2b 100644 --- a/ext/node/polyfills/_tls_wrap.ts +++ b/ext/node/polyfills/_tls_wrap.ts @@ -154,6 +154,13 @@ export class TLSSocket extends net.Socket { const afterConnect = handle.afterConnect; handle.afterConnect = async (req: any, status: number) => { options.hostname ??= undefined; // coerce to undefined if null, startTls expects hostname to be undefined + if (tlssock._isNpmAgent) { + // skips the TLS handshake for @npmcli/agent as it's handled by + // onSocket handler of ClientRequest object. + tlssock.emit("secure"); + tlssock.removeListener("end", onConnectEnd); + return afterConnect.call(handle, req, status); + } try { const conn = await Deno.startTls(handle[kStreamBaseField], options); diff --git a/ext/node/polyfills/http.ts b/ext/node/polyfills/http.ts index 948a3527bd..e911535be5 100644 --- a/ext/node/polyfills/http.ts +++ b/ext/node/polyfills/http.ts @@ -5,16 +5,17 @@ import { core, primordials } from "ext:core/mod.js"; import { + op_node_http_await_response, op_node_http_fetch_response_upgrade, - op_node_http_fetch_send, - op_node_http_request, + op_node_http_request_with_conn, + op_tls_start, } from "ext:core/ops"; import { TextEncoder } from "ext:deno_web/08_text_encoding.js"; import { setTimeout } from "ext:deno_web/02_timers.js"; import { _normalizeArgs, - // createConnection, + createConnection, ListenOptions, Socket, } from "node:net"; @@ -48,9 +49,10 @@ import { kOutHeaders } from "ext:deno_node/internal/http.ts"; import { _checkIsHttpToken as checkIsHttpToken } from "node:_http_common"; import { Agent, globalAgent } from "node:_http_agent"; import { urlToHttpOptions } from "ext:deno_node/internal/url.ts"; -import { kEmptyObject } from "ext:deno_node/internal/util.mjs"; +import { kEmptyObject, once } from "ext:deno_node/internal/util.mjs"; import { constants, TCP } from "ext:deno_node/internal_binding/tcp_wrap.ts"; -import { notImplemented, warnNotImplemented } from "ext:deno_node/_utils.ts"; +import { kStreamBaseField } from "ext:deno_node/internal_binding/stream_wrap.ts"; +import { notImplemented } from "ext:deno_node/_utils.ts"; import { connResetException, ERR_HTTP_HEADERS_SENT, @@ -62,7 +64,6 @@ import { } from "ext:deno_node/internal/errors.ts"; import { getTimerDuration } from "ext:deno_node/internal/timers.mjs"; import { serve, upgradeHttpRaw } from "ext:deno_http/00_serve.ts"; -import { createHttpClient } from "ext:deno_fetch/22_http_client.js"; import { headersEntries } from "ext:deno_fetch/20_headers.js"; import { timerId } from "ext:deno_web/03_abort_signal.js"; import { clearTimeout as webClearTimeout } from "ext:deno_web/02_timers.js"; @@ -148,6 +149,10 @@ class FakeSocket extends EventEmitter { } } +function emitErrorEvent(request, error) { + request.emit("error", error); +} + /** ClientRequest represents the http(s) request from the client */ class ClientRequest extends OutgoingMessage { defaultProtocol = "http:"; @@ -160,6 +165,8 @@ class ClientRequest extends OutgoingMessage { useChunkedEncodingByDefault: boolean; path: string; _req: { requestRid: number; cancelHandleRid: number | null } | undefined; + _encrypted = false; + socket: Socket; constructor( input: string | URL, @@ -382,17 +389,11 @@ class ClientRequest extends OutgoingMessage { delete optsWithoutSignal.signal; } - if (options!.createConnection) { - warnNotImplemented("ClientRequest.options.createConnection"); - } - if (options!.lookup) { notImplemented("ClientRequest.options.lookup"); } - // initiate connection - // TODO(crowlKats): finish this - /*if (this.agent) { + if (this.agent) { this.agent.addRequest(this, optsWithoutSignal); } else { // No agent, default to Connection:close. @@ -422,8 +423,7 @@ class ClientRequest extends OutgoingMessage { debug("CLIENT use net.createConnection", optsWithoutSignal); this.onSocket(createConnection(optsWithoutSignal)); } - }*/ - this.onSocket(new FakeSocket({ encrypted: this._encrypted })); + } } _writeHeader() { @@ -437,9 +437,6 @@ class ClientRequest extends OutgoingMessage { } } - const client = this._getClient() ?? createHttpClient({ http2: false }); - this._client = client; - if ( this.method === "POST" || this.method === "PATCH" || this.method === "PUT" ) { @@ -455,17 +452,29 @@ class ClientRequest extends OutgoingMessage { this._bodyWriteRid = resourceForReadableStream(readable); } - this._req = op_node_http_request( - this.method, - url, - headers, - client[internalRidSymbol], - this._bodyWriteRid, - ); - (async () => { try { - const res = await op_node_http_fetch_send(this._req.requestRid); + const parsedUrl = new URL(url); + let baseConnRid = + this.socket._handle[kStreamBaseField][internalRidSymbol]; + if (this._encrypted) { + [baseConnRid] = op_tls_start({ + rid: baseConnRid, + hostname: parsedUrl.hostname, + caCerts: [], + alpnProtocols: ["http/1.0", "http/1.1"], + }); + } + this._req = await op_node_http_request_with_conn( + this.method, + url, + headers, + this._bodyWriteRid, + baseConnRid, + this._encrypted, + ); + this._flushBuffer(); + const res = await op_node_http_await_response(this._req!.requestRid); if (this._req.cancelHandleRid !== null) { core.tryClose(this._req.cancelHandleRid); } @@ -473,7 +482,6 @@ class ClientRequest extends OutgoingMessage { this._timeout.removeEventListener("abort", this._timeoutCb); webClearTimeout(this._timeout[timerId]); } - this._client.close(); const incoming = new IncomingMessageForClient(this.socket); incoming.req = this; this.res = incoming; @@ -512,12 +520,9 @@ class ClientRequest extends OutgoingMessage { if (this.method === "CONNECT") { throw new Error("not implemented CONNECT"); } - const upgradeRid = await op_node_http_fetch_response_upgrade( res.responseRid, ); - assert(typeof res.remoteAddrIp !== "undefined"); - assert(typeof res.remoteAddrIp !== "undefined"); const conn = new UpgradedConn( upgradeRid, { @@ -543,13 +548,11 @@ class ClientRequest extends OutgoingMessage { this._closed = true; this.emit("close"); } else { - { - incoming._bodyRid = res.responseRid; - } + incoming._bodyRid = res.responseRid; this.emit("response", incoming); } } catch (err) { - if (this._req.cancelHandleRid !== null) { + if (this._req && this._req.cancelHandleRid !== null) { core.tryClose(this._req.cancelHandleRid); } @@ -592,11 +595,54 @@ class ClientRequest extends OutgoingMessage { return undefined; } - // TODO(bartlomieju): handle error - onSocket(socket, _err) { + onSocket(socket, err) { nextTick(() => { - this.socket = socket; - this.emit("socket", socket); + // deno-lint-ignore no-this-alias + const req = this; + if (req.destroyed || err) { + req.destroyed = true; + + // deno-lint-ignore no-inner-declarations + function _destroy(req, err) { + if (!req.aborted && !err) { + err = new connResetException("socket hang up"); + } + if (err) { + emitErrorEvent(req, err); + } + req._closed = true; + req.emit("close"); + } + + if (socket) { + if (!err && req.agent && !socket.destroyed) { + socket.emit("free"); + } else { + finished(socket.destroy(err || req[kError]), (er) => { + if (er?.code === "ERR_STREAM_PREMATURE_CLOSE") { + er = null; + } + _destroy(req, er || err); + }); + return; + } + } + + _destroy(req, err || req[kError]); + } else { + // Note: this code is specific to deno to initiate a request. + const onConnect = () => { + // Flush the internal buffers once socket is ready. + this._flushHeaders(); + }; + this.socket = socket; + this.emit("socket", socket); + if (socket.readyState === "opening") { + socket.on("connect", onConnect); + } else { + onConnect(); + } + } }); } @@ -618,14 +664,20 @@ class ClientRequest extends OutgoingMessage { if (chunk) { this.write_(chunk, encoding, null, true); } else if (!this._headerSent) { - this._contentLength = 0; - this._implicitHeader(); - this._send("", "latin1"); + if ( + (this.socket && !this.socket.connecting) || // socket is not connecting, or + (!this.socket && this.outputData.length === 0) // no data to send + ) { + this._contentLength = 0; + this._implicitHeader(); + this._send("", "latin1"); + } } - (async () => { + const finish = async () => { try { + await this._bodyWriter.ready; await this._bodyWriter?.close(); - } catch (_) { + } catch { // The readable stream resource is dropped right after // read is complete closing the writable stream resource. // If we try to close the writer again, it will result in an @@ -633,10 +685,20 @@ class ClientRequest extends OutgoingMessage { } try { cb?.(); - } catch (_) { + } catch { // } - })(); + }; + + if (this.socket && this._bodyWriter) { + finish(); + } else { + this.on("drain", () => { + if (this.outputData.length === 0) { + finish(); + } + }); + } return this; } @@ -658,11 +720,6 @@ class ClientRequest extends OutgoingMessage { } this.destroyed = true; - const rid = this._client?.[internalRidSymbol]; - if (rid) { - core.tryClose(rid); - } - // Request might be closed before we actually made it if (this._req !== undefined && this._req.cancelHandleRid !== null) { core.tryClose(this._req.cancelHandleRid); diff --git a/ext/node/polyfills/https.ts b/ext/node/polyfills/https.ts index f60c5e471a..fd700173eb 100644 --- a/ext/node/polyfills/https.ts +++ b/ext/node/polyfills/https.ts @@ -112,7 +112,7 @@ export const globalAgent = new Agent({ /** HttpsClientRequest class loosely follows http.ClientRequest class API. */ class HttpsClientRequest extends ClientRequest { - override _encrypted: true; + override _encrypted = true; override defaultProtocol = "https:"; override _getClient(): Deno.HttpClient | undefined { if (caCerts === null) { diff --git a/ext/node/polyfills/internal_binding/cares_wrap.ts b/ext/node/polyfills/internal_binding/cares_wrap.ts index 6feb7faf0d..cbd0bb8ef6 100644 --- a/ext/node/polyfills/internal_binding/cares_wrap.ts +++ b/ext/node/polyfills/internal_binding/cares_wrap.ts @@ -36,7 +36,6 @@ import { } from "ext:deno_node/internal_binding/async_wrap.ts"; import { ares_strerror } from "ext:deno_node/internal_binding/ares.ts"; import { notImplemented } from "ext:deno_node/_utils.ts"; -import { isWindows } from "ext:deno_node/_util/os.ts"; interface LookupAddress { address: string; @@ -68,7 +67,7 @@ export function getaddrinfo( _hints: number, verbatim: boolean, ): number { - let addresses: string[] = []; + const addresses: string[] = []; // TODO(cmorten): use hints // REF: https://nodejs.org/api/dns.html#dns_supported_getaddrinfo_flags @@ -107,13 +106,6 @@ export function getaddrinfo( }); } - // TODO(@bartlomieju): Forces IPv4 as a workaround for Deno not - // aligning with Node on implicit binding on Windows - // REF: https://github.com/denoland/deno/issues/10762 - if (isWindows && hostname === "localhost") { - addresses = addresses.filter((address) => isIPv4(address)); - } - req.oncomplete(error, addresses); })(); diff --git a/ext/node/polyfills/net.ts b/ext/node/polyfills/net.ts index 2b01125190..b2b0c9857c 100644 --- a/ext/node/polyfills/net.ts +++ b/ext/node/polyfills/net.ts @@ -986,16 +986,20 @@ function _lookupAndConnect( } else { self._unrefTimer(); - defaultTriggerAsyncIdScope( - self[asyncIdSymbol], - _internalConnect, - self, - ip, - port, - addressType, - localAddress, - localPort, - ); + defaultTriggerAsyncIdScope(self[asyncIdSymbol], nextTick, () => { + if (self.connecting) { + defaultTriggerAsyncIdScope( + self[asyncIdSymbol], + _internalConnect, + self, + ip, + port, + addressType, + localAddress, + localPort, + ); + } + }); } }, ); @@ -1197,6 +1201,9 @@ export class Socket extends Duplex { _host: string | null = null; // deno-lint-ignore no-explicit-any _parent: any = null; + // The flag for detecting if it's called in @npmcli/agent + // See discussions in https://github.com/denoland/deno/pull/25470 for more details. + _isNpmAgent = false; autoSelectFamilyAttemptedAddresses: AddressInfo[] | undefined = undefined; constructor(options: SocketOptions | number) { @@ -1217,6 +1224,19 @@ export class Socket extends Duplex { super(options); + // Note: If the socket is created from @npmcli/agent, the 'socket' event + // on ClientRequest object happens after 'connect' event on Socket object. + // That swaps the sequence of op_node_http_request_with_conn() call and + // initial socket read. That causes op_node_http_request_with_conn() not + // working. + // To avoid the above situation, we detect the socket created from + // @npmcli/agent and pause the socket (and also skips the startTls call + // if it's TLSSocket) + this._isNpmAgent = new Error().stack?.includes("@npmcli/agent") || false; + if (this._isNpmAgent) { + this.pause(); + } + if (options.handle) { this._handle = options.handle; this[asyncIdSymbol] = _getNewAsyncId(this._handle); diff --git a/ext/node/polyfills/v8.ts b/ext/node/polyfills/v8.ts index 9df199865e..e24a79ab5a 100644 --- a/ext/node/polyfills/v8.ts +++ b/ext/node/polyfills/v8.ts @@ -227,6 +227,7 @@ function arrayBufferViewTypeToIndex(abView: ArrayBufferView) { // Index 10 is FastBuffer. if (type === "[object BigInt64Array]") return 11; if (type === "[object BigUint64Array]") return 12; + if (type === "[object Float16Array]") return 13; return -1; } export class DefaultSerializer extends Serializer { @@ -276,6 +277,7 @@ function arrayBufferViewIndexToType(index: number): any { if (index === 10) return Buffer; if (index === 11) return BigInt64Array; if (index === 12) return BigUint64Array; + if (index === 13) return Float16Array; return undefined; } diff --git a/ext/telemetry/Cargo.toml b/ext/telemetry/Cargo.toml index 87b330b322..f3d4bbd336 100644 --- a/ext/telemetry/Cargo.toml +++ b/ext/telemetry/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_telemetry" -version = "0.3.0" +version = "0.4.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/telemetry/lib.rs b/ext/telemetry/lib.rs index 9612401014..816e838743 100644 --- a/ext/telemetry/lib.rs +++ b/ext/telemetry/lib.rs @@ -97,13 +97,28 @@ deno_core::extension!( ); #[derive(Debug, Clone, Serialize, Deserialize)] -pub struct OtelConfig { +pub struct OtelRuntimeConfig { pub runtime_name: Cow<'static, str>, pub runtime_version: Cow<'static, str>, +} + +#[derive(Default, Debug, Clone, Serialize, Deserialize)] +pub struct OtelConfig { + pub tracing_enabled: bool, pub console: OtelConsoleConfig, pub deterministic: bool, } +impl OtelConfig { + pub fn as_v8(&self) -> Box<[u8]> { + Box::new([ + self.tracing_enabled as u8, + self.console as u8, + self.deterministic as u8, + ]) + } +} + #[derive(Debug, Clone, Copy, Serialize, Deserialize)] #[repr(u8)] pub enum OtelConsoleConfig { @@ -112,14 +127,9 @@ pub enum OtelConsoleConfig { Replace = 2, } -impl Default for OtelConfig { +impl Default for OtelConsoleConfig { fn default() -> Self { - Self { - runtime_name: Cow::Borrowed(env!("CARGO_PKG_NAME")), - runtime_version: Cow::Borrowed(env!("CARGO_PKG_VERSION")), - console: OtelConsoleConfig::Capture, - deterministic: false, - } + Self::Ignore } } @@ -411,16 +421,14 @@ static BUILT_IN_INSTRUMENTATION_SCOPE: OnceCell< opentelemetry::InstrumentationScope, > = OnceCell::new(); -pub fn init(config: OtelConfig) -> anyhow::Result<()> { +pub fn init(config: OtelRuntimeConfig) -> anyhow::Result<()> { // Parse the `OTEL_EXPORTER_OTLP_PROTOCOL` variable. The opentelemetry_* // crates don't do this automatically. // TODO(piscisaureus): enable GRPC support. let protocol = match env::var("OTEL_EXPORTER_OTLP_PROTOCOL").as_deref() { Ok("http/protobuf") => Protocol::HttpBinary, Ok("http/json") => Protocol::HttpJson, - Ok("") | Err(env::VarError::NotPresent) => { - return Ok(()); - } + Ok("") | Err(env::VarError::NotPresent) => Protocol::HttpBinary, Ok(protocol) => { return Err(anyhow!( "Env var OTEL_EXPORTER_OTLP_PROTOCOL specifies an unsupported protocol: {}", @@ -732,9 +740,9 @@ fn op_otel_instrumentation_scope_enter( #[op2(fast)] fn op_otel_instrumentation_scope_enter_builtin(state: &mut OpState) { - state.put(InstrumentationScope( - BUILT_IN_INSTRUMENTATION_SCOPE.get().unwrap().clone(), - )); + if let Some(scope) = BUILT_IN_INSTRUMENTATION_SCOPE.get() { + state.put(InstrumentationScope(scope.clone())); + } } #[op2(fast)] @@ -749,6 +757,9 @@ fn op_otel_log( let Some(Processors { logs, .. }) = OTEL_PROCESSORS.get() else { return; }; + let Some(instrumentation_scope) = BUILT_IN_INSTRUMENTATION_SCOPE.get() else { + return; + }; // Convert the integer log level that ext/console uses to the corresponding // OpenTelemetry log severity. @@ -776,10 +787,7 @@ fn op_otel_log( ); } - logs.emit( - &mut log_record, - BUILT_IN_INSTRUMENTATION_SCOPE.get().unwrap(), - ); + logs.emit(&mut log_record, instrumentation_scope); } fn owned_string<'s>( diff --git a/ext/telemetry/telemetry.ts b/ext/telemetry/telemetry.ts index e9e38d1592..d1335f65b5 100644 --- a/ext/telemetry/telemetry.ts +++ b/ext/telemetry/telemetry.ts @@ -220,6 +220,7 @@ function submitSpan( startTime: number, endTime: number, ) { + if (!TRACING_ENABLED) return; if (!(traceFlags & TRACE_FLAG_SAMPLED)) return; // TODO(@lucacasonato): `resource` is ignored for now, should we implement it? @@ -949,15 +950,15 @@ const otelConsoleConfig = { }; export function bootstrap( - config: [] | [ + config: [ + 0 | 1, typeof otelConsoleConfig[keyof typeof otelConsoleConfig], - number, + 0 | 1, ], ): void { - if (config.length === 0) return; - const { 0: consoleConfig, 1: deterministic } = config; + const { 0: tracingEnabled, 1: consoleConfig, 2: deterministic } = config; - TRACING_ENABLED = true; + TRACING_ENABLED = tracingEnabled === 1; DETERMINISTIC = deterministic === 1; switch (consoleConfig) { diff --git a/ext/tls/Cargo.toml b/ext/tls/Cargo.toml index 239c8f0834..fc75b0a05d 100644 --- a/ext/tls/Cargo.toml +++ b/ext/tls/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_tls" -version = "0.168.0" +version = "0.169.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/url/Cargo.toml b/ext/url/Cargo.toml index 8d0f951869..d30332d0ff 100644 --- a/ext/url/Cargo.toml +++ b/ext/url/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_url" -version = "0.181.0" +version = "0.182.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/web/01_dom_exception.js b/ext/web/01_dom_exception.js index 38e4d088e5..db2996e0c6 100644 --- a/ext/web/01_dom_exception.js +++ b/ext/web/01_dom_exception.js @@ -9,14 +9,15 @@ import { primordials } from "ext:core/mod.js"; const { + Error, ErrorPrototype, - ErrorCaptureStackTrace, ObjectDefineProperty, ObjectCreate, ObjectEntries, ObjectHasOwn, ObjectPrototypeIsPrototypeOf, ObjectSetPrototypeOf, + ReflectConstruct, Symbol, SymbolFor, } = primordials; @@ -107,12 +108,14 @@ class DOMException { ); const code = nameToCodeMapping[name] ?? 0; - this[_message] = message; - this[_name] = name; - this[_code] = code; - this[webidl.brand] = webidl.brand; + // execute Error constructor to have stack property and [[ErrorData]] internal slot + const error = ReflectConstruct(Error, [], new.target); + error[_message] = message; + error[_name] = name; + error[_code] = code; + error[webidl.brand] = webidl.brand; - ErrorCaptureStackTrace(this, DOMException); + return error; } get message() { diff --git a/ext/web/Cargo.toml b/ext/web/Cargo.toml index c05bc2889c..ac2b14fbed 100644 --- a/ext/web/Cargo.toml +++ b/ext/web/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_web" -version = "0.212.0" +version = "0.213.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/webgpu/Cargo.toml b/ext/webgpu/Cargo.toml index a6eea01ced..9621c085e8 100644 --- a/ext/webgpu/Cargo.toml +++ b/ext/webgpu/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_webgpu" -version = "0.148.0" +version = "0.149.0" authors = ["the Deno authors"] edition.workspace = true license = "MIT" diff --git a/ext/webidl/Cargo.toml b/ext/webidl/Cargo.toml index 07ee546be8..38eff7b66f 100644 --- a/ext/webidl/Cargo.toml +++ b/ext/webidl/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_webidl" -version = "0.181.0" +version = "0.182.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/websocket/Cargo.toml b/ext/websocket/Cargo.toml index 5ba2172414..cb72618cad 100644 --- a/ext/websocket/Cargo.toml +++ b/ext/websocket/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_websocket" -version = "0.186.0" +version = "0.187.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/ext/webstorage/Cargo.toml b/ext/webstorage/Cargo.toml index 5e74700b84..700a252016 100644 --- a/ext/webstorage/Cargo.toml +++ b/ext/webstorage/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_webstorage" -version = "0.176.0" +version = "0.177.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/resolvers/deno/Cargo.toml b/resolvers/deno/Cargo.toml index ce5bee3839..2966b5fef6 100644 --- a/resolvers/deno/Cargo.toml +++ b/resolvers/deno/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_resolver" -version = "0.13.0" +version = "0.14.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/resolvers/deno/lib.rs b/resolvers/deno/lib.rs index 661caf836d..a74ca614a3 100644 --- a/resolvers/deno/lib.rs +++ b/resolvers/deno/lib.rs @@ -355,16 +355,16 @@ impl< }) .map_err(|err| { match err.into_kind() { - ResolveReqWithSubPathErrorKind::MissingPackageNodeModulesFolder( - err, - ) => err.into(), - ResolveReqWithSubPathErrorKind::ResolvePkgFolderFromDenoReq( - err, - ) => err.into(), - ResolveReqWithSubPathErrorKind::PackageSubpathResolve(err) => { - err.into() + ResolveReqWithSubPathErrorKind::MissingPackageNodeModulesFolder( + err, + ) => err.into(), + ResolveReqWithSubPathErrorKind::ResolvePkgFolderFromDenoReq( + err, + ) => err.into(), + ResolveReqWithSubPathErrorKind::PackageSubpathResolve(err) => { + err.into() + } } - } }); } } diff --git a/resolvers/deno/npm/byonm.rs b/resolvers/deno/npm/byonm.rs index 771f23ce23..6e1be35ca0 100644 --- a/resolvers/deno/npm/byonm.rs +++ b/resolvers/deno/npm/byonm.rs @@ -205,9 +205,9 @@ impl ByonmNpmResolver { } // attempt to resolve the npm specifier from the referrer's package.json, - if let Ok(file_path) = url_to_file_path(referrer) { - let mut current_path = file_path.as_path(); - while let Some(dir_path) = current_path.parent() { + let maybe_referrer_path = url_to_file_path(referrer).ok(); + if let Some(file_path) = maybe_referrer_path { + for dir_path in file_path.as_path().ancestors().skip(1) { let package_json_path = dir_path.join("package.json"); if let Some(pkg_json) = self.load_pkg_json(&package_json_path)? { if let Some(alias) = @@ -216,11 +216,10 @@ impl ByonmNpmResolver { return Ok(Some((pkg_json, alias))); } } - current_path = dir_path; } } - // otherwise, fall fallback to the project's package.json + // fall fallback to the project's package.json if let Some(root_node_modules_dir) = &self.root_node_modules_dir { let root_pkg_json_path = root_node_modules_dir.parent().unwrap().join("package.json"); @@ -232,6 +231,58 @@ impl ByonmNpmResolver { } } + // now try to resolve based on the closest node_modules directory + let maybe_referrer_path = url_to_file_path(referrer).ok(); + let search_node_modules = |node_modules: &Path| { + if req.version_req.tag().is_some() { + return None; + } + + let pkg_folder = node_modules.join(&req.name); + if let Ok(Some(dep_pkg_json)) = + self.load_pkg_json(&pkg_folder.join("package.json")) + { + if dep_pkg_json.name.as_ref() == Some(&req.name) { + let matches_req = dep_pkg_json + .version + .as_ref() + .and_then(|v| Version::parse_from_npm(v).ok()) + .map(|version| req.version_req.matches(&version)) + .unwrap_or(true); + if matches_req { + return Some((dep_pkg_json, req.name.clone())); + } + } + } + None + }; + if let Some(file_path) = &maybe_referrer_path { + for dir_path in file_path.as_path().ancestors().skip(1) { + if let Some(result) = + search_node_modules(&dir_path.join("node_modules")) + { + return Ok(Some(result)); + } + } + } + + // and finally check the root node_modules directory + if let Some(root_node_modules_dir) = &self.root_node_modules_dir { + let already_searched = maybe_referrer_path + .as_ref() + .and_then(|referrer_path| { + root_node_modules_dir + .parent() + .map(|root_dir| referrer_path.starts_with(root_dir)) + }) + .unwrap_or(false); + if !already_searched { + if let Some(result) = search_node_modules(root_node_modules_dir) { + return Ok(Some(result)); + } + } + } + Ok(None) } diff --git a/resolvers/node/Cargo.toml b/resolvers/node/Cargo.toml index 111d67ad59..e175bcfafa 100644 --- a/resolvers/node/Cargo.toml +++ b/resolvers/node/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "node_resolver" -version = "0.20.0" +version = "0.21.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/resolvers/npm_cache/Cargo.toml b/resolvers/npm_cache/Cargo.toml index fab102dee8..1cc7237025 100644 --- a/resolvers/npm_cache/Cargo.toml +++ b/resolvers/npm_cache/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_npm_cache" -version = "0.1.0" +version = "0.2.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/resolvers/npm_cache/tarball_extract.rs b/resolvers/npm_cache/tarball_extract.rs index 262618d905..c4c614b35f 100644 --- a/resolvers/npm_cache/tarball_extract.rs +++ b/resolvers/npm_cache/tarball_extract.rs @@ -219,8 +219,9 @@ fn get_atomic_dir_path(file_path: &Path) -> PathBuf { } fn gen_rand_path_component() -> String { - (0..4).fold(String::new(), |mut output, _| { - output.push_str(&format!("{:02x}", rand::random::())); + use std::fmt::Write; + (0..4).fold(String::with_capacity(8), |mut output, _| { + write!(&mut output, "{:02x}", rand::random::()).unwrap(); output }) } diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index 317e619bf7..cb12abb141 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_runtime" -version = "0.190.0" +version = "0.191.0" authors.workspace = true edition.workspace = true license.workspace = true diff --git a/runtime/errors.rs b/runtime/errors.rs index 22ba640bcf..3f8e900851 100644 --- a/runtime/errors.rs +++ b/runtime/errors.rs @@ -712,7 +712,6 @@ fn get_fetch_error(error: &FetchError) -> &'static str { FetchError::ClientSend(_) => "TypeError", FetchError::RequestBuilderHook(_) => "TypeError", FetchError::Io(e) => get_io_error_class(e), - FetchError::Hyper(e) => get_hyper_error_class(e), } } @@ -1083,6 +1082,7 @@ mod node { pub use deno_node::ops::crypto::SignEd25519Error; pub use deno_node::ops::crypto::VerifyEd25519Error; pub use deno_node::ops::fs::FsError; + pub use deno_node::ops::http::ConnError; pub use deno_node::ops::http2::Http2Error; pub use deno_node::ops::idna::IdnaError; pub use deno_node::ops::ipc::IpcError; @@ -1538,6 +1538,24 @@ mod node { pub fn get_verify_ed25519_error(_: &VerifyEd25519Error) -> &'static str { "TypeError" } + + pub fn get_conn_error(e: &ConnError) -> &'static str { + match e { + ConnError::Resource(e) => get_error_class_name(e).unwrap_or("Error"), + ConnError::Permission(e) => get_permission_check_error_class(e), + ConnError::InvalidUrl(_) => "TypeError", + ConnError::InvalidHeaderName(_) => "TypeError", + ConnError::InvalidHeaderValue(_) => "TypeError", + ConnError::Url(e) => get_url_parse_error_class(e), + ConnError::Method(_) => "TypeError", + ConnError::Io(e) => get_io_error_class(e), + ConnError::Hyper(e) => super::get_hyper_error_class(e), + ConnError::TlsStreamBusy => "Busy", + ConnError::TcpStreamBusy => "Busy", + ConnError::ReuniteTcp(_) => "Error", + ConnError::Canceled(_) => "Error", + } + } } fn get_os_error(error: &OsError) -> &'static str { @@ -1730,6 +1748,10 @@ pub fn get_error_class_name(e: &AnyError) -> Option<&'static str> { e.downcast_ref::() .map(node::get_verify_ed25519_error) }) + .or_else(|| { + e.downcast_ref::() + .map(node::get_conn_error) + }) .or_else(|| e.downcast_ref::().map(get_napi_error_class)) .or_else(|| e.downcast_ref::().map(get_web_error_class)) .or_else(|| { diff --git a/runtime/lib.rs b/runtime/lib.rs index 1ce325964f..1f449dc69a 100644 --- a/runtime/lib.rs +++ b/runtime/lib.rs @@ -46,7 +46,7 @@ pub use worker_bootstrap::BootstrapOptions; pub use worker_bootstrap::WorkerExecutionMode; pub use worker_bootstrap::WorkerLogLevel; -mod shared; +pub mod shared; pub use shared::runtime; pub struct UnstableGranularFlag { diff --git a/runtime/permissions/Cargo.toml b/runtime/permissions/Cargo.toml index 0140f0594e..dc46b03310 100644 --- a/runtime/permissions/Cargo.toml +++ b/runtime/permissions/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "deno_permissions" -version = "0.41.0" +version = "0.42.0" authors.workspace = true edition.workspace = true license.workspace = true @@ -14,6 +14,7 @@ name = "deno_permissions" path = "lib.rs" [dependencies] +capacity_builder.workspace = true deno_core.workspace = true deno_path_util.workspace = true deno_terminal.workspace = true diff --git a/runtime/permissions/lib.rs b/runtime/permissions/lib.rs index a0b901d200..a1a217738d 100644 --- a/runtime/permissions/lib.rs +++ b/runtime/permissions/lib.rs @@ -1,5 +1,6 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +use capacity_builder::StringBuilder; use deno_core::parking_lot::Mutex; use deno_core::serde::de; use deno_core::serde::Deserialize; @@ -179,13 +180,18 @@ impl PermissionState { (Ok(()), false, false) } PermissionState::Prompt if prompt => { - let msg = format!( - "{} access{}", - name, - info() - .map(|info| { format!(" to {info}") }) - .unwrap_or_default(), - ); + let msg = { + let info = info(); + StringBuilder::build(|builder| { + builder.append(name); + builder.append(" access"); + if let Some(info) = &info { + builder.append(" to "); + builder.append(info); + } + }) + .unwrap() + }; match permission_prompt(&msg, name, api_name, true) { PromptResponse::Allow => { Self::log_perm_access(name, info); @@ -344,11 +350,11 @@ pub trait QueryDescriptor: Debug { fn overlaps_deny(&self, other: &Self::DenyDesc) -> bool; } -fn format_display_name(display_name: Cow) -> String { +fn format_display_name(display_name: Cow) -> Cow { if display_name.starts_with('<') && display_name.ends_with('>') { - display_name.into_owned() + display_name } else { - format!("\"{}\"", display_name) + Cow::Owned(format!("\"{}\"", display_name)) } } @@ -424,7 +430,7 @@ impl UnaryPermission { .check2( TQuery::flag_name(), api_name, - || desc.map(|d| format_display_name(d.display_name())), + || desc.map(|d| format_display_name(d.display_name()).into_owned()), self.prompt, ); if prompted { @@ -487,12 +493,17 @@ impl UnaryPermission { if !self.prompt { return PermissionState::Denied; } - let mut message = String::with_capacity(40); - message.push_str(&format!("{} access", TQuery::flag_name())); - if let Some(desc) = desc { - message - .push_str(&format!(" to {}", format_display_name(desc.display_name()))); - } + let maybe_formatted_display_name = + desc.map(|d| format_display_name(d.display_name())); + let message = StringBuilder::build(|builder| { + builder.append(TQuery::flag_name()); + builder.append(" access"); + if let Some(display_name) = &maybe_formatted_display_name { + builder.append(" to "); + builder.append(display_name) + } + }) + .unwrap(); match permission_prompt( &message, TQuery::flag_name(), diff --git a/runtime/worker_bootstrap.rs b/runtime/worker_bootstrap.rs index 4a8c5dba86..2020c2bc8d 100644 --- a/runtime/worker_bootstrap.rs +++ b/runtime/worker_bootstrap.rs @@ -119,8 +119,7 @@ pub struct BootstrapOptions { // Used by `deno serve` pub serve_port: Option, pub serve_host: Option, - // OpenTelemetry output options. If `None`, OpenTelemetry is disabled. - pub otel_config: Option, + pub otel_config: OtelConfig, } impl Default for BootstrapOptions { @@ -155,7 +154,7 @@ impl Default for BootstrapOptions { mode: WorkerExecutionMode::None, serve_port: Default::default(), serve_host: Default::default(), - otel_config: None, + otel_config: Default::default(), } } } @@ -225,11 +224,7 @@ impl BootstrapOptions { self.serve_host.as_deref(), serve_is_main, serve_worker_count, - if let Some(otel_config) = self.otel_config.as_ref() { - Box::new([otel_config.console as u8, otel_config.deterministic as u8]) - } else { - Box::new([]) - }, + self.otel_config.as_v8(), ); bootstrap.serialize(ser).unwrap() diff --git a/tests/integration/check_tests.rs b/tests/integration/check_tests.rs index 178ac6493f..b98d719fca 100644 --- a/tests/integration/check_tests.rs +++ b/tests/integration/check_tests.rs @@ -3,23 +3,9 @@ use deno_lockfile::NewLockfileOptions; use deno_semver::jsr::JsrDepPackageReq; use test_util as util; -use test_util::itest; use util::TestContext; use util::TestContextBuilder; -itest!(check_all { - args: "check --allow-import --quiet --all check/all/check_all.ts", - output: "check/all/check_all.out", - http_server: true, - exit_code: 1, -}); - -itest!(check_all_local { - args: "check --allow-import --quiet check/all/check_all.ts", - output_str: Some(""), - http_server: true, -}); - #[test] fn cache_switching_config_then_no_config() { let context = TestContext::default(); diff --git a/tests/integration/compile_tests.rs b/tests/integration/compile_tests.rs index fa6364a136..a34d2cdd1d 100644 --- a/tests/integration/compile_tests.rs +++ b/tests/integration/compile_tests.rs @@ -846,21 +846,6 @@ testing[WILDCARD]this .assert_matches_text("2\n"); } -#[test] -fn compile_npm_file_system() { - run_npm_bin_compile_test(RunNpmBinCompileOptions { - input_specifier: "compile/npm_fs/main.ts", - copy_temp_dir: Some("compile/npm_fs"), - compile_args: vec!["-A"], - run_args: vec![], - output_file: "compile/npm_fs/main.out", - node_modules_local: true, - input_name: Some("binary"), - expected_name: "binary", - exit_code: 0, - }); -} - #[test] fn compile_npm_bin_esm() { run_npm_bin_compile_test(RunNpmBinCompileOptions { @@ -906,21 +891,6 @@ fn compile_npm_cowsay_main() { }); } -#[test] -fn compile_npm_vfs_implicit_read_permissions() { - run_npm_bin_compile_test(RunNpmBinCompileOptions { - input_specifier: "compile/vfs_implicit_read_permission/main.ts", - copy_temp_dir: Some("compile/vfs_implicit_read_permission"), - compile_args: vec![], - run_args: vec![], - output_file: "compile/vfs_implicit_read_permission/main.out", - node_modules_local: false, - input_name: Some("binary"), - expected_name: "binary", - exit_code: 0, - }); -} - #[test] fn compile_npm_no_permissions() { run_npm_bin_compile_test(RunNpmBinCompileOptions { @@ -1045,6 +1015,7 @@ fn compile_node_modules_symlink_outside() { let symlink_target_dir = temp_dir.path().join("some_folder"); project_dir.join("node_modules").create_dir_all(); symlink_target_dir.create_dir_all(); + symlink_target_dir.join("file.txt").write("5"); let symlink_target_file = temp_dir.path().join("target.txt"); symlink_target_file.write("5"); let symlink_dir = project_dir.join("node_modules").join("symlink_dir"); @@ -1119,6 +1090,11 @@ Compile file:///[WILDCARD]/main.ts to [WILDCARD] Warning Failed resolving symlink. Ignoring. Path: [WILDCARD] Message: [WILDCARD]) + +Embedded File System + +[WILDCARD] + "#, ); diff --git a/tests/integration/lsp_tests.rs b/tests/integration/lsp_tests.rs index decc635ffa..13a3c0d69b 100644 --- a/tests/integration/lsp_tests.rs +++ b/tests/integration/lsp_tests.rs @@ -2701,7 +2701,7 @@ fn lsp_hover_dependency() { "uri": "file:///a/file.ts", "languageId": "typescript", "version": 1, - "text": "import * as a from \"http://127.0.0.1:4545/xTypeScriptTypes.js\";\n// @deno-types=\"http://127.0.0.1:4545/type_definitions/foo.d.ts\"\nimport * as b from \"http://127.0.0.1:4545/type_definitions/foo.js\";\nimport * as c from \"http://127.0.0.1:4545/subdir/type_reference.js\";\nimport * as d from \"http://127.0.0.1:4545/subdir/mod1.ts\";\nimport * as e from \"data:application/typescript;base64,ZXhwb3J0IGNvbnN0IGEgPSAiYSI7CgpleHBvcnQgZW51bSBBIHsKICBBLAogIEIsCiAgQywKfQo=\";\nimport * as f from \"./file_01.ts\";\nimport * as g from \"http://localhost:4545/x/a/mod.ts\";\nimport * as h from \"./mod🦕.ts\";\n\nconsole.log(a, b, c, d, e, f, g, h);\n" + "text": "import * as a from \"http://127.0.0.1:4545/xTypeScriptTypes.js\";\n// @ts-types=\"http://127.0.0.1:4545/type_definitions/foo.d.ts\"\nimport * as b from \"http://127.0.0.1:4545/type_definitions/foo.js\";\nimport * as c from \"http://127.0.0.1:4545/subdir/type_reference.js\";\nimport * as d from \"http://127.0.0.1:4545/subdir/mod1.ts\";\nimport * as e from \"data:application/typescript;base64,ZXhwb3J0IGNvbnN0IGEgPSAiYSI7CgpleHBvcnQgZW51bSBBIHsKICBBLAogIEIsCiAgQywKfQo=\";\nimport * as f from \"./file_01.ts\";\nimport * as g from \"http://localhost:4545/x/a/mod.ts\";\nimport * as h from \"./mod🦕.ts\";\n\nconsole.log(a, b, c, d, e, f, g, h);\n" } }), ); @@ -6269,7 +6269,7 @@ fn lsp_code_actions_deno_types_for_npm() { res, json!([ { - "title": "Add @deno-types directive for \"@types/react\"", + "title": "Add @ts-types directive for \"@types/react\"", "kind": "quickfix", "edit": { "changes": { @@ -6279,7 +6279,7 @@ fn lsp_code_actions_deno_types_for_npm() { "start": { "line": 0, "character": 0 }, "end": { "line": 0, "character": 0 }, }, - "newText": "// @deno-types=\"@types/react\"\n", + "newText": "// @ts-types=\"@types/react\"\n", }, ], }, @@ -6322,7 +6322,7 @@ fn lsp_code_actions_deno_types_for_npm() { res, json!([ { - "title": "Add @deno-types directive for \"npm:@types/react@^18.3.10\"", + "title": "Add @ts-types directive for \"npm:@types/react@^18.3.10\"", "kind": "quickfix", "edit": { "changes": { @@ -6332,7 +6332,7 @@ fn lsp_code_actions_deno_types_for_npm() { "start": { "line": 0, "character": 0 }, "end": { "line": 0, "character": 0 }, }, - "newText": "// @deno-types=\"npm:@types/react@^18.3.10\"\n", + "newText": "// @ts-types=\"npm:@types/react@^18.3.10\"\n", }, ], }, @@ -8310,7 +8310,7 @@ fn lsp_npm_auto_import_with_deno_types() { temp_dir.write( "other.ts", r#" - // @deno-types="@types/lz-string" + // @ts-types="@types/lz-string" import "lz-string"; "#, ); @@ -8358,7 +8358,7 @@ fn lsp_npm_auto_import_with_deno_types() { "start": { "line": 0, "character": 0 }, "end": { "line": 0, "character": 0 }, }, - "newText": "// @deno-types=\"@types/lz-string\"\nimport { compressToBase64 } from \"lz-string\";\n", + "newText": "// @ts-types=\"@types/lz-string\"\nimport { compressToBase64 } from \"lz-string\";\n", }, ], }), @@ -8391,7 +8391,7 @@ fn lsp_npm_auto_import_with_deno_types() { "start": { "line": 0, "character": 0 }, "end": { "line": 0, "character": 0 }, }, - "newText": "// @deno-types=\"@types/react\"\nimport { createRef } from \"react\";\n", + "newText": "// @ts-types=\"@types/react\"\nimport { createRef } from \"react\";\n", }, ], }), @@ -10226,7 +10226,7 @@ fn lsp_cache_location() { "uri": "file:///a/file.ts", "languageId": "typescript", "version": 1, - "text": "import * as a from \"http://127.0.0.1:4545/xTypeScriptTypes.js\";\n// @deno-types=\"http://127.0.0.1:4545/type_definitions/foo.d.ts\"\nimport * as b from \"http://127.0.0.1:4545/type_definitions/foo.js\";\nimport * as c from \"http://127.0.0.1:4545/subdir/type_reference.js\";\nimport * as d from \"http://127.0.0.1:4545/subdir/mod1.ts\";\nimport * as e from \"data:application/typescript;base64,ZXhwb3J0IGNvbnN0IGEgPSAiYSI7CgpleHBvcnQgZW51bSBBIHsKICBBLAogIEIsCiAgQywKfQo=\";\nimport * as f from \"./file_01.ts\";\nimport * as g from \"http://localhost:4545/x/a/mod.ts\";\n\nconsole.log(a, b, c, d, e, f, g);\n" + "text": "import * as a from \"http://127.0.0.1:4545/xTypeScriptTypes.js\";\n// @ts-types=\"http://127.0.0.1:4545/type_definitions/foo.d.ts\"\nimport * as b from \"http://127.0.0.1:4545/type_definitions/foo.js\";\nimport * as c from \"http://127.0.0.1:4545/subdir/type_reference.js\";\nimport * as d from \"http://127.0.0.1:4545/subdir/mod1.ts\";\nimport * as e from \"data:application/typescript;base64,ZXhwb3J0IGNvbnN0IGEgPSAiYSI7CgpleHBvcnQgZW51bSBBIHsKICBBLAogIEIsCiAgQywKfQo=\";\nimport * as f from \"./file_01.ts\";\nimport * as g from \"http://localhost:4545/x/a/mod.ts\";\n\nconsole.log(a, b, c, d, e, f, g);\n" } })); assert_eq!(diagnostics.all().len(), 6); @@ -10319,7 +10319,7 @@ fn lsp_tls_cert() { "uri": "file:///a/file.ts", "languageId": "typescript", "version": 1, - "text": "import * as a from \"https://localhost:5545/xTypeScriptTypes.js\";\n// @deno-types=\"https://localhost:5545/type_definitions/foo.d.ts\"\nimport * as b from \"https://localhost:5545/type_definitions/foo.js\";\nimport * as c from \"https://localhost:5545/subdir/type_reference.js\";\nimport * as d from \"https://localhost:5545/subdir/mod1.ts\";\nimport * as e from \"data:application/typescript;base64,ZXhwb3J0IGNvbnN0IGEgPSAiYSI7CgpleHBvcnQgZW51bSBBIHsKICBBLAogIEIsCiAgQywKfQo=\";\nimport * as f from \"./file_01.ts\";\nimport * as g from \"http://localhost:4545/x/a/mod.ts\";\n\nconsole.log(a, b, c, d, e, f, g);\n" + "text": "import * as a from \"https://localhost:5545/xTypeScriptTypes.js\";\n// @ts-types=\"https://localhost:5545/type_definitions/foo.d.ts\"\nimport * as b from \"https://localhost:5545/type_definitions/foo.js\";\nimport * as c from \"https://localhost:5545/subdir/type_reference.js\";\nimport * as d from \"https://localhost:5545/subdir/mod1.ts\";\nimport * as e from \"data:application/typescript;base64,ZXhwb3J0IGNvbnN0IGEgPSAiYSI7CgpleHBvcnQgZW51bSBBIHsKICBBLAogIEIsCiAgQywKfQo=\";\nimport * as f from \"./file_01.ts\";\nimport * as g from \"http://localhost:4545/x/a/mod.ts\";\n\nconsole.log(a, b, c, d, e, f, g);\n" } })); let diagnostics = diagnostics.all(); @@ -10784,7 +10784,7 @@ fn lsp_diagnostics_deno_types() { "uri": "file:///a/file.ts", "languageId": "typescript", "version": 1, - "text": "/// \n/// \n/// "); + let file = source_file(temp_dir.path().join("file.html"), " "); let mut client = context.new_lsp_command().build(); client.initialize_default(); let res = client.write_request( "textDocument/formatting", json!({ - "textDocument": { "uri": html_file.url() }, + "textDocument": { "uri": file.url() }, "options": { "tabSize": 2, "insertSpaces": true, @@ -11627,13 +11625,13 @@ fn lsp_format_html() { fn lsp_format_css() { let context = TestContextBuilder::new().use_temp_cwd().build(); let temp_dir = context.temp_dir(); - let css_file = source_file(temp_dir.path().join("file.css"), " foo {}"); + let file = source_file(temp_dir.path().join("file.css"), " foo {}"); let mut client = context.new_lsp_command().build(); client.initialize_default(); let res = client.write_request( "textDocument/formatting", json!({ - "textDocument": { "uri": css_file.url() }, + "textDocument": { "uri": file.url() }, "options": { "tabSize": 2, "insertSpaces": true, @@ -11666,13 +11664,13 @@ fn lsp_format_css() { fn lsp_format_yaml() { let context = TestContextBuilder::new().use_temp_cwd().build(); let temp_dir = context.temp_dir(); - let yaml_file = source_file(temp_dir.path().join("file.yaml"), " foo: 1"); + let file = source_file(temp_dir.path().join("file.yaml"), " foo: 1"); let mut client = context.new_lsp_command().build(); client.initialize_default(); let res = client.write_request( "textDocument/formatting", json!({ - "textDocument": { "uri": yaml_file.url() }, + "textDocument": { "uri": file.url() }, "options": { "tabSize": 2, "insertSpaces": true, @@ -11701,6 +11699,201 @@ fn lsp_format_yaml() { client.shutdown(); } +#[test] +fn lsp_format_sql() { + let context = TestContextBuilder::new().use_temp_cwd().build(); + let temp_dir = context.temp_dir(); + temp_dir.write( + "deno.json", + json!({ + "unstable": ["fmt-sql"], + }) + .to_string(), + ); + let file = source_file( + temp_dir.path().join("file.sql"), + " CREATE TABLE item (id int NOT NULL IDENTITY(1, 1))", + ); + let mut client = context.new_lsp_command().build(); + client.initialize_default(); + let res = client.write_request( + "textDocument/formatting", + json!({ + "textDocument": { "uri": file.url() }, + "options": { + "tabSize": 2, + "insertSpaces": true, + }, + }), + ); + assert_eq!( + res, + json!([ + { + "range": { + "start": { "line": 0, "character": 0 }, + "end": { "line": 0, "character": 2 }, + }, + "newText": "", + }, + { + "range": { + "start": { "line": 0, "character": 52 }, + "end": { "line": 0, "character": 52 }, + }, + "newText": "\n", + }, + ]), + ); + client.shutdown(); +} + +#[test] +fn lsp_format_component() { + let context = TestContextBuilder::new().use_temp_cwd().build(); + let temp_dir = context.temp_dir(); + temp_dir.write( + "deno.json", + json!({ + "unstable": ["fmt-component"], + }) + .to_string(), + ); + let svelte_file = source_file( + temp_dir.path().join("file.svelte"), + " \n", + ); + let vue_file = source_file( + temp_dir.path().join("file.vue"), + " \n", + ); + let astro_file = source_file( + temp_dir.path().join("file.astro"), + " ---\n// foo\n---\n\n", + ); + let vento_file = source_file( + temp_dir.path().join("file.vto"), + " {{ layout \"foo.vto\" }}\n

Foo!

\n{{ /layout }}\n", + ); + let nunjucks_file = source_file( + temp_dir.path().join("file.njk"), + " {% block header %}\n Foo\n{% endblock %}\n", + ); + let mut client = context.new_lsp_command().build(); + client.initialize_default(); + let res = client.write_request( + "textDocument/formatting", + json!({ + "textDocument": { "uri": svelte_file.url() }, + "options": { + "tabSize": 2, + "insertSpaces": true, + }, + }), + ); + assert_eq!( + res, + json!([ + { + "range": { + "start": { "line": 0, "character": 0 }, + "end": { "line": 0, "character": 2 }, + }, + "newText": "", + }, + ]), + ); + let res = client.write_request( + "textDocument/formatting", + json!({ + "textDocument": { "uri": vue_file.url() }, + "options": { + "tabSize": 2, + "insertSpaces": true, + }, + }), + ); + assert_eq!( + res, + json!([ + { + "range": { + "start": { "line": 0, "character": 0 }, + "end": { "line": 0, "character": 2 }, + }, + "newText": "", + }, + ]), + ); + let res = client.write_request( + "textDocument/formatting", + json!({ + "textDocument": { "uri": astro_file.url() }, + "options": { + "tabSize": 2, + "insertSpaces": true, + }, + }), + ); + assert_eq!( + res, + json!([ + { + "range": { + "start": { "line": 0, "character": 0 }, + "end": { "line": 0, "character": 2 }, + }, + "newText": "", + }, + ]), + ); + let res = client.write_request( + "textDocument/formatting", + json!({ + "textDocument": { "uri": vento_file.url() }, + "options": { + "tabSize": 2, + "insertSpaces": true, + }, + }), + ); + assert_eq!( + res, + json!([ + { + "range": { + "start": { "line": 0, "character": 0 }, + "end": { "line": 0, "character": 2 }, + }, + "newText": "", + }, + ]), + ); + let res = client.write_request( + "textDocument/formatting", + json!({ + "textDocument": { "uri": nunjucks_file.url() }, + "options": { + "tabSize": 2, + "insertSpaces": true, + }, + }), + ); + assert_eq!( + res, + json!([ + { + "range": { + "start": { "line": 0, "character": 0 }, + "end": { "line": 0, "character": 2 }, + }, + "newText": "", + }, + ]), + ); + client.shutdown(); +} + #[test] fn lsp_format_with_config() { let context = TestContextBuilder::new().use_temp_cwd().build(); @@ -15782,7 +15975,7 @@ fn lsp_sloppy_imports() { "import * as b from './b.js';\n", // this one's types resolve to a .d.ts file and we don't // bother warning about it because it's a bit complicated - // to explain to use @deno-types in a diagnostic + // to explain to use @ts-types in a diagnostic "import * as c from './c.js';\n", "console.log(a)\n", "console.log(b);\n", diff --git a/tests/node_compat/config.jsonc b/tests/node_compat/config.jsonc index 105341109c..cda2923789 100644 --- a/tests/node_compat/config.jsonc +++ b/tests/node_compat/config.jsonc @@ -565,9 +565,7 @@ "test-handle-wrap-close-abort.js", "test-http-abort-before-end.js", "test-http-addrequest-localaddress.js", - "test-http-agent-false.js", "test-http-agent-getname.js", - "test-http-agent-keepalive-delay.js", "test-http-agent-maxtotalsockets.js", "test-http-agent-no-protocol.js", "test-http-agent-null.js", @@ -590,7 +588,6 @@ "test-http-client-race.js", "test-http-client-read-in-error.js", "test-http-client-reject-unexpected-agent.js", - "test-http-client-timeout-connect-listener.js", "test-http-client-timeout-with-data.js", "test-http-client-unescaped-path.js", "test-http-client-upload-buf.js", @@ -604,7 +601,6 @@ "test-http-date-header.js", "test-http-decoded-auth.js", "test-http-default-encoding.js", - "test-http-dump-req-when-res-ends.js", "test-http-end-throw-socket-handling.js", "test-http-eof-on-connect.js", "test-http-extra-response.js", @@ -622,7 +618,6 @@ "test-http-hex-write.js", "test-http-highwatermark.js", "test-http-host-headers.js", - "test-http-hostname-typechecking.js", "test-http-incoming-message-destroy.js", "test-http-invalid-path-chars.js", "test-http-invalidheaderfield.js", @@ -1292,10 +1287,7 @@ "test-buffer-creation-regression.js", "test-child-process-exit.js", "test-http-server-keep-alive-timeout-slow-server.js", - "test-net-better-error-messages-port.js", - "test-net-connect-handle-econnrefused.js", "test-net-connect-local-error.js", - "test-net-reconnect-error.js", "test-net-response-size.js", "test-net-server-bind.js", "test-tls-lookup.js", diff --git a/tests/node_compat/runner/TODO.md b/tests/node_compat/runner/TODO.md index 09d68aded7..8ad00c9bfd 100644 --- a/tests/node_compat/runner/TODO.md +++ b/tests/node_compat/runner/TODO.md @@ -1,7 +1,7 @@ # Remaining Node Tests -1163 tests out of 3681 have been ported from Node 20.11.1 (31.59% ported, 68.92% remaining). +1155 tests out of 3681 have been ported from Node 20.11.1 (31.38% ported, 69.14% remaining). NOTE: This file should not be manually edited. Please edit `tests/node_compat/config.json` and run `deno task setup` in `tests/node_compat/runner` dir instead. @@ -792,6 +792,8 @@ NOTE: This file should not be manually edited. Please edit `tests/node_compat/co - [parallel/test-http-agent-destroyed-socket.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-agent-destroyed-socket.js) - [parallel/test-http-agent-domain-reused-gc.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-agent-domain-reused-gc.js) - [parallel/test-http-agent-error-on-idle.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-agent-error-on-idle.js) +- [parallel/test-http-agent-false.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-agent-false.js) +- [parallel/test-http-agent-keepalive-delay.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-agent-keepalive-delay.js) - [parallel/test-http-agent-keepalive.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-agent-keepalive.js) - [parallel/test-http-agent-maxsockets-respected.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-agent-maxsockets-respected.js) - [parallel/test-http-agent-maxsockets.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-agent-maxsockets.js) @@ -848,6 +850,7 @@ NOTE: This file should not be manually edited. Please edit `tests/node_compat/co - [parallel/test-http-client-set-timeout.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-client-set-timeout.js) - [parallel/test-http-client-spurious-aborted.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-client-spurious-aborted.js) - [parallel/test-http-client-timeout-agent.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-client-timeout-agent.js) +- [parallel/test-http-client-timeout-connect-listener.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-client-timeout-connect-listener.js) - [parallel/test-http-client-timeout-event.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-client-timeout-event.js) - [parallel/test-http-client-timeout-on-connect.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-client-timeout-on-connect.js) - [parallel/test-http-client-timeout-option-listeners.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-client-timeout-option-listeners.js) @@ -865,6 +868,7 @@ NOTE: This file should not be manually edited. Please edit `tests/node_compat/co - [parallel/test-http-destroyed-socket-write2.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-destroyed-socket-write2.js) - [parallel/test-http-dns-error.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-dns-error.js) - [parallel/test-http-double-content-length.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-double-content-length.js) +- [parallel/test-http-dump-req-when-res-ends.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-dump-req-when-res-ends.js) - [parallel/test-http-early-hints-invalid-argument.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-early-hints-invalid-argument.js) - [parallel/test-http-early-hints.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-early-hints.js) - [parallel/test-http-exceptions.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-exceptions.js) @@ -876,6 +880,7 @@ NOTE: This file should not be manually edited. Please edit `tests/node_compat/co - [parallel/test-http-header-badrequest.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-header-badrequest.js) - [parallel/test-http-header-overflow.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-header-overflow.js) - [parallel/test-http-host-header-ipv6-fail.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-host-header-ipv6-fail.js) +- [parallel/test-http-hostname-typechecking.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-hostname-typechecking.js) - [parallel/test-http-incoming-matchKnownFields.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-incoming-matchKnownFields.js) - [parallel/test-http-incoming-message-connection-setter.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-incoming-message-connection-setter.js) - [parallel/test-http-incoming-message-options.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-http-incoming-message-options.js) @@ -2508,9 +2513,12 @@ NOTE: This file should not be manually edited. Please edit `tests/node_compat/co - [sequential/test-inspector-port-cluster.js](https://github.com/nodejs/node/tree/v20.11.1/test/sequential/test-inspector-port-cluster.js) - [sequential/test-module-loading.js](https://github.com/nodejs/node/tree/v20.11.1/test/sequential/test-module-loading.js) - [sequential/test-net-GH-5504.js](https://github.com/nodejs/node/tree/v20.11.1/test/sequential/test-net-GH-5504.js) +- [sequential/test-net-better-error-messages-port.js](https://github.com/nodejs/node/tree/v20.11.1/test/sequential/test-net-better-error-messages-port.js) - [sequential/test-net-connect-econnrefused.js](https://github.com/nodejs/node/tree/v20.11.1/test/sequential/test-net-connect-econnrefused.js) +- [sequential/test-net-connect-handle-econnrefused.js](https://github.com/nodejs/node/tree/v20.11.1/test/sequential/test-net-connect-handle-econnrefused.js) - [sequential/test-net-listen-shared-ports.js](https://github.com/nodejs/node/tree/v20.11.1/test/sequential/test-net-listen-shared-ports.js) - [sequential/test-net-localport.js](https://github.com/nodejs/node/tree/v20.11.1/test/sequential/test-net-localport.js) +- [sequential/test-net-reconnect-error.js](https://github.com/nodejs/node/tree/v20.11.1/test/sequential/test-net-reconnect-error.js) - [sequential/test-net-server-address.js](https://github.com/nodejs/node/tree/v20.11.1/test/sequential/test-net-server-address.js) - [sequential/test-next-tick-error-spin.js](https://github.com/nodejs/node/tree/v20.11.1/test/sequential/test-next-tick-error-spin.js) - [sequential/test-perf-hooks.js](https://github.com/nodejs/node/tree/v20.11.1/test/sequential/test-perf-hooks.js) diff --git a/tests/node_compat/test/parallel/test-http-agent-false.js b/tests/node_compat/test/parallel/test-http-agent-false.js deleted file mode 100644 index 60dc16d9b0..0000000000 --- a/tests/node_compat/test/parallel/test-http-agent-false.js +++ /dev/null @@ -1,53 +0,0 @@ -// deno-fmt-ignore-file -// deno-lint-ignore-file - -// Copyright Joyent and Node contributors. All rights reserved. MIT license. -// Taken from Node 20.11.1 -// This file is automatically generated by `tests/node_compat/runner/setup.ts`. Do not modify this file manually. - -// Copyright Joyent, Inc. and other Node contributors. -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the -// "Software"), to deal in the Software without restriction, including -// without limitation the rights to use, copy, modify, merge, publish, -// distribute, sublicense, and/or sell copies of the Software, and to permit -// persons to whom the Software is furnished to do so, subject to the -// following conditions: -// -// The above copyright notice and this permission notice shall be included -// in all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN -// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE -// USE OR OTHER DEALINGS IN THE SOFTWARE. - -'use strict'; -const common = require('../common'); -const http = require('http'); - -// Sending `agent: false` when `port: null` is also passed in (i.e. the result -// of a `url.parse()` call with the default port used, 80 or 443), should not -// result in an assertion error... -const opts = { - host: '127.0.0.1', - port: null, - path: '/', - method: 'GET', - agent: false -}; - -// We just want an "error" (no local HTTP server on port 80) or "response" -// to happen (user happens ot have HTTP server running on port 80). -// As long as the process doesn't crash from a C++ assertion then we're good. -const req = http.request(opts); - -// Will be called by either the response event or error event, not both -const oneResponse = common.mustCall(); -req.on('response', oneResponse); -req.on('error', oneResponse); -req.end(); diff --git a/tests/node_compat/test/parallel/test-http-agent-keepalive-delay.js b/tests/node_compat/test/parallel/test-http-agent-keepalive-delay.js deleted file mode 100644 index 7cc6120d73..0000000000 --- a/tests/node_compat/test/parallel/test-http-agent-keepalive-delay.js +++ /dev/null @@ -1,43 +0,0 @@ -// deno-fmt-ignore-file -// deno-lint-ignore-file - -// Copyright Joyent and Node contributors. All rights reserved. MIT license. -// Taken from Node 20.11.1 -// This file is automatically generated by `tests/node_compat/runner/setup.ts`. Do not modify this file manually. - -'use strict'; - -const common = require('../common'); -const assert = require('assert'); -const http = require('http'); -const { Agent } = require('_http_agent'); - -const agent = new Agent({ - keepAlive: true, - keepAliveMsecs: 1000, -}); - -const server = http.createServer(common.mustCall((req, res) => { - res.end('ok'); -})); - -server.listen(0, common.mustCall(() => { - const createConnection = agent.createConnection; - agent.createConnection = (options, ...args) => { - assert.strictEqual(options.keepAlive, true); - assert.strictEqual(options.keepAliveInitialDelay, agent.keepAliveMsecs); - return createConnection.call(agent, options, ...args); - }; - http.get({ - host: 'localhost', - port: server.address().port, - agent: agent, - path: '/' - }, common.mustCall((res) => { - // for emit end event - res.on('data', () => {}); - res.on('end', () => { - server.close(); - }); - })); -})); diff --git a/tests/node_compat/test/parallel/test-http-client-timeout-connect-listener.js b/tests/node_compat/test/parallel/test-http-client-timeout-connect-listener.js deleted file mode 100644 index c151d16556..0000000000 --- a/tests/node_compat/test/parallel/test-http-client-timeout-connect-listener.js +++ /dev/null @@ -1,49 +0,0 @@ -// deno-fmt-ignore-file -// deno-lint-ignore-file - -// Copyright Joyent and Node contributors. All rights reserved. MIT license. -// Taken from Node 20.11.1 -// This file is automatically generated by `tests/node_compat/runner/setup.ts`. Do not modify this file manually. - -'use strict'; -const common = require('../common'); - -// This test ensures that `ClientRequest.prototype.setTimeout()` does -// not add a listener for the `'connect'` event to the socket if the -// socket is already connected. - -const assert = require('assert'); -const http = require('http'); - -// Maximum allowed value for timeouts. -const timeout = 2 ** 31 - 1; - -const server = http.createServer((req, res) => { - res.end(); -}); - -server.listen(0, common.mustCall(() => { - const agent = new http.Agent({ keepAlive: true, maxSockets: 1 }); - const options = { port: server.address().port, agent: agent }; - - doRequest(options, common.mustCall(() => { - const req = doRequest(options, common.mustCall(() => { - agent.destroy(); - server.close(); - })); - - req.on('socket', common.mustCall((socket) => { - assert.strictEqual(socket.listenerCount('connect'), 0); - })); - })); -})); - -function doRequest(options, callback) { - const req = http.get(options, (res) => { - res.on('end', callback); - res.resume(); - }); - - req.setTimeout(timeout); - return req; -} diff --git a/tests/node_compat/test/parallel/test-http-dump-req-when-res-ends.js b/tests/node_compat/test/parallel/test-http-dump-req-when-res-ends.js deleted file mode 100644 index 3b94250f5a..0000000000 --- a/tests/node_compat/test/parallel/test-http-dump-req-when-res-ends.js +++ /dev/null @@ -1,73 +0,0 @@ -// deno-fmt-ignore-file -// deno-lint-ignore-file - -// Copyright Joyent and Node contributors. All rights reserved. MIT license. -// Taken from Node 20.11.1 -// This file is automatically generated by `tests/node_compat/runner/setup.ts`. Do not modify this file manually. - -'use strict'; - -const { mustCall } = require('../common'); - -const fs = require('fs'); -const http = require('http'); -const { strictEqual } = require('assert'); - -const server = http.createServer(mustCall(function(req, res) { - strictEqual(req.socket.listenerCount('data'), 1); - req.socket.once('data', mustCall(function() { - // Ensure that a chunk of data is received before calling `res.end()`. - res.end('hello world'); - })); - // This checks if the request gets dumped - // resume will be triggered by res.end(). - req.on('resume', mustCall(function() { - // There is no 'data' event handler anymore - // it gets automatically removed when dumping the request. - strictEqual(req.listenerCount('data'), 0); - req.on('data', mustCall()); - })); - - // We explicitly pause the stream - // so that the following on('data') does not cause - // a resume. - req.pause(); - req.on('data', function() {}); - - // Start sending the response. - res.flushHeaders(); -})); - -server.listen(0, mustCall(function() { - const req = http.request({ - method: 'POST', - port: server.address().port - }); - - // Send the http request without waiting - // for the body. - req.flushHeaders(); - - req.on('response', mustCall(function(res) { - // Pipe the body as soon as we get the headers of the - // response back. - fs.createReadStream(__filename).pipe(req); - - res.resume(); - - // On some platforms the `'end'` event might not be emitted because the - // socket could be destroyed by the other peer while data is still being - // sent. In this case the 'aborted'` event is emitted instead of `'end'`. - // `'close'` is used here because it is always emitted and does not - // invalidate the test. - res.on('close', function() { - server.close(); - }); - })); - - req.on('error', function() { - // An error can happen if there is some data still - // being sent, as the other side is calling .destroy() - // this is safe to ignore. - }); -})); diff --git a/tests/node_compat/test/parallel/test-http-hostname-typechecking.js b/tests/node_compat/test/parallel/test-http-hostname-typechecking.js deleted file mode 100644 index e42384504b..0000000000 --- a/tests/node_compat/test/parallel/test-http-hostname-typechecking.js +++ /dev/null @@ -1,49 +0,0 @@ -// deno-fmt-ignore-file -// deno-lint-ignore-file - -// Copyright Joyent and Node contributors. All rights reserved. MIT license. -// Taken from Node 20.11.1 -// This file is automatically generated by `tests/node_compat/runner/setup.ts`. Do not modify this file manually. - -'use strict'; - -const common = require('../common'); -const assert = require('assert'); -const http = require('http'); - -// All of these values should cause http.request() to throw synchronously -// when passed as the value of either options.hostname or options.host -const vals = [{}, [], NaN, Infinity, -Infinity, true, false, 1, 0, new Date()]; - -vals.forEach((v) => { - const received = common.invalidArgTypeHelper(v); - assert.throws( - () => http.request({ hostname: v }), - { - code: 'ERR_INVALID_ARG_TYPE', - name: 'TypeError', - message: 'The "options.hostname" property must be of ' + - 'type string or one of undefined or null.' + - received - } - ); - - assert.throws( - () => http.request({ host: v }), - { - code: 'ERR_INVALID_ARG_TYPE', - name: 'TypeError', - message: 'The "options.host" property must be of ' + - 'type string or one of undefined or null.' + - received - } - ); -}); - -// These values are OK and should not throw synchronously. -// Only testing for 'hostname' validation so ignore connection errors. -const dontCare = () => {}; -['', undefined, null].forEach((v) => { - http.request({ hostname: v }).on('error', dontCare).end(); - http.request({ host: v }).on('error', dontCare).end(); -}); diff --git a/tests/node_compat/test/sequential/test-net-better-error-messages-port.js b/tests/node_compat/test/sequential/test-net-better-error-messages-port.js deleted file mode 100644 index f718ca3f84..0000000000 --- a/tests/node_compat/test/sequential/test-net-better-error-messages-port.js +++ /dev/null @@ -1,24 +0,0 @@ -// deno-fmt-ignore-file -// deno-lint-ignore-file - -// Copyright Joyent and Node contributors. All rights reserved. MIT license. -// Taken from Node 20.11.1 -// This file is automatically generated by `tests/node_compat/runner/setup.ts`. Do not modify this file manually. - -'use strict'; -const common = require('../common'); -const net = require('net'); -const assert = require('assert'); - -const c = net.createConnection(common.PORT); - -c.on('connect', common.mustNotCall()); - -c.on('error', common.mustCall(function(error) { - // Family autoselection might be skipped if only a single address is returned by DNS. - const failedAttempt = Array.isArray(error.errors) ? error.errors[0] : error; - - assert.strictEqual(failedAttempt.code, 'ECONNREFUSED'); - assert.strictEqual(failedAttempt.port, common.PORT); - assert.match(failedAttempt.address, /^(127\.0\.0\.1|::1)$/); -})); diff --git a/tests/node_compat/test/sequential/test-net-connect-handle-econnrefused.js b/tests/node_compat/test/sequential/test-net-connect-handle-econnrefused.js deleted file mode 100644 index 629705564b..0000000000 --- a/tests/node_compat/test/sequential/test-net-connect-handle-econnrefused.js +++ /dev/null @@ -1,39 +0,0 @@ -// deno-fmt-ignore-file -// deno-lint-ignore-file - -// Copyright Joyent and Node contributors. All rights reserved. MIT license. -// Taken from Node 20.11.1 -// This file is automatically generated by `tests/node_compat/runner/setup.ts`. Do not modify this file manually. - -// Copyright Joyent, Inc. and other Node contributors. -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the -// "Software"), to deal in the Software without restriction, including -// without limitation the rights to use, copy, modify, merge, publish, -// distribute, sublicense, and/or sell copies of the Software, and to permit -// persons to whom the Software is furnished to do so, subject to the -// following conditions: -// -// The above copyright notice and this permission notice shall be included -// in all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN -// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE -// USE OR OTHER DEALINGS IN THE SOFTWARE. - -'use strict'; -const common = require('../common'); -const net = require('net'); -const assert = require('assert'); - -const c = net.createConnection(common.PORT); -c.on('connect', common.mustNotCall()); -c.on('error', common.mustCall((e) => { - assert.strictEqual(c.connecting, false); - assert.strictEqual(e.code, 'ECONNREFUSED'); -})); diff --git a/tests/node_compat/test/sequential/test-net-reconnect-error.js b/tests/node_compat/test/sequential/test-net-reconnect-error.js deleted file mode 100644 index 450a0798bf..0000000000 --- a/tests/node_compat/test/sequential/test-net-reconnect-error.js +++ /dev/null @@ -1,50 +0,0 @@ -// deno-fmt-ignore-file -// deno-lint-ignore-file - -// Copyright Joyent and Node contributors. All rights reserved. MIT license. -// Taken from Node 20.11.1 -// This file is automatically generated by `tests/node_compat/runner/setup.ts`. Do not modify this file manually. - -// Copyright Joyent, Inc. and other Node contributors. -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the -// "Software"), to deal in the Software without restriction, including -// without limitation the rights to use, copy, modify, merge, publish, -// distribute, sublicense, and/or sell copies of the Software, and to permit -// persons to whom the Software is furnished to do so, subject to the -// following conditions: -// -// The above copyright notice and this permission notice shall be included -// in all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN -// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE -// USE OR OTHER DEALINGS IN THE SOFTWARE. - -'use strict'; -const common = require('../common'); -const net = require('net'); -const assert = require('assert'); -const N = 20; -let disconnectCount = 0; - -const c = net.createConnection(common.PORT); - -c.on('connect', common.mustNotCall('client should not have connected')); - -c.on('error', common.mustCall((error) => { - // Family autoselection might be skipped if only a single address is returned by DNS. - const actualError = Array.isArray(error.errors) ? error.errors[0] : error; - - assert.strictEqual(actualError.code, 'ECONNREFUSED'); -}, N + 1)); - -c.on('close', common.mustCall(() => { - if (disconnectCount++ < N) - c.connect(common.PORT); // reconnect -}, N + 1)); diff --git a/tests/specs/add/add_with_subpath/wrong_constraint_jsr.out b/tests/specs/add/add_with_subpath/wrong_constraint_jsr.out index 7de7008e93..5f6661920a 100644 --- a/tests/specs/add/add_with_subpath/wrong_constraint_jsr.out +++ b/tests/specs/add/add_with_subpath/wrong_constraint_jsr.out @@ -1,4 +1,4 @@ -error: Failed to parse package required: jsr:@std/testing/bdd@1 +error: Failed to parse package: jsr:@std/testing/bdd@1 Caused by: - Invalid package specifier 'jsr:@std/testing/bdd@1'. Did you mean to write 'jsr:@std/testing@1/bdd'? + Invalid package specifier 'jsr:@std/testing/bdd@1'. Did you mean to write 'jsr:@std/testing@1/bdd'? If not, add a version requirement to the specifier. diff --git a/tests/specs/add/add_with_subpath/wrong_constraint_npm.out b/tests/specs/add/add_with_subpath/wrong_constraint_npm.out index 4adcf9ef6a..2298810bec 100644 --- a/tests/specs/add/add_with_subpath/wrong_constraint_npm.out +++ b/tests/specs/add/add_with_subpath/wrong_constraint_npm.out @@ -1,4 +1,4 @@ -error: Failed to parse package required: npm:preact/hooks@10 +error: Failed to parse package: npm:preact/hooks@10 Caused by: - Invalid package specifier 'npm:preact/hooks@10'. Did you mean to write 'npm:preact@10/hooks'? + Invalid package specifier 'npm:preact/hooks@10'. Did you mean to write 'npm:preact@10/hooks'? If not, add a version requirement to the specifier. diff --git a/tests/specs/check/check_all/__test__.jsonc b/tests/specs/check/check_all/__test__.jsonc new file mode 100644 index 0000000000..101e5aab9e --- /dev/null +++ b/tests/specs/check/check_all/__test__.jsonc @@ -0,0 +1,5 @@ +{ + "args": "check --allow-import --quiet --all check_all.ts", + "output": "check_all.out", + "exitCode": 1 +} diff --git a/tests/testdata/check/all/check_all.out b/tests/specs/check/check_all/check_all.out similarity index 100% rename from tests/testdata/check/all/check_all.out rename to tests/specs/check/check_all/check_all.out diff --git a/tests/testdata/check/all/check_all.ts b/tests/specs/check/check_all/check_all.ts similarity index 100% rename from tests/testdata/check/all/check_all.ts rename to tests/specs/check/check_all/check_all.ts diff --git a/tests/specs/check/check_all_local/__test__.jsonc b/tests/specs/check/check_all_local/__test__.jsonc new file mode 100644 index 0000000000..0cd1c16acb --- /dev/null +++ b/tests/specs/check/check_all_local/__test__.jsonc @@ -0,0 +1,5 @@ +{ + "args": "check --allow-import --quiet check_all_local.ts", + "output": "", + "exitCode": 0 +} diff --git a/tests/specs/check/check_all_local/check_all_local.ts b/tests/specs/check/check_all_local/check_all_local.ts new file mode 100644 index 0000000000..2ae8c2692c --- /dev/null +++ b/tests/specs/check/check_all_local/check_all_local.ts @@ -0,0 +1,3 @@ +import * as a from "http://localhost:4545/subdir/type_error.ts"; + +console.log(a.a); diff --git a/tests/specs/cli/otel_basic/main.ts b/tests/specs/cli/otel_basic/main.ts index ccba126cc1..634727cea7 100644 --- a/tests/specs/cli/otel_basic/main.ts +++ b/tests/specs/cli/otel_basic/main.ts @@ -13,6 +13,7 @@ const server = Deno.serve( const command = new Deno.Command(Deno.execPath(), { args: ["run", "-A", "-q", "--unstable-otel", Deno.args[0]], env: { + OTEL_DENO: "true", DENO_UNSTABLE_OTEL_DETERMINISTIC: "1", OTEL_EXPORTER_OTLP_PROTOCOL: "http/json", OTEL_EXPORTER_OTLP_ENDPOINT: `http://localhost:${port}`, diff --git a/tests/specs/compile/env_vars_support/compile.out b/tests/specs/compile/env_vars_support/compile.out index 2d004e7cb1..02332ea0eb 100644 --- a/tests/specs/compile/env_vars_support/compile.out +++ b/tests/specs/compile/env_vars_support/compile.out @@ -2,3 +2,9 @@ Warning Parsing failed within the specified environment file: environment.env at Check [WILDCARD]main.ts Compile [WILDCARD]main.ts to out[WILDCARD] Warning Environment variables from the file "environment.env" were embedded in the generated executable file + +Embedded File System + +out[WILDLINE] +└── main.ts + diff --git a/tests/specs/compile/global_npm_cache_implicit_read_permission/__test__.jsonc b/tests/specs/compile/global_npm_cache_implicit_read_permission/__test__.jsonc new file mode 100644 index 0000000000..d346c3ad20 --- /dev/null +++ b/tests/specs/compile/global_npm_cache_implicit_read_permission/__test__.jsonc @@ -0,0 +1,22 @@ +{ + "tempDir": true, + "steps": [{ + "if": "unix", + "args": "compile --output main main.ts", + "output": "compile.out" + }, { + "if": "unix", + "commandName": "./main", + "args": [], + "output": "main.out" + }, { + "if": "windows", + "args": "compile --output main.exe main.ts", + "output": "compile.out" + }, { + "if": "windows", + "commandName": "./main.exe", + "args": [], + "output": "main.out" + }] +} diff --git a/tests/specs/compile/global_npm_cache_implicit_read_permission/compile.out b/tests/specs/compile/global_npm_cache_implicit_read_permission/compile.out new file mode 100644 index 0000000000..c29c878593 --- /dev/null +++ b/tests/specs/compile/global_npm_cache_implicit_read_permission/compile.out @@ -0,0 +1,47 @@ +[WILDCARD] +Compile file:///[WILDLINE]/main.ts to [WILDLINE] + +Embedded File System + +main[WILDLINE] +├─┬ .deno_compile_node_modules +│ └─┬ localhost +│ ├─┬ ansi-regex +│ │ ├── 3.0.1/* +│ │ └── 5.0.1/* +│ ├── ansi-styles/4.3.0/* +│ ├── camelcase/5.3.1/* +│ ├── cliui/6.0.0/* +│ ├── color-convert/2.0.1/* +│ ├── color-name/1.1.4/* +│ ├── cowsay/1.5.0/* +│ ├── decamelize/1.2.0/* +│ ├── emoji-regex/8.0.0/* +│ ├── find-up/4.1.0/* +│ ├── get-caller-file/2.0.5/* +│ ├── get-stdin/8.0.0/* +│ ├─┬ is-fullwidth-code-point +│ │ ├── 2.0.0/* +│ │ └── 3.0.0/* +│ ├── locate-path/5.0.0/* +│ ├── p-limit/2.3.0/* +│ ├── p-locate/4.1.0/* +│ ├── p-try/2.2.0/* +│ ├── path-exists/4.0.0/* +│ ├── require-directory/2.1.1/* +│ ├── require-main-filename/2.0.0/* +│ ├── set-blocking/2.0.0/* +│ ├─┬ string-width +│ │ ├── 2.1.1/* +│ │ └── 4.2.3/* +│ ├─┬ strip-ansi +│ │ ├── 4.0.0/* +│ │ └── 6.0.1/* +│ ├── strip-final-newline/2.0.0/* +│ ├── which-module/2.0.0/* +│ ├── wrap-ansi/6.2.0/* +│ ├── y18n/4.0.3/* +│ ├── yargs/15.4.1/* +│ └── yargs-parser/18.1.3/* +└── main.ts + diff --git a/tests/testdata/compile/vfs_implicit_read_permission/main.out b/tests/specs/compile/global_npm_cache_implicit_read_permission/main.out similarity index 100% rename from tests/testdata/compile/vfs_implicit_read_permission/main.out rename to tests/specs/compile/global_npm_cache_implicit_read_permission/main.out diff --git a/tests/testdata/compile/vfs_implicit_read_permission/main.ts b/tests/specs/compile/global_npm_cache_implicit_read_permission/main.ts similarity index 100% rename from tests/testdata/compile/vfs_implicit_read_permission/main.ts rename to tests/specs/compile/global_npm_cache_implicit_read_permission/main.ts diff --git a/tests/specs/compile/include/data_files/non_existent.out b/tests/specs/compile/include/data_files/non_existent.out index a88b441ba8..54bc69ef09 100644 --- a/tests/specs/compile/include/data_files/non_existent.out +++ b/tests/specs/compile/include/data_files/non_existent.out @@ -3,4 +3,5 @@ error: Writing deno compile executable to temporary file 'main[WILDLINE]' Caused by: 0: Including [WILDLINE]does_not_exist.txt - 1: [WILDLINE] + 1: Reading [WILDLINE]does_not_exist.txt + 2: [WILDLINE] diff --git a/tests/specs/compile/include/folder_ts_file/__test__.jsonc b/tests/specs/compile/include/folder_ts_file/__test__.jsonc new file mode 100644 index 0000000000..f02ed1efc3 --- /dev/null +++ b/tests/specs/compile/include/folder_ts_file/__test__.jsonc @@ -0,0 +1,25 @@ +{ + "tempDir": true, + "steps": [{ + "if": "unix", + // notice how the math folder is not included + "args": "compile --allow-read=data --include src --output main main.js", + "output": "[WILDCARD]" + }, { + "if": "unix", + "commandName": "./main", + "args": [], + "output": "output.out", + "exitCode": 0 + }, { + "if": "windows", + "args": "compile --allow-read=data --include src --output main.exe main.js", + "output": "[WILDCARD]" + }, { + "if": "windows", + "commandName": "./main.exe", + "args": [], + "output": "output.out", + "exitCode": 0 + }] +} diff --git a/tests/specs/compile/include/folder_ts_file/main.js b/tests/specs/compile/include/folder_ts_file/main.js new file mode 100644 index 0000000000..23b490e390 --- /dev/null +++ b/tests/specs/compile/include/folder_ts_file/main.js @@ -0,0 +1,14 @@ +const mathDir = import.meta.dirname + "/math"; +const files = Array.from( + Deno.readDirSync(mathDir).map((entry) => mathDir + "/" + entry.name), +); +files.sort(); +for (const file of files) { + console.log(file); +} + +function nonAnalyzable() { + return "./src/main.ts"; +} + +await import(nonAnalyzable()); diff --git a/tests/specs/compile/include/folder_ts_file/math/add.ts b/tests/specs/compile/include/folder_ts_file/math/add.ts new file mode 100644 index 0000000000..3b399665dc --- /dev/null +++ b/tests/specs/compile/include/folder_ts_file/math/add.ts @@ -0,0 +1,3 @@ +export function add(a: number, b: number) { + return a + b; +} diff --git a/tests/specs/compile/include/folder_ts_file/output.out b/tests/specs/compile/include/folder_ts_file/output.out new file mode 100644 index 0000000000..959e3d5c76 --- /dev/null +++ b/tests/specs/compile/include/folder_ts_file/output.out @@ -0,0 +1,2 @@ +[WILDLINE]add.ts +3 diff --git a/tests/specs/compile/include/folder_ts_file/src/main.ts b/tests/specs/compile/include/folder_ts_file/src/main.ts new file mode 100644 index 0000000000..38868c3d82 --- /dev/null +++ b/tests/specs/compile/include/folder_ts_file/src/main.ts @@ -0,0 +1,2 @@ +import { add } from "../math/add.ts"; +console.log(add(1, 2)); diff --git a/tests/specs/compile/include/symlink_twice/__test__.jsonc b/tests/specs/compile/include/symlink_twice/__test__.jsonc index ebdf824f43..f0f57292a6 100644 --- a/tests/specs/compile/include/symlink_twice/__test__.jsonc +++ b/tests/specs/compile/include/symlink_twice/__test__.jsonc @@ -6,7 +6,7 @@ }, { "if": "unix", "args": "compile --allow-read=data --include . --output main link.js", - "output": "[WILDCARD]" + "output": "compile.out" }, { "if": "unix", "commandName": "./main", @@ -16,7 +16,7 @@ }, { "if": "windows", "args": "compile --allow-read=data --include . --output main.exe link.js", - "output": "[WILDCARD]" + "output": "compile.out" }, { "if": "windows", "commandName": "./main.exe", diff --git a/tests/specs/compile/include/symlink_twice/compile.out b/tests/specs/compile/include/symlink_twice/compile.out new file mode 100644 index 0000000000..c57eb9b2f1 --- /dev/null +++ b/tests/specs/compile/include/symlink_twice/compile.out @@ -0,0 +1,9 @@ +Compile [WILDLINE] + +Embedded File System + +main[WILDLINE] +├── index.js +├── link.js --> index.js +└── setup.js + diff --git a/tests/specs/compile/include/symlink_twice/setup.js b/tests/specs/compile/include/symlink_twice/setup.js index 3e713dd63e..4c7cebfaf5 100644 --- a/tests/specs/compile/include/symlink_twice/setup.js +++ b/tests/specs/compile/include/symlink_twice/setup.js @@ -1,3 +1,2 @@ -Deno.mkdirSync("data"); Deno.writeTextFileSync("index.js", "console.log(1);"); Deno.symlinkSync("index.js", "link.js"); diff --git a/tests/specs/compile/npm_fs/__test__.jsonc b/tests/specs/compile/npm_fs/__test__.jsonc new file mode 100644 index 0000000000..a8198bfb5d --- /dev/null +++ b/tests/specs/compile/npm_fs/__test__.jsonc @@ -0,0 +1,24 @@ +{ + "tempDir": true, + // use this so the vfs output is all in the same folder + "canonicalizedTempDir": true, + "steps": [{ + "if": "unix", + "args": "compile -A --output main main.ts", + "output": "compile.out" + }, { + "if": "unix", + "commandName": "./main", + "args": [], + "output": "main.out" + }, { + "if": "windows", + "args": "compile -A --output main.exe main.ts", + "output": "compile.out" + }, { + "if": "windows", + "commandName": "./main.exe", + "args": [], + "output": "main.out" + }] +} diff --git a/tests/specs/compile/npm_fs/compile.out b/tests/specs/compile/npm_fs/compile.out new file mode 100644 index 0000000000..4944146788 --- /dev/null +++ b/tests/specs/compile/npm_fs/compile.out @@ -0,0 +1,8 @@ +[WILDCARD] + +Embedded File System + +main[WILDLINE] +├── main.ts +└── node_modules/* + diff --git a/tests/specs/compile/npm_fs/deno.json b/tests/specs/compile/npm_fs/deno.json new file mode 100644 index 0000000000..fbd70ec480 --- /dev/null +++ b/tests/specs/compile/npm_fs/deno.json @@ -0,0 +1,3 @@ +{ + "nodeModulesDir": "auto" +} diff --git a/tests/testdata/compile/npm_fs/main.out b/tests/specs/compile/npm_fs/main.out similarity index 100% rename from tests/testdata/compile/npm_fs/main.out rename to tests/specs/compile/npm_fs/main.out diff --git a/tests/testdata/compile/npm_fs/main.ts b/tests/specs/compile/npm_fs/main.ts similarity index 100% rename from tests/testdata/compile/npm_fs/main.ts rename to tests/specs/compile/npm_fs/main.ts diff --git a/tests/specs/compile/package_json_type/compile.out b/tests/specs/compile/package_json_type/compile.out index 913e363c3e..0370192809 100644 --- a/tests/specs/compile/package_json_type/compile.out +++ b/tests/specs/compile/package_json_type/compile.out @@ -1,2 +1,6 @@ Check file:///[WILDLINE]/main.js Compile file:///[WILDLINE] + +Embedded File System + +[WILDCARD] diff --git a/tests/specs/install/entrypoint_only_used_packages/__test__.jsonc b/tests/specs/install/entrypoint_only_used_packages/__test__.jsonc new file mode 100644 index 0000000000..9d24eb4de4 --- /dev/null +++ b/tests/specs/install/entrypoint_only_used_packages/__test__.jsonc @@ -0,0 +1,13 @@ +{ + "tempDir": true, + "steps": [ + { + "args": "install --unstable-npm-lazy-caching --entrypoint main.ts", + "output": "install-entrypoint.out" + }, + { + "args": "install", + "output": "install.out" + } + ] +} diff --git a/tests/specs/install/entrypoint_only_used_packages/install-entrypoint.out b/tests/specs/install/entrypoint_only_used_packages/install-entrypoint.out new file mode 100644 index 0000000000..5f6c8247a5 --- /dev/null +++ b/tests/specs/install/entrypoint_only_used_packages/install-entrypoint.out @@ -0,0 +1,6 @@ +[UNORDERED_START] +Download http://localhost:4260/@denotest%2fadd +Download http://localhost:4260/@denotest%2fsay-hello +Download http://localhost:4260/@denotest/add/1.0.0.tgz +[UNORDERED_END] +Initialize @denotest/add@1.0.0 diff --git a/tests/specs/install/entrypoint_only_used_packages/install.out b/tests/specs/install/entrypoint_only_used_packages/install.out new file mode 100644 index 0000000000..5ea41bed4c --- /dev/null +++ b/tests/specs/install/entrypoint_only_used_packages/install.out @@ -0,0 +1,2 @@ +Download http://localhost:4260/@denotest/say-hello/1.0.0.tgz +Initialize @denotest/say-hello@1.0.0 diff --git a/tests/specs/install/entrypoint_only_used_packages/main.ts b/tests/specs/install/entrypoint_only_used_packages/main.ts new file mode 100644 index 0000000000..1ca631410f --- /dev/null +++ b/tests/specs/install/entrypoint_only_used_packages/main.ts @@ -0,0 +1,3 @@ +import { add } from "@denotest/add"; + +console.log(add(1, 2)); diff --git a/tests/specs/install/entrypoint_only_used_packages/package.json b/tests/specs/install/entrypoint_only_used_packages/package.json new file mode 100644 index 0000000000..2623bcc82e --- /dev/null +++ b/tests/specs/install/entrypoint_only_used_packages/package.json @@ -0,0 +1,6 @@ +{ + "dependencies": { + "@denotest/add": "1.0.0", + "@denotest/say-hello": "1.0.0" + } +} diff --git a/tests/specs/lockfile/external_import_map/__test__.jsonc b/tests/specs/lockfile/external_import_map/__test__.jsonc new file mode 100644 index 0000000000..2bdffed334 --- /dev/null +++ b/tests/specs/lockfile/external_import_map/__test__.jsonc @@ -0,0 +1,10 @@ +{ + "tempDir": true, + "steps": [{ + "args": "run -A main.ts", + "output": "[WILDCARD]" + }, { + "args": ["eval", "console.log(Deno.readTextFileSync('deno.lock').trim())"], + "output": "deno.lock.out" + }] +} diff --git a/tests/specs/lockfile/external_import_map/deno.json b/tests/specs/lockfile/external_import_map/deno.json new file mode 100644 index 0000000000..ee44ba9472 --- /dev/null +++ b/tests/specs/lockfile/external_import_map/deno.json @@ -0,0 +1,3 @@ +{ + "importMap": "import_map.json" +} diff --git a/tests/specs/lockfile/external_import_map/deno.lock.out b/tests/specs/lockfile/external_import_map/deno.lock.out new file mode 100644 index 0000000000..c811061125 --- /dev/null +++ b/tests/specs/lockfile/external_import_map/deno.lock.out @@ -0,0 +1,17 @@ +{ + "version": "4", + "specifiers": { + "jsr:@denotest/add@1.0.0": "1.0.0" + }, + "jsr": { + "@denotest/add@1.0.0": { + "integrity": "[WILDLINE]" + } + }, + "workspace": { + "dependencies": [ + "jsr:@denotest/add@1.0.0", + "npm:@denotest/esm-basic@1.0.0" + ] + } +} diff --git a/tests/specs/lockfile/external_import_map/import_map.json b/tests/specs/lockfile/external_import_map/import_map.json new file mode 100644 index 0000000000..069b294ce4 --- /dev/null +++ b/tests/specs/lockfile/external_import_map/import_map.json @@ -0,0 +1,10 @@ +{ + "imports": { + "@denotest/add": "jsr:@denotest/add@1.0.0" + }, + "scopes": { + "/foo/": { + "@denotest/esm-basic": "npm:@denotest/esm-basic@1.0.0" + } + } +} diff --git a/tests/specs/lockfile/external_import_map/main.ts b/tests/specs/lockfile/external_import_map/main.ts new file mode 100644 index 0000000000..b75bbc03ed --- /dev/null +++ b/tests/specs/lockfile/external_import_map/main.ts @@ -0,0 +1,2 @@ +import { add } from "@denotest/add"; +console.log(add(1, 2)); diff --git a/tests/specs/mod.rs b/tests/specs/mod.rs index f5820e4d88..985a6c7c40 100644 --- a/tests/specs/mod.rs +++ b/tests/specs/mod.rs @@ -118,6 +118,12 @@ struct MultiStepMetaData { /// steps. #[serde(default)] pub temp_dir: bool, + /// Whether the temporary directory should be canonicalized. + /// + /// This should be used sparingly, but is sometimes necessary + /// on the CI. + #[serde(default)] + pub canonicalized_temp_dir: bool, /// Whether the temporary directory should be symlinked to another path. #[serde(default)] pub symlinked_temp_dir: bool, @@ -144,6 +150,8 @@ struct SingleTestMetaData { #[serde(default)] pub temp_dir: bool, #[serde(default)] + pub canonicalized_temp_dir: bool, + #[serde(default)] pub symlinked_temp_dir: bool, #[serde(default)] pub repeat: Option, @@ -159,6 +167,7 @@ impl SingleTestMetaData { base: self.base, cwd: None, temp_dir: self.temp_dir, + canonicalized_temp_dir: self.canonicalized_temp_dir, symlinked_temp_dir: self.symlinked_temp_dir, repeat: self.repeat, envs: Default::default(), @@ -326,6 +335,13 @@ fn test_context_from_metadata( builder = builder.cwd(cwd.to_string_lossy()); } + if metadata.canonicalized_temp_dir { + // not actually deprecated, we just want to discourage its use + #[allow(deprecated)] + { + builder = builder.use_canonicalized_temp_dir(); + } + } if metadata.symlinked_temp_dir { // not actually deprecated, we just want to discourage its use // because it's mostly used for testing purposes locally diff --git a/tests/specs/npm/byonm_npm_specifier_in_node_modules/__test__.jsonc b/tests/specs/npm/byonm_npm_specifier_in_node_modules/__test__.jsonc new file mode 100644 index 0000000000..e2c5495387 --- /dev/null +++ b/tests/specs/npm/byonm_npm_specifier_in_node_modules/__test__.jsonc @@ -0,0 +1,18 @@ +{ + "tests": { + "matches": { + "args": "run -A matches.ts", + "output": "5\n" + }, + "not_matches": { + "args": "run -A not_matches.ts", + "output": "not_matches.out", + "exitCode": 1 + }, + "not_matches_aliased": { + "args": "run -A not_matches_aliased.ts", + "output": "not_matches_aliased.out", + "exitCode": 1 + } + } +} diff --git a/tests/specs/npm/byonm_npm_specifier_in_node_modules/matches.ts b/tests/specs/npm/byonm_npm_specifier_in_node_modules/matches.ts new file mode 100644 index 0000000000..986e7baf4c --- /dev/null +++ b/tests/specs/npm/byonm_npm_specifier_in_node_modules/matches.ts @@ -0,0 +1,3 @@ +import { add } from "npm:package@1"; + +console.log(add(2, 3)); diff --git a/tests/specs/npm/byonm_npm_specifier_in_node_modules/node_modules/aliased/index.js b/tests/specs/npm/byonm_npm_specifier_in_node_modules/node_modules/aliased/index.js new file mode 100644 index 0000000000..efe826ba65 --- /dev/null +++ b/tests/specs/npm/byonm_npm_specifier_in_node_modules/node_modules/aliased/index.js @@ -0,0 +1,3 @@ +module.exports.add = function(a, b) { + return a + b; +}; diff --git a/tests/specs/npm/byonm_npm_specifier_in_node_modules/node_modules/aliased/package.json b/tests/specs/npm/byonm_npm_specifier_in_node_modules/node_modules/aliased/package.json new file mode 100644 index 0000000000..618960872f --- /dev/null +++ b/tests/specs/npm/byonm_npm_specifier_in_node_modules/node_modules/aliased/package.json @@ -0,0 +1,4 @@ +{ + "name": "not-same-name", + "version": "1.0.0" +} \ No newline at end of file diff --git a/tests/specs/npm/byonm_npm_specifier_in_node_modules/node_modules/package/index.js b/tests/specs/npm/byonm_npm_specifier_in_node_modules/node_modules/package/index.js new file mode 100644 index 0000000000..efe826ba65 --- /dev/null +++ b/tests/specs/npm/byonm_npm_specifier_in_node_modules/node_modules/package/index.js @@ -0,0 +1,3 @@ +module.exports.add = function(a, b) { + return a + b; +}; diff --git a/tests/specs/npm/byonm_npm_specifier_in_node_modules/node_modules/package/package.json b/tests/specs/npm/byonm_npm_specifier_in_node_modules/node_modules/package/package.json new file mode 100644 index 0000000000..5723987e9f --- /dev/null +++ b/tests/specs/npm/byonm_npm_specifier_in_node_modules/node_modules/package/package.json @@ -0,0 +1,4 @@ +{ + "name": "package", + "version": "1.0.0" +} \ No newline at end of file diff --git a/tests/specs/npm/byonm_npm_specifier_in_node_modules/not_matches.out b/tests/specs/npm/byonm_npm_specifier_in_node_modules/not_matches.out new file mode 100644 index 0000000000..c549e13ff2 --- /dev/null +++ b/tests/specs/npm/byonm_npm_specifier_in_node_modules/not_matches.out @@ -0,0 +1,2 @@ +error: Could not find a matching package for 'npm:package@2' in the node_modules directory. Ensure you have all your JSR and npm dependencies listed in your deno.json or package.json, then run `deno install`. Alternatively, turn on auto-install by specifying `"nodeModulesDir": "auto"` in your deno.json file. + at file:///[WILDLINE] diff --git a/tests/specs/npm/byonm_npm_specifier_in_node_modules/not_matches.ts b/tests/specs/npm/byonm_npm_specifier_in_node_modules/not_matches.ts new file mode 100644 index 0000000000..a337bd7d86 --- /dev/null +++ b/tests/specs/npm/byonm_npm_specifier_in_node_modules/not_matches.ts @@ -0,0 +1,3 @@ +import { add } from "npm:package@2"; // won't match 2 + +console.log(add(2, 3)); diff --git a/tests/specs/npm/byonm_npm_specifier_in_node_modules/not_matches_aliased.out b/tests/specs/npm/byonm_npm_specifier_in_node_modules/not_matches_aliased.out new file mode 100644 index 0000000000..93b52f64bc --- /dev/null +++ b/tests/specs/npm/byonm_npm_specifier_in_node_modules/not_matches_aliased.out @@ -0,0 +1,2 @@ +error: Could not find a matching package for 'npm:aliased@1' in the node_modules directory. Ensure you have all your JSR and npm dependencies listed in your deno.json or package.json, then run `deno install`. Alternatively, turn on auto-install by specifying `"nodeModulesDir": "auto"` in your deno.json file. + at file:///[WILDLINE] diff --git a/tests/specs/npm/byonm_npm_specifier_in_node_modules/not_matches_aliased.ts b/tests/specs/npm/byonm_npm_specifier_in_node_modules/not_matches_aliased.ts new file mode 100644 index 0000000000..85fa11e319 --- /dev/null +++ b/tests/specs/npm/byonm_npm_specifier_in_node_modules/not_matches_aliased.ts @@ -0,0 +1,3 @@ +import { add } from "npm:aliased@1"; + +console.log(add(2, 3)); diff --git a/tests/specs/npm/byonm_npm_specifier_in_node_modules/package.json b/tests/specs/npm/byonm_npm_specifier_in_node_modules/package.json new file mode 100644 index 0000000000..2c63c08510 --- /dev/null +++ b/tests/specs/npm/byonm_npm_specifier_in_node_modules/package.json @@ -0,0 +1,2 @@ +{ +} diff --git a/tests/specs/npm/error_version_after_subpath/error_version_after_subpath/main.out b/tests/specs/npm/error_version_after_subpath/error_version_after_subpath/main.out index 4e5f03e23c..1804936f0f 100644 --- a/tests/specs/npm/error_version_after_subpath/error_version_after_subpath/main.out +++ b/tests/specs/npm/error_version_after_subpath/error_version_after_subpath/main.out @@ -1,2 +1,2 @@ -error: Invalid package specifier 'npm:react-dom/server@18.2.0'. Did you mean to write 'npm:react-dom@18.2.0/server'? +error: Invalid package specifier 'npm:react-dom/server@18.2.0'. Did you mean to write 'npm:react-dom@18.2.0/server'? If not, add a version requirement to the specifier. at [WILDCARD]/error_version_after_subpath/main.js:1:8 diff --git a/tests/specs/run/jsx_import_source/__test__.jsonc b/tests/specs/run/jsx_import_source/__test__.jsonc index 55a895fc8f..cbda2dd32e 100644 --- a/tests/specs/run/jsx_import_source/__test__.jsonc +++ b/tests/specs/run/jsx_import_source/__test__.jsonc @@ -1,4 +1,5 @@ { + "tempDir": true, "tests": { "jsx_import_source_error": { "args": "run --config jsx/deno-jsx-error.jsonc --check jsx_import_source_no_pragma.tsx", diff --git a/tests/specs/run/lazy_npm/__test__.jsonc b/tests/specs/run/lazy_npm/__test__.jsonc new file mode 100644 index 0000000000..8212addd5c --- /dev/null +++ b/tests/specs/run/lazy_npm/__test__.jsonc @@ -0,0 +1,9 @@ +{ + "tempDir": true, + "steps": [ + { + "args": "run --unstable-npm-lazy-caching -A main.ts", + "output": "main.out" + } + ] +} diff --git a/tests/specs/run/lazy_npm/deno.json b/tests/specs/run/lazy_npm/deno.json new file mode 100644 index 0000000000..fbd70ec480 --- /dev/null +++ b/tests/specs/run/lazy_npm/deno.json @@ -0,0 +1,3 @@ +{ + "nodeModulesDir": "auto" +} diff --git a/tests/specs/run/lazy_npm/main.out b/tests/specs/run/lazy_npm/main.out new file mode 100644 index 0000000000..d1aab68e90 --- /dev/null +++ b/tests/specs/run/lazy_npm/main.out @@ -0,0 +1,7 @@ +[UNORDERED_START] +Download http://localhost:4260/@denotest%2fadd +Download http://localhost:4260/@denotest%2fsay-hello +Download http://localhost:4260/@denotest/add/1.0.0.tgz +[UNORDERED_END] +Initialize @denotest/add@1.0.0 +3 diff --git a/tests/specs/run/lazy_npm/main.ts b/tests/specs/run/lazy_npm/main.ts new file mode 100644 index 0000000000..1ca631410f --- /dev/null +++ b/tests/specs/run/lazy_npm/main.ts @@ -0,0 +1,3 @@ +import { add } from "@denotest/add"; + +console.log(add(1, 2)); diff --git a/tests/specs/run/lazy_npm/package.json b/tests/specs/run/lazy_npm/package.json new file mode 100644 index 0000000000..2623bcc82e --- /dev/null +++ b/tests/specs/run/lazy_npm/package.json @@ -0,0 +1,6 @@ +{ + "dependencies": { + "@denotest/add": "1.0.0", + "@denotest/say-hello": "1.0.0" + } +} diff --git a/tests/specs/schema.json b/tests/specs/schema.json index 2b35d9bd7d..77ffc59530 100644 --- a/tests/specs/schema.json +++ b/tests/specs/schema.json @@ -36,6 +36,9 @@ "flaky": { "type": "boolean" }, + "canonicalizedTempDir": { + "type": "boolean" + }, "symlinkedTempDir": { "type": "boolean" }, @@ -66,6 +69,12 @@ "tempDir": { "type": "boolean" }, + "canonicalizedTempDir": { + "type": "boolean" + }, + "symlinkedTempDir": { + "type": "boolean" + }, "base": { "type": "string" }, @@ -94,6 +103,12 @@ "tempDir": { "type": "boolean" }, + "canonicalizedTempDir": { + "type": "boolean" + }, + "symlinkedTempDir": { + "type": "boolean" + }, "base": { "type": "string" }, diff --git a/tests/specs/update/deno_json/__test__.jsonc b/tests/specs/update/deno_json/__test__.jsonc index 8b4aa26b5c..7983d2c56f 100644 --- a/tests/specs/update/deno_json/__test__.jsonc +++ b/tests/specs/update/deno_json/__test__.jsonc @@ -26,6 +26,16 @@ { "args": "outdated", "output": "outdated.out" + }, + { + // Filtering that matches nothing, should exit cleanly + "args": "outdated foobar", + "output": "" + }, + { + // Respect `--quiet flag and don't print hint how to update + "args": "outdated --quiet", + "output": "outdated_quiet.out" } ] }, @@ -38,6 +48,11 @@ { "args": "outdated --compatible", "output": "outdated_compatible.out" + }, + { + // Filtering that matches nothing, should exit cleanly + "args": "outdated --compatible foobar", + "output": "" } ] }, diff --git a/tests/specs/update/deno_json/outdated.out b/tests/specs/update/deno_json/outdated.out index 07ff9f3416..dd86ed8e87 100644 --- a/tests/specs/update/deno_json/outdated.out +++ b/tests/specs/update/deno_json/outdated.out @@ -13,3 +13,6 @@ ├────────────────────────────────────────────────┼─────────┼────────┼────────┤ │ npm:@denotest/breaking-change-between-versions │ 1.0.0 │ 1.0.0 │ 2.0.0 │ └────────────────────────────────────────────────┴─────────┴────────┴────────┘ + +Run deno outdated --update --latest to update to the latest available versions, +or deno outdated --help for more information. diff --git a/tests/specs/update/deno_json/outdated_compatible.out b/tests/specs/update/deno_json/outdated_compatible.out index 54511a537f..8b67e0750e 100644 --- a/tests/specs/update/deno_json/outdated_compatible.out +++ b/tests/specs/update/deno_json/outdated_compatible.out @@ -5,3 +5,6 @@ ├──────────────────────────────────┼─────────┼────────┼────────┤ │ npm:@denotest/has-patch-versions │ 0.1.0 │ 0.1.1 │ 0.2.0 │ └──────────────────────────────────┴─────────┴────────┴────────┘ + +Run deno outdated --update to update to the latest compatible versions, +or deno outdated --help for more information. diff --git a/tests/specs/update/deno_json/outdated_quiet.out b/tests/specs/update/deno_json/outdated_quiet.out new file mode 100644 index 0000000000..07ff9f3416 --- /dev/null +++ b/tests/specs/update/deno_json/outdated_quiet.out @@ -0,0 +1,15 @@ +┌────────────────────────────────────────────────┬─────────┬────────┬────────┐ +│ Package │ Current │ Update │ Latest │ +├────────────────────────────────────────────────┼─────────┼────────┼────────┤ +│ jsr:@denotest/multiple-exports │ 0.2.0 │ 0.2.0 │ 1.0.0 │ +├────────────────────────────────────────────────┼─────────┼────────┼────────┤ +│ jsr:@denotest/subtract │ 0.2.0 │ 0.2.0 │ 1.0.0 │ +├────────────────────────────────────────────────┼─────────┼────────┼────────┤ +│ jsr:@denotest/add │ 0.2.0 │ 0.2.1 │ 1.0.0 │ +├────────────────────────────────────────────────┼─────────┼────────┼────────┤ +│ npm:@denotest/has-patch-versions │ 0.1.0 │ 0.1.1 │ 0.2.0 │ +├────────────────────────────────────────────────┼─────────┼────────┼────────┤ +│ npm:@denotest/bin │ 0.6.0 │ 0.6.0 │ 1.0.0 │ +├────────────────────────────────────────────────┼─────────┼────────┼────────┤ +│ npm:@denotest/breaking-change-between-versions │ 1.0.0 │ 1.0.0 │ 2.0.0 │ +└────────────────────────────────────────────────┴─────────┴────────┴────────┘ diff --git a/tests/specs/update/external_import_map/__test__.jsonc b/tests/specs/update/external_import_map/__test__.jsonc new file mode 100644 index 0000000000..66dc55cc57 --- /dev/null +++ b/tests/specs/update/external_import_map/__test__.jsonc @@ -0,0 +1,24 @@ +{ + "tempDir": true, + "steps": [ + { + "args": "i", + "output": "[WILDCARD]" + }, + { + "args": "outdated", + "output": "outdated.out" + }, + { + "args": "outdated --update --latest", + "output": "update.out" + }, + { + "args": [ + "eval", + "console.log(Deno.readTextFileSync('import_map.json').trim())" + ], + "output": "import_map.json.out" + } + ] +} diff --git a/tests/specs/update/external_import_map/deno.json b/tests/specs/update/external_import_map/deno.json new file mode 100644 index 0000000000..ee44ba9472 --- /dev/null +++ b/tests/specs/update/external_import_map/deno.json @@ -0,0 +1,3 @@ +{ + "importMap": "import_map.json" +} diff --git a/tests/specs/update/external_import_map/deno.lock b/tests/specs/update/external_import_map/deno.lock new file mode 100644 index 0000000000..940f2527d8 --- /dev/null +++ b/tests/specs/update/external_import_map/deno.lock @@ -0,0 +1,33 @@ +{ + "version": "4", + "specifiers": { + "jsr:@denotest/add@0.2": "0.2.0", + "jsr:@denotest/subtract@0.2": "0.2.0", + "npm:@denotest/breaking-change-between-versions@1.0.0": "1.0.0", + "npm:@denotest/has-patch-versions@0.1": "0.1.0" + }, + "jsr": { + "@denotest/add@0.2.0": { + "integrity": "a9076d30ecb42b2fc6dd95e7055fbf4e6358b53f550741bd7f60089d19f68848" + }, + "@denotest/subtract@0.2.0": { + "integrity": "c9650fc559ab2430effc0c7fb1540e3aa89888fbdd926335ccfdeac57eb3a64d" + } + }, + "npm": { + "@denotest/breaking-change-between-versions@1.0.0": { + "integrity": "sha512-bzMGYx+DxxPlI74n/VsDAN7Db1BY7Sz2XqxXruMo9dEznsBZu7Ez3i8YQ8n0leTxAiiMk1RCG4zQHPG1aj3xRw==" + }, + "@denotest/has-patch-versions@0.1.0": { + "integrity": "sha512-H/MBo0jKDdMsX4AAGEGQbZj70nfNe3oUNZXbohYHhqf9EfpLnXp/7FC29ZdfV4+p6VjEcOGdCtXc6rilE6iYpg==" + } + }, + "workspace": { + "dependencies": [ + "jsr:@denotest/add@0.2", + "jsr:@denotest/subtract@0.2", + "npm:@denotest/breaking-change-between-versions@1.0.0", + "npm:@denotest/has-patch-versions@0.1" + ] + } +} diff --git a/tests/specs/update/external_import_map/import_map.json b/tests/specs/update/external_import_map/import_map.json new file mode 100644 index 0000000000..9cc5c77e24 --- /dev/null +++ b/tests/specs/update/external_import_map/import_map.json @@ -0,0 +1,8 @@ +{ + "imports": { + "@denotest/add": "jsr:@denotest/add@^0.2.0", + "@denotest/subtract": "jsr:@denotest/subtract@^0.2.0", + "@denotest/breaking-change-between-versions": "npm:@denotest/breaking-change-between-versions@1.0.0", + "@denotest/has-patch-versions": "npm:@denotest/has-patch-versions@^0.1.0" + } +} diff --git a/tests/specs/update/external_import_map/import_map.json.out b/tests/specs/update/external_import_map/import_map.json.out new file mode 100644 index 0000000000..b4e24decbc --- /dev/null +++ b/tests/specs/update/external_import_map/import_map.json.out @@ -0,0 +1,8 @@ +{ + "imports": { + "@denotest/add": "jsr:@denotest/add@^1.0.0", + "@denotest/subtract": "jsr:@denotest/subtract@^1.0.0", + "@denotest/breaking-change-between-versions": "npm:@denotest/breaking-change-between-versions@^2.0.0", + "@denotest/has-patch-versions": "npm:@denotest/has-patch-versions@^0.2.0" + } +} diff --git a/tests/specs/update/external_import_map/main.ts b/tests/specs/update/external_import_map/main.ts new file mode 100644 index 0000000000..b008bb0d41 --- /dev/null +++ b/tests/specs/update/external_import_map/main.ts @@ -0,0 +1 @@ +import { add } from "@denotest/add"; diff --git a/tests/specs/update/external_import_map/outdated.out b/tests/specs/update/external_import_map/outdated.out new file mode 100644 index 0000000000..8752b132bf --- /dev/null +++ b/tests/specs/update/external_import_map/outdated.out @@ -0,0 +1,14 @@ +┌────────────────────────────────────────────────┬─────────┬────────┬────────┐ +│ Package │ Current │ Update │ Latest │ +├────────────────────────────────────────────────┼─────────┼────────┼────────┤ +│ jsr:@denotest/subtract │ 0.2.0 │ 0.2.0 │ 1.0.0 │ +├────────────────────────────────────────────────┼─────────┼────────┼────────┤ +│ jsr:@denotest/add │ 0.2.0 │ 0.2.1 │ 1.0.0 │ +├────────────────────────────────────────────────┼─────────┼────────┼────────┤ +│ npm:@denotest/has-patch-versions │ 0.1.0 │ 0.1.1 │ 0.2.0 │ +├────────────────────────────────────────────────┼─────────┼────────┼────────┤ +│ npm:@denotest/breaking-change-between-versions │ 1.0.0 │ 1.0.0 │ 2.0.0 │ +└────────────────────────────────────────────────┴─────────┴────────┴────────┘ + +Run deno outdated --update --latest to update to the latest available versions, +or deno outdated --help for more information. diff --git a/tests/specs/update/external_import_map/update.out b/tests/specs/update/external_import_map/update.out new file mode 100644 index 0000000000..d075cff296 --- /dev/null +++ b/tests/specs/update/external_import_map/update.out @@ -0,0 +1,11 @@ +[UNORDERED_START] +Download http://127.0.0.1:4250/@denotest/subtract/1.0.0/mod.ts +Download http://127.0.0.1:4250/@denotest/add/1.0.0/mod.ts +Download http://localhost:4260/@denotest/has-patch-versions/0.2.0.tgz +Download http://localhost:4260/@denotest/breaking-change-between-versions/2.0.0.tgz +[UNORDERED_END] +Updated 4 dependencies: + - jsr:@denotest/add 0.2.0 -> 1.0.0 + - jsr:@denotest/subtract 0.2.0 -> 1.0.0 + - npm:@denotest/breaking-change-between-versions 1.0.0 -> 2.0.0 + - npm:@denotest/has-patch-versions 0.1.0 -> 0.2.0 diff --git a/tests/specs/update/mixed_workspace/__test__.jsonc b/tests/specs/update/mixed_workspace/__test__.jsonc index 8c846467d4..9810e15bdd 100644 --- a/tests/specs/update/mixed_workspace/__test__.jsonc +++ b/tests/specs/update/mixed_workspace/__test__.jsonc @@ -26,6 +26,11 @@ { "args": "outdated", "output": "print_outdated/root.out" + }, + { + // Filtering that matches nothing, should exit cleanly + "args": "outdated foobar", + "output": "" } ] }, @@ -38,6 +43,11 @@ { "args": "outdated --recursive", "output": "print_outdated/recursive.out" + }, + { + // Filtering that matches nothing, should exit cleanly + "args": "outdated foobar", + "output": "" } ] }, diff --git a/tests/specs/update/mixed_workspace/print_outdated/member_a.out b/tests/specs/update/mixed_workspace/print_outdated/member_a.out index 8699aac2bf..9abfd127c0 100644 --- a/tests/specs/update/mixed_workspace/print_outdated/member_a.out +++ b/tests/specs/update/mixed_workspace/print_outdated/member_a.out @@ -7,3 +7,6 @@ ├────────────────────────────────────────────────┼─────────┼────────┼────────┤ │ npm:@denotest/breaking-change-between-versions │ 1.0.0 │ 1.0.0 │ 2.0.0 │ └────────────────────────────────────────────────┴─────────┴────────┴────────┘ + +Run deno outdated --update --latest to update to the latest available versions, +or deno outdated --help for more information. diff --git a/tests/specs/update/mixed_workspace/print_outdated/member_b.out b/tests/specs/update/mixed_workspace/print_outdated/member_b.out index fc8ef320a8..80548e4ac3 100644 --- a/tests/specs/update/mixed_workspace/print_outdated/member_b.out +++ b/tests/specs/update/mixed_workspace/print_outdated/member_b.out @@ -5,3 +5,6 @@ ├──────────────────────────────────┼─────────┼────────┼────────┤ │ npm:@denotest/bin │ 0.6.0 │ 0.6.0 │ 1.0.0 │ └──────────────────────────────────┴─────────┴────────┴────────┘ + +Run deno outdated --update --latest to update to the latest available versions, +or deno outdated --help for more information. diff --git a/tests/specs/update/mixed_workspace/print_outdated/recursive.out b/tests/specs/update/mixed_workspace/print_outdated/recursive.out index ca03776a9c..1c4f4d06e2 100644 --- a/tests/specs/update/mixed_workspace/print_outdated/recursive.out +++ b/tests/specs/update/mixed_workspace/print_outdated/recursive.out @@ -13,3 +13,6 @@ ├────────────────────────────────────────────────┼─────────┼────────┼────────┤ │ npm:@denotest/breaking-change-between-versions │ 1.0.0 │ 1.0.0 │ 2.0.0 │ └────────────────────────────────────────────────┴─────────┴────────┴────────┘ + +Run deno outdated --update --latest to update to the latest available versions, +or deno outdated --help for more information. diff --git a/tests/specs/update/mixed_workspace/print_outdated/root.out b/tests/specs/update/mixed_workspace/print_outdated/root.out index c7934edc79..ef8df6dbe1 100644 --- a/tests/specs/update/mixed_workspace/print_outdated/root.out +++ b/tests/specs/update/mixed_workspace/print_outdated/root.out @@ -3,3 +3,6 @@ ├────────────────────────┼─────────┼────────┼────────┤ │ jsr:@denotest/subtract │ 0.2.0 │ 0.2.0 │ 1.0.0 │ └────────────────────────┴─────────┴────────┴────────┘ + +Run deno outdated --update --latest to update to the latest available versions, +or deno outdated --help for more information. diff --git a/tests/specs/update/no_config_file/__test__.jsonc b/tests/specs/update/no_config_file/__test__.jsonc new file mode 100644 index 0000000000..1032e79279 --- /dev/null +++ b/tests/specs/update/no_config_file/__test__.jsonc @@ -0,0 +1,6 @@ +{ + "tempDir": true, + "args": "outdated", + "exitCode": 1, + "output": "error: No deno.json or package.json in \"[WILDLINE]\".\n" +} diff --git a/tests/specs/update/package_json/__test__.jsonc b/tests/specs/update/package_json/__test__.jsonc index 19d576dfc0..b86b9956cc 100644 --- a/tests/specs/update/package_json/__test__.jsonc +++ b/tests/specs/update/package_json/__test__.jsonc @@ -25,6 +25,11 @@ { "args": "outdated", "output": "outdated.out" + }, + { + // Filtering that matches nothing, should exit cleanly + "args": "outdated foobar", + "output": "" } ] }, @@ -37,6 +42,11 @@ { "args": "outdated --compatible", "output": "outdated_compatible.out" + }, + { + // Filtering that matches nothing, should exit cleanly + "args": "outdated --compatible foobar", + "output": "" } ] }, diff --git a/tests/specs/update/package_json/outdated.out b/tests/specs/update/package_json/outdated.out index d672aace7f..cf61af8065 100644 --- a/tests/specs/update/package_json/outdated.out +++ b/tests/specs/update/package_json/outdated.out @@ -7,3 +7,6 @@ ├────────────────────────────────────────────────┼─────────┼────────┼────────┤ │ npm:@denotest/breaking-change-between-versions │ 1.0.0 │ 1.0.0 │ 2.0.0 │ └────────────────────────────────────────────────┴─────────┴────────┴────────┘ + +Run deno outdated --update --latest to update to the latest available versions, +or deno outdated --help for more information. diff --git a/tests/specs/update/package_json/outdated_compatible.out b/tests/specs/update/package_json/outdated_compatible.out index 8a682c461c..812e9ba0f4 100644 --- a/tests/specs/update/package_json/outdated_compatible.out +++ b/tests/specs/update/package_json/outdated_compatible.out @@ -3,3 +3,6 @@ ├──────────────────────────────────┼─────────┼────────┼────────┤ │ npm:@denotest/has-patch-versions │ 0.1.0 │ 0.1.1 │ 0.2.0 │ └──────────────────────────────────┴─────────┴────────┴────────┘ + +Run deno outdated --update to update to the latest compatible versions, +or deno outdated --help for more information. diff --git a/tests/testdata/compile/node_modules_symlink_outside/main_compile_file.out b/tests/testdata/compile/node_modules_symlink_outside/main_compile_file.out index e5b39a7527..633c2cca62 100644 --- a/tests/testdata/compile/node_modules_symlink_outside/main_compile_file.out +++ b/tests/testdata/compile/node_modules_symlink_outside/main_compile_file.out @@ -1,2 +1,5 @@ Compile file:///[WILDCARD]/node_modules_symlink_outside/main.ts to [WILDCARD] -Warning Symlink target is outside '[WILDCARD]compile'. Inlining symlink at '[WILDCARD]node_modules_symlink_outside[WILDCARD]node_modules[WILDCARD]test.txt' to '[WILDCARD]target.txt' as file. + +Embedded File System + +[WILDCARD] diff --git a/tests/testdata/compile/node_modules_symlink_outside/main_compile_folder.out b/tests/testdata/compile/node_modules_symlink_outside/main_compile_folder.out index 1a7eb0a4f2..61f0a2456a 100644 --- a/tests/testdata/compile/node_modules_symlink_outside/main_compile_folder.out +++ b/tests/testdata/compile/node_modules_symlink_outside/main_compile_folder.out @@ -3,4 +3,13 @@ Download http://localhost:4260/@denotest/esm-basic/1.0.0.tgz Initialize @denotest/esm-basic@1.0.0 Check file:///[WILDCARD]/node_modules_symlink_outside/main.ts Compile file:///[WILDCARD]/node_modules_symlink_outside/main.ts to [WILDLINE] -Warning Symlink target is outside '[WILDLINE]compile'. Excluding symlink at '[WILDLINE]node_modules_symlink_outside[WILDLINE]node_modules[WILDLINE]symlink_dir' with target '[WILDLINE]some_folder'. + +Embedded File System + +bin[WILDLINE] +├─┬ compile +│ └─┬ node_modules_symlink_outside +│ ├── main.ts +│ └── node_modules/* +└── some_folder/* + diff --git a/tests/unit_node/http_test.ts b/tests/unit_node/http_test.ts index 048ddf30f5..e6c36eea19 100644 --- a/tests/unit_node/http_test.ts +++ b/tests/unit_node/http_test.ts @@ -499,7 +499,6 @@ Deno.test("[node/http] send request with non-chunked body", async () => { assert(socket.writable); assert(socket.readable); socket.setKeepAlive(); - socket.destroy(); socket.setTimeout(100); }); req.write("hello "); @@ -512,6 +511,11 @@ Deno.test("[node/http] send request with non-chunked body", async () => { // in order to not cause a flaky test sanitizer failure await new Promise((resolve) => setTimeout(resolve, 100)), ]); + + if (Deno.build.os === "windows") { + // FIXME(kt3k): This is necessary for preventing op leak on windows + await new Promise((resolve) => setTimeout(resolve, 4000)); + } }); Deno.test("[node/http] send request with chunked body", async () => { @@ -559,6 +563,11 @@ Deno.test("[node/http] send request with chunked body", async () => { req.end(); await servePromise; + + if (Deno.build.os === "windows") { + // FIXME(kt3k): This is necessary for preventing op leak on windows + await new Promise((resolve) => setTimeout(resolve, 4000)); + } }); Deno.test("[node/http] send request with chunked body as default", async () => { @@ -604,6 +613,11 @@ Deno.test("[node/http] send request with chunked body as default", async () => { req.end(); await servePromise; + + if (Deno.build.os === "windows") { + // FIXME(kt3k): This is necessary for preventing op leak on windows + await new Promise((resolve) => setTimeout(resolve, 4000)); + } }); Deno.test("[node/http] ServerResponse _implicitHeader", async () => { @@ -689,7 +703,7 @@ Deno.test("[node/http] ClientRequest handle non-string headers", async () => { assertEquals(headers!["1"], "2"); }); -Deno.test("[node/http] ClientRequest uses HTTP/1.1", async () => { +Deno.test("[node/https] ClientRequest uses HTTP/1.1", async () => { let body = ""; const { promise, resolve, reject } = Promise.withResolvers(); const req = https.request("https://localhost:5545/http_version", { @@ -800,8 +814,9 @@ Deno.test("[node/http] ClientRequest search params", async () => { let body = ""; const { promise, resolve, reject } = Promise.withResolvers(); const req = http.request({ - host: "localhost:4545", - path: "search_params?foo=bar", + host: "localhost", + port: 4545, + path: "/search_params?foo=bar", }, (resp) => { resp.on("data", (chunk) => { body += chunk; @@ -1011,28 +1026,50 @@ Deno.test( Deno.test( "[node/http] client destroy before sending request should not error", - () => { + async () => { + const { resolve, promise } = Promise.withResolvers(); const request = http.request("http://localhost:5929/"); // Calling this would throw request.destroy(); + request.on("error", (e) => { + assertEquals(e.message, "socket hang up"); + }); + request.on("close", () => resolve()); + await promise; + + if (Deno.build.os === "windows") { + // FIXME(kt3k): This is necessary for preventing op leak on windows + await new Promise((resolve) => setTimeout(resolve, 4000)); + } }, ); +const isWindows = Deno.build.os === "windows"; + Deno.test( "[node/http] destroyed requests should not be sent", + { sanitizeResources: !isWindows, sanitizeOps: !isWindows }, async () => { let receivedRequest = false; - const server = Deno.serve(() => { + const requestClosed = Promise.withResolvers(); + const ac = new AbortController(); + const server = Deno.serve({ port: 0, signal: ac.signal }, () => { receivedRequest = true; return new Response(null); }); const request = http.request(`http://localhost:${server.addr.port}/`); request.destroy(); request.end("hello"); - - await new Promise((r) => setTimeout(r, 500)); + request.on("error", (err) => { + assert(err.message.includes("socket hang up")); + ac.abort(); + }); + request.on("close", () => { + requestClosed.resolve(); + }); + await requestClosed.promise; assertEquals(receivedRequest, false); - await server.shutdown(); + await server.finished; }, ); @@ -1060,22 +1097,33 @@ Deno.test("[node/https] node:https exports globalAgent", async () => { ); }); -Deno.test("[node/http] node:http request.setHeader(header, null) doesn't throw", () => { +Deno.test("[node/http] node:http request.setHeader(header, null) doesn't throw", async () => { { - const req = http.request("http://localhost:4545/"); - req.on("error", () => {}); + const { promise, resolve } = Promise.withResolvers(); + const req = http.request("http://localhost:4545/", (res) => { + res.on("data", () => {}); + res.on("end", () => { + resolve(); + }); + }); // @ts-expect-error - null is not a valid header value req.setHeader("foo", null); req.end(); - req.destroy(); + await promise; } { - const req = https.request("https://localhost:4545/"); - req.on("error", () => {}); + const { promise, resolve } = Promise.withResolvers(); + const req = http.request("http://localhost:4545/", (res) => { + res.on("data", () => {}); + res.on("end", () => { + resolve(); + }); + }); // @ts-expect-error - null is not a valid header value req.setHeader("foo", null); req.end(); - req.destroy(); + + await promise; } }); diff --git a/tests/unit_node/util_test.ts b/tests/unit_node/util_test.ts index 6267018b12..af174b0f4d 100644 --- a/tests/unit_node/util_test.ts +++ b/tests/unit_node/util_test.ts @@ -224,7 +224,7 @@ Deno.test({ fn() { assert(util.types.isNativeError(new Error())); assert(util.types.isNativeError(new TypeError())); - assert(!util.types.isNativeError(new DOMException())); + assert(util.types.isNativeError(new DOMException())); }, }); diff --git a/tests/util/server/src/lib.rs b/tests/util/server/src/lib.rs index 953896cffd..531944bf6a 100644 --- a/tests/util/server/src/lib.rs +++ b/tests/util/server/src/lib.rs @@ -816,15 +816,17 @@ pub fn wildcard_match_detailed( } let actual_next_text = ¤t_text[max_current_text_found_index..]; - let max_next_text_len = 40; - let next_text_len = - std::cmp::min(max_next_text_len, actual_next_text.len()); + let next_text_len = actual_next_text + .chars() + .take(40) + .map(|c| c.len_utf8()) + .sum::(); output_lines.push(format!( "==== NEXT ACTUAL TEXT ====\n{}{}", colors::red(annotate_whitespace( &actual_next_text[..next_text_len] )), - if actual_next_text.len() > max_next_text_len { + if actual_next_text.len() > next_text_len { "[TRUNCATED]" } else { "" diff --git a/tests/util/server/src/lsp.rs b/tests/util/server/src/lsp.rs index d34deb2161..92169ee644 100644 --- a/tests/util/server/src/lsp.rs +++ b/tests/util/server/src/lsp.rs @@ -1290,6 +1290,14 @@ impl SourceFile { "html" => "html", "css" => "css", "yaml" => "yaml", + "sql" => "sql", + "svelte" => "svelte", + "vue" => "vue", + "astro" => "astro", + "vto" => "vento", + "vento" => "vento", + "njk" => "nunjucks", + "nunjucks" => "nunjucks", other => panic!("unsupported file extension: {other}"), }; Self { diff --git a/tests/wpt/runner/expectation.json b/tests/wpt/runner/expectation.json index 5776fdb486..6025ce42a4 100644 --- a/tests/wpt/runner/expectation.json +++ b/tests/wpt/runner/expectation.json @@ -3594,16 +3594,13 @@ "DOMException-constructor-behavior.any.html": true, "DOMException-constructor-behavior.any.worker.html": true, "DOMException-custom-bindings.any.html": true, - "DOMException-custom-bindings.any.worker.html": true + "DOMException-custom-bindings.any.worker.html": true, + "exceptions.html": false }, "class-string-interface.any.html": true, "class-string-interface.any.worker.html": true, - "class-string-iterator-prototype-object.any.html": [ - "Object.prototype.toString applied after deleting @@toStringTag" - ], - "class-string-iterator-prototype-object.any.worker.html": [ - "Object.prototype.toString applied after deleting @@toStringTag" - ], + "class-string-iterator-prototype-object.any.html": true, + "class-string-iterator-prototype-object.any.worker.html": true, "class-string-named-properties-object.window.html": false, "global-immutable-prototype.any.html": [ "Setting to a different prototype" @@ -9754,7 +9751,6 @@ "structured-cloning-error-stack-optional.sub.window.html": [ "page-created Error (cross-site iframe)", "page-created Error (same-origin iframe)", - "page-created DOMException (structuredClone())", "page-created DOMException (cross-site iframe)", "page-created DOMException (same-origin iframe)", "JS-engine-created TypeError (cross-site iframe)", @@ -9762,8 +9758,7 @@ "web API-created TypeError (cross-site iframe)", "web API-created TypeError (same-origin iframe)", "web API-created DOMException (cross-site iframe)", - "web API-created DOMException (same-origin iframe)", - "page-created DOMException (worker)" + "web API-created DOMException (same-origin iframe)" ], "transfer-errors.window.html": false, "window-postmessage.window.html": false diff --git a/tools/deno.lock.json b/tools/deno.lock.json index 46c09ce24b..3d21e5c9f6 100644 --- a/tools/deno.lock.json +++ b/tools/deno.lock.json @@ -1,81 +1,365 @@ { - "version": "3", - "packages": { - "specifiers": { - "jsr:@david/dax@0.41.0": "jsr:@david/dax@0.41.0", - "jsr:@david/which@^0.4.1": "jsr:@david/which@0.4.1", - "jsr:@deno/patchver@0.1.0": "jsr:@deno/patchver@0.1.0", - "jsr:@std/assert@^0.221.0": "jsr:@std/assert@0.221.0", - "jsr:@std/bytes@^0.221.0": "jsr:@std/bytes@0.221.0", - "jsr:@std/fmt@1": "jsr:@std/fmt@1.0.0", - "jsr:@std/fmt@^0.221.0": "jsr:@std/fmt@0.221.0", - "jsr:@std/fs@0.221.0": "jsr:@std/fs@0.221.0", - "jsr:@std/io@0.221.0": "jsr:@std/io@0.221.0", - "jsr:@std/io@^0.221.0": "jsr:@std/io@0.221.0", - "jsr:@std/path@0.221.0": "jsr:@std/path@0.221.0", - "jsr:@std/path@^0.221.0": "jsr:@std/path@0.221.0", - "jsr:@std/streams@0.221.0": "jsr:@std/streams@0.221.0", - "jsr:@std/yaml@^0.221": "jsr:@std/yaml@0.221.0" + "version": "4", + "specifiers": { + "jsr:@david/dax@0.41.0": "0.41.0", + "jsr:@david/dax@0.42": "0.42.0", + "jsr:@david/path@0.2": "0.2.0", + "jsr:@david/which@~0.4.1": "0.4.1", + "jsr:@deno/patchver@0.1.0": "0.1.0", + "jsr:@std/assert@0.221": "0.221.0", + "jsr:@std/bytes@0.221": "0.221.0", + "jsr:@std/fmt@0.221": "0.221.0", + "jsr:@std/fmt@1": "1.0.0", + "jsr:@std/fs@0.221.0": "0.221.0", + "jsr:@std/fs@1": "1.0.5", + "jsr:@std/io@0.221": "0.221.0", + "jsr:@std/io@0.221.0": "0.221.0", + "jsr:@std/path@0.221": "0.221.0", + "jsr:@std/path@0.221.0": "0.221.0", + "jsr:@std/path@1": "1.0.8", + "jsr:@std/path@^1.0.7": "1.0.8", + "jsr:@std/streams@0.221": "0.221.0", + "jsr:@std/streams@0.221.0": "0.221.0", + "jsr:@std/yaml@0.221": "0.221.0", + "npm:decompress@4.2.1": "4.2.1" + }, + "jsr": { + "@david/dax@0.41.0": { + "integrity": "9e1ecf66a0415962cc8ad3ba4e3fa93ce0f1a1cc797dd95c36fdfb6977dc7fc8", + "dependencies": [ + "jsr:@david/which", + "jsr:@std/fmt@0.221", + "jsr:@std/fs@0.221.0", + "jsr:@std/io@0.221.0", + "jsr:@std/path@0.221.0", + "jsr:@std/streams@0.221.0" + ] }, - "jsr": { - "@david/dax@0.41.0": { - "integrity": "9e1ecf66a0415962cc8ad3ba4e3fa93ce0f1a1cc797dd95c36fdfb6977dc7fc8", - "dependencies": [ - "jsr:@david/which@^0.4.1", - "jsr:@std/fmt@^0.221.0", - "jsr:@std/fs@0.221.0", - "jsr:@std/io@0.221.0", - "jsr:@std/path@0.221.0", - "jsr:@std/streams@0.221.0" - ] - }, - "@david/which@0.4.1": { - "integrity": "896a682b111f92ab866cc70c5b4afab2f5899d2f9bde31ed00203b9c250f225e" - }, - "@deno/patchver@0.1.0": { - "integrity": "3102aa1b751a9fb85ef6cf7d4c0a1ec6624c85a77facc140c5748d82126d66a6" - }, - "@std/assert@0.221.0": { - "integrity": "a5f1aa6e7909dbea271754fd4ab3f4e687aeff4873b4cef9a320af813adb489a" - }, - "@std/bytes@0.221.0": { - "integrity": "64a047011cf833890a4a2ab7293ac55a1b4f5a050624ebc6a0159c357de91966" - }, - "@std/fmt@0.221.0": { - "integrity": "379fed69bdd9731110f26b9085aeb740606b20428ce6af31ef6bd45ef8efa62a" - }, - "@std/fmt@1.0.0": { - "integrity": "8a95c9fdbb61559418ccbc0f536080cf43341655e1444f9d375a66886ceaaa3d" - }, - "@std/fs@0.221.0": { - "integrity": "028044450299de8ed5a716ade4e6d524399f035513b85913794f4e81f07da286", - "dependencies": [ - "jsr:@std/assert@^0.221.0", - "jsr:@std/path@^0.221.0" - ] - }, - "@std/io@0.221.0": { - "integrity": "faf7f8700d46ab527fa05cc6167f4b97701a06c413024431c6b4d207caa010da", - "dependencies": [ - "jsr:@std/assert@^0.221.0", - "jsr:@std/bytes@^0.221.0" - ] - }, - "@std/path@0.221.0": { - "integrity": "0a36f6b17314ef653a3a1649740cc8db51b25a133ecfe838f20b79a56ebe0095", - "dependencies": [ - "jsr:@std/assert@^0.221.0" - ] - }, - "@std/streams@0.221.0": { - "integrity": "47f2f74634b47449277c0ee79fe878da4424b66bd8975c032e3afdca88986e61", - "dependencies": [ - "jsr:@std/io@^0.221.0" - ] - }, - "@std/yaml@0.221.0": { - "integrity": "bac8913ee4f6fc600d4b92cc020f755070e22687ad242341f31d123ff690ae98" - } + "@david/dax@0.42.0": { + "integrity": "0c547c9a20577a6072b90def194c159c9ddab82280285ebfd8268a4ebefbd80b", + "dependencies": [ + "jsr:@david/path", + "jsr:@david/which", + "jsr:@std/fmt@1", + "jsr:@std/fs@1", + "jsr:@std/io@0.221", + "jsr:@std/path@1", + "jsr:@std/streams@0.221" + ] + }, + "@david/path@0.2.0": { + "integrity": "f2d7aa7f02ce5a55e27c09f9f1381794acb09d328f8d3c8a2e3ab3ffc294dccd", + "dependencies": [ + "jsr:@std/fs@1", + "jsr:@std/path@1" + ] + }, + "@david/which@0.4.1": { + "integrity": "896a682b111f92ab866cc70c5b4afab2f5899d2f9bde31ed00203b9c250f225e" + }, + "@deno/patchver@0.1.0": { + "integrity": "3102aa1b751a9fb85ef6cf7d4c0a1ec6624c85a77facc140c5748d82126d66a6" + }, + "@std/assert@0.221.0": { + "integrity": "a5f1aa6e7909dbea271754fd4ab3f4e687aeff4873b4cef9a320af813adb489a" + }, + "@std/bytes@0.221.0": { + "integrity": "64a047011cf833890a4a2ab7293ac55a1b4f5a050624ebc6a0159c357de91966" + }, + "@std/fmt@0.221.0": { + "integrity": "379fed69bdd9731110f26b9085aeb740606b20428ce6af31ef6bd45ef8efa62a" + }, + "@std/fmt@1.0.0": { + "integrity": "8a95c9fdbb61559418ccbc0f536080cf43341655e1444f9d375a66886ceaaa3d" + }, + "@std/fs@0.221.0": { + "integrity": "028044450299de8ed5a716ade4e6d524399f035513b85913794f4e81f07da286", + "dependencies": [ + "jsr:@std/assert", + "jsr:@std/path@0.221" + ] + }, + "@std/fs@1.0.5": { + "integrity": "41806ad6823d0b5f275f9849a2640d87e4ef67c51ee1b8fb02426f55e02fd44e", + "dependencies": [ + "jsr:@std/path@^1.0.7" + ] + }, + "@std/io@0.221.0": { + "integrity": "faf7f8700d46ab527fa05cc6167f4b97701a06c413024431c6b4d207caa010da", + "dependencies": [ + "jsr:@std/assert", + "jsr:@std/bytes" + ] + }, + "@std/path@0.221.0": { + "integrity": "0a36f6b17314ef653a3a1649740cc8db51b25a133ecfe838f20b79a56ebe0095", + "dependencies": [ + "jsr:@std/assert" + ] + }, + "@std/path@1.0.8": { + "integrity": "548fa456bb6a04d3c1a1e7477986b6cffbce95102d0bb447c67c4ee70e0364be" + }, + "@std/streams@0.221.0": { + "integrity": "47f2f74634b47449277c0ee79fe878da4424b66bd8975c032e3afdca88986e61", + "dependencies": [ + "jsr:@std/io@0.221" + ] + }, + "@std/yaml@0.221.0": { + "integrity": "bac8913ee4f6fc600d4b92cc020f755070e22687ad242341f31d123ff690ae98" + } + }, + "npm": { + "base64-js@1.5.1": { + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" + }, + "bl@1.2.3": { + "integrity": "sha512-pvcNpa0UU69UT341rO6AYy4FVAIkUHuZXRIWbq+zHnsVcRzDDjIAhGuuYoi0d//cwIwtt4pkpKycWEfjdV+vww==", + "dependencies": [ + "readable-stream", + "safe-buffer@5.2.1" + ] + }, + "buffer-alloc-unsafe@1.1.0": { + "integrity": "sha512-TEM2iMIEQdJ2yjPJoSIsldnleVaAk1oW3DBVUykyOLsEsFmEc9kn+SFFPz+gl54KQNxlDnAwCXosOS9Okx2xAg==" + }, + "buffer-alloc@1.2.0": { + "integrity": "sha512-CFsHQgjtW1UChdXgbyJGtnm+O/uLQeZdtbDo8mfUgYXCHSM1wgrVxXm6bSyrUuErEb+4sYVGCzASBRot7zyrow==", + "dependencies": [ + "buffer-alloc-unsafe", + "buffer-fill" + ] + }, + "buffer-crc32@0.2.13": { + "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==" + }, + "buffer-fill@1.0.0": { + "integrity": "sha512-T7zexNBwiiaCOGDg9xNX9PBmjrubblRkENuptryuI64URkXDFum9il/JGL8Lm8wYfAXpredVXXZz7eMHilimiQ==" + }, + "buffer@5.7.1": { + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "dependencies": [ + "base64-js", + "ieee754" + ] + }, + "commander@2.20.3": { + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==" + }, + "core-util-is@1.0.3": { + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==" + }, + "decompress-tar@4.1.1": { + "integrity": "sha512-JdJMaCrGpB5fESVyxwpCx4Jdj2AagLmv3y58Qy4GE6HMVjWz1FeVQk1Ct4Kye7PftcdOo/7U7UKzYBJgqnGeUQ==", + "dependencies": [ + "file-type@5.2.0", + "is-stream", + "tar-stream" + ] + }, + "decompress-tarbz2@4.1.1": { + "integrity": "sha512-s88xLzf1r81ICXLAVQVzaN6ZmX4A6U4z2nMbOwobxkLoIIfjVMBg7TeguTUXkKeXni795B6y5rnvDw7rxhAq9A==", + "dependencies": [ + "decompress-tar", + "file-type@6.2.0", + "is-stream", + "seek-bzip", + "unbzip2-stream" + ] + }, + "decompress-targz@4.1.1": { + "integrity": "sha512-4z81Znfr6chWnRDNfFNqLwPvm4db3WuZkqV+UgXQzSngG3CEKdBkw5jrv3axjjL96glyiiKjsxJG3X6WBZwX3w==", + "dependencies": [ + "decompress-tar", + "file-type@5.2.0", + "is-stream" + ] + }, + "decompress-unzip@4.0.1": { + "integrity": "sha512-1fqeluvxgnn86MOh66u8FjbtJpAFv5wgCT9Iw8rcBqQcCo5tO8eiJw7NNTrvt9n4CRBVq7CstiS922oPgyGLrw==", + "dependencies": [ + "file-type@3.9.0", + "get-stream", + "pify@2.3.0", + "yauzl" + ] + }, + "decompress@4.2.1": { + "integrity": "sha512-e48kc2IjU+2Zw8cTb6VZcJQ3lgVbS4uuB1TfCHbiZIP/haNXm+SVyhu+87jts5/3ROpd82GSVCoNs/z8l4ZOaQ==", + "dependencies": [ + "decompress-tar", + "decompress-tarbz2", + "decompress-targz", + "decompress-unzip", + "graceful-fs", + "make-dir", + "pify@2.3.0", + "strip-dirs" + ] + }, + "end-of-stream@1.4.4": { + "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "dependencies": [ + "once" + ] + }, + "fd-slicer@1.1.0": { + "integrity": "sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g==", + "dependencies": [ + "pend" + ] + }, + "file-type@3.9.0": { + "integrity": "sha512-RLoqTXE8/vPmMuTI88DAzhMYC99I8BWv7zYP4A1puo5HIjEJ5EX48ighy4ZyKMG9EDXxBgW6e++cn7d1xuFghA==" + }, + "file-type@5.2.0": { + "integrity": "sha512-Iq1nJ6D2+yIO4c8HHg4fyVb8mAJieo1Oloy1mLLaB2PvezNedhBVm+QU7g0qM42aiMbRXTxKKwGD17rjKNJYVQ==" + }, + "file-type@6.2.0": { + "integrity": "sha512-YPcTBDV+2Tm0VqjybVd32MHdlEGAtuxS3VAYsumFokDSMG+ROT5wawGlnHDoz7bfMcMDt9hxuXvXwoKUx2fkOg==" + }, + "fs-constants@1.0.0": { + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" + }, + "get-stream@2.3.1": { + "integrity": "sha512-AUGhbbemXxrZJRD5cDvKtQxLuYaIbNtDTK8YqupCI393Q2KSTreEsLUN3ZxAWFGiKTzL6nKuzfcIvieflUX9qA==", + "dependencies": [ + "object-assign", + "pinkie-promise" + ] + }, + "graceful-fs@4.2.11": { + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==" + }, + "ieee754@1.2.1": { + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==" + }, + "inherits@2.0.4": { + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "is-natural-number@4.0.1": { + "integrity": "sha512-Y4LTamMe0DDQIIAlaer9eKebAlDSV6huy+TWhJVPlzZh2o4tRP5SQWFlLn5N0To4mDD22/qdOq+veo1cSISLgQ==" + }, + "is-stream@1.1.0": { + "integrity": "sha512-uQPm8kcs47jx38atAcWTVxyltQYoPT68y9aWYdV6yWXSyW8mzSat0TL6CiWdZeCdF3KrAvpVtnHbTv4RN+rqdQ==" + }, + "isarray@1.0.0": { + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==" + }, + "make-dir@1.3.0": { + "integrity": "sha512-2w31R7SJtieJJnQtGc7RVL2StM2vGYVfqUOvUDxH6bC6aJTxPxTF0GnIgCyu7tjockiUWAYQRbxa7vKn34s5sQ==", + "dependencies": [ + "pify@3.0.0" + ] + }, + "object-assign@4.1.1": { + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==" + }, + "once@1.4.0": { + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dependencies": [ + "wrappy" + ] + }, + "pend@1.2.0": { + "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==" + }, + "pify@2.3.0": { + "integrity": "sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog==" + }, + "pify@3.0.0": { + "integrity": "sha512-C3FsVNH1udSEX48gGX1xfvwTWfsYWj5U+8/uK15BGzIGrKoUpghX8hWZwa/OFnakBiiVNmBvemTJR5mcy7iPcg==" + }, + "pinkie-promise@2.0.1": { + "integrity": "sha512-0Gni6D4UcLTbv9c57DfxDGdr41XfgUjqWZu492f0cIGr16zDU06BWP/RAEvOuo7CQ0CNjHaLlM59YJJFm3NWlw==", + "dependencies": [ + "pinkie" + ] + }, + "pinkie@2.0.4": { + "integrity": "sha512-MnUuEycAemtSaeFSjXKW/aroV7akBbY+Sv+RkyqFjgAe73F+MR0TBWKBRDkmfWq/HiFmdavfZ1G7h4SPZXaCSg==" + }, + "process-nextick-args@2.0.1": { + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" + }, + "readable-stream@2.3.8": { + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "dependencies": [ + "core-util-is", + "inherits", + "isarray", + "process-nextick-args", + "safe-buffer@5.1.2", + "string_decoder", + "util-deprecate" + ] + }, + "safe-buffer@5.1.2": { + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "safe-buffer@5.2.1": { + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" + }, + "seek-bzip@1.0.6": { + "integrity": "sha512-e1QtP3YL5tWww8uKaOCQ18UxIT2laNBXHjV/S2WYCiK4udiv8lkG89KRIoCjUagnAmCBurjF4zEVX2ByBbnCjQ==", + "dependencies": [ + "commander" + ] + }, + "string_decoder@1.1.1": { + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dependencies": [ + "safe-buffer@5.1.2" + ] + }, + "strip-dirs@2.1.0": { + "integrity": "sha512-JOCxOeKLm2CAS73y/U4ZeZPTkE+gNVCzKt7Eox84Iej1LT/2pTWYpZKJuxwQpvX1LiZb1xokNR7RLfuBAa7T3g==", + "dependencies": [ + "is-natural-number" + ] + }, + "tar-stream@1.6.2": { + "integrity": "sha512-rzS0heiNf8Xn7/mpdSVVSMAWAoy9bfb1WOTYC78Z0UQKeKa/CWS8FOq0lKGNa8DWKAn9gxjCvMLYc5PGXYlK2A==", + "dependencies": [ + "bl", + "buffer-alloc", + "end-of-stream", + "fs-constants", + "readable-stream", + "to-buffer", + "xtend" + ] + }, + "through@2.3.8": { + "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==" + }, + "to-buffer@1.1.1": { + "integrity": "sha512-lx9B5iv7msuFYE3dytT+KE5tap+rNYw+K4jVkb9R/asAb+pbBSM17jtunHplhBe6RRJdZx3Pn2Jph24O32mOVg==" + }, + "unbzip2-stream@1.4.3": { + "integrity": "sha512-mlExGW4w71ebDJviH16lQLtZS32VKqsSfk80GCfUlwT/4/hNRFsoscrF/c++9xinkMzECL1uL9DDwXqFWkruPg==", + "dependencies": [ + "buffer", + "through" + ] + }, + "util-deprecate@1.0.2": { + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" + }, + "wrappy@1.0.2": { + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" + }, + "xtend@4.0.2": { + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==" + }, + "yauzl@2.10.0": { + "integrity": "sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==", + "dependencies": [ + "buffer-crc32", + "fd-slicer" + ] } }, "remote": { diff --git a/tools/lint.js b/tools/lint.js index 2312cde272..5bc3f2654f 100755 --- a/tools/lint.js +++ b/tools/lint.js @@ -211,7 +211,7 @@ async function ensureNoNewITests() { "bench_tests.rs": 0, "cache_tests.rs": 0, "cert_tests.rs": 0, - "check_tests.rs": 2, + "check_tests.rs": 0, "compile_tests.rs": 0, "coverage_tests.rs": 0, "eval_tests.rs": 0, diff --git a/tools/release/npm/.gitignore b/tools/release/npm/.gitignore new file mode 100644 index 0000000000..1521c8b765 --- /dev/null +++ b/tools/release/npm/.gitignore @@ -0,0 +1 @@ +dist diff --git a/tools/release/npm/bin.cjs b/tools/release/npm/bin.cjs new file mode 100644 index 0000000000..984aa350f1 --- /dev/null +++ b/tools/release/npm/bin.cjs @@ -0,0 +1,54 @@ +#!/usr/bin/env node +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +// @ts-check +const path = require("path"); +const child_process = require("child_process"); +const os = require("os"); +const fs = require("fs"); + +const exePath = path.join( + __dirname, + os.platform() === "win32" ? "deno.exe" : "deno", +); + +if (!fs.existsSync(exePath)) { + try { + const resolvedExePath = require("./install_api.cjs").runInstall(); + runDenoExe(resolvedExePath); + } catch (err) { + if (err !== undefined && typeof err.message === "string") { + console.error(err.message); + } else { + console.error(err); + } + process.exit(1); + } +} else { + runDenoExe(exePath); +} + +/** @param exePath {string} */ +function runDenoExe(exePath) { + const result = child_process.spawnSync( + exePath, + process.argv.slice(2), + { stdio: "inherit" }, + ); + if (result.error) { + throw result.error; + } + + throwIfNoExePath(); + + process.exitCode = result.status; + + function throwIfNoExePath() { + if (!fs.existsSync(exePath)) { + throw new Error( + "Could not find exe at path '" + exePath + + "'. Maybe try running deno again.", + ); + } + } +} diff --git a/tools/release/npm/build.ts b/tools/release/npm/build.ts new file mode 100755 index 0000000000..b1f1c45cbf --- /dev/null +++ b/tools/release/npm/build.ts @@ -0,0 +1,237 @@ +#!/usr/bin/env -S deno run -A --lock=tools/deno.lock.json +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +// NOTICE: This deployment/npm folder was lifted from https://github.com/dprint/dprint/blob/0ba79811cc96d2dee8e0cf766a8c8c0fc44879c2/deployment/npm/ +// with permission (Copyright 2019-2023 David Sherret) +import $ from "jsr:@david/dax@^0.42.0"; +// @ts-types="npm:@types/decompress@4.2.7" +import decompress from "npm:decompress@4.2.1"; +import { parseArgs } from "@std/cli/parse-args"; + +interface Package { + zipFileName: string; + os: "win32" | "darwin" | "linux"; + cpu: "x64" | "arm64"; + libc?: "glibc" | "musl"; +} + +const args = parseArgs(Deno.args, { + boolean: ["publish"], +}); +const packages: Package[] = [{ + zipFileName: "deno-x86_64-pc-windows-msvc.zip", + os: "win32", + cpu: "x64", +}, { + // use x64_64 until there's an arm64 build + zipFileName: "deno-x86_64-pc-windows-msvc.zip", + os: "win32", + cpu: "arm64", +}, { + zipFileName: "deno-x86_64-apple-darwin.zip", + os: "darwin", + cpu: "x64", +}, { + zipFileName: "deno-aarch64-apple-darwin.zip", + os: "darwin", + cpu: "arm64", +}, { + zipFileName: "deno-x86_64-unknown-linux-gnu.zip", + os: "linux", + cpu: "x64", + libc: "glibc", +}, { + zipFileName: "deno-aarch64-unknown-linux-gnu.zip", + os: "linux", + cpu: "arm64", + libc: "glibc", +}]; + +const markdownText = `# Deno + +[Deno](https://www.deno.com) +([/ˈdiːnoʊ/](http://ipa-reader.xyz/?text=%CB%88di%CB%90no%CA%8A), pronounced +\`dee-no\`) is a JavaScript, TypeScript, and WebAssembly runtime with secure +defaults and a great developer experience. It's built on [V8](https://v8.dev/), +[Rust](https://www.rust-lang.org/), and [Tokio](https://tokio.rs/). + +Learn more about the Deno runtime +[in the documentation](https://docs.deno.com/runtime/manual). +`; + +const currentDir = $.path(import.meta.url).parentOrThrow(); +const rootDir = currentDir.parentOrThrow().parentOrThrow().parentOrThrow(); +const outputDir = currentDir.join("./dist"); +const scopeDir = outputDir.join("@deno"); +const denoDir = outputDir.join("deno"); +const version = resolveVersion(); + +$.logStep(`Publishing ${version}...`); + +await $`rm -rf ${outputDir}`; +await $`mkdir -p ${denoDir} ${scopeDir}`; + +// setup Deno packages +{ + $.logStep(`Setting up deno ${version}...`); + const pkgJson = { + "name": "deno", + "version": version, + "description": "A modern runtime for JavaScript and TypeScript.", + "bin": "bin.cjs", + "repository": { + "type": "git", + "url": "git+https://github.com/denoland/deno.git", + }, + "keywords": [ + "runtime", + "typescript", + ], + "author": "the Deno authors", + "license": "MIT", + "bugs": { + "url": "https://github.com/denoland/deno/issues", + }, + "homepage": "https://deno.com", + // for yarn berry (https://github.com/dprint/dprint/issues/686) + "preferUnplugged": true, + "scripts": { + "postinstall": "node ./install.cjs", + }, + optionalDependencies: packages + .map((pkg) => `@deno/${getPackageNameNoScope(pkg)}`) + .reduce((obj, pkgName) => ({ ...obj, [pkgName]: version }), {}), + }; + currentDir.join("bin.cjs").copyFileToDirSync(denoDir); + currentDir.join("install_api.cjs").copyFileToDirSync(denoDir); + currentDir.join("install.cjs").copyFileToDirSync(denoDir); + denoDir.join("package.json").writeJsonPrettySync(pkgJson); + rootDir.join("LICENSE.md").copyFileSync(denoDir.join("LICENSE")); + denoDir.join("README.md").writeTextSync(markdownText); + // ensure the test files don't get published + denoDir.join(".npmignore").writeTextSync("deno\ndeno.exe\n"); + + // setup each binary package + for (const pkg of packages) { + const pkgName = getPackageNameNoScope(pkg); + $.logStep(`Setting up @deno/${pkgName}...`); + const pkgDir = scopeDir.join(pkgName); + const zipPath = pkgDir.join("output.zip"); + + await $`mkdir -p ${pkgDir}`; + + // download and extract the zip file + const zipUrl = + `https://github.com/denoland/deno/releases/download/v${version}/${pkg.zipFileName}`; + await $.request(zipUrl).showProgress().pipeToPath(zipPath); + await decompress(zipPath.toString(), pkgDir.toString()); + zipPath.removeSync(); + + // create the package.json and readme + pkgDir.join("README.md").writeTextSync( + `# @denoland/${pkgName}\n\n${pkgName} distribution of [Deno](https://deno.land).\n`, + ); + pkgDir.join("package.json").writeJsonPrettySync({ + "name": `@deno/${pkgName}`, + "version": version, + "description": `${pkgName} distribution of Deno`, + "repository": { + "type": "git", + "url": "git+https://github.com/denoland/deno.git", + }, + // force yarn to unpack + "preferUnplugged": true, + "author": "David Sherret", + "license": "MIT", + "bugs": { + "url": "https://github.com/denoland/deno/issues", + }, + "homepage": "https://deno.land", + "os": [pkg.os], + "cpu": [pkg.cpu], + libc: pkg.libc == null ? undefined : [pkg.libc], + }); + } +} + +// verify that the package is created correctly +{ + $.logStep("Verifying packages..."); + const testPlatform = Deno.build.os == "windows" + ? (Deno.build.arch === "x86_64" ? "@deno/win32-x64" : "@deno/win32-arm64") + : Deno.build.os === "darwin" + ? (Deno.build.arch === "x86_64" ? "@deno/darwin-x64" : "@deno/darwin-arm64") + : "@deno/linux-x64-glibc"; + outputDir.join("package.json").writeJsonPrettySync({ + workspaces: [ + "deno", + // There seems to be a bug with npm workspaces where this doesn't + // work, so for now make some assumptions and only include the package + // that works on the CI for the current operating system + // ...packages.map(p => `@deno/${getPackageNameNoScope(p)}`), + testPlatform, + ], + }); + + const denoExe = Deno.build.os === "windows" ? "deno.exe" : "deno"; + await $`npm install`.cwd(denoDir); + + // ensure the post-install script adds the executable to the deno package, + // which is necessary for faster caching and to ensure the vscode extension + // picks it up + if (!denoDir.join(denoExe).existsSync()) { + throw new Error("Deno executable did not exist after post install"); + } + + // run once after post install created deno, once with a simulated readonly file system, once creating the cache and once with + await $`node bin.cjs -v && rm ${denoExe} && DENO_SIMULATED_READONLY_FILE_SYSTEM=1 node bin.cjs -v && node bin.cjs -v && node bin.cjs -v` + .cwd(denoDir); + + if (!denoDir.join(denoExe).existsSync()) { + throw new Error("Deno executable did not exist when lazily initialized"); + } +} + +// publish if necessary +if (args.publish) { + for (const pkg of packages) { + const pkgName = getPackageNameNoScope(pkg); + $.logStep(`Publishing @deno/${pkgName}...`); + if (await checkPackagePublished(`@deno/${pkgName}`)) { + $.logLight(" Already published."); + continue; + } + const pkgDir = scopeDir.join(pkgName); + await $`cd ${pkgDir} && npm publish --provenance --access public`; + } + + $.logStep(`Publishing deno...`); + await $`cd ${denoDir} && npm publish --provenance --access public`; +} + +function getPackageNameNoScope(name: Package) { + const libc = name.libc == null ? "" : `-${name.libc}`; + return `${name.os}-${name.cpu}${libc}`; +} + +function resolveVersion() { + const firstArg = args._[0]; + if ( + firstArg != null && + typeof firstArg === "string" && + firstArg.trim().length > 0 + ) { + return firstArg; + } + const version = (rootDir.join("cli/Cargo.toml").readTextSync().match( + /version = "(.*?)"/, + ))?.[1]; + if (version == null) { + throw new Error("Could not resolve version."); + } + return version; +} + +async function checkPackagePublished(pkgName: string) { + const result = await $`npm info ${pkgName}@${version}`.quiet().noThrow(); + return result.code === 0; +} diff --git a/tools/release/npm/install.cjs b/tools/release/npm/install.cjs new file mode 100644 index 0000000000..8bf9aabe47 --- /dev/null +++ b/tools/release/npm/install.cjs @@ -0,0 +1,5 @@ +// @ts-check +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +"use strict"; + +require("./install_api.cjs").runInstall(); diff --git a/tools/release/npm/install_api.cjs b/tools/release/npm/install_api.cjs new file mode 100644 index 0000000000..026d8ccc45 --- /dev/null +++ b/tools/release/npm/install_api.cjs @@ -0,0 +1,196 @@ +// @ts-check +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +"use strict"; + +const fs = require("fs"); +const os = require("os"); +const path = require("path"); +/** @type {string | undefined} */ +let cachedIsMusl = undefined; + +module.exports = { + runInstall() { + const denoFileName = os.platform() === "win32" ? "deno.exe" : "deno"; + const targetExecutablePath = path.join( + __dirname, + denoFileName, + ); + + if (fs.existsSync(targetExecutablePath)) { + return targetExecutablePath; + } + + const target = getTarget(); + const sourcePackagePath = path.dirname( + require.resolve("@deno/" + target + "/package.json"), + ); + const sourceExecutablePath = path.join(sourcePackagePath, denoFileName); + + if (!fs.existsSync(sourceExecutablePath)) { + throw new Error( + "Could not find executable for @deno/" + target + " at " + + sourceExecutablePath, + ); + } + + try { + if (process.env.DPRINT_SIMULATED_READONLY_FILE_SYSTEM === "1") { + console.warn("Simulating readonly file system for testing."); + throw new Error("Throwing for testing purposes."); + } + + // in order to make things faster the next time we run and to allow the + // deno vscode extension to easily pick this up, copy the executable + // into the deno package folder + hardLinkOrCopy(sourceExecutablePath, targetExecutablePath); + if (os.platform() !== "win32") { + // chomd +x + chmodX(targetExecutablePath); + } + return targetExecutablePath; + } catch (err) { + // this may fail on readonly file systems... in this case, fall + // back to using the resolved package path + if (process.env.DENO_DEBUG === "1") { + console.warn( + "Failed to copy executable from " + + sourceExecutablePath + " to " + targetExecutablePath + + ". Using resolved package path instead.", + err, + ); + } + // use the path found in the specific package + try { + chmodX(sourceExecutablePath); + } catch (_err) { + // ignore + } + return sourceExecutablePath; + } + }, +}; + +/** @filePath {string} */ +function chmodX(filePath) { + const perms = fs.statSync(filePath).mode; + fs.chmodSync(filePath, perms | 0o111); +} + +function getTarget() { + const platform = os.platform(); + if (platform === "linux") { + return platform + "-" + getArch() + "-" + getLinuxFamily(); + } else { + return platform + "-" + getArch(); + } +} + +function getArch() { + const arch = os.arch(); + if (arch !== "arm64" && arch !== "x64") { + throw new Error( + "Unsupported architecture " + os.arch() + + ". Only x64 and aarch64 binaries are available.", + ); + } + return arch; +} + +function getLinuxFamily() { + if (getIsMusl()) { + throw new Error( + "Musl is not supported. It's one of our priorities. Please upvote this issue: https://github.com/denoland/deno/issues/3711", + ); + // return "musl"; + } + return "glibc"; + + function getIsMusl() { + // code adapted from https://github.com/lovell/detect-libc + // Copyright Apache 2.0 license, the detect-libc maintainers + if (cachedIsMusl == null) { + cachedIsMusl = innerGet(); + } + return cachedIsMusl; + + function innerGet() { + try { + if (os.platform() !== "linux") { + return false; + } + return isProcessReportMusl() || isConfMusl(); + } catch (err) { + // just in case + console.warn("Error checking if musl.", err); + return false; + } + } + + function isProcessReportMusl() { + if (!process.report) { + return false; + } + const rawReport = process.report.getReport(); + const report = typeof rawReport === "string" + ? JSON.parse(rawReport) + : rawReport; + if (!report || !(report.sharedObjects instanceof Array)) { + return false; + } + return report.sharedObjects.some((o) => + o.includes("libc.musl-") || o.includes("ld-musl-") + ); + } + + function isConfMusl() { + const output = getCommandOutput(); + const [_, ldd1] = output.split(/[\r\n]+/); + return ldd1 && ldd1.includes("musl"); + } + + function getCommandOutput() { + try { + const command = + "getconf GNU_LIBC_VERSION 2>&1 || true; ldd --version 2>&1 || true"; + return require("child_process").execSync(command, { encoding: "utf8" }); + } catch (_err) { + return ""; + } + } + } +} + +/** + * @param sourcePath {string} + * @param destinationPath {string} + */ +function hardLinkOrCopy(sourcePath, destinationPath) { + try { + fs.linkSync(sourcePath, destinationPath); + } catch { + atomicCopyFile(sourcePath, destinationPath); + } +} + +/** + * @param sourcePath {string} + * @param destinationPath {string} + */ +function atomicCopyFile(sourcePath, destinationPath) { + const crypto = require("crypto"); + const rand = crypto.randomBytes(4).toString("hex"); + const tempFilePath = destinationPath + "." + rand; + fs.copyFileSync(sourcePath, tempFilePath); + try { + fs.renameSync(tempFilePath, destinationPath); + } catch (err) { + // will maybe throw when another process had already done this + // so just ignore and delete the created temporary file + try { + fs.unlinkSync(tempFilePath); + } catch (_err2) { + // ignore + } + throw err; + } +}