diff --git a/Cargo.lock b/Cargo.lock index df6196d22f..caafc8f612 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1622,6 +1622,7 @@ dependencies = [ "bytes", "data-url", "deno_core", + "deno_path_util", "deno_permissions", "deno_tls", "dyn-clone", diff --git a/ext/fetch/Cargo.toml b/ext/fetch/Cargo.toml index 2ad19bffde..959fac574a 100644 --- a/ext/fetch/Cargo.toml +++ b/ext/fetch/Cargo.toml @@ -18,6 +18,7 @@ base64.workspace = true bytes.workspace = true data-url.workspace = true deno_core.workspace = true +deno_path_util.workspace = true deno_permissions.workspace = true deno_tls.workspace = true dyn-clone = "1" diff --git a/ext/fetch/lib.rs b/ext/fetch/lib.rs index 7a525053b3..a3f5d03e64 100644 --- a/ext/fetch/lib.rs +++ b/ext/fetch/lib.rs @@ -41,6 +41,8 @@ use deno_core::OpState; use deno_core::RcRef; use deno_core::Resource; use deno_core::ResourceId; +use deno_path_util::url_from_file_path; +use deno_path_util::PathToUrlError; use deno_permissions::PermissionCheckError; use deno_tls::rustls::RootCertStore; use deno_tls::Proxy; @@ -172,6 +174,8 @@ pub enum FetchError { NetworkError, #[error("Fetching files only supports the GET method: received {0}")] FsNotGet(Method), + #[error(transparent)] + PathToUrl(#[from] PathToUrlError), #[error("Invalid URL {0}")] InvalidUrl(Url), #[error(transparent)] @@ -436,7 +440,7 @@ where let permissions = state.borrow_mut::(); let path = permissions.check_read(&path, "fetch()")?; let url = match path { - Cow::Owned(path) => Url::from_file_path(path).unwrap(), + Cow::Owned(path) => url_from_file_path(&path)?, Cow::Borrowed(_) => url, }; diff --git a/ext/node/polyfills/_fs/_fs_readFile.ts b/ext/node/polyfills/_fs/_fs_readFile.ts index 029e57c502..b1bc53675d 100644 --- a/ext/node/polyfills/_fs/_fs_readFile.ts +++ b/ext/node/polyfills/_fs/_fs_readFile.ts @@ -10,7 +10,7 @@ import { TextOptionsArgument, } from "ext:deno_node/_fs/_fs_common.ts"; import { Buffer } from "node:buffer"; -import { readAll } from "ext:deno_io/12_io.js"; +import { readAll, readAllSync } from "ext:deno_io/12_io.js"; import { FileHandle } from "ext:deno_node/internal/fs/handle.ts"; import { pathFromURL } from "ext:deno_web/00_infra.js"; import { @@ -39,7 +39,7 @@ type TextCallback = (err: Error | null, data?: string) => void; type BinaryCallback = (err: Error | null, data?: Buffer) => void; type GenericCallback = (err: Error | null, data?: string | Buffer) => void; type Callback = TextCallback | BinaryCallback | GenericCallback; -type Path = string | URL | FileHandle; +type Path = string | URL | FileHandle | number; export function readFile( path: Path, @@ -76,6 +76,9 @@ export function readFile( if (path instanceof FileHandle) { const fsFile = new FsFile(path.fd, Symbol.for("Deno.internal.FsFile")); p = readAll(fsFile); + } else if (typeof path === "number") { + const fsFile = new FsFile(path, Symbol.for("Deno.internal.FsFile")); + p = readAll(fsFile); } else { p = Deno.readFile(path); } @@ -106,23 +109,28 @@ export function readFilePromise( } export function readFileSync( - path: string | URL, + path: string | URL | number, opt: TextOptionsArgument, ): string; export function readFileSync( - path: string | URL, + path: string | URL | number, opt?: BinaryOptionsArgument, ): Buffer; export function readFileSync( - path: string | URL, + path: string | URL | number, opt?: FileOptionsArgument, ): string | Buffer { path = path instanceof URL ? pathFromURL(path) : path; let data; - try { - data = Deno.readFileSync(path); - } catch (err) { - throw denoErrorToNodeError(err, { path, syscall: "open" }); + if (typeof path === "number") { + const fsFile = new FsFile(path, Symbol.for("Deno.internal.FsFile")); + data = readAllSync(fsFile); + } else { + try { + data = Deno.readFileSync(path); + } catch (err) { + throw denoErrorToNodeError(err, { path, syscall: "open" }); + } } const encoding = getEncoding(opt); if (encoding && encoding !== "binary") { diff --git a/runtime/errors.rs b/runtime/errors.rs index 4268fbd502..22ba640bcf 100644 --- a/runtime/errors.rs +++ b/runtime/errors.rs @@ -696,6 +696,7 @@ fn get_fetch_error(error: &FetchError) -> &'static str { FetchError::Permission(e) => get_permission_check_error_class(e), FetchError::NetworkError => "TypeError", FetchError::FsNotGet(_) => "TypeError", + FetchError::PathToUrl(_) => "TypeError", FetchError::InvalidUrl(_) => "TypeError", FetchError::InvalidHeaderName(_) => "TypeError", FetchError::InvalidHeaderValue(_) => "TypeError", diff --git a/tests/node_compat/common.ts b/tests/node_compat/common.ts index 75a06566a6..2982095a29 100644 --- a/tests/node_compat/common.ts +++ b/tests/node_compat/common.ts @@ -2,6 +2,9 @@ import { partition } from "@std/collections/partition"; import { join } from "@std/path"; import * as JSONC from "@std/jsonc"; +import { walk } from "@std/fs/walk"; +import { relative } from "@std/path/posix/relative"; + /** * The test suite matches the folders inside the `test` folder inside the * node repo @@ -61,3 +64,130 @@ export function partitionParallelTestPaths( const partitions = partition(testPaths, (p) => !!p.match(PARALLEL_PATTERN)); return { parallel: partitions[0], sequential: partitions[1] }; } + +export const NODE_IGNORED_TEST_DIRS = [ + "addons", + "async-hooks", + "cctest", + "common", + "doctool", + "embedding", + "fixtures", + "fuzzers", + "js-native-api", + "node-api", + "overlapped-checker", + "report", + "testpy", + "tick-processor", + "tools", + "v8-updates", + "wasi", + "wpt", +]; + +export const VENDORED_NODE_TEST = new URL( + "./runner/suite/test/", + import.meta.url, +); +export const NODE_COMPAT_TEST_DEST_URL = new URL( + "./test/", + import.meta.url, +); + +export async function getNodeTests(): Promise { + const paths: string[] = []; + const rootPath = VENDORED_NODE_TEST.href.slice(7); + for await ( + const item of walk(VENDORED_NODE_TEST, { exts: [".js"] }) + ) { + const path = relative(rootPath, item.path); + if (NODE_IGNORED_TEST_DIRS.every((dir) => !path.startsWith(dir))) { + paths.push(path); + } + } + + return paths.sort(); +} + +export async function getDenoTests() { + const paths: string[] = []; + const rootPath = NODE_COMPAT_TEST_DEST_URL.href.slice(7); + for await ( + const item of walk(NODE_COMPAT_TEST_DEST_URL, { exts: [".js"] }) + ) { + const path = relative(rootPath, item.path); + paths.push(path); + } + + return paths.sort(); +} + +let testSerialId = 0; +const cwd = new URL(".", import.meta.url); + +export async function runNodeCompatTestCase( + testCase: string, + signal?: AbortSignal, +) { + const v8Flags = ["--stack-size=4000"]; + const testSource = await Deno.readTextFile(testCase); + const envVars: Record = {}; + const knownGlobals: string[] = []; + parseFlags(testSource).forEach((flag) => { + switch (flag) { + case "--expose_externalize_string": + v8Flags.push("--expose-externalize-string"); + knownGlobals.push("createExternalizableString"); + break; + case "--expose-gc": + v8Flags.push("--expose-gc"); + knownGlobals.push("gc"); + break; + default: + break; + } + }); + if (knownGlobals.length > 0) { + envVars["NODE_TEST_KNOWN_GLOBALS"] = knownGlobals.join(","); + } + // TODO(nathanwhit): once we match node's behavior on executing + // `node:test` tests when we run a file, we can remove this + const usesNodeTest = testSource.includes("node:test"); + const args = [ + usesNodeTest ? "test" : "run", + "-A", + "--quiet", + "--unstable-unsafe-proto", + "--unstable-bare-node-builtins", + "--unstable-fs", + "--v8-flags=" + v8Flags.join(), + ]; + if (usesNodeTest) { + // deno test typechecks by default + we want to pass script args + args.push("--no-check", "runner.ts", "--", testCase); + } else { + args.push("runner.ts", testCase); + } + + // Pipe stdout in order to output each test result as Deno.test output + // That way the tests will respect the `--quiet` option when provided + return new Deno.Command(Deno.execPath(), { + args, + env: { + TEST_SERIAL_ID: String(testSerialId++), + ...envVars, + }, + cwd, + stdout: "piped", + stderr: "piped", + signal, + }).spawn(); +} + +/** Parses the special "Flags:"" syntax in Node.js test files */ +function parseFlags(source: string): string[] { + const line = /^\/\/ Flags: (.+)$/um.exec(source); + if (line == null) return []; + return line[1].split(" "); +} diff --git a/tests/node_compat/config.jsonc b/tests/node_compat/config.jsonc index 04cb4e6e2d..105341109c 100644 --- a/tests/node_compat/config.jsonc +++ b/tests/node_compat/config.jsonc @@ -348,7 +348,6 @@ "test-dgram-connect-send-default-host.js", "test-dgram-connect-send-empty-array.js", "test-dgram-connect-send-empty-buffer.js", - "test-dgram-connect-send-empty-packet.js", "test-dgram-connect-send-multi-buffer-copy.js", "test-dgram-connect-send-multi-string-array.js", "test-dgram-connect.js", @@ -503,6 +502,7 @@ "test-fs-readdir-stack-overflow.js", "test-fs-readdir.js", "test-fs-readfile-empty.js", + "test-fs-readfile-fd.js", "test-fs-readfile-unlink.js", "test-fs-readfile-zero-byte-liar.js", "test-fs-readfilesync-enoent.js", diff --git a/tests/node_compat/runner/TODO.md b/tests/node_compat/runner/TODO.md index f6393a5e1f..09d68aded7 100644 --- a/tests/node_compat/runner/TODO.md +++ b/tests/node_compat/runner/TODO.md @@ -1,7 +1,7 @@ # Remaining Node Tests -1120 tests out of 3681 have been ported from Node 20.11.1 (30.43% ported, 69.68% remaining). +1163 tests out of 3681 have been ported from Node 20.11.1 (31.59% ported, 68.92% remaining). NOTE: This file should not be manually edited. Please edit `tests/node_compat/config.json` and run `deno task setup` in `tests/node_compat/runner` dir instead. @@ -223,18 +223,12 @@ NOTE: This file should not be manually edited. Please edit `tests/node_compat/co - [parallel/test-child-process-constructor.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-constructor.js) - [parallel/test-child-process-cwd.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-cwd.js) - [parallel/test-child-process-destroy.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-destroy.js) -- [parallel/test-child-process-detached.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-detached.js) - [parallel/test-child-process-disconnect.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-disconnect.js) - [parallel/test-child-process-env.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-env.js) - [parallel/test-child-process-exec-any-shells-windows.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-exec-any-shells-windows.js) -- [parallel/test-child-process-exec-encoding.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-exec-encoding.js) -- [parallel/test-child-process-exec-std-encoding.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-exec-std-encoding.js) - [parallel/test-child-process-exec-timeout-expire.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-exec-timeout-expire.js) - [parallel/test-child-process-exec-timeout-kill.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-exec-timeout-kill.js) -- [parallel/test-child-process-exec-timeout-not-expired.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-exec-timeout-not-expired.js) - [parallel/test-child-process-execFile-promisified-abortController.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-execFile-promisified-abortController.js) -- [parallel/test-child-process-execfile.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-execfile.js) -- [parallel/test-child-process-exit-code.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-exit-code.js) - [parallel/test-child-process-fork-abort-signal.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-fork-abort-signal.js) - [parallel/test-child-process-fork-and-spawn.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-fork-and-spawn.js) - [parallel/test-child-process-fork-args.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-fork-args.js) @@ -249,15 +243,12 @@ NOTE: This file should not be manually edited. Please edit `tests/node_compat/co - [parallel/test-child-process-fork-net-socket.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-fork-net-socket.js) - [parallel/test-child-process-fork-net.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-fork-net.js) - [parallel/test-child-process-fork-no-shell.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-fork-no-shell.js) -- [parallel/test-child-process-fork-ref.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-fork-ref.js) -- [parallel/test-child-process-fork-ref2.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-fork-ref2.js) - [parallel/test-child-process-fork-stdio-string-variant.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-fork-stdio-string-variant.js) - [parallel/test-child-process-fork-stdio.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-fork-stdio.js) - [parallel/test-child-process-fork-timeout-kill-signal.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-fork-timeout-kill-signal.js) - [parallel/test-child-process-fork.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-fork.js) - [parallel/test-child-process-http-socket-leak.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-http-socket-leak.js) - [parallel/test-child-process-internal.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-internal.js) -- [parallel/test-child-process-ipc.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-ipc.js) - [parallel/test-child-process-no-deprecation.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-no-deprecation.js) - [parallel/test-child-process-pipe-dataflow.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-pipe-dataflow.js) - [parallel/test-child-process-promisified.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-promisified.js) @@ -276,19 +267,15 @@ NOTE: This file should not be manually edited. Please edit `tests/node_compat/co - [parallel/test-child-process-spawn-shell.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-spawn-shell.js) - [parallel/test-child-process-spawn-timeout-kill-signal.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-spawn-timeout-kill-signal.js) - [parallel/test-child-process-spawn-typeerror.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-spawn-typeerror.js) -- [parallel/test-child-process-spawnsync-env.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-spawnsync-env.js) - [parallel/test-child-process-spawnsync-input.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-spawnsync-input.js) - [parallel/test-child-process-spawnsync-kill-signal.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-spawnsync-kill-signal.js) - [parallel/test-child-process-spawnsync-shell.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-spawnsync-shell.js) - [parallel/test-child-process-spawnsync-timeout.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-spawnsync-timeout.js) - [parallel/test-child-process-stdin.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-stdin.js) - [parallel/test-child-process-stdio-big-write-end.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-stdio-big-write-end.js) -- [parallel/test-child-process-stdio-inherit.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-stdio-inherit.js) - [parallel/test-child-process-stdio-merge-stdouts-into-cat.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-stdio-merge-stdouts-into-cat.js) - [parallel/test-child-process-stdio-reuse-readable-stdio.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-stdio-reuse-readable-stdio.js) - [parallel/test-child-process-stdio.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-stdio.js) -- [parallel/test-child-process-stdout-flush-exit.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-stdout-flush-exit.js) -- [parallel/test-child-process-stdout-flush.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-stdout-flush.js) - [parallel/test-child-process-stdout-ipc.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-stdout-ipc.js) - [parallel/test-child-process-uid-gid.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-uid-gid.js) - [parallel/test-child-process-validate-stdio.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-validate-stdio.js) @@ -394,7 +381,6 @@ NOTE: This file should not be manually edited. Please edit `tests/node_compat/co - [parallel/test-common.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-common.js) - [parallel/test-console-clear.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-console-clear.js) - [parallel/test-console-count.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-console-count.js) -- [parallel/test-console-instance.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-console-instance.js) - [parallel/test-console-issue-43095.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-console-issue-43095.js) - [parallel/test-console-methods.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-console-methods.js) - [parallel/test-console-stdio-setters.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-console-stdio-setters.js) @@ -503,23 +489,20 @@ NOTE: This file should not be manually edited. Please edit `tests/node_compat/co - [parallel/test-dgram-cluster-bind-error.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-dgram-cluster-bind-error.js) - [parallel/test-dgram-cluster-close-during-bind.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-dgram-cluster-close-during-bind.js) - [parallel/test-dgram-cluster-close-in-listening.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-dgram-cluster-close-in-listening.js) +- [parallel/test-dgram-connect-send-empty-packet.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-dgram-connect-send-empty-packet.js) - [parallel/test-dgram-create-socket-handle-fd.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-dgram-create-socket-handle-fd.js) - [parallel/test-dgram-create-socket-handle.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-dgram-create-socket-handle.js) -- [parallel/test-dgram-custom-lookup.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-dgram-custom-lookup.js) - [parallel/test-dgram-deprecation-error.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-dgram-deprecation-error.js) - [parallel/test-dgram-exclusive-implicit-bind.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-dgram-exclusive-implicit-bind.js) -- [parallel/test-dgram-ipv6only.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-dgram-ipv6only.js) - [parallel/test-dgram-membership.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-dgram-membership.js) - [parallel/test-dgram-multicast-loopback.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-dgram-multicast-loopback.js) - [parallel/test-dgram-multicast-set-interface.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-dgram-multicast-set-interface.js) - [parallel/test-dgram-multicast-setTTL.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-dgram-multicast-setTTL.js) - [parallel/test-dgram-send-address-types.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-dgram-send-address-types.js) -- [parallel/test-dgram-send-cb-quelches-error.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-dgram-send-cb-quelches-error.js) - [parallel/test-dgram-send-queue-info.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-dgram-send-queue-info.js) - [parallel/test-dgram-sendto.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-dgram-sendto.js) - [parallel/test-dgram-setBroadcast.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-dgram-setBroadcast.js) - [parallel/test-dgram-setTTL.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-dgram-setTTL.js) -- [parallel/test-dgram-socket-buffer-size.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-dgram-socket-buffer-size.js) - [parallel/test-dgram-udp6-link-local-address.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-dgram-udp6-link-local-address.js) - [parallel/test-dgram-unref-in-cluster.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-dgram-unref-in-cluster.js) - [parallel/test-diagnostics-channel-http-server-start.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-diagnostics-channel-http-server-start.js) @@ -544,11 +527,9 @@ NOTE: This file should not be manually edited. Please edit `tests/node_compat/co - [parallel/test-dns-perf_hooks.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-dns-perf_hooks.js) - [parallel/test-dns-resolve-promises.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-dns-resolve-promises.js) - [parallel/test-dns-resolveany-bad-ancount.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-dns-resolveany-bad-ancount.js) -- [parallel/test-dns-resolveany.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-dns-resolveany.js) - [parallel/test-dns-set-default-order.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-dns-set-default-order.js) - [parallel/test-dns-setlocaladdress.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-dns-setlocaladdress.js) - [parallel/test-dns-setserver-when-querying.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-dns-setserver-when-querying.js) -- [parallel/test-dns.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-dns.js) - [parallel/test-domain-abort-on-uncaught.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-domain-abort-on-uncaught.js) - [parallel/test-domain-add-remove.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-domain-add-remove.js) - [parallel/test-domain-async-id-map-leak.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-domain-async-id-map-leak.js) @@ -711,7 +692,6 @@ NOTE: This file should not be manually edited. Please edit `tests/node_compat/co - [parallel/test-fs-readdir-types.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-fs-readdir-types.js) - [parallel/test-fs-readdir-ucs2.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-fs-readdir-ucs2.js) - [parallel/test-fs-readfile-error.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-fs-readfile-error.js) -- [parallel/test-fs-readfile-fd.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-fs-readfile-fd.js) - [parallel/test-fs-readfile-flags.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-fs-readfile-flags.js) - [parallel/test-fs-readfile-pipe-large.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-fs-readfile-pipe-large.js) - [parallel/test-fs-readfile-pipe.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-fs-readfile-pipe.js) @@ -1462,8 +1442,6 @@ NOTE: This file should not be manually edited. Please edit `tests/node_compat/co - [parallel/test-net-child-process-connect-reset.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-net-child-process-connect-reset.js) - [parallel/test-net-client-bind-twice.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-net-client-bind-twice.js) - [parallel/test-net-connect-abort-controller.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-net-connect-abort-controller.js) -- [parallel/test-net-connect-buffer.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-net-connect-buffer.js) -- [parallel/test-net-connect-buffer2.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-net-connect-buffer2.js) - [parallel/test-net-connect-keepalive.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-net-connect-keepalive.js) - [parallel/test-net-connect-memleak.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-net-connect-memleak.js) - [parallel/test-net-connect-nodelay.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-net-connect-nodelay.js) @@ -1484,7 +1462,6 @@ NOTE: This file should not be manually edited. Please edit `tests/node_compat/co - [parallel/test-net-onread-static-buffer.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-net-onread-static-buffer.js) - [parallel/test-net-perf_hooks.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-net-perf_hooks.js) - [parallel/test-net-pingpong.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-net-pingpong.js) -- [parallel/test-net-server-call-listen-multiple-times.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-net-server-call-listen-multiple-times.js) - [parallel/test-net-server-drop-connections.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-net-server-drop-connections.js) - [parallel/test-net-server-keepalive.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-net-server-keepalive.js) - [parallel/test-net-server-listen-handle.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-net-server-listen-handle.js) @@ -1493,14 +1470,12 @@ NOTE: This file should not be manually edited. Please edit `tests/node_compat/co - [parallel/test-net-server-nodelay.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-net-server-nodelay.js) - [parallel/test-net-server-reset.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-net-server-reset.js) - [parallel/test-net-server-simultaneous-accepts-produce-warning-once.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-net-server-simultaneous-accepts-produce-warning-once.js) -- [parallel/test-net-server-try-ports.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-net-server-try-ports.js) - [parallel/test-net-socket-byteswritten.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-net-socket-byteswritten.js) - [parallel/test-net-socket-connect-invalid-autoselectfamilyattempttimeout.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-net-socket-connect-invalid-autoselectfamilyattempttimeout.js) - [parallel/test-net-socket-constructor.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-net-socket-constructor.js) - [parallel/test-net-socket-local-address.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-net-socket-local-address.js) - [parallel/test-net-socket-reset-send.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-net-socket-reset-send.js) - [parallel/test-net-socket-reset-twice.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-net-socket-reset-twice.js) -- [parallel/test-net-socket-timeout.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-net-socket-timeout.js) - [parallel/test-net-stream.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-net-stream.js) - [parallel/test-net-throttle.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-net-throttle.js) - [parallel/test-net-write-after-close.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-net-write-after-close.js) @@ -1662,7 +1637,6 @@ NOTE: This file should not be manually edited. Please edit `tests/node_compat/co - [parallel/test-readline-input-onerror.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-readline-input-onerror.js) - [parallel/test-readline-interface-no-trailing-newline.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-readline-interface-no-trailing-newline.js) - [parallel/test-readline-interface-recursive-writes.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-readline-interface-recursive-writes.js) -- [parallel/test-readline-interface.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-readline-interface.js) - [parallel/test-readline-promises-interface.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-readline-promises-interface.js) - [parallel/test-readline-promises-tab-complete.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-readline-promises-tab-complete.js) - [parallel/test-readline-tab-complete.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-readline-tab-complete.js) @@ -2103,7 +2077,6 @@ NOTE: This file should not be manually edited. Please edit `tests/node_compat/co - [parallel/test-tracing-no-crash.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-tracing-no-crash.js) - [parallel/test-tty-backwards-api.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-tty-backwards-api.js) - [parallel/test-tty-stdin-pipe.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-tty-stdin-pipe.js) -- [parallel/test-ttywrap-invalid-fd.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-ttywrap-invalid-fd.js) - [parallel/test-ttywrap-stack.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-ttywrap-stack.js) - [parallel/test-unhandled-exception-rethrow-error.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-unhandled-exception-rethrow-error.js) - [parallel/test-unhandled-exception-with-worker-inuse.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-unhandled-exception-with-worker-inuse.js) @@ -2126,7 +2099,6 @@ NOTE: This file should not be manually edited. Please edit `tests/node_compat/co - [parallel/test-v8-flag-type-check.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-v8-flag-type-check.js) - [parallel/test-v8-flags.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-v8-flags.js) - [parallel/test-v8-getheapsnapshot-twice.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-v8-getheapsnapshot-twice.js) -- [parallel/test-v8-serdes.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-v8-serdes.js) - [parallel/test-v8-serialize-leak.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-v8-serialize-leak.js) - [parallel/test-v8-startup-snapshot-api.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-v8-startup-snapshot-api.js) - [parallel/test-v8-stats.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-v8-stats.js) diff --git a/tests/node_compat/runner/challenge_new_test.ts b/tests/node_compat/runner/challenge_new_test.ts new file mode 100644 index 0000000000..313bf60490 --- /dev/null +++ b/tests/node_compat/runner/challenge_new_test.ts @@ -0,0 +1,72 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +// deno-lint-ignore-file no-console + +import { deadline } from "@std/async/deadline"; +import { ensureDir } from "@std/fs/ensure-dir"; +import { copy } from "@std/fs/copy"; +import { withoutAll } from "@std/collections/without-all"; +import { + getDenoTests, + getNodeTests, + NODE_COMPAT_TEST_DEST_URL, + runNodeCompatTestCase, + VENDORED_NODE_TEST, +} from "../common.ts"; +import { fromFileUrl } from "@std/path/from-file-url"; + +/** The timeout ms for single test execution. If a single test didn't finish in this timeout milliseconds, the test is considered as failure */ +const TIMEOUT = 2000; + +async function main() { + const remainingTests = withoutAll(await getNodeTests(), await getDenoTests()); + + console.log(`Remaining tests: ${remainingTests.length}`); + const success = [] as string[]; + let i = 0; + + Deno.addSignalListener("SIGINT", () => { + console.log(`Success: ${success.length}`); + for (const testPath of success) { + console.log(testPath); + } + Deno.exit(1); + }); + + for (const testPath of remainingTests) { + i++; + const source = new URL(testPath, VENDORED_NODE_TEST); + const dest = new URL(testPath, NODE_COMPAT_TEST_DEST_URL); + + await ensureDir(new URL(".", dest)); + await copy(source, dest); + const num = String(i).padStart(4, " "); + try { + const cp = await runNodeCompatTestCase( + fromFileUrl(dest), + AbortSignal.timeout(TIMEOUT), + ); + const result = await deadline(cp.output(), TIMEOUT + 1000); + if (result.code === 0) { + console.log(`${num} %cPASS`, "color: green", testPath); + success.push(testPath); + } else { + console.log(`${num} %cFAIL`, "color: red", testPath); + } + } catch (e) { + if (e instanceof DOMException && e.name === "TimeoutError") { + console.log(`${num} %cFAIL`, "color: red", testPath); + } else { + console.log(`Unexpected Error`, e); + } + } finally { + await Deno.remove(dest); + } + } + console.log(`Success: ${success.length}`); + for (const testPath of success) { + console.log(testPath); + } + Deno.exit(0); +} + +await main(); diff --git a/tests/node_compat/runner/setup.ts b/tests/node_compat/runner/setup.ts index eeae3ae92d..b655633ead 100755 --- a/tests/node_compat/runner/setup.ts +++ b/tests/node_compat/runner/setup.ts @@ -10,63 +10,20 @@ import { SEPARATOR } from "@std/path/constants"; import { ensureFile } from "@std/fs/ensure-file"; import { writeAll } from "@std/io/write-all"; import { withoutAll } from "@std/collections/without-all"; -import { relative } from "@std/path/posix/relative"; import { version } from "./suite/node_version.ts"; - -import { config, ignoreList } from "../common.ts"; +import { + config, + getDenoTests, + getNodeTests, + ignoreList, + NODE_COMPAT_TEST_DEST_URL, + VENDORED_NODE_TEST, +} from "../common.ts"; const encoder = new TextEncoder(); const NODE_VERSION = version; -export const NODE_IGNORED_TEST_DIRS = [ - "addons", - "async-hooks", - "cctest", - "common", - "doctool", - "embedding", - "fixtures", - "fuzzers", - "js-native-api", - "node-api", - "overlapped-checker", - "report", - "testpy", - "tick-processor", - "tools", - "v8-updates", - "wasi", - "wpt", -]; - -export const VENDORED_NODE_TEST = new URL("./suite/test/", import.meta.url); -export const NODE_COMPAT_TEST_DEST_URL = new URL( - "../test/", - import.meta.url, -); - -export async function getNodeTests(): Promise { - const paths: string[] = []; - const rootPath = VENDORED_NODE_TEST.href.slice(7); - for await ( - const item of walk(VENDORED_NODE_TEST, { exts: [".js"] }) - ) { - const path = relative(rootPath, item.path); - if (NODE_IGNORED_TEST_DIRS.every((dir) => !path.startsWith(dir))) { - paths.push(path); - } - } - - return paths.sort(); -} - -export function getDenoTests() { - return Object.entries(config.tests) - .filter(([testDir]) => !NODE_IGNORED_TEST_DIRS.includes(testDir)) - .flatMap(([testDir, tests]) => tests.map((test) => testDir + "/" + test)); -} - async function updateToDo() { using file = await Deno.open(new URL("./TODO.md", import.meta.url), { write: true, @@ -75,7 +32,7 @@ async function updateToDo() { }); const nodeTests = await getNodeTests(); - const portedTests = getDenoTests(); + const portedTests = await getDenoTests(); const remainingTests = withoutAll(nodeTests, portedTests); const numPorted = portedTests.length; const numMissing = remainingTests.length; @@ -167,7 +124,9 @@ await copyTests(); await updateToDo(); if (Deno.args[0] === "--check") { - const cmd = new Deno.Command("git", { args: ["status", "-s"] }); + const cmd = new Deno.Command("git", { + args: ["status", "-s", "tests/node_compat/test"], + }); const { stdout } = await cmd.output(); if (stdout.length > 0) { diff --git a/tests/node_compat/test.ts b/tests/node_compat/test.ts index 6cb41d2e45..52d2b17169 100644 --- a/tests/node_compat/test.ts +++ b/tests/node_compat/test.ts @@ -24,6 +24,7 @@ import { config, getPathsFromTestSuites, partitionParallelTestPaths, + runNodeCompatTestCase, } from "./common.ts"; // If the test case is invoked like @@ -40,7 +41,6 @@ const testPaths = partitionParallelTestPaths( testPaths.sequential = distinct(testPaths.sequential); testPaths.parallel = distinct(testPaths.parallel); -const cwd = new URL(".", import.meta.url); const windowsIgnorePaths = new Set( getPathsFromTestSuites(config.windowsIgnore), ); @@ -49,13 +49,6 @@ const darwinIgnorePaths = new Set( ); const decoder = new TextDecoder(); -let testSerialId = 0; - -function parseFlags(source: string): string[] { - const line = /^\/\/ Flags: (.+)$/um.exec(source); - if (line == null) return []; - return line[1].split(" "); -} async function runTest(t: Deno.TestContext, path: string): Promise { // If filter patterns are given and any pattern doesn't match @@ -77,60 +70,7 @@ async function runTest(t: Deno.TestContext, path: string): Promise { sanitizeExit: false, fn: async () => { const testCase = join(toolsPath, "test", path); - - const v8Flags = ["--stack-size=4000"]; - const testSource = await Deno.readTextFile(testCase); - const envVars: Record = {}; - const knownGlobals: string[] = []; - parseFlags(testSource).forEach((flag) => { - switch (flag) { - case "--expose_externalize_string": - v8Flags.push("--expose-externalize-string"); - knownGlobals.push("createExternalizableString"); - break; - case "--expose-gc": - v8Flags.push("--expose-gc"); - knownGlobals.push("gc"); - break; - default: - break; - } - }); - if (knownGlobals.length > 0) { - envVars["NODE_TEST_KNOWN_GLOBALS"] = knownGlobals.join(","); - } - // TODO(nathanwhit): once we match node's behavior on executing - // `node:test` tests when we run a file, we can remove this - const usesNodeTest = testSource.includes("node:test"); - const args = [ - usesNodeTest ? "test" : "run", - "-A", - "--quiet", - //"--unsafely-ignore-certificate-errors", - "--unstable-unsafe-proto", - "--unstable-bare-node-builtins", - "--unstable-fs", - "--v8-flags=" + v8Flags.join(), - ]; - if (usesNodeTest) { - // deno test typechecks by default + we want to pass script args - args.push("--no-check", "runner.ts", "--", testCase); - } else { - args.push("runner.ts", testCase); - } - - // Pipe stdout in order to output each test result as Deno.test output - // That way the tests will respect the `--quiet` option when provided - const command = new Deno.Command(Deno.execPath(), { - args, - env: { - TEST_SERIAL_ID: String(testSerialId++), - ...envVars, - }, - cwd, - stdout: "piped", - stderr: "piped", - }).spawn(); + const command = await runNodeCompatTestCase(testCase); const warner = setTimeout(() => { console.error(`Test is running slow: ${testCase}`); }, 2 * 60_000); diff --git a/tests/node_compat/test/parallel/test-dgram-connect-send-empty-packet.js b/tests/node_compat/test/parallel/test-dgram-connect-send-empty-packet.js deleted file mode 100644 index 8c4d52af82..0000000000 --- a/tests/node_compat/test/parallel/test-dgram-connect-send-empty-packet.js +++ /dev/null @@ -1,35 +0,0 @@ -// deno-fmt-ignore-file -// deno-lint-ignore-file - -// Copyright Joyent and Node contributors. All rights reserved. MIT license. -// Taken from Node 20.11.1 -// This file is automatically generated by `tests/node_compat/runner/setup.ts`. Do not modify this file manually. - -'use strict'; -const common = require('../common'); - -const assert = require('assert'); -const dgram = require('dgram'); - -const client = dgram.createSocket('udp4'); - -client.bind(0, common.mustCall(function() { - client.connect(client.address().port, common.mustCall(() => { - client.on('message', common.mustCall(callback)); - const buf = Buffer.alloc(1); - - const interval = setInterval(function() { - client.send(buf, 0, 0, common.mustCall(callback)); - }, 10); - - function callback(firstArg) { - // If client.send() callback, firstArg should be null. - // If client.on('message') listener, firstArg should be a 0-length buffer. - if (firstArg instanceof Buffer) { - assert.strictEqual(firstArg.length, 0); - clearInterval(interval); - client.close(); - } - } - })); -})); diff --git a/tests/node_compat/test/parallel/test-fs-readfile-fd.js b/tests/node_compat/test/parallel/test-fs-readfile-fd.js new file mode 100644 index 0000000000..7edfd1d6a9 --- /dev/null +++ b/tests/node_compat/test/parallel/test-fs-readfile-fd.js @@ -0,0 +1,101 @@ +// deno-fmt-ignore-file +// deno-lint-ignore-file + +// Copyright Joyent and Node contributors. All rights reserved. MIT license. +// Taken from Node 20.11.1 +// This file is automatically generated by `tests/node_compat/runner/setup.ts`. Do not modify this file manually. + +'use strict'; +const common = require('../common'); + +// Test fs.readFile using a file descriptor. + +const fixtures = require('../common/fixtures'); +const assert = require('assert'); +const fs = require('fs'); +const fn = fixtures.path('empty.txt'); +const tmpdir = require('../common/tmpdir'); +tmpdir.refresh(); + +tempFd(function(fd, close) { + fs.readFile(fd, function(err, data) { + assert.ok(data); + close(); + }); +}); + +tempFd(function(fd, close) { + fs.readFile(fd, 'utf8', function(err, data) { + assert.strictEqual(data, ''); + close(); + }); +}); + +tempFdSync(function(fd) { + assert.ok(fs.readFileSync(fd)); +}); + +tempFdSync(function(fd) { + assert.strictEqual(fs.readFileSync(fd, 'utf8'), ''); +}); + +function tempFd(callback) { + fs.open(fn, 'r', function(err, fd) { + assert.ifError(err); + callback(fd, function() { + fs.close(fd, function(err) { + assert.ifError(err); + }); + }); + }); +} + +function tempFdSync(callback) { + const fd = fs.openSync(fn, 'r'); + callback(fd); + fs.closeSync(fd); +} + +{ + // This test makes sure that `readFile()` always reads from the current + // position of the file, instead of reading from the beginning of the file, + // when used with file descriptors. + + const filename = tmpdir.resolve('test.txt'); + fs.writeFileSync(filename, 'Hello World'); + + { + // Tests the fs.readFileSync(). + const fd = fs.openSync(filename, 'r'); + + // Read only five bytes, so that the position moves to five. + const buf = Buffer.alloc(5); + assert.strictEqual(fs.readSync(fd, buf, 0, 5), 5); + assert.strictEqual(buf.toString(), 'Hello'); + + // readFileSync() should read from position five, instead of zero. + assert.strictEqual(fs.readFileSync(fd).toString(), ' World'); + + fs.closeSync(fd); + } + + { + // Tests the fs.readFile(). + fs.open(filename, 'r', common.mustSucceed((fd) => { + const buf = Buffer.alloc(5); + + // Read only five bytes, so that the position moves to five. + fs.read(fd, buf, 0, 5, null, common.mustSucceed((bytes) => { + assert.strictEqual(bytes, 5); + assert.strictEqual(buf.toString(), 'Hello'); + + fs.readFile(fd, common.mustSucceed((data) => { + // readFile() should read from position five, instead of zero. + assert.strictEqual(data.toString(), ' World'); + + fs.closeSync(fd); + })); + })); + })); + } +}