chore: check node_compat config diff in CI (#19119)

.dprint.json

@@ -48,7 +48,7 @@
     "test_util/wpt",
     "third_party",
     "tools/node_compat/TODO.md",
-    "tools/node_compat/versions",
+    "tools/node_compat/node",
     "tools/wpt/expectation.json",
     "tools/wpt/manifest.json",
     "ext/websocket/autobahn/reports"

.github/workflows/ci.generate.ts

@@ -362,6 +362,10 @@ const ci = {
           ...submoduleStep("./test_util/wpt"),
           if: "matrix.wpt",
         },
+        {
+          ...submoduleStep("./tools/node_compat/node"),
+          if: "matrix.job == 'lint'",
+        },
         {
           name: "Create source tarballs (release, linux)",
           if: [
@@ -541,6 +545,12 @@ const ci = {
           run:
             "deno run --unstable --allow-write --allow-read --allow-run ./tools/lint.js",
         },
+        {
+          name: "node_compat/setup.ts --check",
+          if: "matrix.job == 'lint'",
+          run:
+            "deno run --allow-write --allow-read --allow-run=git ./tools/node_compat/setup.ts --check",
+        },
         {
           name: "Build debug",
           if: "matrix.job == 'test' && matrix.profile == 'debug'",

.github/workflows/ci.yml

@@ -120,6 +120,9 @@ jobs:
       - name: Clone submodule ./test_util/wpt
         run: git submodule update --init --recursive --depth=1 -- ./test_util/wpt
         if: '!(github.event_name == ''pull_request'' && matrix.skip_pr) && (matrix.wpt)'
+      - name: Clone submodule ./tools/node_compat/node
+        run: git submodule update --init --recursive --depth=1 -- ./tools/node_compat/node
+        if: '!(github.event_name == ''pull_request'' && matrix.skip_pr) && (matrix.job == ''lint'')'
       - name: 'Create source tarballs (release, linux)'
         if: |-
           !(github.event_name == 'pull_request' && matrix.skip_pr) && (startsWith(matrix.os, 'ubuntu') &&
@@ -328,6 +331,9 @@ jobs:
       - name: lint.js
         if: '!(github.event_name == ''pull_request'' && matrix.skip_pr) && (matrix.job == ''lint'')'
         run: deno run --unstable --allow-write --allow-read --allow-run ./tools/lint.js
+      - name: node_compat/setup.ts --check
+        if: '!(github.event_name == ''pull_request'' && matrix.skip_pr) && (matrix.job == ''lint'')'
+        run: deno run --allow-write --allow-read --allow-run=git ./tools/node_compat/setup.ts --check
       - name: Build debug
         if: '!(github.event_name == ''pull_request'' && matrix.skip_pr) && (matrix.job == ''test'' && matrix.profile == ''debug'')'
         run: cargo build --locked --all-targets

.gitmodules

@@ -10,3 +10,6 @@
 	path = test_util/wpt
 	url = https://github.com/web-platform-tests/wpt.git
 
+[submodule "tools/node_compat/node"]
+	path = tools/node_compat/node
+	url = https://github.com/denoland/node_test.git

tools/node_compat/TODO.md

@@ -3,7 +3,7 @@
 
 NOTE: This file should not be manually edited. Please edit 'cli/tests/node_compat/config.json' and run 'tools/node_compat/setup.ts' instead.
 
-Total: 2934
+Total: 2935
 
 - [abort/test-abort-backtrace.js](https://github.com/nodejs/node/tree/v18.12.1/test/abort/test-abort-backtrace.js)
 - [abort/test-abort-fatal-error.js](https://github.com/nodejs/node/tree/v18.12.1/test/abort/test-abort-fatal-error.js)
@@ -1815,6 +1815,7 @@ Total: 2934
 - [parallel/test-process-env.js](https://github.com/nodejs/node/tree/v18.12.1/test/parallel/test-process-env.js)
 - [parallel/test-process-euid-egid.js](https://github.com/nodejs/node/tree/v18.12.1/test/parallel/test-process-euid-egid.js)
 - [parallel/test-process-exception-capture-errors.js](https://github.com/nodejs/node/tree/v18.12.1/test/parallel/test-process-exception-capture-errors.js)
+- [parallel/test-process-exception-capture-should-abort-on-uncaught-setflagsfromstring.js](https://github.com/nodejs/node/tree/v18.12.1/test/parallel/test-process-exception-capture-should-abort-on-uncaught-setflagsfromstring.js)
 - [parallel/test-process-exception-capture-should-abort-on-uncaught.js](https://github.com/nodejs/node/tree/v18.12.1/test/parallel/test-process-exception-capture-should-abort-on-uncaught.js)
 - [parallel/test-process-exception-capture.js](https://github.com/nodejs/node/tree/v18.12.1/test/parallel/test-process-exception-capture.js)
 - [parallel/test-process-exec-argv.js](https://github.com/nodejs/node/tree/v18.12.1/test/parallel/test-process-exec-argv.js)

tools/node_compat/node

@@ -0,0 +1 @@
+Subproject commit d0d9c1ba9d3facf1086438e21d6d329c599e5a3b

tools/node_compat/setup.ts

@@ -1,23 +1,11 @@
-#!/usr/bin/env -S deno run --allow-read=. --allow-write=. --allow-net=nodejs.org
+#!/usr/bin/env -S deno run --allow-read=. --allow-write=. --allow-run=git
 // Copyright 2018-2023 the Deno authors. All rights reserved. MIT license.
 
-/** This script downloads Node.js source tarball, extracts it and copies the
+/** This copies the test files according to the config file `cli/tests/node_compat/config.jsonc` */
- * test files according to the config file `cli/tests/node_compat/config.json`
- */
 
-import { Foras, gunzip } from "https://deno.land/x/denoflate@2.0.2/deno/mod.ts";
-import { Untar } from "../../test_util/std/archive/untar.ts";
 import { walk } from "../../test_util/std/fs/walk.ts";
-import {
+import { sep } from "../../test_util/std/path/mod.ts";
-  dirname,
-  fromFileUrl,
-  join,
-  sep,
-} from "../../test_util/std/path/mod.ts";
 import { ensureFile } from "../../test_util/std/fs/ensure_file.ts";
-import { Buffer } from "../../test_util/std/io/buffer.ts";
-import { copy } from "../../test_util/std/streams/copy.ts";
-import { readAll } from "../../test_util/std/streams/read_all.ts";
 import { writeAll } from "../../test_util/std/streams/write_all.ts";
 import { withoutAll } from "../../test_util/std/collections/without_all.ts";
 import { relative } from "../../test_util/std/path/posix.ts";
@@ -27,8 +15,6 @@ import { config, ignoreList } from "../../cli/tests/node_compat/common.ts";
 const encoder = new TextEncoder();
 
 const NODE_VERSION = config.nodeVersion;
-const NODE_NAME = "node-v" + NODE_VERSION;
-const NODE_ARCHIVE_NAME = `${NODE_NAME}.tar.gz`;
 
 const NODE_IGNORED_TEST_DIRS = [
   "addons",
@@ -51,25 +37,17 @@ const NODE_IGNORED_TEST_DIRS = [
   "wpt",
 ];
 
-const NODE_TARBALL_URL =
+const VENDORED_NODE_TEST = new URL("node/test/", import.meta.url);
-  `https://nodejs.org/dist/v${NODE_VERSION}/${NODE_ARCHIVE_NAME}`;
-const NODE_VERSIONS_ROOT = new URL("versions/", import.meta.url);
-const NODE_TARBALL_LOCAL_URL = new URL(NODE_ARCHIVE_NAME, NODE_VERSIONS_ROOT);
-// local dir url where we copy the node tests
-const NODE_LOCAL_ROOT_URL = new URL(NODE_NAME, NODE_VERSIONS_ROOT);
-const NODE_LOCAL_TEST_URL = new URL(NODE_NAME + "/test/", NODE_VERSIONS_ROOT);
 const NODE_COMPAT_TEST_DEST_URL = new URL(
   "../../cli/tests/node_compat/test/",
   import.meta.url,
 );
 
-Foras.initSyncBundledOnce();
-
 async function getNodeTests(): Promise<string[]> {
   const paths: string[] = [];
-  const rootPath = NODE_LOCAL_TEST_URL.href.slice(7);
+  const rootPath = VENDORED_NODE_TEST.href.slice(7);
   for await (
-    const item of walk(NODE_LOCAL_TEST_URL, { exts: [".js"] })
+    const item of walk(VENDORED_NODE_TEST, { exts: [".js"] })
   ) {
     const path = relative(rootPath, item.path);
     if (NODE_IGNORED_TEST_DIRS.every((dir) => !path.startsWith(dir))) {
@@ -125,33 +103,6 @@ async function clearTests() {
   }
 }
 
-async function decompressTests() {
-  console.log(`Decompressing ${NODE_ARCHIVE_NAME}...`);
-
-  const compressedFile = await Deno.open(NODE_TARBALL_LOCAL_URL);
-
-  const buffer = new Buffer(gunzip(await readAll(compressedFile)));
-  compressedFile.close();
-
-  const tar = new Untar(buffer);
-  const outFolder = dirname(fromFileUrl(NODE_TARBALL_LOCAL_URL));
-  const testsFolder = `${NODE_NAME}/test`;
-
-  for await (const entry of tar) {
-    if (entry.type !== "file") continue;
-    if (!entry.fileName.startsWith(testsFolder)) continue;
-    const path = join(outFolder, entry.fileName);
-    await ensureFile(path);
-    const file = await Deno.open(path, {
-      create: true,
-      truncate: true,
-      write: true,
-    });
-    await copy(entry, file);
-    file.close();
-  }
-}
-
 /** Checks if file has entry in config.json */
 function hasEntry(file: string, suite: string) {
   return Array.isArray(config.tests[suite]) &&
@@ -161,12 +112,12 @@ function hasEntry(file: string, suite: string) {
 async function copyTests() {
   console.log("Copying test files...");
 
-  for await (const entry of walk(NODE_LOCAL_TEST_URL, { skip: ignoreList })) {
+  for await (const entry of walk(VENDORED_NODE_TEST, { skip: ignoreList })) {
     const fragments = entry.path.split(sep);
     // suite is the directory name after test/. For example, if the file is
-    // "node-v18.12.1/test/fixtures/policy/main.mjs"
+    // "node_comapt/node/test/fixtures/policy/main.mjs"
     // then suite is "fixtures/policy"
-    const suite = fragments.slice(fragments.indexOf(NODE_NAME) + 2, -1)
+    const suite = fragments.slice(fragments.indexOf("node_compat") + 3, -1)
       .join("/");
     if (!hasEntry(entry.name, suite)) {
       continue;
@@ -180,8 +131,9 @@ async function copyTests() {
       write: true,
     });
     const srcFile = await Deno.open(
-      new URL(`${suite}/${entry.name}`, NODE_LOCAL_TEST_URL),
+      new URL(`${suite}/${entry.name}`, VENDORED_NODE_TEST),
     );
+    // Add header to js files
     if (dest.pathname.endsWith("js")) {
       await writeAll(
         destFile,
@@ -199,44 +151,19 @@ async function copyTests() {
   }
 }
 
-/** Downloads Node tarball  */
-async function downloadFile() {
-  console.log(
-    `Downloading ${NODE_TARBALL_URL} in "${NODE_TARBALL_LOCAL_URL}" ...`,
-  );
-  const response = await fetch(NODE_TARBALL_URL);
-  if (!response.ok) {
-    throw new Error(`Request failed with status ${response.status}`);
-  }
-  await ensureFile(NODE_TARBALL_LOCAL_URL);
-  const file = await Deno.open(NODE_TARBALL_LOCAL_URL, {
-    truncate: true,
-    write: true,
-    create: true,
-  });
-  await response.body.pipeTo(file.writable);
-}
-
 // main
 
-try {
-  Deno.lstatSync(NODE_TARBALL_LOCAL_URL);
-} catch (e) {
-  if (!(e instanceof Deno.errors.NotFound)) {
-    throw e;
-  }
-  await downloadFile();
-}
-
-try {
-  Deno.lstatSync(NODE_LOCAL_ROOT_URL);
-} catch (e) {
-  if (!(e instanceof Deno.errors.NotFound)) {
-    throw e;
-  }
-  await decompressTests();
-}
-
 await clearTests();
 await copyTests();
 await updateToDo();
+
+if (Deno.args[0] === "--check") {
+  const cmd = new Deno.Command("git", { args: ["status", "-s"] });
+  const { stdout } = await cmd.output();
+
+  if (stdout.length > 0) {
+    console.log("The following files have been changed:");
+    console.log(new TextDecoder().decode(stdout));
+    Deno.exit(1);
+  }
+}