Clean up tools/

- Factor out tools/util.py
- Move js/*.py to tools.
- Rewrite tools/format.sh in python.
- Run lint first in travis.

.gitignore

@@ -1,5 +1,6 @@
 # build
 /out/
+*.pyc
 
 # npm deps
 node_modules

.travis.yml

@@ -44,8 +44,8 @@ install:
  # Travis hangs without -j2 argument to ninja.
  - ninja -j2 -C $BUILD_PATH mock_runtime_test handlers_test deno_cc deno
 script:
+ - ./tools/lint.py
  - $BUILD_PATH/mock_runtime_test
  - $BUILD_PATH/handlers_test
  - $BUILD_PATH/deno_cc foo bar
  - $BUILD_PATH/deno meow
- - ./tools/lint.py

BUILD.gn

@@ -177,7 +177,7 @@ run_node("run_tsc") {
 # be removed at some point. If msg.fps is changed, commit changes to the
 # generated JS files. The stamp file is just to make gn work.
 action("flatbufferjs") {
-  script = "js/flatbufferjs_hack.py"
+  script = "//tools/flatbufferjs_hack.py"
   sources = [
     "src/msg.fbs",
   ]

build_extra/deno.gni

@@ -1,7 +1,7 @@
 template("run_node") {
   action(target_name) {
     forward_variables_from(invoker, "*")
-    script = "//js/run_node.py"
+    script = "//tools/run_node.py"
   }
 }
 

build_extra/flatbuffers/BUILD.gn

@@ -100,7 +100,6 @@ flatbuffer("flatbuffers_samplebuffer") {
   ]
   flatc_include_dirs = [ "$fb_src/tests/include_test" ]
 }
-
 # test("flatbuffers_unittest") {
 #   sources = [
 #     "src/tests/test.cpp",

build_extra/flatbuffers/flatbuffer.gni

@@ -188,15 +188,21 @@ template("ts_flatbuffer") {
   copy_name = target_name + "_copy"
 
   copy(copy_name) {
-    sources = [ "$flatbuffers_source_location/js/flatbuffers.js" ]
+    sources = [
-    outputs = [ "$target_gen_dir/flatbuffers.js" ]
+      "$flatbuffers_source_location/js/flatbuffers.js",
+    ]
+    outputs = [
+      "$target_gen_dir/flatbuffers.js",
+    ]
   }
 
   compiled_action_foreach(target_name) {
     tool = "$flatbuffers_build_location:flatc"
 
     sources = invoker.sources
-    deps = [ ":" + copy_name ]
+    deps = [
+      ":" + copy_name,
+    ]
 
     out_dir = target_gen_dir
 

js/run_node.py

@@ -1,40 +0,0 @@
-#!/usr/bin/env python
-"""
-gn can only run python scripts. This launches a subprocess Node process.
-The working dir of this program is out/Debug/ (AKA root_build_dir)
-Before running node, we symlink js/node_modules to out/Debug/node_modules.
-"""
-import subprocess
-import sys
-import os
-
-
-def symlink(target, name, target_is_dir=False):
-    if os.name == "nt":
-        import ctypes
-        CreateSymbolicLinkW = ctypes.windll.kernel32.CreateSymbolicLinkW
-        CreateSymbolicLinkW.restype = ctypes.c_ubyte
-        CreateSymbolicLinkW.argtypes = (ctypes.c_wchar_p, ctypes.c_wchar_p,
-                                        ctypes.c_uint32)
-
-        flags = 0x02  # SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE
-        if (target_is_dir):
-            flags |= 0x01  # SYMBOLIC_LINK_FLAG_DIRECTORY
-        if not CreateSymbolicLinkW(name, target, flags):
-            raise ctypes.WinError()
-    else:
-        os.symlink(target, name)
-
-
-root_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
-third_party_path = os.path.join(root_path, "third_party")
-target_abs = os.path.join(third_party_path, "node_modules")
-target_rel = os.path.relpath(target_abs)
-
-if not os.path.exists("node_modules"):
-    if os.path.lexists("node_modules"):
-        os.unlink("node_modules")
-    symlink(target_rel, "node_modules", True)
-
-args = ["node"] + sys.argv[1:]
-sys.exit(subprocess.call(args))

tools/build_third_party.py

@@ -10,58 +10,22 @@
 import os
 from os.path import join
 import subprocess
+from util import run, remove_and_symlink
 
 root_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
 third_party_path = join(root_path, "third_party")
 
-
+try:
-def main():
-    try:
     os.makedirs(third_party_path)
-    except:
+except:
     pass
-    os.chdir(third_party_path)
+os.chdir(third_party_path)
-    remove_and_symlink(join("..", "gclient_config.py"), ".gclient")
+remove_and_symlink(join("..", "gclient_config.py"), ".gclient")
-    remove_and_symlink(join("..", "package.json"), "package.json")
+remove_and_symlink(join("..", "package.json"), "package.json")
-    remove_and_symlink(join("..", "yarn.lock"), "yarn.lock")
+remove_and_symlink(join("..", "yarn.lock"), "yarn.lock")
-    remove_and_symlink(join("v8", "third_party", "googletest"), "googletest")
+remove_and_symlink(join("v8", "third_party", "googletest"), "googletest")
-    remove_and_symlink(join("v8", "third_party", "jinja2"), "jinja2")
+remove_and_symlink(join("v8", "third_party", "jinja2"), "jinja2")
-    remove_and_symlink(join("v8", "third_party", "llvm-build"), "llvm-build")
+remove_and_symlink(join("v8", "third_party", "llvm-build"), "llvm-build")
-    remove_and_symlink(join("v8", "third_party", "markupsafe"), "markupsafe")
+remove_and_symlink(join("v8", "third_party", "markupsafe"), "markupsafe")
-    run(["gclient", "sync", "--no-history"])
+run(["gclient", "sync", "--no-history"])
-    run(["yarn"])
+run(["yarn"])
-
-
-def run(args):
-    print " ".join(args)
-    env = os.environ.copy()
-    subprocess.check_call(args, env=env)
-
-
-def remove_and_symlink(target, name):
-    try:
-        os.unlink(name)
-    except:
-        pass
-    os.symlink(target, name)
-
-
-def symlink(target, name, target_is_dir=False):
-    if os.name == "nt":
-        import ctypes
-        CreateSymbolicLinkW = ctypes.windll.kernel32.CreateSymbolicLinkW
-        CreateSymbolicLinkW.restype = ctypes.c_ubyte
-        CreateSymbolicLinkW.argtypes = (ctypes.c_wchar_p, ctypes.c_wchar_p,
-                                        ctypes.c_uint32)
-
-        flags = 0x02  # SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE
-        if (target_is_dir):
-            flags |= 0x01  # SYMBOLIC_LINK_FLAG_DIRECTORY
-        if not CreateSymbolicLinkW(name, target, flags):
-            raise ctypes.WinError()
-    else:
-        os.symlink(target, name)
-
-
-if '__main__' == __name__:
-    main()

tools/flatbufferjs_hack.py

@@ -8,6 +8,7 @@ import subprocess
 import sys
 import os
 import shutil
+import util
 
 # TODO(ry) Ideally flatc output files should be written into target_gen_dir, but
 # its difficult to get this working in a way that parcel can resolve their
@@ -22,12 +23,4 @@ stamp_file = sys.argv[3]
 
 shutil.copyfile(src, dst)
 
-
+util.touch(stamp_file)
-def touch(fname):
-    if os.path.exists(fname):
-        os.utime(fname, None)
-    else:
-        open(fname, 'a').close()
-
-
-touch(stamp_file)

tools/format.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python
+
+import os
+from glob import glob
+from util import run
+
+root_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+
+os.chdir(root_path)
+# TODO(ry) Install clang-format in third_party.
+run(["clang-format", "-i", "-style", "Google"] + glob("src/*.cc") +
+    glob("src/*.h"))
+for fn in ["BUILD.gn", ".gn"] + glob("build_extra/**/*.gn*"):
+    run(["gn", "format", fn])
+# TODO(ry) Install yapf in third_party.
+run(["yapf", "-i"] + glob("tools/*.py"))
+# TODO(ry) Install prettier in third_party.
+run([
+    "prettier", "--write", "js/deno.d.ts", "js/main.ts", "js/mock_runtime.js",
+    "tsconfig.json"
+])
+# Do not format these.
+#  js/msg_generated.ts
+#  js/flatbuffers.js
+run(["rustfmt", "--write-mode", "overwrite"] + glob("src/*.rs"))

tools/format.sh

@@ -1,24 +0,0 @@
-#!/bin/sh
-set -e
-cd `dirname "$0"`/..
-clang-format -i -style Google src/*.cc src/*.h
-
-gn format BUILD.gn
-gn format build_extra/deno.gni
-gn format build_extra/rust/rust.gni
-gn format build_extra/rust/BUILD.gn
-gn format .gn
-
-yapf -i js/*.py
-yapf -i tools/*.py
-
-prettier --write \
-  js/deno.d.ts \
-  js/main.ts \
-  js/mock_runtime.js \
-  tsconfig.json
-# Do not format these.
-#  js/msg_generated.ts
-#  js/flatbuffers.js
-
-rustfmt --write-mode overwrite src/*.rs

tools/lint.py

@@ -2,7 +2,7 @@
 # Does google-lint on c++ files and ts-lint on typescript files
 
 import os
-import subprocess
+from util import run
 
 root_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
 third_party_path = os.path.join(root_path, "third_party")
@@ -10,21 +10,9 @@ cpplint = os.path.join(third_party_path, "cpplint", "cpplint.py")
 tslint = os.path.join(third_party_path, "node_modules", "tslint", "bin",
                       "tslint")
 
-
+os.chdir(root_path)
-def run(args):
+run([
-    print(" ".join(args))
+    "python", cpplint, "--filter=-build/include_subdir", "--repository=src",
-    env = os.environ.copy()
+    "--extensions=cc,h", "--recursive", "src/."
-    subprocess.check_call(args, env=env)
+])
-
+run(["node", tslint, "-p", ".", "--exclude", "js/msg_generated.ts"])
-
-def main():
-    os.chdir(root_path)
-    run([
-        "python", cpplint, "--filter=-build/include_subdir",
-        "--repository=src", "--extensions=cc,h", "--recursive", "src/."
-    ])
-    run(["node", tslint, "-p", ".", "--exclude", "js/msg_generated.ts"])
-
-
-if __name__ == "__main__":
-    main()

tools/run_node.py

@@ -0,0 +1,19 @@
+#!/usr/bin/env python
+"""
+gn can only run python scripts. This launches a subprocess Node process.
+The working dir of this program is out/Debug/ (AKA root_build_dir)
+Before running node, we symlink js/node_modules to out/Debug/node_modules.
+"""
+import subprocess
+import sys
+import os
+import util
+
+root_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+tools_path = os.path.join(root_path, "tools")
+third_party_path = os.path.join(root_path, "third_party")
+target_abs = os.path.join(third_party_path, "node_modules")
+target_rel = os.path.relpath(target_abs)
+
+util.remove_and_symlink(target_rel, "node_modules", True)
+util.run(["node"] + sys.argv[1:])

tools/util.py

@@ -0,0 +1,42 @@
+# Copyright 2018 Ryan Dahl <ry@tinyclouds.org>
+# All rights reserved. MIT License.
+import os
+import subprocess
+
+
+def run(args):
+    print " ".join(args)
+    env = os.environ.copy()
+    subprocess.check_call(args, env=env)
+
+
+def remove_and_symlink(target, name, target_is_dir=False):
+    try:
+        os.unlink(name)
+    except:
+        pass
+    symlink(target, name, target_is_dir)
+
+
+def symlink(target, name, target_is_dir=False):
+    if os.name == "nt":
+        import ctypes
+        CreateSymbolicLinkW = ctypes.windll.kernel32.CreateSymbolicLinkW
+        CreateSymbolicLinkW.restype = ctypes.c_ubyte
+        CreateSymbolicLinkW.argtypes = (ctypes.c_wchar_p, ctypes.c_wchar_p,
+                                        ctypes.c_uint32)
+
+        flags = 0x02  # SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE
+        if (target_is_dir):
+            flags |= 0x01  # SYMBOLIC_LINK_FLAG_DIRECTORY
+        if not CreateSymbolicLinkW(name, target, flags):
+            raise ctypes.WinError()
+    else:
+        os.symlink(target, name)
+
+
+def touch(fname):
+    if os.path.exists(fname):
+        os.utime(fname, None)
+    else:
+        open(fname, 'a').close()