build: replace js2c.py with js2c.cc

PR-URL: https://github.com/nodejs/node/pull/46997
Reviewed-By: Yagiz Nizipli <yagiz@nizipli.com>

deps/simdutf/simdutf.gyp

@@ -7,6 +7,7 @@
   'targets': [
     {
       'target_name': 'simdutf',
+      'toolsets': ['host', 'target'],
       'type': 'static_library',
       'include_dirs': ['.'],
       'direct_dependent_settings': {

deps/uv/uv.gyp

@@ -155,6 +155,7 @@
   'targets': [
     {
       'target_name': 'libuv',
+      'toolsets': ['host', 'target'],
       'type': '<(uv_library)',
       'include_dirs': [
         'include',

node.gyp

@@ -27,7 +27,7 @@
     'node_lib_target_name%': 'libnode',
     'node_intermediate_lib_type%': 'static_library',
     'node_builtin_modules_path%': '',
-    # We list the deps/ files out instead of globbing them in js2c.py since we
+    # We list the deps/ files out instead of globbing them in js2c.cc since we
     # only include a subset of all the files under these directories.
     # The lengths of their file names combined should not exceed the
     # Windows command length limit or there would be an error.
@@ -362,6 +362,7 @@
       'src/quic/transportparams.h',
     ],
     'node_mksnapshot_exec': '<(PRODUCT_DIR)/<(EXECUTABLE_PREFIX)node_mksnapshot<(EXECUTABLE_SUFFIX)',
+    'node_js2c_exec': '<(PRODUCT_DIR)/<(EXECUTABLE_PREFIX)node_js2c<(EXECUTABLE_SUFFIX)',
     'conditions': [
       ['GENERATOR == "ninja"', {
         'node_text_start_object_path': 'src/large_pages/node_text_start.node_text_start.o'
@@ -770,6 +771,7 @@
         'deps/uvwasi/uvwasi.gyp:uvwasi',
         'deps/simdutf/simdutf.gyp:simdutf',
         'deps/ada/ada.gyp:ada',
+        'node_js2c#host',
       ],
 
       'sources': [
@@ -925,8 +927,7 @@
           'action_name': 'node_js2c',
           'process_outputs_as_sources': 1,
           'inputs': [
-            # Put the code first so it's a dependency and can be used for invocation.
+            '<(node_js2c_exec)',
-            'tools/js2c.py',
             '<@(library_files)',
             '<@(deps_files)',
             'config.gypi'
@@ -935,12 +936,9 @@
             '<(SHARED_INTERMEDIATE_DIR)/node_javascript.cc',
           ],
           'action': [
-            '<(python)',
+            '<(node_js2c_exec)',
-            'tools/js2c.py',
-            '--directory',
-            'lib',
-            '--target',
             '<@(_outputs)',
+            'lib',
             'config.gypi',
             '<@(deps_files)',
           ],
@@ -1179,8 +1177,9 @@
     {
       'target_name': 'node_js2c',
       'type': 'executable',
+      'toolsets': ['host'],
       'dependencies': [
-        'deps/simdutf/simdutf.gyp:simdutf',
+        'deps/simdutf/simdutf.gyp:simdutf#host',
       ],
       'include_dirs': [
         'tools'
@@ -1191,7 +1190,7 @@
       ],
       'conditions': [
         [ 'node_shared_libuv=="false"', {
-          'dependencies': [ 'deps/uv/uv.gyp:libuv' ],
+          'dependencies': [ 'deps/uv/uv.gyp:libuv#host' ],
         }],
         [ 'debug_node=="true"', {
           'cflags!': [ '-O3' ],

test/tools/test_js2c.py

@@ -1,14 +0,0 @@
-import unittest
-import sys, os
-sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__),
-                                             '..', '..', 'tools')))
-from js2c import NormalizeFileName
-
-class Js2ctest(unittest.TestCase):
-    def testNormalizeFileName(self):
-        self.assertEqual(NormalizeFileName('dir/mod.js'), 'mod')
-        self.assertEqual(NormalizeFileName('deps/mod.js'), 'internal/deps/mod')
-        self.assertEqual(NormalizeFileName('mod.js'), 'mod')
-
-if __name__ == '__main__':
-    unittest.main()

tools/js2c.py

@@ -1,268 +0,0 @@
-#!/usr/bin/env python
-#
-# Copyright 2006-2008 the V8 project authors. All rights reserved.
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are
-# met:
-#
-#     * Redistributions of source code must retain the above copyright
-#       notice, this list of conditions and the following disclaimer.
-#     * Redistributions in binary form must reproduce the above
-#       copyright notice, this list of conditions and the following
-#       disclaimer in the documentation and/or other materials provided
-#       with the distribution.
-#     * Neither the name of Google Inc. nor the names of its
-#       contributors may be used to endorse or promote products derived
-#       from this software without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-"""
-This is a utility for converting JavaScript source code into uint16_t[],
-that are used for embedding JavaScript code into the Node.js binary.
-"""
-import argparse
-import os
-import re
-import functools
-import codecs
-import utils
-
-def ReadFile(filename):
-  if is_verbose:
-    print(filename)
-  with codecs.open(filename, "r", "utf-8") as f:
-    lines = f.read()
-    return lines
-
-
-TEMPLATE = """
-#include "env-inl.h"
-#include "node_builtins.h"
-#include "node_external_reference.h"
-#include "node_internals.h"
-
-namespace node {{
-
-namespace builtins {{
-{0}
-namespace {{
-const ThreadsafeCopyOnWrite<BuiltinSourceMap> global_source_map {{
-  BuiltinSourceMap {{
-{1}
-  }}  // BuiltinSourceMap
-}};  // ThreadsafeCopyOnWrite
-}}  // anonymous namespace
-
-void BuiltinLoader::LoadJavaScriptSource() {{
-  source_ = global_source_map;
-}}
-
-void RegisterExternalReferencesForInternalizedBuiltinCode(
-  ExternalReferenceRegistry* registry) {{
-{2}
-}}
-
-UnionBytes BuiltinLoader::GetConfig() {{
-  return UnionBytes(&{3});
-}}
-
-}}  // namespace builtins
-
-}}  // namespace node
-"""
-
-ONE_BYTE_STRING = """
-static const uint8_t {0}[] = {{
-{1}
-}};
-
-static StaticExternalOneByteResource {2}({0}, {3}, nullptr);
-"""
-
-TWO_BYTE_STRING = """
-static const uint16_t {0}[] = {{
-{1}
-}};
-
-static StaticExternalTwoByteResource {2}({0}, {3}, nullptr);
-"""
-
-INITIALIZER = '    {{"{0}", UnionBytes(&{1}) }},'
-
-REGISTRATION = '  registry->Register(&{0});'
-
-CONFIG_GYPI_ID = 'config_raw'
-
-CONFIG_GYPI_RESOURCE_ID = 'config_resource'
-
-SLUGGER_RE = re.compile(r'[.\-/]')
-
-is_verbose = False
-
-def GetDefinition(var, source, resource_var):
-  template = ONE_BYTE_STRING
-  code_points = [ord(c) for c in source]
-  if any(c > 127 for c in code_points):
-    template = TWO_BYTE_STRING
-    # Treat non-ASCII as UTF-8 and encode as UTF-16 Little Endian.
-    encoded_source = bytearray(source, 'utf-16le')
-    code_points = [
-      encoded_source[i] + (encoded_source[i + 1] * 256)
-      for i in range(0, len(encoded_source), 2)
-    ]
-
-  elements_s = ['%s' % x for x in code_points]
-  array_content = ','.join(elements_s) + ','
-  length = len(code_points)
-  definition = template.format(var, array_content, resource_var, length)
-
-  return definition
-
-
-def AddModule(filename, definitions, initializers, registrations):
-  code = ReadFile(filename)
-  name = NormalizeFileName(filename)
-  slug = SLUGGER_RE.sub('_', name)
-  var = slug + '_raw'
-  resource_var = slug + '_resource'
-  definition = GetDefinition(var, code, resource_var)
-  initializer = INITIALIZER.format(name, resource_var)
-  registration = REGISTRATION.format(resource_var)
-  definitions.append(definition)
-  initializers.append(initializer)
-  registrations.append(registration)
-
-def NormalizeFileName(filename):
-  split = filename.split('/')
-  if split[0] == 'deps':
-    split = ['internal'] + split
-  else:  # `lib/**/*.js` so drop the 'lib' part
-    split = split[1:]
-  if len(split):
-    filename = '/'.join(split)
-  return os.path.splitext(filename)[0]
-
-
-def JS2C(source_files, target):
-  # Build source code lines
-  definitions = []
-  initializers = []
-  registrations = []
-
-  for filename in source_files['.js']:
-    AddModule(filename, definitions, initializers, registrations)
-  for filename in source_files['.mjs']:
-    AddModule(filename, definitions, initializers, registrations)
-
-  config_def = handle_config_gypi(source_files['config.gypi'])
-  definitions.append(config_def)
-
-  # Emit result
-  definitions = ''.join(definitions)
-  initializers = '\n'.join(initializers)
-  registrations = '\n'.join(registrations)
-  out = TEMPLATE.format(definitions, initializers,
-                        registrations, CONFIG_GYPI_RESOURCE_ID)
-  write_if_chaged(out, target)
-
-
-def handle_config_gypi(config_filename):
-  # if its a gypi file we're going to want it as json
-  # later on anyway, so get it out of the way now
-  config = ReadFile(config_filename)
-  config = jsonify(config)
-  config_def = GetDefinition(CONFIG_GYPI_ID, config, CONFIG_GYPI_RESOURCE_ID)
-  return config_def
-
-
-def jsonify(config):
-  # 1. string comments
-  config = re.sub(r'#.*?\n', '', config)
-  # 2. join multiline strings
-  config = re.sub(r"'$\s+'", '', config, flags=re.M)
-  # 3. normalize string literals from ' into "
-  config = re.sub('\'', '"', config)
-  # 4. turn pseudo-booleans strings into Booleans
-  config = re.sub('"true"', 'true', config)
-  config = re.sub('"false"', 'false', config)
-  return config
-
-
-def write_if_chaged(content, target):
-  if os.path.exists(target):
-    with open(target, 'rt') as existing:
-      old_content = existing.read()
-  else:
-    old_content = ''
-  if old_content == content:
-    os.utime(target, None)
-    return
-  with open(target, "wt") as output:
-    output.write(content)
-
-
-def SourceFileByExt(files_by_ext, filename):
-  """
-  :type files_by_ext: dict
-  :type filename: str
-  :rtype: dict
-  """
-  ext = os.path.splitext(filename)[-1]
-  files_by_ext.setdefault(ext, []).append(filename)
-  return files_by_ext
-
-def main():
-  parser = argparse.ArgumentParser(
-    description='Convert code files into `uint16_t[]`s',
-    fromfile_prefix_chars='@'
-  )
-  parser.add_argument('--target', help='output file')
-  parser.add_argument(
-      '--directory',
-      default=None,
-      help='input file directory')
-  parser.add_argument(
-      '--root',
-      default=None,
-      help='root directory containing the sources')
-  parser.add_argument('--verbose', action='store_true', help='output file')
-  parser.add_argument('sources', nargs='*', help='input files')
-  options = parser.parse_args()
-  global is_verbose
-  is_verbose = options.verbose
-  sources = options.sources
-
-  if options.root is not None:
-    os.chdir(options.root)
-
-  if options.directory is not None:
-    js_files = utils.SearchFiles(options.directory, 'js')
-    mjs_files = utils.SearchFiles(options.directory, 'mjs')
-    sources = js_files + mjs_files + options.sources
-
-  source_files = functools.reduce(SourceFileByExt, sources, {})
-
-  # Should have exactly 3 types: `.js`, `.mjs` and `.gypi`
-  assert len(source_files) == 3
-  # Currently config.gypi is the only `.gypi` file allowed
-  assert len(source_files['.gypi']) == 1
-  assert os.path.basename(source_files['.gypi'][0]) == 'config.gypi'
-  source_files['config.gypi'] = source_files.pop('.gypi')[0]
-  source_files['.js'] = sorted(source_files['.js'])
-  source_files['.mjs'] = sorted(source_files['.mjs'])
-
-  JS2C(source_files, options.target)
-
-if __name__ == "__main__":
-  main()