archive/node
1
0
Fork 0
mirror of https://github.com/nodejs/node.git synced 2025-08-15 13:48:44 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

build: replace js2c.py with js2c.cc

Browse source 
PR-URL: https://github.com/nodejs/node/pull/46997
Reviewed-By: Yagiz Nizipli <yagiz@nizipli.com>
This commit is contained in:
Joyee Cheung 2023-03-07 17:46:04 +01:00
parent 7fba2c223d
commit 4da7bc915c
No known key found for this signature in database
GPG key ID: 92B78A53C8303B8D
5 changed files with 11 additions and 292 deletions
Download patch file Download diff file Expand all files Collapse all files

1
deps/simdutf/simdutf.gyp vendored
View file

@ -7,6 +7,7 @@
'targets': [
{
'target_name': 'simdutf',
'toolsets': ['host', 'target'],
'type': 'static_library',
'include_dirs': ['.'],
'direct_dependent_settings': {

1
deps/uv/uv.gyp vendored
View file

@ -155,6 +155,7 @@
'targets': [
{
'target_name': 'libuv',
'toolsets': ['host', 'target'],
'type': '<(uv_library)',
'include_dirs': [
'include',

19
node.gyp
View file

@ -27,7 +27,7 @@
'node_lib_target_name%': 'libnode',
'node_intermediate_lib_type%': 'static_library',
'node_builtin_modules_path%': '',
# We list the deps/ files out instead of globbing them in js2c.py since we
# We list the deps/ files out instead of globbing them in js2c.cc since we
# only include a subset of all the files under these directories.
# The lengths of their file names combined should not exceed the
# Windows command length limit or there would be an error.
@ -362,6 +362,7 @@
'src/quic/transportparams.h',
],
'node_mksnapshot_exec': '<(PRODUCT_DIR)/<(EXECUTABLE_PREFIX)node_mksnapshot<(EXECUTABLE_SUFFIX)',
'node_js2c_exec': '<(PRODUCT_DIR)/<(EXECUTABLE_PREFIX)node_js2c<(EXECUTABLE_SUFFIX)',
'conditions': [
['GENERATOR == "ninja"', {
'node_text_start_object_path': 'src/large_pages/node_text_start.node_text_start.o'
@ -770,6 +771,7 @@
'deps/uvwasi/uvwasi.gyp:uvwasi',
'deps/simdutf/simdutf.gyp:simdutf',
'deps/ada/ada.gyp:ada',
'node_js2c#host',
],
'sources': [
@ -925,8 +927,7 @@
'action_name': 'node_js2c',
'process_outputs_as_sources': 1,
'inputs': [
# Put the code first so it's a dependency and can be used for invocation.
'tools/js2c.py',
'<(node_js2c_exec)',
'<@(library_files)',
'<@(deps_files)',
'config.gypi'
@ -935,12 +936,9 @@
'<(SHARED_INTERMEDIATE_DIR)/node_javascript.cc',
],
'action': [
'<(python)',
'tools/js2c.py',
'--directory',
'lib',
'--target',
'<(node_js2c_exec)',
'<@(_outputs)',
'lib',
'config.gypi',
'<@(deps_files)',
],
@ -1179,8 +1177,9 @@
{
'target_name': 'node_js2c',
'type': 'executable',
'toolsets': ['host'],
'dependencies': [
'deps/simdutf/simdutf.gyp:simdutf',
'deps/simdutf/simdutf.gyp:simdutf#host',
],
'include_dirs': [
'tools'
@ -1191,7 +1190,7 @@
],
'conditions': [
[ 'node_shared_libuv=="false"', {
'dependencies': [ 'deps/uv/uv.gyp:libuv' ],
'dependencies': [ 'deps/uv/uv.gyp:libuv#host' ],
}],
[ 'debug_node=="true"', {
'cflags!': [ '-O3' ],

14
test/tools/test_js2c.py
View file

@ -1,14 +0,0 @@
import unittest
import sys, os
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__),
'..', '..', 'tools')))
from js2c import NormalizeFileName
class Js2ctest(unittest.TestCase):
def testNormalizeFileName(self):
self.assertEqual(NormalizeFileName('dir/mod.js'), 'mod')
self.assertEqual(NormalizeFileName('deps/mod.js'), 'internal/deps/mod')
self.assertEqual(NormalizeFileName('mod.js'), 'mod')
if __name__ == '__main__':
unittest.main()

268
tools/js2c.py
View file

@ -1,268 +0,0 @@
#!/usr/bin/env python
#
# Copyright 2006-2008 the V8 project authors. All rights reserved.
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following
# disclaimer in the documentation and/or other materials provided
# with the distribution.
# * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived
# from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""
This is a utility for converting JavaScript source code into uint16_t[],
that are used for embedding JavaScript code into the Node.js binary.
"""
import argparse
import os
import re
import functools
import codecs
import utils
def ReadFile(filename):
if is_verbose:
print(filename)
with codecs.open(filename, "r", "utf-8") as f:
lines = f.read()
return lines
TEMPLATE = """
#include "env-inl.h"
#include "node_builtins.h"
#include "node_external_reference.h"
#include "node_internals.h"
namespace node {{
namespace builtins {{
{0}
namespace {{
const ThreadsafeCopyOnWrite<BuiltinSourceMap> global_source_map {{
BuiltinSourceMap {{
{1}
}} // BuiltinSourceMap
}}; // ThreadsafeCopyOnWrite
}} // anonymous namespace
void BuiltinLoader::LoadJavaScriptSource() {{
source_ = global_source_map;
}}
void RegisterExternalReferencesForInternalizedBuiltinCode(
ExternalReferenceRegistry* registry) {{
{2}
}}
UnionBytes BuiltinLoader::GetConfig() {{
return UnionBytes(&{3});
}}
}} // namespace builtins
}} // namespace node
"""
ONE_BYTE_STRING = """
static const uint8_t {0}[] = {{
{1}
}};
static StaticExternalOneByteResource {2}({0}, {3}, nullptr);
"""
TWO_BYTE_STRING = """
static const uint16_t {0}[] = {{
{1}
}};
static StaticExternalTwoByteResource {2}({0}, {3}, nullptr);
"""
INITIALIZER = ' {{"{0}", UnionBytes(&{1}) }},'
REGISTRATION = ' registry->Register(&{0});'
CONFIG_GYPI_ID = 'config_raw'
CONFIG_GYPI_RESOURCE_ID = 'config_resource'
SLUGGER_RE = re.compile(r'[.\-/]')
is_verbose = False
def GetDefinition(var, source, resource_var):
template = ONE_BYTE_STRING
code_points = [ord(c) for c in source]
if any(c > 127 for c in code_points):
template = TWO_BYTE_STRING
# Treat non-ASCII as UTF-8 and encode as UTF-16 Little Endian.
encoded_source = bytearray(source, 'utf-16le')
code_points = [
encoded_source[i] + (encoded_source[i + 1] * 256)
for i in range(0, len(encoded_source), 2)
]
elements_s = ['%s' % x for x in code_points]
array_content = ','.join(elements_s) + ','
length = len(code_points)
definition = template.format(var, array_content, resource_var, length)
return definition
def AddModule(filename, definitions, initializers, registrations):
code = ReadFile(filename)
name = NormalizeFileName(filename)
slug = SLUGGER_RE.sub('_', name)
var = slug + '_raw'
resource_var = slug + '_resource'
definition = GetDefinition(var, code, resource_var)
initializer = INITIALIZER.format(name, resource_var)
registration = REGISTRATION.format(resource_var)
definitions.append(definition)
initializers.append(initializer)
registrations.append(registration)
def NormalizeFileName(filename):
split = filename.split('/')
if split[0] == 'deps':
split = ['internal'] + split
else: # `lib/**/*.js` so drop the 'lib' part
split = split[1:]
if len(split):
filename = '/'.join(split)
return os.path.splitext(filename)[0]
def JS2C(source_files, target):
# Build source code lines
definitions = []
initializers = []
registrations = []
for filename in source_files['.js']:
AddModule(filename, definitions, initializers, registrations)
for filename in source_files['.mjs']:
AddModule(filename, definitions, initializers, registrations)
config_def = handle_config_gypi(source_files['config.gypi'])
definitions.append(config_def)
# Emit result
definitions = ''.join(definitions)
initializers = '\n'.join(initializers)
registrations = '\n'.join(registrations)
out = TEMPLATE.format(definitions, initializers,
registrations, CONFIG_GYPI_RESOURCE_ID)
write_if_chaged(out, target)
def handle_config_gypi(config_filename):
# if its a gypi file we're going to want it as json
# later on anyway, so get it out of the way now
config = ReadFile(config_filename)
config = jsonify(config)
config_def = GetDefinition(CONFIG_GYPI_ID, config, CONFIG_GYPI_RESOURCE_ID)
return config_def
def jsonify(config):
# 1. string comments
config = re.sub(r'#.*?\n', '', config)
# 2. join multiline strings
config = re.sub(r"'$\s+'", '', config, flags=re.M)
# 3. normalize string literals from ' into "
config = re.sub('\'', '"', config)
# 4. turn pseudo-booleans strings into Booleans
config = re.sub('"true"', 'true', config)
config = re.sub('"false"', 'false', config)
return config
def write_if_chaged(content, target):
if os.path.exists(target):
with open(target, 'rt') as existing:
old_content = existing.read()
else:
old_content = ''
if old_content == content:
os.utime(target, None)
return
with open(target, "wt") as output:
output.write(content)
def SourceFileByExt(files_by_ext, filename):
"""
:type files_by_ext: dict
:type filename: str
:rtype: dict
"""
ext = os.path.splitext(filename)[-1]
files_by_ext.setdefault(ext, []).append(filename)
return files_by_ext
def main():
parser = argparse.ArgumentParser(
description='Convert code files into `uint16_t[]`s',
fromfile_prefix_chars='@'
)
parser.add_argument('--target', help='output file')
parser.add_argument(
'--directory',
default=None,
help='input file directory')
parser.add_argument(
'--root',
default=None,
help='root directory containing the sources')
parser.add_argument('--verbose', action='store_true', help='output file')
parser.add_argument('sources', nargs='*', help='input files')
options = parser.parse_args()
global is_verbose
is_verbose = options.verbose
sources = options.sources
if options.root is not None:
os.chdir(options.root)
if options.directory is not None:
js_files = utils.SearchFiles(options.directory, 'js')
mjs_files = utils.SearchFiles(options.directory, 'mjs')
sources = js_files + mjs_files + options.sources
source_files = functools.reduce(SourceFileByExt, sources, {})
# Should have exactly 3 types: `.js`, `.mjs` and `.gypi`
assert len(source_files) == 3
# Currently config.gypi is the only `.gypi` file allowed
assert len(source_files['.gypi']) == 1
assert os.path.basename(source_files['.gypi'][0]) == 'config.gypi'
source_files['config.gypi'] = source_files.pop('.gypi')[0]
source_files['.js'] = sorted(source_files['.js'])
source_files['.mjs'] = sorted(source_files['.mjs'])
JS2C(source_files, options.target)
if __name__ == "__main__":
main()