feat(gyp): update gyp to v0.13.0

This commit is contained in:
Kevin Adler 2022-08-18 10:16:04 -05:00 committed by Richard Lau
parent c379a744c6
commit 3e2a5324f1
13 changed files with 275 additions and 48 deletions

View file

@ -11,11 +11,11 @@ jobs:
max-parallel: 8
matrix:
os: [macos-latest, ubuntu-latest] # , windows-latest]
python-version: [3.6, 3.7, 3.8, 3.9]
python-version: ["3.7", "3.8", "3.9", "3.10"]
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
uses: actions/setup-python@v3
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies

View file

@ -8,23 +8,23 @@ jobs:
fail-fast: false
matrix:
os: [macos-latest, ubuntu-latest, windows-latest]
python: [3.6, 3.9]
python: ["3.7", "3.10"]
runs-on: ${{ matrix.os }}
steps:
- name: Clone gyp-next
uses: actions/checkout@v2
uses: actions/checkout@v3
with:
path: gyp-next
- name: Clone nodejs/node-gyp
uses: actions/checkout@v2
uses: actions/checkout@v3
with:
repository: nodejs/node-gyp
path: node-gyp
- uses: actions/setup-node@v2
- uses: actions/setup-node@v3
with:
node-version: 14.x
- uses: actions/setup-python@v2
- uses: actions/setup-python@v3
with:
python-version: ${{ matrix.python }}
- name: Install dependencies

View file

@ -7,11 +7,11 @@ jobs:
runs-on: windows-latest
steps:
- name: Clone gyp-next
uses: actions/checkout@v2
uses: actions/checkout@v3
with:
path: gyp-next
- name: Clone nodejs/node
uses: actions/checkout@v2
uses: actions/checkout@v3
with:
repository: nodejs/node
path: node

View file

@ -1,5 +1,45 @@
# Changelog
## [0.13.0](https://www.github.com/nodejs/gyp-next/compare/v0.12.1...v0.13.0) (2022-05-11)
### Features
* add PRODUCT_DIR_ABS variable ([#151](https://www.github.com/nodejs/gyp-next/issues/151)) ([80d2626](https://www.github.com/nodejs/gyp-next/commit/80d26263581db829b61b312a7bdb5cc791df7824))
### Bug Fixes
* execvp: printf: Argument list too long ([#147](https://www.github.com/nodejs/gyp-next/issues/147)) ([c4e14f3](https://www.github.com/nodejs/gyp-next/commit/c4e14f301673fadbac3ab7882d0b5f4d02530cb9))
### [0.12.1](https://www.github.com/nodejs/gyp-next/compare/v0.12.0...v0.12.1) (2022-04-06)
### Bug Fixes
* **msvs:** avoid fixing path for arguments with "=" ([#143](https://www.github.com/nodejs/gyp-next/issues/143)) ([7e8f16e](https://www.github.com/nodejs/gyp-next/commit/7e8f16eb165e042e64bec98fa6c2a0232a42c26b))
## [0.12.0](https://www.github.com/nodejs/gyp-next/compare/v0.11.0...v0.12.0) (2022-04-04)
### Features
* support building shared libraries on z/OS ([#137](https://www.github.com/nodejs/gyp-next/issues/137)) ([293bcfa](https://www.github.com/nodejs/gyp-next/commit/293bcfa4c25c6adb743377adafc45a80fee492c6))
## [0.11.0](https://www.github.com/nodejs/gyp-next/compare/v0.10.1...v0.11.0) (2022-03-04)
### Features
* Add proper support for IBM i ([#140](https://www.github.com/nodejs/gyp-next/issues/140)) ([fdda4a3](https://www.github.com/nodejs/gyp-next/commit/fdda4a3038b8a7042ad960ce7a223687c24a21b1))
### [0.10.1](https://www.github.com/nodejs/gyp-next/compare/v0.10.0...v0.10.1) (2021-11-24)
### Bug Fixes
* **make:** only generate makefile for multiple toolsets if requested ([#133](https://www.github.com/nodejs/gyp-next/issues/133)) ([f463a77](https://www.github.com/nodejs/gyp-next/commit/f463a77705973289ea38fec1b244c922ac438e26))
## [0.10.0](https://www.github.com/nodejs/gyp-next/compare/v0.9.6...v0.10.0) (2021-08-26)

View file

@ -103,6 +103,18 @@ def Load(
for (key, val) in generator.generator_default_variables.items():
default_variables.setdefault(key, val)
output_dir = params["options"].generator_output or params["options"].toplevel_dir
if default_variables["GENERATOR"] == "ninja":
default_variables.setdefault(
"PRODUCT_DIR_ABS",
os.path.join(output_dir, "out", default_variables["build_type"]),
)
else:
default_variables.setdefault(
"PRODUCT_DIR_ABS",
os.path.join(output_dir, default_variables["CONFIGURATION_NAME"]),
)
# Give the generator the opportunity to set additional variables based on
# the params it will receive in the output phase.
if getattr(generator, "CalculateVariables", None):

View file

@ -454,6 +454,8 @@ def GetFlavor(params):
return "aix"
if sys.platform.startswith(("os390", "zos")):
return "zos"
if sys.platform == "os400":
return "os400"
return "linux"
@ -463,9 +465,13 @@ def CopyTool(flavor, out_path, generator_flags={}):
to |out_path|."""
# aix and solaris just need flock emulation. mac and win use more complicated
# support scripts.
prefix = {"aix": "flock", "solaris": "flock", "mac": "mac", "win": "win"}.get(
flavor, None
)
prefix = {
"aix": "flock",
"os400": "flock",
"solaris": "flock",
"mac": "mac",
"win": "win",
}.get(flavor, None)
if not prefix:
return

View file

@ -41,7 +41,7 @@ class FlockTool:
# with EBADF, that's why we use this F_SETLK
# hack instead.
fd = os.open(lockfile, os.O_WRONLY | os.O_NOCTTY | os.O_CREAT, 0o666)
if sys.platform.startswith("aix"):
if sys.platform.startswith("aix") or sys.platform == "os400":
# Python on AIX is compiled with LARGEFILE support, which changes the
# struct size.
op = struct.pack("hhIllqq", fcntl.F_WRLCK, 0, 0, 0, 0, 0, 0)

View file

@ -50,7 +50,7 @@ generator_default_variables = {
}
# Make supports multiple toolsets
generator_supports_multiple_toolsets = True
generator_supports_multiple_toolsets = gyp.common.CrossCompileRequested()
# Request sorted dependencies in the order from dependents to dependencies.
generator_wants_sorted_dependencies = False
@ -99,6 +99,8 @@ def CalculateVariables(default_variables, params):
default_variables.setdefault("OS", operating_system)
if flavor == "aix":
default_variables.setdefault("SHARED_LIB_SUFFIX", ".a")
elif flavor == "zos":
default_variables.setdefault("SHARED_LIB_SUFFIX", ".x")
else:
default_variables.setdefault("SHARED_LIB_SUFFIX", ".so")
default_variables.setdefault("SHARED_LIB_DIR", "$(builddir)/lib.$(TOOLSET)")
@ -154,6 +156,31 @@ cmd_alink_thin = rm -f $@ && $(AR.$(TOOLSET)) crsT $@ $(filter %.o,$^)
quiet_cmd_link = LINK($(TOOLSET)) $@
cmd_link = $(LINK.$(TOOLSET)) -o $@ $(GYP_LDFLAGS) $(LDFLAGS.$(TOOLSET)) -Wl,--start-group $(LD_INPUTS) $(LIBS) -Wl,--end-group
# Note: this does not handle spaces in paths
define xargs
$(1) $(word 1,$(2))
$(if $(word 2,$(2)),$(call xargs,$(1),$(wordlist 2,$(words $(2)),$(2))))
endef
define write-to-file
@: >$(1)
$(call xargs,@printf "%s\\n" >>$(1),$(2))
endef
OBJ_FILE_LIST := ar-file-list
define create_archive
rm -f $(1) $(1).$(OBJ_FILE_LIST); mkdir -p `dirname $(1)`
$(call write-to-file,$(1).$(OBJ_FILE_LIST),$(filter %.o,$(2)))
$(AR.$(TOOLSET)) crs $(1) @$(1).$(OBJ_FILE_LIST)
endef
define create_thin_archive
rm -f $(1) $(OBJ_FILE_LIST); mkdir -p `dirname $(1)`
$(call write-to-file,$(1).$(OBJ_FILE_LIST),$(filter %.o,$(2)))
$(AR.$(TOOLSET)) crsT $(1) @$(1).$(OBJ_FILE_LIST)
endef
# We support two kinds of shared objects (.so):
# 1) shared_library, which is just bundling together many dependent libraries
# into a link line.
@ -198,6 +225,31 @@ cmd_alink = rm -f $@ && $(AR.$(TOOLSET)) crs $@ $(filter %.o,$^)
quiet_cmd_alink_thin = AR($(TOOLSET)) $@
cmd_alink_thin = rm -f $@ && $(AR.$(TOOLSET)) crsT $@ $(filter %.o,$^)
# Note: this does not handle spaces in paths
define xargs
$(1) $(word 1,$(2))
$(if $(word 2,$(2)),$(call xargs,$(1),$(wordlist 2,$(words $(2)),$(2))))
endef
define write-to-file
@: >$(1)
$(call xargs,@printf "%s\\n" >>$(1),$(2))
endef
OBJ_FILE_LIST := ar-file-list
define create_archive
rm -f $(1) $(1).$(OBJ_FILE_LIST); mkdir -p `dirname $(1)`
$(call write-to-file,$(1).$(OBJ_FILE_LIST),$(filter %.o,$(2)))
$(AR.$(TOOLSET)) crs $(1) @$(1).$(OBJ_FILE_LIST)
endef
define create_thin_archive
rm -f $(1) $(OBJ_FILE_LIST); mkdir -p `dirname $(1)`
$(call write-to-file,$(1).$(OBJ_FILE_LIST),$(filter %.o,$(2)))
$(AR.$(TOOLSET)) crsT $(1) @$(1).$(OBJ_FILE_LIST)
endef
# Due to circular dependencies between libraries :(, we wrap the
# special "figure out circular dependencies" flags around the entire
# input list during linking.
@ -237,6 +289,24 @@ cmd_solink_module = $(LINK.$(TOOLSET)) -shared $(GYP_LDFLAGS) $(LDFLAGS.$(TOOLSE
""" # noqa: E501
LINK_COMMANDS_OS400 = """\
quiet_cmd_alink = AR($(TOOLSET)) $@
cmd_alink = rm -f $@ && $(AR.$(TOOLSET)) -X64 crs $@ $(filter %.o,$^)
quiet_cmd_alink_thin = AR($(TOOLSET)) $@
cmd_alink_thin = rm -f $@ && $(AR.$(TOOLSET)) -X64 crs $@ $(filter %.o,$^)
quiet_cmd_link = LINK($(TOOLSET)) $@
cmd_link = $(LINK.$(TOOLSET)) $(GYP_LDFLAGS) $(LDFLAGS.$(TOOLSET)) -o $@ $(LD_INPUTS) $(LIBS)
quiet_cmd_solink = SOLINK($(TOOLSET)) $@
cmd_solink = $(LINK.$(TOOLSET)) -shared $(GYP_LDFLAGS) $(LDFLAGS.$(TOOLSET)) -o $@ $(LD_INPUTS) $(LIBS)
quiet_cmd_solink_module = SOLINK_MODULE($(TOOLSET)) $@
cmd_solink_module = $(LINK.$(TOOLSET)) -shared $(GYP_LDFLAGS) $(LDFLAGS.$(TOOLSET)) -o $@ $(filter-out FORCE_DO_CMD, $^) $(LIBS)
""" # noqa: E501
LINK_COMMANDS_OS390 = """\
quiet_cmd_alink = AR($(TOOLSET)) $@
cmd_alink = rm -f $@ && $(AR.$(TOOLSET)) crs $@ $(filter %.o,$^)
@ -248,10 +318,10 @@ quiet_cmd_link = LINK($(TOOLSET)) $@
cmd_link = $(LINK.$(TOOLSET)) $(GYP_LDFLAGS) $(LDFLAGS.$(TOOLSET)) -o $@ $(LD_INPUTS) $(LIBS)
quiet_cmd_solink = SOLINK($(TOOLSET)) $@
cmd_solink = $(LINK.$(TOOLSET)) $(GYP_LDFLAGS) $(LDFLAGS.$(TOOLSET)) -o $@ $(LD_INPUTS) $(LIBS) -Wl,DLL
cmd_solink = $(LINK.$(TOOLSET)) $(GYP_LDFLAGS) $(LDFLAGS.$(TOOLSET)) -Wl,DLL -o $(patsubst %.x,%.so,$@) $(LD_INPUTS) $(LIBS) && if [ -f $(notdir $@) ]; then /bin/cp $(notdir $@) $@; else true; fi
quiet_cmd_solink_module = SOLINK_MODULE($(TOOLSET)) $@
cmd_solink_module = $(LINK.$(TOOLSET)) $(GYP_LDFLAGS) $(LDFLAGS.$(TOOLSET)) -o $@ $(filter-out FORCE_DO_CMD, $^) $(LIBS) -Wl,DLL
cmd_solink_module = $(LINK.$(TOOLSET)) $(GYP_LDFLAGS) $(LDFLAGS.$(TOOLSET)) -o $@ $(filter-out FORCE_DO_CMD, $^) $(LIBS)
""" # noqa: E501
@ -400,6 +470,9 @@ quiet_cmd_copy = COPY $@
# send stderr to /dev/null to ignore messages when linking directories.
cmd_copy = ln -f "$<" "$@" 2>/dev/null || (rm -rf "$@" && cp %(copy_archive_args)s "$<" "$@")
quiet_cmd_symlink = SYMLINK $@
cmd_symlink = ln -sf "$<" "$@"
%(link_commands)s
""" # noqa: E501
r"""
@ -981,12 +1054,20 @@ $(obj).$(TOOLSET)/$(TARGET)/%%.o: $(obj)/%%%s FORCE_DO_CMD
# libraries, but until everything is made cross-compile safe, also use
# target libraries.
# TODO(piman): when everything is cross-compile safe, remove lib.target
self.WriteLn(
"cmd_%s = LD_LIBRARY_PATH=$(builddir)/lib.host:"
"$(builddir)/lib.target:$$LD_LIBRARY_PATH; "
"export LD_LIBRARY_PATH; "
"%s%s" % (name, cd_action, command)
)
if self.flavor == "zos" or self.flavor == "aix":
self.WriteLn(
"cmd_%s = LIBPATH=$(builddir)/lib.host:"
"$(builddir)/lib.target:$$LIBPATH; "
"export LIBPATH; "
"%s%s" % (name, cd_action, command)
)
else:
self.WriteLn(
"cmd_%s = LD_LIBRARY_PATH=$(builddir)/lib.host:"
"$(builddir)/lib.target:$$LD_LIBRARY_PATH; "
"export LD_LIBRARY_PATH; "
"%s%s" % (name, cd_action, command)
)
self.WriteLn()
outputs = [self.Absolutify(o) for o in outputs]
# The makefile rules are all relative to the top dir, but the gyp actions
@ -1480,6 +1561,8 @@ $(obj).$(TOOLSET)/$(TARGET)/%%.o: $(obj)/%%%s FORCE_DO_CMD
target_prefix = "lib"
if self.flavor == "aix":
target_ext = ".a"
elif self.flavor == "zos":
target_ext = ".x"
else:
target_ext = ".so"
elif self.type == "none":
@ -1560,6 +1643,14 @@ $(obj).$(TOOLSET)/$(TARGET)/%%.o: $(obj)/%%%s FORCE_DO_CMD
# link_deps.extend(spec.get('libraries', []))
return (gyp.common.uniquer(deps), gyp.common.uniquer(link_deps))
def GetSharedObjectFromSidedeck(self, sidedeck):
"""Return the shared object files based on sidedeck"""
return re.sub(r"\.x$", ".so", sidedeck)
def GetUnversionedSidedeckFromSidedeck(self, sidedeck):
"""Return the shared object files based on sidedeck"""
return re.sub(r"\.\d+\.x$", ".x", sidedeck)
def WriteDependencyOnExtraOutputs(self, target, extra_outputs):
self.WriteMakeRule(
[self.output_binary],
@ -1768,21 +1859,35 @@ $(obj).$(TOOLSET)/$(TARGET)/%%.o: $(obj)/%%%s FORCE_DO_CMD
self.flavor not in ("mac", "openbsd", "netbsd", "win")
and not self.is_standalone_static_library
):
self.WriteDoCmd(
[self.output_binary],
link_deps,
"alink_thin",
part_of_all,
postbuilds=postbuilds,
)
if self.flavor in ("linux", "android"):
self.WriteMakeRule(
[self.output_binary],
link_deps,
actions=["$(call create_thin_archive,$@,$^)"],
)
else:
self.WriteDoCmd(
[self.output_binary],
link_deps,
"alink_thin",
part_of_all,
postbuilds=postbuilds,
)
else:
self.WriteDoCmd(
[self.output_binary],
link_deps,
"alink",
part_of_all,
postbuilds=postbuilds,
)
if self.flavor in ("linux", "android"):
self.WriteMakeRule(
[self.output_binary],
link_deps,
actions=["$(call create_archive,$@,$^)"],
)
else:
self.WriteDoCmd(
[self.output_binary],
link_deps,
"alink",
part_of_all,
postbuilds=postbuilds,
)
elif self.type == "shared_library":
self.WriteLn(
"%s: LD_INPUTS := %s"
@ -1798,6 +1903,17 @@ $(obj).$(TOOLSET)/$(TARGET)/%%.o: $(obj)/%%%s FORCE_DO_CMD
part_of_all,
postbuilds=postbuilds,
)
# z/OS has a .so target as well as a sidedeck .x target
if self.flavor == "zos":
self.WriteLn(
"%s: %s"
% (
QuoteSpaces(
self.GetSharedObjectFromSidedeck(self.output_binary)
),
QuoteSpaces(self.output_binary),
)
)
elif self.type == "loadable_module":
for link_dep in link_deps:
assert " " not in link_dep, (
@ -1855,7 +1971,9 @@ $(obj).$(TOOLSET)/$(TARGET)/%%.o: $(obj)/%%%s FORCE_DO_CMD
else:
file_desc = "executable"
install_path = self._InstallableTargetInstallPath()
installable_deps = [self.output]
installable_deps = []
if self.flavor != "zos":
installable_deps.append(self.output)
if (
self.flavor == "mac"
and "product_dir" not in spec
@ -1880,7 +1998,30 @@ $(obj).$(TOOLSET)/$(TARGET)/%%.o: $(obj)/%%%s FORCE_DO_CMD
comment="Copy this to the %s output path." % file_desc,
part_of_all=part_of_all,
)
installable_deps.append(install_path)
if self.flavor != "zos":
installable_deps.append(install_path)
if self.flavor == "zos" and self.type == "shared_library":
# lib.target/libnode.so has a dependency on $(obj).target/libnode.so
self.WriteDoCmd(
[self.GetSharedObjectFromSidedeck(install_path)],
[self.GetSharedObjectFromSidedeck(self.output)],
"copy",
comment="Copy this to the %s output path." % file_desc,
part_of_all=part_of_all,
)
# Create a symlink of libnode.x to libnode.version.x
self.WriteDoCmd(
[self.GetUnversionedSidedeckFromSidedeck(install_path)],
[install_path],
"symlink",
comment="Symlnk this to the %s output path." % file_desc,
part_of_all=part_of_all,
)
# Place libnode.version.so and libnode.x symlink in lib.target dir
installable_deps.append(self.GetSharedObjectFromSidedeck(install_path))
installable_deps.append(
self.GetUnversionedSidedeckFromSidedeck(install_path)
)
if self.output != self.alias and self.alias != self.target:
self.WriteMakeRule(
[self.alias],
@ -1888,7 +2029,18 @@ $(obj).$(TOOLSET)/$(TARGET)/%%.o: $(obj)/%%%s FORCE_DO_CMD
comment="Short alias for building this %s." % file_desc,
phony=True,
)
if part_of_all:
if self.flavor == "zos" and self.type == "shared_library":
# Make sure that .x symlink target is run
self.WriteMakeRule(
["all"],
[
self.GetUnversionedSidedeckFromSidedeck(install_path),
self.GetSharedObjectFromSidedeck(install_path),
],
comment='Add %s to "all" target.' % file_desc,
phony=True,
)
elif part_of_all:
self.WriteMakeRule(
["all"],
[install_path],
@ -2184,6 +2336,9 @@ $(obj).$(TOOLSET)/$(TARGET)/%%.o: $(obj)/%%%s FORCE_DO_CMD
# # Install all shared libs into a common directory (per toolset) for
# # convenient access with LD_LIBRARY_PATH.
# return "$(builddir)/lib.%s/%s" % (self.toolset, self.alias)
if self.flavor == "zos" and self.type == "shared_library":
return "$(builddir)/lib.%s/%s" % (self.toolset, self.alias)
return "$(builddir)/" + self.alias
@ -2351,6 +2506,16 @@ def GenerateOutput(target_list, target_dicts, data, params):
"flock_index": 2,
}
)
elif flavor == "os400":
copy_archive_arguments = "-pPRf"
header_params.update(
{
"copy_archive_args": copy_archive_arguments,
"link_commands": LINK_COMMANDS_OS400,
"flock": "./gyp-flock-tool flock",
"flock_index": 2,
}
)
build_file, _, _ = gyp.common.ParseQualifiedTarget(target_list[0])
make_global_settings_array = data[build_file].get("make_global_settings", [])

View file

@ -423,12 +423,15 @@ def _BuildCommandLineForRuleRaw(
command.insert(0, "call")
# Fix the paths
# TODO(quote): This is a really ugly heuristic, and will miss path fixing
# for arguments like "--arg=path" or "/opt:path".
# If the argument starts with a slash or dash, it's probably a command line
# switch
# for arguments like "--arg=path", arg=path, or "/opt:path".
# If the argument starts with a slash or dash, or contains an equal sign,
# it's probably a command line switch.
# Return the path with forward slashes because the command using it might
# not support backslashes.
arguments = [i if (i[:1] in "/-") else _FixPath(i, "/") for i in cmd[1:]]
arguments = [
i if (i[:1] in "/-" or "=" in i) else _FixPath(i, "/")
for i in cmd[1:]
]
arguments = [i.replace("$(InputDir)", "%INPUTDIR%") for i in arguments]
arguments = [MSVSSettings.FixVCMacroSlashes(i) for i in arguments]
if quote_cmd:

View file

@ -2112,8 +2112,8 @@ def GetDefaultConcurrentLinks():
ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(stat))
# VS 2015 uses 20% more working set than VS 2013 and can consume all RAM
# on a 64 GB machine.
mem_limit = max(1, stat.ullTotalPhys // (5 * (2 ** 30))) # total / 5GB
# on a 64 GiB machine.
mem_limit = max(1, stat.ullTotalPhys // (5 * (2 ** 30))) # total / 5GiB
hard_cap = max(1, int(os.environ.get("GYP_LINK_CONCURRENCY_MAX", 2 ** 32)))
return min(mem_limit, hard_cap)
elif sys.platform.startswith("linux"):

View file

@ -15,7 +15,7 @@ with open(path.join(here, "README.md")) as in_file:
setup(
name="gyp-next",
version="0.10.0",
version="0.13.0",
description="A fork of the GYP build system for use in the Node.js projects",
long_description=long_description,
long_description_content_type="text/markdown",

View file

@ -116,6 +116,7 @@ def main(argv=None):
else:
format_list = {
"aix5": ["make"],
"os400": ["make"],
"freebsd7": ["make"],
"freebsd8": ["make"],
"openbsd5": ["make"],

View file

@ -90,7 +90,7 @@ def count_braces(line):
"""
open_braces = ["[", "(", "{"]
close_braces = ["]", ")", "}"]
closing_prefix_re = re.compile(r"(.*?[^\s\]\}\)]+.*?)([\]\}\)],?)\s*$")
closing_prefix_re = re.compile(r"[^\s\]\}\)]\s*[\]\}\)]+,?\s*$")
cnt = 0
stripline = COMMENT_RE.sub(r"", line)
stripline = QUOTE_RE.sub(r"''", stripline)