From 14912d850dfae3ceb1f5e90b810aee862d5417cf Mon Sep 17 00:00:00 2001 From: Refael Ackermann Date: Mon, 14 Jan 2019 18:43:16 -0500 Subject: [PATCH 1/3] tools: refactor js2c.py for maximal Python3 compatibility * simplify js2c semantics --- node.gyp | 7 +- tools/js2c.py | 309 ++++++++++++++++++++++++++++---------------------- 2 files changed, 176 insertions(+), 140 deletions(-) diff --git a/node.gyp b/node.gyp index 554a8349755ffc..bc7c77ef437a29 100644 --- a/node.gyp +++ b/node.gyp @@ -791,6 +791,8 @@ 'action_name': 'node_js2c', 'process_outputs_as_sources': 1, 'inputs': [ + # Put the code first so it's a dependency and can be used for invocation. + 'tools/js2c.py', '<@(library_files)', 'config.gypi', 'tools/check_macros.py' @@ -810,9 +812,8 @@ }] ], 'action': [ - 'python', 'tools/js2c.py', - '<@(_outputs)', - '<@(_inputs)', + 'python', '<@(_inputs)', + '--target', '<@(_outputs)', ], }, ], diff --git a/tools/js2c.py b/tools/js2c.py index eff44940c57ec6..09ec6278a1e4b8 100755 --- a/tools/js2c.py +++ b/tools/js2c.py @@ -27,44 +27,37 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# This is a utility for converting JavaScript source code into C-style -# char arrays. It is used for embedded JavaScript code in the V8 -# library. - +""" +This is a utility for converting JavaScript source code into uint16_t[], +that are used for embeding JavaScript code into the Node.js binary. +""" +import argparse import os import re import sys -import string -import hashlib - -try: - xrange # Python 2 -except NameError: - xrange = range # Python 3 - - -def ToCArray(elements, step=10): - slices = (elements[i:i+step] for i in xrange(0, len(elements), step)) - slices = map(lambda s: ','.join(str(x) for x in s), slices) - return ',\n'.join(slices) +import functools +import codecs def ReadFile(filename): - file = open(filename, "rt") - try: - lines = file.read() - finally: - file.close() - return lines + if is_verbose: + print(filename) + with codecs.open(filename, "r", "utf-8") as f: + lines = f.read() + return lines + +def ReadMacroFiles(filenames): + """ -def ReadLines(filename): + :rtype: List(str) + """ result = [] - for line in open(filename, "rt"): - if '#' in line: - line = line[:line.index('#')] - line = line.strip() - if len(line) > 0: - result.append(line) + for filename in filenames: + with open(filename, "rt") as f: + # strip python-like comments and whitespace padding + lines = [line.split('#')[0].strip() for line in f] + # filter empty lines + result.extend(filter(bool, lines)) return result @@ -77,6 +70,7 @@ def ExpandConstants(lines, constants): def ExpandMacros(lines, macros): def expander(s): return ExpandMacros(s, macros) + for name, macro in macros.items(): name_pattern = re.compile("\\b%s\\(" % name) pattern_match = name_pattern.search(lines, 0) @@ -89,13 +83,15 @@ def expander(s): last_match = end arg_index = [0] # Wrap state into array, to work around Python "scoping" mapping = {} - def add_arg(str): + + def add_arg(s): # Remember to expand recursively in the arguments if arg_index[0] >= len(macro.args): return - replacement = expander(str.strip()) + replacement = expander(s.strip()) mapping[macro.args[arg_index[0]]] = replacement arg_index[0] += 1 + while end < len(lines) and height > 0: # We don't count commas at higher nesting levels. if lines[end] == ',' and height == 1: @@ -107,10 +103,11 @@ def add_arg(str): height = height - 1 end = end + 1 # Remember to add the last match. - add_arg(lines[last_match:end-1]) - if arg_index[0] < len(macro.args) -1: + add_arg(lines[last_match:end - 1]) + if arg_index[0] < len(macro.args) - 1: lineno = lines.count(os.linesep, 0, start) + 1 - raise Exception('line %s: Too few arguments for macro "%s"' % (lineno, name)) + raise Exception( + 'line %s: Too few arguments for macro "%s"' % (lineno, name)) result = macro.expand(mapping) # Replace the occurrence of the macro with the expansion lines = lines[:start] + result + lines[end:] @@ -122,34 +119,39 @@ class TextMacro: def __init__(self, args, body): self.args = args self.body = body + def expand(self, mapping): result = self.body for key, value in mapping.items(): - result = result.replace(key, value) + result = result.replace(key, value) return result + class PythonMacro: def __init__(self, args, fun): self.args = args self.fun = fun + def expand(self, mapping): args = [] for arg in self.args: args.append(mapping[arg]) return str(self.fun(*args)) + CONST_PATTERN = re.compile('^const\s+([a-zA-Z0-9_]+)\s*=\s*([^;]*);$') MACRO_PATTERN = re.compile('^macro\s+([a-zA-Z0-9_]+)\s*\(([^)]*)\)\s*=\s*([^;]*);$') PYTHON_MACRO_PATTERN = re.compile('^python\s+macro\s+([a-zA-Z0-9_]+)\s*\(([^)]*)\)\s*=\s*([^;]*);$') -def ReadMacros(lines): - constants = { } - macros = { } + +def ReadMacros(macro_files): + lines = ReadMacroFiles(macro_files) + constants = {} + macros = {} for line in lines: - hash = line.find('#') - if hash != -1: line = line[:hash] - line = line.strip() - if len(line) is 0: continue + line = line.split('#')[0].strip() + if len(line) is 0: + continue const_match = CONST_PATTERN.match(line) if const_match: name = const_match.group(1) @@ -159,20 +161,20 @@ def ReadMacros(lines): macro_match = MACRO_PATTERN.match(line) if macro_match: name = macro_match.group(1) - args = map(string.strip, macro_match.group(2).split(',')) + args = [p.strip() for p in macro_match.group(2).split(',')] body = macro_match.group(3).strip() macros[name] = TextMacro(args, body) else: python_match = PYTHON_MACRO_PATTERN.match(line) if python_match: name = python_match.group(1) - args = map(string.strip, python_match.group(2).split(',')) + args = [p.strip() for p in macro_match.group(2).split(',')] body = python_match.group(3).strip() fun = eval("lambda " + ",".join(args) + ': ' + body) macros[name] = PythonMacro(args, fun) else: raise Exception("Illegal line: " + line) - return (constants, macros) + return constants, macros TEMPLATE = """ @@ -183,14 +185,14 @@ def ReadMacros(lines): namespace native_module {{ -{definitions} +{0} void NativeModuleLoader::LoadJavaScriptSource() {{ - {initializers} + {1} }} UnionBytes NativeModuleLoader::GetConfig() {{ - return UnionBytes(config_raw, arraysize(config_raw)); // config.gypi + return UnionBytes(config_raw, {2}); // config.gypi }} }} // namespace native_module @@ -198,104 +200,137 @@ def ReadMacros(lines): }} // namespace node """ -ONE_BYTE_STRING = """ -static const uint8_t {var}[] = {{ {data} }}; -""" - TWO_BYTE_STRING = """ -static const uint16_t {var}[] = {{ {data} }}; -""" - - -INITIALIZER = """ -source_.emplace( - "{module}", - UnionBytes({var}, arraysize({var})) -); +static const uint16_t {0}[] = {{ +{1} +}}; """ -def JS2C(source, target): - modules = [] - consts = {} - macros = {} - macro_lines = [] - - for s in source: - if (os.path.split(str(s))[1]).endswith('macros.py'): - macro_lines.extend(ReadLines(str(s))) - else: - modules.append(s) - +INITIALIZER = 'source_.emplace("{0}", UnionBytes{{{1}, {2}}});' + +CONFIG_GYPI_ID = 'config_raw' + +SLUGGER_RE =re.compile('[.\-/]') + +is_verbose = False + +def GetDefinition(var, source, step=30): + encoded_source = bytearray(source, 'utf-16le') + code_points = [encoded_source[i] + (encoded_source[i+1] * 256) for i in range(0, len(encoded_source), 2)] + # For easier debugging, align to the common 3 char for code-points. + elements_s = ['%3s' % x for x in code_points] + # Put no more then `step` code-points in a line. + slices = [elements_s[i:i + step] for i in range(0, len(elements_s), step)] + lines = [','.join(s) for s in slices] + array_content = ',\n'.join(lines) + definition = TWO_BYTE_STRING.format(var, array_content) + return definition, len(code_points) + + +def AddModule(filename, consts, macros, definitions, initializers): + code = ReadFile(filename) + code = ExpandConstants(code, consts) + code = ExpandMacros(code, macros) + name = NormalizeFileName(filename) + slug = SLUGGER_RE.sub('_', name) + var = slug + '_raw' + definition, size = GetDefinition(var, code) + initializer = INITIALIZER.format(name, var, size) + definitions.append(definition) + initializers.append(initializer) + +def NormalizeFileName(filename): + split = filename.split(os.path.sep) + if split[0] == 'deps': + split = ['internal'] + split + else: # `lib/**/*.js` so drop the 'lib' part + split = split[1:] + filename = '/'.join(split) + return os.path.splitext(filename)[0] + + +def JS2C(source_files, target): # Process input from all *macro.py files - (consts, macros) = ReadMacros(macro_lines) + consts, macros = ReadMacros(source_files['.py']) # Build source code lines definitions = [] initializers = [] - def GetDefinition(var, source): - # Treat non-ASCII as UTF-8 and convert it to UTF-16. - if any(ord(c) > 127 for c in source): - source = map(ord, source.decode('utf-8').encode('utf-16be')) - source = [source[i] * 256 + source[i+1] for i in xrange(0, len(source), 2)] - source = ToCArray(source) - return TWO_BYTE_STRING.format(var=var, data=source) - else: - source = ToCArray(map(ord, source), step=20) - return ONE_BYTE_STRING.format(var=var, data=source) - - def AddModule(module, source): - var = '%s_raw' % (module.replace('-', '_').replace('/', '_')) - definition = GetDefinition(var, source) - initializer = INITIALIZER.format(module=module, - var=var) - definitions.append(definition) - initializers.append(initializer) - - for name in modules: - lines = ReadFile(str(name)) - lines = ExpandConstants(lines, consts) - lines = ExpandMacros(lines, macros) - - # On Windows, "./foo.bar" in the .gyp file is passed as "foo.bar" - # so don't assume there is always a slash in the file path. - if '/' in name or '\\' in name: - split = re.split('/|\\\\', name) - if split[0] == 'deps': - split = ['internal'] + split - else: - split = split[1:] - name = '/'.join(split) - - # if its a gypi file we're going to want it as json - # later on anyway, so get it out of the way now - if name.endswith('.gypi'): - # Currently only config.gypi is allowed - assert name == 'config.gypi' - lines = re.sub(r'\'true\'', 'true', lines) - lines = re.sub(r'\'false\'', 'false', lines) - lines = re.sub(r'#.*?\n', '', lines) - lines = re.sub(r'\'', '"', lines) - definition = GetDefinition('config_raw', lines) - definitions.append(definition) - else: - AddModule(name.split('.', 1)[0], lines) + for filename in source_files['.js']: + AddModule(filename, consts, macros, definitions, initializers) + + config_def, config_size = handle_config_gypi(source_files['config.gypi']) + definitions.append(config_def) # Emit result - output = open(str(target[0]), "w") - output.write( - TEMPLATE.format(definitions=''.join(definitions), - initializers=''.join(initializers))) - output.close() - -def main(): - natives = sys.argv[1] - source_files = sys.argv[2:] - if source_files[-2] == '-t': - global TEMPLATE - TEMPLATE = source_files[-1] - source_files = source_files[:-2] - JS2C(source_files, [natives]) + definitions = ''.join(definitions) + initializers = '\n '.join(initializers) + out = TEMPLATE.format(definitions, initializers, config_size) + write_if_chaged(out, target) + + +def handle_config_gypi(config_filename): + # if its a gypi file we're going to want it as json + # later on anyway, so get it out of the way now + config = ReadFile(config_filename) + config = jsonify(config) + config_def, config_size = GetDefinition(CONFIG_GYPI_ID, config) + return config_def, config_size + + +def jsonify(config): + # 1. string comments + config = re.sub(r'#.*?\n', '', config) + # 3. normalize string literals from ' into " + config = re.sub('\'', '"', config) + # 2. turn pseudo-booleans strings into Booleans + config = re.sub('"true"', 'true', config) + config = re.sub('"false"', 'false', config) + return config + + +def write_if_chaged(content, target): + if os.path.exists(target): + with open(target, 'rt') as existing: + old_content = existing.read() + else: + old_content = '' + if old_content == content: + return + with open(target, "wt") as output: + output.write(content) + + +def SourceFileByExt(files_by_ext, filename): + """ + :type files_by_ext: dict + :type filename: str + :rtype: dict + """ + ext = os.path.splitext(filename)[-1] + files_by_ext.setdefault(ext, []).append(filename) + return files_by_ext + +def main(args): + parser = argparse.ArgumentParser( + description='Convert code files into `uint16_t[]`s', + fromfile_prefix_chars='@' + ) + parser.add_argument('--target', help='output file') + parser.add_argument('--verbose', action='store_true', help='output file') + parser.add_argument('sources', nargs='*', help='input files') + options = parser.parse_args(args) + global is_verbose + is_verbose = options.verbose + source_files = functools.reduce(SourceFileByExt, options.sources, {}) + # Should have exactly 3 types: `.js`, `.py`, and `.gypi` + assert len(source_files) == 3 + # Currently config.gypi is the only `.gypi` file allowed + assert source_files['.gypi'] == ['config.gypi'] + source_files['config.gypi'] = source_files.pop('.gypi')[0] + JS2C(source_files, options.target) + if __name__ == "__main__": - main() + main(sys.argv[1:]) From 38a33624e0f30c5ddb35f5ca60554e3835620ece Mon Sep 17 00:00:00 2001 From: Refael Ackermann Date: Thu, 4 Apr 2019 21:07:55 -0400 Subject: [PATCH 2/3] [temp]encode unicode code-points in code --- deps/acorn/acorn/dist/acorn.js | 16 +++++------ .../node-inspect/lib/internal/inspect_repl.js | 2 +- lib/internal/cli_table.js | 26 +++++++++--------- lib/internal/timers.js | 27 ++----------------- 4 files changed, 24 insertions(+), 47 deletions(-) diff --git a/deps/acorn/acorn/dist/acorn.js b/deps/acorn/acorn/dist/acorn.js index 44d95c5fae74db..53620b8e309240 100644 --- a/deps/acorn/acorn/dist/acorn.js +++ b/deps/acorn/acorn/dist/acorn.js @@ -362,7 +362,7 @@ var defaultOptions = { // cause Acorn to call that function with object in the same // format as tokens returned from `tokenizer().getToken()`. Note // that you are not allowed to call the parser from the - // callback—that will corrupt its internal state. + // callback-that will corrupt its internal state. onToken: null, // A function can be passed as `onComment` option, which will // cause Acorn to call that function with `(block, text, start, @@ -373,7 +373,7 @@ var defaultOptions = { // When the `locations` option is on, two more parameters are // passed, the full `{line, column}` locations of the start and // end of the comments. Note that you are not allowed to call the - // parser from the callback—that will corrupt its internal state. + // parser from the callback-that will corrupt its internal state. onComment: null, // Nodes have their start and end characters offsets recorded in // `start` and `end` properties (directly on the node, rather than @@ -1810,7 +1810,7 @@ pp$2.parseMaybeDefault = function(startPos, startLoc, left) { return this.finishNode(node, "AssignmentPattern") }; -// Verify that a node is an lval — something that can be assigned +// Verify that a node is an lval - something that can be assigned // to. // bindingType can be either: // 'var' indicating that the lval creates a 'var' binding @@ -1882,7 +1882,7 @@ pp$2.checkLVal = function(expr, bindingType, checkClashes) { // of constructs (for example, the fact that `!x[1]` means `!(x[1])` // instead of `(!x)[1]` is handled by the fact that the parser // function that parses unary prefix operators is called first, and -// in turn calls the function that parses `[]` subscripts — that +// in turn calls the function that parses `[]` subscripts - that // way, it'll receive the node for `x[1]` already parsed, and wraps // *that* in the unary operator node. // @@ -1897,8 +1897,8 @@ pp$2.checkLVal = function(expr, bindingType, checkClashes) { var pp$3 = Parser.prototype; // Check if property name clashes with already added. -// Object/class getters and setters are not allowed to clash — -// either with each other or with an init property — and in +// Object/class getters and setters are not allowed to clash - +// either with each other or with an init property - and in // strict mode, init properties are also not allowed to be repeated. pp$3.checkPropClash = function(prop, propHash, refDestructuringErrors) { @@ -2188,7 +2188,7 @@ pp$3.parseSubscript = function(base, startPos, startLoc, noCalls, maybeAsyncArro return base }; -// Parse an atomic expression — either a single token that is an +// Parse an atomic expression - either a single token that is an // expression, an expression started by a keyword like `function` or // `new`, or an expression wrapped in punctuation like `()`, `[]`, // or `{}`. @@ -2381,7 +2381,7 @@ pp$3.parseParenArrowList = function(startPos, startLoc, exprList) { }; // New's precedence is slightly tricky. It must allow its argument to -// be a `[]` or dot subscript expression, but not a call — at least, +// be a `[]` or dot subscript expression, but not a call - at least, // not without wrapping it in parentheses. Thus, it uses the noCalls // argument to parseSubscripts to prevent it from consuming the // argument list. diff --git a/deps/node-inspect/lib/internal/inspect_repl.js b/deps/node-inspect/lib/internal/inspect_repl.js index 38fe4684cf6d71..855e5176a006dd 100644 --- a/deps/node-inspect/lib/internal/inspect_repl.js +++ b/deps/node-inspect/lib/internal/inspect_repl.js @@ -359,7 +359,7 @@ function createRepl(inspector) { [util.inspect.custom](depth, { stylize }) { const { startTime, endTime } = this.data; - return stylize(`[Profile ${endTime - startTime}μs]`, 'special'); + return stylize(`[Profile ${endTime - startTime}\u3bcss]`, 'special'); } save(filename = 'node.cpuprofile') { diff --git a/lib/internal/cli_table.js b/lib/internal/cli_table.js index b2d2779e5f9cdc..849c9af0b89d00 100644 --- a/lib/internal/cli_table.js +++ b/lib/internal/cli_table.js @@ -14,19 +14,19 @@ const HasOwnProperty = Function.call.bind(Object.prototype.hasOwnProperty); // Refs: https://github.com/nodejs/node/issues/10673 const tableChars = { /* eslint-disable node-core/non-ascii-character */ - middleMiddle: '─', - rowMiddle: '┼', - topRight: '┐', - topLeft: '┌', - leftMiddle: '├', - topMiddle: '┬', - bottomRight: '┘', - bottomLeft: '└', - bottomMiddle: '┴', - rightMiddle: '┤', - left: '│ ', - right: ' │', - middle: ' │ ', + middleMiddle: '\u2500', + rowMiddle: '\u253c', + topRight: '\u2510', + topLeft: '\u250c', + leftMiddle: '\u251c', + topMiddle: '\u252c', + bottomRight: '\u2518', + bottomLeft: '\u2514', + bottomMiddle: '\u2534', + rightMiddle: '\u2524', + left: '\u2502 ', + right: ' \u2502', + middle: ' \u2502 ', /* eslint-enable node-core/non-ascii-character */ }; diff --git a/lib/internal/timers.js b/lib/internal/timers.js index 0c7388b7bbd6d2..128241197104ee 100644 --- a/lib/internal/timers.js +++ b/lib/internal/timers.js @@ -29,29 +29,6 @@ // Object maps are kept which contain linked lists keyed by their duration in // milliseconds. // -/* eslint-disable node-core/non-ascii-character */ -// -// ╔════ > Object Map -// ║ -// ╠══ -// ║ lists: { '40': { }, '320': { etc } } (keys of millisecond duration) -// ╚══ ┌────┘ -// │ -// ╔══ │ -// ║ TimersList { _idleNext: { }, _idlePrev: (self) } -// ║ ┌────────────────┘ -// ║ ╔══ │ ^ -// ║ ║ { _idleNext: { }, _idlePrev: { }, _onTimeout: (callback) } -// ║ ║ ┌───────────┘ -// ║ ║ │ ^ -// ║ ║ { _idleNext: { etc }, _idlePrev: { }, _onTimeout: (callback) } -// ╠══ ╠══ -// ║ ║ -// ║ ╚════ > Actual JavaScript timeouts -// ║ -// ╚════ > Linked List -// -/* eslint-enable node-core/non-ascii-character */ // // With this, virtually constant-time insertion (append), removal, and timeout // is possible in the JavaScript layer. Any one list of timers is able to be @@ -66,8 +43,8 @@ // always be due to timeout at a later time. // // Less-than constant time operations are thus contained in two places: -// The PriorityQueue — an efficient binary heap implementation that does all -// operations in worst-case O(log n) time — which manages the order of expiring +// The PriorityQueue - an efficient binary heap implementation that does all +// operations in worst-case O(log n) time - which manages the order of expiring // Timeout lists and the object map lookup of a specific list by the duration of // timers within (or creation of a new list). However, these operations combined // have shown to be trivial in comparison to other timers architectures. From d5fd4e7683520f3f94e25cbc793d50c3dd3a503d Mon Sep 17 00:00:00 2001 From: Refael Ackermann Date: Thu, 17 Jan 2019 17:58:22 -0500 Subject: [PATCH 3/3] src: simplify Javascript code embedding * use `string_view` instead of `UnionBytes` --- common.gypi | 6 ++- deps/gtest/gtest.gyp | 1 + node.gyp | 7 ++- node.gypi | 3 ++ src/node_javascript.h | 32 +++++++++++ src/node_native_module.cc | 68 ++++++++++++++--------- src/node_native_module.h | 23 ++++---- src/node_union_bytes.h | 111 -------------------------------------- tools/js2c.py | 47 ++++++++-------- 9 files changed, 126 insertions(+), 172 deletions(-) create mode 100644 src/node_javascript.h delete mode 100644 src/node_union_bytes.h diff --git a/common.gypi b/common.gypi index 11b5d3b7aeb11d..8f57974e771edc 100644 --- a/common.gypi +++ b/common.gypi @@ -231,6 +231,8 @@ 'SuppressStartupBanner': 'true', 'WarnAsError': 'false', 'WarningLevel': 3, # /W3 + 'AdditionalOptions': [ '/std:c++17' ], + }, 'VCLinkerTool': { 'conditions': [ @@ -334,7 +336,7 @@ }], [ 'OS in "linux freebsd openbsd solaris android aix cloudabi"', { 'cflags': [ '-Wall', '-Wextra', '-Wno-unused-parameter', ], - 'cflags_cc': [ '-fno-rtti', '-fno-exceptions', '-std=gnu++1y' ], + 'cflags_cc': [ '-fno-rtti', '-fno-exceptions', '-std=gnu++1z' ], 'ldflags': [ '-rdynamic' ], 'target_conditions': [ # The 1990s toolchain on SmartOS can't handle thin archives. @@ -461,7 +463,7 @@ ['clang==1', { 'xcode_settings': { 'GCC_VERSION': 'com.apple.compilers.llvm.clang.1_0', - 'CLANG_CXX_LANGUAGE_STANDARD': 'gnu++1y', # -std=gnu++1y + 'CLANG_CXX_LANGUAGE_STANDARD': 'gnu++1z', # -std=gnu++1z 'CLANG_CXX_LIBRARY': 'libc++', }, }], diff --git a/deps/gtest/gtest.gyp b/deps/gtest/gtest.gyp index c7c850c52cb83f..2d0d35eaac1764 100644 --- a/deps/gtest/gtest.gyp +++ b/deps/gtest/gtest.gyp @@ -7,6 +7,7 @@ 'direct_dependent_settings': { 'include_dirs': ['include'], }, + 'defines': [ 'GTEST_LANG_CXX11' ], 'include_dirs': ['.', 'include'], 'sources': [ 'src/gtest-death-test.cc', diff --git a/node.gyp b/node.gyp index bc7c77ef437a29..71edc409c06db8 100644 --- a/node.gyp +++ b/node.gyp @@ -523,6 +523,7 @@ 'src/node_http2_state.h', 'src/node_i18n.h', 'src/node_internals.h', + 'src/node_javascript.h', 'src/node_messaging.h', 'src/node_metadata.h', 'src/node_mutex.h', @@ -538,7 +539,6 @@ 'src/node_revert.h', 'src/node_root_certs.h', 'src/node_stat_watcher.h', - 'src/node_union_bytes.h', 'src/node_url.h', 'src/node_version.h', 'src/node_v8_platform-inl.h', @@ -1031,7 +1031,10 @@ '<(SHARED_INTERMEDIATE_DIR)', # for node_natives.h ], - 'defines': [ 'NODE_WANT_INTERNALS=1' ], + 'defines': [ + 'NODE_WANT_INTERNALS=1', + 'GTEST_LANG_CXX11' + ], 'sources': [ 'test/cctest/node_test_fixture.cc', diff --git a/node.gypi b/node.gypi index b53ccfd6bebbd8..6e3c0f0a688d2b 100644 --- a/node.gypi +++ b/node.gypi @@ -78,6 +78,9 @@ '<(_msvs_precompiled_header)', '<(_msvs_precompiled_source)', ], + 'include_dirs': [ + 'tools/msvs/pch', + ], }, { # POSIX 'defines': [ '__POSIX__' ], }], diff --git a/src/node_javascript.h b/src/node_javascript.h new file mode 100644 index 00000000000000..b21425f35dace0 --- /dev/null +++ b/src/node_javascript.h @@ -0,0 +1,32 @@ +#pragma once + +#include +#include + +#if __has_include() +#include +using std::string_view; +#else +#include +using std::experimental::string_view; +#endif + + + +namespace node { + +namespace native_module { + +using NativeModuleRecordMap = std::map; + +class JavascriptEmbeddedCode { + public: + JavascriptEmbeddedCode(); + protected: + string_view config_; + NativeModuleRecordMap source_; +}; + +} // namespace native_module + +} // namespace node diff --git a/src/node_native_module.cc b/src/node_native_module.cc index 08c0ab16e3d8fa..4d4d3c5e752079 100644 --- a/src/node_native_module.cc +++ b/src/node_native_module.cc @@ -1,5 +1,8 @@ #include "node_native_module.h" #include "node_errors.h" +#include "env.h" + +#include namespace node { @@ -16,18 +19,15 @@ using v8::DEFAULT; using v8::EscapableHandleScope; using v8::Function; using v8::FunctionCallbackInfo; -using v8::HandleScope; using v8::Integer; using v8::IntegrityLevel; using v8::Isolate; using v8::Local; -using v8::Maybe; using v8::MaybeLocal; using v8::Name; using v8::None; using v8::Object; using v8::PropertyCallbackInfo; -using v8::Script; using v8::ScriptCompiler; using v8::ScriptOrigin; using v8::Set; @@ -36,6 +36,32 @@ using v8::String; using v8::Uint8Array; using v8::Value; +class UInt8SpanResource + : public String::ExternalOneByteStringResource { + public: + explicit UInt8SpanResource(string_view span) : span_(span) {} + UInt8SpanResource(const UInt8SpanResource&) = delete; + UInt8SpanResource(const UInt8SpanResource&&) = delete; + UInt8SpanResource& operator=(const UInt8SpanResource&) = delete; + UInt8SpanResource& operator=(const UInt8SpanResource&&) = delete; + + ~UInt8SpanResource() override = default; + void Dispose() override { delete this; } + + const char* data() const override { return span_.data(); } + size_t length() const override { return span_.size(); } + + static v8::Local ToStringChecked(v8::Isolate* isolate, + string_view span) { + return + v8::String::NewExternalOneByte(isolate, new UInt8SpanResource{span}) + .ToLocalChecked(); + } + + private: + const string_view span_; +}; + void NativeModuleLoader::InitializeModuleCategories() { if (module_categories_.is_initialized) { DCHECK(!module_categories_.can_be_required.empty()); @@ -83,22 +109,18 @@ void NativeModuleLoader::InitializeModuleCategories() { "internal/v8_prof_processor", }; - for (auto const& x : source_) { + for (const auto& x : source_) { const std::string& id = x.first; for (auto const& prefix : prefixes) { if (prefix.length() > id.length()) { continue; } if (id.find(prefix) == 0) { - module_categories_.cannot_be_required.emplace(id); + module_categories_.cannot_be_required.insert(id); } } - } - - for (auto const& x : source_) { - const std::string& id = x.first; if (0 == module_categories_.cannot_be_required.count(id)) { - module_categories_.can_be_required.emplace(id); + module_categories_.can_be_required.insert(id); } } @@ -111,8 +133,11 @@ Local MapToObject(Local context, Isolate* isolate = context->GetIsolate(); Local out = Object::New(isolate); for (auto const& x : in) { - Local key = OneByteString(isolate, x.first.c_str(), x.first.size()); - out->Set(context, key, x.second.ToStringChecked(isolate)).FromJust(); + const Local key = OneByteString( + isolate, x.first.c_str(), x.first.size()); + const Local val = UInt8SpanResource::ToStringChecked( + isolate, x.second); + out->Set(context, key, val).Check(); } return out; } @@ -206,18 +231,14 @@ void NativeModuleLoader::ConfigStringGetter( per_process::native_module_loader.GetConfigString(info.GetIsolate())); } -Local NativeModuleLoader::GetSourceObject( - Local context) const { +Local NativeModuleLoader::GetSourceObject(Local context) + const { return MapToObject(context, source_); } -Local NativeModuleLoader::GetConfigString(Isolate* isolate) const { - return config_.ToStringChecked(isolate); -} - -NativeModuleLoader::NativeModuleLoader() : config_(GetConfig()) { - LoadJavaScriptSource(); - LoadCodeCache(); +Local NativeModuleLoader::GetConfigString(Isolate* isolate) + const { + return UInt8SpanResource::ToStringChecked(isolate, config_);; } // This is supposed to be run only by the main thread in @@ -294,9 +315,8 @@ MaybeLocal NativeModuleLoader::LookupAndCompile( Isolate* isolate = context->GetIsolate(); EscapableHandleScope scope(isolate); - const auto source_it = source_.find(id); - CHECK_NE(source_it, source_.end()); - Local source = source_it->second.ToStringChecked(isolate); + const Local source = UInt8SpanResource::ToStringChecked( + isolate, source_.at(id)); std::string filename_s = id + std::string(".js"); Local filename = diff --git a/src/node_native_module.h b/src/node_native_module.h index 587c59022afc56..c5789a67043e7a 100644 --- a/src/node_native_module.h +++ b/src/node_native_module.h @@ -3,18 +3,19 @@ #if defined(NODE_WANT_INTERNALS) && NODE_WANT_INTERNALS -#include -#include -#include #include "env.h" +#include "node_javascript.h" #include "node_mutex.h" -#include "node_union_bytes.h" #include "v8.h" +#include +#include +#include + namespace node { + namespace native_module { -using NativeModuleRecordMap = std::map; using NativeModuleCacheMap = std::unordered_map>; @@ -26,13 +27,16 @@ using NativeModuleCacheMap = // This class should not depend on a particular isolate, context, or // environment. Rather it should take them as arguments when necessary. // The instances of this class are per-process. -class NativeModuleLoader { +class NativeModuleLoader : public JavascriptEmbeddedCode { public: - NativeModuleLoader(); + NativeModuleLoader() = default; + ~NativeModuleLoader() = default; // TODO(joyeecheung): maybe we should make this a singleton, instead of // putting it in per_process. NativeModuleLoader(const NativeModuleLoader&) = delete; + NativeModuleLoader(const NativeModuleLoader&&) = delete; NativeModuleLoader& operator=(const NativeModuleLoader&) = delete; + NativeModuleLoader& operator=(const NativeModuleLoader&&) = delete; static void Initialize(v8::Local target, v8::Local unused, @@ -75,12 +79,11 @@ class NativeModuleLoader { // Generated by tools/js2c.py as node_javascript.cc void LoadJavaScriptSource(); // Loads data into source_ - UnionBytes GetConfig(); // Return data for config.gypi // Generated by tools/generate_code_cache.js as node_code_cache.cc when // the build is configured with --code-cache-path=.... They are noops // in node_code_cache_stub.cc - void LoadCodeCache(); // Loads data into code_cache_ + void LoadCodeCache(); // Loads data into code_cache_ // Compile a script as a NativeModule that can be loaded via // NativeModule.p.require in JS land. @@ -96,9 +99,7 @@ class NativeModuleLoader { ModuleCategories module_categories_; - NativeModuleRecordMap source_; NativeModuleCacheMap code_cache_; - UnionBytes config_; // Used to synchronize access to the code cache map Mutex code_cache_mutex_; diff --git a/src/node_union_bytes.h b/src/node_union_bytes.h deleted file mode 100644 index 0034f184cc38ca..00000000000000 --- a/src/node_union_bytes.h +++ /dev/null @@ -1,111 +0,0 @@ - -#ifndef SRC_NODE_UNION_BYTES_H_ -#define SRC_NODE_UNION_BYTES_H_ - -#if defined(NODE_WANT_INTERNALS) && NODE_WANT_INTERNALS - -// A union of const uint8_t* or const uint16_t* data that can be -// turned into external v8::String when given an isolate. - -#include "env.h" -#include "v8.h" - -namespace node { - -class NonOwningExternalOneByteResource - : public v8::String::ExternalOneByteStringResource { - public: - explicit NonOwningExternalOneByteResource(const uint8_t* data, size_t length) - : data_(data), length_(length) {} - ~NonOwningExternalOneByteResource() override = default; - - const char* data() const override { - return reinterpret_cast(data_); - } - size_t length() const override { return length_; } - - NonOwningExternalOneByteResource(const NonOwningExternalOneByteResource&) = - delete; - NonOwningExternalOneByteResource& operator=( - const NonOwningExternalOneByteResource&) = delete; - - private: - const uint8_t* data_; - size_t length_; -}; - -class NonOwningExternalTwoByteResource - : public v8::String::ExternalStringResource { - public: - explicit NonOwningExternalTwoByteResource(const uint16_t* data, size_t length) - : data_(data), length_(length) {} - ~NonOwningExternalTwoByteResource() override = default; - - const uint16_t* data() const override { return data_; } - size_t length() const override { return length_; } - - NonOwningExternalTwoByteResource(const NonOwningExternalTwoByteResource&) = - delete; - NonOwningExternalTwoByteResource& operator=( - const NonOwningExternalTwoByteResource&) = delete; - - private: - const uint16_t* data_; - size_t length_; -}; - -// Similar to a v8::String, but it's independent from Isolates -// and can be materialized in Isolates as external Strings -// via ToStringChecked. The data pointers are owned by the caller. -class UnionBytes { - public: - UnionBytes(const uint16_t* data, size_t length) - : is_one_byte_(false), two_bytes_(data), length_(length) {} - UnionBytes(const uint8_t* data, size_t length) - : is_one_byte_(true), one_bytes_(data), length_(length) {} - - UnionBytes(const UnionBytes&) = default; - UnionBytes& operator=(const UnionBytes&) = default; - UnionBytes(UnionBytes&&) = default; - UnionBytes& operator=(UnionBytes&&) = default; - - bool is_one_byte() const { return is_one_byte_; } - const uint16_t* two_bytes_data() const { - CHECK(!is_one_byte_); - CHECK_NOT_NULL(two_bytes_); - return two_bytes_; - } - const uint8_t* one_bytes_data() const { - CHECK(is_one_byte_); - CHECK_NOT_NULL(one_bytes_); - return one_bytes_; - } - v8::Local ToStringChecked(v8::Isolate* isolate) const { - if (is_one_byte_) { - CHECK_NOT_NULL(one_bytes_); - NonOwningExternalOneByteResource* source = - new NonOwningExternalOneByteResource(one_bytes_, length_); - return v8::String::NewExternalOneByte(isolate, source).ToLocalChecked(); - } else { - CHECK_NOT_NULL(two_bytes_); - NonOwningExternalTwoByteResource* source = - new NonOwningExternalTwoByteResource(two_bytes_, length_); - return v8::String::NewExternalTwoByte(isolate, source).ToLocalChecked(); - } - } - size_t length() { return length_; } - - private: - bool is_one_byte_; - union { - const uint8_t* one_bytes_; - const uint16_t* two_bytes_; - }; - size_t length_; -}; - -} // namespace node - -#endif // defined(NODE_WANT_INTERNALS) && NODE_WANT_INTERNALS - -#endif // SRC_NODE_UNION_BYTES_H_ diff --git a/tools/js2c.py b/tools/js2c.py index 09ec6278a1e4b8..db196d03e5fb18 100755 --- a/tools/js2c.py +++ b/tools/js2c.py @@ -29,7 +29,7 @@ """ This is a utility for converting JavaScript source code into uint16_t[], -that are used for embeding JavaScript code into the Node.js binary. +that are used for embedding JavaScript code into the Node.js binary. """ import argparse import os @@ -178,8 +178,10 @@ def ReadMacros(macro_files): TEMPLATE = """ -#include "node_native_module.h" -#include "node_internals.h" +#include "node_javascript.h" + +#include +using namespace std::string_literals; namespace node {{ @@ -187,12 +189,11 @@ def ReadMacros(macro_files): {0} -void NativeModuleLoader::LoadJavaScriptSource() {{ - {1} -}} - -UnionBytes NativeModuleLoader::GetConfig() {{ - return UnionBytes(config_raw, {2}); // config.gypi +JavascriptEmbeddedCode::JavascriptEmbeddedCode() : + config_{{config_raw}}, + source_{{ + {1} + }} {{ }} }} // namespace native_module @@ -200,13 +201,14 @@ def ReadMacros(macro_files): }} // namespace node """ -TWO_BYTE_STRING = """ -static const uint16_t {0}[] = {{ +CHAR_STRING_AND_STRING_VIEW = """ +static const char {0}_arr[] = {{ {1} }}; +static const string_view {0} = {{ {0}_arr }}; """ -INITIALIZER = 'source_.emplace("{0}", UnionBytes{{{1}, {2}}});' +INITIALIZER = '{{"{0}"s, {1}}}' CONFIG_GYPI_ID = 'config_raw' @@ -215,15 +217,16 @@ def ReadMacros(macro_files): is_verbose = False def GetDefinition(var, source, step=30): - encoded_source = bytearray(source, 'utf-16le') - code_points = [encoded_source[i] + (encoded_source[i+1] * 256) for i in range(0, len(encoded_source), 2)] + code_points = bytearray(source, 'utf-8') # For easier debugging, align to the common 3 char for code-points. elements_s = ['%3s' % x for x in code_points] + # Put a nul terminator. + elements_s.append('0') # Put no more then `step` code-points in a line. slices = [elements_s[i:i + step] for i in range(0, len(elements_s), step)] lines = [','.join(s) for s in slices] array_content = ',\n'.join(lines) - definition = TWO_BYTE_STRING.format(var, array_content) + definition = CHAR_STRING_AND_STRING_VIEW.format(var, array_content) return definition, len(code_points) @@ -234,8 +237,8 @@ def AddModule(filename, consts, macros, definitions, initializers): name = NormalizeFileName(filename) slug = SLUGGER_RE.sub('_', name) var = slug + '_raw' - definition, size = GetDefinition(var, code) - initializer = INITIALIZER.format(name, var, size) + definition, _ = GetDefinition(var, code) + initializer = INITIALIZER.format(name, var) definitions.append(definition) initializers.append(initializer) @@ -260,14 +263,14 @@ def JS2C(source_files, target): for filename in source_files['.js']: AddModule(filename, consts, macros, definitions, initializers) - config_def, config_size = handle_config_gypi(source_files['config.gypi']) + config_def, _ = handle_config_gypi(source_files['config.gypi']) definitions.append(config_def) # Emit result definitions = ''.join(definitions) - initializers = '\n '.join(initializers) - out = TEMPLATE.format(definitions, initializers, config_size) - write_if_chaged(out, target) + initializers = ',\n '.join(initializers) + out = TEMPLATE.format(definitions, initializers) + write_if_changed(out, target) def handle_config_gypi(config_filename): @@ -290,7 +293,7 @@ def jsonify(config): return config -def write_if_chaged(content, target): +def write_if_changed(content, target): if os.path.exists(target): with open(target, 'rt') as existing: old_content = existing.read()