From 28e96c5c55fe38df4e8a3aa01d2fcc56250c3ec8 Mon Sep 17 00:00:00 2001 From: Andrew Bradley Date: Sat, 16 Oct 2021 16:10:51 -0400 Subject: [PATCH 1/7] initial commit --- development-docs/ts-sys.ts | 36 ++++++++++++++ src/configuration.ts | 27 +++++----- src/diagnostics.ts | 22 ++++++++ src/fs.ts | 81 ++++++++++++++++++++++++++++++ src/index.ts | 94 +++++++++++++---------------------- src/module-type-classifier.ts | 1 - src/resolver-functions.ts | 1 + 7 files changed, 190 insertions(+), 72 deletions(-) create mode 100644 development-docs/ts-sys.ts create mode 100644 src/diagnostics.ts create mode 100644 src/fs.ts diff --git a/development-docs/ts-sys.ts b/development-docs/ts-sys.ts new file mode 100644 index 000000000..747b11102 --- /dev/null +++ b/development-docs/ts-sys.ts @@ -0,0 +1,36 @@ +import ts = require("typescript"); +import { getPatternFromSpec } from "../src/ts-internals"; + +// Notes and demos to understand `ts.sys` + +console.dir(ts.sys.getCurrentDirectory()); +// Gets names (not paths) of all directories that are direct children of given path +// Never throws +// Accepts trailing `/` or not +console.dir(ts.sys.getDirectories(ts.sys.getCurrentDirectory())); + +///// + +// Returns array of absolute paths +// Never returns directories; only files + +// Values can have period or not; are interpreted as a suffix ('o.svg' matches logo.svg; seems to also match if you embed / directory delimiters) +// [''] is the same as undefined; returns everything +const extensions: string[] | undefined = ['']; +// Supports wildcards; ts-style globs? +const exclude: string[] | undefined = undefined; +const include: string[] | undefined = ['*/????????????']; +// Depth == 0 is the same as undefined: unlimited depth +// Depth == 1 is only direct children of directory +const depth: number | undefined = undefined; +console.dir(ts.sys.readDirectory(ts.sys.getCurrentDirectory(), extensions, exclude, include, depth)); + +// To overlay virtual filesystem contents over `ts.sys.readDirectory`, try this: +// start with array of all virtual files +// Filter by those having base directory prefix +// if extensions is array, do an `endsWith` filter +// if exclude is an array, use `getPatternFromSpec` and filter out anything that matches +// if include is an array, use `getPatternFromSpec` and filter out anything that does not match at least one +// if depth is non-zero, count the number of directory delimiters following the base directory prefix + +console.log(getPatternFromSpec('foo/*/bar', ts.sys.getCurrentDirectory())); diff --git a/src/configuration.ts b/src/configuration.ts index a970b49c4..3b6e09fd6 100644 --- a/src/configuration.ts +++ b/src/configuration.ts @@ -1,5 +1,6 @@ import { resolve, dirname } from 'path'; import type * as _ts from 'typescript'; +import type { FsReader } from './fs'; import { CreateOptions, DEFAULTS, @@ -61,6 +62,7 @@ function fixConfig(ts: TSCommon, config: _ts.ParsedCommandLine) { export function readConfig( cwd: string, ts: TSCommon, + fsReader: FsReader, rawApiOptions: CreateOptions ): { /** @@ -90,8 +92,6 @@ export function readConfig( const projectSearchDir = resolve(cwd, rawApiOptions.projectSearchDir ?? cwd); const { - fileExists = ts.sys.fileExists, - readFile = ts.sys.readFile, skipProject = DEFAULTS.skipProject, project = DEFAULTS.project, } = rawApiOptions; @@ -100,7 +100,7 @@ export function readConfig( if (!skipProject) { configFilePath = project ? resolve(cwd, project) - : ts.findConfigFile(projectSearchDir, fileExists); + : ts.findConfigFile(projectSearchDir, fsReader.fileExists); if (configFilePath) { let pathToNextConfigInChain = configFilePath; @@ -109,7 +109,10 @@ export function readConfig( // Follow chain of "extends" while (true) { - const result = ts.readConfigFile(pathToNextConfigInChain, readFile); + const result = ts.readConfigFile( + pathToNextConfigInChain, + fsReader.readFile + ); // Return diagnostics. if (result.error) { @@ -133,10 +136,10 @@ export function readConfig( const resolvedExtendedConfigPath = tsInternals.getExtendsConfigPath( c.extends, { - fileExists, - readDirectory: ts.sys.readDirectory, - readFile, - useCaseSensitiveFileNames: ts.sys.useCaseSensitiveFileNames, + fileExists: fsReader.fileExists, + readDirectory: fsReader.readDirectory, + readFile: fsReader.readFile, + useCaseSensitiveFileNames: fsReader.useCaseSensitiveFileNames, trace, }, bp, @@ -226,10 +229,10 @@ export function readConfig( ts.parseJsonConfigFileContent( config, { - fileExists, - readFile, - readDirectory: ts.sys.readDirectory, - useCaseSensitiveFileNames: ts.sys.useCaseSensitiveFileNames, + fileExists: fsReader.fileExists, + readFile: fsReader.readFile, + readDirectory: fsReader.readDirectory, + useCaseSensitiveFileNames: fsReader.useCaseSensitiveFileNames, }, basePath, undefined, diff --git a/src/diagnostics.ts b/src/diagnostics.ts new file mode 100644 index 000000000..8e669626c --- /dev/null +++ b/src/diagnostics.ts @@ -0,0 +1,22 @@ +import { env } from '.'; +import { yn } from './util'; + +/** + * Debugging `ts-node`. + */ +const shouldDebug = yn(env.TS_NODE_DEBUG); +/** @internal */ +export const debug = shouldDebug + ? (...args: any) => + console.log(`[ts-node ${new Date().toISOString()}]`, ...args) + : () => undefined; +/** @internal */ +export const debugFn = shouldDebug + ? (key: string, fn: (arg: T) => U) => { + let i = 0; + return (x: T) => { + debug(key, x, ++i); + return fn(x); + }; + } + : (_: string, fn: (arg: T) => U) => fn; diff --git a/src/fs.ts b/src/fs.ts new file mode 100644 index 000000000..5f3f3fcdf --- /dev/null +++ b/src/fs.ts @@ -0,0 +1,81 @@ +import type * as _ts from 'typescript'; +import { debugFn } from './diagnostics'; +import { cachedLookup } from './util'; + +// Types of fs implementation: +// +// Cached fs +// Proxies to real filesystem, caches results in-memory. +// Has invalidation APIs to support `delete require.cache[foo]` +// +// Overlay fs +// Is writable. Written contents remain in memory. +// Written contents can be serialized / deserialized. +// Read calls return from in-memory, proxy to another FS if not found. + +/* + +require('./dist/bar.js') +dist/bar.js exists +preferTsExts=true we should resolve to ./src/bar.ts +preferTsExts=false we should resolve to ./dist/bar.js + - read file from the filesystem? + - read file from the overlay fs? + +*/ + +/** + * @internal + * Since `useCaseSensitiveFileNames` is required to know how to cache, we expose it on the interface + */ +export type FsReader = Pick< + _ts.System, + | 'directoryExists' + | 'fileExists' + | 'getDirectories' + | 'readDirectory' + | 'readFile' + | 'realpath' + | 'resolvePath' + | 'useCaseSensitiveFileNames' +>; +/** since I've never hit code that needs these functions implemented */ +type FullFsReader = FsReader & + Pick<_ts.System, 'getFileSize' | 'getModifiedTime'>; +type FsWriter = Pick< + _ts.System, + 'createDirectory' | 'deleteFile' | 'setModifiedTime' | 'writeFile' +>; +type FsWatcher = Pick<_ts.System, 'watchDirectory' | 'watchFile'>; + +// Start with no caching; then add it bit by bit +/** @internal */ +export function createCachedFsReader(reader: FsReader) { + // TODO if useCaseSensitive is false, then lowercase all cache keys? + + const fileContentsCache = new Map(); + const normalizeFileCacheKey = reader.useCaseSensitiveFileNames + ? (key: string) => key + : (key: string) => key.toLowerCase(); + + function invalidateFileContents() {} + function invalidateFileExistence() {} + + return { + ...reader, + directoryExists: cachedLookup( + debugFn('directoryExists', reader.directoryExists) + ), + fileExists: cachedLookup(debugFn('fileExists', reader.fileExists)), + getDirectories: cachedLookup( + debugFn('getDirectories', reader.getDirectories) + ), + readFile: cachedLookup(debugFn('readFile', reader.readFile)), + realpath: reader.realpath + ? cachedLookup(debugFn('realpath', reader.realpath)) + : undefined, + resolvePath: cachedLookup(debugFn('resolvePath', reader.resolvePath)), + invalidateFileContents, + invalidateFileExistence, + }; +} diff --git a/src/index.ts b/src/index.ts index 15fbb0ad0..a5f149759 100644 --- a/src/index.ts +++ b/src/index.ts @@ -8,14 +8,7 @@ import { BaseError } from 'make-error'; import type * as _ts from 'typescript'; import type { Transpiler, TranspilerFactory } from './transpilers/types'; -import { - assign, - cachedLookup, - normalizeSlashes, - parse, - split, - yn, -} from './util'; +import { assign, normalizeSlashes, parse, split, yn } from './util'; import { readConfig } from './configuration'; import type { TSCommon, TSInternal } from './ts-compiler-types'; import { @@ -23,6 +16,9 @@ import { ModuleTypeClassifier, } from './module-type-classifier'; import { createResolverFunctions } from './resolver-functions'; +import type { createEsmHooks as createEsmHooksFn } from './esm'; +import { createCachedFsReader } from './fs'; +import { debug } from './diagnostics'; export { TSCommon }; export { @@ -146,25 +142,6 @@ export interface ProcessEnv { */ export const INSPECT_CUSTOM = util.inspect.custom || 'inspect'; -/** - * Debugging `ts-node`. - */ -const shouldDebug = yn(env.TS_NODE_DEBUG); -/** @internal */ -export const debug = shouldDebug - ? (...args: any) => - console.log(`[ts-node ${new Date().toISOString()}]`, ...args) - : () => undefined; -const debugFn = shouldDebug - ? (key: string, fn: (arg: T) => U) => { - let i = 0; - return (x: T) => { - debug(key, x, ++i); - return fn(x); - }; - } - : (_: string, fn: (arg: T) => U) => fn; - /** * Export the current version. */ @@ -556,13 +533,24 @@ export function create(rawOptions: CreateOptions = {}): Service { rawOptions.projectSearchDir ?? rawOptions.project ?? cwd ); + const cachedFsReader = createCachedFsReader({ + directoryExists: ts.sys.directoryExists, + fileExists: rawOptions.fileExists ?? ts.sys.fileExists, + getDirectories: ts.sys.getDirectories, + readDirectory: ts.sys.readDirectory, + readFile: rawOptions.readFile ?? ts.sys.readFile, + resolvePath: ts.sys.resolvePath, + realpath: ts.sys.realpath, + useCaseSensitiveFileNames: ts.sys.useCaseSensitiveFileNames, + }); + // Read config file and merge new options between env and CLI options. const { configFilePath, config, tsNodeOptionsFromTsconfig, optionBasePaths, - } = readConfig(cwd, ts, rawOptions); + } = readConfig(cwd, ts, cachedFsReader, rawOptions); const options = assign( {}, DEFAULTS, @@ -602,8 +590,6 @@ export function create(rawOptions: CreateOptions = {}): Service { ({ compiler, ts } = loadCompiler(options.compiler, configFilePath)); } - const readFile = options.readFile || ts.sys.readFile; - const fileExists = options.fileExists || ts.sys.fileExists; // typeCheck can override transpileOnly, useful for CLI flag to override config file const transpileOnly = options.transpileOnly === true && options.typeCheck !== true; @@ -651,7 +637,8 @@ export function create(rawOptions: CreateOptions = {}): Service { const diagnosticHost: _ts.FormatDiagnosticsHost = { getNewLine: () => ts.sys.newLine, getCurrentDirectory: () => cwd, - getCanonicalFileName: ts.sys.useCaseSensitiveFileNames + // TODO replace with `ts.createGetCanonicalFileName`? + getCanonicalFileName: cachedFsReader.useCaseSensitiveFileNames ? (x) => x : (x) => x.toLowerCase(), }; @@ -778,7 +765,7 @@ export function create(rawOptions: CreateOptions = {}): Service { ) => TypeInfo; const getCanonicalFileName = ((ts as unknown) as TSInternal).createGetCanonicalFileName( - ts.sys.useCaseSensitiveFileNames + cachedFsReader.useCaseSensitiveFileNames ); const moduleTypeClassifier = createModuleTypeClassifier({ @@ -790,7 +777,7 @@ export function create(rawOptions: CreateOptions = {}): Service { if (!transpileOnly) { const fileContents = new Map(); const rootFileNames = new Set(config.fileNames); - const cachedReadFile = cachedLookup(debugFn('readFile', readFile)); + const cachedReadFile = cachedFsReader.readFile; // Use language services by default (TODO: invert next major version). if (!options.compilerHost) { @@ -834,19 +821,14 @@ export function create(rawOptions: CreateOptions = {}): Service { return ts.ScriptSnapshot.fromString(contents); }, readFile: cachedReadFile, - readDirectory: ts.sys.readDirectory, - getDirectories: cachedLookup( - debugFn('getDirectories', ts.sys.getDirectories) - ), - fileExists: cachedLookup(debugFn('fileExists', fileExists)), - directoryExists: cachedLookup( - debugFn('directoryExists', ts.sys.directoryExists) - ), - realpath: ts.sys.realpath - ? cachedLookup(debugFn('realpath', ts.sys.realpath)) - : undefined, + readDirectory: cachedFsReader.readDirectory, + getDirectories: cachedFsReader.getDirectories, + fileExists: cachedFsReader.fileExists, + directoryExists: cachedFsReader.directoryExists, + realpath: cachedFsReader.realpath, getNewLine: () => ts.sys.newLine, - useCaseSensitiveFileNames: () => ts.sys.useCaseSensitiveFileNames, + useCaseSensitiveFileNames: () => + cachedFsReader.useCaseSensitiveFileNames, getCurrentDirectory: () => cwd, getCompilationSettings: () => config.options, getDefaultLibFileName: () => ts.getDefaultLibFilePath(config.options), @@ -871,7 +853,7 @@ export function create(rawOptions: CreateOptions = {}): Service { serviceHost.resolveTypeReferenceDirectives = resolveTypeReferenceDirectives; const registry = ts.createDocumentRegistry( - ts.sys.useCaseSensitiveFileNames, + cachedFsReader.useCaseSensitiveFileNames, cwd ); const service = ts.createLanguageService(serviceHost, registry); @@ -962,6 +944,7 @@ export function create(rawOptions: CreateOptions = {}): Service { }; } else { const sys: _ts.System & _ts.FormatDiagnosticsHost = { + // TODO splat all cachedFsReader methods into here ...ts.sys, ...diagnosticHost, readFile: (fileName: string) => { @@ -971,18 +954,12 @@ export function create(rawOptions: CreateOptions = {}): Service { if (contents) fileContents.set(fileName, contents); return contents; }, - readDirectory: ts.sys.readDirectory, - getDirectories: cachedLookup( - debugFn('getDirectories', ts.sys.getDirectories) - ), - fileExists: cachedLookup(debugFn('fileExists', fileExists)), - directoryExists: cachedLookup( - debugFn('directoryExists', ts.sys.directoryExists) - ), - resolvePath: cachedLookup(debugFn('resolvePath', ts.sys.resolvePath)), - realpath: ts.sys.realpath - ? cachedLookup(debugFn('realpath', ts.sys.realpath)) - : undefined, + readDirectory: cachedFsReader.readDirectory, + getDirectories: cachedFsReader.getDirectories, + fileExists: cachedFsReader.fileExists, + directoryExists: cachedFsReader.directoryExists, + resolvePath: cachedFsReader.resolvePath, + realpath: cachedFsReader.realpath, }; const host: _ts.CompilerHost = ts.createIncrementalCompilerHost @@ -1486,7 +1463,6 @@ function getTokenAtPosition( } } -import type { createEsmHooks as createEsmHooksFn } from './esm'; export const createEsmHooks: typeof createEsmHooksFn = ( tsNodeService: Service ) => require('./esm').createEsmHooks(tsNodeService); diff --git a/src/module-type-classifier.ts b/src/module-type-classifier.ts index dfe153289..07f8d54ae 100644 --- a/src/module-type-classifier.ts +++ b/src/module-type-classifier.ts @@ -1,4 +1,3 @@ -import { dirname } from 'path'; import { getPatternFromSpec } from './ts-internals'; import { cachedLookup, normalizeSlashes } from './util'; diff --git a/src/resolver-functions.ts b/src/resolver-functions.ts index e76528155..e83f2e50e 100644 --- a/src/resolver-functions.ts +++ b/src/resolver-functions.ts @@ -65,6 +65,7 @@ export function createResolverFunctions(kwargs: { // .js is switched on-demand if ( resolvedModule.isExternalLibraryImport && + // TODO should include tsx, mts, and cts? ((resolvedFileName.endsWith('.ts') && !resolvedFileName.endsWith('.d.ts')) || isFileKnownToBeInternal(resolvedFileName) || From e8f2bd0919b07be2683957a5e4d63809f05c7074 Mon Sep 17 00:00:00 2001 From: Andrew Bradley Date: Sat, 16 Oct 2021 16:18:47 -0400 Subject: [PATCH 2/7] fix circular dependency issue --- src/diagnostics.ts | 3 +-- src/index.ts | 4 +--- src/repl.ts | 3 ++- src/util.ts | 4 ++++ 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/diagnostics.ts b/src/diagnostics.ts index 8e669626c..bee56aad5 100644 --- a/src/diagnostics.ts +++ b/src/diagnostics.ts @@ -1,5 +1,4 @@ -import { env } from '.'; -import { yn } from './util'; +import { env, yn } from './util'; /** * Debugging `ts-node`. diff --git a/src/index.ts b/src/index.ts index a5f149759..f869f6aab 100644 --- a/src/index.ts +++ b/src/index.ts @@ -8,7 +8,7 @@ import { BaseError } from 'make-error'; import type * as _ts from 'typescript'; import type { Transpiler, TranspilerFactory } from './transpilers/types'; -import { assign, normalizeSlashes, parse, split, yn } from './util'; +import { assign, env, normalizeSlashes, parse, split, yn } from './util'; import { readConfig } from './configuration'; import type { TSCommon, TSInternal } from './ts-compiler-types'; import { @@ -102,8 +102,6 @@ declare global { } } -/** @internal */ -export const env = process.env as ProcessEnv; /** * Declare all env vars, to aid discoverability. * If an env var affects ts-node's behavior, it should not be buried somewhere in our codebase. diff --git a/src/repl.ts b/src/repl.ts index 0ef51017d..afc85f5f2 100644 --- a/src/repl.ts +++ b/src/repl.ts @@ -8,13 +8,14 @@ import { start as nodeReplStart, } from 'repl'; import { Context, createContext, Script } from 'vm'; -import { Service, CreateOptions, TSError, env } from './index'; +import { Service, CreateOptions, TSError } from './index'; import { readFileSync, statSync } from 'fs'; import { Console } from 'console'; import * as assert from 'assert'; import type * as tty from 'tty'; import type * as Module from 'module'; import { builtinModules } from 'module'; +import { env } from './util'; // Lazy-loaded. let _processTopLevelAwait: (src: string) => string | null; diff --git a/src/util.ts b/src/util.ts index 23e19bba2..f5ba324e1 100644 --- a/src/util.ts +++ b/src/util.ts @@ -4,6 +4,10 @@ import { } from 'module'; import type _createRequire from 'create-require'; import * as ynModule from 'yn'; +import type { ProcessEnv } from '.'; + +/** @internal */ +export const env = process.env as ProcessEnv; /** @internal */ export const createRequire = From c00942f2e5b46636572d5d3c9b8594130b168bfb Mon Sep 17 00:00:00 2001 From: Andrew Bradley Date: Sun, 17 Oct 2021 19:45:31 -0400 Subject: [PATCH 3/7] changes --- .gitignore | 1 + development-docs/caches.md | 32 +++ dist-raw/node-cjs-helpers.d.ts | 1 - dist-raw/node-cjs-loader-utils.js | 219 ++++++++++---------- dist-raw/node-esm-default-get-format.js | 9 +- dist-raw/node-esm-resolve-implementation.js | 19 +- dist-raw/node-internal-fs.js | 36 ++-- dist-raw/node-package-json-reader.js | 9 +- package.json | 2 +- src/configuration.ts | 4 +- src/esm.ts | 9 +- src/fs.ts | 41 +++- src/index.ts | 106 ++++++---- tsconfig.build-dist-raw.json | 16 ++ tsconfig.build.json | 20 ++ tsconfig.json | 6 +- 16 files changed, 345 insertions(+), 185 deletions(-) create mode 100644 development-docs/caches.md delete mode 100644 dist-raw/node-cjs-helpers.d.ts create mode 100644 tsconfig.build-dist-raw.json create mode 100644 tsconfig.build.json diff --git a/.gitignore b/.gitignore index f8e97ed5f..215656b70 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ tsconfig.schemastore-schema.json /website/static/api /tsconfig.tsbuildinfo /temp +/.tmp diff --git a/development-docs/caches.md b/development-docs/caches.md new file mode 100644 index 000000000..ec6b3e8cc --- /dev/null +++ b/development-docs/caches.md @@ -0,0 +1,32 @@ +Lots of caching in ts-node. + +## Caches + +### FS cache: +caches results of primitive ts.sys.readFile, etc operations +Shared across compiler and config loader + +### fileContents (and fileVersions) cache: +sits in front of fs cache. +Node.js module loading mechanism reads file contents from disk. That's put into this cache. + +### Output cache: +Caches the emitted JS syntax from compilation. +Has appended //# sourcemap comments. +source-map-support reads from here before fallback to filesystem. + +### source-map-support cache: +caches fs.readFile calls +overlayed by output cache above +overlayed by sourcesContents from parsed sourcemaps + +### SourceFile cache: (does not exist today) +for Compiler API codepath +to avoid re-parsing SourceFile repeatedly + +## Questions + +If both: +- source-map-support caches a sourcesContents string of a .ts file +- cachedFsReader caches the same .ts file from disk +...which is used by source-map-support? Does it matter since they should be identical? diff --git a/dist-raw/node-cjs-helpers.d.ts b/dist-raw/node-cjs-helpers.d.ts deleted file mode 100644 index a57c2f831..000000000 --- a/dist-raw/node-cjs-helpers.d.ts +++ /dev/null @@ -1 +0,0 @@ -export function addBuiltinLibsToObject(object: any): void; diff --git a/dist-raw/node-cjs-loader-utils.js b/dist-raw/node-cjs-loader-utils.js index b7ec0d531..d2a1aabf9 100644 --- a/dist-raw/node-cjs-loader-utils.js +++ b/dist-raw/node-cjs-loader-utils.js @@ -3,131 +3,136 @@ // Each function and variable below must have a comment linking to the source in node's github repo. const path = require('path'); -const packageJsonReader = require('./node-package-json-reader'); const {JSONParse} = require('./node-primordials'); const {normalizeSlashes} = require('../dist/util'); -module.exports.assertScriptCanLoadAsCJSImpl = assertScriptCanLoadAsCJSImpl; +/** @param {ReturnType} packageJsonReader */ +function createNodeCjsLoaderUtils(packageJsonReader) { -/** - * copied from Module._extensions['.js'] - * https://github.com/nodejs/node/blob/v15.3.0/lib/internal/modules/cjs/loader.js#L1113-L1120 - * @param {import('../src/index').Service} service - * @param {NodeJS.Module} module - * @param {string} filename - */ -function assertScriptCanLoadAsCJSImpl(service, module, filename) { - const pkg = readPackageScope(filename); + /** + * copied from Module._extensions['.js'] + * https://github.com/nodejs/node/blob/v15.3.0/lib/internal/modules/cjs/loader.js#L1113-L1120 + * @param {import('../src/index').Service} service + * @param {NodeJS.Module} module + * @param {string} filename + */ + function assertScriptCanLoadAsCJSImpl(service, module, filename) { + const pkg = readPackageScope(filename); - // ts-node modification: allow our configuration to override - const tsNodeClassification = service.moduleTypeClassifier.classifyModule(normalizeSlashes(filename)); - if(tsNodeClassification.moduleType === 'cjs') return; + // ts-node modification: allow our configuration to override + const tsNodeClassification = service.moduleTypeClassifier.classifyModule(normalizeSlashes(filename)); + if(tsNodeClassification.moduleType === 'cjs') return; - // Function require shouldn't be used in ES modules. - if (tsNodeClassification.moduleType === 'esm' || (pkg && pkg.data && pkg.data.type === 'module')) { - const parentPath = module.parent && module.parent.filename; - const packageJsonPath = pkg ? path.resolve(pkg.path, 'package.json') : null; - throw createErrRequireEsm(filename, parentPath, packageJsonPath); + // Function require shouldn't be used in ES modules. + if (tsNodeClassification.moduleType === 'esm' || (pkg && pkg.data && pkg.data.type === 'module')) { + const parentPath = module.parent && module.parent.filename; + const packageJsonPath = pkg ? path.resolve(pkg.path, 'package.json') : null; + throw createErrRequireEsm(filename, parentPath, packageJsonPath); + } } -} -// Copied from https://github.com/nodejs/node/blob/2d5d77306f6dff9110c1f77fefab25f973415770/lib/internal/modules/cjs/loader.js#L285-L301 -function readPackageScope(checkPath) { - const rootSeparatorIndex = checkPath.indexOf(path.sep); - let separatorIndex; - while ( - (separatorIndex = checkPath.lastIndexOf(path.sep)) > rootSeparatorIndex - ) { - checkPath = checkPath.slice(0, separatorIndex); - if (checkPath.endsWith(path.sep + 'node_modules')) - return false; - const pjson = readPackage(checkPath); - if (pjson) return { - path: checkPath, - data: pjson - }; + // Copied from https://github.com/nodejs/node/blob/2d5d77306f6dff9110c1f77fefab25f973415770/lib/internal/modules/cjs/loader.js#L285-L301 + function readPackageScope(checkPath) { + const rootSeparatorIndex = checkPath.indexOf(path.sep); + let separatorIndex; + while ( + (separatorIndex = checkPath.lastIndexOf(path.sep)) > rootSeparatorIndex + ) { + checkPath = checkPath.slice(0, separatorIndex); + if (checkPath.endsWith(path.sep + 'node_modules')) + return false; + const pjson = readPackage(checkPath); + if (pjson) return { + path: checkPath, + data: pjson + }; + } + return false; } - return false; -} -// Copied from https://github.com/nodejs/node/blob/2d5d77306f6dff9110c1f77fefab25f973415770/lib/internal/modules/cjs/loader.js#L249 -const packageJsonCache = new Map(); + // Copied from https://github.com/nodejs/node/blob/2d5d77306f6dff9110c1f77fefab25f973415770/lib/internal/modules/cjs/loader.js#L249 + const packageJsonCache = new Map(); -// Copied from https://github.com/nodejs/node/blob/v15.3.0/lib/internal/modules/cjs/loader.js#L275-L304 -function readPackage(requestPath) { - const jsonPath = path.resolve(requestPath, 'package.json'); + // Copied from https://github.com/nodejs/node/blob/v15.3.0/lib/internal/modules/cjs/loader.js#L275-L304 + function readPackage(requestPath) { + const jsonPath = path.resolve(requestPath, 'package.json'); - const existing = packageJsonCache.get(jsonPath); - if (existing !== undefined) return existing; + const existing = packageJsonCache.get(jsonPath); + if (existing !== undefined) return existing; - const result = packageJsonReader.read(jsonPath); - const json = result.containsKeys === false ? '{}' : result.string; - if (json === undefined) { - packageJsonCache.set(jsonPath, false); - return false; - } + const result = packageJsonReader.read(jsonPath); + const json = result.containsKeys === false ? '{}' : result.string; + if (json === undefined) { + packageJsonCache.set(jsonPath, false); + return false; + } - try { - const parsed = JSONParse(json); - const filtered = { - name: parsed.name, - main: parsed.main, - exports: parsed.exports, - imports: parsed.imports, - type: parsed.type - }; - packageJsonCache.set(jsonPath, filtered); - return filtered; - } catch (e) { - e.path = jsonPath; - e.message = 'Error parsing ' + jsonPath + ': ' + e.message; - throw e; + try { + const parsed = JSONParse(json); + const filtered = { + name: parsed.name, + main: parsed.main, + exports: parsed.exports, + imports: parsed.imports, + type: parsed.type + }; + packageJsonCache.set(jsonPath, filtered); + return filtered; + } catch (e) { + e.path = jsonPath; + e.message = 'Error parsing ' + jsonPath + ': ' + e.message; + throw e; + } } -} -// Native ERR_REQUIRE_ESM Error is declared here: -// https://github.com/nodejs/node/blob/2d5d77306f6dff9110c1f77fefab25f973415770/lib/internal/errors.js#L1294-L1313 -// Error class factory is implemented here: -// function E: https://github.com/nodejs/node/blob/2d5d77306f6dff9110c1f77fefab25f973415770/lib/internal/errors.js#L323-L341 -// function makeNodeErrorWithCode: https://github.com/nodejs/node/blob/2d5d77306f6dff9110c1f77fefab25f973415770/lib/internal/errors.js#L251-L278 -// The code below should create an error that matches the native error as closely as possible. -// Third-party libraries which attempt to catch the native ERR_REQUIRE_ESM should recognize our imitation error. -function createErrRequireEsm(filename, parentPath, packageJsonPath) { - const code = 'ERR_REQUIRE_ESM' - const err = new Error(getMessage(filename, parentPath, packageJsonPath)) - // Set `name` to be used in stack trace, generate stack trace with that name baked in, then re-declare the `name` field. - // This trick is copied from node's source. - err.name = `Error [${ code }]` - err.stack - Object.defineProperty(err, 'name', { - value: 'Error', - enumerable: false, - writable: true, - configurable: true - }) - err.code = code - return err + // Native ERR_REQUIRE_ESM Error is declared here: + // https://github.com/nodejs/node/blob/2d5d77306f6dff9110c1f77fefab25f973415770/lib/internal/errors.js#L1294-L1313 + // Error class factory is implemented here: + // function E: https://github.com/nodejs/node/blob/2d5d77306f6dff9110c1f77fefab25f973415770/lib/internal/errors.js#L323-L341 + // function makeNodeErrorWithCode: https://github.com/nodejs/node/blob/2d5d77306f6dff9110c1f77fefab25f973415770/lib/internal/errors.js#L251-L278 + // The code below should create an error that matches the native error as closely as possible. + // Third-party libraries which attempt to catch the native ERR_REQUIRE_ESM should recognize our imitation error. + function createErrRequireEsm(filename, parentPath, packageJsonPath) { + const code = 'ERR_REQUIRE_ESM' + const err = new Error(getMessage(filename, parentPath, packageJsonPath)) + // Set `name` to be used in stack trace, generate stack trace with that name baked in, then re-declare the `name` field. + // This trick is copied from node's source. + err.name = `Error [${ code }]` + err.stack + Object.defineProperty(err, 'name', { + value: 'Error', + enumerable: false, + writable: true, + configurable: true + }) + err.code = code + return err - // Copy-pasted from https://github.com/nodejs/node/blob/b533fb3508009e5f567cc776daba8fbf665386a6/lib/internal/errors.js#L1293-L1311 - // so that our error message is identical to the native message. - function getMessage(filename, parentPath = null, packageJsonPath = null) { - const ext = path.extname(filename) - let msg = `Must use import to load ES Module: ${filename}`; - if (parentPath && packageJsonPath) { - const path = require('path'); - const basename = path.basename(filename) === path.basename(parentPath) ? - filename : path.basename(filename); - msg += - '\nrequire() of ES modules is not supported.\nrequire() of ' + - `${filename} ${parentPath ? `from ${parentPath} ` : ''}` + - `is an ES module file as it is a ${ext} file whose nearest parent ` + - `package.json contains "type": "module" which defines all ${ext} ` + - 'files in that package scope as ES modules.\nInstead ' + - 'change the requiring code to use ' + - 'import(), or remove "type": "module" from ' + - `${packageJsonPath}.\n`; + // Copy-pasted from https://github.com/nodejs/node/blob/b533fb3508009e5f567cc776daba8fbf665386a6/lib/internal/errors.js#L1293-L1311 + // so that our error message is identical to the native message. + function getMessage(filename, parentPath = null, packageJsonPath = null) { + const ext = path.extname(filename) + let msg = `Must use import to load ES Module: ${filename}`; + if (parentPath && packageJsonPath) { + const path = require('path'); + const basename = path.basename(filename) === path.basename(parentPath) ? + filename : path.basename(filename); + msg += + '\nrequire() of ES modules is not supported.\nrequire() of ' + + `${filename} ${parentPath ? `from ${parentPath} ` : ''}` + + `is an ES module file as it is a ${ext} file whose nearest parent ` + + `package.json contains "type": "module" which defines all ${ext} ` + + 'files in that package scope as ES modules.\nInstead ' + + 'change the requiring code to use ' + + 'import(), or remove "type": "module" from ' + + `${packageJsonPath}.\n`; + return msg; + } return msg; } - return msg; } + + return {assertScriptCanLoadAsCJSImpl}; } + +module.exports.createNodeCjsLoaderUtils = createNodeCjsLoaderUtils; diff --git a/dist-raw/node-esm-default-get-format.js b/dist-raw/node-esm-default-get-format.js index d8af956f3..8f3784bf6 100644 --- a/dist-raw/node-esm-default-get-format.js +++ b/dist-raw/node-esm-default-get-format.js @@ -15,7 +15,6 @@ const experimentalJsonModules = getOptionValue('--experimental-json-modules'); const experimentalSpeciferResolution = getOptionValue('--experimental-specifier-resolution'); const experimentalWasmModules = getOptionValue('--experimental-wasm-modules'); -const { getPackageType } = require('./node-esm-resolve-implementation.js').createResolve({tsExtensions: [], jsExtensions: []}); const { URL, fileURLToPath } = require('url'); const { ERR_UNKNOWN_FILE_EXTENSION } = require('./node-errors').codes; @@ -41,6 +40,9 @@ if (experimentalWasmModules) if (experimentalJsonModules) extensionFormatMap['.json'] = legacyExtensionFormatMap['.json'] = 'json'; +function createDefaultGetFormat(getPackageType) { + +// Intentionally unindented to simplify the diff function defaultGetFormat(url, context, defaultGetFormatUnused) { if (StringPrototypeStartsWith(url, 'node:')) { return { format: 'builtin' }; @@ -80,4 +82,7 @@ function defaultGetFormat(url, context, defaultGetFormatUnused) { } return { format: null }; } -exports.defaultGetFormat = defaultGetFormat; +return {defaultGetFormat}; +} + +exports.createDefaultGetFormat = createDefaultGetFormat; diff --git a/dist-raw/node-esm-resolve-implementation.js b/dist-raw/node-esm-resolve-implementation.js index 04ea84668..d89ed4fb9 100644 --- a/dist-raw/node-esm-resolve-implementation.js +++ b/dist-raw/node-esm-resolve-implementation.js @@ -95,16 +95,26 @@ const { const CJSModule = Module; // const packageJsonReader = require('internal/modules/package_json_reader'); -const packageJsonReader = require('./node-package-json-reader'); + +const { createDefaultGetFormat } = require('./node-esm-default-get-format'); + const userConditions = getOptionValue('--conditions'); const DEFAULT_CONDITIONS = ObjectFreeze(['node', 'import', ...userConditions]); const DEFAULT_CONDITIONS_SET = new SafeSet(DEFAULT_CONDITIONS); const pendingDeprecation = getOptionValue('--pending-deprecation'); +/** + * @param {{ + * tsExtensions: string[]; + * jsExtensions: string[]; + * preferTsExts: boolean; + * packageJsonReader: ReturnType; + * }} opts + */ function createResolve(opts) { -// TODO receive cached fs implementations here -const {tsExtensions, jsExtensions, preferTsExts} = opts; +const {tsExtensions, jsExtensions, preferTsExts, packageJsonReader} = opts; +const {defaultGetFormat} = createDefaultGetFormat(getPackageType); const emittedPackageWarnings = new SafeSet(); function emitFolderMapDeprecation(match, pjsonUrl, isExports, base) { @@ -971,7 +981,8 @@ return { encodedSepRegEx, getPackageType, packageExportsResolve, - packageImportsResolve + packageImportsResolve, + defaultGetFormat }; } module.exports = { diff --git a/dist-raw/node-internal-fs.js b/dist-raw/node-internal-fs.js index d9a2528dd..e649dfc27 100644 --- a/dist-raw/node-internal-fs.js +++ b/dist-raw/node-internal-fs.js @@ -1,22 +1,24 @@ const fs = require('fs'); -// In node's core, this is implemented in C -// https://github.com/nodejs/node/blob/v15.3.0/src/node_file.cc#L891-L985 -function internalModuleReadJSON(path) { - let string - try { - string = fs.readFileSync(path, 'utf8') - } catch (e) { - if (e.code === 'ENOENT') return [] - throw e +function createNodeInternalModuleReadJSON() { + // In node's core, this is implemented in C + // https://github.com/nodejs/node/blob/v15.3.0/src/node_file.cc#L891-L985 + function internalModuleReadJSON(path) { + let string + try { + string = fs.readFileSync(path, 'utf8') + } catch (e) { + if (e.code === 'ENOENT') return [] + throw e + } + // Node's implementation checks for the presence of relevant keys: main, name, type, exports, imports + // Node does this for performance to skip unnecessary parsing. + // This would slow us down and, based on our usage, we can skip it. + const containsKeys = true + return [string, containsKeys] } - // Node's implementation checks for the presence of relevant keys: main, name, type, exports, imports - // Node does this for performance to skip unnecessary parsing. - // This would slow us down and, based on our usage, we can skip it. - const containsKeys = true - return [string, containsKeys] + + return internalModuleReadJSON; } -module.exports = { - internalModuleReadJSON -}; +module.exports.createNodeInternalModuleReadJSON = createNodeInternalModuleReadJSON; diff --git a/dist-raw/node-package-json-reader.js b/dist-raw/node-package-json-reader.js index e9f82c6f4..179be01ba 100644 --- a/dist-raw/node-package-json-reader.js +++ b/dist-raw/node-package-json-reader.js @@ -2,11 +2,13 @@ 'use strict'; const { SafeMap } = require('./node-primordials'); -const { internalModuleReadJSON } = require('./node-internal-fs'); const { pathToFileURL } = require('url'); const { toNamespacedPath } = require('path'); // const { getOptionValue } = require('./node-options'); +/** @param {ReturnType} internalModuleReadJSON */ +function createNodePackageJsonReader(internalModuleReadJSON) { +// Intentionally un-indented to keep diff small without needing to mess with whitespace-ignoring flags const cache = new SafeMap(); let manifest; @@ -41,4 +43,7 @@ function read(jsonPath) { return result; } -module.exports = { read }; +return { read }; +} + +module.exports.createNodePackageJsonReader = createNodePackageJsonReader; diff --git a/package.json b/package.json index 22aea1e52..a10bb7c2b 100644 --- a/package.json +++ b/package.json @@ -60,7 +60,7 @@ "rebuild": "npm run clean && npm run build", "build": "npm run build-nopack && npm run build-pack", "build-nopack": "npm run build-tsc && npm run build-configSchema", - "build-tsc": "tsc", + "build-tsc": "tsc --build ./tsconfig.build.json", "build-configSchema": "typescript-json-schema --topRef --refs --validationKeywords allOf --out tsconfig.schema.json tsconfig.build-schema.json TsConfigSchema && node --require ./register ./scripts/create-merged-schema", "build-pack": "node ./scripts/build-pack.js", "test-spec": "ava", diff --git a/src/configuration.ts b/src/configuration.ts index 3b6e09fd6..0a7b5764f 100644 --- a/src/configuration.ts +++ b/src/configuration.ts @@ -1,6 +1,6 @@ import { resolve, dirname } from 'path'; import type * as _ts from 'typescript'; -import type { FsReader } from './fs'; +import type { TsSysFsReader } from './fs'; import { CreateOptions, DEFAULTS, @@ -62,7 +62,7 @@ function fixConfig(ts: TSCommon, config: _ts.ParsedCommandLine) { export function readConfig( cwd: string, ts: TSCommon, - fsReader: FsReader, + fsReader: TsSysFsReader, rawApiOptions: CreateOptions ): { /** diff --git a/src/esm.ts b/src/esm.ts index 5502d0155..3964cd294 100644 --- a/src/esm.ts +++ b/src/esm.ts @@ -18,7 +18,6 @@ import { normalizeSlashes } from './util'; const { createResolve, } = require('../dist-raw/node-esm-resolve-implementation'); -const { defaultGetFormat } = require('../dist-raw/node-esm-default-get-format'); // Note: On Windows, URLs look like this: file:///D:/dev/@TypeStrong/ts-node-examples/foo.ts @@ -124,7 +123,13 @@ export function createEsmHooks(tsNodeService: Service) { // otherwise call the old getFormat() hook using node's old built-in defaultGetFormat() that ships with ts-node const format = context.format ?? - (await getFormat(url, context, defaultGetFormat)).format; + ( + await getFormat( + url, + context, + nodeResolveImplementation.defaultGetFormat + ) + ).format; let source = undefined; if (format !== 'builtin' && format !== 'commonjs') { diff --git a/src/fs.ts b/src/fs.ts index 5f3f3fcdf..608cd0456 100644 --- a/src/fs.ts +++ b/src/fs.ts @@ -1,4 +1,5 @@ import type * as _ts from 'typescript'; +import * as fs from 'fs'; import { debugFn } from './diagnostics'; import { cachedLookup } from './util'; @@ -24,11 +25,17 @@ preferTsExts=false we should resolve to ./dist/bar.js */ +/** @internal */ +export type NodeFsReader = { + readFileSync(path: string, encoding: 'utf8'): string; + statSync(path: string): fs.Stats; +}; + /** * @internal * Since `useCaseSensitiveFileNames` is required to know how to cache, we expose it on the interface */ -export type FsReader = Pick< +export type TsSysFsReader = Pick< _ts.System, | 'directoryExists' | 'fileExists' @@ -40,17 +47,20 @@ export type FsReader = Pick< | 'useCaseSensitiveFileNames' >; /** since I've never hit code that needs these functions implemented */ -type FullFsReader = FsReader & +type TsSysFullFsReader = TsSysFsReader & Pick<_ts.System, 'getFileSize' | 'getModifiedTime'>; -type FsWriter = Pick< +type TsSysFsWriter = Pick< _ts.System, 'createDirectory' | 'deleteFile' | 'setModifiedTime' | 'writeFile' >; -type FsWatcher = Pick<_ts.System, 'watchDirectory' | 'watchFile'>; +type TsSysFsWatcher = Pick<_ts.System, 'watchDirectory' | 'watchFile'>; + +/** @internal */ +export type CachedFsReader = ReturnType; // Start with no caching; then add it bit by bit /** @internal */ -export function createCachedFsReader(reader: FsReader) { +export function createCachedFsReader(reader: TsSysFsReader) { // TODO if useCaseSensitive is false, then lowercase all cache keys? const fileContentsCache = new Map(); @@ -61,7 +71,7 @@ export function createCachedFsReader(reader: FsReader) { function invalidateFileContents() {} function invalidateFileExistence() {} - return { + const sys: TsSysFsReader = { ...reader, directoryExists: cachedLookup( debugFn('directoryExists', reader.directoryExists) @@ -75,6 +85,25 @@ export function createCachedFsReader(reader: FsReader) { ? cachedLookup(debugFn('realpath', reader.realpath)) : undefined, resolvePath: cachedLookup(debugFn('resolvePath', reader.resolvePath)), + }; + const enoentError = new Error() as Error & { code: 'ENOENT' }; + enoentError.code = 'ENOENT'; + const nodeFs: NodeFsReader = { + readFileSync(path: string, encoding: 'utf8') { + // TODO It is unnecessarily messy to implement node's `readFileSync` on top of TS's `readFile`. Refactor this. + const ret = sys.readFile(path); + if (typeof ret !== 'string') { + throw enoentError; + } + return ret; + }, + statSync: cachedLookup( + debugFn('statSync', fs.statSync) + ) as NodeFsReader['statSync'], + }; + return { + sys, + nodeFs, invalidateFileContents, invalidateFileExistence, }; diff --git a/src/index.ts b/src/index.ts index f869f6aab..7fd439527 100644 --- a/src/index.ts +++ b/src/index.ts @@ -17,8 +17,11 @@ import { } from './module-type-classifier'; import { createResolverFunctions } from './resolver-functions'; import type { createEsmHooks as createEsmHooksFn } from './esm'; -import { createCachedFsReader } from './fs'; +import { CachedFsReader, createCachedFsReader } from './fs'; import { debug } from './diagnostics'; +import type { createNodeCjsLoaderUtils as _createNodeCjsLoaderUtils } from '../dist-raw/node-cjs-loader-utils'; +import { createNodePackageJsonReader } from '../dist-raw/node-package-json-reader'; +import { createNodeInternalModuleReadJSON as createNodeInternalModuleReadJSON } from '../dist-raw/node-internal-fs'; export { TSCommon }; export { @@ -40,9 +43,15 @@ export type { * Does this version of node obey the package.json "type" field * and throw ERR_REQUIRE_ESM when attempting to require() an ESM modules. */ +// TODO remove this; today we only support node >=12 const engineSupportsPackageTypeField = parseInt(process.versions.node.split('.')[0], 10) >= 12; +const createNodeCjsLoaderUtils = engineSupportsPackageTypeField + ? (require('../dist-raw/node-cjs-loader-utils') + .createNodeCjsLoaderUtils as typeof _createNodeCjsLoaderUtils) + : undefined; + /** @internal */ export function versionGteLt( version: string, @@ -70,22 +79,6 @@ export function versionGteLt( } } -/** - * Assert that script can be loaded as CommonJS when we attempt to require it. - * If it should be loaded as ESM, throw ERR_REQUIRE_ESM like node does. - * - * Loaded conditionally so we don't need to support older node versions - */ -let assertScriptCanLoadAsCJS: ( - service: Service, - module: NodeJS.Module, - filename: string -) => void = engineSupportsPackageTypeField - ? require('../dist-raw/node-cjs-loader-utils').assertScriptCanLoadAsCJSImpl - : () => { - /* noop */ - }; - /** * Registered `ts-node` instance information. */ @@ -434,6 +427,22 @@ export interface Service { installSourceMapSupport(): void; /** @internal */ enableExperimentalEsmLoaderInterop(): void; + /** @internal */ + cachedFsReader: CachedFsReader; + /** @internal */ + nodeInternalModuleReadJson: ReturnType< + typeof import('../dist-raw/node-internal-fs').createNodeInternalModuleReadJSON + >; + /** @internal */ + nodePackageJsonReader: ReturnType< + typeof import('../dist-raw/node-package-json-reader').createNodePackageJsonReader + >; + /** @internal */ + nodeCjsLoaderUtils: + | ReturnType< + typeof import('../dist-raw/node-cjs-loader-utils').createNodeCjsLoaderUtils + > + | undefined; } /** @@ -541,6 +550,11 @@ export function create(rawOptions: CreateOptions = {}): Service { realpath: ts.sys.realpath, useCaseSensitiveFileNames: ts.sys.useCaseSensitiveFileNames, }); + const nodeInternalModuleReadJson = createNodeInternalModuleReadJSON(); + const nodePackageJsonReader = createNodePackageJsonReader( + nodeInternalModuleReadJson + ); + const nodeCjsLoaderUtils = createNodeCjsLoaderUtils?.(nodePackageJsonReader); // Read config file and merge new options between env and CLI options. const { @@ -548,7 +562,7 @@ export function create(rawOptions: CreateOptions = {}): Service { config, tsNodeOptionsFromTsconfig, optionBasePaths, - } = readConfig(cwd, ts, cachedFsReader, rawOptions); + } = readConfig(cwd, ts, cachedFsReader.sys, rawOptions); const options = assign( {}, DEFAULTS, @@ -636,7 +650,7 @@ export function create(rawOptions: CreateOptions = {}): Service { getNewLine: () => ts.sys.newLine, getCurrentDirectory: () => cwd, // TODO replace with `ts.createGetCanonicalFileName`? - getCanonicalFileName: cachedFsReader.useCaseSensitiveFileNames + getCanonicalFileName: cachedFsReader.sys.useCaseSensitiveFileNames ? (x) => x : (x) => x.toLowerCase(), }; @@ -697,7 +711,11 @@ export function create(rawOptions: CreateOptions = {}): Service { } } path = normalizeSlashes(path); - return outputCache.get(path)?.content || ''; + return ( + outputCache.get(path)?.content || + cachedFsReader.sys.readFile(path) || + '' + ); }, redirectConflictingLibrary: true, onConflictingLibraryRedirect( @@ -763,7 +781,7 @@ export function create(rawOptions: CreateOptions = {}): Service { ) => TypeInfo; const getCanonicalFileName = ((ts as unknown) as TSInternal).createGetCanonicalFileName( - cachedFsReader.useCaseSensitiveFileNames + cachedFsReader.sys.useCaseSensitiveFileNames ); const moduleTypeClassifier = createModuleTypeClassifier({ @@ -775,7 +793,6 @@ export function create(rawOptions: CreateOptions = {}): Service { if (!transpileOnly) { const fileContents = new Map(); const rootFileNames = new Set(config.fileNames); - const cachedReadFile = cachedFsReader.readFile; // Use language services by default (TODO: invert next major version). if (!options.compilerHost) { @@ -808,7 +825,7 @@ export function create(rawOptions: CreateOptions = {}): Service { // Read contents into TypeScript memory cache. if (contents === undefined) { - contents = cachedReadFile(fileName); + contents = cachedFsReader.sys.readFile(fileName); if (contents === undefined) return; fileVersions.set(fileName, 1); @@ -818,15 +835,15 @@ export function create(rawOptions: CreateOptions = {}): Service { return ts.ScriptSnapshot.fromString(contents); }, - readFile: cachedReadFile, - readDirectory: cachedFsReader.readDirectory, - getDirectories: cachedFsReader.getDirectories, - fileExists: cachedFsReader.fileExists, - directoryExists: cachedFsReader.directoryExists, - realpath: cachedFsReader.realpath, + readFile: cachedFsReader.sys.readFile, + readDirectory: cachedFsReader.sys.readDirectory, + getDirectories: cachedFsReader.sys.getDirectories, + fileExists: cachedFsReader.sys.fileExists, + directoryExists: cachedFsReader.sys.directoryExists, + realpath: cachedFsReader.sys.realpath, getNewLine: () => ts.sys.newLine, useCaseSensitiveFileNames: () => - cachedFsReader.useCaseSensitiveFileNames, + cachedFsReader.sys.useCaseSensitiveFileNames, getCurrentDirectory: () => cwd, getCompilationSettings: () => config.options, getDefaultLibFileName: () => ts.getDefaultLibFilePath(config.options), @@ -851,7 +868,7 @@ export function create(rawOptions: CreateOptions = {}): Service { serviceHost.resolveTypeReferenceDirectives = resolveTypeReferenceDirectives; const registry = ts.createDocumentRegistry( - cachedFsReader.useCaseSensitiveFileNames, + cachedFsReader.sys.useCaseSensitiveFileNames, cwd ); const service = ts.createLanguageService(serviceHost, registry); @@ -948,16 +965,16 @@ export function create(rawOptions: CreateOptions = {}): Service { readFile: (fileName: string) => { const cacheContents = fileContents.get(fileName); if (cacheContents !== undefined) return cacheContents; - const contents = cachedReadFile(fileName); + const contents = cachedFsReader.sys.readFile(fileName); if (contents) fileContents.set(fileName, contents); return contents; }, - readDirectory: cachedFsReader.readDirectory, - getDirectories: cachedFsReader.getDirectories, - fileExists: cachedFsReader.fileExists, - directoryExists: cachedFsReader.directoryExists, - resolvePath: cachedFsReader.resolvePath, - realpath: cachedFsReader.realpath, + readDirectory: cachedFsReader.sys.readDirectory, + getDirectories: cachedFsReader.sys.getDirectories, + fileExists: cachedFsReader.sys.fileExists, + directoryExists: cachedFsReader.sys.directoryExists, + resolvePath: cachedFsReader.sys.resolvePath, + realpath: cachedFsReader.sys.realpath, }; const host: _ts.CompilerHost = ts.createIncrementalCompilerHost @@ -1266,6 +1283,10 @@ export function create(rawOptions: CreateOptions = {}): Service { addDiagnosticFilter, installSourceMapSupport, enableExperimentalEsmLoaderInterop, + cachedFsReader, + nodeInternalModuleReadJson, + nodePackageJsonReader, + nodeCjsLoaderUtils, }; } @@ -1329,7 +1350,14 @@ function registerExtension( require.extensions[ext] = function (m: any, filename) { if (service.ignored(filename)) return old(m, filename); - assertScriptCanLoadAsCJS(service, m, filename); + // Assert that script can be loaded as CommonJS when we attempt to require it. + // If it should be loaded as ESM, throw ERR_REQUIRE_ESM like node does. + if (engineSupportsPackageTypeField) + service.nodeCjsLoaderUtils!.assertScriptCanLoadAsCJSImpl( + service, + module, + filename + ); const _compile = m._compile; diff --git a/tsconfig.build-dist-raw.json b/tsconfig.build-dist-raw.json new file mode 100644 index 000000000..87b4f35d0 --- /dev/null +++ b/tsconfig.build-dist-raw.json @@ -0,0 +1,16 @@ +{ + "extends": "./tsconfig.json", + "include": [ + "dist-raw", + "src" + ], + "compilerOptions": { + "composite": true, + "allowJs": true, + "emitDeclarationOnly": true, + "stripInternal": false, + "noEmit": false, + "outDir": ".tmp/build-dist-raw", + "tsBuildInfoFile": ".tmp/tsconfig.build-dist-raw.tsbuildinfo", + } +} diff --git a/tsconfig.build.json b/tsconfig.build.json new file mode 100644 index 000000000..f548c18cd --- /dev/null +++ b/tsconfig.build.json @@ -0,0 +1,20 @@ +{ + // In our primary tsconfig.json, + // We set rootDir to `.` to encompass both `src` and `dist-raw`, allowing + // us to typecheck against JSDoc annotations in `dist-raw` + + // When building, however, we must set `rootDir` to `src` and disable `allowJs` + // so that `src`->`dist` paths map correctly. + + "extends": "./tsconfig.json", + "references": [{ + "path": "./tsconfig.build-dist-raw.json" + }], + "compilerOptions": { + "noEmit": false, + "allowJs": false, + "rootDir": "src", + "outDir": "dist", + "tsBuildInfoFile": ".tmp/tsconfig.build.tsbuildinfo", + } +} diff --git a/tsconfig.json b/tsconfig.json index bf59b8f83..23a671966 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,10 +1,12 @@ { "$schema": "./tsconfig.schemastore-schema.json", "compilerOptions": { + "allowJs": true, + "noEmit": true, "target": "es2015", "lib": ["es2015", "dom"], - "rootDir": "src", - "outDir": "dist", + "rootDir": ".", + "tsBuildInfoFile": ".tmp/tsconfig.tsbuildinfo", "module": "commonjs", "moduleResolution": "node", "strict": true, From 28f1f39e88bcc2285454bae9b39e412a4ed39a9b Mon Sep 17 00:00:00 2001 From: Andrew Bradley Date: Sun, 17 Oct 2021 20:48:38 -0400 Subject: [PATCH 4/7] try tweaks to tsconfigs to allow typechecking dist-raw APIs --- package.json | 2 +- tsconfig.build-dist-raw.json | 16 ---------------- tsconfig.build.json | 7 ++++--- tsconfig.declarations.json | 18 ++++++++++++++++++ tsconfig.json | 5 +++++ 5 files changed, 28 insertions(+), 20 deletions(-) delete mode 100644 tsconfig.build-dist-raw.json create mode 100644 tsconfig.declarations.json diff --git a/package.json b/package.json index a10bb7c2b..f6a8a4608 100644 --- a/package.json +++ b/package.json @@ -60,7 +60,7 @@ "rebuild": "npm run clean && npm run build", "build": "npm run build-nopack && npm run build-pack", "build-nopack": "npm run build-tsc && npm run build-configSchema", - "build-tsc": "tsc --build ./tsconfig.build.json", + "build-tsc": "tsc --build ./tsconfig.declarations.json && ( rm -r .tmp/declarations/src 2> /dev/null || true) && tsc --build ./tsconfig.build.json", "build-configSchema": "typescript-json-schema --topRef --refs --validationKeywords allOf --out tsconfig.schema.json tsconfig.build-schema.json TsConfigSchema && node --require ./register ./scripts/create-merged-schema", "build-pack": "node ./scripts/build-pack.js", "test-spec": "ava", diff --git a/tsconfig.build-dist-raw.json b/tsconfig.build-dist-raw.json deleted file mode 100644 index 87b4f35d0..000000000 --- a/tsconfig.build-dist-raw.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "extends": "./tsconfig.json", - "include": [ - "dist-raw", - "src" - ], - "compilerOptions": { - "composite": true, - "allowJs": true, - "emitDeclarationOnly": true, - "stripInternal": false, - "noEmit": false, - "outDir": ".tmp/build-dist-raw", - "tsBuildInfoFile": ".tmp/tsconfig.build-dist-raw.tsbuildinfo", - } -} diff --git a/tsconfig.build.json b/tsconfig.build.json index f548c18cd..a0d8aeac1 100644 --- a/tsconfig.build.json +++ b/tsconfig.build.json @@ -7,13 +7,14 @@ // so that `src`->`dist` paths map correctly. "extends": "./tsconfig.json", - "references": [{ - "path": "./tsconfig.build-dist-raw.json" - }], "compilerOptions": { "noEmit": false, "allowJs": false, "rootDir": "src", + "rootDirs": [ + ".", + ".tmp/declarations" + ], "outDir": "dist", "tsBuildInfoFile": ".tmp/tsconfig.build.tsbuildinfo", } diff --git a/tsconfig.declarations.json b/tsconfig.declarations.json new file mode 100644 index 000000000..c1f566ab4 --- /dev/null +++ b/tsconfig.declarations.json @@ -0,0 +1,18 @@ +{ + // Purpose: to extract .d.ts declarations from dist-raw + "extends": "./tsconfig.json", + "include": [ + "src", + "dist-raw" + ], + "compilerOptions": { + "incremental": true, + "allowJs": true, + "emitDeclarationOnly": true, + "stripInternal": false, + "noEmit": false, + "rootDir": ".", + "outDir": ".tmp/declarations", + "tsBuildInfoFile": ".tmp/tsconfig.declarations.tsbuildinfo", + } +} diff --git a/tsconfig.json b/tsconfig.json index 23a671966..b492cbd71 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,4 +1,9 @@ { + // Role: support editor code intelligence + + // Note: this tsconfig is not used for compiling. + // See tsconfig.declarations.json and tsconfig.build.json + "$schema": "./tsconfig.schemastore-schema.json", "compilerOptions": { "allowJs": true, From b61ef15b6471396647fe9ce2059af7763f2485eb Mon Sep 17 00:00:00 2001 From: Andrew Bradley Date: Sun, 17 Oct 2021 20:52:04 -0400 Subject: [PATCH 5/7] fix --- tsconfig.build-schema.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tsconfig.build-schema.json b/tsconfig.build-schema.json index adf48b4c9..bf10488e9 100644 --- a/tsconfig.build-schema.json +++ b/tsconfig.build-schema.json @@ -1,6 +1,7 @@ { "extends": "./tsconfig.json", "compilerOptions": { - "incremental": false + "incremental": false, + "tsBuildInfoFile": null } } From 79dd7534eb3b2b5bdc1b22bbc6c3ac78c27112a0 Mon Sep 17 00:00:00 2001 From: Andrew Bradley Date: Sun, 17 Oct 2021 21:03:13 -0400 Subject: [PATCH 6/7] fix --- package.json | 4 ++-- tsconfig.build.json | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index f6a8a4608..55ed606ce 100644 --- a/package.json +++ b/package.json @@ -56,11 +56,11 @@ "scripts": { "lint": "prettier --check .", "lint-fix": "prettier --write .", - "clean": "rimraf dist tsconfig.schema.json tsconfig.schemastore-schema.json tsconfig.tsbuildinfo tests/ts-node-packed.tgz", + "clean": "rimraf dist tsconfig.schema.json tsconfig.schemastore-schema.json tests/ts-node-packed.tgz .tmp", "rebuild": "npm run clean && npm run build", "build": "npm run build-nopack && npm run build-pack", "build-nopack": "npm run build-tsc && npm run build-configSchema", - "build-tsc": "tsc --build ./tsconfig.declarations.json && ( rm -r .tmp/declarations/src 2> /dev/null || true) && tsc --build ./tsconfig.build.json", + "build-tsc": "tsc --build ./tsconfig.declarations.json && rimraf .tmp/declarations/src && tsc --build ./tsconfig.build.json", "build-configSchema": "typescript-json-schema --topRef --refs --validationKeywords allOf --out tsconfig.schema.json tsconfig.build-schema.json TsConfigSchema && node --require ./register ./scripts/create-merged-schema", "build-pack": "node ./scripts/build-pack.js", "test-spec": "ava", diff --git a/tsconfig.build.json b/tsconfig.build.json index a0d8aeac1..b48f2254b 100644 --- a/tsconfig.build.json +++ b/tsconfig.build.json @@ -11,11 +11,11 @@ "noEmit": false, "allowJs": false, "rootDir": "src", + "outDir": "dist", + "tsBuildInfoFile": ".tmp/tsconfig.build.tsbuildinfo", "rootDirs": [ ".", ".tmp/declarations" ], - "outDir": "dist", - "tsBuildInfoFile": ".tmp/tsconfig.build.tsbuildinfo", } } From ce39b0f9bf0dfa2fe46dd17dd7178e9f5a82e808 Mon Sep 17 00:00:00 2001 From: Andrew Bradley Date: Sun, 17 Oct 2021 21:27:28 -0400 Subject: [PATCH 7/7] fix --- dist-raw/node-esm-default-get-format.js | 4 +++- dist-raw/node-esm-resolve-implementation.js | 10 ++++------ src/esm.ts | 10 +++++++--- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/dist-raw/node-esm-default-get-format.js b/dist-raw/node-esm-default-get-format.js index 8f3784bf6..5c2fec59c 100644 --- a/dist-raw/node-esm-default-get-format.js +++ b/dist-raw/node-esm-default-get-format.js @@ -82,7 +82,9 @@ function defaultGetFormat(url, context, defaultGetFormatUnused) { } return { format: null }; } -return {defaultGetFormat}; +return { + defaultGetFormat: /** @type {import('../src/esm').GetFormatHook} */(defaultGetFormat) +}; } exports.createDefaultGetFormat = createDefaultGetFormat; diff --git a/dist-raw/node-esm-resolve-implementation.js b/dist-raw/node-esm-resolve-implementation.js index d89ed4fb9..a0f5c40b6 100644 --- a/dist-raw/node-esm-resolve-implementation.js +++ b/dist-raw/node-esm-resolve-implementation.js @@ -108,12 +108,12 @@ const pendingDeprecation = getOptionValue('--pending-deprecation'); * @param {{ * tsExtensions: string[]; * jsExtensions: string[]; - * preferTsExts: boolean; - * packageJsonReader: ReturnType; + * preferTsExts: boolean | undefined; + * nodePackageJsonReader: ReturnType; * }} opts */ function createResolve(opts) { -const {tsExtensions, jsExtensions, preferTsExts, packageJsonReader} = opts; +const {tsExtensions, jsExtensions, preferTsExts, nodePackageJsonReader: packageJsonReader} = opts; const {defaultGetFormat} = createDefaultGetFormat(getPackageType); const emittedPackageWarnings = new SafeSet(); @@ -985,6 +985,4 @@ return { defaultGetFormat }; } -module.exports = { - createResolve -}; +module.exports.createResolve = createResolve; diff --git a/src/esm.ts b/src/esm.ts index 3964cd294..90c140d56 100644 --- a/src/esm.ts +++ b/src/esm.ts @@ -15,9 +15,7 @@ import { import { extname } from 'path'; import * as assert from 'assert'; import { normalizeSlashes } from './util'; -const { - createResolve, -} = require('../dist-raw/node-esm-resolve-implementation'); +import { createResolve } from '../dist-raw/node-esm-resolve-implementation'; // Note: On Windows, URLs look like this: file:///D:/dev/@TypeStrong/ts-node-examples/foo.ts @@ -35,6 +33,11 @@ const { // from node, build our implementation of the *new* API on top of it, and implement the *old* // hooks API as a shim to the *new* API. +/** @internal */ +export type GetFormatHook = NonNullable< + ReturnType['getFormat'] +>; + /** @internal */ export function registerAndCreateEsmHooks(opts?: RegisterOptions) { // Automatically performs registration just like `-r ts-node/register` @@ -50,6 +53,7 @@ export function createEsmHooks(tsNodeService: Service) { const nodeResolveImplementation = createResolve({ ...getExtensions(tsNodeService.config), preferTsExts: tsNodeService.options.preferTsExts, + nodePackageJsonReader: tsNodeService.nodePackageJsonReader, }); // The hooks API changed in node version X so we need to check for backwards compatibility.