build: add ts support in core modules

Author: marco-ippolito

configure.py

@@ -1765,7 +1765,8 @@ def gcc_version_ge(version_checked):
   return True
 
 def configure_node_lib_files(o):
-  o['variables']['node_library_files'] = SearchFiles('lib', 'js')
+  o['variables']['node_library_files'] = SearchFiles('lib', 'js') + \
+    SearchFiles('lib', 'ts')
 
 def configure_node_cctest_sources(o):
   o['variables']['node_cctest_sources'] = [ 'src/node_snapshot_stub.cc' ] + \

node.gyp

@@ -1557,12 +1557,27 @@
       'target_name': 'node_js2c',
       'type': 'executable',
       'toolsets': ['host'],
+      'dependencies': [
+        'tools/v8_gypfiles/v8.gyp:v8_base_without_compiler',
+        'tools/v8_gypfiles/v8.gyp:v8_compiler_for_mksnapshot',
+        'tools/v8_gypfiles/v8.gyp:v8_init',
+        'tools/v8_gypfiles/v8.gyp:v8_snapshot',
+        'tools/v8_gypfiles/v8.gyp:v8_libbase',
+        'tools/v8_gypfiles/v8.gyp:v8_libplatform',
+        'tools/v8_gypfiles/v8.gyp:v8_maybe_icu',
+        'tools/v8_gypfiles/v8.gyp:fp16',
+        'tools/v8_gypfiles/abseil.gyp:abseil',
+      ],
       'include_dirs': [
+        'deps/v8',
+        'deps/v8/include',
         'tools',
         'src',
       ],
       'sources': [
         'tools/js2c.cc',
+        'tools/typescript_transpiler.cc',
+        'tools/typescript_transpiler.h',
         'tools/executable_wrapper.h',
         'src/embedded_data.h',
         'src/embedded_data.cc',

src/node_builtins.cc

@@ -178,10 +178,12 @@ BuiltinLoader::BuiltinCategories BuiltinLoader::GetBuiltinCategories() const {
 }
 
 #ifdef NODE_BUILTIN_MODULES_PATH
-static std::string OnDiskFileName(const char* id) {
+std::string BuiltinLoader::OnDiskFileName(const char* id) const {
   std::string filename = NODE_BUILTIN_MODULES_PATH;
   filename += "/";
 
+  // Save the original id for source map lookup before any pointer arithmetic.
+  const char* original_id = id;
   if (strncmp(id, "internal/deps", strlen("internal/deps")) == 0) {
     id += strlen("internal/");
   } else {
@@ -192,7 +194,15 @@ static std::string OnDiskFileName(const char* id) {
     // V8 tools scripts are .mjs files.
     filename += ".mjs";
   } else {
-    filename += ".js";
+    // Use the pre-built source map to determine the file extension.
+    // This avoids a filesystem probe on every module load in dev mode.
+    auto source = source_.read();
+    auto it = source->find(original_id);
+    if (it != source->end() && it->second.is_typescript) {
+      filename += ".ts";
+    } else {
+      filename += ".js";
+    }
   }
 
   return filename;

src/node_builtins.h

@@ -73,6 +73,7 @@ struct BuiltinSource {
   std::string id;
   UnionBytes source;
   BuiltinSourceType type;
+  bool is_typescript = false;
 };
 
 using BuiltinSourceMap = std::map<std::string, BuiltinSource>;
@@ -203,6 +204,13 @@ class NODE_EXTERN_PRIVATE BuiltinLoader {
   const BuiltinSource* AddExternalizedBuiltin(const char* id,
                                               const char* filename);
 
+#ifdef NODE_BUILTIN_MODULES_PATH
+  // Returns the on-disk path for a builtin module id.
+  // Uses the pre-built source map to determine the file extension rather
+  // than probing the filesystem.
+  std::string OnDiskFileName(const char* id) const;
+#endif
+
   ThreadsafeCopyOnWrite<BuiltinSourceMap> source_;
 
   const UnionBytes config_;

tools/js2c.cc

@@ -8,11 +8,13 @@
 #include <map>
 #include <string>
 #include <string_view>
+#include <unordered_set>
 #include <vector>
 #include "builtin_info.h"
 #include "embedded_data.h"
 #include "executable_wrapper.h"
 #include "simdutf.h"
+#include "typescript_transpiler.h"
 #include "uv.h"
 
 #if defined(_WIN32)
@@ -131,13 +133,14 @@ bool SearchFiles(const std::string& dir,
 
 constexpr std::string_view kMjsSuffix = ".mjs";
 constexpr std::string_view kJsSuffix = ".js";
+constexpr std::string_view kTsSuffix = ".ts";
 constexpr std::string_view kGypiSuffix = ".gypi";
 constexpr std::string_view depsPrefix = "deps/";
 constexpr std::string_view libPrefix = "lib/";
 
 constexpr std::string_view HasAllowedExtensions(
     const std::string_view filename) {
-  for (const auto& ext : {kGypiSuffix, kJsSuffix, kMjsSuffix}) {
+  for (const auto& ext : {kGypiSuffix, kJsSuffix, kMjsSuffix, kTsSuffix}) {
     if (filename.ends_with(ext)) {
       return ext;
     }
@@ -329,11 +332,13 @@ std::string GetFileId(const std::string& filename) {
   size_t end = filename.size();
   size_t start = 0;
   std::string prefix;
-  // Strip .mjs and .js suffix
+  // Strip .mjs, .js and .ts suffix
   if (filename.ends_with(kMjsSuffix)) {
     end -= kMjsSuffix.size();
   } else if (filename.ends_with(kJsSuffix)) {
     end -= kJsSuffix.size();
+  } else if (filename.ends_with(kTsSuffix)) {
+    end -= kTsSuffix.size();
   }
 
   // deps/acorn/acorn/dist/acorn.js -> internal/deps/acorn/acorn/dist/acorn
@@ -670,6 +675,7 @@ Fragment GetDefinition(const std::string& var, const std::vector<char>& code) {
 }
 
 int AddModule(const std::string& filename,
+              TypeScriptTranspiler* transpiler,
               Fragments* definitions,
               Fragments* initializers,
               Fragments* registrations) {
@@ -684,6 +690,21 @@ int AddModule(const std::string& filename,
   if (error != 0) {
     return error;
   }
+
+  if (filename.ends_with(kTsSuffix)) {
+    std::vector<char> transpiled;
+    if (transpiler->Strip(std::string_view(code.data(), code.size()),
+                          filename,
+                          &transpiled) != 0) {
+      fprintf(stderr,
+              "Failed to transpile TypeScript file %s: %s\n",
+              filename.c_str(),
+              std::string(transpiler->LastError()).c_str());
+      return 1;
+    }
+    code = std::move(transpiled);
+  }
+
   std::string file_id = GetFileId(filename);
   std::string var = GetVariableName(file_id);
 
@@ -696,18 +717,20 @@ int AddModule(const std::string& filename,
   // {"internal/deps/v8/tools/tickprocessor-driver",
   //  BuiltinSource{UnionBytes(&fs_resource),
   //                BuiltinSourceType::kSourceTextModule}},
-  Fragment& init_buf = initializers->emplace_back(Fragment(512, 0));
+  Fragment& init_buf = initializers->emplace_back(Fragment(640, 0));
   int init_size = snprintf(init_buf.data(),
                            init_buf.size(),
                            "    {\"%s\","
                            " BuiltinSource{"
                            " \"%s\","
                            " UnionBytes(&%s_resource),"
-                           " BuiltinSourceType::%s} },",
+                           " BuiltinSourceType::%s,"
+                           " %s} },",
                            file_id.c_str(),
                            file_id.c_str(),
                            var.c_str(),
-                           source_type.c_str());
+                           source_type.c_str(),
+                           filename.ends_with(kTsSuffix) ? "true" : "false");
   init_buf.resize(init_size);
 
   // Registrations:
@@ -826,23 +849,39 @@ int AddGypi(const std::string& var,
 
 int JS2C(const FileList& js_files,
          const FileList& mjs_files,
+         const FileList& ts_files,
          const std::string& config,
+         const char* argv0,
          const std::string& dest) {
+  TypeScriptTranspiler transpiler;
   Fragments definitions;
-  definitions.reserve(js_files.size() + mjs_files.size() + 1);
+  definitions.reserve(js_files.size() + mjs_files.size() + ts_files.size() + 1);
   Fragments initializers;
-  initializers.reserve(js_files.size() + mjs_files.size());
+  initializers.reserve(js_files.size() + mjs_files.size() + ts_files.size());
   Fragments registrations;
-  registrations.reserve(js_files.size() + mjs_files.size() + 1);
+  registrations.reserve(js_files.size() + mjs_files.size() + ts_files.size() +
+                        1);
 
-  for (const auto& filename : js_files) {
-    int r = AddModule(filename, &definitions, &initializers, &registrations);
-    if (r != 0) {
-      return r;
-    }
+  if (!ts_files.empty() && transpiler.Initialize(argv0) != 0) {
+    fprintf(stderr,
+            "Failed to initialize TypeScript transpiler: %s\n",
+            std::string(transpiler.LastError()).c_str());
+    return 1;
   }
-  for (const auto& filename : mjs_files) {
-    int r = AddModule(filename, &definitions, &initializers, &registrations);
+
+  auto add_modules = [&](const FileList& files) {
+    for (const auto& filename : files) {
+      int r = AddModule(
+          filename, &transpiler, &definitions, &initializers, &registrations);
+      if (r != 0) {
+        return r;
+      }
+    }
+    return 0;
+  };
+
+  for (const auto* files : {&js_files, &mjs_files, &ts_files}) {
+    int r = add_modules(*files);
     if (r != 0) {
       return r;
     }
@@ -910,7 +949,8 @@ int Main(int argc, char* argv[]) {
     const std::string& file = args[i];
     if (IsDirectory(file, &error)) {
       if (!SearchFiles(file, &file_map, kJsSuffix) ||
-          !SearchFiles(file, &file_map, kMjsSuffix)) {
+          !SearchFiles(file, &file_map, kMjsSuffix) ||
+          !SearchFiles(file, &file_map, kTsSuffix)) {
         return 1;
       }
     } else if (error != 0) {
@@ -927,8 +967,15 @@ int Main(int argc, char* argv[]) {
     }
   }
 
-  // Should have exactly 3 types: `.js`, `.mjs` and `.gypi`.
-  assert(file_map.size() == 3);
+  // Should have at most 4 extension types: `.js`, `.mjs`, `.ts` and `.gypi`.
+  // Any other extension indicates an unexpected file slipped into the inputs.
+  if (file_map.size() > 4) {
+    fprintf(stderr,
+            "Unexpected file types in inputs (expected .js, .mjs, .ts, "
+            ".gypi only)\n");
+    return 1;
+  }
+
   auto gypi_it = file_map.find(".gypi");
   // Currently config.gypi is the only `.gypi` file allowed
   if (gypi_it == file_map.end() || gypi_it->second.size() != 1 ||
@@ -940,19 +987,55 @@ int Main(int argc, char* argv[]) {
   }
   auto js_it = file_map.find(".js");
   auto mjs_it = file_map.find(".mjs");
-  assert(js_it != file_map.end() && mjs_it != file_map.end());
+  auto ts_it = file_map.find(".ts");
+  if (mjs_it != file_map.end()) {
+    auto it = std::find(mjs_it->second.begin(),
+                        mjs_it->second.end(),
+                        "lib/eslint.config_partial.mjs");
+    if (it != mjs_it->second.end()) {
+      mjs_it->second.erase(it);
+    }
+  }
 
-  auto it = std::find(mjs_it->second.begin(),
-                      mjs_it->second.end(),
-                      "lib/eslint.config_partial.mjs");
-  if (it != mjs_it->second.end()) {
-    mjs_it->second.erase(it);
+  if (js_it != file_map.end()) {
+    std::sort(js_it->second.begin(), js_it->second.end());
+  }
+  if (mjs_it != file_map.end()) {
+    std::sort(mjs_it->second.begin(), mjs_it->second.end());
+  }
+  if (ts_it != file_map.end()) {
+    std::sort(ts_it->second.begin(), ts_it->second.end());
   }
 
-  std::sort(js_it->second.begin(), js_it->second.end());
-  std::sort(mjs_it->second.begin(), mjs_it->second.end());
+  static const FileList empty_list;
+  const FileList& js_files =
+      js_it == file_map.end() ? empty_list : js_it->second;
+  const FileList& mjs_files =
+      mjs_it == file_map.end() ? empty_list : mjs_it->second;
+  const FileList& ts_files =
+      ts_it == file_map.end() ? empty_list : ts_it->second;
+
+  // Detect duplicate module IDs across .js/.mjs and .ts file lists.
+  // GetFileId strips the extension, so lib/foo.ts and lib/foo.js would both
+  // resolve to the same ID "foo", causing a silent registration conflict.
+  if (!ts_files.empty()) {
+    std::unordered_set<std::string> known_ids;
+    for (const auto& f : js_files) known_ids.insert(GetFileId(f));
+    for (const auto& f : mjs_files) known_ids.insert(GetFileId(f));
+    for (const auto& f : ts_files) {
+      std::string id = GetFileId(f);
+      if (known_ids.count(id) != 0) {
+        fprintf(stderr,
+                "Duplicate module ID '%s': a .ts file and a .js/.mjs file "
+                "both resolve to the same module ID\n",
+                id.c_str());
+        return 1;
+      }
+    }
+  }
 
-  return JS2C(js_it->second, mjs_it->second, gypi_it->second[0], output);
+  return JS2C(
+      js_files, mjs_files, ts_files, gypi_it->second[0], argv[0], output);
 }
 }  // namespace js2c
 }  // namespace node