Source Maps: Support 5 segment mappings (#6795)

Support 5-segment source mappings, which add a name.

Reference:
https://github.com/tc39/source-map/blob/main/source-map-rev3.md#proposed-format

Author: osa1

src/ir/module-utils.cpp

@@ -38,23 +38,44 @@ static void updateLocationSet(std::set<Function::DebugLocation>& locations,
   std::swap(locations, updatedLocations);
 }
 
+// Update the symbol name indices when moving a set of debug locations from one
+// module to another.
+static void updateSymbolSet(std::set<Function::DebugLocation>& locations,
+                            std::vector<Index>& symbolIndexMap) {
+  std::set<Function::DebugLocation> updatedLocations;
+
+  for (auto iter : locations) {
+    if (iter.symbolNameIndex) {
+      iter.symbolNameIndex = symbolIndexMap[*iter.symbolNameIndex];
+    }
+    updatedLocations.insert(iter);
+  }
+  locations.clear();
+  std::swap(locations, updatedLocations);
+}
+
 // Copies a function into a module. If newName is provided it is used as the
 // name of the function (otherwise the original name is copied). If fileIndexMap
 // is specified, it is used to rename source map filename indices when copying
+// the function from one module to another one. If symbolNameIndexMap is
+// specified, it is used to rename source map symbol name indices when copying
 // the function from one module to another one.
 Function* copyFunction(Function* func,
                        Module& out,
                        Name newName,
-                       std::optional<std::vector<Index>> fileIndexMap) {
-  auto ret = copyFunctionWithoutAdd(func, out, newName, fileIndexMap);
+                       std::optional<std::vector<Index>> fileIndexMap,
+                       std::optional<std::vector<Index>> symbolNameIndexMap) {
+  auto ret = copyFunctionWithoutAdd(
+    func, out, newName, fileIndexMap, symbolNameIndexMap);
   return out.addFunction(std::move(ret));
 }
 
 std::unique_ptr<Function>
 copyFunctionWithoutAdd(Function* func,
                        Module& out,
                        Name newName,
-                       std::optional<std::vector<Index>> fileIndexMap) {
+                       std::optional<std::vector<Index>> fileIndexMap,
+                       std::optional<std::vector<Index>> symbolNameIndexMap) {
   auto ret = std::make_unique<Function>();
   ret->name = newName.is() ? newName : func->name;
   ret->hasExplicitName = func->hasExplicitName;
@@ -76,6 +97,18 @@ copyFunctionWithoutAdd(Function* func,
     updateLocationSet(ret->prologLocation, *fileIndexMap);
     updateLocationSet(ret->epilogLocation, *fileIndexMap);
   }
+  if (symbolNameIndexMap) {
+    for (auto& iter : ret->debugLocations) {
+      if (iter.second) {
+        if (iter.second->symbolNameIndex.has_value()) {
+          iter.second->symbolNameIndex =
+            (*symbolNameIndexMap)[*(iter.second->symbolNameIndex)];
+        }
+      }
+      updateSymbolSet(ret->prologLocation, *symbolNameIndexMap);
+      updateSymbolSet(ret->epilogLocation, *symbolNameIndexMap);
+    }
+  }
   ret->module = func->module;
   ret->base = func->base;
   ret->noFullInline = func->noFullInline;
@@ -199,8 +232,27 @@ void copyModuleItems(const Module& in, Module& out) {
     }
   }
 
+  std::optional<std::vector<Index>> symbolNameIndexMap;
+  if (!in.debugInfoSymbolNames.empty()) {
+    std::unordered_map<std::string, Index> debugInfoSymbolNameIndices;
+    for (Index i = 0; i < out.debugInfoSymbolNames.size(); i++) {
+      debugInfoSymbolNameIndices[out.debugInfoSymbolNames[i]] = i;
+    }
+    symbolNameIndexMap.emplace();
+    for (Index i = 0; i < in.debugInfoSymbolNames.size(); i++) {
+      std::string file = in.debugInfoSymbolNames[i];
+      auto iter = debugInfoSymbolNameIndices.find(file);
+      if (iter == debugInfoSymbolNameIndices.end()) {
+        Index index = out.debugInfoSymbolNames.size();
+        out.debugInfoSymbolNames.push_back(file);
+        debugInfoSymbolNameIndices[file] = index;
+      }
+      symbolNameIndexMap->push_back(debugInfoSymbolNameIndices[file]);
+    }
+  }
+
   for (auto& curr : in.functions) {
-    copyFunction(curr.get(), out, Name(), fileIndexMap);
+    copyFunction(curr.get(), out, Name(), fileIndexMap, symbolNameIndexMap);
   }
   for (auto& curr : in.globals) {
     copyGlobal(curr.get(), out);
@@ -241,6 +293,7 @@ void copyModule(const Module& in, Module& out) {
   out.start = in.start;
   out.customSections = in.customSections;
   out.debugInfoFileNames = in.debugInfoFileNames;
+  out.debugInfoSymbolNames = in.debugInfoSymbolNames;
   out.features = in.features;
 }
 

src/ir/module-utils.h

@@ -25,21 +25,24 @@
 namespace wasm::ModuleUtils {
 
 // Copies a function into a module. If newName is provided it is used as the
-// name of the function (otherwise the original name is copied). If fileIndexMap
-// is specified, it is used to rename source map filename indices when copying
-// the function from one module to another one.
-Function*
-copyFunction(Function* func,
-             Module& out,
-             Name newName = Name(),
-             std::optional<std::vector<Index>> fileIndexMap = std::nullopt);
+// name of the function (otherwise the original name is copied). When specified,
+// fileIndexMap and symbolNameIndexMap are used to rename source map filename
+// and symbol name indices when copying the function from one module to another
+// one.
+Function* copyFunction(
+  Function* func,
+  Module& out,
+  Name newName = Name(),
+  std::optional<std::vector<Index>> fileIndexMap = std::nullopt,
+  std::optional<std::vector<Index>> symbolNameIndexMap = std::nullopt);
 
 // As above, but does not add the copy to the module.
 std::unique_ptr<Function> copyFunctionWithoutAdd(
   Function* func,
   Module& out,
   Name newName = Name(),
-  std::optional<std::vector<Index>> fileIndexMap = std::nullopt);
+  std::optional<std::vector<Index>> fileIndexMap = std::nullopt,
+  std::optional<std::vector<Index>> symbolNameIndexMap = std::nullopt);
 
 Global* copyGlobal(Global* global, Module& out);
 

src/parser/contexts.h

@@ -1398,6 +1398,7 @@ struct ParseDefsCtx : TypeParserCtx<ParseDefsCtx> {
     typeNames;
   const std::unordered_map<Index, Index>& implicitElemIndices;
 
+  std::unordered_map<std::string_view, Index> debugSymbolNameIndices;
   std::unordered_map<std::string_view, Index> debugFileIndices;
 
   // The index of the current module element.
@@ -1777,12 +1778,32 @@ struct ParseDefsCtx : TypeParserCtx<ParseDefsCtx> {
     }
     contents = contents.substr(lineSize + 1);
 
-    lexer = Lexer(contents);
+    auto colSize = contents.find(':');
+    if (colSize == contents.npos) {
+      colSize = contents.size();
+      if (colSize == 0) {
+        return;
+      }
+    }
+    lexer = Lexer(contents.substr(0, colSize));
     auto col = lexer.takeU32();
-    if (!col || !lexer.empty()) {
+    if (!col) {
       return;
     }
 
+    std::optional<BinaryLocation> symbolNameIndex;
+    if (colSize != contents.size()) {
+      contents = contents.substr(colSize + 1);
+      auto symbolName = contents;
+      auto [it, inserted] = debugSymbolNameIndices.insert(
+        {symbolName, debugSymbolNameIndices.size()});
+      if (inserted) {
+        assert(wasm.debugInfoSymbolNames.size() == it->second);
+        wasm.debugInfoSymbolNames.push_back(std::string(symbolName));
+      }
+      symbolNameIndex = it->second;
+    }
+
     // TODO: If we ever parallelize the parse, access to
     // `wasm.debugInfoFileNames` will have to be protected by a lock.
     auto [it, inserted] =
@@ -1792,7 +1813,7 @@ struct ParseDefsCtx : TypeParserCtx<ParseDefsCtx> {
       wasm.debugInfoFileNames.push_back(std::string(file));
     }
     irBuilder.setDebugLocation(
-      Function::DebugLocation({it->second, *line, *col}));
+      Function::DebugLocation({it->second, *line, *col, symbolNameIndex}));
   }
 
   Result<> makeBlock(Index pos,

src/passes/Print.cpp

@@ -2516,7 +2516,15 @@ void PrintSExpression::printDebugLocation(
   } else {
     auto fileName = currModule->debugInfoFileNames[location->fileIndex];
     o << ";;@ " << fileName << ":" << location->lineNumber << ":"
-      << location->columnNumber << '\n';
+      << location->columnNumber;
+
+    if (location->symbolNameIndex) {
+      auto symbolName =
+        currModule->debugInfoSymbolNames[*(location->symbolNameIndex)];
+      o << ":" << symbolName;
+    }
+
+    o << '\n';
   }
   doIndent(o, indent);
 }

src/wasm-binary.h

@@ -1674,6 +1674,7 @@ class WasmBinaryReader {
   // Debug information reading helpers
   void setDebugLocations(std::istream* sourceMap_) { sourceMap = sourceMap_; }
   std::unordered_map<std::string, Index> debugInfoFileIndices;
+  std::unordered_map<std::string, Index> debugInfoSymbolNameIndices;
   void readNextDebugLocation();
   void readSourceMapHeader();
 

src/wasm.h

@@ -2073,22 +2073,25 @@ class Function : public Importable {
   std::unordered_map<Index, Name> localNames;
   std::unordered_map<Name, Index> localIndices;
 
-  // Source maps debugging info: map expression nodes to their file, line, col.
+  // Source maps debugging info: map expression nodes to their file, line, col,
+  // symbol name.
   struct DebugLocation {
     BinaryLocation fileIndex, lineNumber, columnNumber;
+    std::optional<BinaryLocation> symbolNameIndex;
     bool operator==(const DebugLocation& other) const {
       return fileIndex == other.fileIndex && lineNumber == other.lineNumber &&
-             columnNumber == other.columnNumber;
+             columnNumber == other.columnNumber &&
+             symbolNameIndex == other.symbolNameIndex;
     }
     bool operator!=(const DebugLocation& other) const {
       return !(*this == other);
     }
     bool operator<(const DebugLocation& other) const {
-      return fileIndex != other.fileIndex
-               ? fileIndex < other.fileIndex
-               : lineNumber != other.lineNumber
-                   ? lineNumber < other.lineNumber
-                   : columnNumber < other.columnNumber;
+      return fileIndex != other.fileIndex     ? fileIndex < other.fileIndex
+             : lineNumber != other.lineNumber ? lineNumber < other.lineNumber
+             : columnNumber != other.columnNumber
+               ? columnNumber < other.columnNumber
+               : symbolNameIndex < other.symbolNameIndex;
     }
   };
   // One can explicitly set the debug location of an expression to
@@ -2300,6 +2303,7 @@ class Module {
 
   // Source maps debug info.
   std::vector<std::string> debugInfoFileNames;
+  std::vector<std::string> debugInfoSymbolNames;
 
   // `features` are the features allowed to be used in this module and should be
   // respected regardless of the value of`hasFeaturesSection`.

src/wasm/wasm-binary.cpp

@@ -1189,7 +1189,7 @@ void WasmBinaryWriter::writeSymbolMap() {
 }
 
 void WasmBinaryWriter::initializeDebugInfo() {
-  lastDebugLocation = {0, /* lineNumber = */ 1, 0};
+  lastDebugLocation = {0, /* lineNumber = */ 1, 0, std::nullopt};
 }
 
 void WasmBinaryWriter::writeSourceMapProlog() {
@@ -1225,7 +1225,17 @@ void WasmBinaryWriter::writeSourceMapProlog() {
     // TODO respect JSON string encoding, e.g. quotes and control chars.
     *sourceMap << "\"" << wasm->debugInfoFileNames[i] << "\"";
   }
-  *sourceMap << "],\"names\":[],\"mappings\":\"";
+  *sourceMap << "],\"names\":[";
+
+  for (size_t i = 0; i < wasm->debugInfoSymbolNames.size(); i++) {
+    if (i > 0) {
+      *sourceMap << ",";
+    }
+    // TODO respect JSON string encoding, e.g. quotes and control chars.
+    *sourceMap << "\"" << wasm->debugInfoSymbolNames[i] << "\"";
+  }
+
+  *sourceMap << "],\"mappings\":\"";
 }
 
 static void writeBase64VLQ(std::ostream& out, int32_t n) {
@@ -1249,21 +1259,31 @@ static void writeBase64VLQ(std::ostream& out, int32_t n) {
 void WasmBinaryWriter::writeSourceMapEpilog() {
   // write source map entries
   size_t lastOffset = 0;
-  Function::DebugLocation lastLoc = {0, /* lineNumber = */ 1, 0};
+  BinaryLocation lastFileIndex = 0;
+  BinaryLocation lastLineNumber = 1;
+  BinaryLocation lastColumnNumber = 0;
+  BinaryLocation lastSymbolNameIndex = 0;
   for (const auto& [offset, loc] : sourceMapLocations) {
     if (lastOffset > 0) {
       *sourceMap << ",";
     }
     writeBase64VLQ(*sourceMap, int32_t(offset - lastOffset));
     lastOffset = offset;
     if (loc) {
-      // There is debug information for this location, so emit the next 3
-      // fields and update lastLoc.
-      writeBase64VLQ(*sourceMap, int32_t(loc->fileIndex - lastLoc.fileIndex));
-      writeBase64VLQ(*sourceMap, int32_t(loc->lineNumber - lastLoc.lineNumber));
-      writeBase64VLQ(*sourceMap,
-                     int32_t(loc->columnNumber - lastLoc.columnNumber));
-      lastLoc = *loc;
+      writeBase64VLQ(*sourceMap, int32_t(loc->fileIndex - lastFileIndex));
+      lastFileIndex = loc->fileIndex;
+
+      writeBase64VLQ(*sourceMap, int32_t(loc->lineNumber - lastLineNumber));
+      lastLineNumber = loc->lineNumber;
+
+      writeBase64VLQ(*sourceMap, int32_t(loc->columnNumber - lastColumnNumber));
+      lastColumnNumber = loc->columnNumber;
+
+      if (loc->symbolNameIndex) {
+        writeBase64VLQ(*sourceMap,
+                       int32_t(*loc->symbolNameIndex - lastSymbolNameIndex));
+        lastSymbolNameIndex = *loc->symbolNameIndex;
+      }
     }
   }
   *sourceMap << "\"}";
@@ -1716,7 +1736,7 @@ WasmBinaryReader::WasmBinaryReader(Module& wasm,
                                    FeatureSet features,
                                    const std::vector<char>& input)
   : wasm(wasm), allocator(wasm.allocator), input(input), sourceMap(nullptr),
-    nextDebugPos(0), nextDebugLocation{0, 0, 0},
+    nextDebugPos(0), nextDebugLocation{0, 0, 0, std::nullopt},
     nextDebugLocationHasDebugInfo(false), debugLocation() {
   wasm.features = features;
 }
@@ -2885,6 +2905,21 @@ void WasmBinaryReader::readSourceMapHeader() {
     mustReadChar(']');
   }
 
+  if (findField("names")) {
+    skipWhitespace();
+    mustReadChar('[');
+    if (!maybeReadChar(']')) {
+      do {
+        std::string symbol;
+        readString(symbol);
+        Index index = wasm.debugInfoSymbolNames.size();
+        wasm.debugInfoSymbolNames.push_back(symbol);
+        debugInfoSymbolNameIndices[symbol] = index;
+      } while (maybeReadChar(','));
+      mustReadChar(']');
+    }
+  }
+
   if (!findField("mappings")) {
     throw MapParseException("cannot find the 'mappings' field in map");
   }
@@ -2911,7 +2946,12 @@ void WasmBinaryReader::readSourceMapHeader() {
     uint32_t lineNumber =
       readBase64VLQ(*sourceMap) + 1; // adjust zero-based line number
     uint32_t columnNumber = readBase64VLQ(*sourceMap);
-    nextDebugLocation = {fileIndex, lineNumber, columnNumber};
+    std::optional<BinaryLocation> symbolNameIndex;
+    peek = sourceMap->peek();
+    if (!(peek == ',' || peek == '\"')) {
+      symbolNameIndex = readBase64VLQ(*sourceMap);
+    }
+    nextDebugLocation = {fileIndex, lineNumber, columnNumber, symbolNameIndex};
     nextDebugLocationHasDebugInfo = true;
   }
 }
@@ -2966,7 +3006,15 @@ void WasmBinaryReader::readNextDebugLocation() {
     int32_t columnNumberDelta = readBase64VLQ(*sourceMap);
     uint32_t columnNumber = nextDebugLocation.columnNumber + columnNumberDelta;
 
-    nextDebugLocation = {fileIndex, lineNumber, columnNumber};
+    std::optional<BinaryLocation> symbolNameIndex;
+    peek = sourceMap->peek();
+    if (!(peek == ',' || peek == '\"')) {
+      int32_t symbolNameIndexDelta = readBase64VLQ(*sourceMap);
+      symbolNameIndex =
+        nextDebugLocation.symbolNameIndex.value_or(0) + symbolNameIndexDelta;
+    }
+
+    nextDebugLocation = {fileIndex, lineNumber, columnNumber, symbolNameIndex};
     nextDebugLocationHasDebugInfo = true;
   }
 }

src/wasm/wasm.cpp

@@ -1863,6 +1863,9 @@ void Module::updateMaps() {
   assert(tagsMap.size() == tags.size());
 }
 
-void Module::clearDebugInfo() { debugInfoFileNames.clear(); }
+void Module::clearDebugInfo() {
+  debugInfoFileNames.clear();
+  debugInfoSymbolNames.clear();
+}
 
 } // namespace wasm