Skip to content

Experiment with Uint8Array to reduce memory consumption of sourcemap generation. #43987

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 11 commits into from
Closed
2 changes: 2 additions & 0 deletions src/compiler/emitter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -532,12 +532,14 @@ namespace ts {
const bundle = sourceFileOrBundle.kind === SyntaxKind.Bundle ? sourceFileOrBundle : undefined;
const sourceFile = sourceFileOrBundle.kind === SyntaxKind.SourceFile ? sourceFileOrBundle : undefined;
const sourceFiles = bundle ? bundle.sourceFiles : [sourceFile!];
const guessedLength = sourceFile?.text.length ?? reduceLeft(sourceFiles, (acc, file) => acc + file.text.length, 0);

let sourceMapGenerator: SourceMapGenerator | undefined;
if (shouldEmitSourceMaps(mapOptions, sourceFileOrBundle)) {
sourceMapGenerator = createSourceMapGenerator(
host,
getBaseFileName(normalizeSlashes(jsFilePath)),
guessedLength,
getSourceRoot(mapOptions),
getSourceMapDirectory(mapOptions, jsFilePath, sourceFile),
mapOptions);
Expand Down
94 changes: 57 additions & 37 deletions src/compiler/sourcemap.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,17 @@ namespace ts {
extendedDiagnostics?: boolean;
}

export function createSourceMapGenerator(host: EmitHost, file: string, sourceRoot: string, sourcesDirectoryPath: string, generatorOptions: SourceMapGeneratorOptions): SourceMapGenerator {
declare let TextDecoder: undefined | (new() => { decode(buffer: ArrayBuffer | ArrayBufferView): string });
const decoder = typeof Buffer === "undefined" ?
new (typeof TextDecoder !== "undefined" ? TextDecoder : require("util").TextDecoder)("ascii") :
undefined;
const decode: (buffer: ArrayBuffer) => string =
decoder
? buffer => decoder.decode(buffer)
: buffer => (buffer as Buffer).toString("ascii");
let mappingsBuffer: Uint8Array;

export function createSourceMapGenerator(host: EmitHost, file: string, guessedInputLength: number, sourceRoot: string, sourcesDirectoryPath: string, generatorOptions: SourceMapGeneratorOptions): SourceMapGenerator {
const { enter, exit } = generatorOptions.extendedDiagnostics
? performance.createTimer("Source Map", "beforeSourcemap", "afterSourcemap")
: performance.nullTimer;
Expand All @@ -17,7 +27,44 @@ namespace ts {

const names: string[] = [];
let nameToNameIndexMap: ESMap<string, number> | undefined;
let mappings = "";
mappingsBuffer ||= typeof Buffer === undefined ? new Uint8Array(guessedInputLength + 1 >> 1) : Buffer.alloc(guessedInputLength + 1 >> 1);
let lastMappings: string | undefined;
let mappingsPos = 0;
function setMapping(charCode: number) {
// resize to 1.5 * length
if (mappingsPos >= mappingsBuffer.length) {
const oldLength = mappingsBuffer.length + 1;
const replacementBuffer = new Uint8Array(oldLength + ((oldLength + 1) >> 1));
replacementBuffer.set(mappingsBuffer);
mappingsBuffer = replacementBuffer;
}
mappingsBuffer[mappingsPos++] = charCode;
}

function base64VLQFormatEncode(inValue: number) {
// Add a new least significant bit that has the sign of the value.
// if negative number the least significant bit that gets added to the number has value 1
// else least significant bit value that gets added is 0
// eg. -1 changes to binary : 01 [1] => 3
// +1 changes to binary : 01 [0] => 2
if (inValue < 0) {
inValue = ((-inValue) << 1) + 1;
}
else {
inValue = inValue << 1;
}

// Encode 5 bits at a time starting from least significant bits
do {
let currentDigit = inValue & 31; // 11111
inValue = inValue >> 5;
if (inValue > 0) {
// There are still more digits to decode, set the msb (6th bit)
currentDigit = currentDigit | 32;
}
setMapping(base64FormatEncode(currentDigit));
} while (inValue > 0);
}

// Last recorded and encoded mappings
let lastGeneratedLine = 0;
Expand Down Expand Up @@ -221,7 +268,7 @@ namespace ts {
if (lastGeneratedLine < pendingGeneratedLine) {
// Emit line delimiters
do {
mappings += ";";
setMapping(CharacterCodes.semicolon);
lastGeneratedLine++;
lastGeneratedCharacter = 0;
}
Expand All @@ -231,30 +278,30 @@ namespace ts {
Debug.assertEqual(lastGeneratedLine, pendingGeneratedLine, "generatedLine cannot backtrack");
// Emit comma to separate the entry
if (hasLast) {
mappings += ",";
setMapping(CharacterCodes.comma);
}
}

// 1. Relative generated character
mappings += base64VLQFormatEncode(pendingGeneratedCharacter - lastGeneratedCharacter);
base64VLQFormatEncode(pendingGeneratedCharacter - lastGeneratedCharacter);
lastGeneratedCharacter = pendingGeneratedCharacter;

if (hasPendingSource) {
// 2. Relative sourceIndex
mappings += base64VLQFormatEncode(pendingSourceIndex - lastSourceIndex);
base64VLQFormatEncode(pendingSourceIndex - lastSourceIndex);
lastSourceIndex = pendingSourceIndex;

// 3. Relative source line
mappings += base64VLQFormatEncode(pendingSourceLine - lastSourceLine);
base64VLQFormatEncode(pendingSourceLine - lastSourceLine);
lastSourceLine = pendingSourceLine;

// 4. Relative source character
mappings += base64VLQFormatEncode(pendingSourceCharacter - lastSourceCharacter);
base64VLQFormatEncode(pendingSourceCharacter - lastSourceCharacter);
lastSourceCharacter = pendingSourceCharacter;

if (hasPendingName) {
// 5. Relative nameIndex
mappings += base64VLQFormatEncode(pendingNameIndex - lastNameIndex);
base64VLQFormatEncode(pendingNameIndex - lastNameIndex);
lastNameIndex = pendingNameIndex;
}
}
Expand All @@ -265,6 +312,7 @@ namespace ts {

function toJSON(): RawSourceMap {
commitPendingMapping();
const mappings = (lastMappings ??= decode(mappingsBuffer.subarray(0, mappingsPos)));
return {
version: 3,
file,
Expand Down Expand Up @@ -544,34 +592,6 @@ namespace ts {
-1;
}

function base64VLQFormatEncode(inValue: number) {
// Add a new least significant bit that has the sign of the value.
// if negative number the least significant bit that gets added to the number has value 1
// else least significant bit value that gets added is 0
// eg. -1 changes to binary : 01 [1] => 3
// +1 changes to binary : 01 [0] => 2
if (inValue < 0) {
inValue = ((-inValue) << 1) + 1;
}
else {
inValue = inValue << 1;
}

// Encode 5 bits at a time starting from least significant bits
let encodedStr = "";
do {
let currentDigit = inValue & 31; // 11111
inValue = inValue >> 5;
if (inValue > 0) {
// There are still more digits to decode, set the msb (6th bit)
currentDigit = currentDigit | 32;
}
encodedStr = encodedStr + String.fromCharCode(base64FormatEncode(currentDigit));
} while (inValue > 0);

return encodedStr;
}

interface MappedPosition {
generatedPosition: number;
source: string | undefined;
Expand Down