From 21e8ac084bc15d48a31e031f702aaf02ab5fd713 Mon Sep 17 00:00:00 2001 From: David Michon Date: Mon, 10 May 2021 16:58:28 -0700 Subject: [PATCH 1/3] Test normal char code array for source mappings --- src/compiler/sourcemap.ts | 76 +++++++++++++++++++++------------------ 1 file changed, 41 insertions(+), 35 deletions(-) diff --git a/src/compiler/sourcemap.ts b/src/compiler/sourcemap.ts index 0db89a5830fd2..3a25403e9cf64 100644 --- a/src/compiler/sourcemap.ts +++ b/src/compiler/sourcemap.ts @@ -17,6 +17,7 @@ namespace ts { const names: string[] = []; let nameToNameIndexMap: ESMap | undefined; + const mappingCharCodes: number[] = []; let mappings = ""; // Last recorded and encoded mappings @@ -221,7 +222,7 @@ namespace ts { if (lastGeneratedLine < pendingGeneratedLine) { // Emit line delimiters do { - mappings += ";"; + mappingCharCodes.push(CharacterCodes.semicolon); // ';' lastGeneratedLine++; lastGeneratedCharacter = 0; } @@ -231,30 +232,30 @@ namespace ts { Debug.assertEqual(lastGeneratedLine, pendingGeneratedLine, "generatedLine cannot backtrack"); // Emit comma to separate the entry if (hasLast) { - mappings += ","; + mappingCharCodes.push(CharacterCodes.comma); // ',' } } // 1. Relative generated character - mappings += base64VLQFormatEncode(pendingGeneratedCharacter - lastGeneratedCharacter); + appendBase64VLQ(pendingGeneratedCharacter - lastGeneratedCharacter); lastGeneratedCharacter = pendingGeneratedCharacter; if (hasPendingSource) { // 2. Relative sourceIndex - mappings += base64VLQFormatEncode(pendingSourceIndex - lastSourceIndex); + appendBase64VLQ(pendingSourceIndex - lastSourceIndex); lastSourceIndex = pendingSourceIndex; // 3. Relative source line - mappings += base64VLQFormatEncode(pendingSourceLine - lastSourceLine); + appendBase64VLQ(pendingSourceLine - lastSourceLine); lastSourceLine = pendingSourceLine; // 4. Relative source character - mappings += base64VLQFormatEncode(pendingSourceCharacter - lastSourceCharacter); + appendBase64VLQ(pendingSourceCharacter - lastSourceCharacter); lastSourceCharacter = pendingSourceCharacter; if (hasPendingName) { // 5. Relative nameIndex - mappings += base64VLQFormatEncode(pendingNameIndex - lastNameIndex); + appendBase64VLQ(pendingNameIndex - lastNameIndex); lastNameIndex = pendingNameIndex; } } @@ -263,8 +264,16 @@ namespace ts { exit(); } + function serializeMappings(): void { + for (let i = 0, len = mappingCharCodes.length; i < len; i += 1024) { + mappings += String.fromCharCode.apply(undefined, mappingCharCodes.slice(i, i + 1024)); + } + mappingCharCodes.length = 0; + } + function toJSON(): RawSourceMap { commitPendingMapping(); + serializeMappings(); return { version: 3, file, @@ -275,6 +284,31 @@ namespace ts { sourcesContent, }; } + + function appendBase64VLQ(inValue: number): void { + // Add a new least significant bit that has the sign of the value. + // if negative number the least significant bit that gets added to the number has value 1 + // else least significant bit value that gets added is 0 + // eg. -1 changes to binary : 01 [1] => 3 + // +1 changes to binary : 01 [0] => 2 + if (inValue < 0) { + inValue = ((-inValue) << 1) + 1; + } + else { + inValue = inValue << 1; + } + + // Encode 5 bits at a time starting from least significant bits + do { + let currentDigit = inValue & 31; // 11111 + inValue = inValue >> 5; + if (inValue > 0) { + // There are still more digits to decode, set the msb (6th bit) + currentDigit = currentDigit | 32; + } + mappingCharCodes.push(base64FormatEncode(currentDigit)); + } while (inValue > 0); + } } // Sometimes tools can see the following line as a source mapping url comment, so we mangle it a bit (the [M]) @@ -544,34 +578,6 @@ namespace ts { -1; } - function base64VLQFormatEncode(inValue: number) { - // Add a new least significant bit that has the sign of the value. - // if negative number the least significant bit that gets added to the number has value 1 - // else least significant bit value that gets added is 0 - // eg. -1 changes to binary : 01 [1] => 3 - // +1 changes to binary : 01 [0] => 2 - if (inValue < 0) { - inValue = ((-inValue) << 1) + 1; - } - else { - inValue = inValue << 1; - } - - // Encode 5 bits at a time starting from least significant bits - let encodedStr = ""; - do { - let currentDigit = inValue & 31; // 11111 - inValue = inValue >> 5; - if (inValue > 0) { - // There are still more digits to decode, set the msb (6th bit) - currentDigit = currentDigit | 32; - } - encodedStr = encodedStr + String.fromCharCode(base64FormatEncode(currentDigit)); - } while (inValue > 0); - - return encodedStr; - } - interface MappedPosition { generatedPosition: number; source: string | undefined; From 228f3eb80784e81a7ecd008d0143eb67008056c4 Mon Sep 17 00:00:00 2001 From: David Michon Date: Mon, 10 May 2021 17:47:01 -0700 Subject: [PATCH 2/3] Limit buffer size, minor performance tweaks --- src/compiler/sourcemap.ts | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/src/compiler/sourcemap.ts b/src/compiler/sourcemap.ts index 3a25403e9cf64..e79067387118e 100644 --- a/src/compiler/sourcemap.ts +++ b/src/compiler/sourcemap.ts @@ -18,6 +18,8 @@ namespace ts { const names: string[] = []; let nameToNameIndexMap: ESMap | undefined; const mappingCharCodes: number[] = []; + // We will create a string from the char code buffer whenever it exceeds this length + const mappingCommitThreshold = 1000; let mappings = ""; // Last recorded and encoded mappings @@ -221,12 +223,14 @@ namespace ts { // Line/Comma delimiters if (lastGeneratedLine < pendingGeneratedLine) { // Emit line delimiters + // This loop can potentially overflow the stack on the char code conversion if it were a single operation do { mappingCharCodes.push(CharacterCodes.semicolon); // ';' lastGeneratedLine++; - lastGeneratedCharacter = 0; } while (lastGeneratedLine < pendingGeneratedLine); + // Only need to set this once + lastGeneratedCharacter = 0; } else { Debug.assertEqual(lastGeneratedLine, pendingGeneratedLine, "generatedLine cannot backtrack"); @@ -260,20 +264,29 @@ namespace ts { } } + if (mappings.length > mappingCommitThreshold) { + flushMappingBuffer(); + } + hasLast = true; exit(); } - function serializeMappings(): void { - for (let i = 0, len = mappingCharCodes.length; i < len; i += 1024) { - mappings += String.fromCharCode.apply(undefined, mappingCharCodes.slice(i, i + 1024)); + function flushMappingBuffer(): void { + const len = mappingCharCodes.length; + if (len > 0) { + // If there are a very large number of skipped lines in the source mapping, this loop can iterate multiple times + // Otherwise it should always have 1 iteration + for (let i = 0; i < len; i += 1024) { + mappings += String.fromCharCode.apply(undefined, mappingCharCodes.slice(i, i + 1024)); + } + mappingCharCodes.length = 0; } - mappingCharCodes.length = 0; } function toJSON(): RawSourceMap { commitPendingMapping(); - serializeMappings(); + flushMappingBuffer(); return { version: 3, file, From 0d5b28580c347e17d1a7c0fed800762e1a449fb5 Mon Sep 17 00:00:00 2001 From: David Michon Date: Tue, 11 May 2021 08:33:36 -0700 Subject: [PATCH 3/3] Always commit at exactly chunk size --- src/compiler/sourcemap.ts | 31 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/src/compiler/sourcemap.ts b/src/compiler/sourcemap.ts index e79067387118e..6ca09906db47f 100644 --- a/src/compiler/sourcemap.ts +++ b/src/compiler/sourcemap.ts @@ -18,8 +18,6 @@ namespace ts { const names: string[] = []; let nameToNameIndexMap: ESMap | undefined; const mappingCharCodes: number[] = []; - // We will create a string from the char code buffer whenever it exceeds this length - const mappingCommitThreshold = 1000; let mappings = ""; // Last recorded and encoded mappings @@ -213,6 +211,15 @@ namespace ts { || lastNameIndex !== pendingNameIndex; } + function appendMappingCharCode(charCode: number) { + mappingCharCodes.push(charCode); + // String.fromCharCode accepts its arguments on the stack, so we have to chunk the input, + // otherwise we can get stack overflows for large source maps + if (mappingCharCodes.length >= 1024) { + flushMappingBuffer(); + } + } + function commitPendingMapping() { if (!hasPending || !shouldCommitMapping()) { return; @@ -223,9 +230,8 @@ namespace ts { // Line/Comma delimiters if (lastGeneratedLine < pendingGeneratedLine) { // Emit line delimiters - // This loop can potentially overflow the stack on the char code conversion if it were a single operation do { - mappingCharCodes.push(CharacterCodes.semicolon); // ';' + appendMappingCharCode(CharacterCodes.semicolon); lastGeneratedLine++; } while (lastGeneratedLine < pendingGeneratedLine); @@ -236,7 +242,7 @@ namespace ts { Debug.assertEqual(lastGeneratedLine, pendingGeneratedLine, "generatedLine cannot backtrack"); // Emit comma to separate the entry if (hasLast) { - mappingCharCodes.push(CharacterCodes.comma); // ',' + appendMappingCharCode(CharacterCodes.comma); } } @@ -264,22 +270,13 @@ namespace ts { } } - if (mappings.length > mappingCommitThreshold) { - flushMappingBuffer(); - } - hasLast = true; exit(); } function flushMappingBuffer(): void { - const len = mappingCharCodes.length; - if (len > 0) { - // If there are a very large number of skipped lines in the source mapping, this loop can iterate multiple times - // Otherwise it should always have 1 iteration - for (let i = 0; i < len; i += 1024) { - mappings += String.fromCharCode.apply(undefined, mappingCharCodes.slice(i, i + 1024)); - } + if (mappingCharCodes.length > 0) { + mappings += String.fromCharCode.apply(undefined, mappingCharCodes); mappingCharCodes.length = 0; } } @@ -319,7 +316,7 @@ namespace ts { // There are still more digits to decode, set the msb (6th bit) currentDigit = currentDigit | 32; } - mappingCharCodes.push(base64FormatEncode(currentDigit)); + appendMappingCharCode(base64FormatEncode(currentDigit)); } while (inValue > 0); } }