Skip to content

Import the semantic highlighter from typescript-vscode-sh-plugin #39119

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 21 commits into from
Sep 11, 2020
Merged

Import the semantic highlighter from typescript-vscode-sh-plugin #39119

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
1d3a728
Initial import of the vscode semantic highlight code
orta Jun 1, 2020
fa8a499
Adds the ability to test modern semantic classification via strings i…
orta Jun 15, 2020
8a5c3b3
Adds existing tests
orta Jun 16, 2020
2a4113b
Port over the semantic classification tests
orta Jun 17, 2020
8728a1e
Update baselines
orta Jun 19, 2020
78ad1d7
Update src/harness/fourslashImpl.ts
orta Jun 22, 2020
c43374b
Handle feedback from #39119
orta Jun 22, 2020
40e5d22
Consistent formatting in the 2020 classifier
orta Jun 22, 2020
524e475
Update baselines
orta Jun 22, 2020
e1ce709
Apply suggestions from code review
orta Jun 23, 2020
7e35e53
Update src/harness/fourslashImpl.ts
orta Jun 23, 2020
8a7596a
Reafactor after comments
orta Jun 23, 2020
b8742c3
Use 2020 everywhere
orta Jul 6, 2020
473f651
Handle feedback
orta Jul 8, 2020
2b663a8
WIP - don't provide a breaking change
orta Jul 8, 2020
219c64a
Fix all build errors
orta Sep 9, 2020
415b30f
Merge pull request #4 from orta/semantic_2_2
orta Sep 9, 2020
1747860
Merge branch 'master' of https://github.com/microsoft/TypeScript into…
orta Sep 9, 2020
a9035d6
Update baselines
orta Sep 9, 2020
8739cbf
Update src/services/classifier2020.ts
orta Sep 9, 2020
ff748a5
Addresses Ron's feedback
orta Sep 11, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/harness/client.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -747,7 +747,7 @@ namespace ts.server {
return notImplemented();
}

getEncodedSemanticClassifications(_fileName: string, _span: TextSpan): Classifications {
getEncodedSemanticClassifications(_fileName: string, _span: TextSpan, _format?: SemanticClassificationFormat): Classifications {
return notImplemented();
}

Expand Down
76 changes: 70 additions & 6 deletions src/harness/fourslashImpl.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2492,7 +2492,49 @@ namespace FourSlash {
Harness.IO.log(this.spanInfoToString(this.getNameOrDottedNameSpan(pos)!, "**"));
}

private verifyClassifications(expected: { classificationType: string; text: string; textSpan?: TextSpan }[], actual: ts.ClassifiedSpan[], sourceFileText: string) {
private classificationToIdentifier(classification: number){

const tokenTypes: string[] = [];
tokenTypes[ts.classifier.v2020.TokenType.class] = "class";
tokenTypes[ts.classifier.v2020.TokenType.enum] = "enum";
tokenTypes[ts.classifier.v2020.TokenType.interface] = "interface";
tokenTypes[ts.classifier.v2020.TokenType.namespace] = "namespace";
tokenTypes[ts.classifier.v2020.TokenType.typeParameter] = "typeParameter";
tokenTypes[ts.classifier.v2020.TokenType.type] = "type";
tokenTypes[ts.classifier.v2020.TokenType.parameter] = "parameter";
tokenTypes[ts.classifier.v2020.TokenType.variable] = "variable";
tokenTypes[ts.classifier.v2020.TokenType.enumMember] = "enumMember";
tokenTypes[ts.classifier.v2020.TokenType.property] = "property";
tokenTypes[ts.classifier.v2020.TokenType.function] = "function";
tokenTypes[ts.classifier.v2020.TokenType.member] = "member";

const tokenModifiers: string[] = [];
tokenModifiers[ts.classifier.v2020.TokenModifier.async] = "async";
tokenModifiers[ts.classifier.v2020.TokenModifier.declaration] = "declaration";
tokenModifiers[ts.classifier.v2020.TokenModifier.readonly] = "readonly";
tokenModifiers[ts.classifier.v2020.TokenModifier.static] = "static";
tokenModifiers[ts.classifier.v2020.TokenModifier.local] = "local";
tokenModifiers[ts.classifier.v2020.TokenModifier.defaultLibrary] = "defaultLibrary";


function getTokenTypeFromClassification(tsClassification: number): number | undefined {
if (tsClassification > ts.classifier.v2020.TokenEncodingConsts.modifierMask) {
return (tsClassification >> ts.classifier.v2020.TokenEncodingConsts.typeOffset) - 1;
}
return undefined;
}

function getTokenModifierFromClassification(tsClassification: number) {
return tsClassification & ts.classifier.v2020.TokenEncodingConsts.modifierMask;
}

const typeIdx = getTokenTypeFromClassification(classification) || 0;
const modSet = getTokenModifierFromClassification(classification);

return [tokenTypes[typeIdx], ...tokenModifiers.filter((_, i) => modSet & 1 << i)].join(".");
}

private verifyClassifications(expected: { classificationType: string | number, text?: string; textSpan?: TextSpan }[], actual: (ts.ClassifiedSpan | ts.ClassifiedSpan2020)[] , sourceFileText: string) {
if (actual.length !== expected.length) {
this.raiseError("verifyClassifications failed - expected total classifications to be " + expected.length +
", but was " + actual.length +
Expand All @@ -2501,10 +2543,12 @@ namespace FourSlash {

ts.zipWith(expected, actual, (expectedClassification, actualClassification) => {
const expectedType = expectedClassification.classificationType;
if (expectedType !== actualClassification.classificationType) {
const actualType = typeof actualClassification.classificationType === "number" ? this.classificationToIdentifier(actualClassification.classificationType) : actualClassification.classificationType;

if (expectedType !== actualType) {
this.raiseError("verifyClassifications failed - expected classifications type to be " +
expectedType + ", but was " +
actualClassification.classificationType +
actualType +
jsonMismatchString());
}

Expand Down Expand Up @@ -2555,10 +2599,30 @@ namespace FourSlash {
}
}

public verifySemanticClassifications(expected: { classificationType: string; text: string }[]) {
public replaceWithSemanticClassifications(format: ts.SemanticClassificationFormat.TwentyTwenty) {
const actual = this.languageService.getSemanticClassifications(this.activeFile.fileName,
ts.createTextSpan(0, this.activeFile.content.length));
ts.createTextSpan(0, this.activeFile.content.length), format);
const replacement = [`const c2 = classification("2020");`,`verify.semanticClassificationsAre("2020",`];
for (const a of actual) {
const identifier = this.classificationToIdentifier(a.classificationType as number);
const text = this.activeFile.content.slice(a.textSpan.start, a.textSpan.start + a.textSpan.length);
replacement.push(` c2.semanticToken("${identifier}", "${text}"), `);
};
replacement.push(");");

throw new Error("You need to change the source code of fourslash test to use replaceWithSemanticClassifications");

// const fs = require("fs");
// const testfilePath = this.originalInputFileName.slice(1);
// const testfile = fs.readFileSync(testfilePath, "utf8");
// const newfile = testfile.replace("verify.replaceWithSemanticClassifications(\"2020\")", replacement.join("\n"));
// fs.writeFileSync(testfilePath, newfile);
}


public verifySemanticClassifications(format: ts.SemanticClassificationFormat, expected: { classificationType: string | number; text?: string }[]) {
const actual = this.languageService.getSemanticClassifications(this.activeFile.fileName,
ts.createTextSpan(0, this.activeFile.content.length), format);
this.verifyClassifications(expected, actual, this.activeFile.content);
}

Expand Down Expand Up @@ -3859,7 +3923,7 @@ namespace FourSlash {
const cancellation = new FourSlashInterface.Cancellation(state);
// eslint-disable-next-line no-eval
const f = eval(wrappedCode);
f(test, goTo, plugins, verify, edit, debug, format, cancellation, FourSlashInterface.Classification, FourSlashInterface.Completion, verifyOperationIsCancelled);
f(test, goTo, plugins, verify, edit, debug, format, cancellation, FourSlashInterface.classification, FourSlashInterface.Completion, verifyOperationIsCancelled);
}
catch (err) {
// ensure 'source-map-support' is triggered while we still have the handler attached by accessing `error.stack`.
Expand Down
112 changes: 85 additions & 27 deletions src/harness/fourslashInterfaceImpl.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -524,8 +524,12 @@ namespace FourSlashInterface {
/**
* This method *requires* an ordered stream of classifications for a file, and spans are highly recommended.
*/
public semanticClassificationsAre(...classifications: Classification[]) {
this.state.verifySemanticClassifications(classifications);
public semanticClassificationsAre(format: ts.SemanticClassificationFormat, ...classifications: Classification[]) {
this.state.verifySemanticClassifications(format, classifications);
}

public replaceWithSemanticClassifications(format: ts.SemanticClassificationFormat.TwentyTwenty) {
this.state.replaceWithSemanticClassifications(format);
}

public renameInfoSucceeded(displayName?: string, fullDisplayName?: string, kind?: string, kindModifiers?: string, fileToRename?: string, expectedRange?: FourSlash.Range, options?: ts.RenameInfoOptions) {
Expand Down Expand Up @@ -768,109 +772,163 @@ namespace FourSlashInterface {
}
}

interface Classification {
interface OlderClassification {
classificationType: ts.ClassificationTypeNames;
text: string;
textSpan?: FourSlash.TextSpan;
}
export namespace Classification {
export function comment(text: string, position?: number): Classification {

// The VS Code LSP
interface ModernClassification {
classificationType: string;
text?: string;
textSpan?: FourSlash.TextSpan;
}

type Classification = OlderClassification | ModernClassification;

export function classification(format: ts.SemanticClassificationFormat) {

function semanticToken(identifier: string, text: string, _position: number): Classification {
return {
classificationType: identifier,
text
};
}

if (format === ts.SemanticClassificationFormat.TwentyTwenty) {
return {
semanticToken
};
}

// Defaults to the previous semantic classifier factory functions

function comment(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.comment, text, position);
}

export function identifier(text: string, position?: number): Classification {
function identifier(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.identifier, text, position);
}

export function keyword(text: string, position?: number): Classification {
function keyword(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.keyword, text, position);
}

export function numericLiteral(text: string, position?: number): Classification {
function numericLiteral(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.numericLiteral, text, position);
}

export function operator(text: string, position?: number): Classification {
function operator(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.operator, text, position);
}

export function stringLiteral(text: string, position?: number): Classification {
function stringLiteral(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.stringLiteral, text, position);
}

export function whiteSpace(text: string, position?: number): Classification {
function whiteSpace(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.whiteSpace, text, position);
}

export function text(text: string, position?: number): Classification {
function text(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.text, text, position);
}

export function punctuation(text: string, position?: number): Classification {
function punctuation(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.punctuation, text, position);
}

export function docCommentTagName(text: string, position?: number): Classification {
function docCommentTagName(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.docCommentTagName, text, position);
}

export function className(text: string, position?: number): Classification {
function className(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.className, text, position);
}

export function enumName(text: string, position?: number): Classification {
function enumName(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.enumName, text, position);
}

export function interfaceName(text: string, position?: number): Classification {
function interfaceName(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.interfaceName, text, position);
}

export function moduleName(text: string, position?: number): Classification {
function moduleName(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.moduleName, text, position);
}

export function typeParameterName(text: string, position?: number): Classification {
function typeParameterName(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.typeParameterName, text, position);
}

export function parameterName(text: string, position?: number): Classification {
function parameterName(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.parameterName, text, position);
}

export function typeAliasName(text: string, position?: number): Classification {
function typeAliasName(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.typeAliasName, text, position);
}

export function jsxOpenTagName(text: string, position?: number): Classification {
function jsxOpenTagName(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.jsxOpenTagName, text, position);
}

export function jsxCloseTagName(text: string, position?: number): Classification {
function jsxCloseTagName(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.jsxCloseTagName, text, position);
}

export function jsxSelfClosingTagName(text: string, position?: number): Classification {
function jsxSelfClosingTagName(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.jsxSelfClosingTagName, text, position);
}

export function jsxAttribute(text: string, position?: number): Classification {
function jsxAttribute(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.jsxAttribute, text, position);
}

export function jsxText(text: string, position?: number): Classification {
function jsxText(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.jsxText, text, position);
}

export function jsxAttributeStringLiteralValue(text: string, position?: number): Classification {
function jsxAttributeStringLiteralValue(text: string, position?: number): Classification {
return getClassification(ts.ClassificationTypeNames.jsxAttributeStringLiteralValue, text, position);
}

function getClassification(classificationType: ts.ClassificationTypeNames, text: string, position?: number): Classification {
const textSpan = position === undefined ? undefined : { start: position, end: position + text.length };
return { classificationType, text, textSpan };
}

return {
comment,
identifier,
keyword,
numericLiteral,
operator,
stringLiteral,
whiteSpace,
text,
punctuation,
docCommentTagName,
className,
enumName,
interfaceName,
moduleName,
typeParameterName,
parameterName,
typeAliasName,
jsxOpenTagName,
jsxCloseTagName,
jsxSelfClosingTagName,
jsxAttribute,
jsxText,
jsxAttributeStringLiteralValue,
getClassification
};
}

export namespace Completion {
export import SortText = ts.Completions.SortText;
export import CompletionSource = ts.Completions.CompletionSource;
Expand Down
9 changes: 5 additions & 4 deletions src/harness/harnessLanguageService.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -461,14 +461,15 @@ namespace Harness.LanguageService {
getSyntacticClassifications(fileName: string, span: ts.TextSpan): ts.ClassifiedSpan[] {
return unwrapJSONCallResult(this.shim.getSyntacticClassifications(fileName, span.start, span.length));
}
getSemanticClassifications(fileName: string, span: ts.TextSpan): ts.ClassifiedSpan[] {
return unwrapJSONCallResult(this.shim.getSemanticClassifications(fileName, span.start, span.length));
getSemanticClassifications(fileName: string, span: ts.TextSpan, format?: ts.SemanticClassificationFormat): ts.ClassifiedSpan[] {
return unwrapJSONCallResult(this.shim.getSemanticClassifications(fileName, span.start, span.length, format));
}
getEncodedSyntacticClassifications(fileName: string, span: ts.TextSpan): ts.Classifications {
return unwrapJSONCallResult(this.shim.getEncodedSyntacticClassifications(fileName, span.start, span.length));
}
getEncodedSemanticClassifications(fileName: string, span: ts.TextSpan): ts.Classifications {
return unwrapJSONCallResult(this.shim.getEncodedSemanticClassifications(fileName, span.start, span.length));
getEncodedSemanticClassifications(fileName: string, span: ts.TextSpan, format?: ts.SemanticClassificationFormat): ts.Classifications {
const responseFormat = format || ts.SemanticClassificationFormat.Original;
return unwrapJSONCallResult(this.shim.getEncodedSemanticClassifications(fileName, span.start, span.length, responseFormat));
}
getCompletionsAtPosition(fileName: string, position: number, preferences: ts.UserPreferences | undefined): ts.CompletionInfo {
return unwrapJSONCallResult(this.shim.getCompletionsAtPosition(fileName, position, preferences));
Expand Down
Loading