diff --git a/src/compiler/moduleSpecifiers.ts b/src/compiler/moduleSpecifiers.ts
index bc448698c699e..fbed429e31273 100644
--- a/src/compiler/moduleSpecifiers.ts
+++ b/src/compiler/moduleSpecifiers.ts
@@ -93,7 +93,7 @@ namespace ts.moduleSpecifiers {
 
         const info = getInfo(importingSourceFile.path, host);
         const moduleSourceFile = getSourceFileOfNode(moduleSymbol.valueDeclaration || getNonAugmentationDeclaration(moduleSymbol));
-        const modulePaths = getAllModulePaths(files, importingSourceFile.path, moduleSourceFile.fileName, info.getCanonicalFileName, host, redirectTargetsMap);
+        const modulePaths = getAllModulePaths(files, importingSourceFile.path, moduleSourceFile.originalFileName, info.getCanonicalFileName, host, redirectTargetsMap);
 
         const preferences = getPreferences(userPreferences, compilerOptions, importingSourceFile);
         const global = mapDefined(modulePaths, moduleFileName => tryGetModuleNameAsNodeModule(moduleFileName, info, host, compilerOptions));
diff --git a/src/harness/vfs.ts b/src/harness/vfs.ts
index b20c04937f878..5ef5851433a75 100644
--- a/src/harness/vfs.ts
+++ b/src/harness/vfs.ts
@@ -1525,8 +1525,11 @@ namespace vfs {
         return typeof value === "string" || Buffer.isBuffer(value) ? new File(value) : new Directory(value);
     }
 
-    export function formatPatch(patch: FileSet) {
-        return formatPatchWorker("", patch);
+    export function formatPatch(patch: FileSet): string;
+    export function formatPatch(patch: FileSet | undefined): string | null;
+    export function formatPatch(patch: FileSet | undefined) {
+        // tslint:disable-next-line:no-null-keyword
+        return patch ? formatPatchWorker("", patch) : null;
     }
 
     function formatPatchWorker(dirname: string, container: FileSet): string {
@@ -1559,5 +1562,24 @@ namespace vfs {
         }
         return text;
     }
+
+    export function iteratePatch(patch: FileSet | undefined): IterableIterator<[string, string]> | null {
+        // tslint:disable-next-line:no-null-keyword
+        return patch ? Harness.Compiler.iterateOutputs(iteratePatchWorker("", patch)) : null;
+    }
+
+    function* iteratePatchWorker(dirname: string, container: FileSet): IterableIterator<documents.TextDocument> {
+        for (const name of Object.keys(container)) {
+            const entry = normalizeFileSetEntry(container[name]);
+            const file = dirname ? vpath.combine(dirname, name) : name;
+            if (entry instanceof Directory) {
+                yield* ts.arrayFrom(iteratePatchWorker(file, entry.files));
+            }
+            else if (entry instanceof File) {
+                const content = typeof entry.data === "string" ? entry.data : entry.data.toString("utf8");
+                yield new documents.TextDocument(file, content);
+            }
+        }
+    }
 }
 // tslint:enable:no-null-keyword
\ No newline at end of file
diff --git a/src/testRunner/tsconfig.json b/src/testRunner/tsconfig.json
index 8c8e90c8d9fcf..fd9398ce21cf1 100644
--- a/src/testRunner/tsconfig.json
+++ b/src/testRunner/tsconfig.json
@@ -97,6 +97,7 @@
         "unittests/tsbuild/inferredTypeFromTransitiveModule.ts",
         "unittests/tsbuild/lateBoundSymbol.ts",
         "unittests/tsbuild/missingExtendedFile.ts",
+        "unittests/tsbuild/moduleSpecifiers.ts",
         "unittests/tsbuild/outFile.ts",
         "unittests/tsbuild/referencesWithRootDirInParent.ts",
         "unittests/tsbuild/resolveJsonModule.ts",
diff --git a/src/testRunner/unittests/config/projectReferences.ts b/src/testRunner/unittests/config/projectReferences.ts
index 9bbef4e631f89..305cc726c9772 100644
--- a/src/testRunner/unittests/config/projectReferences.ts
+++ b/src/testRunner/unittests/config/projectReferences.ts
@@ -348,5 +348,4 @@ namespace ts {
             });
         });
     });
-
 }
diff --git a/src/testRunner/unittests/tsbuild/moduleSpecifiers.ts b/src/testRunner/unittests/tsbuild/moduleSpecifiers.ts
new file mode 100644
index 0000000000000..ad0a229e876dd
--- /dev/null
+++ b/src/testRunner/unittests/tsbuild/moduleSpecifiers.ts
@@ -0,0 +1,98 @@
+namespace ts {
+    // https://github.com/microsoft/TypeScript/issues/31696
+    it("unittests:: tsbuild:: moduleSpecifiers:: synthesized module specifiers to referenced projects resolve correctly", () => {
+        const baseFs = vfs.createFromFileSystem(Harness.IO, /*ignoreCase*/ false, {
+            files: {
+                "/src/common/nominal.ts": utils.dedent`
+                    export declare type Nominal<T, Name extends string> = T & {
+                        [Symbol.species]: Name;
+                    };
+                    `,
+                "/src/common/tsconfig.json": utils.dedent`
+                    {
+                        "extends": "../../tsconfig.base.json",
+                        "compilerOptions": {
+                            "composite": true
+                        },
+                        "include": ["nominal.ts"]
+                    }`,
+                "/src/sub-project/index.ts": utils.dedent`
+                    import { Nominal } from '../common/nominal';
+
+                    export type MyNominal = Nominal<string, 'MyNominal'>;
+                    `,
+                "/src/sub-project/tsconfig.json": utils.dedent`
+                    {
+                        "extends": "../../tsconfig.base.json",
+                        "compilerOptions": {
+                            "composite": true
+                        },
+                        "references": [
+                            { "path": "../common" }
+                        ],
+                        "include": ["./index.ts"]
+                    }`,
+                "/src/sub-project-2/index.ts": utils.dedent`
+                    import { MyNominal } from '../sub-project/index';
+
+                    const variable = {
+                        key: 'value' as MyNominal,
+                    };
+
+                    export function getVar(): keyof typeof variable {
+                        return 'key';
+                    }
+                    `,
+                "/src/sub-project-2/tsconfig.json": utils.dedent`
+                    {
+                        "extends": "../../tsconfig.base.json",
+                        "compilerOptions": {
+                            "composite": true
+                        },
+                        "references": [
+                            { "path": "../sub-project" }
+                        ],
+                        "include": ["./index.ts"]
+                    }`,
+                "/src/tsconfig.json": utils.dedent`
+                    {
+                        "compilerOptions": {
+                            "composite": true
+                        },
+                        "references": [
+                            { "path": "./sub-project" },
+                            { "path": "./sub-project-2" }
+                        ],
+                        "include": []
+                    }`,
+                "/tsconfig.base.json": utils.dedent`
+                    {
+                        "compilerOptions": {
+                            "skipLibCheck": true,
+                            "rootDir": "./",
+                            "outDir": "lib",
+                            "lib": ["dom", "es2015", "es2015.symbol.wellknown"]
+                        }
+                    }`,
+                "/tsconfig.json": utils.dedent`{
+                    "compilerOptions": {
+                        "composite": true
+                    },
+                    "references": [
+                        { "path": "./src" }
+                    ],
+                    "include": []
+                }`
+            },
+            cwd: "/"
+        });
+        const fs = baseFs.makeReadonly().shadow();
+        const sys = new fakes.System(fs, { executingFilePath: "/", newLine: "\n" });
+        const host = new fakes.SolutionBuilderHost(sys);
+        const builder = createSolutionBuilder(host, ["/tsconfig.json"], { dry: false, force: false, verbose: false });
+        builder.build();
+
+        // Prior to fixing GH31696 the import in `/lib/src/sub-project-2/index.d.ts` was `import("../../lib/src/common/nonterminal")`, which was invalid.
+        Harness.Baseline.runBaseline("tsbuild/moduleSpecifiers/initial-build/resolves-correctly.js", vfs.formatPatch(fs.diff(baseFs)));
+    });
+}
\ No newline at end of file
diff --git a/tests/baselines/reference/tsbuild/moduleSpecifiers/initial-build/resolves-correctly.js b/tests/baselines/reference/tsbuild/moduleSpecifiers/initial-build/resolves-correctly.js
new file mode 100644
index 0000000000000..d8c1cfb360ead
--- /dev/null
+++ b/tests/baselines/reference/tsbuild/moduleSpecifiers/initial-build/resolves-correctly.js
@@ -0,0 +1,348 @@
+//// [/lib/src/common/nominal.d.ts]
+export declare type Nominal<T, Name extends string> = T & {
+    [Symbol.species]: Name;
+};
+
+
+//// [/lib/src/common/nominal.js]
+"use strict";
+exports.__esModule = true;
+
+
+//// [/lib/src/common/tsconfig.tsbuildinfo]
+{
+  "program": {
+    "fileInfos": {
+      "../../../.ts/lib.es5.d.ts": {
+        "version": "354625479304",
+        "signature": "354625479304"
+      },
+      "../../../.ts/lib.es2015.d.ts": {
+        "version": "57263133672",
+        "signature": "57263133672"
+      },
+      "../../../.ts/lib.dom.d.ts": {
+        "version": "-1041975536091",
+        "signature": "-1041975536091"
+      },
+      "../../../.ts/lib.es2015.core.d.ts": {
+        "version": "370321249768",
+        "signature": "370321249768"
+      },
+      "../../../.ts/lib.es2015.collection.d.ts": {
+        "version": "-95997535017",
+        "signature": "-95997535017"
+      },
+      "../../../.ts/lib.es2015.generator.d.ts": {
+        "version": "10837180865",
+        "signature": "10837180865"
+      },
+      "../../../.ts/lib.es2015.iterable.d.ts": {
+        "version": "232404497324",
+        "signature": "232404497324"
+      },
+      "../../../.ts/lib.es2015.promise.d.ts": {
+        "version": "235321148269",
+        "signature": "235321148269"
+      },
+      "../../../.ts/lib.es2015.proxy.d.ts": {
+        "version": "55479865087",
+        "signature": "55479865087"
+      },
+      "../../../.ts/lib.es2015.reflect.d.ts": {
+        "version": "30748787093",
+        "signature": "30748787093"
+      },
+      "../../../.ts/lib.es2015.symbol.d.ts": {
+        "version": "9409688441",
+        "signature": "9409688441"
+      },
+      "../../../.ts/lib.es2015.symbol.wellknown.d.ts": {
+        "version": "-67261006573",
+        "signature": "-67261006573"
+      },
+      "../../../src/common/nominal.ts": {
+        "version": "-24498031910",
+        "signature": "-24498031910"
+      }
+    },
+    "options": {
+      "skipLibCheck": true,
+      "rootDir": "../../..",
+      "outDir": "../..",
+      "lib": [
+        "lib.dom.d.ts",
+        "lib.es2015.d.ts",
+        "lib.es2015.symbol.wellknown.d.ts"
+      ],
+      "composite": true,
+      "configFilePath": "../../../src/common/tsconfig.json"
+    },
+    "referencedMap": {},
+    "exportedModulesMap": {},
+    "semanticDiagnosticsPerFile": [
+      "../../../.ts/lib.dom.d.ts",
+      "../../../.ts/lib.es2015.collection.d.ts",
+      "../../../.ts/lib.es2015.core.d.ts",
+      "../../../.ts/lib.es2015.d.ts",
+      "../../../.ts/lib.es2015.generator.d.ts",
+      "../../../.ts/lib.es2015.iterable.d.ts",
+      "../../../.ts/lib.es2015.promise.d.ts",
+      "../../../.ts/lib.es2015.proxy.d.ts",
+      "../../../.ts/lib.es2015.reflect.d.ts",
+      "../../../.ts/lib.es2015.symbol.d.ts",
+      "../../../.ts/lib.es2015.symbol.wellknown.d.ts",
+      "../../../.ts/lib.es5.d.ts",
+      "../../../src/common/nominal.ts"
+    ]
+  },
+  "version": "FakeTSVersion"
+}
+
+//// [/lib/src/sub-project/index.d.ts]
+import { Nominal } from '../common/nominal';
+export declare type MyNominal = Nominal<string, 'MyNominal'>;
+
+
+//// [/lib/src/sub-project/index.js]
+"use strict";
+exports.__esModule = true;
+
+
+//// [/lib/src/sub-project/tsconfig.tsbuildinfo]
+{
+  "program": {
+    "fileInfos": {
+      "../../../.ts/lib.es5.d.ts": {
+        "version": "354625479304",
+        "signature": "354625479304"
+      },
+      "../../../.ts/lib.es2015.d.ts": {
+        "version": "57263133672",
+        "signature": "57263133672"
+      },
+      "../../../.ts/lib.dom.d.ts": {
+        "version": "-1041975536091",
+        "signature": "-1041975536091"
+      },
+      "../../../.ts/lib.es2015.core.d.ts": {
+        "version": "370321249768",
+        "signature": "370321249768"
+      },
+      "../../../.ts/lib.es2015.collection.d.ts": {
+        "version": "-95997535017",
+        "signature": "-95997535017"
+      },
+      "../../../.ts/lib.es2015.generator.d.ts": {
+        "version": "10837180865",
+        "signature": "10837180865"
+      },
+      "../../../.ts/lib.es2015.iterable.d.ts": {
+        "version": "232404497324",
+        "signature": "232404497324"
+      },
+      "../../../.ts/lib.es2015.promise.d.ts": {
+        "version": "235321148269",
+        "signature": "235321148269"
+      },
+      "../../../.ts/lib.es2015.proxy.d.ts": {
+        "version": "55479865087",
+        "signature": "55479865087"
+      },
+      "../../../.ts/lib.es2015.reflect.d.ts": {
+        "version": "30748787093",
+        "signature": "30748787093"
+      },
+      "../../../.ts/lib.es2015.symbol.d.ts": {
+        "version": "9409688441",
+        "signature": "9409688441"
+      },
+      "../../../.ts/lib.es2015.symbol.wellknown.d.ts": {
+        "version": "-67261006573",
+        "signature": "-67261006573"
+      },
+      "../../../src/common/nominal.ts": {
+        "version": "-24498031910",
+        "signature": "-24498031910"
+      },
+      "../../../src/sub-project/index.ts": {
+        "version": "-22894055505",
+        "signature": "-18559108619"
+      }
+    },
+    "options": {
+      "skipLibCheck": true,
+      "rootDir": "../../..",
+      "outDir": "../..",
+      "lib": [
+        "lib.dom.d.ts",
+        "lib.es2015.d.ts",
+        "lib.es2015.symbol.wellknown.d.ts"
+      ],
+      "composite": true,
+      "configFilePath": "../../../src/sub-project/tsconfig.json"
+    },
+    "referencedMap": {
+      "../../../src/sub-project/index.ts": [
+        "../common/nominal.d.ts"
+      ]
+    },
+    "exportedModulesMap": {
+      "../../../src/sub-project/index.ts": [
+        "../common/nominal.d.ts"
+      ]
+    },
+    "semanticDiagnosticsPerFile": [
+      "../../../.ts/lib.dom.d.ts",
+      "../../../.ts/lib.es2015.collection.d.ts",
+      "../../../.ts/lib.es2015.core.d.ts",
+      "../../../.ts/lib.es2015.d.ts",
+      "../../../.ts/lib.es2015.generator.d.ts",
+      "../../../.ts/lib.es2015.iterable.d.ts",
+      "../../../.ts/lib.es2015.promise.d.ts",
+      "../../../.ts/lib.es2015.proxy.d.ts",
+      "../../../.ts/lib.es2015.reflect.d.ts",
+      "../../../.ts/lib.es2015.symbol.d.ts",
+      "../../../.ts/lib.es2015.symbol.wellknown.d.ts",
+      "../../../.ts/lib.es5.d.ts",
+      "../../../src/common/nominal.ts",
+      "../../../src/sub-project/index.ts"
+    ]
+  },
+  "version": "FakeTSVersion"
+}
+
+//// [/lib/src/sub-project-2/index.d.ts]
+declare const variable: {
+    key: import("../common/nominal").Nominal<string, "MyNominal">;
+};
+export declare function getVar(): keyof typeof variable;
+export {};
+
+
+//// [/lib/src/sub-project-2/index.js]
+"use strict";
+exports.__esModule = true;
+var variable = {
+    key: 'value'
+};
+function getVar() {
+    return 'key';
+}
+exports.getVar = getVar;
+
+
+//// [/lib/src/sub-project-2/tsconfig.tsbuildinfo]
+{
+  "program": {
+    "fileInfos": {
+      "../../../.ts/lib.es5.d.ts": {
+        "version": "354625479304",
+        "signature": "354625479304"
+      },
+      "../../../.ts/lib.es2015.d.ts": {
+        "version": "57263133672",
+        "signature": "57263133672"
+      },
+      "../../../.ts/lib.dom.d.ts": {
+        "version": "-1041975536091",
+        "signature": "-1041975536091"
+      },
+      "../../../.ts/lib.es2015.core.d.ts": {
+        "version": "370321249768",
+        "signature": "370321249768"
+      },
+      "../../../.ts/lib.es2015.collection.d.ts": {
+        "version": "-95997535017",
+        "signature": "-95997535017"
+      },
+      "../../../.ts/lib.es2015.generator.d.ts": {
+        "version": "10837180865",
+        "signature": "10837180865"
+      },
+      "../../../.ts/lib.es2015.iterable.d.ts": {
+        "version": "232404497324",
+        "signature": "232404497324"
+      },
+      "../../../.ts/lib.es2015.promise.d.ts": {
+        "version": "235321148269",
+        "signature": "235321148269"
+      },
+      "../../../.ts/lib.es2015.proxy.d.ts": {
+        "version": "55479865087",
+        "signature": "55479865087"
+      },
+      "../../../.ts/lib.es2015.reflect.d.ts": {
+        "version": "30748787093",
+        "signature": "30748787093"
+      },
+      "../../../.ts/lib.es2015.symbol.d.ts": {
+        "version": "9409688441",
+        "signature": "9409688441"
+      },
+      "../../../.ts/lib.es2015.symbol.wellknown.d.ts": {
+        "version": "-67261006573",
+        "signature": "-67261006573"
+      },
+      "../../../src/common/nominal.ts": {
+        "version": "-24498031910",
+        "signature": "-24498031910"
+      },
+      "../../../src/sub-project/index.ts": {
+        "version": "-18559108619",
+        "signature": "-18559108619"
+      },
+      "../../../src/sub-project-2/index.ts": {
+        "version": "-13939373533",
+        "signature": "-33844181688"
+      }
+    },
+    "options": {
+      "skipLibCheck": true,
+      "rootDir": "../../..",
+      "outDir": "../..",
+      "lib": [
+        "lib.dom.d.ts",
+        "lib.es2015.d.ts",
+        "lib.es2015.symbol.wellknown.d.ts"
+      ],
+      "composite": true,
+      "configFilePath": "../../../src/sub-project-2/tsconfig.json"
+    },
+    "referencedMap": {
+      "../../../src/sub-project-2/index.ts": [
+        "../sub-project/index.d.ts"
+      ],
+      "../../../src/sub-project/index.ts": [
+        "../common/nominal.d.ts"
+      ]
+    },
+    "exportedModulesMap": {
+      "../../../src/sub-project-2/index.ts": [
+        "../common/nominal.d.ts"
+      ],
+      "../../../src/sub-project/index.ts": [
+        "../common/nominal.d.ts"
+      ]
+    },
+    "semanticDiagnosticsPerFile": [
+      "../../../.ts/lib.dom.d.ts",
+      "../../../.ts/lib.es2015.collection.d.ts",
+      "../../../.ts/lib.es2015.core.d.ts",
+      "../../../.ts/lib.es2015.d.ts",
+      "../../../.ts/lib.es2015.generator.d.ts",
+      "../../../.ts/lib.es2015.iterable.d.ts",
+      "../../../.ts/lib.es2015.promise.d.ts",
+      "../../../.ts/lib.es2015.proxy.d.ts",
+      "../../../.ts/lib.es2015.reflect.d.ts",
+      "../../../.ts/lib.es2015.symbol.d.ts",
+      "../../../.ts/lib.es2015.symbol.wellknown.d.ts",
+      "../../../.ts/lib.es5.d.ts",
+      "../../../src/common/nominal.ts",
+      "../../../src/sub-project-2/index.ts",
+      "../../../src/sub-project/index.ts"
+    ]
+  },
+  "version": "FakeTSVersion"
+}
+