fix: Handle path normalization in JSON map.

Author: varungandhi-src

core/Error.cc

@@ -124,7 +124,7 @@ string Error::toString(const GlobalState &gs) const {
     stringstream buf;
     buf << RESET_STYLE << FILE_POS_STYLE << loc.filePosToString(gs) << RESET_STYLE << ": " << ERROR_COLOR
         << restoreColors(header, ERROR_COLOR) << RESET_COLOR;
-    if (what.code != 25900) { // SCIPRubyDebug
+    if (what != scip_indexer::errors::SCIPRubyDebug && what != scip_indexer::errors::SCIPRuby) {
         buf << LOW_NOISE_COLOR << " " << gs.errorUrlBase << what.code << RESET_COLOR;
     }
     if (loc.exists()) {

core/GlobalState.cc

@@ -2156,7 +2156,7 @@ bool GlobalState::shouldReportErrorOn(Loc loc, ErrorClass what) const {
         return false;
     }
     if (this->isSCIPRuby && !this->unsilenceErrors) {
-        if (what.code != 25900) { // SCIPRubyDebug
+        if (what != scip_indexer::errors::SCIPRubyDebug && what != scip_indexer::errors::SCIPRuby) {
             return false;
         }
     }

core/errors/errors.h

@@ -15,3 +15,4 @@
 #include "core/errors/packager.h"
 #include "core/errors/parser.h"
 #include "core/errors/resolver.h"
+#include "core/errors/scip_ruby.h"

core/errors/scip_ruby.h

@@ -0,0 +1,10 @@
+#ifndef SORBET_CORE_ERRORS_SCIP_RUBY_H
+#define SORBET_CORE_ERRORS_SCIP_RUBY_H
+#include "core/Error.h"
+
+namespace sorbet::scip_indexer::errors {
+constexpr sorbet::core::ErrorClass SCIPRubyDebug{25900, sorbet::core::StrictLevel::False};
+constexpr sorbet::core::ErrorClass SCIPRuby{25901, sorbet::core::StrictLevel::False};
+} // namespace sorbet::scip_indexer::errors
+
+#endif // SORBET_CORE_ERRORS_SCIP_RUBY_H

docs/scip-ruby/CLI.md

@@ -56,6 +56,12 @@ If information about the gem being indexed is not present in the JSON file,
 and cannot be inferred from the filesystem, then you can supply
 the `--gem-metadata` argument as described earlier.
 
+If you run into an error message where a path in the JSON file
+is not recognized by `scip-ruby`, you can re-run the indexing command
+with extra arguments `--log-recorded-filepaths --debug-log-file out.log`
+to identify differences between the JSON file
+and paths created by traversing directories.
+
 ## `--index-file <arg>`
 
 The path for emitting the SCIP index. Defaults to `index.scip`.

scip_indexer/Debug.cc

@@ -3,15 +3,14 @@
 
 #include "core/GlobalState.h"
 #include "core/Loc.h"
+#include "core/errors/scip_ruby.h"
 
 #include "scip_indexer/Debug.h"
 
 namespace sorbet::scip_indexer {
 
-constexpr sorbet::core::ErrorClass SCIPRubyDebug{25900, sorbet::core::StrictLevel::False};
-
 void _log_debug(const sorbet::core::GlobalState &gs, sorbet::core::Loc loc, std::string s) {
-    if (auto e = gs.beginError(loc, SCIPRubyDebug)) {
+    if (auto e = gs.beginError(loc, errors::SCIPRubyDebug)) {
         auto lines = absl::StrSplit(s, '\n');
         for (auto line = lines.begin(); line != lines.end(); line++) {
             auto text = std::string(line->begin(), line->length());

scip_indexer/Debug.h

@@ -55,8 +55,6 @@ template <typename V, typename Fn> std::string showVec(const V &v, Fn f) {
     return out.str();
 }
 
-extern const sorbet::core::ErrorClass SCIPRubyDebug;
-
 void _log_debug(const sorbet::core::GlobalState &gs, sorbet::core::Loc loc, std::string s);
 } // namespace sorbet::scip_indexer
 

scip_indexer/SCIPGemMetadata.cc

@@ -9,8 +9,10 @@
 
 #include "absl/strings/str_replace.h"
 
+#include "common/FileSystem.h"
 #include "common/common.h"
 #include "core/FileRef.h"
+#include "core/errors/scip_ruby.h"
 
 // Uses ENFORCE as defined by common/common.h, so put this later
 #include "rapidjson/document.h"
@@ -201,8 +203,10 @@ optional<shared_ptr<GemMetadata>> GemMapping::lookupGemForFile(const core::Globa
     return nullopt;
 }
 
-void GemMapping::populateFromNDJSON(const core::GlobalState &gs, const std::string &ndjsonText) {
-    istringstream input(ndjsonText);
+void GemMapping::populateFromNDJSON(const core::GlobalState &gs, const FileSystem &fs, const std::string &ndjsonPath) {
+    istringstream input(fs.readFile(ndjsonPath));
+    auto currentDir = filesystem::path(fs.getCurrentDir());
+    unsigned errorCount = 0;
     for (string line; getline(input, line);) {
         auto jsonLine = std::string(absl::StripAsciiWhitespace(line));
         if (jsonLine.empty()) {
@@ -215,9 +219,31 @@ void GemMapping::populateFromNDJSON(const core::GlobalState &gs, const std::stri
         }
         if (document.HasMember("path") && document["path"].IsString() && document.HasMember("gem") &&
             document["gem"].IsString()) {
-            // TODO: handle path normalization
-            auto fileRef = gs.findFileByPath(document["path"].GetString());
+            auto jsonPath = document["path"].GetString();
+            auto fileRef = gs.findFileByPath(jsonPath);
             if (!fileRef.exists()) {
+                vector<string> alternatePaths;
+                auto path = filesystem::path(jsonPath);
+                auto normalizedPath = path.lexically_normal();
+                alternatePaths.push_back(normalizedPath);
+                if (normalizedPath.is_absolute()) {
+                    alternatePaths.push_back(string(normalizedPath.lexically_relative(currentDir).lexically_normal()));
+                } else {
+                    alternatePaths.push_back(string((currentDir / normalizedPath).lexically_normal()));
+                }
+                for (auto &altPath : alternatePaths) {
+                    fileRef = gs.findFileByPath(altPath);
+                    if (fileRef.exists()) {
+                        break;
+                    }
+                }
+                errorCount++;
+                if (errorCount < 5) {
+                    if (auto e = gs.beginError(core::Loc(), errors::SCIPRuby)) {
+                        e.setHeader("File path in JSON map doesn't match known paths: {}", jsonPath);
+                        e.addErrorNote("Use --debug-log-filepaths to check the paths being recorded by scip-ruby");
+                    }
+                }
                 continue;
             }
             auto gemMetadata = GemMetadata::tryParse(document["gem"].GetString());

scip_indexer/SCIPGemMetadata.h

@@ -87,7 +87,7 @@ class GemMapping final {
 
     std::optional<std::shared_ptr<GemMetadata>> lookupGemForFile(const core::GlobalState &gs, core::FileRef file) const;
 
-    void populateFromNDJSON(const core::GlobalState &, const std::string &ndjsonContents);
+    void populateFromNDJSON(const core::GlobalState &, const FileSystem &fs, const std::string &ndjsonPath);
 
     void markCurrentGem(GemMetadata gem);
 };

scip_indexer/SCIPIndexer.cc

@@ -30,6 +30,7 @@
 #include "core/Loc.h"
 #include "core/SymbolRef.h"
 #include "core/Symbols.h"
+#include "core/errors/scip_ruby.h"
 #include "main/pipeline/semantic_extension/SemanticExtension.h"
 #include "sorbet_version/sorbet_version.h"
 
@@ -507,6 +508,7 @@ class SCIPState {
         scip::Document document;
         // TODO(varun): Double-check the path code and maybe document it,
         // to make sure its guarantees match what SCIP expects.
+        // FIXME: This can contain absolute paths if an absolute path was passed to scip-ruby.
         document.set_relative_path(string(file.data(gs).path()));
 
         auto occurrences = this->occurrenceMap.find(file);
@@ -1184,7 +1186,7 @@ class SCIPSemanticExtension : public SemanticExtension {
             auto [gem, errors] = scip_indexer::GemMetadata::readFromConfig(OSFileSystem());
             currentGem = gem;
             for (auto &error : errors) {
-                if (auto e = gs.beginError(core::Loc(), scip_indexer::SCIPRubyDebug)) {
+                if (auto e = gs.beginError(core::Loc(), scip_indexer::errors::SCIPRubyDebug)) {
                     e.setHeader("{}: {}",
                                 error.kind == scip_indexer::GemMetadataError::Kind::Error ? "error" : "warning",
                                 error.message);
@@ -1193,7 +1195,7 @@ class SCIPSemanticExtension : public SemanticExtension {
         }
         this->gemMap.markCurrentGem(currentGem);
         if (!this->gemMapPath.empty()) {
-            this->gemMap.populateFromNDJSON(gs, OSFileSystem().readFile(this->gemMapPath));
+            this->gemMap.populateFromNDJSON(gs, OSFileSystem(), this->gemMapPath);
         }
     };