feat: Add support for manually specifiying gems for cross-repo.

Author: varungandhi-src

core/Error.cc

@@ -124,7 +124,7 @@ string Error::toString(const GlobalState &gs) const {
     stringstream buf;
     buf << RESET_STYLE << FILE_POS_STYLE << loc.filePosToString(gs) << RESET_STYLE << ": " << ERROR_COLOR
         << restoreColors(header, ERROR_COLOR) << RESET_COLOR;
-    if (what.code != 25900) { // SCIPRubyDebug
+    if (what != scip_indexer::errors::SCIPRubyDebug && what != scip_indexer::errors::SCIPRuby) {
         buf << LOW_NOISE_COLOR << " " << gs.errorUrlBase << what.code << RESET_COLOR;
     }
     if (loc.exists()) {

core/GlobalState.cc

@@ -13,6 +13,7 @@
 #include "core/hashing/hashing.h"
 #include "core/lsp/Task.h"
 #include "core/lsp/TypecheckEpochManager.h"
+#include <filesystem>
 #include <utility>
 
 #include "absl/strings/str_cat.h"
@@ -1663,14 +1664,21 @@ FileRef GlobalState::enterFile(const shared_ptr<File> &file) {
     }
 
     files.emplace_back(file);
+    // GlobalState initialization guarantees the 0 slot will be taken, so this is OK.
     auto ret = FileRef(filesUsed() - 1);
     fileRefByPath[string(file->path())] = ret;
     return ret;
 }
 
 FileRef GlobalState::enterFile(string_view path, string_view source) {
+    std::string pathBuf;
+    if (this->isSCIPRuby) { // See [NOTE: scip-ruby-path-normalization]
+        pathBuf = string(std::filesystem::path(path).lexically_normal());
+    } else {
+        pathBuf = string(path);
+    }
     return GlobalState::enterFile(
-        make_shared<File>(string(path.begin(), path.end()), string(source.begin(), source.end()), File::Type::Normal));
+        make_shared<File>(move(pathBuf), string(source.begin(), source.end()), File::Type::Normal));
 }
 
 FileRef GlobalState::enterNewFileAt(const shared_ptr<File> &file, FileRef id) {
@@ -1686,7 +1694,13 @@ FileRef GlobalState::enterNewFileAt(const shared_ptr<File> &file, FileRef id) {
 }
 
 FileRef GlobalState::reserveFileRef(string path) {
-    return GlobalState::enterFile(make_shared<File>(move(path), "", File::Type::NotYetRead));
+    std::string pathBuf;
+    if (this->isSCIPRuby) { // See [NOTE: scip-ruby-path-normalization]
+        pathBuf = string(std::filesystem::path(path).lexically_normal());
+    } else {
+        pathBuf = move(path);
+    }
+    return GlobalState::enterFile(make_shared<File>(move(pathBuf), "", File::Type::NotYetRead));
 }
 
 NameRef GlobalState::nextMangledName(ClassOrModuleRef owner, NameRef origName) {
@@ -2155,7 +2169,7 @@ bool GlobalState::shouldReportErrorOn(Loc loc, ErrorClass what) const {
         return false;
     }
     if (this->isSCIPRuby && !this->unsilenceErrors) {
-        if (what.code != 25900) { // SCIPRubyDebug
+        if (what != scip_indexer::errors::SCIPRubyDebug && what != scip_indexer::errors::SCIPRuby) {
             return false;
         }
     }

core/errors/errors.h

@@ -15,3 +15,4 @@
 #include "core/errors/packager.h"
 #include "core/errors/parser.h"
 #include "core/errors/resolver.h"
+#include "core/errors/scip_ruby.h"

core/errors/scip_ruby.h

@@ -0,0 +1,10 @@
+#ifndef SORBET_CORE_ERRORS_SCIP_RUBY_H
+#define SORBET_CORE_ERRORS_SCIP_RUBY_H
+#include "core/Error.h"
+
+namespace sorbet::scip_indexer::errors {
+constexpr sorbet::core::ErrorClass SCIPRubyDebug{25900, sorbet::core::StrictLevel::False};
+constexpr sorbet::core::ErrorClass SCIPRuby{25901, sorbet::core::StrictLevel::False};
+} // namespace sorbet::scip_indexer::errors
+
+#endif // SORBET_CORE_ERRORS_SCIP_RUBY_H

docs/scip-ruby/CLI.md

@@ -20,11 +20,60 @@ For example, with Git, you can use the last tag
 string. For repos which index every commit, you could also use the SHA
 instead (`git rev-parse HEAD`).
 
+## `--gem-map-path <arg>`
+
+At the moment, scip-ruby requires an extra step for cross-repo
+code navigation; you need to supply information about which.
+file belongs to which gem explicitly, using a newline-delimited
+JSON file in the following format:
+
+<!-- 
+TODO: Uncomment this
+By default, `scip-ruby` will attempt to identify which gems the
+ingested files belong to based on the standard layout of paths
+as used by Bundler. If it can't identify the gems for certain files,
+it will print a warning. This may happen if you're using a custom
+build system or different filesystem layout.
+
+To get correct cross-repo code navigation, you can explicitly
+supply information about files and gems using a supplementary
+newline-delimited JSON file in the following format:
+-->
+```json
+{"path": "a/b/c.rb", "gem": "[email protected]"}
+{"path": "a/b/d.rb", "gem": "[email protected]"}
+{"path": "a/x/y.rb", "gem": "[email protected]"}
+...
+```
+
+Then pass the path to the JSON file:
+
+```bash
+scip-ruby --gem-map-path path/to/cross-repo-metadata.json
+```
+
+Paths are interpreted relative to the working directory
+for the `scip-ruby` invocation.
+
+If information about the gem being indexed
+cannot be inferred from the filesystem, then you can supply
+the `--gem-metadata` argument as described earlier.
+
+If you run into an error message where a path in the JSON file
+is not recognized by `scip-ruby`, you can re-run the indexing command
+with extra arguments `--log-recorded-filepaths --debug-log-file out.log`
+to identify differences between the JSON file
+and paths created by traversing directories.
+
+## `--index-file <arg>`
+
+The path for emitting the SCIP index. Defaults to `index.scip`.
+
 ## `--unquiet-errors`
 
 scip-ruby defaults to running in Sorbet's quiet mode, as scip-ruby supports
 indexing `# typed: false` files on a best-effort basis, but Sorbet may
 rightfully flag many errors in those files. The number of errors can be
 overwhelming if there is a large amount of untyped code.
 
-This flag restores Sorbet's default behavior.
+This flag restores Sorbet's default behavior.

scip_indexer/BUILD

@@ -61,6 +61,7 @@ cc_library(
         "@com_google_absl//absl/strings",
         "@com_google_absl//absl/synchronization",
         "@cxxopts",
+        "@rapidjson",
         "@spdlog",
     ],
 )

scip_indexer/Debug.cc

@@ -3,15 +3,14 @@
 
 #include "core/GlobalState.h"
 #include "core/Loc.h"
+#include "core/errors/scip_ruby.h"
 
 #include "scip_indexer/Debug.h"
 
 namespace sorbet::scip_indexer {
 
-constexpr sorbet::core::ErrorClass SCIPRubyDebug{25900, sorbet::core::StrictLevel::False};
-
 void _log_debug(const sorbet::core::GlobalState &gs, sorbet::core::Loc loc, std::string s) {
-    if (auto e = gs.beginError(loc, SCIPRubyDebug)) {
+    if (auto e = gs.beginError(loc, errors::SCIPRubyDebug)) {
         auto lines = absl::StrSplit(s, '\n');
         for (auto line = lines.begin(); line != lines.end(); line++) {
             auto text = std::string(line->begin(), line->length());

scip_indexer/Debug.h

@@ -55,8 +55,6 @@ template <typename V, typename Fn> std::string showVec(const V &v, Fn f) {
     return out.str();
 }
 
-extern const sorbet::core::ErrorClass SCIPRubyDebug;
-
 void _log_debug(const sorbet::core::GlobalState &gs, sorbet::core::Loc loc, std::string s);
 } // namespace sorbet::scip_indexer
 

scip_indexer/SCIPGemMetadata.cc

@@ -8,6 +8,15 @@
 #include <vector>
 
 #include "absl/strings/str_replace.h"
+
+#include "common/FileSystem.h"
+#include "common/common.h"
+#include "core/FileRef.h"
+#include "core/errors/scip_ruby.h"
+
+// Uses ENFORCE as defined by common/common.h, so put this later
+#include "rapidjson/document.h"
+
 #include "scip_indexer/SCIPGemMetadata.h"
 
 using namespace std;
@@ -169,4 +178,95 @@ pair<GemMetadata, vector<GemMetadataError>> GemMetadata::readFromConfig(const Fi
     return {GemMetadata(name.value_or(currentDirName()), version.value_or("latest")), errors};
 }
 
+GemMapping::GemMapping() {
+    // The 'ruby' namespace is reserved by RubyGems.org, so we won't run into any
+    // actual gem called 'ruby', so we use that here. 'stdlib' would not be appropriate
+    // as Sorbet contains RBIs for both core and stdlib modules.
+    // For the version, we're not really doing any versioning for core & stdlib RBIs,
+    // (e.g. handling both 2.7 and 3) so let's stick to latest for now.
+    this->stdlibGem = make_shared<GemMetadata>(GemMetadata::tryParse("ruby@latest").value());
+}
+
+optional<shared_ptr<GemMetadata>> GemMapping::lookupGemForFile(const core::GlobalState &gs, core::FileRef file) const {
+    ENFORCE(file.exists());
+    auto it = this->map.find(file);
+    if (it != this->map.end()) {
+        return it->second;
+    }
+    auto filepath = file.data(gs).path();
+    if (absl::StartsWith(filepath, core::File::URL_PREFIX)) {
+        return this->stdlibGem;
+    }
+    if (this->currentGem.has_value()) {
+        // TODO Should we enforce here in debug builds?
+        // Fallback to this if set, to avoid collisions with other gems.
+        return this->currentGem.value();
+    }
+    return nullopt;
+}
+
+void GemMapping::populateFromNDJSON(const core::GlobalState &gs, const FileSystem &fs, const std::string &ndjsonPath) {
+    istringstream input(fs.readFile(ndjsonPath));
+    auto currentDir = filesystem::path(fs.getCurrentDir());
+    unsigned errorCount = 0;
+    for (string line; getline(input, line);) {
+        auto jsonLine = std::string(absl::StripAsciiWhitespace(line));
+        if (jsonLine.empty()) {
+            continue;
+        }
+        rapidjson::Document document;
+        document.Parse(jsonLine);
+        if (document.HasParseError() || !document.IsObject()) {
+            continue;
+        }
+        if (document.HasMember("path") && document["path"].IsString() && document.HasMember("gem") &&
+            document["gem"].IsString()) {
+            auto jsonPath = document["path"].GetString();
+            auto fileRef = gs.findFileByPath(jsonPath);
+            if (!fileRef.exists()) {
+                vector<string> alternatePaths;
+                auto path = filesystem::path(jsonPath);
+                // [NOTE: scip-ruby-path-normalization]
+                // We normalize paths upon entry into GlobalState, so it is sufficient to only check
+                // different normalized combinations here. One common situation where normalization is
+                // necessary is if you pass '.' as the directory argument, without normalization,
+                // the entered paths would be './'-prefixed, but you could reasonably forget to add
+                // the same in the JSON file.
+                auto normalizedPath = path.lexically_normal();
+                if (normalizedPath != path) {
+                    alternatePaths.push_back(normalizedPath);
+                }
+                if (normalizedPath.is_absolute()) {
+                    alternatePaths.push_back(string(normalizedPath.lexically_relative(currentDir).lexically_normal()));
+                } else {
+                    alternatePaths.push_back(string((currentDir / normalizedPath).lexically_normal()));
+                }
+                bool foundFile = absl::c_any_of(alternatePaths, [&](auto &altPath) -> bool {
+                    fileRef = gs.findFileByPath(altPath);
+                    return fileRef.exists();
+                });
+                if (!foundFile) {
+                    errorCount++;
+                    if (errorCount < 5) {
+                        if (auto e = gs.beginError(core::Loc(), errors::SCIPRuby)) {
+                            e.setHeader("File path in JSON map doesn't match known paths: {}", jsonPath);
+                            e.addErrorNote("Use --debug-log-filepaths to check the paths being recorded by scip-ruby");
+                        }
+                    }
+                    continue;
+                }
+            }
+            auto gemMetadata = GemMetadata::tryParse(document["gem"].GetString());
+            if (!gemMetadata.has_value()) {
+                continue;
+            }
+            this->map[fileRef] = make_shared<GemMetadata>(move(gemMetadata.value()));
+        }
+    }
+}
+
+void GemMapping::markCurrentGem(GemMetadata gem) {
+    this->currentGem = make_shared<GemMetadata>(gem);
+}
+
 } // namespace sorbet::scip_indexer

scip_indexer/SCIPGemMetadata.h

@@ -1,13 +1,17 @@
 #ifndef SORBET_SCIP_GEM_METADATA
 #define SORBET_SCIP_GEM_METADATA
 
+#include <memory>
 #include <optional>
 #include <string>
 #include <utility>
 
 #include "absl/strings/str_split.h"
 
 #include "common/FileSystem.h"
+#include "common/common.h"
+#include "core/FileRef.h"
+#include "core/GlobalState.h"
 
 namespace sorbet::scip_indexer {
 
@@ -71,6 +75,23 @@ class GemMetadata final {
     static std::pair<GemMetadata, std::vector<GemMetadataError>> readFromGemspec(const std::string &);
 };
 
+// Type carrying gem information for each file, which is used during
+// symbol emission to ensure correct symbol names for cross-repo.
+class GemMapping final {
+    std::shared_ptr<GemMetadata> stdlibGem;
+    std::optional<std::shared_ptr<GemMetadata>> currentGem;
+    UnorderedMap<core::FileRef, std::shared_ptr<GemMetadata>> map;
+
+public:
+    GemMapping();
+
+    std::optional<std::shared_ptr<GemMetadata>> lookupGemForFile(const core::GlobalState &gs, core::FileRef file) const;
+
+    void populateFromNDJSON(const core::GlobalState &, const FileSystem &fs, const std::string &ndjsonPath);
+
+    void markCurrentGem(GemMetadata gem);
+};
+
 } // namespace sorbet::scip_indexer
 
 #endif // SORBET_SCIP_GEM_METADATA