WIP: Add support for manually specifiying gems for cross-repo.

Author: varungandhi-src

common/common.cc

@@ -90,7 +90,7 @@ bool sorbet::FileOps::ensureDir(string_view path) {
 
 std::string sorbet::FileOps::getCurrentDir() {
     char buf[MAXPATHLEN + 1];
-    getcwd(buf, sizeof(buf));
+    (void)getcwd(buf, sizeof(buf));
     return std::string(buf);
 }
 

core/GlobalState.cc

@@ -1663,6 +1663,7 @@ FileRef GlobalState::enterFile(const shared_ptr<File> &file) {
     }
 
     files.emplace_back(file);
+    // GlobalState initialization guarantees the 0 slot will be taken, so this is OK.
     auto ret = FileRef(filesUsed() - 1);
     fileRefByPath[string(file->path())] = ret;
     return ret;

docs/scip-ruby/CLI.md

@@ -20,11 +20,51 @@ For example, with Git, you can use the last tag
 string. For repos which index every commit, you could also use the SHA
 instead (`git rev-parse HEAD`).
 
+## `--gem-map-path <arg>`
+
+At the moment, scip-ruby requires an extra step for cross-repo
+code navigation; you need to supply information about which.
+file belongs to which gem explicitly, using a newline-delimited
+JSON file in the following format:
+
+<!-- 
+TODO: Uncomment this
+By default, `scip-ruby` will attempt to identify which gems the
+ingested files belong to based on the standard layout of paths
+as used by Bundler. If it can't identify the gems for certain files,
+it will print a warning. This may happen if you're using a custom
+build system or different filesystem layout.
+
+To get correct cross-repo code navigation, you can explicitly
+supply information about files and gems using a supplementary
+newline-delimited JSON file in the following format:
+-->
+```json
+{"path": "a/b/c.rb", "gem": "[email protected]"}
+{"path": "a/b/d.rb", "gem": "[email protected]"}
+{"path": "a/x/y.rb", "gem": "[email protected]"}
+...
+```
+
+Then pass the path to the JSON file:
+
+```bash
+scip-ruby --gem-map-path path/to/cross-repo-metadata.json
+```
+
+If information about the gem being indexed is not present in the JSON file,
+and cannot be inferred from the filesystem, then you can supply
+the `--gem-metadata` argument as described earlier.
+
+## `--index-file <arg>`
+
+The path for emitting the SCIP index. Defaults to `index.scip`.
+
 ## `--unquiet-errors`
 
 scip-ruby defaults to running in Sorbet's quiet mode, as scip-ruby supports
 indexing `# typed: false` files on a best-effort basis, but Sorbet may
 rightfully flag many errors in those files. The number of errors can be
 overwhelming if there is a large amount of untyped code.
 
-This flag restores Sorbet's default behavior.
+This flag restores Sorbet's default behavior.

scip_indexer/BUILD

@@ -61,6 +61,7 @@ cc_library(
         "@com_google_absl//absl/strings",
         "@com_google_absl//absl/synchronization",
         "@cxxopts",
+        "@rapidjson",
         "@spdlog",
     ],
 )

scip_indexer/SCIPGemMetadata.cc

@@ -8,6 +8,13 @@
 #include <vector>
 
 #include "absl/strings/str_replace.h"
+
+#include "common/common.h"
+#include "core/FileRef.h"
+
+// Uses ENFORCE as defined by common/common.h, so put this later
+#include "rapidjson/document.h"
+
 #include "scip_indexer/SCIPGemMetadata.h"
 
 using namespace std;
@@ -169,4 +176,61 @@ pair<GemMetadata, vector<GemMetadataError>> GemMetadata::readFromConfig(const Fi
     return {GemMetadata(name.value_or(currentDirName()), version.value_or("latest")), errors};
 }
 
+GemMapping::GemMapping() {
+    // The 'ruby' namespace is reserved by RubyGems.org, so we can use that here.
+    // Unfortunately, there are already gems which use the names stdlib and ruby-stdlib,
+    // so not using those here.
+    // FIXME: How do we get the version reliably?
+    this->stdlibGem = make_shared<GemMetadata>(GemMetadata::tryParse("[email protected]").value());
+}
+
+optional<shared_ptr<GemMetadata>> GemMapping::lookupGemForFile(const core::GlobalState &gs, core::FileRef file) const {
+    auto it = this->map.find(file);
+    if (it != this->map.end()) {
+        return it->second;
+    }
+    auto filepath = file.data(gs).path();
+    if (absl::StartsWith(filepath, core::File::URL_PREFIX)) {
+        return this->stdlibGem;
+    }
+    if (this->currentGem.has_value()) {
+        // TODO Should we enforce here in debug builds?
+        // Fallback to this if set, to avoid collisions with other gems.
+        return this->currentGem.value();
+    }
+    return nullopt;
+}
+
+void GemMapping::populateFromNDJSON(const core::GlobalState &gs, const std::string &ndjsonText) {
+    istringstream input(ndjsonText);
+    for (string line; getline(input, line);) {
+        auto jsonLine = std::string(absl::StripAsciiWhitespace(line));
+        if (jsonLine.empty()) {
+            continue;
+        }
+        rapidjson::Document document;
+        document.Parse(jsonLine);
+        if (document.HasParseError() || !document.IsObject()) {
+            continue;
+        }
+        if (document.HasMember("path") && document["path"].IsString() && document.HasMember("gem") &&
+            document["gem"].IsString()) {
+            // TODO: handle path normalization
+            auto fileRef = gs.findFileByPath(document["path"].GetString());
+            if (!fileRef.exists()) {
+                continue;
+            }
+            auto gemMetadata = GemMetadata::tryParse(document["gem"].GetString());
+            if (!gemMetadata.has_value()) {
+                continue;
+            }
+            this->map[fileRef] = make_shared<GemMetadata>(move(gemMetadata.value()));
+        }
+    }
+}
+
+void GemMapping::markCurrentGem(GemMetadata gem) {
+    this->currentGem = make_shared<GemMetadata>(gem);
+}
+
 } // namespace sorbet::scip_indexer

scip_indexer/SCIPGemMetadata.h

@@ -1,13 +1,17 @@
 #ifndef SORBET_SCIP_GEM_METADATA
 #define SORBET_SCIP_GEM_METADATA
 
+#include <memory>
 #include <optional>
 #include <string>
 #include <utility>
 
 #include "absl/strings/str_split.h"
 
 #include "common/FileSystem.h"
+#include "common/common.h"
+#include "core/FileRef.h"
+#include "core/GlobalState.h"
 
 namespace sorbet::scip_indexer {
 
@@ -71,6 +75,23 @@ class GemMetadata final {
     static std::pair<GemMetadata, std::vector<GemMetadataError>> readFromGemspec(const std::string &);
 };
 
+// Type carrying gem information for each file, which is used during
+// symbol emission to ensure correct symbol names for cross-repo.
+class GemMapping final {
+    std::shared_ptr<GemMetadata> stdlibGem;
+    std::optional<std::shared_ptr<GemMetadata>> currentGem;
+    UnorderedMap<core::FileRef, std::shared_ptr<GemMetadata>> map;
+
+public:
+    GemMapping();
+
+    std::optional<std::shared_ptr<GemMetadata>> lookupGemForFile(const core::GlobalState &gs, core::FileRef file) const;
+
+    void populateFromNDJSON(const core::GlobalState &, const std::string &ndjsonContents);
+
+    void markCurrentGem(GemMetadata gem);
+};
+
 } // namespace sorbet::scip_indexer
 
 #endif // SORBET_SCIP_GEM_METADATA

scip_indexer/SCIPIndexer.cc

@@ -181,7 +181,7 @@ class SCIPState {
     //
     // FIXME(varun): This seems redundant, get rid of it.
 
-    GemMetadata gemMetadata;
+    GemMapping gemMap;
 
 public:
     UnorderedMap<core::FileRef, vector<scip::Occurrence>> occurrenceMap;
@@ -207,9 +207,9 @@ class SCIPState {
     vector<scip::SymbolInformation> externalSymbols;
 
 public:
-    SCIPState(GemMetadata metadata)
+    SCIPState(GemMapping gemMap)
         : symbolStringCache(), localOccurrenceCache(), symbolOccurrenceCache(), potentialRefOnlySymbols(),
-          gemMetadata(metadata), occurrenceMap(), emittedSymbols(), symbolMap(), documents(), externalSymbols() {}
+          gemMap(gemMap), occurrenceMap(), emittedSymbols(), symbolMap(), documents(), externalSymbols() {}
     ~SCIPState() = default;
     SCIPState(SCIPState &&) = default;
     SCIPState &operator=(SCIPState &&other) = default;
@@ -241,7 +241,7 @@ class SCIPState {
             status = scip::utils::emitSymbolString(*symbol, output);
         } else {
             scip::Symbol symbol;
-            status = symRef.symbolForExpr(gs, this->gemMetadata, {}, symbol);
+            status = symRef.symbolForExpr(gs, this->gemMap, {}, symbol);
             if (!status.ok()) {
                 return status;
             }
@@ -402,7 +402,7 @@ class SCIPState {
         auto occLoc = loc.has_value() ? core::Loc(file, loc.value()) : symRef.symbolLoc(gs);
         scip::Symbol symbol;
         auto untypedSymRef = symRef.withoutType();
-        auto status = untypedSymRef.symbolForExpr(gs, this->gemMetadata, occLoc, symbol);
+        auto status = untypedSymRef.symbolForExpr(gs, this->gemMap, occLoc, symbol);
         if (!status.ok()) {
             return status;
         }
@@ -1114,18 +1114,40 @@ namespace sorbet::pipeline::semantic_extension {
 using LocalSymbolTable = UnorderedMap<core::LocalVariable, core::Loc>;
 
 class SCIPSemanticExtension : public SemanticExtension {
-public:
     string indexFilePath;
-    string gemMetadataString;
-    scip_indexer::GemMetadata gemMetadata;
+    string gemMetadataArg;
+    string gemMapPath;
+    scip_indexer::GemMapping gemMap;
 
     using SCIPState = sorbet::scip_indexer::SCIPState;
 
+public:
+    using StateMap = UnorderedMap<thread::id, shared_ptr<SCIPState>>;
+
+private:
     mutable struct {
-        UnorderedMap<thread::id, shared_ptr<SCIPState>> states;
+        StateMap states;
         absl::Mutex mtx;
     } mutableState;
 
+public:
+    SCIPSemanticExtension(string indexFilePath, string gemMetadataArg, string gemMapPath,
+                          scip_indexer::GemMapping gemMap, StateMap states)
+        : indexFilePath(indexFilePath), gemMetadataArg(gemMetadataArg), gemMapPath(gemMapPath),
+          gemMap(gemMap), mutableState{states, {}} {}
+
+    ~SCIPSemanticExtension() {}
+
+    virtual unique_ptr<SemanticExtension> deepCopy(const core::GlobalState &from, core::GlobalState &to) override {
+        // FIXME: Technically, this would copy the state too, but we haven't implemented
+        // deep-copying for it because it doesn't matter for scip-ruby.
+        StateMap map;
+        return make_unique<SCIPSemanticExtension>(this->indexFilePath, this->gemMetadataArg, this->gemMapPath,
+                                                  this->gemMap, map);
+    };
+    virtual void merge(const core::GlobalState &from, core::GlobalState &to, core::NameSubstitution &subst) override{};
+
+private:
     // Return a shared_ptr here as we need a stable address to avoid an invalid reference
     // on table resizing, and we need to maintain at least two pointers,
     // an "owning ref" from the table and a mutable borrow from the caller.
@@ -1141,24 +1163,26 @@ class SCIPSemanticExtension : public SemanticExtension {
 
             // We will move the state out later, so use a no-op deleter.
             return mutableState.states[this_thread::get_id()] =
-                       shared_ptr<SCIPState>(new SCIPState(gemMetadata), [](SCIPState *) {});
+                       shared_ptr<SCIPState>(new SCIPState(gemMap), [](SCIPState *) {});
         }
     }
 
     bool doNothing() const {
         return this->indexFilePath.empty();
     }
 
+public:
     void run(core::MutableContext &ctx, ast::ClassDef *cd) const override {}
 
     virtual void prepareForTypechecking(const core::GlobalState &gs) override {
-        auto maybeMetadata = scip_indexer::GemMetadata::tryParse(this->gemMetadataString);
+        auto maybeMetadata = scip_indexer::GemMetadata::tryParse(this->gemMetadataArg);
+        scip_indexer::GemMetadata currentGem;
         if (maybeMetadata.has_value()) {
-            this->gemMetadata = maybeMetadata.value();
+            currentGem = maybeMetadata.value();
         } // TODO: Issue error for incorrect format in string...
-        if (this->gemMetadata.name().empty() || this->gemMetadata.version().empty()) {
-            auto [metadata, errors] = scip_indexer::GemMetadata::readFromConfig(OSFileSystem());
-            this->gemMetadata = metadata;
+        if (currentGem.name().empty() || currentGem.version().empty()) {
+            auto [gem, errors] = scip_indexer::GemMetadata::readFromConfig(OSFileSystem());
+            currentGem = gem;
             for (auto &error : errors) {
                 if (auto e = gs.beginError(core::Loc(), scip_indexer::SCIPRubyDebug)) {
                     e.setHeader("{}: {}",
@@ -1167,6 +1191,10 @@ class SCIPSemanticExtension : public SemanticExtension {
                 }
             }
         }
+        this->gemMap.markCurrentGem(currentGem);
+        if (!this->gemMapPath.empty()) {
+            this->gemMap.populateFromNDJSON(gs, OSFileSystem().readFile(this->gemMapPath));
+        }
     };
 
     virtual void finishTypecheckFile(const core::GlobalState &gs, const core::FileRef &file) const override {
@@ -1269,27 +1297,25 @@ class SCIPSemanticExtension : public SemanticExtension {
         traversal.traverse(cfg);
         scipStateRef.clearFunctionLocalCaches();
     }
-
-    virtual unique_ptr<SemanticExtension> deepCopy(const core::GlobalState &from, core::GlobalState &to) override {
-        return make_unique<SCIPSemanticExtension>(this->indexFilePath, this->gemMetadataString, this->gemMetadata);
-    };
-    virtual void merge(const core::GlobalState &from, core::GlobalState &to, core::NameSubstitution &subst) override{};
-
-    SCIPSemanticExtension(string indexFilePath, string gemMetadataString, scip_indexer::GemMetadata gemMetadata)
-        : indexFilePath(indexFilePath), gemMetadataString(gemMetadataString), gemMetadata(gemMetadata), mutableState() {
-    }
-    ~SCIPSemanticExtension() {}
 };
 
 class SCIPSemanticExtensionProvider : public SemanticExtensionProvider {
 public:
     void injectOptions(cxxopts::Options &optsBuilder) const override {
-        optsBuilder.add_options("indexer")("index-file", "Output SCIP index to a directory, which must already exist",
-                                           cxxopts::value<string>())(
-            "gem-metadata",
+        // clang-format off
+        optsBuilder.add_options("indexer")
+            ("index-file",
+            "Output SCIP index to a directory, which must already exist",
+            cxxopts::value<string>())
+            ("gem-metadata",
             "Metadata in 'name@version' format to be used for cross-repository code navigation. For repositories which "
             "index every commit, the SHA should be used for the version instead of a git tag (or equivalent).",
+            cxxopts::value<string>())
+            ("gem-map-path",
+            "Path to newline delimited JSON file which describes how to map paths to gem name and versions. See the"
+            " scip-ruby docs on GitHub for information about the JSON schema.",
             cxxopts::value<string>());
+        // clang-format on
     };
     unique_ptr<SemanticExtension> readOptions(cxxopts::ParseResult &providedOptions) const override {
         if (providedOptions.count("version") > 0) {
@@ -1308,12 +1334,22 @@ class SCIPSemanticExtensionProvider : public SemanticExtensionProvider {
         } else {
             indexFilePath = "index.scip";
         }
-        auto gemMetadataString =
-            providedOptions.count("gem-metadata") > 0 ? providedOptions["gem-metadata"].as<string>() : "";
-        return make_unique<SCIPSemanticExtension>(indexFilePath, gemMetadataString, scip_indexer::GemMetadata());
+        string gemMapPath{};
+        if (providedOptions.count("gem-map-path") > 0) {
+            gemMapPath = providedOptions["gem-map-path"].as<string>();
+        }
+        string gemMetadataArg{};
+        if (providedOptions.count("gem-metadata") > 0) {
+            gemMetadataArg = providedOptions["gem-metadata"].as<string>();
+        }
+        scip_indexer::GemMapping gemMap{};
+        SCIPSemanticExtension::StateMap stateMap;
+        return make_unique<SCIPSemanticExtension>(indexFilePath, gemMetadataArg, gemMapPath, gemMap, stateMap);
     };
     virtual unique_ptr<SemanticExtension> defaultInstance() const override {
-        return make_unique<SCIPSemanticExtension>("index.scip", "", scip_indexer::GemMetadata());
+        scip_indexer::GemMapping gemMap{};
+        SCIPSemanticExtension::StateMap stateMap;
+        return make_unique<SCIPSemanticExtension>("index.scip", "", "", gemMap, stateMap);
     };
     static vector<SemanticExtensionProvider *> getProviders();
     virtual ~SCIPSemanticExtensionProvider() = default;