Proposed syntax APIs

clang-tools-extra/unittests/clangd/Annotations.cpp

@@ -12,74 +12,41 @@
 namespace clang {
 namespace clangd {
 
-// Crash if the assertion fails, printing the message and testcase.
-// More elegant error handling isn't needed for unit tests.
-static void require(bool Assertion, const char *Msg, llvm::StringRef Code) {
-  if (!Assertion) {
-    llvm::errs() << "Annotated testcase: " << Msg << "\n" << Code << "\n";
-    llvm_unreachable("Annotated testcase assertion failed!");
-  }
+Position Annotations::point(llvm::StringRef Name) const {
+  return offsetToPosition(code(), Base::point(Name));
 }
 
-Annotations::Annotations(llvm::StringRef Text) {
-  auto Here = [this] { return offsetToPosition(Code, Code.size()); };
-  auto Require = [Text](bool Assertion, const char *Msg) {
-    require(Assertion, Msg, Text);
-  };
-  llvm::Optional<llvm::StringRef> Name;
-  llvm::SmallVector<std::pair<llvm::StringRef, Position>, 8> OpenRanges;
+std::vector<Position> Annotations::points(llvm::StringRef Name) const {
+  auto Offsets = Base::points(Name);
 
-  Code.reserve(Text.size());
-  while (!Text.empty()) {
-    if (Text.consume_front("^")) {
-      Points[Name.getValueOr("")].push_back(Here());
-      Name = None;
-      continue;
-    }
-    if (Text.consume_front("[[")) {
-      OpenRanges.emplace_back(Name.getValueOr(""), Here());
-      Name = None;
-      continue;
-    }
-    Require(!Name, "$name should be followed by ^ or [[");
-    if (Text.consume_front("]]")) {
-      Require(!OpenRanges.empty(), "unmatched ]]");
-      Ranges[OpenRanges.back().first].push_back(
-          {OpenRanges.back().second, Here()});
-      OpenRanges.pop_back();
-      continue;
-    }
-    if (Text.consume_front("$")) {
-      Name = Text.take_while(llvm::isAlnum);
-      Text = Text.drop_front(Name->size());
-      continue;
-    }
-    Code.push_back(Text.front());
-    Text = Text.drop_front();
-  }
-  Require(!Name, "unterminated $name");
-  Require(OpenRanges.empty(), "unmatched [[");
-}
+  std::vector<Position> Ps;
+  Ps.reserve(Offsets.size());
+  for (size_t O : Offsets)
+    Ps.push_back(offsetToPosition(code(), O));
 
-Position Annotations::point(llvm::StringRef Name) const {
-  auto I = Points.find(Name);
-  require(I != Points.end() && I->getValue().size() == 1,
-          "expected exactly one point", Code);
-  return I->getValue()[0];
+  return Ps;
 }
-std::vector<Position> Annotations::points(llvm::StringRef Name) const {
-  auto P = Points.lookup(Name);
-  return {P.begin(), P.end()};
+
+static clangd::Range toLSPRange(llvm::StringRef Code, llvm::Range R) {
+  clangd::Range LSPRange;
+  LSPRange.start = offsetToPosition(Code, R.Begin);
+  LSPRange.end = offsetToPosition(Code, R.End);
+  return LSPRange;
 }
-Range Annotations::range(llvm::StringRef Name) const {
-  auto I = Ranges.find(Name);
-  require(I != Ranges.end() && I->getValue().size() == 1,
-          "expected exactly one range", Code);
-  return I->getValue()[0];
+
+clangd::Range Annotations::range(llvm::StringRef Name) const {
+  return toLSPRange(code(), Base::range(Name));
 }
-std::vector<Range> Annotations::ranges(llvm::StringRef Name) const {
-  auto R = Ranges.lookup(Name);
-  return {R.begin(), R.end()};
+
+std::vector<clangd::Range> Annotations::ranges(llvm::StringRef Name) const {
+  auto OffsetRanges = Base::ranges(Name);
+
+  std::vector<clangd::Range> Rs;
+  Rs.reserve(OffsetRanges.size());
+  for (llvm::Range R : OffsetRanges)
+    Rs.push_back(toLSPRange(code(), R));
+
+  return Rs;
 }
 
 } // namespace clangd

clang-tools-extra/unittests/clangd/Annotations.h

@@ -5,64 +5,32 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
-//
-// Annotations lets you mark points and ranges inside source code, for tests:
-//
-//    Annotations Example(R"cpp(
-//       int complete() { x.pri^ }          // ^ indicates a point
-//       void err() { [["hello" == 42]]; }  // [[this is a range]]
-//       $definition^class Foo{};           // points can be named: "definition"
-//       $fail[[static_assert(false, "")]]  // ranges can be named too: "fail"
-//    )cpp");
-//
-//    StringRef Code = Example.code();              // annotations stripped.
-//    std::vector<Position> PP = Example.points();  // all unnamed points
-//    Position P = Example.point();                 // there must be exactly one
-//    Range R = Example.range("fail");              // find named ranges
-//
-// Points/ranges are coordinates into `code()` which is stripped of annotations.
-//
-// Ranges may be nested (and points can be inside ranges), but there's no way
-// to define general overlapping ranges.
-//
+// A clangd-specific version of llvm/Testing/Support/Annotations.h, replaces
+// offsets and offset-based ranges with types from the LSP protocol.
 //===---------------------------------------------------------------------===//
 
 #ifndef LLVM_CLANG_TOOLS_EXTRA_UNITTESTS_CLANGD_ANNOTATIONS_H
 #define LLVM_CLANG_TOOLS_EXTRA_UNITTESTS_CLANGD_ANNOTATIONS_H
 
 #include "Protocol.h"
-#include "llvm/ADT/SmallVector.h"
-#include "llvm/ADT/StringMap.h"
-#include "llvm/ADT/StringRef.h"
+#include "llvm/Testing/Support/Annotations.h"
 
 namespace clang {
 namespace clangd {
 
-class Annotations {
-public:
-  // Parses the annotations from Text. Crashes if it's malformed.
-  Annotations(llvm::StringRef Text);
+/// Same as llvm::Annotations, but adjusts functions to LSP-specific types for
+/// positions and ranges.
+class Annotations : public llvm::Annotations {
+  using Base = llvm::Annotations;
 
-  // The input text with all annotations stripped.
-  // All points and ranges are relative to this stripped text.
-  llvm::StringRef code() const { return Code; }
+public:
+  using llvm::Annotations::Annotations;
 
-  // Returns the position of the point marked by ^ (or $name^) in the text.
-  // Crashes if there isn't exactly one.
   Position point(llvm::StringRef Name = "") const;
-  // Returns the position of all points marked by ^ (or $name^) in the text.
   std::vector<Position> points(llvm::StringRef Name = "") const;
 
-  // Returns the location of the range marked by [[ ]] (or $name[[ ]]).
-  // Crashes if there isn't exactly one.
-  Range range(llvm::StringRef Name = "") const;
-  // Returns the location of all ranges marked by [[ ]] (or $name[[ ]]).
-  std::vector<Range> ranges(llvm::StringRef Name = "") const;
-
-private:
-  std::string Code;
-  llvm::StringMap<llvm::SmallVector<Position, 1>> Points;
-  llvm::StringMap<llvm::SmallVector<Range, 1>> Ranges;
+  clangd::Range range(llvm::StringRef Name = "") const;
+  std::vector<clangd::Range> ranges(llvm::StringRef Name = "") const;
 };
 
 } // namespace clangd

clang/include/clang/AST/ExprCXX.h

@@ -1213,7 +1213,7 @@ class CXXTemporary {
 ///     const S &s_ref = S(); // Requires a CXXBindTemporaryExpr.
 ///   }
 /// \endcode
-class CXXBindTemporaryExpr : public Expr {
+class CXXBindTemporaryExpr final : public Expr {
   CXXTemporary *Temp = nullptr;
   Stmt *SubExpr = nullptr;
 

clang/include/clang/AST/RecursiveASTVisitor.h

@@ -2090,6 +2090,9 @@ DEF_TRAVERSE_DECL(NonTypeTemplateParmDecl, {
 DEF_TRAVERSE_DECL(ParmVarDecl, {
   TRY_TO(TraverseVarHelper(D));
 
+  if (!shouldVisitImplicitCode() && D->hasInheritedDefaultArg())
+    return true;
+
   if (D->hasDefaultArg() && D->hasUninstantiatedDefaultArg() &&
       !D->hasUnparsedDefaultArg())
     TRY_TO(TraverseStmt(D->getUninstantiatedDefaultArg()));

clang/include/clang/Lex/Preprocessor.h

@@ -33,6 +33,7 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/FoldingSet.h"
+#include "llvm/ADT/FunctionExtras.h"
 #include "llvm/ADT/None.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/PointerUnion.h"
@@ -48,8 +49,8 @@
 #include <cassert>
 #include <cstddef>
 #include <cstdint>
-#include <memory>
 #include <map>
+#include <memory>
 #include <string>
 #include <utility>
 #include <vector>
@@ -124,6 +125,7 @@ class Preprocessor {
   friend class VAOptDefinitionContext;
   friend class VariadicMacroScopeGuard;
 
+  llvm::unique_function<void(const clang::Token &)> OnToken;
   std::shared_ptr<PreprocessorOptions> PPOpts;
   DiagnosticsEngine        *Diags;
   LangOptions       &LangOpts;
@@ -911,6 +913,14 @@ class Preprocessor {
   }
   /// \}
 
+  /// Register a function that would be called on each token seen by the
+  /// preprocessor. This is a very low-level hook, the produced token stream is
+  /// tied to the internals of the preprocessor so interpreting result of the
+  /// callback is hard.
+  void setTokenWatcher(llvm::unique_function<void(const clang::Token &)> F) {
+    OnToken = std::move(F);
+  }
+
   bool isMacroDefined(StringRef Id) {
     return isMacroDefined(&Identifiers.get(Id));
   }

clang/include/clang/Tooling/Syntax/Corpus.h

@@ -0,0 +1,64 @@
+//===- Corpus.h - memory arena and bookkeeping for syntax trees --- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_CLANG_TOOLING_SYNTAX_CORPUS_H
+#define LLVM_CLANG_TOOLING_SYNTAX_CORPUS_H
+
+#include "clang/Basic/LangOptions.h"
+#include "clang/Basic/SourceManager.h"
+#include "clang/Tooling/Syntax/TokenBuffer.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/Support/Allocator.h"
+
+namespace clang {
+namespace syntax {
+/// A corpus is a memory arena for a set of syntax trees. In addition, it also
+/// tracks the underlying token buffers, manager, etc.
+class Corpus {
+public:
+  Corpus(SourceManager &SourceMgr, const LangOptions &LangOpts,
+         TokenBuffer MainFile);
+
+  const SourceManager &sourceManager() const { return SourceMgr; }
+  const LangOptions &langOptions() const { return LangOpts; }
+
+  /// Construct a new syntax node of a specified kind. The memory for a node is
+  /// owned by the corpus and will be freed when the corpus is destroyed.
+  template <class TNode, class... Args> TNode *construct(Args &&... As) {
+    static_assert(std::is_trivially_destructible<TNode>::value,
+                  "nodes should be trivially destructible");
+    return new (Allocator) TNode(std::forward<Args>(As)...);
+  }
+
+  const TokenBuffer *findBuffer(FileID FID) const;
+  const TokenBuffer *findBuffer(SourceLocation Loc) const;
+
+  const TokenBuffer &mainFile() const;
+
+  llvm::BumpPtrAllocator &allocator() { return Allocator; }
+
+  /// Get a text for a continuous range of tokens.
+  /// EXPECTS: Tokens are a part of some tokenized buffer from \p Buffers.
+  llvm::StringRef getText(llvm::ArrayRef<syntax::Token> Tokens);
+
+  /// Tokenize the passed buffer. This function runs a lexer in raw mode, i.e.
+  /// the result won't contain any macro expansions, etc.
+  std::pair<FileID, const TokenBuffer &>
+  tokenizeBuffer(std::unique_ptr<llvm::MemoryBuffer> Buffer);
+
+private:
+  SourceManager &SourceMgr;
+  const LangOptions &LangOpts;
+  llvm::DenseMap<FileID, TokenBuffer> Buffers;
+  /// Keeps all the allocated nodes.
+  llvm::BumpPtrAllocator Allocator;
+};
+
+} // namespace syntax
+} // namespace clang
+
+#endif

clang/include/clang/Tooling/Syntax/NodeList.h

@@ -0,0 +1,87 @@
+//===- NodeList.h ---------------------------------------------*- C++ -*-=====//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLING_SYNTAX_NODELIST_H
+#define LLVM_CLANG_TOOLING_SYNTAX_NODELIST_H
+
+#include "llvm/Support/Allocator.h"
+#include <algorithm>
+#include <type_traits>
+
+namespace clang {
+namespace syntax {
+namespace detail {
+/// A vector that requests memory from BumpPtrAllocator and has a trivial
+/// destructor. Syntax tree nodes use it to store children.
+template <class T> class BumpAllocVector {
+  // Make sure the elements are allowed to drop destructors.
+  static_assert(std::is_trivially_destructible<T>::value,
+                "T must be trivially destructible");
+  // Implementation for trivially-copyable types is much simpler.
+  static_assert(std::is_trivially_copyable<T>::value,
+                "T must be trivially copyable.");
+
+public:
+  T *begin() { return Begin; }
+  T *end() { return End; }
+
+  const T *begin() const { return Begin; }
+  const T *end() const { return End; }
+
+  size_t size() const { return End - Begin; }
+  bool empty() const { return begin() == end(); }
+
+  void push_back(llvm::BumpPtrAllocator &A, T Element) {
+    if (StorageEnd == End)
+      Grow(A);
+    *End = Element;
+    ++End;
+  }
+
+  void erase(T *Begin, T *End) {
+    std::ptrdiff_t ErasedSize = End - Begin;
+    for (auto *It = End; It != this->End; ++It) {
+      *Begin = It;
+      ++Begin;
+    }
+    End -= ErasedSize;
+  }
+
+private:
+  void Grow(llvm::BumpPtrAllocator &A) {
+    size_t Size = End - Begin;
+
+    size_t NewCapacity = 2 * (StorageEnd - Begin);
+    if (NewCapacity == 0)
+      NewCapacity = 1;
+
+    T *NewStorage = A.Allocate<T>(NewCapacity);
+    std::copy(Begin, End, NewStorage);
+
+    A.Deallocate(Begin, StorageEnd - Begin);
+
+    Begin = NewStorage;
+    End = NewStorage + Size;
+    StorageEnd = NewStorage + NewCapacity;
+  }
+
+private:
+  T *Begin = nullptr;
+  T *End = nullptr;
+  T *StorageEnd = nullptr;
+};
+} // namespace detail
+
+class Node;
+/// Like vector<Node*>, but allocates all the memory in the BumpPtrAllocator.
+/// Can be dropped without running the destructor.
+using NodeList = detail::BumpAllocVector<Node *>;
+} // namespace syntax
+} // namespace clang
+
+#endif