swift-reflection-dump support for unresolved opaque return types.

Resolve mangled names containing symbolic references to indirect opaque type descriptors from other
dylibs by demangling the referenced symbol name, like we do for other kinds of context descriptor.
Add an OpaqueArchetypeTypeRef that can represent unresolved opaque types in the Reflection library.

Author: jckarter

include/swift/Reflection/TypeRef.h

@@ -316,7 +316,8 @@ class TupleTypeRef final : public TypeRef {
 
 public:
   TupleTypeRef(std::vector<const TypeRef *> Elements, bool Variadic=false)
-    : TypeRef(TypeRefKind::Tuple), Elements(Elements), Variadic(Variadic) {}
+    : TypeRef(TypeRefKind::Tuple), Elements(std::move(Elements)),
+      Variadic(Variadic) {}
 
   template <typename Allocator>
   static const TupleTypeRef *create(Allocator &A,
@@ -338,6 +339,84 @@ class TupleTypeRef final : public TypeRef {
   }
 };
 
+class OpaqueArchetypeTypeRef final : public TypeRef {
+  std::string ID;
+  std::string Description;
+  unsigned Ordinal;
+  // Each ArrayRef in ArgumentLists references into the buffer owned by this
+  // vector, which must not be modified after construction.
+  std::vector<const TypeRef *> AllArgumentsBuf;
+  std::vector<ArrayRef<const TypeRef *>> ArgumentLists;
+  
+  static TypeRefID Profile(StringRef idString,
+                           StringRef description, unsigned ordinal,
+                           ArrayRef<ArrayRef<const TypeRef *>> argumentLists) {
+    TypeRefID ID;
+    ID.addString(idString);
+    ID.addInteger(ordinal);
+    for (auto argList : argumentLists) {
+      ID.addInteger(0u);
+      for (auto arg : argList)
+        ID.addPointer(arg);
+    }
+    
+    return ID;
+  }
+  
+public:
+  OpaqueArchetypeTypeRef(StringRef id,
+                         StringRef description, unsigned ordinal,
+                         ArrayRef<ArrayRef<const TypeRef *>> argumentLists)
+    : TypeRef(TypeRefKind::OpaqueArchetype),
+      ID(id), Description(description), Ordinal(ordinal)
+  {
+    std::vector<unsigned> argumentListLengths;
+    
+    for (auto argList : argumentLists) {
+      argumentListLengths.push_back(argList.size());
+      AllArgumentsBuf.insert(AllArgumentsBuf.end(),
+                             argList.begin(), argList.end());
+    }
+    auto *data = AllArgumentsBuf.data();
+    for (auto length : argumentListLengths) {
+      ArgumentLists.push_back(ArrayRef<const TypeRef *>(data, length));
+      data += length;
+    }
+    assert(data == AllArgumentsBuf.data() + AllArgumentsBuf.size());
+  }
+  
+  template <typename Allocator>
+  static const OpaqueArchetypeTypeRef *create(Allocator &A,
+                                     StringRef id, StringRef description,
+                                     unsigned ordinal,
+                                     ArrayRef<ArrayRef<const TypeRef *>> arguments) {
+    FIND_OR_CREATE_TYPEREF(A, OpaqueArchetypeTypeRef,
+                           id, description, ordinal, arguments);
+  }
+
+  ArrayRef<ArrayRef<const TypeRef *>> getArgumentLists() const {
+    return ArgumentLists;
+  }
+  
+  unsigned getOrdinal() const {
+    return Ordinal;
+  }
+  
+  /// A stable identifier for the opaque type.
+  StringRef getID() const {
+    return ID;
+  }
+  
+  /// A human-digestible, but not necessarily stable, description of the opaque type.
+  StringRef getDescription() const {
+    return Description;
+  }
+  
+  static bool classof(const TypeRef *T) {
+    return T->getKind() == TypeRefKind::OpaqueArchetype;
+  }
+};
+
 class FunctionTypeRef final : public TypeRef {
   using Param = remote::FunctionParam<const TypeRef *>;
 

include/swift/Reflection/TypeRefBuilder.h

@@ -372,6 +372,7 @@ class TypeRefBuilder {
                     unsigned ordinal) {
     // TODO: Produce a type ref for the opaque type if the underlying type isn't
     // available.
+    opaqueDescriptor->dump();
 
     // Try to resolve to the underlying type, if we can.
     if (opaqueDescriptor->getKind() ==
@@ -392,7 +393,15 @@ class TypeRefBuilder {
 
       return underlyingTy->subst(*this, subs);
     }
-    return nullptr;
+    
+    // Otherwise, build a type ref that represents the opaque type.
+    return OpaqueArchetypeTypeRef::create(*this,
+                                          mangleNode(opaqueDescriptor,
+                                                     SymbolicResolver(),
+                                                     Dem),
+                                          nodeToString(opaqueDescriptor),
+                                          ordinal,
+                                          genericArgs);
   }
 
   const TupleTypeRef *

include/swift/Reflection/TypeRefs.def

@@ -30,6 +30,7 @@ TYPEREF(ForeignClass, TypeRef)
 TYPEREF(ObjCClass, TypeRef)
 TYPEREF(ObjCProtocol, TypeRef)
 TYPEREF(Opaque, TypeRef)
+TYPEREF(OpaqueArchetype, TypeRef)
 #define REF_STORAGE(Name, ...) \
   TYPEREF(Name##Storage, TypeRef)
 #include "swift/AST/ReferenceStorage.def"

include/swift/Remote/MetadataReader.h

@@ -1077,11 +1077,16 @@ class MetadataReader {
       demangledSymbol = demangledSymbol->getChild(0);
       assert(demangledSymbol->getKind() == Demangle::Node::Kind::Type);
       break;
-    // We don't handle pointers to other symbols yet.
-    // TODO: Opaque type descriptors could be useful.
+    // Pointers to opaque type descriptors demangle to the name of the opaque
+    // type declaration.
+    case Demangle::Node::Kind::OpaqueTypeDescriptor:
+      demangledSymbol = demangledSymbol->getChild(0);
+      break;
+      // We don't handle pointers to other symbols yet.
     default:
       return nullptr;
     }
+  
     return demangledSymbol;
   }
 

lib/Demangling/Demangler.cpp

@@ -716,8 +716,10 @@ NodePointer Demangler::demangleSymbolicReference(unsigned char rawKind,
     return nullptr;
 
   // Types register as substitutions even when symbolically referenced.
+  // OOPS: Except for opaque type references!
   if (kind == SymbolicReferenceKind::Context &&
-      resolved->getKind() != Node::Kind::OpaqueTypeDescriptorSymbolicReference)
+      resolved->getKind() != Node::Kind::OpaqueTypeDescriptorSymbolicReference &&
+      resolved->getKind() != Node::Kind::OpaqueReturnTypeOf)
     addSubstitution(resolved);
   return resolved;
 }

stdlib/public/Reflection/TypeLowering.cpp

@@ -1584,6 +1584,10 @@ class HasFixedSize
   bool visitOpaqueTypeRef(const OpaqueTypeRef *O) {
     return false;
   }
+
+  bool visitOpaqueArchetypeTypeRef(const OpaqueArchetypeTypeRef *O) {
+    return false;
+  }
 };
 
 bool TypeConverter::hasFixedSize(const TypeRef *TR) {
@@ -1705,6 +1709,10 @@ class HasSingletonMetatype
   MetatypeRepresentation visitOpaqueTypeRef(const OpaqueTypeRef *O) {
     return MetatypeRepresentation::Unknown;
   }
+
+  MetatypeRepresentation visitOpaqueArchetypeTypeRef(const OpaqueArchetypeTypeRef *O) {
+    return MetatypeRepresentation::Unknown;
+  }
 };
 
 class EnumTypeInfoBuilder {
@@ -2153,6 +2161,13 @@ class LowerType
     DEBUG_LOG(fprintf(stderr, "Can't lower opaque TypeRef"));
     return nullptr;
   }
+
+  const TypeInfo *visitOpaqueArchetypeTypeRef(const OpaqueArchetypeTypeRef *O) {
+    // TODO: Provide a hook for the client to try to resolve the opaque archetype
+    // with additional information?
+    DEBUG_LOG(fprintf(stderr, "Can't lower unresolved opaque archetype TypeRef"));
+    return nullptr;
+  }
 };
 
 const TypeInfo *TypeConverter::getTypeInfo(const TypeRef *TR) {

stdlib/public/Reflection/TypeRef.cpp

@@ -256,6 +256,22 @@ class PrintTypeRef : public TypeRefVisitor<PrintTypeRef, void> {
     fprintf(file, ")");
   }
 
+  void visitOpaqueArchetypeTypeRef(const OpaqueArchetypeTypeRef *O) {
+    printHeader("opaque_archetype");
+    printField("id", O->getID());
+    printField("description", O->getDescription());
+    fprintf(file, " ordinal %u ", O->getOrdinal());
+    for (auto argList : O->getArgumentLists()) {
+      fprintf(file, "\n");
+      fprintf(indent(Indent + 2), "args: <");
+      for (auto arg : argList) {
+        printRec(arg);
+      }
+      fprintf(file, ">");
+    }
+    fprintf(file, ")");
+  }
+
   void visitOpaqueTypeRef(const OpaqueTypeRef *O) {
     printHeader("opaque");
     fprintf(file, ")");
@@ -348,6 +364,10 @@ struct TypeRefIsConcrete
   bool visitOpaqueTypeRef(const OpaqueTypeRef *O) {
     return true;
   }
+    
+  bool visitOpaqueArchetypeTypeRef(const OpaqueArchetypeTypeRef *O) {
+    return false;
+  }
 
 #define REF_STORAGE(Name, name, ...) \
   bool visit##Name##StorageTypeRef(const Name##StorageTypeRef *US) { \
@@ -660,6 +680,36 @@ class DemanglingForTypeRef
   Demangle::NodePointer visitOpaqueTypeRef(const OpaqueTypeRef *O) {
     return Dem.createNode(Node::Kind::OpaqueType);
   }
+      
+  Demangle::NodePointer visitOpaqueArchetypeTypeRef(const OpaqueArchetypeTypeRef *O) {
+    auto decl = Dem.demangleSymbol(O->getID());
+    if (!decl)
+      return nullptr;
+    
+    auto index = Dem.createNode(Node::Kind::Index, O->getOrdinal());
+    
+    auto argNodeLists = Dem.createNode(Node::Kind::TypeList);
+    for (auto argList : O->getArgumentLists()) {
+      auto argNodeList = Dem.createNode(Node::Kind::TypeList);
+      
+      for (auto arg : argList) {
+        auto argNode = visit(arg);
+        if (!argNode)
+          return nullptr;
+        
+        argNodeList->addChild(argNode, Dem);
+      }
+      
+      argNodeLists->addChild(argNodeList, Dem);
+    }
+    
+    auto node = Dem.createNode(Node::Kind::OpaqueType);
+    node->addChild(decl, Dem);
+    node->addChild(index, Dem);
+    node->addChild(argNodeLists, Dem);
+    
+    return node;
+  }
 };
 
 Demangle::NodePointer TypeRef::getDemangling(Demangle::Demangler &Dem) const {
@@ -831,6 +881,11 @@ class ThickenMetatype
   const TypeRef *visitOpaqueTypeRef(const OpaqueTypeRef *O) {
     return O;
   }
+
+  const TypeRef *visitOpaqueArchetypeTypeRef(const OpaqueArchetypeTypeRef *O) {
+    return O;
+  }
+
 };
 
 static const TypeRef *
@@ -1009,6 +1064,21 @@ class TypeRefSubstitution
   const TypeRef *visitOpaqueTypeRef(const OpaqueTypeRef *O) {
     return O;
   }
+    
+  const TypeRef *visitOpaqueArchetypeTypeRef(const OpaqueArchetypeTypeRef *O) {
+    std::vector<const TypeRef *> newArgsBuffer;
+    for (auto argList : O->getArgumentLists()) {
+      for (auto arg : argList) {
+        newArgsBuffer.push_back(visit(arg));
+      }
+    }
+    
+    std::vector<ArrayRef<const TypeRef *>> newArgLists;
+    
+    return OpaqueArchetypeTypeRef::create(Builder, O->getID(), O->getDescription(),
+                                          O->getOrdinal(),
+                                          newArgLists);
+  }
 };
 
 const TypeRef *