[Android] Add callback to host-runtime contract for getting assembly data (#112705)

Add a new callback to `host_runtime_contract` that is called default assembly resolution:
```c++
// Probe the host for `path`. Sets pointer to data start and its size, if found.
// Returns true if found, false otherwise. If false, out parameter values are ignored by the runtime.
bool(HOST_CONTRACT_CALLTYPE* external_assembly_probe)(
    const char* path,
    /*out*/ void** data
    /*out*/ int64_t* size);
```

Contributes to #112706

This is a set of minimal changes required to make dotnet/android#9572 (the `.NET for Android` CoreCLR host) work with standard .NET for Android applications.

Author: grendello

eng/native/build-commons.sh

@@ -63,6 +63,20 @@ build_native()
     # All set to commence the build
     echo "Commencing build of \"$target\" target in \"$message\" for $__TargetOS.$__TargetArch.$__BuildType in $intermediatesDir"
 
+    SAVED_CFLAGS="${CFLAGS}"
+    SAVED_CXXFLAGS="${CXXFLAGS}"
+    SAVED_LDFLAGS="${LDFLAGS}"
+
+    # Let users provide additional compiler/linker flags via EXTRA_CFLAGS/EXTRA_CXXFLAGS/EXTRA_LDFLAGS.
+    # If users directly override CFLAG/CXXFLAGS/LDFLAGS, that may lead to some configure tests working incorrectly.
+    # See https://github.com/dotnet/runtime/issues/35727 for more information.
+    # 
+    # These flags MUST be exported before gen-buildsys.sh runs or cmake will ignore them
+    #
+    export CFLAGS="${CFLAGS} ${EXTRA_CFLAGS}"
+    export CXXFLAGS="${CXXFLAGS} ${EXTRA_CXXFLAGS}"
+    export LDFLAGS="${LDFLAGS} ${EXTRA_LDFLAGS}"
+
     if [[ "$targetOS" == osx || "$targetOS" == maccatalyst ]]; then
         if [[ "$hostArch" == x64 ]]; then
             cmakeArgs="-DCMAKE_OSX_ARCHITECTURES=\"x86_64\" $cmakeArgs"
@@ -194,17 +208,6 @@ build_native()
         return
     fi
 
-    SAVED_CFLAGS="${CFLAGS}"
-    SAVED_CXXFLAGS="${CXXFLAGS}"
-    SAVED_LDFLAGS="${LDFLAGS}"
-
-    # Let users provide additional compiler/linker flags via EXTRA_CFLAGS/EXTRA_CXXFLAGS/EXTRA_LDFLAGS.
-    # If users directly override CFLAG/CXXFLAGS/LDFLAGS, that may lead to some configure tests working incorrectly.
-    # See https://github.com/dotnet/runtime/issues/35727 for more information.
-    export CFLAGS="${CFLAGS} ${EXTRA_CFLAGS}"
-    export CXXFLAGS="${CXXFLAGS} ${EXTRA_CXXFLAGS}"
-    export LDFLAGS="${LDFLAGS} ${EXTRA_LDFLAGS}"
-
     local exit_code
     if [[ "$__StaticAnalyzer" == 1 ]]; then
         pushd "$intermediatesDir"

eng/native/gen-buildsys.sh

@@ -106,15 +106,9 @@ if [[ "$host_arch" == "wasm" ]]; then
     fi
 fi
 
-$cmake_command \
-  --no-warn-unused-cli \
-  -G "$generator" \
-  "-DCMAKE_BUILD_TYPE=$buildtype" \
-  "-DCMAKE_INSTALL_PREFIX=$__CMakeBinDir" \
-  $cmake_extra_defines \
-  $__UnprocessedCMakeArgs \
-  "${cmake_extra_defines_wasm[@]}" \
-  -S "$1" \
-  -B "$2"
+buildsys_command="$cmake_command --no-warn-unused-cli -G \"$generator\" \"-DCMAKE_BUILD_TYPE=$buildtype\" \"-DCMAKE_INSTALL_PREFIX=$__CMakeBinDir\" $cmake_extra_defines $__UnprocessedCMakeArgs \"${cmake_extra_defines_wasm[@]}\" -S \"$1\" -B \"$2\""
+buildsys_command=$(echo $buildsys_command | sed 's/""//g')
+echo $buildsys_command
+eval $buildsys_command
 
 # don't add anything after this line so the cmake exit code gets propagated correctly

src/coreclr/dlls/mscoree/exports.cpp

@@ -292,9 +292,10 @@ int coreclr_initialize(
 
     ConstWStringHolder appDomainFriendlyNameW = StringToUnicode(appDomainFriendlyName);
 
-    if (bundleProbe != nullptr)
+    ExternalAssemblyProbeFn* externalAssemblyProbe = hostContract != nullptr ? hostContract->external_assembly_probe : nullptr;
+    if (bundleProbe != nullptr || externalAssemblyProbe != nullptr)
     {
-        static Bundle bundle(exePath, bundleProbe);
+        static Bundle bundle(exePath, bundleProbe, externalAssemblyProbe);
         Bundle::AppBundle = &bundle;
     }
 

src/coreclr/hosts/inc/coreclrhost.h

@@ -150,6 +150,7 @@ CORECLR_HOSTING_API(coreclr_execute_assembly,
 //
 // Callback types used by the hosts
 //
+typedef bool(CORECLR_CALLING_CONVENTION ExternalAssemblyProbeFn)(const char* path, void** data_start, int64_t* size);
 typedef bool(CORECLR_CALLING_CONVENTION BundleProbeFn)(const char* path, int64_t* offset, int64_t* size, int64_t* compressedSize);
 typedef const void* (CORECLR_CALLING_CONVENTION PInvokeOverrideFn)(const char* libraryName, const char* entrypointName);
 

src/coreclr/inc/bundle.h

@@ -18,29 +18,31 @@ class Bundle;
 struct BundleFileLocation
 {
     INT64 Size;
+    void* DataStart;
     INT64 Offset;
     INT64 UncompresedSize;
 
     BundleFileLocation()
-    { 
+    {
         LIMITED_METHOD_CONTRACT;
 
         Size = 0;
-        Offset = 0; 
+        DataStart = nullptr;
+        Offset = 0;
         UncompresedSize = 0;
     }
 
     static BundleFileLocation Invalid() { LIMITED_METHOD_CONTRACT; return BundleFileLocation(); }
 
     const SString &Path() const;
 
-    bool IsValid() const { LIMITED_METHOD_CONTRACT; return Offset != 0; }
+    bool IsValid() const { LIMITED_METHOD_CONTRACT; return DataStart != nullptr || Offset != 0; }
 };
 
 class Bundle
 {
 public:
-    Bundle(LPCSTR bundlePath, BundleProbeFn *probe);
+    Bundle(LPCSTR bundlePath, BundleProbeFn *probe, ExternalAssemblyProbeFn* externalAssemblyProbe = nullptr);
     BundleFileLocation Probe(const SString& path, bool pathIsBundleRelative = false) const;
 
     const SString &Path() const { LIMITED_METHOD_CONTRACT; return m_path; }
@@ -51,9 +53,9 @@ class Bundle
     static BundleFileLocation ProbeAppBundle(const SString& path, bool pathIsBundleRelative = false);
 
 private:
-
-    SString m_path; // The path to single-file executable
+    SString m_path; // The path to single-file executable or package name/id on Android
     BundleProbeFn *m_probe;
+    ExternalAssemblyProbeFn *m_externalAssemblyProbe;
 
     SString m_basePath; // The prefix to denote a path within the bundle
     COUNT_T m_basePathLength;

src/coreclr/pal/src/CMakeLists.txt

@@ -300,8 +300,7 @@ if(CLR_CMAKE_TARGET_LINUX)
   else(NOT CLR_CMAKE_TARGET_ANDROID)
     target_link_libraries(coreclrpal
       PUBLIC
-      ${ANDROID_GLOB}
-      ${LZMA})
+      ${ANDROID_GLOB})
   endif(NOT CLR_CMAKE_TARGET_ANDROID)
 
   target_link_libraries(coreclrpal

src/coreclr/vm/bundle.cpp

@@ -30,15 +30,18 @@ const SString &BundleFileLocation::Path() const
     return Bundle::AppBundle->Path();
 }
 
-Bundle::Bundle(LPCSTR bundlePath, BundleProbeFn *probe)
+Bundle::Bundle(LPCSTR bundlePath, BundleProbeFn *probe, ExternalAssemblyProbeFn* externalAssemblyProbe)
+    : m_probe(probe)
+    , m_externalAssemblyProbe(externalAssemblyProbe)
+    , m_basePathLength(0)
 {
     STANDARD_VM_CONTRACT;
 
-    _ASSERTE(probe != nullptr);
+    _ASSERTE(m_probe != nullptr || m_externalAssemblyProbe != nullptr);
 
+    // On Android this is not a real path, but rather the application's package name
     m_path.SetUTF8(bundlePath);
-    m_probe = probe;
-
+#if !defined(TARGET_ANDROID)
     // The bundle-base path is the directory containing the single-file bundle.
     // When the Probe() function searches within the bundle, it masks out the basePath from the assembly-path (if found).
 
@@ -47,14 +50,13 @@ Bundle::Bundle(LPCSTR bundlePath, BundleProbeFn *probe)
     size_t baseLen = pos - bundlePath + 1; // Include DIRECTORY_SEPARATOR_CHAR_A in m_basePath
     m_basePath.SetUTF8(bundlePath, (COUNT_T)baseLen);
     m_basePathLength = (COUNT_T)baseLen;
+#endif // !TARGET_ANDROID
 }
 
 BundleFileLocation Bundle::Probe(const SString& path, bool pathIsBundleRelative) const
 {
     STANDARD_VM_CONTRACT;
 
-    BundleFileLocation loc;
-
     // Skip over m_base_path, if any. For example:
     //    Bundle.Probe("lib.dll") => m_probe("lib.dll")
     //    Bundle.Probe("path/to/exe/lib.dll") => m_probe("lib.dll")
@@ -77,27 +79,44 @@ BundleFileLocation Bundle::Probe(const SString& path, bool pathIsBundleRelative)
         else
         {
             // This is not a file within the bundle
-            return loc;
+            return BundleFileLocation::Invalid();
         }
     }
 
-    INT64 fileSize = 0;
-    INT64 compressedSize = 0;
-
-    m_probe(utf8Path, &loc.Offset, &fileSize, &compressedSize);
-
-    if (compressedSize)
+    if (m_probe != nullptr)
     {
-        loc.Size = compressedSize;
-        loc.UncompresedSize = fileSize;
+        BundleFileLocation loc;
+        INT64 fileSize = 0;
+        INT64 compressedSize = 0;
+        if (m_probe(utf8Path, &loc.Offset, &fileSize, &compressedSize))
+        {
+            // Found assembly in bundle
+            if (compressedSize)
+            {
+                loc.Size = compressedSize;
+                loc.UncompresedSize = fileSize;
+            }
+            else
+            {
+                loc.Size = fileSize;
+                loc.UncompresedSize = 0;
+            }
+
+            return loc;
+        }
     }
-    else
+
+    if (m_externalAssemblyProbe != nullptr)
     {
-        loc.Size = fileSize;
-        loc.UncompresedSize = 0;
+        BundleFileLocation loc;
+        if (m_externalAssemblyProbe(utf8Path, &loc.DataStart, &loc.Size))
+        {
+            // Found via external assembly probe
+            return loc;
+        }
     }
 
-    return loc;
+    return BundleFileLocation::Invalid();
 }
 
 BundleFileLocation Bundle::ProbeAppBundle(const SString& path, bool pathIsBundleRelative)

src/coreclr/vm/coreassemblyspec.cpp

@@ -89,10 +89,13 @@ STDAPI BinderAcquirePEImage(LPCWSTR             wszAssemblyPath,
         PEImageHolder pImage = PEImage::OpenImage(wszAssemblyPath, MDInternalImport_Default, bundleFileLocation);
 
         // Make sure that the IL image can be opened.
-        hr=pImage->TryOpenFile();
-        if (FAILED(hr))
+        if (pImage->IsFile())
         {
-            goto Exit;
+            hr = pImage->TryOpenFile();
+            if (FAILED(hr))
+            {
+                goto Exit;
+            }
         }
 
         if (pImage)

src/coreclr/vm/peimage.cpp

@@ -863,13 +863,15 @@ HRESULT PEImage::TryOpenFile(bool takeLock)
 {
     STANDARD_VM_CONTRACT;
 
+    _ASSERTE(IsFile());
+
     SimpleWriteLockHolder lock(m_pLayoutLock, takeLock);
 
-    if (m_hFile!=INVALID_HANDLE_VALUE)
+    if (m_hFile != INVALID_HANDLE_VALUE)
         return S_OK;
 
     ErrorModeHolder mode{};
-    m_hFile=WszCreateFile((LPCWSTR)GetPathToLoad(),
+    m_hFile = WszCreateFile((LPCWSTR)GetPathToLoad(),
                           GENERIC_READ
 #if TARGET_WINDOWS
                           // the file may have native code sections, make sure we are allowed to execute the file
@@ -881,7 +883,6 @@ HRESULT PEImage::TryOpenFile(bool takeLock)
                           OPEN_EXISTING,
                           FILE_ATTRIBUTE_NORMAL,
                           NULL);
-
     if (m_hFile != INVALID_HANDLE_VALUE)
             return S_OK;
 

src/coreclr/vm/peimage.h

@@ -138,6 +138,7 @@ class PEImage final
 
     BOOL IsFile();
     BOOL IsInBundle() const;
+    void* GetExternalData(INT64* size);
     INT64 GetOffset() const;
     INT64 GetSize() const;
     INT64 GetUncompressedSize() const;

src/coreclr/vm/peimage.inl

@@ -39,6 +39,16 @@ inline const SString& PEImage::GetPathToLoad()
     return IsInBundle() ? m_bundleFileLocation.Path() : m_path;
 }
 
+inline void* PEImage::GetExternalData(INT64* size)
+{
+    LIMITED_METHOD_CONTRACT;
+
+    _ASSERTE(size != nullptr);
+
+    *size = m_bundleFileLocation.Size;
+    return m_bundleFileLocation.DataStart;
+}
+
 inline INT64 PEImage::GetOffset() const
 {
     LIMITED_METHOD_CONTRACT;
@@ -98,8 +108,7 @@ inline const SString &PEImage::GetModuleFileNameHintForDAC()
 inline BOOL PEImage::IsFile()
 {
     WRAPPER_NO_CONTRACT;
-
-    return !GetPathToLoad().IsEmpty();
+    return m_bundleFileLocation.DataStart == nullptr && !GetPathToLoad().IsEmpty();
 }
 
 //

src/coreclr/vm/peimagelayout.cpp

@@ -616,17 +616,28 @@ FlatImageLayout::FlatImageLayout(PEImage* pOwner)
         PRECONDITION(CheckPointer(pOwner));
     }
     CONTRACTL_END;
-    m_pOwner=pOwner;
-
-    HANDLE hFile = pOwner->GetFileHandle();
-    INT64 offset = pOwner->GetOffset();
-    INT64 size = pOwner->GetSize();
+    m_pOwner = pOwner;
 
 #ifdef LOGGING
     SString ownerPath{ pOwner->GetPath() };
     LOG((LF_LOADER, LL_INFO100, "PEImage: Opening flat %s\n", ownerPath.GetUTF8()));
 #endif // LOGGING
 
+    INT64 dataSize;
+    void* data = pOwner->GetExternalData(&dataSize);
+    if (data != nullptr)
+    {
+        // Image was provided as flat data via external assembly probing.
+        // We do not manage the data - just initialize with it directly.
+        _ASSERTE(dataSize != 0);
+        Init(data, (COUNT_T)dataSize);
+        return;
+    }
+
+    HANDLE hFile = pOwner->GetFileHandle();
+    INT64 offset = pOwner->GetOffset();
+    INT64 size = pOwner->GetSize();
+
     // If a size is not specified, load the whole file
     if (size == 0)
     {

src/coreclr/vm/peimagelayout.h

@@ -81,11 +81,7 @@ class FlatImageLayout : public PEImageLayout
 {
     VPTR_VTABLE_CLASS(FlatImageLayout, PEImageLayout)
         VPTR_UNIQUE(0x59)
-protected:
-    CLRMapViewHolder m_FileView;
 public:
-    HandleHolder m_FileMap;
-
 #ifndef DACCESS_COMPILE
     FlatImageLayout(PEImage* pOwner);
     FlatImageLayout(PEImage* pOwner, const BYTE* array, COUNT_T size);
@@ -94,6 +90,12 @@ class FlatImageLayout : public PEImageLayout
     void* LoadImageByMappingParts(SIZE_T* m_imageParts) const;
 #endif
 #endif
+
+private:
+    // Handles for the mapped image.
+    // These will be null if the image data is not mapped by the runtime (for example, provided via an external assembly probe).
+    CLRMapViewHolder m_FileView;
+    HandleHolder m_FileMap;
 };
 
 // ConvertedImageView is for the case when we construct a loaded

src/coreclr/vm/util.cpp

@@ -15,6 +15,9 @@
 
 #ifndef DACCESS_COMPILE
 
+#if defined(TARGET_ANDROID)
+#include <android/log.h>
+#endif // defined(TARGET_ANDROID)
 
 thread_local size_t t_ThreadType;
 
@@ -177,10 +180,13 @@ void PrintToStdErrA(const char *pszString) {
     }
     CONTRACTL_END
 
-    HANDLE  Handle = GetStdHandle(STD_ERROR_HANDLE);
-
+#if defined(TARGET_ANDROID)
+    __android_log_write(ANDROID_LOG_FATAL, MAIN_CLR_MODULE_NAME_A, pszString);
+#else
+    HANDLE Handle = GetStdHandle(STD_ERROR_HANDLE);
     size_t len = strlen(pszString);
     NPrintToHandleA(Handle, pszString, len);
+#endif // defined(TARGET_ANDROID)
 }
 
 void PrintToStdErrW(const WCHAR *pwzString)