Skip to content

Download and Copy libdatadog binaries for Data-Pipeline integration #6777

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 10 commits into from
Mar 25, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 15 additions & 9 deletions build/cmake/FindLibdatadog.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -6,16 +6,22 @@ include(FetchContent)

set(LIBDATADOG_VERSION "v16.0.3" CACHE STRING "libdatadog version")

if (CMAKE_SYSTEM_PROCESSOR STREQUAL aarch64 OR CMAKE_SYSTEM_PROCESSOR STREQUAL arm64)
if (DEFINED ENV{IsAlpine} AND "$ENV{IsAlpine}" MATCHES "true")
if (CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "arm64")
if (CMAKE_SYSTEM_NAME STREQUAL "Darwin")
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Have you tested this on mac?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

no but before merging, I plan to ask someone with a mac

set(SHA256_LIBDATADOG "2d7933e09dc39706e9c99c7edcff5c60f7567ea2777157596de828f62f39035b" CACHE STRING "libdatadog sha256")
set(FILE_TO_DOWNLOAD libdatadog-aarch64-apple-darwin.tar.gz)
elseif (DEFINED ENV{IsAlpine} AND "$ENV{IsAlpine}" MATCHES "true")
set(SHA256_LIBDATADOG "dd08d3a4dbbd765392121d27b790d7818e80dd28500b554db16e9186b1025ba9" CACHE STRING "libdatadog sha256")
set(FILE_TO_DOWNLOAD libdatadog-aarch64-alpine-linux-musl.tar.gz)
else()
set(SHA256_LIBDATADOG "decc01a2e0f732cabcc56594429a3dbc13678070e07f24891555dcc02df2e516" CACHE STRING "libdatadog sha256")
set(FILE_TO_DOWNLOAD libdatadog-aarch64-unknown-linux-gnu.tar.gz)
endif()
else()
if (DEFINED ENV{IsAlpine} AND "$ENV{IsAlpine}" MATCHES "true")
if (CMAKE_SYSTEM_NAME STREQUAL "Darwin")
set(SHA256_LIBDATADOG "ced5db61e0ca8e974b9d59b0b6833c28e19445a3e4ec3c548fda965806c17560" CACHE STRING "libdatadog sha256")
set(FILE_TO_DOWNLOAD libdatadog-x86_64-apple-darwin.tar.gz)
elseif (DEFINED ENV{IsAlpine} AND "$ENV{IsAlpine}" MATCHES "true")
set(SHA256_LIBDATADOG "8e09afd3cfb5ace85501f37b4bd6378299ebbf71189ccc2173169998b75b4b56" CACHE STRING "libdatadog sha256")
set(FILE_TO_DOWNLOAD libdatadog-${CMAKE_SYSTEM_PROCESSOR}-alpine-linux-musl.tar.gz)
else()
Expand All @@ -24,16 +30,16 @@ else()
endif()
endif()

FetchContent_Declare(libdatadog-${LIBDATADOG_VERSION}
FetchContent_Declare(libdatadog-install
URL https://github.com/DataDog/libdatadog/releases/download/${LIBDATADOG_VERSION}/${FILE_TO_DOWNLOAD}
URL_HASH SHA256=${SHA256_LIBDATADOG}
SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/libdatadog-${LIBDATADOG_VERSION}
SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/libdatadog-install
)
if(NOT libdatadog-${LIBDATADOG_VERSION}_POPULATED)
FetchContent_Populate(libdatadog-${LIBDATADOG_VERSION})
if(NOT libdatadog-install_POPULATED)
FetchContent_Populate(libdatadog-install)
endif()

set(LIBDATADOG_BASE_DIR ${libdatadog-${LIBDATADOG_VERSION}_SOURCE_DIR})
set(LIBDATADOG_BASE_DIR ${libdatadog-install_SOURCE_DIR})

add_library(libdatadog-lib SHARED IMPORTED)

Expand All @@ -42,7 +48,7 @@ set_target_properties(libdatadog-lib PROPERTIES
IMPORTED_LOCATION ${LIBDATADOG_BASE_DIR}/lib/libdatadog_profiling.so
)

add_dependencies(libdatadog-lib libdatadog-${LIBDATADOG_VERSION})
add_dependencies(libdatadog-lib libdatadog-install)

# Override target_link_libraries
function(target_link_libraries target)
Expand Down
68 changes: 1 addition & 67 deletions tracer/build/_build/Build.Profiler.Steps.cs
Original file line number Diff line number Diff line change
Expand Up @@ -45,74 +45,8 @@ partial class Build
.OnlyWhenStatic(() => IsWin)
.Executes(async () =>
{
var vcpkg = await GetVcpkg();
var vcpkg = ToolResolver.GetLocalTool(await GetVcpkg());
vcpkg("integrate install");

async Task<Tool> GetVcpkg()
{
var vcpkgFilePath = string.Empty;

try
{
vcpkgFilePath = ToolPathResolver.GetPathExecutable("vcpkg.exe");
}
catch (ArgumentException)
{ }

if (File.Exists(vcpkgFilePath))
{
return ToolResolver.GetLocalTool(vcpkgFilePath);
}

// Check if already downloaded
var vcpkgRoot = RootDirectory / "artifacts" / "bin" / "vcpkg";
var vcpkgExecPath = vcpkgRoot / "vcpkg.exe";

if (File.Exists(vcpkgExecPath))
{
return ToolResolver.GetLocalTool($"{vcpkgExecPath}");
}

await DownloadAndExtractVcpkg(vcpkgRoot);
Cmd.Value(arguments: $"cmd /c {vcpkgRoot / "bootstrap-vcpkg.bat"}");
return ToolResolver.GetLocalTool($"{vcpkgRoot / "vcpkg.exe"}");
}

async Task DownloadAndExtractVcpkg(AbsolutePath destinationFolder)
{
var nbTries = 0;
var keepTrying = true;
var vcpkgZip = TempDirectory / "vcpkg.zip";
using var client = new HttpClient();
const string vcpkgVersion = "2024.11.16";
while (keepTrying)
{
nbTries++;
try
{
var response = await client.GetAsync($"https://github.com/microsoft/vcpkg/archive/refs/tags/{vcpkgVersion}.zip");
response.EnsureSuccessStatusCode();
await using var stream = await response.Content.ReadAsStreamAsync();
await using var file = File.Create(vcpkgZip);
await stream.CopyToAsync(file);
keepTrying = false;
}
catch (HttpRequestException)
{
if (nbTries > 3)
{
throw;
}
}
}

EnsureExistingParentDirectory(destinationFolder);
var parentFolder = destinationFolder.Parent;

CompressionTasks.UncompressZip(vcpkgZip, parentFolder);

RenameDirectory(parentFolder / $"vcpkg-{vcpkgVersion}", destinationFolder.Name);
}
});

Target CompileProfilerNativeSrcWindows => _ => _
Expand Down
134 changes: 134 additions & 0 deletions tracer/build/_build/Build.Steps.cs
Original file line number Diff line number Diff line change
Expand Up @@ -552,6 +552,73 @@ async Task DownloadWafVersion(string libddwafVersion = null, string uncompressFo
}
});

Target DownloadLibDatadog => _ => _
.Unlisted()
.After(CreateRequiredDirectories)
.Executes(async () =>
{
if (IsLinux || IsOsx)
{
if (!Directory.Exists(NativeBuildDirectory))
{
CMake.Value(
arguments: $"-DCMAKE_CXX_COMPILER=clang++ -DCMAKE_C_COMPILER=clang -B {NativeBuildDirectory} -S {RootDirectory}");
}
// In CI, the folder might already exist. Which means that CMake was already configured
// and libdatadog artifact (correct one) was already downloaded.
// So just reuse it.

}
else if (IsWin)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It looks like this branch always executes, regardless if it's already downloaded. Is that the case? Does it need to?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🤔 in that case, I rely on vcpkg to download it only once (using timestamp).

{
var vcpkgExePath = await GetVcpkg();
var vcpkgRoot = Environment.GetEnvironmentVariable("VCPKG_ROOT") ?? Directory.GetParent(vcpkgExePath).FullName;
var vcpkg = ToolResolver.GetLocalTool(vcpkgExePath);
foreach (var arch in new[] { MSBuildTargetPlatform.x64, MSBuildTargetPlatform.x86 })
{
// This big line is the same generated by VS when installing libdatadog while building the profiler
vcpkg($"install --x-wait-for-lock --triplet \"{arch}-windows\" --vcpkg-root \"{vcpkgRoot}\" \"--x-manifest-root={RootDirectory}\" \"--x-install-root={BuildArtifactsDirectory}\\deps\\vcpkg\\{arch}-windows\" --downloads-root {BuildArtifactsDirectory}\\obj\\vcpkg\\downloads --x-packages-root {BuildArtifactsDirectory}\\obj\\vcpkg\\packages --x-buildtrees-root {BuildArtifactsDirectory}\\obj\\vcpkg/buildtrees --clean-after-build");
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

glad we have you :)

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yeah, I wanted a place more centralized (sharing it with the profiler so we avoid duplicating things and stuff) but nop
😢

}
}
});

Target CopyLibDatadog => _ => _
.Unlisted()
.After(DownloadLibDatadog)
.Executes(() =>
{
if (IsWin)
{
var vcpkgDepsFolder = BuildArtifactsDirectory / "deps" / "vcpkg";
foreach (var architecture in new[] { MSBuildTargetPlatform.x64, MSBuildTargetPlatform.x86 })
{
var source = vcpkgDepsFolder / $"{architecture}-windows" / $"{architecture}-windows";
if (BuildConfiguration == Configuration.Debug)
{
source /= "debug";
}

var dllFile = source / "bin" / "datadog_profiling_ffi.dll";
var dest = MonitoringHomeDirectory / $"win-{architecture}";
CopyFileToDirectory(dllFile, dest, FileExistsPolicy.Overwrite);

var pdbFile = source / "bin" / "datadog_profiling_ffi.pdb";
dest = SymbolsDirectory / $"win-{architecture}";
CopyFileToDirectory(pdbFile, dest, FileExistsPolicy.Overwrite);
}
}
else if (IsLinux || IsOsx)
{
var (destArch, ext) = GetUnixArchitectureAndExtension();

var libdatadogFileName = $"libdatadog_profiling.{ext}";

var source = NativeBuildDirectory / "libdatadog-install" / "lib" / libdatadogFileName;
var dest = MonitoringHomeDirectory / destArch;
CopyFileToDirectory(source, dest, FileExistsPolicy.Overwrite);
}
});

Target CopyNativeFilesForAppSecUnitTests => _ => _
.Unlisted()
.After(Clean)
Expand Down Expand Up @@ -2746,4 +2813,71 @@ private void DotnetBuild(
.SetProcessArgumentConfigurator(arg => arg.Add("/nowarn:NU1701")) //nowarn:NU1701 - Package 'x' was restored using '.NETFramework,Version=v4.6.1' instead of the project target framework '.NETCoreApp,Version=v2.1'.
.CombineWith(projPaths, (settings, projPath) => settings.SetProjectFile(projPath)));
}


private async Task<string> GetVcpkg()
{
var vcpkgFilePath = string.Empty;

try
{
vcpkgFilePath = ToolPathResolver.GetPathExecutable("vcpkg.exe");
}
catch (ArgumentException)
{ }

if (File.Exists(vcpkgFilePath))
{
return vcpkgFilePath;
}

// Check if already downloaded
var vcpkgRoot = RootDirectory / "artifacts" / "bin" / "vcpkg";
var vcpkgExecPath = vcpkgRoot / "vcpkg.exe";

if (File.Exists(vcpkgExecPath))
{
return $"{vcpkgExecPath}";
}

await DownloadAndExtractVcpkg(vcpkgRoot);
Cmd.Value(arguments: $"cmd /c {vcpkgRoot / "bootstrap-vcpkg.bat"}");
return $"{vcpkgRoot / "vcpkg.exe"}";
}

private async Task DownloadAndExtractVcpkg(AbsolutePath destinationFolder)
{
var nbTries = 0;
var keepTrying = true;
var vcpkgZip = TempDirectory / "vcpkg.zip";
using var client = new HttpClient();
const string vcpkgVersion = "2024.11.16";
while (keepTrying)
{
nbTries++;
try
{
var response = await client.GetAsync($"https://github.com/microsoft/vcpkg/archive/refs/tags/{vcpkgVersion}.zip");
response.EnsureSuccessStatusCode();
await using var stream = await response.Content.ReadAsStreamAsync();
await using var file = File.Create(vcpkgZip);
await stream.CopyToAsync(file);
keepTrying = false;
}
catch (HttpRequestException)
{
if (nbTries > 3)
{
throw;
}
}
}

EnsureExistingParentDirectory(destinationFolder);
var parentFolder = destinationFolder.Parent;

CompressionTasks.UncompressZip(vcpkgZip, parentFolder);

RenameDirectory(parentFolder / $"vcpkg-{vcpkgVersion}", destinationFolder.Name);
}
}
4 changes: 4 additions & 0 deletions tracer/build/_build/Build.cs
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,8 @@ void DeleteReparsePoints(string path)
.DependsOn(PublishManagedTracer)
.DependsOn(DownloadLibDdwaf)
.DependsOn(CopyLibDdwaf)
.DependsOn(DownloadLibDatadog)
.DependsOn(CopyLibDatadog)
.DependsOn(CreateMissingNullabilityFile)
.DependsOn(CreateTrimmingFile)
.DependsOn(RegenerateSolutions);
Expand All @@ -219,6 +221,8 @@ void DeleteReparsePoints(string path)
.DependsOn(PublishManagedTracerR2R)
.DependsOn(DownloadLibDdwaf)
.DependsOn(CopyLibDdwaf)
.DependsOn(DownloadLibDatadog)
.DependsOn(CopyLibDatadog)
Comment on lines +224 to +225
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This raises an interesting question actually - should we enable the trace exporter in the AWS Lambda artifacts 🤔 My gut feeling is no, at least initially, as they're very sensitive to size. It's also possible that the lambda layer would strip out this file anyway!

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I will follow you gut on that

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The plan was to cover AWS Lambda as well, Lambda layer I think allows 250MB and specially these are all AL2 instances - we should be good.

My other question related to this, do we test on AL2 instances in our CI?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

(this PR is merged already, but commenting here for visiblity)

should we enable the trace exporter in the AWS Lambda artifacts? My gut feeling is no, at least initially, as they're very sensitive to size. It's also possible that the lambda layer would strip out this file anyway!

Yup, we are already stripping it out (DataDog/dd-trace-dotnet-aws-lambda-layer#10), so including this in the Lambda artifacts is useless at the moment.

.DependsOn(CreateMissingNullabilityFile)
.DependsOn(CreateTrimmingFile);

Expand Down
67 changes: 67 additions & 0 deletions tracer/src/Datadog.Trace/LibDatadog/NativeInterop.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
// <copyright file="NativeInterop.cs" company="Datadog">
// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
// </copyright>

#nullable enable

using System;
using System.Runtime.InteropServices;

namespace Datadog.Trace.LibDatadog;

internal class NativeInterop
{
private const string DllName = "LibDatadog";
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

/question

shouldn't this be name of file?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

in this case, it can be whatever you want: here I use libdatadog for documentation.
Since it will be rewritten by the native side, it does not matter.


internal static class Exporter
{
// [DllImport(DllName, EntryPoint = "ddog_trace_exporter_new")]
// internal static extern ErrorHandle Create(out IntPtr outHandle, SafeHandle config);
Comment on lines +19 to +20
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I assume these are intentionally comment out, as they'll be used shortly by the trace exporter pipeline?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

exactly


[DllImport(DllName, EntryPoint = "ddog_trace_exporter_error_free")]
internal static extern void ReleaseError(IntPtr error);

[DllImport(DllName, EntryPoint = "ddog_trace_exporter_free")]
internal static extern void Release(IntPtr handle);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we can probably add a dummy test in Datadog.Trace.Tests, which calls with some ptr. It will fail but we can make sure library is loaded on all platforms and right entry point is invoked.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This was to see if the native part was able to rewrite it.
The tests should be added when you add data-pipeline and/or config support

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

After rewrite, were you able to call any of the methods?

I want to this on my end as well - just to see library integration is perfect and we can focus on calling the methods.


// [DllImport(DllName, EntryPoint = "ddog_trace_exporter_send")]
// internal static extern ErrorHandle Send(SafeHandle handle, ByteSlice trace, UIntPtr traceCount, ref IntPtr response);
}

internal static class Config
{
[DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_new")]
internal static extern void Create(out IntPtr outHandle);

[DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_free")]
internal static extern void Release(IntPtr handle);

// [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_url")]
// internal static extern ErrorHandle SetUrl(SafeHandle config, CharSlice url);

// [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_tracer_version")]
// internal static extern ErrorHandle SetTracerVersion(SafeHandle config, CharSlice version);

// [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_language")]
// internal static extern ErrorHandle SetLanguage(SafeHandle config, CharSlice lang);

// [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_lang_version")]
// internal static extern ErrorHandle SetLanguageVersion(SafeHandle config, CharSlice version);

// [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_lang_interpreter")]
// internal static extern ErrorHandle SetInterperter(SafeHandle config, CharSlice interpreter);

// [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_hostname")]
// internal static extern ErrorHandle SetHostname(SafeHandle config, CharSlice hostname);

// [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_env")]
// internal static extern ErrorHandle SetEnvironment(SafeHandle config, CharSlice env);

// [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_version")]
// internal static extern ErrorHandle SetVersion(SafeHandle config, CharSlice version);

// [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_service")]
// internal static extern ErrorHandle SetService(SafeHandle config, CharSlice service);
}
}
Loading
Loading