-
Notifications
You must be signed in to change notification settings - Fork 147
Download and Copy libdatadog binaries for Data-Pipeline integration #6777
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
1f96dd7
ad2361c
b013411
d92696a
530727a
5ab0752
f24b965
2f09173
568e767
80f0570
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -552,6 +552,73 @@ async Task DownloadWafVersion(string libddwafVersion = null, string uncompressFo | |
} | ||
}); | ||
|
||
Target DownloadLibDatadog => _ => _ | ||
.Unlisted() | ||
.After(CreateRequiredDirectories) | ||
.Executes(async () => | ||
{ | ||
if (IsLinux || IsOsx) | ||
{ | ||
if (!Directory.Exists(NativeBuildDirectory)) | ||
{ | ||
CMake.Value( | ||
arguments: $"-DCMAKE_CXX_COMPILER=clang++ -DCMAKE_C_COMPILER=clang -B {NativeBuildDirectory} -S {RootDirectory}"); | ||
} | ||
// In CI, the folder might already exist. Which means that CMake was already configured | ||
// and libdatadog artifact (correct one) was already downloaded. | ||
// So just reuse it. | ||
|
||
} | ||
else if (IsWin) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. It looks like this branch always executes, regardless if it's already downloaded. Is that the case? Does it need to? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🤔 in that case, I rely on vcpkg to download it only once (using timestamp). |
||
{ | ||
var vcpkgExePath = await GetVcpkg(); | ||
var vcpkgRoot = Environment.GetEnvironmentVariable("VCPKG_ROOT") ?? Directory.GetParent(vcpkgExePath).FullName; | ||
var vcpkg = ToolResolver.GetLocalTool(vcpkgExePath); | ||
foreach (var arch in new[] { MSBuildTargetPlatform.x64, MSBuildTargetPlatform.x86 }) | ||
{ | ||
// This big line is the same generated by VS when installing libdatadog while building the profiler | ||
vcpkg($"install --x-wait-for-lock --triplet \"{arch}-windows\" --vcpkg-root \"{vcpkgRoot}\" \"--x-manifest-root={RootDirectory}\" \"--x-install-root={BuildArtifactsDirectory}\\deps\\vcpkg\\{arch}-windows\" --downloads-root {BuildArtifactsDirectory}\\obj\\vcpkg\\downloads --x-packages-root {BuildArtifactsDirectory}\\obj\\vcpkg\\packages --x-buildtrees-root {BuildArtifactsDirectory}\\obj\\vcpkg/buildtrees --clean-after-build"); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. glad we have you :) There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. yeah, I wanted a place more centralized (sharing it with the profiler so we avoid duplicating things and stuff) but nop |
||
} | ||
} | ||
}); | ||
|
||
Target CopyLibDatadog => _ => _ | ||
.Unlisted() | ||
.After(DownloadLibDatadog) | ||
.Executes(() => | ||
{ | ||
if (IsWin) | ||
{ | ||
var vcpkgDepsFolder = BuildArtifactsDirectory / "deps" / "vcpkg"; | ||
foreach (var architecture in new[] { MSBuildTargetPlatform.x64, MSBuildTargetPlatform.x86 }) | ||
{ | ||
var source = vcpkgDepsFolder / $"{architecture}-windows" / $"{architecture}-windows"; | ||
if (BuildConfiguration == Configuration.Debug) | ||
{ | ||
source /= "debug"; | ||
} | ||
|
||
var dllFile = source / "bin" / "datadog_profiling_ffi.dll"; | ||
ganeshnj marked this conversation as resolved.
Show resolved
Hide resolved
|
||
var dest = MonitoringHomeDirectory / $"win-{architecture}"; | ||
CopyFileToDirectory(dllFile, dest, FileExistsPolicy.Overwrite); | ||
|
||
var pdbFile = source / "bin" / "datadog_profiling_ffi.pdb"; | ||
dest = SymbolsDirectory / $"win-{architecture}"; | ||
CopyFileToDirectory(pdbFile, dest, FileExistsPolicy.Overwrite); | ||
} | ||
} | ||
else if (IsLinux || IsOsx) | ||
{ | ||
var (destArch, ext) = GetUnixArchitectureAndExtension(); | ||
|
||
var libdatadogFileName = $"libdatadog_profiling.{ext}"; | ||
|
||
var source = NativeBuildDirectory / "libdatadog-install" / "lib" / libdatadogFileName; | ||
var dest = MonitoringHomeDirectory / destArch; | ||
CopyFileToDirectory(source, dest, FileExistsPolicy.Overwrite); | ||
} | ||
}); | ||
|
||
Target CopyNativeFilesForAppSecUnitTests => _ => _ | ||
.Unlisted() | ||
.After(Clean) | ||
|
@@ -2746,4 +2813,71 @@ private void DotnetBuild( | |
.SetProcessArgumentConfigurator(arg => arg.Add("/nowarn:NU1701")) //nowarn:NU1701 - Package 'x' was restored using '.NETFramework,Version=v4.6.1' instead of the project target framework '.NETCoreApp,Version=v2.1'. | ||
.CombineWith(projPaths, (settings, projPath) => settings.SetProjectFile(projPath))); | ||
} | ||
|
||
|
||
private async Task<string> GetVcpkg() | ||
{ | ||
var vcpkgFilePath = string.Empty; | ||
|
||
try | ||
{ | ||
vcpkgFilePath = ToolPathResolver.GetPathExecutable("vcpkg.exe"); | ||
} | ||
catch (ArgumentException) | ||
{ } | ||
|
||
if (File.Exists(vcpkgFilePath)) | ||
{ | ||
return vcpkgFilePath; | ||
} | ||
|
||
// Check if already downloaded | ||
var vcpkgRoot = RootDirectory / "artifacts" / "bin" / "vcpkg"; | ||
var vcpkgExecPath = vcpkgRoot / "vcpkg.exe"; | ||
|
||
if (File.Exists(vcpkgExecPath)) | ||
{ | ||
return $"{vcpkgExecPath}"; | ||
} | ||
|
||
await DownloadAndExtractVcpkg(vcpkgRoot); | ||
Cmd.Value(arguments: $"cmd /c {vcpkgRoot / "bootstrap-vcpkg.bat"}"); | ||
return $"{vcpkgRoot / "vcpkg.exe"}"; | ||
} | ||
|
||
private async Task DownloadAndExtractVcpkg(AbsolutePath destinationFolder) | ||
{ | ||
var nbTries = 0; | ||
var keepTrying = true; | ||
var vcpkgZip = TempDirectory / "vcpkg.zip"; | ||
using var client = new HttpClient(); | ||
const string vcpkgVersion = "2024.11.16"; | ||
while (keepTrying) | ||
{ | ||
nbTries++; | ||
try | ||
{ | ||
var response = await client.GetAsync($"https://github.com/microsoft/vcpkg/archive/refs/tags/{vcpkgVersion}.zip"); | ||
response.EnsureSuccessStatusCode(); | ||
await using var stream = await response.Content.ReadAsStreamAsync(); | ||
await using var file = File.Create(vcpkgZip); | ||
await stream.CopyToAsync(file); | ||
keepTrying = false; | ||
} | ||
catch (HttpRequestException) | ||
{ | ||
if (nbTries > 3) | ||
{ | ||
throw; | ||
} | ||
} | ||
} | ||
|
||
EnsureExistingParentDirectory(destinationFolder); | ||
var parentFolder = destinationFolder.Parent; | ||
|
||
CompressionTasks.UncompressZip(vcpkgZip, parentFolder); | ||
|
||
RenameDirectory(parentFolder / $"vcpkg-{vcpkgVersion}", destinationFolder.Name); | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -205,6 +205,8 @@ void DeleteReparsePoints(string path) | |
.DependsOn(PublishManagedTracer) | ||
.DependsOn(DownloadLibDdwaf) | ||
.DependsOn(CopyLibDdwaf) | ||
.DependsOn(DownloadLibDatadog) | ||
.DependsOn(CopyLibDatadog) | ||
.DependsOn(CreateMissingNullabilityFile) | ||
.DependsOn(CreateTrimmingFile) | ||
.DependsOn(RegenerateSolutions); | ||
|
@@ -219,6 +221,8 @@ void DeleteReparsePoints(string path) | |
.DependsOn(PublishManagedTracerR2R) | ||
.DependsOn(DownloadLibDdwaf) | ||
.DependsOn(CopyLibDdwaf) | ||
.DependsOn(DownloadLibDatadog) | ||
.DependsOn(CopyLibDatadog) | ||
Comment on lines
+224
to
+225
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This raises an interesting question actually - should we enable the trace exporter in the AWS Lambda artifacts 🤔 My gut feeling is no, at least initially, as they're very sensitive to size. It's also possible that the lambda layer would strip out this file anyway! There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I will follow you gut on that There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The plan was to cover AWS Lambda as well, Lambda layer I think allows 250MB and specially these are all AL2 instances - we should be good. My other question related to this, do we test on AL2 instances in our CI? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. (this PR is merged already, but commenting here for visiblity)
Yup, we are already stripping it out (DataDog/dd-trace-dotnet-aws-lambda-layer#10), so including this in the Lambda artifacts is useless at the moment. |
||
.DependsOn(CreateMissingNullabilityFile) | ||
.DependsOn(CreateTrimmingFile); | ||
|
||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,67 @@ | ||
// <copyright file="NativeInterop.cs" company="Datadog"> | ||
// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. | ||
// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. | ||
// </copyright> | ||
|
||
#nullable enable | ||
|
||
using System; | ||
using System.Runtime.InteropServices; | ||
|
||
namespace Datadog.Trace.LibDatadog; | ||
|
||
internal class NativeInterop | ||
{ | ||
private const string DllName = "LibDatadog"; | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. /question shouldn't this be name of file? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. in this case, it can be whatever you want: here I use libdatadog for documentation. |
||
|
||
internal static class Exporter | ||
{ | ||
// [DllImport(DllName, EntryPoint = "ddog_trace_exporter_new")] | ||
// internal static extern ErrorHandle Create(out IntPtr outHandle, SafeHandle config); | ||
Comment on lines
+19
to
+20
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I assume these are intentionally comment out, as they'll be used shortly by the trace exporter pipeline? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. exactly |
||
|
||
[DllImport(DllName, EntryPoint = "ddog_trace_exporter_error_free")] | ||
internal static extern void ReleaseError(IntPtr error); | ||
|
||
[DllImport(DllName, EntryPoint = "ddog_trace_exporter_free")] | ||
internal static extern void Release(IntPtr handle); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. we can probably add a dummy test in Datadog.Trace.Tests, which calls with some ptr. It will fail but we can make sure library is loaded on all platforms and right entry point is invoked. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This was to see if the native part was able to rewrite it. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. After rewrite, were you able to call any of the methods? I want to this on my end as well - just to see library integration is perfect and we can focus on calling the methods. |
||
|
||
// [DllImport(DllName, EntryPoint = "ddog_trace_exporter_send")] | ||
// internal static extern ErrorHandle Send(SafeHandle handle, ByteSlice trace, UIntPtr traceCount, ref IntPtr response); | ||
} | ||
|
||
internal static class Config | ||
{ | ||
[DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_new")] | ||
internal static extern void Create(out IntPtr outHandle); | ||
|
||
[DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_free")] | ||
internal static extern void Release(IntPtr handle); | ||
|
||
// [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_url")] | ||
// internal static extern ErrorHandle SetUrl(SafeHandle config, CharSlice url); | ||
|
||
// [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_tracer_version")] | ||
// internal static extern ErrorHandle SetTracerVersion(SafeHandle config, CharSlice version); | ||
|
||
// [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_language")] | ||
// internal static extern ErrorHandle SetLanguage(SafeHandle config, CharSlice lang); | ||
|
||
// [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_lang_version")] | ||
// internal static extern ErrorHandle SetLanguageVersion(SafeHandle config, CharSlice version); | ||
|
||
// [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_lang_interpreter")] | ||
// internal static extern ErrorHandle SetInterperter(SafeHandle config, CharSlice interpreter); | ||
|
||
// [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_hostname")] | ||
// internal static extern ErrorHandle SetHostname(SafeHandle config, CharSlice hostname); | ||
|
||
// [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_env")] | ||
// internal static extern ErrorHandle SetEnvironment(SafeHandle config, CharSlice env); | ||
|
||
// [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_version")] | ||
// internal static extern ErrorHandle SetVersion(SafeHandle config, CharSlice version); | ||
|
||
// [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_service")] | ||
// internal static extern ErrorHandle SetService(SafeHandle config, CharSlice service); | ||
} | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Have you tested this on mac?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
no but before merging, I plan to ask someone with a mac