Skip to content

Commit

Permalink
Add support to load native libraries from the function.deps.json file.
Browse files Browse the repository at this point in the history 
This allows for probing of more locations for native libraries instead of just the "runtimes\{rid}\native" folder. For example, native assets can also live in a "runtimes\{rid}\nativeassets\{tfm}" folder.

Fix Azure#5187
  • Loading branch information
@eerhardt
eerhardt committed Nov 1, 2019
1 parent 831d69e commit 9c47cf4
Show file tree
Hide file tree
Showing 3 changed files with 104 additions and 12 deletions.
37 changes: 29 additions & 8 deletions src/WebJobs.Script/Description/DotNet/FunctionAssemblyLoadContext.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ public partial class FunctionAssemblyLoadContext : AssemblyLoadContext

private readonly List<string> _probingPaths = new List<string>();
private readonly IDictionary<string, string> _depsAssemblies;
private readonly IDictionary<string, string> _nativeLibraries;

public FunctionAssemblyLoadContext(string basePath)
{
Expand All @@ -38,7 +39,7 @@ public FunctionAssemblyLoadContext(string basePath)
throw new ArgumentNullException(nameof(basePath));
}

_depsAssemblies = InitializeDeps(basePath);
(_depsAssemblies, _nativeLibraries) = InitializeDeps(basePath);

_probingPaths.Add(basePath);
}
Expand All @@ -50,7 +51,7 @@ internal static void ResetSharedContext()
_defaultContext = new Lazy<FunctionAssemblyLoadContext>(CreateSharedContext, true);
}

internal static IDictionary<string, string> InitializeDeps(string basePath)
internal static (IDictionary<string, string> depsAssemblies, IDictionary<string, string> nativeLibraries) InitializeDeps(string basePath)
{
string depsFilePath = Path.Combine(basePath, DotNetConstants.FunctionsDepsFileName);

Expand All @@ -64,16 +65,22 @@ internal static IDictionary<string, string> InitializeDeps(string basePath)
using (Stream file = File.OpenRead(depsFilePath))
{
var depsContext = reader.Read(file);
return depsContext.RuntimeLibraries.SelectMany(l => SelectRuntimeAssemblyGroup(rids, l.RuntimeAssemblyGroups))
var depsAssemblies = depsContext.RuntimeLibraries.SelectMany(l => SelectRuntimeAssemblyGroup(rids, l.RuntimeAssemblyGroups))
.ToDictionary(path => Path.GetFileNameWithoutExtension(path));

// Note the difference here that nativeLibraries has the whole file name, including extension.
var nativeLibraries = depsContext.RuntimeLibraries.SelectMany(l => SelectRuntimeAssemblyGroup(rids, l.NativeLibraryGroups))
.ToDictionary(path => Path.GetFileName(path));

return (depsAssemblies, nativeLibraries);
}
}
catch
{
}
}

return null;
return (null, null);
}

private static IEnumerable<string> SelectRuntimeAssemblyGroup(List<string> rids, IReadOnlyList<RuntimeAssetGroup> runtimeAssemblyGroups)
Expand Down Expand Up @@ -289,7 +296,7 @@ protected virtual Assembly OnResolvingAssembly(AssemblyLoadContext arg1, Assembl
protected override IntPtr LoadUnmanagedDll(string unmanagedDllName)
{
string fileName = GetUnmanagedLibraryFileName(unmanagedDllName);
string filePath = GetRuntimeNativeAssetPath(fileName, true);
string filePath = GetRuntimeNativeAssetPath(fileName);

if (filePath != null)
{
Expand All @@ -299,17 +306,31 @@ protected override IntPtr LoadUnmanagedDll(string unmanagedDllName)
return base.LoadUnmanagedDll(unmanagedDllName);
}

private string GetRuntimeNativeAssetPath(string assetFileName, bool isNativeAsset)
private string GetRuntimeNativeAssetPath(string assetFileName)
{
string basePath = _probingPaths[0];
string ridSubFolder = isNativeAsset ? "native" : string.Empty;
const string ridSubFolder = "native";
string runtimesPath = Path.Combine(basePath, "runtimes");

List<string> rids = DependencyHelper.GetRuntimeFallbacks();

return rids.Select(r => Path.Combine(runtimesPath, r, ridSubFolder, assetFileName))
string result = rids.Select(r => Path.Combine(runtimesPath, r, ridSubFolder, assetFileName))
.Union(_probingPaths)
.FirstOrDefault(p => File.Exists(p));

if (result == null && _nativeLibraries != null)
{
if (_nativeLibraries.TryGetValue(assetFileName, out string relativePath))
{
string nativeLibraryFullPath = Path.Combine(basePath, relativePath);
if (File.Exists(nativeLibraryFullPath))
{
result = nativeLibraryFullPath;
}
}
}

return result;
}

internal string GetUnmanagedLibraryFileName(string unmanagedLibraryName)
Expand Down
38 changes: 34 additions & 4 deletions test/WebJobs.Script.Tests/Description/DotNet/FunctionAssemblyLoadContextTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,16 +44,46 @@ public void InitializeDeps_LoadsExpectedDependencies()
{
string depsPath = Path.Combine(Directory.GetCurrentDirectory(), "Description", "DotNet", "TestFiles", "DepsFiles");

IDictionary<string, string> assemblies = FunctionAssemblyLoadContext.InitializeDeps(depsPath);
(IDictionary<string, string> depsAssemblies, IDictionary<string, string> nativeLibraries) =
FunctionAssemblyLoadContext.InitializeDeps(depsPath);

string testRid = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "win" : "unix";

// Ensure runtime specific dependencies are resolved, with appropriate RID
Assert.Contains($"runtimes/{testRid}/lib/netstandard2.0/System.Private.ServiceModel.dll", assemblies.Values);
Assert.Contains($"runtimes/{testRid}/lib/netstandard1.3/System.Text.Encoding.CodePages.dll", assemblies.Values);
Assert.Contains($"runtimes/{testRid}/lib/netstandard2.0/System.Private.ServiceModel.dll", depsAssemblies.Values);
Assert.Contains($"runtimes/{testRid}/lib/netstandard1.3/System.Text.Encoding.CodePages.dll", depsAssemblies.Values);

// Ensure flattened dependency has expected path
Assert.Contains($"Microsoft.Azure.WebJobs.Host.Storage.dll", assemblies.Values);
Assert.Contains($"Microsoft.Azure.WebJobs.Host.Storage.dll", depsAssemblies.Values);

// Ensure native libraries are resolved, with appropriate RID and path
string nativeRid;
string nativePrefix;
string nativeExtension;
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
nativeRid = "win-";
nativePrefix = string.Empty;
nativeExtension = "dll";
}
else
{
nativePrefix = "lib";
if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
{
nativeRid = "osx-";
nativeExtension = "dylib";
}
else
{
nativeRid = "linux-";
nativeExtension = "so";
}
}

nativeRid += Environment.Is64BitProcess ? "x64" : "x86";

Assert.Contains($"runtimes/{nativeRid}/nativeassets/netstandard2.0/{nativePrefix}CpuMathNative.{nativeExtension}", nativeLibraries.Values);
}
}
}
41 changes: 41 additions & 0 deletions test/WebJobs.Script.Tests/Description/DotNet/TestFiles/DepsFiles/function.deps.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
".NETCoreApp,Version=v2.1": {
"FunctionApp66/1.0.0": {
"dependencies": {
"Microsoft.ML.CpuMath": "1.4.0",
"Microsoft.NET.Sdk.Functions": "1.0.23",
"System.ServiceModel.Http": "4.5.3"
},
Expand Down Expand Up @@ -685,6 +686,39 @@
}
}
},
"Microsoft.ML.CpuMath/1.4.0": {
"dependencies": {
"System.Memory": "4.5.0"
},
"runtime": {
"lib/netstandard2.0/Microsoft.ML.CpuMath.dll": {
"assemblyVersion": "1.0.0.0",
"fileVersion": "1.4.28230.4"
}
},
"runtimeTargets": {
"runtimes/linux-x64/nativeassets/netstandard2.0/libCpuMathNative.so": {
"rid": "linux-x64",
"assetType": "native",
"fileVersion": "0.0.0.0"
},
"runtimes/osx-x64/nativeassets/netstandard2.0/libCpuMathNative.dylib": {
"rid": "osx-x64",
"assetType": "native",
"fileVersion": "0.0.0.0"
},
"runtimes/win-x64/nativeassets/netstandard2.0/CpuMathNative.dll": {
"rid": "win-x64",
"assetType": "native",
"fileVersion": "1.4.28230.4"
},
"runtimes/win-x86/nativeassets/netstandard2.0/CpuMathNative.dll": {
"rid": "win-x86",
"assetType": "native",
"fileVersion": "1.4.28230.4"
}
}
},
"Microsoft.Net.Http.Headers/2.1.0": {
"dependencies": {
"Microsoft.Extensions.Primitives": "2.1.0",
Expand Down Expand Up @@ -1713,6 +1747,13 @@
"path": "microsoft.extensions.primitives/2.1.0",
"hashPath": "microsoft.extensions.primitives.2.1.0.nupkg.sha512"
},
"Microsoft.ML.CpuMath/1.4.0": {
"type": "package",
"serviceable": true,
"sha512": "sha512-eRDY1KMcDc4iT075MOWlQL4RBrvI9GYl0wD/LiCR4h49hej61qHk88FW3ij3kBRrxSG9ssG8jJJJCNaQeGUaZg==",
"path": "microsoft.ml.cpumath/1.4.0",
"hashPath": "microsoft.ml.cpumath.1.4.0.nupkg.sha512"
},
"Microsoft.Net.Http.Headers/2.1.0": {
"type": "package",
"serviceable": true,
Expand Down

0 comments on commit 9c47cf4

Please sign in to comment.