[release/10.0.1xx] Support for exempting native libraries from JNI preload

Documentation/docs-mobile/building-apps/build-items.md

@@ -33,7 +33,7 @@ The following MSBuild metadata are required:
 - `%(JavaArtifact)`: The group and artifact id of the Java library matching the specifed POM
   file in the form `{GroupId}:{ArtifactId}`.
 - `%(JavaVersion)`: The version of the Java library matching the specified POM file.
-  
+
 See the [Java Dependency Resolution documentation](../features/maven/java-dependency-verification.md)
 for more details.
 
@@ -279,9 +279,9 @@ installing app bundles.
 
 ## AndroidMavenLibrary
 
-`<AndroidMavenLibrary>` allows a Maven artifact to be specified which will 
-automatically be downloaded and added to a .NET for Android binding project. 
-This can be useful to simplify maintenance of .NET for Android bindings for artifacts 
+`<AndroidMavenLibrary>` allows a Maven artifact to be specified which will
+automatically be downloaded and added to a .NET for Android binding project.
+This can be useful to simplify maintenance of .NET for Android bindings for artifacts
 hosted in Maven.
 
 ```xml
@@ -336,6 +336,20 @@ used to specify the ABI that the library targets. Thus, if you add
 </ItemGroup>
 ```
 
+## AndroidNativeLibraryNoJniPreload
+
+Every native library included in this item group will be exempt from the
+JNI library preload mechanism. By default, all such libraries will be loaded
+by the runtime early during application startup in order to assure their
+proper initialization. However, in some cases it might not be the desired
+behavior and this item group allows exclusion of libraries from this process
+on an individual basis.
+
+Some framework libraries which must be loaded at application startup will not
+be affected if included in this item group.
+
+See also [`$(AndroidIgnoreAllJniPreload)`](build-properties.md#androidignorealljnipreload)
+
 ## AndroidPackagingOptionsExclude
 
 A set of file glob compatible items which will allow for items to be

Documentation/docs-mobile/building-apps/build-properties.md

@@ -689,6 +689,20 @@ The most common values for this property are:
 > [`@(AndroidEnvironment)`](build-items.md#androidenvironment)
 > will take precedence.
 
+## AndroidIgnoreAllJniPreload
+
+A boolean value which, if set to `true`, exempts all the native JNI libraries
+from being preloaded at application startup.  By default, all such libraries
+will be loaded by the runtime early during application startup in order to
+assure their proper initialization. However, in some cases it might not be the
+desired behavior and this property allows you to effectively disable it.
+
+Some framework libraries which must be loaded at application startup will not
+be affected by this property.
+
+See also [`@(AndroidNativeLibraryNoJniPreload)`](build-items.md#androidnativelibrarynojnipreload)
+for a more fine-grained way to exempt libraries from the preload mechanism.
+
 ## AndroidIncludeWrapSh
 
 A boolean value that indicates whether the Android wrapper script

build-tools/xaprepare/xaprepare/Application/KnownProperties.cs

@@ -55,6 +55,7 @@ static class KnownProperties
 		public const string PkgXamarin_LibZipSharp              = "PkgXamarin_LibZipSharp";
 		public const string ProductVersion                      = "ProductVersion";
 		public const string RuntimeRedistDirName                = "_RuntimeRedistDirName";
+		public const string TestOutputDirectory                 = "TestOutputDirectory";
 		public const string XABuildToolsFolder                  = "XABuildToolsFolder";
 		public const string XABuildToolsVersion                 = "XABuildToolsVersion";
 		public const string XABuildToolsPackagePrefixMacOS      = "XABuildToolsPackagePrefixMacOS";

build-tools/xaprepare/xaprepare/Application/Properties.Defaults.cs.in

@@ -59,6 +59,7 @@ namespace Xamarin.Android.Prepare
 			properties.Add (KnownProperties.PkgXamarin_LibZipSharp,              StripQuotes (@"@PkgXamarin_LibZipSharp@"));
 			properties.Add (KnownProperties.ProductVersion,                      StripQuotes ("@ProductVersion@"));
 			properties.Add (KnownProperties.RuntimeRedistDirName,                StripQuotes ("@_RuntimeRedistDirName@"));
+			properties.Add (KnownProperties.TestOutputDirectory,                 StripQuotes (@"@TestOutputDirectory@"));
 			properties.Add (KnownProperties.XABuildToolsFolder,                  StripQuotes (@"@XABuildToolsFolder@"));
 			properties.Add (KnownProperties.XABuildToolsVersion,                 StripQuotes ("@XABuildToolsVersion@"));
 			properties.Add (KnownProperties.XABuildToolsPackagePrefixMacOS,      StripQuotes ("@XABuildToolsPackagePrefixMacOS@"));

build-tools/xaprepare/xaprepare/Steps/Step_GenerateFiles.cs

@@ -127,6 +127,7 @@ GeneratedFile GetCmakePresetsCommon (Context context, string sourcesDir)
 				{ "@NDK_X86_API_NET@",           BuildAndroidPlatforms.NdkMinimumAPI.ToString () },
 				{ "@NDK_X86_64_API_NET@",        BuildAndroidPlatforms.NdkMinimumAPI.ToString () },
 				{ "@XA_BUILD_CONFIGURATION@",    context.Configuration },
+				{ "@XA_TEST_OUTPUT_DIR@", Utilities.EscapePathSeparators (props.GetRequiredValue (KnownProperties.TestOutputDirectory)) },
 			};
 
 			return new GeneratedPlaceholdersFile (

build-tools/xaprepare/xaprepare/xaprepare.targets

@@ -92,6 +92,7 @@
       <Replacement Include="@PkgXamarin_LibZipSharp@=$(PkgXamarin_LibZipSharp)" />
       <Replacement Include="@ProductVersion@=$(ProductVersion)" />
       <Replacement Include="@_RuntimeRedistDirName@=$(_RuntimeRedistDirName)" />
+      <Replacement Include="@TestOutputDirectory@=$(TestOutputDirectory)" />
       <Replacement Include="@XABuildToolsFolder@=$(XABuildToolsFolder)" />
       <Replacement Include="@XABuildToolsVersion@=$(XABuildToolsVersion)" />
       <Replacement Include="@XABuildToolsPackagePrefixMacOS@=$(XABuildToolsPackagePrefixMacOS)" />

src/Xamarin.Android.Build.Tasks/Tasks/GenerateNativeApplicationConfigSources.cs

@@ -28,6 +28,8 @@ public class GenerateNativeApplicationConfigSources : AndroidTask
 		public ITaskItem[] ResolvedAssemblies { get; set; } = [];
 
 		public ITaskItem[]? NativeLibraries { get; set; }
+		public ITaskItem[]? NativeLibrariesNoJniPreload { get; set; }
+		public ITaskItem[]? NativeLibrariesAlwaysJniPreload { get; set; }
 
 		public ITaskItem[]? MonoComponents { get; set; }
 
@@ -253,6 +255,8 @@ public override bool RunTask ()
 					NumberOfAssembliesInApk = assemblyCount,
 					BundledAssemblyNameWidth = assemblyNameWidth,
 					NativeLibraries = uniqueNativeLibraries,
+					NativeLibrariesNoJniPreload = NativeLibrariesNoJniPreload,
+					NativeLibrariesAlwaysJniPreload = NativeLibrariesAlwaysJniPreload,
 					AndroidRuntimeJNIEnvToken = android_runtime_jnienv_class_token,
 					JNIEnvInitializeToken = jnienv_initialize_method_token,
 					JNIEnvRegisterJniNativesToken = jnienv_registerjninatives_method_token,
@@ -279,6 +283,8 @@ public override bool RunTask ()
 					BundledAssemblyNameWidth = assemblyNameWidth,
 					MonoComponents = (MonoComponent)monoComponents,
 					NativeLibraries = uniqueNativeLibraries,
+					NativeLibrariesNoJniPreload = NativeLibrariesNoJniPreload,
+					NativeLibrariesAlwaysJniPreload = NativeLibrariesAlwaysJniPreload,
 					HaveAssemblyStore = UseAssemblyStore,
 					AndroidRuntimeJNIEnvToken = android_runtime_jnienv_class_token,
 					JNIEnvInitializeToken = jnienv_initialize_method_token,

src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest3.cs

@@ -0,0 +1,269 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using NUnit.Framework;
+using Xamarin.Android.Tasks;
+using Xamarin.Android.Tools;
+using Xamarin.ProjectTools;
+
+namespace Xamarin.Android.Build.Tests;
+
+[Parallelizable (ParallelScope.Children)]
+public partial class BuildTest3 : BaseTest
+{
+	const int ExpectedJniPreloadIndexStride = 4;
+	const string JniPreloadSourceLibraryName = "libtest-jni-library.so";
+
+	[Test]
+	public void NativeLibraryJniPreload_NoDuplicates ()
+	{
+		const string MyLibKeep1 = "libMyStuffKeep.so";
+		const string MyLibKeep2 = "libMyStuffKeep.so";
+
+		List<EnvironmentHelper.JniPreloads>? allPreloads = NativeLibraryJniPreload_CommonInitAndGetPreloads (
+			(XamarinAndroidApplicationProject proj, AndroidTargetArch[] supportedArches) => {
+				NativeLibraryJniPreload_AddNativeLibraries (proj, supportedArches, MyLibKeep1, MyLibKeep2);
+			}
+		);
+		if (allPreloads == null) {
+			return;
+		}
+
+		NativeLibraryJniPreload_VerifyLibs (allPreloads, new List<string> { MyLibKeep1 });
+	}
+
+	[Test]
+	public void NativeLibraryJniPreload_IncludeCustomLibraries ()
+	{
+		const string MyLib = "libMyStuff.so";
+
+		List<EnvironmentHelper.JniPreloads>? allPreloads = NativeLibraryJniPreload_CommonInitAndGetPreloads (
+			(XamarinAndroidApplicationProject proj, AndroidTargetArch[] supportedArches) => {
+				NativeLibraryJniPreload_AddNativeLibraries (proj, supportedArches, MyLib);
+			}
+		);
+		if (allPreloads == null) {
+			return;
+		}
+
+		NativeLibraryJniPreload_VerifyLibs (allPreloads, new List<string> { MyLib });
+	}
+
+	[Test]
+	public void NativeLibraryJniPreload_ExcludeSomeCustomLibraries ()
+	{
+		const string MyLibKeep = "libMyStuffKeep.so";
+		const string MyLibExempt = "libMyStuffExempt.so";
+
+		List<EnvironmentHelper.JniPreloads>? allPreloads = NativeLibraryJniPreload_CommonInitAndGetPreloads (
+			(XamarinAndroidApplicationProject proj, AndroidTargetArch[] supportedArches) => {
+				NativeLibraryJniPreload_AddNativeLibraries (proj, supportedArches, MyLibKeep, MyLibExempt);
+				proj.OtherBuildItems.Add (
+					new AndroidItem.AndroidNativeLibraryNoJniPreload (MyLibExempt)
+				);
+			}
+		);
+		if (allPreloads == null) {
+			return;
+		}
+
+		NativeLibraryJniPreload_VerifyLibs (allPreloads, new List<string> { MyLibKeep });
+	}
+
+	[Test]
+	public void NativeLibraryJniPreload_ExcludeAllCustomLibraries ()
+	{
+		const string MyLibExempt1 = "libMyStuffExempt1.so";
+		const string MyLibExempt2 = "libMyStuffExempt2.so";
+
+		List<EnvironmentHelper.JniPreloads>? allPreloads = NativeLibraryJniPreload_CommonInitAndGetPreloads (
+			(XamarinAndroidApplicationProject proj, AndroidTargetArch[] supportedArches) => {
+				NativeLibraryJniPreload_AddNativeLibraries (proj, supportedArches, MyLibExempt1, MyLibExempt2);
+				proj.OtherBuildItems.Add (
+					new AndroidItem.AndroidNativeLibraryNoJniPreload (MyLibExempt1)
+				);
+				proj.OtherBuildItems.Add (
+					new AndroidItem.AndroidNativeLibraryNoJniPreload (MyLibExempt2)
+				);
+			}
+		);
+		if (allPreloads == null) {
+			return;
+		}
+
+		NativeLibraryJniPreload_VerifyDefaults (allPreloads);
+	}
+
+	[Test]
+	public void NativeLibraryJniPreload_AddSomeCustomLibrariesAndIgnoreAll ()
+	{
+		List<EnvironmentHelper.JniPreloads>? allPreloads = NativeLibraryJniPreload_CommonInitAndGetPreloads (
+			(XamarinAndroidApplicationProject proj, AndroidTargetArch[] supportedArches) => {
+				NativeLibraryJniPreload_AddNativeLibraries (proj, supportedArches, "libMyStuffOne.so", "libMyStuffTwo.so");
+				proj.SetProperty ("AndroidIgnoreAllJniPreload", "true");
+			}
+		);
+		if (allPreloads == null) {
+			return;
+		}
+
+		// With `$(AndroidIgnoreAllJniPreload)=true` we still must have the defaults in the generated code.
+		NativeLibraryJniPreload_VerifyDefaults (allPreloads);
+	}
+
+	[Test]
+	public void NativeLibraryJniPreload_AddSomeCustomLibrariesAndIgnoreAllByName ()
+	{
+		const string MyLibExemptOne = "libMyStuffExemptOne.so";
+		const string MyLibExemptTwo = "libMyStuffExemptTwo.so";
+
+		List<EnvironmentHelper.JniPreloads>? allPreloads = NativeLibraryJniPreload_CommonInitAndGetPreloads (
+			(XamarinAndroidApplicationProject proj, AndroidTargetArch[] supportedArches) => {
+				NativeLibraryJniPreload_AddNativeLibraries (proj, supportedArches, MyLibExemptOne, MyLibExemptTwo);
+				proj.OtherBuildItems.Add (
+					new AndroidItem.AndroidNativeLibraryNoJniPreload (MyLibExemptOne)
+				);
+				proj.OtherBuildItems.Add (
+					new AndroidItem.AndroidNativeLibraryNoJniPreload (MyLibExemptTwo)
+				);
+			}
+		);
+		if (allPreloads == null) {
+			return;
+		}
+
+		// With all custom libraries ignored, we still must have the defaults in the generated code.
+		NativeLibraryJniPreload_VerifyDefaults (allPreloads);
+	}
+
+	void NativeLibraryJniPreload_AddNativeLibraries (XamarinAndroidApplicationProject proj, AndroidTargetArch[] supportedArches, string libName, params string[]? moreLibNames)
+	{
+		var libNames = new List<string> {
+			libName,
+		};
+		if (moreLibNames != null && moreLibNames.Length > 0) {
+			libNames.AddRange (moreLibNames);
+		}
+
+		foreach (AndroidTargetArch arch in supportedArches) {
+			string libPath = Path.Combine (XABuildPaths.TestAssemblyOutputDirectory, MonoAndroidHelper.ArchToRid (arch), JniPreloadSourceLibraryName);
+			Assert.IsTrue (File.Exists (libPath), $"Native library '{libPath}' does not exist.");
+
+			foreach (string lib in libNames) {
+				string abi = MonoAndroidHelper.ArchToAbi (arch);
+				proj.OtherBuildItems.Add (
+					new AndroidItem.AndroidNativeLibrary ($"native/{abi}/{lib}") {
+						BinaryContent = () => File.ReadAllBytes (libPath),
+						MetadataValues = $"Link={abi}/{lib}",
+					}
+				);
+			}
+		}
+	}
+
+	[Test]
+	public void NativeLibraryJniPreload_IgnoreAll_PreservesRequired ()
+	{
+		List<EnvironmentHelper.JniPreloads>? allPreloads = NativeLibraryJniPreload_CommonInitAndGetPreloads (
+			(XamarinAndroidApplicationProject proj, AndroidTargetArch[] supportedArches) => {
+				proj.SetProperty ("AndroidIgnoreAllJniPreload", "true");
+			}
+		);
+
+		// With `$(AndroidIgnoreAllJniPreload)=true` we still must have the defaults in the generated code.
+		NativeLibraryJniPreload_VerifyDefaults (allPreloads);
+	}
+
+	[Test]
+	public void NativeLibraryJniPreload_DefaultsWork ()
+	{
+		List<EnvironmentHelper.JniPreloads>? allPreloads = NativeLibraryJniPreload_CommonInitAndGetPreloads ();
+		NativeLibraryJniPreload_VerifyDefaults (allPreloads);
+	}
+
+	void NativeLibraryJniPreload_VerifyDefaults (List<EnvironmentHelper.JniPreloads>? allPreloads)
+	{
+		NativeLibraryJniPreload_VerifyLibs (allPreloads, additionalLibs: null);
+	}
+
+	void NativeLibraryJniPreload_VerifyLibs (List<EnvironmentHelper.JniPreloads>? allPreloads, List<string>? additionalLibs)
+	{
+		if (allPreloads == null) {
+			return;
+		}
+
+		int numberOfLibs = 1;
+		if (additionalLibs != null) {
+			numberOfLibs += additionalLibs.Count;
+		}
+
+		int ExpectedEntryCount = ExpectedJniPreloadIndexStride * numberOfLibs;
+		foreach (EnvironmentHelper.JniPreloads preloads in allPreloads) {
+			Assert.IsTrue (preloads.IndexStride == (uint)ExpectedJniPreloadIndexStride, $"JNI preloads index stride should be {ExpectedJniPreloadIndexStride}, was {preloads.IndexStride} instead. Source file: {preloads.SourceFile}");
+			Assert.IsTrue (preloads.Entries.Count == ExpectedEntryCount, $"JNI preloads index entry count should be {ExpectedEntryCount}, was {preloads.Entries.Count} instead. Source file: {preloads.SourceFile}");
+
+			// DSO cache entries are sorted based on their **mutated name's** 64-bit xxHash, which
+			// won't change but builds may add/remove libraries and, thus, change the indexes after
+			// sorting. For that reason we don't verify the index values and use them just for reporting.
+			//
+			// Also, all the entries will point to the same library name. Name variations aren't
+			// stored directly in the DSO cache, just their hashes which are used for lookup at run time.
+			//
+			// We use a Dictionary<> here because there might be more libraries to preload in the future.
+			var expectedLibNames = new Dictionary<string, uint> (StringComparer.Ordinal) {
+				{ "libSystem.Security.Cryptography.Native.Android.so", 0 },
+			};
+
+			if (additionalLibs != null) {
+				foreach (string extraLib in additionalLibs) {
+					expectedLibNames.Add (extraLib, 0);
+				}
+			}
+
+			for (int i = 0; i < preloads.Entries.Count; i++) {
+				EnvironmentHelper.JniPreloadsEntry entry = preloads.Entries[i];
+				Assert.IsFalse (entry.LibraryName == "libmonodroid.so", $"JNI preloads entry at index {i} refers to the .NET for Android native runtime. It must never be preloaded. Source file: {preloads.SourceFile}");
+				Assert.IsTrue (expectedLibNames.ContainsKey (entry.LibraryName), $"JNI preloads entry at index {i}, referring to library at DSO cache index {entry.Index} has unexpected name '{entry.LibraryName}';  Source file: {preloads.SourceFile}");
+				expectedLibNames[entry.LibraryName]++;
+			}
+
+			foreach (var kvp in expectedLibNames) {
+				Assert.IsTrue (kvp.Value == ExpectedJniPreloadIndexStride, $"JNI preloads entry '{kvp.Key}' should have {ExpectedJniPreloadIndexStride} instances, it had {kvp.Value} instead. Source file: {preloads.SourceFile}");
+			}
+		}
+	}
+
+	List<EnvironmentHelper.JniPreloads>? NativeLibraryJniPreload_CommonInitAndGetPreloads (Action<XamarinAndroidApplicationProject, AndroidTargetArch[]>? configureProject = null)
+	{
+		const bool isRelease = true;
+		const AndroidRuntime runtime = AndroidRuntime.MonoVM;
+
+		AndroidTargetArch[] supportedArches = new [] {
+			AndroidTargetArch.Arm64,
+			AndroidTargetArch.X86_64,
+		};
+
+		var proj = new XamarinAndroidApplicationProject {
+			IsRelease = isRelease,
+		};
+		proj.SetRuntime (runtime);
+		proj.SetRuntimeIdentifiers (supportedArches);
+		configureProject?.Invoke (proj, supportedArches);
+
+		using var builder = CreateApkBuilder ();
+		Assert.IsTrue (builder.Build (proj), "Build should have succeeded.");
+
+		string objDirPath = Path.Combine (Root, builder.ProjectDirectory, proj.IntermediateOutputPath);
+		List<EnvironmentHelper.EnvironmentFile> envFiles = EnvironmentHelper.GatherEnvironmentFiles (
+			objDirPath,
+			String.Join (";", supportedArches.Select (arch => MonoAndroidHelper.ArchToAbi (arch))),
+			true
+		);
+
+		EnvironmentHelper.ApplicationConfig app_config = EnvironmentHelper.ReadApplicationConfig (envFiles);
+		uint numberOfDsoCacheEntries = app_config.number_of_dso_cache_entries;
+
+		return EnvironmentHelper.ReadJniPreloads (envFiles, numberOfDsoCacheEntries, runtime);
+	}
+}