GraalVM updates to extract the shared lib with native image instead of package as resource

.github/workflows/ci.yml

@@ -192,7 +192,7 @@ jobs:
           python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')"
           python builder.pyz build -p ${{ env.PACKAGE_NAME }} downstream
 
-  macos: 
+  macos:
     runs-on: macos-14 #latest
     steps:
       - name: Checkout Sources
@@ -204,7 +204,7 @@ jobs:
           python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')"
           chmod a+x builder
           ./builder build -p ${{ env.PACKAGE_NAME }} --spec=downstream
-          python3 codebuild/macos_compatibility_check.py 
+          python3 codebuild/macos_compatibility_check.py
 
   macos-x64:
     runs-on: macos-14-large #latest

.github/workflows/docs.yml

@@ -17,7 +17,6 @@ jobs:
         uses: actions/checkout@v3
         with:
           submodules: true
-
       - name: Update docs branch
         run: |
           ./make-docs.sh

android/crt/build.gradle

@@ -76,6 +76,7 @@ android {
         main {
             java.srcDir '../../src/main/java'
             java.srcDir 'src/main/java'
+            java.exclude '**/GraalVMNativeFeature.java'
         }
         androidTest {
             setRoot '../../src/test'

javadoc.options

@@ -6,7 +6,7 @@
 -bottom 'Copyright © Amazon.com, Inc. or its affiliates. All Rights Reserved.'
 -sourcepath src/main/java
 -notimestamp
--subpackages software.amazon.awssdk.crt
+-exclude software.amazon.awssdk.crt.internal
 -quiet
 -Xdoclint:all
 -Xwerror

make-docs.sh

@@ -2,12 +2,12 @@
 
 set -e
 
-pushd $(dirname $0) > /dev/null
+pushd $(dirname $0) >/dev/null
 
 # clean
 rm -rf docs/
 
 # build
-javadoc @javadoc.options
+mvn clean javadoc:javadoc -Prelease
 
-popd > /dev/null
+popd >/dev/null

pom.xml

@@ -292,7 +292,13 @@
                     <plugin>
                         <groupId>org.apache.maven.plugins</groupId>
                         <artifactId>maven-javadoc-plugin</artifactId>
-                        <version>2.9.1</version>
+                        <version>3.4.0</version>
+                        <configuration>
+                            <reportOutputDirectory>${project.basedir}/docs</reportOutputDirectory>
+                            <additionalJOptions>
+                                @${project.basedir}/javadoc.options
+                            </additionalJOptions>
+                        </configuration>
                         <executions>
                             <execution>
                                 <id>attach-javadocs</id>
@@ -351,6 +357,13 @@
             <version>1.4</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.graalvm.sdk</groupId>
+            <artifactId>graal-sdk</artifactId>
+            <version>21.3.11</version>          <!-- The latest version supports JDK8 -->
+            <scope>provided</scope>
+            <optional>true</optional>
+        </dependency>
     </dependencies>
     <build>
         <resources>

src/main/java/software/amazon/awssdk/crt/CRT.java

@@ -4,16 +4,15 @@
  */
 package software.amazon.awssdk.crt;
 
+import software.amazon.awssdk.crt.internal.ExtractLib;
 import software.amazon.awssdk.crt.io.ClientBootstrap;
 import software.amazon.awssdk.crt.io.EventLoopGroup;
 import software.amazon.awssdk.crt.io.HostResolver;
 
 import java.io.BufferedReader;
 import java.io.File;
-import java.io.FileOutputStream;
 import java.io.FilenameFilter;
 import java.io.IOException;
-import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.lang.reflect.InvocationTargetException;
 import java.util.*;
@@ -29,7 +28,7 @@ public final class CRT {
     private static final String CRT_ARCH_OVERRIDE_SYSTEM_PROPERTY = "aws.crt.arch";
     private static final String CRT_ARCH_OVERRIDE_ENVIRONMENT_VARIABLE = "AWS_CRT_ARCH";
 
-    private static final String CRT_LIB_NAME = "aws-crt-jni";
+    public static final String CRT_LIB_NAME = "aws-crt-jni";
     public static final int AWS_CRT_SUCCESS = 0;
     private static final CrtPlatform s_platform;
 
@@ -246,112 +245,65 @@ private static List<String> runProcess(String[] cmdArray) throws IOException {
     }
 
     private static void extractAndLoadLibrary(String path) {
+        // Check java.io.tmpdir
+        String tmpdirPath;
+        File tmpdirFile;
         try {
-            // Check java.io.tmpdir
-            String tmpdirPath;
-            File tmpdirFile;
-            try {
-                tmpdirFile = new File(path).getAbsoluteFile();
-                tmpdirPath = tmpdirFile.getAbsolutePath();
-                if (tmpdirFile.exists()) {
-                    if (!tmpdirFile.isDirectory()) {
-                        throw new IOException("not a directory: " + tmpdirPath);
-                    }
-                } else {
-                    tmpdirFile.mkdirs();
-                }
-
-                if (!tmpdirFile.canRead() || !tmpdirFile.canWrite()) {
-                    throw new IOException("access denied: " + tmpdirPath);
-                }
-            } catch (Exception ex) {
-                String msg = "Invalid directory: " + path;
-                throw new IOException(msg, ex);
-            }
-
-            String libraryName = System.mapLibraryName(CRT_LIB_NAME);
-
-            // Prefix the lib we'll extract to disk
-            String tempSharedLibPrefix = "AWSCRT_";
-
-            File tempSharedLib = File.createTempFile(tempSharedLibPrefix, libraryName, tmpdirFile);
-            if (!tempSharedLib.setExecutable(true, true)) {
-                throw new CrtRuntimeException("Unable to make shared library executable by owner only");
-            }
-            if (!tempSharedLib.setWritable(true, true)) {
-                throw new CrtRuntimeException("Unable to make shared library writeable by owner only");
-            }
-            if (!tempSharedLib.setReadable(true, true)) {
-                throw new CrtRuntimeException("Unable to make shared library readable by owner only");
-            }
-
-			// The temp lib file should be deleted when we're done with it.
-			// Ask Java to try and delete it on exit. We call this immediately
-			// so that if anything goes wrong writing the file to disk, or
-			// loading it as a shared lib, it will still get cleaned up.
-			tempSharedLib.deleteOnExit();
-
-            // Unfortunately File.deleteOnExit() won't work on Windows, where
-            // files cannot be deleted while they're in use. On Windows, once
-            // our .dll is loaded, it can't be deleted by this process.
-            //
-            // The Windows-only solution to this problem is to scan on startup
-            // for old instances of the .dll and try to delete them. If another
-            // process is still using the .dll, the delete will fail, which is fine.
-            String os = getOSIdentifier();
-            if (os.equals("windows")) {
-                tryDeleteOldLibrariesFromTempDir(tmpdirFile, tempSharedLibPrefix, libraryName);
-            }
-
-            // open a stream to read the shared lib contents from this JAR
-            String libResourcePath = "/" + os + "/" + getArchIdentifier() + "/" +  getCRuntime(os) + "/" + libraryName;
-            // Check whether there is a platform specific resource path to use
-            CrtPlatform platform = getPlatformImpl();
-            if (platform != null){
-                String platformLibResourcePath = platform.getResourcePath(getCRuntime(os), libraryName);
-                if (platformLibResourcePath != null){
-                    libResourcePath = platformLibResourcePath;
+            tmpdirFile = new File(path).getAbsoluteFile();
+            tmpdirPath = tmpdirFile.getAbsolutePath();
+            if (tmpdirFile.exists()) {
+                if (!tmpdirFile.isDirectory()) {
+                    throw new IOException("not a directory: " + tmpdirPath);
                 }
+            } else {
+                tmpdirFile.mkdirs();
             }
 
-            try (InputStream in = CRT.class.getResourceAsStream(libResourcePath)) {
-                if (in == null) {
-                    throw new IOException("Unable to open library in jar for AWS CRT: " + libResourcePath);
-                }
-
-                // Copy from jar stream to temp file
-                try (FileOutputStream out = new FileOutputStream(tempSharedLib)) {
-                    int read;
-                    byte [] bytes = new byte[1024];
-                    while ((read = in.read(bytes)) != -1){
-                        out.write(bytes, 0, read);
-                    }
-                }
+            if (!tmpdirFile.canRead() || !tmpdirFile.canWrite()) {
+                throw new IOException("access denied: " + tmpdirPath);
             }
+        } catch (IOException ex) {
+            CrtRuntimeException rex = new CrtRuntimeException("Invalid directory: " + path);
+            rex.initCause(ex);
+            throw rex;
+        }
 
-            if (!tempSharedLib.setWritable(false)) {
-                throw new CrtRuntimeException("Unable to make shared library read-only");
-            }
+        String libraryName = System.mapLibraryName(CRT_LIB_NAME);
 
-            // load the shared lib from the temp path
-            System.load(tempSharedLib.getAbsolutePath());
-        } catch (CrtRuntimeException crtex) {
-            System.err.println("Unable to initialize AWS CRT: " + crtex);
-            crtex.printStackTrace();
-            throw crtex;
-        } catch (UnknownPlatformException upe) {
-            System.err.println("Unable to determine platform for AWS CRT: " + upe);
-            upe.printStackTrace();
-            CrtRuntimeException rex = new CrtRuntimeException("Unable to determine platform for AWS CRT");
-            rex.initCause(upe);
-            throw rex;
-        } catch (Exception ex) {
-            System.err.println("Unable to unpack AWS CRT lib: " + ex);
+        // Prefix the lib we'll extract to disk
+        String tempSharedLibPrefix = "AWSCRT_";
+        File tempSharedLib = null;
+        try{
+            tempSharedLib = File.createTempFile(tempSharedLibPrefix, libraryName, tmpdirFile);
+        }
+        catch (IOException ex){
+            System.err.println("Unable to create temp file to extract AWS CRT library" + ex);
             ex.printStackTrace();
-            CrtRuntimeException rex = new CrtRuntimeException("Unable to unpack AWS CRT library");
+            CrtRuntimeException rex = new CrtRuntimeException("Unable to create temp file to extract AWS CRT library");
             rex.initCause(ex);
             throw rex;
         }
+        // The temp lib file should be deleted when we're done with it.
+        // Ask Java to try and delete it on exit. We call this immediately
+        // so that if anything goes wrong writing the file to disk, or
+        // loading it as a shared lib, it will still get cleaned up.
+        tempSharedLib.deleteOnExit();
+
+        // Unfortunately File.deleteOnExit() won't work on Windows, where
+        // files cannot be deleted while they're in use. On Windows, once
+        // our .dll is loaded, it can't be deleted by this process.
+        //
+        // The Windows-only solution to this problem is to scan on startup
+        // for old instances of the .dll and try to delete them. If another
+        // process is still using the .dll, the delete will fail, which is fine.
+        String os = getOSIdentifier();
+        if (os.equals("windows")) {
+            tryDeleteOldLibrariesFromTempDir(tmpdirFile, tempSharedLibPrefix, libraryName);
+        }
+        ExtractLib.extractLibrary(tempSharedLib);
+        // load the shared lib from the temp path
+        System.load(tempSharedLib.getAbsolutePath());
+
     }
 
     private static void loadLibraryFromJar() {

src/main/java/software/amazon/awssdk/crt/internal/ExtractLib.java

@@ -0,0 +1,107 @@
+/**
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0.
+ */
+package software.amazon.awssdk.crt.internal;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+
+import software.amazon.awssdk.crt.CRT;
+import software.amazon.awssdk.crt.CRT.UnknownPlatformException;
+import software.amazon.awssdk.crt.CrtPlatform;
+import software.amazon.awssdk.crt.CrtRuntimeException;
+
+/**
+ * Helper to extract JNI shared lib from Jar.
+ * Internal API, not for external usage.
+ */
+public class ExtractLib {
+
+    /**
+     * Extract the CRT JNI library on current platform to a specific File
+     *
+     * @param extractFile the File extracting to
+     */
+    public static void extractLibrary(File extractFile) {
+
+        try {
+            if (!extractFile.setExecutable(true, true)) {
+                throw new CrtRuntimeException("Unable to make shared library executable by owner only");
+            }
+            if (!extractFile.setWritable(true, true)) {
+                throw new CrtRuntimeException("Unable to make shared library writeable by owner only");
+            }
+            if (!extractFile.setReadable(true, true)) {
+                throw new CrtRuntimeException("Unable to make shared library readable by owner only");
+            }
+            String libraryName = System.mapLibraryName(CRT.CRT_LIB_NAME);
+            String os = CRT.getOSIdentifier();
+            // open a stream to read the shared lib contents from this JAR
+            String libResourcePath = "/" + os + "/" + CRT.getArchIdentifier() + "/" + CRT.getCRuntime(os) + "/"
+                    + libraryName;
+            // Check whether there is a platform specific resource path to use
+            CrtPlatform platform = CRT.getPlatformImpl();
+            if (platform != null) {
+                String platformLibResourcePath = platform.getResourcePath(CRT.getCRuntime(os), libraryName);
+                if (platformLibResourcePath != null) {
+                    libResourcePath = platformLibResourcePath;
+                }
+            }
+            try (InputStream in = CRT.class.getResourceAsStream(libResourcePath)) {
+                if (in == null) {
+                    throw new IOException("Unable to open library in jar for AWS CRT: " + libResourcePath);
+                }
+
+                // Copy from jar stream to temp file
+                try (FileOutputStream out = new FileOutputStream(extractFile)) {
+                    int read;
+                    byte[] bytes = new byte[1024];
+                    while ((read = in.read(bytes)) != -1) {
+                        out.write(bytes, 0, read);
+                    }
+                }
+            }
+            if (!extractFile.setWritable(false)) {
+                throw new CrtRuntimeException("Unable to make shared library read-only");
+            }
+        } catch (CrtRuntimeException crtex) {
+            System.err.println("Unable to initialize AWS CRT: " + crtex);
+            crtex.printStackTrace();
+            throw crtex;
+        } catch (UnknownPlatformException upe) {
+            System.err.println("Unable to determine platform for AWS CRT: " + upe);
+            upe.printStackTrace();
+            CrtRuntimeException rex = new CrtRuntimeException("Unable to determine platform for AWS CRT");
+            rex.initCause(upe);
+            throw rex;
+        } catch (Exception ex) {
+            System.err.println("Unable to unpack AWS CRT lib: " + ex);
+            ex.printStackTrace();
+            CrtRuntimeException rex = new CrtRuntimeException("Unable to unpack AWS CRT library");
+            rex.initCause(ex);
+            throw rex;
+        }
+    }
+
+    /**
+     * Extract the CRT JNI library on current platform to a specific path.
+     *
+     * @param path the path extracting to
+     */
+    public static void extractLibrary(String path) {
+        String libraryName = System.mapLibraryName(CRT.CRT_LIB_NAME);
+        File extractFile = new File(path, libraryName);
+        try {
+            extractFile.createNewFile();
+        } catch (Exception ex) {
+            CrtRuntimeException rex = new CrtRuntimeException(
+                    "Unable to create file on path:" + extractFile.getAbsolutePath());
+            rex.initCause(ex);
+            throw rex;
+        }
+        extractLibrary(extractFile);
+    }
+}