Skip to content

Commit a95248e

Browse files
chris-allanchris-allan
committed
Refactor NativeLibLoader to use a folder based workflow (Windows compat)
A folder based workflow for loading native libraries has several advantages. Noteably, such a workflow is essential for Windows support as currently the Intel Thread Building Blocks library dependency cannot be statically linked to TileDB on the platform. Consequently, `tbb.dll` is linked dynamically and its filename must be preserved for DLL resolution to happen successful. In contrast multi-classloader support has now, I think, been lost. However, there are to my knowledge no tests for that currently.
1 parent a1c1e35 commit a95248e

File tree

2 files changed

+108
-108
lines changed

2 files changed

+108
-108
lines changed

‎src/main/java/io/tiledb/libtiledb/NativeLibLoader.java

Lines changed: 54 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -27,9 +27,15 @@
2727
import java.io.FileOutputStream;
2828
import java.io.IOException;
2929
import java.io.InputStream;
30+
import java.nio.file.Files;
31+
import java.nio.file.Path;
32+
import java.nio.file.attribute.PosixFileAttributeView;
33+
import java.nio.file.attribute.PosixFilePermission;
34+
import java.util.Comparator;
3035
import java.util.Locale;
3136
import java.util.Properties;
32-
import java.util.UUID;
37+
import java.util.Set;
38+
import java.util.stream.Stream;
3339

3440
/** Helper class that finds native libraries embedded as resources and loads them dynamically. */
3541
public class NativeLibLoader {
@@ -39,6 +45,26 @@ public class NativeLibLoader {
3945
/** Path (relative to jar) where native libraries are located. */
4046
private static final String LIB_RESOURCE_DIR = "/lib";
4147

48+
/** Temporary directory where native libraries will be extracted. */
49+
private static Path tempDir;
50+
51+
static {
52+
try {
53+
tempDir = Files.createTempDirectory("tileDbNativeLibLoader");
54+
} catch (IOException e) {
55+
e.printStackTrace(System.err);
56+
}
57+
Runtime.getRuntime().addShutdownHook(new Thread(() -> {
58+
try (Stream<Path> walk = Files.walk(tempDir)) {
59+
walk.sorted(Comparator.reverseOrder())
60+
.map(Path::toFile)
61+
.forEach(File::delete);
62+
} catch (IOException e) {
63+
e.printStackTrace(System.err);
64+
}
65+
}));
66+
}
67+
4268
/** Finds and loads native TileDB. */
4369
static void loadNativeTileDB() {
4470
try {
@@ -218,56 +244,39 @@ private static String getOSClassifier() {
218244
*
219245
* @param libraryDir Path of directory containing native library
220246
* @param libraryName Name of native library
221-
* @param targetDir Path of target directory to extract library to
222247
* @param mapLibraryName If true, transform libraryName with System.mapLibraryName
223248
* @return File pointing to the extracted library
224249
*/
225-
private static File extractLibraryFile(
226-
String libraryDir, String libraryName, String targetDir, boolean mapLibraryName) {
250+
private static Path extractLibraryFile(
251+
String libraryDir, String libraryName, boolean mapLibraryName) {
227252
String libraryFileName = mapLibraryName ? System.mapLibraryName(libraryName) : libraryName;
228253
String nativeLibraryFilePath = libraryDir + "/" + libraryFileName;
229-
230-
// Attach UUID to the native library file to ensure multiple class loaders can read the
231-
// native lib multiple times.
232-
String uuid = UUID.randomUUID().toString();
233-
String extractedLibFileName = String.format("%s-%s-%s", libraryName, uuid, libraryFileName);
234-
File extractedLibFile = new File(targetDir, extractedLibFileName);
254+
Path extractedLibFile = tempDir.resolve(libraryFileName);
235255

236256
try {
237257
// Extract a native library file into the target directory
238-
InputStream reader = null;
239-
FileOutputStream writer = null;
240-
try {
241-
reader = NativeLibLoader.class.getResourceAsStream(nativeLibraryFilePath);
242-
try {
243-
writer = new FileOutputStream(extractedLibFile);
258+
try (InputStream reader = NativeLibLoader.class.getResourceAsStream(nativeLibraryFilePath);
259+
FileOutputStream writer = new FileOutputStream(extractedLibFile.toFile())) {
244260

245-
byte[] buffer = new byte[8192];
246-
int bytesRead = 0;
247-
while ((bytesRead = reader.read(buffer)) != -1) {
248-
writer.write(buffer, 0, bytesRead);
249-
}
250-
} finally {
251-
if (writer != null) {
252-
writer.close();
253-
}
254-
}
255-
} finally {
256-
if (reader != null) {
257-
reader.close();
261+
byte[] buffer = new byte[8192];
262+
int bytesRead = 0;
263+
while ((bytesRead = reader.read(buffer)) != -1) {
264+
writer.write(buffer, 0, bytesRead);
258265
}
259-
260-
// Delete the extracted lib file on JVM exit.
261-
extractedLibFile.deleteOnExit();
262266
}
263267

264-
// Set executable (x) flag to enable Java to load the native library
265-
boolean success =
266-
extractedLibFile.setReadable(true)
267-
&& extractedLibFile.setWritable(true, true)
268-
&& extractedLibFile.setExecutable(true);
269-
if (!success) {
270-
// Setting file flag may fail, but in this case another error will be thrown in later phase
268+
// Set executable (x) flag to enable Java to load the native library on
269+
// UNIX platforms
270+
PosixFileAttributeView view = Files.getFileAttributeView(
271+
extractedLibFile, PosixFileAttributeView.class);
272+
if (view != null) {
273+
// On a UNIX platform
274+
Set<PosixFilePermission> permissions =
275+
view.readAttributes().permissions();
276+
permissions.add(PosixFilePermission.OWNER_READ);
277+
permissions.add(PosixFilePermission.OWNER_WRITE);
278+
permissions.add(PosixFilePermission.OWNER_EXECUTE);
279+
view.setPermissions(permissions);
271280
}
272281

273282
// Check whether the contents are properly copied from the resource folder
@@ -276,7 +285,7 @@ private static File extractLibraryFile(
276285
InputStream extractedLibIn = null;
277286
try {
278287
nativeIn = NativeLibLoader.class.getResourceAsStream(nativeLibraryFilePath);
279-
extractedLibIn = new FileInputStream(extractedLibFile);
288+
extractedLibIn = new FileInputStream(extractedLibFile.toFile());
280289

281290
if (!contentsEquals(nativeIn, extractedLibIn)) {
282291
throw new IOException(
@@ -292,7 +301,7 @@ private static File extractLibraryFile(
292301
}
293302
}
294303

295-
return new File(targetDir, extractedLibFileName);
304+
return extractedLibFile;
296305
} catch (IOException e) {
297306
e.printStackTrace(System.err);
298307
return null;
@@ -306,7 +315,7 @@ private static File extractLibraryFile(
306315
* @param mapLibraryName If true, transform libraryName with System.mapLibraryName
307316
* @return File pointing to the extracted library
308317
*/
309-
private static File findNativeLibrary(String libraryName, boolean mapLibraryName) {
318+
private static Path findNativeLibrary(String libraryName, boolean mapLibraryName) {
310319
String mappedLibraryName = mapLibraryName ? System.mapLibraryName(libraryName) : libraryName;
311320
String libDir = LIB_RESOURCE_DIR + "/" + getOSClassifier();
312321
String libPath = libDir + "/" + mappedLibraryName;
@@ -316,17 +325,8 @@ private static File findNativeLibrary(String libraryName, boolean mapLibraryName
316325
return null;
317326
}
318327

319-
// Temporary folder for the extracted native lib.
320-
File tempFolder = new File(System.getProperty("java.io.tmpdir"));
321-
if (!tempFolder.exists()) {
322-
boolean created = tempFolder.mkdirs();
323-
if (!created) {
324-
// if created == false, it will fail eventually in the later part
325-
}
326-
}
327-
328328
// Extract and load a native library inside the jar file
329-
return extractLibraryFile(libDir, libraryName, tempFolder.getAbsolutePath(), mapLibraryName);
329+
return extractLibraryFile(libDir, libraryName, mapLibraryName);
330330
}
331331

332332
/**
@@ -336,10 +336,10 @@ private static File findNativeLibrary(String libraryName, boolean mapLibraryName
336336
* @param mapLibraryName If true, transform libraryName with System.mapLibraryName
337337
*/
338338
private static void loadNativeLib(String libraryName, boolean mapLibraryName) {
339-
File nativeLibFile = findNativeLibrary(libraryName, mapLibraryName);
339+
Path nativeLibFile = findNativeLibrary(libraryName, mapLibraryName);
340340
if (nativeLibFile != null) {
341341
// Load extracted or specified native library.
342-
System.load(nativeLibFile.getAbsolutePath());
342+
System.load(nativeLibFile.toString());
343343
} else {
344344
// Try loading preinstalled library (in the path -Djava.library.path)
345345
System.loadLibrary(libraryName);

‎swig/customCode/NativeLibLoader.java

Lines changed: 54 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -27,9 +27,15 @@
2727
import java.io.FileOutputStream;
2828
import java.io.IOException;
2929
import java.io.InputStream;
30+
import java.nio.file.Files;
31+
import java.nio.file.Path;
32+
import java.nio.file.attribute.PosixFileAttributeView;
33+
import java.nio.file.attribute.PosixFilePermission;
34+
import java.util.Comparator;
3035
import java.util.Locale;
3136
import java.util.Properties;
32-
import java.util.UUID;
37+
import java.util.Set;
38+
import java.util.stream.Stream;
3339

3440
/** Helper class that finds native libraries embedded as resources and loads them dynamically. */
3541
public class NativeLibLoader {
@@ -39,6 +45,26 @@ public class NativeLibLoader {
3945
/** Path (relative to jar) where native libraries are located. */
4046
private static final String LIB_RESOURCE_DIR = "/lib";
4147

48+
/** Temporary directory where native libraries will be extracted. */
49+
private static Path tempDir;
50+
51+
static {
52+
try {
53+
tempDir = Files.createTempDirectory("tileDbNativeLibLoader");
54+
} catch (IOException e) {
55+
e.printStackTrace(System.err);
56+
}
57+
Runtime.getRuntime().addShutdownHook(new Thread(() -> {
58+
try (Stream<Path> walk = Files.walk(tempDir)) {
59+
walk.sorted(Comparator.reverseOrder())
60+
.map(Path::toFile)
61+
.forEach(File::delete);
62+
} catch (IOException e) {
63+
e.printStackTrace(System.err);
64+
}
65+
}));
66+
}
67+
4268
/** Finds and loads native TileDB. */
4369
static void loadNativeTileDB() {
4470
try {
@@ -218,56 +244,39 @@ private static String getOSClassifier() {
218244
*
219245
* @param libraryDir Path of directory containing native library
220246
* @param libraryName Name of native library
221-
* @param targetDir Path of target directory to extract library to
222247
* @param mapLibraryName If true, transform libraryName with System.mapLibraryName
223248
* @return File pointing to the extracted library
224249
*/
225-
private static File extractLibraryFile(
226-
String libraryDir, String libraryName, String targetDir, boolean mapLibraryName) {
250+
private static Path extractLibraryFile(
251+
String libraryDir, String libraryName, boolean mapLibraryName) {
227252
String libraryFileName = mapLibraryName ? System.mapLibraryName(libraryName) : libraryName;
228253
String nativeLibraryFilePath = libraryDir + "/" + libraryFileName;
229-
230-
// Attach UUID to the native library file to ensure multiple class loaders can read the
231-
// native lib multiple times.
232-
String uuid = UUID.randomUUID().toString();
233-
String extractedLibFileName = String.format("%s-%s-%s", libraryName, uuid, libraryFileName);
234-
File extractedLibFile = new File(targetDir, extractedLibFileName);
254+
Path extractedLibFile = tempDir.resolve(libraryFileName);
235255

236256
try {
237257
// Extract a native library file into the target directory
238-
InputStream reader = null;
239-
FileOutputStream writer = null;
240-
try {
241-
reader = NativeLibLoader.class.getResourceAsStream(nativeLibraryFilePath);
242-
try {
243-
writer = new FileOutputStream(extractedLibFile);
258+
try (InputStream reader = NativeLibLoader.class.getResourceAsStream(nativeLibraryFilePath);
259+
FileOutputStream writer = new FileOutputStream(extractedLibFile.toFile())) {
244260

245-
byte[] buffer = new byte[8192];
246-
int bytesRead = 0;
247-
while ((bytesRead = reader.read(buffer)) != -1) {
248-
writer.write(buffer, 0, bytesRead);
249-
}
250-
} finally {
251-
if (writer != null) {
252-
writer.close();
253-
}
254-
}
255-
} finally {
256-
if (reader != null) {
257-
reader.close();
261+
byte[] buffer = new byte[8192];
262+
int bytesRead = 0;
263+
while ((bytesRead = reader.read(buffer)) != -1) {
264+
writer.write(buffer, 0, bytesRead);
258265
}
259-
260-
// Delete the extracted lib file on JVM exit.
261-
extractedLibFile.deleteOnExit();
262266
}
263267

264-
// Set executable (x) flag to enable Java to load the native library
265-
boolean success =
266-
extractedLibFile.setReadable(true)
267-
&& extractedLibFile.setWritable(true, true)
268-
&& extractedLibFile.setExecutable(true);
269-
if (!success) {
270-
// Setting file flag may fail, but in this case another error will be thrown in later phase
268+
// Set executable (x) flag to enable Java to load the native library on
269+
// UNIX platforms
270+
PosixFileAttributeView view = Files.getFileAttributeView(
271+
extractedLibFile, PosixFileAttributeView.class);
272+
if (view != null) {
273+
// On a UNIX platform
274+
Set<PosixFilePermission> permissions =
275+
view.readAttributes().permissions();
276+
permissions.add(PosixFilePermission.OWNER_READ);
277+
permissions.add(PosixFilePermission.OWNER_WRITE);
278+
permissions.add(PosixFilePermission.OWNER_EXECUTE);
279+
view.setPermissions(permissions);
271280
}
272281

273282
// Check whether the contents are properly copied from the resource folder
@@ -276,7 +285,7 @@ private static File extractLibraryFile(
276285
InputStream extractedLibIn = null;
277286
try {
278287
nativeIn = NativeLibLoader.class.getResourceAsStream(nativeLibraryFilePath);
279-
extractedLibIn = new FileInputStream(extractedLibFile);
288+
extractedLibIn = new FileInputStream(extractedLibFile.toFile());
280289

281290
if (!contentsEquals(nativeIn, extractedLibIn)) {
282291
throw new IOException(
@@ -292,7 +301,7 @@ private static File extractLibraryFile(
292301
}
293302
}
294303

295-
return new File(targetDir, extractedLibFileName);
304+
return extractedLibFile;
296305
} catch (IOException e) {
297306
e.printStackTrace(System.err);
298307
return null;
@@ -306,7 +315,7 @@ private static File extractLibraryFile(
306315
* @param mapLibraryName If true, transform libraryName with System.mapLibraryName
307316
* @return File pointing to the extracted library
308317
*/
309-
private static File findNativeLibrary(String libraryName, boolean mapLibraryName) {
318+
private static Path findNativeLibrary(String libraryName, boolean mapLibraryName) {
310319
String mappedLibraryName = mapLibraryName ? System.mapLibraryName(libraryName) : libraryName;
311320
String libDir = LIB_RESOURCE_DIR + "/" + getOSClassifier();
312321
String libPath = libDir + "/" + mappedLibraryName;
@@ -316,17 +325,8 @@ private static File findNativeLibrary(String libraryName, boolean mapLibraryName
316325
return null;
317326
}
318327

319-
// Temporary folder for the extracted native lib.
320-
File tempFolder = new File(System.getProperty("java.io.tmpdir"));
321-
if (!tempFolder.exists()) {
322-
boolean created = tempFolder.mkdirs();
323-
if (!created) {
324-
// if created == false, it will fail eventually in the later part
325-
}
326-
}
327-
328328
// Extract and load a native library inside the jar file
329-
return extractLibraryFile(libDir, libraryName, tempFolder.getAbsolutePath(), mapLibraryName);
329+
return extractLibraryFile(libDir, libraryName, mapLibraryName);
330330
}
331331

332332
/**
@@ -336,10 +336,10 @@ private static File findNativeLibrary(String libraryName, boolean mapLibraryName
336336
* @param mapLibraryName If true, transform libraryName with System.mapLibraryName
337337
*/
338338
private static void loadNativeLib(String libraryName, boolean mapLibraryName) {
339-
File nativeLibFile = findNativeLibrary(libraryName, mapLibraryName);
339+
Path nativeLibFile = findNativeLibrary(libraryName, mapLibraryName);
340340
if (nativeLibFile != null) {
341341
// Load extracted or specified native library.
342-
System.load(nativeLibFile.getAbsolutePath());
342+
System.load(nativeLibFile.toString());
343343
} else {
344344
// Try loading preinstalled library (in the path -Djava.library.path)
345345
System.loadLibrary(libraryName);

0 commit comments

Comments
 (0)