diff --git a/src/main/c/generated/tiledb_wrap.cxx b/src/main/c/generated/tiledb_wrap.cxx index b8a6fd5a..baaee3fd 100644 --- a/src/main/c/generated/tiledb_wrap.cxx +++ b/src/main/c/generated/tiledb_wrap.cxx @@ -7016,6 +7016,34 @@ SWIGEXPORT jint JNICALL Java_io_tiledb_libtiledb_tiledbJNI_tiledb_1query_1set_1b return jresult; } +SWIGEXPORT jint JNICALL Java_io_tiledb_libtiledb_tiledbJNI_tiledb_1query_1set_1buffer_1nio(JNIEnv *jenv, jclass jcls, jlong jarg1, jlong jarg2, jstring jarg3, jobject jarg4, jlong jarg5) { + void* buffer = (void *)jenv->GetDirectBufferAddress(jarg4); + + jint jresult = 0 ; + tiledb_ctx_t *arg1 = (tiledb_ctx_t *) 0 ; + tiledb_query_t *arg2 = (tiledb_query_t *) 0 ; + char *arg3 = (char *) 0 ; + void *arg4 = (void *) 0 ; + uint64_t *arg5 = (uint64_t *) 0 ; + int32_t result; + + (void)jenv; + (void)jcls; + arg1 = *(tiledb_ctx_t **)&jarg1; + arg2 = *(tiledb_query_t **)&jarg2; + arg3 = 0; + if (jarg3) { + arg3 = (char *)jenv->GetStringUTFChars(jarg3, 0); + if (!arg3) return 0; + } + arg4 = *(void **)&jarg4; + arg5 = *(uint64_t **)&jarg5; + result = (int32_t)tiledb_query_set_buffer(arg1,arg2,(char const *)arg3, buffer,arg5); + jresult = (jint)result; + if (arg3) jenv->ReleaseStringUTFChars(jarg3, (const char *)arg3); + return jresult; +} + SWIGEXPORT jint JNICALL Java_io_tiledb_libtiledb_tiledbJNI_tiledb_1query_1set_1buffer_1var(JNIEnv *jenv, jclass jcls, jlong jarg1, jlong jarg2, jstring jarg3, jlong jarg4, jlong jarg5, jlong jarg6, jlong jarg7) { jint jresult = 0 ; @@ -9503,6 +9531,34 @@ SWIGEXPORT jint JNICALL Java_io_tiledb_libtiledb_tiledbJNI_tiledb_1stats_1dump_1 } +SWIGEXPORT jint JNICALL Java_io_tiledb_libtiledb_tiledbJNI_tiledb_1stats_1raw_1dump(JNIEnv *jenv, jclass jcls, jlong jarg1) { + jint jresult = 0 ; + FILE *arg1 = (FILE *) 0 ; + int32_t result; + + (void)jenv; + (void)jcls; + arg1 = *(FILE **)&jarg1; + result = (int32_t)tiledb_stats_raw_dump(arg1); + jresult = (jint)result; + return jresult; +} + + +SWIGEXPORT jint JNICALL Java_io_tiledb_libtiledb_tiledbJNI_tiledb_1stats_1raw_1dump_1str(JNIEnv *jenv, jclass jcls, jlong jarg1) { + jint jresult = 0 ; + char **arg1 = (char **) 0 ; + int32_t result; + + (void)jenv; + (void)jcls; + arg1 = *(char ***)&jarg1; + result = (int32_t)tiledb_stats_raw_dump_str(arg1); + jresult = (jint)result; + return jresult; +} + + SWIGEXPORT jint JNICALL Java_io_tiledb_libtiledb_tiledbJNI_tiledb_1stats_1free_1str(JNIEnv *jenv, jclass jcls, jlong jarg1) { jint jresult = 0 ; char **arg1 = (char **) 0 ; diff --git a/src/main/java/io/tiledb/java/api/Query.java b/src/main/java/io/tiledb/java/api/Query.java index a8b293bb..3f143a0c 100644 --- a/src/main/java/io/tiledb/java/api/Query.java +++ b/src/main/java/io/tiledb/java/api/Query.java @@ -28,6 +28,8 @@ import io.tiledb.libtiledb.*; import java.math.BigInteger; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -58,6 +60,7 @@ public class Query implements AutoCloseable { private NativeArray subarray; private Map buffers_; + private Map byteBuffers_; private Map> var_buffers_; private Map> buffer_sizes_; @@ -78,6 +81,7 @@ public Query(Array array, QueryType type) throws TileDBError { this.querypp = _querypp; this.queryp = tiledb.tiledb_query_tpp_value(_querypp); this.buffers_ = Collections.synchronizedMap(new HashMap<>()); + this.byteBuffers_ = Collections.synchronizedMap(new HashMap<>()); this.var_buffers_ = Collections.synchronizedMap(new HashMap<>()); this.buffer_sizes_ = Collections.synchronizedMap(new HashMap<>()); } @@ -467,6 +471,82 @@ public synchronized Query setBuffer(String attr, NativeArray buffer, long buffer return this; } + /** + * * Sets a NIO ByteBuffer + * + * @param attr The attribute + * @param bufferElements + * @return The NIO ByteBuffer + * @throws TileDBError + */ + public synchronized ByteBuffer setBuffer(String attr, long bufferElements) throws TileDBError { + if (bufferElements <= 0) { + throw new TileDBError("Number of buffer elements must be >= 1"); + } + + Datatype dt; + + try (ArraySchema schema = array.getSchema()) { + try (Domain domain = schema.getDomain()) { + if (domain.hasDimension(attr)) { + dt = domain.getDimension(attr).getType(); + } else { + try (Attribute attribute = schema.getAttribute(attr)) { + dt = attribute.getType(); + } + } + } + } + + int size = Util.castLongToInt(bufferElements * dt.getNativeSize()); + + ByteBuffer buffer = ByteBuffer.allocateDirect(size); + + // Set the byte order to the native system's native order + buffer.order(ByteOrder.nativeOrder()); + + this.byteBuffers_.put(attr, buffer); + + this.setBuffer(attr, buffer); + + return buffer; + } + + /** + * * Sets a NIO ByteBuffer + * + * @param attr The attribute + * @param buffer The input NIO ByteBuffer + * @return The NIO ByteBuffer + * @throws TileDBError + */ + public synchronized ByteBuffer setBuffer(String attr, ByteBuffer buffer) throws TileDBError { + if (buffer.capacity() <= 0) { + throw new TileDBError("Number of buffer elements must be >= 1"); + } + + if (!buffer.isDirect()) { + throw new TileDBError( + "The ByteBuffer provided is not direct. Please provide a direct buffer (ByteBuffer.allocateDirect(...))"); + } + + if (!buffer.order().equals(ByteOrder.nativeOrder())) { + // TODO: Add a logger component to Query class and a WARN here + buffer.order(ByteOrder.nativeOrder()); + } + + this.byteBuffers_.put(attr, buffer); + + uint64_tArray values_array_size = new uint64_tArray(1); + values_array_size.setitem(0, BigInteger.valueOf(buffer.capacity())); + + ctx.handleError( + tiledb.tiledb_query_set_buffer_nio( + ctx.getCtxp(), queryp, attr, buffer, values_array_size.cast())); + + return buffer; + } + /** * Sets a buffer for a variable-sized getAttribute. * @@ -885,6 +965,18 @@ public Object getBuffer(String attr) throws TileDBError { } } + /** + * Retrieves the ByteBuffer of attribute attr + * + * @param attr The attribute name + * @return The ByteBuffer + * @throws TileDBError A TileDB exception + */ + public ByteBuffer getByteBuffer(String attr) throws TileDBError { + if (byteBuffers_.containsKey(attr)) return byteBuffers_.get(attr); + else throw new TileDBError("ByteBuffer does not exist for attribute: " + attr); + } + /** * Return an array containing offsets for a variable attribute buffer * diff --git a/src/main/java/io/tiledb/java/api/Util.java b/src/main/java/io/tiledb/java/api/Util.java index 7671b6fb..6b960a48 100644 --- a/src/main/java/io/tiledb/java/api/Util.java +++ b/src/main/java/io/tiledb/java/api/Util.java @@ -29,4 +29,11 @@ public static String[] bytesToStrings(long[] offsets, byte[] data) { return results; } + + public static int castLongToInt(long num) throws TileDBError { + if (num > Integer.MAX_VALUE) + throw new TileDBError(num + " is larger that the integer max value"); + + return (int) num; + } } diff --git a/src/main/java/io/tiledb/libtiledb/tiledb.java b/src/main/java/io/tiledb/libtiledb/tiledb.java index 37d21b93..f917253f 100644 --- a/src/main/java/io/tiledb/libtiledb/tiledb.java +++ b/src/main/java/io/tiledb/libtiledb/tiledb.java @@ -8,6 +8,8 @@ package io.tiledb.libtiledb; +import java.nio.ByteBuffer; + public class tiledb implements tiledbConstants { public static SWIGTYPE_p_p_char new_charpArray(int nelements) { long cPtr = tiledbJNI.new_charpArray(nelements); @@ -2131,6 +2133,20 @@ public static int tiledb_query_set_buffer( SWIGTYPE_p_unsigned_long_long.getCPtr(buffer_size)); } + public static int tiledb_query_set_buffer_nio( + SWIGTYPE_p_tiledb_ctx_t ctx, + SWIGTYPE_p_tiledb_query_t query, + String name, + ByteBuffer buffer, + SWIGTYPE_p_unsigned_long_long buffer_size) { + return tiledbJNI.tiledb_query_set_buffer_nio( + SWIGTYPE_p_tiledb_ctx_t.getCPtr(ctx), + SWIGTYPE_p_tiledb_query_t.getCPtr(query), + name, + buffer, + SWIGTYPE_p_unsigned_long_long.getCPtr(buffer_size)); + } + public static int tiledb_query_set_buffer_var( SWIGTYPE_p_tiledb_ctx_t ctx, SWIGTYPE_p_tiledb_query_t query, @@ -3155,6 +3171,14 @@ public static int tiledb_stats_dump_str(SWIGTYPE_p_p_char out) { return tiledbJNI.tiledb_stats_dump_str(SWIGTYPE_p_p_char.getCPtr(out)); } + public static int tiledb_stats_raw_dump(SWIGTYPE_p_FILE out) { + return tiledbJNI.tiledb_stats_raw_dump(SWIGTYPE_p_FILE.getCPtr(out)); + } + + public static int tiledb_stats_raw_dump_str(SWIGTYPE_p_p_char out) { + return tiledbJNI.tiledb_stats_raw_dump_str(SWIGTYPE_p_p_char.getCPtr(out)); + } + public static int tiledb_stats_free_str(SWIGTYPE_p_p_char out) { return tiledbJNI.tiledb_stats_free_str(SWIGTYPE_p_p_char.getCPtr(out)); } diff --git a/src/main/java/io/tiledb/libtiledb/tiledbConstants.java b/src/main/java/io/tiledb/libtiledb/tiledbConstants.java index 60cb558f..a1d1ea57 100644 --- a/src/main/java/io/tiledb/libtiledb/tiledbConstants.java +++ b/src/main/java/io/tiledb/libtiledb/tiledbConstants.java @@ -14,5 +14,5 @@ public interface tiledbConstants { public static final int TILEDB_OOM = (-2); public static final int TILEDB_VERSION_MAJOR = 2; public static final int TILEDB_VERSION_MINOR = 0; - public static final int TILEDB_VERSION_PATCH = 0; + public static final int TILEDB_VERSION_PATCH = 5; } diff --git a/src/main/java/io/tiledb/libtiledb/tiledbJNI.java b/src/main/java/io/tiledb/libtiledb/tiledbJNI.java index 0ba76f87..af41d83c 100644 --- a/src/main/java/io/tiledb/libtiledb/tiledbJNI.java +++ b/src/main/java/io/tiledb/libtiledb/tiledbJNI.java @@ -8,6 +8,8 @@ package io.tiledb.libtiledb; +import java.nio.ByteBuffer; + public class tiledbJNI { static { @@ -920,6 +922,9 @@ public static final native int tiledb_array_schema_has_attribute( public static final native int tiledb_query_set_buffer( long jarg1, long jarg2, String jarg3, long jarg4, long jarg5); + public static final native int tiledb_query_set_buffer_nio( + long jarg1, long jarg2, String jarg3, ByteBuffer jarg4, long jarg5); + public static final native int tiledb_query_set_buffer_var( long jarg1, long jarg2, String jarg3, long jarg4, long jarg5, long jarg6, long jarg7); @@ -1188,6 +1193,10 @@ public static final native int tiledb_uri_to_path( public static final native int tiledb_stats_dump_str(long jarg1); + public static final native int tiledb_stats_raw_dump(long jarg1); + + public static final native int tiledb_stats_raw_dump_str(long jarg1); + public static final native int tiledb_stats_free_str(long jarg1); public static final native int tiledb_dimension_dump_stdout(long jarg1, long jarg2); diff --git a/src/test/java/io/tiledb/java/api/QueryTest.java b/src/test/java/io/tiledb/java/api/QueryTest.java index 25da5971..0dd27df7 100644 --- a/src/test/java/io/tiledb/java/api/QueryTest.java +++ b/src/test/java/io/tiledb/java/api/QueryTest.java @@ -8,6 +8,8 @@ import static io.tiledb.java.api.QueryType.TILEDB_WRITE; import java.math.BigInteger; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.util.HashMap; import org.junit.Assert; import org.junit.Before; @@ -173,6 +175,271 @@ public void arrayReadDimensionsTest() throws Exception { } } + public static class NIODenseTests { + + @Rule public TemporaryFolder temp = new TemporaryFolder(); + + private String arrayURI; + + @Before + public void setup() throws Exception { + ctx = new Context(); + arrayURI = temp.getRoot().toPath().resolve("query").toString(); + arrayCreate(); + arrayWrite(); + } + + public void arrayCreate() throws Exception { + // The array will be 4x4 with dimensions "rows" and "cols", with domain [1,4]. + Dimension rows = + new Dimension<>(ctx, "rows", Integer.class, new Pair(1, 4), 2); + Dimension cols = + new Dimension<>(ctx, "cols", Integer.class, new Pair(1, 4), 2); + + // Create and set getDomain + Domain domain = new Domain(ctx); + domain.addDimension(rows); + domain.addDimension(cols); + + // Add two attributes "a1" and "a2", so each (i,j) cell can store + // a character on "a1" and a vector of two floats on "a2". + Attribute a1 = new Attribute(ctx, "a1", Character.class); + Attribute a2 = new Attribute(ctx, "a2", Float.class); + a2.setCellValNum(2); + + ArraySchema schema = new ArraySchema(ctx, TILEDB_DENSE); + schema.setTileOrder(TILEDB_ROW_MAJOR); + schema.setCellOrder(TILEDB_ROW_MAJOR); + schema.setDomain(domain); + schema.addAttribute(a1); + schema.addAttribute(a2); + + Array.create(arrayURI, schema); + } + + public void arrayWrite() throws Exception { + String str = "abcdefghijklmnop"; + float[] floatArr = + new float[] { + 0.1f, 0.2f, 1.1f, 1.2f, 2.1f, 2.2f, 3.1f, 3.2f, + 4.1f, 4.2f, 5.1f, 5.2f, 6.1f, 6.2f, 7.1f, 7.2f, + 8.1f, 8.2f, 9.1f, 9.2f, 10.1f, 10.2f, 11.1f, 11.2f, + 12.1f, 12.2f, 13.1f, 13.2f, 14.1f, 14.2f, 15.1f, 15.2f + }; + + ByteBuffer a1 = ByteBuffer.allocateDirect(1 * str.length()); + ByteBuffer a2 = ByteBuffer.allocateDirect(4 * floatArr.length); + + a1.order(ByteOrder.nativeOrder()); + a2.order(ByteOrder.nativeOrder()); + + for (int i = 0; i < floatArr.length; ++i) { + if (i < str.length()) a1.put((byte) str.charAt(i)); + a2.putFloat(floatArr[i]); + } + + // Create query + try (Array array = new Array(ctx, arrayURI, TILEDB_WRITE); + Query query = new Query(array)) { + + query.setLayout(TILEDB_ROW_MAJOR); + query.setBuffer("a1", a1); + query.setBuffer("a2", a2); + // Submit query + query.submit(); + } + } + + @Test + public void queryTestNIOReadArray() throws Exception { + Array array = new Array(ctx, arrayURI, TILEDB_READ); + + Query query = new Query(array, TILEDB_READ); + + int bufferSize = 4; + + ByteBuffer d1 = query.setBuffer("rows", bufferSize); + ByteBuffer d2 = query.setBuffer("cols", bufferSize); + + query.addRange(0, 1, 4); + query.addRange(1, 1, 4); + + query.setLayout(TILEDB_ROW_MAJOR); + + int[] d1_result = new int[16]; + int[] d2_result = new int[16]; + int idx = 0; + + while (query.getQueryStatus() != QueryStatus.TILEDB_COMPLETED) { + query.submit(); + + while (d1.hasRemaining() && d2.hasRemaining()) { + d1_result[idx] = d1.getInt(); + d2_result[idx] = d2.getInt(); + idx++; + } + d1.clear(); + d2.clear(); + } + + Assert.assertArrayEquals( + new int[] {1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4}, d1_result); + Assert.assertArrayEquals( + new int[] {1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4}, d2_result); + } + + @Test + public void arrayReadTest() throws Exception { + // Create array and query + try (Array array = new Array(ctx, arrayURI, TILEDB_READ); + Query query = new Query(array, TILEDB_READ)) { + + // Slice only rows 1, 2 and cols 2, 3, 4 + query.addRange(0, 1, 2); + query.addRange(1, 2, 4); + query.setLayout(TILEDB_ROW_MAJOR); + + ByteBuffer dim1Buffer = query.setBuffer("rows", 3); + ByteBuffer dim2Buffer = query.setBuffer("cols", 3); + ByteBuffer a1Buffer = query.setBuffer("a1", 3); + ByteBuffer a2Buffer = query.setBuffer("a2", 6); + + // Submit query + query.submit(); + + int[] dim1 = new int[3]; + int[] dim2 = new int[3]; + byte[] a1 = new byte[3]; + float[] a2 = new float[6]; + + int idx = 0; + while (dim1Buffer.hasRemaining()) { + dim1[idx++] = dim1Buffer.getInt(); + } + + idx = 0; + while (dim2Buffer.hasRemaining()) { + dim2[idx++] = dim2Buffer.getInt(); + } + + idx = 0; + while (a1Buffer.hasRemaining()) { + a1[idx++] = a1Buffer.get(); + } + + idx = 0; + while (a2Buffer.hasRemaining()) { + a2[idx++] = a2Buffer.getFloat(); + } + + Assert.assertArrayEquals(new int[] {1, 1, 1}, dim1); + Assert.assertArrayEquals(new int[] {2, 3, 4}, dim2); + Assert.assertArrayEquals(new byte[] {'b', 'c', 'd'}, a1); + Assert.assertArrayEquals(new float[] {1.1f, 1.2f, 2.1f, 2.2f, 3.1f, 3.2f}, a2, 0.01f); + } + } + + @Test + public void arrayReadTestCustomBufferWithDifferentOrder() throws Exception { + // Create array and query + try (Array array = new Array(ctx, arrayURI, TILEDB_READ); + Query query = new Query(array, TILEDB_READ)) { + + // Slice only rows 1, 2 and cols 2, 3, 4 + query.addRange(0, 1, 2); + query.addRange(1, 2, 4); + query.setLayout(TILEDB_ROW_MAJOR); + + // Set the opposite byte order from the native system + ByteOrder order = + ByteOrder.nativeOrder().equals(ByteOrder.BIG_ENDIAN) + ? ByteOrder.LITTLE_ENDIAN + : ByteOrder.BIG_ENDIAN; + + ByteBuffer dim1Buffer = + query.setBuffer("rows", ByteBuffer.allocateDirect(3 * 4).order(order)); + ByteBuffer dim2Buffer = + query.setBuffer("cols", ByteBuffer.allocateDirect(3 * 4).order(order)); + ByteBuffer a1Buffer = query.setBuffer("a1", ByteBuffer.allocateDirect(3).order(order)); + ByteBuffer a2Buffer = query.setBuffer("a2", ByteBuffer.allocateDirect(6 * 4).order(order)); + + // Submit query + query.submit(); + + int[] dim1 = new int[3]; + int[] dim2 = new int[3]; + byte[] a1 = new byte[3]; + float[] a2 = new float[6]; + + int idx = 0; + while (dim1Buffer.hasRemaining()) { + dim1[idx++] = dim1Buffer.getInt(); + } + + idx = 0; + while (dim2Buffer.hasRemaining()) { + dim2[idx++] = dim2Buffer.getInt(); + } + + idx = 0; + while (a1Buffer.hasRemaining()) { + a1[idx++] = a1Buffer.get(); + } + + idx = 0; + while (a2Buffer.hasRemaining()) { + a2[idx++] = a2Buffer.getFloat(); + } + + Assert.assertArrayEquals(new int[] {1, 1, 1}, dim1); + Assert.assertArrayEquals(new int[] {2, 3, 4}, dim2); + Assert.assertArrayEquals(new byte[] {'b', 'c', 'd'}, a1); + Assert.assertArrayEquals(new float[] {1.1f, 1.2f, 2.1f, 2.2f, 3.1f, 3.2f}, a2, 0.01f); + } + } + + @Test + public void queryTestNIOGetByteBuffer() throws Exception { + Array array = new Array(ctx, arrayURI, TILEDB_READ); + + Query query = new Query(array, TILEDB_READ); + + int bufferSize = 16; + + query.setBuffer("rows", bufferSize); + + Assert.assertEquals(query.getByteBuffer("rows").capacity(), bufferSize * 4); + Assert.assertTrue(query.getByteBuffer("rows").isDirect()); + Assert.assertEquals(query.getByteBuffer("rows").order(), ByteOrder.nativeOrder()); + } + + @Test() + public void queryTestNIOGetByteBuffeErrors() throws Exception { + Array array = new Array(ctx, arrayURI, TILEDB_READ); + + Query query = new Query(array, TILEDB_READ); + + int bufferSize = 16; + + try { + query.setBuffer("rows", ByteBuffer.allocate(bufferSize)); + Assert.fail("An exception should be thrown in the ByteBuffer provided is not direct"); + } catch (TileDBError error) { + + } + + // Set the opposite byte order from the native system + ByteOrder order = + ByteOrder.nativeOrder().equals(ByteOrder.BIG_ENDIAN) + ? ByteOrder.LITTLE_ENDIAN + : ByteOrder.BIG_ENDIAN; + + // The Byte Order should be automatically changed to the native order + ByteBuffer b = query.setBuffer("rows", ByteBuffer.allocateDirect(bufferSize).order(order)); + Assert.assertEquals(b.order(), ByteOrder.nativeOrder()); + } + } + public static class SparseTests { @Rule public TemporaryFolder temp = new TemporaryFolder();