diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/transform/schema/Schema.java b/datavec/datavec-api/src/main/java/org/datavec/api/transform/schema/Schema.java index 1c16ebcceebb..0e5b3285923e 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/transform/schema/Schema.java +++ b/datavec/datavec-api/src/main/java/org/datavec/api/transform/schema/Schema.java @@ -710,6 +710,17 @@ public Builder addColumn(ColumnMetaData metaData) { return this; } + + /** + * Add a bytes column representing + * arbitrary bytes string data + * @param name the name of the column + * @return + */ + public Builder addColumnBytes(String name) { + return addColumn(new BinaryMetaData(name)); + } + /** * Add a String column with no restrictions on the allowable values. * diff --git a/datavec/datavec-api/src/main/java/org/datavec/api/util/ndarray/RecordConverter.java b/datavec/datavec-api/src/main/java/org/datavec/api/util/ndarray/RecordConverter.java index c55d4d3bb0ef..e3730843b7d6 100644 --- a/datavec/datavec-api/src/main/java/org/datavec/api/util/ndarray/RecordConverter.java +++ b/datavec/datavec-api/src/main/java/org/datavec/api/util/ndarray/RecordConverter.java @@ -127,7 +127,7 @@ public static INDArray toArray(Collection record) { INDArray arr = Nd4j.create(1, length); int k = 0; - for (Writable w : record ) { + for (Writable w : record) { if (w instanceof NDArrayWritable) { INDArray toPut = ((NDArrayWritable) w).get(); arr.put(new INDArrayIndex[] {NDArrayIndex.point(0), diff --git a/datavec/datavec-arrow/pom.xml b/datavec/datavec-arrow/pom.xml index 04420a5e967d..88ee22574430 100644 --- a/datavec/datavec-arrow/pom.xml +++ b/datavec/datavec-arrow/pom.xml @@ -27,8 +27,29 @@ jar datavec-arrow - + + + + org.apache.maven.plugins + maven-compiler-plugin + + 1.8 + 1.8 + + + + + + org.nd4j + nd4j-arrow + ${nd4j.version} + + + org.bytedeco + arrow-platform + ${arrow.javacpp.version} + org.datavec datavec-api diff --git a/datavec/datavec-arrow/src/main/java/org/datavec/arrow/table/DataVecArrowUtils.java b/datavec/datavec-arrow/src/main/java/org/datavec/arrow/table/DataVecArrowUtils.java new file mode 100644 index 000000000000..709dc2e0b1f0 --- /dev/null +++ b/datavec/datavec-arrow/src/main/java/org/datavec/arrow/table/DataVecArrowUtils.java @@ -0,0 +1,585 @@ +/* + * Copyright (c) 2019 Konduit KK + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.datavec.arrow.table; + +import org.apache.commons.lang3.ArrayUtils; +import org.bytedeco.arrow.*; +import org.bytedeco.arrow.global.arrow; +import org.bytedeco.javacpp.BytePointer; +import org.bytedeco.javacpp.LongPointer; +import org.datavec.api.transform.schema.Schema; +import org.datavec.api.transform.schema.Schema.Builder; +import org.datavec.arrow.table.column.DataVecColumn; +import org.datavec.arrow.table.column.impl.*; +import org.nd4j.arrow.ByteDecoArrowSerde; +import org.nd4j.base.Preconditions; +import org.nd4j.linalg.api.buffer.DataBuffer; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.shade.guava.primitives.*; + +import java.util.List; +import java.util.TimeZone; + +import static org.bytedeco.arrow.global.arrow.*; +import static org.nd4j.arrow.ByteDecoArrowSerde.fromArrowBuffer; + +/** + * Utilities for interop between data vec types + * and arrow types. + * + * @author Adam Gibson + */ +public class DataVecArrowUtils { + + + /** + * Returns the number of elements in the given + * {@link FlatArray}. + * This accesses buffer[buffer.length - 1] and returns its size + * @param flatArray the flat array to return the number + * of elements for + * @return + */ + public static long numberOfElementsInBuffer(FlatArray flatArray) { + long indexOfBuffer = flatArray.data().buffers().size() - 1; + Preconditions.checkState(flatArray.data().length() > 1,"Flat array size must be at least size 2."); + return flatArray.data().buffers().get(indexOfBuffer).size(); + } + + + /** + * + * @param schema + * @param data + * @return + */ + public static Table tableFromSchema(Schema schema,ChunkedArrayVector data) { + return tableFromSchema(schema,data,data.size()); + } + + /** + * + * @param schema + * @param data + * @param numRows + * @return + */ + public static Table tableFromSchema(Schema schema,ChunkedArrayVector data,long numRows) { + return Table.Make(toArrowSchema(schema),data,numRows); + } + + /** + * + * @param schema + * @param arrayVector + * @param numRows + * @return + */ + public static Table tableFromSchema(Schema schema, ArrayVector arrayVector,long numRows) { + return Table.Make(toArrowSchema(schema),arrayVector,numRows); + } + + /** + * + * @param schema + * @param arrayVector + * @return + */ + public static Table tableFromSchema(Schema schema, ArrayVector arrayVector) { + return tableFromSchema(schema,arrayVector,arrayVector.size()); + } + + + /** + * Convert an existing data vec {@link Schema} + * to an {@link org.bytedeco.arrow.Schema } + * @param schema the input schema + * @return the arrow schema + */ + public static org.bytedeco.arrow.Schema toArrowSchema(Schema schema) { + Field[] fields = new Field[schema.numColumns()]; + FieldVector schemaVector = null; + for(int i = 0; i < schema.numColumns(); i++) { + switch(schema.getType(i)) { + case Double: + fields[i] = new Field(schema.getName(i),float64()); + break; + case NDArray: + fields[i] = new Field(schema.getName(i),binary()); + break; + case Bytes: + fields[i] = new Field(schema.getName(i),binary()); + break; + case String: + fields[i] = new Field(schema.getName(i),utf8()); + break; + case Integer: + fields[i] = new Field(schema.getName(i),int32()); + break; + case Time: + //note datavec times are stored as longs + fields[i] = new Field(schema.getName(i),int64()); + break; + case Categorical: + fields[i] = new Field(schema.getName(i),utf8()); + break; + case Float: + fields[i] = new Field(schema.getName(i),float32()); + break; + case Long: + fields[i] = new Field(schema.getName(i),int64()); + break; + case Boolean: + fields[i] = new Field(schema.getName(i),_boolean()); + break; + } + } + + schemaVector = new FieldVector(fields); + return new org.bytedeco.arrow.Schema(schemaVector); + } + + + /** + * Convert the given input + * to a boolean array + * @param array the input + * @return the equivalent boolean data + */ + public static boolean[] convertArrayToBoolean(FlatArray array) { + BooleanArray primitiveArray = (BooleanArray) array; + long length = numberOfElementsInBuffer(array); + boolean[] ret = new boolean[(int) length]; + for(int i = 0; i < ret.length; i++) { + ret[i] = primitiveArray.Value(i); + } + + return ret; + } + + /** + * Convert the given input + * to a float array + * @param array the input + * @return the equivalent float data + */ + public static float[] convertArrayToFloat(FlatArray array) { + FloatArray primitiveArray = (FloatArray) array; + long length = numberOfElementsInBuffer(array); + float[] ret = new float[(int) length]; + for(int i = 0; i < ret.length; i++) { + ret[i] = primitiveArray.Value(i); + } + + return ret; + } + + /** + * Convert the given input + * to a double array + * @param array the input + * @return the equivalent double data + */ + public static double[] convertArrayToDouble(FlatArray array) { + DoubleArray primitiveArray = (DoubleArray) array; + long length = numberOfElementsInBuffer(array); + double[] ret = new double[(int) length]; + for(int i = 0; i < ret.length; i++) { + ret[i] = primitiveArray.Value(i); + } + return ret; + } + + + /** + * Find the element at a particular ror + * in the {@link StringArray} + * @param stringArray the string array to get the item from + * @param i the index + * @return the string at the specified index + */ + public static String elementAt(StringArray stringArray,long i) { + long numElements = numberOfElementsInBuffer(stringArray); + return elementAt(stringArray,i,numElements); + } + + /** + * Find the element at a particular ror + * in the {@link StringArray} + * @param stringArray the string array to get the item from + * @param i the index + * @param length the number of elements + * @return the string at the specified index + */ + public static String elementAt(StringArray stringArray,long i,long length) { + long valLength = stringArray.value_length(i); + long offset = stringArray.value_offset(i); + ArrowBuffer currData = stringArray.value_data(); + //offsets: each begin/end for each element that isn't the last + long offsetSize = (length + 1) * 8; + return currData.data().position(offset + offsetSize) + .capacity(valLength) + .limit(offset + offsetSize + valLength) + .getString(); + } + + /** + * Convert the given input + * to a string array + * @param array the input + * @return the equivalent string data + */ + public static String[] convertArrayToString(FlatArray array) { + StringArray primitiveArray = (StringArray) array; + long length = numberOfElementsInBuffer(array); + String[] ret = new String[(int) length]; + for(int i = 0; i < ret.length; i++) { + ret[i] = elementAt(primitiveArray,i); + } + + return ret; + } + + /** + * Convert the given input + * to a long array + * @param array the input + * @return the equivalent long data + */ + public static long[] convertArrayToLong(FlatArray array) { + Int64Array primitiveArray = (Int64Array) array; + long length = numberOfElementsInBuffer(array); + long[] ret = new long[(int) length]; + for(int i = 0; i < ret.length; i++) { + ret[i] = primitiveArray.Value(i); + } + + return ret; + } + + /** + * Convert the given input + * to a int array + * @param array the input + * @return the equivalent int data + */ + public static int[] convertArrayToInt(FlatArray array) { + Int32Array primitiveArray = (Int32Array) array; + long length = numberOfElementsInBuffer(array); + int[] ret = new int[(int) length]; + for(int i = 0; i < ret.length; i++) { + ret[i] = primitiveArray.Value(i); + } + return ret; + } + + /** + * Convert a boolean array to a {@link BooleanArray} + * @param input the input + * @return the converted array + */ + public static FlatArray convertBooleanArray(boolean[] input) { + DataBuffer dataBuffer = Nd4j.createBufferOfType(org.nd4j.linalg.api.buffer.DataType.BOOL,input); + ArrowBuffer arrowBuffer = new ArrowBuffer(new BytePointer(dataBuffer.pointer()),input.length); + return ByteDecoArrowSerde.createArrayFromArrayData(arrowBuffer,dataBuffer.dataType()); + } + + /** + * Convert a boolean array to a {@link BooleanArray} + * @param input the input + * @return the converted array + */ + public static FlatArray convertBooleanArray(Boolean[] input) { + return convertBooleanArray(ArrayUtils.toPrimitive(input)); + } + + + /** + * Convert a boolean array to a {@link BooleanArray} + * @param input the input + * @return the converted array + */ + public static FlatArray convertBooleanArray(List input) { + return convertBooleanArray(Booleans.toArray(input)); + } + + /** + * Convert a long array to a {@link Int64Array} + * @param input the input + * @return the converted array + */ + public static FlatArray convertLongArray(Long[] input) { + return convertLongArray(ArrayUtils.toPrimitive(input)); + } + + /** + * Convert a long array to a {@link Int64Array} + * @param input the input + * @return the converted array + */ + public static FlatArray convertLongArray(List input) { + return convertLongArray(Longs.toArray(input)); + } + + + /** + * Convert a long array to a {@link Int64Array} + * @param input the input + * @return the converted array + */ + public static FlatArray convertLongArray(long[] input) { + DataBuffer dataBuffer = Nd4j.createBuffer(input); + ArrowBuffer arrowBuffer = new ArrowBuffer(new BytePointer(dataBuffer.pointer()),input.length); + return ByteDecoArrowSerde.createArrayFromArrayData(arrowBuffer,dataBuffer.dataType()); + } + + /** + * Convert a double array to a {@link DoubleArray} + * @param input the input + * @return the converted array + */ + public static FlatArray convertDoubleArray(double[] input) { + DataBuffer dataBuffer = Nd4j.createBuffer(input); + ArrowBuffer arrowBuffer = new ArrowBuffer(new BytePointer(dataBuffer.pointer()),dataBuffer.length()); + return ByteDecoArrowSerde.createArrayFromArrayData(arrowBuffer,dataBuffer.dataType()); + } + + + /** + * Convert a double array to a {@link DoubleArray} + * @param input the input + * @return the converted array + */ + public static FlatArray convertDoubleArray(Double[] input) { + return convertDoubleArray(ArrayUtils.toPrimitive(input)); + } + + /** + * Convert a double array to a {@link DoubleArray} + * @param input the input + * @return the converted array + */ + public static FlatArray convertDoubleArray(List input) { + return convertDoubleArray(Doubles.toArray(input)); + } + + /** + * Convert a float array to a {@link FloatArray} + * @param input the input + * @return the converted array + */ + public static FlatArray convertFloatArray(float[] input) { + DataBuffer dataBuffer = Nd4j.createBuffer(input); + ArrowBuffer arrowBuffer = new ArrowBuffer(new BytePointer(dataBuffer.pointer()),input.length); + return ByteDecoArrowSerde.createArrayFromArrayData(arrowBuffer,dataBuffer.dataType()); + } + + /** + * Convert a float array to a {@link FloatArray} + * @param input the input + * @return the converted array + */ + public static FlatArray convertFloatArray(Float[] input) { + return convertFloatArray(ArrayUtils.toPrimitive(input)); + } + + + /** + * Convert a float array to a {@link FloatArray} + * @param input the input + * @return the converted array + */ + public static FlatArray convertFloatArray(List input) { + return convertFloatArray(Floats.toArray(input)); + } + + /** + * Convert an int array to a {@link Int32Array} + * @param input the input + * @return the converted array + */ + public static FlatArray convertIntArray(int[] input) { + DataBuffer dataBuffer = Nd4j.createBuffer(input); + ArrowBuffer arrowBuffer = new ArrowBuffer(new BytePointer(dataBuffer.pointer()),input.length); + return ByteDecoArrowSerde.createArrayFromArrayData(arrowBuffer,dataBuffer.dataType()); + } + + + + /** + * Convert an int array to a {@link Int32Array} + * @param input the input + * @return the converted array + */ + public static FlatArray convertIntArray(Integer[] input) { + return convertIntArray(ArrayUtils.toPrimitive(input)); + } + + /** + * Convert an int array to a {@link Int32Array} + * @param input the input + * @return the converted array + */ + public static FlatArray convertIntArray(List input) { + return convertIntArray(Ints.toArray(input)); + } + + + + /** + * Convert a string array to a {@link PrimitiveArray} + * @param input the input data + * @return the converted array + */ + public static FlatArray convertStringArray(List input) { + return convertStringArray(input.toArray(new String[input.size()])); + } + + /** + * Convert a string array to a {@link PrimitiveArray} + * @param input the input data + * @return the converted array + */ + public static FlatArray convertStringArray(String[] input) { + DataBuffer dataBuffer = Nd4j.createBufferOfType(org.nd4j.linalg.api.buffer.DataType.UTF8,input); + BytePointer bytePointer = new BytePointer(dataBuffer.pointer()); + ArrowBuffer offsets = ByteDecoArrowSerde.arrowBufferForStringOffsets(dataBuffer).getFirst(); + ArrowBuffer arrowBuffer = new ArrowBuffer(bytePointer,dataBuffer.length()); + return ByteDecoArrowSerde.createArrayFromArrayData(input.length, arrowBuffer, offsets, dataBuffer.dataType()); + } + + + + /** + * Convert a {@link org.bytedeco.arrow.Schema } + * to a datavec {@link Schema} + * @param schema the input schema + * @return the {@link Schema} + */ + public static Schema toDataVecSchema(org.bytedeco.arrow.Schema schema) { + Schema.Builder schemaBuilder = new Builder(); + for(int i = 0; i < schema.num_fields(); i++) { + Field field = schema.field(i); + DataType dataType = field.type(); + if(dataType.equals(arrow._boolean())) { + schemaBuilder.addColumnBoolean(field.name()); + } + else if(dataType.equals(arrow.uint8())) { + schemaBuilder.addColumnInteger(field.name()); + } + else if(dataType.equals(arrow.uint16())) { + schemaBuilder.addColumnInteger(field.name()); + } + else if(dataType.equals(arrow.uint32())) { + schemaBuilder.addColumnLong(field.name()); + } + else if(dataType.equals(arrow.uint64())) { + schemaBuilder.addColumnLong(field.name()); + } + else if(dataType.equals(arrow.int8())) { + schemaBuilder.addColumnInteger(field.name()); + } + else if(dataType.equals(arrow.int16())) { + schemaBuilder.addColumnInteger(field.name()); + } + else if(dataType.equals(arrow.int32())) { + schemaBuilder.addColumnInteger(field.name()); + } + else if(dataType.equals(int64())) { + schemaBuilder.addColumnLong(field.name()); + } + else if(dataType.equals(arrow.float16())) { + schemaBuilder.addColumnFloat(field.name()); + } + else if(dataType.equals(arrow.float32())) { + schemaBuilder.addColumnFloat(field.name()); + } + else if(dataType.equals(float64())) { + schemaBuilder.addColumnDouble(field.name()); + } + else if(dataType.equals(arrow.date32()) || dataType.equals(arrow.date64())) { + schemaBuilder.addColumnTime(field.name(), TimeZone.getTimeZone("UTC")); + } + else if(dataType.equals(arrow.day_time_interval())) { + throw new IllegalArgumentException("Unable to convert type " + dataType.name()); + + } + else if(dataType.equals(arrow.large_utf8())) { + schemaBuilder.addColumnString(field.name()); + } + else if(dataType.equals(arrow.utf8())) { + schemaBuilder.addColumnString(field.name()); + } + else if(dataType.equals(arrow.binary())) { + schemaBuilder.addColumnBytes(field.name()); + } + else { + throw new IllegalArgumentException("Unable to convert type " + dataType.name()); + } + } + + return schemaBuilder.build(); + } + + + /** + * Convert a set of {@link PrimitiveArray} + * to {@link DataVecColumn} + * @param primitiveArrays the primitive arrays + * @param names the names of the columns + * @return the equivalent {@link DataVecColumn}s + * given the types of {@link PrimitiveArray} + */ + public static DataVecColumn[] convertPrimitiveArraysToColumns(FlatArray[] primitiveArrays, String[] names) { + Preconditions.checkState(primitiveArrays != null && names != null && primitiveArrays.length == names.length, + "Arrays and names must not be null and must be same length arrays"); + DataVecColumn[] ret = new DataVecColumn[primitiveArrays.length]; + for(int i = 0; i < ret.length; i++) { + switch(ByteDecoArrowSerde.dataBufferTypeTypeForArrow(primitiveArrays[i].data().type())) { + case UTF8: + StringColumn stringColumn = new StringColumn(names[i],primitiveArrays[i]); + ret[i] = stringColumn; + break; + case INT: + IntColumn intColumn = new IntColumn(names[i],primitiveArrays[i]); + ret[i] = intColumn; + break; + case DOUBLE: + DoubleColumn doubleColumn = new DoubleColumn(names[i],primitiveArrays[i]); + ret[i] = doubleColumn; + break; + case LONG: + LongColumn longColumn = new LongColumn(names[i],primitiveArrays[i]); + ret[i] = longColumn; + break; + case BOOL: + BooleanColumn booleanColumn = new BooleanColumn(names[i],primitiveArrays[i]); + ret[i] = booleanColumn; + break; + case FLOAT: + FloatColumn floatColumn = new FloatColumn(names[i],primitiveArrays[i]); + ret[i] = floatColumn; + break; + + } + } + + return ret; + } + +} diff --git a/datavec/datavec-arrow/src/main/java/org/datavec/arrow/table/DataVecTable.java b/datavec/datavec-arrow/src/main/java/org/datavec/arrow/table/DataVecTable.java new file mode 100644 index 000000000000..20327076ec08 --- /dev/null +++ b/datavec/datavec-arrow/src/main/java/org/datavec/arrow/table/DataVecTable.java @@ -0,0 +1,282 @@ +/* + * Copyright (c) 2019 Konduit KK + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.datavec.arrow.table; + +import org.bytedeco.arrow.*; +import org.datavec.api.transform.schema.Schema; +import org.datavec.api.transform.schema.Schema.Builder; +import org.datavec.arrow.table.column.DataVecColumn; +import org.datavec.arrow.table.column.impl.*; +import org.datavec.arrow.table.row.Row; +import org.datavec.arrow.table.row.RowImpl; +import org.nd4j.base.Preconditions; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.TimeZone; + +/** + * A table representing a data frame like datastructure + * for accessing columnar data + * + * @author Adam Gibson + */ +public class DataVecTable { + + private Table table; + private Schema schema; + private Map columns; + + private DataVecTable(Table table) { + this.table = table; + this.schema = DataVecArrowUtils.toDataVecSchema(table.schema()); + this.columns = new LinkedHashMap<>(); + for(int i = 0; i < schema.numColumns(); i++) { + switch(schema.getType(i)) { + case String: + columns.put(schema.getName(i),new StringColumn(schema.getName(i),table.column(i))); + break; + case Boolean: + columns.put(schema.getName(i),new BooleanColumn(schema.getName(i),table.column(i))); + break; + case Long: + columns.put(schema.getName(i),new LongColumn(schema.getName(i),table.column(i))); + break; + case Float: + columns.put(schema.getName(i),new FloatColumn(schema.getName(i),table.column(i))); + break; + case Double: + columns.put(schema.getName(i),new DoubleColumn(schema.getName(i),table.column(i))); + break; + case Categorical: + columns.put(schema.getName(i),new StringColumn(schema.getName(i),table.column(i))); + break; + case Integer: + columns.put(schema.getName(i),new IntColumn(schema.getName(i),table.column(i))); + break; + case Bytes: + columns.put(schema.getName(i),new StringColumn(schema.getName(i),table.column(i))); + break; + case Time: + columns.put(schema.getName(i),new StringColumn(schema.getName(i),table.column(i))); + break; + case NDArray: + columns.put(schema.getName(i),new StringColumn(schema.getName(i),table.column(i))); + break; + } + } + + } + + + + public DataVecTable addRow(Row row) { + Preconditions.checkState(schema.getColumnNames().equals(row.columnNames())); + Array[] inputData = new Array[schema.numColumns()]; + for(int i = 0; i < schema.numColumns(); i++) { + + } + //ArrayVector arrayVector = new ArrayVector(inputData); + throw new UnsupportedOperationException(); + } + + /** + * Returns the arrow schema {@link org.bytedeco.arrow.Schema} + * @return + */ + public org.bytedeco.arrow.Schema arrowSchema() { + return DataVecArrowUtils.toArrowSchema(schema); + } + + /** + * Returns the {@link Schema} + * for this tbale + * @return + */ + public Schema schema() { + return schema; + } + + /** + * Get the name of the column + * at the specified index + * @param index the indes to get the column name at + * @return the name of the column at the specified index + */ + public String columnNameAt(int index) { + return schema.getName(index); + } + + /** + * Returns the column of the table + * at the given index + * @param columnIndex the index of the column + * to get + * @return the column at the specified index + */ + public DataVecColumn column(int columnIndex) { + return column(schema.getName(columnIndex)); + } + + + /** + * Returns the column in the table with + * the given name + * @param name the name of the column + * @return the column with the given name + */ + public DataVecColumn column(String name) { + Preconditions.checkState(columns.containsKey(name),"No column named " + name + " present in table!"); + return columns.get(name); + } + + + + /** + * Create a {@link Row} + * using this table given the row number + * @param rowNum the row number + * @return + */ + public Row row(int rowNum) { + Row row = new RowImpl(this,rowNum); + return row; + } + + /** + * Create a {@link DataVecTable} + * using the given {@link Table} + * @param table the table to use + * @return the created table + */ + public static DataVecTable create(Table table) { + return new DataVecTable(table); + } + + + /** + * Create a {@link DataVecTable} + * based on the columns + * @param columns the input columns + * @return the created table + */ + public static DataVecTable create(DataVecColumn...columns) { + Preconditions.checkNotNull(columns,"Passed in column array was null!"); + Schema.Builder schemaBuilder = new Builder(); + ArrayVector arrayVector = null; + Array[] arrays = new Array[columns.length]; + for(int i = 0; i < columns.length; i++) { + Preconditions.checkNotNull(columns[i],"Column " + i + " was null!"); + switch(columns[i].type()) { + case Boolean: + schemaBuilder.addColumnBoolean(columns[i].name()); + break; + case Float: + schemaBuilder.addColumnFloat(columns[i].name()); + break; + case Double: + schemaBuilder.addColumnDouble(columns[i].name()); + break; + case Integer: + schemaBuilder.addColumnInteger(columns[i].name()); + break; + case String: + schemaBuilder.addColumnString(columns[i].name()); + break; + case Long: + schemaBuilder.addColumnLong(columns[i].name()); + break; + case Time: + schemaBuilder.addColumnTime(columns[i].name(), TimeZone.getDefault()); + break; + } + + FlatArray flatArray = columns[i].values(); + arrays[i] = flatArray; + } + + arrayVector = new ArrayVector(arrays); + Schema dataVecSchema = schemaBuilder.build(); + org.bytedeco.arrow.Schema arrowSchema = DataVecArrowUtils.toArrowSchema(dataVecSchema); + Table table = Table.Make(arrowSchema,arrayVector); + return new DataVecTable(table); + } + + + /** + * Returns the number of rows in the table + * @return + */ + public long numRows() { + return columns.get(schema.getName(0)).rows(); + } + + /** + * Returns the number of columns in the table + * @return + */ + public long numColumns() { + return table.num_columns(); + } + + + /** + * Create a {@link DataVecColumn} of the specified type + * @param name the name of the column + * @param dataWith the data to create teh column with + * @param the type + * @return + */ + public static DataVecColumn createColumnOfType(String name,T[] dataWith) { + Class clazz = (Class) dataWith[0].getClass(); + DataVecColumn ret = null; + if(clazz.equals(Boolean.class)) { + Boolean[] casted = (Boolean[]) dataWith; + ret = (DataVecColumn) new BooleanColumn(name,casted); + } + else if(clazz.equals(Double.class)) { + Double[] casted = (Double[]) dataWith; + ret = (DataVecColumn) new DoubleColumn(name,casted); + + } + else if(clazz.equals(Float.class)) { + Float[] casted = (Float[]) dataWith; + ret = (DataVecColumn) new FloatColumn(name,casted); + } + else if(clazz.equals(String.class)) { + String[] casted = (String[]) dataWith; + ret = (DataVecColumn) new StringColumn(name,casted); + } + else if(clazz.equals(Long.class)) { + Long[] casted = (Long[]) dataWith; + ret = (DataVecColumn) new LongColumn(name,casted); + } + else if(clazz.equals(Integer.class)) { + Integer[] casted = (Integer[]) dataWith; + ret = (DataVecColumn) new IntColumn(name,casted); + + } + else { + throw new IllegalArgumentException("Illegal type " + clazz.getName()); + } + + return ret; + + } + +} diff --git a/datavec/datavec-arrow/src/main/java/org/datavec/arrow/table/column/BaseDataVecColumn.java b/datavec/datavec-arrow/src/main/java/org/datavec/arrow/table/column/BaseDataVecColumn.java new file mode 100644 index 000000000000..887d129ebb42 --- /dev/null +++ b/datavec/datavec-arrow/src/main/java/org/datavec/arrow/table/column/BaseDataVecColumn.java @@ -0,0 +1,133 @@ +/* + * Copyright (c) 2019 Konduit KK + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.datavec.arrow.table.column; + +import org.bytedeco.arrow.ChunkedArray; +import org.bytedeco.arrow.FlatArray; +import org.datavec.arrow.table.DataVecArrowUtils; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +import static org.nd4j.arrow.Nd4jArrowOpRunner.runOpOn; + +/** + * Abstract class for the column. + * @param the type of the class + * + * @author Adam Gibson + */ +public abstract class BaseDataVecColumn implements DataVecColumn { + + protected String name; + protected FlatArray values; + protected ChunkedArray chunkedArray; + protected long length; + + public BaseDataVecColumn(String name,List input) { + setValues(input); + this.name = name; + } + + public BaseDataVecColumn(String name,T[] input) { + setValues(input); + this.name = name; + } + + public BaseDataVecColumn(String name,ChunkedArray chunkedArray) { + this.name = name; + this.chunkedArray = chunkedArray; + } + + public BaseDataVecColumn(String name, FlatArray values) { + this.name = name; + this.chunkedArray = new ChunkedArray(values); + this.values = values; + } + + @Override + public long rows() { + return length; + } + + @Override + public String name() { + return name; + } + + @Override + public FlatArray values() { + return values; + } + + @Override + public DataVecColumn[] op(String opName, DataVecColumn[] columnParams, String[] outputColumnNames, Object... otherArgs) { + FlatArray[] primitiveArrays = new FlatArray[columnParams.length]; + for(int i = 0; i < columnParams.length; i++) { + primitiveArrays[i] = columnParams[i].values(); + } + + return DataVecArrowUtils.convertPrimitiveArraysToColumns(runOpOn(primitiveArrays, opName, otherArgs),outputColumnNames); + } + + + @Override + public boolean contains(T input) { + for(int i = 0; i < rows(); i++) { + if(elementAtRow(i).equals(input)) + return true; + } + return false; + } + + @Override + public ChunkedArray chunkedValues() { + return chunkedArray; + } + + @Override + public Iterator iterator() { + return new ColumnIterator<>(this); + } + + @Override + public List toList() { + List ret = new ArrayList<>(); + for(T item : this) { + ret.add(item); + } + + return ret; + } + + /** + * Set the values for this column using + * the specified array + * @param values the array of values to use + */ + public abstract void setValues(List values); + + /** + * Set the values for this column using + * the specified array + * @param values the array of values to use + */ + public abstract void setValues(T[] values); + +} diff --git a/datavec/datavec-arrow/src/main/java/org/datavec/arrow/table/column/ColumnIterator.java b/datavec/datavec-arrow/src/main/java/org/datavec/arrow/table/column/ColumnIterator.java new file mode 100644 index 000000000000..ef77493fff8c --- /dev/null +++ b/datavec/datavec-arrow/src/main/java/org/datavec/arrow/table/column/ColumnIterator.java @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2019 Konduit KK + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.datavec.arrow.table.column; + +import org.datavec.arrow.table.column.DataVecColumn; + +import java.util.Iterator; + +/** + * Simple iterator over a column. + * @param + */ +public class ColumnIterator implements Iterator { + + private DataVecColumn dataVecColumn; + private int currRow; + + public ColumnIterator(DataVecColumn dataVecColumn) { + this.dataVecColumn = dataVecColumn; + } + + @Override + public boolean hasNext() { + return currRow < dataVecColumn.rows(); + } + + @Override + public T next() { + T ret = dataVecColumn.elementAtRow(currRow); + currRow++; + return ret; + } +} diff --git a/datavec/datavec-arrow/src/main/java/org/datavec/arrow/table/column/DataVecColumn.java b/datavec/datavec-arrow/src/main/java/org/datavec/arrow/table/column/DataVecColumn.java new file mode 100644 index 000000000000..0d6ad5ec1ac0 --- /dev/null +++ b/datavec/datavec-arrow/src/main/java/org/datavec/arrow/table/column/DataVecColumn.java @@ -0,0 +1,122 @@ +/* + * Copyright (c) 2019 Konduit KK + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.datavec.arrow.table.column; + +import org.bytedeco.arrow.ChunkedArray; +import org.bytedeco.arrow.DataType; +import org.bytedeco.arrow.FlatArray; +import org.datavec.api.transform.ColumnType; +import org.nd4j.linalg.api.ndarray.INDArray; + +import java.util.Comparator; +import java.util.List; + +/** + * A column abstraction on top of {@link org.nd4j.linalg.api.ndarray.INDArray} + * @param + * + * @author Adam Gibson + */ +public interface DataVecColumn extends Iterable, Comparator { + + + /** + * Converts this column to a java list. + * @return + */ + List toList(); + + /** + * Converts this column to an {@link INDArray} + * @return + */ + INDArray toNdArray(); + + /** + * Returns the element at the given row number + * @param rowNumber the element at the given row number + * @return + */ + T elementAtRow(int rowNumber); + + /** + * The column type + * @return + */ + ColumnType type(); + + /** + * The arrow representation + * of the values for the column + * @return the values + */ + FlatArray values(); + + /** + * + * @return + */ + ChunkedArray chunkedValues(); + + /** + * + * @return + */ + DataType arrowDataType(); + + /** + * The column name + * @return + */ + String name(); + + DataVecColumn[] op(String opName, DataVecColumn[] columnParams, String[] outputColumnNames, Object... otherArgs); + + /** + * Returns true if the input is contained in the + * column or not + * @param input the input to test for + * @return + */ + boolean contains(T input); + + /** + * Returns true if the given row is null + * @param row the row to test for + * @return + */ + default boolean rowIsNull(int row) { + return values().IsNull(row); + } + + /** + * Returns the number of missing values + * @return + */ + default long numValuesMissing() { + return values().null_count(); + } + + /** + * Returns the number of rows in the column + * @return + */ + default long rows() { + return values().data().length(); + } +} diff --git a/datavec/datavec-arrow/src/main/java/org/datavec/arrow/table/column/impl/BooleanColumn.java b/datavec/datavec-arrow/src/main/java/org/datavec/arrow/table/column/impl/BooleanColumn.java new file mode 100644 index 000000000000..fb3b037b29f1 --- /dev/null +++ b/datavec/datavec-arrow/src/main/java/org/datavec/arrow/table/column/impl/BooleanColumn.java @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2019 Konduit KK + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.datavec.arrow.table.column.impl; + +import org.bytedeco.arrow.BooleanArray; +import org.bytedeco.arrow.ChunkedArray; +import org.bytedeco.arrow.DataType; +import org.bytedeco.arrow.FlatArray; +import org.bytedeco.arrow.global.arrow; +import org.datavec.api.transform.ColumnType; +import org.datavec.arrow.table.DataVecArrowUtils; +import org.datavec.arrow.table.column.BaseDataVecColumn; +import org.nd4j.arrow.ByteDecoArrowSerde; +import org.nd4j.linalg.api.buffer.DataBuffer; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; + +import java.util.Iterator; +import java.util.List; + +/** + * Boolean type column + * + * @author Adam Gibson + */ +public class BooleanColumn extends BaseDataVecColumn { + + private BooleanArray booleanArray; + + public BooleanColumn(String name, ChunkedArray chunkedArray) { + super(name, chunkedArray); + this.booleanArray = new BooleanArray(chunkedArray.chunk(0)); + this.length = booleanArray.data().buffers().get()[1].size(); + + } + + public BooleanColumn(String name, FlatArray values) { + super(name, values); + this.booleanArray = (BooleanArray) values; + this.length = booleanArray.data().buffers().get()[1].size(); + } + + public BooleanColumn(String name, Boolean[] input) { + super(name, input); + } + + public BooleanColumn(String name, List input) { + super(name, input); + } + + @Override + public void setValues(Boolean[] values) { + this.values = DataVecArrowUtils.convertBooleanArray(values); + this.chunkedArray = new ChunkedArray(this.values); + this.booleanArray = (BooleanArray) this.values; + this.length = booleanArray.data().buffers().get()[1].size(); + + } + + @Override + public void setValues(List values) { + this.values = DataVecArrowUtils.convertBooleanArray(values); + this.chunkedArray = new ChunkedArray(this.values); + this.booleanArray = (BooleanArray) this.values; + this.length = booleanArray.data().buffers().get()[1].size(); + + } + + @Override + public INDArray toNdArray() { + DataBuffer dataBuffer = ByteDecoArrowSerde.fromArrowBuffer(booleanArray.values(),arrowDataType()); + INDArray ret = Nd4j.create(dataBuffer); + return ret; + } + + @Override + public Boolean elementAtRow(int rowNumber) { + return booleanArray.Value(rowNumber); + } + + @Override + public ColumnType type() { + return ColumnType.Boolean; + } + + @Override + public DataType arrowDataType() { + return arrow._boolean(); + } + + @Override + public int compare(Boolean o1, Boolean o2) { + return Boolean.compare(o1,o2); + } +} diff --git a/datavec/datavec-arrow/src/main/java/org/datavec/arrow/table/column/impl/DoubleColumn.java b/datavec/datavec-arrow/src/main/java/org/datavec/arrow/table/column/impl/DoubleColumn.java new file mode 100644 index 000000000000..2999628dfe44 --- /dev/null +++ b/datavec/datavec-arrow/src/main/java/org/datavec/arrow/table/column/impl/DoubleColumn.java @@ -0,0 +1,109 @@ +/* + * Copyright (c) 2019 Konduit KK + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.datavec.arrow.table.column.impl; + +import org.bytedeco.arrow.ChunkedArray; +import org.bytedeco.arrow.DataType; +import org.bytedeco.arrow.DoubleArray; +import org.bytedeco.arrow.FlatArray; +import org.datavec.api.transform.ColumnType; +import org.datavec.arrow.table.DataVecArrowUtils; +import org.datavec.arrow.table.column.BaseDataVecColumn; +import org.nd4j.arrow.ByteDecoArrowSerde; +import org.nd4j.linalg.api.buffer.DataBuffer; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; + +import java.util.Iterator; +import java.util.List; + +import static org.bytedeco.arrow.global.arrow.float64; + +/** + * Double type column + * + * @author Adam Gibson + */ +public class DoubleColumn extends BaseDataVecColumn { + + private DoubleArray doubleArray; + + public DoubleColumn(String name, ChunkedArray chunkedArray) { + super(name, chunkedArray); + this.doubleArray = new DoubleArray(chunkedArray.chunk(0)); + this.length = doubleArray.data().buffers().get()[1].size(); + } + + public DoubleColumn(String name, FlatArray values) { + super(name, values); + this.doubleArray = (DoubleArray) values; + this.length = doubleArray.data().buffers().get()[1].size(); + } + + public DoubleColumn(String name, Double[] input) { + super(name, input); + } + + public DoubleColumn(String name, List input) { + super(name, input); + } + + @Override + public void setValues(Double[] values) { + this.values = DataVecArrowUtils.convertDoubleArray(values); + this.chunkedArray = new ChunkedArray(this.values); + this.doubleArray = (DoubleArray) this.values; + this.length = doubleArray.data().buffers().get()[1].size(); + } + + @Override + public void setValues(List values) { + this.values = DataVecArrowUtils.convertDoubleArray(values); + this.chunkedArray = new ChunkedArray(this.values); + this.doubleArray = (DoubleArray) this.values; + this.length = doubleArray.data().buffers().get()[1].size(); + } + + @Override + public INDArray toNdArray() { + DataBuffer dataBuffer = ByteDecoArrowSerde.fromArrowBuffer(doubleArray.values(),arrowDataType()); + INDArray ret = Nd4j.create(dataBuffer); + return ret; + } + + @Override + public Double elementAtRow(int rowNumber) { + return doubleArray.Value(rowNumber); + } + + @Override + public ColumnType type() { + return ColumnType.Double; + } + + @Override + public DataType arrowDataType() { + return float64(); + } + + + @Override + public int compare(Double o1, Double o2) { + return Double.compare(o1,o2); + } +} diff --git a/datavec/datavec-arrow/src/main/java/org/datavec/arrow/table/column/impl/FloatColumn.java b/datavec/datavec-arrow/src/main/java/org/datavec/arrow/table/column/impl/FloatColumn.java new file mode 100644 index 000000000000..f8f8f215e375 --- /dev/null +++ b/datavec/datavec-arrow/src/main/java/org/datavec/arrow/table/column/impl/FloatColumn.java @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2019 Konduit KK + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.datavec.arrow.table.column.impl; + +import org.bytedeco.arrow.ChunkedArray; +import org.bytedeco.arrow.DataType; +import org.bytedeco.arrow.FlatArray; +import org.bytedeco.arrow.FloatArray; +import org.datavec.api.transform.ColumnType; +import org.datavec.arrow.table.DataVecArrowUtils; +import org.datavec.arrow.table.column.BaseDataVecColumn; +import org.nd4j.arrow.ByteDecoArrowSerde; +import org.nd4j.linalg.api.buffer.DataBuffer; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; + +import java.util.Iterator; +import java.util.List; + +import static org.bytedeco.arrow.global.arrow.float32; + +/** + * Float type column + * + * @author Adam Gibson + */ +public class FloatColumn extends BaseDataVecColumn { + + private FloatArray floatArray; + + public FloatColumn(String name, ChunkedArray chunkedArray) { + super(name, chunkedArray); + this.floatArray = new FloatArray(chunkedArray.chunk(0)); + this.length = floatArray.data().buffers().get()[1].size(); + } + + public FloatColumn(String name, FlatArray values) { + super(name, values); + this.floatArray = (FloatArray) values; + this.length = floatArray.data().buffers().get()[1].size(); + + } + + public FloatColumn(String name, Float[] input) { + super(name, input); + } + + public FloatColumn(String name, List input) { + super(name, input); + } + + @Override + public void setValues(Float[] values) { + this.values = DataVecArrowUtils.convertFloatArray(values); + this.chunkedArray = new ChunkedArray(this.values); + this.floatArray = (FloatArray) this.values; + this.length = floatArray.data().buffers().get()[1].size(); + + } + + @Override + public void setValues(List values) { + this.values = DataVecArrowUtils.convertFloatArray(values); + this.chunkedArray = new ChunkedArray(this.values); + this.floatArray = (FloatArray) this.values; + this.length = floatArray.data().buffers().get()[1].size(); + } + + @Override + public INDArray toNdArray() { + DataBuffer dataBuffer = ByteDecoArrowSerde.fromArrowBuffer(floatArray.values(),arrowDataType()); + INDArray ret = Nd4j.create(dataBuffer); + return ret; + } + + @Override + public Float elementAtRow(int rowNumber) { + return floatArray.Value(rowNumber); + } + + @Override + public ColumnType type() { + return ColumnType.Float; + } + + @Override + public DataType arrowDataType() { + return float32(); + } + + @Override + public int compare(Float o1, Float o2) { + return Float.compare(o1,o2); + } +} diff --git a/datavec/datavec-arrow/src/main/java/org/datavec/arrow/table/column/impl/IntColumn.java b/datavec/datavec-arrow/src/main/java/org/datavec/arrow/table/column/impl/IntColumn.java new file mode 100644 index 000000000000..207ed643d3d7 --- /dev/null +++ b/datavec/datavec-arrow/src/main/java/org/datavec/arrow/table/column/impl/IntColumn.java @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2019 Konduit KK + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.datavec.arrow.table.column.impl; + +import org.bytedeco.arrow.*; +import org.datavec.api.transform.ColumnType; +import org.datavec.arrow.table.DataVecArrowUtils; +import org.datavec.arrow.table.column.BaseDataVecColumn; +import org.nd4j.arrow.ByteDecoArrowSerde; +import org.nd4j.linalg.api.buffer.DataBuffer; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; + +import java.util.Iterator; +import java.util.List; + +import static org.bytedeco.arrow.global.arrow.int32; + +/** + * Int type column + * + * @author Adam Gibson + */ +public class IntColumn extends BaseDataVecColumn { + + private Int32Array intArray; + + public IntColumn(String name, ChunkedArray chunkedArray) { + super(name, chunkedArray); + this.intArray = new Int32Array(chunkedArray.chunk(0)); + this.length = intArray.data().buffers().get()[1].size(); + } + + public IntColumn(String name, FlatArray values) { + super(name, values); + this.intArray = (Int32Array) values; + this.length = intArray.data().buffers().get()[1].size(); + + } + + public IntColumn(String name, Integer[] input) { + super(name, input); + } + + public IntColumn(String name, List input) { + super(name, input); + } + + @Override + public void setValues(Integer[] values) { + this.values = DataVecArrowUtils.convertIntArray(values); + this.chunkedArray = new ChunkedArray(new ArrayVector(this.values)); + this.intArray = (Int32Array) this.values; + this.length = intArray.data().buffers().get()[1].size(); + + } + + @Override + public void setValues(List values) { + this.values = DataVecArrowUtils.convertIntArray(values); + this.chunkedArray = new ChunkedArray(new ArrayVector(this.values)); + this.intArray = (Int32Array) this.values; + this.length = intArray.data().buffers().get()[1].size(); + } + + @Override + public INDArray toNdArray() { + DataBuffer dataBuffer = ByteDecoArrowSerde.fromArrowBuffer(intArray.values(),arrowDataType()); + INDArray ret = Nd4j.create(dataBuffer); + return ret; + } + + @Override + public Integer elementAtRow(int rowNumber) { + return intArray.Value(rowNumber); + } + + @Override + public ColumnType type() { + return ColumnType.Integer; + } + + @Override + public DataType arrowDataType() { + return int32(); + } + + @Override + public int compare(Integer o1, Integer o2) { + return Integer.compare(o1,o2); + } +} diff --git a/datavec/datavec-arrow/src/main/java/org/datavec/arrow/table/column/impl/LongColumn.java b/datavec/datavec-arrow/src/main/java/org/datavec/arrow/table/column/impl/LongColumn.java new file mode 100644 index 000000000000..46fc2122eb62 --- /dev/null +++ b/datavec/datavec-arrow/src/main/java/org/datavec/arrow/table/column/impl/LongColumn.java @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2019 Konduit KK + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.datavec.arrow.table.column.impl; + +import org.bytedeco.arrow.ChunkedArray; +import org.bytedeco.arrow.DataType; +import org.bytedeco.arrow.FlatArray; +import org.bytedeco.arrow.Int64Array; +import org.datavec.api.transform.ColumnType; +import org.datavec.arrow.table.DataVecArrowUtils; +import org.datavec.arrow.table.column.BaseDataVecColumn; +import org.nd4j.arrow.ByteDecoArrowSerde; +import org.nd4j.linalg.api.buffer.DataBuffer; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; + +import java.util.Iterator; +import java.util.List; + +import static org.bytedeco.arrow.global.arrow.int64; + +/** + * Long type column + * + * @author Adam Gibson + */ +public class LongColumn extends BaseDataVecColumn { + + private Int64Array int64Array; + + public LongColumn(String name, ChunkedArray chunkedArray) { + super(name, chunkedArray); + this.int64Array = new Int64Array(chunkedArray.chunk(0)); + this.length = int64Array.data().buffers().get()[1].size(); + } + + public LongColumn(String name, FlatArray values) { + super(name, values); + this.int64Array = (Int64Array) values; + this.length = int64Array.data().buffers().get()[1].size(); + } + + public LongColumn(String name, List input) { + super(name, input); + } + + public LongColumn(String name, Long[] input) { + super(name, input); + } + + @Override + public void setValues(Long[] values) { + this.values = DataVecArrowUtils.convertLongArray(values); + this.chunkedArray = new ChunkedArray(this.values); + this.int64Array = (Int64Array) this.values; + this.length = int64Array.data().buffers().get()[1].size(); + } + + @Override + public void setValues(List values) { + this.values = DataVecArrowUtils.convertLongArray(values); + this.chunkedArray = new ChunkedArray(this.values); + this.int64Array = (Int64Array) this.values; + this.length = int64Array.data().buffers().get()[1].size(); + } + + @Override + public INDArray toNdArray() { + DataBuffer dataBuffer = ByteDecoArrowSerde.fromArrowBuffer(int64Array.values(),arrowDataType()); + INDArray ret = Nd4j.create(dataBuffer); + return ret; + } + + @Override + public Long elementAtRow(int rowNumber) { + return int64Array.Value(rowNumber); + } + + @Override + public ColumnType type() { + return ColumnType.Long; + } + + @Override + public DataType arrowDataType() { + return int64(); + } + + @Override + public int compare(Long o1, Long o2) { + return Long.compare(o1,o2); + } + + +} diff --git a/datavec/datavec-arrow/src/main/java/org/datavec/arrow/table/column/impl/StringColumn.java b/datavec/datavec-arrow/src/main/java/org/datavec/arrow/table/column/impl/StringColumn.java new file mode 100644 index 000000000000..484725660f18 --- /dev/null +++ b/datavec/datavec-arrow/src/main/java/org/datavec/arrow/table/column/impl/StringColumn.java @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2019 Konduit KK + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.datavec.arrow.table.column.impl; + +import org.bytedeco.arrow.ChunkedArray; +import org.bytedeco.arrow.DataType; +import org.bytedeco.arrow.FlatArray; +import org.bytedeco.arrow.StringArray; +import org.bytedeco.javacpp.IntPointer; +import org.datavec.api.transform.ColumnType; +import org.datavec.arrow.table.DataVecArrowUtils; +import org.datavec.arrow.table.column.BaseDataVecColumn; +import org.nd4j.arrow.ByteDecoArrowSerde; +import org.nd4j.linalg.api.buffer.DataBuffer; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; + +import java.util.Iterator; +import java.util.List; + +import static org.bytedeco.arrow.global.arrow.utf8; + +/** + * String type column + * @author Adam Gibson + */ +public class StringColumn extends BaseDataVecColumn { + + private StringArray stringArray; + + public StringColumn(String name, ChunkedArray chunkedArray) { + super(name, chunkedArray); + this.stringArray = new StringArray(chunkedArray.chunk(0)); + this.length = stringArray.data().buffers().get()[2].size(); + } + + public StringColumn(String name, FlatArray values) { + super(name, values); + this.stringArray = (StringArray) values; + this.length = stringArray.data().buffers().get()[2].size(); + + + } + + public StringColumn(String name, List input) { + super(name, input); + } + + public StringColumn(String name, String[] input) { + super(name, input); + } + + @Override + public void setValues(String[] values) { + this.values = DataVecArrowUtils.convertStringArray(values); + this.chunkedArray = new ChunkedArray(this.values); + this.stringArray = (StringArray) this.values; + this.length = stringArray.data().buffers().get()[2].size(); + } + + @Override + public INDArray toNdArray() { + DataBuffer dataBuffer = ByteDecoArrowSerde.fromArrowBuffer(stringArray.value_data(),arrowDataType()); + INDArray ret = Nd4j.create(dataBuffer); + return ret; + } + + @Override + public void setValues(List values) { + this.values = DataVecArrowUtils.convertStringArray(values); + this.chunkedArray = new ChunkedArray(this.values); + this.stringArray = (StringArray) this.values; + this.length = stringArray.data().buffers().get()[2].size(); + } + + @Override + public String elementAtRow(int rowNumber) { + return DataVecArrowUtils.elementAt(stringArray,rowNumber,length); + } + + @Override + public ColumnType type() { + return ColumnType.String; + } + + + @Override + public DataType arrowDataType() { + return utf8(); + } + + @Override + public int compare(String o1, String o2) { + return o1.compareTo(o2); + } +} diff --git a/datavec/datavec-arrow/src/main/java/org/datavec/arrow/table/row/Row.java b/datavec/datavec-arrow/src/main/java/org/datavec/arrow/table/row/Row.java new file mode 100644 index 000000000000..7b529083fa28 --- /dev/null +++ b/datavec/datavec-arrow/src/main/java/org/datavec/arrow/table/row/Row.java @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2019 Konduit KK + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.datavec.arrow.table.row; + +import org.datavec.arrow.table.DataVecTable; + +import java.util.List; + +/** + * Represents a row in a {@link DataVecTable} + * This row is just a view of the underlying data. + * + * @author Adam Gibson + */ +public interface Row { + + /** + * The underlying {@link DataVecTable} + * of the row + * @return + */ + DataVecTable table(); + + /** + * The row number of the row + * @return + */ + int rowNumber(); + + /** + * Get the element at a particular column + * @param column the index of the column to get the element at + * @param the type of return + * @return + */ + T elementAtColumn(int column); + + /** + * + * @param columnName + * @param + * @return + */ + T elementAtColumn(String columnName); + + /** + * The column names of the row + * @return + */ + List columnNames(); + +} diff --git a/datavec/datavec-arrow/src/main/java/org/datavec/arrow/table/row/RowImpl.java b/datavec/datavec-arrow/src/main/java/org/datavec/arrow/table/row/RowImpl.java new file mode 100644 index 000000000000..558997e74573 --- /dev/null +++ b/datavec/datavec-arrow/src/main/java/org/datavec/arrow/table/row/RowImpl.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2019 Konduit KK + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.datavec.arrow.table.row; + +import org.datavec.arrow.table.DataVecTable; +import org.datavec.arrow.table.column.DataVecColumn; + +import java.util.List; + +/** + * Row implementation. + * Represents multiple {@link org.datavec.arrow.table.column.DataVecColumn} + * that have all + * of the same row index. + * + * @author Adam Gibson + */ +public class RowImpl implements Row { + + private DataVecTable table; + private int rowNum; + + /** + * An implementation of a row. + * @param table the table to provide the view for + * @param rowNum the row number representative for the + * view + */ + public RowImpl(DataVecTable table, int rowNum) { + this.table = table; + this.rowNum = rowNum; + } + + @Override + public DataVecTable table() { + return table; + } + + @Override + public int rowNumber() { + return rowNum; + } + + @Override + public T elementAtColumn(int column) { + return elementAtColumn(table.columnNameAt(column)); + } + + @Override + public T elementAtColumn(String columnName) { + DataVecColumn column = table.column(columnName); + return column.elementAtRow(rowNumber()); + } + + @Override + public List columnNames() { + return table.schema().getColumnNames(); + } +} diff --git a/datavec/datavec-arrow/src/test/java/org/datavec/arrow/table/DataVecArrowUtilsTest.java b/datavec/datavec-arrow/src/test/java/org/datavec/arrow/table/DataVecArrowUtilsTest.java new file mode 100644 index 000000000000..587bd432484a --- /dev/null +++ b/datavec/datavec-arrow/src/test/java/org/datavec/arrow/table/DataVecArrowUtilsTest.java @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2019 Konduit KK + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.datavec.arrow.table; + +import org.bytedeco.arrow.FlatArray; +import org.junit.Test; +import org.nd4j.linalg.api.buffer.DataType; + + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + +public class DataVecArrowUtilsTest { + + @Test + public void testToArrayDataConversion() { + for(DataType dataType : DataType.values()) { + switch(dataType) { + case UINT32: + break; + case UBYTE: + break; + case BOOL: + boolean[] inputBoolean = {true}; + FlatArray primitiveArrayBoolean = DataVecArrowUtils.convertBooleanArray(inputBoolean); + boolean[] booleans = DataVecArrowUtils.convertArrayToBoolean(primitiveArrayBoolean); + assertArrayEquals(inputBoolean,booleans); + break; + case LONG: + long[] input = {1}; + FlatArray primitiveArrayLong = DataVecArrowUtils.convertLongArray(input); + long[] longs = DataVecArrowUtils.convertArrayToLong(primitiveArrayLong); + assertArrayEquals(input,longs); + break; + case UNKNOWN: + break; + case SHORT: + break; + case DOUBLE: + double[] inputDouble = {1.0,2.0,3.0}; + FlatArray primitiveArrayDouble = DataVecArrowUtils.convertDoubleArray(inputDouble); + assertEquals(inputDouble.length,DataVecArrowUtils.numberOfElementsInBuffer(primitiveArrayDouble)); + double[] doubles = DataVecArrowUtils.convertArrayToDouble(primitiveArrayDouble); + assertArrayEquals(inputDouble,doubles,1e-3); + break; + case UTF8: + String[] inputString = {"input","input2","input3"}; + FlatArray primitiveArray = DataVecArrowUtils.convertStringArray(inputString); + assertEquals(inputString.length,DataVecArrowUtils.numberOfElementsInBuffer(primitiveArray)); + + String[] strings = DataVecArrowUtils.convertArrayToString(primitiveArray); + assertArrayEquals(inputString,strings); + break; + case BFLOAT16: + break; + case UINT16: + break; + case INT: + int[] ret = {1}; + FlatArray primitiveArray1 = DataVecArrowUtils.convertIntArray(ret); + int[] ints = DataVecArrowUtils.convertArrayToInt(primitiveArray1); + assertArrayEquals(ret,ints); + break; + case BYTE: + break; + case UINT64: + break; + case HALF: + break; + case FLOAT: + float[] retFloat = {1.0f}; + FlatArray primitiveArrayFloat = DataVecArrowUtils.convertFloatArray(retFloat); + float[] floats = DataVecArrowUtils.convertArrayToFloat(primitiveArrayFloat); + assertArrayEquals(retFloat,floats,1e-3f); + break; + case COMPRESSED: + break; + } + } + } +} diff --git a/datavec/datavec-arrow/src/test/java/org/datavec/arrow/table/TableTests.java b/datavec/datavec-arrow/src/test/java/org/datavec/arrow/table/TableTests.java new file mode 100644 index 000000000000..0d688e9d3475 --- /dev/null +++ b/datavec/datavec-arrow/src/test/java/org/datavec/arrow/table/TableTests.java @@ -0,0 +1,122 @@ +/* + * Copyright (c) 2019 Konduit KK + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.datavec.arrow.table; + +import org.datavec.api.transform.ColumnType; +import org.datavec.arrow.table.column.DataVecColumn; +import org.datavec.arrow.table.column.impl.*; +import org.datavec.arrow.table.row.Row; +import org.junit.Test; +import org.nd4j.linalg.api.ndarray.INDArray; + +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class TableTests { + + @Test + public void testTable() { + int count = 0; + ColumnType[] columnTypes = new ColumnType[] { + ColumnType.Integer, + ColumnType.Double, + ColumnType.Float, + ColumnType.Boolean, + ColumnType.String + }; + + DataVecColumn[] dataVecColumns = new DataVecColumn[columnTypes.length]; + DataVecColumn[] dataVecColumnsList = new DataVecColumn[columnTypes.length]; + + for(ColumnType columnType : columnTypes) { + switch(columnType) { + case Double: + dataVecColumns[count] = new DoubleColumn(columnType.name().toLowerCase(),new Double[]{1.0}); + dataVecColumnsList[count] = new DoubleColumn(columnType.name().toLowerCase(), Arrays.asList(1.0)); + break; + case Float: + dataVecColumns[count] = new FloatColumn(columnType.name().toLowerCase(),new Float[]{1.0f}); + dataVecColumnsList[count] = new FloatColumn(columnType.name().toLowerCase(),Arrays.asList(1.0f)); + break; + case Boolean: + dataVecColumns[count] = new BooleanColumn(columnType.name().toLowerCase(),new Boolean[]{true}); + dataVecColumnsList[count] = new BooleanColumn(columnType.name().toLowerCase(),Arrays.asList(true)); + break; + case String: + dataVecColumns[count] = new StringColumn(columnType.name().toLowerCase(),new String[]{"1.0"}); + dataVecColumnsList[count] = new StringColumn(columnType.name().toLowerCase(),Arrays.asList("1.0")); + break; + case Long: + dataVecColumns[count] = new LongColumn(columnType.name().toLowerCase(),new Long[]{1L}); + dataVecColumnsList[count] = new LongColumn(columnType.name().toLowerCase(),Arrays.asList(1L)); + break; + case Integer: + dataVecColumns[count] = new IntColumn(columnType.name().toUpperCase(),new Integer[]{1}); + dataVecColumnsList[count] = new IntColumn(columnType.name().toUpperCase(),Arrays.asList(1)); + break; + + } + + assertEquals(1,dataVecColumns[count].rows()); + assertEquals("Column type of " + columnType + " has wrong number of rows",1,dataVecColumns[count].rows()); + count++; + } + + DataVecTable dataVecTable1 = DataVecTable.create(dataVecColumns); + assertEquals(columnTypes.length,dataVecTable1.numColumns()); + DataVecTable dataVecTableList = DataVecTable.create(dataVecColumnsList); + + Row row = dataVecTable1.row(0); + Row row2 = dataVecTableList.row(0); + assertEquals(1.0d,row.elementAtColumn("double"),1e-3); + assertEquals(1.0f,row.elementAtColumn("float"),1e-3f); + assertEquals("1.0",row.elementAtColumn("string")); + assertEquals(true, row.elementAtColumn("boolean")); + + assertEquals(1.0d,row2.elementAtColumn("double"),1e-3); + assertEquals(1.0f,row2.elementAtColumn("float"),1e-3f); + assertEquals("1.0",row2.elementAtColumn("string")); + assertEquals(true, row2.elementAtColumn("boolean")); + + + + for(int i = 0; i < row.columnNames().size(); i++) { + assertTrue(row.elementAtColumn(i).equals(row.elementAtColumn(row.columnNames().get(i)))); + assertTrue(dataVecTable1.column(i).contains(row.elementAtColumn(i))); + INDArray arr2 = dataVecTable1.column(i).toNdArray(); + assertEquals(dataVecTable1.column(1).rows(),arr2.length()); + List list = dataVecTable1.column(i).toList(); + assertEquals(1,list.size()); + + + assertTrue(row2.elementAtColumn(i).equals(row2.elementAtColumn(row2.columnNames().get(i)))); + assertTrue(dataVecTableList.column(i).contains(row2.elementAtColumn(i))); + arr2 = dataVecTableList.column(i).toNdArray(); + assertEquals(dataVecTableList.column(1).rows(),arr2.length()); + list = dataVecTableList.column(i).toList(); + assertEquals(1,list.size()); + + } + + assertEquals(1,dataVecTable1.numRows()); + + } +} diff --git a/datavec/datavec-arrow/src/test/java/org/datavec/arrow/table/column/impl/ColumnTests.java b/datavec/datavec-arrow/src/test/java/org/datavec/arrow/table/column/impl/ColumnTests.java new file mode 100644 index 000000000000..33003130aaa3 --- /dev/null +++ b/datavec/datavec-arrow/src/test/java/org/datavec/arrow/table/column/impl/ColumnTests.java @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2019 Konduit KK + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.datavec.arrow.table.column.impl; + +import org.datavec.arrow.table.DataVecTable; +import org.datavec.arrow.table.column.DataVecColumn; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class ColumnTests { + + @Test + public void testBooleanColumn() { + assertColumnInput(new Boolean[]{true,false}); + } + + @Test + public void testDoubleColumn() { + assertColumnInput(new Double[]{1.0,2.0}); + + } + + @Test + public void testFloatColumn() { + assertColumnInput(new Float[]{1.0f,2.0f}); + + } + + + @Test + public void testIntColumn() { + assertColumnInput(new Integer[]{1,2}); + + } + + @Test + public void testLongColumn() { + assertColumnInput(new Long[]{1L,2L}); + + } + + @Test + public void testStringColumn() { + assertColumnInput(new String[]{"12","22"}); + + } + + + private void assertColumnInput(T[] inputData) { + DataVecColumn column = DataVecTable.createColumnOfType("test",inputData); + assertEquals(inputData.length,column.rows()); + for(int i = 0; i < inputData.length; i++) { + assertEquals(inputData[i],column.elementAtRow(i)); + } + + column.op("reduce_sum",new DataVecColumn[]{column},new String[]{"test"},null); + + } + +} diff --git a/datavec/datavec-local/src/main/java/org/datavec/local/transforms/LocalTransformExecutor.java b/datavec/datavec-local/src/main/java/org/datavec/local/transforms/LocalTransformExecutor.java index f0d882f92247..91a90e3791ab 100644 --- a/datavec/datavec-local/src/main/java/org/datavec/local/transforms/LocalTransformExecutor.java +++ b/datavec/datavec-local/src/main/java/org/datavec/local/transforms/LocalTransformExecutor.java @@ -453,7 +453,6 @@ public boolean test(Map.Entry>> stringListEntry, Map } } - } }); diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/buffer/DataBuffer.java b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/buffer/DataBuffer.java index c6e1af9d08b8..efee1400a4d5 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/buffer/DataBuffer.java +++ b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/buffer/DataBuffer.java @@ -452,6 +452,14 @@ enum AllocationMode { long[] asLong(); + boolean[] asBoolean(); + + boolean getBool(long i); + + String getUtf8(long i); + + String[] asUtf8(); + /** * Get element i in the buffer as a double * @@ -508,11 +516,38 @@ enum AllocationMode { */ void put(long i, int element); + /** + * Insert the element + * @param i the index + * @param element the element + */ void put(long i, long element); + /** + * + * @param i + * @param element + */ void put(long i, boolean element); + /** + * Assign an element in the buffer to the specified index + * + * @param i the index + * @param element the element to assign + */ + void put(long i, String element); + + + /** + * The total byte length of the array. + * Typically it's the data type * the number of elements. + * For strings, it's variable. + * @return + */ + long byteLength(); + /** * Returns the length of the buffer * @@ -590,6 +625,12 @@ enum AllocationMode { */ void assign(DataBuffer... buffers); + /** + * Returns an n + 1 length buffer + * @return + */ + DataBuffer binaryOffsets(); + /** * Assign the given buffers to this buffer * based on the given offsets and strides. diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/buffer/DataType.java b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/buffer/DataType.java index c48b7577c270..0b4393a1f965 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/buffer/DataType.java +++ b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/buffer/DataType.java @@ -63,6 +63,8 @@ public enum DataType { BOOL, UTF8, + UTF16, + UTF32, COMPRESSED, BFLOAT16, UINT16, @@ -93,6 +95,9 @@ public static DataType fromInt(int type) { case 13: return UINT32; case 14: return UINT64; case 17: return BFLOAT16; + case 50: return UTF8; + case 51: return UTF16; + case 52: return UTF32; default: throw new UnsupportedOperationException("Unknown data type: [" + type + "]"); } } @@ -113,6 +118,8 @@ public int toInt() { case UINT64: return 14; case BFLOAT16: return 17; case UTF8: return 50; + case UTF16: return 51; + case UTF32: return 52; default: throw new UnsupportedOperationException("Non-covered data type: [" + this + "]"); } } @@ -131,19 +138,23 @@ public boolean isIntType(){ return this == LONG || this == INT || this == SHORT || this == UBYTE || this == BYTE || this == UINT16 || this == UINT32 || this == UINT64; } + public boolean isStringType() { + return this == UTF8 || this == UTF16 || this == UTF32; + } + /** * Return true if the value is numerical.
* Equivalent to {@code this != UTF8 && this != COMPRESSED && this != UNKNOWN}
* Note: Boolean values are considered numerical (0/1)
*/ public boolean isNumerical(){ - return this != UTF8 && this != BOOL && this != COMPRESSED && this != UNKNOWN; + return !this.isStringType() && this != BOOL && this != COMPRESSED && this != UNKNOWN; } /** * @return True if the datatype is a numerical type and is signed (supports negative values) */ - public boolean isSigned(){ + public boolean isSigned() { switch (this){ case DOUBLE: case FLOAT: @@ -170,7 +181,7 @@ public boolean isSigned(){ /** * @return the max number of significant decimal digits */ - public int precision(){ + public int precision() { switch (this){ case DOUBLE: return 17; @@ -200,7 +211,7 @@ public int precision(){ /** * @return For fixed-width types, this returns the number of bytes per array element */ - public int width(){ + public int width() { switch (this){ case DOUBLE: case LONG: @@ -220,6 +231,8 @@ public int width(){ case BOOL: return 1; case UTF8: + case UTF16: + case UTF32: case COMPRESSED: case UNKNOWN: default: @@ -227,7 +240,7 @@ public int width(){ } } - public static DataType fromNumpy(String numpyDtypeName){ + public static DataType fromNumpy(String numpyDtypeName) { switch (numpyDtypeName.toLowerCase()){ case "bool": return BOOL; case "byte": diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/buffer/factory/DataBufferFactory.java b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/buffer/factory/DataBufferFactory.java index 3ca03ebe2c4f..518b771020fd 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/buffer/factory/DataBufferFactory.java +++ b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/buffer/factory/DataBufferFactory.java @@ -47,6 +47,22 @@ public interface DataBufferFactory { */ DataBuffer.AllocationMode allocationMode(); + /** + * Create a buffer of a specified data type + * @param dataType the data type to create + * @param length the length of the buffer + * @return the created data buffer + */ + DataBuffer createBufferOfType(DataType dataType, long length); + + /** + * Create a buffer of a specified data type + * @param dataType the data type to create + * @param input the input to use (should be a type of array) + * @return the created data buffer + */ + DataBuffer createBufferOfType(DataType dataType,Object input); + /** * Create a databuffer wrapping another one * this allows you to create a view of a buffer diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/buffer/util/DataTypeUtil.java b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/buffer/util/DataTypeUtil.java index 15624209f49d..9f707b44cd5f 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/buffer/util/DataTypeUtil.java +++ b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/buffer/util/DataTypeUtil.java @@ -40,17 +40,26 @@ public class DataTypeUtil { */ public static int lengthForDtype(DataType type) { switch (type) { + case LONG: + case UINT64: case DOUBLE: return 8; case FLOAT: - return 4; case INT: + case UINT32: return 4; + case UINT16: + case SHORT: + case BFLOAT16: case HALF: return 2; - case LONG: - return 8; + case BOOL: + case UTF8: + case BYTE: + case UBYTE: + return 1; case COMPRESSED: + case UNKNOWN: default: throw new IllegalArgumentException("Illegal opType for length"); diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ndarray/BaseNDArray.java b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ndarray/BaseNDArray.java index 3edbd2682117..27d4681e23e2 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ndarray/BaseNDArray.java +++ b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/api/ndarray/BaseNDArray.java @@ -212,7 +212,6 @@ public BaseNDArray(DataBuffer buffer, long[] shape, long[] stride, char ordering setShapeInformation(Nd4j.getShapeInfoProvider().createShapeInformation(shape, stride, Shape.elementWiseStride(shape, stride, ordering == 'f'), ordering, type, false)); init(shape, stride); - // Shape.setElementWiseStride(this.shapeInfo(),Shape.elementWiseStride(shape, stride, ordering == 'f')); } public BaseNDArray(DataBuffer buffer, long[] shape, long[] stride, char ordering, DataType type, MemoryWorkspace workspace) { @@ -220,7 +219,6 @@ public BaseNDArray(DataBuffer buffer, long[] shape, long[] stride, char ordering setShapeInformation(Nd4j.getShapeInfoProvider().createShapeInformation(shape, stride, Shape.elementWiseStride(shape, stride, ordering == 'f'), ordering, type, false)); init(shape, stride); - // Shape.setElementWiseStride(this.shapeInfo(),Shape.elementWiseStride(shape, stride, ordering == 'f')); } public BaseNDArray(DataBuffer buffer, DataType dataType, long[] shape, long[] stride, long offset, char ordering) { @@ -228,7 +226,6 @@ public BaseNDArray(DataBuffer buffer, DataType dataType, long[] shape, long[] s setShapeInformation(Nd4j.getShapeInfoProvider().createShapeInformation(shape, stride, Shape.elementWiseStride(shape, stride, ordering == 'f'), ordering, dataType, false)); init(shape, stride); - // Shape.setElementWiseStride(this.shapeInfo(),Shape.elementWiseStride(shape, stride, ordering == 'f')); } /** @@ -3697,14 +3694,15 @@ public INDArray reshape(char order, long... newShape) { } @Override - public INDArray reshape(char order, boolean enforceView, long... newShape){ + public INDArray reshape(char order, boolean enforceView, long... newShape) { Nd4j.getCompressor().autoDecompress(this); // special case for empty reshape - if (this.length() == 1 && (newShape == null || newShape.length == 0) && this.elementWiseStride() == 1) { + if (this.length() < 2 && (newShape == null || newShape.length == 0) && this.elementWiseStride() == 1) { return Nd4j.create(this.data(), new int[0], new int[0], 0); } + if (newShape == null || newShape.length < 1) throw new ND4JIllegalStateException( "Can't reshape(long...) without shape arguments. Got empty shape instead."); @@ -5494,7 +5492,7 @@ public boolean isB() { @Override public boolean isS() { - return dataType() == DataType.UTF8; + return dataType().isStringType(); } @Override diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/compression/CompressedDataBuffer.java b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/compression/CompressedDataBuffer.java index 7cbfb0d70674..66ec553e68af 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/compression/CompressedDataBuffer.java +++ b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/compression/CompressedDataBuffer.java @@ -84,6 +84,36 @@ public void write(DataOutputStream out) throws IOException { } } + @Override + public boolean[] asBoolean() { + return new boolean[0]; + } + + @Override + public boolean getBool(long i) { + return false; + } + + @Override + public String getUtf8(long i) { + return null; + } + + @Override + public String[] asUtf8() { + return new String[0]; + } + + @Override + public void put(long i, String element) { + + } + + @Override + public long byteLength() { + return 0; + } + @Override protected void setIndexer(Indexer indexer) { // no-op @@ -149,6 +179,11 @@ public DataBuffer dup() { return nBuf; } + @Override + public DataBuffer binaryOffsets() { + throw new UnsupportedOperationException(); + } + @Override public long length() { return compressionDescriptor.getNumberOfElements(); diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/factory/Nd4j.java b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/factory/Nd4j.java index 5da64dadb5a4..c32049d29395 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/factory/Nd4j.java +++ b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/linalg/factory/Nd4j.java @@ -371,7 +371,7 @@ private static INDArray appendImpl(INDArray arr, int padAmount, double val, int INDArray concatArray = Nd4j.valueArrayOf(paShape, val, arr.dataType()); return appendFlag ? Nd4j.concat(axis, arr, concatArray) : Nd4j.concat(axis, concatArray, arr); } - + /** * Expand the array dimensions. * This is equivalent to @@ -873,7 +873,7 @@ public static INDArray matmul(INDArray a, INDArray b, INDArray result){ * See {@link #matmul(INDArray, INDArray, INDArray, boolean, boolean, boolean)} */ public static INDArray matmul(INDArray a, INDArray b, boolean transposeA, boolean transposeB, boolean transposeResult){ - return matmul(a, b, null, transposeA, transposeB, transposeResult); + return matmul(a, b, null, transposeA, transposeB, transposeResult); } /** @@ -1061,6 +1061,37 @@ public static INDArray mean(INDArray compute, int dimension) { public static DataBuffer createBuffer(DataBuffer underlyingBuffer, long offset, long length) { return DATA_BUFFER_FACTORY_INSTANCE.create(underlyingBuffer, offset, length); } + /** + * Create a buffer of a specified data type + * @param dataType the data type to create + * @param offset the offset to create + * @param length the length of the buffer + * @return the created data buffer + */ + public static DataBuffer createBufferOfType(DataType dataType, long offset, long length) { + return DATA_BUFFER_FACTORY_INSTANCE.createBufferOfType(dataType,length); + } + + /** + * Create a buffer of a specified data type + * @param dataType the data type to create + * @param input the input to use (should be a type of array) + * @return the created data buffer + */ + public static DataBuffer createBufferOfType(DataType dataType,Object input) { + return DATA_BUFFER_FACTORY_INSTANCE.createBufferOfType(dataType,input); + } + + + /** + * Create a buffer of a specified data type + * @param dataType the data type to create + * @param length the input to use (should be a type of array) + * @return the created data buffer + */ + public static DataBuffer createBufferOfType(DataType dataType,long length) { + return DATA_BUFFER_FACTORY_INSTANCE.createBufferOfType(dataType,length); + } /** * Create a buffer equal of length prod(shape) @@ -1071,8 +1102,7 @@ public static DataBuffer createBuffer(DataBuffer underlyingBuffer, long offset, */ public static DataBuffer createBuffer(int[] shape, DataType type, long offset) { int length = ArrayUtil.prod(shape); - return type == DataType.DOUBLE ? createBuffer(new double[length], offset) - : createBuffer(new float[length], offset); + return createBufferOfType(type,offset,length); } /** @@ -1103,7 +1133,7 @@ public static DataBuffer createBuffer(byte[] data, int length, long offset) { ret = DATA_BUFFER_FACTORY_INSTANCE.createFloat(offset, data, length); return ret; } - + /** * Creates a buffer of the specified length based on the data opType * @@ -1140,6 +1170,8 @@ private static Indexer getIndexerByType(Pointer pointer, DataType dataType) { return ShortIndexer.create((ShortPointer) pointer); case BYTE: return ByteIndexer.create((BytePointer) pointer); + case UTF8: + return ByteIndexer.create((BytePointer) pointer); case UBYTE: return UByteIndexer.create((BytePointer) pointer); case BOOL: @@ -1201,6 +1233,7 @@ private static Pointer getPointer(@NonNull Pointer pointer, @NonNull DataType da nPointer = new ShortPointer(pointer); break; case BYTE: + case UTF8: nPointer = new BytePointer(pointer); break; case UBYTE: @@ -2647,7 +2680,7 @@ public static INDArray readBinary(File read) throws IOException { public static void clearNans(INDArray arr) { getExecutioner().exec(new ReplaceNans(arr, Nd4j.EPS_THRESHOLD)); } - + /** * Reverses the passed in matrix such that m[0] becomes m[m.length - 1] etc * @@ -2747,7 +2780,7 @@ public static INDArray choice(@NonNull INDArray source, @NonNull INDArray probs, public static INDArray choice(INDArray source, INDArray probs, INDArray target) { return choice(source, probs, target, Nd4j.getRandom()); } - + // @see tag works well here. /** * This method returns new INDArray instance, sampled from Source array with probabilities given in Probs. @@ -3741,7 +3774,7 @@ public static INDArray empty(DataType type) { try(MemoryWorkspace ignored = Nd4j.getMemoryManager().scopeOutOfWorkspaces()){ val ret = INSTANCE.empty(type); EMPTY_ARRAYS[type.ordinal()] = ret; - } + } } return EMPTY_ARRAYS[type.ordinal()]; } @@ -4095,7 +4128,7 @@ public static INDArray create(DataBuffer data, long... shape) { * @param buffer data data buffer used for initialisation. * @return the created ndarray. */ - public static INDArray create(DataBuffer buffer) { + public static INDArray create(DataBuffer buffer) { return INSTANCE.create(buffer); } @@ -4247,7 +4280,7 @@ public static INDArray create(@NonNull int[] shape, char ordering) { if(shape.length == 0) return Nd4j.scalar(dataType(), 0.0); - return INSTANCE.create(shape, ordering); + return INSTANCE.create(shape, ordering); } // used often. @@ -4832,7 +4865,7 @@ public static INDArray pullRows(INDArray source, INDArray destination, int sourc public static INDArray stack(int axis, @NonNull INDArray... values){ Preconditions.checkArgument(values != null && values.length > 0, "No inputs: %s", (Object[]) values); Preconditions.checkState(axis >= -(values[0].rank()+1) && axis < values[0].rank()+1, "Invalid axis: must be between " + - "%s (inclusive) and %s (exclusive) for rank %s input, got %s", -(values[0].rank()+1), values[0].rank()+1, + "%s (inclusive) and %s (exclusive) for rank %s input, got %s", -(values[0].rank()+1), values[0].rank()+1, values[0].rank(), axis); Stack stack = new Stack(values, null, axis); @@ -5853,7 +5886,7 @@ public static INDArray createFromFlatArray(FlatArray array) { } } - + public static DataType defaultFloatingPointType() { return defaultFloatingPointDataType.get(); } @@ -6585,7 +6618,7 @@ public static INDArray[] exec(CustomOp op, OpContext context){ @Deprecated public static void scatterUpdate(ScatterUpdate.UpdateOp op, @NonNull INDArray array, @NonNull INDArray indices, @NonNull INDArray updates, int... axis) { Preconditions.checkArgument(indices.dataType() == DataType.INT || indices.dataType() == DataType.LONG, - "Indices should have INT data type"); + "Indices should have INT data type"); Preconditions.checkArgument(array.dataType() == updates.dataType(), "Array and updates should have the same data type"); getExecutioner().scatterUpdate(op, array, indices, updates, axis); } diff --git a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/serde/jackson/shaded/NDArrayTextSerializer.java b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/serde/jackson/shaded/NDArrayTextSerializer.java index c395959d3cba..d7339fef31dc 100644 --- a/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/serde/jackson/shaded/NDArrayTextSerializer.java +++ b/nd4j/nd4j-backends/nd4j-api-parent/nd4j-api/src/main/java/org/nd4j/serde/jackson/shaded/NDArrayTextSerializer.java @@ -19,8 +19,10 @@ import lombok.val; +import org.nd4j.linalg.api.buffer.DataBuffer; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.api.shape.Shape; +import org.nd4j.serde.base64.Nd4jBase64; import org.nd4j.shade.jackson.core.JsonGenerator; import org.nd4j.shade.jackson.databind.JsonSerializer; import org.nd4j.shade.jackson.databind.SerializerProvider; @@ -77,9 +79,10 @@ public void serialize(INDArray arr, JsonGenerator jg, SerializerProvider seriali jg.writeNumber(v); break; case UTF8: - val n = arr.length(); - for( int j=0; j references = new ArrayList<>(); - @Getter - protected long numWords = 0; - /** * Meant for creating another view of a buffer * @@ -67,12 +64,12 @@ public CudaUtf8Buffer(long length) { public CudaUtf8Buffer(long length, boolean initialize) { super((length + 1) * 8, 1, initialize); - numWords = length; + this.length = length; } public CudaUtf8Buffer(long length, boolean initialize, MemoryWorkspace workspace) { super((length + 1) * 8, 1, initialize, workspace); - numWords = length; + this.length = length; } public CudaUtf8Buffer(int[] ints, boolean copy, MemoryWorkspace workspace) { @@ -83,10 +80,11 @@ public CudaUtf8Buffer(byte[] data, long numWords) { super(data.length, 1, false); lazyAllocateHostPointer(); - - val bp = (BytePointer) pointer; + ptrDataBuffer.syncToPrimary(); + val bp = new BytePointer(ptrDataBuffer.primaryBuffer()); bp.put(data); - this.numWords = numWords; + ptrDataBuffer.tickHostWrite(); + this.length = numWords; } public CudaUtf8Buffer(double[] data, boolean copy) { @@ -127,9 +125,9 @@ public CudaUtf8Buffer(int length, int elementSize, long offset) { public CudaUtf8Buffer(DataBuffer underlyingBuffer, long length, long offset) { super(underlyingBuffer, length, offset); - this.numWords = length; + this.length = length; - Preconditions.checkArgument(((CudaUtf8Buffer) underlyingBuffer).numWords == numWords, "String array can't be a view"); + Preconditions.checkArgument(((CudaUtf8Buffer) underlyingBuffer).length == length, "String array can't be a view"); } public CudaUtf8Buffer(@NonNull Collection strings) { @@ -141,7 +139,7 @@ public CudaUtf8Buffer(@NonNull Collection strings) { val headerPointer = new LongPointer(this.pointer); val dataPointer = new BytePointer(this.pointer); - numWords = strings.size(); + length = strings.size(); long cnt = 0; long currentLength = 0; @@ -163,12 +161,25 @@ public CudaUtf8Buffer(@NonNull Collection strings) { allocationPoint.tickHostWrite(); } + @Override + public long byteLength() { + this.ptrDataBuffer.syncToPrimary(); + val headerPointer = new LongPointer(this.ptrDataBuffer.primaryBuffer()); + val headerLen = length(); + + // buffer byteLen is a sum of header (which is long) and data (which is byte) + val bytesLast = headerPointer.get(headerLen) + (headerLen + 1 ) * 8; + return bytesLast; + } + public String getString(long index) { - if (index > numWords) - throw new IllegalArgumentException("Requested index [" + index + "] is above actual number of words stored: [" + numWords + "]"); + if (index > length()) + throw new IllegalArgumentException("Requested index [" + index + "] is above actual number of words stored: [" + length() + "]"); - val headerPointer = new LongPointer(this.pointer); - val dataPointer = (BytePointer) (this.pointer); + this.ptrDataBuffer.syncToPrimary(); + val _pointer = this.ptrDataBuffer.primaryBuffer(); + val headerPointer = new LongPointer(_pointer); + val dataPointer = new BytePointer(_pointer); val start = headerPointer.get(index); val end = headerPointer.get(index+1); @@ -179,7 +190,7 @@ public String getString(long index) { val dataLength = (int) (end - start); val bytes = new byte[dataLength]; - val headerLength = (numWords + 1) * 8; + val headerLength = (length() + 1) * 8; for (int e = 0; e < dataLength; e++) { val idx = headerLength + start + e; @@ -223,6 +234,16 @@ private static long stringBufferRequiredLength(@NonNull Collection strin return size; } + @Override + public String[] asUtf8() { + val result = new String[(int) length()]; + + for (int e = 0; e < length; e++) + result[e] = getString(e); + + return result; + } + public void put(long index, Pointer pointer) { throw new UnsupportedOperationException(); //references.add(pointer); @@ -238,5 +259,8 @@ protected void initTypeAndSize() { type = DataType.UTF8; } - + @Override + public String getUtf8(long i) { + return getString(i); + } } diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/factory/CudaDataBufferFactory.java b/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/factory/CudaDataBufferFactory.java index 5083a2bf91d1..2d3ae93a7b6d 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/factory/CudaDataBufferFactory.java +++ b/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/factory/CudaDataBufferFactory.java @@ -24,6 +24,7 @@ import org.bytedeco.javacpp.indexer.*; import org.nd4j.linalg.api.buffer.DataBuffer; import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.buffer.*; import org.nd4j.linalg.api.buffer.factory.DataBufferFactory; import org.nd4j.linalg.api.memory.MemoryWorkspace; import org.nd4j.linalg.exception.ND4JIllegalStateException; @@ -31,6 +32,7 @@ import org.nd4j.linalg.util.ArrayUtil; import java.nio.ByteBuffer; +import java.util.Arrays; /** * Creates cuda buffers @@ -132,6 +134,89 @@ public DataBuffer create(DataBuffer underlyingBuffer, long offset, long length) } } + @Override + public DataBuffer createBufferOfType(DataType dataType, long length) { + switch(dataType) { + case FLOAT: + return new CudaFloatDataBuffer(length); + case HALF: + return new CudaHalfDataBuffer(length); + case UINT64: + return new CudaUInt64DataBuffer(length); + case INT: + return new CudaIntDataBuffer(length); + case UINT16: + return new CudaUInt16DataBuffer(length); + case BFLOAT16: + return new CudaBfloat16DataBuffer(length); + case UTF8: + return new CudaUtf8Buffer(length); + case DOUBLE: + return new CudaDoubleDataBuffer(length); + case LONG: + return new CudaLongDataBuffer(length); + case BOOL: + return new CudaBoolDataBuffer(length); + case UINT32: + return new CudaUInt32DataBuffer(length); + case UBYTE: + return new CudaUByteDataBuffer(length); + case BYTE: + return new CudaByteDataBuffer(length); + case SHORT: + return new CudaShortDataBuffer(length); + case COMPRESSED: + case UNKNOWN: + default: + throw new IllegalArgumentException("Illegal type " + dataType); + } + } + + @Override + public DataBuffer createBufferOfType(DataType dataType, Object input) { + switch(dataType) { + case FLOAT: + float[] inputFloatArr = (float[]) input; + return new CudaFloatDataBuffer(inputFloatArr); + case INT: + int[] inputIntArr = (int[]) input; + return new CudaIntDataBuffer(inputIntArr); + case UTF8: + String[] inputStringArr = (String[]) input; + return new CudaUtf8Buffer(Arrays.asList(inputStringArr)); + case DOUBLE: + double[] inputDoubleArr = (double[]) input; + return new CudaDoubleDataBuffer(inputDoubleArr); + case LONG: + long[] inputLongArr = (long[]) input; + return new CudaLongDataBuffer(inputLongArr); + case BOOL: + boolean[] inputBooleanArr = (boolean[]) input; + CudaBoolDataBuffer retBuffer = new CudaBoolDataBuffer(inputBooleanArr.length); + for(int i = 0; i < inputBooleanArr.length; i++) { + retBuffer.put(i,inputBooleanArr[i]); + } + return retBuffer; + case SHORT: + short[] inputShortArr = (short[]) input; + CudaShortDataBuffer retShortBuffer = new CudaShortDataBuffer(inputShortArr.length); + for(int i = 0; i < inputShortArr.length; i++) { + retShortBuffer.putByDestinationType(i,inputShortArr[i],DataType.SHORT); + } + return retShortBuffer; + case UBYTE: + case COMPRESSED: + case UINT32: + case UNKNOWN: + case HALF: + case UINT64: + case UINT16: + case BFLOAT16: + default: + throw new IllegalArgumentException("Illegal data type " + dataType); + + } + } /** * This method will create new DataBuffer of the same dataType & same length * diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/NDArray.java b/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/NDArray.java index 5d8af851a249..beaabe601b6a 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/NDArray.java +++ b/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/NDArray.java @@ -511,9 +511,10 @@ protected int stringBuffer(FlatBufferBuilder builder, DataBuffer buffer) { // writing length first val t = length(); val ptr = (BytePointer) ub.pointer(); + val ub_len = ub.byteLength(); // now write all strings as bytes - for (int i = 0; i < ub.length(); i++) { + for (int i = 0; i < ub_len; i++) { dos.writeByte(ptr.get(i)); } diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/BaseCpuDataBuffer.java b/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/BaseCpuDataBuffer.java index 7abc8e7be860..598ca7219e79 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/BaseCpuDataBuffer.java +++ b/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/BaseCpuDataBuffer.java @@ -49,6 +49,35 @@ protected BaseCpuDataBuffer() { } + @Override + public boolean[] asBoolean() { + return new boolean[0]; + } + + @Override + public boolean getBool(long i) { + return false; + } + + @Override + public String getUtf8(long i) { + return null; + } + + @Override + public String[] asUtf8() { + throw new UnsupportedOperationException(); + } + + @Override + public void put(long i, String element) { + throw new UnsupportedOperationException(); + } + + @Override + public long byteLength() { + return length * dataType().width(); + } @Override public String getUniqueId() { @@ -424,6 +453,27 @@ public void actualizePointerAndIndexer() { throw new IllegalArgumentException("Unknown datatype: " + dataType()); } + /** + * Returns the offsets for each element + * in the buffer. + * This is only used in variable length + * binary buffers. + * @return + */ + @Override + public DataBuffer binaryOffsets() { + val headerPointer = new LongPointer(this.pointer); + val offsetBuffer = Nd4j.createBufferOfType(DataType.INT32,length() + 1); + long stringByteLength = 0; + for(int i = 0; i < length(); i++) { + offsetBuffer.put(i,headerPointer.get(i)); + stringByteLength += getUtf8(i).length(); + } + + offsetBuffer.put(length(),stringByteLength); + return offsetBuffer; + } + @Override public Pointer addressPointer() { // we're fetching actual pointer right from C++ diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/DefaultDataBufferFactory.java b/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/DefaultDataBufferFactory.java index 54b02e309e20..b151118c9153 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/DefaultDataBufferFactory.java +++ b/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/DefaultDataBufferFactory.java @@ -31,6 +31,7 @@ import org.nd4j.linalg.util.ArrayUtil; import java.nio.ByteBuffer; +import java.util.Arrays; /** * Normal data buffer creation @@ -89,12 +90,52 @@ public DataBuffer create(DataBuffer underlyingBuffer, long offset, long length) } else if (underlyingBuffer.dataType() == DataType.HALF) { return new HalfBuffer(underlyingBuffer, length, offset); } else if (underlyingBuffer.dataType() == DataType.UTF8) { + Utf8Buffer utf8Buffer = (Utf8Buffer) underlyingBuffer; return new Utf8Buffer(underlyingBuffer, length, offset); } return null; } + + + @Override + public DataBuffer createBufferOfType(DataType dataType, Object input) { + switch(dataType) { + case FLOAT: + float[] inputFloatArr = (float[]) input; + return new FloatBuffer(inputFloatArr); + case INT: + int[] inputIntArr = (int[]) input; + return new IntBuffer(inputIntArr); + case UTF8: + String[] inputStringArr = (String[]) input; + return new Utf8Buffer(Arrays.asList(inputStringArr)); + case DOUBLE: + double[] inputDoubleArr = (double[]) input; + return new DoubleBuffer(inputDoubleArr); + case LONG: + long[] inputLongArr = (long[]) input; + return new LongBuffer(inputLongArr); + case BOOL: + boolean[] inputBooleanArr = (boolean[]) input; + BoolBuffer retBuffer = new BoolBuffer(inputBooleanArr.length); + for(int i = 0; i < inputBooleanArr.length; i++) { + retBuffer.put(i,inputBooleanArr[i]); + } + return retBuffer; + case COMPRESSED: + case UINT32: + case UNKNOWN: + case UINT64: + case UINT16: + case BFLOAT16: + default: + throw new IllegalArgumentException("Illegal data type " + dataType); + + } + } + @Override public DataBuffer createDouble(long offset, int length) { return new DoubleBuffer(length, 8, offset); @@ -110,6 +151,45 @@ public DataBuffer createInt(long offset, int length) { return new IntBuffer(length, 4, offset); } + @Override + public DataBuffer createBufferOfType(DataType dataType, long length) { + switch(dataType) { + case FLOAT: + return new FloatBuffer(length); + case HALF: + return new HalfBuffer(length); + case UINT64: + return new UInt64Buffer(length); + case INT: + return new IntBuffer(length); + case UINT16: + return new UInt16Buffer(length); + case BFLOAT16: + return new BFloat16Buffer(length); + case UTF8: + return new Utf8Buffer(length); + case DOUBLE: + return new DoubleBuffer(length); + case LONG: + return new LongBuffer(length); + case BOOL: + return new BoolBuffer(length); + case UINT32: + return new UInt32Buffer(length); + case UBYTE: + return new UInt8Buffer(length); + case BYTE: + return new Int8Buffer(length); + case SHORT: + return new Int16Buffer(length); + case COMPRESSED: + case UNKNOWN: + default: + throw new IllegalArgumentException("Illegal type " + dataType); + } + } + + @Override public DataBuffer createDouble(long offset, int[] data) { return createDouble(offset, data, true); @@ -731,8 +811,11 @@ public DataBuffer create(Pointer pointer, DataType type, long length, @NonNull I return new FloatBuffer(pointer, indexer, length); case DOUBLE: return new DoubleBuffer(pointer, indexer, length); + case UTF8: + return new Utf8Buffer(pointer,indexer,length); + } - throw new IllegalArgumentException("Invalid opType " + type); + throw new IllegalArgumentException("Invalid data type for creation " + type); } @Override diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/Utf8Buffer.java b/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/Utf8Buffer.java index bc5758b1dce7..b80e00a3bd96 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/Utf8Buffer.java +++ b/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/Utf8Buffer.java @@ -17,7 +17,6 @@ package org.nd4j.linalg.cpu.nativecpu.buffer; -import lombok.Getter; import lombok.NonNull; import lombok.val; import org.bytedeco.javacpp.BytePointer; @@ -42,9 +41,6 @@ public class Utf8Buffer extends BaseCpuDataBuffer { protected Collection references = new ArrayList<>(); - @Getter - protected long numWords = 0; - /** * Meant for creating another view of a buffer * @@ -65,7 +61,7 @@ public Utf8Buffer(long length, boolean initialize) { * Special case: we're creating empty buffer for length strings, each of 0 chars */ super((length + 1) * 8, true); - numWords = length; + this.length = length; } public Utf8Buffer(long length, boolean initialize, MemoryWorkspace workspace) { @@ -74,7 +70,7 @@ public Utf8Buffer(long length, boolean initialize, MemoryWorkspace workspace) { */ super((length + 1) * 8, true, workspace); - numWords = length; + this.length = length; } public Utf8Buffer(ByteBuffer buffer, DataType dataType, long length, long offset) { @@ -90,7 +86,7 @@ public Utf8Buffer(byte[] data, long numWords) { val bp = (BytePointer) pointer; bp.put(data); - this.numWords = numWords; + this.length = numWords; } public Utf8Buffer(double[] data, boolean copy) { @@ -131,7 +127,7 @@ public Utf8Buffer(int length, int elementSize, long offset) { public Utf8Buffer(DataBuffer underlyingBuffer, long length, long offset) { super(underlyingBuffer, length, offset); - this.numWords = length; + this.length = length; } public Utf8Buffer(@NonNull Collection strings) { @@ -142,7 +138,7 @@ public Utf8Buffer(@NonNull Collection strings) { val headerPointer = new LongPointer(this.pointer); val dataPointer = new BytePointer(this.pointer); - numWords = strings.size(); + length = strings.size(); long cnt = 0; long currentLength = 0; @@ -163,23 +159,49 @@ public Utf8Buffer(@NonNull Collection strings) { headerPointer.put(cnt, currentLength); } + @Override + public String getUtf8(long i) { + return getString(i); + } + + @Override + public long byteLength() { + val headerPointer = new LongPointer(this.ptrDataBuffer.primaryBuffer()); + val headerLen = length(); + + // buffer byteLen is a sum of header (which is long) and data (which is byte) + val bytesLast = headerPointer.get(headerLen) + (headerLen + 1 ) * 8; + return bytesLast; + } + + @Override + public String[] asUtf8() { + val result = new String[(int) length()]; + + for (int e = 0; e < length; e++) + result[e] = getString(e); + + return result; + } + public String getString(long index) { - if (index > numWords) - throw new IllegalArgumentException("Requested index [" + index + "] is above actual number of words stored: [" + numWords + "]"); + if (index > length()) + throw new IllegalArgumentException("Requested index [" + index + "] is above actual number of words stored: [" + length() + "]"); - val headerPointer = new LongPointer(this.pointer); - val dataPointer = (BytePointer) (this.pointer); + val _pointer = this.ptrDataBuffer.primaryBuffer(); + val headerPointer = new LongPointer(_pointer); + val dataPointer = new BytePointer(_pointer); val start = headerPointer.get(index); - val end = headerPointer.get(index+1); + val end = headerPointer.get(index + 1); if (end - start > Integer.MAX_VALUE) - throw new IllegalStateException("Array is too long for Java"); + throw new IllegalStateException("Array is too long for GraphRunnerJava"); val dataLength = (int) (end - start); val bytes = new byte[dataLength]; - val headerLength = (numWords + 1) * 8; + val headerLength = (length() + 1) * 8; for (int e = 0; e < dataLength; e++) { val idx = headerLength + start + e; diff --git a/nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/imports/ByteOrderTests.java b/nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/imports/ByteOrderTests.java index 5ac5c9410a29..0924d0b8a1a0 100644 --- a/nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/imports/ByteOrderTests.java +++ b/nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/imports/ByteOrderTests.java @@ -20,6 +20,7 @@ import lombok.extern.slf4j.Slf4j; import lombok.val; import org.junit.After; +import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -165,6 +166,7 @@ public void testVectorEncoding_2() { } @Test + @Ignore public void testStringEncoding_1() { val strings = Arrays.asList("alpha", "beta", "gamma"); val vector = Nd4j.create(strings, 3); diff --git a/nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/linalg/DataTypeTest.java b/nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/linalg/DataTypeTest.java index d3c033abe981..c89f4decd478 100644 --- a/nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/linalg/DataTypeTest.java +++ b/nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/linalg/DataTypeTest.java @@ -40,7 +40,7 @@ public DataTypeTest(Nd4jBackend backend) { @Test public void testDataTypes() throws Exception { for (val type : DataType.values()) { - if (DataType.UTF8.equals(type) || DataType.UNKNOWN.equals(type) || DataType.COMPRESSED.equals(type)) + if (type.isStringType() || DataType.UNKNOWN.equals(type) || DataType.COMPRESSED.equals(type)) continue; val in1 = Nd4j.ones(type, 10, 10); diff --git a/nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/linalg/Nd4jTestsC.java b/nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/linalg/Nd4jTestsC.java index da91fb6cf44a..fa51eb8702b8 100644 --- a/nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/linalg/Nd4jTestsC.java +++ b/nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/linalg/Nd4jTestsC.java @@ -7460,11 +7460,11 @@ public void testAssignInvalid(){ @Test public void testEmptyCasting(){ for(val from : DataType.values()) { - if (from == DataType.UTF8 || from == DataType.UNKNOWN || from == DataType.COMPRESSED) + if (from.isStringType() || from == DataType.UNKNOWN || from == DataType.COMPRESSED) continue; for(val to : DataType.values()){ - if (to == DataType.UTF8 || to == DataType.UNKNOWN || to == DataType.COMPRESSED) + if (to.isStringType() || to == DataType.UNKNOWN || to == DataType.COMPRESSED) continue; INDArray emptyFrom = Nd4j.empty(from); diff --git a/nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/linalg/api/buffer/DataBufferTests.java b/nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/linalg/api/buffer/DataBufferTests.java index b7660bc6e0e2..e430ad0cf17b 100644 --- a/nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/linalg/api/buffer/DataBufferTests.java +++ b/nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/linalg/api/buffer/DataBufferTests.java @@ -284,7 +284,7 @@ public void testCreateTypedBuffer() { for (String sourceType : new String[]{"int", "long", "float", "double", "short", "byte", "boolean"}) { for (DataType dt : DataType.values()) { - if (dt == DataType.UTF8 || dt == DataType.COMPRESSED || dt == DataType.UNKNOWN) { + if (dt.isStringType() || dt == DataType.COMPRESSED || dt == DataType.UNKNOWN) { continue; } diff --git a/nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/linalg/options/ArrayOptionsTests.java b/nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/linalg/options/ArrayOptionsTests.java index f4281aadca2d..5539349c0187 100644 --- a/nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/linalg/options/ArrayOptionsTests.java +++ b/nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/linalg/options/ArrayOptionsTests.java @@ -75,7 +75,7 @@ public void testArrayType_3() { public void testDataTypesToFromLong(){ for(DataType dt : DataType.values()){ - if(dt == DataType.UNKNOWN) + if(dt == DataType.UNKNOWN || dt.isStringType()) continue; String s = dt.toString(); long l = 0; diff --git a/nd4j/nd4j-serde/nd4j-arrow/pom.xml b/nd4j/nd4j-serde/nd4j-arrow/pom.xml index 3a768c1a5679..c80d9f3262dc 100644 --- a/nd4j/nd4j-serde/nd4j-arrow/pom.xml +++ b/nd4j/nd4j-serde/nd4j-arrow/pom.xml @@ -29,6 +29,11 @@ nd4j-arrow + + org.bytedeco + arrow-platform + ${arrow.javacpp.version} + junit junit @@ -80,6 +85,11 @@ nd4j-native ${project.version} + + org.nd4j + nd4j-common-tests + ${project.version} + @@ -127,6 +137,11 @@ nd4j-cuda-10.2 ${project.version} + + org.nd4j + nd4j-common-tests + ${project.version} + diff --git a/nd4j/nd4j-serde/nd4j-arrow/src/main/java/org/nd4j/arrow/ArrowSerde.java b/nd4j/nd4j-serde/nd4j-arrow/src/main/java/org/nd4j/arrow/ArrowSerde.java index 2c05ab918d0f..3f6fa88265bc 100644 --- a/nd4j/nd4j-serde/nd4j-arrow/src/main/java/org/nd4j/arrow/ArrowSerde.java +++ b/nd4j/nd4j-serde/nd4j-arrow/src/main/java/org/nd4j/arrow/ArrowSerde.java @@ -18,6 +18,8 @@ import com.google.flatbuffers.FlatBufferBuilder; import org.apache.arrow.flatbuf.*; +import org.apache.arrow.flatbuf.Tensor; +import org.apache.arrow.flatbuf.Type; import org.nd4j.linalg.api.buffer.DataBuffer; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.ndarray.INDArray; @@ -25,6 +27,8 @@ import org.nd4j.linalg.factory.Nd4j; import org.nd4j.linalg.util.ArrayUtil; + + /** * Conversion to and from arrow {@link Tensor} * and {@link INDArray} @@ -113,7 +117,7 @@ public static int addDataForArr(FlatBufferBuilder bufferBuilder, INDArray arr) { * Convert the given {@link INDArray} * data type to the proper data type for the tensor. * @param bufferBuilder the buffer builder in use - * @param arr the array to conver tthe data type for + * @param arr the array to convert the data type for */ public static void addTypeTypeRelativeToNDArray(FlatBufferBuilder bufferBuilder,INDArray arr) { switch(arr.data().dataType()) { @@ -122,11 +126,21 @@ public static void addTypeTypeRelativeToNDArray(FlatBufferBuilder bufferBuilder, Tensor.addTypeType(bufferBuilder,Type.Int); break; case FLOAT: - Tensor.addTypeType(bufferBuilder,Type.FloatingPoint); + Tensor.addTypeType(bufferBuilder, Type.FloatingPoint); break; case DOUBLE: Tensor.addTypeType(bufferBuilder,Type.Decimal); break; + case HALF: + Tensor.addTypeType(bufferBuilder,Type.FloatingPoint); + break; + case BOOL: + Tensor.addTypeType(bufferBuilder,Type.Bool); + break; + case UTF8: + Tensor.addTypeType(bufferBuilder,Type.Utf8); + break; + } } @@ -167,7 +181,7 @@ public static long[] getArrowStrides(INDArray arr) { /** - * Create thee databuffer type frm the given type, + * Create thee {@link DataType} type frm the given type, * relative to the bytes in arrow in class: * {@link Type} * @param type the type to create the nd4j {@link DataType} from @@ -188,6 +202,10 @@ else if(type == Type.Int) { else if(elementSize == 8) { return DataType.LONG; } + + } + else if(type == Type.Utf8) { + return DataType.UTF8; } else { throw new IllegalArgumentException("Only valid types are Type.Decimal and Type.Int"); diff --git a/nd4j/nd4j-serde/nd4j-arrow/src/main/java/org/nd4j/arrow/ByteDecoArrowSerde.java b/nd4j/nd4j-serde/nd4j-arrow/src/main/java/org/nd4j/arrow/ByteDecoArrowSerde.java new file mode 100644 index 000000000000..34aabfac659e --- /dev/null +++ b/nd4j/nd4j-serde/nd4j-arrow/src/main/java/org/nd4j/arrow/ByteDecoArrowSerde.java @@ -0,0 +1,384 @@ +/******************************************************************************* + * Copyright (c) 2019 Konduit KK + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.nd4j.arrow; +import org.bytedeco.arrow.global.arrow; +import org.bytedeco.arrow.*; +import org.bytedeco.javacpp.BytePointer; +import org.nd4j.base.Preconditions; +import org.nd4j.linalg.api.buffer.DataBuffer; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.primitives.Pair; +import org.nd4j.linalg.util.ArrayUtil; + + +/** + * Arrow serialization utilities + * using the javacpp arrow bindings. + * + * @author Adam Gibson + */ +public class ByteDecoArrowSerde { + + /** + * Convert a {@link Tensor} + * to an {@link INDArray} + * @param tensor the input tensor + * @return the equivalent {@link INDArray} + */ + public static INDArray fromTensor(Tensor tensor) { + long[] shape = new long[tensor.ndim()]; + long[] stride = new long[tensor.ndim()]; + + long bufferCapacity = 1; + for(int i = 0; i < tensor.ndim(); i++) { + shape[i] = tensor.shape().get(i); + stride[i] = tensor.strides().get(i); + bufferCapacity *= shape[i]; + } + + + org.nd4j.linalg.api.buffer.DataType dtype = dataBufferTypeTypeForArrow(tensor.type()); + //buffer capacity needs to be initialized properly, otherwise defaults to zero + ArrowBuffer arrowBuffer = tensor.data().capacity(bufferCapacity); + DataBuffer buffer = fromArrowBuffer(arrowBuffer,arrowDataTypeForNd4j(dtype)); + Preconditions.checkState(buffer.length() == ArrayUtil.prod(shape),"Data buffer creation from arrow failed. Data buffer is empty and not the same length as the shape."); + INDArray arr = Nd4j.create(buffer,shape,stride,0); + return arr; + } + + /** + * + * @param input + * @return + */ + public static Tensor toTensor(INDArray input) { + if(input.dataType() == org.nd4j.linalg.api.buffer.DataType.BOOL) + throw new IllegalArgumentException("Arrow does not currently support converting boolean arrays to tensors."); + ArrowBuffer arrowBuffer = fromNd4jBuffer(input.data()).getFirst(); + long[] shape = input.shape(); + long[] stride = input.stride(); + if(shape.length == 0) { + shape = new long[] {1}; + stride = new long[] {1}; + } + + Tensor ret = new Tensor(arrowDataTypeForNd4j(input.dataType()),arrowBuffer,shape,stride); + ret.data().capacity(arrowBuffer.capacity()); + ret.data().limit(arrowBuffer.limit()); + return ret; + } + + + + /** + * Convert a {@link org.nd4j.linalg.api.buffer.DataType} + * to an arrow {@link DataType} + * @param dataType the input data type + * @return the equivalent arrow data type + */ + public static DataType arrowDataTypeForNd4j(org.nd4j.linalg.api.buffer.DataType dataType) { + switch(dataType) { + case UINT64: + return arrow.uint64(); + case COMPRESSED: + throw new IllegalArgumentException("Unable to convert data type " + dataType.name()); + case UINT16: + return arrow.uint16(); + case UBYTE: + return arrow.uint8(); + case SHORT: + return arrow.int16(); + case BYTE: + return arrow.int8(); + case FLOAT: + return arrow.float32(); + case LONG: + return arrow.int64(); + case BOOL: + return arrow._boolean(); + case UTF8: + return arrow.utf8(); + case INT: + return arrow.int32(); + case HALF: + return arrow.float16(); + case DOUBLE: + return arrow.float64(); + case UNKNOWN: + throw new IllegalArgumentException("Unable to convert data type " + dataType.name()); + case BFLOAT16: + return arrow.float16(); + case UINT32: + return arrow.uint32(); + default: + throw new IllegalArgumentException("Unable to convert data type " + dataType.name()); + } + + } + + /** + * Convert the input {@link DataType} + * to the nd4j equivalent of {@link org.nd4j.linalg.api.buffer.DataType} + * @param dataType the input data type + * @return the equivalent nd4j data type + */ + public static org.nd4j.linalg.api.buffer.DataType dataBufferTypeTypeForArrow(DataType dataType) { + if(dataType.equals(arrow._boolean())) { + return org.nd4j.linalg.api.buffer.DataType.BOOL; + } + else if(dataType.equals(arrow.uint8())) { + return org.nd4j.linalg.api.buffer.DataType.UBYTE; + } + else if(dataType.equals(arrow.uint16())) { + return org.nd4j.linalg.api.buffer.DataType.UINT16; + } + else if(dataType.equals(arrow.uint32())) { + return org.nd4j.linalg.api.buffer.DataType.UINT32; + } + else if(dataType.equals(arrow.uint64())) { + return org.nd4j.linalg.api.buffer.DataType.UINT64; + + } + else if(dataType.equals(arrow.int8())) { + return org.nd4j.linalg.api.buffer.DataType.BYTE; + } + else if(dataType.equals(arrow.int16())) { + return org.nd4j.linalg.api.buffer.DataType.SHORT; + } + else if(dataType.equals(arrow.int32())) { + return org.nd4j.linalg.api.buffer.DataType.INT; + } + else if(dataType.equals(arrow.int64())) { + return org.nd4j.linalg.api.buffer.DataType.LONG; + } + else if(dataType.equals(arrow.float16())) { + return org.nd4j.linalg.api.buffer.DataType.HALF; + } + else if(dataType.equals(arrow.float32())) { + return org.nd4j.linalg.api.buffer.DataType.FLOAT; + } + else if(dataType.equals(arrow.float64())) { + return org.nd4j.linalg.api.buffer.DataType.DOUBLE; + } + else if(dataType.equals(arrow.date32())) { + throw new IllegalArgumentException("Unable to convert type " + dataType.name()); + } + else if(dataType.equals(arrow.date64())) { + throw new IllegalArgumentException("Unable to convert type " + dataType.name()); + } + else if(dataType.equals(arrow.day_time_interval())) { + throw new IllegalArgumentException("Unable to convert type " + dataType.name()); + + } + else if(dataType.equals(arrow.large_utf8())) { + return org.nd4j.linalg.api.buffer.DataType.UTF8; + } + else if(dataType.equals(arrow.utf8())) { + return org.nd4j.linalg.api.buffer.DataType.UTF8; + } + else if(dataType.equals(arrow.binary())) { + return org.nd4j.linalg.api.buffer.DataType.BYTE; + } + else { + throw new IllegalArgumentException("Unable to convert type " + dataType.name()); + } + } + + /** + * + * @param arrowBuffer + * @param dataType + * @return + */ + public static DataBuffer fromArrowBuffer(ArrowBuffer arrowBuffer,DataType dataType) { + org.nd4j.linalg.api.buffer.DataType dataType1 = dataBufferTypeTypeForArrow(dataType); + if(dataType1 != org.nd4j.linalg.api.buffer.DataType.UTF8) { + BytePointer bytePointer = arrowBuffer.data().capacity(arrowBuffer.size() * dataType1.width()); + return Nd4j.createBuffer(bytePointer,arrowBuffer.size(),dataBufferTypeTypeForArrow(dataType)); + + } + else { + BytePointer bytePointer = arrowBuffer.data(); + return Nd4j.createBuffer(bytePointer,arrowBuffer.size(),dataBufferTypeTypeForArrow(dataType)); + + } + } + + /** + * Create a {@link Pair} + * of {@link ArrowBuffer} and {@link org.nd4j.linalg.api.buffer.DataType} + * based on the input {@link DataBuffer} + * @param dataBuffer the input data buffer + * @return the pair + */ + public static Pair fromNd4jBuffer(DataBuffer dataBuffer) { + BytePointer bytePointer = new BytePointer(dataBuffer.pointer()); + ArrowBuffer arrowBuffer = new ArrowBuffer(bytePointer,dataBuffer.length()); + return Pair.of(arrowBuffer,arrowDataTypeForNd4j(dataBuffer.dataType())); + } + + + /** + * Creates an {@link INDArray} from an arrow {@link Array} + * @param array the input {@link Array} + * @return the equivalent {@link INDArray} zero copied + */ + public static INDArray ndarrayFromArrowArray(FlatArray array) { + if(array instanceof PrimitiveArray) { + PrimitiveArray primitiveArray = (PrimitiveArray) array; + ArrowBuffer arrowBuffer = primitiveArray.values(); + DataBuffer nd4jBuffer = fromArrowBuffer(arrowBuffer,array.data().type()); + return Nd4j.create(nd4jBuffer,1,nd4jBuffer.length()); + } + else { + StringArray stringArray = (StringArray) array; + ArrowBuffer arrowBuffer = stringArray.value_data(); + DataBuffer nd4jBuffer = fromArrowBuffer(arrowBuffer,array.data().type()); + return Nd4j.create(nd4jBuffer,1,nd4jBuffer.length()); + } + + } + + /** + * Create an {@link Array} + * from the given {@link INDArray} + * with zero copy + * @param input the input {@link INDArray} + * @return the equivalent wrapped {@link Array} + * for the given input {@link INDArray} + */ + public static FlatArray arrayFromExistingINDArray(INDArray input) { + Pair fromNd4jBuffer = fromNd4jBuffer(input.data()); + ArrowBuffer arrowBuffer = fromNd4jBuffer.getFirst(); + return createArrayFromArrayData(arrowBuffer,input.dataType()); + } + + + /** + * Create an {@link ArrowBuffer} and {@link org.nd4j.linalg.api.buffer.DataType} + * pair for the offsets of a string/utf8 buffer. The databuffer + * will come from {@link Utf8Buffer#binaryOffsets()} + * @param stringBuffer the input data buffer where offsets are. The + * input data buffer must be a Utf8 buffer + * @return the arrow buffer for the offsets accompanied by the data type + */ + public static Pair arrowBufferForStringOffsets(DataBuffer stringBuffer) { + Preconditions.checkState(stringBuffer.dataType() == org.nd4j.linalg.api.buffer.DataType.UTF8,"Passed in data buffer has to be a utf8 buffer."); + DataBuffer offsets = stringBuffer.binaryOffsets(); + return fromNd4jBuffer(offsets); + } + + /** + * Create an {@link Array} + * with the passed in {@link ArrayData} + * + * @param numElements the number of elements in the array + * @param arrowBuffer the array data to create the {@link Array} from + * @param offsets the offsets for each string + * @param dataType the {@link DataType} for the array + * @return the created {@link Array} + */ + public static FlatArray createArrayFromArrayData(long numElements, ArrowBuffer arrowBuffer, ArrowBuffer offsets, org.nd4j.linalg.api.buffer.DataType dataType) { + switch (dataType) { + case UTF8: + //note the size - 1 here is due to appending the final boundary + ArrowBuffer nullVectorBitMap = new ArrowBuffer(new byte[(int) numElements],numElements); + nullVectorBitMap.fill(1); + return new StringArray(numElements,offsets,arrowBuffer,nullVectorBitMap,0,0); + default: + throw new IllegalArgumentException("Illegal type for array creation. For other data types, please avoid specifying offsets." + dataType); + + } + } + + /** + * Create an {@link Array} + * with the passed in {@link ArrayData} + * @param arrowBuffer the array data to create the {@link Array} from + * @param dataType the {@link DataType} for the array + * @return the created {@link Array} + */ + public static FlatArray createArrayFromArrayData(ArrowBuffer arrowBuffer, org.nd4j.linalg.api.buffer.DataType dataType) { + ArrayData arrayData = arrayDataFromArrowBuffer(arrowBuffer,arrowDataTypeForNd4j(dataType), true); + FlatArray flatArray = null; + switch (dataType) { + case DOUBLE: + flatArray = new DoubleArray(arrayData); + break; + case BOOL: + flatArray = new BooleanArray(arrayData); + break; + case FLOAT: + flatArray = new FloatArray(arrayData); + break; + case INT: + flatArray = new Int32Array(arrayData); + break; + case UTF8: + throw new UnsupportedOperationException("Please use createArrayFromArrayData that forces specifications of offsets."); + case LONG: + flatArray = new Int64Array(arrayData); + break; + case UINT32: + flatArray = new UInt32Array(arrayData); + break; + case HALF: + flatArray = new HalfFloatArray(arrayData); + break; + case UINT64: + flatArray = new UInt64Array(arrayData); + break; + case BYTE: + flatArray = new BinaryArray(arrayData); + break; + case UINT16: + flatArray = new UInt16Array(arrayData); + break; + + + } + + return flatArray; + } + + + + /** + * Create array data for a given arrow buffer and data type + * @param arrowBuffer + * @param dataType + * @param nullBitMaskIncluded + * @return + */ + public static ArrayData arrayDataFromArrowBuffer(ArrowBuffer arrowBuffer, DataType dataType, boolean nullBitMaskIncluded) { + if(nullBitMaskIncluded) { + ArrowBuffer nullVectorBitMap = new ArrowBuffer(new byte[(int) arrowBuffer.size()],arrowBuffer.size()); + //all items are present + nullVectorBitMap.fill(1); + ArrowBufferVector arrowBufferVector = new ArrowBufferVector(nullVectorBitMap,arrowBuffer); + return ArrayData.Make(dataType,arrowBufferVector.size(),arrowBufferVector,0,0); + } + else { + ArrowBufferVector arrowBufferVector = new ArrowBufferVector(arrowBuffer); + return ArrayData.Make(dataType,arrowBufferVector.size(),arrowBufferVector,0,0); + } + + } + + +} diff --git a/nd4j/nd4j-serde/nd4j-arrow/src/main/java/org/nd4j/arrow/Nd4jArrowOpRunner.java b/nd4j/nd4j-serde/nd4j-arrow/src/main/java/org/nd4j/arrow/Nd4jArrowOpRunner.java new file mode 100644 index 000000000000..9cf60cd477c9 --- /dev/null +++ b/nd4j/nd4j-serde/nd4j-arrow/src/main/java/org/nd4j/arrow/Nd4jArrowOpRunner.java @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2019 Konduit KK + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.nd4j.arrow; + +import org.bytedeco.arrow.FlatArray; +import org.bytedeco.arrow.PrimitiveArray; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.api.ops.DynamicCustomOp; +import org.nd4j.linalg.api.ops.DynamicCustomOp.DynamicCustomOpsBuilder; +import org.nd4j.linalg.factory.Nd4j; + +import static org.nd4j.arrow.ByteDecoArrowSerde.ndarrayFromArrowArray; + +/** + * Runs {@link DynamicCustomOp} + * on arrow based data types. + * + * @author Adam Gibson + */ +public class Nd4jArrowOpRunner { + + /** + * Runs operations + * @param array the input {@link PrimitiveArray} + * @param opName the op name to run + * @param args the args (booleans, integers,..) + * @return the {@link PrimitiveArray} equivalents + * from the outputs from the execution of {@link DynamicCustomOp} + * derived from the input names. + */ + public static FlatArray[] runOpOn(FlatArray[] array, String opName, Object...args) { + DynamicCustomOpsBuilder opBuilder = DynamicCustomOp.builder(opName); + + if (args != null) + for(Object arg : args) { + if(arg instanceof Integer || arg instanceof Long) { + Number integer = (Number) arg; + opBuilder.addIntegerArguments(integer.longValue()); + } + else if(arg instanceof Float || arg instanceof Double) { + Number floatArg = (Number) arg; + opBuilder.addFloatingPointArguments(floatArg.doubleValue()); + } + else if(arg instanceof Boolean) { + Boolean boolArg = (Boolean) arg; + opBuilder.addBooleanArguments(boolArg); + } + } + + INDArray[] inputs = new INDArray[array.length]; + for(int i = 0; i < inputs.length; i++) { + inputs[i] = ndarrayFromArrowArray(array[i]); + } + + opBuilder.addInputs(inputs); + + DynamicCustomOp build = opBuilder.build(); + Nd4j.getExecutioner().exec(build); + INDArray[] ret = build.outputArguments().toArray(new INDArray[0]); + FlatArray[] outputArrays = new FlatArray[ret.length]; + for(int i = 0; i < ret.length; i++) { + outputArrays[i] = ByteDecoArrowSerde.arrayFromExistingINDArray(ret[i]); + } + + return outputArrays; + } + + + +} diff --git a/nd4j/nd4j-serde/nd4j-arrow/src/test/java/org/nd4j/arrow/ArrowSerdeTest.java b/nd4j/nd4j-serde/nd4j-arrow/src/test/java/org/nd4j/arrow/ArrowSerdeTest.java index eee1521bd5e1..6c3328ea87ff 100644 --- a/nd4j/nd4j-serde/nd4j-arrow/src/test/java/org/nd4j/arrow/ArrowSerdeTest.java +++ b/nd4j/nd4j-serde/nd4j-arrow/src/test/java/org/nd4j/arrow/ArrowSerdeTest.java @@ -18,13 +18,12 @@ import org.apache.arrow.flatbuf.Tensor; import org.junit.Test; -import org.nd4j.BaseND4JTest; import org.nd4j.linalg.api.ndarray.INDArray; import org.nd4j.linalg.factory.Nd4j; import static org.junit.Assert.assertEquals; -public class ArrowSerdeTest extends BaseND4JTest { +public class ArrowSerdeTest { @Test public void testBackAndForth() { diff --git a/nd4j/nd4j-serde/nd4j-arrow/src/test/java/org/nd4j/arrow/ByteDecoArrowSerdeTests.java b/nd4j/nd4j-serde/nd4j-arrow/src/test/java/org/nd4j/arrow/ByteDecoArrowSerdeTests.java new file mode 100644 index 000000000000..f6b87a678d52 --- /dev/null +++ b/nd4j/nd4j-serde/nd4j-arrow/src/test/java/org/nd4j/arrow/ByteDecoArrowSerdeTests.java @@ -0,0 +1,92 @@ +/******************************************************************************* + * Copyright (c) 2019 Konduit KK + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.nd4j.arrow; + +import org.bytedeco.arrow.ArrowBuffer; +import org.bytedeco.arrow.FlatArray; +import org.bytedeco.arrow.Tensor; +import org.junit.Test; +import org.nd4j.linalg.api.buffer.DataBuffer; +import org.nd4j.linalg.api.buffer.DataType; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; +import org.nd4j.linalg.primitives.Pair; + +import static org.junit.Assert.assertEquals; + +public class ByteDecoArrowSerdeTests { + + + @Test + public void testBufferConversion() { + for(DataType value : DataType.values()) { + if(value != DataType.UTF8 && value != DataType.COMPRESSED && value != DataType.BFLOAT16 && value != DataType.UNKNOWN) + assertBufferCreation(Nd4j.createBuffer(new int[]{1,1},value,0)); + } + + } + + @Test + public void testStringOffsetsGeneration() { + DataBuffer dataBuffer = Nd4j.createBufferOfType(DataType.UTF8,new String[]{"hello1","hello2"}); + DataBuffer offsets = dataBuffer.binaryOffsets(); + //note that the offsets is number of elements + 1 + assertEquals(dataBuffer.length() + 1,offsets.length()); + } + + @Test + public void testToTensor() { + for(DataType value : DataType.values()) { + //note arrow does not support boolean conversion + if(value == DataType.UTF8 || value == DataType.BOOL || value == DataType.COMPRESSED || value == DataType.UNKNOWN || value == DataType.BFLOAT16) + continue; + + INDArray arr = Nd4j.create(Nd4j.createBuffer(new int[]{1,1},value,0)); + Tensor convert = ByteDecoArrowSerde.toTensor(arr); + INDArray convertedBack = ByteDecoArrowSerde.fromTensor(convert).reshape(1,1); + assertEquals("Arrays of data type " + value + " were not equal",arr,convertedBack); + } + } + + @Test + public void testToFromTensorDataTypes() { + for(DataType dataType : DataType.values()) { + if(dataType == DataType.COMPRESSED || dataType == DataType.BFLOAT16 || dataType == DataType.UNKNOWN) + continue; + + org.bytedeco.arrow.DataType dataType1 = ByteDecoArrowSerde.arrowDataTypeForNd4j(dataType); + DataType dataType2 = ByteDecoArrowSerde.dataBufferTypeTypeForArrow(dataType1); + + assertEquals(dataType,dataType2); + } + } + + private void assertBufferCreation(DataBuffer buffer) { + Pair arrowBuffer = ByteDecoArrowSerde.fromNd4jBuffer(buffer); + assertEquals(buffer.dataType(),ByteDecoArrowSerde.dataBufferTypeTypeForArrow(arrowBuffer.getRight())); + DataBuffer buffer1 = ByteDecoArrowSerde.fromArrowBuffer(arrowBuffer.getFirst(), arrowBuffer.getRight()); + assertEquals(buffer1,buffer1); + } + + @Test + public void testConvertToNdArray() { + INDArray arr = Nd4j.scalar(1.0).reshape(1,1); + FlatArray array1 = ByteDecoArrowSerde.arrayFromExistingINDArray(arr); + INDArray convertBack = ByteDecoArrowSerde.ndarrayFromArrowArray(array1).reshape(1,1); + assertEquals(arr,convertBack); + } +} diff --git a/nd4j/nd4j-serde/nd4j-arrow/src/test/java/org/nd4j/arrow/Nd4jArrowOpRunnerTest.java b/nd4j/nd4j-serde/nd4j-arrow/src/test/java/org/nd4j/arrow/Nd4jArrowOpRunnerTest.java new file mode 100644 index 000000000000..2b5e4e7d5fa4 --- /dev/null +++ b/nd4j/nd4j-serde/nd4j-arrow/src/test/java/org/nd4j/arrow/Nd4jArrowOpRunnerTest.java @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2019 Konduit KK + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.nd4j.arrow; + +import org.bytedeco.arrow.FlatArray; +import org.bytedeco.arrow.PrimitiveArray; +import org.junit.Test; +import org.nd4j.linalg.api.ndarray.INDArray; +import org.nd4j.linalg.factory.Nd4j; + +import static org.junit.Assert.assertEquals; + +public class Nd4jArrowOpRunnerTest { + + @Test + public void testOpExec() { + INDArray arr = Nd4j.scalar(1.0); + INDArray arr2 = Nd4j.scalar(2.0); + FlatArray conversionOne = ByteDecoArrowSerde.arrayFromExistingINDArray(arr); + FlatArray conversionTwo = ByteDecoArrowSerde.arrayFromExistingINDArray(arr2); + INDArray verifyFirst = ByteDecoArrowSerde.ndarrayFromArrowArray(conversionOne).reshape(new long[0]); + INDArray verifySecond = ByteDecoArrowSerde.ndarrayFromArrowArray(conversionTwo).reshape(new long[0]); + assertEquals(arr,verifyFirst); + assertEquals(arr2,verifySecond); + FlatArray[] primitiveArrays = Nd4jArrowOpRunner.runOpOn(new FlatArray[]{conversionOne, conversionOne}, "add"); + INDArray outputArr = ByteDecoArrowSerde.ndarrayFromArrowArray(primitiveArrays[0]); + assertEquals(2.0,outputArr.sumNumber().doubleValue(),1e-3); + + + } + + +} diff --git a/pom.xml b/pom.xml index 5a8d49d8883b..8976db70e9cf 100644 --- a/pom.xml +++ b/pom.xml @@ -230,7 +230,7 @@ 1.9.13 5.1 - 0.11.0 + 0.15.1 1.7.7 2.8.0 4.0 @@ -297,6 +297,7 @@ 1.18.2 ${numpy.version}-${javacpp-presets.version} + ${arrow.version}-${javacpp.version} 0.3.9 2020.0 4.2.0