Introduce 'BIGINT' to utilize native BigInt

README.md

@@ -48,6 +48,7 @@ is a simple example that shows how to instantiate a `ParquetSchema` object:
 var schema = new parquet.ParquetSchema({
   name: { type: 'UTF8' },
   quantity: { type: 'INT64' },
+  bigint: { type: 'BIGINT' }, // To utilize BigInt instead of node-int64
   price: { type: 'DOUBLE' },
   date: { type: 'TIMESTAMP_MILLIS' },
   in_stock: { type: 'BOOLEAN' }

lib/codec/plain.js

@@ -49,7 +49,15 @@ function decodeValues_INT32(cursor, count) {
 function encodeValues_INT64(values) {
   let buf = Buffer.alloc(8 * values.length);
   for (let i = 0; i < values.length; i++) {
-    //console.log(typeof values[i]);
+    buf.writeBigInt64LE(BigInt(values[i]), i * 8);
+  }
+
+  return buf;
+}
+
+function encodeValues_BIGINT(values) {
+  let buf = Buffer.alloc(8 * values.length);
+  for (let i = 0; i < values.length; i++) {
     buf.writeBigInt64LE(BigInt(values[i]), i * 8);
   }
 
@@ -67,6 +75,17 @@ function decodeValues_INT64(cursor, count) {
   return values;
 }
 
+function decodeValues_BIGINT(cursor, count) {
+  let values = [];
+
+  for (let i = 0; i < count; ++i) {
+    values.push(cursor.buffer.readBigInt64LE(cursor.offset));
+    cursor.offset += 8;
+  }
+
+  return values;
+}
+
 function encodeValues_INT96(values) {
   let buf = Buffer.alloc(12 * values.length);
 
@@ -235,6 +254,9 @@ exports.encodeValues = function(type, values, opts) {
     case 'FIXED_LEN_BYTE_ARRAY':
       return encodeValues_FIXED_LEN_BYTE_ARRAY(values, opts);
 
+    case 'BIGINT':
+      return encodeValues_BIGINT(values);
+
     default:
       throw 'unsupported type: ' + type;
 
@@ -268,6 +290,9 @@ exports.decodeValues = function(type, cursor, count, opts) {
     case 'FIXED_LEN_BYTE_ARRAY':
       return decodeValues_FIXED_LEN_BYTE_ARRAY(cursor, count, opts);
 
+    case 'BIGINT':
+      return decodeValues_BIGINT(cursor, count);
+
     default:
       throw 'unsupported type: ' + type;
 

lib/codec/rle.js

@@ -44,6 +44,9 @@ exports.encodeValues = function(type, values, opts) {
     case 'INT64':
       values = values.map((x) => parseInt(x, 10));
       break;
+    case 'BIGINT':
+      values = values.map((x) => BigInt(x));
+      break;
 
     default:
       throw 'unsupported type: ' + type;

lib/reader.js

@@ -11,6 +11,10 @@ const parquet_compression = require('./compression')
 const parquet_types = require('./types');
 const BufferReader = require('./bufferReader');
 
+const parquet_thrift_Type = Object.assign({
+  BIGINT: 2
+}, parquet_thrift.Type);
+
 /**
  * Parquet File Magic String
  */
@@ -640,7 +644,7 @@ class ParquetEnvelopeReader {
 
     let field = schema.findField(colChunk.meta_data.path_in_schema);
     let type = parquet_util.getThriftEnum(
-      parquet_thrift.Type,
+      parquet_thrift_Type,
       colChunk.meta_data.type);
 
     let compression = parquet_util.getThriftEnum(
@@ -1058,7 +1062,7 @@ function decodeSchema(schemaElements) {
       schema = schema[schemaElement.name].fields;
     } else {
       let logicalType = parquet_util.getThriftEnum(
-        parquet_thrift.Type,
+        parquet_thrift_Type,
         schemaElement.type);
 
       if (schemaElement.converted_type != null) {

lib/types.js

@@ -15,6 +15,11 @@ const PARQUET_LOGICAL_TYPES = {
     primitiveType: 'INT64',
     toPrimitive: toPrimitive_INT64
   },
+  'BIGINT': {
+    primitiveType: 'BIGINT',
+    toPrimitive: toPrimitive_BIGINT,
+    fromPrimitive: fromPrimitive_BIGINT
+  },
   'INT96': {
     primitiveType: 'INT96',
     toPrimitive: toPrimitive_INT96
@@ -253,6 +258,16 @@ function toPrimitive_INT64(value) {
   return v;
 }
 
+function toPrimitive_BIGINT(value) {
+  const v = BigInt(value)
+
+  return v;
+}
+
+function fromPrimitive_BIGINT(value) {
+  return value.toString();
+}
+
 function toPrimitive_UINT64(value) {
   const v = parseInt(value, 10);
   if (v < 0 || isNaN(v)) {

lib/util.js

@@ -100,16 +100,18 @@ exports.getBitWidth = function (val) {
   }
 }
 
-/**
- * FIXME not ideal that this is linear
- */
+const reverseEnums = new Map();
 exports.getThriftEnum = function (klass, value) {
-  for (let k in klass) {
-    if (klass[k] === value) {
-      return k;
+  let reverse = reverseEnums.get(klass)
+  if (!reverse) {
+    reverse = {};
+    for (let k in klass) {
+      reverse[klass[k]] = k;
     }
+    reverseEnums.set(klass, reverse);
   }
-
+  const ret = reverse[value]
+  if (ret) return ret;
   throw 'Invalid ENUM value';
 }
 

lib/writer.js

@@ -312,7 +312,7 @@ class ParquetTransformer extends stream.Transform {
   }
 
   _flush(callback) {
-    this.writer.close(callback)
+    this.writer.close()
     .then(d => callback(null, d), callback);
   }