From ac5bde44b45ae16efc6403ebe33e5b0fa59de0c8 Mon Sep 17 00:00:00 2001 From: Vatsal Mevada Date: Mon, 18 Nov 2019 13:21:42 +0530 Subject: [PATCH 01/12] Changes to provide details of tables stored in external hive metastore as part of SYS.HIVETABLES VTI. Pending: Changes are tested by picking first available snappysession from ConnectionIdToSnappySession map instead of picking the correct SnappySession for given connection id. --- .../internal/cache/ExternalTableMetaData.java | 2 + .../internal/engine/GfxdDataSerializable.java | 3 + .../internal/engine/GfxdSerializable.java | 2 +- .../gemfirexd/internal/engine/Misc.java | 4 + .../internal/engine/diag/HiveTablesVTI.java | 21 ++++- .../distributed/GfxdDistributionAdvisor.java | 19 +++- .../internal/engine/store/GemFireStore.java | 2 + .../snappy/CallbackFactoryProvider.java | 7 ++ .../internal/snappy/ClusterCallbacks.java | 10 ++ .../ExternalHiveTablesCollectorFunction.java | 49 ++++++++++ .../ExternalHiveTablesCollectorResult.java | 94 +++++++++++++++++++ 11 files changed, 210 insertions(+), 3 deletions(-) create mode 100644 gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/hivetables/ExternalHiveTablesCollectorFunction.java create mode 100644 gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/hivetables/dto/ExternalHiveTablesCollectorResult.java diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/ExternalTableMetaData.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/ExternalTableMetaData.java index 62d6323c8..199afc3ad 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/ExternalTableMetaData.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/ExternalTableMetaData.java @@ -69,6 +69,8 @@ public ExternalTableMetaData(String entityName, public List columns; public boolean hasDependentSampleTables; + public ExternalTableMetaData() { } + @Override public String toString() { return "ObjectMetadata(name=" + this.entityName + ", schema=" + this.schema + diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/GfxdDataSerializable.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/GfxdDataSerializable.java index cc4c6473d..c2d16e693 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/GfxdDataSerializable.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/GfxdDataSerializable.java @@ -89,6 +89,7 @@ import com.pivotal.gemfirexd.internal.engine.ui.SnappyRegionStatsCollectorResult; import com.pivotal.gemfirexd.internal.impl.store.raw.data.GfxdJarMessage; import com.pivotal.gemfirexd.internal.snappy.LeadNodeExecutionContext; +import com.pivotal.gemfirexd.internal.snappy.hivetables.dto.ExternalHiveTablesCollectorResult; import com.pivotal.gemfirexd.tools.planexporter.ExecutionPlanMessage; /** @@ -270,6 +271,8 @@ public static synchronized boolean initTypes() { () -> new SnappyResultHolder()); DSFIDFactory.registerGemFireXDClass(SNAPPY_REGION_STATS_RESULT, () -> new SnappyRegionStatsCollectorResult()); + DSFIDFactory.registerGemFireXDClass(HIVE_TABLES_COLLECTOR_RESULT, + () -> new ExternalHiveTablesCollectorResult()); DSFIDFactory.registerGemFireXDClass(MEMBER_STATISTICS_MESSAGE, () -> new MemberStatisticsMessage()); DSFIDFactory.registerGemFireXDClass(MEMBER_LOGS_MESSAGE, diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/GfxdSerializable.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/GfxdSerializable.java index cd78c7782..99a722839 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/GfxdSerializable.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/GfxdSerializable.java @@ -326,7 +326,7 @@ public interface GfxdSerializable extends GfxdDSFID { byte SNAPPY_REGION_STATS_RESULT = 111; - // 112 is unused -- use for new IDs + byte HIVE_TABLES_COLLECTOR_RESULT= 112; byte COLUMN_FORMAT_KEY = 113; diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/Misc.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/Misc.java index 8b1192b4c..57cb9dc35 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/Misc.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/Misc.java @@ -275,6 +275,10 @@ public static Set getLeadNode() { throw new NoMemberFoundException("SnappyData Lead node is not available"); } + public static boolean isLead(){ + return GemFireXDUtils.getGfxdAdvisor().getMyProfile().hasSparkURL(); + } + /** * Check if {@link GemFireCache} is closed or is in the process of closing and * throw {@link CacheClosedException} if so. diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/diag/HiveTablesVTI.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/diag/HiveTablesVTI.java index 4b50982cd..741d32ea5 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/diag/HiveTablesVTI.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/diag/HiveTablesVTI.java @@ -21,10 +21,15 @@ import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.sql.Types; +import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.Iterator; +import java.util.List; import java.util.ListIterator; +import java.util.concurrent.TimeUnit; +import com.gemstone.gemfire.cache.execute.FunctionService; import com.gemstone.gemfire.internal.cache.ExternalTableMetaData; import com.gemstone.gemfire.internal.shared.SystemProperties; import com.pivotal.gemfirexd.internal.catalog.ExternalCatalog; @@ -40,6 +45,8 @@ import com.pivotal.gemfirexd.internal.shared.common.SharedUtils; import com.pivotal.gemfirexd.internal.shared.common.reference.Limits; import com.pivotal.gemfirexd.internal.shared.common.reference.SQLState; +import com.pivotal.gemfirexd.internal.snappy.hivetables.ExternalHiveTablesCollectorFunction; +import com.pivotal.gemfirexd.internal.snappy.hivetables.dto.ExternalHiveTablesCollectorResult; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -69,11 +76,16 @@ public boolean next() { if (!GfxdDataDictionary.SKIP_CATALOG_OPS.get().skipHiveCatalogCalls && (hiveCatalog = Misc.getMemStore().getExternalCatalog()) != null) { try { - this.tableMetas = hiveCatalog.getCatalogTables().iterator(); + List catalogTables = hiveCatalog.getCatalogTables(); + if (!Misc.isLead()) { + catalogTables.addAll(getExternalHiveTables()); + } + this.tableMetas = catalogTables.iterator(); } catch (Exception e) { // log and move on logger.warn("ERROR in retrieving Hive tables: " + e.toString()); this.tableMetas = Collections.emptyIterator(); + throw new RuntimeException(e); } } else { this.tableMetas = Collections.emptyIterator(); @@ -96,6 +108,13 @@ public boolean next() { } } + private Collection getExternalHiveTables() throws InterruptedException { + ArrayList result = (ArrayList)FunctionService.onMembers(Misc.getLeadNode()) + .withArgs(0).execute(ExternalHiveTablesCollectorFunction.ID) + .getResult(300, TimeUnit.SECONDS); + return ((ExternalHiveTablesCollectorResult)(result).get(0)).getTablesMetadata(); + } + @Override protected Object getObjectForColumn(int columnNumber) { String provider = this.currentTableMeta.shortProvider; diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/GfxdDistributionAdvisor.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/GfxdDistributionAdvisor.java index 80f5a9366..547e81439 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/GfxdDistributionAdvisor.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/GfxdDistributionAdvisor.java @@ -1336,6 +1336,13 @@ public static final class GfxdProfile extends DistributionAdvisor.Profile */ private long usableHeap; + /** + * Flag indicating whether the external hive catalog is enabled. By default it's false. + * Set to true when hive session is initialized at + * org.apache.spark.sql.SnappyContext#newHiveSession(). + */ + private boolean hiveEnabled; + /** for deserialization */ public GfxdProfile() { this.initialized = true; @@ -1450,7 +1457,7 @@ public final void setInitialized(boolean initialized) { public final boolean getInitialized() { return this.initialized; } - + public final String getLocale() { return this.dbLocaleStr; } @@ -1463,6 +1470,14 @@ public final long getUsableHeap() { return this.usableHeap; } + public void setHiveEnabled(boolean hiveEnabled) { + this.hiveEnabled = hiveEnabled; + } + + public boolean isHiveEnabled() { + return hiveEnabled; + } + @Override public void processIncoming(DistributionManager dm, String adviseePath, boolean removeProfile, boolean exchangeProfiles, @@ -1549,6 +1564,7 @@ public final void toData(DataOutput out) throws IOException { } out.writeLong(this.catalogSchemaVersion.get()); out.writeLong(this.usableHeap); + out.writeBoolean(this.hiveEnabled); } @Override @@ -1589,6 +1605,7 @@ public final void fromData(DataInput in) throws IOException, if ((this.flags & F_HAS_USABLE_HEAP) != 0) { this.usableHeap = in.readLong(); } + this.hiveEnabled = in.readBoolean(); } @Override diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/store/GemFireStore.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/store/GemFireStore.java index 7c50eedc7..8e99222a5 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/store/GemFireStore.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/store/GemFireStore.java @@ -99,6 +99,7 @@ import com.pivotal.gemfirexd.internal.engine.GemFireXDQueryTimeStatistics; import com.pivotal.gemfirexd.internal.engine.GfxdConstants; import com.pivotal.gemfirexd.internal.engine.GfxdDataSerializable; +import com.pivotal.gemfirexd.internal.snappy.hivetables.ExternalHiveTablesCollectorFunction; import com.pivotal.gemfirexd.internal.engine.Misc; import com.pivotal.gemfirexd.internal.engine.SigThreadDumpHandler; import com.pivotal.gemfirexd.internal.engine.access.GemFireTransaction; @@ -1202,6 +1203,7 @@ else if (!hostData) { FunctionService.registerFunction(new QueryCancelFunction()); FunctionService.registerFunction(new SnappyRegionStatsCollectorFunction()); FunctionService.registerFunction(new DiskStoreIDs.DiskStoreIDFunction()); + FunctionService.registerFunction(new ExternalHiveTablesCollectorFunction()); final ConnectionSignaller signaller = ConnectionSignaller.getInstance(); if (logger.fineEnabled()) { diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/CallbackFactoryProvider.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/CallbackFactoryProvider.java index 2db1bdb08..9dcf1dd9f 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/CallbackFactoryProvider.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/CallbackFactoryProvider.java @@ -17,12 +17,14 @@ package com.pivotal.gemfirexd.internal.snappy; +import java.util.Collection; import java.util.HashSet; import java.util.Iterator; import java.util.List; import com.gemstone.gemfire.distributed.internal.membership.InternalDistributedMember; import com.gemstone.gemfire.internal.ByteArrayDataInput; +import com.gemstone.gemfire.internal.cache.ExternalTableMetaData; import com.gemstone.gemfire.internal.shared.Version; import com.gemstone.gemfire.internal.snappy.StoreCallbacks; import com.pivotal.gemfirexd.internal.iapi.sql.ParameterValueSet; @@ -107,6 +109,11 @@ public String getClusterType() { @Override public void setLeadClassLoader() { } + + @Override + public Collection getHiveTablesMetadata(long connectionId) { + return null; + } }; public static ClusterCallbacks getClusterCallbacks() { diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/ClusterCallbacks.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/ClusterCallbacks.java index 6a9e375a9..00ac66502 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/ClusterCallbacks.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/ClusterCallbacks.java @@ -17,12 +17,14 @@ package com.pivotal.gemfirexd.internal.snappy; +import java.util.Collection; import java.util.HashSet; import java.util.Iterator; import java.util.List; import com.gemstone.gemfire.distributed.internal.membership.InternalDistributedMember; import com.gemstone.gemfire.internal.ByteArrayDataInput; +import com.gemstone.gemfire.internal.cache.ExternalTableMetaData; import com.gemstone.gemfire.internal.shared.Version; import com.pivotal.gemfirexd.internal.iapi.sql.ParameterValueSet; import com.pivotal.gemfirexd.internal.iapi.types.DataValueDescriptor; @@ -68,4 +70,12 @@ Iterator getRowIterator(DataValueDescriptor[] dvds, int[] types, String getClusterType(); void setLeadClassLoader(); + + /** + * Used for fetching the metadata of hive tables stored in external hive metastore from lead. + * @param connectionId connection id of current connection + * @return Collection of {@link ExternalTableMetaData} representing necessary information to + * required by SYS.HIVETABLES VTI. + */ + Collection getHiveTablesMetadata(long connectionId); } diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/hivetables/ExternalHiveTablesCollectorFunction.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/hivetables/ExternalHiveTablesCollectorFunction.java new file mode 100644 index 000000000..923d90623 --- /dev/null +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/hivetables/ExternalHiveTablesCollectorFunction.java @@ -0,0 +1,49 @@ +package com.pivotal.gemfirexd.internal.snappy.hivetables; + +import java.util.Collection; + +import com.gemstone.gemfire.cache.execute.Function; +import com.gemstone.gemfire.cache.execute.FunctionContext; +import com.gemstone.gemfire.internal.cache.ExternalTableMetaData; +import com.pivotal.gemfirexd.internal.snappy.CallbackFactoryProvider; +import com.pivotal.gemfirexd.internal.snappy.hivetables.dto.ExternalHiveTablesCollectorResult; + +/** + * Server node(s) use this function to retrieve metadata of hive tables stored + * in external hive metastore (if configured) which is available on lead node. + * Returns {@link ExternalHiveTablesCollectorResult}. + */ +public class ExternalHiveTablesCollectorFunction implements Function { + + public static final String ID = "ExternalHiveTablesCollectorFunction"; + + @Override + public boolean hasResult() { + return true; + } + + @Override + public void execute(FunctionContext context) { + //todo[vatsal] : pass the correct connection id from server + Collection hiveTablesMetadata = + CallbackFactoryProvider.getClusterCallbacks().getHiveTablesMetadata(1); + context.getResultSender().lastResult(new ExternalHiveTablesCollectorResult(hiveTablesMetadata)); + } + + @Override + public String getId() { + return ID; + } + + //todo[vatsal] : verify this return value + @Override + public boolean optimizeForWrite() { + return false; + } + + //todo[vatsal] : verify this return value + @Override + public boolean isHA() { + return false; + } +} diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/hivetables/dto/ExternalHiveTablesCollectorResult.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/hivetables/dto/ExternalHiveTablesCollectorResult.java new file mode 100644 index 000000000..dbeb85a0f --- /dev/null +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/hivetables/dto/ExternalHiveTablesCollectorResult.java @@ -0,0 +1,94 @@ +package com.pivotal.gemfirexd.internal.snappy.hivetables.dto; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import com.gemstone.gemfire.DataSerializer; +import com.gemstone.gemfire.internal.InternalDataSerializer; +import com.gemstone.gemfire.internal.cache.ExternalTableMetaData; +import com.pivotal.gemfirexd.internal.engine.GfxdDataSerializable; +import com.pivotal.gemfirexd.internal.engine.diag.HiveTablesVTI; +import com.pivotal.gemfirexd.internal.snappy.hivetables.ExternalHiveTablesCollectorFunction; + +/** + * Result returned by {@link ExternalHiveTablesCollectorFunction}. Contains Collection of + * {@link ExternalTableMetaData}. + *
+ * Important Note: Only contains those fields of {@link ExternalTableMetaData} which are + * required by {@link HiveTablesVTI} are serialized. Remaining fields will be null. + */ +public class ExternalHiveTablesCollectorResult extends GfxdDataSerializable { + + private Collection tablesMetadata; + + public ExternalHiveTablesCollectorResult(Collection tablesMetadata) { + this.tablesMetadata = tablesMetadata; + } + + public ExternalHiveTablesCollectorResult() { + } + + @Override + public byte getGfxdID() { + return HIVE_TABLES_COLLECTOR_RESULT; + } + + public Collection getTablesMetadata() { + return tablesMetadata; + } + + @Override + public void toData(DataOutput out) throws IOException { + InternalDataSerializer.writeArrayLength(tablesMetadata.size(), out); + for (ExternalTableMetaData md : tablesMetadata) { + DataSerializer.writeString(md.entityName, out); + DataSerializer.writeObject(md.schema, out); + DataSerializer.writeString(md.tableType, out); + DataSerializer.writeString(md.compressionCodec, out); + DataSerializer.writeString(md.shortProvider, out); + DataSerializer.writeString(md.dataSourcePath, out); + DataSerializer.writeString(md.viewText, out); + InternalDataSerializer.writeArrayLength(md.columns.size(), out); + for (ExternalTableMetaData.Column c : md.columns) { + DataSerializer.writeString(c.name, out); + out.writeInt(c.typeId); + DataSerializer.writeString(c.typeName, out); + out.writeInt(c.precision); + out.writeInt(c.scale); + out.writeInt(c.maxWidth); + out.writeBoolean(c.nullable); + } + } + } + + @Override + public void fromData(DataInput out) throws IOException, + ClassNotFoundException { + int tableMetadataSize = InternalDataSerializer.readArrayLength(out); + this.tablesMetadata = new ArrayList<>(tableMetadataSize); + for (int i = 0; i < tableMetadataSize; i++) { + ExternalTableMetaData md = new ExternalTableMetaData(); + md.entityName = DataSerializer.readString(out); + md.schema = DataSerializer.readObject(out); + md.tableType = DataSerializer.readString(out); + md.compressionCodec = DataSerializer.readString(out); + md.shortProvider = DataSerializer.readString(out); + md.dataSourcePath = DataSerializer.readString(out); + md.viewText = DataSerializer.readString(out); + int columnsSize = InternalDataSerializer.readArrayLength(out); + List columns = new ArrayList<>(columnsSize); + for (int j = 0; j < columnsSize; j++) { + ExternalTableMetaData.Column c = new ExternalTableMetaData.Column( + DataSerializer.readString(out), out.readInt(), DataSerializer.readString(out), + out.readInt(), out.readInt(), out.readInt(), out.readBoolean()); + columns.add(c); + } + md.columns = columns; + this.tablesMetadata.add(md); + } + } +} From 817dd9f73613ea43ed5b64b29c7f907e348036fb Mon Sep 17 00:00:00 2001 From: Vatsal Mevada Date: Mon, 18 Nov 2019 13:25:02 +0530 Subject: [PATCH 02/12] Fetching external hive tables only when external hive metastore support is enabled --- .../pivotal/gemfirexd/internal/engine/diag/HiveTablesVTI.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/diag/HiveTablesVTI.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/diag/HiveTablesVTI.java index 741d32ea5..3946d7620 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/diag/HiveTablesVTI.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/diag/HiveTablesVTI.java @@ -36,6 +36,7 @@ import com.pivotal.gemfirexd.internal.engine.GfxdVTITemplate; import com.pivotal.gemfirexd.internal.engine.GfxdVTITemplateNoAllNodesRoute; import com.pivotal.gemfirexd.internal.engine.Misc; +import com.pivotal.gemfirexd.internal.engine.distributed.utils.GemFireXDUtils; import com.pivotal.gemfirexd.internal.engine.jdbc.GemFireXDRuntimeException; import com.pivotal.gemfirexd.internal.iapi.sql.ResultColumnDescriptor; import com.pivotal.gemfirexd.internal.iapi.types.HarmonySerialClob; @@ -77,7 +78,7 @@ public boolean next() { (hiveCatalog = Misc.getMemStore().getExternalCatalog()) != null) { try { List catalogTables = hiveCatalog.getCatalogTables(); - if (!Misc.isLead()) { + if (!Misc.isLead() && GemFireXDUtils.getMyProfile(true).isHiveEnabled()) { catalogTables.addAll(getExternalHiveTables()); } this.tableMetas = catalogTables.iterator(); From 7073218c3576033373d0fde31ff66b0ec2194ac6 Mon Sep 17 00:00:00 2001 From: Vatsal Mevada Date: Mon, 18 Nov 2019 13:57:11 +0530 Subject: [PATCH 03/12] Reverting some code which was added for debugging. --- .../pivotal/gemfirexd/internal/engine/diag/HiveTablesVTI.java | 1 - 1 file changed, 1 deletion(-) diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/diag/HiveTablesVTI.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/diag/HiveTablesVTI.java index 3946d7620..c95aa4707 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/diag/HiveTablesVTI.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/diag/HiveTablesVTI.java @@ -86,7 +86,6 @@ public boolean next() { // log and move on logger.warn("ERROR in retrieving Hive tables: " + e.toString()); this.tableMetas = Collections.emptyIterator(); - throw new RuntimeException(e); } } else { this.tableMetas = Collections.emptyIterator(); From f283d299c9c21318311d9c82b333e0a5453f7019 Mon Sep 17 00:00:00 2001 From: Vatsal Mevada Date: Mon, 18 Nov 2019 18:10:40 +0530 Subject: [PATCH 04/12] code refactoring --- .../internal/engine/diag/HiveTablesVTI.java | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/diag/HiveTablesVTI.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/diag/HiveTablesVTI.java index c95aa4707..70c2567c9 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/diag/HiveTablesVTI.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/diag/HiveTablesVTI.java @@ -32,6 +32,7 @@ import com.gemstone.gemfire.cache.execute.FunctionService; import com.gemstone.gemfire.internal.cache.ExternalTableMetaData; import com.gemstone.gemfire.internal.shared.SystemProperties; +import com.google.common.collect.Iterators; import com.pivotal.gemfirexd.internal.catalog.ExternalCatalog; import com.pivotal.gemfirexd.internal.engine.GfxdVTITemplate; import com.pivotal.gemfirexd.internal.engine.GfxdVTITemplateNoAllNodesRoute; @@ -78,10 +79,8 @@ public boolean next() { (hiveCatalog = Misc.getMemStore().getExternalCatalog()) != null) { try { List catalogTables = hiveCatalog.getCatalogTables(); - if (!Misc.isLead() && GemFireXDUtils.getMyProfile(true).isHiveEnabled()) { - catalogTables.addAll(getExternalHiveTables()); - } - this.tableMetas = catalogTables.iterator(); + Collection externalHiveTables = getExternalHiveTables(); + this.tableMetas = Iterators.concat(catalogTables.iterator(), externalHiveTables.iterator()); } catch (Exception e) { // log and move on logger.warn("ERROR in retrieving Hive tables: " + e.toString()); @@ -109,10 +108,14 @@ public boolean next() { } private Collection getExternalHiveTables() throws InterruptedException { - ArrayList result = (ArrayList)FunctionService.onMembers(Misc.getLeadNode()) - .withArgs(0).execute(ExternalHiveTablesCollectorFunction.ID) - .getResult(300, TimeUnit.SECONDS); - return ((ExternalHiveTablesCollectorResult)(result).get(0)).getTablesMetadata(); + if (!Misc.isLead() && GemFireXDUtils.getMyProfile(true).isHiveEnabled()) { + ArrayList result = (ArrayList)FunctionService.onMembers(Misc.getLeadNode()) + .withArgs(0).execute(ExternalHiveTablesCollectorFunction.ID) + .getResult(10, TimeUnit.SECONDS); + return ((ExternalHiveTablesCollectorResult)(result).get(0)).getTablesMetadata(); + } else { + return Collections.emptyList(); + } } @Override From c2dbb05711f5dea15259a0089d2f2c9859dfe280 Mon Sep 17 00:00:00 2001 From: Vatsal Mevada Date: Tue, 19 Nov 2019 12:02:36 +0530 Subject: [PATCH 05/12] - passing connection id from server while calling `ExternalHiveTablesCollectorFunction`. - added copyright headers --- .../internal/engine/ConnectionAwareVTI.java | 28 +++++++++++++++++++ .../internal/engine/diag/HiveTablesVTI.java | 11 ++++++-- .../distributed/GfxdDistributionAdvisor.java | 1 + .../impl/sql/execute/VTIResultSet.java | 24 ++++++++++++++++ .../ExternalHiveTablesCollectorFunction.java | 20 +++++++++++-- .../ExternalHiveTablesCollectorResult.java | 17 +++++++++++ 6 files changed, 97 insertions(+), 4 deletions(-) create mode 100644 gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ConnectionAwareVTI.java diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ConnectionAwareVTI.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ConnectionAwareVTI.java new file mode 100644 index 000000000..28ed7fad3 --- /dev/null +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ConnectionAwareVTI.java @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2017-2019 TIBCO Software Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +package com.pivotal.gemfirexd.internal.engine; + +import com.pivotal.gemfirexd.internal.iapi.sql.conn.LanguageConnectionContext; + +/** + * VTIs requiring details available from {@link LanguageConnectionContext} should implement this + * interface. + */ +public interface ConnectionAwareVTI { + void setConnectionContext(LanguageConnectionContext lcc); +} diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/diag/HiveTablesVTI.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/diag/HiveTablesVTI.java index 70c2567c9..238e1e375 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/diag/HiveTablesVTI.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/diag/HiveTablesVTI.java @@ -34,12 +34,14 @@ import com.gemstone.gemfire.internal.shared.SystemProperties; import com.google.common.collect.Iterators; import com.pivotal.gemfirexd.internal.catalog.ExternalCatalog; +import com.pivotal.gemfirexd.internal.engine.ConnectionAwareVTI; import com.pivotal.gemfirexd.internal.engine.GfxdVTITemplate; import com.pivotal.gemfirexd.internal.engine.GfxdVTITemplateNoAllNodesRoute; import com.pivotal.gemfirexd.internal.engine.Misc; import com.pivotal.gemfirexd.internal.engine.distributed.utils.GemFireXDUtils; import com.pivotal.gemfirexd.internal.engine.jdbc.GemFireXDRuntimeException; import com.pivotal.gemfirexd.internal.iapi.sql.ResultColumnDescriptor; +import com.pivotal.gemfirexd.internal.iapi.sql.conn.LanguageConnectionContext; import com.pivotal.gemfirexd.internal.iapi.types.HarmonySerialClob; import com.pivotal.gemfirexd.internal.impl.jdbc.EmbedResultSetMetaData; import com.pivotal.gemfirexd.internal.impl.jdbc.Util; @@ -57,7 +59,7 @@ * in a de-normalized form. */ public class HiveTablesVTI extends GfxdVTITemplate - implements GfxdVTITemplateNoAllNodesRoute { + implements GfxdVTITemplateNoAllNodesRoute, ConnectionAwareVTI { private final Logger logger = LoggerFactory.getLogger(getClass().getName()); @@ -65,6 +67,7 @@ public class HiveTablesVTI extends GfxdVTITemplate private ExternalTableMetaData currentTableMeta; private ListIterator currentTableColumns; private ExternalTableMetaData.Column currentTableColumn; + private LanguageConnectionContext lcc; @Override public ResultSetMetaData getMetaData() throws SQLException { @@ -110,7 +113,7 @@ public boolean next() { private Collection getExternalHiveTables() throws InterruptedException { if (!Misc.isLead() && GemFireXDUtils.getMyProfile(true).isHiveEnabled()) { ArrayList result = (ArrayList)FunctionService.onMembers(Misc.getLeadNode()) - .withArgs(0).execute(ExternalHiveTablesCollectorFunction.ID) + .withArgs(lcc.getConnectionId()).execute(ExternalHiveTablesCollectorFunction.ID) .getResult(10, TimeUnit.SECONDS); return ((ExternalHiveTablesCollectorResult)(result).get(0)).getTablesMetadata(); } else { @@ -256,4 +259,8 @@ public Clob getClob(int columnNumber) throws SQLException { private static final ResultSetMetaData metadata = new EmbedResultSetMetaData( columnInfo); + + public void setConnectionContext(LanguageConnectionContext lcc) { + this.lcc = lcc; + } } diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/GfxdDistributionAdvisor.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/GfxdDistributionAdvisor.java index 547e81439..7b206b9de 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/GfxdDistributionAdvisor.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/GfxdDistributionAdvisor.java @@ -1625,6 +1625,7 @@ public final void fillInToString(StringBuilder sb) { sb.append("; numProcessors=").append(this.numProcessors); sb.append("; catalogVersion=").append(this.catalogSchemaVersion.get()); sb.append("; usableHeap=").append(this.usableHeap); + sb.append("; hiveEnabled=").append(this.hiveEnabled); } } diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/sql/execute/VTIResultSet.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/sql/execute/VTIResultSet.java index 2bfb6ae4e..06166cb0d 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/sql/execute/VTIResultSet.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/sql/execute/VTIResultSet.java @@ -38,10 +38,30 @@ Licensed to the Apache Software Foundation (ASF) under one or more * LICENSE file. */ +/* + * Changes for SnappyData data platform. + * + * Portions Copyright (c) 2017-2019 TIBCO Software Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + package com.pivotal.gemfirexd.internal.impl.sql.execute; import com.gemstone.gemfire.internal.cache.TXState; import com.pivotal.gemfirexd.internal.catalog.TypeDescriptor; +import com.pivotal.gemfirexd.internal.engine.ConnectionAwareVTI; import com.pivotal.gemfirexd.internal.engine.GfxdConstants; import com.pivotal.gemfirexd.internal.engine.UpdateVTITemplate; import com.pivotal.gemfirexd.internal.engine.distributed.utils.GemFireXDUtils; @@ -876,6 +896,10 @@ private void setSharedStateInUserVTI() throws SQLException { q.setQualifiers(this, pushedQualifiers); } + + if (userVTI instanceof ConnectionAwareVTI) { + ((ConnectionAwareVTI)userVTI).setConnectionContext(lcc); + } } @Override diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/hivetables/ExternalHiveTablesCollectorFunction.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/hivetables/ExternalHiveTablesCollectorFunction.java index 923d90623..707677841 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/hivetables/ExternalHiveTablesCollectorFunction.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/hivetables/ExternalHiveTablesCollectorFunction.java @@ -1,3 +1,20 @@ +/* + * Copyright (c) 2017-2019 TIBCO Software Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + package com.pivotal.gemfirexd.internal.snappy.hivetables; import java.util.Collection; @@ -24,9 +41,8 @@ public boolean hasResult() { @Override public void execute(FunctionContext context) { - //todo[vatsal] : pass the correct connection id from server Collection hiveTablesMetadata = - CallbackFactoryProvider.getClusterCallbacks().getHiveTablesMetadata(1); + CallbackFactoryProvider.getClusterCallbacks().getHiveTablesMetadata((Long)context.getArguments()); context.getResultSender().lastResult(new ExternalHiveTablesCollectorResult(hiveTablesMetadata)); } diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/hivetables/dto/ExternalHiveTablesCollectorResult.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/hivetables/dto/ExternalHiveTablesCollectorResult.java index dbeb85a0f..29d8d1d5c 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/hivetables/dto/ExternalHiveTablesCollectorResult.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/hivetables/dto/ExternalHiveTablesCollectorResult.java @@ -1,3 +1,20 @@ +/* + * Copyright (c) 2017-2019 TIBCO Software Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + package com.pivotal.gemfirexd.internal.snappy.hivetables.dto; import java.io.DataInput; From 4a56abba1544ad31ff4ce42dbe83ec48c197b31d Mon Sep 17 00:00:00 2001 From: Vatsal Mevada Date: Wed, 20 Nov 2019 15:49:00 +0530 Subject: [PATCH 06/12] Fixing profile update issue. --- .../pivotal/gemfirexd/internal/engine/Misc.java | 10 ++++++++++ .../internal/engine/diag/HiveTablesVTI.java | 9 +++++++-- .../distributed/GfxdDistributionAdvisor.java | 16 ++++++++-------- .../internal/engine/store/GemFireStore.java | 7 +++++++ 4 files changed, 32 insertions(+), 10 deletions(-) diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/Misc.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/Misc.java index 57cb9dc35..282155c27 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/Misc.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/Misc.java @@ -275,6 +275,16 @@ public static Set getLeadNode() { throw new NoMemberFoundException("SnappyData Lead node is not available"); } + /** + * + * @return Optional of profile of primary lead node if primary lead exists in cluster + * Optional#empty if primary lead node does not exist in cluster + */ + public static Optional getPrimaryLeadProfile(){ + return getLeadNode().stream().map(GemFireXDUtils::getGfxdProfile) + .filter(GfxdDistributionAdvisor.GfxdProfile::hasSparkURL).findFirst(); + } + public static boolean isLead(){ return GemFireXDUtils.getGfxdAdvisor().getMyProfile().hasSparkURL(); } diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/diag/HiveTablesVTI.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/diag/HiveTablesVTI.java index 238e1e375..fb9bcb9e8 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/diag/HiveTablesVTI.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/diag/HiveTablesVTI.java @@ -27,6 +27,7 @@ import java.util.Iterator; import java.util.List; import java.util.ListIterator; +import java.util.Optional; import java.util.concurrent.TimeUnit; import com.gemstone.gemfire.cache.execute.FunctionService; @@ -38,7 +39,7 @@ import com.pivotal.gemfirexd.internal.engine.GfxdVTITemplate; import com.pivotal.gemfirexd.internal.engine.GfxdVTITemplateNoAllNodesRoute; import com.pivotal.gemfirexd.internal.engine.Misc; -import com.pivotal.gemfirexd.internal.engine.distributed.utils.GemFireXDUtils; +import com.pivotal.gemfirexd.internal.engine.distributed.GfxdDistributionAdvisor; import com.pivotal.gemfirexd.internal.engine.jdbc.GemFireXDRuntimeException; import com.pivotal.gemfirexd.internal.iapi.sql.ResultColumnDescriptor; import com.pivotal.gemfirexd.internal.iapi.sql.conn.LanguageConnectionContext; @@ -111,7 +112,11 @@ public boolean next() { } private Collection getExternalHiveTables() throws InterruptedException { - if (!Misc.isLead() && GemFireXDUtils.getMyProfile(true).isHiveEnabled()) { + GfxdDistributionAdvisor.GfxdProfile primaryLeadProfile = Misc.getPrimaryLeadProfile() + .orElseThrow(() -> { + throw new IllegalStateException("Lead not available"); + }); + if (!Misc.isLead() && primaryLeadProfile.isHiveSessionInitialized()) { ArrayList result = (ArrayList)FunctionService.onMembers(Misc.getLeadNode()) .withArgs(lcc.getConnectionId()).execute(ExternalHiveTablesCollectorFunction.ID) .getResult(10, TimeUnit.SECONDS); diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/GfxdDistributionAdvisor.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/GfxdDistributionAdvisor.java index 7b206b9de..b4e7440cf 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/GfxdDistributionAdvisor.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/GfxdDistributionAdvisor.java @@ -1341,7 +1341,7 @@ public static final class GfxdProfile extends DistributionAdvisor.Profile * Set to true when hive session is initialized at * org.apache.spark.sql.SnappyContext#newHiveSession(). */ - private boolean hiveEnabled; + private boolean hiveSessionInitialized; /** for deserialization */ public GfxdProfile() { @@ -1470,12 +1470,12 @@ public final long getUsableHeap() { return this.usableHeap; } - public void setHiveEnabled(boolean hiveEnabled) { - this.hiveEnabled = hiveEnabled; + public void setHiveSessionInitialized(boolean hiveSessionInitialized) { + this.hiveSessionInitialized = hiveSessionInitialized; } - public boolean isHiveEnabled() { - return hiveEnabled; + public boolean isHiveSessionInitialized() { + return hiveSessionInitialized; } @Override @@ -1564,7 +1564,7 @@ public final void toData(DataOutput out) throws IOException { } out.writeLong(this.catalogSchemaVersion.get()); out.writeLong(this.usableHeap); - out.writeBoolean(this.hiveEnabled); + out.writeBoolean(this.hiveSessionInitialized); } @Override @@ -1605,7 +1605,7 @@ public final void fromData(DataInput in) throws IOException, if ((this.flags & F_HAS_USABLE_HEAP) != 0) { this.usableHeap = in.readLong(); } - this.hiveEnabled = in.readBoolean(); + this.hiveSessionInitialized = in.readBoolean(); } @Override @@ -1625,7 +1625,7 @@ public final void fillInToString(StringBuilder sb) { sb.append("; numProcessors=").append(this.numProcessors); sb.append("; catalogVersion=").append(this.catalogSchemaVersion.get()); sb.append("; usableHeap=").append(this.usableHeap); - sb.append("; hiveEnabled=").append(this.hiveEnabled); + sb.append("; hiveSessionInitialized=").append(this.hiveSessionInitialized); } } diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/store/GemFireStore.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/store/GemFireStore.java index 8e99222a5..9beb55096 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/store/GemFireStore.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/store/GemFireStore.java @@ -2691,6 +2691,8 @@ public static final class StoreAdvisee implements DistributionAdvisee { */ private volatile SortedSet serverGroups; + private boolean hiveSessionInitialized; + private final CancelCriterion stopper = new CancelCriterion() { @Override @@ -2776,6 +2778,10 @@ final synchronized void setVMKind(VMKind newKind) { + " GemFireXD not booted or closed down."); } + public void setHiveSessionInitialized(boolean hiveSessionInitialized) { + this.hiveSessionInitialized = hiveSessionInitialized; + } + /** * Get the server groups of this VM. */ @@ -2804,6 +2810,7 @@ public void fillInProfile(Profile p) { .isDataDictionaryPersistent()); profile.setLocale(Misc.getMemStoreBooting().getLocale()); profile.serialNumber = getSerialNumber(); + profile.setHiveSessionInitialized(hiveSessionInitialized); } @Override From 48597d1cb18c11136217ab6113255c53020b189f Mon Sep 17 00:00:00 2001 From: Vatsal Mevada Date: Wed, 20 Nov 2019 18:53:32 +0530 Subject: [PATCH 07/12] Passing current schema as argument of ExternalHiveTablesCollectorFunction which will be used to retrieve hive tables available in current schema. --- .../gemfirexd/internal/engine/Misc.java | 18 ++++++++++++++++++ .../internal/engine/diag/HiveTablesVTI.java | 3 ++- .../snappy/CallbackFactoryProvider.java | 2 +- .../internal/snappy/ClusterCallbacks.java | 3 ++- .../ExternalHiveTablesCollectorFunction.java | 5 ++++- 5 files changed, 27 insertions(+), 4 deletions(-) diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/Misc.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/Misc.java index 282155c27..c7f63d594 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/Misc.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/Misc.java @@ -15,6 +15,24 @@ * LICENSE file. */ +/* + * Changes for SnappyData data platform. + * + * Portions Copyright (c) 2017-2019 TIBCO Software Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ package com.pivotal.gemfirexd.internal.engine; import java.io.CharArrayWriter; diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/diag/HiveTablesVTI.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/diag/HiveTablesVTI.java index fb9bcb9e8..486f62680 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/diag/HiveTablesVTI.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/diag/HiveTablesVTI.java @@ -117,8 +117,9 @@ private Collection getExternalHiveTables() throws Interru throw new IllegalStateException("Lead not available"); }); if (!Misc.isLead() && primaryLeadProfile.isHiveSessionInitialized()) { + Object[] args = new Object[] { lcc.getConnectionId(), lcc.getCurrentSchemaName() }; ArrayList result = (ArrayList)FunctionService.onMembers(Misc.getLeadNode()) - .withArgs(lcc.getConnectionId()).execute(ExternalHiveTablesCollectorFunction.ID) + .withArgs(args).execute(ExternalHiveTablesCollectorFunction.ID) .getResult(10, TimeUnit.SECONDS); return ((ExternalHiveTablesCollectorResult)(result).get(0)).getTablesMetadata(); } else { diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/CallbackFactoryProvider.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/CallbackFactoryProvider.java index 9dcf1dd9f..e733fb7f5 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/CallbackFactoryProvider.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/CallbackFactoryProvider.java @@ -111,7 +111,7 @@ public void setLeadClassLoader() { } @Override - public Collection getHiveTablesMetadata(long connectionId) { + public Collection getHiveTablesMetadata(long connectionId, String schema) { return null; } }; diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/ClusterCallbacks.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/ClusterCallbacks.java index 00ac66502..3e8869a5b 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/ClusterCallbacks.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/ClusterCallbacks.java @@ -74,8 +74,9 @@ Iterator getRowIterator(DataValueDescriptor[] dvds, int[] types, /** * Used for fetching the metadata of hive tables stored in external hive metastore from lead. * @param connectionId connection id of current connection + * @param schema current schema of the connection * @return Collection of {@link ExternalTableMetaData} representing necessary information to * required by SYS.HIVETABLES VTI. */ - Collection getHiveTablesMetadata(long connectionId); + Collection getHiveTablesMetadata(long connectionId, String schema); } diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/hivetables/ExternalHiveTablesCollectorFunction.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/hivetables/ExternalHiveTablesCollectorFunction.java index 707677841..3821e86a2 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/hivetables/ExternalHiveTablesCollectorFunction.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/hivetables/ExternalHiveTablesCollectorFunction.java @@ -41,8 +41,11 @@ public boolean hasResult() { @Override public void execute(FunctionContext context) { + Object[] args = (Object[])context.getArguments(); + Long connectionId = (Long)args[0]; + String schema = (String)args[1]; Collection hiveTablesMetadata = - CallbackFactoryProvider.getClusterCallbacks().getHiveTablesMetadata((Long)context.getArguments()); + CallbackFactoryProvider.getClusterCallbacks().getHiveTablesMetadata(connectionId, schema); context.getResultSender().lastResult(new ExternalHiveTablesCollectorResult(hiveTablesMetadata)); } From 231da277d1cade94f64ee1c8a35c4a0d636d854a Mon Sep 17 00:00:00 2001 From: Vatsal Mevada Date: Wed, 20 Nov 2019 19:15:49 +0530 Subject: [PATCH 08/12] Using spaces for indentation instead of tabs --- .../gemfirexd/internal/impl/sql/execute/VTIResultSet.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/sql/execute/VTIResultSet.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/sql/execute/VTIResultSet.java index 06166cb0d..c53a611cb 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/sql/execute/VTIResultSet.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/sql/execute/VTIResultSet.java @@ -897,9 +897,9 @@ private void setSharedStateInUserVTI() throws SQLException { q.setQualifiers(this, pushedQualifiers); } - if (userVTI instanceof ConnectionAwareVTI) { - ((ConnectionAwareVTI)userVTI).setConnectionContext(lcc); - } + if (userVTI instanceof ConnectionAwareVTI) { + ((ConnectionAwareVTI)userVTI).setConnectionContext(lcc); + } } @Override From 2b17af09faa81e72b7381d22014eebbb50c45adc Mon Sep 17 00:00:00 2001 From: Vatsal Mevada Date: Thu, 21 Nov 2019 16:18:39 +0530 Subject: [PATCH 09/12] Fetching hive tables from all schemas instead of tables from only current schema. --- .../internal/engine/ConnectionAwareVTI.java | 28 -------- .../internal/engine/diag/HiveTablesVTI.java | 14 +--- .../impl/sql/execute/VTIResultSet.java | 72 +++++++------------ .../snappy/CallbackFactoryProvider.java | 2 +- .../internal/snappy/ClusterCallbacks.java | 4 +- .../ExternalHiveTablesCollectorFunction.java | 5 +- 6 files changed, 30 insertions(+), 95 deletions(-) delete mode 100644 gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ConnectionAwareVTI.java diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ConnectionAwareVTI.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ConnectionAwareVTI.java deleted file mode 100644 index 28ed7fad3..000000000 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ConnectionAwareVTI.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2017-2019 TIBCO Software Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ - -package com.pivotal.gemfirexd.internal.engine; - -import com.pivotal.gemfirexd.internal.iapi.sql.conn.LanguageConnectionContext; - -/** - * VTIs requiring details available from {@link LanguageConnectionContext} should implement this - * interface. - */ -public interface ConnectionAwareVTI { - void setConnectionContext(LanguageConnectionContext lcc); -} diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/diag/HiveTablesVTI.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/diag/HiveTablesVTI.java index 486f62680..8194b8660 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/diag/HiveTablesVTI.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/diag/HiveTablesVTI.java @@ -27,7 +27,6 @@ import java.util.Iterator; import java.util.List; import java.util.ListIterator; -import java.util.Optional; import java.util.concurrent.TimeUnit; import com.gemstone.gemfire.cache.execute.FunctionService; @@ -35,7 +34,6 @@ import com.gemstone.gemfire.internal.shared.SystemProperties; import com.google.common.collect.Iterators; import com.pivotal.gemfirexd.internal.catalog.ExternalCatalog; -import com.pivotal.gemfirexd.internal.engine.ConnectionAwareVTI; import com.pivotal.gemfirexd.internal.engine.GfxdVTITemplate; import com.pivotal.gemfirexd.internal.engine.GfxdVTITemplateNoAllNodesRoute; import com.pivotal.gemfirexd.internal.engine.Misc; @@ -59,8 +57,7 @@ * A virtual table that shows the hive tables and their columns * in a de-normalized form. */ -public class HiveTablesVTI extends GfxdVTITemplate - implements GfxdVTITemplateNoAllNodesRoute, ConnectionAwareVTI { +public class HiveTablesVTI extends GfxdVTITemplate implements GfxdVTITemplateNoAllNodesRoute { private final Logger logger = LoggerFactory.getLogger(getClass().getName()); @@ -117,10 +114,9 @@ private Collection getExternalHiveTables() throws Interru throw new IllegalStateException("Lead not available"); }); if (!Misc.isLead() && primaryLeadProfile.isHiveSessionInitialized()) { - Object[] args = new Object[] { lcc.getConnectionId(), lcc.getCurrentSchemaName() }; ArrayList result = (ArrayList)FunctionService.onMembers(Misc.getLeadNode()) - .withArgs(args).execute(ExternalHiveTablesCollectorFunction.ID) - .getResult(10, TimeUnit.SECONDS); + .execute(ExternalHiveTablesCollectorFunction.ID) + .getResult(30, TimeUnit.SECONDS); return ((ExternalHiveTablesCollectorResult)(result).get(0)).getTablesMetadata(); } else { return Collections.emptyList(); @@ -265,8 +261,4 @@ public Clob getClob(int columnNumber) throws SQLException { private static final ResultSetMetaData metadata = new EmbedResultSetMetaData( columnInfo); - - public void setConnectionContext(LanguageConnectionContext lcc) { - this.lcc = lcc; - } } diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/sql/execute/VTIResultSet.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/sql/execute/VTIResultSet.java index c53a611cb..ba81402b5 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/sql/execute/VTIResultSet.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/sql/execute/VTIResultSet.java @@ -38,30 +38,10 @@ Licensed to the Apache Software Foundation (ASF) under one or more * LICENSE file. */ -/* - * Changes for SnappyData data platform. - * - * Portions Copyright (c) 2017-2019 TIBCO Software Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ - package com.pivotal.gemfirexd.internal.impl.sql.execute; import com.gemstone.gemfire.internal.cache.TXState; import com.pivotal.gemfirexd.internal.catalog.TypeDescriptor; -import com.pivotal.gemfirexd.internal.engine.ConnectionAwareVTI; import com.pivotal.gemfirexd.internal.engine.GfxdConstants; import com.pivotal.gemfirexd.internal.engine.UpdateVTITemplate; import com.pivotal.gemfirexd.internal.engine.distributed.utils.GemFireXDUtils; @@ -139,7 +119,7 @@ Specified isolation level of SELECT (scan). If not set or not application, it will be set to ExecutionContext.UNSPECIFIED_ISOLATION_LEVEL */ private int scanIsolationLevel = ExecutionContext.UNSPECIFIED_ISOLATION_LEVEL; - + // // class interface // @@ -156,10 +136,10 @@ Specified isolation level of SELECT (scan). If not set or double optimizerEstimatedCost, boolean isDerbyStyleTableFunction, String returnType - ) + ) throws StandardException { - super(activation, resultSetNumber, + super(activation, resultSetNumber, optimizerEstimatedRowCount, optimizerEstimatedCost); this.row = row; this.constructor = constructor; @@ -183,7 +163,7 @@ Specified isolation level of SELECT (scan). If not set or getSavedObject(ctcNumber)); recordConstructorTime(); - + // GemStone changes BEGIN printResultSetHierarchy(); // GemStone changes END @@ -199,7 +179,7 @@ Specified isolation level of SELECT (scan). If not set or * * @exception StandardException thrown if activation closed. */ - public void openCore() throws StandardException + public void openCore() throws StandardException { beginTime = statisticsTimingOn ? XPLAINUtil.nanoTime() : 0; if (SanityManager.DEBUG) @@ -239,7 +219,7 @@ public void openCore() throws StandardException UpdatableVTIConstantAction constants = (UpdatableVTIConstantAction) activation.getConstantAction(); ((DeferModification) userPS).modificationNotify( constants.statementType, constants.deferred); } - + if ((fastPath != null) && fastPath.executeAsFastPath()) ; else @@ -288,10 +268,10 @@ private boolean[] setNullableColumnList() throws SQLException, StandardException if ( isDerbyStyleTableFunction ) { int count = getAllocatedRow().nColumns() + 1; - + runtimeNullableColumn = new boolean[ count ]; for ( int i = 0; i < count; i++ ) { runtimeNullableColumn[ i ] = true; } - + return runtimeNullableColumn; } @@ -310,8 +290,8 @@ private boolean[] setNullableColumnList() throws SQLException, StandardException /** * If the VTI is a version2 vti that does not * need to be instantiated multiple times then - * we simply close the current ResultSet and - * create a new one via a call to + * we simply close the current ResultSet and + * create a new one via a call to * PreparedStatement.executeQuery(). * * @see NoPutResultSet#openCore @@ -344,7 +324,7 @@ public void reopenCore() throws StandardException else { close(false); - openCore(); + openCore(); } } @@ -354,13 +334,13 @@ public void reopenCore() throws StandardException * * @exception StandardException thrown on failure. */ - public ExecRow getNextRowCore() throws StandardException + public ExecRow getNextRowCore() throws StandardException { ExecRow result = null; beginTime = statisticsTimingOn ? XPLAINUtil.nanoTime() : 0; - - if ( isOpen ) + + if ( isOpen ) { try { @@ -411,7 +391,7 @@ else if (action == IFastPath.NEED_RS) { return result; } - + /** * @see com.pivotal.gemfirexd.internal.iapi.sql.ResultSet#close @@ -618,7 +598,7 @@ public void populateFromResultSet(ExecRow row) } columns[index].setValueFromResultSet( - userVTI, rsColNumber, + userVTI, rsColNumber, /* last parameter is whether or * not the column is nullable */ @@ -739,13 +719,13 @@ private TypeDescriptor thawReturnType( String ice ) TypeDescriptor td = (TypeDescriptor) fiis.readObject(); return td; - + } catch (Throwable t) { throw StandardException.unexpectedUserException( t ); } } - + /** *

* Cast the value coming out of the user-coded ResultSet. The @@ -774,7 +754,7 @@ private void cast( DataTypeDescriptor dtd, DataValueDescriptor dvd ) int width; if ( typeID.isNumericTypeId() ) { width = dtd.getPrecision(); } else { width = dtd.getMaximumWidth(); } - + vsdv.setWidth( width, dtd.getScale(), false ); } } @@ -796,7 +776,7 @@ private void castLongvarchar( DataTypeDescriptor dtd, DataValueDescriptor dvd dvd.setValue( dvd.getString().substring( 0, TypeId.LONGVARCHAR_MAXWIDTH ) ); } } - + /** *

* Truncate long varbinary values to the legal maximum. @@ -811,11 +791,11 @@ private void castLongvarbinary( DataTypeDescriptor dtd, DataValueDescriptor d byte[] result = new byte[ TypeId.LONGVARBIT_MAXWIDTH ]; System.arraycopy( original, 0, result, 0, TypeId.LONGVARBIT_MAXWIDTH ); - + dvd.setValue( result ); } } - + /** *

* Set the correct precision and scale for a decimal value. @@ -825,7 +805,7 @@ private void castDecimal( DataTypeDescriptor dtd, DataValueDescriptor dvd ) throws StandardException { VariableSizeDataValue vsdv = (VariableSizeDataValue) dvd; - + vsdv.setWidth( dtd.getPrecision(), dtd.getScale(), false ); } @@ -883,7 +863,7 @@ private void setSharedStateInUserVTI() throws SQLException { ((com.pivotal.gemfirexd.internal.engine.GfxdVTITemplate)userVTI) .setSharedState(compileTimeConstants); } - + if (userVTI instanceof com.pivotal.gemfirexd.internal.vti.Pushable) { com.pivotal.gemfirexd.internal.vti.Pushable p = (com.pivotal.gemfirexd.internal.vti.Pushable)userVTI; if (referencedColumns != null) { @@ -896,10 +876,6 @@ private void setSharedStateInUserVTI() throws SQLException { q.setQualifiers(this, pushedQualifiers); } - - if (userVTI instanceof ConnectionAwareVTI) { - ((ConnectionAwareVTI)userVTI).setConnectionContext(lcc); - } } @Override diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/CallbackFactoryProvider.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/CallbackFactoryProvider.java index e733fb7f5..6cf660efd 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/CallbackFactoryProvider.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/CallbackFactoryProvider.java @@ -111,7 +111,7 @@ public void setLeadClassLoader() { } @Override - public Collection getHiveTablesMetadata(long connectionId, String schema) { + public Collection getHiveTablesMetadata() { return null; } }; diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/ClusterCallbacks.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/ClusterCallbacks.java index 3e8869a5b..69e5263b6 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/ClusterCallbacks.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/ClusterCallbacks.java @@ -73,10 +73,8 @@ Iterator getRowIterator(DataValueDescriptor[] dvds, int[] types, /** * Used for fetching the metadata of hive tables stored in external hive metastore from lead. - * @param connectionId connection id of current connection - * @param schema current schema of the connection * @return Collection of {@link ExternalTableMetaData} representing necessary information to * required by SYS.HIVETABLES VTI. */ - Collection getHiveTablesMetadata(long connectionId, String schema); + Collection getHiveTablesMetadata(); } diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/hivetables/ExternalHiveTablesCollectorFunction.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/hivetables/ExternalHiveTablesCollectorFunction.java index 3821e86a2..c390502e7 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/hivetables/ExternalHiveTablesCollectorFunction.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/hivetables/ExternalHiveTablesCollectorFunction.java @@ -41,11 +41,8 @@ public boolean hasResult() { @Override public void execute(FunctionContext context) { - Object[] args = (Object[])context.getArguments(); - Long connectionId = (Long)args[0]; - String schema = (String)args[1]; Collection hiveTablesMetadata = - CallbackFactoryProvider.getClusterCallbacks().getHiveTablesMetadata(connectionId, schema); + CallbackFactoryProvider.getClusterCallbacks().getHiveTablesMetadata(); context.getResultSender().lastResult(new ExternalHiveTablesCollectorResult(hiveTablesMetadata)); } From 473709ffed20e945a849be2edd5984791c531da1 Mon Sep 17 00:00:00 2001 From: Vatsal Mevada Date: Thu, 21 Nov 2019 16:33:42 +0530 Subject: [PATCH 10/12] fixing formatting and removing unused code --- .../internal/engine/diag/HiveTablesVTI.java | 4 +- .../impl/sql/execute/VTIResultSet.java | 48 +++++++++---------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/diag/HiveTablesVTI.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/diag/HiveTablesVTI.java index 8194b8660..42a0d12fb 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/diag/HiveTablesVTI.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/diag/HiveTablesVTI.java @@ -57,7 +57,8 @@ * A virtual table that shows the hive tables and their columns * in a de-normalized form. */ -public class HiveTablesVTI extends GfxdVTITemplate implements GfxdVTITemplateNoAllNodesRoute { +public class HiveTablesVTI extends GfxdVTITemplate implements + GfxdVTITemplateNoAllNodesRoute { private final Logger logger = LoggerFactory.getLogger(getClass().getName()); @@ -65,7 +66,6 @@ public class HiveTablesVTI extends GfxdVTITemplate implements GfxdVTITemplateNoA private ExternalTableMetaData currentTableMeta; private ListIterator currentTableColumns; private ExternalTableMetaData.Column currentTableColumn; - private LanguageConnectionContext lcc; @Override public ResultSetMetaData getMetaData() throws SQLException { diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/sql/execute/VTIResultSet.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/sql/execute/VTIResultSet.java index ba81402b5..2bfb6ae4e 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/sql/execute/VTIResultSet.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/sql/execute/VTIResultSet.java @@ -119,7 +119,7 @@ Specified isolation level of SELECT (scan). If not set or not application, it will be set to ExecutionContext.UNSPECIFIED_ISOLATION_LEVEL */ private int scanIsolationLevel = ExecutionContext.UNSPECIFIED_ISOLATION_LEVEL; - + // // class interface // @@ -136,10 +136,10 @@ Specified isolation level of SELECT (scan). If not set or double optimizerEstimatedCost, boolean isDerbyStyleTableFunction, String returnType - ) + ) throws StandardException { - super(activation, resultSetNumber, + super(activation, resultSetNumber, optimizerEstimatedRowCount, optimizerEstimatedCost); this.row = row; this.constructor = constructor; @@ -163,7 +163,7 @@ Specified isolation level of SELECT (scan). If not set or getSavedObject(ctcNumber)); recordConstructorTime(); - + // GemStone changes BEGIN printResultSetHierarchy(); // GemStone changes END @@ -179,7 +179,7 @@ Specified isolation level of SELECT (scan). If not set or * * @exception StandardException thrown if activation closed. */ - public void openCore() throws StandardException + public void openCore() throws StandardException { beginTime = statisticsTimingOn ? XPLAINUtil.nanoTime() : 0; if (SanityManager.DEBUG) @@ -219,7 +219,7 @@ public void openCore() throws StandardException UpdatableVTIConstantAction constants = (UpdatableVTIConstantAction) activation.getConstantAction(); ((DeferModification) userPS).modificationNotify( constants.statementType, constants.deferred); } - + if ((fastPath != null) && fastPath.executeAsFastPath()) ; else @@ -268,10 +268,10 @@ private boolean[] setNullableColumnList() throws SQLException, StandardException if ( isDerbyStyleTableFunction ) { int count = getAllocatedRow().nColumns() + 1; - + runtimeNullableColumn = new boolean[ count ]; for ( int i = 0; i < count; i++ ) { runtimeNullableColumn[ i ] = true; } - + return runtimeNullableColumn; } @@ -290,8 +290,8 @@ private boolean[] setNullableColumnList() throws SQLException, StandardException /** * If the VTI is a version2 vti that does not * need to be instantiated multiple times then - * we simply close the current ResultSet and - * create a new one via a call to + * we simply close the current ResultSet and + * create a new one via a call to * PreparedStatement.executeQuery(). * * @see NoPutResultSet#openCore @@ -324,7 +324,7 @@ public void reopenCore() throws StandardException else { close(false); - openCore(); + openCore(); } } @@ -334,13 +334,13 @@ public void reopenCore() throws StandardException * * @exception StandardException thrown on failure. */ - public ExecRow getNextRowCore() throws StandardException + public ExecRow getNextRowCore() throws StandardException { ExecRow result = null; beginTime = statisticsTimingOn ? XPLAINUtil.nanoTime() : 0; - - if ( isOpen ) + + if ( isOpen ) { try { @@ -391,7 +391,7 @@ else if (action == IFastPath.NEED_RS) { return result; } - + /** * @see com.pivotal.gemfirexd.internal.iapi.sql.ResultSet#close @@ -598,7 +598,7 @@ public void populateFromResultSet(ExecRow row) } columns[index].setValueFromResultSet( - userVTI, rsColNumber, + userVTI, rsColNumber, /* last parameter is whether or * not the column is nullable */ @@ -719,13 +719,13 @@ private TypeDescriptor thawReturnType( String ice ) TypeDescriptor td = (TypeDescriptor) fiis.readObject(); return td; - + } catch (Throwable t) { throw StandardException.unexpectedUserException( t ); } } - + /** *

* Cast the value coming out of the user-coded ResultSet. The @@ -754,7 +754,7 @@ private void cast( DataTypeDescriptor dtd, DataValueDescriptor dvd ) int width; if ( typeID.isNumericTypeId() ) { width = dtd.getPrecision(); } else { width = dtd.getMaximumWidth(); } - + vsdv.setWidth( width, dtd.getScale(), false ); } } @@ -776,7 +776,7 @@ private void castLongvarchar( DataTypeDescriptor dtd, DataValueDescriptor dvd dvd.setValue( dvd.getString().substring( 0, TypeId.LONGVARCHAR_MAXWIDTH ) ); } } - + /** *

* Truncate long varbinary values to the legal maximum. @@ -791,11 +791,11 @@ private void castLongvarbinary( DataTypeDescriptor dtd, DataValueDescriptor d byte[] result = new byte[ TypeId.LONGVARBIT_MAXWIDTH ]; System.arraycopy( original, 0, result, 0, TypeId.LONGVARBIT_MAXWIDTH ); - + dvd.setValue( result ); } } - + /** *

* Set the correct precision and scale for a decimal value. @@ -805,7 +805,7 @@ private void castDecimal( DataTypeDescriptor dtd, DataValueDescriptor dvd ) throws StandardException { VariableSizeDataValue vsdv = (VariableSizeDataValue) dvd; - + vsdv.setWidth( dtd.getPrecision(), dtd.getScale(), false ); } @@ -863,7 +863,7 @@ private void setSharedStateInUserVTI() throws SQLException { ((com.pivotal.gemfirexd.internal.engine.GfxdVTITemplate)userVTI) .setSharedState(compileTimeConstants); } - + if (userVTI instanceof com.pivotal.gemfirexd.internal.vti.Pushable) { com.pivotal.gemfirexd.internal.vti.Pushable p = (com.pivotal.gemfirexd.internal.vti.Pushable)userVTI; if (referencedColumns != null) { From e28f4233421eed2935484c94bf26a05de41782f0 Mon Sep 17 00:00:00 2001 From: Vatsal Mevada Date: Thu, 21 Nov 2019 16:42:31 +0530 Subject: [PATCH 11/12] minor: code formatting --- .../pivotal/gemfirexd/internal/engine/diag/HiveTablesVTI.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/diag/HiveTablesVTI.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/diag/HiveTablesVTI.java index 42a0d12fb..b7beed268 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/diag/HiveTablesVTI.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/diag/HiveTablesVTI.java @@ -57,8 +57,8 @@ * A virtual table that shows the hive tables and their columns * in a de-normalized form. */ -public class HiveTablesVTI extends GfxdVTITemplate implements - GfxdVTITemplateNoAllNodesRoute { +public class HiveTablesVTI extends GfxdVTITemplate + implements GfxdVTITemplateNoAllNodesRoute { private final Logger logger = LoggerFactory.getLogger(getClass().getName()); From 2f3e020fc3ad3cdf03dce66e96d7be9ecb9c28c2 Mon Sep 17 00:00:00 2001 From: Vatsal Mevada Date: Fri, 22 Nov 2019 17:27:08 +0530 Subject: [PATCH 12/12] fixing failing test: AnalyzeSerializablesJUnitTest.testDataSerializables --- .../resources/codeAnalysis/sanctionedDataSerializables.txt | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/gemfirexd/tools/src/test/resources/codeAnalysis/sanctionedDataSerializables.txt b/gemfirexd/tools/src/test/resources/codeAnalysis/sanctionedDataSerializables.txt index 977127d5d..aa4ee9718 100644 --- a/gemfirexd/tools/src/test/resources/codeAnalysis/sanctionedDataSerializables.txt +++ b/gemfirexd/tools/src/test/resources/codeAnalysis/sanctionedDataSerializables.txt @@ -135,8 +135,8 @@ fromData,53,2a2bb700152bb9001601003d1c047e99000c2a2bb8001788b500051c057e9900112a toData,71,2a2bb700112ab40009412ab4000a10bf7e360420140007949900091504058036042b1504b9001202002ab6000b99000c2ab40005852bb8001320140007949900082b20b80014b1 com/pivotal/gemfirexd/internal/engine/distributed/GfxdDistributionAdvisor$GfxdProfile,2 -fromData,216,2a2bb700552a2bb900560100b80057b500012bb800583d1c02a0000b2a01b5001ea700262abb005959b7005ab5001e1c8402ff9e00142ab4001e2bb8005bb9005c020057a7ffeb2a2bb900560100b5001c2a2bb9005d0100b500032ab4001c057e99000b2a2bb8005bb5001f2ab4001c10087e99000d2a2bb9005e0100b500072ab7001b2ab6002199000b2a2bb8005bb500082ab4001c10107e9900172abb000b592bb9005f0100b7000cb5000da700152abb000b592bb9005e010085b7000cb5000d2ab4001c10207e99000d2a2bb9005f0100b50015b1 -toData,236,2a2bb700462b2ab40001b60047b9004802002ab4001ec7000b022bb80049a700352ab4001eb9004a01002bb800492ab4001eb9004b01004d2cb9003e01009900152cb9003f0100c0004c4e2d2bb8004da7ffe82bb8004e4db2004f2cb600509b000704a70004033eb200512cb600509e000704a700040336042ab4001c36051d99000b150510fd7e913605150499000b150510f77e9136052b1505b9004802002b2ab40003b9005202001d9a000b2ab4001f2bb8004d15049a000d2b2ab40007b9005302002ab6002199000b2ab400082bb8004d2b2ab4000db60022b9005403002b2ab40015b900540300b1 +fromData,226,2a2bb700562a2bb900570100b80058b500012bb800593d1c02a0000b2a01b5001ea700262abb005a59b7005bb5001e1c8402ff9e00142ab4001e2bb8005cb9005d020057a7ffeb2a2bb900570100b5001c2a2bb9005e0100b500032ab4001c057e99000b2a2bb8005cb5001f2ab4001c10087e99000d2a2bb9005f0100b500072ab7001b2ab6002199000b2a2bb8005cb500082ab4001c10107e9900172abb000b592bb900600100b7000cb5000da700152abb000b592bb9005f010085b7000cb5000d2ab4001c10207e99000d2a2bb900600100b500152a2bb9005e0100b50025b1 +toData,246,2a2bb700472b2ab40001b60048b9004902002ab4001ec7000b022bb8004aa700352ab4001eb9004b01002bb8004a2ab4001eb9004c01004d2cb9003f01009900152cb900400100c0004d4e2d2bb8004ea7ffe82bb8004f4db200502cb600519b000704a70004033eb200522cb600519e000704a700040336042ab4001c36051d99000b150510fd7e913605150499000b150510f77e9136052b1505b9004902002b2ab40003b9005302001d9a000b2ab4001f2bb8004e15049a000d2b2ab40007b9005402002ab6002199000b2ab400082bb8004e2b2ab4000db60022b9005503002b2ab40015b9005503002b2ab40025b900530200b1 com/pivotal/gemfirexd/internal/engine/distributed/GfxdDumpLocalResultMessage,2 fromData,16,2a2bb7005a2a2bb9005b0100b50002b1 @@ -546,3 +546,6 @@ com/pivotal/gemfirexd/tools/planexporter/ExecutionPlanMessage$ExecutionPlanReply fromData,14,2a2bb700052a2bb80006b50001b1 toData,14,2a2bb700072ab400012bb80008b1 +com/pivotal/gemfirexd/internal/snappy/hivetables/dto/ExternalHiveTablesCollectorResult,2 +fromData,212,2bb800203d2abb0021591cb70022b50002033e1d1ca200bebb000959b700233a0419042bb80024b5000a19042bb80025b5000c19042bb80024b5000e19042bb80024b5000f19042bb80024b5001019042bb80024b5001119042bb80024b500122bb800203605bb0021591505b700223a0603360715071505a20042bb0016592bb800242bb9002601002bb800242bb9002601002bb9002601002bb9002601002bb900270100b700283a0819061908b90029020057840701a7ffbd19041906b500132ab400021904b9002a020057840301a7ff43b1 +toData,224,2ab40002b9000401002bb800052ab40002b9000601004d2cb9000701009900c22cb900080100c000094e2db4000a2bb8000b2db4000c2bb8000d2db4000e2bb8000b2db4000f2bb8000b2db400102bb8000b2db400112bb8000b2db400122bb8000b2db40013b9001401002bb800052db40013b9001501003a041904b90007010099005b1904b900080100c000163a051905b400172bb8000b2b1905b40018b9001902001905b4001a2bb8000b2b1905b4001bb9001902002b1905b4001cb9001902002b1905b4001db9001902002b1905b4001eb9001f0200a7ffa1a7ff3bb1 \ No newline at end of file