From fccebf5b11992daf9a1a7dd8482ef6b149ba4752 Mon Sep 17 00:00:00 2001 From: ramesh Date: Tue, 19 Mar 2019 16:22:43 +0530 Subject: [PATCH] support hive 3 serde interfaces/remove deprecated hive api references --- build.gradle | 4 ++-- hive/src/main/java/com/mongodb/hadoop/hive/BSONSerDe.java | 4 ++-- .../java/com/mongodb/hadoop/hive/MongoStorageHandler.java | 6 +++--- .../com/mongodb/hadoop/hive/input/HiveMongoInputFormat.java | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/build.gradle b/build.gradle index b451744a..3b9bfdf6 100644 --- a/build.gradle +++ b/build.gradle @@ -8,9 +8,9 @@ apply plugin: 'com.github.ben-manes.versions' ext.configDir = new File(rootDir, 'config') ext.hadoopBinaries = "${rootDir}/hadoop-binaries".toString() ext.javaDriverVersion = '3.2.1' -ext.hiveVersion = System.getenv("HIVE_VERSION") ?: '1.2.1' +ext.hiveVersion = System.getenv("HIVE_VERSION") ?: '3.1.1' ext.pigVersion = System.getenv("PIG_VERSION") ?: '0.15.0' -ext.hadoopVersion = System.getenv("HADOOP_VERSION") ?: '2.7.2' +ext.hadoopVersion = System.getenv("HADOOP_VERSION") ?: '3.1.0' ext.sparkVersion = System.getenv("SPARK_VERSION") ?: '1.6.0' ext.scalaVersion = System.getenv("SCALA_VERSION") ?: '2.11' ext.isHadoopV1 = "$hadoopVersion".startsWith("1.") diff --git a/hive/src/main/java/com/mongodb/hadoop/hive/BSONSerDe.java b/hive/src/main/java/com/mongodb/hadoop/hive/BSONSerDe.java index 00573ed0..0e25ac63 100644 --- a/hive/src/main/java/com/mongodb/hadoop/hive/BSONSerDe.java +++ b/hive/src/main/java/com/mongodb/hadoop/hive/BSONSerDe.java @@ -22,7 +22,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.serde.serdeConstants; -import org.apache.hadoop.hive.serde2.SerDe; +import org.apache.hadoop.hive.serde2.AbstractSerDe; import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.SerDeStats; import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; @@ -62,7 +62,7 @@ * The BSONSerDe class deserializes (parses) and serializes object from BSON to Hive represented object. It's initialized with the hive * columns and hive recognized types as well as other config variables mandated by the StorageHanders. */ -public class BSONSerDe implements SerDe { +public class BSONSerDe extends AbstractSerDe { private static final Log LOG = LogFactory.getLog(BSONSerDe.class); // stores the 1-to-1 mapping of MongoDB fields to hive columns diff --git a/hive/src/main/java/com/mongodb/hadoop/hive/MongoStorageHandler.java b/hive/src/main/java/com/mongodb/hadoop/hive/MongoStorageHandler.java index c203a25a..b632ca20 100644 --- a/hive/src/main/java/com/mongodb/hadoop/hive/MongoStorageHandler.java +++ b/hive/src/main/java/com/mongodb/hadoop/hive/MongoStorageHandler.java @@ -25,7 +25,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.metastore.HiveMetaHook; -import org.apache.hadoop.hive.metastore.MetaStoreUtils; +import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.ql.index.IndexPredicateAnalyzer; @@ -36,8 +36,8 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; import org.apache.hadoop.hive.ql.plan.TableDesc; import org.apache.hadoop.hive.serde.serdeConstants; +import org.apache.hadoop.hive.serde2.AbstractSerDe; import org.apache.hadoop.hive.serde2.Deserializer; -import org.apache.hadoop.hive.serde2.SerDe; import org.apache.hadoop.mapred.InputFormat; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.OutputFormat; @@ -84,7 +84,7 @@ public HiveMetaHook getMetaHook() { } @Override - public Class getSerDeClass() { + public Class getSerDeClass() { return BSONSerDe.class; } diff --git a/hive/src/main/java/com/mongodb/hadoop/hive/input/HiveMongoInputFormat.java b/hive/src/main/java/com/mongodb/hadoop/hive/input/HiveMongoInputFormat.java index eb003498..287ca9a6 100644 --- a/hive/src/main/java/com/mongodb/hadoop/hive/input/HiveMongoInputFormat.java +++ b/hive/src/main/java/com/mongodb/hadoop/hive/input/HiveMongoInputFormat.java @@ -32,7 +32,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.ql.exec.Utilities; +import org.apache.hadoop.hive.ql.exec.SerializationUtilities; import org.apache.hadoop.hive.ql.index.IndexPredicateAnalyzer; import org.apache.hadoop.hive.ql.index.IndexSearchCondition; import org.apache.hadoop.hive.ql.io.HiveInputFormat; @@ -131,7 +131,7 @@ DBObject getFilter( String serializedExpr = conf.get(TableScanDesc.FILTER_EXPR_CONF_STR); if (serializedExpr != null) { ExprNodeGenericFuncDesc expr = - Utilities.deserializeExpression(serializedExpr); + SerializationUtilities.deserializeExpression(serializedExpr); IndexPredicateAnalyzer analyzer = IndexPredicateAnalyzer.createAnalyzer(false);