add a udf to improve performance by replacing array_contain

src/main/java/brickhouse/udf/collect/CollectHyphenPartOneUDF.java

@@ -0,0 +1,45 @@
+package brickhouse.udf.collect;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+import java.util.HashSet;
+
+import org.apache.hadoop.hive.ql.exec.UDF;
+
+import java.util.Arrays;
+
+/**
+ * Retain strings in the first array only if _.split("-")[0]
+ * is in the second array.
+ */
+
+public class CollectHyphenPartOneUDF extends UDF {
+
+    // in user_session_daily.dw_user_session_daily, the filterSet
+    // stays constant once initialized; static filterSet can be used by
+    // multiple rows.
+    private static Set<String> filterSet = null;
+
+    public List<String> evaluate(List<String> array1, List<String> array2) {
+        if(array1 == null) return null;
+        if(filterSet == null) {
+            filterSet = new HashSet<String>(array2);
+        }
+        List<String> res = new ArrayList<String>();
+        for(String s : array1) {
+            if(s == null) continue;
+            if(filterSet.contains(s.split("-")[0])) {
+                res.add(s);
+            }
+        }
+        return res;
+    }
+
+    // mvn exec:java -Dexec.mainClass="brickhouse.udf.collect.CollectHyphenPartOneUDF"
+    public static void main(String[] args) {
+        CollectHyphenPartOneUDF myudf = new CollectHyphenPartOneUDF();
+        // should return ["a-1", "b-2"]
+        System.out.println(myudf.evaluate(Arrays.asList("a-1", "b-2", "c-3"), Arrays.asList("a", "b")));
+    }
+}

src/main/resources/brickhouse.hql

@@ -16,6 +16,7 @@ CREATE TEMPORARY FUNCTION cast_array AS 'brickhouse.udf.collect.CastArrayUDF';
 CREATE TEMPORARY FUNCTION cast_map AS 'brickhouse.udf.collect.CastMapUDF';
 CREATE TEMPORARY FUNCTION combine AS 'brickhouse.udf.collect.CombineUDF';
 CREATE TEMPORARY FUNCTION combine_unique AS 'brickhouse.udf.collect.CombineUniqueUDAF';
+CREATE TEMPORARY FUNCTION collect_hyphen_part_one AS 'brickhouse.udf.collect.CollectHyphenPartOneUDF';
 CREATE TEMPORARY FUNCTION conditional_emit AS 'brickhouse.udf.collect.ConditionalEmit';
 CREATE TEMPORARY FUNCTION join_array AS 'brickhouse.udf.collect.JoinArrayUDF';
 CREATE TEMPORARY FUNCTION map_filter_keys AS 'brickhouse.udf.collect.MapFilterKeysUDF';