Add support for creating custom Graph Schemas for JanusGraph

.gitignore

@@ -66,3 +66,4 @@ pom.xml.tag
 # Maven
 log/
 target/
+dependency-reduced-pom.xml

README-custom-graph.md

@@ -0,0 +1,89 @@
+# How to build a custom graph?
+
+Here is an example of how to build a custom graph for your dataset. Lets say we have following data:
+
+```
+Event -> FlowTuple
+FlowTuple -> SourceIP
+FlowTuple -> DestinationIP
+```
+
+In the above exapmle we have some JSON data that has properties for an Event, FlowTuple, and IP Addresses.
+
+Checkout following file: [azurensg_sample_record.json](./src/main/resources/azurensg_sample_record.json)
+
+
+So now we can store the relationship among these entities as follows:
+
+```
+    Event     -> hasFlowTuple -> FlowTuple (via jsonId)
+    FlowTuple -> hasSourceIp  -> IP (via ipSrcAddr)
+    FlowTuple -> hasDestIp    -> IP (via ipDstAddr)
+```
+
+Checkout following file for more details: [GlobalSchema.scala](./src/main/scala/metron/graph/GlobalSchema.scala)
+
+
+# How to import sample data into graph?
+
+## Setup a Janus Graph instance locally
+
+Steps defined here: https://gist.github.com/tuxdna/166dd41902c59ca0470252b5bf4f3dcd
+
+
+## Build and run the Graph importer
+
+Build the project
+
+```
+export CP=`mvn dependency:build-classpath | grep -A1 "Dependencies classpath:" | tail -1`
+mvn clean compile
+java -cp $CP:target/classes:target/test-classes metron.graph.TestMain ./src/main/config/graph-hbase-config.properties
+```
+
+
+[metron.graph.TestMain](./src/main/scala/metron/graph/TestMain.scala) will perform following steps:
+
+ * Connect to graph as specified in [graph-hbase-config.properties](./src/main/config/graph-hbase-config.properties)
+ * Initialize Graph Schema with properties and indices as defined in [GraphManager.scala](./src/main/scala/metron/graph/GraphManager.scala)
+ * Read a single JSON record from [azurensg_sample_record.json](./src/main/resources/azurensg_sample_record.json), and add it to graph as defined in [AzureNSGDataImporter.scala](./src/main/scala/metron/graph/AzureNSGDataImporter.scala)
+
+
+## Query some data from Graph
+
+
+[QueryURL](http://localhost:8182/?gremlin=g.V().has('data_source_type','azurensg').hasLabel('flow_tuple').has('ipDstAddr','100.10.2.6').valueMap('ipSrcAddr','ipDstAddr','protocol'))
+
+
+```
+$ wget -q -O - "http://localhost:8182/?gremlin=g.V().has('data_source_type','azurensg').hasLabel('flow_tuple').has('ipDstAddr','100.10.2.6').valueMap('ipSrcAddr','ipDstAddr','protocol')" | python -mjson.tool
+{
+    "requestId": "db04e13c-15a2-4ccb-9f3b-57a822490712",
+    "result": {
+        "data": [
+            {
+                "ipDstAddr": [
+                    "100.10.2.6"
+                ],
+                "ipSrcAddr": [
+                    "100.10.1.2"
+                ],
+                "protocol": [
+                    "T"
+                ]
+            }
+        ],
+        "meta": {}
+    },
+    "status": {
+        "attributes": {},
+        "code": 200,
+        "message": ""
+    }
+}
+```
+
+
+# TODOs
+
+Integrating with Apache Storm Bolt

pom.xml

@@ -13,6 +13,9 @@
 
 	<properties>
 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+		<scala.version>2.12.4</scala.version>
+		<janusgraph.version>0.3.1</janusgraph.version>
+		<hadoop.version>2.7.2</hadoop.version>
 	</properties>
 
 	<dependencies>
@@ -57,7 +60,7 @@
 		<dependency>
 			<groupId>org.janusgraph</groupId>
 			<artifactId>janusgraph-cassandra</artifactId>
-			<version>0.3.1</version>
+			<version>${janusgraph.version}</version>
 
 			<exclusions>
 				<exclusion>
@@ -91,7 +94,7 @@
 		<dependency>
 			<groupId>org.janusgraph</groupId>
 			<artifactId>janusgraph-hbase</artifactId>
-			<version>0.3.1</version>
+			<version>${janusgraph.version}</version>
 
 			<exclusions>
 				<exclusion>
@@ -124,7 +127,7 @@
 		<dependency>
 			<groupId>org.janusgraph</groupId>
 			<artifactId>janusgraph-solr</artifactId>
-			<version>0.3.1</version>
+			<version>${janusgraph.version}</version>
 			<exclusions>
 				<exclusion>
 					<artifactId>servlet-api</artifactId>
@@ -156,7 +159,7 @@
 		<dependency>
 			<groupId>org.janusgraph</groupId>
 			<artifactId>janusgraph-es</artifactId>
-			<version>0.3.1</version>
+			<version>${janusgraph.version}</version>
 
 			<exclusions>
 				<exclusion>
@@ -190,7 +193,7 @@
 		<dependency>
 			<groupId>org.janusgraph</groupId>
 			<artifactId>janusgraph-core</artifactId>
-			<version>0.3.1</version>
+			<version>${janusgraph.version}</version>
 
 			<exclusions>
 				<exclusion>
@@ -380,6 +383,20 @@
 		</dependency>
 
 
+		<dependency>
+			<groupId>org.scala-lang</groupId>
+			<artifactId>scala-reflect</artifactId>
+			<version>${scala.version}</version>
+		</dependency>
+
+		<dependency>
+			<groupId>org.apache.hbase</groupId>
+			<artifactId>hbase-client</artifactId>
+			<version>1.1.1</version>
+			<type>jar</type>
+		</dependency>
+
+
 	</dependencies>
 
 
@@ -388,6 +405,9 @@
 			<resource>
 				<directory>src/test/resources</directory>
 			</resource>
+			<resource>
+				<directory>src/main/resources</directory>
+			</resource>
 		</resources>
 		<plugins>
 
@@ -451,6 +471,27 @@
 			</plugin>
 
 
+			<plugin>
+				<groupId>net.alchim31.maven</groupId>
+				<artifactId>scala-maven-plugin</artifactId>
+				<executions>
+					<execution>
+						<id>scala-compile-first</id>
+						<phase>process-resources</phase>
+						<goals>
+							<goal>add-source</goal>
+							<goal>compile</goal>
+						</goals>
+					</execution>
+					<execution>
+						<id>scala-test-compile</id>
+						<phase>process-test-resources</phase>
+						<goals>
+							<goal>testCompile</goal>
+						</goals>
+					</execution>
+				</executions>
+			</plugin>
 
 
 			<plugin>
@@ -463,6 +504,33 @@
 				</configuration>
 			</plugin>
 
+
+			<plugin>
+				<artifactId>maven-assembly-plugin</artifactId>
+				<version>2.5.3</version>
+				<configuration>
+					<descriptorRefs>
+						<descriptorRef>jar-with-dependencies</descriptorRef>
+					</descriptorRefs>
+					<archive>
+						<manifest>
+							<mainClass>metron.graph.GraphTopology</mainClass>
+						</manifest>
+					</archive>
+					<descriptors>
+						<descriptor>src/main/assembly/assembly.xml</descriptor>
+					</descriptors>
+				</configuration>
+				<executions>
+					<execution>
+						<phase>package</phase>
+						<goals>
+							<goal>single</goal>
+						</goals>
+					</execution>
+				</executions>
+			</plugin>
+
 		</plugins>
 
 	</build>

src/main/assembly/assembly.xml

@@ -0,0 +1,42 @@
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3"
+          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+          xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3 http://maven.apache.org/xsd/assembly-1.1.3.xsd">
+    <id>tarball</id>
+    <formats>
+        <format>tar.gz</format>
+    </formats>
+    <includeBaseDirectory>false</includeBaseDirectory>
+    <baseDirectory>${project.artifactId}</baseDirectory>
+    <files>
+        <file>
+            <source>${project.basedir}/src/main/config/graphtopology_config.conf</source>
+	    <outputDirectory>stormgraph</outputDirectory>
+            <destName>graphtopology_config.conf</destName>
+        </file>
+    </files>
+    <fileSets>
+        <fileSet>
+            <directory>${project.basedir}/target</directory>
+            <outputDirectory>stormgraph</outputDirectory>
+            <includes>
+                <include>${project.artifactId}-${project.version}.jar</include>
+            </includes>
+        </fileSet>
+    </fileSets>
+</assembly>

src/main/config/graph-hbase-config.properties

@@ -0,0 +1,9 @@
+gremlin.graph=org.janusgraph.core.JanusGraphFactory
+storage.backend=hbase
+storage.hostname=localhost
+storage.port=2181
+storage.hbase.table=janus_graph1
+cache.db-cache=true
+cache.db-cache-clean-wait=20
+cache.db-cache-time=180000
+cache.db-cache-size=0.5

src/main/config/graphtopology_config.conf

@@ -0,0 +1,55 @@
+#generic topology settings
+
+top.debug = true
+top.generatorSpoutEnabled = false
+top.localDeploy = false
+top.numWorkers = 1
+top.name = SiaStormGraph
+top.spout.name = GraphSpout
+top.spout.parallelism = 1
+top.mapperbolt.name = JanusMapper
+top.mapperbolt.parallelism = 1
+top.graphbolt.name = JanusBolt
+top.graphbolt.parallelism = 1
+
+#settings for generator spout
+
+top.spout.generator.sleep = 1000
+top.spout.generator.randSize = 120
+top.spout.generator.outTupleName = raw
+top.spout.generator.sourceFieldName = ip_src
+top.spout.generator.destFieldName = ip_dst
+top.spout.generator.userField = username
+
+#settings for kafka spout
+
+top.spout.kafka.bootStrapServers = localhost:6667
+top.spout.kafka.topic = indexing
+top.spout.kafka.consumerGroupId = graphSpout
+top.spout.kafka.offsetCommitPeriodMs = 10000
+top.spout.kafka.retry.initialDelay = 500
+top.spout.kafka.retry.delayPeriod = 2
+top.spout.kafka.retry.maxDelay = 10
+top.spout.kafka.maxUncommittedOffsets = 1000000
+#name of tuples that kafka spout outputs are set below
+top.spout.kafka.tupleFieldTopic = topic
+top.spout.kafka.tupleFieldPartition = partition
+top.spout.kafka.tupleFieldOffset = offset
+top.spout.kafka.tupleFieldKey = key
+top.spout.kafka.tupleFieldValue = value
+
+#settings for mapper bolt
+
+top.mapperbolt.tupleToLookFor = value
+top.mapperbolt.allowedEdges = connectsTo, uses, usedBy
+top.mapperbolt.allowedVertexTypes = valueKey
+top.mapperbolt.mappings = ip_src, ip_dst, connectsTo, host, host;username, ip_src, uses, user, host
+
+#settings for JanusBolt
+
+top.graphbolt.backEndConfigLocation = /path/to/graph.properties
+top.graphbolt.ttlDays = 5
+
+#settings for HDFS Access
+top.hdfs.uri = "hdfs://localhost:8020"
+