From 6d441b9cd595f9839a27f2451b9fb04e6e264dfb Mon Sep 17 00:00:00 2001
From: Denise Kersjes <denise.kersjes@hotmail.com>
Date: Thu, 9 Aug 2018 15:55:06 +0200
Subject: [PATCH 01/27] Initial commit

Duplicate of HAIL.java, but with TODO's in it to connect Hail with IRCT using Livy.
---
 .../hms/dbmi/bd2k/picsure/ri/LivyHAIL.java    | 659 ++++++++++++++++++
 1 file changed, 659 insertions(+)
 create mode 100644 IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java

diff --git a/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java b/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
new file mode 100644
index 00000000..b89239a9
--- /dev/null
+++ b/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
@@ -0,0 +1,659 @@
+package edu.harvard.hms.dbmi.bd2k.picsure.ri;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonMappingException;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.JsonNodeFactory;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import edu.harvard.hms.dbmi.bd2k.irct.IRCTApplication;
+import edu.harvard.hms.dbmi.bd2k.irct.exception.ResourceInterfaceException;
+import edu.harvard.hms.dbmi.bd2k.irct.model.find.FindInformationInterface;
+import edu.harvard.hms.dbmi.bd2k.irct.model.ontology.Entity;
+import edu.harvard.hms.dbmi.bd2k.irct.model.ontology.OntologyRelationship;
+import edu.harvard.hms.dbmi.bd2k.irct.model.query.Query;
+import edu.harvard.hms.dbmi.bd2k.irct.model.query.WhereClause;
+import edu.harvard.hms.dbmi.bd2k.irct.model.resource.PrimitiveDataType;
+import edu.harvard.hms.dbmi.bd2k.irct.model.resource.ResourceState;
+import edu.harvard.hms.dbmi.bd2k.irct.model.resource.implementation.PathResourceImplementationInterface;
+import edu.harvard.hms.dbmi.bd2k.irct.model.resource.implementation.QueryResourceImplementationInterface;
+import edu.harvard.hms.dbmi.bd2k.irct.model.result.Data;
+import edu.harvard.hms.dbmi.bd2k.irct.model.result.Result;
+import edu.harvard.hms.dbmi.bd2k.irct.model.result.ResultDataType;
+import edu.harvard.hms.dbmi.bd2k.irct.model.result.ResultStatus;
+import edu.harvard.hms.dbmi.bd2k.irct.model.result.exception.PersistableException;
+import edu.harvard.hms.dbmi.bd2k.irct.model.result.exception.ResultSetException;
+import edu.harvard.hms.dbmi.bd2k.irct.model.result.tabular.Column;
+import edu.harvard.hms.dbmi.bd2k.irct.model.result.tabular.FileResultSet;
+import edu.harvard.hms.dbmi.bd2k.irct.model.security.User;
+import edu.harvard.hms.dbmi.bd2k.util.Utility;
+import org.apache.http.HttpEntity;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.utils.URIBuilder;
+import org.apache.http.entity.ContentType;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.util.EntityUtils;
+import org.apache.log4j.Logger;
+import us.monoid.json.JSONException;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UnsupportedEncodingException;
+import java.net.URISyntaxException;
+import java.util.*;
+
+/**
+ * A resource implementation of a data source that communicates with a HAIL proxy via HTTP
+ */
+public class LivyHAIL implements QueryResourceImplementationInterface,
+        PathResourceImplementationInterface {
+    Logger logger = Logger.getLogger(this.getClass());
+
+
+    private static final String PATH_NAME = "pui";
+
+    protected String resourceName;
+    protected String resourceURL;
+
+    protected ResourceState resourceState;
+
+//    Map<Entity> allPathEntities;
+
+    @Override
+    public void setup(Map<String, String> parameters) throws ResourceInterfaceException {
+
+        if (logger.isDebugEnabled())
+            logger.debug("setup for Hail" +
+                    " Starting...");
+
+        String errorString = "";
+        this.resourceName = parameters.get("resourceName");
+        if (this.resourceName == null) {
+            logger.error("setup() `resourceName` parameter is missing.");
+            errorString += " resourceName";
+        }
+
+        String tempResourceURL = parameters.get("resourceURL");
+        if (tempResourceURL == null) {
+            logger.error("setup() `resourceURL` parameter is missing.");
+            errorString += " resourceURL";
+        } else {
+            resourceURL = (tempResourceURL.endsWith("/")) ? tempResourceURL.substring(0, tempResourceURL.length() - 1) : tempResourceURL;
+        }
+
+        if (!errorString.isEmpty()) {
+            throw new ResourceInterfaceException("Hail Interface setup() is missing:" + errorString);
+        }
+
+        resourceState = ResourceState.READY;
+        logger.debug("setup() for " + resourceName +
+                " Finished. " + resourceName +
+                " is in READY state.");
+    }
+
+    @Override
+    public String getType() {
+        return "Hail";
+    }
+
+    @Override
+    public List<Entity> getPathRelationship(Entity path, OntologyRelationship relationship, User user) {
+        logger.debug("getPathRelationship() Starting");
+        List<Entity> entities = new ArrayList<Entity>();
+
+        // Split the path into components. The first component is the Hail resource name, the rest is
+        // a URL path like string.
+        String p = path.getPui();
+        logger.debug("getPathRelationship() pui:" + p);
+        if (p.indexOf('/', 2) == -1) {
+            // This is a request for the root
+
+            // Call the external URL
+            InputStream is = simpleRestCall(this.resourceURL);
+
+            // Parse the response, as JSON mime type
+            ObjectMapper objectMapper = IRCTApplication.objectMapper;
+            JsonNode responseJsonNode;
+            try {
+                responseJsonNode = objectMapper.readTree(is);
+                String responseStatus = responseJsonNode.get("status").textValue();
+                Entity e = new Entity();
+
+                e.setPui("objectIdVal");
+                e.setName("objectNameVal");
+                e.setDisplayName("objectDisplayNameVal status:" + responseStatus);
+
+                entities.add(e);
+            } catch (JsonMappingException jme) {
+                logger.error("getPathRelationship() Exception:" + jme.getMessage());
+                throw new RuntimeException("Could not parse JSON response from `" + resourceName + "` resource");
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+
+
+        } else {
+            String objectPath = p.substring(p.indexOf('/', 2));
+            logger.debug("getPathRelationship() objectPath: " + objectPath);
+
+            Entity e = new Entity();
+
+            e.setPui("objectIdVal");
+            e.setName("objectNameVal");
+            e.setDisplayName("objectDisplayNameVal");
+
+            entities.add(e);
+        }
+
+        logger.debug("getPathRelationship() Finished");
+        return entities;
+    }
+
+
+    private List<Entity> retrieveAllPathTree() {
+        String urlString = resourceURL + "/tree";
+
+        CloseableHttpClient httpClient = IRCTApplication.CLOSEABLE_HTTP_CLIENT;
+        HttpGet httpGet = new HttpGet(urlString);
+
+        CloseableHttpResponse response = null;
+
+        List<Entity> entities = null;
+
+        try {
+            response = httpClient.execute(httpGet);
+            HttpEntity entity = response.getEntity();
+
+//            entities = parseAllHailPathJsonNode(IRCTApplication.objectMapper
+//                    .readTree(entity
+//                            .getContent()));
+
+            EntityUtils.consume(entity);
+        } catch (IOException ex) {
+            logger.error("IOException when retrieving all path from Hail API:" + urlString +
+                    " with exception message: " + ex.getMessage());
+        } finally {
+            try {
+                if (response != null)
+                    response.close();
+            } catch (IOException ex) {
+                logger.error("GNOME - IOExcpetion when closing http response: " + ex.getMessage());
+            }
+        }
+
+        return entities;
+    }
+
+    /**
+     * @param pathNode
+     * @return null if nothing
+     */
+    private TreeMap<String, JsonNode> parseAllHailPathJsonNode(JsonNode pathNode) {
+        TreeMap<String, JsonNode> entityTreeMap = null;
+        return entityTreeMap;
+    }
+
+    @Override
+    public List<Entity> find(Entity path, FindInformationInterface findInformation, User user) {
+        logger.debug("find() starting");
+        List<Entity> returns = new ArrayList<Entity>();
+        logger.debug("find() finished");
+        return returns;
+    }
+
+    @Override
+    public Result runQuery(User user, Query query, Result result) throws ResourceInterfaceException {
+        logger.debug("runQuery() *** STARTING ***");
+
+        if (result == null)
+            logger.error("runQuery() `result` object is null, still.");
+
+
+        List<WhereClause> whereClauses = query.getClausesOfType(WhereClause.class);
+        result.setResultStatus(ResultStatus.CREATED);
+        result.setMessage("Started running the query.");
+
+        // Convert the predicate fields into variables on the Hail request
+        // These variables will be used when rendering the Hail template
+        // into an actual script.
+        Map<String, String> hailVariables = new HashMap<String, String>();
+        for (WhereClause whereClause : whereClauses) {
+
+            for (String fieldName : whereClause.getStringValues().keySet()) {
+                hailVariables.put(fieldName, whereClause.getStringValues().get(fieldName));
+                logger.debug("runQuery() field:" + fieldName + "=" + whereClause.getStringValues().get(fieldName));
+            }
+
+            // Convert the predicate value to a hail template
+            String hailTemplate = whereClause.getPredicateType().getName();
+            logger.debug("runQuery() hailTemplate:" + hailTemplate);
+            hailVariables.put("template", hailTemplate);
+
+            // Use the pui from the where clause to set the study name in the list of variables.
+            hailVariables.put("study", Utility.getURLFromPui(whereClause
+                    .getField()
+                    .getPui(), resourceName));
+        }
+
+        // hailVariables now contains at least 'template' and 'study' fields, but not necessarily with valid values
+//TODO Create a session
+        // Send the JSON request to the remote datasource, as an HTTP POST, with
+        // `variables` as the body of the request.
+        logger.debug("runQuery() starting hail job submission");
+        Date starttime = new Date();
+        //TODO Read pyspark template file and substitute placeholders with hailVaiables (gene, significance and sample ids)
+        JsonNode nd = restPOST(this.resourceURL + "/statements", hailVariables);
+        logger.debug("runQUery() hail job submission finished");
+
+        // Parse JSON and evaluate if this is an error, or whatnot
+        HailResponse hailResponse = new HailResponse(nd);
+        if (hailResponse.isError()) {
+            logger.error("runQuery() Hail job failed, due to " + hailResponse.getErrorMessage() + ".");
+            result.setResultStatus(ResultStatus.ERROR);
+            result.setMessage(hailResponse.getErrorMessage());
+        } else {
+            logger.debug("runQuery() Hail job started. UUID:" + hailResponse.getJobUUID());
+            result.setStartTime(starttime);
+            result.setResourceActionId(hailResponse.getJobUUID());
+            result.setResultStatus(ResultStatus.RUNNING);
+            result.setMessage(hailResponse.getHailMessage());
+        }
+
+        logger.debug("runQuery() finished");
+        return result;
+    }
+
+    @Override
+    public Result getResults(User user, Result result) throws ResourceInterfaceException {
+        logger.debug("getResults() starting");
+
+        String hailJobUUID = result.getResourceActionId();
+        logger.debug("getResults() getting result for " + hailJobUUID);
+
+        //TODO Get result from the output json file
+        JsonNode nd = restGET(resourceURL + "/statemnts" + hailJobUUID);
+        HailResponse hailResponse = new HailResponse(nd);
+        logger.debug("getResults() finished parsing Hail response.");
+
+        if (hailResponse.isError()) {
+            logger.debug("getResults() Hail error message:" + hailResponse.getErrorMessage());
+            result.setResultStatus(ResultStatus.ERROR);
+            result.setMessage(hailResponse.getErrorMessage());
+        } else {
+            logger.debug("getResults() jobStatus: " + hailResponse.getJobStatus());
+
+            if (hailResponse.getJobStatus().equalsIgnoreCase("running")) {
+                result.setResultStatus(ResultStatus.RUNNING);
+            }
+            if (hailResponse.getJobStatus().equalsIgnoreCase("finished")) {
+                logger.debug("getResults() setting result status to COMPLETE");
+                result.setResultStatus(ResultStatus.COMPLETE);
+
+                // Parse and persist the data
+                nd = restGET(resourceURL + "/data?id=" + result.getResourceActionId());
+                logger.debug("getResults() parsing returned actual data");
+                try {
+                    parseData(result, nd);
+                } catch (PersistableException pe) {
+                    logger.error("getResults() Unable to persist data");
+                    result.setResultStatus(ResultStatus.ERROR);
+                    result.setMessage(pe.getMessage());
+                } catch (ResultSetException re) {
+                    logger.error("getResults() Cannot parse HAIL response");
+                    result.setResultStatus(ResultStatus.ERROR);
+                    result.setMessage(re.getMessage());
+                }
+            }
+        }
+        logger.debug("getResults() finished");
+        return result;
+    }
+
+    @Override
+    public ResourceState getState() {
+
+
+//TODO Make call to the livy API and map livy state to the ResourceState
+        return resourceState;
+    }
+
+    @Override
+    public ResultDataType getQueryDataType(Query query) {
+        return ResultDataType.TABULAR;
+    }
+
+    private InputStream simpleRestCall(String urlString, Map<String, String> payload) {
+        logger.debug("simpleRestCall() Starting");
+
+        HttpEntity restEntity = null;
+        CloseableHttpClient restClient = HttpClientBuilder.create().build();
+        URIBuilder builder = null;
+        HttpGet get = null;
+        try {
+            builder = new URIBuilder(urlString);
+            for (String fieldName : payload.keySet()) {
+                logger.debug("simpleRestCall() add `" + fieldName + "` to payload as `" + payload.get(fieldName) + "`.");
+                builder.setParameter(fieldName, payload.get(fieldName));
+            }
+            get = new HttpGet(builder.build());
+            get.addHeader("Content-Type", ContentType.APPLICATION_JSON.toString());
+        } catch (URISyntaxException e) {
+            throw new RuntimeException("Invalid URL generated:" + urlString);
+        }
+        CloseableHttpResponse restResponse = null;
+        try {
+            restResponse = restClient.execute(get);
+            restEntity = restResponse.getEntity();
+            // https://stackoverflow.com/questions/15969037/why-did-the-author-use-entityutils-consumehttpentity#15970985
+            EntityUtils.consume(restEntity);
+            logger.debug("simpleRestCall() released entity resource.");
+        } catch (IOException ex) {
+            logger.error("simpleRestCall() IOException: Cannot execute POST with URL: " + urlString);
+        } finally {
+            try {
+                if (restResponse != null)
+                    restResponse.close();
+            } catch (Exception ex) {
+                logger.error("simpleRestCall() finallyException: " + ex.getMessage());
+            }
+        }
+        logger.debug("simpleRestCall() finished.");
+
+        if (restEntity != null) {
+            try {
+                return restEntity.getContent();
+            } catch (IOException ioex) {
+                logger.error("simpleRestCall() Exception:" + ioex);
+
+            } finally {
+
+            }
+        }
+        return null;
+    }
+
+    private InputStream simpleRestCall(String urlString) {
+        logger.debug("restCall() Starting");
+        HttpEntity restEntity = null;
+        CloseableHttpClient restClient = IRCTApplication.CLOSEABLE_HTTP_CLIENT;
+
+        HttpGet get = new HttpGet(urlString);
+        get.addHeader("Content-Type", ContentType.APPLICATION_JSON.toString());
+
+        try (CloseableHttpResponse restResponse = restClient.execute(get)) {
+
+            restEntity = restResponse.getEntity();
+
+            // https://stackoverflow.com/questions/15969037/why-did-the-author-use-entityutils-consumehttpentity#15970985
+            EntityUtils.consume(restEntity);
+            logger.debug("restCall() released entity resource.");
+        } catch (IOException ex) {
+            logger.error("restCall() IOException: Cannot execute POST with URL: " + urlString);
+        }
+        logger.debug("restCall() finished.");
+
+        if (restEntity != null) {
+            try {
+                return restEntity.getContent();
+            } catch (IOException ioex) {
+                logger.error("restCall() Exception:" + ioex);
+
+            } finally {
+
+            }
+        }
+        return null;
+    }
+
+    /**
+     * HTTP POST with JSON body, constructed from `payload` Map, and parse the
+     * returned stream into a JsonNode object for later parsing. Protocoll errors
+     * are captured as well.
+     * Any protocol or parsing error will be thrown as RuntimeException
+     *
+     * @param urlString
+     * @param payload
+     * @return
+     * @throws JSONException
+     * @throws IOException
+     */
+
+    //TODO Change/overload call to receive arbitrary body content
+    private JsonNode restPOST(String urlString, Map<String, String> payload) {
+        logger.debug("restPOST() Starting ");
+        JsonNode responseObject = null;
+
+
+        ObjectMapper objectMapper = IRCTApplication.objectMapper;
+        CloseableHttpClient restClient = IRCTApplication.CLOSEABLE_HTTP_CLIENT;
+
+
+        HttpPost post = new HttpPost((urlString));
+        post.addHeader("Content-Type", ContentType.APPLICATION_JSON.toString());
+
+        try {
+            post.setEntity(
+                    new StringEntity(objectMapper
+                            .writeValueAsString(payload)));
+        } catch (JsonProcessingException e) {
+            throw new ResourceInterfaceException("Hail - restPOST() cannot parse payload map to json string for request body: " + payload + ", with message: " + e.getMessage());
+        } catch (UnsupportedEncodingException e) {
+            throw new ResourceInterfaceException("Hail - restPOST() the encoding is not supported by apache httppost: " + e.getMessage());
+        }
+
+
+        try (CloseableHttpResponse restResponse = restClient.execute(post)) {
+
+            if (restResponse.getStatusLine().getStatusCode() != 200) {
+                logger.error("restPost() Error status response from RESTful call:" + restResponse.getStatusLine().getStatusCode());
+            }
+            if (restResponse == null) {
+                logger.error("restPOST() restResponse is null");
+            }
+            HttpEntity restEntity = restResponse.getEntity();
+            if (restEntity == null) {
+                logger.error("restEntity is null");
+            }
+
+            if (restResponse.getLastHeader("Content-type").getValue().equalsIgnoreCase("application/json")) {
+                // Convert JSON response into Java object
+                responseObject = objectMapper
+                        .readTree(restEntity
+                                .getContent());
+
+            } else {
+                logger.debug("restPOST() Response is not JSON mime type.");
+                JsonNodeFactory factory = JsonNodeFactory.instance;
+                ObjectNode rootNode = factory.objectNode();
+
+                rootNode.put("status", "ok");
+                rootNode.put("message", "nonJSON data received");
+                rootNode.put("data", EntityUtils.toString(restEntity));
+
+            }
+            logger.debug("restPOST() finished parsing data");
+
+            // https://stackoverflow.com/questions/15969037/why-did-the-author-use-entityutils-consumehttpentity#15970985
+            EntityUtils.consume(restEntity);
+            logger.debug("restPOST() released entity resource.");
+        } catch (IOException e) {
+            throw new ResourceInterfaceException("Hail - restPost() Could not connect to resource URL." + e.getMessage());
+        }
+
+        logger.debug("restPOST() finished.");
+        return responseObject;
+    }
+
+    /**
+     * HTTP GET, parse the returned stream into a JsonNode object for
+     * later parsing.
+     *
+     * @param urlString
+     * @return
+     * @throws ResourceInterfaceException
+     */
+    private JsonNode restGET(String urlString) throws ResourceInterfaceException {
+        logger.debug("restGET() Starting");
+        JsonNode responseObject = null;
+
+        CloseableHttpResponse restResponse = null;
+        try {
+            ObjectMapper objectMapper = IRCTApplication.objectMapper;
+            CloseableHttpClient restClient = HttpClientBuilder.create().build();
+
+            HttpGet get = new HttpGet((urlString));
+            restResponse = restClient.execute(get);
+
+            if (restResponse == null) {
+                logger.error("restResponse is null");
+                throw new ResourceInterfaceException("Could not get Hail response");
+            }
+
+            if (restResponse.getStatusLine().getStatusCode() != 200) {
+                throw new ResourceInterfaceException("Could not get Hail response (" + restResponse.getStatusLine().getStatusCode() + ")");
+            }
+            HttpEntity restEntity = restResponse.getEntity();
+            if (restEntity == null) {
+                logger.error("restEntity is null");
+                throw new ResourceInterfaceException("Could not get Hail response");
+            }
+
+            if (restEntity.getContentType().getValue().equalsIgnoreCase("application/json")) {
+                // Convert JSON response into Java object
+                responseObject = objectMapper
+                        .readTree(restEntity
+                                .getContent());
+            } else {
+                // Process non JSON data, which happens if we are streaming down actual data
+                System.out.println(EntityUtils.toString(restEntity));
+            }
+            logger.debug("restGET() finished parsing data");
+
+            // https://stackoverflow.com/questions/15969037/why-did-the-author-use-entityutils-consumehttpentity#15970985
+            EntityUtils.consume(restEntity);
+            logger.debug("restGET() released entity resource.");
+        } catch (IOException ex) {
+            logger.error("restGET() IOException: " + urlString + " " + ex.getMessage());
+            throw new ResourceInterfaceException("Could not get Hail response (" + ex.getMessage() + ")");
+        } finally {
+            try {
+                if (restResponse != null)
+                    restResponse.close();
+            } catch (Exception ex) {
+                logger.error("restGET() finallyException: " + ex.getMessage());
+            }
+        }
+        logger.debug("restGET() finished.");
+        return responseObject;
+    }
+
+    private void parseData(Result result, JsonNode responseJsonNode)
+            throws PersistableException, ResultSetException {
+        FileResultSet frs = (FileResultSet) result.getData();
+
+        //Expected to be a string in TSV format.
+        //If not, a ResultSetException will most likely occur while appending rows or columns
+        String tsv = responseJsonNode.asText();
+        //first line is header, data starts at second line
+        String[] allRows = tsv.split("\n");
+        String[] headers = allRows[0].split("\t");
+        for (int i = 0; i < headers.length; i++) {
+            frs.appendColumn(new Column(headers[i], PrimitiveDataType.STRING));
+        }
+        for (int i = 1; i < allRows.length; i++) {
+            String[] row = allRows[i].split("\t");
+            frs.appendRow();
+            for (int j = 0; j < row.length; j++) {
+                frs.updateString(j, row[j]);
+            }
+        }
+
+        result.setData(frs);
+        result.setDataType(ResultDataType.TABULAR);
+        result.setMessage("Data has been downloaded");
+
+    }
+
+    class PathElement {
+        String pui;
+
+    }
+
+    class HailResponse {
+
+        private String errorMessage = "";
+        private String hailMessage = "";
+        private String jobStatus = "";
+        private Data hailData = null;
+
+        public HailResponse(JsonNode rootNode) {
+            logger.debug("HailResponse() constructor");
+
+            if (rootNode.get("status") == null) {
+                logger.error("HailResponse() 'status' field is mandatory, but it is missing.");
+            } else {
+                logger.debug("HailResponse() 'status' field has data in it.");
+                // Start parsing a Hail response
+                if (rootNode.get("status").textValue().equalsIgnoreCase("ok")) {
+                    logger.debug("HailResponse() Success message from Hail.");
+
+                    // Success response from Hail
+                    if (rootNode.get("job") != null) {
+                        // If this is a job response
+                        logger.debug("HailResponse() parse job details.");
+                        this.jobUUID = rootNode.get("job").get("id").textValue();
+                        this.hailMessage = rootNode.get("job").get("state").textValue();
+                        this.jobStatus = rootNode.get("job").get("state").textValue();
+
+                    } else {
+                        logger.debug("HailResponse() parse data details.");
+                        this.hailMessage = "Parsing Hail data";
+                    }
+
+                } else {
+                    logger.debug("HailResponse() Error message from Hail" + rootNode.get("message").textValue());
+
+                    // Error response from Hail
+                    this.errorMessage = rootNode.get("message").textValue();
+                }
+            }
+            logger.debug("HailResponse() finished");
+        }
+
+        public String getJobUUID() {
+            return jobUUID;
+        }
+
+        private String jobUUID = "";
+
+        public boolean isError() {
+            return !this.errorMessage.isEmpty();
+        }
+
+        public String getErrorMessage() {
+            return this.errorMessage;
+        }
+
+        public void setError(String errorMsg) {
+            this.errorMessage = errorMsg;
+        }
+
+        public Data getData() {
+            return this.hailData;
+        }
+
+        public String getJobStatus() {
+            return jobStatus;
+        }
+
+        public String getHailMessage() {
+            return hailMessage;
+        }
+
+
+    }
+}

From 8ef3f2c938718c1127c43fdd3d40e93f2c9d21bb Mon Sep 17 00:00:00 2001
From: Denise Kersjes <denise.kersjes@hotmail.com>
Date: Fri, 10 Aug 2018 14:08:55 +0200
Subject: [PATCH 02/27] Add Hail resource

---
 .../irct/sql/hail_resource_function.sql       | 69 +++++++++++++++++++
 1 file changed, 69 insertions(+)
 create mode 100644 deployments/irct/sql/hail_resource_function.sql

diff --git a/deployments/irct/sql/hail_resource_function.sql b/deployments/irct/sql/hail_resource_function.sql
new file mode 100644
index 00000000..a0f58e3e
--- /dev/null
+++ b/deployments/irct/sql/hail_resource_function.sql
@@ -0,0 +1,69 @@
+CREATE OR REPLACE FUNCTION add_hail_resource(
+  resourceName varchar,
+  resourceURL varchar
+) RETURNS VOID AS $$
+
+
+DECLARE
+  -- SET THE RESOURCE VARIABLES
+  resourceId integer := (select COALESCE(max(id), 1) from IRCT_Resource) + 1;
+  --resourceName varchar := ( COALESCE(resourceName, '') = '', '{{ resourceName }}', resourceName);
+  --resourceURL varchar := ( COALESCE(resourceURL, '') = '', '{{ resourceURL}}', resourceURL);
+
+  -- SET THE RESOURCE PREDICATES
+  predicatetype_filter_id integer := (select COALESCE(max(id), 1) from IRCT_PredicateType) + 1;
+
+  -- SET THE FIELDS
+  gene_field_id integer := (select COALESCE(max(id), 0) from IRCT_Field) + 1;
+  sign_field_id integer := gene_field_id + 1;
+  subject_field_id integer := sign_field_id + 1;
+
+BEGIN
+  -- INSERT THE RESOURCE VARIABLE
+  insert into IRCT_Resource(id, implementingInterface, name, ontologyType) values
+    (resourceId, 'edu.harvard.hms.dbmi.bd2k.picsure.ri.LivyHAIL', resourceName, 'TREE');
+
+  -- INSERT THE RESOURCE PARAMETERS
+  insert into IRCT_resource_parameters(id, name, value) values(resourceId, 'resourceName', resourceName);
+  insert into IRCT_resource_parameters(id, name, value) values(resourceId, 'resourceURL', resourceURL);
+
+  -- INSERT RESOURCE DATA TYPES
+  ---- insert into PredicateType_dataTypes(PredicateType_id, dataTypes) values (predicate_type_id, 'edu.harvard.hms.dbmi.bd2k.irct.model.resource.PrimitiveDataType:STRING');
+  --insert into PredicateType_dataTypes(PredicateType_id, dataTypes) values
+  --  (predicatetype_filter_id, 'edu.harvard.hms.dbmi.bd2k.irct.model.resource.PrimitiveDataType:STRING');
+
+  -- INSERT RESOURCE RELATIONSHIPS
+
+  -- INSERT RESOURCE LogicalOperators
+
+
+  -- FILTER predicate
+  insert into IRCT_PredicateType(id, defaultPredicate, description, displayName, name) values
+    (predicatetype_filter_id, false, 'Execute filter predicate', 'filter', 'filter');
+  insert into IRCT_Resource_PredicateType(Resource_Id, supportedPredicates_id) values
+    (resourceId, predicatetype_filter_id);
+
+  -- INSERT FIELDS
+  insert into IRCT_Field(id, description, name, path, required, relationship) values
+    (gene_field_id, 'Gene', 'Gene', 'gene',false,NULL);
+  insert into IRCT_Field_dataTypes(Field_id, dataTypes)
+    values(gene_field_id, 'edu.harvard.hms.dbmi.bd2k.irct.model.resource.PrimitiveDataType:STRING');
+  insert into IRCT_PredicateType_Field(PredicateType_id, fields_id) values
+    (predicatetype_filter_id, gene_field_id);
+
+  insert into IRCT_Field(id, description, name, path, required, relationship) values
+    (sign_field_id, 'Significance', 'Significance', 'significance',false,NULL);
+  insert into IRCT_Field_dataTypes(Field_id, dataTypes)
+    values(sign_field_id, 'edu.harvard.hms.dbmi.bd2k.irct.model.resource.PrimitiveDataType:STRING');
+  insert into IRCT_PredicateType_Field(PredicateType_id, fields_id) values
+    (predicatetype_filter_id, sign_field_id);
+
+  insert into IRCT_Field(id, description, name, path, required, relationship) values
+    (subject_field_id, 'Subject ID', 'Subject ID', 'subject_id',false,NULL);
+  insert into IRCT_Field_dataTypes(Field_id, dataTypes)
+    values(subject_field_id, 'edu.harvard.hms.dbmi.bd2k.irct.model.resource.PrimitiveDataType:STRING');
+  insert into IRCT_PredicateType_Field(PredicateType_id, fields_id) values
+    (predicatetype_filter_id, subject_field_id);
+
+END;
+$$ LANGUAGE plpgsql

From 03cb499615e22ca1cc758c1b0c25a3be34613a36 Mon Sep 17 00:00:00 2001
From: Denise Kersjes <denise.kersjes@hotmail.com>
Date: Wed, 22 Aug 2018 11:57:23 +0200
Subject: [PATCH 03/27] Suppress the warning of duplication code (from
 HAIL.java)

---
 .../main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java | 1 +
 1 file changed, 1 insertion(+)

diff --git a/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java b/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
index b89239a9..ca6ccf5c 100644
--- a/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
+++ b/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
@@ -49,6 +49,7 @@
 /**
  * A resource implementation of a data source that communicates with a HAIL proxy via HTTP
  */
+@SuppressWarnings("Duplicates")
 public class LivyHAIL implements QueryResourceImplementationInterface,
         PathResourceImplementationInterface {
     Logger logger = Logger.getLogger(this.getClass());

From 8905157ea78da4cf5df5a979ba7f4ce0292856d4 Mon Sep 17 00:00:00 2001
From: jeremy <jeremy.geo@gmail.com>
Date: Wed, 22 Aug 2018 16:31:37 +0200
Subject: [PATCH 04/27] ignore

---
 .gitignore | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/.gitignore b/.gitignore
index 62068f66..341ab484 100644
--- a/.gitignore
+++ b/.gitignore
@@ -15,6 +15,9 @@ scratch/
 .DS_Store
 
 deploy.sh
-IRCT-RI/src/main/.DS_Store
-.idea/
-*.iml
+IRCT-RI/src/main/.DS_Store
+.idea/
+*.iml
+
+deployments/hail/data/*
+!deployments/hail/data/.gitkeep
\ No newline at end of file

From d65f0c9bf82c9be858e5c29a470721f00c4a34e0 Mon Sep 17 00:00:00 2001
From: Denise Kersjes <denise.kersjes@hotmail.com>
Date: Thu, 23 Aug 2018 11:31:13 +0200
Subject: [PATCH 05/27] Create a Livy session via IRCT

---
 .../hms/dbmi/bd2k/picsure/ri/LivyHAIL.java       | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java b/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
index ca6ccf5c..74816d0f 100644
--- a/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
+++ b/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
@@ -219,8 +219,7 @@ public Result runQuery(User user, Query query, Result result) throws ResourceInt
         result.setMessage("Started running the query.");
 
         // Convert the predicate fields into variables on the Hail request
-        // These variables will be used when rendering the Hail template
-        // into an actual script.
+        // These variables will be used when rendering the Hail template into an actual script.
         Map<String, String> hailVariables = new HashMap<String, String>();
         for (WhereClause whereClause : whereClauses) {
 
@@ -240,14 +239,17 @@ public Result runQuery(User user, Query query, Result result) throws ResourceInt
                     .getPui(), resourceName));
         }
 
+        //TODO Create a session
+        // Initialize a new session
+        JsonNode sessionResponse = restPOST(this.resourceURL + "/sessions", new HashMap<String, String>());
+        logger.info(sessionResponse.get("id"));
+
         // hailVariables now contains at least 'template' and 'study' fields, but not necessarily with valid values
-//TODO Create a session
-        // Send the JSON request to the remote datasource, as an HTTP POST, with
-        // `variables` as the body of the request.
+        // Send the JSON request to the remote datasource, as an HTTP POST, with `variables` as the body of the request.
         logger.debug("runQuery() starting hail job submission");
         Date starttime = new Date();
         //TODO Read pyspark template file and substitute placeholders with hailVaiables (gene, significance and sample ids)
-        JsonNode nd = restPOST(this.resourceURL + "/statements", hailVariables);
+        JsonNode nd = restPOST(this.resourceURL + "/sessions", hailVariables);
         logger.debug("runQUery() hail job submission finished");
 
         // Parse JSON and evaluate if this is an error, or whatnot
@@ -276,7 +278,7 @@ public Result getResults(User user, Result result) throws ResourceInterfaceExcep
         logger.debug("getResults() getting result for " + hailJobUUID);
 
         //TODO Get result from the output json file
-        JsonNode nd = restGET(resourceURL + "/statemnts" + hailJobUUID);
+        JsonNode nd = restGET(resourceURL + "/statements" + hailJobUUID);
         HailResponse hailResponse = new HailResponse(nd);
         logger.debug("getResults() finished parsing Hail response.");
 

From 81c21f47a06ad9cfae9ef81237ad85ad61e08479 Mon Sep 17 00:00:00 2001
From: Denise Kersjes <denise.kersjes@hotmail.com>
Date: Thu, 23 Aug 2018 11:45:54 +0200
Subject: [PATCH 06/27] Create a livy session via IRCT

---
 .../main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java | 1 -
 1 file changed, 1 deletion(-)

diff --git a/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java b/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
index 74816d0f..c757f2d8 100644
--- a/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
+++ b/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
@@ -239,7 +239,6 @@ public Result runQuery(User user, Query query, Result result) throws ResourceInt
                     .getPui(), resourceName));
         }
 
-        //TODO Create a session
         // Initialize a new session
         JsonNode sessionResponse = restPOST(this.resourceURL + "/sessions", new HashMap<String, String>());
         logger.info(sessionResponse.get("id"));

From c52fc83fcfb8b81c2d40b1514412a7c9679f2d6f Mon Sep 17 00:00:00 2001
From: Denise Kersjes <denise.kersjes@hotmail.com>
Date: Fri, 24 Aug 2018 10:04:55 +0200
Subject: [PATCH 07/27] Create a POST request with a PySpark template
 containing Hail variables

---
 .../hms/dbmi/bd2k/picsure/ri/LivyHAIL.java    | 47 ++++++++++++++++---
 1 file changed, 41 insertions(+), 6 deletions(-)

diff --git a/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java b/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
index c757f2d8..6e60f5bc 100644
--- a/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
+++ b/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
@@ -62,6 +62,10 @@ public class LivyHAIL implements QueryResourceImplementationInterface,
 
     protected ResourceState resourceState;
 
+    protected String SessionID;
+
+    private String dataFile = "~/glowing-bear-backend/deployments/hail/data/example_data_PMC.maf";
+
 //    Map<Entity> allPathEntities;
 
     @Override
@@ -90,10 +94,15 @@ public void setup(Map<String, String> parameters) throws ResourceInterfaceExcept
             throw new ResourceInterfaceException("Hail Interface setup() is missing:" + errorString);
         }
 
+        // Initialize a new session
+        JsonNode sessionResponse = restPOST(this.resourceURL + "/sessions", new HashMap<String, String>());
+        SessionID = sessionResponse.get("id").toString();
+
         resourceState = ResourceState.READY;
         logger.debug("setup() for " + resourceName +
                 " Finished. " + resourceName +
                 " is in READY state.");
+
     }
 
     @Override
@@ -237,18 +246,21 @@ public Result runQuery(User user, Query query, Result result) throws ResourceInt
             hailVariables.put("study", Utility.getURLFromPui(whereClause
                     .getField()
                     .getPui(), resourceName));
-        }
 
-        // Initialize a new session
-        JsonNode sessionResponse = restPOST(this.resourceURL + "/sessions", new HashMap<String, String>());
-        logger.info(sessionResponse.get("id"));
+            // Add the dataset to the haiLVariables
+            hailVariables.put("dataset", dataFile);
+        }
 
         // hailVariables now contains at least 'template' and 'study' fields, but not necessarily with valid values
         // Send the JSON request to the remote datasource, as an HTTP POST, with `variables` as the body of the request.
         logger.debug("runQuery() starting hail job submission");
         Date starttime = new Date();
-        //TODO Read pyspark template file and substitute placeholders with hailVaiables (gene, significance and sample ids)
-        JsonNode nd = restPOST(this.resourceURL + "/sessions", hailVariables);
+
+        // Read the PySpark template file
+        Map<java.lang.String,java.lang.String> filledTemplate = GenerateQuery(hailVariables);
+
+        JsonNode nd = restPOST(this.resourceURL + "/sessions/" + SessionID + "/statements", filledTemplate);
+
         logger.debug("runQUery() hail job submission finished");
 
         // Parse JSON and evaluate if this is an error, or whatnot
@@ -580,6 +592,29 @@ private void parseData(Result result, JsonNode responseJsonNode)
 
     }
 
+    private java.util.Map GenerateQuery(Map variables) {
+
+        // Get the specified variables out of the HashMap hailVariables
+        Object dataSet = variables.get("dataset");
+        Object gene = variables.get("gene");
+        Object significance = variables.get("significance");
+        Object subjectIds = variables.get("subject_id");
+
+        // Fill in the template with the desired variables
+        String template = "mt = hl.import_table('" + dataSet + "')\n" +
+                "new_data = mt.filter(mt.Hugo_symbol=='" + gene + "')&" +
+                "(mt.CLIN_SIG=='" + significance + "')&" +
+                "(mt.Tumor_Sample_Barcode=='" + subjectIds + "')\n" +
+                "new_data.show()";
+
+        // Create a HashMap to specify where the data code can be found for the post request
+        Map<java.lang.String, java.lang.String> filledTemplate = new HashMap();
+        filledTemplate.put("code", template);
+
+        return filledTemplate;
+
+    }
+
     class PathElement {
         String pui;
 

From ceeafc978f1133ce6cb380fa960879b5e66fce5d Mon Sep 17 00:00:00 2001
From: Denise Kersjes <denise.kersjes@hotmail.com>
Date: Mon, 27 Aug 2018 09:51:14 +0200
Subject: [PATCH 08/27] Map the Livy state to the ResourceState

---
 .../hms/dbmi/bd2k/picsure/ri/LivyHAIL.java    | 60 ++++++++++++++-----
 1 file changed, 44 insertions(+), 16 deletions(-)

diff --git a/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java b/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
index 6e60f5bc..5dcdd74c 100644
--- a/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
+++ b/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
@@ -62,7 +62,7 @@ public class LivyHAIL implements QueryResourceImplementationInterface,
 
     protected ResourceState resourceState;
 
-    protected String SessionID;
+    protected String sessionID;
 
     private String dataFile = "~/glowing-bear-backend/deployments/hail/data/example_data_PMC.maf";
 
@@ -96,13 +96,12 @@ public void setup(Map<String, String> parameters) throws ResourceInterfaceExcept
 
         // Initialize a new session
         JsonNode sessionResponse = restPOST(this.resourceURL + "/sessions", new HashMap<String, String>());
-        SessionID = sessionResponse.get("id").toString();
+        sessionID = sessionResponse.get("id").toString();
 
         resourceState = ResourceState.READY;
         logger.debug("setup() for " + resourceName +
                 " Finished. " + resourceName +
                 " is in READY state.");
-
     }
 
     @Override
@@ -222,6 +221,19 @@ public Result runQuery(User user, Query query, Result result) throws ResourceInt
         if (result == null)
             logger.error("runQuery() `result` object is null, still.");
 
+        try {
+            // Wait for it to be either ready or fail
+            while (resourceState != ResourceState.COMPLETE) {
+                Thread.sleep(3000);
+                updateResourceState();
+            }
+            result = getResults(user, result);
+            result.setResultStatus(ResultStatus.RUNNING);
+        }
+        catch (InterruptedException | UnsupportedOperationException e) {
+            result.setResultStatus(ResultStatus.ERROR);
+            result.setMessage(e.getMessage());
+        }
 
         List<WhereClause> whereClauses = query.getClausesOfType(WhereClause.class);
         result.setResultStatus(ResultStatus.CREATED);
@@ -257,11 +269,11 @@ public Result runQuery(User user, Query query, Result result) throws ResourceInt
         Date starttime = new Date();
 
         // Read the PySpark template file
-        Map<java.lang.String,java.lang.String> filledTemplate = GenerateQuery(hailVariables);
+        Map<java.lang.String,java.lang.String> filledTemplate = generateQuery(hailVariables);
 
-        JsonNode nd = restPOST(this.resourceURL + "/sessions/" + SessionID + "/statements", filledTemplate);
+        JsonNode nd = restPOST(this.resourceURL + "/sessions/" + sessionID + "/statements", filledTemplate);
 
-        logger.debug("runQUery() hail job submission finished");
+        logger.debug("runQuery() hail job submission finished");
 
         // Parse JSON and evaluate if this is an error, or whatnot
         HailResponse hailResponse = new HailResponse(nd);
@@ -328,12 +340,7 @@ public Result getResults(User user, Result result) throws ResourceInterfaceExcep
     }
 
     @Override
-    public ResourceState getState() {
-
-
-//TODO Make call to the livy API and map livy state to the ResourceState
-        return resourceState;
-    }
+    public ResourceState getState() { return resourceState; }
 
     @Override
     public ResultDataType getQueryDataType(Query query) {
@@ -592,7 +599,7 @@ private void parseData(Result result, JsonNode responseJsonNode)
 
     }
 
-    private java.util.Map GenerateQuery(Map variables) {
+    private java.util.Map generateQuery(Map variables) {
 
         // Get the specified variables out of the HashMap hailVariables
         Object dataSet = variables.get("dataset");
@@ -608,11 +615,32 @@ private java.util.Map GenerateQuery(Map variables) {
                 "new_data.show()";
 
         // Create a HashMap to specify where the data code can be found for the post request
-        Map<java.lang.String, java.lang.String> filledTemplate = new HashMap();
-        filledTemplate.put("code", template);
+        HashMap<java.lang.String, java.lang.String> postTemplate = new HashMap<>();
+        postTemplate.put("code", template);
+
+        return postTemplate;
 
-        return filledTemplate;
+    }
 
+    public void updateResourceState() {
+
+        // Create a dictionary to map the state of Livy to the ResourceState
+        HashMap<String, ResourceState> stateMapping = new HashMap<>();
+            stateMapping.put("not_started", ResourceState.READY);
+            stateMapping.put("starting", ResourceState.RUNNING);
+            stateMapping.put("idle", ResourceState.COMPLETE);
+            stateMapping.put("busy", ResourceState.RUNNING);
+            stateMapping.put("shutting_down", ResourceState.RUNNING);
+            stateMapping.put("error", null);
+            stateMapping.put("dead", null);
+            stateMapping.put("success", ResourceState.COMPLETE);
+
+        // Get the state of the session
+        JsonNode request = restGET(this.resourceURL + "/sessions/" + sessionID);
+        String requestState = request.get("state").asText();
+
+        // Convert the Livy state to the one of IRCT and set the new resource state
+        resourceState = stateMapping.get(requestState);
     }
 
     class PathElement {

From 25ab838d2faf19d18322e29e89738b29f9ed5081 Mon Sep 17 00:00:00 2001
From: Denise Kersjes <denise.kersjes@hotmail.com>
Date: Mon, 27 Aug 2018 10:17:58 +0200
Subject: [PATCH 09/27] Reformat the code

---
 .../hms/dbmi/bd2k/picsure/ri/LivyHAIL.java    | 38 +++++++------------
 1 file changed, 14 insertions(+), 24 deletions(-)

diff --git a/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java b/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
index 5dcdd74c..3779fa9d 100644
--- a/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
+++ b/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
@@ -54,12 +54,10 @@ public class LivyHAIL implements QueryResourceImplementationInterface,
         PathResourceImplementationInterface {
     Logger logger = Logger.getLogger(this.getClass());
 
-
     private static final String PATH_NAME = "pui";
 
     protected String resourceName;
     protected String resourceURL;
-
     protected ResourceState resourceState;
 
     protected String sessionID;
@@ -144,7 +142,6 @@ public List<Entity> getPathRelationship(Entity path, OntologyRelationship relati
                 throw new RuntimeException(e);
             }
 
-
         } else {
             String objectPath = p.substring(p.indexOf('/', 2));
             logger.debug("getPathRelationship() objectPath: " + objectPath);
@@ -162,7 +159,6 @@ public List<Entity> getPathRelationship(Entity path, OntologyRelationship relati
         return entities;
     }
 
-
     private List<Entity> retrieveAllPathTree() {
         String urlString = resourceURL + "/tree";
 
@@ -229,8 +225,7 @@ public Result runQuery(User user, Query query, Result result) throws ResourceInt
             }
             result = getResults(user, result);
             result.setResultStatus(ResultStatus.RUNNING);
-        }
-        catch (InterruptedException | UnsupportedOperationException e) {
+        } catch (InterruptedException | UnsupportedOperationException e) {
             result.setResultStatus(ResultStatus.ERROR);
             result.setMessage(e.getMessage());
         }
@@ -269,7 +264,7 @@ public Result runQuery(User user, Query query, Result result) throws ResourceInt
         Date starttime = new Date();
 
         // Read the PySpark template file
-        Map<java.lang.String,java.lang.String> filledTemplate = generateQuery(hailVariables);
+        Map<java.lang.String, java.lang.String> filledTemplate = generateQuery(hailVariables);
 
         JsonNode nd = restPOST(this.resourceURL + "/sessions/" + sessionID + "/statements", filledTemplate);
 
@@ -340,7 +335,9 @@ public Result getResults(User user, Result result) throws ResourceInterfaceExcep
     }
 
     @Override
-    public ResourceState getState() { return resourceState; }
+    public ResourceState getState() {
+        return resourceState;
+    }
 
     @Override
     public ResultDataType getQueryDataType(Query query) {
@@ -448,11 +445,9 @@ private JsonNode restPOST(String urlString, Map<String, String> payload) {
         logger.debug("restPOST() Starting ");
         JsonNode responseObject = null;
 
-
         ObjectMapper objectMapper = IRCTApplication.objectMapper;
         CloseableHttpClient restClient = IRCTApplication.CLOSEABLE_HTTP_CLIENT;
 
-
         HttpPost post = new HttpPost((urlString));
         post.addHeader("Content-Type", ContentType.APPLICATION_JSON.toString());
 
@@ -466,7 +461,6 @@ private JsonNode restPOST(String urlString, Map<String, String> payload) {
             throw new ResourceInterfaceException("Hail - restPOST() the encoding is not supported by apache httppost: " + e.getMessage());
         }
 
-
         try (CloseableHttpResponse restResponse = restClient.execute(post)) {
 
             if (restResponse.getStatusLine().getStatusCode() != 200) {
@@ -596,7 +590,6 @@ private void parseData(Result result, JsonNode responseJsonNode)
         result.setData(frs);
         result.setDataType(ResultDataType.TABULAR);
         result.setMessage("Data has been downloaded");
-
     }
 
     private java.util.Map generateQuery(Map variables) {
@@ -619,21 +612,20 @@ private java.util.Map generateQuery(Map variables) {
         postTemplate.put("code", template);
 
         return postTemplate;
-
     }
 
-    public void updateResourceState() {
+    private void updateResourceState() {
 
         // Create a dictionary to map the state of Livy to the ResourceState
         HashMap<String, ResourceState> stateMapping = new HashMap<>();
-            stateMapping.put("not_started", ResourceState.READY);
-            stateMapping.put("starting", ResourceState.RUNNING);
-            stateMapping.put("idle", ResourceState.COMPLETE);
-            stateMapping.put("busy", ResourceState.RUNNING);
-            stateMapping.put("shutting_down", ResourceState.RUNNING);
-            stateMapping.put("error", null);
-            stateMapping.put("dead", null);
-            stateMapping.put("success", ResourceState.COMPLETE);
+        stateMapping.put("not_started", ResourceState.READY);
+        stateMapping.put("starting", ResourceState.RUNNING);
+        stateMapping.put("idle", ResourceState.COMPLETE);
+        stateMapping.put("busy", ResourceState.RUNNING);
+        stateMapping.put("shutting_down", ResourceState.RUNNING);
+        stateMapping.put("error", null);
+        stateMapping.put("dead", null);
+        stateMapping.put("success", ResourceState.COMPLETE);
 
         // Get the state of the session
         JsonNode request = restGET(this.resourceURL + "/sessions/" + sessionID);
@@ -645,7 +637,6 @@ public void updateResourceState() {
 
     class PathElement {
         String pui;
-
     }
 
     class HailResponse {
@@ -719,6 +710,5 @@ public String getHailMessage() {
             return hailMessage;
         }
 
-
     }
 }

From 760759e4c6dc7347db940a6b3ab94a31a0eaf5e6 Mon Sep 17 00:00:00 2001
From: Denise Kersjes <denise.kersjes@hotmail.com>
Date: Mon, 27 Aug 2018 15:05:18 +0200
Subject: [PATCH 10/27] Set the session kind to PySpark

---
 .../java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java  | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java b/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
index 3779fa9d..d030dfb8 100644
--- a/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
+++ b/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
@@ -93,7 +93,10 @@ public void setup(Map<String, String> parameters) throws ResourceInterfaceExcept
         }
 
         // Initialize a new session
-        JsonNode sessionResponse = restPOST(this.resourceURL + "/sessions", new HashMap<String, String>());
+        HashMap<String, String> kindSpecified = new HashMap<>();
+        kindSpecified.put("kind" , "pyspark");
+
+        JsonNode sessionResponse = restPOST(this.resourceURL + "/sessions", kindSpecified);
         sessionID = sessionResponse.get("id").toString();
 
         resourceState = ResourceState.READY;

From f067cbda587a72b86ab93d95b47a4169e1d487b3 Mon Sep 17 00:00:00 2001
From: Denise Kersjes <denise.kersjes@hotmail.com>
Date: Tue, 28 Aug 2018 16:24:35 +0200
Subject: [PATCH 11/27] Get the Hail output result from the output JSON file

---
 .../hms/dbmi/bd2k/picsure/ri/LivyHAIL.java    | 59 ++++++++++++-------
 1 file changed, 38 insertions(+), 21 deletions(-)

diff --git a/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java b/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
index d030dfb8..116f7275 100644
--- a/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
+++ b/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
@@ -62,7 +62,7 @@ public class LivyHAIL implements QueryResourceImplementationInterface,
 
     protected String sessionID;
 
-    private String dataFile = "~/glowing-bear-backend/deployments/hail/data/example_data_PMC.maf";
+    private String dataFile = "/app/data/example_data_PMC.maf";
 
 //    Map<Entity> allPathEntities;
 
@@ -226,8 +226,6 @@ public Result runQuery(User user, Query query, Result result) throws ResourceInt
                 Thread.sleep(3000);
                 updateResourceState();
             }
-            result = getResults(user, result);
-            result.setResultStatus(ResultStatus.RUNNING);
         } catch (InterruptedException | UnsupportedOperationException e) {
             result.setResultStatus(ResultStatus.ERROR);
             result.setMessage(e.getMessage());
@@ -298,8 +296,8 @@ public Result getResults(User user, Result result) throws ResourceInterfaceExcep
         String hailJobUUID = result.getResourceActionId();
         logger.debug("getResults() getting result for " + hailJobUUID);
 
-        //TODO Get result from the output json file
-        JsonNode nd = restGET(resourceURL + "/statements" + hailJobUUID);
+        JsonNode nd = restGET(resourceURL + "/sessions/" + sessionID + "/statements/" + hailJobUUID);
+
         HailResponse hailResponse = new HailResponse(nd);
         logger.debug("getResults() finished parsing Hail response.");
 
@@ -310,15 +308,13 @@ public Result getResults(User user, Result result) throws ResourceInterfaceExcep
         } else {
             logger.debug("getResults() jobStatus: " + hailResponse.getJobStatus());
 
-            if (hailResponse.getJobStatus().equalsIgnoreCase("running")) {
+            if (hailResponse.getJobStatus().equalsIgnoreCase("RUNNING")) {
                 result.setResultStatus(ResultStatus.RUNNING);
             }
-            if (hailResponse.getJobStatus().equalsIgnoreCase("finished")) {
+            if (hailResponse.getJobStatus().equalsIgnoreCase("AVAILABLE")) {
                 logger.debug("getResults() setting result status to COMPLETE");
                 result.setResultStatus(ResultStatus.COMPLETE);
 
-                // Parse and persist the data
-                nd = restGET(resourceURL + "/data?id=" + result.getResourceActionId());
                 logger.debug("getResults() parsing returned actual data");
                 try {
                     parseData(result, nd);
@@ -443,7 +439,6 @@ private InputStream simpleRestCall(String urlString) {
      * @throws IOException
      */
 
-    //TODO Change/overload call to receive arbitrary body content
     private JsonNode restPOST(String urlString, Map<String, String> payload) {
         logger.debug("restPOST() Starting ");
         JsonNode responseObject = null;
@@ -575,7 +570,8 @@ private void parseData(Result result, JsonNode responseJsonNode)
 
         //Expected to be a string in TSV format.
         //If not, a ResultSetException will most likely occur while appending rows or columns
-        String tsv = responseJsonNode.asText();
+        String tsv = responseJsonNode.get("output").get("data").asText();
+
         //first line is header, data starts at second line
         String[] allRows = tsv.split("\n");
         String[] headers = allRows[0].split("\t");
@@ -604,10 +600,12 @@ private java.util.Map generateQuery(Map variables) {
         Object subjectIds = variables.get("subject_id");
 
         // Fill in the template with the desired variables
-        String template = "mt = hl.import_table('" + dataSet + "')\n" +
-                "new_data = mt.filter(mt.Hugo_symbol=='" + gene + "')&" +
+        String template = "import hail as hl\n" +
+                "hl.init(sc)\n" +
+                "mt = hl.import_table('" + dataSet + "')\n" +
+                "new_data = mt.filter((mt.Hugo_Symbol=='" + gene + "')&" +
                 "(mt.CLIN_SIG=='" + significance + "')&" +
-                "(mt.Tumor_Sample_Barcode=='" + subjectIds + "')\n" +
+                "(mt.Tumor_Sample_Barcode=='" + subjectIds + "'))\n" +
                 "new_data.show()";
 
         // Create a HashMap to specify where the data code can be found for the post request
@@ -652,25 +650,30 @@ class HailResponse {
         public HailResponse(JsonNode rootNode) {
             logger.debug("HailResponse() constructor");
 
-            if (rootNode.get("status") == null) {
+            if (rootNode.get("state") == null) {
                 logger.error("HailResponse() 'status' field is mandatory, but it is missing.");
             } else {
                 logger.debug("HailResponse() 'status' field has data in it.");
                 // Start parsing a Hail response
-                if (rootNode.get("status").textValue().equalsIgnoreCase("ok")) {
+                if (rootNode.get("state").textValue().equalsIgnoreCase("waiting") ||
+                        rootNode.get("state").textValue().equalsIgnoreCase("running") ||
+                                rootNode.get("state").textValue().equalsIgnoreCase("available")) {
                     logger.debug("HailResponse() Success message from Hail.");
 
                     // Success response from Hail
-                    if (rootNode.get("job") != null) {
+                    if (rootNode.get("output") != null) {
                         // If this is a job response
                         logger.debug("HailResponse() parse job details.");
-                        this.jobUUID = rootNode.get("job").get("id").textValue();
-                        this.hailMessage = rootNode.get("job").get("state").textValue();
-                        this.jobStatus = rootNode.get("job").get("state").textValue();
-
+                        this.jobUUID = rootNode.get("id").toString();
+                        this.hailMessage = rootNode.get("output").textValue();
+                        String state = rootNode.get("state").textValue();
+                        mapResultState(state);
                     } else {
                         logger.debug("HailResponse() parse data details.");
+                        this.jobUUID = rootNode.get("id").toString();
                         this.hailMessage = "Parsing Hail data";
+                        String state = rootNode.get("state").textValue();
+                        mapResultState(state);
                     }
 
                 } else {
@@ -683,6 +686,20 @@ public HailResponse(JsonNode rootNode) {
             logger.debug("HailResponse() finished");
         }
 
+        public void mapResultState(String stateLivy) {
+
+            // Create a dictionary to map the result state of Livy to the one of IRCT
+            HashMap<String, String> resultStateMapping = new HashMap<>();
+            resultStateMapping.put("waiting", "");
+            resultStateMapping.put("running", "RUNNING");
+            resultStateMapping.put("available", "AVAILABLE");
+            resultStateMapping.put("error", "ERROR");
+            resultStateMapping.put("cancelling", "ERROR");
+            resultStateMapping.put("cancelled", "ERROR");
+
+            this.jobStatus = resultStateMapping.get(stateLivy);
+          }
+
         public String getJobUUID() {
             return jobUUID;
         }

From 816530790cd3c5ed368a70402686bdbb92d822be Mon Sep 17 00:00:00 2001
From: Denise Kersjes <denise.kersjes@hotmail.com>
Date: Tue, 28 Aug 2018 18:00:17 +0200
Subject: [PATCH 12/27] Export the data output

---
 .../hms/dbmi/bd2k/picsure/ri/LivyHAIL.java     | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java b/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
index 116f7275..a49d74a9 100644
--- a/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
+++ b/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
@@ -62,7 +62,11 @@ public class LivyHAIL implements QueryResourceImplementationInterface,
 
     protected String sessionID;
 
-    private String dataFile = "/app/data/example_data_PMC.maf";
+    protected String inputFileDir = "/app/data/";
+    protected String outputFileDir = "/app/data/output/";
+    // Choose your desired input file and output file name
+    protected String inputFileName = "/example_data_PMC.maf";
+    protected String outputFileName = "BRCA2_benign";
 
 //    Map<Entity> allPathEntities;
 
@@ -255,8 +259,11 @@ public Result runQuery(User user, Query query, Result result) throws ResourceInt
                     .getField()
                     .getPui(), resourceName));
 
-            // Add the dataset to the haiLVariables
-            hailVariables.put("dataset", dataFile);
+            // Add the desired input file and the output file name to the haiLVariables
+            String inputFile = inputFileDir + inputFileName;
+            String outputFile = outputFileDir + outputFileName;
+            hailVariables.put("dataset", inputFile);
+            hailVariables.put("output_name", outputFile);
         }
 
         // hailVariables now contains at least 'template' and 'study' fields, but not necessarily with valid values
@@ -598,15 +605,16 @@ private java.util.Map generateQuery(Map variables) {
         Object gene = variables.get("gene");
         Object significance = variables.get("significance");
         Object subjectIds = variables.get("subject_id");
+        Object outputDir = variables.get("output_name");
 
         // Fill in the template with the desired variables
         String template = "import hail as hl\n" +
-                "hl.init(sc)\n" +
+                "hl.init(sc, quiet=True, idempotent=True)\n" +
                 "mt = hl.import_table('" + dataSet + "')\n" +
                 "new_data = mt.filter((mt.Hugo_Symbol=='" + gene + "')&" +
                 "(mt.CLIN_SIG=='" + significance + "')&" +
                 "(mt.Tumor_Sample_Barcode=='" + subjectIds + "'))\n" +
-                "new_data.show()";
+                "new_data.export(output='" + outputDir + "', types_file='maf')";
 
         // Create a HashMap to specify where the data code can be found for the post request
         HashMap<java.lang.String, java.lang.String> postTemplate = new HashMap<>();

From 17728a049a8c4e75b8cb67f740969050297c6770 Mon Sep 17 00:00:00 2001
From: Denise Kersjes <denise.kersjes@hotmail.com>
Date: Thu, 30 Aug 2018 16:21:21 +0200
Subject: [PATCH 13/27] Reformat the Hail template to handle multiple variables
 in the fields (gene, significance, subjectID)

---
 .../hms/dbmi/bd2k/picsure/ri/LivyHAIL.java    | 53 ++++++++++++++-----
 1 file changed, 41 insertions(+), 12 deletions(-)

diff --git a/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java b/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
index a49d74a9..b7554ec1 100644
--- a/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
+++ b/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
@@ -65,7 +65,7 @@ public class LivyHAIL implements QueryResourceImplementationInterface,
     protected String inputFileDir = "/app/data/";
     protected String outputFileDir = "/app/data/output/";
     // Choose your desired input file and output file name
-    protected String inputFileName = "/example_data_PMC.maf";
+    protected String inputFileName = "example_data_PMC.maf";
     protected String outputFileName = "BRCA2_benign";
 
 //    Map<Entity> allPathEntities;
@@ -577,7 +577,7 @@ private void parseData(Result result, JsonNode responseJsonNode)
 
         //Expected to be a string in TSV format.
         //If not, a ResultSetException will most likely occur while appending rows or columns
-        String tsv = responseJsonNode.get("output").get("data").asText();
+        String tsv = responseJsonNode.get("output").get("data").get("text/plain").asText();
 
         //first line is header, data starts at second line
         String[] allRows = tsv.split("\n");
@@ -601,20 +601,28 @@ private void parseData(Result result, JsonNode responseJsonNode)
     private java.util.Map generateQuery(Map variables) {
 
         // Get the specified variables out of the HashMap hailVariables
-        Object dataSet = variables.get("dataset");
-        Object gene = variables.get("gene");
-        Object significance = variables.get("significance");
-        Object subjectIds = variables.get("subject_id");
-        Object outputDir = variables.get("output_name");
+        String dataSet = variables.get("dataset").toString();
+        String gene = variables.get("gene").toString();
+        String significance = variables.get("significance").toString();
+        String subjectIds = variables.get("subject_id").toString();
+        String outputDir = variables.get("output_name").toString();
+
+        // Parse multiple variables into one string suitable for the Hail template
+        String genesHailFormat = parseMultipleVariables(gene, "Hugo_Symbol");
+        String significancesHailFormat = parseMultipleVariables(significance, "CLIN_SIG");
+        String idsHailFormat = parseMultipleVariables(subjectIds, "Tumor_Sample_Barcode");
 
         // Fill in the template with the desired variables
-        String template = "import hail as hl\n" +
+        String template = "import warnings\n" +
+                "warnings.filterwarnings('ignore')\n" +
+                "import hail as hl\n" +
                 "hl.init(sc, quiet=True, idempotent=True)\n" +
                 "mt = hl.import_table('" + dataSet + "')\n" +
-                "new_data = mt.filter((mt.Hugo_Symbol=='" + gene + "')&" +
-                "(mt.CLIN_SIG=='" + significance + "')&" +
-                "(mt.Tumor_Sample_Barcode=='" + subjectIds + "'))\n" +
-                "new_data.export(output='" + outputDir + "', types_file='maf')";
+                "new_data = mt.filter((" + genesHailFormat + ") & " +
+                                     "(" + significancesHailFormat + ") & " +
+                                     "(" + idsHailFormat + "))\n" +
+                "new_data.export(output='" + outputDir + "', types_file='maf')\n" +
+                "new_data.to_pandas().to_string()";
 
         // Create a HashMap to specify where the data code can be found for the post request
         HashMap<java.lang.String, java.lang.String> postTemplate = new HashMap<>();
@@ -623,6 +631,27 @@ private java.util.Map generateQuery(Map variables) {
         return postTemplate;
     }
 
+    private String parseMultipleVariables(String variable, String columnName) {
+
+        // The variables are strings that can consist of multiple elements separated by commas
+        String[] splittedVariables = variable.split(",");
+
+        StringBuilder hailFormat = new StringBuilder();
+        int i = 0;
+        // Create for every variable a string in where the column is set to the variable name
+        for (String var : splittedVariables) {
+            String part = "(mt." + columnName + "=='" + var + "')";
+            if (i++ == splittedVariables.length-1) {
+                hailFormat.append(part);
+            } else {
+                // The string should be extended with the or operator until the last element
+                hailFormat.append(part + " | ");
+            }
+        }
+
+        return hailFormat.toString();
+    }
+
     private void updateResourceState() {
 
         // Create a dictionary to map the state of Livy to the ResourceState

From 3593c4fe4b81b62628b6b4b6dd66f4b32dac22bf Mon Sep 17 00:00:00 2001
From: Denise Kersjes <denise.kersjes@hotmail.com>
Date: Fri, 31 Aug 2018 13:23:59 +0200
Subject: [PATCH 14/27] Parse the output data in correct table format

---
 .../hms/dbmi/bd2k/picsure/ri/LivyHAIL.java     | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java b/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
index b7554ec1..c00465ce 100644
--- a/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
+++ b/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
@@ -575,20 +575,26 @@ private void parseData(Result result, JsonNode responseJsonNode)
             throws PersistableException, ResultSetException {
         FileResultSet frs = (FileResultSet) result.getData();
 
-        //Expected to be a string in TSV format.
-        //If not, a ResultSetException will most likely occur while appending rows or columns
+        // Expected to be a string in TSV format.
+        // If not, a ResultSetException will most likely occur while appending rows or columns
         String tsv = responseJsonNode.get("output").get("data").get("text/plain").asText();
+        // Tabs in the tab-separated text are translated to (multiple) spaces after getting it from the JsonNode,
+        // so the spaces are first replaced by a tab
+        String parsedTsv = tsv.replaceAll("^ +| +$|( )+", "\t");
 
-        //first line is header, data starts at second line
-        String[] allRows = tsv.split("\n");
+        // First line is header, data starts at second line
+        String[] allRows = parsedTsv.split("\\\\n");
         String[] headers = allRows[0].split("\t");
-        for (int i = 0; i < headers.length; i++) {
+
+        // The header line started with a tab, so skip the first column
+        for (int i = 1; i < headers.length; i++) {
             frs.appendColumn(new Column(headers[i], PrimitiveDataType.STRING));
         }
         for (int i = 1; i < allRows.length; i++) {
             String[] row = allRows[i].split("\t");
             frs.appendRow();
-            for (int j = 0; j < row.length; j++) {
+            // A data line start with the line number, so the first element is skipped
+            for (int j = 1; j < row.length; j++) {
                 frs.updateString(j, row[j]);
             }
         }

From 0ad0f8802cf768b2d1ac07fd8b6cbf796ef48e93 Mon Sep 17 00:00:00 2001
From: Denise Kersjes <denise.kersjes@hotmail.com>
Date: Tue, 4 Sep 2018 09:21:43 +0200
Subject: [PATCH 15/27] Soft-coded the datafile selection

---
 .../hms/dbmi/bd2k/picsure/ri/LivyHAIL.java      | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java b/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
index c00465ce..6387a70a 100644
--- a/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
+++ b/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
@@ -61,8 +61,9 @@ public class LivyHAIL implements QueryResourceImplementationInterface,
     protected ResourceState resourceState;
 
     protected String sessionID;
+    protected String inputFile;
 
-    protected String inputFileDir = "/app/data/";
+    protected String dataFileDir = "/app/data/";
     protected String outputFileDir = "/app/data/output/";
     // Choose your desired input file and output file name
     protected String inputFileName = "example_data_PMC.maf";
@@ -259,10 +260,11 @@ public Result runQuery(User user, Query query, Result result) throws ResourceInt
                     .getField()
                     .getPui(), resourceName));
 
-            // Add the desired input file and the output file name to the haiLVariables
-            String inputFile = inputFileDir + inputFileName;
+            // Add the desired input file to the haiLVariables, which is at the second position of the PUI
+            inputFile = whereClause.getField().getPui().split("/")[2];
+
+            // TODO remove export file when output through API is as expected
             String outputFile = outputFileDir + outputFileName;
-            hailVariables.put("dataset", inputFile);
             hailVariables.put("output_name", outputFile);
         }
 
@@ -607,23 +609,22 @@ private void parseData(Result result, JsonNode responseJsonNode)
     private java.util.Map generateQuery(Map variables) {
 
         // Get the specified variables out of the HashMap hailVariables
-        String dataSet = variables.get("dataset").toString();
         String gene = variables.get("gene").toString();
         String significance = variables.get("significance").toString();
         String subjectIds = variables.get("subject_id").toString();
         String outputDir = variables.get("output_name").toString();
 
         // Parse multiple variables into one string suitable for the Hail template
-        String genesHailFormat = parseMultipleVariables(gene, "Hugo_Symbol");
+        String genesHailFormat = parseMultipleVariables(gene, "GENE");
         String significancesHailFormat = parseMultipleVariables(significance, "CLIN_SIG");
-        String idsHailFormat = parseMultipleVariables(subjectIds, "Tumor_Sample_Barcode");
+        String idsHailFormat = parseMultipleVariables(subjectIds, "ID");
 
         // Fill in the template with the desired variables
         String template = "import warnings\n" +
                 "warnings.filterwarnings('ignore')\n" +
                 "import hail as hl\n" +
                 "hl.init(sc, quiet=True, idempotent=True)\n" +
-                "mt = hl.import_table('" + dataSet + "')\n" +
+                "mt = hl.import_table(" + inputFile + ")\n" +
                 "new_data = mt.filter((" + genesHailFormat + ") & " +
                                      "(" + significancesHailFormat + ") & " +
                                      "(" + idsHailFormat + "))\n" +

From 21bf1b18af217b18fa3129466ab06aa9c4bf81ca Mon Sep 17 00:00:00 2001
From: Denise Kersjes <denise.kersjes@hotmail.com>
Date: Wed, 5 Sep 2018 11:42:02 +0200
Subject: [PATCH 16/27] Hail and datafiles are imported during the IRCT set up
 instead of during the query

---
 .../hms/dbmi/bd2k/picsure/ri/LivyHAIL.java    | 72 ++++++++++---------
 1 file changed, 40 insertions(+), 32 deletions(-)

diff --git a/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java b/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
index 6387a70a..f56f6223 100644
--- a/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
+++ b/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
@@ -64,10 +64,6 @@ public class LivyHAIL implements QueryResourceImplementationInterface,
     protected String inputFile;
 
     protected String dataFileDir = "/app/data/";
-    protected String outputFileDir = "/app/data/output/";
-    // Choose your desired input file and output file name
-    protected String inputFileName = "example_data_PMC.maf";
-    protected String outputFileName = "BRCA2_benign";
 
 //    Map<Entity> allPathEntities;
 
@@ -97,17 +93,29 @@ public void setup(Map<String, String> parameters) throws ResourceInterfaceExcept
             throw new ResourceInterfaceException("Hail Interface setup() is missing:" + errorString);
         }
 
-        // Initialize a new session
+        // Initialize a new PySpark session
         HashMap<String, String> kindSpecified = new HashMap<>();
         kindSpecified.put("kind" , "pyspark");
-
         JsonNode sessionResponse = restPOST(this.resourceURL + "/sessions", kindSpecified);
         sessionID = sessionResponse.get("id").toString();
 
-        resourceState = ResourceState.READY;
+        try {
+            // Wait for the session to be either ready or fail
+            while (resourceState != ResourceState.COMPLETE) {
+                Thread.sleep(3000);
+                updateResourceState();
+            }
+        } catch (InterruptedException | UnsupportedOperationException e) {
+            logger.error("Session state is not ready");
+        }
+
+        // Initialize hail and load all datafiles as a hail table
+        Map<java.lang.String, java.lang.String> dataTemplate = loadDataTemplate();
+        restPOST(this.resourceURL + "/sessions/" + sessionID + "/statements", dataTemplate);
+
         logger.debug("setup() for " + resourceName +
                 " Finished. " + resourceName +
-                " is in READY state.");
+                " is in COMPLETE state.");
     }
 
     @Override
@@ -225,17 +233,6 @@ public Result runQuery(User user, Query query, Result result) throws ResourceInt
         if (result == null)
             logger.error("runQuery() `result` object is null, still.");
 
-        try {
-            // Wait for it to be either ready or fail
-            while (resourceState != ResourceState.COMPLETE) {
-                Thread.sleep(3000);
-                updateResourceState();
-            }
-        } catch (InterruptedException | UnsupportedOperationException e) {
-            result.setResultStatus(ResultStatus.ERROR);
-            result.setMessage(e.getMessage());
-        }
-
         List<WhereClause> whereClauses = query.getClausesOfType(WhereClause.class);
         result.setResultStatus(ResultStatus.CREATED);
         result.setMessage("Started running the query.");
@@ -262,10 +259,6 @@ public Result runQuery(User user, Query query, Result result) throws ResourceInt
 
             // Add the desired input file to the haiLVariables, which is at the second position of the PUI
             inputFile = whereClause.getField().getPui().split("/")[2];
-
-            // TODO remove export file when output through API is as expected
-            String outputFile = outputFileDir + outputFileName;
-            hailVariables.put("output_name", outputFile);
         }
 
         // hailVariables now contains at least 'template' and 'study' fields, but not necessarily with valid values
@@ -612,7 +605,6 @@ private java.util.Map generateQuery(Map variables) {
         String gene = variables.get("gene").toString();
         String significance = variables.get("significance").toString();
         String subjectIds = variables.get("subject_id").toString();
-        String outputDir = variables.get("output_name").toString();
 
         // Parse multiple variables into one string suitable for the Hail template
         String genesHailFormat = parseMultipleVariables(gene, "GENE");
@@ -620,15 +612,10 @@ private java.util.Map generateQuery(Map variables) {
         String idsHailFormat = parseMultipleVariables(subjectIds, "ID");
 
         // Fill in the template with the desired variables
-        String template = "import warnings\n" +
-                "warnings.filterwarnings('ignore')\n" +
-                "import hail as hl\n" +
-                "hl.init(sc, quiet=True, idempotent=True)\n" +
-                "mt = hl.import_table(" + inputFile + ")\n" +
-                "new_data = mt.filter((" + genesHailFormat + ") & " +
+        String template =
+                "new_data = all_data_files['" + inputFile + "'].filter((" + genesHailFormat + ") & " +
                                      "(" + significancesHailFormat + ") & " +
                                      "(" + idsHailFormat + "))\n" +
-                "new_data.export(output='" + outputDir + "', types_file='maf')\n" +
                 "new_data.to_pandas().to_string()";
 
         // Create a HashMap to specify where the data code can be found for the post request
@@ -647,7 +634,7 @@ private String parseMultipleVariables(String variable, String columnName) {
         int i = 0;
         // Create for every variable a string in where the column is set to the variable name
         for (String var : splittedVariables) {
-            String part = "(mt." + columnName + "=='" + var + "')";
+            String part = "(all_data_files['" + inputFile + "']." + columnName + "=='" + var + "')";
             if (i++ == splittedVariables.length-1) {
                 hailFormat.append(part);
             } else {
@@ -659,6 +646,27 @@ private String parseMultipleVariables(String variable, String columnName) {
         return hailFormat.toString();
     }
 
+    private java.util.Map loadDataTemplate() {
+
+        // Create a PySpark that imports hail and load all datafiles as a hail table
+        String pysparkFormatTemplate =
+                "import warnings\n" +
+                "warnings.filterwarnings('ignore')\n" +
+                "import hail as hl\n" +
+                "hl.init(sc, quiet=True, idempotent=True)\n" +
+                "import os\n" +
+                "all_data_files = {}\n" +
+                "for data_file in os.listdir('" + dataFileDir + "'):\n" +
+                "\tif data_file.endswith('.maf'):\n" +
+                "\t\tall_data_files[data_file] = hl.import_table('" + dataFileDir + "'+data_file)\n";
+
+        // Create a HashMap to specify where the data code can be found for the post request
+        HashMap<java.lang.String, java.lang.String> postDataTemplate = new HashMap<>();
+        postDataTemplate.put("code", pysparkFormatTemplate);
+
+        return postDataTemplate;
+    }
+
     private void updateResourceState() {
 
         // Create a dictionary to map the state of Livy to the ResourceState

From 32dc34d96e085964f672789ce66995c48b9ff88f Mon Sep 17 00:00:00 2001
From: Denise Kersjes <denise.kersjes@hotmail.com>
Date: Wed, 5 Sep 2018 15:32:19 +0200
Subject: [PATCH 17/27] Removed unnecessary code

---
 .../hms/dbmi/bd2k/picsure/ri/LivyHAIL.java    | 203 +-----------------
 1 file changed, 1 insertion(+), 202 deletions(-)

diff --git a/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java b/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
index f56f6223..f6dfff99 100644
--- a/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
+++ b/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
@@ -50,8 +50,7 @@
  * A resource implementation of a data source that communicates with a HAIL proxy via HTTP
  */
 @SuppressWarnings("Duplicates")
-public class LivyHAIL implements QueryResourceImplementationInterface,
-        PathResourceImplementationInterface {
+public class LivyHAIL implements QueryResourceImplementationInterface {
     Logger logger = Logger.getLogger(this.getClass());
 
     private static final String PATH_NAME = "pui";
@@ -65,8 +64,6 @@ public class LivyHAIL implements QueryResourceImplementationInterface,
 
     protected String dataFileDir = "/app/data/";
 
-//    Map<Entity> allPathEntities;
-
     @Override
     public void setup(Map<String, String> parameters) throws ResourceInterfaceException {
 
@@ -123,109 +120,6 @@ public String getType() {
         return "Hail";
     }
 
-    @Override
-    public List<Entity> getPathRelationship(Entity path, OntologyRelationship relationship, User user) {
-        logger.debug("getPathRelationship() Starting");
-        List<Entity> entities = new ArrayList<Entity>();
-
-        // Split the path into components. The first component is the Hail resource name, the rest is
-        // a URL path like string.
-        String p = path.getPui();
-        logger.debug("getPathRelationship() pui:" + p);
-        if (p.indexOf('/', 2) == -1) {
-            // This is a request for the root
-
-            // Call the external URL
-            InputStream is = simpleRestCall(this.resourceURL);
-
-            // Parse the response, as JSON mime type
-            ObjectMapper objectMapper = IRCTApplication.objectMapper;
-            JsonNode responseJsonNode;
-            try {
-                responseJsonNode = objectMapper.readTree(is);
-                String responseStatus = responseJsonNode.get("status").textValue();
-                Entity e = new Entity();
-
-                e.setPui("objectIdVal");
-                e.setName("objectNameVal");
-                e.setDisplayName("objectDisplayNameVal status:" + responseStatus);
-
-                entities.add(e);
-            } catch (JsonMappingException jme) {
-                logger.error("getPathRelationship() Exception:" + jme.getMessage());
-                throw new RuntimeException("Could not parse JSON response from `" + resourceName + "` resource");
-            } catch (IOException e) {
-                throw new RuntimeException(e);
-            }
-
-        } else {
-            String objectPath = p.substring(p.indexOf('/', 2));
-            logger.debug("getPathRelationship() objectPath: " + objectPath);
-
-            Entity e = new Entity();
-
-            e.setPui("objectIdVal");
-            e.setName("objectNameVal");
-            e.setDisplayName("objectDisplayNameVal");
-
-            entities.add(e);
-        }
-
-        logger.debug("getPathRelationship() Finished");
-        return entities;
-    }
-
-    private List<Entity> retrieveAllPathTree() {
-        String urlString = resourceURL + "/tree";
-
-        CloseableHttpClient httpClient = IRCTApplication.CLOSEABLE_HTTP_CLIENT;
-        HttpGet httpGet = new HttpGet(urlString);
-
-        CloseableHttpResponse response = null;
-
-        List<Entity> entities = null;
-
-        try {
-            response = httpClient.execute(httpGet);
-            HttpEntity entity = response.getEntity();
-
-//            entities = parseAllHailPathJsonNode(IRCTApplication.objectMapper
-//                    .readTree(entity
-//                            .getContent()));
-
-            EntityUtils.consume(entity);
-        } catch (IOException ex) {
-            logger.error("IOException when retrieving all path from Hail API:" + urlString +
-                    " with exception message: " + ex.getMessage());
-        } finally {
-            try {
-                if (response != null)
-                    response.close();
-            } catch (IOException ex) {
-                logger.error("GNOME - IOExcpetion when closing http response: " + ex.getMessage());
-            }
-        }
-
-        return entities;
-    }
-
-    /**
-     * @param pathNode
-     * @return null if nothing
-     */
-    private TreeMap<String, JsonNode> parseAllHailPathJsonNode(JsonNode pathNode) {
-        TreeMap<String, JsonNode> entityTreeMap = null;
-        return entityTreeMap;
-    }
-
-    @Override
-    public List<Entity> find(Entity path, FindInformationInterface findInformation, User user) {
-        logger.debug("find() starting");
-        List<Entity> returns = new ArrayList<Entity>();
-        logger.debug("find() finished");
-        return returns;
-    }
-
     @Override
     public Result runQuery(User user, Query query, Result result) throws ResourceInterfaceException {
         logger.debug("runQuery() *** STARTING ***");
@@ -345,89 +239,6 @@ public ResultDataType getQueryDataType(Query query) {
         return ResultDataType.TABULAR;
     }
 
-    private InputStream simpleRestCall(String urlString, Map<String, String> payload) {
-        logger.debug("simpleRestCall() Starting");
-
-        HttpEntity restEntity = null;
-        CloseableHttpClient restClient = HttpClientBuilder.create().build();
-        URIBuilder builder = null;
-        HttpGet get = null;
-        try {
-            builder = new URIBuilder(urlString);
-            for (String fieldName : payload.keySet()) {
-                logger.debug("simpleRestCall() add `" + fieldName + "` to payload as `" + payload.get(fieldName) + "`.");
-                builder.setParameter(fieldName, payload.get(fieldName));
-            }
-            get = new HttpGet(builder.build());
-            get.addHeader("Content-Type", ContentType.APPLICATION_JSON.toString());
-        } catch (URISyntaxException e) {
-            throw new RuntimeException("Invalid URL generated:" + urlString);
-        }
-        CloseableHttpResponse restResponse = null;
-        try {
-            restResponse = restClient.execute(get);
-            restEntity = restResponse.getEntity();
-            // https://stackoverflow.com/questions/15969037/why-did-the-author-use-entityutils-consumehttpentity#15970985
-            EntityUtils.consume(restEntity);
-            logger.debug("simpleRestCall() released entity resource.");
-        } catch (IOException ex) {
-            logger.error("simpleRestCall() IOException: Cannot execute POST with URL: " + urlString);
-        } finally {
-            try {
-                if (restResponse != null)
-                    restResponse.close();
-            } catch (Exception ex) {
-                logger.error("simpleRestCall() finallyException: " + ex.getMessage());
-            }
-        }
-        logger.debug("simpleRestCall() finished.");
-
-        if (restEntity != null) {
-            try {
-                return restEntity.getContent();
-            } catch (IOException ioex) {
-                logger.error("simpleRestCall() Exception:" + ioex);
-
-            } finally {
-
-            }
-        }
-        return null;
-    }
-
-    private InputStream simpleRestCall(String urlString) {
-        logger.debug("restCall() Starting");
-        HttpEntity restEntity = null;
-        CloseableHttpClient restClient = IRCTApplication.CLOSEABLE_HTTP_CLIENT;
-
-        HttpGet get = new HttpGet(urlString);
-        get.addHeader("Content-Type", ContentType.APPLICATION_JSON.toString());
-
-        try (CloseableHttpResponse restResponse = restClient.execute(get)) {
-
-            restEntity = restResponse.getEntity();
-
-            // https://stackoverflow.com/questions/15969037/why-did-the-author-use-entityutils-consumehttpentity#15970985
-            EntityUtils.consume(restEntity);
-            logger.debug("restCall() released entity resource.");
-        } catch (IOException ex) {
-            logger.error("restCall() IOException: Cannot execute POST with URL: " + urlString);
-        }
-        logger.debug("restCall() finished.");
-
-        if (restEntity != null) {
-            try {
-                return restEntity.getContent();
-            } catch (IOException ioex) {
-                logger.error("restCall() Exception:" + ioex);
-
-            } finally {
-
-            }
-        }
-        return null;
-    }
-
     /**
      * HTTP POST with JSON body, constructed from `payload` Map, and parse the
      * returned stream into a JsonNode object for later parsing. Protocoll errors
@@ -688,10 +499,6 @@ private void updateResourceState() {
         resourceState = stateMapping.get(requestState);
     }
 
-    class PathElement {
-        String pui;
-    }
-
     class HailResponse {
 
         private String errorMessage = "";
@@ -766,14 +573,6 @@ public String getErrorMessage() {
             return this.errorMessage;
         }
 
-        public void setError(String errorMsg) {
-            this.errorMessage = errorMsg;
-        }
-
-        public Data getData() {
-            return this.hailData;
-        }
-
         public String getJobStatus() {
             return jobStatus;
         }

From 0069e50678994c5750c440b8d50f459289be2258 Mon Sep 17 00:00:00 2001
From: Denise Kersjes <denise.kersjes@hotmail.com>
Date: Thu, 6 Sep 2018 14:37:38 +0200
Subject: [PATCH 18/27] Handle .tsv files and remove unused import statements

---
 .../harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java  | 13 +++----------
 1 file changed, 3 insertions(+), 10 deletions(-)

diff --git a/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java b/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
index f6dfff99..cecbcae7 100644
--- a/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
+++ b/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
@@ -1,21 +1,16 @@
 package edu.harvard.hms.dbmi.bd2k.picsure.ri;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.JsonMappingException;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.JsonNodeFactory;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import edu.harvard.hms.dbmi.bd2k.irct.IRCTApplication;
 import edu.harvard.hms.dbmi.bd2k.irct.exception.ResourceInterfaceException;
-import edu.harvard.hms.dbmi.bd2k.irct.model.find.FindInformationInterface;
-import edu.harvard.hms.dbmi.bd2k.irct.model.ontology.Entity;
-import edu.harvard.hms.dbmi.bd2k.irct.model.ontology.OntologyRelationship;
 import edu.harvard.hms.dbmi.bd2k.irct.model.query.Query;
 import edu.harvard.hms.dbmi.bd2k.irct.model.query.WhereClause;
 import edu.harvard.hms.dbmi.bd2k.irct.model.resource.PrimitiveDataType;
 import edu.harvard.hms.dbmi.bd2k.irct.model.resource.ResourceState;
-import edu.harvard.hms.dbmi.bd2k.irct.model.resource.implementation.PathResourceImplementationInterface;
 import edu.harvard.hms.dbmi.bd2k.irct.model.resource.implementation.QueryResourceImplementationInterface;
 import edu.harvard.hms.dbmi.bd2k.irct.model.result.Data;
 import edu.harvard.hms.dbmi.bd2k.irct.model.result.Result;
@@ -31,7 +26,6 @@
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.client.methods.HttpPost;
-import org.apache.http.client.utils.URIBuilder;
 import org.apache.http.entity.ContentType;
 import org.apache.http.entity.StringEntity;
 import org.apache.http.impl.client.CloseableHttpClient;
@@ -41,9 +35,7 @@
 import us.monoid.json.JSONException;
 
 import java.io.IOException;
-import java.io.InputStream;
 import java.io.UnsupportedEncodingException;
-import java.net.URISyntaxException;
 import java.util.*;
 
 /**
@@ -401,7 +393,8 @@ private void parseData(Result result, JsonNode responseJsonNode)
             frs.appendRow();
             // A data line start with the line number, so the first element is skipped
             for (int j = 1; j < row.length; j++) {
-                frs.updateString(j, row[j]);
+                // ColumnIndex starts counting at 0
+                frs.updateString(j-1, row[j]);
             }
         }
 
@@ -468,7 +461,7 @@ private java.util.Map loadDataTemplate() {
                 "import os\n" +
                 "all_data_files = {}\n" +
                 "for data_file in os.listdir('" + dataFileDir + "'):\n" +
-                "\tif data_file.endswith('.maf'):\n" +
+                "\tif data_file.endswith('.maf') or data_file.endswith('.tsv'):\n" +
                 "\t\tall_data_files[data_file] = hl.import_table('" + dataFileDir + "'+data_file)\n";
 
         // Create a HashMap to specify where the data code can be found for the post request

From 60407645a6f3e953089e372e83863caab2c70633 Mon Sep 17 00:00:00 2001
From: Denise Kersjes <denise.kersjes@hotmail.com>
Date: Fri, 7 Sep 2018 16:40:14 +0200
Subject: [PATCH 19/27] Add json dependency

---
 IRCT-RI/pom.xml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/IRCT-RI/pom.xml b/IRCT-RI/pom.xml
index 201a09d9..08207147 100644
--- a/IRCT-RI/pom.xml
+++ b/IRCT-RI/pom.xml
@@ -68,6 +68,11 @@
 			<artifactId>jaxp-ri</artifactId>
 			<version>1.4.5</version>
 			<scope>runtime</scope>
+ 		</dependency>
+		<dependency>
+			<groupId>org.json</groupId>
+			<artifactId>json</artifactId>
+			<version>20180813</version>
 		</dependency>
 	</dependencies>
 	<build>

From a2fc02bb20135e2209af2ef45b0f66038158a8d3 Mon Sep 17 00:00:00 2001
From: Denise Kersjes <denise.kersjes@hotmail.com>
Date: Fri, 7 Sep 2018 16:50:31 +0200
Subject: [PATCH 20/27] Solve issue of parsing the output

Previous multiple tabs in a row  where seen as one tab. This issue was solved by converting the output to JSON instead of a string, and rewrite the parser.
---
 .../hms/dbmi/bd2k/picsure/ri/LivyHAIL.java    | 43 +++++++++++--------
 1 file changed, 26 insertions(+), 17 deletions(-)

diff --git a/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java b/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
index cecbcae7..6a25fe0d 100644
--- a/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
+++ b/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
@@ -32,6 +32,8 @@
 import org.apache.http.impl.client.HttpClientBuilder;
 import org.apache.http.util.EntityUtils;
 import org.apache.log4j.Logger;
+import org.json.JSONArray;
+import org.json.JSONObject;
 import us.monoid.json.JSONException;
 
 import java.io.IOException;
@@ -375,26 +377,33 @@ private void parseData(Result result, JsonNode responseJsonNode)
 
         // Expected to be a string in TSV format.
         // If not, a ResultSetException will most likely occur while appending rows or columns
-        String tsv = responseJsonNode.get("output").get("data").get("text/plain").asText();
-        // Tabs in the tab-separated text are translated to (multiple) spaces after getting it from the JsonNode,
-        // so the spaces are first replaced by a tab
-        String parsedTsv = tsv.replaceAll("^ +| +$|( )+", "\t");
-
-        // First line is header, data starts at second line
-        String[] allRows = parsedTsv.split("\\\\n");
-        String[] headers = allRows[0].split("\t");
-
-        // The header line started with a tab, so skip the first column
-        for (int i = 1; i < headers.length; i++) {
-            frs.appendColumn(new Column(headers[i], PrimitiveDataType.STRING));
+        JsonNode queryOutput = responseJsonNode.get("output").get("data").get("text/plain");
+        // Remove the first and the last apostrophes to avoid json in string format
+        String jsonString = queryOutput.textValue().replaceAll("^'|'$", "");
+
+        JSONObject jsonObject = new JSONObject(jsonString);
+        logger.debug("Parsed json object: " + jsonObject);
+
+        // Get the header
+        JSONArray fields = jsonObject.getJSONObject("schema").getJSONArray("fields");
+        // Header starts with 'index', which do not have to include in the end output table
+        for (int i = 1; i < fields.length(); i++) {
+            JSONObject field = fields.getJSONObject(i);
+            frs.appendColumn(new Column(field.getString("name"),
+                             PrimitiveDataType.valueOf(field.getString("type").toUpperCase())));
         }
-        for (int i = 1; i < allRows.length; i++) {
-            String[] row = allRows[i].split("\t");
+        // Get the data rows
+        JSONArray rows = jsonObject.getJSONArray("data");
+        // Rows are starting with the 'index', which do not have to include in the end output table
+        for (int i = 1; i < rows.length(); i++) {
+            JSONObject row = rows.getJSONObject(i);
             frs.appendRow();
             // A data line start with the line number, so the first element is skipped
-            for (int j = 1; j < row.length; j++) {
+            for (int j = 1; j < row.length(); j++) {
+                String header = fields.getJSONObject(j).getString("name");
+                String value = row.getString(header);
                 // ColumnIndex starts counting at 0
-                frs.updateString(j-1, row[j]);
+                frs.updateString(j-1, value);
             }
         }
 
@@ -420,7 +429,7 @@ private java.util.Map generateQuery(Map variables) {
                 "new_data = all_data_files['" + inputFile + "'].filter((" + genesHailFormat + ") & " +
                                      "(" + significancesHailFormat + ") & " +
                                      "(" + idsHailFormat + "))\n" +
-                "new_data.to_pandas().to_string()";
+                "new_data.to_pandas().to_json(orient='table')";
 
         // Create a HashMap to specify where the data code can be found for the post request
         HashMap<java.lang.String, java.lang.String> postTemplate = new HashMap<>();

From c1de650ab77a03152af46c760e30be74bf573ec4 Mon Sep 17 00:00:00 2001
From: Denise Kersjes <denise.kersjes@hotmail.com>
Date: Mon, 10 Sep 2018 16:36:39 +0200
Subject: [PATCH 21/27] Resolve the issue of skipping one data row while
 parsing the json

---
 .../hms/dbmi/bd2k/picsure/ri/LivyHAIL.java    | 26 +++++++++++--------
 1 file changed, 15 insertions(+), 11 deletions(-)

diff --git a/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java b/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
index 6a25fe0d..14b3e670 100644
--- a/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
+++ b/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
@@ -95,6 +95,7 @@ public void setup(Map<String, String> parameters) throws ResourceInterfaceExcept
             while (resourceState != ResourceState.COMPLETE) {
                 Thread.sleep(3000);
                 updateResourceState();
+                logger.debug(resourceName + " is in state" + resourceState);
             }
         } catch (InterruptedException | UnsupportedOperationException e) {
             logger.error("Session state is not ready");
@@ -386,24 +387,27 @@ private void parseData(Result result, JsonNode responseJsonNode)
 
         // Get the header
         JSONArray fields = jsonObject.getJSONObject("schema").getJSONArray("fields");
-        // Header starts with 'index', which do not have to include in the end output table
-        for (int i = 1; i < fields.length(); i++) {
+        for (int i = 0; i < fields.length(); i++) {
             JSONObject field = fields.getJSONObject(i);
-            frs.appendColumn(new Column(field.getString("name"),
-                             PrimitiveDataType.valueOf(field.getString("type").toUpperCase())));
+            // Remove the index column, which is autonatically added by converting to JSON
+            if (!field.equals("index")) {
+                PrimitiveDataType dataType = PrimitiveDataType.valueOf(field.getString("type").toUpperCase());
+                frs.appendColumn(new Column(field.getString("name"), dataType));
+            }
         }
         // Get the data rows
         JSONArray rows = jsonObject.getJSONArray("data");
-        // Rows are starting with the 'index', which do not have to include in the end output table
-        for (int i = 1; i < rows.length(); i++) {
+        for (int i = 0; i < rows.length(); i++) {
             JSONObject row = rows.getJSONObject(i);
             frs.appendRow();
-            // A data line start with the line number, so the first element is skipped
-            for (int j = 1; j < row.length(); j++) {
+            // Get for every row the data per column
+            for (int j = 0; j < row.length(); j++) {
                 String header = fields.getJSONObject(j).getString("name");
-                String value = row.getString(header);
-                // ColumnIndex starts counting at 0
-                frs.updateString(j-1, value);
+                // Skip the 'index' column, because it is not added to the header
+                if (!header.equals("index")) {
+                    String value = row.get(header).toString();
+                    frs.updateString(j, value);
+                }
             }
         }
 

From ab31193fe672e613c2aedf6758d35584bf4c1104 Mon Sep 17 00:00:00 2001
From: Denise Kersjes <denise.kersjes@hotmail.com>
Date: Mon, 10 Sep 2018 19:56:48 +0200
Subject: [PATCH 22/27] Remove 'index' column from parsed header

---
 .../edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java     | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java b/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
index 14b3e670..726febe3 100644
--- a/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
+++ b/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
@@ -389,8 +389,8 @@ private void parseData(Result result, JsonNode responseJsonNode)
         JSONArray fields = jsonObject.getJSONObject("schema").getJSONArray("fields");
         for (int i = 0; i < fields.length(); i++) {
             JSONObject field = fields.getJSONObject(i);
-            // Remove the index column, which is autonatically added by converting to JSON
-            if (!field.equals("index")) {
+            // Remove the index column, which is automatically added by converting to JSON
+            if (!field.get("name").equals("index")) {
                 PrimitiveDataType dataType = PrimitiveDataType.valueOf(field.getString("type").toUpperCase());
                 frs.appendColumn(new Column(field.getString("name"), dataType));
             }
@@ -403,10 +403,9 @@ private void parseData(Result result, JsonNode responseJsonNode)
             // Get for every row the data per column
             for (int j = 0; j < row.length(); j++) {
                 String header = fields.getJSONObject(j).getString("name");
-                // Skip the 'index' column, because it is not added to the header
                 if (!header.equals("index")) {
                     String value = row.get(header).toString();
-                    frs.updateString(j, value);
+                    frs.updateString(j-1, value);
                 }
             }
         }

From 44fcedaca8bace789f99b9728272349f34785608 Mon Sep 17 00:00:00 2001
From: Denise Kersjes <denise.kersjes@hotmail.com>
Date: Wed, 12 Sep 2018 14:54:59 +0200
Subject: [PATCH 23/27] Add documentation to the (existing) functions

---
 .../hms/dbmi/bd2k/picsure/ri/LivyHAIL.java    | 176 +++++++++++++++---
 1 file changed, 148 insertions(+), 28 deletions(-)

diff --git a/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java b/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
index 726febe3..67d7f981 100644
--- a/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
+++ b/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
@@ -12,7 +12,6 @@
 import edu.harvard.hms.dbmi.bd2k.irct.model.resource.PrimitiveDataType;
 import edu.harvard.hms.dbmi.bd2k.irct.model.resource.ResourceState;
 import edu.harvard.hms.dbmi.bd2k.irct.model.resource.implementation.QueryResourceImplementationInterface;
-import edu.harvard.hms.dbmi.bd2k.irct.model.result.Data;
 import edu.harvard.hms.dbmi.bd2k.irct.model.result.Result;
 import edu.harvard.hms.dbmi.bd2k.irct.model.result.ResultDataType;
 import edu.harvard.hms.dbmi.bd2k.irct.model.result.ResultStatus;
@@ -34,7 +33,6 @@
 import org.apache.log4j.Logger;
 import org.json.JSONArray;
 import org.json.JSONObject;
-import us.monoid.json.JSONException;
 
 import java.io.IOException;
 import java.io.UnsupportedEncodingException;
@@ -47,17 +45,22 @@
 public class LivyHAIL implements QueryResourceImplementationInterface {
     Logger logger = Logger.getLogger(this.getClass());
 
-    private static final String PATH_NAME = "pui";
-
     protected String resourceName;
     protected String resourceURL;
     protected ResourceState resourceState;
 
-    protected String sessionID;
-    protected String inputFile;
+    private String sessionID;
+    private String inputFile;
 
-    protected String dataFileDir = "/app/data/";
+    private String dataFileDir = "/app/data/";
 
+    /**
+     * Set up the resource parameters, initialize a PySpark session, start the
+     * Hail library and load all available data files as Hail tables.
+     *
+     * @param parameters Map<String, String> of setup parameters
+     * @throws ResourceInterfaceException if a set up parameter is missing
+     */
     @Override
     public void setup(Map<String, String> parameters) throws ResourceInterfaceException {
 
@@ -65,6 +68,7 @@ public void setup(Map<String, String> parameters) throws ResourceInterfaceExcept
             logger.debug("setup for Hail" +
                     " Starting...");
 
+        // Check the resource name
         String errorString = "";
         this.resourceName = parameters.get("resourceName");
         if (this.resourceName == null) {
@@ -72,6 +76,7 @@ public void setup(Map<String, String> parameters) throws ResourceInterfaceExcept
             errorString += " resourceName";
         }
 
+        // Check the resource URL
         String tempResourceURL = parameters.get("resourceURL");
         if (tempResourceURL == null) {
             logger.error("setup() `resourceURL` parameter is missing.");
@@ -110,11 +115,27 @@ public void setup(Map<String, String> parameters) throws ResourceInterfaceExcept
                 " is in COMPLETE state.");
     }
 
+    /**
+     * A string representation of the type of resource implementation this is.
+     *
+     * @return Type of resource
+     */
     @Override
     public String getType() {
         return "Hail";
     }
 
+    /**
+     * Run the given query. It converts the predicate fields into variables on the Hail request,
+     * create a POST request with the variables set in a Hail function using a PySpark template.
+     *
+     * @param user A basic user representation. It can be associated with a session in EE 7. The
+     * userId, and name are the same in this implementation.
+     * @param query A query against any individual or group of resources
+     * @param result Execution that is run on the IRCT
+     * @return Execution that is run on the IRCT with set parameters for receiving it
+     * @throws ResourceInterfaceException when there is an error in the Resource Interface
+     */
     @Override
     public Result runQuery(User user, Query query, Result result) throws ResourceInterfaceException {
         logger.debug("runQuery() *** STARTING ***");
@@ -128,7 +149,7 @@ public Result runQuery(User user, Query query, Result result) throws ResourceInt
 
         // Convert the predicate fields into variables on the Hail request
         // These variables will be used when rendering the Hail template into an actual script.
-        Map<String, String> hailVariables = new HashMap<String, String>();
+        Map<String, String> hailVariables = new HashMap<>();
         for (WhereClause whereClause : whereClauses) {
 
             for (String fieldName : whereClause.getStringValues().keySet()) {
@@ -155,9 +176,8 @@ public Result runQuery(User user, Query query, Result result) throws ResourceInt
         logger.debug("runQuery() starting hail job submission");
         Date starttime = new Date();
 
-        // Read the PySpark template file
+        // Read the PySpark template file and create a request with it
         Map<java.lang.String, java.lang.String> filledTemplate = generateQuery(hailVariables);
-
         JsonNode nd = restPOST(this.resourceURL + "/sessions/" + sessionID + "/statements", filledTemplate);
 
         logger.debug("runQuery() hail job submission finished");
@@ -180,6 +200,15 @@ public Result runQuery(User user, Query query, Result result) throws ResourceInt
         return result;
     }
 
+    /**
+     * Perform GET Request on the Hail job, and parse the data as a JSON table.
+     *
+     * @param user A basic user representation. It can be associated with a session in EE 7. The
+     * userId, and name are the same in this implementation.
+     * @param result Execution that is run on the IRCT with set parameters for receiving it
+     * @return Result of the Hail query
+     * @throws ResourceInterfaceException when there is an error in the Resource Interface
+     */
     @Override
     public Result getResults(User user, Result result) throws ResourceInterfaceException {
         logger.debug("getResults() starting");
@@ -187,15 +216,18 @@ public Result getResults(User user, Result result) throws ResourceInterfaceExcep
         String hailJobUUID = result.getResourceActionId();
         logger.debug("getResults() getting result for " + hailJobUUID);
 
+        // Create a GET request on the Hail job.
         JsonNode nd = restGET(resourceURL + "/sessions/" + sessionID + "/statements/" + hailJobUUID);
 
         HailResponse hailResponse = new HailResponse(nd);
         logger.debug("getResults() finished parsing Hail response.");
 
+        // Check for errors
         if (hailResponse.isError()) {
             logger.debug("getResults() Hail error message:" + hailResponse.getErrorMessage());
             result.setResultStatus(ResultStatus.ERROR);
             result.setMessage(hailResponse.getErrorMessage());
+        // Get the results when no errors are occurred
         } else {
             logger.debug("getResults() jobStatus: " + hailResponse.getJobStatus());
 
@@ -208,6 +240,7 @@ public Result getResults(User user, Result result) throws ResourceInterfaceExcep
 
                 logger.debug("getResults() parsing returned actual data");
                 try {
+                    // Parse the output
                     parseData(result, nd);
                 } catch (PersistableException pe) {
                     logger.error("getResults() Unable to persist data");
@@ -224,11 +257,26 @@ public Result getResults(User user, Result result) throws ResourceInterfaceExcep
         return result;
     }
 
+    /**
+     * Get the state of the resource.
+     * Note that this function is not used, but part is of the
+     * 'QueryResourceImplementationInterface'.
+     *
+     * @return the resource state
+     */
     @Override
     public ResourceState getState() {
         return resourceState;
     }
 
+    /**
+     * Get the result data type.
+     * Note that this function is not used, but part is of the
+     * 'QueryResourceImplementationInterface'.
+     *
+     * @param query Query to run
+     * @return Result data type
+     */
     @Override
     public ResultDataType getQueryDataType(Query query) {
         return ResultDataType.TABULAR;
@@ -236,17 +284,14 @@ public ResultDataType getQueryDataType(Query query) {
 
     /**
      * HTTP POST with JSON body, constructed from `payload` Map, and parse the
-     * returned stream into a JsonNode object for later parsing. Protocoll errors
+     * returned stream into a JsonNode object for later parsing. Protocol errors
      * are captured as well.
-     * Any protocol or parsing error will be thrown as RuntimeException
+     * Any protocol or parsing error will be thrown as RuntimeException.
      *
-     * @param urlString
-     * @param payload
-     * @return
-     * @throws JSONException
-     * @throws IOException
+     * @param urlString String referring to the Resource URL
+     * @param payload Map<String, String> of Livy parameters of the request body
+     * @return JsonNode containing the elements inside a JSON stream
      */
-
     private JsonNode restPOST(String urlString, Map<String, String> payload) {
         logger.debug("restPOST() Starting ");
         JsonNode responseObject = null;
@@ -254,9 +299,11 @@ private JsonNode restPOST(String urlString, Map<String, String> payload) {
         ObjectMapper objectMapper = IRCTApplication.objectMapper;
         CloseableHttpClient restClient = IRCTApplication.CLOSEABLE_HTTP_CLIENT;
 
+        // Create the HTTP post request
         HttpPost post = new HttpPost((urlString));
         post.addHeader("Content-Type", ContentType.APPLICATION_JSON.toString());
 
+        // Set the request body as a string entity
         try {
             post.setEntity(
                     new StringEntity(objectMapper
@@ -267,6 +314,7 @@ private JsonNode restPOST(String urlString, Map<String, String> payload) {
             throw new ResourceInterfaceException("Hail - restPOST() the encoding is not supported by apache httppost: " + e.getMessage());
         }
 
+        // Try to close the request and otherwise raise an error
         try (CloseableHttpResponse restResponse = restClient.execute(post)) {
 
             if (restResponse.getStatusLine().getStatusCode() != 200) {
@@ -310,12 +358,11 @@ private JsonNode restPOST(String urlString, Map<String, String> payload) {
     }
 
     /**
-     * HTTP GET, parse the returned stream into a JsonNode object for
-     * later parsing.
+     * HTTP GET, parse the returned stream into a JsonNode object for later parsing.
      *
-     * @param urlString
-     * @return
-     * @throws ResourceInterfaceException
+     * @param urlString String referring to the Resource URL
+     * @return JsonNode object of the JSON response
+     * @throws ResourceInterfaceException when there is an error in the Resource Interface
      */
     private JsonNode restGET(String urlString) throws ResourceInterfaceException {
         logger.debug("restGET() Starting");
@@ -326,9 +373,11 @@ private JsonNode restGET(String urlString) throws ResourceInterfaceException {
             ObjectMapper objectMapper = IRCTApplication.objectMapper;
             CloseableHttpClient restClient = HttpClientBuilder.create().build();
 
+            // Executes HTTP request using the given Resource URL
             HttpGet get = new HttpGet((urlString));
             restResponse = restClient.execute(get);
 
+            // Check for errors
             if (restResponse == null) {
                 logger.error("restResponse is null");
                 throw new ResourceInterfaceException("Could not get Hail response");
@@ -372,6 +421,14 @@ private JsonNode restGET(String urlString) throws ResourceInterfaceException {
         return responseObject;
     }
 
+    /**
+     * Parse the output element of the JsonNode into a JSON Table.
+     *
+     * @param result Execution that is run on the IRCT with set parameters for receiving it
+     * @param responseJsonNode Output of GET request containing JsonNode elements
+     * @throws PersistableException when a problem occours by the persistence provider
+     * @throws ResultSetException when the data can not be converted to the result table
+     */
     private void parseData(Result result, JsonNode responseJsonNode)
             throws PersistableException, ResultSetException {
         FileResultSet frs = (FileResultSet) result.getData();
@@ -415,7 +472,14 @@ private void parseData(Result result, JsonNode responseJsonNode)
         result.setMessage("Data has been downloaded");
     }
 
-    private java.util.Map generateQuery(Map variables) {
+    /**
+     * Get the Hail variables and put them in the PySpark template with the correct Hail
+     * function.
+     *
+     * @param variables <String, String> of Hail variables
+     * @return Map<java.lang.String, java.lang.String> of the Livy request body 'code'
+     */
+    private java.util.Map<java.lang.String, java.lang.String> generateQuery(Map variables) {
 
         // Get the specified variables out of the HashMap hailVariables
         String gene = variables.get("gene").toString();
@@ -441,6 +505,14 @@ private java.util.Map generateQuery(Map variables) {
         return postTemplate;
     }
 
+    /**
+     * Check if a Hail variable consist of multiple values and parse the values to a
+     * PySpark template that can be read by Hail.
+     *
+     * @param variable String of Hail variables separated by comma's
+     * @param columnName String referring the to column name in the data file
+     * @return String readable by Hail using PySpark
+     */
     private String parseMultipleVariables(String variable, String columnName) {
 
         // The variables are strings that can consist of multiple elements separated by commas
@@ -462,7 +534,14 @@ private String parseMultipleVariables(String variable, String columnName) {
         return hailFormat.toString();
     }
 
-    private java.util.Map loadDataTemplate() {
+    /**
+     * Create a PySpark template that loads the Hail library and read all data files
+     * as a Hail table.
+     *
+     * @return java.util.Map<java.lang.String, java.lang.String> of the Livy request
+     * body 'code'
+     */
+    private java.util.Map<java.lang.String, java.lang.String> loadDataTemplate() {
 
         // Create a PySpark that imports hail and load all datafiles as a hail table
         String pysparkFormatTemplate =
@@ -483,9 +562,12 @@ private java.util.Map loadDataTemplate() {
         return postDataTemplate;
     }
 
+    /**
+     * Map the resource state of Livy to the one of IRCT.
+     */
     private void updateResourceState() {
 
-        // Create a dictionary to map the state of Livy to the ResourceState
+        // Create a dictionary to map the state of Livy to the IRCT ResourceState
         HashMap<String, ResourceState> stateMapping = new HashMap<>();
         stateMapping.put("not_started", ResourceState.READY);
         stateMapping.put("starting", ResourceState.RUNNING);
@@ -504,13 +586,20 @@ private void updateResourceState() {
         resourceState = stateMapping.get(requestState);
     }
 
+    /**
+     * Get the Hail response of the POST request.
+     */
     class HailResponse {
 
         private String errorMessage = "";
         private String hailMessage = "";
         private String jobStatus = "";
-        private Data hailData = null;
 
+        /**
+         * Get JsonNode elements out of the response of the POST request.
+         *
+         * @param rootNode JsonNode containing the elements inside a JSON stream
+         */
         public HailResponse(JsonNode rootNode) {
             logger.debug("HailResponse() constructor");
 
@@ -526,13 +615,14 @@ public HailResponse(JsonNode rootNode) {
 
                     // Success response from Hail
                     if (rootNode.get("output") != null) {
-                        // If this is a job response
+                        // Get JsonNode elements if this is a job response
                         logger.debug("HailResponse() parse job details.");
                         this.jobUUID = rootNode.get("id").toString();
                         this.hailMessage = rootNode.get("output").textValue();
                         String state = rootNode.get("state").textValue();
                         mapResultState(state);
                     } else {
+                        // Else get the available elements out to get the output of te response later
                         logger.debug("HailResponse() parse data details.");
                         this.jobUUID = rootNode.get("id").toString();
                         this.hailMessage = "Parsing Hail data";
@@ -550,6 +640,11 @@ public HailResponse(JsonNode rootNode) {
             logger.debug("HailResponse() finished");
         }
 
+        /**
+         * Map the result State of Livy to the one of IRCT.
+         *
+         * @param stateLivy String that represent the result state of Livy
+         */
         public void mapResultState(String stateLivy) {
 
             // Create a dictionary to map the result state of Livy to the one of IRCT
@@ -564,24 +659,49 @@ public void mapResultState(String stateLivy) {
             this.jobStatus = resultStateMapping.get(stateLivy);
           }
 
+        /**
+         * Get the Hail job number.
+         *
+         * @return String representing the Hail job number
+         */
         public String getJobUUID() {
             return jobUUID;
         }
 
         private String jobUUID = "";
 
+        /**
+         * Check if an error occurs.
+         *
+         * @return boolean that refers if there is an error message or not
+         */
         public boolean isError() {
             return !this.errorMessage.isEmpty();
         }
 
+        /**
+         * Get the error message.
+         *
+         * @return String with the error message
+         */
         public String getErrorMessage() {
             return this.errorMessage;
         }
 
+        /**
+         * Get the IRCT status of the Hail job.
+         *
+         * @return String that represents the state a result is in
+         */
         public String getJobStatus() {
             return jobStatus;
         }
 
+        /**
+         * Get the JsonNode element 'output' out of the JSON stream.
+         *
+         * @return String of the JsonNode element 'output'
+         */
         public String getHailMessage() {
             return hailMessage;
         }

From a5c976c4ba1dffe454f86aae4c9db64a35ffa630 Mon Sep 17 00:00:00 2001
From: Denise Kersjes <denise.kersjes@hotmail.com>
Date: Wed, 12 Sep 2018 16:51:16 +0200
Subject: [PATCH 24/27] Add assumptions

---
 .../edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java    | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java b/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
index 67d7f981..9ab814fb 100644
--- a/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
+++ b/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
@@ -474,7 +474,9 @@ private void parseData(Result result, JsonNode responseJsonNode)
 
     /**
      * Get the Hail variables and put them in the PySpark template with the correct Hail
-     * function.
+     * function. In here, the function 'filter' is used.
+     * NOTE: this function contains assumptions about the data. It assumes that the data
+     * file as the column names 'GENE', 'CLIN_SIG', and 'ID'.
      *
      * @param variables <String, String> of Hail variables
      * @return Map<java.lang.String, java.lang.String> of the Livy request body 'code'
@@ -508,6 +510,8 @@ private java.util.Map<java.lang.String, java.lang.String> generateQuery(Map vari
     /**
      * Check if a Hail variable consist of multiple values and parse the values to a
      * PySpark template that can be read by Hail.
+     * NOTE: this function contains assumptions. If a Hail variables contains multiple
+     * values, it is assumed that these are separated by comma's.
      *
      * @param variable String of Hail variables separated by comma's
      * @param columnName String referring the to column name in the data file
@@ -537,6 +541,8 @@ private String parseMultipleVariables(String variable, String columnName) {
     /**
      * Create a PySpark template that loads the Hail library and read all data files
      * as a Hail table.
+     * NOTE: this function contains assumptions. It is assumed that the data files
+     * are tab-separated file of the format maf or tsv.
      *
      * @return java.util.Map<java.lang.String, java.lang.String> of the Livy request
      * body 'code'

From 40a2162774b7ec0131d2f7c6606cde80c8430dd9 Mon Sep 17 00:00:00 2001
From: Denise Kersjes <denise.kersjes@hotmail.com>
Date: Mon, 17 Sep 2018 11:36:23 +0200
Subject: [PATCH 25/27] Revert "Add Hail resource"

This reverts commit 8ef3f2c938718c1127c43fdd3d40e93f2c9d21bb.
---
 .../irct/sql/hail_resource_function.sql       | 69 -------------------
 1 file changed, 69 deletions(-)
 delete mode 100644 deployments/irct/sql/hail_resource_function.sql

diff --git a/deployments/irct/sql/hail_resource_function.sql b/deployments/irct/sql/hail_resource_function.sql
deleted file mode 100644
index a0f58e3e..00000000
--- a/deployments/irct/sql/hail_resource_function.sql
+++ /dev/null
@@ -1,69 +0,0 @@
-CREATE OR REPLACE FUNCTION add_hail_resource(
-  resourceName varchar,
-  resourceURL varchar
-) RETURNS VOID AS $$
-
-
-DECLARE
-  -- SET THE RESOURCE VARIABLES
-  resourceId integer := (select COALESCE(max(id), 1) from IRCT_Resource) + 1;
-  --resourceName varchar := ( COALESCE(resourceName, '') = '', '{{ resourceName }}', resourceName);
-  --resourceURL varchar := ( COALESCE(resourceURL, '') = '', '{{ resourceURL}}', resourceURL);
-
-  -- SET THE RESOURCE PREDICATES
-  predicatetype_filter_id integer := (select COALESCE(max(id), 1) from IRCT_PredicateType) + 1;
-
-  -- SET THE FIELDS
-  gene_field_id integer := (select COALESCE(max(id), 0) from IRCT_Field) + 1;
-  sign_field_id integer := gene_field_id + 1;
-  subject_field_id integer := sign_field_id + 1;
-
-BEGIN
-  -- INSERT THE RESOURCE VARIABLE
-  insert into IRCT_Resource(id, implementingInterface, name, ontologyType) values
-    (resourceId, 'edu.harvard.hms.dbmi.bd2k.picsure.ri.LivyHAIL', resourceName, 'TREE');
-
-  -- INSERT THE RESOURCE PARAMETERS
-  insert into IRCT_resource_parameters(id, name, value) values(resourceId, 'resourceName', resourceName);
-  insert into IRCT_resource_parameters(id, name, value) values(resourceId, 'resourceURL', resourceURL);
-
-  -- INSERT RESOURCE DATA TYPES
-  ---- insert into PredicateType_dataTypes(PredicateType_id, dataTypes) values (predicate_type_id, 'edu.harvard.hms.dbmi.bd2k.irct.model.resource.PrimitiveDataType:STRING');
-  --insert into PredicateType_dataTypes(PredicateType_id, dataTypes) values
-  --  (predicatetype_filter_id, 'edu.harvard.hms.dbmi.bd2k.irct.model.resource.PrimitiveDataType:STRING');
-
-  -- INSERT RESOURCE RELATIONSHIPS
-
-  -- INSERT RESOURCE LogicalOperators
-
-
-  -- FILTER predicate
-  insert into IRCT_PredicateType(id, defaultPredicate, description, displayName, name) values
-    (predicatetype_filter_id, false, 'Execute filter predicate', 'filter', 'filter');
-  insert into IRCT_Resource_PredicateType(Resource_Id, supportedPredicates_id) values
-    (resourceId, predicatetype_filter_id);
-
-  -- INSERT FIELDS
-  insert into IRCT_Field(id, description, name, path, required, relationship) values
-    (gene_field_id, 'Gene', 'Gene', 'gene',false,NULL);
-  insert into IRCT_Field_dataTypes(Field_id, dataTypes)
-    values(gene_field_id, 'edu.harvard.hms.dbmi.bd2k.irct.model.resource.PrimitiveDataType:STRING');
-  insert into IRCT_PredicateType_Field(PredicateType_id, fields_id) values
-    (predicatetype_filter_id, gene_field_id);
-
-  insert into IRCT_Field(id, description, name, path, required, relationship) values
-    (sign_field_id, 'Significance', 'Significance', 'significance',false,NULL);
-  insert into IRCT_Field_dataTypes(Field_id, dataTypes)
-    values(sign_field_id, 'edu.harvard.hms.dbmi.bd2k.irct.model.resource.PrimitiveDataType:STRING');
-  insert into IRCT_PredicateType_Field(PredicateType_id, fields_id) values
-    (predicatetype_filter_id, sign_field_id);
-
-  insert into IRCT_Field(id, description, name, path, required, relationship) values
-    (subject_field_id, 'Subject ID', 'Subject ID', 'subject_id',false,NULL);
-  insert into IRCT_Field_dataTypes(Field_id, dataTypes)
-    values(subject_field_id, 'edu.harvard.hms.dbmi.bd2k.irct.model.resource.PrimitiveDataType:STRING');
-  insert into IRCT_PredicateType_Field(PredicateType_id, fields_id) values
-    (predicatetype_filter_id, subject_field_id);
-
-END;
-$$ LANGUAGE plpgsql

From 08ed4ef6143d6479a6733dbffcd01763faf73338 Mon Sep 17 00:00:00 2001
From: Denise Kersjes <denise.kersjes@hotmail.com>
Date: Mon, 17 Sep 2018 11:41:15 +0200
Subject: [PATCH 26/27] Revert "ignore"

This reverts commit 8905157ea78da4cf5df5a979ba7f4ce0292856d4.
---
 .gitignore | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/.gitignore b/.gitignore
index 341ab484..62068f66 100644
--- a/.gitignore
+++ b/.gitignore
@@ -15,9 +15,6 @@ scratch/
 .DS_Store
 
 deploy.sh
-IRCT-RI/src/main/.DS_Store
-.idea/
-*.iml
-
-deployments/hail/data/*
-!deployments/hail/data/.gitkeep
\ No newline at end of file
+IRCT-RI/src/main/.DS_Store
+.idea/
+*.iml

From f2eec8b08ea0caa76eed7a20baa584ff54ab4cd6 Mon Sep 17 00:00:00 2001
From: Denise Kersjes <denise.kersjes@hotmail.com>
Date: Mon, 17 Sep 2018 12:21:01 +0200
Subject: [PATCH 27/27] Add note about the docker image usage

---
 .../java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java     | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java b/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
index 9ab814fb..e533a34a 100644
--- a/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
+++ b/IRCT-RI/src/main/java/edu/harvard/hms/dbmi/bd2k/picsure/ri/LivyHAIL.java
@@ -52,6 +52,8 @@ public class LivyHAIL implements QueryResourceImplementationInterface {
     private String sessionID;
     private String inputFile;
 
+    // This data directory is created in our docker compose file by defining the
+    // volume './hail/data:/app/data'. Docker image: https://github.com/thehyve/livy-hail-docker
     private String dataFileDir = "/app/data/";
 
     /**