diff --git a/.github/workflows/pic-sure-hpds-test.yml b/.github/workflows/pic-sure-hpds-test.yml new file mode 100644 index 000000000..a5993483b --- /dev/null +++ b/.github/workflows/pic-sure-hpds-test.yml @@ -0,0 +1,22 @@ +name: pic-sure-hpds tests +on: + pull_request: + paths: ['services/pic-sure-hpds/**'] + push: + branches: [main] + paths: ['services/pic-sure-hpds/**'] +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd #v6.0.2 + - name: Set up JDK 25 + uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 #v5.2.0 + with: + java-version: '25' + distribution: 'temurin' + cache: maven + - name: Test with Maven + run: mvn -B -pl services/pic-sure-hpds/common,services/pic-sure-hpds/data,services/pic-sure-hpds/etl,services/pic-sure-hpds/processing,services/pic-sure-hpds/service,services/pic-sure-hpds/genomic-processor,services/pic-sure-hpds/war,services/pic-sure-hpds/docker -am verify + env: + GITHUB_TOKEN: ${{ github.token }} diff --git a/pom.xml b/pom.xml index 98ce6343e..c2cc33acf 100644 --- a/pom.xml +++ b/pom.xml @@ -24,6 +24,7 @@ services/picsure-dictionary/dictionaryweights services/pic-sure-logging services/pic-sure-auth-microapp + services/pic-sure-hpds diff --git a/services/pic-sure-hpds/common/pom.xml b/services/pic-sure-hpds/common/pom.xml index 75af6f689..5d08375bf 100644 --- a/services/pic-sure-hpds/common/pom.xml +++ b/services/pic-sure-hpds/common/pom.xml @@ -6,7 +6,7 @@ pic-sure-hpds edu.harvard.hms.dbmi.avillach.hpds - 3.0.0-SNAPSHOT + ${revision} common diff --git a/services/pic-sure-hpds/common/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/crypto/Crypto.java b/services/pic-sure-hpds/common/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/crypto/Crypto.java index e9245a42a..a6d014519 100644 --- a/services/pic-sure-hpds/common/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/crypto/Crypto.java +++ b/services/pic-sure-hpds/common/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/crypto/Crypto.java @@ -26,89 +26,95 @@ public class Crypto { - public static final String DEFAULT_KEY_NAME = "DEFAULT"; - - // This needs to be set in a static initializer block to be overridable in tests. - private static final String DEFAULT_ENCRYPTION_KEY_PATH; - static{ - DEFAULT_ENCRYPTION_KEY_PATH = "/opt/local/hpds/encryption_key"; - } - - private static final Logger LOGGER = LoggerFactory.getLogger(Crypto.class); - - private static final HashMap keys = new HashMap(); - - public static void loadDefaultKey() { - loadKey(DEFAULT_KEY_NAME, DEFAULT_ENCRYPTION_KEY_PATH); - } - - public static void loadKey(String keyName, String filePath) { - try { - setKey(keyName, IOUtils.toString(new FileInputStream(filePath), Charset.forName("UTF-8")).trim().getBytes()); - LOGGER.info("****LOADED CRYPTO KEY****"); - } catch (IOException e) { - LOGGER.error("****CRYPTO KEY NOT FOUND****", e); - } - } - - public static byte[] encryptData(byte[] plaintextBytes) { - return encryptData(DEFAULT_KEY_NAME, plaintextBytes); - } - - public static byte[] encryptData(String keyName, byte[] plaintextBytes) { - byte[] key = keys.get(keyName); - SecureRandom secureRandom = new SecureRandom(); - SecretKey secretKey = new SecretKeySpec(key, "AES"); - byte[] iv = new byte[12]; //NEVER REUSE THIS IV WITH SAME KEY - secureRandom.nextBytes(iv); - byte[] cipherText; - Cipher cipher; - try { - cipher = Cipher.getInstance("AES/GCM/NoPadding"); - GCMParameterSpec parameterSpec = new GCMParameterSpec(128, iv); //128 bit auth tag length - cipher.init(Cipher.ENCRYPT_MODE, secretKey, parameterSpec); - cipherText = new byte[cipher.getOutputSize(plaintextBytes.length)]; - cipher.doFinal(plaintextBytes, 0, plaintextBytes.length, cipherText, 0); - LOGGER.debug("Length of cipherText : " + cipherText.length); - ByteBuffer byteBuffer = ByteBuffer.allocate(4 + iv.length + cipherText.length); - byteBuffer.putInt(iv.length); - byteBuffer.put(iv); - byteBuffer.put(cipherText); - byte[] cipherMessage = byteBuffer.array(); - return cipherMessage; - } catch (NoSuchAlgorithmException | NoSuchPaddingException | InvalidKeyException | InvalidAlgorithmParameterException | ShortBufferException | IllegalBlockSizeException | BadPaddingException e) { - throw new RuntimeException("Exception while trying to encrypt data : ", e); - } - } - - public static byte[] decryptData(byte[] encrypted) { - return decryptData(DEFAULT_KEY_NAME, encrypted); - } - - public static byte[] decryptData(String keyName, byte[] encrypted) { - byte[] key = keys.get(keyName); - ByteBuffer byteBuffer = ByteBuffer.wrap(encrypted); - int ivLength = byteBuffer.getInt(); - byte[] iv = new byte[ivLength]; - byteBuffer.get(iv); - byte[] cipherText = new byte[byteBuffer.remaining()]; - byteBuffer.get(cipherText); - Cipher cipher; - try { - cipher = Cipher.getInstance("AES/GCM/NoPadding"); - cipher.init(Cipher.DECRYPT_MODE, new SecretKeySpec(key, "AES"), new GCMParameterSpec(128, iv)); - return cipher.doFinal(cipherText); - } catch (NoSuchAlgorithmException | NoSuchPaddingException | InvalidKeyException | InvalidAlgorithmParameterException | IllegalBlockSizeException | BadPaddingException e) { - throw new RuntimeException("Exception caught trying to decrypt data : " + e, e); - } - } - - private static void setKey(String keyName, byte[] key) { - keys.put(keyName, key); - } - - public static boolean hasKey(String keyName) { - return keys.containsKey(keyName); - } + public static final String DEFAULT_KEY_NAME = "DEFAULT"; + + // This needs to be set in a static initializer block to be overridable in tests. + private static final String DEFAULT_ENCRYPTION_KEY_PATH; + static { + DEFAULT_ENCRYPTION_KEY_PATH = "/opt/local/hpds/encryption_key"; + } + + private static final Logger LOGGER = LoggerFactory.getLogger(Crypto.class); + + private static final HashMap keys = new HashMap(); + + public static void loadDefaultKey() { + loadKey(DEFAULT_KEY_NAME, DEFAULT_ENCRYPTION_KEY_PATH); + } + + public static void loadKey(String keyName, String filePath) { + try { + setKey(keyName, IOUtils.toString(new FileInputStream(filePath), Charset.forName("UTF-8")).trim().getBytes()); + LOGGER.info("****LOADED CRYPTO KEY****"); + } catch (IOException e) { + LOGGER.error("****CRYPTO KEY NOT FOUND****", e); + } + } + + public static byte[] encryptData(byte[] plaintextBytes) { + return encryptData(DEFAULT_KEY_NAME, plaintextBytes); + } + + public static byte[] encryptData(String keyName, byte[] plaintextBytes) { + byte[] key = keys.get(keyName); + SecureRandom secureRandom = new SecureRandom(); + SecretKey secretKey = new SecretKeySpec(key, "AES"); + byte[] iv = new byte[12]; // NEVER REUSE THIS IV WITH SAME KEY + secureRandom.nextBytes(iv); + byte[] cipherText; + Cipher cipher; + try { + cipher = Cipher.getInstance("AES/GCM/NoPadding"); + GCMParameterSpec parameterSpec = new GCMParameterSpec(128, iv); // 128 bit auth tag length + cipher.init(Cipher.ENCRYPT_MODE, secretKey, parameterSpec); + cipherText = new byte[cipher.getOutputSize(plaintextBytes.length)]; + cipher.doFinal(plaintextBytes, 0, plaintextBytes.length, cipherText, 0); + LOGGER.debug("Length of cipherText : " + cipherText.length); + ByteBuffer byteBuffer = ByteBuffer.allocate(4 + iv.length + cipherText.length); + byteBuffer.putInt(iv.length); + byteBuffer.put(iv); + byteBuffer.put(cipherText); + byte[] cipherMessage = byteBuffer.array(); + return cipherMessage; + } catch ( + NoSuchAlgorithmException | NoSuchPaddingException | InvalidKeyException | InvalidAlgorithmParameterException + | ShortBufferException | IllegalBlockSizeException | BadPaddingException e + ) { + throw new RuntimeException("Exception while trying to encrypt data : ", e); + } + } + + public static byte[] decryptData(byte[] encrypted) { + return decryptData(DEFAULT_KEY_NAME, encrypted); + } + + public static byte[] decryptData(String keyName, byte[] encrypted) { + byte[] key = keys.get(keyName); + ByteBuffer byteBuffer = ByteBuffer.wrap(encrypted); + int ivLength = byteBuffer.getInt(); + byte[] iv = new byte[ivLength]; + byteBuffer.get(iv); + byte[] cipherText = new byte[byteBuffer.remaining()]; + byteBuffer.get(cipherText); + Cipher cipher; + try { + cipher = Cipher.getInstance("AES/GCM/NoPadding"); + cipher.init(Cipher.DECRYPT_MODE, new SecretKeySpec(key, "AES"), new GCMParameterSpec(128, iv)); + return cipher.doFinal(cipherText); + } catch ( + NoSuchAlgorithmException | NoSuchPaddingException | InvalidKeyException | InvalidAlgorithmParameterException + | IllegalBlockSizeException | BadPaddingException e + ) { + throw new RuntimeException("Exception caught trying to decrypt data : " + e, e); + } + } + + private static void setKey(String keyName, byte[] key) { + keys.put(keyName, key); + } + + public static boolean hasKey(String keyName) { + return keys.containsKey(keyName); + } } diff --git a/services/pic-sure-hpds/common/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/exception/NotEnoughMemoryException.java b/services/pic-sure-hpds/common/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/exception/NotEnoughMemoryException.java index f75631ea9..5cb595abb 100644 --- a/services/pic-sure-hpds/common/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/exception/NotEnoughMemoryException.java +++ b/services/pic-sure-hpds/common/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/exception/NotEnoughMemoryException.java @@ -2,11 +2,13 @@ public class NotEnoughMemoryException extends RuntimeException { - private static final long serialVersionUID = 2592915631853567560L; + private static final long serialVersionUID = 2592915631853567560L; - public NotEnoughMemoryException() { - super("Not enough available heap space to allocate results array." - + "Please reduce ID_BATCH_SIZE or CACHE_SIZE system property or increase the heap available to this application. " - + "This query will be retried up to 3 times."); - } + public NotEnoughMemoryException() { + super( + "Not enough available heap space to allocate results array." + + "Please reduce ID_BATCH_SIZE or CACHE_SIZE system property or increase the heap available to this application. " + + "This query will be retried up to 3 times." + ); + } } diff --git a/services/pic-sure-hpds/common/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/exception/ResultSetTooLargeException.java b/services/pic-sure-hpds/common/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/exception/ResultSetTooLargeException.java index 8d7dd1b19..051b6f049 100644 --- a/services/pic-sure-hpds/common/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/exception/ResultSetTooLargeException.java +++ b/services/pic-sure-hpds/common/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/exception/ResultSetTooLargeException.java @@ -1,21 +1,20 @@ package edu.harvard.hms.dbmi.avillach.hpds.exception; public class ResultSetTooLargeException extends RuntimeException { - /** - * This exception is thrown when a result set would be larger in memory than - * Integer.MAX_INT bytes. This is a JVM limitation... but seriously people... - * - * You actually don't need that much data on the client side, work with us - * to implement whatever filtering you need. - * - * @param multiplesOfCapacity The number of pieces to split this result to actually have it work. - */ - public ResultSetTooLargeException(long multiplesOfCapacity) { - super("The result set is too large to handle in one request. " - + "We have a plan to fix this limitation, but for now try " - + "splitting your selected fields array in " + multiplesOfCapacity + " equal pieces " - + "and running " + multiplesOfCapacity + " queries. Additionally, do you actually " - + "need all this data? Do you have the compute resources " - + "to handle processing it?"); - } + /** + * This exception is thrown when a result set would be larger in memory than Integer.MAX_INT bytes. This is a JVM limitation... but + * seriously people... + * + * You actually don't need that much data on the client side, work with us to implement whatever filtering you need. + * + * @param multiplesOfCapacity The number of pieces to split this result to actually have it work. + */ + public ResultSetTooLargeException(long multiplesOfCapacity) { + super( + "The result set is too large to handle in one request. " + "We have a plan to fix this limitation, but for now try " + + "splitting your selected fields array in " + multiplesOfCapacity + " equal pieces " + "and running " + multiplesOfCapacity + + " queries. Additionally, do you actually " + "need all this data? Do you have the compute resources " + + "to handle processing it?" + ); + } } diff --git a/services/pic-sure-hpds/common/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/storage/FileBackedByteIndexedStorage.java b/services/pic-sure-hpds/common/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/storage/FileBackedByteIndexedStorage.java index b86b671f9..6c9ce95e2 100644 --- a/services/pic-sure-hpds/common/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/storage/FileBackedByteIndexedStorage.java +++ b/services/pic-sure-hpds/common/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/storage/FileBackedByteIndexedStorage.java @@ -5,68 +5,68 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.function.Function; -public abstract class FileBackedByteIndexedStorage implements Serializable { - private static final long serialVersionUID = -7297090745384302635L; - protected transient RandomAccessFile storage; - protected ConcurrentHashMap index; - protected File storageFile; - protected boolean completed = false; +public abstract class FileBackedByteIndexedStorage implements Serializable { + private static final long serialVersionUID = -7297090745384302635L; + protected transient RandomAccessFile storage; + protected ConcurrentHashMap index; + protected File storageFile; + protected boolean completed = false; - public FileBackedByteIndexedStorage(Class keyClass, Class valueClass, File storageFile) throws FileNotFoundException { - this.index = new ConcurrentHashMap(); - this.storageFile = storageFile; - this.storage = new RandomAccessFile(this.storageFile, "rw"); - } + public FileBackedByteIndexedStorage(Class keyClass, Class valueClass, File storageFile) throws FileNotFoundException { + this.index = new ConcurrentHashMap(); + this.storageFile = storageFile; + this.storage = new RandomAccessFile(this.storageFile, "rw"); + } - public void updateStorageDirectory(File storageDirectory) { - if (!storageDirectory.isDirectory()) { - throw new IllegalArgumentException("storageDirectory is not a directory"); - } - String currentStoreageFilename = storageFile.getName(); - storageFile = new File(storageDirectory, currentStoreageFilename); - } + public void updateStorageDirectory(File storageDirectory) { + if (!storageDirectory.isDirectory()) { + throw new IllegalArgumentException("storageDirectory is not a directory"); + } + String currentStoreageFilename = storageFile.getName(); + storageFile = new File(storageDirectory, currentStoreageFilename); + } - public Set keys(){ - return index.keySet(); - } + public Set keys() { + return index.keySet(); + } - public void put(K key, V value) { - if(completed) { - throw new RuntimeException("A completed FileBackedByteIndexedStorage cannot be modified."); - } - Long[] recordIndex; - try (ByteArrayOutputStream out = writeObject(value)) { - recordIndex = new Long[2]; - synchronized (storage) { - storage.seek(storage.length()); - recordIndex[0] = storage.getFilePointer(); - storage.write(out.toByteArray()); - recordIndex[1] = storage.getFilePointer() - recordIndex[0]; - } - } catch (IOException e) { - throw new UncheckedIOException(e); - } - index.put(key, recordIndex); - } + public void put(K key, V value) { + if (completed) { + throw new RuntimeException("A completed FileBackedByteIndexedStorage cannot be modified."); + } + Long[] recordIndex; + try (ByteArrayOutputStream out = writeObject(value)) { + recordIndex = new Long[2]; + synchronized (storage) { + storage.seek(storage.length()); + recordIndex[0] = storage.getFilePointer(); + storage.write(out.toByteArray()); + recordIndex[1] = storage.getFilePointer() - recordIndex[0]; + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + index.put(key, recordIndex); + } - public void load(Iterable values, Function mapper) { - //make sure we start fresh - if(this.storageFile.exists()) { - this.storageFile.delete(); - } - try (RandomAccessFile storage = new RandomAccessFile(storageFile, "rw");) { - this.storage = storage; - for(V value : values) { - put(mapper.apply(value), value); - } - complete(); - } catch (IOException e) { + public void load(Iterable values, Function mapper) { + // make sure we start fresh + if (this.storageFile.exists()) { + this.storageFile.delete(); + } + try (RandomAccessFile storage = new RandomAccessFile(storageFile, "rw");) { + this.storage = storage; + for (V value : values) { + put(mapper.apply(value), value); + } + complete(); + } catch (IOException e) { throw new UncheckedIOException(e); } } - public void open() { + public void open() { try { this.storage = new RandomAccessFile(this.storageFile, "rwd"); } catch (FileNotFoundException e) { @@ -74,48 +74,48 @@ public void open() { } } - public void complete() { - this.completed = true; - } + public void complete() { + this.completed = true; + } - public V get(K key) { - try { - // todo: make this class immutable and remove this lock/check altogether - synchronized(this) { - if(this.storage==null) { - this.open(); - } - } - Long[] offsetsInStorage = index.get(key); - if(offsetsInStorage != null) { - Long offsetInStorage = index.get(key)[0]; - int offsetLength = index.get(key)[1].intValue(); - if(offsetInStorage != null && offsetLength>0) { - byte[] buffer = new byte[offsetLength]; - synchronized(storage) { - storage.seek(offsetInStorage); - storage.readFully(buffer); - } - V readObject = readObject(buffer); - return readObject; - }else { - return null; - } - } else { - return null; - } - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } + public V get(K key) { + try { + // todo: make this class immutable and remove this lock/check altogether + synchronized (this) { + if (this.storage == null) { + this.open(); + } + } + Long[] offsetsInStorage = index.get(key); + if (offsetsInStorage != null) { + Long offsetInStorage = index.get(key)[0]; + int offsetLength = index.get(key)[1].intValue(); + if (offsetInStorage != null && offsetLength > 0) { + byte[] buffer = new byte[offsetLength]; + synchronized (storage) { + storage.seek(offsetInStorage); + storage.readFully(buffer); + } + V readObject = readObject(buffer); + return readObject; + } else { + return null; + } + } else { + return null; + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } - protected abstract V readObject(byte[] buffer); + protected abstract V readObject(byte[] buffer); - protected abstract ByteArrayOutputStream writeObject(V value) throws IOException; + protected abstract ByteArrayOutputStream writeObject(V value) throws IOException; - public V getOrELse(K key, V defaultValue) { - V result = get(key); - return result == null ? defaultValue : result; - } + public V getOrELse(K key, V defaultValue) { + V result = get(key); + return result == null ? defaultValue : result; + } } diff --git a/services/pic-sure-hpds/common/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/storage/FileBackedJavaIndexedStorage.java b/services/pic-sure-hpds/common/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/storage/FileBackedJavaIndexedStorage.java index d70c65a2d..dc30ada05 100644 --- a/services/pic-sure-hpds/common/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/storage/FileBackedJavaIndexedStorage.java +++ b/services/pic-sure-hpds/common/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/storage/FileBackedJavaIndexedStorage.java @@ -4,15 +4,16 @@ import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; -public class FileBackedJavaIndexedStorage extends FileBackedByteIndexedStorage { +public class FileBackedJavaIndexedStorage extends FileBackedByteIndexedStorage { public FileBackedJavaIndexedStorage(Class keyClass, Class valueClass, File storageFile) throws FileNotFoundException { super(keyClass, valueClass, storageFile); } protected ByteArrayOutputStream writeObject(V value) throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); - try (GZIPOutputStream gzipOutputStream = new GZIPOutputStream(out); - ObjectOutputStream oos = new ObjectOutputStream(gzipOutputStream)) { + try ( + GZIPOutputStream gzipOutputStream = new GZIPOutputStream(out); ObjectOutputStream oos = new ObjectOutputStream(gzipOutputStream) + ) { oos.writeObject(value); oos.flush(); } diff --git a/services/pic-sure-hpds/common/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/storage/FileBackedJsonIndexStorage.java b/services/pic-sure-hpds/common/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/storage/FileBackedJsonIndexStorage.java index f93a206fd..89e5133df 100644 --- a/services/pic-sure-hpds/common/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/storage/FileBackedJsonIndexStorage.java +++ b/services/pic-sure-hpds/common/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/storage/FileBackedJsonIndexStorage.java @@ -7,7 +7,7 @@ import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; -public abstract class FileBackedJsonIndexStorage extends FileBackedByteIndexedStorage { +public abstract class FileBackedJsonIndexStorage extends FileBackedByteIndexedStorage { private static final long serialVersionUID = -1086729119489479152L; protected transient ObjectMapper objectMapper = new ObjectMapper(); diff --git a/services/pic-sure-hpds/common/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/crypto/CryptoDefaultKeyTest.java b/services/pic-sure-hpds/common/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/crypto/CryptoDefaultKeyTest.java index 8f93aeea7..d5f59fd73 100644 --- a/services/pic-sure-hpds/common/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/crypto/CryptoDefaultKeyTest.java +++ b/services/pic-sure-hpds/common/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/crypto/CryptoDefaultKeyTest.java @@ -13,38 +13,37 @@ @Disabled // We should rewrite the crypto class to make it more testable, these tests don't work on certain JDKs public class CryptoDefaultKeyTest { - - String TEST_MESSAGE = "This is a test."; - - @BeforeTestClass - public static void overrideDefaultKeyLocation() throws IllegalArgumentException, IllegalAccessException { - } - - @Test - public void testCryptoLoadsKey() throws IllegalArgumentException, IllegalAccessException, NoSuchFieldException, SecurityException { - Field field = Crypto.class.getDeclaredField("DEFAULT_ENCRYPTION_KEY_PATH"); - field.setAccessible(true); - Field modifiersField = Field.class.getDeclaredField("modifiers"); + + String TEST_MESSAGE = "This is a test."; + + @BeforeTestClass + public static void overrideDefaultKeyLocation() throws IllegalArgumentException, IllegalAccessException {} + + @Test + public void testCryptoLoadsKey() throws IllegalArgumentException, IllegalAccessException, NoSuchFieldException, SecurityException { + Field field = Crypto.class.getDeclaredField("DEFAULT_ENCRYPTION_KEY_PATH"); + field.setAccessible(true); + Field modifiersField = Field.class.getDeclaredField("modifiers"); modifiersField.setAccessible(true); modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL); - field.set(Crypto.class, new File("src/test/resources/test_encryption_key").getAbsolutePath()); + field.set(Crypto.class, new File("src/test/resources/test_encryption_key").getAbsolutePath()); Crypto.loadDefaultKey(); assertTrue(Crypto.hasKey(Crypto.DEFAULT_KEY_NAME)); - } + } - @Test - public void testEncryptDecrypt() throws IllegalArgumentException, IllegalAccessException, NoSuchFieldException, SecurityException { - Field field = Crypto.class.getDeclaredField("DEFAULT_ENCRYPTION_KEY_PATH"); - field.setAccessible(true); - Field modifiersField = Field.class.getDeclaredField("modifiers"); + @Test + public void testEncryptDecrypt() throws IllegalArgumentException, IllegalAccessException, NoSuchFieldException, SecurityException { + Field field = Crypto.class.getDeclaredField("DEFAULT_ENCRYPTION_KEY_PATH"); + field.setAccessible(true); + Field modifiersField = Field.class.getDeclaredField("modifiers"); modifiersField.setAccessible(true); modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL); - field.set(Crypto.class, new File("src/test/resources/test_encryption_key").getAbsolutePath()); - Crypto.loadDefaultKey(); - byte[] ciphertext = Crypto.encryptData(TEST_MESSAGE.getBytes()); - assertTrue(!new String(ciphertext).contentEquals(TEST_MESSAGE)); - String plaintext = new String(Crypto.decryptData(ciphertext)); - assertEquals(plaintext, TEST_MESSAGE); - } - + field.set(Crypto.class, new File("src/test/resources/test_encryption_key").getAbsolutePath()); + Crypto.loadDefaultKey(); + byte[] ciphertext = Crypto.encryptData(TEST_MESSAGE.getBytes()); + assertTrue(!new String(ciphertext).contentEquals(TEST_MESSAGE)); + String plaintext = new String(Crypto.decryptData(ciphertext)); + assertEquals(plaintext, TEST_MESSAGE); + } + } diff --git a/services/pic-sure-hpds/common/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/crypto/CryptoNamedKeyTest.java b/services/pic-sure-hpds/common/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/crypto/CryptoNamedKeyTest.java index 435b51508..d18d1495e 100644 --- a/services/pic-sure-hpds/common/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/crypto/CryptoNamedKeyTest.java +++ b/services/pic-sure-hpds/common/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/crypto/CryptoNamedKeyTest.java @@ -15,49 +15,50 @@ @Disabled // We should rewrite the crypto class to make it more testable, these tests don't work on certain JDKs public class CryptoNamedKeyTest { - private static final String TEST_NAMED_ENCRYPTIOON_KEY_PATH = "src/test/resources/test_named_encryption_key"; - - String TEST_MESSAGE = "This is a test."; - - String TEST_NAMED_KEY = "TEST_NAMED_KEY"; - - @BeforeTestClass - public static void overrideDefaultKeyLocation() throws IllegalArgumentException, IllegalAccessException, NoSuchFieldException, SecurityException { - Field field = Crypto.class.getDeclaredField("DEFAULT_ENCRYPTION_KEY_PATH"); - field.setAccessible(true); - Field modifiersField = Field.class.getDeclaredField("modifiers"); - modifiersField.setAccessible(true); - modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL); - field.set(Crypto.class, new File("src/test/resources/test_encryption_key").getAbsolutePath()); - Crypto.loadDefaultKey(); - } - - @Test - public void testNamedKeyEncryptDecrypt() { - Crypto.loadKey("TEST_NAMED_KEY", TEST_NAMED_ENCRYPTIOON_KEY_PATH); - byte[] ciphertext = Crypto.encryptData(TEST_MESSAGE.getBytes()); - assertTrue(!new String(ciphertext).contentEquals(TEST_MESSAGE)); - String plaintext = new String(Crypto.decryptData(ciphertext)); - assertEquals(plaintext, TEST_MESSAGE); - } - - @Test - public void testNamedKeyEncryptNotUsingDefaultKey() { - assertThrows(AEADBadTagException.class, () -> { - Crypto.loadKey(TEST_NAMED_KEY, TEST_NAMED_ENCRYPTIOON_KEY_PATH); - byte[] ciphertext = Crypto.encryptData(TEST_NAMED_KEY, TEST_MESSAGE.getBytes()); - assertFalse(new String(ciphertext).contentEquals(TEST_MESSAGE)); - Crypto.decryptData(ciphertext); - }); - } - - @Test - public void testNamedKeyDecryptNotUsingDefaultKey() { - assertThrows(AEADBadTagException.class, () -> { - Crypto.loadKey(TEST_NAMED_KEY, TEST_NAMED_ENCRYPTIOON_KEY_PATH); - byte[] ciphertext = Crypto.encryptData(TEST_MESSAGE.getBytes()); - assertTrue(!new String(ciphertext).contentEquals(TEST_MESSAGE)); - Crypto.decryptData(TEST_NAMED_KEY, ciphertext); - }); - } + private static final String TEST_NAMED_ENCRYPTIOON_KEY_PATH = "src/test/resources/test_named_encryption_key"; + + String TEST_MESSAGE = "This is a test."; + + String TEST_NAMED_KEY = "TEST_NAMED_KEY"; + + @BeforeTestClass + public static void overrideDefaultKeyLocation() + throws IllegalArgumentException, IllegalAccessException, NoSuchFieldException, SecurityException { + Field field = Crypto.class.getDeclaredField("DEFAULT_ENCRYPTION_KEY_PATH"); + field.setAccessible(true); + Field modifiersField = Field.class.getDeclaredField("modifiers"); + modifiersField.setAccessible(true); + modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL); + field.set(Crypto.class, new File("src/test/resources/test_encryption_key").getAbsolutePath()); + Crypto.loadDefaultKey(); + } + + @Test + public void testNamedKeyEncryptDecrypt() { + Crypto.loadKey("TEST_NAMED_KEY", TEST_NAMED_ENCRYPTIOON_KEY_PATH); + byte[] ciphertext = Crypto.encryptData(TEST_MESSAGE.getBytes()); + assertTrue(!new String(ciphertext).contentEquals(TEST_MESSAGE)); + String plaintext = new String(Crypto.decryptData(ciphertext)); + assertEquals(plaintext, TEST_MESSAGE); + } + + @Test + public void testNamedKeyEncryptNotUsingDefaultKey() { + assertThrows(AEADBadTagException.class, () -> { + Crypto.loadKey(TEST_NAMED_KEY, TEST_NAMED_ENCRYPTIOON_KEY_PATH); + byte[] ciphertext = Crypto.encryptData(TEST_NAMED_KEY, TEST_MESSAGE.getBytes()); + assertFalse(new String(ciphertext).contentEquals(TEST_MESSAGE)); + Crypto.decryptData(ciphertext); + }); + } + + @Test + public void testNamedKeyDecryptNotUsingDefaultKey() { + assertThrows(AEADBadTagException.class, () -> { + Crypto.loadKey(TEST_NAMED_KEY, TEST_NAMED_ENCRYPTIOON_KEY_PATH); + byte[] ciphertext = Crypto.encryptData(TEST_MESSAGE.getBytes()); + assertTrue(!new String(ciphertext).contentEquals(TEST_MESSAGE)); + Crypto.decryptData(TEST_NAMED_KEY, ciphertext); + }); + } } diff --git a/services/pic-sure-hpds/data/pom.xml b/services/pic-sure-hpds/data/pom.xml index c1f5f2f48..abcff078a 100644 --- a/services/pic-sure-hpds/data/pom.xml +++ b/services/pic-sure-hpds/data/pom.xml @@ -5,7 +5,7 @@ pic-sure-hpds edu.harvard.hms.dbmi.avillach.hpds - 3.0.0-SNAPSHOT + ${revision} data diff --git a/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/BucketIndexBySample.java b/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/BucketIndexBySample.java index cf653a72b..afa68be0d 100644 --- a/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/BucketIndexBySample.java +++ b/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/BucketIndexBySample.java @@ -13,184 +13,179 @@ import edu.harvard.hms.dbmi.avillach.hpds.storage.FileBackedByteIndexedStorage; public class BucketIndexBySample implements Serializable { - private static final long serialVersionUID = -1230735595028630687L; - private static final String STORAGE_FILE_NAME = "BucketIndexBySampleStorage.javabin"; - - List patientIds; - ArrayList contigSet; - - transient Logger log = LoggerFactory.getLogger(BucketIndexBySample.class); - - /** - * Threadsafe Map of patientNum to a BigInteger which acts as a bitmask of the buckets in which each patient - * has a variant. The bits in the BigInteger are indexed by the bucketList. To find the offset of a bucket's - * bit in the mask use patientBucketMask.get(patientId).testBit(Collections - */ - private FileBackedByteIndexedStorage patientBucketMasks; - - /** - * ArrayList containing all bucket keys in the dataset used as an index for the patientBucketMask offsets. - * This list is in natural sort order so Collections.binarySearch should be used instead of indexOf when - * finding the offset of a given bucket. - */ - private ArrayList bucketList = new ArrayList(); - - public BucketIndexBySample(VariantStore variantStore, String storageDir) throws FileNotFoundException { - log.info("Creating new Bucket Index by Sample"); - final String storageFileStr = storageDir + STORAGE_FILE_NAME; - - contigSet = new ArrayList(variantStore.getVariantMaskStorage().keySet()); - - //Create a bucketList, containing keys for all buckets in the variantStore - for(String contig: contigSet){ - FileBackedByteIndexedStorage> contigStore = variantStore.getVariantMaskStorage().get(contig); - if(contigStore != null && contigStore.keys() != null) { - bucketList.addAll(contigStore.keys().stream().map( - (Integer bucket)->{ - return contig + ":" + bucket; - }).collect(Collectors.toList())); - log.debug("Found " + contigStore.keys().size() + " buckets in contig " + contig); - } else { - log.debug("null entry for contig " + contig); - } - } - - // bucketList must be sorted so we can later use binarySearch to find offsets for specific buckets - // in the patientBucketMask records - Collections.sort(bucketList); - - log.debug("Found " + bucketList.size() + " total buckets"); - - // get all patientIds as Integers, eventually this should be fixed in variantStore so they are - // Integers to begin with, which would mean reloading all variant data everywhere so that will - // have to wait. - patientIds = Arrays.stream(variantStore.getPatientIds()).map(id->{ - return Integer.parseInt(id); - }).collect(Collectors.toList()); - - // create empty char arrays for each patient - char[][] patientBucketCharMasks = new char[patientIds.size()][bucketList.size()]; - for(int x = 0;x{ - FileBackedByteIndexedStorage> contigStore = - variantStore.getVariantMaskStorage().get(contig); - if(contigStore != null && contigStore.keys() != null) { - contigStore.keys().stream().forEach( - (Integer bucket)->{ - String bucketKey = contig + ":" + bucket; - - // Create a bitmask with 1 values for each patient who has any variant in this bucket - VariantMask[] patientMaskForBucket = {new VariantMaskSparseImpl(Set.of())}; - contigStore.get(bucket).values().forEach((VariableVariantMasks masks)->{ - if(masks.heterozygousMask!=null) { - patientMaskForBucket[0] = patientMaskForBucket[0].union(masks.heterozygousMask); - } - //add hetreo no call bits to mask - if(masks.heterozygousNoCallMask!=null) { - patientMaskForBucket[0] = patientMaskForBucket[0].union(masks.heterozygousNoCallMask); - } - if(masks.homozygousMask!=null) { - patientMaskForBucket[0] = patientMaskForBucket[0].union(masks.homozygousMask); - } - }); - - // For each patient set the patientBucketCharMask entry to 0 or 1 if they have a variant in the bucket. - int indexOfBucket = Collections.binarySearch(bucketList, bucketKey) + 2; //patientBucketCharMasks has bookend bits - for(int x = 0; x < patientIds.size(); x++) { - if(patientMaskForBucket[0].testBit(x)) { - patientBucketCharMasks[x][indexOfBucket] = '1'; - }else { - patientBucketCharMasks[x][indexOfBucket] = '0'; - } - } - }); - } else { - log.info("null entry for contig " + contig); - } - log.info("completed contig " + contig); - }); - - // populate patientBucketMasks with bucketMasks for each patient - patientBucketMasks = new FileBackedJavaIndexedStorage(Integer.class, BigInteger.class, new File(storageFileStr)); - - int[] processedPatients = new int[1]; - patientIds.parallelStream().forEach((patientId)->{ - BigInteger patientMask = new BigInteger(new String(patientBucketCharMasks[patientIds.indexOf(patientId)]),2); - patientBucketMasks.put(patientId, patientMask); - processedPatients[0] += 1; - int processedPatientsCount = processedPatients[0]; - if (processedPatientsCount % 1000 == 0) { - log.debug("wrote " + processedPatientsCount + " patient bucket masks"); - } - }); - patientBucketMasks.complete(); - log.info("Done creating patient bucket masks"); - } - - /** - * Given a set of variants and a set of patientNums, filter out variants which no patients in the patientSet - * have any variant in the bucket. This operation is extremely fast and cuts down on processing by excluding - * variants for queries where not all patients are included. - * - * @param variantSet - * @param patientSet - * @return - * @throws IOException - */ - public Set filterVariantSetForPatientSet(Set variantSet, Collection patientSet) throws IOException{ - - BigInteger patientBucketMask = patientSet.stream().findFirst() - .map(id -> patientBucketMasks.get(id)) - .orElseGet(() -> new BigInteger(new String(emptyBucketMaskChar()),2)); - - BigInteger _defaultMask = patientBucketMask; - List patientBucketmasksForSet = patientSet.parallelStream() - .map((patientNum)-> patientBucketMasks.get(patientNum)) - .collect(Collectors.toList()); - for(BigInteger patientMask : patientBucketmasksForSet) { - patientBucketMask = patientMask.or(patientBucketMask); - } - - BigInteger _bucketMask = patientBucketMask; - int maxIndex = bucketList.size() - 1; //use to invert testBit index - return variantSet.parallelStream().filter((variantSpec)->{ - String bucketKey = variantSpec.split(",")[0] + ":" + (Integer.parseInt(variantSpec.split(",")[1])/1000); - - //testBit uses inverted indexes include +2 offset for bookends - int bucketKeyIndex = Collections.binarySearch(bucketList, bucketKey); - if (bucketKeyIndex < 0) { - return false; - } - return _bucketMask.testBit(maxIndex - bucketKeyIndex + 2); - }).collect(Collectors.toSet()); - } - - private char[] _emptyBucketMaskChar = null; - - /** - * Produce an empty patientBucketMask char[] by cloning a momoized empty patientBucketMask after the - * first has been created. - * - * @return - */ - private char[] emptyBucketMaskChar() { - if(_emptyBucketMaskChar == null) { - char[] bucketMaskChar = new char[bucketList.size()+4]; - bucketMaskChar[0] = '1'; - bucketMaskChar[1] = '1'; - bucketMaskChar[bucketMaskChar.length-1] = '1'; - bucketMaskChar[bucketMaskChar.length-2]='1'; - for(int x = 2;x patientIds; + ArrayList contigSet; + + transient Logger log = LoggerFactory.getLogger(BucketIndexBySample.class); + + /** + * Threadsafe Map of patientNum to a BigInteger which acts as a bitmask of the buckets in which each patient has a variant. The bits in + * the BigInteger are indexed by the bucketList. To find the offset of a bucket's bit in the mask use + * patientBucketMask.get(patientId).testBit(Collections + */ + private FileBackedByteIndexedStorage patientBucketMasks; + + /** + * ArrayList containing all bucket keys in the dataset used as an index for the patientBucketMask offsets. This list is in natural sort + * order so Collections.binarySearch should be used instead of indexOf when finding the offset of a given bucket. + */ + private ArrayList bucketList = new ArrayList(); + + public BucketIndexBySample(VariantStore variantStore, String storageDir) throws FileNotFoundException { + log.info("Creating new Bucket Index by Sample"); + final String storageFileStr = storageDir + STORAGE_FILE_NAME; + + contigSet = new ArrayList(variantStore.getVariantMaskStorage().keySet()); + + // Create a bucketList, containing keys for all buckets in the variantStore + for (String contig : contigSet) { + FileBackedByteIndexedStorage> contigStore = + variantStore.getVariantMaskStorage().get(contig); + if (contigStore != null && contigStore.keys() != null) { + bucketList.addAll(contigStore.keys().stream().map((Integer bucket) -> { + return contig + ":" + bucket; + }).collect(Collectors.toList())); + log.debug("Found " + contigStore.keys().size() + " buckets in contig " + contig); + } else { + log.debug("null entry for contig " + contig); + } + } + + // bucketList must be sorted so we can later use binarySearch to find offsets for specific buckets + // in the patientBucketMask records + Collections.sort(bucketList); + + log.debug("Found " + bucketList.size() + " total buckets"); + + // get all patientIds as Integers, eventually this should be fixed in variantStore so they are + // Integers to begin with, which would mean reloading all variant data everywhere so that will + // have to wait. + patientIds = Arrays.stream(variantStore.getPatientIds()).map(id -> { + return Integer.parseInt(id); + }).collect(Collectors.toList()); + + // create empty char arrays for each patient + char[][] patientBucketCharMasks = new char[patientIds.size()][bucketList.size()]; + for (int x = 0; x < patientBucketCharMasks.length; x++) { + patientBucketCharMasks[x] = emptyBucketMaskChar(); + } + contigSet.parallelStream().forEach((contig) -> { + FileBackedByteIndexedStorage> contigStore = + variantStore.getVariantMaskStorage().get(contig); + if (contigStore != null && contigStore.keys() != null) { + contigStore.keys().stream().forEach((Integer bucket) -> { + String bucketKey = contig + ":" + bucket; + + // Create a bitmask with 1 values for each patient who has any variant in this bucket + VariantMask[] patientMaskForBucket = {new VariantMaskSparseImpl(Set.of())}; + contigStore.get(bucket).values().forEach((VariableVariantMasks masks) -> { + if (masks.heterozygousMask != null) { + patientMaskForBucket[0] = patientMaskForBucket[0].union(masks.heterozygousMask); + } + // add hetreo no call bits to mask + if (masks.heterozygousNoCallMask != null) { + patientMaskForBucket[0] = patientMaskForBucket[0].union(masks.heterozygousNoCallMask); + } + if (masks.homozygousMask != null) { + patientMaskForBucket[0] = patientMaskForBucket[0].union(masks.homozygousMask); + } + }); + + // For each patient set the patientBucketCharMask entry to 0 or 1 if they have a variant in the bucket. + int indexOfBucket = Collections.binarySearch(bucketList, bucketKey) + 2; // patientBucketCharMasks has bookend bits + for (int x = 0; x < patientIds.size(); x++) { + if (patientMaskForBucket[0].testBit(x)) { + patientBucketCharMasks[x][indexOfBucket] = '1'; + } else { + patientBucketCharMasks[x][indexOfBucket] = '0'; + } + } + }); + } else { + log.info("null entry for contig " + contig); + } + log.info("completed contig " + contig); + }); + + // populate patientBucketMasks with bucketMasks for each patient + patientBucketMasks = new FileBackedJavaIndexedStorage(Integer.class, BigInteger.class, new File(storageFileStr)); + + int[] processedPatients = new int[1]; + patientIds.parallelStream().forEach((patientId) -> { + BigInteger patientMask = new BigInteger(new String(patientBucketCharMasks[patientIds.indexOf(patientId)]), 2); + patientBucketMasks.put(patientId, patientMask); + processedPatients[0] += 1; + int processedPatientsCount = processedPatients[0]; + if (processedPatientsCount % 1000 == 0) { + log.debug("wrote " + processedPatientsCount + " patient bucket masks"); + } + }); + patientBucketMasks.complete(); + log.info("Done creating patient bucket masks"); + } + + /** + * Given a set of variants and a set of patientNums, filter out variants which no patients in the patientSet have any variant in the + * bucket. This operation is extremely fast and cuts down on processing by excluding variants for queries where not all patients are + * included. + * + * @param variantSet + * @param patientSet + * @return + * @throws IOException + */ + public Set filterVariantSetForPatientSet(Set variantSet, Collection patientSet) throws IOException { + + BigInteger patientBucketMask = patientSet.stream().findFirst().map(id -> patientBucketMasks.get(id)) + .orElseGet(() -> new BigInteger(new String(emptyBucketMaskChar()), 2)); + + BigInteger _defaultMask = patientBucketMask; + List patientBucketmasksForSet = + patientSet.parallelStream().map((patientNum) -> patientBucketMasks.get(patientNum)).collect(Collectors.toList()); + for (BigInteger patientMask : patientBucketmasksForSet) { + patientBucketMask = patientMask.or(patientBucketMask); + } + + BigInteger _bucketMask = patientBucketMask; + int maxIndex = bucketList.size() - 1; // use to invert testBit index + return variantSet.parallelStream().filter((variantSpec) -> { + String bucketKey = variantSpec.split(",")[0] + ":" + (Integer.parseInt(variantSpec.split(",")[1]) / 1000); + + // testBit uses inverted indexes include +2 offset for bookends + int bucketKeyIndex = Collections.binarySearch(bucketList, bucketKey); + if (bucketKeyIndex < 0) { + return false; + } + return _bucketMask.testBit(maxIndex - bucketKeyIndex + 2); + }).collect(Collectors.toSet()); + } + + private char[] _emptyBucketMaskChar = null; + + /** + * Produce an empty patientBucketMask char[] by cloning a momoized empty patientBucketMask after the first has been created. + * + * @return + */ + private char[] emptyBucketMaskChar() { + if (_emptyBucketMaskChar == null) { + char[] bucketMaskChar = new char[bucketList.size() + 4]; + bucketMaskChar[0] = '1'; + bucketMaskChar[1] = '1'; + bucketMaskChar[bucketMaskChar.length - 1] = '1'; + bucketMaskChar[bucketMaskChar.length - 2] = '1'; + for (int x = 2; x < bucketMaskChar.length - 2; x++) { + bucketMaskChar[x] = '0'; + } + _emptyBucketMaskChar = bucketMaskChar; + } + return _emptyBucketMaskChar.clone(); + } + + public void updateStorageDirectory(File storageDirectory) { + patientBucketMasks.updateStorageDirectory(storageDirectory); + } +} diff --git a/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/CompressedIndex.java b/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/CompressedIndex.java index edf7c2bd7..224b9273b 100644 --- a/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/CompressedIndex.java +++ b/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/CompressedIndex.java @@ -23,168 +23,158 @@ public class CompressedIndex implements Serializable { - /** - * - */ - private static final long serialVersionUID = 5089713203903957829L; - private static final Logger log = LoggerFactory.getLogger(CompressedIndex.class); - public Float min = Float.MAX_VALUE, max = Float.MIN_VALUE; - private HashMap, byte[]> compressedRangeMap; - private int valueCount; - - public TreeMap> buildContinuousValuesMap(FileBackedByteIndexedStorage allValues) { - TreeMap> continuousValueMap = new TreeMap<>(); - for(String key : allValues.keys()) { - try{ - Float DoubleValue = Float.parseFloat(key.trim()); - TreeSet currentValues = continuousValueMap.get(DoubleValue); - if(currentValues == null) { - currentValues = new TreeSet<>(); - continuousValueMap.put(DoubleValue, currentValues); - } - currentValues.add(key); - continuousValueMap.put(DoubleValue, currentValues); - setMinAndMax(DoubleValue); - }catch(NumberFormatException e) { - String[] valuesInKey = key.split(","); - if(valuesInKey.length > 1) { - for(String value : valuesInKey) { - try { - Float DoubleValue = Float.parseFloat(value.trim()); - TreeSet currentValues = continuousValueMap.get(DoubleValue); - if(currentValues == null) { - currentValues = new TreeSet<>(); - continuousValueMap.put(DoubleValue, currentValues); - } - currentValues.add(key); - continuousValueMap.put(DoubleValue, currentValues); - setMinAndMax(DoubleValue); - }catch(NumberFormatException e3) { - log.info("Unable to parse value : " + value.trim()); - } - } - } - } - } - this.setValueCount(continuousValueMap.size()); - return continuousValueMap; - } - - private void setMinAndMax(Float DoubleValue) { - if(min > DoubleValue) { - min = DoubleValue; - } - if(max < DoubleValue) { - max = DoubleValue; - } - } - - public void buildIndex(TreeMap> continuousValueMap) { - Set continuousValuesMapKeys = new TreeSet(continuousValueMap.keySet()); - List> partitions = Lists.partition(new ArrayList(continuousValuesMapKeys), 1000); - HashMap, TreeMap>> rangeMap = new HashMap<>(); - List partition = partitions.get(0); - SortedMap> partitionMap = continuousValueMap.subMap(partition.get(0), partition.get(partition.size()-1)); - rangeMap.put( - Range.openClosed(partition.get(0), partition.get(partition.size()-1)), - new TreeMap<>(partitionMap)); - for(int x = 0; x < partitions.size()-1; x++) { - partition = partitions.get(x); - partitionMap = continuousValueMap.subMap(partition.get(0), partition.get(partition.size()-1)); - rangeMap.put(Range.openClosed(partition.get(0), partitions.get(x+1).get(0)), new TreeMap<>(partitionMap)); - } - partition = partitions.get(partitions.size()-1); - partitionMap = continuousValueMap.subMap(partition.get(0), partition.get(partition.size()-1)); - rangeMap.put(Range.openClosed(partition.get(0), partition.get(partition.size()-1)), new TreeMap<>(partitionMap)); - compressedRangeMap = new HashMap<>(rangeMap.entrySet().stream().collect( - Collectors.toMap( - (key)->{return key.getKey();}, - (part)->{ - byte[] compressed = null; - try( - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - GZIPOutputStream gzos = new GZIPOutputStream(baos); - ObjectOutputStream out = new ObjectOutputStream(gzos); - ){ - out.writeObject(part.getValue()); - out.flush(); - out.close(); - gzos.flush(); - gzos.close(); - compressed = baos.toByteArray(); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - return compressed; - }) - )); - } - - public List getValuesInRange(Range targetRange) { - - log.debug("Getting valuesInRange : " + targetRange); - // Get a list of ranges that are connected to the target range - List connectedRanges = compressedRangeMap.keySet().stream().filter((range)->{ - return range.isConnected(targetRange); - }).collect(Collectors.toList()); - - // Get a list of ranges that enclose the target range completely - List enclosingRanges = connectedRanges.stream().filter((range)->{ - return targetRange.encloses(range); - }).collect(Collectors.toList()); - - List valuesInRange = new ArrayList<>(); - - // Add all variants from enclosing ranges - enclosingRanges.forEach( - range->{ - TreeMap> continousValueMap = retrieveRangeMap(range); - continousValueMap.values().forEach( - variantSet->{ - valuesInRange.addAll(variantSet); - }); - }); - - // We already added all variants in the enclosing ranges - connectedRanges.removeAll(enclosingRanges); - - connectedRanges.forEach( - range ->{ - TreeMap> continousValueMap = retrieveRangeMap(range); - log.debug("Searching within : " + range.lowerEndpoint() + " : " + range.upperEndpoint()); - continousValueMap.entrySet().stream().forEach( - entry->{ - if(targetRange.contains(entry.getKey())) { - log.debug("Adding : " + entry.getValue().first() + " : " + entry.getValue().last()); - valuesInRange.addAll(entry.getValue()); - } - }); - }); - return valuesInRange; - } - - private TreeMap> retrieveRangeMap(Range range) { - TreeMap> continousValueMap = null; - try( - ByteArrayInputStream bais = new ByteArrayInputStream(compressedRangeMap.get(range)); - GZIPInputStream gzis = new GZIPInputStream(bais); - ObjectInputStream ois = new ObjectInputStream(gzis); - ){ - continousValueMap = (TreeMap>)ois.readObject(); - } catch (IOException e) { - throw new UncheckedIOException(e); - } catch (ClassNotFoundException e) { - throw new RuntimeException(e); - } - return continousValueMap; - } - - public int getValueCount() { - return valueCount; - } - - public void setValueCount(int valueCount) { - this.valueCount = valueCount; - } + /** + * + */ + private static final long serialVersionUID = 5089713203903957829L; + private static final Logger log = LoggerFactory.getLogger(CompressedIndex.class); + public Float min = Float.MAX_VALUE, max = Float.MIN_VALUE; + private HashMap, byte[]> compressedRangeMap; + private int valueCount; + + public TreeMap> buildContinuousValuesMap(FileBackedByteIndexedStorage allValues) { + TreeMap> continuousValueMap = new TreeMap<>(); + for (String key : allValues.keys()) { + try { + Float DoubleValue = Float.parseFloat(key.trim()); + TreeSet currentValues = continuousValueMap.get(DoubleValue); + if (currentValues == null) { + currentValues = new TreeSet<>(); + continuousValueMap.put(DoubleValue, currentValues); + } + currentValues.add(key); + continuousValueMap.put(DoubleValue, currentValues); + setMinAndMax(DoubleValue); + } catch (NumberFormatException e) { + String[] valuesInKey = key.split(","); + if (valuesInKey.length > 1) { + for (String value : valuesInKey) { + try { + Float DoubleValue = Float.parseFloat(value.trim()); + TreeSet currentValues = continuousValueMap.get(DoubleValue); + if (currentValues == null) { + currentValues = new TreeSet<>(); + continuousValueMap.put(DoubleValue, currentValues); + } + currentValues.add(key); + continuousValueMap.put(DoubleValue, currentValues); + setMinAndMax(DoubleValue); + } catch (NumberFormatException e3) { + log.info("Unable to parse value : " + value.trim()); + } + } + } + } + } + this.setValueCount(continuousValueMap.size()); + return continuousValueMap; + } + + private void setMinAndMax(Float DoubleValue) { + if (min > DoubleValue) { + min = DoubleValue; + } + if (max < DoubleValue) { + max = DoubleValue; + } + } + + public void buildIndex(TreeMap> continuousValueMap) { + Set continuousValuesMapKeys = new TreeSet(continuousValueMap.keySet()); + List> partitions = Lists.partition(new ArrayList(continuousValuesMapKeys), 1000); + HashMap, TreeMap>> rangeMap = new HashMap<>(); + List partition = partitions.get(0); + SortedMap> partitionMap = continuousValueMap.subMap(partition.get(0), partition.get(partition.size() - 1)); + rangeMap.put(Range.openClosed(partition.get(0), partition.get(partition.size() - 1)), new TreeMap<>(partitionMap)); + for (int x = 0; x < partitions.size() - 1; x++) { + partition = partitions.get(x); + partitionMap = continuousValueMap.subMap(partition.get(0), partition.get(partition.size() - 1)); + rangeMap.put(Range.openClosed(partition.get(0), partitions.get(x + 1).get(0)), new TreeMap<>(partitionMap)); + } + partition = partitions.get(partitions.size() - 1); + partitionMap = continuousValueMap.subMap(partition.get(0), partition.get(partition.size() - 1)); + rangeMap.put(Range.openClosed(partition.get(0), partition.get(partition.size() - 1)), new TreeMap<>(partitionMap)); + compressedRangeMap = new HashMap<>(rangeMap.entrySet().stream().collect(Collectors.toMap((key) -> { + return key.getKey(); + }, (part) -> { + byte[] compressed = null; + try ( + ByteArrayOutputStream baos = new ByteArrayOutputStream(); GZIPOutputStream gzos = + new GZIPOutputStream(baos); ObjectOutputStream out = new ObjectOutputStream(gzos); + ) { + out.writeObject(part.getValue()); + out.flush(); + out.close(); + gzos.flush(); + gzos.close(); + compressed = baos.toByteArray(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + return compressed; + }))); + } + + public List getValuesInRange(Range targetRange) { + + log.debug("Getting valuesInRange : " + targetRange); + // Get a list of ranges that are connected to the target range + List connectedRanges = compressedRangeMap.keySet().stream().filter((range) -> { + return range.isConnected(targetRange); + }).collect(Collectors.toList()); + + // Get a list of ranges that enclose the target range completely + List enclosingRanges = connectedRanges.stream().filter((range) -> { + return targetRange.encloses(range); + }).collect(Collectors.toList()); + + List valuesInRange = new ArrayList<>(); + + // Add all variants from enclosing ranges + enclosingRanges.forEach(range -> { + TreeMap> continousValueMap = retrieveRangeMap(range); + continousValueMap.values().forEach(variantSet -> { + valuesInRange.addAll(variantSet); + }); + }); + + // We already added all variants in the enclosing ranges + connectedRanges.removeAll(enclosingRanges); + + connectedRanges.forEach(range -> { + TreeMap> continousValueMap = retrieveRangeMap(range); + log.debug("Searching within : " + range.lowerEndpoint() + " : " + range.upperEndpoint()); + continousValueMap.entrySet().stream().forEach(entry -> { + if (targetRange.contains(entry.getKey())) { + log.debug("Adding : " + entry.getValue().first() + " : " + entry.getValue().last()); + valuesInRange.addAll(entry.getValue()); + } + }); + }); + return valuesInRange; + } + + private TreeMap> retrieveRangeMap(Range range) { + TreeMap> continousValueMap = null; + try ( + ByteArrayInputStream bais = new ByteArrayInputStream(compressedRangeMap.get(range)); GZIPInputStream gzis = + new GZIPInputStream(bais); ObjectInputStream ois = new ObjectInputStream(gzis); + ) { + continousValueMap = (TreeMap>) ois.readObject(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } catch (ClassNotFoundException e) { + throw new RuntimeException(e); + } + return continousValueMap; + } + + public int getValueCount() { + return valueCount; + } + + public void setValueCount(int valueCount) { + this.valueCount = valueCount; + } } diff --git a/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/FileBackedByteIndexedInfoStore.java b/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/FileBackedByteIndexedInfoStore.java index 3d730765c..35b6de942 100644 --- a/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/FileBackedByteIndexedInfoStore.java +++ b/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/FileBackedByteIndexedInfoStore.java @@ -15,142 +15,142 @@ public class FileBackedByteIndexedInfoStore implements Serializable { - private static final long serialVersionUID = 6478256007934827195L; - private static final Logger log = LoggerFactory.getLogger(FileBackedByteIndexedInfoStore.class); - public final String column_key; - public final String description; - public boolean isContinuous; - public Float min = Float.MAX_VALUE, max = Float.MIN_VALUE; - - private FileBackedByteIndexedStorage allValues; - public TreeMap> continuousValueMap; - - public CompressedIndex continuousValueIndex; - - public FileBackedByteIndexedStorage getAllValues() { - return allValues; - } - - public List search(String term) { - if(isContinuous) { - return new ArrayList(); - }else { - return allValues.keys().stream().filter((value)->{ - String lowerTerm = term.toLowerCase(Locale.ENGLISH); - return value.toLowerCase(Locale.ENGLISH).contains(lowerTerm); - }).collect(Collectors.toList()); - } - } - - public void addEntry(String value, Integer[] variantIds) throws IOException { - allValues.put(value, variantIds); - } - - - public void complete() { - this.allValues.complete(); - } - - public FileBackedByteIndexedInfoStore(File storageFolder, InfoStore infoStore) throws IOException { - this.allValues = new FileBackedStorageVariantIndexImpl(new File(storageFolder, infoStore.column_key + "_infoStoreStorage.javabin")); - this.description = infoStore.description; - this.column_key = infoStore.column_key; - this.isContinuous = infoStore.isNumeric(); - this.allValues.open(); - if(isContinuous) { - normalizeNumericStore(infoStore); - } - TreeSet sortedKeys = new TreeSet(infoStore.allValues.keySet()); - log.debug(infoStore.column_key + " : " + sortedKeys.size() + " values"); - int x = 0; - for(String key : sortedKeys){ - if(key.contentEquals(".")) { - log.debug("Skipping . value for " + infoStore.column_key); - }else { - if(x%10000 == 0) { - log.debug(infoStore.column_key + " " + ((((double)x) / sortedKeys.size()) * 100) + "% done"); - } - ConcurrentSkipListSet variantIds = infoStore.allValues.get(key); - addEntry(key, variantIds.toArray(new Integer[variantIds.size()])); - x++; - } - } - this.allValues.complete(); - if(isContinuous) { - log.debug(this.column_key + " is continuous, building continuousValueIndex and nulling continuousValueMap."); - this.continuousValueIndex = new CompressedIndex(); - TreeMap> continuousValueMap = this.continuousValueIndex.buildContinuousValuesMap(this.allValues); - this.continuousValueIndex.buildIndex(continuousValueMap); - this.continuousValueMap = null; - } - } - - private static void normalizeNumericStore(InfoStore store) { - TreeSet allKeys = new TreeSet(store.allValues.keySet()); - - ConcurrentHashMap> normalizedValues = new ConcurrentHashMap<>(); - for(String key : allKeys) { - String[] keys = key.split(","); - ConcurrentSkipListSet variantIds = store.allValues.get(key); - if(key.contentEquals(".")) { - //don't add it - }else if(keyHasMultipleValues(keys)) { - for(String value : keys) { - if(value.contentEquals(".")) { - - }else { - ConcurrentSkipListSet normalizedVariantIds = normalizedValues.get(value); - if(normalizedVariantIds == null) { - normalizedVariantIds = variantIds; - }else { - normalizedVariantIds.addAll(variantIds); - } - normalizedValues.put(value, normalizedVariantIds); - } - } - }else { - if(key.contentEquals(".")) { - - }else { - ConcurrentSkipListSet normalizedVariantIds = normalizedValues.get(key); - if(normalizedVariantIds == null) { - normalizedVariantIds = variantIds; - }else { - normalizedVariantIds.addAll(variantIds); - } - normalizedValues.put(key, normalizedVariantIds); - } - } - - } - store.allValues = normalizedValues; - } - - private static boolean keyHasMultipleValues(String[] keys) { - int x = 0; - for(String k : keys) { - if(k == null || k.isEmpty()) { - - }else { - x++; - } - } - return x>1; - } - - public void updateStorageDirectory(File storageDirectory) { - allValues.updateStorageDirectory(storageDirectory); - } - - public void write(File outputFile) { - try( - FileOutputStream fos = new FileOutputStream(outputFile); - GZIPOutputStream gzos = new GZIPOutputStream(fos); - ObjectOutputStream oos = new ObjectOutputStream(gzos);) { - oos.writeObject(this); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } + private static final long serialVersionUID = 6478256007934827195L; + private static final Logger log = LoggerFactory.getLogger(FileBackedByteIndexedInfoStore.class); + public final String column_key; + public final String description; + public boolean isContinuous; + public Float min = Float.MAX_VALUE, max = Float.MIN_VALUE; + + private FileBackedByteIndexedStorage allValues; + public TreeMap> continuousValueMap; + + public CompressedIndex continuousValueIndex; + + public FileBackedByteIndexedStorage getAllValues() { + return allValues; + } + + public List search(String term) { + if (isContinuous) { + return new ArrayList(); + } else { + return allValues.keys().stream().filter((value) -> { + String lowerTerm = term.toLowerCase(Locale.ENGLISH); + return value.toLowerCase(Locale.ENGLISH).contains(lowerTerm); + }).collect(Collectors.toList()); + } + } + + public void addEntry(String value, Integer[] variantIds) throws IOException { + allValues.put(value, variantIds); + } + + + public void complete() { + this.allValues.complete(); + } + + public FileBackedByteIndexedInfoStore(File storageFolder, InfoStore infoStore) throws IOException { + this.allValues = new FileBackedStorageVariantIndexImpl(new File(storageFolder, infoStore.column_key + "_infoStoreStorage.javabin")); + this.description = infoStore.description; + this.column_key = infoStore.column_key; + this.isContinuous = infoStore.isNumeric(); + this.allValues.open(); + if (isContinuous) { + normalizeNumericStore(infoStore); + } + TreeSet sortedKeys = new TreeSet(infoStore.allValues.keySet()); + log.debug(infoStore.column_key + " : " + sortedKeys.size() + " values"); + int x = 0; + for (String key : sortedKeys) { + if (key.contentEquals(".")) { + log.debug("Skipping . value for " + infoStore.column_key); + } else { + if (x % 10000 == 0) { + log.debug(infoStore.column_key + " " + ((((double) x) / sortedKeys.size()) * 100) + "% done"); + } + ConcurrentSkipListSet variantIds = infoStore.allValues.get(key); + addEntry(key, variantIds.toArray(new Integer[variantIds.size()])); + x++; + } + } + this.allValues.complete(); + if (isContinuous) { + log.debug(this.column_key + " is continuous, building continuousValueIndex and nulling continuousValueMap."); + this.continuousValueIndex = new CompressedIndex(); + TreeMap> continuousValueMap = this.continuousValueIndex.buildContinuousValuesMap(this.allValues); + this.continuousValueIndex.buildIndex(continuousValueMap); + this.continuousValueMap = null; + } + } + + private static void normalizeNumericStore(InfoStore store) { + TreeSet allKeys = new TreeSet(store.allValues.keySet()); + + ConcurrentHashMap> normalizedValues = new ConcurrentHashMap<>(); + for (String key : allKeys) { + String[] keys = key.split(","); + ConcurrentSkipListSet variantIds = store.allValues.get(key); + if (key.contentEquals(".")) { + // don't add it + } else if (keyHasMultipleValues(keys)) { + for (String value : keys) { + if (value.contentEquals(".")) { + + } else { + ConcurrentSkipListSet normalizedVariantIds = normalizedValues.get(value); + if (normalizedVariantIds == null) { + normalizedVariantIds = variantIds; + } else { + normalizedVariantIds.addAll(variantIds); + } + normalizedValues.put(value, normalizedVariantIds); + } + } + } else { + if (key.contentEquals(".")) { + + } else { + ConcurrentSkipListSet normalizedVariantIds = normalizedValues.get(key); + if (normalizedVariantIds == null) { + normalizedVariantIds = variantIds; + } else { + normalizedVariantIds.addAll(variantIds); + } + normalizedValues.put(key, normalizedVariantIds); + } + } + + } + store.allValues = normalizedValues; + } + + private static boolean keyHasMultipleValues(String[] keys) { + int x = 0; + for (String k : keys) { + if (k == null || k.isEmpty()) { + + } else { + x++; + } + } + return x > 1; + } + + public void updateStorageDirectory(File storageDirectory) { + allValues.updateStorageDirectory(storageDirectory); + } + + public void write(File outputFile) { + try ( + FileOutputStream fos = new FileOutputStream(outputFile); GZIPOutputStream gzos = + new GZIPOutputStream(fos); ObjectOutputStream oos = new ObjectOutputStream(gzos); + ) { + oos.writeObject(this); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } } diff --git a/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/InfoStore.java b/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/InfoStore.java index 272d32caf..8669df967 100644 --- a/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/InfoStore.java +++ b/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/InfoStore.java @@ -13,79 +13,79 @@ import java.util.stream.Collectors; public class InfoStore implements Serializable { - - private static final long serialVersionUID = 6478256007934827195L; - private static final Logger log = LoggerFactory.getLogger(InfoStore.class); - public final String column_key; - public final String description; - public ConcurrentHashMap> allValues = new ConcurrentHashMap<>(); - private String prefix; + private static final long serialVersionUID = 6478256007934827195L; + private static final Logger log = LoggerFactory.getLogger(InfoStore.class); + public final String column_key; + public final String description; - public List search(String term) { - if(isNumeric()) { - return new ArrayList(); - }else { - return allValues.keySet().stream().filter((value)->{ - String lowerTerm = term.toLowerCase(Locale.ENGLISH); - return value.toLowerCase(Locale.ENGLISH).contains(lowerTerm); - }).collect(Collectors.toList()); - } - } + public ConcurrentHashMap> allValues = new ConcurrentHashMap<>(); + private String prefix; - public InfoStore(String description, String delimiter, String key) { - this.prefix = key + "="; - this.description = description; - this.column_key = key; - } + public List search(String term) { + if (isNumeric()) { + return new ArrayList(); + } else { + return allValues.keySet().stream().filter((value) -> { + String lowerTerm = term.toLowerCase(Locale.ENGLISH); + return value.toLowerCase(Locale.ENGLISH).contains(lowerTerm); + }).collect(Collectors.toList()); + } + } - public boolean isNumeric() { - int nonNumericCount = 0; - int numericCount = 0; - log.debug("Testing for numeric : " + this.column_key + " : " + allValues.size() + " values"); - KeySetView> allKeys = allValues.keySet(); - for(String key : allKeys){ - try { - Double.parseDouble(key); - numericCount++; - }catch(NumberFormatException e) { - String[] keys = key.split(","); - for(String key2 : keys) { - try { - Double.parseDouble(key2); - numericCount++; - }catch(NumberFormatException e2) { - if(key.contentEquals(".")) { - nonNumericCount++; - } - } - } - } - if(nonNumericCount > 1) { - log.debug(this.column_key + " is not numeric"); - return false; - } - if(numericCount > 10000 || numericCount > (allKeys.size()/2)) { - log.debug(this.column_key + " is numeric"); - return true; - } - }; - log.debug(this.column_key + " is not numeric"); - return false; - } + public InfoStore(String description, String delimiter, String key) { + this.prefix = key + "="; + this.description = description; + this.column_key = key; + } - public void processRecord(Integer variantId, String[] infoValues) { - for(String value : infoValues) { - if(value.startsWith(prefix)) { - String valueWithoutkey = value.substring(prefix.length()); - ConcurrentSkipListSet entriesForValue = allValues.get(valueWithoutkey); - if(entriesForValue == null) { - entriesForValue = new ConcurrentSkipListSet<>(); - allValues.put(valueWithoutkey, entriesForValue); - } - entriesForValue.add(variantId); - } - } - } + public boolean isNumeric() { + int nonNumericCount = 0; + int numericCount = 0; + log.debug("Testing for numeric : " + this.column_key + " : " + allValues.size() + " values"); + KeySetView> allKeys = allValues.keySet(); + for (String key : allKeys) { + try { + Double.parseDouble(key); + numericCount++; + } catch (NumberFormatException e) { + String[] keys = key.split(","); + for (String key2 : keys) { + try { + Double.parseDouble(key2); + numericCount++; + } catch (NumberFormatException e2) { + if (key.contentEquals(".")) { + nonNumericCount++; + } + } + } + } + if (nonNumericCount > 1) { + log.debug(this.column_key + " is not numeric"); + return false; + } + if (numericCount > 10000 || numericCount > (allKeys.size() / 2)) { + log.debug(this.column_key + " is numeric"); + return true; + } + } ; + log.debug(this.column_key + " is not numeric"); + return false; + } -} \ No newline at end of file + public void processRecord(Integer variantId, String[] infoValues) { + for (String value : infoValues) { + if (value.startsWith(prefix)) { + String valueWithoutkey = value.substring(prefix.length()); + ConcurrentSkipListSet entriesForValue = allValues.get(valueWithoutkey); + if (entriesForValue == null) { + entriesForValue = new ConcurrentSkipListSet<>(); + allValues.put(valueWithoutkey, entriesForValue); + } + entriesForValue.add(variantId); + } + } + } + +} diff --git a/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/VariableVariantMasks.java b/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/VariableVariantMasks.java index 7c56828f5..cebb3fffd 100644 --- a/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/VariableVariantMasks.java +++ b/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/VariableVariantMasks.java @@ -13,257 +13,251 @@ public class VariableVariantMasks implements Serializable { - private static Logger log = LoggerFactory.getLogger(VariableVariantMasks.class); - - private static final long serialVersionUID = 6225420483804601477L; - private static final String oneone = "11"; - private static final char one = '1'; - private static final char zero = '0'; - private static final String hetero = "0/1"; - private static final String heteroDel = "1/0"; - private static final String heteroPhased = "0|1"; - private static final String heteroPhased2 = "1|0"; - private static final String homo = "1/1"; - private static final String homoPhased = "1|1"; - private static final String homoNoCall = "./."; - private static final String heteroNoCall = "./1"; - - private static Map emptyBitmaskMap = new ConcurrentHashMap<>(); - - public static int SPARSE_VARIANT_THRESHOLD = 5; - - /* - * These indices result from the char values of the 3 characters in a VCF sample - * record summed as integers % 7 - * - * This allows us to not actually do string comparisons, instead we add 3 values, - * do one modulus operation, then use the result as the index into the result array. - */ - - // ./0 = (46 + 47 + 48) % 7 = 1 - // 0|. = (48 + 124 + 46) % 7 = 1 - // .|0 = (46 + 124 + 48) % 7 = 1 - public static final int HETERO_NOCALL_REF_CHAR_INDEX = 1; - - // ./1 = (46 + 47 + 49) % 7 = 2 - // 1|. = (49 + 124 + 46) % 7 = 2 - // .|1 = (46 + 124 + 49) % 7 = 2 - // ./1 = (46 + 47 + 49) % 7 = 2 - // 1|. = (49 + 124 + 46) % 7 = 2 - // .|1 = (46 + 124 + 49) % 7 = 2 - public static final int HETERO_NOCALL_VARIANT_CHAR_INDEX = 2; - - // 0/0 = (48 + 47 + 48) % 7 = 3 - // 0|0 = (48 + 124 + 48) % 7 = 3 - public static final int ZERO_ZERO_CHAR_INDEX = 3; - - // 0/1 = (48 + 47 + 49) % 7 = 4 - // 1|0 = (49 + 124 + 48) % 7 = 4 - // 0|1 = (48 + 124 + 49) % 7 = 4 - public static final int ZERO_ONE_CHAR_INDEX = 4; - - // 1/1 = (49 + 47 + 49) % 7 = 5 - // 1|1 = (49 + 124 + 49) % 7 = 5 - public static final int ONE_ONE_CHAR_INDEX = 5; - - // ./. = (46 + 47 + 46) % 7 = 6 - // .|. = (46 + 124 + 46) % 7 = 6 - public static final int HOMO_NOCALL_CHAR_INDEX = 6; - - public VariableVariantMasks(char[][] maskValues) { - String heteroMaskStringRaw = new String(maskValues[ZERO_ONE_CHAR_INDEX]); - String homoMaskStringRaw = new String(maskValues[ONE_ONE_CHAR_INDEX]); - String heteroNoCallMaskStringRaw = new String(maskValues[HETERO_NOCALL_VARIANT_CHAR_INDEX]); - String homoNoCallMaskStringRaw = new String(maskValues[HOMO_NOCALL_CHAR_INDEX]); - - heterozygousMask = variantMaskFromRawString(heteroMaskStringRaw); - homozygousMask = variantMaskFromRawString(homoMaskStringRaw); - heterozygousNoCallMask = variantMaskFromRawString(heteroNoCallMaskStringRaw); - homozygousNoCallMask = variantMaskFromRawString(homoNoCallMaskStringRaw); - } - - private VariantMask variantMaskFromRawString(String maskStringRaw) { - if (!maskStringRaw.contains("1")) { - return null; - } - - VariantMask variantMask; - BigInteger bitmask = new BigInteger(oneone + new StringBuilder(maskStringRaw).reverse() + oneone, 2); - if (bitmask.bitCount() - 4 > VariableVariantMasks.SPARSE_VARIANT_THRESHOLD) { - variantMask = new VariantMaskBitmaskImpl(bitmask); - } else { - Set patientIndexes = new HashSet<>(); - for(int i = 0; i < bitmask.bitLength() - 4; i++) { - // i + 2 because the mask is padded with 2 bits on each end - if (bitmask.testBit(i + 2)) { - patientIndexes.add(i); - } - } - variantMask = new VariantMaskSparseImpl(patientIndexes); - } - return variantMask; - } - - public VariableVariantMasks() { - } - - @JsonProperty("ho") - @JsonInclude(JsonInclude.Include.NON_NULL) - public VariantMask homozygousMask; - - @JsonProperty("he") - @JsonInclude(JsonInclude.Include.NON_NULL) - public VariantMask heterozygousMask; - - @JsonProperty("hon") - @JsonInclude(JsonInclude.Include.NON_NULL) - public VariantMask homozygousNoCallMask; - - @JsonProperty("hen") - @JsonInclude(JsonInclude.Include.NON_NULL) - public VariantMask heterozygousNoCallMask; - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - VariableVariantMasks that = (VariableVariantMasks) o; - return Objects.equals(homozygousMask, that.homozygousMask) && Objects.equals(heterozygousMask, that.heterozygousMask) && Objects.equals(homozygousNoCallMask, that.homozygousNoCallMask) && Objects.equals(heterozygousNoCallMask, that.heterozygousNoCallMask); - } - - @Override - public int hashCode() { - return Objects.hash(homozygousMask, heterozygousMask, homozygousNoCallMask, heterozygousNoCallMask); - } - - public static BigInteger emptyBitmask(int length) { - BigInteger emptyBitmask = emptyBitmaskMap.get(length); - if (emptyBitmask == null) { - String emptyVariantMask = ""; - for (int i = 0; i < length; i++) { - emptyVariantMask = emptyVariantMask + "0"; - } - BigInteger newEmptyBitmask = new BigInteger("11" + emptyVariantMask + "11", 2); - emptyBitmaskMap.put(length, newEmptyBitmask); - return newEmptyBitmask; - } - return emptyBitmask; - } - - public static VariableVariantMasks append(VariableVariantMasks masks1, int length1, VariableVariantMasks masks2, int length2) { - VariableVariantMasks appendedMasks = new VariableVariantMasks(); - appendedMasks.homozygousMask = appendMask(masks1.homozygousMask, masks2.homozygousMask, length1, length2); - appendedMasks.heterozygousMask = appendMask(masks1.heterozygousMask, masks2.heterozygousMask, length1, length2); - appendedMasks.homozygousNoCallMask = appendMask(masks1.homozygousNoCallMask, masks2.homozygousNoCallMask, length1, length2); - appendedMasks.heterozygousNoCallMask = appendMask(masks1.heterozygousNoCallMask, masks2.heterozygousNoCallMask, length1, length2); - return appendedMasks; - } - - public static VariantMask appendMask(VariantMask variantMask1, VariantMask variantMask2, int length1, int length2) { - if (variantMask1 == null) { - variantMask1 = VariantMask.emptyInstance(); - } - if (variantMask2 == null) { - variantMask2 = VariantMask.emptyInstance(); - } - if (variantMask1.equals(VariantMask.emptyInstance()) && variantMask2.equals(VariantMask.emptyInstance())) { - return null; - } - - if (variantMask1 instanceof VariantMaskSparseImpl) { - if (variantMask2 instanceof VariantMaskSparseImpl) { - return append((VariantMaskSparseImpl) variantMask1, (VariantMaskSparseImpl) variantMask2, length1, length2); - } else if (variantMask2 instanceof VariantMaskBitmaskImpl) { - return append((VariantMaskSparseImpl) variantMask1, (VariantMaskBitmaskImpl) variantMask2, length1, length2); - } else { - throw new RuntimeException("Unknown VariantMask implementation"); - } - } - else if (variantMask1 instanceof VariantMaskBitmaskImpl) { - if (variantMask2 instanceof VariantMaskSparseImpl) { - return append((VariantMaskBitmaskImpl) variantMask1, (VariantMaskSparseImpl) variantMask2, length1, length2); - } else if (variantMask2 instanceof VariantMaskBitmaskImpl) { - return append((VariantMaskBitmaskImpl) variantMask1, (VariantMaskBitmaskImpl) variantMask2, length1, length2); - } else { - throw new RuntimeException("Unknown VariantMask implementation"); - } - } - else { - throw new RuntimeException("Unknown VariantMask implementation"); - } - } - - private static VariantMask append(VariantMaskSparseImpl variantMask1, VariantMaskBitmaskImpl variantMask2, int length1, int length2) { - BigInteger mask1 = emptyBitmask(length1); - for (Integer patientIndex : variantMask1.patientIndexes) { - mask1 = mask1.setBit(patientIndex + 2); - } - String binaryMask1 = mask1.toString(2); - - String binaryMask2 = variantMask2.bitmask.toString(2); - if (binaryMask2.length() - 4 != length2) { - throw new IllegalArgumentException("Bitmask does not match length (" + length2 + "): " + variantMask2.bitmask); - } - - String appendedString = binaryMask2.substring(0, binaryMask2.length() - 2) + - binaryMask1.substring(2); - return new VariantMaskBitmaskImpl(new BigInteger(appendedString, 2)); - } - - private static VariantMask append(VariantMaskBitmaskImpl variantMask1, VariantMaskSparseImpl variantMask2, int length1, int length2) { - String binaryMask1 = variantMask1.bitmask.toString(2); - if (binaryMask1.length() - 4 != length1) { - throw new IllegalArgumentException("Bitmask does not match length (" + length1 + "): " + variantMask1.bitmask); - } - - BigInteger mask2 = emptyBitmask(length2); - for (Integer patientId : variantMask2.patientIndexes) { - mask2 = mask2.setBit(patientId + 2); - } - String binaryMask2 = mask2.toString(2); - - String appendedString = binaryMask2.substring(0, binaryMask2.length() - 2) + - binaryMask1.substring(2); - return new VariantMaskBitmaskImpl(new BigInteger(appendedString, 2)); - } - - private static VariantMask append(VariantMaskBitmaskImpl variantMask1, VariantMaskBitmaskImpl variantMask2, int length1, int length2) { - String binaryMask1 = variantMask1.bitmask.toString(2); - String binaryMask2 = variantMask2.bitmask.toString(2); - - if (binaryMask1.length() - 4 != length1) { - throw new IllegalArgumentException("Bitmask does not match length (" + length1 + "): " + variantMask1.bitmask); - } - if (binaryMask2.length() - 4 != length2) { - throw new IllegalArgumentException("Bitmask does not match length (" + length2 + "): " + variantMask2.bitmask); - } - - String appendedString = binaryMask2.substring(0, binaryMask2.length() - 2) + - binaryMask1.substring(2); - BigInteger bitmask = new BigInteger(appendedString, 2); - return new VariantMaskBitmaskImpl(bitmask); - } - - private static VariantMask append(VariantMaskSparseImpl variantMask1, VariantMaskSparseImpl variantMask2, int length1, int length2) { - if (variantMask1.patientIndexes.size() + variantMask2.patientIndexes.size() > SPARSE_VARIANT_THRESHOLD) { - BigInteger mask = emptyBitmask(length1 + length2); - for (Integer patientId : variantMask1.patientIndexes) { - mask = mask.setBit(patientId + 2); - } - // We start writing mask 2 where mask 1 ends. So the 0th index of mask 2 is now following the last bit of mask 1 - for (Integer patientId : variantMask2.patientIndexes) { - mask = mask.setBit(patientId + length1 + 2); - } - return new VariantMaskBitmaskImpl(mask); - } - else { - Set patientIndexSet = new HashSet<>(); - patientIndexSet.addAll(variantMask1.patientIndexes); - // The indexes for mask 2 are shifted by the length of mask 1, corresponding to the corresponding patient id array - // for mask 2 being appended to those of mask 1 - patientIndexSet.addAll(variantMask2.patientIndexes.stream().map(i -> i + length1).collect(Collectors.toSet())); - return new VariantMaskSparseImpl(patientIndexSet); - } - } + private static Logger log = LoggerFactory.getLogger(VariableVariantMasks.class); + + private static final long serialVersionUID = 6225420483804601477L; + private static final String oneone = "11"; + private static final char one = '1'; + private static final char zero = '0'; + private static final String hetero = "0/1"; + private static final String heteroDel = "1/0"; + private static final String heteroPhased = "0|1"; + private static final String heteroPhased2 = "1|0"; + private static final String homo = "1/1"; + private static final String homoPhased = "1|1"; + private static final String homoNoCall = "./."; + private static final String heteroNoCall = "./1"; + + private static Map emptyBitmaskMap = new ConcurrentHashMap<>(); + + public static int SPARSE_VARIANT_THRESHOLD = 5; + + /* + * These indices result from the char values of the 3 characters in a VCF sample record summed as integers % 7 + * + * This allows us to not actually do string comparisons, instead we add 3 values, do one modulus operation, then use the result as the + * index into the result array. + */ + + // ./0 = (46 + 47 + 48) % 7 = 1 + // 0|. = (48 + 124 + 46) % 7 = 1 + // .|0 = (46 + 124 + 48) % 7 = 1 + public static final int HETERO_NOCALL_REF_CHAR_INDEX = 1; + + // ./1 = (46 + 47 + 49) % 7 = 2 + // 1|. = (49 + 124 + 46) % 7 = 2 + // .|1 = (46 + 124 + 49) % 7 = 2 + // ./1 = (46 + 47 + 49) % 7 = 2 + // 1|. = (49 + 124 + 46) % 7 = 2 + // .|1 = (46 + 124 + 49) % 7 = 2 + public static final int HETERO_NOCALL_VARIANT_CHAR_INDEX = 2; + + // 0/0 = (48 + 47 + 48) % 7 = 3 + // 0|0 = (48 + 124 + 48) % 7 = 3 + public static final int ZERO_ZERO_CHAR_INDEX = 3; + + // 0/1 = (48 + 47 + 49) % 7 = 4 + // 1|0 = (49 + 124 + 48) % 7 = 4 + // 0|1 = (48 + 124 + 49) % 7 = 4 + public static final int ZERO_ONE_CHAR_INDEX = 4; + + // 1/1 = (49 + 47 + 49) % 7 = 5 + // 1|1 = (49 + 124 + 49) % 7 = 5 + public static final int ONE_ONE_CHAR_INDEX = 5; + + // ./. = (46 + 47 + 46) % 7 = 6 + // .|. = (46 + 124 + 46) % 7 = 6 + public static final int HOMO_NOCALL_CHAR_INDEX = 6; + + public VariableVariantMasks(char[][] maskValues) { + String heteroMaskStringRaw = new String(maskValues[ZERO_ONE_CHAR_INDEX]); + String homoMaskStringRaw = new String(maskValues[ONE_ONE_CHAR_INDEX]); + String heteroNoCallMaskStringRaw = new String(maskValues[HETERO_NOCALL_VARIANT_CHAR_INDEX]); + String homoNoCallMaskStringRaw = new String(maskValues[HOMO_NOCALL_CHAR_INDEX]); + + heterozygousMask = variantMaskFromRawString(heteroMaskStringRaw); + homozygousMask = variantMaskFromRawString(homoMaskStringRaw); + heterozygousNoCallMask = variantMaskFromRawString(heteroNoCallMaskStringRaw); + homozygousNoCallMask = variantMaskFromRawString(homoNoCallMaskStringRaw); + } + + private VariantMask variantMaskFromRawString(String maskStringRaw) { + if (!maskStringRaw.contains("1")) { + return null; + } + + VariantMask variantMask; + BigInteger bitmask = new BigInteger(oneone + new StringBuilder(maskStringRaw).reverse() + oneone, 2); + if (bitmask.bitCount() - 4 > VariableVariantMasks.SPARSE_VARIANT_THRESHOLD) { + variantMask = new VariantMaskBitmaskImpl(bitmask); + } else { + Set patientIndexes = new HashSet<>(); + for (int i = 0; i < bitmask.bitLength() - 4; i++) { + // i + 2 because the mask is padded with 2 bits on each end + if (bitmask.testBit(i + 2)) { + patientIndexes.add(i); + } + } + variantMask = new VariantMaskSparseImpl(patientIndexes); + } + return variantMask; + } + + public VariableVariantMasks() {} + + @JsonProperty("ho") + @JsonInclude(JsonInclude.Include.NON_NULL) + public VariantMask homozygousMask; + + @JsonProperty("he") + @JsonInclude(JsonInclude.Include.NON_NULL) + public VariantMask heterozygousMask; + + @JsonProperty("hon") + @JsonInclude(JsonInclude.Include.NON_NULL) + public VariantMask homozygousNoCallMask; + + @JsonProperty("hen") + @JsonInclude(JsonInclude.Include.NON_NULL) + public VariantMask heterozygousNoCallMask; + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + VariableVariantMasks that = (VariableVariantMasks) o; + return Objects.equals(homozygousMask, that.homozygousMask) && Objects.equals(heterozygousMask, that.heterozygousMask) + && Objects.equals(homozygousNoCallMask, that.homozygousNoCallMask) + && Objects.equals(heterozygousNoCallMask, that.heterozygousNoCallMask); + } + + @Override + public int hashCode() { + return Objects.hash(homozygousMask, heterozygousMask, homozygousNoCallMask, heterozygousNoCallMask); + } + + public static BigInteger emptyBitmask(int length) { + BigInteger emptyBitmask = emptyBitmaskMap.get(length); + if (emptyBitmask == null) { + String emptyVariantMask = ""; + for (int i = 0; i < length; i++) { + emptyVariantMask = emptyVariantMask + "0"; + } + BigInteger newEmptyBitmask = new BigInteger("11" + emptyVariantMask + "11", 2); + emptyBitmaskMap.put(length, newEmptyBitmask); + return newEmptyBitmask; + } + return emptyBitmask; + } + + public static VariableVariantMasks append(VariableVariantMasks masks1, int length1, VariableVariantMasks masks2, int length2) { + VariableVariantMasks appendedMasks = new VariableVariantMasks(); + appendedMasks.homozygousMask = appendMask(masks1.homozygousMask, masks2.homozygousMask, length1, length2); + appendedMasks.heterozygousMask = appendMask(masks1.heterozygousMask, masks2.heterozygousMask, length1, length2); + appendedMasks.homozygousNoCallMask = appendMask(masks1.homozygousNoCallMask, masks2.homozygousNoCallMask, length1, length2); + appendedMasks.heterozygousNoCallMask = appendMask(masks1.heterozygousNoCallMask, masks2.heterozygousNoCallMask, length1, length2); + return appendedMasks; + } + + public static VariantMask appendMask(VariantMask variantMask1, VariantMask variantMask2, int length1, int length2) { + if (variantMask1 == null) { + variantMask1 = VariantMask.emptyInstance(); + } + if (variantMask2 == null) { + variantMask2 = VariantMask.emptyInstance(); + } + if (variantMask1.equals(VariantMask.emptyInstance()) && variantMask2.equals(VariantMask.emptyInstance())) { + return null; + } + + if (variantMask1 instanceof VariantMaskSparseImpl) { + if (variantMask2 instanceof VariantMaskSparseImpl) { + return append((VariantMaskSparseImpl) variantMask1, (VariantMaskSparseImpl) variantMask2, length1, length2); + } else if (variantMask2 instanceof VariantMaskBitmaskImpl) { + return append((VariantMaskSparseImpl) variantMask1, (VariantMaskBitmaskImpl) variantMask2, length1, length2); + } else { + throw new RuntimeException("Unknown VariantMask implementation"); + } + } else if (variantMask1 instanceof VariantMaskBitmaskImpl) { + if (variantMask2 instanceof VariantMaskSparseImpl) { + return append((VariantMaskBitmaskImpl) variantMask1, (VariantMaskSparseImpl) variantMask2, length1, length2); + } else if (variantMask2 instanceof VariantMaskBitmaskImpl) { + return append((VariantMaskBitmaskImpl) variantMask1, (VariantMaskBitmaskImpl) variantMask2, length1, length2); + } else { + throw new RuntimeException("Unknown VariantMask implementation"); + } + } else { + throw new RuntimeException("Unknown VariantMask implementation"); + } + } + + private static VariantMask append(VariantMaskSparseImpl variantMask1, VariantMaskBitmaskImpl variantMask2, int length1, int length2) { + BigInteger mask1 = emptyBitmask(length1); + for (Integer patientIndex : variantMask1.patientIndexes) { + mask1 = mask1.setBit(patientIndex + 2); + } + String binaryMask1 = mask1.toString(2); + + String binaryMask2 = variantMask2.bitmask.toString(2); + if (binaryMask2.length() - 4 != length2) { + throw new IllegalArgumentException("Bitmask does not match length (" + length2 + "): " + variantMask2.bitmask); + } + + String appendedString = binaryMask2.substring(0, binaryMask2.length() - 2) + binaryMask1.substring(2); + return new VariantMaskBitmaskImpl(new BigInteger(appendedString, 2)); + } + + private static VariantMask append(VariantMaskBitmaskImpl variantMask1, VariantMaskSparseImpl variantMask2, int length1, int length2) { + String binaryMask1 = variantMask1.bitmask.toString(2); + if (binaryMask1.length() - 4 != length1) { + throw new IllegalArgumentException("Bitmask does not match length (" + length1 + "): " + variantMask1.bitmask); + } + + BigInteger mask2 = emptyBitmask(length2); + for (Integer patientId : variantMask2.patientIndexes) { + mask2 = mask2.setBit(patientId + 2); + } + String binaryMask2 = mask2.toString(2); + + String appendedString = binaryMask2.substring(0, binaryMask2.length() - 2) + binaryMask1.substring(2); + return new VariantMaskBitmaskImpl(new BigInteger(appendedString, 2)); + } + + private static VariantMask append(VariantMaskBitmaskImpl variantMask1, VariantMaskBitmaskImpl variantMask2, int length1, int length2) { + String binaryMask1 = variantMask1.bitmask.toString(2); + String binaryMask2 = variantMask2.bitmask.toString(2); + + if (binaryMask1.length() - 4 != length1) { + throw new IllegalArgumentException("Bitmask does not match length (" + length1 + "): " + variantMask1.bitmask); + } + if (binaryMask2.length() - 4 != length2) { + throw new IllegalArgumentException("Bitmask does not match length (" + length2 + "): " + variantMask2.bitmask); + } + + String appendedString = binaryMask2.substring(0, binaryMask2.length() - 2) + binaryMask1.substring(2); + BigInteger bitmask = new BigInteger(appendedString, 2); + return new VariantMaskBitmaskImpl(bitmask); + } + + private static VariantMask append(VariantMaskSparseImpl variantMask1, VariantMaskSparseImpl variantMask2, int length1, int length2) { + if (variantMask1.patientIndexes.size() + variantMask2.patientIndexes.size() > SPARSE_VARIANT_THRESHOLD) { + BigInteger mask = emptyBitmask(length1 + length2); + for (Integer patientId : variantMask1.patientIndexes) { + mask = mask.setBit(patientId + 2); + } + // We start writing mask 2 where mask 1 ends. So the 0th index of mask 2 is now following the last bit of mask 1 + for (Integer patientId : variantMask2.patientIndexes) { + mask = mask.setBit(patientId + length1 + 2); + } + return new VariantMaskBitmaskImpl(mask); + } else { + Set patientIndexSet = new HashSet<>(); + patientIndexSet.addAll(variantMask1.patientIndexes); + // The indexes for mask 2 are shifted by the length of mask 1, corresponding to the corresponding patient id array + // for mask 2 being appended to those of mask 1 + patientIndexSet.addAll(variantMask2.patientIndexes.stream().map(i -> i + length1).collect(Collectors.toSet())); + return new VariantMaskSparseImpl(patientIndexSet); + } + } } diff --git a/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/VariantMask.java b/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/VariantMask.java index cca6ee847..376fcc432 100644 --- a/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/VariantMask.java +++ b/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/VariantMask.java @@ -8,16 +8,11 @@ import java.util.Set; import java.util.stream.Collectors; -@JsonTypeInfo( - use = JsonTypeInfo.Id.NAME, - include = JsonTypeInfo.As.PROPERTY, - property = "type", - defaultImpl = VariantMaskBitmaskImpl.class +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type", defaultImpl = VariantMaskBitmaskImpl.class) +@JsonSubTypes( + {@JsonSubTypes.Type(value = VariantMaskBitmaskImpl.class, name = "b"), + @JsonSubTypes.Type(value = VariantMaskSparseImpl.class, name = "s")} ) -@JsonSubTypes({ - @JsonSubTypes.Type(value = VariantMaskBitmaskImpl.class, name = "b"), - @JsonSubTypes.Type(value = VariantMaskSparseImpl.class, name = "s") -}) public interface VariantMask { VariantMask intersection(VariantMask variantMask); diff --git a/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/VariantMaskBitmaskImpl.java b/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/VariantMaskBitmaskImpl.java index fb6b81b86..8fa2922ce 100644 --- a/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/VariantMaskBitmaskImpl.java +++ b/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/VariantMaskBitmaskImpl.java @@ -36,7 +36,7 @@ public VariantMaskBitmaskImpl(BigInteger bitmask) { public VariantMask intersection(VariantMask variantMask) { if (variantMask instanceof VariantMaskBitmaskImpl) { return intersection((VariantMaskBitmaskImpl) variantMask); - } else if (variantMask instanceof VariantMaskSparseImpl) { + } else if (variantMask instanceof VariantMaskSparseImpl) { return variantMask.intersection(this); } else { throw new RuntimeException("Unknown VariantMask implementation"); @@ -47,7 +47,7 @@ public VariantMask intersection(VariantMask variantMask) { public VariantMask union(VariantMask variantMask) { if (variantMask instanceof VariantMaskBitmaskImpl) { return union((VariantMaskBitmaskImpl) variantMask); - } else if (variantMask instanceof VariantMaskSparseImpl) { + } else if (variantMask instanceof VariantMaskSparseImpl) { return variantMask.union(this); } else { throw new RuntimeException("Unknown VariantMask implementation"); @@ -67,8 +67,8 @@ public int bitCount() { @Override public Set patientMaskToPatientIdSet(List patientIds) { Set ids = new HashSet<>(); - for(int x = 0;x < bitmask.bitLength()-4;x++) { - if(testBit(x)) { + for (int x = 0; x < bitmask.bitLength() - 4; x++) { + if (testBit(x)) { String patientId = patientIds.get(x).trim(); ids.add(Integer.parseInt(patientId)); } @@ -86,6 +86,7 @@ public boolean isEmpty() { private VariantMask union(VariantMaskBitmaskImpl variantMaskBitmask) { return new VariantMaskBitmaskImpl(variantMaskBitmask.bitmask.or(this.bitmask)); } + private VariantMask intersection(VariantMaskBitmaskImpl variantMaskBitmask) { // we could consider using a sparse variant index here if we are ever going to be storing the // result of this anywhere @@ -107,8 +108,6 @@ public int hashCode() { @Override public String toString() { - return "VariantMaskBitmaskImpl{" + - "bitmask=" + bitmask.toString() + - '}'; + return "VariantMaskBitmaskImpl{" + "bitmask=" + bitmask.toString() + '}'; } } diff --git a/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/VariantMaskSparseImpl.java b/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/VariantMaskSparseImpl.java index bf020aeea..9b78c51da 100644 --- a/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/VariantMaskSparseImpl.java +++ b/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/VariantMaskSparseImpl.java @@ -25,16 +25,14 @@ public Set getPatientIndexes() { @Override public VariantMask intersection(VariantMask variantMask) { - return new VariantMaskSparseImpl(this.patientIndexes.stream() - .filter(variantMask::testBit) - .collect(Collectors.toSet())); + return new VariantMaskSparseImpl(this.patientIndexes.stream().filter(variantMask::testBit).collect(Collectors.toSet())); } @Override public VariantMask union(VariantMask variantMask) { if (variantMask instanceof VariantMaskBitmaskImpl) { return union((VariantMaskBitmaskImpl) variantMask); - } else if (variantMask instanceof VariantMaskSparseImpl) { + } else if (variantMask instanceof VariantMaskSparseImpl) { return union((VariantMaskSparseImpl) variantMask); } else { throw new RuntimeException("Unknown VariantMask implementation"); @@ -53,11 +51,7 @@ public int bitCount() { @Override public Set patientMaskToPatientIdSet(List patientIds) { - return patientIndexes.stream() - .map(patientIds::get) - .map(String::trim) - .map(Integer::parseInt) - .collect(Collectors.toSet()); + return patientIndexes.stream().map(patientIds::get).map(String::trim).map(Integer::parseInt).collect(Collectors.toSet()); } @Override diff --git a/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/VariantMasks.java b/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/VariantMasks.java index 5647b613c..a0d024984 100644 --- a/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/VariantMasks.java +++ b/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/VariantMasks.java @@ -11,238 +11,240 @@ import java.util.Objects; public class VariantMasks implements Serializable { - private static final long serialVersionUID = 6225420483804601477L; - private static final String oneone = "11"; - private static final char one = '1'; - private static final char zero = '0'; - private static final String hetero = "0/1"; - private static final String heteroDel = "1/0"; - private static final String heteroPhased = "0|1"; - private static final String heteroPhased2 = "1|0"; - private static final String homo = "1/1"; - private static final String homoPhased = "1|1"; - private static final String homoNoCall = "./."; - private static final String heteroNoCall = "./1"; - - /* - * These indices result from the char values of the 3 characters in a VCF sample - * record summed as integers % 7 - * - * This allows us to not actually do string comparisons, instead we add 3 values, - * do one modulus operation, then use the result as the index into the result array. - */ - - // ./0 = (46 + 47 + 48) % 7 = 1 - // 0|. = (48 + 124 + 46) % 7 = 1 - // .|0 = (46 + 124 + 48) % 7 = 1 - public static final int HETERO_NOCALL_REF_CHAR_INDEX = 1; - - // ./1 = (46 + 47 + 49) % 7 = 2 - // 1|. = (49 + 124 + 46) % 7 = 2 - // .|1 = (46 + 124 + 49) % 7 = 2 - // ./1 = (46 + 47 + 49) % 7 = 2 - // 1|. = (49 + 124 + 46) % 7 = 2 - // .|1 = (46 + 124 + 49) % 7 = 2 - public static final int HETERO_NOCALL_VARIANT_CHAR_INDEX = 2; - - // 0/0 = (48 + 47 + 48) % 7 = 3 - // 0|0 = (48 + 124 + 48) % 7 = 3 - public static final int ZERO_ZERO_CHAR_INDEX = 3; - - // 0/1 = (48 + 47 + 49) % 7 = 4 - // 1|0 = (49 + 124 + 48) % 7 = 4 - // 0|1 = (48 + 124 + 49) % 7 = 4 - public static final int ZERO_ONE_CHAR_INDEX = 4; - - // 1/1 = (49 + 47 + 49) % 7 = 5 - // 1|1 = (49 + 124 + 49) % 7 = 5 - public static final int ONE_ONE_CHAR_INDEX = 5; - - // ./. = (46 + 47 + 46) % 7 = 6 - // .|. = (46 + 124 + 46) % 7 = 6 - public static final int HOMO_NOCALL_CHAR_INDEX = 6; - - public VariantMasks(String[] values) { - char[] homozygousBits = new char[values.length]; - char[] heterozygousBits = new char[values.length]; - char[] homozygousNoCallBits = new char[values.length]; - char[] heterozygousNoCallBits = new char[values.length]; - boolean hasHetero = false; - boolean hasHomo = false; - boolean hasHeteroNoCall = false; - boolean hasHomoNoCall = false; - - for(int x = 0;x metadata, instead of the - * metadata -> variant-spec map that is used for searching and identifying patients. + * This class will create a set of FBBIS objects that allow lookups of variant-spec -> metadata, instead of the metadata -> variant-spec map + * that is used for searching and identifying patients. * - * The loading and reading of this class must take place separately; the flush() function will write out the in-memory contents to - * a fast, disk-based backing store. + * The loading and reading of this class must take place separately; the flush() function will write out the in-memory contents to a fast, + * disk-based backing store. */ public class VariantMetadataIndex implements Serializable { - public static final String VARIANT_METADATA_FILENAME = "VariantMetadata.javabin"; - public static String VARIANT_METADATA_BIN_FILE = "/opt/local/hpds/all/" + VARIANT_METADATA_FILENAME; - - private static final long serialVersionUID = 5917054606643971537L; - private static Logger log = LoggerFactory.getLogger(VariantMetadataIndex.class); - - // (String) contig --> (Integer) Bucket --> (String) variant spec --> INFO column data[]. - private final Map> > indexMap = new HashMap<>(); - - public static final String VARIANT_METADATA_STORAGE_FILE_PREFIX = "VariantMetadataStorage"; - private String fileStoragePrefix = "/opt/local/hpds/all/" + VARIANT_METADATA_STORAGE_FILE_PREFIX; - - /** - * This map allows us to load millions of variants without re-writing the fbbis each time (which would blow up the disk space). - * We need to remember to flush() between each contig this gets saved to the fbbis array. - */ - private transient Map> > loadingMap = new HashMap<>(); - - /** - * This constructor should only be used for testing; we expect the files to be in the default locations in production - * @param storageFile - * @throws IOException - */ - public VariantMetadataIndex(String storageFile) throws IOException { - fileStoragePrefix = storageFile; - } - - /** - * creates a default metadata index that maps variant spec -> metadata using an array of one file per contig. - * @throws IOException - */ - public VariantMetadataIndex() throws IOException { - } - - /** - * This will always return a value, including an empty array if there is no data or an error. - * @param variantSpec - * @return - */ - public Set findBySingleVariantSpec(String variantSpec, VariantBucketHolder bucketCache) { - try { - String[] segments = variantSpec.split(","); - if (segments.length < 2) { - log.error("Less than 2 segments found in this variant : " + variantSpec); - } - - int chrOffset = Integer.parseInt(segments[1]) / VariantStore.BUCKET_SIZE; - String contig = segments[0]; - - //see if we can reuse the cache - if (bucketCache.lastValue == null || !contig.contentEquals(bucketCache.lastContig) - || chrOffset != bucketCache.lastChunkOffset) { - FileBackedByteIndexedStorage> ContigFbbis = indexMap.get(contig); - if(ContigFbbis == null) { - return Set.of(); - } - bucketCache.lastValue = ContigFbbis.get(chrOffset); - bucketCache.lastContig = contig; - bucketCache.lastChunkOffset = chrOffset; - } - - if( bucketCache.lastValue != null) { - if(bucketCache.lastValue.get(variantSpec) == null) { - log.warn("No variant data found for spec " + variantSpec); - return Set.of(); - } - return Set.of(bucketCache.lastValue.get(variantSpec)); - } - log.warn("No bucket found for spec " + variantSpec + " in bucket " + chrOffset); - return Set.of(); - - } catch (UncheckedIOException e) { - log.warn("IOException caught looking up variantSpec : " + variantSpec, e); - return Set.of(); - } - } - - public Map> findByMultipleVariantSpec(Collection varientSpecList) { -// log.debug("SPEC list " + varientSpecList.size() + " :: " + Arrays.deepToString(varientSpecList.toArray())); - - VariantBucketHolder bucketCache = new VariantBucketHolder(); - return varientSpecList.stream().collect(Collectors.toMap( - variant->{return variant;}, - variant->{return findBySingleVariantSpec(variant, bucketCache);} - )); - } - - /** - * Get/put symmetry is broken here, since we want the ETL process to build the fbbis objects, so we only - * have to write them to disk once. The data will be written to disk only when the flush() method is called. - * - * @param variantSpec - * @throws IOException - */ - public void put(String variantSpec, String metaData ) throws IOException { - - String[] segments = variantSpec.split(","); - if (segments.length < 2) { - log.error("Less than 2 segments found in this variant : " + variantSpec); - } - - int chrOffset = Integer.parseInt(segments[1]) / VariantStore.BUCKET_SIZE; - String contig = segments[0]; - - ConcurrentHashMap> contigMap = loadingMap.get(contig); - if(contigMap == null) { - contigMap = new ConcurrentHashMap>(); - loadingMap.put(contig, contigMap); - } - - ConcurrentHashMap bucket = contigMap.get(chrOffset); - if(bucket == null) { - bucket = new ConcurrentHashMap(); - contigMap.put(chrOffset, bucket); - } - - List existingRecords = new ArrayList(); - if(bucket.get(variantSpec) != null) { - Collections.addAll(existingRecords, bucket.get(variantSpec)); - } - existingRecords.add(metaData); - bucket.put(variantSpec, existingRecords.toArray(new String[existingRecords.size()])); - } - - /** - * This will write out the current contents of the in-memory cache to file based storage. it should be called - * between processing each contig so that the memory usage doesn't overwhelm the system. - * - * This will overwrite any existing data, so it should only be called once per contig. - * @throws IOException - */ - public synchronized void flush() throws IOException { - - for(String contig : loadingMap.keySet()) { - log.debug("writing contig " + contig); - - FileBackedByteIndexedStorage> contigFbbis = indexMap.get(contig); - if(contigFbbis == null) { - log.info("creating new file for " + contig); - String filePath = fileStoragePrefix + "_" + contig + ".bin"; - contigFbbis = new FileBackedJavaIndexedStorage(Integer.class, ConcurrentHashMap.class, new File(filePath)); - indexMap.put(contig, contigFbbis); - } - - ConcurrentHashMap> contigMap = loadingMap.get(contig); - for(Integer bucketNumber : contigMap.keySet()) { - //make sure we don't lose any existing data - ConcurrentHashMap bucketStorage = contigFbbis.get(bucketNumber); - if(bucketStorage == null) { - bucketStorage = contigMap.get(bucketNumber); - } else { - bucketStorage.putAll(contigMap.get(bucketNumber)); - } - - contigFbbis.put(bucketNumber, bucketStorage); - } - - log.debug("Saved " + contig + " to FBBIS"); - } - //now reset the map - loadingMap = new HashMap> >(); - } - - public void complete() throws IOException { - flush(); - - for(String contig : indexMap.keySet()) { - FileBackedByteIndexedStorage> contigFbbis = indexMap.get(contig); - contigFbbis.complete(); - } - - } - - public static VariantMetadataIndex createInstance(String metadataIndexPath) { - try(ObjectInputStream in = new ObjectInputStream(new GZIPInputStream( - new FileInputStream(metadataIndexPath + VARIANT_METADATA_FILENAME)))){ - VariantMetadataIndex variantMetadataIndex = (VariantMetadataIndex) in.readObject(); - variantMetadataIndex.updateStorageDirectory(new File(metadataIndexPath)); - return variantMetadataIndex; - } catch(Exception e) { - // todo: handle exceptions better - log.info("No Metadata Index found at " + metadataIndexPath); - return null; - } - } - - public static void merge(VariantMetadataIndex variantMetadataIndex1, VariantMetadataIndex variantMetadataIndex2, String outputDirectory) throws IOException { - VariantMetadataIndex merged = new VariantMetadataIndex(outputDirectory + VARIANT_METADATA_STORAGE_FILE_PREFIX); - if (!variantMetadataIndex1.indexMap.keySet().equals(variantMetadataIndex2.indexMap.keySet())) { - log.warn("Merging incompatible variant indexes. Index1 keys: " + Joiner.on(",").join(variantMetadataIndex1.indexMap.keySet()) + ". Index 2 keys: " + Joiner.on(",").join(variantMetadataIndex2.indexMap.keySet())); - throw new IllegalStateException("Cannot merge variant metadata index with different contig keys"); - } - for (String contig : variantMetadataIndex1.indexMap.keySet()) { - String filePath = outputDirectory + VARIANT_METADATA_STORAGE_FILE_PREFIX + "_" + contig + ".bin"; - FileBackedByteIndexedStorage> mergedFbbis = new FileBackedJavaIndexedStorage(Integer.class, ConcurrentHashMap.class, new File(filePath)); - - // Store the merged result here because FileBackedByteIndexedStorage must be written all at once - Map> mergedStagedFbbis = new HashMap<>(); - - FileBackedByteIndexedStorage> fbbis1 = variantMetadataIndex1.indexMap.get(contig); - FileBackedByteIndexedStorage> fbbis2 = variantMetadataIndex2.indexMap.get(contig); - - fbbis1.keys().forEach(key -> { - mergedStagedFbbis.put(key, fbbis1.get(key)); - }); - fbbis2.keys().forEach(key -> { - ConcurrentHashMap metadataMap = mergedStagedFbbis.get(key); - if (metadataMap == null) { - mergedStagedFbbis.put(key, fbbis2.get(key)); - } else { - metadataMap.putAll(fbbis2.get(key)); - } - }); - - mergedStagedFbbis.forEach(mergedFbbis::put); - mergedFbbis.complete(); - merged.indexMap.put(contig, mergedFbbis); - } - - try(ObjectOutputStream out = new ObjectOutputStream(new GZIPOutputStream(Files.newOutputStream(new File(outputDirectory + VARIANT_METADATA_FILENAME).toPath())))){ - out.writeObject(merged); - out.flush(); - } - } - - public void updateStorageDirectory(File genomicDataDirectory) { - indexMap.values().forEach(value -> value.updateStorageDirectory(genomicDataDirectory)); - } -} \ No newline at end of file + public static final String VARIANT_METADATA_FILENAME = "VariantMetadata.javabin"; + public static String VARIANT_METADATA_BIN_FILE = "/opt/local/hpds/all/" + VARIANT_METADATA_FILENAME; + + private static final long serialVersionUID = 5917054606643971537L; + private static Logger log = LoggerFactory.getLogger(VariantMetadataIndex.class); + + // (String) contig --> (Integer) Bucket --> (String) variant spec --> INFO column data[]. + private final Map>> indexMap = new HashMap<>(); + + public static final String VARIANT_METADATA_STORAGE_FILE_PREFIX = "VariantMetadataStorage"; + private String fileStoragePrefix = "/opt/local/hpds/all/" + VARIANT_METADATA_STORAGE_FILE_PREFIX; + + /** + * This map allows us to load millions of variants without re-writing the fbbis each time (which would blow up the disk space). We need + * to remember to flush() between each contig this gets saved to the fbbis array. + */ + private transient Map>> loadingMap = new HashMap<>(); + + /** + * This constructor should only be used for testing; we expect the files to be in the default locations in production + * @param storageFile + * @throws IOException + */ + public VariantMetadataIndex(String storageFile) throws IOException { + fileStoragePrefix = storageFile; + } + + /** + * creates a default metadata index that maps variant spec -> metadata using an array of one file per contig. + * @throws IOException + */ + public VariantMetadataIndex() throws IOException {} + + /** + * This will always return a value, including an empty array if there is no data or an error. + * @param variantSpec + * @return + */ + public Set findBySingleVariantSpec(String variantSpec, VariantBucketHolder bucketCache) { + try { + String[] segments = variantSpec.split(","); + if (segments.length < 2) { + log.error("Less than 2 segments found in this variant : " + variantSpec); + } + + int chrOffset = Integer.parseInt(segments[1]) / VariantStore.BUCKET_SIZE; + String contig = segments[0]; + + // see if we can reuse the cache + if ( + bucketCache.lastValue == null || !contig.contentEquals(bucketCache.lastContig) || chrOffset != bucketCache.lastChunkOffset + ) { + FileBackedByteIndexedStorage> ContigFbbis = indexMap.get(contig); + if (ContigFbbis == null) { + return Set.of(); + } + bucketCache.lastValue = ContigFbbis.get(chrOffset); + bucketCache.lastContig = contig; + bucketCache.lastChunkOffset = chrOffset; + } + + if (bucketCache.lastValue != null) { + if (bucketCache.lastValue.get(variantSpec) == null) { + log.warn("No variant data found for spec " + variantSpec); + return Set.of(); + } + return Set.of(bucketCache.lastValue.get(variantSpec)); + } + log.warn("No bucket found for spec " + variantSpec + " in bucket " + chrOffset); + return Set.of(); + + } catch (UncheckedIOException e) { + log.warn("IOException caught looking up variantSpec : " + variantSpec, e); + return Set.of(); + } + } + + public Map> findByMultipleVariantSpec(Collection varientSpecList) { + // log.debug("SPEC list " + varientSpecList.size() + " :: " + Arrays.deepToString(varientSpecList.toArray())); + + VariantBucketHolder bucketCache = new VariantBucketHolder(); + return varientSpecList.stream().collect(Collectors.toMap(variant -> { + return variant; + }, variant -> { + return findBySingleVariantSpec(variant, bucketCache); + })); + } + + /** + * Get/put symmetry is broken here, since we want the ETL process to build the fbbis objects, so we only have to write them to disk + * once. The data will be written to disk only when the flush() method is called. + * + * @param variantSpec + * @throws IOException + */ + public void put(String variantSpec, String metaData) throws IOException { + + String[] segments = variantSpec.split(","); + if (segments.length < 2) { + log.error("Less than 2 segments found in this variant : " + variantSpec); + } + + int chrOffset = Integer.parseInt(segments[1]) / VariantStore.BUCKET_SIZE; + String contig = segments[0]; + + ConcurrentHashMap> contigMap = loadingMap.get(contig); + if (contigMap == null) { + contigMap = new ConcurrentHashMap>(); + loadingMap.put(contig, contigMap); + } + + ConcurrentHashMap bucket = contigMap.get(chrOffset); + if (bucket == null) { + bucket = new ConcurrentHashMap(); + contigMap.put(chrOffset, bucket); + } + + List existingRecords = new ArrayList(); + if (bucket.get(variantSpec) != null) { + Collections.addAll(existingRecords, bucket.get(variantSpec)); + } + existingRecords.add(metaData); + bucket.put(variantSpec, existingRecords.toArray(new String[existingRecords.size()])); + } + + /** + * This will write out the current contents of the in-memory cache to file based storage. it should be called between processing each + * contig so that the memory usage doesn't overwhelm the system. + * + * This will overwrite any existing data, so it should only be called once per contig. + * @throws IOException + */ + public synchronized void flush() throws IOException { + + for (String contig : loadingMap.keySet()) { + log.debug("writing contig " + contig); + + FileBackedByteIndexedStorage> contigFbbis = indexMap.get(contig); + if (contigFbbis == null) { + log.info("creating new file for " + contig); + String filePath = fileStoragePrefix + "_" + contig + ".bin"; + contigFbbis = new FileBackedJavaIndexedStorage(Integer.class, ConcurrentHashMap.class, new File(filePath)); + indexMap.put(contig, contigFbbis); + } + + ConcurrentHashMap> contigMap = loadingMap.get(contig); + for (Integer bucketNumber : contigMap.keySet()) { + // make sure we don't lose any existing data + ConcurrentHashMap bucketStorage = contigFbbis.get(bucketNumber); + if (bucketStorage == null) { + bucketStorage = contigMap.get(bucketNumber); + } else { + bucketStorage.putAll(contigMap.get(bucketNumber)); + } + + contigFbbis.put(bucketNumber, bucketStorage); + } + + log.debug("Saved " + contig + " to FBBIS"); + } + // now reset the map + loadingMap = new HashMap>>(); + } + + public void complete() throws IOException { + flush(); + + for (String contig : indexMap.keySet()) { + FileBackedByteIndexedStorage> contigFbbis = indexMap.get(contig); + contigFbbis.complete(); + } + + } + + public static VariantMetadataIndex createInstance(String metadataIndexPath) { + try ( + ObjectInputStream in = + new ObjectInputStream(new GZIPInputStream(new FileInputStream(metadataIndexPath + VARIANT_METADATA_FILENAME))) + ) { + VariantMetadataIndex variantMetadataIndex = (VariantMetadataIndex) in.readObject(); + variantMetadataIndex.updateStorageDirectory(new File(metadataIndexPath)); + return variantMetadataIndex; + } catch (Exception e) { + // todo: handle exceptions better + log.info("No Metadata Index found at " + metadataIndexPath); + return null; + } + } + + public static void merge(VariantMetadataIndex variantMetadataIndex1, VariantMetadataIndex variantMetadataIndex2, String outputDirectory) + throws IOException { + VariantMetadataIndex merged = new VariantMetadataIndex(outputDirectory + VARIANT_METADATA_STORAGE_FILE_PREFIX); + if (!variantMetadataIndex1.indexMap.keySet().equals(variantMetadataIndex2.indexMap.keySet())) { + log.warn( + "Merging incompatible variant indexes. Index1 keys: " + Joiner.on(",").join(variantMetadataIndex1.indexMap.keySet()) + + ". Index 2 keys: " + Joiner.on(",").join(variantMetadataIndex2.indexMap.keySet()) + ); + throw new IllegalStateException("Cannot merge variant metadata index with different contig keys"); + } + for (String contig : variantMetadataIndex1.indexMap.keySet()) { + String filePath = outputDirectory + VARIANT_METADATA_STORAGE_FILE_PREFIX + "_" + contig + ".bin"; + FileBackedByteIndexedStorage> mergedFbbis = + new FileBackedJavaIndexedStorage(Integer.class, ConcurrentHashMap.class, new File(filePath)); + + // Store the merged result here because FileBackedByteIndexedStorage must be written all at once + Map> mergedStagedFbbis = new HashMap<>(); + + FileBackedByteIndexedStorage> fbbis1 = variantMetadataIndex1.indexMap.get(contig); + FileBackedByteIndexedStorage> fbbis2 = variantMetadataIndex2.indexMap.get(contig); + + fbbis1.keys().forEach(key -> { + mergedStagedFbbis.put(key, fbbis1.get(key)); + }); + fbbis2.keys().forEach(key -> { + ConcurrentHashMap metadataMap = mergedStagedFbbis.get(key); + if (metadataMap == null) { + mergedStagedFbbis.put(key, fbbis2.get(key)); + } else { + metadataMap.putAll(fbbis2.get(key)); + } + }); + + mergedStagedFbbis.forEach(mergedFbbis::put); + mergedFbbis.complete(); + merged.indexMap.put(contig, mergedFbbis); + } + + try ( + ObjectOutputStream out = new ObjectOutputStream( + new GZIPOutputStream(Files.newOutputStream(new File(outputDirectory + VARIANT_METADATA_FILENAME).toPath())) + ) + ) { + out.writeObject(merged); + out.flush(); + } + } + + public void updateStorageDirectory(File genomicDataDirectory) { + indexMap.values().forEach(value -> value.updateStorageDirectory(genomicDataDirectory)); + } +} diff --git a/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/VariantSpec.java b/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/VariantSpec.java index 833597895..dcaedce25 100644 --- a/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/VariantSpec.java +++ b/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/VariantSpec.java @@ -7,114 +7,116 @@ public class VariantSpec implements Serializable, Comparable { - public class VariantCoords implements Serializable { - public String chromosome; - public Integer offset; - public String name; - public String ref; - public String alt; - public int qual; - public String format; - public String gene; - public String consequence; + public class VariantCoords implements Serializable { + public String chromosome; + public Integer offset; + public String name; + public String ref; + public String alt; + public int qual; + public String format; + public String gene; + public String consequence; - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - VariantCoords that = (VariantCoords) o; - return qual == that.qual && Objects.equals(chromosome, that.chromosome) && Objects.equals(offset, that.offset) && Objects.equals(name, that.name) && Objects.equals(ref, that.ref) && Objects.equals(alt, that.alt) && Objects.equals(format, that.format) && Objects.equals(gene, that.gene) && Objects.equals(consequence, that.consequence); - } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + VariantCoords that = (VariantCoords) o; + return qual == that.qual && Objects.equals(chromosome, that.chromosome) && Objects.equals(offset, that.offset) + && Objects.equals(name, that.name) && Objects.equals(ref, that.ref) && Objects.equals(alt, that.alt) + && Objects.equals(format, that.format) && Objects.equals(gene, that.gene) && Objects.equals(consequence, that.consequence); + } - @Override - public int hashCode() { - return Objects.hash(chromosome, offset, name, ref, alt, qual, format, gene, consequence); - } - } - public static int CHR = 0, OFF = 1, NAME = 2, REF = 3, ALT = 4, QUAL = 5, FILTER = 6, INFO = 7, FORMAT = 8, DATA = 9; - public long heteroOffset; - public long homoOffset; - public VariantCoords metadata; + @Override + public int hashCode() { + return Objects.hash(chromosome, offset, name, ref, alt, qual, format, gene, consequence); + } + } - public VariantSpec(CSVRecord r) { - this.metadata = new VariantCoords(); - this.metadata.chromosome = r.get(CHR); - this.metadata.offset = Integer.parseInt(r.get(OFF)); - this.metadata.name = r.get(NAME); - this.metadata.ref = r.get(REF); - this.metadata.alt = r.get(ALT); - try { - this.metadata.qual = Integer.parseInt(r.get(QUAL)); - }catch(NumberFormatException e) { - this.metadata.qual = -1; - } + public static int CHR = 0, OFF = 1, NAME = 2, REF = 3, ALT = 4, QUAL = 5, FILTER = 6, INFO = 7, FORMAT = 8, DATA = 9; + public long heteroOffset; + public long homoOffset; + public VariantCoords metadata; - String[] variantInfo = r.get(INFO).split("[=;]"); - String gene = "NULL"; - String consequence = "NULL"; - for (int i = 0; i < variantInfo.length; i = i + 2) { - if ("Gene_with_variant".equals(variantInfo[i])) { - gene = variantInfo[i + 1]; - } - if ("Variant_consequence_calculated".equals(variantInfo[i])) { - consequence = variantInfo[i + 1]; - } - } - this.metadata.gene = gene; - this.metadata.consequence = consequence; - } + public VariantSpec(CSVRecord r) { + this.metadata = new VariantCoords(); + this.metadata.chromosome = r.get(CHR); + this.metadata.offset = Integer.parseInt(r.get(OFF)); + this.metadata.name = r.get(NAME); + this.metadata.ref = r.get(REF); + this.metadata.alt = r.get(ALT); + try { + this.metadata.qual = Integer.parseInt(r.get(QUAL)); + } catch (NumberFormatException e) { + this.metadata.qual = -1; + } - public VariantSpec(String variant) { - this.metadata = new VariantCoords(); - String[] segments = variant.split(","); - this.metadata.chromosome = segments[0]; - this.metadata.offset = Integer.parseInt(segments[1]); - this.metadata.name = null; - this.metadata.ref = segments[2]; - this.metadata.alt = segments[3]; - this.metadata.qual = -1; - this.metadata.gene = segments[4]; - this.metadata.consequence = segments[5]; - } + String[] variantInfo = r.get(INFO).split("[=;]"); + String gene = "NULL"; + String consequence = "NULL"; + for (int i = 0; i < variantInfo.length; i = i + 2) { + if ("Gene_with_variant".equals(variantInfo[i])) { + gene = variantInfo[i + 1]; + } + if ("Variant_consequence_calculated".equals(variantInfo[i])) { + consequence = variantInfo[i + 1]; + } + } + this.metadata.gene = gene; + this.metadata.consequence = consequence; + } - public String specNotation() { - return this.metadata.chromosome + "," - + this.metadata.offset + "," + - this.metadata.ref + "," + this.metadata.alt + "," + this.metadata.gene + "," + this.metadata.consequence; - } + public VariantSpec(String variant) { + this.metadata = new VariantCoords(); + String[] segments = variant.split(","); + this.metadata.chromosome = segments[0]; + this.metadata.offset = Integer.parseInt(segments[1]); + this.metadata.name = null; + this.metadata.ref = segments[2]; + this.metadata.alt = segments[3]; + this.metadata.qual = -1; + this.metadata.gene = segments[4]; + this.metadata.consequence = segments[5]; + } - public int compareTo(VariantSpec o) { - int ret = 0; - ret = this.metadata.chromosome.compareTo(o.metadata.chromosome); - if (ret == 0) { - ret = this.metadata.offset.compareTo(o.metadata.offset); - } - if (ret == 0) { - ret = this.metadata.ref.compareTo(o.metadata.ref); - } - if (ret == 0) { - ret = this.metadata.alt.compareTo(o.metadata.alt); - } - if (ret == 0) { - ret = this.metadata.gene.compareTo(o.metadata.gene); - } - if (ret == 0) { - ret = this.metadata.consequence.compareTo(o.metadata.consequence); - } + public String specNotation() { + return this.metadata.chromosome + "," + this.metadata.offset + "," + this.metadata.ref + "," + this.metadata.alt + "," + + this.metadata.gene + "," + this.metadata.consequence; + } - return ret; - } + public int compareTo(VariantSpec o) { + int ret = 0; + ret = this.metadata.chromosome.compareTo(o.metadata.chromosome); + if (ret == 0) { + ret = this.metadata.offset.compareTo(o.metadata.offset); + } + if (ret == 0) { + ret = this.metadata.ref.compareTo(o.metadata.ref); + } + if (ret == 0) { + ret = this.metadata.alt.compareTo(o.metadata.alt); + } + if (ret == 0) { + ret = this.metadata.gene.compareTo(o.metadata.gene); + } + if (ret == 0) { + ret = this.metadata.consequence.compareTo(o.metadata.consequence); + } - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - VariantSpec that = (VariantSpec) o; - return heteroOffset == that.heteroOffset && homoOffset == that.homoOffset && Objects.equals(metadata, that.metadata); - } + return ret; + } - @Override - public int hashCode() { - return Objects.hash(heteroOffset, homoOffset, metadata); - } -} \ No newline at end of file + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + VariantSpec that = (VariantSpec) o; + return heteroOffset == that.heteroOffset && homoOffset == that.homoOffset && Objects.equals(metadata, that.metadata); + } + + @Override + public int hashCode() { + return Objects.hash(heteroOffset, homoOffset, metadata); + } +} diff --git a/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/VariantStore.java b/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/VariantStore.java index 312bede2b..32e24c5a9 100644 --- a/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/VariantStore.java +++ b/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/VariantStore.java @@ -15,152 +15,167 @@ import java.util.zip.GZIPOutputStream; public class VariantStore implements Serializable { - private static final long serialVersionUID = -6970128712587609414L; - public static final String VARIANT_STORE_JAVABIN_FILENAME = "variantStore.javabin"; - public static final String VARIANT_SPEC_INDEX_JAVABIN_FILENAME = "variantSpecIndex.javabin"; - private static Logger log = LoggerFactory.getLogger(VariantStore.class); - public static final int BUCKET_SIZE = 1000; - - private BigInteger emptyBitmask; - private String[] patientIds; - - private transient String[] variantSpecIndex; - - private Map>> variantMaskStorage = new TreeMap<>(); - - public Map>> getVariantMaskStorage() { - return variantMaskStorage; - } - - public void setVariantMaskStorage(Map>> variantMaskStorage) { - this.variantMaskStorage = variantMaskStorage; - } - - public String[] getVariantSpecIndex() { - return variantSpecIndex; - } - public void setVariantSpecIndex(String[] variantSpecIndex) { - this.variantSpecIndex = variantSpecIndex; - } - - public static VariantStore readInstance(String genomicDataDirectory) throws IOException, ClassNotFoundException { - try (GZIPInputStream gzipInputStream = new GZIPInputStream(new FileInputStream(genomicDataDirectory + VARIANT_STORE_JAVABIN_FILENAME)); - ObjectInputStream ois = new ObjectInputStream(gzipInputStream)) { - VariantStore variantStore = (VariantStore) ois.readObject(); - ois.close(); - variantStore.getVariantMaskStorage().values().forEach(store -> { - store.updateStorageDirectory(new File(genomicDataDirectory)); - }); - variantStore.open(); - variantStore.setVariantSpecIndex(loadVariantIndexFromFile(genomicDataDirectory)); - return variantStore; - } - } - - public void writeInstance(String genomicDirectory) { - try (FileOutputStream fos = new FileOutputStream(new File(genomicDirectory, VARIANT_STORE_JAVABIN_FILENAME)); - GZIPOutputStream gzos = new GZIPOutputStream(fos); - ObjectOutputStream oos = new ObjectOutputStream(gzos);) { - oos.writeObject(this); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - try (FileOutputStream fos = new FileOutputStream(new File(genomicDirectory, VARIANT_SPEC_INDEX_JAVABIN_FILENAME)); - GZIPOutputStream gzos = new GZIPOutputStream(fos); - ObjectOutputStream oos = new ObjectOutputStream(gzos);) { - oos.writeObject(Arrays.asList(variantSpecIndex)); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - public String[] getPatientIds() { - return patientIds; - } - - public Optional getMasks(String variant, VariantBucketHolder bucketCache) throws IOException { - String[] segments = variant.split(","); - if (segments.length < 2) { - log.error("Less than 2 segments found in this variant : " + variant); - } - - int chrOffset = Integer.parseInt(segments[1]) / BUCKET_SIZE; - String contig = segments[0]; - - if (bucketCache.lastValue != null && contig.contentEquals(bucketCache.lastContig) - && chrOffset == bucketCache.lastChunkOffset) { - // TODO : This is a temporary efficiency hack, NOT THREADSAFE!!! - } else { - // todo: don't bother doing a lookup if this node does not have the chromosome specified - FileBackedJsonIndexStorage> indexedStorage = variantMaskStorage.get(contig); - if (indexedStorage == null) { - return Optional.empty(); - } - bucketCache.lastValue = indexedStorage.get(chrOffset); - bucketCache.lastContig = contig; - bucketCache.lastChunkOffset = chrOffset; - } - return bucketCache.lastValue == null ? Optional.empty() : Optional.ofNullable(bucketCache.lastValue.get(variant)); - } - public List getMasksForDbSnpSpec(String variant) { - String[] segments = variant.split(","); - if (segments.length < 2) { - log.error("Less than 2 segments found in this variant : " + variant); - } - - int chrOffset = Integer.parseInt(segments[1]) / BUCKET_SIZE; - String contig = segments[0]; - - // todo: don't bother doing a lookup if this node does not have the chromosome specified - FileBackedJsonIndexStorage> indexedStorage = variantMaskStorage.get(contig); - if (indexedStorage == null) { - return List.of(); - } else { - ConcurrentHashMap specToMaskMap = indexedStorage.getOrELse(chrOffset, new ConcurrentHashMap<>()); - return specToMaskMap.entrySet().stream() - .filter(entry -> entry.getKey().startsWith(variant)) - .map(Map.Entry::getValue) - .collect(Collectors.toList()); - } - } - - public void open() { - variantMaskStorage.values().stream().forEach((fbbis -> { - if (fbbis != null) { - fbbis.open(); - } - })); - } - - public VariantStore() { - - } - - public void setPatientIds(String[] patientIds) { - this.patientIds = patientIds; - } - - public BigInteger emptyBitmask() { - if (emptyBitmask == null || emptyBitmask.testBit(emptyBitmask.bitLength() / 2)) { - String emptyVariantMask = ""; - for (String patientId : patientIds) { - emptyVariantMask = emptyVariantMask + "0"; - } - emptyBitmask = new BigInteger("11" + emptyVariantMask + "11", 2); - } - return emptyBitmask; - } - - @SuppressWarnings("unchecked") - public static String[] loadVariantIndexFromFile(String genomicDataDirectory) { - try (ObjectInputStream objectInputStream = new ObjectInputStream(new GZIPInputStream(new FileInputStream(genomicDataDirectory + "/" + VARIANT_SPEC_INDEX_JAVABIN_FILENAME)));){ - - List variants = (List) objectInputStream.readObject(); - return variants.toArray(new String[0]); - - } catch (IOException | ClassNotFoundException e) { - throw new RuntimeException(e); - } + private static final long serialVersionUID = -6970128712587609414L; + public static final String VARIANT_STORE_JAVABIN_FILENAME = "variantStore.javabin"; + public static final String VARIANT_SPEC_INDEX_JAVABIN_FILENAME = "variantSpecIndex.javabin"; + private static Logger log = LoggerFactory.getLogger(VariantStore.class); + public static final int BUCKET_SIZE = 1000; + + private BigInteger emptyBitmask; + private String[] patientIds; + + private transient String[] variantSpecIndex; + + private Map>> variantMaskStorage = + new TreeMap<>(); + + public Map>> getVariantMaskStorage() { + return variantMaskStorage; + } + + public void setVariantMaskStorage( + Map>> variantMaskStorage + ) { + this.variantMaskStorage = variantMaskStorage; + } + + public String[] getVariantSpecIndex() { + return variantSpecIndex; + } + + public void setVariantSpecIndex(String[] variantSpecIndex) { + this.variantSpecIndex = variantSpecIndex; + } + + public static VariantStore readInstance(String genomicDataDirectory) throws IOException, ClassNotFoundException { + try ( + GZIPInputStream gzipInputStream = + new GZIPInputStream(new FileInputStream(genomicDataDirectory + VARIANT_STORE_JAVABIN_FILENAME)); ObjectInputStream ois = + new ObjectInputStream(gzipInputStream) + ) { + VariantStore variantStore = (VariantStore) ois.readObject(); + ois.close(); + variantStore.getVariantMaskStorage().values().forEach(store -> { + store.updateStorageDirectory(new File(genomicDataDirectory)); + }); + variantStore.open(); + variantStore.setVariantSpecIndex(loadVariantIndexFromFile(genomicDataDirectory)); + return variantStore; + } + } + + public void writeInstance(String genomicDirectory) { + try ( + FileOutputStream fos = new FileOutputStream(new File(genomicDirectory, VARIANT_STORE_JAVABIN_FILENAME)); GZIPOutputStream gzos = + new GZIPOutputStream(fos); ObjectOutputStream oos = new ObjectOutputStream(gzos); + ) { + oos.writeObject(this); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + try ( + FileOutputStream fos = + new FileOutputStream(new File(genomicDirectory, VARIANT_SPEC_INDEX_JAVABIN_FILENAME)); GZIPOutputStream gzos = + new GZIPOutputStream(fos); ObjectOutputStream oos = new ObjectOutputStream(gzos); + ) { + oos.writeObject(Arrays.asList(variantSpecIndex)); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + public String[] getPatientIds() { + return patientIds; + } + + public Optional getMasks(String variant, VariantBucketHolder bucketCache) + throws IOException { + String[] segments = variant.split(","); + if (segments.length < 2) { + log.error("Less than 2 segments found in this variant : " + variant); + } + + int chrOffset = Integer.parseInt(segments[1]) / BUCKET_SIZE; + String contig = segments[0]; + + if (bucketCache.lastValue != null && contig.contentEquals(bucketCache.lastContig) && chrOffset == bucketCache.lastChunkOffset) { + // TODO : This is a temporary efficiency hack, NOT THREADSAFE!!! + } else { + // todo: don't bother doing a lookup if this node does not have the chromosome specified + FileBackedJsonIndexStorage> indexedStorage = + variantMaskStorage.get(contig); + if (indexedStorage == null) { + return Optional.empty(); + } + bucketCache.lastValue = indexedStorage.get(chrOffset); + bucketCache.lastContig = contig; + bucketCache.lastChunkOffset = chrOffset; + } + return bucketCache.lastValue == null ? Optional.empty() : Optional.ofNullable(bucketCache.lastValue.get(variant)); + } + + public List getMasksForDbSnpSpec(String variant) { + String[] segments = variant.split(","); + if (segments.length < 2) { + log.error("Less than 2 segments found in this variant : " + variant); + } + + int chrOffset = Integer.parseInt(segments[1]) / BUCKET_SIZE; + String contig = segments[0]; + + // todo: don't bother doing a lookup if this node does not have the chromosome specified + FileBackedJsonIndexStorage> indexedStorage = + variantMaskStorage.get(contig); + if (indexedStorage == null) { + return List.of(); + } else { + ConcurrentHashMap specToMaskMap = indexedStorage.getOrELse(chrOffset, new ConcurrentHashMap<>()); + return specToMaskMap.entrySet().stream().filter(entry -> entry.getKey().startsWith(variant)).map(Map.Entry::getValue) + .collect(Collectors.toList()); + } + } + + public void open() { + variantMaskStorage.values().stream().forEach((fbbis -> { + if (fbbis != null) { + fbbis.open(); + } + })); + } + + public VariantStore() { + + } + + public void setPatientIds(String[] patientIds) { + this.patientIds = patientIds; + } + + public BigInteger emptyBitmask() { + if (emptyBitmask == null || emptyBitmask.testBit(emptyBitmask.bitLength() / 2)) { + String emptyVariantMask = ""; + for (String patientId : patientIds) { + emptyVariantMask = emptyVariantMask + "0"; + } + emptyBitmask = new BigInteger("11" + emptyVariantMask + "11", 2); + } + return emptyBitmask; + } + + @SuppressWarnings("unchecked") + public static String[] loadVariantIndexFromFile(String genomicDataDirectory) { + try ( + ObjectInputStream objectInputStream = new ObjectInputStream( + new GZIPInputStream(new FileInputStream(genomicDataDirectory + "/" + VARIANT_SPEC_INDEX_JAVABIN_FILENAME)) + ); + ) { + + List variants = (List) objectInputStream.readObject(); + return variants.toArray(new String[0]); + + } catch (IOException | ClassNotFoundException e) { + throw new RuntimeException(e); + } } } diff --git a/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/caching/VariantBucketHolder.java b/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/caching/VariantBucketHolder.java index 83018c812..c23cd4331 100644 --- a/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/caching/VariantBucketHolder.java +++ b/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/caching/VariantBucketHolder.java @@ -3,7 +3,7 @@ import java.util.concurrent.ConcurrentHashMap; public class VariantBucketHolder { - public ConcurrentHashMap lastValue; - public String lastContig; - public int lastChunkOffset; + public ConcurrentHashMap lastValue; + public String lastContig; + public int lastChunkOffset; } diff --git a/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/phenotype/KeyAndValue.java b/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/phenotype/KeyAndValue.java index 9124b7890..2a11a0655 100644 --- a/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/phenotype/KeyAndValue.java +++ b/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/phenotype/KeyAndValue.java @@ -4,59 +4,59 @@ public class KeyAndValue> implements Serializable, Comparable> { - private static final long serialVersionUID = -3714097509955396941L; - - int key; - - V value; - - Long timestamp; - - public KeyAndValue() { - - } - - public KeyAndValue(int key, V value) { - this.key = key; - this.value = value; - this.setTimestamp(Long.MIN_VALUE); - } - - - public KeyAndValue(int key, V value, Long timestamp) { - this.key = key; - this.value = value; - this.setTimestamp(timestamp); - } - - public V getValue() { - return value; - } - - public KeyAndValue setValue(V value) { - this.value = value; - return this; - } - - public Integer getKey() { - return key; - } - - public KeyAndValue setKey(Integer key) { - this.key = key; - return this; - } - - @Override - public int compareTo(KeyAndValue o) { - return this.key - o.key; - } - - public Long getTimestamp() { - return timestamp; - } - - public void setTimestamp(Long timestamp) { - this.timestamp = timestamp; - } + private static final long serialVersionUID = -3714097509955396941L; + + int key; + + V value; + + Long timestamp; + + public KeyAndValue() { + + } + + public KeyAndValue(int key, V value) { + this.key = key; + this.value = value; + this.setTimestamp(Long.MIN_VALUE); + } + + + public KeyAndValue(int key, V value, Long timestamp) { + this.key = key; + this.value = value; + this.setTimestamp(timestamp); + } + + public V getValue() { + return value; + } + + public KeyAndValue setValue(V value) { + this.value = value; + return this; + } + + public Integer getKey() { + return key; + } + + public KeyAndValue setKey(Integer key) { + this.key = key; + return this; + } + + @Override + public int compareTo(KeyAndValue o) { + return this.key - o.key; + } + + public Long getTimestamp() { + return timestamp; + } + + public void setTimestamp(Long timestamp) { + this.timestamp = timestamp; + } } diff --git a/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/phenotype/PhenoCube.java b/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/phenotype/PhenoCube.java index dfdf408b7..810cf194a 100644 --- a/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/phenotype/PhenoCube.java +++ b/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/phenotype/PhenoCube.java @@ -7,231 +7,229 @@ public class PhenoCube> implements Serializable { - private static final long serialVersionUID = -2584728054717230390L; - public final String name; - public final Class vType; - - int columnWidth; - - V[] lowDimValues; - boolean isLowDim = false; - - private KeyAndValue[] sortedByKey; - - private TreeMap> categoryMap; - - private transient List> loadingMap = Collections.synchronizedList(new ArrayList<>()); - - public PhenoCube(String name, Class vType){ - this.vType = vType; - this.name = name; - } - - public void add(Integer key, V value, Date date) { - loadingMap.add(new KeyAndValue(key, value, date!=null?date.getTime():null)); - } - - public V getValueForKey(Integer key) { - int idx = Arrays.binarySearch(sortedByKey(), new KeyAndValue(key, null)); - if(idx < 0) return null; - return sortedByKey()[idx].value; - } - - public Set getKeysForValue(V value) { - - if(isStringType()) { - Set keys = categoryMap.get(value); - return keys == null ? new TreeSet() : keys; - } else { - int minIndex; - KeyAndValue keyAndValue = new KeyAndValue(1, value); - - if(value == null) { - return new TreeSet(); - } else { - KeyAndValue[] sortedByValue = sortedByValue(); - - int minSearchIndex = Arrays.binarySearch(sortedByValue, keyAndValue, (a,b)->{ - return a.value.compareTo(b.value); - }); - minIndex = seekForMinIndex(Math.abs(minSearchIndex), keyAndValue, sortedByValue); - - List keys = new ArrayList(); - for(int x = minIndex; x < sortedByValue.length && value.equals(sortedByValue[x].value); x++) { - keys.add(sortedByValue[x].key); - } - - return new TreeSet (keys); - } - } - - } - - public Set getKeysForRange(V min, V max) { - KeyAndValue[] entries = getEntriesForValueRange(min, max); - Set keys = new HashSet<>(); - for(KeyAndValue entry : entries) { - keys.add(entry.key); - } - return keys; - } - - public V[] getValuesForRange(V min, V max) { - KeyAndValue[] entries = getEntriesForValueRange(min, max); - @SuppressWarnings("unchecked") - V[] values = (V[]) Array.newInstance(vType, entries.length); - for(int x = 0;x[] getEntriesForValueRange(V min, V max) { - KeyAndValue minKeyAndValue = new KeyAndValue(1, min); - KeyAndValue maxKeyAndValue = new KeyAndValue(2, max); - - int minIndex; - int maxIndex; - - KeyAndValue[] sortedByValue = sortedByValue(); - if(min == null) { - minIndex = 0; - } else { - int minSearchIndex = Arrays.binarySearch(sortedByValue, minKeyAndValue, (a,b)->{ - return a.value.compareTo(b.value); - }); - minIndex = seekForMinIndex(Math.abs(minSearchIndex), minKeyAndValue, sortedByValue); - } - - if(max == null) { - maxIndex = sortedByValue.length; - } else { - int maxSearchIndex = Arrays.binarySearch(sortedByValue, maxKeyAndValue, (a,b)->{ - return a.value.compareTo(b.value); - }); - /* - * Arrays.binarySearch returns the insertion index of the new value if there is - * no exact match for the value. - * - * Sometimes this is an index that already has a value higher than our max, which - * would be shifted to the right on insertion. To still make use of the advantage - * of the binary search we invert this insertion index AND decrement it once. - * - * This prevents us from including one extra value in these cases. - */ - if(maxSearchIndex < 0) { - maxSearchIndex = (maxSearchIndex * -1)-1; - } - maxIndex = seekForMaxIndex(maxSearchIndex, maxKeyAndValue, sortedByValue); - } - - return Arrays.copyOfRange(sortedByValue, minIndex, maxIndex); - } - - private int seekForMinIndex(int minSearchIndex, KeyAndValue minEntry, KeyAndValue[] sortedByValue) { - - minSearchIndex--; - - Comparator> comparator = (a,b)->{ - return a.value.compareTo(b.value); - }; - while(minSearchIndex > -1 && comparator.compare(sortedByValue[minSearchIndex], minEntry)>=0) { - minSearchIndex--; - } - return Math.max(0, minSearchIndex+1); - } - - private int seekForMaxIndex(int maxSearchIndex, KeyAndValue maxEntry, KeyAndValue[] sortedByValue) { - Comparator> comparator = (a,b)->{ - return a.value.compareTo(b.value); - }; - while(maxSearchIndex < sortedByValue.length && comparator.compare(maxEntry, sortedByValue[maxSearchIndex])>=0) { - maxSearchIndex++; - } - return maxSearchIndex; - } - - public boolean isStringType() { - return vType.equals(String.class); - } - - public KeyAndValue[] sortedByValue() { - KeyAndValue[] sortedByValue = Arrays.copyOf(sortedByKey(), sortedByKey().length); - Arrays.sort(sortedByValue, (KeyAndValue o1, KeyAndValue o2) -> { - return o1.value.compareTo(o2.value); - }); - return sortedByValue; - } - - public KeyAndValue[] sortedByTimestamp() { - KeyAndValue[] sortedByTimestamp = Arrays.copyOf(sortedByKey(), sortedByKey().length); - Arrays.sort(sortedByTimestamp, (KeyAndValue o1, KeyAndValue o2) -> { - return o1.getTimestamp()==null? -1 : o1.getTimestamp().compareTo(o2.getTimestamp()); - }); - return sortedByTimestamp; - } - - public List keyBasedIndex() { - return Arrays.asList(sortedByKey()).stream().map((KeyAndValue kv)->{ - return kv.key; - }).collect(Collectors.toList()); - } - - public List> keyBasedArray() { - return Arrays.asList(sortedByKey()).stream().map((KeyAndValue kv)->{ - return kv.value; - }).collect(Collectors.toList()); - } - - public KeyAndValue[] sortedByKey() { - return sortedByKey; - } - - public PhenoCube setSortedByKey(KeyAndValue[] sortedByKey) { - this.sortedByKey = sortedByKey; - return this; - } - - public int getColumnWidth() { - return columnWidth; - } - - public PhenoCube setColumnWidth(int columnWidth) { - this.columnWidth = columnWidth; - return this; - } - - public TreeMap> getCategoryMap() { - return categoryMap; - } - - public void setCategoryMap(TreeMap> categorySetMap) { - this.categoryMap = categorySetMap; - } - - public List> getLoadingMap() { - return loadingMap; - } - - public void setLoadingMap(List> newMap) { - this.loadingMap= newMap ; - } - - public List> getValuesForKeys(Set patientIds) { - patientIds = new TreeSet<>(patientIds); - - List> values = new ArrayList<>(); - int x = 0; - for(Integer id : patientIds) { - while(x < sortedByKey.length && sortedByKey[x].key vType; + + int columnWidth; + + V[] lowDimValues; + boolean isLowDim = false; + + private KeyAndValue[] sortedByKey; + + private TreeMap> categoryMap; + + private transient List> loadingMap = Collections.synchronizedList(new ArrayList<>()); + + public PhenoCube(String name, Class vType) { + this.vType = vType; + this.name = name; + } + + public void add(Integer key, V value, Date date) { + loadingMap.add(new KeyAndValue(key, value, date != null ? date.getTime() : null)); + } + + public V getValueForKey(Integer key) { + int idx = Arrays.binarySearch(sortedByKey(), new KeyAndValue(key, null)); + if (idx < 0) return null; + return sortedByKey()[idx].value; + } + + public Set getKeysForValue(V value) { + + if (isStringType()) { + Set keys = categoryMap.get(value); + return keys == null ? new TreeSet() : keys; + } else { + int minIndex; + KeyAndValue keyAndValue = new KeyAndValue(1, value); + + if (value == null) { + return new TreeSet(); + } else { + KeyAndValue[] sortedByValue = sortedByValue(); + + int minSearchIndex = Arrays.binarySearch(sortedByValue, keyAndValue, (a, b) -> { + return a.value.compareTo(b.value); + }); + minIndex = seekForMinIndex(Math.abs(minSearchIndex), keyAndValue, sortedByValue); + + List keys = new ArrayList(); + for (int x = minIndex; x < sortedByValue.length && value.equals(sortedByValue[x].value); x++) { + keys.add(sortedByValue[x].key); + } + + return new TreeSet(keys); + } + } + + } + + public Set getKeysForRange(V min, V max) { + KeyAndValue[] entries = getEntriesForValueRange(min, max); + Set keys = new HashSet<>(); + for (KeyAndValue entry : entries) { + keys.add(entry.key); + } + return keys; + } + + public V[] getValuesForRange(V min, V max) { + KeyAndValue[] entries = getEntriesForValueRange(min, max); + @SuppressWarnings("unchecked") + V[] values = (V[]) Array.newInstance(vType, entries.length); + for (int x = 0; x < entries.length; x++) { + values[x] = entries[x].value; + } + return values; + } + + public KeyAndValue[] getEntriesForValueRange(V min, V max) { + KeyAndValue minKeyAndValue = new KeyAndValue(1, min); + KeyAndValue maxKeyAndValue = new KeyAndValue(2, max); + + int minIndex; + int maxIndex; + + KeyAndValue[] sortedByValue = sortedByValue(); + if (min == null) { + minIndex = 0; + } else { + int minSearchIndex = Arrays.binarySearch(sortedByValue, minKeyAndValue, (a, b) -> { + return a.value.compareTo(b.value); + }); + minIndex = seekForMinIndex(Math.abs(minSearchIndex), minKeyAndValue, sortedByValue); + } + + if (max == null) { + maxIndex = sortedByValue.length; + } else { + int maxSearchIndex = Arrays.binarySearch(sortedByValue, maxKeyAndValue, (a, b) -> { + return a.value.compareTo(b.value); + }); + /* + * Arrays.binarySearch returns the insertion index of the new value if there is no exact match for the value. + * + * Sometimes this is an index that already has a value higher than our max, which would be shifted to the right on insertion. To + * still make use of the advantage of the binary search we invert this insertion index AND decrement it once. + * + * This prevents us from including one extra value in these cases. + */ + if (maxSearchIndex < 0) { + maxSearchIndex = (maxSearchIndex * -1) - 1; + } + maxIndex = seekForMaxIndex(maxSearchIndex, maxKeyAndValue, sortedByValue); + } + + return Arrays.copyOfRange(sortedByValue, minIndex, maxIndex); + } + + private int seekForMinIndex(int minSearchIndex, KeyAndValue minEntry, KeyAndValue[] sortedByValue) { + + minSearchIndex--; + + Comparator> comparator = (a, b) -> { + return a.value.compareTo(b.value); + }; + while (minSearchIndex > -1 && comparator.compare(sortedByValue[minSearchIndex], minEntry) >= 0) { + minSearchIndex--; + } + return Math.max(0, minSearchIndex + 1); + } + + private int seekForMaxIndex(int maxSearchIndex, KeyAndValue maxEntry, KeyAndValue[] sortedByValue) { + Comparator> comparator = (a, b) -> { + return a.value.compareTo(b.value); + }; + while (maxSearchIndex < sortedByValue.length && comparator.compare(maxEntry, sortedByValue[maxSearchIndex]) >= 0) { + maxSearchIndex++; + } + return maxSearchIndex; + } + + public boolean isStringType() { + return vType.equals(String.class); + } + + public KeyAndValue[] sortedByValue() { + KeyAndValue[] sortedByValue = Arrays.copyOf(sortedByKey(), sortedByKey().length); + Arrays.sort(sortedByValue, (KeyAndValue o1, KeyAndValue o2) -> { + return o1.value.compareTo(o2.value); + }); + return sortedByValue; + } + + public KeyAndValue[] sortedByTimestamp() { + KeyAndValue[] sortedByTimestamp = Arrays.copyOf(sortedByKey(), sortedByKey().length); + Arrays.sort(sortedByTimestamp, (KeyAndValue o1, KeyAndValue o2) -> { + return o1.getTimestamp() == null ? -1 : o1.getTimestamp().compareTo(o2.getTimestamp()); + }); + return sortedByTimestamp; + } + + public List keyBasedIndex() { + return Arrays.asList(sortedByKey()).stream().map((KeyAndValue kv) -> { + return kv.key; + }).collect(Collectors.toList()); + } + + public List> keyBasedArray() { + return Arrays.asList(sortedByKey()).stream().map((KeyAndValue kv) -> { + return kv.value; + }).collect(Collectors.toList()); + } + + public KeyAndValue[] sortedByKey() { + return sortedByKey; + } + + public PhenoCube setSortedByKey(KeyAndValue[] sortedByKey) { + this.sortedByKey = sortedByKey; + return this; + } + + public int getColumnWidth() { + return columnWidth; + } + + public PhenoCube setColumnWidth(int columnWidth) { + this.columnWidth = columnWidth; + return this; + } + + public TreeMap> getCategoryMap() { + return categoryMap; + } + + public void setCategoryMap(TreeMap> categorySetMap) { + this.categoryMap = categorySetMap; + } + + public List> getLoadingMap() { + return loadingMap; + } + + public void setLoadingMap(List> newMap) { + this.loadingMap = newMap; + } + + public List> getValuesForKeys(Set patientIds) { + patientIds = new TreeSet<>(patientIds); + + List> values = new ArrayList<>(); + int x = 0; + for (Integer id : patientIds) { + while (x < sortedByKey.length && sortedByKey[x].key < id) { + x++; + } + while (x < sortedByKey.length && sortedByKey[x].key == id) { + values.add(sortedByKey[x]); + x++; + } + } + return values; + } } diff --git a/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/phenotype/TimelineEvent.java b/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/phenotype/TimelineEvent.java index d9766b2fc..a348cf11c 100644 --- a/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/phenotype/TimelineEvent.java +++ b/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/phenotype/TimelineEvent.java @@ -4,35 +4,39 @@ public class TimelineEvent { - public TimelineEvent(KeyAndValue value2, long startTime) { - this.timestamp = value2.timestamp-startTime; - this.patient_num = value2.key; - this.value = value2.value.toString(); - } - - long timestamp; - int patient_num; - String value; - - public long getTimestamp() { - return timestamp; - } - public void setTimestamp(long timestamp) { - this.timestamp = timestamp; - } - public int getPatient_num() { - return patient_num; - } - public void setPatient_num(int patient_num) { - this.patient_num = patient_num; - } - public String getValue() { - return value; - } - public void setValue(String value) { - this.value = value; - } - - public static Comparator timestampComparator = - Comparator.comparing(TimelineEvent::getTimestamp); + public TimelineEvent(KeyAndValue value2, long startTime) { + this.timestamp = value2.timestamp - startTime; + this.patient_num = value2.key; + this.value = value2.value.toString(); + } + + long timestamp; + int patient_num; + String value; + + public long getTimestamp() { + return timestamp; + } + + public void setTimestamp(long timestamp) { + this.timestamp = timestamp; + } + + public int getPatient_num() { + return patient_num; + } + + public void setPatient_num(int patient_num) { + this.patient_num = patient_num; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + + public static Comparator timestampComparator = Comparator.comparing(TimelineEvent::getTimestamp); } diff --git a/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/storage/FileBackedStorageVariantIndexImpl.java b/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/storage/FileBackedStorageVariantIndexImpl.java index c29755de9..d4b00542b 100644 --- a/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/storage/FileBackedStorageVariantIndexImpl.java +++ b/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/storage/FileBackedStorageVariantIndexImpl.java @@ -14,8 +14,7 @@ public FileBackedStorageVariantIndexImpl(File storageFile) throws FileNotFoundEx super(storageFile); } - private static final TypeReference typeRef - = new TypeReference() {}; + private static final TypeReference typeRef = new TypeReference() {}; @Override public TypeReference getTypeReference() { diff --git a/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/storage/FileBackedStorageVariantMasksImpl.java b/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/storage/FileBackedStorageVariantMasksImpl.java index bdd94a518..d667be057 100644 --- a/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/storage/FileBackedStorageVariantMasksImpl.java +++ b/services/pic-sure-hpds/data/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/storage/FileBackedStorageVariantMasksImpl.java @@ -10,14 +10,16 @@ import java.io.Serializable; import java.util.concurrent.ConcurrentHashMap; -public class FileBackedStorageVariantMasksImpl extends FileBackedJsonIndexStorage> implements Serializable { +public class FileBackedStorageVariantMasksImpl extends FileBackedJsonIndexStorage> + implements Serializable { private static final long serialVersionUID = -1086729119489479152L; public FileBackedStorageVariantMasksImpl(File storageFile) throws FileNotFoundException { super(storageFile); } - private static final TypeReference> typeRef - = new TypeReference>() {}; + + private static final TypeReference> typeRef = + new TypeReference>() {}; @Override public TypeReference> getTypeReference() { diff --git a/services/pic-sure-hpds/data/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/VariableVariantMasksTest.java b/services/pic-sure-hpds/data/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/VariableVariantMasksTest.java index 6d00f5388..31a2d4cca 100644 --- a/services/pic-sure-hpds/data/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/VariableVariantMasksTest.java +++ b/services/pic-sure-hpds/data/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/VariableVariantMasksTest.java @@ -11,34 +11,41 @@ class VariableVariantMasksTest { @Test void appendMask_bitmaskImpl() { - BigInteger bitmask1 = new BigInteger("141128266537977664750676276621047930360647583370951385898970922432171641921591683601425661017874251620531446414001258481539117879583763813143907296662202568713257464719895142475222938754588184582833582945368733841208297077207159012919514593576142503835920156756519924916168242089239884330492690734761017788145472426679068150988914755013607752960816306592042734726724967292489382922774340753414934719473579642086577795740766564256929075326123036793589965882761953873677505969126336030099232419417531157749845223962002657158936982730111297974575206081858525359109608528262092083167977823315667519770531233235502931655993522963196732342798609807510543882273123542784003837220002062547832711681922161218010724226003150184777238854967623299970205873235909960398972219305810619458430315886318451687126866584390511424196256376003247717440484524668930504599413091194844245946142655802347109092922312106655798606759916931458730259588797621736772069847398957694392864416289605234581063232645315000563794292589544477569850783743632035729362609737922579552116678396740867191143283379987783383546964540333832022555110531730782339840166704631278663393147496643156685747417468473443887224102514355150110201463928311558935840684668441340082266117306282514075566574248767903320202249460451453420775463375086967869642395460246215345716051454824012618762098616749770441010729935100690497726813080659572063353154599964806728241328892738034442472863741591198849392657539785115946383059858121434147389809099727185407701653069953116828987459279642700621389676861517273949473071786975738343769782464350475716221573322857987917669867491560145456280976454077958400126376423873081869441464609193216436634914151839730185428390533788824490041131307997850622739795832813220163889874229711053167483400593799762457890098932032534121358716659875839129363445294836150352418351095797845291006220927734160887236196809213101473795"); + BigInteger bitmask1 = new BigInteger( + "141128266537977664750676276621047930360647583370951385898970922432171641921591683601425661017874251620531446414001258481539117879583763813143907296662202568713257464719895142475222938754588184582833582945368733841208297077207159012919514593576142503835920156756519924916168242089239884330492690734761017788145472426679068150988914755013607752960816306592042734726724967292489382922774340753414934719473579642086577795740766564256929075326123036793589965882761953873677505969126336030099232419417531157749845223962002657158936982730111297974575206081858525359109608528262092083167977823315667519770531233235502931655993522963196732342798609807510543882273123542784003837220002062547832711681922161218010724226003150184777238854967623299970205873235909960398972219305810619458430315886318451687126866584390511424196256376003247717440484524668930504599413091194844245946142655802347109092922312106655798606759916931458730259588797621736772069847398957694392864416289605234581063232645315000563794292589544477569850783743632035729362609737922579552116678396740867191143283379987783383546964540333832022555110531730782339840166704631278663393147496643156685747417468473443887224102514355150110201463928311558935840684668441340082266117306282514075566574248767903320202249460451453420775463375086967869642395460246215345716051454824012618762098616749770441010729935100690497726813080659572063353154599964806728241328892738034442472863741591198849392657539785115946383059858121434147389809099727185407701653069953116828987459279642700621389676861517273949473071786975738343769782464350475716221573322857987917669867491560145456280976454077958400126376423873081869441464609193216436634914151839730185428390533788824490041131307997850622739795832813220163889874229711053167483400593799762457890098932032534121358716659875839129363445294836150352418351095797845291006220927734160887236196809213101473795" + ); VariantMask mask1 = new VariantMaskBitmaskImpl(bitmask1); - BigInteger bitmask2 = new BigInteger("30986998537042903080350519548408047200882037374471443267447678987916097706689940814664693588471382927433911322922595655683"); + BigInteger bitmask2 = new BigInteger( + "30986998537042903080350519548408047200882037374471443267447678987916097706689940814664693588471382927433911322922595655683" + ); VariantMask mask2 = new VariantMaskBitmaskImpl(bitmask2); - BigInteger expected = new BigInteger("364428449062309565827240887371711117987574656019257508470944944605332301145049360947562613021825062352285550919581217473685332023988542714833984833584283993391279871691386067405572061472644828810015780133411925154335939452908415258964249206427971383684062861395477403040282307619178797392994855011010859741065713642937722555386553472167057412349499978682577859360114784650082282703011601887907648540462571623667509259030122346885116531561265495702316349933315144109402391832141090339015610768872781857606433548246400953322494469603219947770538558126916689855755477351331412849701993441724311536424509298383050084506206211767917732037267909332303742403672014600658318302593423194411973298402659688783532220840192809967726287753517722498547930875524876017589557370530361981799944335286739781131150443454009586308444310074980028948209472950776329093810066130759876455221806685349198404676070598652587630369814761497643023695920944554335463560556407863991874650393356359742532476840044738380631950001000889884841365536456755318891308758448828432322546827241786950989285280451703830359048521243704376576346418597615317052525572950500220422542100551941365526524572447973251506942320379483402120949518538372004537133062107782710822000749441206410146350956094013529342546401955361956752745578930129151472060056501664176391306323733630924234020572479814310832686356211870629168524382442619274880885230138170761194502147197237097179031772802871639774718532255582374871483153552524216278948609829889298505566944209429410943844015411379772232511986448333683276938151761604643176820833900087800015896793760190738573447504231241253241247411492029282990960833205840126804844971397564707229362865231720213379844987578324282466174232636950775929742330163059024377711272017786328444659381735736216404131832568759328992585474109193095809959345744181054394252160618715444166335074662636279289524790651946664188322635565248044921668500523867959008028963420119948417112017996816651287475428019005246560397815063629879127215220996440067"); + BigInteger expected = new BigInteger( + "364428449062309565827240887371711117987574656019257508470944944605332301145049360947562613021825062352285550919581217473685332023988542714833984833584283993391279871691386067405572061472644828810015780133411925154335939452908415258964249206427971383684062861395477403040282307619178797392994855011010859741065713642937722555386553472167057412349499978682577859360114784650082282703011601887907648540462571623667509259030122346885116531561265495702316349933315144109402391832141090339015610768872781857606433548246400953322494469603219947770538558126916689855755477351331412849701993441724311536424509298383050084506206211767917732037267909332303742403672014600658318302593423194411973298402659688783532220840192809967726287753517722498547930875524876017589557370530361981799944335286739781131150443454009586308444310074980028948209472950776329093810066130759876455221806685349198404676070598652587630369814761497643023695920944554335463560556407863991874650393356359742532476840044738380631950001000889884841365536456755318891308758448828432322546827241786950989285280451703830359048521243704376576346418597615317052525572950500220422542100551941365526524572447973251506942320379483402120949518538372004537133062107782710822000749441206410146350956094013529342546401955361956752745578930129151472060056501664176391306323733630924234020572479814310832686356211870629168524382442619274880885230138170761194502147197237097179031772802871639774718532255582374871483153552524216278948609829889298505566944209429410943844015411379772232511986448333683276938151761604643176820833900087800015896793760190738573447504231241253241247411492029282990960833205840126804844971397564707229362865231720213379844987578324282466174232636950775929742330163059024377711272017786328444659381735736216404131832568759328992585474109193095809959345744181054394252160618715444166335074662636279289524790651946664188322635565248044921668500523867959008028963420119948417112017996816651287475428019005246560397815063629879127215220996440067" + ); // length should equal the length of the binary representation - 4. Bitmasks are padded with 11 on each end VariantMask merged = VariableVariantMasks.appendMask(mask1, mask2, 6282, 400); - assertEquals(expected, ((VariantMaskBitmaskImpl)merged).getBitmask()); + assertEquals(expected, ((VariantMaskBitmaskImpl) merged).getBitmask()); } + @Test void appendMask_bitmaskImpl_invalidLength() { - BigInteger bitmask1 = new BigInteger("141128266537977664750676276621047930360647583370951385898970922432171641921591683601425661017874251620531446414001258481539117879583763813143907296662202568713257464719895142475222938754588184582833582945368733841208297077207159012919514593576142503835920156756519924916168242089239884330492690734761017788145472426679068150988914755013607752960816306592042734726724967292489382922774340753414934719473579642086577795740766564256929075326123036793589965882761953873677505969126336030099232419417531157749845223962002657158936982730111297974575206081858525359109608528262092083167977823315667519770531233235502931655993522963196732342798609807510543882273123542784003837220002062547832711681922161218010724226003150184777238854967623299970205873235909960398972219305810619458430315886318451687126866584390511424196256376003247717440484524668930504599413091194844245946142655802347109092922312106655798606759916931458730259588797621736772069847398957694392864416289605234581063232645315000563794292589544477569850783743632035729362609737922579552116678396740867191143283379987783383546964540333832022555110531730782339840166704631278663393147496643156685747417468473443887224102514355150110201463928311558935840684668441340082266117306282514075566574248767903320202249460451453420775463375086967869642395460246215345716051454824012618762098616749770441010729935100690497726813080659572063353154599964806728241328892738034442472863741591198849392657539785115946383059858121434147389809099727185407701653069953116828987459279642700621389676861517273949473071786975738343769782464350475716221573322857987917669867491560145456280976454077958400126376423873081869441464609193216436634914151839730185428390533788824490041131307997850622739795832813220163889874229711053167483400593799762457890098932032534121358716659875839129363445294836150352418351095797845291006220927734160887236196809213101473795"); + BigInteger bitmask1 = new BigInteger( + "141128266537977664750676276621047930360647583370951385898970922432171641921591683601425661017874251620531446414001258481539117879583763813143907296662202568713257464719895142475222938754588184582833582945368733841208297077207159012919514593576142503835920156756519924916168242089239884330492690734761017788145472426679068150988914755013607752960816306592042734726724967292489382922774340753414934719473579642086577795740766564256929075326123036793589965882761953873677505969126336030099232419417531157749845223962002657158936982730111297974575206081858525359109608528262092083167977823315667519770531233235502931655993522963196732342798609807510543882273123542784003837220002062547832711681922161218010724226003150184777238854967623299970205873235909960398972219305810619458430315886318451687126866584390511424196256376003247717440484524668930504599413091194844245946142655802347109092922312106655798606759916931458730259588797621736772069847398957694392864416289605234581063232645315000563794292589544477569850783743632035729362609737922579552116678396740867191143283379987783383546964540333832022555110531730782339840166704631278663393147496643156685747417468473443887224102514355150110201463928311558935840684668441340082266117306282514075566574248767903320202249460451453420775463375086967869642395460246215345716051454824012618762098616749770441010729935100690497726813080659572063353154599964806728241328892738034442472863741591198849392657539785115946383059858121434147389809099727185407701653069953116828987459279642700621389676861517273949473071786975738343769782464350475716221573322857987917669867491560145456280976454077958400126376423873081869441464609193216436634914151839730185428390533788824490041131307997850622739795832813220163889874229711053167483400593799762457890098932032534121358716659875839129363445294836150352418351095797845291006220927734160887236196809213101473795" + ); VariantMask mask1 = new VariantMaskBitmaskImpl(bitmask1); - BigInteger bitmask2 = new BigInteger("30986998537042903080350519548408047200882037374471443267447678987916097706689940814664693588471382927433911322922595655683"); + BigInteger bitmask2 = new BigInteger( + "30986998537042903080350519548408047200882037374471443267447678987916097706689940814664693588471382927433911322922595655683" + ); VariantMask mask2 = new VariantMaskBitmaskImpl(bitmask2); - BigInteger expected = new BigInteger("364428449062309565827240887371711117987574656019257508470944944605332301145049360947562613021825062352285550919581217473685332023988542714833984833584283993391279871691386067405572061472644828810015780133411925154335939452908415258964249206427971383684062861395477403040282307619178797392994855011010859741065713642937722555386553472167057412349499978682577859360114784650082282703011601887907648540462571623667509259030122346885116531561265495702316349933315144109402391832141090339015610768872781857606433548246400953322494469603219947770538558126916689855755477351331412849701993441724311536424509298383050084506206211767917732037267909332303742403672014600658318302593423194411973298402659688783532220840192809967726287753517722498547930875524876017589557370530361981799944335286739781131150443454009586308444310074980028948209472950776329093810066130759876455221806685349198404676070598652587630369814761497643023695920944554335463560556407863991874650393356359742532476840044738380631950001000889884841365536456755318891308758448828432322546827241786950989285280451703830359048521243704376576346418597615317052525572950500220422542100551941365526524572447973251506942320379483402120949518538372004537133062107782710822000749441206410146350956094013529342546401955361956752745578930129151472060056501664176391306323733630924234020572479814310832686356211870629168524382442619274880885230138170761194502147197237097179031772802871639774718532255582374871483153552524216278948609829889298505566944209429410943844015411379772232511986448333683276938151761604643176820833900087800015896793760190738573447504231241253241247411492029282990960833205840126804844971397564707229362865231720213379844987578324282466174232636950775929742330163059024377711272017786328444659381735736216404131832568759328992585474109193095809959345744181054394252160618715444166335074662636279289524790651946664188322635565248044921668500523867959008028963420119948417112017996816651287475428019005246560397815063629879127215220996440067"); + BigInteger expected = new BigInteger( + "364428449062309565827240887371711117987574656019257508470944944605332301145049360947562613021825062352285550919581217473685332023988542714833984833584283993391279871691386067405572061472644828810015780133411925154335939452908415258964249206427971383684062861395477403040282307619178797392994855011010859741065713642937722555386553472167057412349499978682577859360114784650082282703011601887907648540462571623667509259030122346885116531561265495702316349933315144109402391832141090339015610768872781857606433548246400953322494469603219947770538558126916689855755477351331412849701993441724311536424509298383050084506206211767917732037267909332303742403672014600658318302593423194411973298402659688783532220840192809967726287753517722498547930875524876017589557370530361981799944335286739781131150443454009586308444310074980028948209472950776329093810066130759876455221806685349198404676070598652587630369814761497643023695920944554335463560556407863991874650393356359742532476840044738380631950001000889884841365536456755318891308758448828432322546827241786950989285280451703830359048521243704376576346418597615317052525572950500220422542100551941365526524572447973251506942320379483402120949518538372004537133062107782710822000749441206410146350956094013529342546401955361956752745578930129151472060056501664176391306323733630924234020572479814310832686356211870629168524382442619274880885230138170761194502147197237097179031772802871639774718532255582374871483153552524216278948609829889298505566944209429410943844015411379772232511986448333683276938151761604643176820833900087800015896793760190738573447504231241253241247411492029282990960833205840126804844971397564707229362865231720213379844987578324282466174232636950775929742330163059024377711272017786328444659381735736216404131832568759328992585474109193095809959345744181054394252160618715444166335074662636279289524790651946664188322635565248044921668500523867959008028963420119948417112017996816651287475428019005246560397815063629879127215220996440067" + ); // length should equal the length of the binary representation - 4. Bitmasks are padded with 11 on each end VariantMask merged = VariableVariantMasks.appendMask(mask1, mask2, 6282, 400); - assertThrows( - IllegalArgumentException.class, - () -> VariableVariantMasks.appendMask(mask1, mask2, 6283, 400) - ); - assertThrows( - IllegalArgumentException.class, - () -> VariableVariantMasks.appendMask(mask1, mask2, 6282, 399) - ); + assertThrows(IllegalArgumentException.class, () -> VariableVariantMasks.appendMask(mask1, mask2, 6283, 400)); + assertThrows(IllegalArgumentException.class, () -> VariableVariantMasks.appendMask(mask1, mask2, 6282, 399)); } @Test @@ -50,6 +57,7 @@ void appendMask_sparseImpl() { VariantMask expected = new VariantMaskSparseImpl(Set.of(1, 10, 99, 101, 199)); assertEquals(expected, merged); } + @Test void appendMask_sparseImplOneEmpty() { VariantMask mask1 = new VariantMaskSparseImpl(Set.of()); @@ -92,4 +100,4 @@ void appendMask_bitmaskSparse() { System.out.println(((VariantMaskBitmaskImpl) merged).getBitmask().toString(2)); assertEquals(expected, merged); } -} \ No newline at end of file +} diff --git a/services/pic-sure-hpds/data/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/VariantMaskPerformanceTest.java b/services/pic-sure-hpds/data/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/VariantMaskPerformanceTest.java index aa4702197..ddb7894f4 100644 --- a/services/pic-sure-hpds/data/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/VariantMaskPerformanceTest.java +++ b/services/pic-sure-hpds/data/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/VariantMaskPerformanceTest.java @@ -7,36 +7,45 @@ public class VariantMaskPerformanceTest { /** - * This test shows the ideal maximum size for a sparse variant is between 5-10 items. The slight decrease in performance - * is generally worth the drastic reduction in disks space + * This test shows the ideal maximum size for a sparse variant is between 5-10 items. The slight decrease in performance is generally + * worth the drastic reduction in disks space */ - //@Test + // @Test public void test() { VariantMaskBitmaskImpl mask100k = new VariantMaskBitmaskImpl(generateRandomBitmask(100_000)); VariantMaskBitmaskImpl mask100k2 = new VariantMaskBitmaskImpl(generateRandomBitmask(100_000)); VariantMaskBitmaskImpl mask1m = new VariantMaskBitmaskImpl(generateRandomBitmask(1_000_000)); VariantMaskBitmaskImpl mask1m2 = new VariantMaskBitmaskImpl(generateRandomBitmask(1_000_000)); VariantMaskSparseImpl sparseMask100k = new VariantMaskSparseImpl(Set.of(100, 200, 400, 50_000, 90_000)); - VariantMaskSparseImpl sparseMask100k2 = new VariantMaskSparseImpl(Set.of(100, 101, 200, 300, 400, 1000, 20_000, 30_000, 50_000, 90_000)); - VariantMaskSparseImpl sparseMask1m = new VariantMaskSparseImpl(Set.of(100, 200, 400, 50_000, 90_000, 300_000, 420_000, 555_555, 867_530, 999_999)); + VariantMaskSparseImpl sparseMask100k2 = + new VariantMaskSparseImpl(Set.of(100, 101, 200, 300, 400, 1000, 20_000, 30_000, 50_000, 90_000)); + VariantMaskSparseImpl sparseMask1m = + new VariantMaskSparseImpl(Set.of(100, 200, 400, 50_000, 90_000, 300_000, 420_000, 555_555, 867_530, 999_999)); long time = System.currentTimeMillis(); for (int k = 0; k < 1000; k++) { VariantMask and = mask100k.union(mask100k2); } - System.out.println(mask100k.getBitmask().bitLength() + " bitmask union completed in " + (System.currentTimeMillis() - time) + " ms"); + System.out + .println(mask100k.getBitmask().bitLength() + " bitmask union completed in " + (System.currentTimeMillis() - time) + " ms"); time = System.currentTimeMillis(); for (int k = 0; k < 1000; k++) { VariantMask and = sparseMask100k.union(mask100k); } - System.out.println(mask100k.getBitmask().bitLength() + " bitmask and " + sparseMask100k.patientIndexes.size() + " sparse union completed in " + (System.currentTimeMillis() - time) + " ms"); + System.out.println( + mask100k.getBitmask().bitLength() + " bitmask and " + sparseMask100k.patientIndexes.size() + " sparse union completed in " + + (System.currentTimeMillis() - time) + " ms" + ); time = System.currentTimeMillis(); for (int k = 0; k < 1000; k++) { VariantMask and = sparseMask100k2.union(mask100k); } - System.out.println(mask100k.getBitmask().bitLength() + " bitmask and " + sparseMask100k2.patientIndexes.size() + " sparse union completed in " + (System.currentTimeMillis() - time) + " ms"); + System.out.println( + mask100k.getBitmask().bitLength() + " bitmask and " + sparseMask100k2.patientIndexes.size() + " sparse union completed in " + + (System.currentTimeMillis() - time) + " ms" + ); time = System.currentTimeMillis(); for (int k = 0; k < 1000; k++) { @@ -48,13 +57,19 @@ public void test() { for (int k = 0; k < 1000; k++) { VariantMask and = sparseMask100k.union(mask1m); } - System.out.println(mask1m.getBitmask().bitLength() + " bitmask and " + sparseMask100k.patientIndexes.size() + " sparse union completed in " + (System.currentTimeMillis() - time) + " ms"); + System.out.println( + mask1m.getBitmask().bitLength() + " bitmask and " + sparseMask100k.patientIndexes.size() + " sparse union completed in " + + (System.currentTimeMillis() - time) + " ms" + ); time = System.currentTimeMillis(); for (int k = 0; k < 1000; k++) { VariantMask and = sparseMask1m.union(mask1m); } - System.out.println(mask1m.getBitmask().bitLength() + " bitmask and " + sparseMask1m.patientIndexes.size() + " sparse union completed in " + (System.currentTimeMillis() - time) + " ms"); + System.out.println( + mask1m.getBitmask().bitLength() + " bitmask and " + sparseMask1m.patientIndexes.size() + " sparse union completed in " + + (System.currentTimeMillis() - time) + " ms" + ); } diff --git a/services/pic-sure-hpds/data/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/VariantMaskTest.java b/services/pic-sure-hpds/data/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/VariantMaskTest.java index d6ad8b4a5..4a8896e0c 100644 --- a/services/pic-sure-hpds/data/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/VariantMaskTest.java +++ b/services/pic-sure-hpds/data/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/VariantMaskTest.java @@ -17,6 +17,7 @@ public void intersection_bitmaskVsBitmask() { assertEquals(expected, mask1.intersection(mask2)); } + @Test public void intersection_bitmaskVsSparse() { // this is essentially a mask for patients 0, 2, 3, 7 (there is 11 padding on both ends) @@ -26,6 +27,7 @@ public void intersection_bitmaskVsSparse() { assertEquals(expected, mask1.intersection(mask2)); } + @Test public void intersection_sparseVsBitmask() { VariantMask mask1 = new VariantMaskSparseImpl(Set.of(4, 7)); diff --git a/services/pic-sure-hpds/data/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/VariantMasksSerializationTest.java b/services/pic-sure-hpds/data/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/VariantMasksSerializationTest.java index 6bdbb3fc5..1f01d5bd3 100644 --- a/services/pic-sure-hpds/data/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/VariantMasksSerializationTest.java +++ b/services/pic-sure-hpds/data/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/VariantMasksSerializationTest.java @@ -17,25 +17,25 @@ public void testFieldMaxLength() throws JsonProcessingException { VariantMasks variantMasks = new VariantMasks(); StringBuilder homozygousStr = new StringBuilder(); - for (int i = 0 ; i < 50000; i++) { + for (int i = 0; i < 50000; i++) { homozygousStr.append(ThreadLocalRandom.current().nextInt(0, 10)); } variantMasks.homozygousMask = new BigInteger(homozygousStr.toString()); StringBuilder heterozygousStr = new StringBuilder(); - for (int i = 0 ; i < 50000; i++) { + for (int i = 0; i < 50000; i++) { heterozygousStr.append(ThreadLocalRandom.current().nextInt(0, 10)); } variantMasks.heterozygousMask = new BigInteger(heterozygousStr.toString()); StringBuilder homozygousNoCallStr = new StringBuilder(); - for (int i = 0 ; i < 50000; i++) { + for (int i = 0; i < 50000; i++) { homozygousNoCallStr.append(ThreadLocalRandom.current().nextInt(0, 10)); } variantMasks.homozygousNoCallMask = new BigInteger(homozygousNoCallStr.toString()); StringBuilder heterozygousNoCallStr = new StringBuilder(); - for (int i = 0 ; i < 50000; i++) { + for (int i = 0; i < 50000; i++) { heterozygousNoCallStr.append(ThreadLocalRandom.current().nextInt(0, 10)); } variantMasks.heterozygousNoCallMask = new BigInteger(heterozygousNoCallStr.toString()); diff --git a/services/pic-sure-hpds/data/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/data/phenotype/ColumnMetaTest.java b/services/pic-sure-hpds/data/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/data/phenotype/ColumnMetaTest.java index e4fb85de4..5e2cd30d2 100644 --- a/services/pic-sure-hpds/data/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/data/phenotype/ColumnMetaTest.java +++ b/services/pic-sure-hpds/data/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/data/phenotype/ColumnMetaTest.java @@ -9,33 +9,22 @@ /** * Unit tests for ColumnMeta timestamp field functionality. * - *

These tests validate the addition of timestamp tracking fields to ColumnMeta: - *

    - *
  • hasTimestamp: boolean flag indicating if temporal data is present
    • - *
  • timestampMin: earliest timestamp value (epoch milliseconds)
    • - *
  • timestampMax: latest timestamp value (epoch milliseconds)
    • - *
+ *

These tests validate the addition of timestamp tracking fields to ColumnMeta:

  • hasTimestamp: boolean flag indicating if + * temporal data is present
  • timestampMin: earliest timestamp value (epoch milliseconds)
  • timestampMax: latest + * timestamp value (epoch milliseconds)
* - *

The implementation maintains backward compatibility with existing serialized - * ColumnMeta objects that do not contain timestamp fields. New fields default - * to hasTimestamp=false and timestampMin/timestampMax=null when deserializing - * legacy data. + *

The implementation maintains backward compatibility with existing serialized ColumnMeta objects that do not contain timestamp fields. + * New fields default to hasTimestamp=false and timestampMin/timestampMax=null when deserializing legacy data. * * @see ColumnMeta */ class ColumnMetaTest { /** - * Verifies that newly instantiated ColumnMeta objects have correct default values - * for timestamp fields. + * Verifies that newly instantiated ColumnMeta objects have correct default values for timestamp fields. * - *

This ensures backward compatibility when working with datasets that do not - * contain temporal data. Default values should be: - *

    - *
  • hasTimestamp: false
    • - *
  • timestampMin: null
    • - *
  • timestampMax: null
    • - *
+ *

This ensures backward compatibility when working with datasets that do not contain temporal data. Default values should be:

    + *
  • hasTimestamp: false
  • timestampMin: null
  • timestampMax: null
*/ @Test void shouldCreateColumnMetaWithDefaultTimestampValues() { @@ -49,15 +38,11 @@ void shouldCreateColumnMetaWithDefaultTimestampValues() { /** * Validates that timestamp field setters and getters function correctly. * - *

Tests the basic contract of the timestamp accessor methods to ensure - * values can be set and retrieved as expected. + *

Tests the basic contract of the timestamp accessor methods to ensure values can be set and retrieved as expected. */ @Test void shouldSetAndGetTimestampFields() { - ColumnMeta columnMeta = new ColumnMeta() - .setHasTimestamp(true) - .setTimestampMin(1000000L) - .setTimestampMax(2000000L); + ColumnMeta columnMeta = new ColumnMeta().setHasTimestamp(true).setTimestampMin(1000000L).setTimestampMax(2000000L); assertTrue(columnMeta.hasTimestamp()); assertEquals(1000000L, columnMeta.getTimestampMin()); @@ -65,22 +50,15 @@ void shouldSetAndGetTimestampFields() { } /** - * Confirms that timestamp setters follow the builder pattern and can be chained - * with other ColumnMeta setters. + * Confirms that timestamp setters follow the builder pattern and can be chained with other ColumnMeta setters. * - *

The builder pattern allows for fluent configuration of ColumnMeta objects, - * improving code readability when initializing metadata with multiple properties. + *

The builder pattern allows for fluent configuration of ColumnMeta objects, improving code readability when initializing metadata + * with multiple properties. */ @Test void shouldSupportMethodChaining() { - ColumnMeta columnMeta = new ColumnMeta() - .setName("test_column") - .setHasTimestamp(true) - .setTimestampMin(1000000L) - .setTimestampMax(2000000L) - .setCategorical(false) - .setObservationCount(100) - .setPatientCount(50); + ColumnMeta columnMeta = new ColumnMeta().setName("test_column").setHasTimestamp(true).setTimestampMin(1000000L) + .setTimestampMax(2000000L).setCategorical(false).setObservationCount(100).setPatientCount(50); assertEquals("test_column", columnMeta.getName()); assertTrue(columnMeta.hasTimestamp()); @@ -92,29 +70,20 @@ void shouldSupportMethodChaining() { } /** - * Verifies that ColumnMeta objects with timestamp fields can be serialized and - * deserialized correctly. + * Verifies that ColumnMeta objects with timestamp fields can be serialized and deserialized correctly. * - *

This test ensures that the timestamp fields are properly included in the - * Java serialization process. When a ColumnMeta object containing timestamp data - * is serialized to disk (e.g., columnMeta.javabin) and later deserialized, all - * timestamp field values should be preserved. + *

This test ensures that the timestamp fields are properly included in the Java serialization process. When a ColumnMeta object + * containing timestamp data is serialized to disk (e.g., columnMeta.javabin) and later deserialized, all timestamp field values should + * be preserved. * - *

Uses realistic epoch millisecond values: - *

    - *
  • 1609459200000L = 2021-01-01 00:00:00 UTC
    • - *
  • 1640995200000L = 2022-01-01 00:00:00 UTC
    • - *
+ *

Uses realistic epoch millisecond values:

  • 1609459200000L = 2021-01-01 00:00:00 UTC
  • 1640995200000L = 2022-01-01 + * 00:00:00 UTC
*/ @Test void shouldSerializeWithTimestampFields() throws IOException, ClassNotFoundException { - ColumnMeta original = new ColumnMeta() - .setName("timestamped_column") - .setHasTimestamp(true) - .setTimestampMin(1609459200000L) // 2021-01-01 - .setTimestampMax(1640995200000L) // 2022-01-01 - .setObservationCount(500) - .setPatientCount(250); + ColumnMeta original = new ColumnMeta().setName("timestamped_column").setHasTimestamp(true).setTimestampMin(1609459200000L) // 2021-01-01 + .setTimestampMax(1640995200000L) // 2022-01-01 + .setObservationCount(500).setPatientCount(250); ByteArrayOutputStream baos = new ByteArrayOutputStream(); try (ObjectOutputStream oos = new ObjectOutputStream(baos)) { @@ -136,30 +105,19 @@ void shouldSerializeWithTimestampFields() throws IOException, ClassNotFoundExcep } /** - * Tests backward compatibility by deserializing ColumnMeta objects that were - * serialized before timestamp fields were added. + * Tests backward compatibility by deserializing ColumnMeta objects that were serialized before timestamp fields were added. * - *

This is a critical test for production deployment. When HPDS loads existing - * columnMeta.javabin files created by older versions of the software, it must - * handle the absence of timestamp fields gracefully. Java serialization supports - * this by initializing new fields with their default values: - *

    - *
  • hasTimestamp: false (primitive boolean default)
    • - *
  • timestampMin: null (object reference default)
    • - *
  • timestampMax: null (object reference default)
    • - *
+ *

This is a critical test for production deployment. When HPDS loads existing columnMeta.javabin files created by older versions of + * the software, it must handle the absence of timestamp fields gracefully. Java serialization supports this by initializing new fields + * with their default values:

  • hasTimestamp: false (primitive boolean default)
  • timestampMin: null (object reference + * default)
  • timestampMax: null (object reference default)
* - *

This test simulates loading legacy data by creating a ColumnMeta object, - * serializing it, and verifying that timestamp fields receive proper defaults - * upon deserialization. + *

This test simulates loading legacy data by creating a ColumnMeta object, serializing it, and verifying that timestamp fields + * receive proper defaults upon deserialization. */ @Test void shouldDeserializeLegacyColumnMetaWithoutTimestampFields() throws IOException, ClassNotFoundException { - ColumnMeta legacy = new ColumnMeta() - .setName("legacy_column") - .setCategorical(true) - .setObservationCount(1000) - .setPatientCount(500); + ColumnMeta legacy = new ColumnMeta().setName("legacy_column").setCategorical(true).setObservationCount(1000).setPatientCount(500); ByteArrayOutputStream baos = new ByteArrayOutputStream(); try (ObjectOutputStream oos = new ObjectOutputStream(baos)) { @@ -186,16 +144,12 @@ void shouldDeserializeLegacyColumnMetaWithoutTimestampFields() throws IOExceptio /** * Validates that timestamp min/max fields can be explicitly set to null. * - *

This may occur when a column has temporal observations but timestamp values - * are not yet calculated or are unavailable. The hasTimestamp flag can be true - * while min/max values remain null. + *

This may occur when a column has temporal observations but timestamp values are not yet calculated or are unavailable. The + * hasTimestamp flag can be true while min/max values remain null. */ @Test void shouldHandleNullTimestampMinMax() { - ColumnMeta columnMeta = new ColumnMeta() - .setHasTimestamp(true) - .setTimestampMin(null) - .setTimestampMax(null); + ColumnMeta columnMeta = new ColumnMeta().setHasTimestamp(true).setTimestampMin(null).setTimestampMax(null); assertTrue(columnMeta.hasTimestamp()); assertNull(columnMeta.getTimestampMin()); @@ -205,15 +159,12 @@ void shouldHandleNullTimestampMinMax() { /** * Tests that timestamp min/max values can be set independently of the hasTimestamp flag. * - *

While this may represent an inconsistent state in production usage, the fields - * are designed to be independently settable for flexibility. This test ensures the - * setters do not have hidden dependencies on each other. + *

While this may represent an inconsistent state in production usage, the fields are designed to be independently settable for + * flexibility. This test ensures the setters do not have hidden dependencies on each other. */ @Test void shouldAllowTimestampFieldsWithoutHasTimestampFlag() { - ColumnMeta columnMeta = new ColumnMeta() - .setTimestampMin(1000000L) - .setTimestampMax(2000000L); + ColumnMeta columnMeta = new ColumnMeta().setTimestampMin(1000000L).setTimestampMax(2000000L); assertFalse(columnMeta.hasTimestamp()); assertEquals(1000000L, columnMeta.getTimestampMin()); diff --git a/services/pic-sure-hpds/data/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/data/phenotype/PhenoCubeTest.java b/services/pic-sure-hpds/data/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/data/phenotype/PhenoCubeTest.java index ff20a474a..efa4faf4f 100644 --- a/services/pic-sure-hpds/data/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/data/phenotype/PhenoCubeTest.java +++ b/services/pic-sure-hpds/data/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/data/phenotype/PhenoCubeTest.java @@ -11,12 +11,8 @@ class PhenoCubeTest { @Test void shouldGetValuesForKeys() { - KeyAndValue[] sortedByKey = { - new KeyAndValue<>(1, "a"), - new KeyAndValue<>(1, "b"), - new KeyAndValue<>(2, "c"), - new KeyAndValue<>(3, "d"), - }; + KeyAndValue[] sortedByKey = + {new KeyAndValue<>(1, "a"), new KeyAndValue<>(1, "b"), new KeyAndValue<>(2, "c"), new KeyAndValue<>(3, "d"),}; PhenoCube subject = new PhenoCube<>("phill the phenocube", String.class); subject.setSortedByKey(sortedByKey); @@ -26,4 +22,4 @@ void shouldGetValuesForKeys() { Assertions.assertEquals(expected, actual); } -} \ No newline at end of file +} diff --git a/services/pic-sure-hpds/docker/pic-sure-hpds/Dockerfile b/services/pic-sure-hpds/docker/pic-sure-hpds/Dockerfile index 3f3b8035d..556a564ea 100644 --- a/services/pic-sure-hpds/docker/pic-sure-hpds/Dockerfile +++ b/services/pic-sure-hpds/docker/pic-sure-hpds/Dockerfile @@ -2,6 +2,6 @@ FROM amazoncorretto:25-alpine WORKDIR /app -COPY /service/target/service-*-SNAPSHOT.jar /service.jar +COPY /service/target/service-*.jar /service.jar ENTRYPOINT ["sh", "-c", "java ${JAVA_OPTS} -jar /service.jar"] \ No newline at end of file diff --git a/services/pic-sure-hpds/docker/pom.xml b/services/pic-sure-hpds/docker/pom.xml index 7f27d595c..036daa76c 100644 --- a/services/pic-sure-hpds/docker/pom.xml +++ b/services/pic-sure-hpds/docker/pom.xml @@ -5,7 +5,7 @@ pic-sure-hpds edu.harvard.hms.dbmi.avillach.hpds - 3.0.0-SNAPSHOT + ${revision} docker diff --git a/services/pic-sure-hpds/etl/pom.xml b/services/pic-sure-hpds/etl/pom.xml index 9a84f5ea0..0a80af06f 100644 --- a/services/pic-sure-hpds/etl/pom.xml +++ b/services/pic-sure-hpds/etl/pom.xml @@ -6,7 +6,7 @@ pic-sure-hpds edu.harvard.hms.dbmi.avillach.hpds - 3.0.0-SNAPSHOT + ${revision} etl diff --git a/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/util/ConvertMergedInfoStoresToFBBIS.java b/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/util/ConvertMergedInfoStoresToFBBIS.java index 6ef9e736c..a82b3176f 100644 --- a/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/util/ConvertMergedInfoStoresToFBBIS.java +++ b/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/util/ConvertMergedInfoStoresToFBBIS.java @@ -1,4 +1,5 @@ package edu.harvard.hms.dbmi.avillach.hpds.data.genotype.util; + import edu.harvard.hms.dbmi.avillach.hpds.etl.genotype.NewVCFLoader; import java.io.FileNotFoundException; @@ -7,10 +8,10 @@ public class ConvertMergedInfoStoresToFBBIS { - public static void main(String[] args) throws IOException { + public static void main(String[] args) throws IOException { - new NewVCFLoader().convertInfoStoresToByteIndexed(); + new NewVCFLoader().convertInfoStoresToByteIndexed(); - } + } } diff --git a/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/util/HideAnnotationCategoryValue.java b/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/util/HideAnnotationCategoryValue.java index 3a5074f05..cad9d2c1a 100644 --- a/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/util/HideAnnotationCategoryValue.java +++ b/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/util/HideAnnotationCategoryValue.java @@ -17,29 +17,30 @@ import edu.harvard.hms.dbmi.avillach.hpds.data.phenotype.PhenoCube; public class HideAnnotationCategoryValue { - protected static LoadingCache> store; - - protected static TreeSet allIds; - - public static void main(String[] args) throws ClassNotFoundException, IOException { - String infoStoreToModify = args[0]; - String valueToScrub = args[1]; - - String infoStoreFilename = "/opt/local/hpds/all/" + infoStoreToModify.trim(); - - FileInputStream fis = new FileInputStream(infoStoreFilename); - GZIPInputStream gis = new GZIPInputStream(fis); - ObjectInputStream ois = new ObjectInputStream(gis); - - FileBackedByteIndexedInfoStore infoStore = (FileBackedByteIndexedInfoStore) ois.readObject(); - infoStore.getAllValues().keys().remove(valueToScrub); - - FileOutputStream fos = new FileOutputStream(infoStoreFilename); - GZIPOutputStream gos = new GZIPOutputStream(fos); - ObjectOutputStream oos = new ObjectOutputStream(gos); - - oos.writeObject(infoStore); - oos.flush();oos.close(); - } - -} \ No newline at end of file + protected static LoadingCache> store; + + protected static TreeSet allIds; + + public static void main(String[] args) throws ClassNotFoundException, IOException { + String infoStoreToModify = args[0]; + String valueToScrub = args[1]; + + String infoStoreFilename = "/opt/local/hpds/all/" + infoStoreToModify.trim(); + + FileInputStream fis = new FileInputStream(infoStoreFilename); + GZIPInputStream gis = new GZIPInputStream(fis); + ObjectInputStream ois = new ObjectInputStream(gis); + + FileBackedByteIndexedInfoStore infoStore = (FileBackedByteIndexedInfoStore) ois.readObject(); + infoStore.getAllValues().keys().remove(valueToScrub); + + FileOutputStream fos = new FileOutputStream(infoStoreFilename); + GZIPOutputStream gos = new GZIPOutputStream(fos); + ObjectOutputStream oos = new ObjectOutputStream(gos); + + oos.writeObject(infoStore); + oos.flush(); + oos.close(); + } + +} diff --git a/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/util/ReSplitMergeInfoStores.java b/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/util/ReSplitMergeInfoStores.java index f49035aee..b773baffd 100644 --- a/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/util/ReSplitMergeInfoStores.java +++ b/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/util/ReSplitMergeInfoStores.java @@ -7,14 +7,14 @@ public class ReSplitMergeInfoStores { - public static void main(String[] args) throws IOException { + public static void main(String[] args) throws IOException { - NewVCFLoader newVCFLoader = new NewVCFLoader(); - - newVCFLoader.splitInfoStoresByColumn(); + NewVCFLoader newVCFLoader = new NewVCFLoader(); - newVCFLoader.convertInfoStoresToByteIndexed(); + newVCFLoader.splitInfoStoresByColumn(); - } + newVCFLoader.convertInfoStoresToByteIndexed(); + + } } diff --git a/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/util/RemapPatientIds.java b/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/util/RemapPatientIds.java index 22605f5d9..926e75de9 100644 --- a/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/util/RemapPatientIds.java +++ b/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/genotype/util/RemapPatientIds.java @@ -19,43 +19,45 @@ import edu.harvard.hms.dbmi.avillach.hpds.data.phenotype.PhenoCube; public class RemapPatientIds { - protected static LoadingCache> store; - - protected static TreeMap metaStoreSource; - - protected static TreeSet allIds; - - private static final int PATIENT_NUM = 0; - - private static final int CONCEPT_PATH = 1; - - private static final int NUMERIC_VALUE = 2; - - private static final int TEXT_VALUE = 3; - - public static void main(String[] args) throws ClassNotFoundException, FileNotFoundException, IOException { - VariantStore variantStore = VariantStore.readInstance("/opt/local/hpds/all/"); - - String[] oldPatientIds = variantStore.getPatientIds(); - String[] newPatientIds = new String[oldPatientIds.length]; - - CsvReader reader = new CsvReader(); - CsvContainer csv = reader.read(new FileReader("/opt/local/hpds/mappings.csv")); - for(CsvRow row : csv.getRows()) { - String oldId = row.getField(0); - String newId = row.getField(1); - for(int x = 0;x> store; + + protected static TreeMap metaStoreSource; + + protected static TreeSet allIds; + + private static final int PATIENT_NUM = 0; + + private static final int CONCEPT_PATH = 1; + + private static final int NUMERIC_VALUE = 2; + + private static final int TEXT_VALUE = 3; + + public static void main(String[] args) throws ClassNotFoundException, FileNotFoundException, IOException { + VariantStore variantStore = VariantStore.readInstance("/opt/local/hpds/all/"); + + String[] oldPatientIds = variantStore.getPatientIds(); + String[] newPatientIds = new String[oldPatientIds.length]; + + CsvReader reader = new CsvReader(); + CsvContainer csv = reader.read(new FileReader("/opt/local/hpds/mappings.csv")); + for (CsvRow row : csv.getRows()) { + String oldId = row.getField(0); + String newId = row.getField(1); + for (int x = 0; x < oldPatientIds.length; x++) { + if (oldPatientIds[x].contentEquals(oldId.trim())) { + newPatientIds[x] = newId; + System.out.println(oldId + ", " + newId); + } + } + } + variantStore.setPatientIds(newPatientIds); + + ObjectOutputStream out = + new ObjectOutputStream(new GZIPOutputStream(new FileOutputStream("/opt/local/hpds/all/variantStoreRemapped.javabin"))); + out.writeObject(variantStore); + out.flush(); + out.close(); + } + } diff --git a/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/phenotype/util/DumpSourceCSV.java b/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/phenotype/util/DumpSourceCSV.java index 03d892d2c..e2dd7df9d 100644 --- a/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/phenotype/util/DumpSourceCSV.java +++ b/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/phenotype/util/DumpSourceCSV.java @@ -24,94 +24,95 @@ import edu.harvard.hms.dbmi.avillach.hpds.etl.LoadingStore; public class DumpSourceCSV { - protected static LoadingCache> store; - - protected static TreeMap metaStoreSource; - - protected static TreeSet allIds; - - private static final int PATIENT_NUM = 0; - - private static final int CONCEPT_PATH = 1; - - private static final int NUMERIC_VALUE = 2; - - private static final int TEXT_VALUE = 3; - - private static final int TIMESTAMP = 4; - - public static void main(String[] args) throws ClassNotFoundException, FileNotFoundException, IOException { - Object[] metadata = loadMetadata(); - metaStoreSource = (TreeMap) metadata[0]; - allIds = (TreeSet) metadata[1]; - store = initializeCache(); - FileWriter fWriter = new FileWriter("/opt/local/hpds/allConcepts.csv"); - CSVPrinter writer = CSVFormat.DEFAULT.print(fWriter); - writer.printRecord(ImmutableList.of(new String[] {"PATIENT_NUM","CONCEPT_PATH","NUMERIC_VALUE","TEXT_VALUE","TIMESTAMP"})); - metaStoreSource.keySet().forEach((String key)->{ - try { - PhenoCube cube = store.get(key); - ArrayList cubeLines = new ArrayList<>(); - for(KeyAndValue kv : cube.sortedByKey()) { - String[] line = new String[5]; - line[PATIENT_NUM] = kv.getKey().toString(); - line[CONCEPT_PATH] = key; - line[NUMERIC_VALUE] = cube.isStringType() ? "" : kv.getValue().toString(); - line[TEXT_VALUE] = cube.isStringType() ? kv.getValue().toString() : ""; - line[TIMESTAMP] = kv.getTimestamp() == null ? null : kv.getTimestamp().toString(); - cubeLines.add(line); - } - writer.printRecords(cubeLines); - } catch(ExecutionException e) { - throw new RuntimeException(e); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - }); - writer.flush(); - writer.close(); - } - - static LoadingStore loadingStoreSource = new LoadingStore(); - - protected static Object[] loadMetadata() { - try (ObjectInputStream objectInputStream = new ObjectInputStream(new GZIPInputStream(new FileInputStream("/opt/local/hpds/columnMeta.javabin")));){ - TreeMap metastore = (TreeMap) objectInputStream.readObject(); - TreeMap metastoreScrubbed = new TreeMap(); - for(Entry entry : metastore.entrySet()) { - metastoreScrubbed.put(entry.getKey().replaceAll("\\ufffd",""), entry.getValue()); - } - Set allIds = (TreeSet) objectInputStream.readObject(); - return new Object[] {metastoreScrubbed, allIds}; - } catch (IOException | ClassNotFoundException e) { - throw new RuntimeException("Could not load metastore"); - } - } - - protected static LoadingCache> initializeCache() throws ClassNotFoundException, FileNotFoundException, IOException { - return CacheBuilder.newBuilder() - .maximumSize(100) - .build( - new CacheLoader>() { - public PhenoCube load(String key) throws Exception { - try(RandomAccessFile allObservationsStore = new RandomAccessFile("/opt/local/hpds/allObservationsStore.javabin", "r");){ - ColumnMeta columnMeta = metaStoreSource.get(key); - if(columnMeta != null) { - allObservationsStore.seek(columnMeta.getAllObservationsOffset()); - int length = (int) (columnMeta.getAllObservationsLength() - columnMeta.getAllObservationsOffset()); - byte[] buffer = new byte[length]; - allObservationsStore.read(buffer); - allObservationsStore.close(); - ObjectInputStream inStream = new ObjectInputStream(new ByteArrayInputStream(Crypto.decryptData(buffer))); - PhenoCube ret = (PhenoCube)inStream.readObject(); - inStream.close(); - return ret; - }else { - System.out.println("ColumnMeta not found for : [" + key + "]"); - return null; - } - } - } - }); - } + protected static LoadingCache> store; + + protected static TreeMap metaStoreSource; + + protected static TreeSet allIds; + + private static final int PATIENT_NUM = 0; + + private static final int CONCEPT_PATH = 1; + + private static final int NUMERIC_VALUE = 2; + + private static final int TEXT_VALUE = 3; + + private static final int TIMESTAMP = 4; + + public static void main(String[] args) throws ClassNotFoundException, FileNotFoundException, IOException { + Object[] metadata = loadMetadata(); + metaStoreSource = (TreeMap) metadata[0]; + allIds = (TreeSet) metadata[1]; + store = initializeCache(); + FileWriter fWriter = new FileWriter("/opt/local/hpds/allConcepts.csv"); + CSVPrinter writer = CSVFormat.DEFAULT.print(fWriter); + writer.printRecord(ImmutableList.of(new String[] {"PATIENT_NUM", "CONCEPT_PATH", "NUMERIC_VALUE", "TEXT_VALUE", "TIMESTAMP"})); + metaStoreSource.keySet().forEach((String key) -> { + try { + PhenoCube cube = store.get(key); + ArrayList cubeLines = new ArrayList<>(); + for (KeyAndValue kv : cube.sortedByKey()) { + String[] line = new String[5]; + line[PATIENT_NUM] = kv.getKey().toString(); + line[CONCEPT_PATH] = key; + line[NUMERIC_VALUE] = cube.isStringType() ? "" : kv.getValue().toString(); + line[TEXT_VALUE] = cube.isStringType() ? kv.getValue().toString() : ""; + line[TIMESTAMP] = kv.getTimestamp() == null ? null : kv.getTimestamp().toString(); + cubeLines.add(line); + } + writer.printRecords(cubeLines); + } catch (ExecutionException e) { + throw new RuntimeException(e); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }); + writer.flush(); + writer.close(); + } + + static LoadingStore loadingStoreSource = new LoadingStore(); + + protected static Object[] loadMetadata() { + try ( + ObjectInputStream objectInputStream = + new ObjectInputStream(new GZIPInputStream(new FileInputStream("/opt/local/hpds/columnMeta.javabin"))); + ) { + TreeMap metastore = (TreeMap) objectInputStream.readObject(); + TreeMap metastoreScrubbed = new TreeMap(); + for (Entry entry : metastore.entrySet()) { + metastoreScrubbed.put(entry.getKey().replaceAll("\\ufffd", ""), entry.getValue()); + } + Set allIds = (TreeSet) objectInputStream.readObject(); + return new Object[] {metastoreScrubbed, allIds}; + } catch (IOException | ClassNotFoundException e) { + throw new RuntimeException("Could not load metastore"); + } + } + + protected static LoadingCache> initializeCache() + throws ClassNotFoundException, FileNotFoundException, IOException { + return CacheBuilder.newBuilder().maximumSize(100).build(new CacheLoader>() { + public PhenoCube load(String key) throws Exception { + try (RandomAccessFile allObservationsStore = new RandomAccessFile("/opt/local/hpds/allObservationsStore.javabin", "r");) { + ColumnMeta columnMeta = metaStoreSource.get(key); + if (columnMeta != null) { + allObservationsStore.seek(columnMeta.getAllObservationsOffset()); + int length = (int) (columnMeta.getAllObservationsLength() - columnMeta.getAllObservationsOffset()); + byte[] buffer = new byte[length]; + allObservationsStore.read(buffer); + allObservationsStore.close(); + ObjectInputStream inStream = new ObjectInputStream(new ByteArrayInputStream(Crypto.decryptData(buffer))); + PhenoCube ret = (PhenoCube) inStream.readObject(); + inStream.close(); + return ret; + } else { + System.out.println("ColumnMeta not found for : [" + key + "]"); + return null; + } + } + } + }); + } } diff --git a/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/phenotype/util/FixCategoricalConcepts.java b/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/phenotype/util/FixCategoricalConcepts.java index a651f3977..c6e51ad95 100644 --- a/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/phenotype/util/FixCategoricalConcepts.java +++ b/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/phenotype/util/FixCategoricalConcepts.java @@ -22,166 +22,179 @@ @SuppressWarnings({"unchecked", "rawtypes"}) public class FixCategoricalConcepts { - - private static LoadingStore store = new LoadingStore(); - - private static Logger log = LoggerFactory.getLogger(FixCategoricalConcepts.class); - - protected static LoadingCache> sourceStore; - - protected static TreeMap sourceMetaStore; - - private static final int PATIENT_NUM = 0; - - private static final int CONCEPT_PATH = 1; - - private static final int NUMERIC_VALUE = 2; - - private static final int TEXT_VALUE = 3; - - private static final int TIMESTAMP = 4; - - private static String HPDS_DIRECTORY = "/opt/local/hpds/"; - - public static void main(String[] args) throws IOException, ClassNotFoundException, ExecutionException { - sourceStore = initializeCache(); - Object[] metadata = loadMetadata(); - sourceMetaStore = (TreeMap) metadata[0]; - store.allObservationsStore = new RandomAccessFile(HPDS_DIRECTORY + "allObservationsStore.javabin", "rw"); - initialLoad(); - store.saveStore(HPDS_DIRECTORY); - } - - private static void initialLoad() throws IOException, ExecutionException { - final PhenoCube[] currentConcept = new PhenoCube[1]; - Map> conceptsToMerge = new TreeMap>(); - for(String conceptPath : sourceMetaStore.keySet()) { - if(conceptPath.endsWith("/")) { - //this is a potentially bugged categorical concept - String correctConcept = conceptPath.substring(0, conceptPath.lastIndexOf("\\")); - List sisterConcepts = conceptsToMerge.get(correctConcept); - if(sisterConcepts == null) { - sisterConcepts = new ArrayList(); - conceptsToMerge.put(correctConcept, sisterConcepts); - } - sisterConcepts.add(conceptPath); - }else { - conceptsToMerge.put(conceptPath, List.of(conceptPath)); - } - } - for(String conceptPath : conceptsToMerge.keySet()) { - for(String concept : conceptsToMerge.get(conceptPath)) { - PhenoCube cubeForPath = sourceStore.get(concept); - for(KeyAndValue entry : cubeForPath.sortedByKey()) { - processRecord(currentConcept, List.of( - entry.getKey().toString(), - conceptPath, - cubeForPath.isStringType()?"":entry.getValue().toString(), - cubeForPath.isStringType()?entry.getValue().toString():"")); - } - } - } - } - - private static void processRecord(final PhenoCube[] currentConcept, List record) { - try { - String conceptPathFromRow = record.get(CONCEPT_PATH); - String[] segments = conceptPathFromRow.split("\\\\"); - for(int x = 0;x> initializeCache() throws ClassNotFoundException, FileNotFoundException, IOException { - return CacheBuilder.newBuilder() - .maximumSize(10) - .build( - new CacheLoader>() { - public PhenoCube load(String key) throws Exception { - try(RandomAccessFile allObservationsStore = new RandomAccessFile("/opt/local/source/allObservationsStore.javabin", "r");){ - ColumnMeta columnMeta = sourceMetaStore.get(key); - if(columnMeta != null) { - allObservationsStore.seek(columnMeta.getAllObservationsOffset()); - int length = (int) (columnMeta.getAllObservationsLength() - columnMeta.getAllObservationsOffset()); - byte[] buffer = new byte[length]; - allObservationsStore.read(buffer); - allObservationsStore.close(); - ObjectInputStream inStream = new ObjectInputStream(new ByteArrayInputStream(Crypto.decryptData(buffer))); - PhenoCube ret = (PhenoCube)inStream.readObject(); - inStream.close(); - return ret; - }else { - System.out.println("ColumnMeta not found for : [" + key + "]"); - return null; - } - } - } - }); - } - - protected static Object[] loadMetadata() { - try (ObjectInputStream objectInputStream = new ObjectInputStream(new GZIPInputStream(new FileInputStream("/opt/local/source/columnMeta.javabin")));){ - TreeMap metastore = (TreeMap) objectInputStream.readObject(); - TreeMap metastoreScrubbed = new TreeMap(); - for(Entry entry : metastore.entrySet()) { - metastoreScrubbed.put(entry.getKey().replaceAll("\\ufffd",""), entry.getValue()); - } - Set allIds = (TreeSet) objectInputStream.readObject(); - return new Object[] {metastoreScrubbed, allIds}; - } catch (IOException | ClassNotFoundException e) { - log.warn("************************************************"); - log.warn("************************************************"); - log.warn("Could not load metastore", e); - log.warn("If you meant to include phenotype data of any kind, please check that the file /opt/local/source/columnMeta.javabin exists and is readable by the service."); - log.warn("************************************************"); - log.warn("************************************************"); - return new Object[] {new TreeMap(), new TreeSet()}; - } - } + + private static LoadingStore store = new LoadingStore(); + + private static Logger log = LoggerFactory.getLogger(FixCategoricalConcepts.class); + + protected static LoadingCache> sourceStore; + + protected static TreeMap sourceMetaStore; + + private static final int PATIENT_NUM = 0; + + private static final int CONCEPT_PATH = 1; + + private static final int NUMERIC_VALUE = 2; + + private static final int TEXT_VALUE = 3; + + private static final int TIMESTAMP = 4; + + private static String HPDS_DIRECTORY = "/opt/local/hpds/"; + + public static void main(String[] args) throws IOException, ClassNotFoundException, ExecutionException { + sourceStore = initializeCache(); + Object[] metadata = loadMetadata(); + sourceMetaStore = (TreeMap) metadata[0]; + store.allObservationsStore = new RandomAccessFile(HPDS_DIRECTORY + "allObservationsStore.javabin", "rw"); + initialLoad(); + store.saveStore(HPDS_DIRECTORY); + } + + private static void initialLoad() throws IOException, ExecutionException { + final PhenoCube[] currentConcept = new PhenoCube[1]; + Map> conceptsToMerge = new TreeMap>(); + for (String conceptPath : sourceMetaStore.keySet()) { + if (conceptPath.endsWith("/")) { + // this is a potentially bugged categorical concept + String correctConcept = conceptPath.substring(0, conceptPath.lastIndexOf("\\")); + List sisterConcepts = conceptsToMerge.get(correctConcept); + if (sisterConcepts == null) { + sisterConcepts = new ArrayList(); + conceptsToMerge.put(correctConcept, sisterConcepts); + } + sisterConcepts.add(conceptPath); + } else { + conceptsToMerge.put(conceptPath, List.of(conceptPath)); + } + } + for (String conceptPath : conceptsToMerge.keySet()) { + for (String concept : conceptsToMerge.get(conceptPath)) { + PhenoCube cubeForPath = sourceStore.get(concept); + for (KeyAndValue entry : cubeForPath.sortedByKey()) { + processRecord( + currentConcept, + List.of( + entry.getKey().toString(), conceptPath, cubeForPath.isStringType() ? "" : entry.getValue().toString(), + cubeForPath.isStringType() ? entry.getValue().toString() : "" + ) + ); + } + } + } + } + + private static void processRecord(final PhenoCube[] currentConcept, List record) { + try { + String conceptPathFromRow = record.get(CONCEPT_PATH); + String[] segments = conceptPathFromRow.split("\\\\"); + for (int x = 0; x < segments.length; x++) { + segments[x] = segments[x].trim(); + } + conceptPathFromRow = String.join("\\", segments) + "\\"; + conceptPathFromRow = conceptPathFromRow.replaceAll("\\ufffd", ""); + String textValueFromRow = record.get(TEXT_VALUE) == null ? null : record.get(TEXT_VALUE).trim(); + if (textValueFromRow != null) { + textValueFromRow = textValueFromRow.replaceAll("\\ufffd", ""); + } + String conceptPath = + conceptPathFromRow.endsWith("\\" + textValueFromRow + "\\") ? conceptPathFromRow.replaceAll("\\\\[^\\\\]*\\\\$", "\\\\") + : conceptPathFromRow; + // This is not getDouble because we need to handle null values, not coerce them into 0s + String numericValue = record.get(NUMERIC_VALUE); + if ((numericValue == null || numericValue.isEmpty()) && textValueFromRow != null) { + try { + numericValue = Double.parseDouble(textValueFromRow) + ""; + } catch (NumberFormatException e) { + + } + } + boolean isAlpha = (numericValue == null || numericValue.isEmpty()); + if (currentConcept[0] == null || !currentConcept[0].name.equals(conceptPath)) { + System.out.println(conceptPath); + try { + currentConcept[0] = store.store.get(conceptPath); + } catch (InvalidCacheLoadException e) { + currentConcept[0] = new PhenoCube(conceptPath, isAlpha ? String.class : Double.class); + store.store.put(conceptPath, currentConcept[0]); + } + } + String value = isAlpha ? record.get(TEXT_VALUE) : numericValue; + + if ( + value != null && !value.trim().isEmpty() + && ((isAlpha && currentConcept[0].vType == String.class) || (!isAlpha && currentConcept[0].vType == Double.class)) + ) { + value = value.trim(); + currentConcept[0] + .setColumnWidth(isAlpha ? Math.max(currentConcept[0].getColumnWidth(), value.getBytes().length) : Double.BYTES); + currentConcept[0].add( + Integer.parseInt(record.get(PATIENT_NUM)), isAlpha ? value : Double.parseDouble(value), new Date(record.get(TIMESTAMP)) + ); + store.allIds.add(Integer.parseInt(record.get(PATIENT_NUM))); + } + } catch (ExecutionException e) { + log.error("Error processing record", e); + } + } + + /** + * Load the variantStore object from disk and build the PhenoCube cache. + * + * @return + * @throws ClassNotFoundException + * @throws FileNotFoundException + * @throws IOException + */ + protected static LoadingCache> initializeCache() + throws ClassNotFoundException, FileNotFoundException, IOException { + return CacheBuilder.newBuilder().maximumSize(10).build(new CacheLoader>() { + public PhenoCube load(String key) throws Exception { + try (RandomAccessFile allObservationsStore = new RandomAccessFile("/opt/local/source/allObservationsStore.javabin", "r");) { + ColumnMeta columnMeta = sourceMetaStore.get(key); + if (columnMeta != null) { + allObservationsStore.seek(columnMeta.getAllObservationsOffset()); + int length = (int) (columnMeta.getAllObservationsLength() - columnMeta.getAllObservationsOffset()); + byte[] buffer = new byte[length]; + allObservationsStore.read(buffer); + allObservationsStore.close(); + ObjectInputStream inStream = new ObjectInputStream(new ByteArrayInputStream(Crypto.decryptData(buffer))); + PhenoCube ret = (PhenoCube) inStream.readObject(); + inStream.close(); + return ret; + } else { + System.out.println("ColumnMeta not found for : [" + key + "]"); + return null; + } + } + } + }); + } + + protected static Object[] loadMetadata() { + try ( + ObjectInputStream objectInputStream = + new ObjectInputStream(new GZIPInputStream(new FileInputStream("/opt/local/source/columnMeta.javabin"))); + ) { + TreeMap metastore = (TreeMap) objectInputStream.readObject(); + TreeMap metastoreScrubbed = new TreeMap(); + for (Entry entry : metastore.entrySet()) { + metastoreScrubbed.put(entry.getKey().replaceAll("\\ufffd", ""), entry.getValue()); + } + Set allIds = (TreeSet) objectInputStream.readObject(); + return new Object[] {metastoreScrubbed, allIds}; + } catch (IOException | ClassNotFoundException e) { + log.warn("************************************************"); + log.warn("************************************************"); + log.warn("Could not load metastore", e); + log.warn( + "If you meant to include phenotype data of any kind, please check that the file /opt/local/source/columnMeta.javabin exists and is readable by the service." + ); + log.warn("************************************************"); + log.warn("************************************************"); + return new Object[] {new TreeMap(), new TreeSet()}; + } + } } diff --git a/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/phenotype/util/RekeyDataset.java b/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/phenotype/util/RekeyDataset.java index 75b35ce32..de230b1dc 100644 --- a/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/phenotype/util/RekeyDataset.java +++ b/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/phenotype/util/RekeyDataset.java @@ -1,4 +1,4 @@ - package edu.harvard.hms.dbmi.avillach.hpds.data.phenotype.util; +package edu.harvard.hms.dbmi.avillach.hpds.data.phenotype.util; import java.io.*; import java.util.Map.Entry; @@ -23,91 +23,94 @@ @SuppressWarnings({"unchecked"}) public class RekeyDataset { - private static final String SOURCE = "SOURCE"; - - private static LoadingStore store = new LoadingStore(); - - private static Logger log = LoggerFactory.getLogger(RekeyDataset.class); - - protected static LoadingCache> sourceStore; - - protected static TreeMap sourceMetaStore; - - private static String HPDS_DIRECTORY = "/opt/local/hpds/"; - - public static void main(String[] args) throws IOException, ClassNotFoundException, ExecutionException { - Crypto.loadDefaultKey(); - Crypto.loadKey(SOURCE, "/opt/local/source/encryption_key"); - sourceStore = initializeCache(); - Object[] metadata = loadMetadata(); - sourceMetaStore = (TreeMap) metadata[0]; - store.allObservationsStore = new RandomAccessFile(HPDS_DIRECTORY + "allObservationsStore.javabin", "rw"); - store.allIds = (TreeSet) metadata[1]; - initialLoad(); - - store.saveStore(HPDS_DIRECTORY); - } - - private static void initialLoad() throws IOException, ExecutionException { - for(String conceptPath : sourceMetaStore.keySet()) { - PhenoCube cube = sourceStore.get(conceptPath); - store.store.put(conceptPath, cube); - } - } - - /** - * Build the PhenoCube cache. - * - * @return - * @throws ClassNotFoundException - * @throws FileNotFoundException - * @throws IOException - */ - protected static LoadingCache> initializeCache() throws ClassNotFoundException, FileNotFoundException, IOException { - return CacheBuilder.newBuilder() - .maximumSize(10) - .build( - new CacheLoader>() { - public PhenoCube load(String key) throws Exception { - try(RandomAccessFile allObservationsStore = new RandomAccessFile("/opt/local/source/allObservationsStore.javabin", "r");){ - ColumnMeta columnMeta = sourceMetaStore.get(key); - if(columnMeta != null) { - allObservationsStore.seek(columnMeta.getAllObservationsOffset()); - int length = (int) (columnMeta.getAllObservationsLength() - columnMeta.getAllObservationsOffset()); - byte[] buffer = new byte[length]; - allObservationsStore.read(buffer); - allObservationsStore.close(); - ObjectInputStream inStream = new ObjectInputStream(new ByteArrayInputStream(Crypto.decryptData(SOURCE, buffer))); - PhenoCube ret = (PhenoCube)inStream.readObject(); - inStream.close(); - return ret; - }else { - System.out.println("ColumnMeta not found for : [" + key + "]"); - return null; - } - } - } - }); - } - - protected static Object[] loadMetadata() { - try (ObjectInputStream objectInputStream = new ObjectInputStream(new GZIPInputStream(new FileInputStream("/opt/local/source/columnMeta.javabin")));){ - TreeMap metastore = (TreeMap) objectInputStream.readObject(); - TreeMap metastoreScrubbed = new TreeMap(); - for(Entry entry : metastore.entrySet()) { - metastoreScrubbed.put(entry.getKey().replaceAll("\\ufffd",""), entry.getValue()); - } - Set allIds = (TreeSet) objectInputStream.readObject(); - return new Object[] {metastoreScrubbed, allIds}; - } catch (IOException | ClassNotFoundException e) { - log.warn("************************************************"); - log.warn("************************************************"); - log.warn("Could not load metastore", e); - log.warn("If you meant to include phenotype data of any kind, please check that the file /opt/local/source/columnMeta.javabin exists and is readable by the service."); - log.warn("************************************************"); - log.warn("************************************************"); - return new Object[] {new TreeMap(), new TreeSet()}; - } - } + private static final String SOURCE = "SOURCE"; + + private static LoadingStore store = new LoadingStore(); + + private static Logger log = LoggerFactory.getLogger(RekeyDataset.class); + + protected static LoadingCache> sourceStore; + + protected static TreeMap sourceMetaStore; + + private static String HPDS_DIRECTORY = "/opt/local/hpds/"; + + public static void main(String[] args) throws IOException, ClassNotFoundException, ExecutionException { + Crypto.loadDefaultKey(); + Crypto.loadKey(SOURCE, "/opt/local/source/encryption_key"); + sourceStore = initializeCache(); + Object[] metadata = loadMetadata(); + sourceMetaStore = (TreeMap) metadata[0]; + store.allObservationsStore = new RandomAccessFile(HPDS_DIRECTORY + "allObservationsStore.javabin", "rw"); + store.allIds = (TreeSet) metadata[1]; + initialLoad(); + + store.saveStore(HPDS_DIRECTORY); + } + + private static void initialLoad() throws IOException, ExecutionException { + for (String conceptPath : sourceMetaStore.keySet()) { + PhenoCube cube = sourceStore.get(conceptPath); + store.store.put(conceptPath, cube); + } + } + + /** + * Build the PhenoCube cache. + * + * @return + * @throws ClassNotFoundException + * @throws FileNotFoundException + * @throws IOException + */ + protected static LoadingCache> initializeCache() + throws ClassNotFoundException, FileNotFoundException, IOException { + return CacheBuilder.newBuilder().maximumSize(10).build(new CacheLoader>() { + public PhenoCube load(String key) throws Exception { + try (RandomAccessFile allObservationsStore = new RandomAccessFile("/opt/local/source/allObservationsStore.javabin", "r");) { + ColumnMeta columnMeta = sourceMetaStore.get(key); + if (columnMeta != null) { + allObservationsStore.seek(columnMeta.getAllObservationsOffset()); + int length = (int) (columnMeta.getAllObservationsLength() - columnMeta.getAllObservationsOffset()); + byte[] buffer = new byte[length]; + allObservationsStore.read(buffer); + allObservationsStore.close(); + ObjectInputStream inStream = new ObjectInputStream(new ByteArrayInputStream(Crypto.decryptData(SOURCE, buffer))); + PhenoCube ret = (PhenoCube) inStream.readObject(); + inStream.close(); + return ret; + } else { + System.out.println("ColumnMeta not found for : [" + key + "]"); + return null; + } + } + } + }); + } + + protected static Object[] loadMetadata() { + try ( + ObjectInputStream objectInputStream = + new ObjectInputStream(new GZIPInputStream(new FileInputStream("/opt/local/source/columnMeta.javabin"))); + ) { + TreeMap metastore = (TreeMap) objectInputStream.readObject(); + TreeMap metastoreScrubbed = new TreeMap(); + for (Entry entry : metastore.entrySet()) { + metastoreScrubbed.put(entry.getKey().replaceAll("\\ufffd", ""), entry.getValue()); + } + Set allIds = (TreeSet) objectInputStream.readObject(); + return new Object[] {metastoreScrubbed, allIds}; + } catch (IOException | ClassNotFoundException e) { + log.warn("************************************************"); + log.warn("************************************************"); + log.warn("Could not load metastore", e); + log.warn( + "If you meant to include phenotype data of any kind, please check that the file /opt/local/source/columnMeta.javabin exists and is readable by the service." + ); + log.warn("************************************************"); + log.warn("************************************************"); + return new Object[] {new TreeMap(), new TreeSet()}; + } + } } diff --git a/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/phenotype/util/RemapIds.java b/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/phenotype/util/RemapIds.java index 6733bc6de..3d18719d1 100644 --- a/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/phenotype/util/RemapIds.java +++ b/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/phenotype/util/RemapIds.java @@ -22,169 +22,184 @@ @SuppressWarnings({"unchecked", "rawtypes"}) public class RemapIds { - - private static LoadingStore store = new LoadingStore(); - - private static Logger log = LoggerFactory.getLogger(RemapIds.class); - - protected static LoadingCache> sourceStore; - - protected static TreeMap sourceMetaStore; - - private static HashMap patientIdMap; - - private static final int PATIENT_NUM = 0; - - private static final int CONCEPT_PATH = 1; - - private static final int NUMERIC_VALUE = 2; - - private static final int TEXT_VALUE = 3; - - private static final int TIMESTAMP = 4; - - private static String HPDS_DIRECTORY = "/opt/local/hpds/"; - - public static void main(String[] args) throws IOException, ClassNotFoundException, ExecutionException { - loadPatientIdMap(); - sourceStore = initializeCache(); - Object[] metadata = loadMetadata(); - sourceMetaStore = (TreeMap) metadata[0]; - store.allObservationsStore = new RandomAccessFile("/opt/local/hpds/allObservationsStore.javabin", "rw"); - initialLoad(); - store.saveStore(HPDS_DIRECTORY); - } - - private static void loadPatientIdMap() throws IOException { - patientIdMap = new HashMap(); - BufferedReader reader = new BufferedReader(new FileReader("/opt/local/source/idMapping.csv")); - String line = reader.readLine(); - while(line!=null) { - String[] mapping = line.split(","); - patientIdMap.put(mapping[0].replaceAll("\"",""),mapping[1].replaceAll("\"","")); - line = reader.readLine(); - } - reader.close(); - } - - private static void initialLoad() throws IOException, ExecutionException { - final PhenoCube[] currentConcept = new PhenoCube[1]; - for(String conceptPath : sourceMetaStore.keySet()) { - PhenoCube cubeForPath = sourceStore.get(conceptPath); - for(KeyAndValue entry : cubeForPath.sortedByKey()) { - processRecord(currentConcept, List.of( - entry.getKey().toString(), - conceptPath, - cubeForPath.isStringType()?"":entry.getValue().toString(), - cubeForPath.isStringType()?entry.getValue().toString():"", entry.getTimestamp() == null ? null : entry.getTimestamp().toString())); - } - } - } - - private static void processRecord(final PhenoCube[] currentConcept, List record) { - try { - String conceptPathFromRow = record.get(CONCEPT_PATH); - String[] segments = conceptPathFromRow.split("\\\\"); - for(int x = 0;x> initializeCache() throws ClassNotFoundException, FileNotFoundException, IOException { - return CacheBuilder.newBuilder() - .maximumSize(10) - .build( - new CacheLoader>() { - public PhenoCube load(String key) throws Exception { - try(RandomAccessFile allObservationsStore = new RandomAccessFile("/opt/local/source/allObservationsStore.javabin", "r");){ - ColumnMeta columnMeta = sourceMetaStore.get(key); - if(columnMeta != null) { - allObservationsStore.seek(columnMeta.getAllObservationsOffset()); - int length = (int) (columnMeta.getAllObservationsLength() - columnMeta.getAllObservationsOffset()); - byte[] buffer = new byte[length]; - allObservationsStore.read(buffer); - allObservationsStore.close(); - ObjectInputStream inStream = new ObjectInputStream(new ByteArrayInputStream(Crypto.decryptData(buffer))); - PhenoCube ret = (PhenoCube)inStream.readObject(); - inStream.close(); - return ret; - }else { - System.out.println("ColumnMeta not found for : [" + key + "]"); - return null; - } - } - } - }); - } - - protected static Object[] loadMetadata() { - try (ObjectInputStream objectInputStream = new ObjectInputStream(new GZIPInputStream(new FileInputStream("/opt/local/source/columnMeta.javabin")));){ - TreeMap metastore = (TreeMap) objectInputStream.readObject(); - TreeMap metastoreScrubbed = new TreeMap(); - for(Entry entry : metastore.entrySet()) { - metastoreScrubbed.put(entry.getKey().replaceAll("\\ufffd",""), entry.getValue()); - } - Set allIds = (TreeSet) objectInputStream.readObject(); - return new Object[] {metastoreScrubbed, allIds}; - } catch (IOException | ClassNotFoundException e) { - log.warn("************************************************"); - log.warn("************************************************"); - log.warn("Could not load metastore", e); - log.warn("If you meant to include phenotype data of any kind, please check that the file /opt/local/source/columnMeta.javabin exists and is readable by the service."); - log.warn("************************************************"); - log.warn("************************************************"); - return new Object[] {new TreeMap(), new TreeSet()}; - } - } + + private static LoadingStore store = new LoadingStore(); + + private static Logger log = LoggerFactory.getLogger(RemapIds.class); + + protected static LoadingCache> sourceStore; + + protected static TreeMap sourceMetaStore; + + private static HashMap patientIdMap; + + private static final int PATIENT_NUM = 0; + + private static final int CONCEPT_PATH = 1; + + private static final int NUMERIC_VALUE = 2; + + private static final int TEXT_VALUE = 3; + + private static final int TIMESTAMP = 4; + + private static String HPDS_DIRECTORY = "/opt/local/hpds/"; + + public static void main(String[] args) throws IOException, ClassNotFoundException, ExecutionException { + loadPatientIdMap(); + sourceStore = initializeCache(); + Object[] metadata = loadMetadata(); + sourceMetaStore = (TreeMap) metadata[0]; + store.allObservationsStore = new RandomAccessFile("/opt/local/hpds/allObservationsStore.javabin", "rw"); + initialLoad(); + store.saveStore(HPDS_DIRECTORY); + } + + private static void loadPatientIdMap() throws IOException { + patientIdMap = new HashMap(); + BufferedReader reader = new BufferedReader(new FileReader("/opt/local/source/idMapping.csv")); + String line = reader.readLine(); + while (line != null) { + String[] mapping = line.split(","); + patientIdMap.put(mapping[0].replaceAll("\"", ""), mapping[1].replaceAll("\"", "")); + line = reader.readLine(); + } + reader.close(); + } + + private static void initialLoad() throws IOException, ExecutionException { + final PhenoCube[] currentConcept = new PhenoCube[1]; + for (String conceptPath : sourceMetaStore.keySet()) { + PhenoCube cubeForPath = sourceStore.get(conceptPath); + for (KeyAndValue entry : cubeForPath.sortedByKey()) { + processRecord( + currentConcept, + List.of( + entry.getKey().toString(), conceptPath, cubeForPath.isStringType() ? "" : entry.getValue().toString(), + cubeForPath.isStringType() ? entry.getValue().toString() : "", + entry.getTimestamp() == null ? null : entry.getTimestamp().toString() + ) + ); + } + } + } + + private static void processRecord(final PhenoCube[] currentConcept, List record) { + try { + String conceptPathFromRow = record.get(CONCEPT_PATH); + String[] segments = conceptPathFromRow.split("\\\\"); + for (int x = 0; x < segments.length; x++) { + segments[x] = segments[x].trim(); + } + conceptPathFromRow = String.join("\\", segments) + "\\"; + conceptPathFromRow = conceptPathFromRow.replaceAll("\\ufffd", ""); + String textValueFromRow = record.get(TEXT_VALUE) == null ? null : record.get(TEXT_VALUE).trim(); + if (textValueFromRow != null) { + textValueFromRow = textValueFromRow.replaceAll("\\ufffd", ""); + } + String conceptPath = + conceptPathFromRow.endsWith("\\" + textValueFromRow + "\\") ? conceptPathFromRow.replaceAll("\\\\[^\\\\]*\\\\$", "\\\\") + : conceptPathFromRow; + // This is not getDouble because we need to handle null values, not coerce them into 0s + String numericValue = record.get(NUMERIC_VALUE); + if ((numericValue == null || numericValue.isEmpty()) && textValueFromRow != null) { + try { + numericValue = Double.parseDouble(textValueFromRow) + ""; + } catch (NumberFormatException e) { + + } + } + boolean isAlpha = (numericValue == null || numericValue.isEmpty()); + if (currentConcept[0] == null || !currentConcept[0].name.equals(conceptPath)) { + System.out.println(conceptPath); + try { + currentConcept[0] = store.store.get(conceptPath); + } catch (InvalidCacheLoadException e) { + currentConcept[0] = new PhenoCube(conceptPath, isAlpha ? String.class : Double.class); + store.store.put(conceptPath, currentConcept[0]); + } + } + String value = isAlpha ? record.get(TEXT_VALUE) : numericValue; + + if ( + value != null && !value.trim().isEmpty() + && ((isAlpha && currentConcept[0].vType == String.class) || (!isAlpha && currentConcept[0].vType == Double.class)) + ) { + value = value.trim(); + currentConcept[0] + .setColumnWidth(isAlpha ? Math.max(currentConcept[0].getColumnWidth(), value.getBytes().length) : Double.BYTES); + int patientId = Integer.parseInt(mapId(record.get(PATIENT_NUM))); + currentConcept[0].add( + patientId, isAlpha ? value : Double.parseDouble(value), + record.get(TIMESTAMP) == null ? null : new Date(Long.parseLong(record.get(TIMESTAMP))) + ); + store.allIds.add(patientId); + } + } catch (ExecutionException e) { + log.error("Error processing record", e); + } + } + + private static String mapId(String string) { + return patientIdMap.containsKey(string) ? patientIdMap.get(string) : string; + } + + /** + * Load the variantStore object from disk and build the PhenoCube cache. + * + * @return + * @throws ClassNotFoundException + * @throws FileNotFoundException + * @throws IOException + */ + protected static LoadingCache> initializeCache() + throws ClassNotFoundException, FileNotFoundException, IOException { + return CacheBuilder.newBuilder().maximumSize(10).build(new CacheLoader>() { + public PhenoCube load(String key) throws Exception { + try (RandomAccessFile allObservationsStore = new RandomAccessFile("/opt/local/source/allObservationsStore.javabin", "r");) { + ColumnMeta columnMeta = sourceMetaStore.get(key); + if (columnMeta != null) { + allObservationsStore.seek(columnMeta.getAllObservationsOffset()); + int length = (int) (columnMeta.getAllObservationsLength() - columnMeta.getAllObservationsOffset()); + byte[] buffer = new byte[length]; + allObservationsStore.read(buffer); + allObservationsStore.close(); + ObjectInputStream inStream = new ObjectInputStream(new ByteArrayInputStream(Crypto.decryptData(buffer))); + PhenoCube ret = (PhenoCube) inStream.readObject(); + inStream.close(); + return ret; + } else { + System.out.println("ColumnMeta not found for : [" + key + "]"); + return null; + } + } + } + }); + } + + protected static Object[] loadMetadata() { + try ( + ObjectInputStream objectInputStream = + new ObjectInputStream(new GZIPInputStream(new FileInputStream("/opt/local/source/columnMeta.javabin"))); + ) { + TreeMap metastore = (TreeMap) objectInputStream.readObject(); + TreeMap metastoreScrubbed = new TreeMap(); + for (Entry entry : metastore.entrySet()) { + metastoreScrubbed.put(entry.getKey().replaceAll("\\ufffd", ""), entry.getValue()); + } + Set allIds = (TreeSet) objectInputStream.readObject(); + return new Object[] {metastoreScrubbed, allIds}; + } catch (IOException | ClassNotFoundException e) { + log.warn("************************************************"); + log.warn("************************************************"); + log.warn("Could not load metastore", e); + log.warn( + "If you meant to include phenotype data of any kind, please check that the file /opt/local/source/columnMeta.javabin exists and is readable by the service." + ); + log.warn("************************************************"); + log.warn("************************************************"); + return new Object[] {new TreeMap(), new TreeSet()}; + } + } } diff --git a/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/phenotype/util/RemoveConceptFromMetadata.java b/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/phenotype/util/RemoveConceptFromMetadata.java index 3a4c2efd3..208525b69 100644 --- a/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/phenotype/util/RemoveConceptFromMetadata.java +++ b/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/phenotype/util/RemoveConceptFromMetadata.java @@ -18,6 +18,7 @@ import java.util.zip.GZIPOutputStream; import edu.harvard.hms.dbmi.avillach.hpds.data.phenotype.ColumnMeta; + /** * * This util will remove a concept path from the columnMetadata.java @@ -27,58 +28,58 @@ * */ public class RemoveConceptFromMetadata { - - protected static final String COLUMN_META_FILE = "/opt/local/hpds/columnMeta.javabin"; - - protected static final String CONCEPTS_TO_REMOVE = "/opt/local/hpds/conceptsToRemove.txt"; - protected static Set allIds; - - public static void main(String[] args) throws ClassNotFoundException, FileNotFoundException, IOException { - - if(!Files.exists(Paths.get(CONCEPTS_TO_REMOVE))) { - throw new RuntimeException("Columns to remove file - " + CONCEPTS_TO_REMOVE + " - does not exist!"); - - } - if(!Files.exists(Paths.get(COLUMN_META_FILE))) { - throw new RuntimeException("Column Metadata file - " + COLUMN_META_FILE + " - does not exist!"); - } - - TreeMap metadata = removeMetadata(Paths.get(CONCEPTS_TO_REMOVE)); - - ObjectOutputStream metaOut = new ObjectOutputStream(new GZIPOutputStream(new FileOutputStream(new File(COLUMN_META_FILE)))); - - metaOut.writeObject(metadata); - - metaOut.writeObject(allIds); - metaOut.flush(); - metaOut.close(); - } - - protected static TreeMap removeMetadata(Path filePath) { - - try (ObjectInputStream objectInputStream = new ObjectInputStream(new GZIPInputStream(new FileInputStream(COLUMN_META_FILE)))){ - - TreeMap metastore = (TreeMap) objectInputStream.readObject(); - - allIds = (TreeSet) objectInputStream.readObject(); - - System.out.println("allIds size = " + allIds.size()); - - try(BufferedReader reader = Files.newBufferedReader(filePath)) { - - String conceptPathToRemove; - - while((conceptPathToRemove = reader.readLine()) != null) { - - metastore.remove(conceptPathToRemove); - - } - - } - - return metastore; - } catch (IOException | ClassNotFoundException e) { - throw new RuntimeException("Could not load metastore"); - } - } + + protected static final String COLUMN_META_FILE = "/opt/local/hpds/columnMeta.javabin"; + + protected static final String CONCEPTS_TO_REMOVE = "/opt/local/hpds/conceptsToRemove.txt"; + protected static Set allIds; + + public static void main(String[] args) throws ClassNotFoundException, FileNotFoundException, IOException { + + if (!Files.exists(Paths.get(CONCEPTS_TO_REMOVE))) { + throw new RuntimeException("Columns to remove file - " + CONCEPTS_TO_REMOVE + " - does not exist!"); + + } + if (!Files.exists(Paths.get(COLUMN_META_FILE))) { + throw new RuntimeException("Column Metadata file - " + COLUMN_META_FILE + " - does not exist!"); + } + + TreeMap metadata = removeMetadata(Paths.get(CONCEPTS_TO_REMOVE)); + + ObjectOutputStream metaOut = new ObjectOutputStream(new GZIPOutputStream(new FileOutputStream(new File(COLUMN_META_FILE)))); + + metaOut.writeObject(metadata); + + metaOut.writeObject(allIds); + metaOut.flush(); + metaOut.close(); + } + + protected static TreeMap removeMetadata(Path filePath) { + + try (ObjectInputStream objectInputStream = new ObjectInputStream(new GZIPInputStream(new FileInputStream(COLUMN_META_FILE)))) { + + TreeMap metastore = (TreeMap) objectInputStream.readObject(); + + allIds = (TreeSet) objectInputStream.readObject(); + + System.out.println("allIds size = " + allIds.size()); + + try (BufferedReader reader = Files.newBufferedReader(filePath)) { + + String conceptPathToRemove; + + while ((conceptPathToRemove = reader.readLine()) != null) { + + metastore.remove(conceptPathToRemove); + + } + + } + + return metastore; + } catch (IOException | ClassNotFoundException e) { + throw new RuntimeException("Could not load metastore"); + } + } } diff --git a/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/phenotype/util/RenameCategories.java b/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/phenotype/util/RenameCategories.java index 268844e85..414c1437c 100644 --- a/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/phenotype/util/RenameCategories.java +++ b/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/phenotype/util/RenameCategories.java @@ -24,76 +24,79 @@ public class RenameCategories { - protected static LoadingCache> store; + protected static LoadingCache> store; - protected static TreeMap metaStoreSource; + protected static TreeMap metaStoreSource; - protected static TreeMap metaStoreTarget; + protected static TreeMap metaStoreTarget; - protected static TreeSet allIds; - - public static void main(String[] args) throws ClassNotFoundException, FileNotFoundException, IOException { - String conceptPath = args[0]; - String oldCategoryName = args[1]; - String newCategoryName = args[2]; - Object[] metadata = loadMetadata(); - metaStoreSource = (TreeMap) metadata[0]; - allIds = (TreeSet) metadata[1]; - ColumnMeta meta = metaStoreSource.remove(conceptPath); - meta.getCategoryValues().remove(oldCategoryName); - meta.getCategoryValues().add(newCategoryName); - Collections.sort(meta.getCategoryValues()); - store = initializeCache(); - LoadingStore loadingStoreTarget = new LoadingStore(); - metaStoreSource.keySet().forEach((String key)->{ - try { - loadingStoreTarget.store.put(key, store.get(key)); - } catch (ExecutionException e) { - throw new RuntimeException(e); - } - }); - } - - static LoadingStore loadingStoreSource = new LoadingStore(); - - protected static Object[] loadMetadata() { - try (ObjectInputStream objectInputStream = new ObjectInputStream(new FileInputStream("/opt/local/hpds/source/columnMeta.javabin"));){ - TreeMap metastore = (TreeMap) objectInputStream.readObject(); - TreeMap metastoreScrubbed = new TreeMap(); - for(Entry entry : metastore.entrySet()) { - metastoreScrubbed.put(entry.getKey().replaceAll("\\ufffd",""), entry.getValue()); - } - Set allIds = (TreeSet) objectInputStream.readObject(); - return new Object[] {metastoreScrubbed, allIds}; - } catch (IOException | ClassNotFoundException e) { - throw new RuntimeException("Could not load metastore"); - } - } - - protected static LoadingCache> initializeCache() throws ClassNotFoundException, FileNotFoundException, IOException { - return CacheBuilder.newBuilder() - .maximumSize(100) - .build( - new CacheLoader>() { - public PhenoCube load(String key) throws Exception { - try(RandomAccessFile allObservationsStore = new RandomAccessFile("/opt/local/hpds/source/allObservationsStore.javabin", "r");){ - ColumnMeta columnMeta = metaStoreSource.get(key); - if(columnMeta != null) { - allObservationsStore.seek(columnMeta.getAllObservationsOffset()); - int length = (int) (columnMeta.getAllObservationsLength() - columnMeta.getAllObservationsOffset()); - byte[] buffer = new byte[length]; - allObservationsStore.read(buffer); - allObservationsStore.close(); - ObjectInputStream inStream = new ObjectInputStream(new ByteArrayInputStream(Crypto.decryptData(buffer))); - PhenoCube ret = (PhenoCube)inStream.readObject(); - inStream.close(); - return ret; - }else { - System.out.println("ColumnMeta not found for : [" + key + "]"); - return null; - } - } - } - }); - } + protected static TreeSet allIds; + + public static void main(String[] args) throws ClassNotFoundException, FileNotFoundException, IOException { + String conceptPath = args[0]; + String oldCategoryName = args[1]; + String newCategoryName = args[2]; + Object[] metadata = loadMetadata(); + metaStoreSource = (TreeMap) metadata[0]; + allIds = (TreeSet) metadata[1]; + ColumnMeta meta = metaStoreSource.remove(conceptPath); + meta.getCategoryValues().remove(oldCategoryName); + meta.getCategoryValues().add(newCategoryName); + Collections.sort(meta.getCategoryValues()); + store = initializeCache(); + LoadingStore loadingStoreTarget = new LoadingStore(); + metaStoreSource.keySet().forEach((String key) -> { + try { + loadingStoreTarget.store.put(key, store.get(key)); + } catch (ExecutionException e) { + throw new RuntimeException(e); + } + }); + } + + static LoadingStore loadingStoreSource = new LoadingStore(); + + protected static Object[] loadMetadata() { + try ( + ObjectInputStream objectInputStream = new ObjectInputStream(new FileInputStream("/opt/local/hpds/source/columnMeta.javabin")); + ) { + TreeMap metastore = (TreeMap) objectInputStream.readObject(); + TreeMap metastoreScrubbed = new TreeMap(); + for (Entry entry : metastore.entrySet()) { + metastoreScrubbed.put(entry.getKey().replaceAll("\\ufffd", ""), entry.getValue()); + } + Set allIds = (TreeSet) objectInputStream.readObject(); + return new Object[] {metastoreScrubbed, allIds}; + } catch (IOException | ClassNotFoundException e) { + throw new RuntimeException("Could not load metastore"); + } + } + + protected static LoadingCache> initializeCache() + throws ClassNotFoundException, FileNotFoundException, IOException { + return CacheBuilder.newBuilder().maximumSize(100).build(new CacheLoader>() { + public PhenoCube load(String key) throws Exception { + try ( + RandomAccessFile allObservationsStore = + new RandomAccessFile("/opt/local/hpds/source/allObservationsStore.javabin", "r"); + ) { + ColumnMeta columnMeta = metaStoreSource.get(key); + if (columnMeta != null) { + allObservationsStore.seek(columnMeta.getAllObservationsOffset()); + int length = (int) (columnMeta.getAllObservationsLength() - columnMeta.getAllObservationsOffset()); + byte[] buffer = new byte[length]; + allObservationsStore.read(buffer); + allObservationsStore.close(); + ObjectInputStream inStream = new ObjectInputStream(new ByteArrayInputStream(Crypto.decryptData(buffer))); + PhenoCube ret = (PhenoCube) inStream.readObject(); + inStream.close(); + return ret; + } else { + System.out.println("ColumnMeta not found for : [" + key + "]"); + return null; + } + } + } + }); + } } diff --git a/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/LoadingStore.java b/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/LoadingStore.java index 55ab6c749..e2dca71ec 100644 --- a/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/LoadingStore.java +++ b/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/LoadingStore.java @@ -21,190 +21,205 @@ import edu.harvard.hms.dbmi.avillach.hpds.data.phenotype.ColumnMeta; import edu.harvard.hms.dbmi.avillach.hpds.data.phenotype.KeyAndValue; import edu.harvard.hms.dbmi.avillach.hpds.data.phenotype.PhenoCube; - + public class LoadingStore { - - public RandomAccessFile allObservationsStore; - - TreeMap metadataMap = new TreeMap<>(); - - private static Logger log = LoggerFactory.getLogger(LoadingStore.class); - - public LoadingCache store = CacheBuilder.newBuilder() - .maximumSize(16) - .removalListener(new RemovalListener() { - - @Override - public void onRemoval(RemovalNotification arg0) { - log.debug("removing " + arg0.getKey()); - if(arg0.getValue().getLoadingMap()!=null) { - complete(arg0.getValue()); - } - try { - ColumnMeta columnMeta = new ColumnMeta().setName(arg0.getKey()).setWidthInBytes(arg0.getValue().getColumnWidth()).setCategorical(arg0.getValue().isStringType()); - - columnMeta.setAllObservationsOffset(allObservationsStore.getFilePointer()); - columnMeta.setObservationCount(arg0.getValue().sortedByKey().length); - columnMeta.setPatientCount(Arrays.stream(arg0.getValue().sortedByKey()).map((kv)->{return kv.getKey();}).collect(Collectors.toSet()).size()); - if(columnMeta.isCategorical()) { - columnMeta.setCategoryValues(new ArrayList(new TreeSet(arg0.getValue().keyBasedArray()))); - } else { - List map = (List) arg0.getValue().keyBasedArray().stream().map((value)->{return (Double) value;}).collect(Collectors.toList()); - double min = Double.MAX_VALUE; - double max = Double.MIN_VALUE; - for(double f : map) { - min = Double.min(min, f); - max = Double.max(max, f); - } - columnMeta.setMin(min); - columnMeta.setMax(max); - } - ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); - - ObjectOutputStream out = new ObjectOutputStream(byteStream); - out.writeObject(arg0.getValue()); - out.flush(); - out.close(); - - allObservationsStore.write(Crypto.encryptData(byteStream.toByteArray())); - columnMeta.setAllObservationsLength(allObservationsStore.getFilePointer()); - metadataMap.put(columnMeta.getName(), columnMeta); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - private > void complete(PhenoCube cube) { - ArrayList> entryList = new ArrayList>( - cube.getLoadingMap().stream().map((entry)->{ - return new KeyAndValue(entry.getKey(), entry.getValue(), entry.getTimestamp()); - }).collect(Collectors.toList())); - - List> sortedByKey = entryList.stream() - .sorted(Comparator.comparing(KeyAndValue::getKey)) - .collect(Collectors.toList()); - cube.setSortedByKey(sortedByKey.toArray(new KeyAndValue[0])); - - if(cube.isStringType()) { - TreeMap> categoryMap = new TreeMap<>(); - for(KeyAndValue entry : cube.sortedByValue()) { - if(!categoryMap.containsKey(entry.getValue())) { - categoryMap.put(entry.getValue(), new LinkedList()); - } - categoryMap.get(entry.getValue()).add(entry.getKey()); - } - TreeMap> categorySetMap = new TreeMap<>(); - categoryMap.entrySet().stream().forEach((entry)->{ - categorySetMap.put(entry.getKey(), new TreeSet(entry.getValue())); - }); - cube.setCategoryMap(categorySetMap); - } - - } - }) - .build( - new CacheLoader() { - public PhenoCube load(String key) throws Exception { - log.debug("Loading cube: " + key); - return null; - } - }); - - public TreeSet allIds = new TreeSet(); - - public void saveStore(String hpdsDirectory) throws IOException { - System.out.println("Invalidating store"); - store.invalidateAll(); - store.cleanUp(); - System.out.println("Writing metadata"); - ObjectOutputStream metaOut = new ObjectOutputStream(new GZIPOutputStream(new FileOutputStream(hpdsDirectory + "columnMeta.javabin"))); - metaOut.writeObject(metadataMap); - metaOut.writeObject(allIds); - metaOut.flush(); - metaOut.close(); - System.out.println("Closing Store"); - allObservationsStore.close(); - dumpStatsAndColumnMeta(hpdsDirectory); - } - - public void dumpStats() { - System.out.println("Dumping Stats"); - try (ObjectInputStream objectInputStream = new ObjectInputStream(new GZIPInputStream(new FileInputStream("/opt/local/hpds/columnMeta.javabin")));){ - TreeMap metastore = (TreeMap) objectInputStream.readObject(); - Set allIds = (TreeSet) objectInputStream.readObject(); - - long totalNumberOfObservations = 0; - - System.out.println("\n\nConceptPath\tObservationCount\tMinNumValue\tMaxNumValue\tCategoryValues"); - for(String key : metastore.keySet()) { - ColumnMeta columnMeta = metastore.get(key); - System.out.println(String.join("\t", key.toString(), columnMeta.getObservationCount()+"", - columnMeta.getMin()==null ? "NaN" : columnMeta.getMin().toString(), - columnMeta.getMax()==null ? "NaN" : columnMeta.getMax().toString(), - columnMeta.getCategoryValues() == null ? "NUMERIC CONCEPT" : String.join(",", - columnMeta.getCategoryValues() - .stream().map((value)->{return value==null ? "NULL_VALUE" : "\""+value+"\"";}).collect(Collectors.toList())))); - totalNumberOfObservations += columnMeta.getObservationCount(); - } - - System.out.println("Total Number of Concepts : " + metastore.size()); - System.out.println("Total Number of Patients : " + allIds.size()); - System.out.println("Total Number of Observations : " + totalNumberOfObservations); - - } catch (IOException | ClassNotFoundException e) { - throw new RuntimeException("Could not load metastore"); - } - } - - /** - * This method will display counts for the objects stored in the metadata. - * This will also write out a csv file used by the data dictionary importer. - */ - public void dumpStatsAndColumnMeta(String hpdsDirectory) { - try (ObjectInputStream objectInputStream = - new ObjectInputStream(new GZIPInputStream(new FileInputStream(hpdsDirectory + "columnMeta.javabin")))){ - TreeMap metastore = (TreeMap) objectInputStream.readObject(); - try(BufferedWriter writer = Files.newBufferedWriter(Paths.get(hpdsDirectory + "columnMeta.csv"), StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING)) { - CSVPrinter printer = new CSVPrinter(writer, CSVFormat.DEFAULT); - for(String key : metastore.keySet()) { - ColumnMeta columnMeta = metastore.get(key); - Object[] columnMetaOut = new Object[11]; - - StringBuilder listQuoted = new StringBuilder(); - AtomicInteger x = new AtomicInteger(1); - - if(columnMeta.getCategoryValues() != null){ - if(!columnMeta.getCategoryValues().isEmpty()) { - columnMeta.getCategoryValues().forEach(string -> { - listQuoted.append(string); - if(x.get() != columnMeta.getCategoryValues().size()) listQuoted.append("µ"); - x.incrementAndGet(); - }); - } - } - - columnMetaOut[0] = columnMeta.getName(); - columnMetaOut[1] = String.valueOf(columnMeta.getWidthInBytes()); - columnMetaOut[2] = String.valueOf(columnMeta.getColumnOffset()); - columnMetaOut[3] = String.valueOf(columnMeta.isCategorical()); - // this should nest the list of values in a list inside the String array. - columnMetaOut[4] = listQuoted; - columnMetaOut[5] = String.valueOf(columnMeta.getMin()); - columnMetaOut[6] = String.valueOf(columnMeta.getMax()); - columnMetaOut[7] = String.valueOf(columnMeta.getAllObservationsOffset()); - columnMetaOut[8] = String.valueOf(columnMeta.getAllObservationsLength()); - columnMetaOut[9] = String.valueOf(columnMeta.getObservationCount()); - columnMetaOut[10] = String.valueOf(columnMeta.getPatientCount()); - - printer.printRecord(columnMetaOut); - } - - writer.flush(); + + public RandomAccessFile allObservationsStore; + + TreeMap metadataMap = new TreeMap<>(); + + private static Logger log = LoggerFactory.getLogger(LoadingStore.class); + + public LoadingCache store = + CacheBuilder.newBuilder().maximumSize(16).removalListener(new RemovalListener() { + + @Override + public void onRemoval(RemovalNotification arg0) { + log.debug("removing " + arg0.getKey()); + if (arg0.getValue().getLoadingMap() != null) { + complete(arg0.getValue()); + } + try { + ColumnMeta columnMeta = new ColumnMeta().setName(arg0.getKey()).setWidthInBytes(arg0.getValue().getColumnWidth()) + .setCategorical(arg0.getValue().isStringType()); + + columnMeta.setAllObservationsOffset(allObservationsStore.getFilePointer()); + columnMeta.setObservationCount(arg0.getValue().sortedByKey().length); + columnMeta.setPatientCount(Arrays.stream(arg0.getValue().sortedByKey()).map((kv) -> { + return kv.getKey(); + }).collect(Collectors.toSet()).size()); + if (columnMeta.isCategorical()) { + columnMeta.setCategoryValues(new ArrayList(new TreeSet(arg0.getValue().keyBasedArray()))); + } else { + List map = (List) arg0.getValue().keyBasedArray().stream().map((value) -> { + return (Double) value; + }).collect(Collectors.toList()); + double min = Double.MAX_VALUE; + double max = Double.MIN_VALUE; + for (double f : map) { + min = Double.min(min, f); + max = Double.max(max, f); + } + columnMeta.setMin(min); + columnMeta.setMax(max); + } + ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); + + ObjectOutputStream out = new ObjectOutputStream(byteStream); + out.writeObject(arg0.getValue()); + out.flush(); + out.close(); + + allObservationsStore.write(Crypto.encryptData(byteStream.toByteArray())); + columnMeta.setAllObservationsLength(allObservationsStore.getFilePointer()); + metadataMap.put(columnMeta.getName(), columnMeta); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private > void complete(PhenoCube cube) { + ArrayList> entryList = new ArrayList>(cube.getLoadingMap().stream().map((entry) -> { + return new KeyAndValue(entry.getKey(), entry.getValue(), entry.getTimestamp()); + }).collect(Collectors.toList())); + + List> sortedByKey = + entryList.stream().sorted(Comparator.comparing(KeyAndValue::getKey)).collect(Collectors.toList()); + cube.setSortedByKey(sortedByKey.toArray(new KeyAndValue[0])); + + if (cube.isStringType()) { + TreeMap> categoryMap = new TreeMap<>(); + for (KeyAndValue entry : cube.sortedByValue()) { + if (!categoryMap.containsKey(entry.getValue())) { + categoryMap.put(entry.getValue(), new LinkedList()); + } + categoryMap.get(entry.getValue()).add(entry.getKey()); + } + TreeMap> categorySetMap = new TreeMap<>(); + categoryMap.entrySet().stream().forEach((entry) -> { + categorySetMap.put(entry.getKey(), new TreeSet(entry.getValue())); + }); + cube.setCategoryMap(categorySetMap); + } + + } + }).build(new CacheLoader() { + public PhenoCube load(String key) throws Exception { + log.debug("Loading cube: " + key); + return null; + } + }); + + public TreeSet allIds = new TreeSet(); + + public void saveStore(String hpdsDirectory) throws IOException { + System.out.println("Invalidating store"); + store.invalidateAll(); + store.cleanUp(); + System.out.println("Writing metadata"); + ObjectOutputStream metaOut = + new ObjectOutputStream(new GZIPOutputStream(new FileOutputStream(hpdsDirectory + "columnMeta.javabin"))); + metaOut.writeObject(metadataMap); + metaOut.writeObject(allIds); + metaOut.flush(); + metaOut.close(); + System.out.println("Closing Store"); + allObservationsStore.close(); + dumpStatsAndColumnMeta(hpdsDirectory); + } + + public void dumpStats() { + System.out.println("Dumping Stats"); + try ( + ObjectInputStream objectInputStream = + new ObjectInputStream(new GZIPInputStream(new FileInputStream("/opt/local/hpds/columnMeta.javabin"))); + ) { + TreeMap metastore = (TreeMap) objectInputStream.readObject(); + Set allIds = (TreeSet) objectInputStream.readObject(); + + long totalNumberOfObservations = 0; + + System.out.println("\n\nConceptPath\tObservationCount\tMinNumValue\tMaxNumValue\tCategoryValues"); + for (String key : metastore.keySet()) { + ColumnMeta columnMeta = metastore.get(key); + System.out.println( + String.join( + "\t", key.toString(), columnMeta.getObservationCount() + "", + columnMeta.getMin() == null ? "NaN" : columnMeta.getMin().toString(), + columnMeta.getMax() == null ? "NaN" : columnMeta.getMax().toString(), + columnMeta.getCategoryValues() == null ? "NUMERIC CONCEPT" + : String.join(",", columnMeta.getCategoryValues().stream().map((value) -> { + return value == null ? "NULL_VALUE" : "\"" + value + "\""; + }).collect(Collectors.toList())) + ) + ); + totalNumberOfObservations += columnMeta.getObservationCount(); + } + + System.out.println("Total Number of Concepts : " + metastore.size()); + System.out.println("Total Number of Patients : " + allIds.size()); + System.out.println("Total Number of Observations : " + totalNumberOfObservations); + + } catch (IOException | ClassNotFoundException e) { + throw new RuntimeException("Could not load metastore"); + } + } + + /** + * This method will display counts for the objects stored in the metadata. This will also write out a csv file used by the data + * dictionary importer. + */ + public void dumpStatsAndColumnMeta(String hpdsDirectory) { + try ( + ObjectInputStream objectInputStream = + new ObjectInputStream(new GZIPInputStream(new FileInputStream(hpdsDirectory + "columnMeta.javabin"))) + ) { + TreeMap metastore = (TreeMap) objectInputStream.readObject(); + try ( + BufferedWriter writer = Files.newBufferedWriter( + Paths.get(hpdsDirectory + "columnMeta.csv"), StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING + ) + ) { + CSVPrinter printer = new CSVPrinter(writer, CSVFormat.DEFAULT); + for (String key : metastore.keySet()) { + ColumnMeta columnMeta = metastore.get(key); + Object[] columnMetaOut = new Object[11]; + + StringBuilder listQuoted = new StringBuilder(); + AtomicInteger x = new AtomicInteger(1); + + if (columnMeta.getCategoryValues() != null) { + if (!columnMeta.getCategoryValues().isEmpty()) { + columnMeta.getCategoryValues().forEach(string -> { + listQuoted.append(string); + if (x.get() != columnMeta.getCategoryValues().size()) listQuoted.append("µ"); + x.incrementAndGet(); + }); + } + } + + columnMetaOut[0] = columnMeta.getName(); + columnMetaOut[1] = String.valueOf(columnMeta.getWidthInBytes()); + columnMetaOut[2] = String.valueOf(columnMeta.getColumnOffset()); + columnMetaOut[3] = String.valueOf(columnMeta.isCategorical()); + // this should nest the list of values in a list inside the String array. + columnMetaOut[4] = listQuoted; + columnMetaOut[5] = String.valueOf(columnMeta.getMin()); + columnMetaOut[6] = String.valueOf(columnMeta.getMax()); + columnMetaOut[7] = String.valueOf(columnMeta.getAllObservationsOffset()); + columnMetaOut[8] = String.valueOf(columnMeta.getAllObservationsLength()); + columnMetaOut[9] = String.valueOf(columnMeta.getObservationCount()); + columnMetaOut[10] = String.valueOf(columnMeta.getPatientCount()); + + printer.printRecord(columnMetaOut); + } + + writer.flush(); } - } catch (IOException | ClassNotFoundException e) { - throw new RuntimeException("Could not load metastore", e); - } - } + } catch (IOException | ClassNotFoundException e) { + throw new RuntimeException("Could not load metastore", e); + } + } } diff --git a/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/genotype/GenomicDatasetFinalizer.java b/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/genotype/GenomicDatasetFinalizer.java index 9313f6b86..95a93c05c 100644 --- a/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/genotype/GenomicDatasetFinalizer.java +++ b/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/genotype/GenomicDatasetFinalizer.java @@ -32,7 +32,9 @@ public GenomicDatasetFinalizer(String genomicDirectory, int maxThreads) { throw new IllegalArgumentException("Path " + genomicDirectory + " does not contain any directories"); } if (chromosomeDirectories.length > 50) { - throw new IllegalArgumentException("Number of chromosome partitions exceeds maximum of 50 (" + chromosomeDirectories.length + ")"); + throw new IllegalArgumentException( + "Number of chromosome partitions exceeds maximum of 50 (" + chromosomeDirectories.length + ")" + ); } sem = new Semaphore(maxThreads); } @@ -50,10 +52,9 @@ public void processDirectory() { String bucketIndexFilename = bucketIndexDirectory + "BucketIndexBySample.javabin"; log.info("creating new " + bucketIndexFilename); try ( - FileOutputStream fos = new FileOutputStream(bucketIndexFilename); - GZIPOutputStream gzos = new GZIPOutputStream(fos); - ObjectOutputStream oos = new ObjectOutputStream(gzos); - ){ + FileOutputStream fos = new FileOutputStream(bucketIndexFilename); GZIPOutputStream gzos = + new GZIPOutputStream(fos); ObjectOutputStream oos = new ObjectOutputStream(gzos); + ) { oos.writeObject(bucketIndexBySample); oos.flush(); } diff --git a/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/genotype/GenomicDatasetMerger.java b/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/genotype/GenomicDatasetMerger.java index ae3a06e6d..4df7cd0f7 100644 --- a/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/genotype/GenomicDatasetMerger.java +++ b/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/genotype/GenomicDatasetMerger.java @@ -33,7 +33,10 @@ public class GenomicDatasetMerger { private final VariantStore mergedVariantStore; - public GenomicDatasetMerger(VariantStore variantStore1, VariantStore variantStore2, Map infoStores1, Map infoStores2, String outputDirectory) { + public GenomicDatasetMerger( + VariantStore variantStore1, VariantStore variantStore2, Map infoStores1, + Map infoStores2, String outputDirectory + ) { this.variantStore1 = variantStore1; this.variantStore1PatientCount = variantStore1.getPatientIds().length; this.variantStore2 = variantStore2; @@ -52,38 +55,40 @@ private void validate() { log.error(String.join(", ", variantStore2.getVariantMaskStorage().keySet())); throw new IllegalStateException("Unable to merge variant stores with different numbers of chromosomes"); } - Sets.SetView patientIntersection = Sets.intersection(Sets.newHashSet(variantStore1.getPatientIds()), Sets.newHashSet(variantStore2.getPatientIds())); + Sets.SetView patientIntersection = + Sets.intersection(Sets.newHashSet(variantStore1.getPatientIds()), Sets.newHashSet(variantStore2.getPatientIds())); if (!patientIntersection.isEmpty()) { throw new IllegalStateException("Cannot merge genomic datasets containing the same patient id"); } } - public VariantStore mergeVariantStore(Map>> mergedChromosomeMasks) { + public VariantStore mergeVariantStore( + Map>> mergedChromosomeMasks + ) { mergedVariantStore.setVariantMaskStorage(mergedChromosomeMasks); return mergedVariantStore; } + public void mergePatients() { mergedVariantStore.setPatientIds(mergePatientIds()); } /** - * Since both sets of variant indexes reference a different variant spec list, we need to re-index - * the values in the second set of variant indexes. + * Since both sets of variant indexes reference a different variant spec list, we need to re-index the values in the second set of + * variant indexes. * - * For each variant in the second list of variants: - * If it exists in list one, update any references to it by index to it's index in list 1 - * Otherwise, append it to list one, and update any references to it by index to it's new position in list 1 + * For each variant in the second list of variants: If it exists in list one, update any references to it by index to it's index in list + * 1 Otherwise, append it to list one, and update any references to it by index to it's new position in list 1 * * Ex: * - * variantIndex1 = ["chr1,1000,A,G", "chr1,1002,A,G", "chr1,2000,A,G"] - * variantIndex2 = ["chr1,1000,A,G", "chr1,1004,A,G", "chr1,3000,A,G"] + * variantIndex1 = ["chr1,1000,A,G", "chr1,1002,A,G", "chr1,2000,A,G"] variantIndex2 = ["chr1,1000,A,G", "chr1,1004,A,G", + * "chr1,3000,A,G"] * - * GeneWithVariant_store1 = [0, 1] - * GeneWithVariant_store2 = [0, 1, 2] + * GeneWithVariant_store1 = [0, 1] GeneWithVariant_store2 = [0, 1, 2] * - * mergedVariantIndex = ["chr1,1000,A,G", "chr1,1002,A,G", "chr1,2000,A,G", "chr1,1004,A,G", "chr1,3000,A,G"] - * GeneWithVariant_merged = [0, 1, 3, 4] + * mergedVariantIndex = ["chr1,1000,A,G", "chr1,1002,A,G", "chr1,2000,A,G", "chr1,1004,A,G", "chr1,3000,A,G"] GeneWithVariant_merged = + * [0, 1, 3, 4] * * @return * @throws IOException @@ -123,7 +128,7 @@ public Map mergeVariantIndexes() throws if (!infoStores1.keySet().equals(infoStores2.keySet())) { throw new IllegalStateException("Info stores do not match"); } - //for (Map.Entry infoStores1Entry : infoStores1.entrySet()) { + // for (Map.Entry infoStores1Entry : infoStores1.entrySet()) { infoStores1.entrySet().stream().forEach(infoStores1Entry -> { FileBackedByteIndexedInfoStore infoStore2 = infoStores2.get(infoStores1Entry.getKey()); @@ -133,8 +138,8 @@ public Map mergeVariantIndexes() throws Sets.SetView allKeys = Sets.union(allValuesStore1.keys(), allValuesStore2.keys()); for (String key : allKeys) { - Set store1Values = Set.of(allValuesStore1.getOrELse(key, new Integer[]{})); - Set store2Values = Set.of(allValuesStore2.getOrELse(key, new Integer[]{})); + Set store1Values = Set.of(allValuesStore1.getOrELse(key, new Integer[] {})); + Set store2Values = Set.of(allValuesStore2.getOrELse(key, new Integer[] {})); Set remappedValuesStore2 = store2Values.stream().map(value -> remappedIndexes[value]).collect(Collectors.toSet()); Set mergedValues = Sets.union(store1Values, remappedValuesStore2); @@ -163,7 +168,7 @@ public Map mergeVariantIndexes() throws */ private String[] mergePatientIds() { return Stream.concat(Arrays.stream(variantStore1.getPatientIds()), Arrays.stream(variantStore2.getPatientIds())) - .toArray(String[]::new); + .toArray(String[]::new); } /** @@ -171,7 +176,8 @@ private String[] mergePatientIds() { * @return */ public Map>> mergeChromosomeMasks() { - Map>> mergedMaskStorage = new ConcurrentHashMap<>(); + Map>> mergedMaskStorage = + new ConcurrentHashMap<>(); variantStore1.getVariantMaskStorage().keySet().forEach(chromosome -> { try { mergedMaskStorage.put(chromosome, mergeChromosomeMask(chromosome)); @@ -183,52 +189,27 @@ public Map { - * "chr22,10001031,A,G" -> "10101010", - * "chr22,10001143,G,A" -> "10101010" - * }, - * 10002 -> { - * "chr22,10002031,A,G" -> "10101010", - * "chr22,10002143,G,A" -> "10101010" - * } - * } - * variantMaskStorage2: { - * 10001 -> { - * "chr22,10001031,A,G" -> "00001111", - * "chr22,10001213,A,G" -> "00001111" - * }, - * 10003 -> { - * "chr22,10003031,A,G" -> "00001111", - * "chr22,10003213,A,G" -> "00001111" - * } + * Examples: variantMaskStorage1: { 10001 -> { "chr22,10001031,A,G" -> "10101010", "chr22,10001143,G,A" -> "10101010" }, 10002 -> { + * "chr22,10002031,A,G" -> "10101010", "chr22,10002143,G,A" -> "10101010" } } variantMaskStorage2: { 10001 -> { "chr22,10001031,A,G" -> + * "00001111", "chr22,10001213,A,G" -> "00001111" }, 10003 -> { "chr22,10003031,A,G" -> "00001111", "chr22,10003213,A,G" -> "00001111" } * } * - * mergedVariantMaskStorage: { - * 10001 -> { - * "chr22,10001031,A,G" -> "1010101000001111", - * "chr22,10001213,A,G" -> "0000000000001111", - * "chr22,10001143,G,A" -> "1010101000000000" - * }, - * 10002 -> { - * "chr22,10002031,A,G" -> "1010101000000000", - * "chr22,10002143,G,A" -> "1010101000000000" - * } - * 10003 -> { - * "chr22,10003031,A,G" -> "0000000000001111", - * "chr22,10003213,A,G" -> "0000000000001111" - * } - * } + * mergedVariantMaskStorage: { 10001 -> { "chr22,10001031,A,G" -> "1010101000001111", "chr22,10001213,A,G" -> "0000000000001111", + * "chr22,10001143,G,A" -> "1010101000000000" }, 10002 -> { "chr22,10002031,A,G" -> "1010101000000000", "chr22,10002143,G,A" -> + * "1010101000000000" } 10003 -> { "chr22,10003031,A,G" -> "0000000000001111", "chr22,10003213,A,G" -> "0000000000001111" } } */ - public FileBackedJsonIndexStorage> mergeChromosomeMask(String chromosome) throws FileNotFoundException { - FileBackedJsonIndexStorage> variantMaskStorage1 = variantStore1.getVariantMaskStorage().get(chromosome); - FileBackedJsonIndexStorage> variantMaskStorage2 = variantStore2.getVariantMaskStorage().get(chromosome); + public FileBackedJsonIndexStorage> mergeChromosomeMask(String chromosome) + throws FileNotFoundException { + FileBackedJsonIndexStorage> variantMaskStorage1 = + variantStore1.getVariantMaskStorage().get(chromosome); + FileBackedJsonIndexStorage> variantMaskStorage2 = + variantStore2.getVariantMaskStorage().get(chromosome); - FileBackedJsonIndexStorage> merged = new FileBackedStorageVariantMasksImpl(new File(outputDirectory + chromosome + "masks.bin")); + FileBackedJsonIndexStorage> merged = + new FileBackedStorageVariantMasksImpl(new File(outputDirectory + chromosome + "masks.bin")); variantMaskStorage1.keys().stream().forEach(key -> { Map masks1 = variantMaskStorage1.get(key); Map masks2 = variantMaskStorage2.get(key); @@ -245,14 +226,16 @@ public FileBackedJsonIndexStorage entry : masks2.entrySet()) { if (!mergedMasks.containsKey(entry.getKey())) { - VariableVariantMasks appendedMasks = VariableVariantMasks.append(new VariableVariantMasks(), variantStore1PatientCount, entry.getValue(), variantStore2PatientCount); + VariableVariantMasks appendedMasks = VariableVariantMasks + .append(new VariableVariantMasks(), variantStore1PatientCount, entry.getValue(), variantStore2PatientCount); mergedMasks.put(entry.getKey(), appendedMasks); } } @@ -269,7 +252,8 @@ public FileBackedJsonIndexStorage masks2 = variantMaskStorage2.get(key); for (Map.Entry entry : masks2.entrySet()) { if (!mergedMasks.containsKey(entry.getKey())) { - VariableVariantMasks appendedMasks = VariableVariantMasks.append(new VariableVariantMasks(), variantStore1PatientCount, entry.getValue(), variantStore2PatientCount); + VariableVariantMasks appendedMasks = VariableVariantMasks + .append(new VariableVariantMasks(), variantStore1PatientCount, entry.getValue(), variantStore2PatientCount); mergedMasks.put(entry.getKey(), appendedMasks); } } diff --git a/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/genotype/GenomicDatasetMergerRunner.java b/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/genotype/GenomicDatasetMergerRunner.java index 0d8af0df7..7e2db7262 100644 --- a/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/genotype/GenomicDatasetMergerRunner.java +++ b/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/genotype/GenomicDatasetMergerRunner.java @@ -27,13 +27,13 @@ public class GenomicDatasetMergerRunner { private static String genomicDirectory2; /** - * args[0]: directory containing genomic dataset 1 - * args[1]: directory containing genomic dataset 2 - * args[2]: output directory + * args[0]: directory containing genomic dataset 1 args[1]: directory containing genomic dataset 2 args[2]: output directory */ public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException { if (args.length != 3) { - throw new IllegalArgumentException("Three arguments must be provided: source directory 1, source directory 2, output directory"); + throw new IllegalArgumentException( + "Three arguments must be provided: source directory 1, source directory 2, output directory" + ); } genomicDirectory1 = args[0]; genomicDirectory2 = args[1]; @@ -42,10 +42,14 @@ public static void main(String[] args) throws IOException, ClassNotFoundExceptio Map infoStores1 = loadInfoStores(genomicDirectory1); Map infoStores2 = loadInfoStores(genomicDirectory2); - GenomicDatasetMerger genomicDatasetMerger = new GenomicDatasetMerger(VariantStore.readInstance(genomicDirectory1),VariantStore.readInstance(genomicDirectory2), infoStores1, infoStores2, outputDirectory); + GenomicDatasetMerger genomicDatasetMerger = new GenomicDatasetMerger( + VariantStore.readInstance(genomicDirectory1), VariantStore.readInstance(genomicDirectory2), infoStores1, infoStores2, + outputDirectory + ); genomicDatasetMerger.mergePatients(); - Map>> mergedChromosomeMasks = genomicDatasetMerger.mergeChromosomeMasks(); + Map>> mergedChromosomeMasks = + genomicDatasetMerger.mergeChromosomeMasks(); VariantStore mergedVariantStore = genomicDatasetMerger.mergeVariantStore(mergedChromosomeMasks); Map variantIndexes = genomicDatasetMerger.mergeVariantIndexes(); @@ -63,22 +67,22 @@ public static void main(String[] args) throws IOException, ClassNotFoundExceptio private static Map loadInfoStores(String directory) { Map infoStores = new HashMap<>(); File genomicDataDirectory = new File(directory); - if(genomicDataDirectory.exists() && genomicDataDirectory.isDirectory()) { - Arrays.stream(genomicDataDirectory.list((file, filename)->{return filename.endsWith(INFO_STORE_JAVABIN_SUFFIX);})) - .forEach((String filename)->{ - try ( - FileInputStream fis = new FileInputStream(directory + filename); - GZIPInputStream gis = new GZIPInputStream(fis); - ObjectInputStream ois = new ObjectInputStream(gis) - ){ - log.info("loading " + filename); - FileBackedByteIndexedInfoStore infoStore = (FileBackedByteIndexedInfoStore) ois.readObject(); - infoStore.updateStorageDirectory(genomicDataDirectory); - infoStores.put(filename.replace("_" + INFO_STORE_JAVABIN_SUFFIX, ""), infoStore); - } catch (IOException | ClassNotFoundException e) { - throw new RuntimeException(e); - } - }); + if (genomicDataDirectory.exists() && genomicDataDirectory.isDirectory()) { + Arrays.stream(genomicDataDirectory.list((file, filename) -> { + return filename.endsWith(INFO_STORE_JAVABIN_SUFFIX); + })).forEach((String filename) -> { + try ( + FileInputStream fis = new FileInputStream(directory + filename); GZIPInputStream gis = + new GZIPInputStream(fis); ObjectInputStream ois = new ObjectInputStream(gis) + ) { + log.info("loading " + filename); + FileBackedByteIndexedInfoStore infoStore = (FileBackedByteIndexedInfoStore) ois.readObject(); + infoStore.updateStorageDirectory(genomicDataDirectory); + infoStores.put(filename.replace("_" + INFO_STORE_JAVABIN_SUFFIX, ""), infoStore); + } catch (IOException | ClassNotFoundException e) { + throw new RuntimeException(e); + } + }); } return infoStores; } diff --git a/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/genotype/SplitChromosomeVariantMetadataLoader.java b/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/genotype/SplitChromosomeVariantMetadataLoader.java index b17b81887..f69162d2e 100644 --- a/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/genotype/SplitChromosomeVariantMetadataLoader.java +++ b/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/genotype/SplitChromosomeVariantMetadataLoader.java @@ -23,84 +23,85 @@ /** * - * This loader will read in the metadata associated with each variant and build a VariantMetadataIndex that - * can be used to populate data in the PIC-SURE Varaint Explorer. + * This loader will read in the metadata associated with each variant and build a VariantMetadataIndex that can be used to populate data in + * the PIC-SURE Varaint Explorer. */ public class SplitChromosomeVariantMetadataLoader { - public static final String DEFAULT_TSV_FILENAME = "vcfIndex.tsv"; - public static final String DEFAULT_HPDS_DIRECTORY = "/opt/local/hpds/"; - - public static final String DEFAULT_HPDS_DATA_DIRECTORY = DEFAULT_HPDS_DIRECTORY + "all/"; - private static Logger log = LoggerFactory.getLogger(SplitChromosomeVariantMetadataLoader.class); - - //fields to allow tests to override default file location - private static String hpdsDataPath = null; - - private static final int - ANNOTATED_FLAG_COLUMN = 2, - GZIP_FLAG_COLUMN=3, - FILE_COLUMN = 0; - - public static void main(String[] args) throws IOException { - File vcfIndexFile; - - log.info(new File(".").getAbsolutePath()); - if(args.length > 0 && new File(args[0]).exists()) { - log.info("using path from command line"); - vcfIndexFile = new File(args[0]); - hpdsDataPath = args[1]; - }else { - hpdsDataPath = DEFAULT_HPDS_DATA_DIRECTORY; - vcfIndexFile = new File(DEFAULT_HPDS_DIRECTORY + DEFAULT_TSV_FILENAME); - } - - List vcfIndexFiles = new ArrayList<>(); - - try(CSVParser parser = CSVParser.parse(vcfIndexFile, Charset.forName("UTF-8"), CSVFormat.DEFAULT.withDelimiter('\t').withSkipHeaderRecord(true))) { - final boolean[] horribleHeaderSkipFlag = {false}; - parser.forEach((CSVRecord r)->{ - if(horribleHeaderSkipFlag[0]) { - File vcfFileLocal = new File(r.get(FILE_COLUMN)); - if(Integer.parseInt(r.get(ANNOTATED_FLAG_COLUMN).trim())==1) { - VcfInputFile vcfInput = new VcfInputFile(vcfFileLocal, (Integer.parseInt(r.get(GZIP_FLAG_COLUMN).trim())==1)); - vcfIndexFiles.add(vcfInput); - } - }else { - horribleHeaderSkipFlag[0] = true; - } - }); - } - - vcfIndexFiles.stream().forEach(SplitChromosomeVariantMetadataLoader::createVariantMetadataIndexForContig); - - - } - - private static void createVariantMetadataIndexForContig(VcfInputFile vcfInput) { - try { - String contig = vcfInput.currentContig; - VariantMetadataIndex metadataIndex = new VariantMetadataIndex(hpdsDataPath + contig + "/" + VARIANT_METADATA_STORAGE_FILE_PREFIX); - - while (vcfInput.hasNextVariant()) { - metadataIndex.put(vcfInput.currentVariantSpec, vcfInput.currentMetaData); - vcfInput.nextVariant(); - } - metadataIndex.put(vcfInput.currentVariantSpec, vcfInput.currentMetaData); - - metadataIndex.complete(); - - //store this in a path per contig (or a preset path - String binfilePath = hpdsDataPath + contig + "/" + VARIANT_METADATA_FILENAME; - - try(ObjectOutputStream out = new ObjectOutputStream(new GZIPOutputStream(new FileOutputStream(new File(binfilePath))))){ - out.writeObject(metadataIndex); - out.flush(); - } - - log.info("Finished processing: "+ vcfInput.fileName); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } + public static final String DEFAULT_TSV_FILENAME = "vcfIndex.tsv"; + public static final String DEFAULT_HPDS_DIRECTORY = "/opt/local/hpds/"; + + public static final String DEFAULT_HPDS_DATA_DIRECTORY = DEFAULT_HPDS_DIRECTORY + "all/"; + private static Logger log = LoggerFactory.getLogger(SplitChromosomeVariantMetadataLoader.class); + + // fields to allow tests to override default file location + private static String hpdsDataPath = null; + + private static final int ANNOTATED_FLAG_COLUMN = 2, GZIP_FLAG_COLUMN = 3, FILE_COLUMN = 0; + + public static void main(String[] args) throws IOException { + File vcfIndexFile; + + log.info(new File(".").getAbsolutePath()); + if (args.length > 0 && new File(args[0]).exists()) { + log.info("using path from command line"); + vcfIndexFile = new File(args[0]); + hpdsDataPath = args[1]; + } else { + hpdsDataPath = DEFAULT_HPDS_DATA_DIRECTORY; + vcfIndexFile = new File(DEFAULT_HPDS_DIRECTORY + DEFAULT_TSV_FILENAME); + } + + List vcfIndexFiles = new ArrayList<>(); + + try ( + CSVParser parser = + CSVParser.parse(vcfIndexFile, Charset.forName("UTF-8"), CSVFormat.DEFAULT.withDelimiter('\t').withSkipHeaderRecord(true)) + ) { + final boolean[] horribleHeaderSkipFlag = {false}; + parser.forEach((CSVRecord r) -> { + if (horribleHeaderSkipFlag[0]) { + File vcfFileLocal = new File(r.get(FILE_COLUMN)); + if (Integer.parseInt(r.get(ANNOTATED_FLAG_COLUMN).trim()) == 1) { + VcfInputFile vcfInput = new VcfInputFile(vcfFileLocal, (Integer.parseInt(r.get(GZIP_FLAG_COLUMN).trim()) == 1)); + vcfIndexFiles.add(vcfInput); + } + } else { + horribleHeaderSkipFlag[0] = true; + } + }); + } + + vcfIndexFiles.stream().forEach(SplitChromosomeVariantMetadataLoader::createVariantMetadataIndexForContig); + + + } + + private static void createVariantMetadataIndexForContig(VcfInputFile vcfInput) { + try { + String contig = vcfInput.currentContig; + VariantMetadataIndex metadataIndex = + new VariantMetadataIndex(hpdsDataPath + contig + "/" + VARIANT_METADATA_STORAGE_FILE_PREFIX); + + while (vcfInput.hasNextVariant()) { + metadataIndex.put(vcfInput.currentVariantSpec, vcfInput.currentMetaData); + vcfInput.nextVariant(); + } + metadataIndex.put(vcfInput.currentVariantSpec, vcfInput.currentMetaData); + + metadataIndex.complete(); + + // store this in a path per contig (or a preset path + String binfilePath = hpdsDataPath + contig + "/" + VARIANT_METADATA_FILENAME; + + try (ObjectOutputStream out = new ObjectOutputStream(new GZIPOutputStream(new FileOutputStream(new File(binfilePath))))) { + out.writeObject(metadataIndex); + out.flush(); + } + + log.info("Finished processing: " + vcfInput.fileName); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } } diff --git a/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/genotype/VCFIndexBuilder.java b/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/genotype/VCFIndexBuilder.java index 9b14d049f..7be3220af 100644 --- a/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/genotype/VCFIndexBuilder.java +++ b/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/genotype/VCFIndexBuilder.java @@ -25,7 +25,9 @@ public class VCFIndexBuilder { private static final Joiner COMMA_JOINER = Joiner.on(","); - public VCFIndexBuilder(File vcfPatientMappingFile, File patientUUIDToIdMappingFile, File vcfIndexOutputDirectory, Set validPatientType) { + public VCFIndexBuilder( + File vcfPatientMappingFile, File patientUUIDToIdMappingFile, File vcfIndexOutputDirectory, Set validPatientType + ) { this.vcfPatientMappingFile = vcfPatientMappingFile; this.patientUUIDToIdMappingFile = patientUUIDToIdMappingFile; this.vcfIndexOutputDirectory = vcfIndexOutputDirectory; @@ -40,12 +42,7 @@ public static void main(String[] args) { if (!vcfIndexOutputDirectoryFile.isDirectory()) { throw new IllegalArgumentException("Argument 3 must be a valid directory"); } - new VCFIndexBuilder( - new File(args[0]), - new File(args[1]), - vcfIndexOutputDirectoryFile, - Set.of(args[3].split(",")) - ).run(); + new VCFIndexBuilder(new File(args[0]), new File(args[1]), vcfIndexOutputDirectoryFile, Set.of(args[3].split(","))).run(); } private void run() { @@ -53,28 +50,22 @@ private void run() { patientUUIDToPatientIdMapping = new HashMap<>(); fileToPatientListMap = new HashMap<>(); try { - Files.lines(patientUUIDToIdMappingFile.toPath()) - .skip(1) - .map(l -> l.split(",")) - .filter(columns -> columns.length == 2) - .forEach(columns -> { - patientUUIDToPatientIdMapping.put(columns[0], columns[1]); - }); - - Files.lines(vcfPatientMappingFile.toPath()) - .skip(1) - .map(l -> l.split(",")) - .filter(columns -> validPatientTypes.contains(columns[4])) - .forEach(columns -> { - String patientUuid = columns[0]; - String vcfFile = columns[1].substring(columns[1].lastIndexOf("/") + 1); - List patientList = Optional.ofNullable(fileToPatientListMap.get(vcfFile)).orElseGet(ArrayList::new); - patientList.add(patientUuid); - - if (patientUUIDToPatientIdMapping.get(patientUuid) != null) { - fileToPatientListMap.put(vcfFile, patientList); - } - }); + Files.lines(patientUUIDToIdMappingFile.toPath()).skip(1).map(l -> l.split(",")).filter(columns -> columns.length == 2) + .forEach(columns -> { + patientUUIDToPatientIdMapping.put(columns[0], columns[1]); + }); + + Files.lines(vcfPatientMappingFile.toPath()).skip(1).map(l -> l.split(",")) + .filter(columns -> validPatientTypes.contains(columns[4])).forEach(columns -> { + String patientUuid = columns[0]; + String vcfFile = columns[1].substring(columns[1].lastIndexOf("/") + 1); + List patientList = Optional.ofNullable(fileToPatientListMap.get(vcfFile)).orElseGet(ArrayList::new); + patientList.add(patientUuid); + + if (patientUUIDToPatientIdMapping.get(patientUuid) != null) { + fileToPatientListMap.put(vcfFile, patientList); + } + }); } catch (IOException e) { throw new UncheckedIOException(e); } @@ -92,24 +83,23 @@ private void writeVcfIndexes() { groupToVcfMapping.put(baseFile, vcfFiles); } - groupToVcfMapping.keySet() - .stream() - .forEach(vcfGroup -> { - writeVcfIndex(vcfGroup, groupToVcfMapping.get(vcfGroup)); - }); + groupToVcfMapping.keySet().stream().forEach(vcfGroup -> { + writeVcfIndex(vcfGroup, groupToVcfMapping.get(vcfGroup)); + }); } private void writeVcfIndex(String vcfGroup, List vcfFiles) { try { FileWriter fileWriter = new FileWriter(vcfIndexOutputDirectory.getAbsolutePath() + "/" + vcfGroup + "-vcfIndex.tsv"); - fileWriter.write("\"vcf_path\"\t\"chromosome\"\t\"isAnnotated\"\t\"isGzipped\"\t\"sample_ids\"\t\"patient_ids\"\t\"sample_relationship\"\t\"related_sample_ids\"\n"); + fileWriter.write( + "\"vcf_path\"\t\"chromosome\"\t\"isAnnotated\"\t\"isGzipped\"\t\"sample_ids\"\t\"patient_ids\"\t\"sample_relationship\"\t\"related_sample_ids\"\n" + ); for (String vcfFile : vcfFiles) { - Set validPatientUUIDs = fileToPatientListMap.get(vcfFile) - .stream() - .filter(patientUUIDToPatientIdMapping::containsKey) - .collect(Collectors.toSet()); - List patentIds = validPatientUUIDs.stream().map(patientUUIDToPatientIdMapping::get).filter(Objects::nonNull).toList(); + Set validPatientUUIDs = fileToPatientListMap.get(vcfFile).stream() + .filter(patientUUIDToPatientIdMapping::containsKey).collect(Collectors.toSet()); + List patentIds = + validPatientUUIDs.stream().map(patientUUIDToPatientIdMapping::get).filter(Objects::nonNull).toList(); fileWriter.write("\"" + VCF_INDEX_DIRECTORY + "/" + vcfFile + "\"\t\"" + extractChromosome(vcfFile) + "\"\t\"1\"\t\"1\"\t"); fileWriter.write("\"" + COMMA_JOINER.join(validPatientUUIDs) + "\"\t\"" + COMMA_JOINER.join(patentIds) + "\"\n"); } diff --git a/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/genotype/VCFPerPatientInfoStoreSplitter.java b/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/genotype/VCFPerPatientInfoStoreSplitter.java index 4172beadb..9cdae6c8f 100644 --- a/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/genotype/VCFPerPatientInfoStoreSplitter.java +++ b/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/genotype/VCFPerPatientInfoStoreSplitter.java @@ -12,95 +12,94 @@ import edu.harvard.hms.dbmi.avillach.hpds.data.genotype.InfoStore; public class VCFPerPatientInfoStoreSplitter { - private static Logger logger = LoggerFactory.getLogger(NewVCFLoader.class); -// private static File mergedFolder = new File("/opt/local/hpds/merged"); + private static Logger logger = LoggerFactory.getLogger(NewVCFLoader.class); + // private static File mergedFolder = new File("/opt/local/hpds/merged"); - public static void splitAll(File javabinDir, File mergedFolder) throws ClassNotFoundException, FileNotFoundException, IOException, - InterruptedException, ExecutionException { - ConcurrentHashMap mergedFiles = new ConcurrentHashMap(); + public static void splitAll(File javabinDir, File mergedFolder) + throws ClassNotFoundException, FileNotFoundException, IOException, InterruptedException, ExecutionException { + ConcurrentHashMap mergedFiles = new ConcurrentHashMap(); - File[] inputFiles = javabinDir.listFiles((file) -> { - return file.getName().endsWith("_infoStores.javabin"); - }); + File[] inputFiles = javabinDir.listFiles((file) -> { + return file.getName().endsWith("_infoStores.javabin"); + }); - mergedFolder.mkdir(); + mergedFolder.mkdir(); - for (File file : inputFiles) { - try (FileInputStream fisi = new FileInputStream(file); - GZIPInputStream gzisi = new GZIPInputStream(fisi); - ObjectInputStream objectInputStream = new ObjectInputStream(gzisi);) { - ConcurrentHashMap store = (ConcurrentHashMap) objectInputStream - .readObject(); - logger.info("Loaded " + file.getName()); - store.keySet().parallelStream().forEach((key) -> { - if (key.contentEquals("PU") || key.contentEquals("IPU")) { + for (File file : inputFiles) { + try ( + FileInputStream fisi = new FileInputStream(file); GZIPInputStream gzisi = + new GZIPInputStream(fisi); ObjectInputStream objectInputStream = new ObjectInputStream(gzisi); + ) { + ConcurrentHashMap store = (ConcurrentHashMap) objectInputStream.readObject(); + logger.info("Loaded " + file.getName()); + store.keySet().parallelStream().forEach((key) -> { + if (key.contentEquals("PU") || key.contentEquals("IPU")) { - } else { - try { - InfoStore partitionInfoStore = store.get(key); - File mergedStoreFile; - if (mergedFiles.containsKey(key)) { - mergedStoreFile = mergedFiles.get(key); - InfoStore mergedInfoStore = loadMergedStore(mergedFiles, key); - for (String value : partitionInfoStore.allValues.keySet()) { - if (mergedInfoStore.allValues.containsKey(value)) { - mergedInfoStore.allValues.get(value) - .addAll(partitionInfoStore.allValues.get(value)); - } else { - mergedInfoStore.allValues.put(value, partitionInfoStore.allValues.get(value)); - } - } - writeStore(mergedFiles, key, mergedInfoStore); - logger.info("Merged " + key + " for partition " + file.getName()); - } else { - mergedStoreFile = new File(mergedFolder, key + "_infoStore.javabin"); - mergedFiles.put(key, mergedStoreFile); - writeStore(mergedFiles, key, partitionInfoStore); - logger.info("Adding new store to merged set : " + key); - } - } catch (IOException e) { - logger.error("Error processing file [" + file.getName() + "], key [" + key + "]", e); - } catch (ClassNotFoundException e) { - logger.error("Error processing file [" + file.getName() + "], key [" + key + "]", e); - } - } + } else { + try { + InfoStore partitionInfoStore = store.get(key); + File mergedStoreFile; + if (mergedFiles.containsKey(key)) { + mergedStoreFile = mergedFiles.get(key); + InfoStore mergedInfoStore = loadMergedStore(mergedFiles, key); + for (String value : partitionInfoStore.allValues.keySet()) { + if (mergedInfoStore.allValues.containsKey(value)) { + mergedInfoStore.allValues.get(value).addAll(partitionInfoStore.allValues.get(value)); + } else { + mergedInfoStore.allValues.put(value, partitionInfoStore.allValues.get(value)); + } + } + writeStore(mergedFiles, key, mergedInfoStore); + logger.info("Merged " + key + " for partition " + file.getName()); + } else { + mergedStoreFile = new File(mergedFolder, key + "_infoStore.javabin"); + mergedFiles.put(key, mergedStoreFile); + writeStore(mergedFiles, key, partitionInfoStore); + logger.info("Adding new store to merged set : " + key); + } + } catch (IOException e) { + logger.error("Error processing file [" + file.getName() + "], key [" + key + "]", e); + } catch (ClassNotFoundException e) { + logger.error("Error processing file [" + file.getName() + "], key [" + key + "]", e); + } + } - }); - } catch (FileNotFoundException e) { - logger.error("Error processing file [" + file.getName() + "] : infoStores.javabin file not found", e); - } catch (IOException e) { - logger.error("Error processing file [" + file.getName() + "]", e); - } catch (ClassNotFoundException e) { - logger.error("Error processing file [" + file.getName() + "]", e); - } - } + }); + } catch (FileNotFoundException e) { + logger.error("Error processing file [" + file.getName() + "] : infoStores.javabin file not found", e); + } catch (IOException e) { + logger.error("Error processing file [" + file.getName() + "]", e); + } catch (ClassNotFoundException e) { + logger.error("Error processing file [" + file.getName() + "]", e); + } + } - } + } - private static InfoStore loadMergedStore(ConcurrentHashMap mergedFiles, String key) - throws FileNotFoundException, IOException, ClassNotFoundException { - FileInputStream fis = new FileInputStream(mergedFiles.get(key)); - GZIPInputStream gzis = new GZIPInputStream(fis); - ObjectInputStream ois = new ObjectInputStream(gzis); - InfoStore mergedInfoStore = (InfoStore) ois.readObject(); - ois.close(); - gzis.close(); - fis.close(); - return mergedInfoStore; - } + private static InfoStore loadMergedStore(ConcurrentHashMap mergedFiles, String key) + throws FileNotFoundException, IOException, ClassNotFoundException { + FileInputStream fis = new FileInputStream(mergedFiles.get(key)); + GZIPInputStream gzis = new GZIPInputStream(fis); + ObjectInputStream ois = new ObjectInputStream(gzis); + InfoStore mergedInfoStore = (InfoStore) ois.readObject(); + ois.close(); + gzis.close(); + fis.close(); + return mergedInfoStore; + } - private static synchronized void writeStore(ConcurrentHashMap mergedFiles, String key, - InfoStore mergedInfoStore) throws FileNotFoundException, IOException { - FileOutputStream fos = new FileOutputStream(mergedFiles.get(key)); - GZIPOutputStream gzos = new GZIPOutputStream(fos); - ObjectOutputStream oos = new ObjectOutputStream(gzos); - oos.writeObject(mergedInfoStore); - oos.flush(); - oos.close(); - gzos.flush(); - gzos.close(); - fos.flush(); - fos.close(); - } + private static synchronized void writeStore(ConcurrentHashMap mergedFiles, String key, InfoStore mergedInfoStore) + throws FileNotFoundException, IOException { + FileOutputStream fos = new FileOutputStream(mergedFiles.get(key)); + GZIPOutputStream gzos = new GZIPOutputStream(fos); + ObjectOutputStream oos = new ObjectOutputStream(gzos); + oos.writeObject(mergedInfoStore); + oos.flush(); + oos.close(); + gzos.flush(); + gzos.close(); + fos.flush(); + fos.close(); + } } diff --git a/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/genotype/VCFPerPatientInfoStoreToFBBIISConverter.java b/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/genotype/VCFPerPatientInfoStoreToFBBIISConverter.java index cdfe3cdd9..6837e738e 100644 --- a/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/genotype/VCFPerPatientInfoStoreToFBBIISConverter.java +++ b/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/genotype/VCFPerPatientInfoStoreToFBBIISConverter.java @@ -13,45 +13,46 @@ import edu.harvard.hms.dbmi.avillach.hpds.data.genotype.InfoStore; public class VCFPerPatientInfoStoreToFBBIISConverter { - private static Logger logger = LoggerFactory.getLogger(NewVCFLoader.class); - - public static void convertAll(String inputPath, String outputPath) throws ClassNotFoundException, - FileNotFoundException, IOException, InterruptedException, ExecutionException { - File[] inputFiles = new File(inputPath).listFiles((path) -> { - return path.getName().endsWith("infoStore.javabin"); - }); - Arrays.sort(inputFiles); - - File outputFolder = new File(outputPath); - - List inputFileList = Arrays.asList(inputFiles); - inputFileList.stream().forEach((file) -> { - convert(outputFolder, file); - }); - } - - public static void convert(File outputFolder, File file) { - logger.info("Converting InfoStore file: " + file.getAbsolutePath()); - try (FileInputStream fisi = new FileInputStream(file); - GZIPInputStream gzisi = new GZIPInputStream(fisi); - ObjectInputStream objectInputStream = new ObjectInputStream(gzisi);) { - InfoStore store = (InfoStore) objectInputStream.readObject(); - - if (store.allValues.size() > 0) { - FileBackedByteIndexedInfoStore fbbiis = new FileBackedByteIndexedInfoStore(outputFolder, store); - - fbbiis.write(new File(outputFolder, file.getName())); - logger.info("Completed converting InfoStore file: " + file.getAbsolutePath()); - } else { - logger.info("Skipping empty InfoStore file: " + file.getAbsolutePath() + ""); - } - } catch (FileNotFoundException e) { - logger.error("InfoStore file not found: " + file.getAbsolutePath()); - } catch (IOException e) { - logger.error("Error converting InfoStore file: " + file.getAbsolutePath()); - } catch (ClassNotFoundException e) { - logger.error("Error converting InfoStore file: " + file.getAbsolutePath()); - } - } - -} \ No newline at end of file + private static Logger logger = LoggerFactory.getLogger(NewVCFLoader.class); + + public static void convertAll(String inputPath, String outputPath) + throws ClassNotFoundException, FileNotFoundException, IOException, InterruptedException, ExecutionException { + File[] inputFiles = new File(inputPath).listFiles((path) -> { + return path.getName().endsWith("infoStore.javabin"); + }); + Arrays.sort(inputFiles); + + File outputFolder = new File(outputPath); + + List inputFileList = Arrays.asList(inputFiles); + inputFileList.stream().forEach((file) -> { + convert(outputFolder, file); + }); + } + + public static void convert(File outputFolder, File file) { + logger.info("Converting InfoStore file: " + file.getAbsolutePath()); + try ( + FileInputStream fisi = new FileInputStream(file); GZIPInputStream gzisi = + new GZIPInputStream(fisi); ObjectInputStream objectInputStream = new ObjectInputStream(gzisi); + ) { + InfoStore store = (InfoStore) objectInputStream.readObject(); + + if (store.allValues.size() > 0) { + FileBackedByteIndexedInfoStore fbbiis = new FileBackedByteIndexedInfoStore(outputFolder, store); + + fbbiis.write(new File(outputFolder, file.getName())); + logger.info("Completed converting InfoStore file: " + file.getAbsolutePath()); + } else { + logger.info("Skipping empty InfoStore file: " + file.getAbsolutePath() + ""); + } + } catch (FileNotFoundException e) { + logger.error("InfoStore file not found: " + file.getAbsolutePath()); + } catch (IOException e) { + logger.error("Error converting InfoStore file: " + file.getAbsolutePath()); + } catch (ClassNotFoundException e) { + logger.error("Error converting InfoStore file: " + file.getAbsolutePath()); + } + } + +} diff --git a/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/genotype/VariantMetadataLoader.java b/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/genotype/VariantMetadataLoader.java index f4b5a81e0..e38799c3f 100644 --- a/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/genotype/VariantMetadataLoader.java +++ b/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/genotype/VariantMetadataLoader.java @@ -19,86 +19,86 @@ /** * - * This loader will read in the metadata associated with each variant and build a VariantMetadataIndex that - * can be used to populate data in the PIC-SURE Varaint Explorer. + * This loader will read in the metadata associated with each variant and build a VariantMetadataIndex that can be used to populate data in + * the PIC-SURE Varaint Explorer. */ public class VariantMetadataLoader { - - private static Logger log = LoggerFactory.getLogger(VariantMetadataLoader.class); - - private static VariantMetadataIndex metadataIndex; - - //fields to allow tests to override default file location - private static String storagePathForTests = null; - private static String variantIndexPathForTests = null; - - private static final int - ANNOTATED_FLAG_COLUMN = 2, - GZIP_FLAG_COLUMN=3, - FILE_COLUMN = 0; - - public static void main(String[] args) throws IOException { - File vcfIndexFile; - - log.info(new File(".").getAbsolutePath()); - if(args.length > 0 && new File(args[0]).exists()) { - log.info("using path from command line"); - vcfIndexFile = new File(args[0]); - variantIndexPathForTests = args[1]; - storagePathForTests = args[2]; - metadataIndex = new VariantMetadataIndex(storagePathForTests); - }else { - metadataIndex = new VariantMetadataIndex(); - vcfIndexFile = new File("/opt/local/hpds/vcfIndex.tsv"); - } - - TreeSet vcfFileTree = new TreeSet<>(); - - try(CSVParser parser = CSVParser.parse(vcfIndexFile, Charset.forName("UTF-8"), CSVFormat.DEFAULT.withDelimiter('\t').withSkipHeaderRecord(true))) { - final boolean[] horribleHeaderSkipFlag = {false}; - parser.forEach((CSVRecord r)->{ - if(horribleHeaderSkipFlag[0]) { - File vcfFileLocal = new File(r.get(FILE_COLUMN)); - if(Integer.parseInt(r.get(ANNOTATED_FLAG_COLUMN).trim())==1) { - VcfInputFile vcfInput = new VcfInputFile(vcfFileLocal, (Integer.parseInt(r.get(GZIP_FLAG_COLUMN).trim())==1)); - vcfFileTree.add(vcfInput); - } - }else { - horribleHeaderSkipFlag[0] = true; - } - }); - } - - String currentContig = ""; - while( vcfFileTree.size() > 0) { - - //find and remove the lowest element - VcfInputFile vcfInput = vcfFileTree.pollFirst(); - - //write to disk each time the contig changes - if(! currentContig.equals(vcfInput.currentContig)) { - metadataIndex.flush(); - } - - currentContig = vcfInput.currentContig; - metadataIndex.put(vcfInput.currentVariantSpec, vcfInput.currentMetaData); - - if(vcfInput.hasNextVariant()) { - vcfInput.nextVariant(); - vcfFileTree.add(vcfInput); - } else { - log.info("Finished processing: "+ vcfInput.fileName); - } - } - - metadataIndex.complete(); - - //store this in a path per contig (or a preset path - String binfilePath = variantIndexPathForTests == null ? VariantMetadataIndex.VARIANT_METADATA_BIN_FILE : variantIndexPathForTests; - - try(ObjectOutputStream out = new ObjectOutputStream(new GZIPOutputStream(new FileOutputStream(new File(binfilePath))))){ - out.writeObject(metadataIndex); - out.flush(); - } - } + + private static Logger log = LoggerFactory.getLogger(VariantMetadataLoader.class); + + private static VariantMetadataIndex metadataIndex; + + // fields to allow tests to override default file location + private static String storagePathForTests = null; + private static String variantIndexPathForTests = null; + + private static final int ANNOTATED_FLAG_COLUMN = 2, GZIP_FLAG_COLUMN = 3, FILE_COLUMN = 0; + + public static void main(String[] args) throws IOException { + File vcfIndexFile; + + log.info(new File(".").getAbsolutePath()); + if (args.length > 0 && new File(args[0]).exists()) { + log.info("using path from command line"); + vcfIndexFile = new File(args[0]); + variantIndexPathForTests = args[1]; + storagePathForTests = args[2]; + metadataIndex = new VariantMetadataIndex(storagePathForTests); + } else { + metadataIndex = new VariantMetadataIndex(); + vcfIndexFile = new File("/opt/local/hpds/vcfIndex.tsv"); + } + + TreeSet vcfFileTree = new TreeSet<>(); + + try ( + CSVParser parser = + CSVParser.parse(vcfIndexFile, Charset.forName("UTF-8"), CSVFormat.DEFAULT.withDelimiter('\t').withSkipHeaderRecord(true)) + ) { + final boolean[] horribleHeaderSkipFlag = {false}; + parser.forEach((CSVRecord r) -> { + if (horribleHeaderSkipFlag[0]) { + File vcfFileLocal = new File(r.get(FILE_COLUMN)); + if (Integer.parseInt(r.get(ANNOTATED_FLAG_COLUMN).trim()) == 1) { + VcfInputFile vcfInput = new VcfInputFile(vcfFileLocal, (Integer.parseInt(r.get(GZIP_FLAG_COLUMN).trim()) == 1)); + vcfFileTree.add(vcfInput); + } + } else { + horribleHeaderSkipFlag[0] = true; + } + }); + } + + String currentContig = ""; + while (vcfFileTree.size() > 0) { + + // find and remove the lowest element + VcfInputFile vcfInput = vcfFileTree.pollFirst(); + + // write to disk each time the contig changes + if (!currentContig.equals(vcfInput.currentContig)) { + metadataIndex.flush(); + } + + currentContig = vcfInput.currentContig; + metadataIndex.put(vcfInput.currentVariantSpec, vcfInput.currentMetaData); + + if (vcfInput.hasNextVariant()) { + vcfInput.nextVariant(); + vcfFileTree.add(vcfInput); + } else { + log.info("Finished processing: " + vcfInput.fileName); + } + } + + metadataIndex.complete(); + + // store this in a path per contig (or a preset path + String binfilePath = variantIndexPathForTests == null ? VariantMetadataIndex.VARIANT_METADATA_BIN_FILE : variantIndexPathForTests; + + try (ObjectOutputStream out = new ObjectOutputStream(new GZIPOutputStream(new FileOutputStream(new File(binfilePath))))) { + out.writeObject(metadataIndex); + out.flush(); + } + } } diff --git a/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/genotype/VcfInputFile.java b/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/genotype/VcfInputFile.java index eb7fc87c2..b29fe1d2d 100644 --- a/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/genotype/VcfInputFile.java +++ b/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/genotype/VcfInputFile.java @@ -33,15 +33,16 @@ public class VcfInputFile implements Comparable { public VcfInputFile(File vcfFile, boolean gzipped) { fileName = vcfFile.getName(); log.info("Processing VCF file: " + fileName); - try{ - Reader reader = new InputStreamReader( gzipped ? new GZIPInputStream(new FileInputStream(vcfFile)) : new FileInputStream(vcfFile)); + try { + Reader reader = + new InputStreamReader(gzipped ? new GZIPInputStream(new FileInputStream(vcfFile)) : new FileInputStream(vcfFile)); parser = new CSVParser(reader, CSVFormat.DEFAULT.withDelimiter('\t').withSkipHeaderRecord(false)); iterator = parser.iterator(); - while(iterator.hasNext()) { + while (iterator.hasNext()) { CSVRecord csvRecord = iterator.next(); - //skip all header rows - if(csvRecord.get(0).startsWith("#")) { + // skip all header rows + if (csvRecord.get(0).startsWith("#")) { continue; } @@ -52,7 +53,7 @@ public VcfInputFile(File vcfFile, boolean gzipped) { break; } - }catch(IOException e) { + } catch (IOException e) { log.error("Error processing VCF file: " + vcfFile.getName(), e); } @@ -64,8 +65,8 @@ boolean hasNextVariant() { void nextVariant() { CSVRecord csvRecord = iterator.next(); - //skip all header rows - if(csvRecord.get(0).startsWith("#")) { + // skip all header rows + if (csvRecord.get(0).startsWith("#")) { return; } @@ -76,13 +77,13 @@ void nextVariant() { } /** - * These files will be sorted by the current variant spec. We need to make sure they are never actually - * equal values (since the TreeSet used to keep them sorted enforces uniqueness) + * These files will be sorted by the current variant spec. We need to make sure they are never actually equal values (since the TreeSet + * used to keep them sorted enforces uniqueness) */ @Override public int compareTo(VcfInputFile arg0) { - return (currentVariantSpec + iterator.toString()).compareTo(arg0.currentVariantSpec + arg0.iterator.toString()); + return (currentVariantSpec + iterator.toString()).compareTo(arg0.currentVariantSpec + arg0.iterator.toString()); } -} \ No newline at end of file +} diff --git a/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/phenotype/columnmeta/CreateColumnmetaCSV.java b/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/phenotype/columnmeta/CreateColumnmetaCSV.java index c1ea0a5b0..2911afda2 100644 --- a/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/phenotype/columnmeta/CreateColumnmetaCSV.java +++ b/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/phenotype/columnmeta/CreateColumnmetaCSV.java @@ -11,16 +11,16 @@ public class CreateColumnmetaCSV { private static LoadingStore store = new LoadingStore(); - private static Logger log = LoggerFactory.getLogger(CSVLoader.class); + private static Logger log = LoggerFactory.getLogger(CSVLoader.class); - private static String HPDS_DIRECTORY = "/opt/local/hpds/"; + private static String HPDS_DIRECTORY = "/opt/local/hpds/"; - public static void main(String[] args) throws IOException { - if (args.length > 0) { - HPDS_DIRECTORY = args[0] + "/"; - } - store.allObservationsStore = new RandomAccessFile(HPDS_DIRECTORY + "allObservationsStore.javabin", "rw"); - store.dumpStatsAndColumnMeta(HPDS_DIRECTORY); - } + public static void main(String[] args) throws IOException { + if (args.length > 0) { + HPDS_DIRECTORY = args[0] + "/"; + } + store.allObservationsStore = new RandomAccessFile(HPDS_DIRECTORY + "allObservationsStore.javabin", "rw"); + store.dumpStatsAndColumnMeta(HPDS_DIRECTORY); + } } diff --git a/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/phenotype/config/CSVConfig.java b/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/phenotype/config/CSVConfig.java index cb6016d53..e79927873 100644 --- a/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/phenotype/config/CSVConfig.java +++ b/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/phenotype/config/CSVConfig.java @@ -1,9 +1,8 @@ package edu.harvard.hms.dbmi.avillach.hpds.etl.phenotype.config; /** - * Configuration class for CSV processing. - * The names are using snake_case to match the JSON keys in the config file. - * This is used to map the JSON keys to the Java fields. + * Configuration class for CSV processing. The names are using snake_case to match the JSON keys in the config file. This is used to map the + * JSON keys to the Java fields. */ public class CSVConfig { diff --git a/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/phenotype/config/ConfigLoader.java b/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/phenotype/config/ConfigLoader.java index a1c39adfd..7f46fbfac 100644 --- a/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/phenotype/config/ConfigLoader.java +++ b/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/phenotype/config/ConfigLoader.java @@ -17,16 +17,15 @@ public class ConfigLoader { private Map csvConfigMap = new HashMap<>(); /** - * Constructor for ConfigLoader. It loads the configuration from a JSON file located in the INPUT_DIR. - * The JSON file should contain a mapping of CSV file names to their respective configurations. + * Constructor for ConfigLoader. It loads the configuration from a JSON file located in the INPUT_DIR. The JSON file should contain a + * mapping of CSV file names to their respective configurations. */ public ConfigLoader() { File file = new File(INPUT_DIR); if (file.exists()) { ObjectMapper objectMapper = new ObjectMapper(); try { - this.csvConfigMap = objectMapper.readValue(new File(INPUT_DIR + "/config.json"), new TypeReference<>() { - }); + this.csvConfigMap = objectMapper.readValue(new File(INPUT_DIR + "/config.json"), new TypeReference<>() {}); // Remove the ".csv" extension from the keys in the map if they exist for (String key : csvConfigMap.keySet()) { diff --git a/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/phenotype/csv/CSVLoaderNewSearch.java b/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/phenotype/csv/CSVLoaderNewSearch.java index 59f3620de..ec4712fbf 100644 --- a/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/phenotype/csv/CSVLoaderNewSearch.java +++ b/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/phenotype/csv/CSVLoaderNewSearch.java @@ -41,7 +41,8 @@ public static void main(String[] args) throws IOException { private static void initialLoad() throws IOException { Crypto.loadDefaultKey(); Reader in = new FileReader(HPDS_DIRECTORY + "allConcepts.csv"); - Iterable records = CSVFormat.DEFAULT.withSkipHeaderRecord().withFirstRecordAsHeader().parse(new BufferedReader(in, 1024 * 1024)); + Iterable records = + CSVFormat.DEFAULT.withSkipHeaderRecord().withFirstRecordAsHeader().parse(new BufferedReader(in, 1024 * 1024)); CSVConfig csvConfig = configLoader.getConfigFor("allConcepts"); final PhenoCube[] currentConcept = new PhenoCube[1]; @@ -67,10 +68,13 @@ private static void processRecord(final PhenoCube[] currentConcept, CSVRecord re String value = isAlpha ? record.get(CSVParserUtil.TEXT_VALUE) : numericValue; currentConcept[0] = getPhenoCube(currentConcept[0], conceptPath, isAlpha); - if (value != null && !value.trim().isEmpty() && - ((isAlpha && currentConcept[0].vType == String.class) || (!isAlpha && currentConcept[0].vType == Double.class))) { + if ( + value != null && !value.trim().isEmpty() + && ((isAlpha && currentConcept[0].vType == String.class) || (!isAlpha && currentConcept[0].vType == Double.class)) + ) { value = value.trim(); - currentConcept[0].setColumnWidth(isAlpha ? Math.max(currentConcept[0].getColumnWidth(), value.getBytes().length) : Double.BYTES); + currentConcept[0] + .setColumnWidth(isAlpha ? Math.max(currentConcept[0].getColumnWidth(), value.getBytes().length) : Double.BYTES); int patientId = Integer.parseInt(record.get(CSVParserUtil.PATIENT_NUM)); Date date = null; if (record.size() > 4 && record.get(CSVParserUtil.DATETIME) != null && !record.get(CSVParserUtil.DATETIME).isEmpty()) { diff --git a/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/phenotype/litecsv/LowRAMMultiCSVLoader.java b/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/phenotype/litecsv/LowRAMMultiCSVLoader.java index c9258b6db..c1bd460bf 100644 --- a/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/phenotype/litecsv/LowRAMMultiCSVLoader.java +++ b/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/phenotype/litecsv/LowRAMMultiCSVLoader.java @@ -39,7 +39,7 @@ public static void main(String[] args) { maxChunkSize = Double.parseDouble(parts[1]); log.info("Configured to use a max chunk size of {} GB", maxChunkSize); } catch (NumberFormatException e) { - throw new IllegalArgumentException("Invalid max chunk size " + maxChunkSize); + throw new IllegalArgumentException("Invalid max chunk size " + maxChunkSize); } } } diff --git a/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/phenotype/sequential/PhenoRecord.java b/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/phenotype/sequential/PhenoRecord.java index ad89dae53..297796130 100644 --- a/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/phenotype/sequential/PhenoRecord.java +++ b/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/phenotype/sequential/PhenoRecord.java @@ -13,75 +13,84 @@ */ public class PhenoRecord { - private int patientNum; - private String conceptPath; - private String numericValue; - private String textValue; - private Date dateTime; - - public int getPatientNum() { - return patientNum; - } - public void setPatientNum(int patientNum) { - this.patientNum = patientNum; - } - public String getConceptPath() { - return conceptPath; - } - public void setConceptPath(String conceptPath) { - this.conceptPath = conceptPath; - } - public String getNumericValue() { - return numericValue; - } - public void setNumericValue(String numericValue) { - this.numericValue = numericValue; - } - public String getTextValue() { - return textValue; - } - public void setTextValue(String textValue) { - this.textValue = textValue; - } - public Date getDateTime() { - return dateTime; - } - public void setDateTime(Date dateTime) { - this.dateTime = dateTime; - } - - - //SQL uses 1-based arrays - private static final int PATIENT_NUM_COL_SQL = 1; - private static final int CONCEPT_PATH_COL_SQL = 2; - private static final int NUMERIC_VALUE_COL_SQL = 3; - private static final int TEXT_VALUE_COL_SQL = 4; - private static final int DATETIME_COL_SQL = 5; - - // SQL constructor - public PhenoRecord (ResultSet result) throws SQLException { - conceptPath = result.getString(CONCEPT_PATH_COL_SQL); - dateTime = result.getDate(DATETIME_COL_SQL); - numericValue = result.getString(NUMERIC_VALUE_COL_SQL); - patientNum = result.getInt(PATIENT_NUM_COL_SQL); - textValue = result.getString(TEXT_VALUE_COL_SQL) == null ? null : result.getString(TEXT_VALUE_COL_SQL).trim(); - } - - //CSV uses 0-based arrays - private static final int PATIENT_NUM_COL = 0; - private static final int CONCEPT_PATH_COL = 1; - private static final int NUMERIC_VALUE_COL = 2; - private static final int TEXT_VALUE_COL = 3; - private static final int DATETIME_COL = 4; - - //CSV constructor - public PhenoRecord (CSVRecord record) { - conceptPath = record.get(CONCEPT_PATH_COL); - if(record.size()>DATETIME_COL && record.get(DATETIME_COL) != null && ! record.get(DATETIME_COL).isEmpty()) { - dateTime = new Date(Long.parseLong(record.get(DATETIME_COL))); - } - numericValue = record.get(NUMERIC_VALUE_COL); - patientNum = Integer.parseInt(record.get(PATIENT_NUM_COL)); - textValue =record.get(TEXT_VALUE_COL) == null ? null : record.get(TEXT_VALUE_COL).trim(); - } + private int patientNum; + private String conceptPath; + private String numericValue; + private String textValue; + private Date dateTime; + + public int getPatientNum() { + return patientNum; + } + + public void setPatientNum(int patientNum) { + this.patientNum = patientNum; + } + + public String getConceptPath() { + return conceptPath; + } + + public void setConceptPath(String conceptPath) { + this.conceptPath = conceptPath; + } + + public String getNumericValue() { + return numericValue; + } + + public void setNumericValue(String numericValue) { + this.numericValue = numericValue; + } + + public String getTextValue() { + return textValue; + } + + public void setTextValue(String textValue) { + this.textValue = textValue; + } + + public Date getDateTime() { + return dateTime; + } + + public void setDateTime(Date dateTime) { + this.dateTime = dateTime; + } + + + // SQL uses 1-based arrays + private static final int PATIENT_NUM_COL_SQL = 1; + private static final int CONCEPT_PATH_COL_SQL = 2; + private static final int NUMERIC_VALUE_COL_SQL = 3; + private static final int TEXT_VALUE_COL_SQL = 4; + private static final int DATETIME_COL_SQL = 5; + + // SQL constructor + public PhenoRecord(ResultSet result) throws SQLException { + conceptPath = result.getString(CONCEPT_PATH_COL_SQL); + dateTime = result.getDate(DATETIME_COL_SQL); + numericValue = result.getString(NUMERIC_VALUE_COL_SQL); + patientNum = result.getInt(PATIENT_NUM_COL_SQL); + textValue = result.getString(TEXT_VALUE_COL_SQL) == null ? null : result.getString(TEXT_VALUE_COL_SQL).trim(); + } + + // CSV uses 0-based arrays + private static final int PATIENT_NUM_COL = 0; + private static final int CONCEPT_PATH_COL = 1; + private static final int NUMERIC_VALUE_COL = 2; + private static final int TEXT_VALUE_COL = 3; + private static final int DATETIME_COL = 4; + + // CSV constructor + public PhenoRecord(CSVRecord record) { + conceptPath = record.get(CONCEPT_PATH_COL); + if (record.size() > DATETIME_COL && record.get(DATETIME_COL) != null && !record.get(DATETIME_COL).isEmpty()) { + dateTime = new Date(Long.parseLong(record.get(DATETIME_COL))); + } + numericValue = record.get(NUMERIC_VALUE_COL); + patientNum = Integer.parseInt(record.get(PATIENT_NUM_COL)); + textValue = record.get(TEXT_VALUE_COL) == null ? null : record.get(TEXT_VALUE_COL).trim(); + } } diff --git a/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/phenotype/sequential/SequentialLoadingStore.java b/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/phenotype/sequential/SequentialLoadingStore.java index 370c6a782..9ecb38ee1 100644 --- a/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/phenotype/sequential/SequentialLoadingStore.java +++ b/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/phenotype/sequential/SequentialLoadingStore.java @@ -15,214 +15,223 @@ import edu.harvard.hms.dbmi.avillach.hpds.data.phenotype.ColumnMeta; import edu.harvard.hms.dbmi.avillach.hpds.data.phenotype.KeyAndValue; import edu.harvard.hms.dbmi.avillach.hpds.data.phenotype.PhenoCube; - + /** * This class provides similar functioanlity to the LoadingStore class, but is designed with sequential loading in mind. * - * This will write out partial cubes to individual files instead of assuming that they can immediately be - * sequenced into the allObservationStore; This will allow us to pick them back up and add more patients - * to existing stores so that we can better handle fragmented data. + * This will write out partial cubes to individual files instead of assuming that they can immediately be sequenced into the + * allObservationStore; This will allow us to pick them back up and add more patients to existing stores so that we can better handle + * fragmented data. * * * @author nchu * */ public class SequentialLoadingStore { - - private static final String COLUMNMETA_FILENAME = "/opt/local/hpds/columnMeta.javabin"; - protected static final String OBSERVATIONS_FILENAME = "/opt/local/hpds/allObservationsStore.javabin"; - protected static final String OBS_TEMP_FILENAME = "/opt/local/hpds/allObservationsTemp.javabin"; - - public RandomAccessFile allObservationsStore; - public RandomAccessFile allObservationsTemp; - - TreeMap metadataMap = new TreeMap<>(); - - private static Logger log = LoggerFactory.getLogger(SequentialLoadingStore.class); - - public SequentialLoadingStore() { - try { - allObservationsTemp = new RandomAccessFile(OBS_TEMP_FILENAME, "rw"); - } catch (FileNotFoundException e) { - throw new UncheckedIOException(e); - } - } - - public LoadingCache loadingCache = CacheBuilder.newBuilder() - .maximumSize(16) - .removalListener(new RemovalListener() { - - @Override - public void onRemoval(RemovalNotification cubeRemoval) { - if(cubeRemoval.getValue().getLoadingMap()!=null) { - try(ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); - ObjectOutputStream out = new ObjectOutputStream(byteStream);) { - ColumnMeta columnMeta = new ColumnMeta().setName(cubeRemoval.getKey()).setWidthInBytes(cubeRemoval.getValue().getColumnWidth()).setCategorical(cubeRemoval.getValue().isStringType()); - columnMeta.setAllObservationsOffset(allObservationsTemp.getFilePointer()); - //write out the basic key/value map for loading; this will be compacted and finalized after all concepts are read in. - out.writeObject(cubeRemoval.getValue().getLoadingMap()); out.flush(); - - allObservationsTemp.write(byteStream.toByteArray()); - columnMeta.setAllObservationsLength(allObservationsTemp.getFilePointer()); - metadataMap.put(columnMeta.getName(), columnMeta); - } catch (IOException e1) { - throw new UncheckedIOException(e1); - } - } - } - }) - .build( - new CacheLoader() { - public PhenoCube load(String key) throws Exception { - ColumnMeta columnMeta = metadataMap.get(key); - if(columnMeta != null) { - log.debug("Loading concept : [" + key + "]"); - return getCubeFromTemp(columnMeta); - }else { - return null; - } - } - }); - - public TreeSet allIds = new TreeSet(); - - public void saveStore() throws FileNotFoundException, IOException, ClassNotFoundException { - log.info("flushing temp storage"); - loadingCache.invalidateAll(); - loadingCache.cleanUp(); - - allObservationsStore = new RandomAccessFile(OBSERVATIONS_FILENAME, "rw"); - //we dumped it all in a temp file; now sort all the data and compress it into the real Store - for(String concept : metadataMap.keySet()) { - ColumnMeta columnMeta = metadataMap.get(concept); - log.debug("Writing concept : [" + concept + "]"); - PhenoCube cube = getCubeFromTemp(columnMeta); - complete(columnMeta, cube); - write(columnMeta, cube); - } - allObservationsStore.close(); - - log.info("Writing metadata"); - ObjectOutputStream metaOut = new ObjectOutputStream(new GZIPOutputStream(new FileOutputStream(new File(COLUMNMETA_FILENAME)))); - metaOut.writeObject(metadataMap); - metaOut.writeObject(allIds); - metaOut.flush(); - metaOut.close(); - - log.info("Cleaning up temporary file"); - - allObservationsTemp.close(); - File tempFile = new File(OBS_TEMP_FILENAME); - tempFile.delete(); - } - - /** - * calculate the min/max, categories, observation count, etc for this cube and write it to disk - * @param columnMeta - * @param cube - * @throws IOException - */ - private void write(ColumnMeta columnMeta, PhenoCube cube) throws IOException { - columnMeta.setAllObservationsOffset(allObservationsStore.getFilePointer()); - - try(ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); - ObjectOutputStream out = new ObjectOutputStream(byteStream);) { - - out.writeObject(cube); out.flush(); - allObservationsStore.write(Crypto.encryptData(byteStream.toByteArray())); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - columnMeta.setAllObservationsLength(allObservationsStore.getFilePointer()); - } - - private PhenoCube getCubeFromTemp(ColumnMeta columnMeta) throws IOException, ClassNotFoundException { - allObservationsTemp.seek(columnMeta.getAllObservationsOffset()); - int length = (int) (columnMeta.getAllObservationsLength() - columnMeta.getAllObservationsOffset()); - byte[] buffer = new byte[length]; - allObservationsTemp.read(buffer); - allObservationsTemp.seek(allObservationsTemp.length()); - ObjectInputStream inStream = new ObjectInputStream(new ByteArrayInputStream(buffer)); - - PhenoCube cube = new PhenoCube(columnMeta.getName() , columnMeta.isCategorical() ? String.class : Double.class); - cube.setLoadingMap((List)inStream.readObject()); - cube.setColumnWidth(columnMeta.getWidthInBytes()); - inStream.close(); - return cube; - } - - private > void complete(ColumnMeta columnMeta, PhenoCube cube) { - ArrayList> entryList = new ArrayList>( - cube.getLoadingMap().stream().map((entry)->{ - return new KeyAndValue(entry.getKey(), entry.getValue(), entry.getTimestamp()); - }).collect(Collectors.toList())); - - List> sortedByKey = entryList.stream() - .sorted(Comparator.comparing(KeyAndValue::getKey)) - .collect(Collectors.toList()); - cube.setSortedByKey(sortedByKey.toArray(new KeyAndValue[0])); - - if(cube.isStringType()) { - TreeMap> categoryMap = new TreeMap<>(); - for(KeyAndValue entry : cube.sortedByValue()) { - if(!categoryMap.containsKey(entry.getValue())) { - categoryMap.put(entry.getValue(), new LinkedList()); - } - categoryMap.get(entry.getValue()).add(entry.getKey()); - } - TreeMap> categorySetMap = new TreeMap<>(); - categoryMap.entrySet().stream().forEach((entry)->{ - categorySetMap.put(entry.getKey(), new TreeSet(entry.getValue())); - }); - cube.setCategoryMap(categorySetMap); - } - - columnMeta.setObservationCount(cube.sortedByKey().length); - columnMeta.setPatientCount(Arrays.stream(cube.sortedByKey()).map((kv)->{return kv.getKey();}).collect(Collectors.toSet()).size()); - if(columnMeta.isCategorical()) { - columnMeta.setCategoryValues(new ArrayList(new TreeSet((List)cube.keyBasedArray()))); - } else { - List map = (List) cube.keyBasedArray().stream().map((value)->{return (Double) value;}).collect(Collectors.toList()); - double min = Double.MAX_VALUE; - double max = Double.MIN_VALUE; - for(double f : map) { - min = Double.min(min, f); - max = Double.max(max, f); - } - columnMeta.setMin(min); - columnMeta.setMax(max); - } - - } - - public void dumpStats() { - log.info("Dumping Stats"); - try (ObjectInputStream objectInputStream = new ObjectInputStream(new GZIPInputStream(new FileInputStream(COLUMNMETA_FILENAME)));){ - TreeMap metastore = (TreeMap) objectInputStream.readObject(); - Set allIds = (TreeSet) objectInputStream.readObject(); - - long totalNumberOfObservations = 0; - - log.info("\n\nConceptPath\tObservationCount\tMinNumValue\tMaxNumValue\tCategoryValues"); - for(String key : metastore.keySet()) { - ColumnMeta columnMeta = metastore.get(key); - log.info(String.join("\t", key.toString(), columnMeta.getObservationCount()+"", - columnMeta.getMin()==null ? "NaN" : columnMeta.getMin().toString(), - columnMeta.getMax()==null ? "NaN" : columnMeta.getMax().toString(), - columnMeta.getCategoryValues() == null ? "NUMERIC CONCEPT" : String.join(",", - columnMeta.getCategoryValues() - .stream().map((value)->{return value==null ? "NULL_VALUE" : "\""+value+"\"";}).collect(Collectors.toList())))); - totalNumberOfObservations += columnMeta.getObservationCount(); - } - - log.info("Total Number of Concepts : " + metastore.size()); - log.info("Total Number of Patients : " + allIds.size()); - log.info("Total Number of Observations : " + totalNumberOfObservations); - - } catch (IOException | ClassNotFoundException e) { - throw new RuntimeException("Could not load metastore"); - } - } + + private static final String COLUMNMETA_FILENAME = "/opt/local/hpds/columnMeta.javabin"; + protected static final String OBSERVATIONS_FILENAME = "/opt/local/hpds/allObservationsStore.javabin"; + protected static final String OBS_TEMP_FILENAME = "/opt/local/hpds/allObservationsTemp.javabin"; + + public RandomAccessFile allObservationsStore; + public RandomAccessFile allObservationsTemp; + + TreeMap metadataMap = new TreeMap<>(); + + private static Logger log = LoggerFactory.getLogger(SequentialLoadingStore.class); + + public SequentialLoadingStore() { + try { + allObservationsTemp = new RandomAccessFile(OBS_TEMP_FILENAME, "rw"); + } catch (FileNotFoundException e) { + throw new UncheckedIOException(e); + } + } + + public LoadingCache loadingCache = + CacheBuilder.newBuilder().maximumSize(16).removalListener(new RemovalListener() { + + @Override + public void onRemoval(RemovalNotification cubeRemoval) { + if (cubeRemoval.getValue().getLoadingMap() != null) { + try ( + ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); ObjectOutputStream out = + new ObjectOutputStream(byteStream); + ) { + ColumnMeta columnMeta = new ColumnMeta().setName(cubeRemoval.getKey()) + .setWidthInBytes(cubeRemoval.getValue().getColumnWidth()).setCategorical(cubeRemoval.getValue().isStringType()); + columnMeta.setAllObservationsOffset(allObservationsTemp.getFilePointer()); + // write out the basic key/value map for loading; this will be compacted and finalized after all concepts are read + // in. + out.writeObject(cubeRemoval.getValue().getLoadingMap()); + out.flush(); + + allObservationsTemp.write(byteStream.toByteArray()); + columnMeta.setAllObservationsLength(allObservationsTemp.getFilePointer()); + metadataMap.put(columnMeta.getName(), columnMeta); + } catch (IOException e1) { + throw new UncheckedIOException(e1); + } + } + } + }).build(new CacheLoader() { + public PhenoCube load(String key) throws Exception { + ColumnMeta columnMeta = metadataMap.get(key); + if (columnMeta != null) { + log.debug("Loading concept : [" + key + "]"); + return getCubeFromTemp(columnMeta); + } else { + return null; + } + } + }); + + public TreeSet allIds = new TreeSet(); + + public void saveStore() throws FileNotFoundException, IOException, ClassNotFoundException { + log.info("flushing temp storage"); + loadingCache.invalidateAll(); + loadingCache.cleanUp(); + + allObservationsStore = new RandomAccessFile(OBSERVATIONS_FILENAME, "rw"); + // we dumped it all in a temp file; now sort all the data and compress it into the real Store + for (String concept : metadataMap.keySet()) { + ColumnMeta columnMeta = metadataMap.get(concept); + log.debug("Writing concept : [" + concept + "]"); + PhenoCube cube = getCubeFromTemp(columnMeta); + complete(columnMeta, cube); + write(columnMeta, cube); + } + allObservationsStore.close(); + + log.info("Writing metadata"); + ObjectOutputStream metaOut = new ObjectOutputStream(new GZIPOutputStream(new FileOutputStream(new File(COLUMNMETA_FILENAME)))); + metaOut.writeObject(metadataMap); + metaOut.writeObject(allIds); + metaOut.flush(); + metaOut.close(); + + log.info("Cleaning up temporary file"); + + allObservationsTemp.close(); + File tempFile = new File(OBS_TEMP_FILENAME); + tempFile.delete(); + } + + /** + * calculate the min/max, categories, observation count, etc for this cube and write it to disk + * @param columnMeta + * @param cube + * @throws IOException + */ + private void write(ColumnMeta columnMeta, PhenoCube cube) throws IOException { + columnMeta.setAllObservationsOffset(allObservationsStore.getFilePointer()); + + try (ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); ObjectOutputStream out = new ObjectOutputStream(byteStream);) { + + out.writeObject(cube); + out.flush(); + allObservationsStore.write(Crypto.encryptData(byteStream.toByteArray())); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + columnMeta.setAllObservationsLength(allObservationsStore.getFilePointer()); + } + + private PhenoCube getCubeFromTemp(ColumnMeta columnMeta) throws IOException, ClassNotFoundException { + allObservationsTemp.seek(columnMeta.getAllObservationsOffset()); + int length = (int) (columnMeta.getAllObservationsLength() - columnMeta.getAllObservationsOffset()); + byte[] buffer = new byte[length]; + allObservationsTemp.read(buffer); + allObservationsTemp.seek(allObservationsTemp.length()); + ObjectInputStream inStream = new ObjectInputStream(new ByteArrayInputStream(buffer)); + + PhenoCube cube = new PhenoCube(columnMeta.getName(), columnMeta.isCategorical() ? String.class : Double.class); + cube.setLoadingMap((List) inStream.readObject()); + cube.setColumnWidth(columnMeta.getWidthInBytes()); + inStream.close(); + return cube; + } + + private > void complete(ColumnMeta columnMeta, PhenoCube cube) { + ArrayList> entryList = new ArrayList>(cube.getLoadingMap().stream().map((entry) -> { + return new KeyAndValue(entry.getKey(), entry.getValue(), entry.getTimestamp()); + }).collect(Collectors.toList())); + + List> sortedByKey = + entryList.stream().sorted(Comparator.comparing(KeyAndValue::getKey)).collect(Collectors.toList()); + cube.setSortedByKey(sortedByKey.toArray(new KeyAndValue[0])); + + if (cube.isStringType()) { + TreeMap> categoryMap = new TreeMap<>(); + for (KeyAndValue entry : cube.sortedByValue()) { + if (!categoryMap.containsKey(entry.getValue())) { + categoryMap.put(entry.getValue(), new LinkedList()); + } + categoryMap.get(entry.getValue()).add(entry.getKey()); + } + TreeMap> categorySetMap = new TreeMap<>(); + categoryMap.entrySet().stream().forEach((entry) -> { + categorySetMap.put(entry.getKey(), new TreeSet(entry.getValue())); + }); + cube.setCategoryMap(categorySetMap); + } + + columnMeta.setObservationCount(cube.sortedByKey().length); + columnMeta.setPatientCount(Arrays.stream(cube.sortedByKey()).map((kv) -> { + return kv.getKey(); + }).collect(Collectors.toSet()).size()); + if (columnMeta.isCategorical()) { + columnMeta.setCategoryValues(new ArrayList(new TreeSet((List) cube.keyBasedArray()))); + } else { + List map = (List) cube.keyBasedArray().stream().map((value) -> { + return (Double) value; + }).collect(Collectors.toList()); + double min = Double.MAX_VALUE; + double max = Double.MIN_VALUE; + for (double f : map) { + min = Double.min(min, f); + max = Double.max(max, f); + } + columnMeta.setMin(min); + columnMeta.setMax(max); + } + + } + + public void dumpStats() { + log.info("Dumping Stats"); + try (ObjectInputStream objectInputStream = new ObjectInputStream(new GZIPInputStream(new FileInputStream(COLUMNMETA_FILENAME)));) { + TreeMap metastore = (TreeMap) objectInputStream.readObject(); + Set allIds = (TreeSet) objectInputStream.readObject(); + + long totalNumberOfObservations = 0; + + log.info("\n\nConceptPath\tObservationCount\tMinNumValue\tMaxNumValue\tCategoryValues"); + for (String key : metastore.keySet()) { + ColumnMeta columnMeta = metastore.get(key); + log.info( + String.join( + "\t", key.toString(), columnMeta.getObservationCount() + "", + columnMeta.getMin() == null ? "NaN" : columnMeta.getMin().toString(), + columnMeta.getMax() == null ? "NaN" : columnMeta.getMax().toString(), + columnMeta.getCategoryValues() == null ? "NUMERIC CONCEPT" + : String.join(",", columnMeta.getCategoryValues().stream().map((value) -> { + return value == null ? "NULL_VALUE" : "\"" + value + "\""; + }).collect(Collectors.toList())) + ) + ); + totalNumberOfObservations += columnMeta.getObservationCount(); + } + + log.info("Total Number of Concepts : " + metastore.size()); + log.info("Total Number of Patients : " + allIds.size()); + log.info("Total Number of Observations : " + totalNumberOfObservations); + + } catch (IOException | ClassNotFoundException e) { + throw new RuntimeException("Could not load metastore"); + } + } } diff --git a/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/phenotype/sql/SQLLoader.java b/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/phenotype/sql/SQLLoader.java index 594334700..6def83fa4 100644 --- a/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/phenotype/sql/SQLLoader.java +++ b/services/pic-sure-hpds/etl/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/etl/phenotype/sql/SQLLoader.java @@ -23,207 +23,216 @@ public class SQLLoader { - private static final Logger log = LoggerFactory.getLogger(SQLLoader.class); - - private static final SimpleDateFormat ORACLE_DATE_FORMAT = new SimpleDateFormat("dd-MMM-yy"); - - static JdbcTemplate template; - - private static LoadingStore store = new LoadingStore(); - - private static final int PATIENT_NUM = 1; - - private static final int CONCEPT_PATH = 2; - - private static final int NUMERIC_VALUE = 3; - - private static final int TEXT_VALUE = 4; - - private static final int DATETIME = 5; - - private static final Properties props = new Properties(); - - private static String HPDS_DIRECTORY = "/opt/local/hpds/"; - - public static void main(String[] args) throws IOException { - props.load(new FileInputStream(HPDS_DIRECTORY + "sql.properties")); - template = new JdbcTemplate(new DriverManagerDataSource(prop("datasource.url"), prop("datasource.user"), prop("datasource.password"))); - store.allObservationsStore = new RandomAccessFile(HPDS_DIRECTORY + "allObservationsStore.javabin", "rw"); - Crypto.loadDefaultKey(); - initialLoad(); - store.saveStore(HPDS_DIRECTORY); - store.dumpStats(); - } - - private static String prop(String key) { - return props.getProperty(key); - } - - private static void initialLoad() throws IOException { - final PhenoCube[] currentConcept = new PhenoCube[1]; - String loadQuery = IOUtils.toString(new FileInputStream("/opt/local/hpds/loadQuery.sql"), Charset.forName("UTF-8")); -// -// ArrayBlockingQueue abq = new ArrayBlockingQueue<>(1000); -// -// ExecutorService chunkWriteEx = Executors.newFixedThreadPool(32); -// -// boolean[] stillProcessingRecords = {true}; -// -// for(int x = 0;x<32;x++) { -// chunkWriteEx.submit(new Runnable() { -// PhenoCube currentConcept = null; -// String[] arg0; -// @Override -// public void run() { -// while(stillProcessingRecords[0]) { -// try { -// arg0 = abq.poll(1, TimeUnit.SECONDS); -// if(arg0 != null) { -// String conceptPathFromRow = arg0[CONCEPT_PATH]; -// String[] segments = conceptPathFromRow.split("\\\\"); -// for(int x = 0;x abq = new ArrayBlockingQueue<>(1000); + // + // ExecutorService chunkWriteEx = Executors.newFixedThreadPool(32); + // + // boolean[] stillProcessingRecords = {true}; + // + // for(int x = 0;x<32;x++) { + // chunkWriteEx.submit(new Runnable() { + // PhenoCube currentConcept = null; + // String[] arg0; + // @Override + // public void run() { + // while(stillProcessingRecords[0]) { + // try { + // arg0 = abq.poll(1, TimeUnit.SECONDS); + // if(arg0 != null) { + // String conceptPathFromRow = arg0[CONCEPT_PATH]; + // String[] segments = conceptPathFromRow.split("\\\\"); + // for(int x = 0;x expectedMetas = - List.of(new CategoricalMeta("\\TEST_DATASET_NAME\\foo\\1\\", 2, 2, List.of("lav", "val")), new NumericMeta("\\TEST_DATASET_NAME\\foo\\2\\", 2, 2, 0.0, 99.0)); + List expectedMetas = List.of( + new CategoricalMeta("\\TEST_DATASET_NAME\\foo\\1\\", 2, 2, List.of("lav", "val")), + new NumericMeta("\\TEST_DATASET_NAME\\foo\\2\\", 2, 2, 0.0, 99.0) + ); verifyStoreMeta(expectedMetas, testDir.getAbsolutePath() + "/columnMeta.javabin"); } diff --git a/services/pic-sure-hpds/etl/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/etl/phenotype/litecsv/LowRAMMultiCSVLoaderTest.java b/services/pic-sure-hpds/etl/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/etl/phenotype/litecsv/LowRAMMultiCSVLoaderTest.java index 7568a2a92..d4a003ca8 100644 --- a/services/pic-sure-hpds/etl/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/etl/phenotype/litecsv/LowRAMMultiCSVLoaderTest.java +++ b/services/pic-sure-hpds/etl/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/etl/phenotype/litecsv/LowRAMMultiCSVLoaderTest.java @@ -16,7 +16,7 @@ class LowRAMMultiCSVLoaderTest { void shouldFilterOutNonCSVs(@TempDir File testDir) throws IOException { Path csvPath = Path.of(testDir.getAbsolutePath(), "test.txt"); RandomAccessFile largeFile = new RandomAccessFile(csvPath.toString(), "rw"); - largeFile.setLength(6L*1024); + largeFile.setLength(6L * 1024); LowRAMCSVProcessor processor = Mockito.mock(LowRAMCSVProcessor.class); LowRAMLoadingStore store = Mockito.mock(LowRAMLoadingStore.class); @@ -32,11 +32,10 @@ void shouldFilterOutNonCSVs(@TempDir File testDir) throws IOException { void shouldProcessSmallCSVs(@TempDir File testDir) throws IOException { Path csvPath = Path.of(testDir.getAbsolutePath(), "test.csv"); RandomAccessFile largeFile = new RandomAccessFile(csvPath.toString(), "rw"); - largeFile.setLength(6L*1024); + largeFile.setLength(6L * 1024); LowRAMCSVProcessor processor = Mockito.mock(LowRAMCSVProcessor.class); - Mockito.when(processor.process(Mockito.any())) - .thenReturn(new IngestStatus(csvPath, 10, 10, 10L)); + Mockito.when(processor.process(Mockito.any())).thenReturn(new IngestStatus(csvPath, 10, 10, 10L)); LowRAMLoadingStore store = Mockito.mock(LowRAMLoadingStore.class); LowRAMMultiCSVLoader subject = new LowRAMMultiCSVLoader(store, processor, testDir.getAbsolutePath()); @@ -50,11 +49,10 @@ void shouldProcessSmallCSVs(@TempDir File testDir) throws IOException { void shouldProcessSmallCSVs_withReducedChunkSize(@TempDir File testDir) throws IOException { Path csvPath = Path.of(testDir.getAbsolutePath(), "test.csv"); RandomAccessFile largeFile = new RandomAccessFile(csvPath.toString(), "rw"); - largeFile.setLength(6L*1024); + largeFile.setLength(6L * 1024); LowRAMCSVProcessor processor = Mockito.mock(LowRAMCSVProcessor.class); - Mockito.when(processor.process(Mockito.any())) - .thenReturn(new IngestStatus(csvPath, 10, 10, 10L)); + Mockito.when(processor.process(Mockito.any())).thenReturn(new IngestStatus(csvPath, 10, 10, 10L)); LowRAMLoadingStore store = Mockito.mock(LowRAMLoadingStore.class); LowRAMMultiCSVLoader subject = new LowRAMMultiCSVLoader(store, processor, testDir.getAbsolutePath()); @@ -63,4 +61,4 @@ void shouldProcessSmallCSVs_withReducedChunkSize(@TempDir File testDir) throws I Assertions.assertEquals(0, actual); Mockito.verify(processor, Mockito.times(1)).process(Mockito.any()); } -} \ No newline at end of file +} diff --git a/services/pic-sure-hpds/genomic-processor/pom.xml b/services/pic-sure-hpds/genomic-processor/pom.xml index 691dfba74..1291045e5 100644 --- a/services/pic-sure-hpds/genomic-processor/pom.xml +++ b/services/pic-sure-hpds/genomic-processor/pom.xml @@ -5,7 +5,7 @@ pic-sure-hpds edu.harvard.hms.dbmi.avillach.hpds - 3.0.0-SNAPSHOT + ${revision} 4.0.0 diff --git a/services/pic-sure-hpds/genomic-processor/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/genomic/HpdsGenomicProcessorApplication.java b/services/pic-sure-hpds/genomic-processor/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/genomic/HpdsGenomicProcessorApplication.java index 2bfa1a0b0..18c06cc20 100644 --- a/services/pic-sure-hpds/genomic-processor/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/genomic/HpdsGenomicProcessorApplication.java +++ b/services/pic-sure-hpds/genomic-processor/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/genomic/HpdsGenomicProcessorApplication.java @@ -6,8 +6,8 @@ @SpringBootApplication public class HpdsGenomicProcessorApplication { - public static void main(String[] args) { - SpringApplication.run(HpdsGenomicProcessorApplication.class, args); - } + public static void main(String[] args) { + SpringApplication.run(HpdsGenomicProcessorApplication.class, args); + } } diff --git a/services/pic-sure-hpds/genomic-processor/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/genomic/ServletInitializer.java b/services/pic-sure-hpds/genomic-processor/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/genomic/ServletInitializer.java index 83040e11e..09c14b2c7 100644 --- a/services/pic-sure-hpds/genomic-processor/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/genomic/ServletInitializer.java +++ b/services/pic-sure-hpds/genomic-processor/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/genomic/ServletInitializer.java @@ -5,9 +5,9 @@ public class ServletInitializer extends SpringBootServletInitializer { - @Override - protected SpringApplicationBuilder configure(SpringApplicationBuilder application) { - return application.sources(HpdsGenomicProcessorApplication.class); - } + @Override + protected SpringApplicationBuilder configure(SpringApplicationBuilder application) { + return application.sources(HpdsGenomicProcessorApplication.class); + } } diff --git a/services/pic-sure-hpds/genomic-processor/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/genomic/config/ApplicationConfig.java b/services/pic-sure-hpds/genomic-processor/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/genomic/config/ApplicationConfig.java index 740cd9d6a..129f12117 100644 --- a/services/pic-sure-hpds/genomic-processor/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/genomic/config/ApplicationConfig.java +++ b/services/pic-sure-hpds/genomic-processor/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/genomic/config/ApplicationConfig.java @@ -11,7 +11,7 @@ @PropertySource("classpath:application.properties") public class ApplicationConfig { - @Value( "${hpds.genomicDataDirectory}" ) + @Value("${hpds.genomicDataDirectory}") private String genomicDataDirectory; @Bean diff --git a/services/pic-sure-hpds/genomic-processor/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/genomic/HpdsGenomicProcessorApplicationTests.java b/services/pic-sure-hpds/genomic-processor/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/genomic/HpdsGenomicProcessorApplicationTests.java index 8b9ec249c..f4c1e2015 100644 --- a/services/pic-sure-hpds/genomic-processor/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/genomic/HpdsGenomicProcessorApplicationTests.java +++ b/services/pic-sure-hpds/genomic-processor/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/genomic/HpdsGenomicProcessorApplicationTests.java @@ -8,8 +8,7 @@ @SpringBootTest class HpdsGenomicProcessorApplicationTests { - @Test - void contextLoads() { - } + @Test + void contextLoads() {} } diff --git a/services/pic-sure-hpds/pom.xml b/services/pic-sure-hpds/pom.xml index 1626ecce7..073fd3ce9 100644 --- a/services/pic-sure-hpds/pom.xml +++ b/services/pic-sure-hpds/pom.xml @@ -4,14 +4,13 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - org.springframework.boot - spring-boot-starter-parent - 3.5.3 - + edu.harvard.hms.dbmi.avillach + pic-sure-api + ${revision} + ../../pom.xml edu.harvard.hms.dbmi.avillach.hpds pic-sure-hpds - 3.0.0-SNAPSHOT pom pic-sure-hpds @@ -145,6 +144,18 @@ + + + commons-logging + commons-logging + 1.2 + + + org.apache.commons + commons-compress + 1.26.0 + org.springframework.boot spring-boot-starter-tomcat @@ -155,11 +166,6 @@ common ${project.version} - - edu.harvard.hms.dbmi.avillach.hpds - pic-sure-hpds-model - 1.0.0-SNAPSHOT - edu.harvard.hms.dbmi.avillach.hpds service @@ -180,11 +186,6 @@ data ${project.version} - - edu.harvard.hms.dbmi.avillach - pic-sure-api-model - 1.0.0-SNAPSHOT - com.google.guava guava @@ -230,27 +231,11 @@ activation 1.1.1 - - org.mockito - mockito-core - 5.18.0 - test - com.oracle.database.jdbc ojdbc10 19.17.0.0 - - io.projectreactor.netty - reactor-netty - 1.2.8 - - - com.fasterxml.jackson.core - jackson-databind - 2.15.3 - org.apache.avro avro @@ -278,11 +263,6 @@ swagger-annotations 2.2.40 - - edu.harvard.dbmi.avillach - pic-sure-logging-client - 1.0.0 - diff --git a/services/pic-sure-hpds/processing/pom.xml b/services/pic-sure-hpds/processing/pom.xml index 3abb2234a..6d5fc1344 100644 --- a/services/pic-sure-hpds/processing/pom.xml +++ b/services/pic-sure-hpds/processing/pom.xml @@ -5,7 +5,7 @@ pic-sure-hpds edu.harvard.hms.dbmi.avillach.hpds - 3.0.0-SNAPSHOT + ${revision} processing diff --git a/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/ColumnSorter.java b/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/ColumnSorter.java index 03b3cb73a..17cba67ff 100644 --- a/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/ColumnSorter.java +++ b/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/ColumnSorter.java @@ -30,12 +30,10 @@ public List sortInfoColumns(Set columns) { if (infoColumnsOrder.isEmpty()) { return new ArrayList<>(columns); } - return columns.stream() - .filter(infoColumnsOrder::containsKey) - .sorted((a, b) -> Integer.compare( - infoColumnsOrder.getOrDefault(a, Integer.MAX_VALUE), - infoColumnsOrder.getOrDefault(b, Integer.MAX_VALUE) - )) - .collect(Collectors.toList()); + return columns.stream().filter(infoColumnsOrder::containsKey) + .sorted( + (a, b) -> Integer + .compare(infoColumnsOrder.getOrDefault(a, Integer.MAX_VALUE), infoColumnsOrder.getOrDefault(b, Integer.MAX_VALUE)) + ).collect(Collectors.toList()); } } diff --git a/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/DenseVariantIndex.java b/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/DenseVariantIndex.java index bb3130a81..8ec68832d 100644 --- a/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/DenseVariantIndex.java +++ b/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/DenseVariantIndex.java @@ -57,8 +57,7 @@ public VariantIndex intersection(VariantIndex variantIndex) { public Set mapToVariantSpec(String[] variantIndex) { ConcurrentHashMap setMap = new ConcurrentHashMap<>(variantIndexMask.length / 10); for (int i = 0; i < variantIndexMask.length; i++) { - if (variantIndexMask[i]) - setMap.put(variantIndex[i], ""); + if (variantIndexMask[i]) setMap.put(variantIndex[i], ""); } return setMap.keySet(); } diff --git a/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/DistributableQuery.java b/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/DistributableQuery.java index 8b0a6e73c..9dba0adcf 100644 --- a/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/DistributableQuery.java +++ b/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/DistributableQuery.java @@ -19,6 +19,7 @@ public class DistributableQuery { public void addRequiredVariantField(String path) { requiredFields.add(path); } + public Set getRequiredFields() { return requiredFields; } @@ -26,16 +27,20 @@ public Set getRequiredFields() { public void addVariantSpecCategoryFilter(String key, String[] values) { categoryFilters.put(key, values); } + public Map getCategoryFilters() { return categoryFilters; } public DistributableQuery setVariantInfoFilters(Collection variantInfoFilters) { this.variantInfoFilters = variantInfoFilters.stream() - .filter(variantInfoFilter -> !variantInfoFilter.categoryVariantInfoFilters.isEmpty() || !variantInfoFilter.numericVariantInfoFilters.isEmpty()) - .collect(Collectors.toList()); + .filter( + variantInfoFilter -> !variantInfoFilter.categoryVariantInfoFilters.isEmpty() + || !variantInfoFilter.numericVariantInfoFilters.isEmpty() + ).collect(Collectors.toList()); return this; } + public List getVariantInfoFilters() { return new ArrayList<>(variantInfoFilters); } diff --git a/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/GenomicProcessorConfig.java b/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/GenomicProcessorConfig.java index 0a1915361..1d6d422c1 100644 --- a/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/GenomicProcessorConfig.java +++ b/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/GenomicProcessorConfig.java @@ -44,7 +44,9 @@ public GenomicProcessor localPatientDistributedGenomicProcessor() { // assumed for now that all first level directories contain a genomic dataset for a group of studies File[] directories = new File(hpdsGenomicDataDirectory).listFiles(File::isDirectory); if (directories.length > 10) { - throw new IllegalArgumentException("Number of genomic partitions by studies exceeds maximum of 10 (" + directories.length + ")"); + throw new IllegalArgumentException( + "Number of genomic partitions by studies exceeds maximum of 10 (" + directories.length + ")" + ); } List studyGroupedGenomicProcessors = new ArrayList<>(); @@ -60,15 +62,14 @@ public GenomicProcessor localPatientDistributedGenomicProcessor() { private static List getGenomicProcessors(File directory) { File[] secondLevelDirectories = directory.listFiles(File::isDirectory); if (secondLevelDirectories.length > 50) { - throw new IllegalArgumentException("Number of chromosome partitions exceeds maximum of 50 (" + secondLevelDirectories.length + ")"); + throw new IllegalArgumentException( + "Number of chromosome partitions exceeds maximum of 50 (" + secondLevelDirectories.length + ")" + ); } - return Flux.fromArray(secondLevelDirectories) - .flatMap(i -> Mono.fromCallable(() -> { - return (GenomicProcessor) new GenomicProcessorNodeImpl(i.getAbsolutePath() + "/"); - }).subscribeOn(Schedulers.boundedElastic())) - .collectList() - .block(); + return Flux.fromArray(secondLevelDirectories).flatMap(i -> Mono.fromCallable(() -> { + return (GenomicProcessor) new GenomicProcessorNodeImpl(i.getAbsolutePath() + "/"); + }).subscribeOn(Schedulers.boundedElastic())).collectList().block(); } @Bean(name = "localPatientOnlyDistributedGenomicProcessor") @@ -77,7 +78,9 @@ public GenomicProcessor localPatientOnlyDistributedGenomicProcessor() { // assumed for now that all first level directories contain a genomic dataset for a group of studies File[] directories = new File(hpdsGenomicDataDirectory).listFiles(File::isDirectory); if (directories.length > 10) { - throw new IllegalArgumentException("Number of genomic partitions by studies exceeds maximum of 10 (" + directories.length + ")"); + throw new IllegalArgumentException( + "Number of genomic partitions by studies exceeds maximum of 10 (" + directories.length + ")" + ); } List studyGroupedGenomicProcessors = new ArrayList<>(); @@ -94,9 +97,10 @@ public GenomicProcessor localPatientOnlyDistributedGenomicProcessor() { @ConditionalOnProperty(prefix = "hpds.genomicProcessor", name = "impl", havingValue = "remote") public GenomicProcessor remoteGenomicProcessor(@Value("${hpds.genomicProcessor.remoteHosts}") List remoteHosts) { List genomicProcessors = Flux.fromIterable(remoteHosts) - .flatMap(remoteHost -> Mono.fromCallable(() -> (GenomicProcessor) new GenomicProcessorRestClient(remoteHost)).subscribeOn(Schedulers.boundedElastic())) - .collectList() - .block(); + .flatMap( + remoteHost -> Mono.fromCallable(() -> (GenomicProcessor) new GenomicProcessorRestClient(remoteHost)) + .subscribeOn(Schedulers.boundedElastic()) + ).collectList().block(); // todo: validate remote processors are valid return new GenomicProcessorParentImpl(genomicProcessors); } diff --git a/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/GenomicProcessorParentImpl.java b/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/GenomicProcessorParentImpl.java index e94b86873..df1002795 100644 --- a/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/GenomicProcessorParentImpl.java +++ b/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/GenomicProcessorParentImpl.java @@ -28,10 +28,8 @@ public class GenomicProcessorParentImpl implements GenomicProcessor { private final LoadingCache> infoStoreValuesCache = CacheBuilder.newBuilder().build(new CacheLoader<>() { @Override public Set load(String conceptPath) { - return nodes.parallelStream() - .map(node -> node.getInfoStoreValues(conceptPath)) - .flatMap(Set::stream) - .collect(Collectors.toSet()); + return nodes.parallelStream().map(node -> node.getInfoStoreValues(conceptPath)).flatMap(Set::stream) + .collect(Collectors.toSet()); } }); @@ -50,8 +48,7 @@ public GenomicProcessorParentImpl(List nodes) { @Override public Mono getPatientMask(DistributableQuery distributableQuery) { Mono result = Flux.just(nodes.toArray(GenomicProcessor[]::new)) - .flatMap(node -> node.getPatientMask(distributableQuery)) - .reduce(VariantMask::union); + .flatMap(node -> node.getPatientMask(distributableQuery)).reduce(VariantMask::union); return result; } @@ -63,22 +60,20 @@ public Set patientMaskToPatientIdSet(VariantMask patientMask) { @Override public VariantMask createMaskForPatientSet(Set patientSubset) { // all nodes have the same patient set -- - VariantMask result = nodes.stream().findFirst() - .map(node -> node.createMaskForPatientSet(patientSubset)) - .orElseGet(VariantMask::emptyInstance); + VariantMask result = + nodes.stream().findFirst().map(node -> node.createMaskForPatientSet(patientSubset)).orElseGet(VariantMask::emptyInstance); return result; } @Override public Mono> getVariantList(DistributableQuery distributableQuery) { Mono> result = Flux.just(nodes.toArray(GenomicProcessor[]::new)) - .flatMap(node -> node.getVariantList(distributableQuery)) - .reduce((variantList1, variantList2) -> { - Set mergedResult = new HashSet<>(variantList1.size() + variantList2.size()); - mergedResult.addAll(variantList1); - mergedResult.addAll(variantList2); - return mergedResult; - }); + .flatMap(node -> node.getVariantList(distributableQuery)).reduce((variantList1, variantList2) -> { + Set mergedResult = new HashSet<>(variantList1.size() + variantList2.size()); + mergedResult.addAll(variantList1); + mergedResult.addAll(variantList2); + return mergedResult; + }); return result; } @@ -89,19 +84,19 @@ public List getPatientIds() { private List initializePatientIds() { List patientIds = Flux.just(nodes.toArray(GenomicProcessor[]::new)) - .flatMap(node -> Mono.fromCallable(node::getPatientIds).subscribeOn(Schedulers.boundedElastic())) - .reduce((patientIds1, patientIds2) -> { - if (patientIds1.size() != patientIds2.size()) { - throw new IllegalStateException("Patient lists from partitions do not match"); - } else { - for (int i = 0; i < patientIds1.size(); i++) { - if (!patientIds1.get(i).equals(patientIds2.get(i))) { - throw new IllegalStateException("Patient lists from partitions do not match"); - } + .flatMap(node -> Mono.fromCallable(node::getPatientIds).subscribeOn(Schedulers.boundedElastic())) + .reduce((patientIds1, patientIds2) -> { + if (patientIds1.size() != patientIds2.size()) { + throw new IllegalStateException("Patient lists from partitions do not match"); + } else { + for (int i = 0; i < patientIds1.size(); i++) { + if (!patientIds1.get(i).equals(patientIds2.get(i))) { + throw new IllegalStateException("Patient lists from partitions do not match"); } } - return patientIds1; - }).block(); + } + return patientIds1; + }).block(); return patientIds; } @@ -124,10 +119,7 @@ public Set getInfoStoreColumns() { } private Set initializeInfoStoreColumns() { - return nodes.parallelStream() - .map(GenomicProcessor::getInfoStoreColumns) - .flatMap(Set::stream) - .collect(Collectors.toSet()); + return nodes.parallelStream().map(GenomicProcessor::getInfoStoreColumns).flatMap(Set::stream).collect(Collectors.toSet()); } @Override @@ -143,26 +135,21 @@ public List getInfoColumnMeta() { @Override public Map> getVariantMetadata(Collection variantList) { ConcurrentHashMap> result = new ConcurrentHashMap<>(); - nodes.stream() - .map(node -> node.getVariantMetadata(variantList)) - .forEach(variantMap -> { - variantMap.entrySet().forEach(entry -> { - Set metadata = result.get(entry.getKey()); - if (metadata != null) { - metadata.addAll(entry.getValue()); - } else { - result.put(entry.getKey(), new HashSet<>(entry.getValue())); - } - }); - }); + nodes.stream().map(node -> node.getVariantMetadata(variantList)).forEach(variantMap -> { + variantMap.entrySet().forEach(entry -> { + Set metadata = result.get(entry.getKey()); + if (metadata != null) { + metadata.addAll(entry.getValue()); + } else { + result.put(entry.getKey(), new HashSet<>(entry.getValue())); + } + }); + }); return result; } private List initInfoColumnsMeta() { - return nodes.parallelStream() - .map(GenomicProcessor::getInfoColumnMeta) - .map(HashSet::new) - .flatMap(Set::stream) - .collect(Collectors.toList()); + return nodes.parallelStream().map(GenomicProcessor::getInfoColumnMeta).map(HashSet::new).flatMap(Set::stream) + .collect(Collectors.toList()); } } diff --git a/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/GenomicProcessorPatientMergingParentImpl.java b/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/GenomicProcessorPatientMergingParentImpl.java index 25d4d68d3..15693bf1b 100644 --- a/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/GenomicProcessorPatientMergingParentImpl.java +++ b/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/GenomicProcessorPatientMergingParentImpl.java @@ -24,10 +24,8 @@ public class GenomicProcessorPatientMergingParentImpl implements GenomicProcesso private final LoadingCache> infoStoreValuesCache = CacheBuilder.newBuilder().build(new CacheLoader<>() { @Override public Set load(String conceptPath) { - return nodes.parallelStream() - .map(node -> node.getInfoStoreValues(conceptPath)) - .flatMap(Set::stream) - .collect(Collectors.toSet()); + return nodes.parallelStream().map(node -> node.getInfoStoreValues(conceptPath)).flatMap(Set::stream) + .collect(Collectors.toSet()); } }); @@ -64,18 +62,16 @@ public SizedResult(T result, int size) { @Override public Mono getPatientMask(DistributableQuery distributableQuery) { - Mono result = Flux.just(nodes.toArray(GenomicProcessor[]::new)) - .flatMapSequential(node -> { - return node.getPatientMask(distributableQuery).map(mask -> { - return new SizedResult<>(mask, node.getPatientIds().size()); - }); - }) - .reduce(this::appendMask) - .map(SizedResult::getResult); + Mono result = Flux.just(nodes.toArray(GenomicProcessor[]::new)).flatMapSequential(node -> { + return node.getPatientMask(distributableQuery).map(mask -> { + return new SizedResult<>(mask, node.getPatientIds().size()); + }); + }).reduce(this::appendMask).map(SizedResult::getResult); return result; } - /** A little bit of a hack for now since the masks don't have sizes at this point and they are needed to merge + /** + * A little bit of a hack for now since the masks don't have sizes at this point and they are needed to merge */ private SizedResult appendMask(SizedResult mask1, SizedResult mask2) { VariantMask variantMask = VariableVariantMasks.appendMask(mask1.result, mask2.result, mask1.size, mask2.size); @@ -89,23 +85,19 @@ public Set patientMaskToPatientIdSet(VariantMask patientMask) { @Override public VariantMask createMaskForPatientSet(Set patientSubset) { - return nodes.stream() - .map(node -> new SizedResult<>(node.createMaskForPatientSet(patientSubset), node.getPatientIds().size())) - .reduce(this::appendMask) - .map(SizedResult::getResult) - .orElseGet(VariantMask::emptyInstance); + return nodes.stream().map(node -> new SizedResult<>(node.createMaskForPatientSet(patientSubset), node.getPatientIds().size())) + .reduce(this::appendMask).map(SizedResult::getResult).orElseGet(VariantMask::emptyInstance); } @Override public Mono> getVariantList(DistributableQuery distributableQuery) { Mono> result = Flux.just(nodes.toArray(GenomicProcessor[]::new)) - .flatMap(node -> node.getVariantList(distributableQuery)) - .reduce((variantList1, variantList2) -> { - Set mergedResult = new HashSet<>(variantList1.size() + variantList2.size()); - mergedResult.addAll(variantList1); - mergedResult.addAll(variantList2); - return mergedResult; - }); + .flatMap(node -> node.getVariantList(distributableQuery)).reduce((variantList1, variantList2) -> { + Set mergedResult = new HashSet<>(variantList1.size() + variantList2.size()); + mergedResult.addAll(variantList1); + mergedResult.addAll(variantList2); + return mergedResult; + }); return result; } @@ -116,12 +108,12 @@ public List getPatientIds() { private List initializePatientIds() { List result = Flux.just(nodes.toArray(GenomicProcessor[]::new)) - .flatMapSequential(node -> Mono.fromCallable(node::getPatientIds).subscribeOn(Schedulers.boundedElastic())) - .reduce((list1, list2) -> { - List concatenatedList = new ArrayList<>(list1); - concatenatedList.addAll(list2); - return concatenatedList; - }).block(); + .flatMapSequential(node -> Mono.fromCallable(node::getPatientIds).subscribeOn(Schedulers.boundedElastic())) + .reduce((list1, list2) -> { + List concatenatedList = new ArrayList<>(list1); + concatenatedList.addAll(list2); + return concatenatedList; + }).block(); Set distinctPatientIds = new HashSet<>(result); if (distinctPatientIds.size() != result.size()) { log.warn((result.size() - distinctPatientIds.size()) + " duplicate patients found in patient partitions"); @@ -132,18 +124,14 @@ private List initializePatientIds() { @Override public Optional getMasks(String path, VariantBucketHolder variantMasksVariantBucketHolder) { - return Optional.ofNullable(Flux.just(nodes.toArray(GenomicProcessor[]::new)) - .flatMapSequential(node -> { - VariableVariantMasks masks = node.getMasks(path, new VariantBucketHolder<>()).orElseGet(VariableVariantMasks::new); - return Mono.fromCallable(() -> new SizedResult<>(masks, node.getPatientIds().size())).subscribeOn(Schedulers.boundedElastic()); - }) - .reduce((masks1, masks2) -> { - int combinedSize = masks1.size + masks2.size; - VariableVariantMasks appendedMasks = VariableVariantMasks.append(masks1.result, masks1.size, masks2.result, masks2.size); - return new SizedResult<>(appendedMasks, combinedSize); - }) - .map(SizedResult::getResult) - .block()); + return Optional.ofNullable(Flux.just(nodes.toArray(GenomicProcessor[]::new)).flatMapSequential(node -> { + VariableVariantMasks masks = node.getMasks(path, new VariantBucketHolder<>()).orElseGet(VariableVariantMasks::new); + return Mono.fromCallable(() -> new SizedResult<>(masks, node.getPatientIds().size())).subscribeOn(Schedulers.boundedElastic()); + }).reduce((masks1, masks2) -> { + int combinedSize = masks1.size + masks2.size; + VariableVariantMasks appendedMasks = VariableVariantMasks.append(masks1.result, masks1.size, masks2.result, masks2.size); + return new SizedResult<>(appendedMasks, combinedSize); + }).map(SizedResult::getResult).block()); } @Override @@ -152,10 +140,7 @@ public Set getInfoStoreColumns() { } private Set initializeInfoStoreColumns() { - return nodes.parallelStream() - .map(GenomicProcessor::getInfoStoreColumns) - .flatMap(Set::stream) - .collect(Collectors.toSet()); + return nodes.parallelStream().map(GenomicProcessor::getInfoStoreColumns).flatMap(Set::stream).collect(Collectors.toSet()); } @Override @@ -170,25 +155,20 @@ public List getInfoColumnMeta() { @Override public Map> getVariantMetadata(Collection variantList) { - return nodes.parallelStream() - .map(node -> node.getVariantMetadata(variantList)) - .reduce((p1, p2) -> { - Map> mapCopy = new HashMap<>(p1); + return nodes.parallelStream().map(node -> node.getVariantMetadata(variantList)).reduce((p1, p2) -> { + Map> mapCopy = new HashMap<>(p1); - for (Map.Entry> map2Entry : p2.entrySet()) { - Set metadataValues = new HashSet<>(mapCopy.getOrDefault(map2Entry.getKey(), new HashSet<>())); - metadataValues.addAll(map2Entry.getValue()); - mapCopy.put(map2Entry.getKey(), metadataValues); - } - return mapCopy; - }).orElseGet(Map::of); + for (Map.Entry> map2Entry : p2.entrySet()) { + Set metadataValues = new HashSet<>(mapCopy.getOrDefault(map2Entry.getKey(), new HashSet<>())); + metadataValues.addAll(map2Entry.getValue()); + mapCopy.put(map2Entry.getKey(), metadataValues); + } + return mapCopy; + }).orElseGet(Map::of); } private List initInfoColumnsMeta() { - return nodes.parallelStream() - .map(GenomicProcessor::getInfoColumnMeta) - .map(HashSet::new) - .flatMap(Set::stream) - .collect(Collectors.toList()); + return nodes.parallelStream().map(GenomicProcessor::getInfoColumnMeta).map(HashSet::new).flatMap(Set::stream) + .collect(Collectors.toList()); } } diff --git a/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/HpdsProcessor.java b/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/HpdsProcessor.java index 97b597a62..bbd48684b 100644 --- a/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/HpdsProcessor.java +++ b/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/HpdsProcessor.java @@ -4,7 +4,7 @@ public interface HpdsProcessor { /** - * This should return a String array of the columns that will be exported in a DATAFRAME or COUNT type query. default is NULL. + * This should return a String array of the columns that will be exported in a DATAFRAME or COUNT type query. default is NULL. * @param query * @return */ diff --git a/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/MultiValueQueryProcessor.java b/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/MultiValueQueryProcessor.java index fd931fced..ae43b067f 100644 --- a/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/MultiValueQueryProcessor.java +++ b/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/MultiValueQueryProcessor.java @@ -33,7 +33,7 @@ public MultiValueQueryProcessor(AbstractProcessor abstractProcessor, @Value("${I @Override public String[] getHeaderRow(Query query) { - String[] header = new String[query.getFields().size()+1]; + String[] header = new String[query.getFields().size() + 1]; header[0] = PATIENT_ID_FIELD_NAME; System.arraycopy(query.getFields().toArray(), 0, header, 1, query.getFields().size()); return header; @@ -43,40 +43,35 @@ public String[] getHeaderRow(Query query) { public void runQuery(Query query, AsyncResult result) { Set idList = abstractProcessor.getPatientSubsetForQuery(query); log.info("Processing " + idList.size() + " rows for result " + result.getId()); - Lists.partition(new ArrayList<>(idList), idBatchSize).stream() - .forEach(patientIds -> { - Map>> pathToPatientToValueMap = buildResult(result, query, new TreeSet<>(patientIds)); - List>> fieldValuesPerPatient = patientIds.stream().map(patientId -> { - List> objectStream = Arrays.stream(getHeaderRow(query)).map(field -> { - if (PATIENT_ID_FIELD_NAME.equals(field)) { - return List.of(patientId.toString()); - } else { - return pathToPatientToValueMap.get(field).get(patientId); - } - }).collect(Collectors.toList()); - return objectStream; - }).collect(Collectors.toList()); - result.appendMultiValueResults(fieldValuesPerPatient); - }); + Lists.partition(new ArrayList<>(idList), idBatchSize).stream().forEach(patientIds -> { + Map>> pathToPatientToValueMap = buildResult(result, query, new TreeSet<>(patientIds)); + List>> fieldValuesPerPatient = patientIds.stream().map(patientId -> { + List> objectStream = Arrays.stream(getHeaderRow(query)).map(field -> { + if (PATIENT_ID_FIELD_NAME.equals(field)) { + return List.of(patientId.toString()); + } else { + return pathToPatientToValueMap.get(field).get(patientId); + } + }).collect(Collectors.toList()); + return objectStream; + }).collect(Collectors.toList()); + result.appendMultiValueResults(fieldValuesPerPatient); + }); result.closeWriter(); } private Map>> buildResult(AsyncResult result, Query query, TreeSet ids) { ConcurrentHashMap>> pathToPatientToValueMap = new ConcurrentHashMap<>(); - List columns = query.getFields().stream() - .map(abstractProcessor.getDictionary()::get) - .filter(Objects::nonNull) - .collect(Collectors.toList()); - List paths = columns.stream() - .map(ColumnMeta::getName) - .collect(Collectors.toList()); + List columns = + query.getFields().stream().map(abstractProcessor.getDictionary()::get).filter(Objects::nonNull).collect(Collectors.toList()); + List paths = columns.stream().map(ColumnMeta::getName).collect(Collectors.toList()); int columnCount = paths.size() + 1; ArrayList columnIndex = abstractProcessor.useResidentCubesFirst(paths, columnCount); // todo: investigate if the parallel stream will thrash the cache if the number of executors is > number of resident cubes - columnIndex.parallelStream().forEach((columnId)->{ - String columnPath = paths.get(columnId-1); + columnIndex.parallelStream().forEach((columnId) -> { + String columnPath = paths.get(columnId - 1); Map> patientIdToValueMap = processColumn(ids, columnPath); pathToPatientToValueMap.put(columnPath, patientIdToValueMap); }); @@ -92,12 +87,12 @@ private Map> processColumn(TreeSet patientIds, St KeyAndValue[] cubeValues = cube.sortedByKey(); int idPointer = 0; - for(int patientId : patientIds) { - while(idPointer < cubeValues.length) { + for (int patientId : patientIds) { + while (idPointer < cubeValues.length) { int key = cubeValues[idPointer].getKey(); - if(key < patientId) { + if (key < patientId) { idPointer++; - } else if(key == patientId){ + } else if (key == patientId) { String value = getResultField(cube, cubeValues, idPointer); patientIdToValueMap.computeIfAbsent(patientId, k -> new ArrayList<>()).add(value); idPointer++; @@ -109,8 +104,7 @@ private Map> processColumn(TreeSet patientIds, St return patientIdToValueMap; } - private String getResultField(PhenoCube cube, KeyAndValue[] cubeValues, - int idPointer) { + private String getResultField(PhenoCube cube, KeyAndValue[] cubeValues, int idPointer) { Comparable value = cubeValues[idPointer].getValue(); return value.toString(); } diff --git a/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/PatientVariantJoinHandler.java b/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/PatientVariantJoinHandler.java index 70b53b5d9..0b5db1abb 100644 --- a/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/PatientVariantJoinHandler.java +++ b/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/PatientVariantJoinHandler.java @@ -24,16 +24,15 @@ public PatientVariantJoinHandler(VariantService variantService) { this.variantService = variantService; } - public VariantMask getPatientIdsForIntersectionOfVariantSets(Set patientSubset, - VariantIndex intersectionOfInfoFilters) { + public VariantMask getPatientIdsForIntersectionOfVariantSets(Set patientSubset, VariantIndex intersectionOfInfoFilters) { - if(!intersectionOfInfoFilters.isEmpty()) { + if (!intersectionOfInfoFilters.isEmpty()) { Set patientsInScope; // todo: don't do this every time - Set patientIds = Arrays.asList( - variantService.getPatientIds()).stream().map((String id)->{ - return Integer.parseInt(id);}).collect(Collectors.toSet()); - if(patientSubset != null) { + Set patientIds = Arrays.asList(variantService.getPatientIds()).stream().map((String id) -> { + return Integer.parseInt(id); + }).collect(Collectors.toSet()); + if (patientSubset != null) { patientsInScope = Sets.intersection(patientIds, patientSubset); } else { // for now, null means there were no phenotypic filters and all patients are eligible @@ -51,27 +50,28 @@ public VariantMask getPatientIdsForIntersectionOfVariantSets(Set patien Set variantsInScope = intersectionOfInfoFilters.mapToVariantSpec(variantService.getVariantIndex()); - /*// todo: determine ideal ratio to bother with this - if (patientsInScope.size() < variantService.getPatientIds().length) { - variantsInScope = variantService.filterVariantSetForPatientSet(variantsInScope, patientsInScope); - }*/ - Collection> values = variantsInScope.stream() - .collect(Collectors.groupingByConcurrent((variantSpec) -> { - return new VariantSpec(variantSpec).metadata.offset / 1000; - })).values(); + /* + * // todo: determine ideal ratio to bother with this if (patientsInScope.size() < variantService.getPatientIds().length) { + * variantsInScope = variantService.filterVariantSetForPatientSet(variantsInScope, patientsInScope); } + */ + Collection> values = variantsInScope.stream().collect(Collectors.groupingByConcurrent((variantSpec) -> { + return new VariantSpec(variantSpec).metadata.offset / 1000; + })).values(); ArrayList> variantBucketsInScope = new ArrayList>(values); - //don't error on small result sets (make sure we have at least one element in each partition) + // don't error on small result sets (make sure we have at least one element in each partition) int partitionSize = variantBucketsInScope.size() / Runtime.getRuntime().availableProcessors(); - List>> variantBucketPartitions = Lists.partition(variantBucketsInScope, partitionSize > 0 ? partitionSize : 1); + List>> variantBucketPartitions = + Lists.partition(variantBucketsInScope, partitionSize > 0 ? partitionSize : 1); - log.debug("found " + variantBucketsInScope.size() + " buckets and partitioned those into " + variantBucketPartitions.size() + " groups"); + log.debug( + "found " + variantBucketsInScope.size() + " buckets and partitioned those into " + variantBucketPartitions.size() + + " groups" + ); VariantMask patientsInScopeMask = new VariantMaskBitmaskImpl(createMaskForPatientSet(patientsInScope)); - for(int x = 0; - x < variantBucketPartitions.size() /*&& matchingPatients[0].bitCount() < patientsInScopeSize + 4*/; - x++) { + for (int x = 0; x < variantBucketPartitions.size() /* && matchingPatients[0].bitCount() < patientsInScopeSize + 4 */; x++) { List> variantBuckets = variantBucketPartitions.get(x); variantBuckets.parallelStream().forEach(variantBucket -> { VariantBucketHolder bucketCache = new VariantBucketHolder<>(); @@ -82,12 +82,12 @@ public VariantMask getPatientIdsForIntersectionOfVariantSets(Set patien VariantMask heterozygousMask = masks.heterozygousMask; VariantMask homozygousMask = masks.homozygousMask; if (heterozygousMask != null) { - synchronized(matchingPatients) { + synchronized (matchingPatients) { matchingPatients[0] = matchingPatients[0].union(heterozygousMask); } } if (homozygousMask != null) { - synchronized(matchingPatients) { + synchronized (matchingPatients) { matchingPatients[0] = matchingPatients[0].union(homozygousMask); } } @@ -95,12 +95,15 @@ public VariantMask getPatientIdsForIntersectionOfVariantSets(Set patien }); if (!missingVariants.isEmpty()) { log.debug(missingVariants.size() + " variant masks not found"); - log.debug("Variants missing masks: " + Joiner.on(",").join( missingVariants.subList(0, Math.min(100, missingVariants.size())))); + log.debug( + "Variants missing masks: " + + Joiner.on(",").join(missingVariants.subList(0, Math.min(100, missingVariants.size()))) + ); } }); } return matchingPatients[0].intersection(patientsInScopeMask); - }else { + } else { // This will happen regularly, for example if someone is searching for a gene that is not on this partition log.debug("No matches found for info filters."); return new VariantMaskSparseImpl(Set.of()); @@ -111,15 +114,15 @@ public VariantMask getPatientIdsForIntersectionOfVariantSets(Set patien public BigInteger createMaskForPatientSet(Set patientSet) { Set patientSubset = patientSet; if (patientSet == null) { - patientSubset = Arrays.asList( - variantService.getPatientIds()).stream().map((String id)->{ - return Integer.parseInt(id);}).collect(Collectors.toSet()); + patientSubset = Arrays.asList(variantService.getPatientIds()).stream().map((String id) -> { + return Integer.parseInt(id); + }).collect(Collectors.toSet()); } - StringBuilder builder = new StringBuilder("11"); //variant bitmasks are bookended with '11' + StringBuilder builder = new StringBuilder("11"); // variant bitmasks are bookended with '11' for (int i = variantService.getPatientIds().length - 1; i >= 0; i--) { Integer idInt = Integer.parseInt(variantService.getPatientIds()[i]); - if(patientSubset.contains(idInt)){ + if (patientSubset.contains(idInt)) { builder.append("1"); } else { builder.append("0"); diff --git a/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/Processor.java b/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/Processor.java index f65735f98..45af248d3 100644 --- a/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/Processor.java +++ b/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/Processor.java @@ -5,6 +5,6 @@ public interface Processor { - public void runQuery(Query query, AsyncResult asyncResult) throws NotEnoughMemoryException; + public void runQuery(Query query, AsyncResult asyncResult) throws NotEnoughMemoryException; } diff --git a/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/ResultStore.java b/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/ResultStore.java index 4fc2f7c42..2c5c52d52 100644 --- a/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/ResultStore.java +++ b/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/ResultStore.java @@ -13,159 +13,161 @@ import edu.harvard.hms.dbmi.avillach.hpds.exception.ResultSetTooLargeException; /** - * This class handles composing a segment of a data export file in memory. The goal here - * is to prevent memory errors during data export and maximize concurrency for processing - * concepts. + * This class handles composing a segment of a data export file in memory. The goal here is to prevent memory errors during data export and + * maximize concurrency for processing concepts. * * Each ResultStore instance allocates a byte[] called resultArray. The size of this array is: * - * (maximum width of a row in the results) * (number of patients in the result segment) - * - * The maximum width of a row in the result is the sum of the maximum widths in bytes of all - * columns. This width is stored when the data is loaded initially in the column meta for the - * concept. + * (maximum width of a row in the results) * (number of patients in the result segment) * - * If the size of the array exceeds the maximum size for an array(Integer.MAX_VALUE) then a - * {@link ResultSetTooLargeException} is thrown with a message that suggests the user select fewer fields. + * The maximum width of a row in the result is the sum of the maximum widths in bytes of all columns. This width is stored when the data is + * loaded initially in the column meta for the concept. * - * If the size of the resultArray exceeds the available heap in the JVM a NotEnoughMemoryException - * is thrown. The only recourse here is to increase the amount of available heap or make the BATCH_SIZE - * system property smaller. One way to increase the amount of available heap is to reduce the CACHE_SIZE - * system property, but that has to be balanced against the performance of non-export queries. Finding - * the ideal settings for these properties is a trial and error process for each dataset and user-base. + * If the size of the array exceeds the maximum size for an array(Integer.MAX_VALUE) then a {@link ResultSetTooLargeException} is thrown + * with a message that suggests the user select fewer fields. * - * The resultArray approach allows us to build the entire CSV segment concurrently for multiple concepts - * without any synchronization overhead. Each concept is written into a dedicated fixed-width region of - * the resultArray. These regions do not overlap and a single thread is used for each concept, so there - * are never two threads writing to the same location at the same time. + * If the size of the resultArray exceeds the available heap in the JVM a NotEnoughMemoryException is thrown. The only recourse here is to + * increase the amount of available heap or make the BATCH_SIZE system property smaller. One way to increase the amount of available heap is + * to reduce the CACHE_SIZE system property, but that has to be balanced against the performance of non-export queries. Finding the ideal + * settings for these properties is a trial and error process for each dataset and user-base. * - * The data is then read out of this fixed width structure into a String[] at the time it is written to - * a temp file for the result by the {@link ResultStoreStream} class. + * The resultArray approach allows us to build the entire CSV segment concurrently for multiple concepts without any synchronization + * overhead. Each concept is written into a dedicated fixed-width region of the resultArray. These regions do not overlap and a single + * thread is used for each concept, so there are never two threads writing to the same location at the same time. + * + * The data is then read out of this fixed width structure into a String[] at the time it is written to a temp file for the result by the + * {@link ResultStoreStream} class. * */ public class ResultStore { - private static Logger log = LoggerFactory.getLogger(ResultStore.class); - - private List columns; - int rowWidth; - private int numRows; - - private static final ColumnMeta PATIENT_ID_COLUMN_META = new ColumnMeta().setColumnOffset(0).setName("PatientId").setWidthInBytes(Integer.BYTES); - byte[] resultArray; - - /** - * - * @param resultId The result id here is only used for logging - * @param columns The ColumnMeta entries involed in the result, these are used to calculate rowWidth - * @param ids The subject ids for in the current batch of the result - * @throws NotEnoughMemoryException If the size of available heap cannot support a byte array of size (rowWidth x numRows) - */ - public ResultStore(String resultId, List columns, TreeSet ids) throws NotEnoughMemoryException { - this.columns = new ArrayList(); - this.numRows = ids.size(); - this.getColumns().add(PATIENT_ID_COLUMN_META); - int rowWidth = Integer.BYTES; - for(ColumnMeta column : columns) { - column.setColumnOffset(rowWidth); - rowWidth += column.getWidthInBytes(); - this.getColumns().add(column); - } - this.rowWidth = rowWidth; - if(1L * this.rowWidth * this.getNumRows() > Integer.MAX_VALUE) { - throw new ResultSetTooLargeException((1L * this.rowWidth * this.getNumRows())/(Integer.MAX_VALUE/2)); - } - try { - log.info("Allocating result array : " + resultId); - resultArray = new byte[this.rowWidth * this.getNumRows()]; - } catch(Error e) { - throw new NotEnoughMemoryException(); - } - log.info("Store created for " + this.getNumRows() + " rows and " + columns.size() + " columns "); - int x = 0; - for(Integer id : ids) { - writeField(0,x++,ByteBuffer.allocate(Integer.BYTES).putInt(id).array()); - } - } - - /** - * Copies fieldValue into the resultArray of this instance using System.arraycopy - * @param column - * @param row - * @param fieldValue - */ - public void writeField(int column, int row, byte[] fieldValue) { - int offset = getFieldOffset(row,column); - try { - System.arraycopy(fieldValue, 0, resultArray, offset, fieldValue.length); - }catch(Exception e) { - log.info("Exception caught writing field : " + column + ", " + row + " : " + fieldValue.length + " bytes into result at offset " + offset + " of " + resultArray.length); - throw e; - } - } - - /** - * Finds the resultArray offset of the passed in row and column - * @param row - * @param column - * @return - */ - private int getFieldOffset(int row, int column) { - int rowOffset = row*rowWidth; - int columnOffset = getColumns().get(column).getColumnOffset(); - return rowOffset + columnOffset; - } - - ByteBuffer wrappedResultArray; - /** - * Populate - * - * @param rowNumber - * @param row - * @throws IOException - */ - public void readRowIntoStringArray(int rowNumber, int[] columnWidths, String[] row) { - if(wrappedResultArray == null) { - wrappedResultArray = ByteBuffer.wrap(resultArray); - } - row[0] = wrappedResultArray.getInt(getFieldOffset(rowNumber, 0)) + ""; - - stringifyRow(rowNumber, columnWidths, row); - } - - /** - * Copy each field of a single row from the resultArray into a String[] to be written out to the CSV - * - * @param rowNumber row number to copy - * @param row String[] to populate with field values - * @param columnBuffers a set of buffers corresponding to each column where each buffer is {@link ColumnMeta.widthInBytes} bytes - */ - private void stringifyRow(int rowNumber, int[] columnWidths, String[] row) { - for(int x = 1;x getColumns() { - return columns; - } + private static Logger log = LoggerFactory.getLogger(ResultStore.class); + + private List columns; + int rowWidth; + private int numRows; + + private static final ColumnMeta PATIENT_ID_COLUMN_META = + new ColumnMeta().setColumnOffset(0).setName("PatientId").setWidthInBytes(Integer.BYTES); + byte[] resultArray; + + /** + * + * @param resultId The result id here is only used for logging + * @param columns The ColumnMeta entries involed in the result, these are used to calculate rowWidth + * @param ids The subject ids for in the current batch of the result + * @throws NotEnoughMemoryException If the size of available heap cannot support a byte array of size (rowWidth x numRows) + */ + public ResultStore(String resultId, List columns, TreeSet ids) throws NotEnoughMemoryException { + this.columns = new ArrayList(); + this.numRows = ids.size(); + this.getColumns().add(PATIENT_ID_COLUMN_META); + int rowWidth = Integer.BYTES; + for (ColumnMeta column : columns) { + column.setColumnOffset(rowWidth); + rowWidth += column.getWidthInBytes(); + this.getColumns().add(column); + } + this.rowWidth = rowWidth; + if (1L * this.rowWidth * this.getNumRows() > Integer.MAX_VALUE) { + throw new ResultSetTooLargeException((1L * this.rowWidth * this.getNumRows()) / (Integer.MAX_VALUE / 2)); + } + try { + log.info("Allocating result array : " + resultId); + resultArray = new byte[this.rowWidth * this.getNumRows()]; + } catch (Error e) { + throw new NotEnoughMemoryException(); + } + log.info("Store created for " + this.getNumRows() + " rows and " + columns.size() + " columns "); + int x = 0; + for (Integer id : ids) { + writeField(0, x++, ByteBuffer.allocate(Integer.BYTES).putInt(id).array()); + } + } + + /** + * Copies fieldValue into the resultArray of this instance using System.arraycopy + * @param column + * @param row + * @param fieldValue + */ + public void writeField(int column, int row, byte[] fieldValue) { + int offset = getFieldOffset(row, column); + try { + System.arraycopy(fieldValue, 0, resultArray, offset, fieldValue.length); + } catch (Exception e) { + log.info( + "Exception caught writing field : " + column + ", " + row + " : " + fieldValue.length + " bytes into result at offset " + + offset + " of " + resultArray.length + ); + throw e; + } + } + + /** + * Finds the resultArray offset of the passed in row and column + * @param row + * @param column + * @return + */ + private int getFieldOffset(int row, int column) { + int rowOffset = row * rowWidth; + int columnOffset = getColumns().get(column).getColumnOffset(); + return rowOffset + columnOffset; + } + + ByteBuffer wrappedResultArray; + + /** + * Populate + * + * @param rowNumber + * @param row + * @throws IOException + */ + public void readRowIntoStringArray(int rowNumber, int[] columnWidths, String[] row) { + if (wrappedResultArray == null) { + wrappedResultArray = ByteBuffer.wrap(resultArray); + } + row[0] = wrappedResultArray.getInt(getFieldOffset(rowNumber, 0)) + ""; + + stringifyRow(rowNumber, columnWidths, row); + } + + /** + * Copy each field of a single row from the resultArray into a String[] to be written out to the CSV + * + * @param rowNumber row number to copy + * @param row String[] to populate with field values + * @param columnBuffers a set of buffers corresponding to each column where each buffer is {@link ColumnMeta.widthInBytes} bytes + */ + private void stringifyRow(int rowNumber, int[] columnWidths, String[] row) { + for (int x = 1; x < row.length; x++) { + ColumnMeta columnMeta = getColumns().get(x); + int fieldOffset = getFieldOffset(rowNumber, x); + if (columnMeta.isCategorical()) { + stringifyString(row, columnWidths, x, fieldOffset); + } else { + stringifyDouble(row, x, fieldOffset); + } + } + } + + DecimalFormat decimalFormat = new DecimalFormat("########.####"); + + private void stringifyDouble(String[] row, int x, int fieldOffset) { + row[x] = Double.valueOf(wrappedResultArray.getDouble(fieldOffset)).toString(); + } + + private void stringifyString(String[] row, int[] columnWidths, int x, int fieldOffset) { + row[x] = new String(Arrays.copyOfRange(resultArray, fieldOffset, fieldOffset + columnWidths[x])).trim(); + } + + public int getNumRows() { + return numRows; + } + + public List getColumns() { + return columns; + } } diff --git a/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/ResultStoreStream.java b/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/ResultStoreStream.java index e57d300a3..e0558190a 100644 --- a/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/ResultStoreStream.java +++ b/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/ResultStoreStream.java @@ -11,119 +11,120 @@ public class ResultStoreStream extends InputStream { - private ResultWriter writer; - private InputStream in; - private int value; - private boolean streamIsClosed = false; - private int numRows; - private String[] originalHeader; - - public ResultStoreStream(String[] header, ResultWriter writer) throws IOException { - this.writer = writer; - this.originalHeader = header; - writeHeader(this.originalHeader); - numRows = 0; - } - - private void writeHeader(String[] header) { - writer.writeHeader(header); - } - - public void appendResultStore(ResultStore results) { - int batchSize = 100; - List entries = new ArrayList<>(batchSize); - for(int x = 0;x entries) { - writer.writeEntity(entries); - } - /** - * Appending data to the writer that supports multiple values per patient/variable combination - */ - public void appendMultiValueResults(List>> entries) { - writer.writeMultiValueEntity(entries); - } - - private List writeResultsToTempFile(ResultStore results, int batchSize, - List entries) { - - List columns = results.getColumns(); - int[] columnWidths = new int[columns.size()]; - for(int x = 0;x entries = new ArrayList<>(batchSize); + for (int x = 0; x < batchSize; x++) { + entries.add(new String[results.getColumns().size()]); + } + writeResultsToTempFile(results, batchSize, entries); + } + + /** + * A more compact method to append data to the temp file without making assumptions about the composition. + * @param entries + */ + public void appendResults(List entries) { + writer.writeEntity(entries); + } + + /** + * Appending data to the writer that supports multiple values per patient/variable combination + */ + public void appendMultiValueResults(List>> entries) { + writer.writeMultiValueEntity(entries); + } + + private List writeResultsToTempFile(ResultStore results, int batchSize, List entries) { + + List columns = results.getColumns(); + int[] columnWidths = new int[columns.size()]; + for (int x = 0; x < columns.size(); x++) { + columnWidths[x] = columns.get(x).getWidthInBytes(); + } + + for (int x = 0; x < (results.getNumRows()); x += batchSize) { + int rowsInBatch = Math.min(batchSize, results.getNumRows() - x); + for (int y = 0; y < rowsInBatch; y++) { + results.readRowIntoStringArray(y + x, columnWidths, entries.get(y)); + } + if (rowsInBatch < batchSize) { + entries = entries.subList(0, rowsInBatch); + } + writer.writeEntity(entries); + numRows += rowsInBatch; + } + return entries; + } + + public void close() { + try { + in.close(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + public void open() { + try { + in = new BufferedInputStream(new FileInputStream(writer.getFile().getAbsolutePath()), 1024 * 1024 * 8); + streamIsClosed = false; + } catch (FileNotFoundException e) { + throw new RuntimeException("temp file for result not found : " + writer.getFile().getAbsolutePath()); + } + } - @Override - public int read() throws IOException { - if(streamIsClosed) { - return -1; - } - value = in.read(); - if(value == -1) { - in.close(); - streamIsClosed = true; - } - return value; - } - - public int getNumRows() { - return numRows; - } - - public long estimatedSize() { - return writer.getFile().length(); - } - - public void closeWriter() { - writer.close(); - } - - public File getFile() { - return writer.getFile(); - } - public Path getTempFilePath() { - return Path.of(getFile().getAbsolutePath()); - } + + @Override + public int read() throws IOException { + if (streamIsClosed) { + return -1; + } + value = in.read(); + if (value == -1) { + in.close(); + streamIsClosed = true; + } + return value; + } + + public int getNumRows() { + return numRows; + } + + public long estimatedSize() { + return writer.getFile().length(); + } + + public void closeWriter() { + writer.close(); + } + + public File getFile() { + return writer.getFile(); + } + + public Path getTempFilePath() { + return Path.of(getFile().getAbsolutePath()); + } } diff --git a/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/SparseVariantIndex.java b/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/SparseVariantIndex.java index befd114ef..0f78d915a 100644 --- a/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/SparseVariantIndex.java +++ b/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/SparseVariantIndex.java @@ -41,13 +41,13 @@ public VariantIndex intersection(VariantIndex variantIndex) { } /** - * Converts a set of variant IDs to a set of String representations of variant spec. This implementation looks - * wonky, but performs much better than other more obvious approaches (ex: Collectors.toSet()) on large sets. + * Converts a set of variant IDs to a set of String representations of variant spec. This implementation looks wonky, but performs much + * better than other more obvious approaches (ex: Collectors.toSet()) on large sets. */ @Override public Set mapToVariantSpec(String[] variantIndex) { ConcurrentHashMap setMap = new ConcurrentHashMap<>(variantIds.size()); - variantIds.stream().parallel().forEach(index-> setMap.put(variantIndex[index], "")); + variantIds.stream().parallel().forEach(index -> setMap.put(variantIndex[index], "")); return setMap.keySet(); } diff --git a/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/VariantIndex.java b/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/VariantIndex.java index 04ad1079c..4e772678e 100644 --- a/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/VariantIndex.java +++ b/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/VariantIndex.java @@ -8,6 +8,7 @@ public abstract class VariantIndex { public abstract VariantIndex union(VariantIndex variantIndex); + public abstract VariantIndex intersection(VariantIndex variantIndex); public abstract Set mapToVariantSpec(String[] variantIndex); @@ -23,9 +24,8 @@ protected VariantIndex union(SparseVariantIndex sparseVariantIndex, DenseVariant protected VariantIndex intersection(SparseVariantIndex sparseVariantIndex, DenseVariantIndex denseVariantIndex) { - Set intersection = sparseVariantIndex.getVariantIds().stream() - .filter(id -> denseVariantIndex.getVariantIndexMask()[id]) - .collect(Collectors.toSet()); + Set intersection = sparseVariantIndex.getVariantIds().stream().filter(id -> denseVariantIndex.getVariantIndexMask()[id]) + .collect(Collectors.toSet()); return new SparseVariantIndex(intersection); } diff --git a/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/VariantIndexCache.java b/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/VariantIndexCache.java index be05501c4..01be22ede 100644 --- a/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/VariantIndexCache.java +++ b/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/VariantIndexCache.java @@ -35,21 +35,22 @@ public class VariantIndexCache { public VariantIndexCache(String[] variantIndex, Map infoStores) { this.variantIndex = variantIndex; this.infoStores = infoStores; - this.infoCache = CacheBuilder.newBuilder() - .weigher(weigher).maximumWeight(10000000000000L).build(cacheLoader); + this.infoCache = CacheBuilder.newBuilder().weigher(weigher).maximumWeight(10000000000000L).build(cacheLoader); } public VariantIndex get(String key) { return infoCache.getUnchecked(key); } + public VariantIndex get(String column, String key) { return infoCache.getUnchecked(columnAndKey(column, key)); } + private String columnAndKey(String column, String key) { return column + COLUMN_AND_KEY_DELIMITER + key; } - private final Weigher weigher = new Weigher(){ + private final Weigher weigher = new Weigher() { @Override public int weigh(String key, VariantIndex value) { if (value instanceof DenseVariantIndex) { @@ -69,16 +70,16 @@ public VariantIndex load(String infoColumn_valueKey) throws IOException { String[] column_and_value = infoColumn_valueKey.split(COLUMN_AND_KEY_DELIMITER); Integer[] variantIndexIntArray = infoStores.get(column_and_value[0]).getAllValues().get(column_and_value[1]); - if ((double)variantIndexIntArray.length / (double)variantIndex.length < MAX_SPARSE_INDEX_RATIO ) { + if ((double) variantIndexIntArray.length / (double) variantIndex.length < MAX_SPARSE_INDEX_RATIO) { Set variantIds = new HashSet<>(); - for(Integer variantIndex : variantIndexIntArray) { + for (Integer variantIndex : variantIndexIntArray) { variantIds.add(variantIndex); } return new SparseVariantIndex(variantIds); } else { boolean[] variantIndexArray = new boolean[variantIndex.length]; int x = 0; - for(Integer variantIndex : variantIndexIntArray) { + for (Integer variantIndex : variantIndexIntArray) { if (variantIndex >= 0) { variantIndexArray[variantIndex] = true; } diff --git a/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/VariantService.java b/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/VariantService.java index 39bbeabd5..cdbd0d3b9 100644 --- a/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/VariantService.java +++ b/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/VariantService.java @@ -44,6 +44,7 @@ public String[] getVariantIndex() { public BucketIndexBySample getBucketIndex() { return bucketIndex; } + public Set filterVariantSetForPatientSet(Set variantSet, Collection patientSet) { try { return bucketIndex.filterVariantSetForPatientSet(variantSet, patientSet); @@ -81,8 +82,8 @@ private VariantStore loadVariantStore() { } public String[] loadVariantIndex() { - //skip if we have no variants - if(variantStore.getPatientIds().length == 0) { + // skip if we have no variants + if (variantStore.getPatientIds().length == 0) { log.warn("No Genomic Data found. Skipping variant Indexing"); return new String[0]; } @@ -100,22 +101,25 @@ public String[] loadVariantIndex() { * @throws InterruptedException */ private void loadGenomicCacheFiles() throws FileNotFoundException, IOException, InterruptedException { - if(bucketIndex==null) { - if(variantIndex==null) { - if(!new File(VARIANT_INDEX_FBBIS_FILE).exists()) { + if (bucketIndex == null) { + if (variantIndex == null) { + if (!new File(VARIANT_INDEX_FBBIS_FILE).exists()) { log.info("Creating new " + VARIANT_INDEX_FBBIS_FILE); this.variantIndex = loadVariantIndex(); FileBackedByteIndexedStorage fbbis = - new FileBackedJavaIndexedStorage<>(Integer.class, String[].class, new File(VARIANT_INDEX_FBBIS_STORAGE_FILE)); - try (ObjectOutputStream oos = new ObjectOutputStream(new GZIPOutputStream(new FileOutputStream(VARIANT_INDEX_FBBIS_FILE))); - ){ + new FileBackedJavaIndexedStorage<>(Integer.class, String[].class, new File(VARIANT_INDEX_FBBIS_STORAGE_FILE)); + try ( + ObjectOutputStream oos = + new ObjectOutputStream(new GZIPOutputStream(new FileOutputStream(VARIANT_INDEX_FBBIS_FILE))); + ) { log.debug("Writing Cache Object in blocks of " + VARIANT_INDEX_BLOCK_SIZE); - int bucketCount = (variantIndex.length / VARIANT_INDEX_BLOCK_SIZE) + 1; //need to handle overflow + int bucketCount = (variantIndex.length / VARIANT_INDEX_BLOCK_SIZE) + 1; // need to handle overflow int index = 0; - for( int i = 0; i < bucketCount; i++) { - int blockSize = i == (bucketCount - 1) ? (variantIndex.length % VARIANT_INDEX_BLOCK_SIZE) : VARIANT_INDEX_BLOCK_SIZE; + for (int i = 0; i < bucketCount; i++) { + int blockSize = + i == (bucketCount - 1) ? (variantIndex.length % VARIANT_INDEX_BLOCK_SIZE) : VARIANT_INDEX_BLOCK_SIZE; String[] variantArrayBlock = new String[blockSize]; System.arraycopy(variantIndex, index, variantArrayBlock, 0, blockSize); @@ -127,33 +131,40 @@ private void loadGenomicCacheFiles() throws FileNotFoundException, IOException, fbbis.complete(); oos.writeObject("" + variantIndex.length); oos.writeObject(fbbis); - oos.flush();oos.close(); + oos.flush(); + oos.close(); } - }else { + } else { ExecutorService ex = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); - try (ObjectInputStream objectInputStream = new ObjectInputStream(new GZIPInputStream(new FileInputStream(VARIANT_INDEX_FBBIS_FILE)));){ + try ( + ObjectInputStream objectInputStream = + new ObjectInputStream(new GZIPInputStream(new FileInputStream(VARIANT_INDEX_FBBIS_FILE))); + ) { Integer variantCount = Integer.parseInt((String) objectInputStream.readObject()); - FileBackedByteIndexedStorage indexStore = (FileBackedByteIndexedStorage) objectInputStream.readObject(); + FileBackedByteIndexedStorage indexStore = + (FileBackedByteIndexedStorage) objectInputStream.readObject(); log.info("loading " + VARIANT_INDEX_FBBIS_FILE); variantIndex = new String[variantCount]; String[] _varaiantIndex2 = variantIndex; - //variant index has to be a single array (we use a binary search for lookups) - //but reading/writing to disk should be batched for performance - int bucketCount = (variantCount / VARIANT_INDEX_BLOCK_SIZE) + 1; //need to handle overflow + // variant index has to be a single array (we use a binary search for lookups) + // but reading/writing to disk should be batched for performance + int bucketCount = (variantCount / VARIANT_INDEX_BLOCK_SIZE) + 1; // need to handle overflow - for( int i = 0; i < bucketCount; i++) { + for (int i = 0; i < bucketCount; i++) { final int _i = i; ex.submit(() -> { String[] variantIndexBucket = indexStore.get(_i); - System.arraycopy(variantIndexBucket, 0, _varaiantIndex2, (_i * VARIANT_INDEX_BLOCK_SIZE), variantIndexBucket.length); + System.arraycopy( + variantIndexBucket, 0, _varaiantIndex2, (_i * VARIANT_INDEX_BLOCK_SIZE), variantIndexBucket.length + ); log.debug("loaded " + (_i * VARIANT_INDEX_BLOCK_SIZE) + " block"); }); } objectInputStream.close(); ex.shutdown(); - while(! ex.awaitTermination(60, TimeUnit.SECONDS)) { + while (!ex.awaitTermination(60, TimeUnit.SECONDS)) { log.info("Waiting for tasks to complete"); Thread.sleep(10000); } @@ -163,22 +174,27 @@ private void loadGenomicCacheFiles() throws FileNotFoundException, IOException, log.info("Found " + variantIndex.length + " total variants."); } } - // todo: not loading bucket index when there is no variant metadata index is a temporary fix for non-variant explorer environments + // todo: not loading bucket index when there is no variant metadata index is a temporary fix for non-variant explorer + // environments // once we start building the bucket index as part of the ETL, we can remove this check and leverage the bucket index // for all genomic queries - if(variantStore.getPatientIds().length > 0 && variantMetadataIndex != null && !new File(BUCKET_INDEX_BY_SAMPLE_FILE).exists()) { + if ( + variantStore.getPatientIds().length > 0 && variantMetadataIndex != null && !new File(BUCKET_INDEX_BY_SAMPLE_FILE).exists() + ) { log.info("creating new " + BUCKET_INDEX_BY_SAMPLE_FILE); bucketIndex = new BucketIndexBySample(variantStore, genomicDataDirectory); try ( - FileOutputStream fos = new FileOutputStream(BUCKET_INDEX_BY_SAMPLE_FILE); - GZIPOutputStream gzos = new GZIPOutputStream(fos); - ObjectOutputStream oos = new ObjectOutputStream(gzos); - ){ + FileOutputStream fos = new FileOutputStream(BUCKET_INDEX_BY_SAMPLE_FILE); GZIPOutputStream gzos = + new GZIPOutputStream(fos); ObjectOutputStream oos = new ObjectOutputStream(gzos); + ) { oos.writeObject(bucketIndex); oos.flush(); } - }else if (new File(BUCKET_INDEX_BY_SAMPLE_FILE).exists()) { - try (ObjectInputStream objectInputStream = new ObjectInputStream(new GZIPInputStream(new FileInputStream(BUCKET_INDEX_BY_SAMPLE_FILE)));){ + } else if (new File(BUCKET_INDEX_BY_SAMPLE_FILE).exists()) { + try ( + ObjectInputStream objectInputStream = + new ObjectInputStream(new GZIPInputStream(new FileInputStream(BUCKET_INDEX_BY_SAMPLE_FILE))); + ) { log.info("loading " + BUCKET_INDEX_BY_SAMPLE_FILE); bucketIndex = (BucketIndexBySample) objectInputStream.readObject(); bucketIndex.updateStorageDirectory(new File(genomicDataDirectory)); @@ -200,6 +216,7 @@ public Optional getMasks(String variantName, VariantBucket throw new UncheckedIOException(e); } } + public List getMasksForDbSnpSpec(String variantName) { return variantStore.getMasksForDbSnpSpec(variantName); } diff --git a/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/audit/AuditAttributes.java b/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/audit/AuditAttributes.java index 95cd40e5e..9bcdf3357 100644 --- a/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/audit/AuditAttributes.java +++ b/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/audit/AuditAttributes.java @@ -6,10 +6,8 @@ import jakarta.servlet.http.HttpServletRequest; /** - * Utility for setting audit metadata on the current request. - * Controllers call {@link #putMetadata} to attach domain-specific context - * (query_id, result_type, etc.) that the {@link AuditLoggingFilter} merges - * into the logging event. + * Utility for setting audit metadata on the current request. Controllers call {@link #putMetadata} to attach domain-specific context + * (query_id, result_type, etc.) that the {@link AuditLoggingFilter} merges into the logging event. */ public final class AuditAttributes { @@ -17,8 +15,7 @@ public final class AuditAttributes { public static final String ACTION = "audit.action"; private static final String METADATA = "audit.metadata"; - private AuditAttributes() { - } + private AuditAttributes() {} public static void putMetadata(HttpServletRequest request, String key, Object value) { if (request != null && key != null && value != null) { diff --git a/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/audit/AuditInterceptor.java b/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/audit/AuditInterceptor.java index 5889f4ff0..96b40e73d 100644 --- a/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/audit/AuditInterceptor.java +++ b/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/audit/AuditInterceptor.java @@ -8,8 +8,8 @@ import jakarta.servlet.http.HttpServletResponse; /** - * Reads {@link AuditEvent} annotations from controller methods and sets - * the event type and action as request attributes for the {@link AuditLoggingFilter}. + * Reads {@link AuditEvent} annotations from controller methods and sets the event type and action as request attributes for the + * {@link AuditLoggingFilter}. */ public class AuditInterceptor implements HandlerInterceptor { diff --git a/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/dictionary/DictionaryService.java b/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/dictionary/DictionaryService.java index c6c10bef7..8ae536dc6 100644 --- a/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/dictionary/DictionaryService.java +++ b/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/dictionary/DictionaryService.java @@ -16,8 +16,7 @@ @ConditionalOnProperty("dictionary.host") public class DictionaryService { - public static final ParameterizedTypeReference> CONCEPT_LIST_TYPE_REFERENCE = new ParameterizedTypeReference<>() { - }; + public static final ParameterizedTypeReference> CONCEPT_LIST_TYPE_REFERENCE = new ParameterizedTypeReference<>() {}; private final String dictionaryHost; private final RestTemplate restTemplate; @@ -32,6 +31,9 @@ public DictionaryService(@Value("${dictionary.host}") String dictionaryHostTempl } public List getConcepts(List conceptPaths) { - return restTemplate.exchange(dictionaryHost + "/pic-sure-api-2/PICSURE/proxy/dictionary-api/concepts/detail", HttpMethod.POST, new HttpEntity<>(conceptPaths), CONCEPT_LIST_TYPE_REFERENCE).getBody(); + return restTemplate.exchange( + dictionaryHost + "/pic-sure-api-2/PICSURE/proxy/dictionary-api/concepts/detail", HttpMethod.POST, + new HttpEntity<>(conceptPaths), CONCEPT_LIST_TYPE_REFERENCE + ).getBody(); } } diff --git a/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/genomic/GenomicProcessorRestClient.java b/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/genomic/GenomicProcessorRestClient.java index 8ecfc0f4f..bc0ac81bf 100644 --- a/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/genomic/GenomicProcessorRestClient.java +++ b/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/genomic/GenomicProcessorRestClient.java @@ -18,26 +18,21 @@ public class GenomicProcessorRestClient implements GenomicProcessor { private final WebClient webClient; - private static final ParameterizedTypeReference> VARIANT_SET_TYPE_REFERENCE = new ParameterizedTypeReference<>(){}; - private static final ParameterizedTypeReference> INFO_COLUMNS_META_TYPE_REFERENCE = new ParameterizedTypeReference<>(){}; - private static final ParameterizedTypeReference> LIST_OF_STRING_TYPE_REFERENCE = new ParameterizedTypeReference<>(){}; - private static final ParameterizedTypeReference> SET_OF_STRING_TYPE_REFERENCE = new ParameterizedTypeReference<>(){}; + private static final ParameterizedTypeReference> VARIANT_SET_TYPE_REFERENCE = new ParameterizedTypeReference<>() {}; + private static final ParameterizedTypeReference> INFO_COLUMNS_META_TYPE_REFERENCE = + new ParameterizedTypeReference<>() {}; + private static final ParameterizedTypeReference> LIST_OF_STRING_TYPE_REFERENCE = new ParameterizedTypeReference<>() {}; + private static final ParameterizedTypeReference> SET_OF_STRING_TYPE_REFERENCE = new ParameterizedTypeReference<>() {}; public GenomicProcessorRestClient(String serviceUrl) { - this.webClient = WebClient.builder() - .baseUrl(serviceUrl) - .defaultHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE) - .build(); + this.webClient = + WebClient.builder().baseUrl(serviceUrl).defaultHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE).build(); } @Override public Mono getPatientMask(DistributableQuery distributableQuery) { - Mono result = webClient.post() - .uri("/patients") - .contentType(MediaType.APPLICATION_JSON) - .body(Mono.just(distributableQuery), DistributableQuery.class) - .retrieve() - .bodyToMono(VariantMask.class); + Mono result = webClient.post().uri("/patients").contentType(MediaType.APPLICATION_JSON) + .body(Mono.just(distributableQuery), DistributableQuery.class).retrieve().bodyToMono(VariantMask.class); return result; } @@ -54,22 +49,14 @@ public VariantMask createMaskForPatientSet(Set patientSubset) { @SuppressWarnings("unchecked") @Override public Mono> getVariantList(DistributableQuery distributableQuery) { - Mono> result = webClient.post() - .uri("/variants") - .contentType(MediaType.APPLICATION_JSON) - .body(Mono.just(distributableQuery), DistributableQuery.class) - .retrieve() - .bodyToMono(VARIANT_SET_TYPE_REFERENCE); + Mono> result = webClient.post().uri("/variants").contentType(MediaType.APPLICATION_JSON) + .body(Mono.just(distributableQuery), DistributableQuery.class).retrieve().bodyToMono(VARIANT_SET_TYPE_REFERENCE); return result; } @Override public List getPatientIds() { - List result = webClient.get() - .uri("/patients/ids") - .retrieve() - .bodyToMono(LIST_OF_STRING_TYPE_REFERENCE) - .block(); + List result = webClient.get().uri("/patients/ids").retrieve().bodyToMono(LIST_OF_STRING_TYPE_REFERENCE).block(); return result; } @@ -80,36 +67,22 @@ public Optional getMasks(String path, VariantBucketHolder< @Override public Set getInfoStoreColumns() { - Set result = webClient.get() - .uri(uriBuilder -> uriBuilder - .path("/info/columns") - .build()) - .retrieve() - .bodyToMono(SET_OF_STRING_TYPE_REFERENCE) - .block(); + Set result = webClient.get().uri(uriBuilder -> uriBuilder.path("/info/columns").build()).retrieve() + .bodyToMono(SET_OF_STRING_TYPE_REFERENCE).block(); return result; } @Override public Set getInfoStoreValues(String conceptPath) { - Set result = webClient.get() - .uri(uriBuilder -> uriBuilder - .path("/info/values") - .queryParam("conceptPath", conceptPath) - .build(conceptPath)) - .retrieve() - .bodyToMono(SET_OF_STRING_TYPE_REFERENCE) - .block(); + Set result = + webClient.get().uri(uriBuilder -> uriBuilder.path("/info/values").queryParam("conceptPath", conceptPath).build(conceptPath)) + .retrieve().bodyToMono(SET_OF_STRING_TYPE_REFERENCE).block(); return result; } @Override public List getInfoColumnMeta() { - List result = webClient.get() - .uri("/info/meta") - .retrieve() - .bodyToMono(INFO_COLUMNS_META_TYPE_REFERENCE) - .block(); + List result = webClient.get().uri("/info/meta").retrieve().bodyToMono(INFO_COLUMNS_META_TYPE_REFERENCE).block(); return result; } diff --git a/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/io/CsvWriter.java b/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/io/CsvWriter.java index 5da10b364..a9123d91a 100644 --- a/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/io/CsvWriter.java +++ b/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/io/CsvWriter.java @@ -40,6 +40,7 @@ public void writeHeader(String[] data) { throw new RuntimeException("IOException while appending to CSV file", e); } } + @Override public void writeEntity(Collection data) { try { @@ -52,14 +53,12 @@ public void writeEntity(Collection data) { @Override public void writeMultiValueEntity(Collection>> data) { List collect = data.stream().map(line -> { - return line.stream() - .map(cell -> { - if (cell == null) { - return ""; - } - return Joiner.on('\t').join(cell); - }) - .toArray(String[]::new); + return line.stream().map(cell -> { + if (cell == null) { + return ""; + } + return Joiner.on('\t').join(cell); + }).toArray(String[]::new); }).toList(); try { csvWriter.write(fileWriter, collect); diff --git a/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/io/ResultWriter.java b/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/io/ResultWriter.java index c0a819299..617c4afa6 100644 --- a/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/io/ResultWriter.java +++ b/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/io/ResultWriter.java @@ -11,6 +11,7 @@ public interface ResultWriter { void writeHeader(String[] data); void writeEntity(Collection data); + void writeMultiValueEntity(Collection>> data); File getFile(); diff --git a/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/patient/PatientProcessor.java b/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/patient/PatientProcessor.java index 5e8db1625..92b9e4ba1 100644 --- a/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/patient/PatientProcessor.java +++ b/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/patient/PatientProcessor.java @@ -24,7 +24,7 @@ public PatientProcessor(AbstractProcessor abstractProcessor) { @Override public String[] getHeaderRow(Query query) { - return new String[]{"PATIENT_NUM"}; + return new String[] {"PATIENT_NUM"}; } @Override @@ -32,9 +32,8 @@ public void runQuery(Query query, AsyncResult asyncResult) { LOG.info("Pulling results for query {}", query.getId()); // floating all this in memory is a bit gross, but the whole list of // patient IDs was already there, so I don't feel too bad - List allPatients = abstractProcessor.getPatientSubsetForQuery(query).stream() - .map(patient -> new String[]{patient.toString()}) - .toList(); + List allPatients = + abstractProcessor.getPatientSubsetForQuery(query).stream().map(patient -> new String[] {patient.toString()}).toList(); LOG.info("Writing results for query {}", query.getId()); asyncResult.appendResults(allPatients); LOG.info("Completed query {}", query.getId()); diff --git a/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/upload/SignUrlService.java b/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/upload/SignUrlService.java index 80b80d341..88325af88 100644 --- a/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/upload/SignUrlService.java +++ b/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/upload/SignUrlService.java @@ -33,10 +33,8 @@ public class SignUrlService { @Autowired public SignUrlService( - @Value("${data-export.s3.bucket-name:}") String bucketName, - @Value("${data-export.s3.region:us-east-1}") String region, - @Value("${data-export.s3.signedUrl-expiry-minutes:60}") int signedUrlExpiryMinutes, - LoggingClient loggingClient + @Value("${data-export.s3.bucket-name:}") String bucketName, @Value("${data-export.s3.region:us-east-1}") String region, + @Value("${data-export.s3.signedUrl-expiry-minutes:60}") int signedUrlExpiryMinutes, LoggingClient loggingClient ) { this.bucketName = bucketName; this.signedUrlExpiryMinutes = signedUrlExpiryMinutes; @@ -45,19 +43,14 @@ public SignUrlService( } public void uploadFile(File file, String objectKey) { - S3Client s3 = S3Client.builder() - .region(this.region) - .build(); + S3Client s3 = S3Client.builder().region(this.region).build(); putS3Object(s3, bucketName, objectKey, file); if (loggingClient != null && loggingClient.isEnabled()) { try { - loggingClient.send(LoggingEvent.builder("DATA_ACCESS") - .action("data.s3.uploaded") - .metadata(Map.of( - "bucket", bucketName, - "object_key", objectKey - )) - .build()); + loggingClient.send( + LoggingEvent.builder("DATA_ACCESS").action("data.s3.uploaded") + .metadata(Map.of("bucket", bucketName, "object_key", objectKey)).build() + ); } catch (Exception e) { log.warn("Failed to send audit log event", e); } @@ -69,11 +62,7 @@ public void uploadFile(File file, String objectKey) { // memory. public void putS3Object(S3Client s3, String bucketName, String objectKey, File file) { Map metadata = new HashMap<>(); - PutObjectRequest putOb = PutObjectRequest.builder() - .bucket(bucketName) - .key(objectKey) - .metadata(metadata) - .build(); + PutObjectRequest putOb = PutObjectRequest.builder().bucket(bucketName).key(objectKey).metadata(metadata).build(); s3.putObject(putOb, RequestBody.fromFile(file)); log.info("Successfully placed " + objectKey + " into bucket " + bucketName); @@ -82,15 +71,12 @@ public void putS3Object(S3Client s3, String bucketName, String objectKey, File f public String createPresignedGetUrl(String keyName) { PresignedGetObjectRequest presignedRequest; try (S3Presigner presigner = S3Presigner.builder().region(region).build()) { - GetObjectRequest objectRequest = GetObjectRequest.builder() - .bucket(bucketName) - .key(keyName) - .build(); + GetObjectRequest objectRequest = GetObjectRequest.builder().bucket(bucketName).key(keyName).build(); - GetObjectPresignRequest presignRequest = GetObjectPresignRequest.builder() - .signatureDuration(Duration.ofMinutes(signedUrlExpiryMinutes)) // The URL will expire in 10 minutes. - .getObjectRequest(objectRequest) - .build(); + GetObjectPresignRequest presignRequest = + GetObjectPresignRequest.builder().signatureDuration(Duration.ofMinutes(signedUrlExpiryMinutes)) // The URL will expire in 10 + // minutes. + .getObjectRequest(objectRequest).build(); presignedRequest = presigner.presignGetObject(presignRequest); } diff --git a/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/v3/CountV3Processor.java b/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/v3/CountV3Processor.java index 3f63d518d..e5d6b2264 100644 --- a/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/v3/CountV3Processor.java +++ b/services/pic-sure-hpds/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/v3/CountV3Processor.java @@ -105,16 +105,13 @@ public Map runCrossCounts(Query query) { } /** - * Returns a separate count for each field in the requiredFields and categoryFilters query. - *

- * The v3 query lets a user add multiple filters on the same variable (e.g. sex=male OR sex=female). Filters are grouped by concept path - * so each variable produces exactly one entry rather than later filters overwriting earlier ones. - *

- * A cross count reports each concept's distribution across the cohort. A category is included when it has members in the cohort, OR when - * it was explicitly named ("called out") by a value filter. This means a value filter never hides cohort members that arrived via an OR - * branch on another concept (sex=male OR age-required still shows females that have an age), while a value the user specifically asked - * for is always shown even when its cohort count is zero. Categories that are neither called out nor present in the cohort are omitted, - * so AND-narrowed concepts do not sprout empty bars. + * Returns a separate count for each field in the requiredFields and categoryFilters query.

The v3 query lets a user add multiple + * filters on the same variable (e.g. sex=male OR sex=female). Filters are grouped by concept path so each variable produces exactly one + * entry rather than later filters overwriting earlier ones.

A cross count reports each concept's distribution across the cohort. A + * category is included when it has members in the cohort, OR when it was explicitly named ("called out") by a value filter. This means + * a value filter never hides cohort members that arrived via an OR branch on another concept (sex=male OR age-required still shows + * females that have an age), while a value the user specifically asked for is always shown even when its cohort count is zero. + * Categories that are neither called out nor present in the cohort are omitted, so AND-narrowed concepts do not sprout empty bars. * * @param query * @return a map of categorical data and their counts @@ -122,8 +119,8 @@ public Map runCrossCounts(Query query) { public Map> runCategoryCrossCounts(Query query) { Set baseQueryPatientSet = queryExecutor.getPatientSubsetForQuery(query); - Map> filtersByConcept = query.allFilters().stream() - .filter(this::isCategoryCrossCountFilter).collect(Collectors.groupingBy(PhenotypicFilter::conceptPath)); + Map> filtersByConcept = query.allFilters().stream().filter(this::isCategoryCrossCountFilter) + .collect(Collectors.groupingBy(PhenotypicFilter::conceptPath)); Map> categoryCounts = new TreeMap<>(); for (Map.Entry> entry : filtersByConcept.entrySet()) { @@ -164,12 +161,11 @@ private boolean isCategoryCrossCountFilter(PhenotypicFilter filter) { } /** - * Returns the distribution of observed values for each continuous concept in the query. - *

- * A continuous concept reports the distribution of every observed value across the cohort. Each filter's min/max only constrains the - * cohort (handled by the query executor); it does not limit which values are shown. So a range filter never hides cohort members that - * arrived via an OR branch on another concept (age>50 OR sex=male still shows the younger ages of the OR'd males), and multiple range - * filters on the same concept collapse to one entry since every patient is counted once from the cube. + * Returns the distribution of observed values for each continuous concept in the query.

A continuous concept reports the + * distribution of every observed value across the cohort. Each filter's min/max only constrains the cohort (handled by the query + * executor); it does not limit which values are shown. So a range filter never hides cohort members that arrived via an OR branch on + * another concept (age>50 OR sex=male still shows the younger ages of the OR'd males), and multiple range filters on the same + * concept collapse to one entry since every patient is counted once from the cube. * * @param query * @return a map of numerical data and their counts @@ -177,8 +173,8 @@ private boolean isCategoryCrossCountFilter(PhenotypicFilter filter) { public Map> runContinuousCrossCounts(Query query) { Set baseQueryPatientSet = queryExecutor.getPatientSubsetForQuery(query); - Set conceptPaths = query.allFilters().stream().filter(this::isContinuousCrossCountFilter) - .map(PhenotypicFilter::conceptPath).collect(Collectors.toCollection(LinkedHashSet::new)); + Set conceptPaths = query.allFilters().stream().filter(this::isContinuousCrossCountFilter).map(PhenotypicFilter::conceptPath) + .collect(Collectors.toCollection(LinkedHashSet::new)); Map> conceptMap = new TreeMap<>(); for (String conceptPath : conceptPaths) { @@ -197,8 +193,7 @@ public Map> runContinuousCrossCounts(Query query) { } private boolean isContinuousCrossCountFilter(PhenotypicFilter phenotypicFilter) { - return Optional.ofNullable(queryExecutor.getDictionary().get(phenotypicFilter.conceptPath())) - .map(meta -> !meta.isCategorical()) + return Optional.ofNullable(queryExecutor.getDictionary().get(phenotypicFilter.conceptPath())).map(meta -> !meta.isCategorical()) .orElse(false); } diff --git a/services/pic-sure-hpds/processing/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/processing/ColumnSorterTest.java b/services/pic-sure-hpds/processing/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/processing/ColumnSorterTest.java index a19a77535..50ed964b5 100644 --- a/services/pic-sure-hpds/processing/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/processing/ColumnSorterTest.java +++ b/services/pic-sure-hpds/processing/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/processing/ColumnSorterTest.java @@ -50,4 +50,4 @@ public void shouldNoOpWithoutConfig() { assertThat(new HashSet<>(expected), is(equalTo(new HashSet<>(actual)))); } -} \ No newline at end of file +} diff --git a/services/pic-sure-hpds/processing/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/processing/CountProcessorTest.java b/services/pic-sure-hpds/processing/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/processing/CountProcessorTest.java index 6db908f13..59ce7f82e 100644 --- a/services/pic-sure-hpds/processing/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/processing/CountProcessorTest.java +++ b/services/pic-sure-hpds/processing/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/processing/CountProcessorTest.java @@ -18,117 +18,89 @@ @ExtendWith(MockitoExtension.class) public class CountProcessorTest { - private final CountProcessor countProcessor; - - private final AbstractProcessor mockAbstractProcessor; - - public CountProcessorTest(@Mock AbstractProcessor mockAbstractProcessor) { - this.mockAbstractProcessor = mockAbstractProcessor; - this.countProcessor = new CountProcessor(mockAbstractProcessor); - } - - @Test - public void testVariantCountWithEmptyQuery() { - Map countResponse = countProcessor.runVariantCount(new Query()); - assertEquals("0",countResponse.get("count") ); - } - - @Test - public void testVariantCountWithEmptyVariantInfoFiltersInQuery() { - Query query = new Query(); - query.setVariantInfoFilters(new ArrayList<>()); - Map countResponse = countProcessor.runVariantCount(query); - assertEquals("0",countResponse.get("count") ); - } - - @Test - public void testVariantCountReturningVariants() throws IOException { - Query query = new Query(); - query.setVariantInfoFilters(List.of(new Query.VariantInfoFilter())); - - when(mockAbstractProcessor.getVariantList(query)).thenReturn(List.of("variant1", "variant2")); - Map countResponse = countProcessor.runVariantCount(query); - assertEquals(2,countResponse.get("count") ); - } - - // todo: test these directly in AbstractProcessor - /* - @Test - public void testVariantCountWithVariantInfoFiltersWithMultipleVariantsWithIntersectingKeys() throws Exception { - ArrayList> data = new ArrayList<>(List.of( - Set.of(1), - Set.of(1, 2))); - TestableCountProcessor t = new TestableCountProcessor(true, data); - - Map categoryVariantInfoFilters = Map.of("FILTERKEY", new String[] { "test1" }); - VariantInfoFilter variantInfoFilter = new VariantInfoFilter(); - variantInfoFilter.categoryVariantInfoFilters = categoryVariantInfoFilters; - - List variantInfoFilters = new ArrayList<>(); - variantInfoFilters.add(variantInfoFilter); - Query q = new Query(); - q.variantInfoFilters = variantInfoFilters; - - Map countResponse = t.runVariantCount(q); - assertEquals(1,countResponse.get("count") ); - } - - @Test - public void testVariantCountWithTwoVariantInfoFiltersWithMultipleVariantsWithIntersectingKeys() throws Exception { - List>> data1 = new ArrayList>>(new ArrayList(List.of( - new ArrayList(List.of(Set.of(1, 2))),new ArrayList(List.of(Set.of(1, 3)))))); - TestableCountProcessor t = new TestableCountProcessor(true, data1); - - Map categoryVariantInfoFilters = Map.of("FILTERKEY", new String[] { "test1" }); - VariantInfoFilter variantInfoFilter = new VariantInfoFilter(); - variantInfoFilter.categoryVariantInfoFilters = categoryVariantInfoFilters; - - VariantInfoFilter variantInfoFilter2 = new VariantInfoFilter(); - variantInfoFilter2.categoryVariantInfoFilters = categoryVariantInfoFilters; - - List variantInfoFilters = new ArrayList<>( - List.of(variantInfoFilter, variantInfoFilter2)); - Query q = new Query(); - q.variantInfoFilters = variantInfoFilters; - - - Map countResponse = t.runVariantCount(q); - assertEquals(3,countResponse.get("count") ); - } - - @Test - public void testVariantCountWithVariantInfoFiltersWithOnlyOneFilterCriteria() throws Exception { - ArrayList> data = new ArrayList(List.of( - Set.of("2,1234,G,T"))); - TestableCountProcessor t = new TestableCountProcessor(true, data); - - Map categoryVariantInfoFilters = Map.of("FILTERKEY", new String[] { "test1" }); - VariantInfoFilter variantInfoFilter = new VariantInfoFilter(); - variantInfoFilter.categoryVariantInfoFilters = categoryVariantInfoFilters; - - List variantInfoFilters = new ArrayList<>(); - variantInfoFilters.add(variantInfoFilter); - Query q = new Query(); - q.variantInfoFilters = variantInfoFilters; - - Map countResponse = t.runVariantCount(q); - assertEquals(1,countResponse.get("count") ); - } - - @Test - public void testVariantCountWithVariantInfoFiltersWhenFiltersDoNotMatchAnyVariants() throws Exception { - TestableCountProcessor t = new TestableCountProcessor(true, new ArrayList>()); - - Map categoryVariantInfoFilters = Map.of("FILTERKEY", new String[] { "test1" }); - VariantInfoFilter variantInfoFilter = new VariantInfoFilter(); - variantInfoFilter.categoryVariantInfoFilters = categoryVariantInfoFilters; - - List variantInfoFilters = new ArrayList<>(); - variantInfoFilters.add(variantInfoFilter); - Query q = new Query(); - - Map countResponse = t.runVariantCount(q); - assertEquals("0",countResponse.get("count") ); - }*/ + private final CountProcessor countProcessor; + + private final AbstractProcessor mockAbstractProcessor; + + public CountProcessorTest(@Mock AbstractProcessor mockAbstractProcessor) { + this.mockAbstractProcessor = mockAbstractProcessor; + this.countProcessor = new CountProcessor(mockAbstractProcessor); + } + + @Test + public void testVariantCountWithEmptyQuery() { + Map countResponse = countProcessor.runVariantCount(new Query()); + assertEquals("0", countResponse.get("count")); + } + + @Test + public void testVariantCountWithEmptyVariantInfoFiltersInQuery() { + Query query = new Query(); + query.setVariantInfoFilters(new ArrayList<>()); + Map countResponse = countProcessor.runVariantCount(query); + assertEquals("0", countResponse.get("count")); + } + + @Test + public void testVariantCountReturningVariants() throws IOException { + Query query = new Query(); + query.setVariantInfoFilters(List.of(new Query.VariantInfoFilter())); + + when(mockAbstractProcessor.getVariantList(query)).thenReturn(List.of("variant1", "variant2")); + Map countResponse = countProcessor.runVariantCount(query); + assertEquals(2, countResponse.get("count")); + } + + // todo: test these directly in AbstractProcessor + /* + * @Test public void testVariantCountWithVariantInfoFiltersWithMultipleVariantsWithIntersectingKeys() throws Exception { + * ArrayList> data = new ArrayList<>(List.of( Set.of(1), Set.of(1, 2))); TestableCountProcessor t = new + * TestableCountProcessor(true, data); + * + * Map categoryVariantInfoFilters = Map.of("FILTERKEY", new String[] { "test1" }); VariantInfoFilter variantInfoFilter + * = new VariantInfoFilter(); variantInfoFilter.categoryVariantInfoFilters = categoryVariantInfoFilters; + * + * List variantInfoFilters = new ArrayList<>(); variantInfoFilters.add(variantInfoFilter); Query q = new Query(); + * q.variantInfoFilters = variantInfoFilters; + * + * Map countResponse = t.runVariantCount(q); assertEquals(1,countResponse.get("count") ); } + * + * @Test public void testVariantCountWithTwoVariantInfoFiltersWithMultipleVariantsWithIntersectingKeys() throws Exception { + * List>> data1 = new ArrayList>>(new ArrayList(List.of( new ArrayList(List.of(Set.of(1, + * 2))),new ArrayList(List.of(Set.of(1, 3)))))); TestableCountProcessor t = new TestableCountProcessor(true, data1); + * + * Map categoryVariantInfoFilters = Map.of("FILTERKEY", new String[] { "test1" }); VariantInfoFilter variantInfoFilter + * = new VariantInfoFilter(); variantInfoFilter.categoryVariantInfoFilters = categoryVariantInfoFilters; + * + * VariantInfoFilter variantInfoFilter2 = new VariantInfoFilter(); variantInfoFilter2.categoryVariantInfoFilters = + * categoryVariantInfoFilters; + * + * List variantInfoFilters = new ArrayList<>( List.of(variantInfoFilter, variantInfoFilter2)); Query q = new Query(); + * q.variantInfoFilters = variantInfoFilters; + * + * + * Map countResponse = t.runVariantCount(q); assertEquals(3,countResponse.get("count") ); } + * + * @Test public void testVariantCountWithVariantInfoFiltersWithOnlyOneFilterCriteria() throws Exception { ArrayList> data = + * new ArrayList(List.of( Set.of("2,1234,G,T"))); TestableCountProcessor t = new TestableCountProcessor(true, data); + * + * Map categoryVariantInfoFilters = Map.of("FILTERKEY", new String[] { "test1" }); VariantInfoFilter variantInfoFilter + * = new VariantInfoFilter(); variantInfoFilter.categoryVariantInfoFilters = categoryVariantInfoFilters; + * + * List variantInfoFilters = new ArrayList<>(); variantInfoFilters.add(variantInfoFilter); Query q = new Query(); + * q.variantInfoFilters = variantInfoFilters; + * + * Map countResponse = t.runVariantCount(q); assertEquals(1,countResponse.get("count") ); } + * + * @Test public void testVariantCountWithVariantInfoFiltersWhenFiltersDoNotMatchAnyVariants() throws Exception { TestableCountProcessor + * t = new TestableCountProcessor(true, new ArrayList>()); + * + * Map categoryVariantInfoFilters = Map.of("FILTERKEY", new String[] { "test1" }); VariantInfoFilter variantInfoFilter + * = new VariantInfoFilter(); variantInfoFilter.categoryVariantInfoFilters = categoryVariantInfoFilters; + * + * List variantInfoFilters = new ArrayList<>(); variantInfoFilters.add(variantInfoFilter); Query q = new Query(); + * + * Map countResponse = t.runVariantCount(q); assertEquals("0",countResponse.get("count") ); } + */ } diff --git a/services/pic-sure-hpds/processing/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/processing/GenomicProcessorParentImplTest.java b/services/pic-sure-hpds/processing/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/processing/GenomicProcessorParentImplTest.java index b1536aea4..85e5fe5f6 100644 --- a/services/pic-sure-hpds/processing/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/processing/GenomicProcessorParentImplTest.java +++ b/services/pic-sure-hpds/processing/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/processing/GenomicProcessorParentImplTest.java @@ -32,9 +32,7 @@ class GenomicProcessorParentImplTest { @BeforeEach public void setup() { - parentProcessor = new GenomicProcessorParentImpl(List.of( - mockProcessor1, mockProcessor2, mockProcessor3 - )); + parentProcessor = new GenomicProcessorParentImpl(List.of(mockProcessor1, mockProcessor2, mockProcessor3)); } @Test @@ -46,6 +44,7 @@ public void patientIdInit_patientsMatch_noException() { mockProcessor1, mockProcessor2, mockProcessor3 )); } + @Test public void patientIdInit_patientsDiffer_exception() { when(mockProcessor1.getPatientIds()).thenReturn(List.of("1", "42", "99")); @@ -62,18 +61,23 @@ public void patientIdInit_patientsDiffer_exception() { @Test public void getPatientMask_validResponses_returnMerged() { DistributableQuery distributableQuery = new DistributableQuery(); - when(mockProcessor1.getPatientMask(distributableQuery)).thenReturn(Mono.just(new VariantMaskBitmaskImpl(new BigInteger("110110000011", 2)))); - when(mockProcessor2.getPatientMask(distributableQuery)).thenReturn(Mono.just(new VariantMaskBitmaskImpl(new BigInteger("110001100011", 2)))); - when(mockProcessor3.getPatientMask(distributableQuery)).thenReturn(Mono.just(new VariantMaskBitmaskImpl(new BigInteger("110000000111", 2)))); + when(mockProcessor1.getPatientMask(distributableQuery)) + .thenReturn(Mono.just(new VariantMaskBitmaskImpl(new BigInteger("110110000011", 2)))); + when(mockProcessor2.getPatientMask(distributableQuery)) + .thenReturn(Mono.just(new VariantMaskBitmaskImpl(new BigInteger("110001100011", 2)))); + when(mockProcessor3.getPatientMask(distributableQuery)) + .thenReturn(Mono.just(new VariantMaskBitmaskImpl(new BigInteger("110000000111", 2)))); VariantMask patientMask = parentProcessor.getPatientMask(distributableQuery).block(); VariantMask expectedPatientMask = new VariantMaskBitmaskImpl(new BigInteger("110111100111", 2)); assertEquals(expectedPatientMask, patientMask); } + @Test public void getPatientMask_oneNode_returnPatients() { DistributableQuery distributableQuery = new DistributableQuery(); - when(mockProcessor1.getPatientMask(distributableQuery)).thenReturn(Mono.just(new VariantMaskBitmaskImpl(new BigInteger("110110000011", 2)))); + when(mockProcessor1.getPatientMask(distributableQuery)) + .thenReturn(Mono.just(new VariantMaskBitmaskImpl(new BigInteger("110110000011", 2)))); parentProcessor = new GenomicProcessorParentImpl(List.of(mockProcessor1)); VariantMask patientMask = parentProcessor.getPatientMask(distributableQuery).block(); @@ -153,4 +157,4 @@ public void getVariantMetadata_overlappingVariants_mergedCorrectly() { assertEquals(Set.of("metadata31", "metadata32"), variantMetadata.get("variant3")); assertEquals(2, variantMetadata.size()); } -} \ No newline at end of file +} diff --git a/services/pic-sure-hpds/processing/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/processing/GenomicProcessorPatientMergingParentImplTest.java b/services/pic-sure-hpds/processing/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/processing/GenomicProcessorPatientMergingParentImplTest.java index a7148bda0..aa1407cce 100644 --- a/services/pic-sure-hpds/processing/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/processing/GenomicProcessorPatientMergingParentImplTest.java +++ b/services/pic-sure-hpds/processing/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/processing/GenomicProcessorPatientMergingParentImplTest.java @@ -39,19 +39,20 @@ public class GenomicProcessorPatientMergingParentImplTest { @BeforeEach public void setup() { - patientMergingParent = new GenomicProcessorPatientMergingParentImpl(List.of( - mockProcessor1, mockProcessor2, mockProcessor3 - )); + patientMergingParent = new GenomicProcessorPatientMergingParentImpl(List.of(mockProcessor1, mockProcessor2, mockProcessor3)); } @Test public void getPatientMask_validResponses_returnMerged() { DistributableQuery distributableQuery = new DistributableQuery(); - when(mockProcessor1.getPatientMask(distributableQuery)).thenReturn(Mono.just(new VariantMaskBitmaskImpl(new BigInteger("11011011", 2)))); + when(mockProcessor1.getPatientMask(distributableQuery)) + .thenReturn(Mono.just(new VariantMaskBitmaskImpl(new BigInteger("11011011", 2)))); when(mockProcessor1.getPatientIds()).thenReturn(List.of("1", "2", "3", "4")); - when(mockProcessor2.getPatientMask(distributableQuery)).thenReturn(Mono.just(new VariantMaskBitmaskImpl(new BigInteger("110001100011", 2)))); + when(mockProcessor2.getPatientMask(distributableQuery)) + .thenReturn(Mono.just(new VariantMaskBitmaskImpl(new BigInteger("110001100011", 2)))); when(mockProcessor2.getPatientIds()).thenReturn(List.of("5", "6", "7", "8", "9", "10", "11", "12")); - when(mockProcessor3.getPatientMask(distributableQuery)).thenReturn(Mono.just(new VariantMaskBitmaskImpl(new BigInteger("11000111", 2)))); + when(mockProcessor3.getPatientMask(distributableQuery)) + .thenReturn(Mono.just(new VariantMaskBitmaskImpl(new BigInteger("11000111", 2)))); when(mockProcessor3.getPatientIds()).thenReturn(List.of("15", "16", "17", "18")); VariantMask patientMask = patientMergingParent.getPatientMask(distributableQuery).block(); VariantMask expectedPatientMask = new VariantMaskBitmaskImpl(new BigInteger("11000100011000011011", 2)); @@ -61,11 +62,14 @@ public void getPatientMask_validResponses_returnMerged() { @Test public void getPatientMask_noPatientResponses_returnMerged() { DistributableQuery distributableQuery = new DistributableQuery(); - when(mockProcessor1.getPatientMask(distributableQuery)).thenReturn(Mono.just(new VariantMaskBitmaskImpl(new BigInteger("11011011", 2)))); + when(mockProcessor1.getPatientMask(distributableQuery)) + .thenReturn(Mono.just(new VariantMaskBitmaskImpl(new BigInteger("11011011", 2)))); when(mockProcessor1.getPatientIds()).thenReturn(List.of("1", "2", "3", "4")); - when(mockProcessor2.getPatientMask(distributableQuery)).thenReturn(Mono.just(new VariantMaskBitmaskImpl(new BigInteger("110000000011", 2)))); + when(mockProcessor2.getPatientMask(distributableQuery)) + .thenReturn(Mono.just(new VariantMaskBitmaskImpl(new BigInteger("110000000011", 2)))); when(mockProcessor2.getPatientIds()).thenReturn(List.of("5", "6", "7", "8", "9", "10", "11", "12")); - when(mockProcessor3.getPatientMask(distributableQuery)).thenReturn(Mono.just(new VariantMaskBitmaskImpl(new BigInteger("11000011", 2)))); + when(mockProcessor3.getPatientMask(distributableQuery)) + .thenReturn(Mono.just(new VariantMaskBitmaskImpl(new BigInteger("11000011", 2)))); when(mockProcessor3.getPatientIds()).thenReturn(List.of("15", "16", "17", "18")); VariantMask patientMask = patientMergingParent.getPatientMask(distributableQuery).block(); VariantMask expectedPatientMask = new VariantMaskBitmaskImpl(new BigInteger("11000000000000011011", 2)); @@ -75,11 +79,14 @@ public void getPatientMask_noPatientResponses_returnMerged() { @Test public void getPatientMask_emptyResponses_returnMerged() { DistributableQuery distributableQuery = new DistributableQuery(); - when(mockProcessor1.getPatientMask(distributableQuery)).thenReturn(Mono.just(new VariantMaskBitmaskImpl(new BigInteger("11011011", 2)))); + when(mockProcessor1.getPatientMask(distributableQuery)) + .thenReturn(Mono.just(new VariantMaskBitmaskImpl(new BigInteger("11011011", 2)))); when(mockProcessor1.getPatientIds()).thenReturn(List.of("1", "2", "3", "4")); - when(mockProcessor2.getPatientMask(distributableQuery)).thenReturn(Mono.just(new VariantMaskBitmaskImpl(new BigInteger("1111", 2)))); + when(mockProcessor2.getPatientMask(distributableQuery)) + .thenReturn(Mono.just(new VariantMaskBitmaskImpl(new BigInteger("1111", 2)))); when(mockProcessor2.getPatientIds()).thenReturn(List.of()); - when(mockProcessor3.getPatientMask(distributableQuery)).thenReturn(Mono.just(new VariantMaskBitmaskImpl(new BigInteger("11000111", 2)))); + when(mockProcessor3.getPatientMask(distributableQuery)) + .thenReturn(Mono.just(new VariantMaskBitmaskImpl(new BigInteger("11000111", 2)))); when(mockProcessor3.getPatientIds()).thenReturn(List.of("5", "6", "7", "8")); VariantMask patientMask = patientMergingParent.getPatientMask(distributableQuery).block(); VariantMask expectedPatientMask = new VariantMaskBitmaskImpl(new BigInteger("110001011011", 2)); @@ -98,6 +105,7 @@ public void patientIdInit_validPatients_noException(CapturedOutput output) { assertFalse(output.getOut().contains("duplicate patients found in patient partitions")); } + @Test public void patientIdInit_invalidPatients_warnMessage(CapturedOutput output) { when(mockProcessor1.getPatientIds()).thenReturn(List.of("1", "42", "99")); @@ -110,6 +118,7 @@ public void patientIdInit_invalidPatients_warnMessage(CapturedOutput output) { assertTrue(output.getOut().contains("1 duplicate patients found in patient partitions")); } + @Test public void patientIdInit_multipleInvalidPatients_warnMessage(CapturedOutput output) { when(mockProcessor1.getPatientIds()).thenReturn(List.of("1", "42", "99")); @@ -156,12 +165,9 @@ public void createMaskForPatientSet_validResponsesOneEmpty_returnMerged() { @Test public void getMasks_validEmptyResponses_returnEmpty() { String path = "chr21,5032061,A,Z,LOC102723996,missense_variant"; - when(mockProcessor1.getMasks(eq(path), any(VariantBucketHolder.class))) - .thenReturn(Optional.empty()); - when(mockProcessor2.getMasks(eq(path), any(VariantBucketHolder.class))) - .thenReturn(Optional.empty()); - when(mockProcessor3.getMasks(eq(path), any(VariantBucketHolder.class))) - .thenReturn(Optional.empty()); + when(mockProcessor1.getMasks(eq(path), any(VariantBucketHolder.class))).thenReturn(Optional.empty()); + when(mockProcessor2.getMasks(eq(path), any(VariantBucketHolder.class))).thenReturn(Optional.empty()); + when(mockProcessor3.getMasks(eq(path), any(VariantBucketHolder.class))).thenReturn(Optional.empty()); Optional masks = patientMergingParent.getMasks(path, new VariantBucketHolder<>()); assertEquals(Optional.of(new VariableVariantMasks()), masks); @@ -170,12 +176,9 @@ public void getMasks_validEmptyResponses_returnEmpty() { @Test public void getMasks_validEmptyAndNullResponses_returnEmpty() { String path = "chr21,5032061,A,Z,LOC102723996,missense_variant"; - when(mockProcessor1.getMasks(eq(path), any(VariantBucketHolder.class))) - .thenReturn(Optional.empty()); - when(mockProcessor2.getMasks(eq(path), any(VariantBucketHolder.class))) - .thenReturn(Optional.of(new VariableVariantMasks())); - when(mockProcessor3.getMasks(eq(path), any(VariantBucketHolder.class))) - .thenReturn(Optional.empty()); + when(mockProcessor1.getMasks(eq(path), any(VariantBucketHolder.class))).thenReturn(Optional.empty()); + when(mockProcessor2.getMasks(eq(path), any(VariantBucketHolder.class))).thenReturn(Optional.of(new VariableVariantMasks())); + when(mockProcessor3.getMasks(eq(path), any(VariantBucketHolder.class))).thenReturn(Optional.empty()); Optional masks = patientMergingParent.getMasks(path, new VariantBucketHolder<>()); assertEquals(Optional.of(new VariableVariantMasks()), masks); @@ -197,18 +200,16 @@ public void getMasks_validResponses_returnMerged() { when(mockProcessor3.getPatientIds()).thenReturn(List.of("15", "16", "17", "18")); - when(mockProcessor1.getMasks(eq(path), any(VariantBucketHolder.class))) - .thenReturn(Optional.of(variableVariantMasks1)); - when(mockProcessor2.getMasks(eq(path), any(VariantBucketHolder.class))) - .thenReturn(Optional.of(variableVariantMasks2)); - when(mockProcessor3.getMasks(eq(path), any(VariantBucketHolder.class))) - .thenReturn(Optional.of(variableVariantMasks3)); + when(mockProcessor1.getMasks(eq(path), any(VariantBucketHolder.class))).thenReturn(Optional.of(variableVariantMasks1)); + when(mockProcessor2.getMasks(eq(path), any(VariantBucketHolder.class))).thenReturn(Optional.of(variableVariantMasks2)); + when(mockProcessor3.getMasks(eq(path), any(VariantBucketHolder.class))).thenReturn(Optional.of(variableVariantMasks3)); Optional masks = patientMergingParent.getMasks(path, new VariantBucketHolder<>()); VariantMask expectedPatientMask = new VariantMaskBitmaskImpl(new BigInteger("11000100011000011011", 2)); assertEquals(expectedPatientMask, masks.get().heterozygousMask); } + @Test public void getMasks_validResponsesSinglePartition_returnResult() { String path = "chr21,5032061,A,Z,LOC102723996,missense_variant"; @@ -218,10 +219,10 @@ public void getMasks_validResponsesSinglePartition_returnResult() { when(mockProcessor1.getPatientIds()).thenReturn(List.of("1", "2", "3", "4")); - when(mockProcessor1.getMasks(eq(path), any(VariantBucketHolder.class))) - .thenReturn(Optional.of(variableVariantMasks1)); + when(mockProcessor1.getMasks(eq(path), any(VariantBucketHolder.class))).thenReturn(Optional.of(variableVariantMasks1)); - Optional masks = new GenomicProcessorPatientMergingParentImpl(List.of(mockProcessor1)).getMasks(path, new VariantBucketHolder<>()); + Optional masks = + new GenomicProcessorPatientMergingParentImpl(List.of(mockProcessor1)).getMasks(path, new VariantBucketHolder<>()); assertEquals(variableVariantMasks1.heterozygousMask, masks.get().heterozygousMask); } @@ -231,10 +232,10 @@ public void getMasks_validResponsesSinglePartitionEmpty_returnEmpty() { String path = "chr21,5032061,A,Z,LOC102723996,missense_variant"; when(mockProcessor1.getPatientIds()).thenReturn(List.of("1", "2", "3", "4")); - when(mockProcessor1.getMasks(eq(path), any(VariantBucketHolder.class))) - .thenReturn(Optional.empty()); + when(mockProcessor1.getMasks(eq(path), any(VariantBucketHolder.class))).thenReturn(Optional.empty()); - Optional masks = new GenomicProcessorPatientMergingParentImpl(List.of(mockProcessor1)).getMasks(path, new VariantBucketHolder<>()); + Optional masks = + new GenomicProcessorPatientMergingParentImpl(List.of(mockProcessor1)).getMasks(path, new VariantBucketHolder<>()); assertNull(masks.get().heterozygousMask); } @@ -252,12 +253,9 @@ public void getMasks_validAndEmptyResponses_returnMerged() { when(mockProcessor3.getPatientIds()).thenReturn(List.of("15", "16", "17", "18")); - when(mockProcessor1.getMasks(eq(path), any(VariantBucketHolder.class))) - .thenReturn(Optional.of(variableVariantMasks1)); - when(mockProcessor2.getMasks(eq(path), any(VariantBucketHolder.class))) - .thenReturn(Optional.empty()); - when(mockProcessor3.getMasks(eq(path), any(VariantBucketHolder.class))) - .thenReturn(Optional.of(variableVariantMasks3)); + when(mockProcessor1.getMasks(eq(path), any(VariantBucketHolder.class))).thenReturn(Optional.of(variableVariantMasks1)); + when(mockProcessor2.getMasks(eq(path), any(VariantBucketHolder.class))).thenReturn(Optional.empty()); + when(mockProcessor3.getMasks(eq(path), any(VariantBucketHolder.class))).thenReturn(Optional.of(variableVariantMasks3)); Optional masks = patientMergingParent.getMasks(path, new VariantBucketHolder<>()); @@ -280,12 +278,9 @@ public void getMasks_validResponsesHomozygous_returnMerged() { when(mockProcessor3.getPatientIds()).thenReturn(List.of("15", "16", "17", "18")); - when(mockProcessor1.getMasks(eq(path), any(VariantBucketHolder.class))) - .thenReturn(Optional.of(variableVariantMasks1)); - when(mockProcessor2.getMasks(eq(path), any(VariantBucketHolder.class))) - .thenReturn(Optional.of(variableVariantMasks2)); - when(mockProcessor3.getMasks(eq(path), any(VariantBucketHolder.class))) - .thenReturn(Optional.of(variableVariantMasks3)); + when(mockProcessor1.getMasks(eq(path), any(VariantBucketHolder.class))).thenReturn(Optional.of(variableVariantMasks1)); + when(mockProcessor2.getMasks(eq(path), any(VariantBucketHolder.class))).thenReturn(Optional.of(variableVariantMasks2)); + when(mockProcessor3.getMasks(eq(path), any(VariantBucketHolder.class))).thenReturn(Optional.of(variableVariantMasks3)); Optional masks = patientMergingParent.getMasks(path, new VariantBucketHolder<>()); @@ -332,4 +327,4 @@ public void getVariantMetadata_overlappingVariants_mergedCorrectly() { assertEquals(Set.of("metadata31", "metadata32"), variantMetadata.get("variant3")); assertEquals(2, variantMetadata.size()); } -} \ No newline at end of file +} diff --git a/services/pic-sure-hpds/processing/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/processing/PatientVariantJoinHandlerTest.java b/services/pic-sure-hpds/processing/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/processing/PatientVariantJoinHandlerTest.java index 1ab10a7b7..ffc1adaca 100644 --- a/services/pic-sure-hpds/processing/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/processing/PatientVariantJoinHandlerTest.java +++ b/services/pic-sure-hpds/processing/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/processing/PatientVariantJoinHandlerTest.java @@ -29,7 +29,9 @@ public class PatientVariantJoinHandlerTest { public static final String[] PATIENT_IDS = {"101", "102", "103", "104", "105", "106", "107", "108"}; public static final Set PATIENT_IDS_INTEGERS = Set.of(PATIENT_IDS).stream().map(Integer::parseInt).collect(Collectors.toSet()); - public static final String[] VARIANT_INDEX = {"16,61642243,A,T,ABC,consequence1", "16,61642252,A,G,ABC,consequence1", "16,61642256,C,T,ABC,consequence2", "16,61642257,G,A,ABC,consequence3", "16,61642258,G,A,ABC,consequence3", "16,61642259,G,A,ABC,consequence1", "16,61642260,G,A,ABC,consequence1", "16,61642261,G,A,ABC,consequence1"}; + public static final String[] VARIANT_INDEX = {"16,61642243,A,T,ABC,consequence1", "16,61642252,A,G,ABC,consequence1", + "16,61642256,C,T,ABC,consequence2", "16,61642257,G,A,ABC,consequence3", "16,61642258,G,A,ABC,consequence3", + "16,61642259,G,A,ABC,consequence1", "16,61642260,G,A,ABC,consequence1", "16,61642261,G,A,ABC,consequence1"}; public PatientVariantJoinHandlerTest(@Mock VariantService variantService) { this.variantService = variantService; @@ -43,7 +45,8 @@ public void getPatientIdsForIntersectionOfVariantSets_allPatientsMatchOneVariant when(variantService.getPatientIds()).thenReturn(PATIENT_IDS); when(variantService.emptyBitmask()).thenReturn(emptyBitmask(PATIENT_IDS)); - VariantMask maskForAllPatients = new VariantMaskBitmaskImpl(patientVariantJoinHandler.createMaskForPatientSet(PATIENT_IDS_INTEGERS)); + VariantMask maskForAllPatients = + new VariantMaskBitmaskImpl(patientVariantJoinHandler.createMaskForPatientSet(PATIENT_IDS_INTEGERS)); VariantMask maskForNoPatients = VariantMask.emptyInstance(); VariableVariantMasks variantMasks = new VariableVariantMasks(); @@ -54,7 +57,8 @@ public void getPatientIdsForIntersectionOfVariantSets_allPatientsMatchOneVariant when(variantService.getMasks(eq(VARIANT_INDEX[2]), any())).thenReturn(Optional.of(emptyVariantMasks)); when(variantService.getMasks(eq(VARIANT_INDEX[4]), any())).thenReturn(Optional.of(emptyVariantMasks)); - Set patientIdsForIntersectionOfVariantSets = patientVariantJoinHandler.getPatientIdsForIntersectionOfVariantSets(null, intersectionOfInfoFilters).patientMaskToPatientIdSet(List.of(PATIENT_IDS)); + Set patientIdsForIntersectionOfVariantSets = patientVariantJoinHandler + .getPatientIdsForIntersectionOfVariantSets(null, intersectionOfInfoFilters).patientMaskToPatientIdSet(List.of(PATIENT_IDS)); // this should be all patients, as all patients match one of the variants assertEquals(PATIENT_IDS_INTEGERS, patientIdsForIntersectionOfVariantSets); } @@ -65,7 +69,8 @@ public void getPatientIdsForIntersectionOfVariantSets_allPatientsMatchOneVariant when(variantService.getPatientIds()).thenReturn(PATIENT_IDS); when(variantService.emptyBitmask()).thenReturn(emptyBitmask(PATIENT_IDS)); - VariantMask maskForAllPatients = new VariantMaskBitmaskImpl(patientVariantJoinHandler.createMaskForPatientSet(PATIENT_IDS_INTEGERS)); + VariantMask maskForAllPatients = + new VariantMaskBitmaskImpl(patientVariantJoinHandler.createMaskForPatientSet(PATIENT_IDS_INTEGERS)); VariantMask maskForNoPatients = VariantMask.emptyInstance(); VariableVariantMasks variantMasks = new VariableVariantMasks(); @@ -76,7 +81,8 @@ public void getPatientIdsForIntersectionOfVariantSets_allPatientsMatchOneVariant when(variantService.getMasks(eq(VARIANT_INDEX[2]), any())).thenReturn(Optional.empty()); when(variantService.getMasks(eq(VARIANT_INDEX[4]), any())).thenReturn(Optional.empty()); - Set patientIdsForIntersectionOfVariantSets = patientVariantJoinHandler.getPatientIdsForIntersectionOfVariantSets(null, intersectionOfInfoFilters).patientMaskToPatientIdSet(List.of(PATIENT_IDS)); + Set patientIdsForIntersectionOfVariantSets = patientVariantJoinHandler + .getPatientIdsForIntersectionOfVariantSets(null, intersectionOfInfoFilters).patientMaskToPatientIdSet(List.of(PATIENT_IDS)); // this should be all patients, as all patients match one of the variants assertEquals(PATIENT_IDS_INTEGERS, patientIdsForIntersectionOfVariantSets); } @@ -94,7 +100,8 @@ public void getPatientIdsForIntersectionOfVariantSets_noPatientsMatchVariants() when(variantService.getMasks(eq(VARIANT_INDEX[2]), any())).thenReturn(Optional.of(emptyVariantMasks)); when(variantService.getMasks(eq(VARIANT_INDEX[4]), any())).thenReturn(Optional.of(emptyVariantMasks)); - Set patientIdsForIntersectionOfVariantSets = patientVariantJoinHandler.getPatientIdsForIntersectionOfVariantSets(Set.of(), intersectionOfInfoFilters).patientMaskToPatientIdSet(List.of(PATIENT_IDS)); + Set patientIdsForIntersectionOfVariantSets = patientVariantJoinHandler + .getPatientIdsForIntersectionOfVariantSets(Set.of(), intersectionOfInfoFilters).patientMaskToPatientIdSet(List.of(PATIENT_IDS)); // this should be empty because all variants masks have no matching patients assertEquals(Set.of(), patientIdsForIntersectionOfVariantSets); } @@ -114,7 +121,8 @@ public void getPatientIdsForIntersectionOfVariantSets_somePatientsMatchVariants( when(variantService.getMasks(eq(VARIANT_INDEX[0]), any())).thenReturn(Optional.of(variantMasks)); when(variantService.getMasks(eq(VARIANT_INDEX[2]), any())).thenReturn(Optional.of(variantMasks2)); - Set patientIdsForIntersectionOfVariantSets = patientVariantJoinHandler.getPatientIdsForIntersectionOfVariantSets(null, intersectionOfInfoFilters).patientMaskToPatientIdSet(List.of(PATIENT_IDS)); + Set patientIdsForIntersectionOfVariantSets = patientVariantJoinHandler + .getPatientIdsForIntersectionOfVariantSets(null, intersectionOfInfoFilters).patientMaskToPatientIdSet(List.of(PATIENT_IDS)); // this should be all patients who match at least one variant assertEquals(Set.of(101, 103, 105), patientIdsForIntersectionOfVariantSets); } @@ -124,7 +132,8 @@ public void getPatientIdsForIntersectionOfVariantSets_noVariants() { VariantIndex intersectionOfInfoFilters = VariantIndex.empty(); when(variantService.getPatientIds()).thenReturn(PATIENT_IDS); - Set patientIdsForIntersectionOfVariantSets = patientVariantJoinHandler.getPatientIdsForIntersectionOfVariantSets(null, intersectionOfInfoFilters).patientMaskToPatientIdSet(List.of(PATIENT_IDS)); + Set patientIdsForIntersectionOfVariantSets = patientVariantJoinHandler + .getPatientIdsForIntersectionOfVariantSets(null, intersectionOfInfoFilters).patientMaskToPatientIdSet(List.of(PATIENT_IDS)); // this should be empty, as there are no variants assertEquals(Set.of(), patientIdsForIntersectionOfVariantSets); } @@ -144,8 +153,11 @@ public void getPatientIdsForIntersectionOfVariantSets_patientSubsetPassed() { when(variantService.getMasks(eq(VARIANT_INDEX[0]), any())).thenReturn(Optional.of(variantMasks)); when(variantService.getMasks(eq(VARIANT_INDEX[2]), any())).thenReturn(Optional.of(variantMasks2)); - Set patientIdsForIntersectionOfVariantSets = patientVariantJoinHandler.getPatientIdsForIntersectionOfVariantSets(Set.of(102, 103, 104, 105, 106), intersectionOfInfoFilters).patientMaskToPatientIdSet(List.of(PATIENT_IDS)); - // this should be the union of patients matching variants (101, 103, 105, 107), intersected with the patient subset parameter (103, 104, 105) which is (103, 105) + Set patientIdsForIntersectionOfVariantSets = + patientVariantJoinHandler.getPatientIdsForIntersectionOfVariantSets(Set.of(102, 103, 104, 105, 106), intersectionOfInfoFilters) + .patientMaskToPatientIdSet(List.of(PATIENT_IDS)); + // this should be the union of patients matching variants (101, 103, 105, 107), intersected with the patient subset parameter (103, + // 104, 105) which is (103, 105) assertEquals(Set.of(103, 105), patientIdsForIntersectionOfVariantSets); } @@ -156,4 +168,4 @@ public BigInteger emptyBitmask(String[] patientIds) { } return new BigInteger("11" + emptyVariantMask + "11", 2); } -} \ No newline at end of file +} diff --git a/services/pic-sure-hpds/processing/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/processing/VariantIndexTest.java b/services/pic-sure-hpds/processing/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/processing/VariantIndexTest.java index da51723e2..61be5e369 100644 --- a/services/pic-sure-hpds/processing/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/processing/VariantIndexTest.java +++ b/services/pic-sure-hpds/processing/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/processing/VariantIndexTest.java @@ -27,22 +27,25 @@ public void testSparseVariantIntersection() { assertEquals(intersection.getClass(), SparseVariantIndex.class); assertEquals(Set.of(3, 5), ((SparseVariantIndex) intersection).getVariantIds()); } + @Test public void testDenseVariantUnion() { - DenseVariantIndex denseVariantIndex1 = new DenseVariantIndex(new boolean[]{true, false, true, false}); - DenseVariantIndex denseVariantIndex2 = new DenseVariantIndex(new boolean[]{true, false, false, true}); + DenseVariantIndex denseVariantIndex1 = new DenseVariantIndex(new boolean[] {true, false, true, false}); + DenseVariantIndex denseVariantIndex2 = new DenseVariantIndex(new boolean[] {true, false, false, true}); VariantIndex union = denseVariantIndex1.union(denseVariantIndex2); assertEquals(union.getClass(), DenseVariantIndex.class); - assertArrayEquals(new boolean[]{true, false, true, true}, ((DenseVariantIndex) union).getVariantIndexMask()); + assertArrayEquals(new boolean[] {true, false, true, true}, ((DenseVariantIndex) union).getVariantIndexMask()); } + @Test public void testDenseVariantIntersection() { - DenseVariantIndex denseVariantIndex1 = new DenseVariantIndex(new boolean[]{true, false, true, false}); - DenseVariantIndex denseVariantIndex2 = new DenseVariantIndex(new boolean[]{true, false, false, true}); + DenseVariantIndex denseVariantIndex1 = new DenseVariantIndex(new boolean[] {true, false, true, false}); + DenseVariantIndex denseVariantIndex2 = new DenseVariantIndex(new boolean[] {true, false, false, true}); VariantIndex intersection = denseVariantIndex1.intersection(denseVariantIndex2); assertEquals(intersection.getClass(), DenseVariantIndex.class); - assertArrayEquals(new boolean[]{true, false, false, false}, ((DenseVariantIndex) intersection).getVariantIndexMask()); + assertArrayEquals(new boolean[] {true, false, false, false}, ((DenseVariantIndex) intersection).getVariantIndexMask()); } + @Test public void testSparseAndDenseUnion() { SparseVariantIndex sparseVariantIndex1 = new SparseVariantIndex(Set.of(0, 2)); @@ -51,6 +54,7 @@ public void testSparseAndDenseUnion() { assertEquals(union.getClass(), DenseVariantIndex.class); assertArrayEquals(new boolean[] {true, true, true, false}, ((DenseVariantIndex) union).getVariantIndexMask()); } + @Test public void testSparseAndDenseIntersection() { SparseVariantIndex sparseVariantIndex1 = new SparseVariantIndex(Set.of(0, 2)); diff --git a/services/pic-sure-hpds/processing/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/processing/dictionary/ConceptTest.java b/services/pic-sure-hpds/processing/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/processing/dictionary/ConceptTest.java index a9dd4b4ed..7ea336751 100644 --- a/services/pic-sure-hpds/processing/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/processing/dictionary/ConceptTest.java +++ b/services/pic-sure-hpds/processing/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/processing/dictionary/ConceptTest.java @@ -13,7 +13,9 @@ public class ConceptTest { @Test public void jsonSerialization() throws JsonProcessingException { - Concept[] concepts = new Concept[]{new Concept("\\demographics\\age\\", "age", "AGE", null, "patient age", Map.of("drs_uri", "[\"a-drs.uri\", \"another-drs.uri\"]"))}; + Concept[] concepts = new Concept[] {new Concept( + "\\demographics\\age\\", "age", "AGE", null, "patient age", Map.of("drs_uri", "[\"a-drs.uri\", \"another-drs.uri\"]") + )}; ObjectMapper objectMapper = new ObjectMapper(); String serialized = objectMapper.writeValueAsString(concepts); @@ -21,4 +23,4 @@ public void jsonSerialization() throws JsonProcessingException { assertEquals(List.of(concepts), List.of(deserialized)); } -} \ No newline at end of file +} diff --git a/services/pic-sure-hpds/processing/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/processing/genomic/GenomicProcessorRestClientTest.java b/services/pic-sure-hpds/processing/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/processing/genomic/GenomicProcessorRestClientTest.java index ff24f62fd..0957d0085 100644 --- a/services/pic-sure-hpds/processing/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/processing/genomic/GenomicProcessorRestClientTest.java +++ b/services/pic-sure-hpds/processing/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/processing/genomic/GenomicProcessorRestClientTest.java @@ -28,10 +28,8 @@ public void simpleTest() { List variantInfoFilters = new ArrayList<>(); Query.VariantInfoFilter variantInfoFilter = new Query.VariantInfoFilter(); - variantInfoFilter.categoryVariantInfoFilters = Map.of( - "Gene_with_variant", new String[]{"BRCA1"}, - "Variant_consequence_calculated", new String[]{"splice_donor_variant"} - ); + variantInfoFilter.categoryVariantInfoFilters = + Map.of("Gene_with_variant", new String[] {"BRCA1"}, "Variant_consequence_calculated", new String[] {"splice_donor_variant"}); variantInfoFilters.add(variantInfoFilter); distributableQuery.setVariantInfoFilters(variantInfoFilters); Set patientIds = IntStream.range(53000, 53635).boxed().collect(Collectors.toSet()); @@ -46,6 +44,7 @@ public void getInfoStoreColumns() { Set infoStoreColumns = genomicProcessorRestClient.getInfoStoreColumns(); assertTrue(infoStoreColumns.contains("Variant_consequence_calculated")); } + @Test public void getInfoStoreValues() { Set infoStoreValues = genomicProcessorRestClient.getInfoStoreValues("Variant_consequence_calculated"); @@ -62,4 +61,4 @@ public void getInfoColumnMeta() { } throw new RuntimeException("Variant_consequence_calculated not found in info column meta"); } -} \ No newline at end of file +} diff --git a/services/pic-sure-hpds/processing/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/processing/io/PfbWriterTest.java b/services/pic-sure-hpds/processing/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/processing/io/PfbWriterTest.java index 36aa67b3c..93437aa0e 100644 --- a/services/pic-sure-hpds/processing/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/processing/io/PfbWriterTest.java +++ b/services/pic-sure-hpds/processing/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/processing/io/PfbWriterTest.java @@ -27,24 +27,28 @@ public class PfbWriterTest { public void writeValidPFB() { PfbWriter pfbWriter = new PfbWriter(new File("target/test-result.avro"), UUID.randomUUID().toString(), dictionaryService); - Mockito.when(dictionaryService.getConcepts(List.of("patient_id", "\\demographics\\age\\", "\\phs123\\stroke\\"))) - .thenReturn(List.of(new Concept("\\demographics\\age\\", "age", "AGE", null, "patient age", Map.of("drs_uri", "[\"a-drs.uri\", \"another-drs.uri\"]")))); + Mockito.when(dictionaryService.getConcepts(List.of("patient_id", "\\demographics\\age\\", "\\phs123\\stroke\\"))).thenReturn( + List.of( + new Concept( + "\\demographics\\age\\", "age", "AGE", null, "patient age", Map.of("drs_uri", "[\"a-drs.uri\", \"another-drs.uri\"]") + ) + ) + ); pfbWriter.writeHeader(new String[] {"patient_id", "\\demographics\\age\\", "\\phs123\\stroke\\"}); List> nullableList = new ArrayList<>(); nullableList.add(List.of("123")); nullableList.add(null); nullableList.add(List.of("Y")); - pfbWriter.writeMultiValueEntity(List.of( - nullableList, - List.of(List.of("456"), List.of("80") ,List.of("N", "Y")), + pfbWriter.writeMultiValueEntity( + List.of(nullableList, List.of(List.of("456"), List.of("80"), List.of("N", "Y")), List.of(List.of(), List.of("75"), List.of())) + ); + pfbWriter.writeMultiValueEntity( + List.of( + List.of(List.of("123"), List.of("80"), List.of("Y")), List.of(List.of("456"), List.of("70"), List.of("N", "Y")), List.of(List.of(), List.of("75"), List.of()) - )); - pfbWriter.writeMultiValueEntity(List.of( - List.of(List.of("123"), List.of("80"), List.of("Y")), - List.of(List.of("456"), List.of("70"),List.of("N", "Y")), - List.of(List.of(), List.of("75"), List.of()) - )); + ) + ); pfbWriter.close(); } @@ -68,4 +72,4 @@ public void formatFieldName_randomGarbage_replaceWithUnderscore() { String formattedName = pfbWriter.formatFieldName("$$$my garbage @vro var!able nam#"); assertEquals("___my_garbage__vro_var_able_nam_", formattedName); } -} \ No newline at end of file +} diff --git a/services/pic-sure-hpds/processing/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/processing/patient/PatientProcessorTest.java b/services/pic-sure-hpds/processing/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/processing/patient/PatientProcessorTest.java index a2cb4bc26..2e620a33a 100644 --- a/services/pic-sure-hpds/processing/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/processing/patient/PatientProcessorTest.java +++ b/services/pic-sure-hpds/processing/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/processing/patient/PatientProcessorTest.java @@ -31,17 +31,15 @@ void shouldProcessPatientQuery() { q.setPicSureId("frank"); q.setExpectedResultType(ResultType.PATIENTS); AsyncResult writeToThis = Mockito.mock(AsyncResult.class); - Mockito.when(abstractProcessor.getPatientSubsetForQuery(q)) - .thenReturn(new TreeSet<>(List.of(1, 2, 42))); + Mockito.when(abstractProcessor.getPatientSubsetForQuery(q)).thenReturn(new TreeSet<>(List.of(1, 2, 42))); subject.runQuery(q, writeToThis); - Mockito.verify(writeToThis, Mockito.times(1)) - .appendResults(Mockito.argThat(strings -> - strings.size() == 3 && - strings.get(0)[0].equals("1") && - strings.get(1)[0].equals("2") && - strings.get(2)[0].equals("42")) - ); + Mockito.verify(writeToThis, Mockito.times(1)).appendResults( + Mockito.argThat( + strings -> strings.size() == 3 && strings.get(0)[0].equals("1") && strings.get(1)[0].equals("2") + && strings.get(2)[0].equals("42") + ) + ); } -} \ No newline at end of file +} diff --git a/services/pic-sure-hpds/processing/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/processing/timeseries/TimeSeriesConversionServiceTest.java b/services/pic-sure-hpds/processing/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/processing/timeseries/TimeSeriesConversionServiceTest.java index ff6c83c18..adadca3d0 100644 --- a/services/pic-sure-hpds/processing/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/processing/timeseries/TimeSeriesConversionServiceTest.java +++ b/services/pic-sure-hpds/processing/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/processing/timeseries/TimeSeriesConversionServiceTest.java @@ -16,4 +16,4 @@ public void shouldConvertToIsoString() { assertEquals(expected, actual); } -} \ No newline at end of file +} diff --git a/services/pic-sure-hpds/processing/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/processing/v3/CountV3ProcessorTest.java b/services/pic-sure-hpds/processing/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/processing/v3/CountV3ProcessorTest.java index 205d714c1..d97125f31 100644 --- a/services/pic-sure-hpds/processing/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/processing/v3/CountV3ProcessorTest.java +++ b/services/pic-sure-hpds/processing/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/processing/v3/CountV3ProcessorTest.java @@ -148,9 +148,7 @@ public void runContinuousCrossCounts_requiredNumericConcept_returnsObservedValue Query query = new Query(List.of(), List.of(), required, List.of(), ResultType.CONTINUOUS_CROSS_COUNT, null, null); PhenoCube cube = new PhenoCube<>(conceptPath, Double.class); - cube.setSortedByKey( - new KeyAndValue[] {new KeyAndValue<>(1, 18.0), new KeyAndValue<>(2, 19.0), new KeyAndValue<>(3, 19.0)} - ); + cube.setSortedByKey(new KeyAndValue[] {new KeyAndValue<>(1, 18.0), new KeyAndValue<>(2, 19.0), new KeyAndValue<>(3, 19.0)}); when(queryExecutor.getPatientSubsetForQuery(query)).thenReturn(Set.of(1, 3)); when(queryExecutor.getDictionary()).thenReturn(Map.of(conceptPath, new ColumnMeta().setName(conceptPath).setCategorical(false))); diff --git a/services/pic-sure-hpds/service/pom.xml b/services/pic-sure-hpds/service/pom.xml index ab5d30342..5a87646c6 100644 --- a/services/pic-sure-hpds/service/pom.xml +++ b/services/pic-sure-hpds/service/pom.xml @@ -5,7 +5,7 @@ pic-sure-hpds edu.harvard.hms.dbmi.avillach.hpds - 3.0.0-SNAPSHOT + ${revision} service diff --git a/services/pic-sure-hpds/service/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/service/HpdsApplication.java b/services/pic-sure-hpds/service/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/service/HpdsApplication.java index b751c2b4b..d2b343956 100644 --- a/services/pic-sure-hpds/service/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/service/HpdsApplication.java +++ b/services/pic-sure-hpds/service/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/service/HpdsApplication.java @@ -19,4 +19,4 @@ public static PropertySourcesPlaceholderConfigurer propertySourcesPlaceholderCon return new PropertySourcesPlaceholderConfigurer(); } -} \ No newline at end of file +} diff --git a/services/pic-sure-hpds/service/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/service/filesharing/FileSharingService.java b/services/pic-sure-hpds/service/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/service/filesharing/FileSharingService.java index 88e131452..93a4abd1f 100644 --- a/services/pic-sure-hpds/service/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/service/filesharing/FileSharingService.java +++ b/services/pic-sure-hpds/service/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/service/filesharing/FileSharingService.java @@ -15,8 +15,7 @@ import java.util.Optional; /** - * Used for sharing data. Given a query, this service will write - * phenotypic and genomic data into a directory + * Used for sharing data. Given a query, this service will write phenotypic and genomic data into a directory */ @Service public class FileSharingService { @@ -29,8 +28,7 @@ public class FileSharingService { private final LoggingClient loggingClient; public FileSharingService( - QueryService queryService, FileSystemService fileWriter, - VariantListProcessor variantListProcessor, LoggingClient loggingClient + QueryService queryService, FileSystemService fileWriter, VariantListProcessor variantListProcessor, LoggingClient loggingClient ) { this.queryService = queryService; this.fileWriter = fileWriter; @@ -71,13 +69,10 @@ public boolean createGenomicData(Query query) { private void sendFileWrittenEvent(String queryId, String dataType) { if (loggingClient != null && loggingClient.isEnabled()) { try { - loggingClient.send(LoggingEvent.builder("DATA_ACCESS") - .action("data.file.written") - .metadata(Map.of( - "query_id", queryId, - "data_type", dataType - )) - .build()); + loggingClient.send( + LoggingEvent.builder("DATA_ACCESS").action("data.file.written") + .metadata(Map.of("query_id", queryId, "data_type", dataType)).build() + ); } catch (Exception e) { LOG.warn("Failed to send audit log event", e); } diff --git a/services/pic-sure-hpds/service/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/service/filesharing/FileSharingV3Service.java b/services/pic-sure-hpds/service/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/service/filesharing/FileSharingV3Service.java index b3e96d580..f9e65535f 100644 --- a/services/pic-sure-hpds/service/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/service/filesharing/FileSharingV3Service.java +++ b/services/pic-sure-hpds/service/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/service/filesharing/FileSharingV3Service.java @@ -28,7 +28,10 @@ public class FileSharingV3Service { private final VariantListV3Processor variantListProcessor; private final LoggingClient loggingClient; - public FileSharingV3Service(QueryV3Service queryService, FileSystemV3Service fileWriter, VariantListV3Processor variantListProcessor, LoggingClient loggingClient) { + public FileSharingV3Service( + QueryV3Service queryService, FileSystemV3Service fileWriter, VariantListV3Processor variantListProcessor, + LoggingClient loggingClient + ) { this.queryService = queryService; this.fileWriter = fileWriter; this.variantListProcessor = variantListProcessor; @@ -63,13 +66,10 @@ public boolean createGenomicData(Query query) { private void sendFileWrittenEvent(String queryId, String dataType) { if (loggingClient != null && loggingClient.isEnabled()) { try { - loggingClient.send(LoggingEvent.builder("DATA_ACCESS") - .action("data.file.written") - .metadata(Map.of( - "query_id", queryId, - "data_type", dataType - )) - .build()); + loggingClient.send( + LoggingEvent.builder("DATA_ACCESS").action("data.file.written") + .metadata(Map.of("query_id", queryId, "data_type", dataType)).build() + ); } catch (Exception e) { LOG.warn("Failed to send audit log event", e); } diff --git a/services/pic-sure-hpds/service/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/service/filesharing/TestDataService.java b/services/pic-sure-hpds/service/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/service/filesharing/TestDataService.java index 92b938f38..f9e5da60f 100644 --- a/services/pic-sure-hpds/service/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/service/filesharing/TestDataService.java +++ b/services/pic-sure-hpds/service/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/service/filesharing/TestDataService.java @@ -8,8 +8,7 @@ import java.util.UUID; /** - * Writes test data to the directory shared with the uploader to test - * uploading features + * Writes test data to the directory shared with the uploader to test uploading features */ @Service public class TestDataService { @@ -25,10 +24,6 @@ public TestDataService(FileSystemService fileSystemService) { public boolean uploadTestFile(String uuid) { LOG.info("Writing test file for uuid {}", uuid); - return fileSystemService.writeResultToFile( - "test_data.txt", - "This is a disposable test file", - uuid - ); + return fileSystemService.writeResultToFile("test_data.txt", "This is a disposable test file", uuid); } } diff --git a/services/pic-sure-hpds/service/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/service/health/HpdsReadinessHealthIndicator.java b/services/pic-sure-hpds/service/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/service/health/HpdsReadinessHealthIndicator.java index 939706b7c..0a250e494 100644 --- a/services/pic-sure-hpds/service/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/service/health/HpdsReadinessHealthIndicator.java +++ b/services/pic-sure-hpds/service/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/service/health/HpdsReadinessHealthIndicator.java @@ -6,9 +6,8 @@ import org.springframework.stereotype.Component; /** - * Deep readiness for HPDS (spec 3.8): UP only when phenotype OR genomic data is - * actually loaded — a real "data ready" signal, not just port-up. HPDS has no - * DataSource, so this replaces the built-in db indicator. + * Deep readiness for HPDS (spec 3.8): UP only when phenotype OR genomic data is actually loaded — a real "data ready" signal, not just + * port-up. HPDS has no DataSource, so this replaces the built-in db indicator. */ @Component("hpdsReadiness") public class HpdsReadinessHealthIndicator implements HealthIndicator { @@ -25,10 +24,7 @@ public Health health() { int phenotypeColumns = abstractProcessor.getDictionary().size(); int genomicColumns = abstractProcessor.getInfoStoreColumns().size(); if (phenotypeColumns > 0 || genomicColumns > 0) { - return Health.up() - .withDetail("phenotypeColumns", phenotypeColumns) - .withDetail("genomicColumns", genomicColumns) - .build(); + return Health.up().withDetail("phenotypeColumns", phenotypeColumns).withDetail("genomicColumns", genomicColumns).build(); } return Health.down().withDetail("reason", "no phenotype or genomic data loaded").build(); } catch (Exception ex) { diff --git a/services/pic-sure-hpds/service/src/main/resources/application.properties b/services/pic-sure-hpds/service/src/main/resources/application.properties index dd1248177..cedc66056 100644 --- a/services/pic-sure-hpds/service/src/main/resources/application.properties +++ b/services/pic-sure-hpds/service/src/main/resources/application.properties @@ -13,8 +13,10 @@ enable_file_sharing=false # See edu.harvard.hms.dbmi.avillach.hpds.data.query.v3.Query.authorizationFilters for more information # hpds.requireAuthorizationFilter=false -# Actuator: deep health readiness (Phase 3). HPDS has no Spring Security, so exposing -# health details here is safe and useful for AIO deployment health checks. -management.endpoints.web.exposure.include=health -management.endpoint.health.show-details=always +# Actuator: OFF BY DEFAULT. HPDS has no Spring Security in front of it, so nothing is exposed unless the +# deployment opts in. AIO's hpds.env sets PICSURE_ACTUATOR_EXPOSURE=health to enable a shallow readiness +# probe; details stay hidden (PICSURE_ACTUATOR_DETAILS defaults to never) so no datasource/disk/component +# internals leak even when health is reachable. +management.endpoints.web.exposure.include=${PICSURE_ACTUATOR_EXPOSURE:none} +management.endpoint.health.show-details=${PICSURE_ACTUATOR_DETAILS:never} management.endpoint.health.probes.enabled=true \ No newline at end of file diff --git a/services/pic-sure-hpds/service/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/service/AuditInterceptorTest.java b/services/pic-sure-hpds/service/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/service/AuditInterceptorTest.java index 52cc847b3..931e4b80d 100644 --- a/services/pic-sure-hpds/service/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/service/AuditInterceptorTest.java +++ b/services/pic-sure-hpds/service/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/service/AuditInterceptorTest.java @@ -26,15 +26,12 @@ void setup() { static class TestController { @AuditEvent(type = "QUERY", action = "query.submitted") - public void queryEndpoint() { - } + public void queryEndpoint() {} @AuditEvent(type = "DATA_ACCESS", action = "query.result") - public void resultEndpoint() { - } + public void resultEndpoint() {} - public void unannotatedEndpoint() { - } + public void unannotatedEndpoint() {} } private HandlerMethod handlerFor(String methodName) throws Exception { diff --git a/services/pic-sure-hpds/service/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/service/AuditWiringTest.java b/services/pic-sure-hpds/service/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/service/AuditWiringTest.java index 928dc0eb4..a738b3845 100644 --- a/services/pic-sure-hpds/service/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/service/AuditWiringTest.java +++ b/services/pic-sure-hpds/service/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/service/AuditWiringTest.java @@ -19,11 +19,7 @@ import org.springframework.test.util.ReflectionTestUtils; @SpringBootTest -@TestPropertySource(properties = { - "SMALL_JOB_LIMIT=100", - "SMALL_TASK_THREADS=2", - "LARGE_TASK_THREADS=2", -}) +@TestPropertySource(properties = {"SMALL_JOB_LIMIT=100", "SMALL_TASK_THREADS=2", "LARGE_TASK_THREADS=2",}) class AuditWiringTest { // -- Bean under verification -------------------------------------------------- diff --git a/services/pic-sure-hpds/service/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/service/PicSureV3ServiceAuditTest.java b/services/pic-sure-hpds/service/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/service/PicSureV3ServiceAuditTest.java index 9a77701ae..5de32549b 100644 --- a/services/pic-sure-hpds/service/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/service/PicSureV3ServiceAuditTest.java +++ b/services/pic-sure-hpds/service/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/service/PicSureV3ServiceAuditTest.java @@ -40,14 +40,8 @@ void setup() { queryService = mock(QueryV3Service.class); queryExecutor = mock(QueryExecutor.class); service = new PicSureV3Service( - queryService, - mock(CountV3Processor.class), - mock(VariantListV3Processor.class), - queryExecutor, - mock(Paginator.class), - mock(SignUrlService.class), - mock(FileSharingV3Service.class), - mock(TestDataService.class) + queryService, mock(CountV3Processor.class), mock(VariantListV3Processor.class), queryExecutor, mock(Paginator.class), + mock(SignUrlService.class), mock(FileSharingV3Service.class), mock(TestDataService.class) ); request = new MockHttpServletRequest(); diff --git a/services/pic-sure-hpds/service/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/service/QueryServiceTest.java b/services/pic-sure-hpds/service/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/service/QueryServiceTest.java index f3aa4ef40..0bf021a53 100644 --- a/services/pic-sure-hpds/service/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/service/QueryServiceTest.java +++ b/services/pic-sure-hpds/service/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/service/QueryServiceTest.java @@ -24,6 +24,7 @@ import java.util.Map; import static org.junit.jupiter.api.Assertions.*; + @ExtendWith(SpringExtension.class) @EnableAutoConfiguration @SpringBootTest(classes = edu.harvard.hms.dbmi.avillach.hpds.service.HpdsApplication.class) @@ -43,7 +44,7 @@ public void dataframeMulti() throws IOException, InterruptedException { Query query = new Query(); List variantInfoFilters = new ArrayList<>(); Query.VariantInfoFilter variantInfoFilter = new Query.VariantInfoFilter(); - variantInfoFilter.categoryVariantInfoFilters = Map.of("Gene_with_variant", new String[]{"LOC102723996", "LOC101928576"}); + variantInfoFilter.categoryVariantInfoFilters = Map.of("Gene_with_variant", new String[] {"LOC102723996", "LOC101928576"}); variantInfoFilters.add(variantInfoFilter); query.setVariantInfoFilters(variantInfoFilters); query.setFields(List.of("\\open_access-1000Genomes\\data\\SYNTHETIC_AGE\\")); @@ -52,7 +53,10 @@ public void dataframeMulti() throws IOException, InterruptedException { AsyncResult asyncResult = queryService.runQuery(query); int retries = 0; - while ((AsyncResult.Status.RUNNING.equals(asyncResult.getStatus()) || AsyncResult.Status.PENDING.equals(asyncResult.getStatus())) && retries < 10) { + while ( + (AsyncResult.Status.RUNNING.equals(asyncResult.getStatus()) || AsyncResult.Status.PENDING.equals(asyncResult.getStatus())) + && retries < 10 + ) { retries++; Thread.sleep(200); } @@ -64,4 +68,4 @@ public void dataframeMulti() throws IOException, InterruptedException { assertEquals(23, csvContainer.getRows().size()); } -} \ No newline at end of file +} diff --git a/services/pic-sure-hpds/service/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/service/filesharing/FileSharingServiceTest.java b/services/pic-sure-hpds/service/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/service/filesharing/FileSharingServiceTest.java index 775839cfe..ff816beb5 100644 --- a/services/pic-sure-hpds/service/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/service/filesharing/FileSharingServiceTest.java +++ b/services/pic-sure-hpds/service/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/service/filesharing/FileSharingServiceTest.java @@ -58,10 +58,8 @@ public void shouldCreatePhenotypicData() { AsyncResult result = new AsyncResult(query, variantListProcessor, resultWriter); result.setStatus(AsyncResult.Status.SUCCESS); - Mockito.when(queryService.getResultFor("my-id")) - .thenReturn(result); - Mockito.when(fileWriter.writeResultToFile("phenotypic_data.csv", result, "my-ps-id")) - .thenReturn(true); + Mockito.when(queryService.getResultFor("my-id")).thenReturn(result); + Mockito.when(fileWriter.writeResultToFile("phenotypic_data.csv", result, "my-ps-id")).thenReturn(true); boolean actual = subject.createPhenotypicData(query); @@ -76,8 +74,7 @@ public void shouldNotCreatePhenotypicData() { AsyncResult result = new AsyncResult(query, variantListProcessor, resultWriter); result.setStatus(AsyncResult.Status.ERROR); - Mockito.when(queryService.getResultFor("my-id")) - .thenReturn(result); + Mockito.when(queryService.getResultFor("my-id")).thenReturn(result); boolean actual = subject.createPhenotypicData(query); @@ -89,10 +86,8 @@ public void shouldCreateGenomicData() throws IOException { Query query = new Query(); query.setPicSureId("my-id"); String vcf = "lol lets put the whole vcf in a string"; - Mockito.when(variantListProcessor.runVcfExcerptQuery(query, true)) - .thenReturn(vcf); - Mockito.when(fileWriter.writeResultToFile("genomic_data.tsv", vcf, "my-id")) - .thenReturn(true); + Mockito.when(variantListProcessor.runVcfExcerptQuery(query, true)).thenReturn(vcf); + Mockito.when(fileWriter.writeResultToFile("genomic_data.tsv", vcf, "my-id")).thenReturn(true); boolean actual = subject.createGenomicData(query); @@ -103,8 +98,7 @@ public void shouldCreateGenomicData() throws IOException { public void shouldNotCreateGenomicData() throws IOException { Query query = new Query(); query.setPicSureId("my-id"); - Mockito.when(variantListProcessor.runVcfExcerptQuery(query, true)) - .thenThrow(new IOException("oh no!")); + Mockito.when(variantListProcessor.runVcfExcerptQuery(query, true)).thenThrow(new IOException("oh no!")); boolean actual = subject.createGenomicData(query); @@ -119,13 +113,11 @@ void shouldCreatePatientsList() { query.setExpectedResultType(ResultType.PATIENTS); AsyncResult result = new AsyncResult(query, patientProcessor, resultWriter); result.setStatus(AsyncResult.Status.SUCCESS); - Mockito.when(queryService.getResultFor("jasdijasd")) - .thenReturn(result); - Mockito.when(fileWriter.writeResultToFile("patients.txt", result, "jasdijasd")) - .thenReturn(true); + Mockito.when(queryService.getResultFor("jasdijasd")).thenReturn(result); + Mockito.when(fileWriter.writeResultToFile("patients.txt", result, "jasdijasd")).thenReturn(true); boolean actual = subject.createPatientList(query); Assertions.assertTrue(actual); } -} \ No newline at end of file +} diff --git a/services/pic-sure-hpds/service/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/service/filesharing/FileSystemServiceTest.java b/services/pic-sure-hpds/service/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/service/filesharing/FileSystemServiceTest.java index 70ba14afb..5911bcbe3 100644 --- a/services/pic-sure-hpds/service/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/service/filesharing/FileSystemServiceTest.java +++ b/services/pic-sure-hpds/service/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/service/filesharing/FileSystemServiceTest.java @@ -17,9 +17,9 @@ public void shouldWriteToFile() throws IOException { FileSystemService subject = new FileSystemService(); ReflectionTestUtils.setField(subject, "sharingRoot", dir); ReflectionTestUtils.setField(subject, "enableFileSharing", true); - String fileContent = "I just got an ad that tried to sell a baguette with moz, dressing, " + - "and tomatoes as a healthy lunch, and that's just so far from the truth that it's bugging me. " + - "Like, come on. It's bread and cheese and oil. I don't care how fresh the tomatoes are."; + String fileContent = "I just got an ad that tried to sell a baguette with moz, dressing, " + + "and tomatoes as a healthy lunch, and that's just so far from the truth that it's bugging me. " + + "Like, come on. It's bread and cheese and oil. I don't care how fresh the tomatoes are."; boolean actual = subject.writeResultToFile("out.tsv", fileContent, "my-id"); String actualContent = Files.readString(dir.resolve("my-id/out.tsv")); @@ -39,4 +39,4 @@ public void shouldNotWriteToFile() throws IOException { assertFalse(actual); } -} \ No newline at end of file +} diff --git a/services/pic-sure-hpds/service/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/service/filesharing/TestDataServiceTest.java b/services/pic-sure-hpds/service/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/service/filesharing/TestDataServiceTest.java index 1123fa175..2d70ff66b 100644 --- a/services/pic-sure-hpds/service/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/service/filesharing/TestDataServiceTest.java +++ b/services/pic-sure-hpds/service/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/service/filesharing/TestDataServiceTest.java @@ -12,7 +12,7 @@ public class TestDataServiceTest { FileSystemService fileSystemService; TestDataService subject; - + @Test public void shouldCreateTestFileForUpload() { fileSystemService = Mockito.mock(FileSystemService.class); @@ -27,4 +27,4 @@ public void shouldCreateTestFileForUpload() { Mockito.verify(fileSystemService, Mockito.times(1)) .writeResultToFile("test_data.txt", "This is a disposable test file", uuid.toString()); } -} \ No newline at end of file +} diff --git a/services/pic-sure-hpds/service/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/service/util/PaginatorTest.java b/services/pic-sure-hpds/service/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/service/util/PaginatorTest.java index 7c13f736f..4cf7614f8 100644 --- a/services/pic-sure-hpds/service/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/service/util/PaginatorTest.java +++ b/services/pic-sure-hpds/service/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/service/util/PaginatorTest.java @@ -67,6 +67,7 @@ public void paginate_invalidPageZero() { paginator.paginate(values, 0, 20); }); } + @Test public void paginate_invalidPageNegative() { assertThrows(IllegalArgumentException.class, () -> { @@ -91,4 +92,4 @@ public void paginate_invalidSizeNegative() { paginator.paginate(values, 1, -5); }); } -} \ No newline at end of file +} diff --git a/services/pic-sure-hpds/service/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/test/AbstractProcessorIntegrationTest.java b/services/pic-sure-hpds/service/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/test/AbstractProcessorIntegrationTest.java index 68ff7c97e..f3eab7dd6 100644 --- a/services/pic-sure-hpds/service/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/test/AbstractProcessorIntegrationTest.java +++ b/services/pic-sure-hpds/service/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/test/AbstractProcessorIntegrationTest.java @@ -54,7 +54,7 @@ public void getPatientSubsetForQuery_validGeneWithVariantQuery() { Query query = new Query(); List variantInfoFilters = new ArrayList<>(); Query.VariantInfoFilter variantInfoFilter = new Query.VariantInfoFilter(); - variantInfoFilter.categoryVariantInfoFilters = Map.of("Gene_with_variant", new String[]{"LOC102723996"}); + variantInfoFilter.categoryVariantInfoFilters = Map.of("Gene_with_variant", new String[] {"LOC102723996"}); variantInfoFilters.add(variantInfoFilter); query.setVariantInfoFilters(variantInfoFilters); @@ -70,7 +70,7 @@ public void getPatientSubsetForQuery_validGeneWithMultipleVariantQuery() { Query query = new Query(); List variantInfoFilters = new ArrayList<>(); Query.VariantInfoFilter variantInfoFilter = new Query.VariantInfoFilter(); - variantInfoFilter.categoryVariantInfoFilters = Map.of("Gene_with_variant", new String[]{"LOC102723996", "LOC101928576"}); + variantInfoFilter.categoryVariantInfoFilters = Map.of("Gene_with_variant", new String[] {"LOC102723996", "LOC101928576"}); variantInfoFilters.add(variantInfoFilter); query.setVariantInfoFilters(variantInfoFilters); @@ -83,7 +83,7 @@ public void getPatientSubsetForQuery_validGeneWithVariantQueryAndNumericQuery() Query query = new Query(); List variantInfoFilters = new ArrayList<>(); Query.VariantInfoFilter variantInfoFilter = new Query.VariantInfoFilter(); - variantInfoFilter.categoryVariantInfoFilters = Map.of("Gene_with_variant", new String[]{"LOC102723996"}); + variantInfoFilter.categoryVariantInfoFilters = Map.of("Gene_with_variant", new String[] {"LOC102723996"}); variantInfoFilters.add(variantInfoFilter); query.setVariantInfoFilters(variantInfoFilters); query.setNumericFilters(Map.of("\\open_access-1000Genomes\\data\\SYNTHETIC_AGE\\", new Filter.DoubleFilter(35.0, 45.0))); @@ -104,10 +104,7 @@ public void getPatientSubsetForQuery_validNumericPhenotypicQuery() { @Test public void getPatientSubsetForQuery_validCategoricalPhenotypicQuery() { Query query = new Query(); - query.setCategoryFilters(Map.of( - "\\open_access-1000Genomes\\data\\POPULATION NAME\\", - new String[] {"Finnish"} - )); + query.setCategoryFilters(Map.of("\\open_access-1000Genomes\\data\\POPULATION NAME\\", new String[] {"Finnish"})); Set idList = abstractProcessor.getPatientSubsetForQuery(query); assertEquals(102, idList.size()); } @@ -115,26 +112,17 @@ public void getPatientSubsetForQuery_validCategoricalPhenotypicQuery() { @Test public void getPatientSubsetForQuery_validMultipleValueCategoricalPhenotypicQuery() { Query query = new Query(); - query.setCategoryFilters(Map.of( - "\\open_access-1000Genomes\\data\\POPULATION NAME\\", - new String[] {"Finnish"} - )); + query.setCategoryFilters(Map.of("\\open_access-1000Genomes\\data\\POPULATION NAME\\", new String[] {"Finnish"})); Set finnishIdList = abstractProcessor.getPatientSubsetForQuery(query); assertEquals(102, finnishIdList.size()); query = new Query(); - query.setCategoryFilters(Map.of( - "\\open_access-1000Genomes\\data\\POPULATION NAME\\", - new String[] {"Colombian"} - )); + query.setCategoryFilters(Map.of("\\open_access-1000Genomes\\data\\POPULATION NAME\\", new String[] {"Colombian"})); Set columbianIdList = abstractProcessor.getPatientSubsetForQuery(query); assertEquals(153, columbianIdList.size()); query = new Query(); - query.setCategoryFilters(Map.of( - "\\open_access-1000Genomes\\data\\POPULATION NAME\\", - new String[] {"Finnish", "Colombian"} - )); + query.setCategoryFilters(Map.of("\\open_access-1000Genomes\\data\\POPULATION NAME\\", new String[] {"Finnish", "Colombian"})); Set bothIdList = abstractProcessor.getPatientSubsetForQuery(query); assertEquals(255, bothIdList.size()); assertEquals(Sets.union(finnishIdList, columbianIdList), bothIdList); @@ -143,26 +131,22 @@ public void getPatientSubsetForQuery_validMultipleValueCategoricalPhenotypicQuer @Test public void getPatientSubsetForQuery_validMultipleCategoricalPhenotypicQuery() { Query query = new Query(); - query.setCategoryFilters(Map.of( - "\\open_access-1000Genomes\\data\\POPULATION NAME\\", - new String[] {"Finnish"} - )); + query.setCategoryFilters(Map.of("\\open_access-1000Genomes\\data\\POPULATION NAME\\", new String[] {"Finnish"})); Set finnishIdList = abstractProcessor.getPatientSubsetForQuery(query); assertEquals(102, finnishIdList.size()); query = new Query(); - query.setCategoryFilters(Map.of( - "\\open_access-1000Genomes\\data\\SEX\\", new String[] {"female"} - )); + query.setCategoryFilters(Map.of("\\open_access-1000Genomes\\data\\SEX\\", new String[] {"female"})); Set femaleIdList = abstractProcessor.getPatientSubsetForQuery(query); assertEquals(2330, femaleIdList.size()); query = new Query(); - query.setCategoryFilters(Map.of( - "\\open_access-1000Genomes\\data\\POPULATION NAME\\", - new String[] {"Finnish"}, - "\\open_access-1000Genomes\\data\\SEX\\", new String[] {"female"} - )); + query.setCategoryFilters( + Map.of( + "\\open_access-1000Genomes\\data\\POPULATION NAME\\", new String[] {"Finnish"}, "\\open_access-1000Genomes\\data\\SEX\\", + new String[] {"female"} + ) + ); Set bothIdList = abstractProcessor.getPatientSubsetForQuery(query); assertEquals(64, bothIdList.size()); assertEquals(Sets.intersection(finnishIdList, femaleIdList), bothIdList); @@ -171,26 +155,18 @@ public void getPatientSubsetForQuery_validMultipleCategoricalPhenotypicQuery() { @Test public void getPatientSubsetForQuery_validMultiplePhenotypicQuery() { Query query = new Query(); - query.setNumericFilters(Map.of( - "\\open_access-1000Genomes\\data\\SYNTHETIC_AGE\\", new Filter.DoubleFilter(35.0, 45.0) - )); + query.setNumericFilters(Map.of("\\open_access-1000Genomes\\data\\SYNTHETIC_AGE\\", new Filter.DoubleFilter(35.0, 45.0))); Set ageIdList = abstractProcessor.getPatientSubsetForQuery(query); assertEquals(562, ageIdList.size()); query = new Query(); - query.setCategoryFilters(Map.of( - "\\open_access-1000Genomes\\data\\SEX\\", new String[] {"male"} - )); + query.setCategoryFilters(Map.of("\\open_access-1000Genomes\\data\\SEX\\", new String[] {"male"})); Set sexIdList = abstractProcessor.getPatientSubsetForQuery(query); assertEquals(2648, sexIdList.size()); query = new Query(); - query.setNumericFilters(Map.of( - "\\open_access-1000Genomes\\data\\SYNTHETIC_AGE\\", new Filter.DoubleFilter(35.0, 45.0) - )); - query.setCategoryFilters(Map.of( - "\\open_access-1000Genomes\\data\\SEX\\", new String[] {"male"} - )); + query.setNumericFilters(Map.of("\\open_access-1000Genomes\\data\\SYNTHETIC_AGE\\", new Filter.DoubleFilter(35.0, 45.0))); + query.setCategoryFilters(Map.of("\\open_access-1000Genomes\\data\\SEX\\", new String[] {"male"})); Set bothIdList = abstractProcessor.getPatientSubsetForQuery(query); assertEquals(269, bothIdList.size()); assertEquals(Sets.intersection(ageIdList, sexIdList), bothIdList); @@ -199,22 +175,20 @@ public void getPatientSubsetForQuery_validMultiplePhenotypicQuery() { @Test public void getPatientSubsetForQuery_validMultipleNumericPhenotypicQuery() { Query query = new Query(); - query.setNumericFilters(Map.of( - "\\open_access-1000Genomes\\data\\SYNTHETIC_AGE\\", new Filter.DoubleFilter(35.0, 45.0) - )); + query.setNumericFilters(Map.of("\\open_access-1000Genomes\\data\\SYNTHETIC_AGE\\", new Filter.DoubleFilter(35.0, 45.0))); Set ageIdList = abstractProcessor.getPatientSubsetForQuery(query); query = new Query(); - query.setNumericFilters(Map.of( - "\\open_access-1000Genomes\\data\\SYNTHETIC_HEIGHT\\", new Filter.DoubleFilter(180.0, null) - )); + query.setNumericFilters(Map.of("\\open_access-1000Genomes\\data\\SYNTHETIC_HEIGHT\\", new Filter.DoubleFilter(180.0, null))); Set heightIdList = abstractProcessor.getPatientSubsetForQuery(query); query = new Query(); - query.setNumericFilters(Map.of( + query.setNumericFilters( + Map.of( "\\open_access-1000Genomes\\data\\SYNTHETIC_AGE\\", new Filter.DoubleFilter(35.0, 45.0), "\\open_access-1000Genomes\\data\\SYNTHETIC_HEIGHT\\", new Filter.DoubleFilter(180.0, null) - )); + ) + ); Set bothIdList = abstractProcessor.getPatientSubsetForQuery(query); assertEquals(Sets.intersection(ageIdList, heightIdList), bothIdList); @@ -250,15 +224,16 @@ public void getPatientSubsetForQuery_validRequiredVariantOldFormat() { @Test public void getPatientSubsetForQuery_invalidVariantSpecQuery() { Query query = new Query(); - query.setCategoryFilters(Map.of("chr21,5061,AAAA,GGGG", new String[]{"0/1", "1/1"})); + query.setCategoryFilters(Map.of("chr21,5061,AAAA,GGGG", new String[] {"0/1", "1/1"})); Set idList = abstractProcessor.getPatientSubsetForQuery(query); assertEquals(0, idList.size()); } + @Test public void getPatientSubsetForQuery_validRequiredVariantOldFormatCategoryFilter() { Query query = new Query(); - query.setCategoryFilters(Map.of("chr21,5032061,A,G", new String[]{"0/1", "1/1"})); + query.setCategoryFilters(Map.of("chr21,5032061,A,G", new String[] {"0/1", "1/1"})); Set idList = abstractProcessor.getPatientSubsetForQuery(query); assertEquals(8, idList.size()); @@ -267,7 +242,7 @@ public void getPatientSubsetForQuery_validRequiredVariantOldFormatCategoryFilter @Test public void getPatientSubsetForQuery_validRequiredVariantOldFormatCategoryFilterHomozygous() { Query query = new Query(); - query.setCategoryFilters(Map.of("chr21,5032061,A,G", new String[]{"1/1"})); + query.setCategoryFilters(Map.of("chr21,5032061,A,G", new String[] {"1/1"})); Set idList = abstractProcessor.getPatientSubsetForQuery(query); assertEquals(3, idList.size()); @@ -276,7 +251,7 @@ public void getPatientSubsetForQuery_validRequiredVariantOldFormatCategoryFilter @Test public void getPatientSubsetForQuery_validRequiredVariantOldFormatCategoryFilterHeterozygous() { Query query = new Query(); - query.setCategoryFilters(Map.of("chr21,5032061,A,G", new String[]{"0/1"})); + query.setCategoryFilters(Map.of("chr21,5032061,A,G", new String[] {"0/1"})); Set idList = abstractProcessor.getPatientSubsetForQuery(query); assertEquals(5, idList.size()); @@ -314,11 +289,17 @@ public void getPatientSubsetForQuery_validAnyRecordOfQuery() { assertEquals(1126, anyRecordOfList.size()); query = new Query(); - query.setAnyRecordOf(List.of("\\open_access-1000Genomes\\data\\THE HUMAN GENOME STRUCTURAL VARIATION CONSORTIUM\\", "\\open_access-1000Genomes\\data\\SYNTHETIC_AGE\\")); + query.setAnyRecordOf( + List.of( + "\\open_access-1000Genomes\\data\\THE HUMAN GENOME STRUCTURAL VARIATION CONSORTIUM\\", + "\\open_access-1000Genomes\\data\\SYNTHETIC_AGE\\" + ) + ); - anyRecordOfList = abstractProcessor.getPatientSubsetForQuery(query); + anyRecordOfList = abstractProcessor.getPatientSubsetForQuery(query); assertEquals(4978, anyRecordOfList.size()); } + public void getPatientSubsetForQuery_validAnyRecordOfAndCategoryQuery() { Query query = new Query(); query.setCategoryFilters(Map.of("\\open_access-1000Genomes\\data\\POPULATION NAME\\", new String[] {"Yoruba"})); @@ -341,21 +322,29 @@ public void getPatientSubsetForQuery_validAnyRecordOfAndCategoryQuery() { assertEquals(Sets.intersection(yorubaIdList, anyRecordOf), bothIdList); } + @Test public void getPatientSubsetForQuery_validAnyRecordOfMultiQuery() { Query query = new Query(); - query.setAnyRecordOfMulti(List.of( - List.of("\\open_access-1000Genomes\\data\\THE HUMAN GENOME STRUCTURAL VARIATION CONSORTIUM\\", "\\open_access-1000Genomes\\data\\SYNTHETIC_AGE\\") - )); + query.setAnyRecordOfMulti( + List.of( + List.of( + "\\open_access-1000Genomes\\data\\THE HUMAN GENOME STRUCTURAL VARIATION CONSORTIUM\\", + "\\open_access-1000Genomes\\data\\SYNTHETIC_AGE\\" + ) + ) + ); Set anyRecordOfList = abstractProcessor.getPatientSubsetForQuery(query); assertEquals(4978, anyRecordOfList.size()); query = new Query(); - query.setAnyRecordOfMulti(List.of( + query.setAnyRecordOfMulti( + List.of( List.of("\\open_access-1000Genomes\\data\\THE HUMAN GENOME STRUCTURAL VARIATION CONSORTIUM\\"), List.of("\\open_access-1000Genomes\\data\\SYNTHETIC_AGE\\", "\\open_access-1000Genomes\\data\\SYNTHETIC_SEX\\") - )); + ) + ); anyRecordOfList = abstractProcessor.getPatientSubsetForQuery(query); assertEquals(1126, anyRecordOfList.size()); @@ -367,7 +356,7 @@ public void getVariantList_validGeneWithVariantQuery() { Query query = new Query(); List variantInfoFilters = new ArrayList<>(); Query.VariantInfoFilter variantInfoFilter = new Query.VariantInfoFilter(); - variantInfoFilter.categoryVariantInfoFilters = Map.of("Gene_with_variant", new String[]{"LOC102723996"}); + variantInfoFilter.categoryVariantInfoFilters = Map.of("Gene_with_variant", new String[] {"LOC102723996"}); variantInfoFilters.add(variantInfoFilter); query.setVariantInfoFilters(variantInfoFilters); @@ -381,19 +370,20 @@ public void getVariantList_invalidGeneQuery() { Query query = new Query(); List variantInfoFilters = new ArrayList<>(); Query.VariantInfoFilter variantInfoFilter = new Query.VariantInfoFilter(); - variantInfoFilter.categoryVariantInfoFilters = Map.of("Gene_with_variant", new String[]{"NOTAGENE"}); + variantInfoFilter.categoryVariantInfoFilters = Map.of("Gene_with_variant", new String[] {"NOTAGENE"}); variantInfoFilters.add(variantInfoFilter); query.setVariantInfoFilters(variantInfoFilters); Collection variantList = abstractProcessor.getVariantList(query); assertEquals(0, variantList.size()); } + @Test public void getVariantList_validGeneWithMultipleVariantQuery() { Query query = new Query(); List variantInfoFilters = new ArrayList<>(); Query.VariantInfoFilter variantInfoFilter = new Query.VariantInfoFilter(); - variantInfoFilter.categoryVariantInfoFilters = Map.of("Gene_with_variant", new String[]{"LOC102723996", "LOC101928576"}); + variantInfoFilter.categoryVariantInfoFilters = Map.of("Gene_with_variant", new String[] {"LOC102723996", "LOC101928576"}); variantInfoFilters.add(variantInfoFilter); query.setVariantInfoFilters(variantInfoFilters); @@ -407,7 +397,7 @@ public void getVariantList_validGeneWithVariantQueryAndNumericQuery() { Query query = new Query(); List variantInfoFilters = new ArrayList<>(); Query.VariantInfoFilter variantInfoFilter = new Query.VariantInfoFilter(); - variantInfoFilter.categoryVariantInfoFilters = Map.of("Gene_with_variant", new String[]{"LOC102723996"}); + variantInfoFilter.categoryVariantInfoFilters = Map.of("Gene_with_variant", new String[] {"LOC102723996"}); variantInfoFilters.add(variantInfoFilter); query.setVariantInfoFilters(variantInfoFilters); query.setNumericFilters(Map.of("\\open_access-1000Genomes\\data\\SYNTHETIC_AGE\\", new Filter.DoubleFilter(35.0, 45.0))); diff --git a/services/pic-sure-hpds/service/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/test/BucketIndexBySampleTest.java b/services/pic-sure-hpds/service/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/test/BucketIndexBySampleTest.java index 193392f3a..c6ea4a8a2 100644 --- a/services/pic-sure-hpds/service/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/test/BucketIndexBySampleTest.java +++ b/services/pic-sure-hpds/service/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/test/BucketIndexBySampleTest.java @@ -19,11 +19,10 @@ import static org.springframework.test.util.AssertionErrors.*; /** - * These tests are in the ETL project so that we can read in data from disk each time instead of storing binfiles - * that may become outdated. + * These tests are in the ETL project so that we can read in data from disk each time instead of storing binfiles that may become outdated. * - * the BucketIndexBySample.filterVariantSetForPatientSet method removes variants base on the patent BUCKET MASK; just because a patient - * does not have a particular variant doesn't mean it will be filtered out e.g., when a patient has a different variant in the same bucket. + * the BucketIndexBySample.filterVariantSetForPatientSet method removes variants base on the patent BUCKET MASK; just because a patient does + * not have a particular variant doesn't mean it will be filtered out e.g., when a patient has a different variant in the same bucket. * * Filtering the specific variants is typically done by the calling function after filtering out the unneeded buckets. * @@ -32,195 +31,196 @@ */ public class BucketIndexBySampleTest { - private static final String STORAGE_DIR = "./target/all/"; - - private static BucketIndexBySample bucketIndexBySampleChr4; - private static BucketIndexBySample bucketIndexBySampleChr20; - private static BucketIndexBySample bucketIndexBySampleChr21; - - //Some known variant specs from the input file. - //Some known variant specs from the input file. These have been designed for testing partially overlapping specs - private static final String spec1 = "chr4,9856624,CAAAAA,C,TVP23A,splice_acceptor_variant"; - private static final String spec2 = "chr4,9856624,CAAA,C,TVP23A,splice_acceptor_variant"; - private static final String spec3 = "chr4,9856624,CA,C,TVP23A,splice_acceptor_variant"; - private static final String spec4 = "chr4,9856624,C,CA,TVP23A,splice_acceptor_variant"; - private static final String spec5 = "chr4,9856624,CAAAAA,CA,TVP23A,splice_acceptor_variant"; - - private static final String spec6 = "chr21,5032061,A,G,LOC102723996,missense_variant"; - private static final String spec6b = "chr21,5032061,A,G,ABCDEF123456,synonymous_variant"; - private static final String spec7 = "chr21,5033914,A,G,LOC102723996,missense_variant"; - private static final String spec8 = "chr21,5033988,C,G,LOC102723996,synonymous_variant"; - private static final String spec9 = "chr21,5034028,C,T,LOC102723996,missense_variant"; - - - //these parameters to the BucketIndexBySample methods are configured by each test - Set variantSet; - List patientSet; - - @BeforeAll - public static void initializeBinfile() throws Exception { - BuildIntegrationTestEnvironment instance = BuildIntegrationTestEnvironment.INSTANCE; - VariantStore variantStoreChr4 = VariantStore.readInstance(STORAGE_DIR + "chr4/"); - bucketIndexBySampleChr4 = new BucketIndexBySample(variantStoreChr4, STORAGE_DIR + "chr4/"); - - VariantStore variantStoreChr20 = VariantStore.readInstance(STORAGE_DIR + "chr20/"); - bucketIndexBySampleChr20 = new BucketIndexBySample(variantStoreChr20, STORAGE_DIR + "chr20/"); - - VariantStore variantStoreChr21 = VariantStore.readInstance(STORAGE_DIR + "chr21/"); - bucketIndexBySampleChr21 = new BucketIndexBySample(variantStoreChr21, STORAGE_DIR + "chr21/"); - } - - @BeforeEach - public void setUpTest() { - //start with fresh, empty collections - variantSet = new HashSet<>(); - patientSet = new ArrayList<>(); - } - - @Test - public void test_filterVariantSetForPatientSet_noPatients() throws IOException { - variantSet.add(spec1); - variantSet.add(spec2); - variantSet.add(spec3); - - Collection filteredVariantSet = bucketIndexBySampleChr4.filterVariantSetForPatientSet(variantSet, patientSet); - - assertTrue("Empty Patient List should filter out all variants", filteredVariantSet.isEmpty()); - } - - @Test - public void test_filterVariantSetForPatientSet_noVariants() throws IOException { - patientSet.add(200392); - patientSet.add(200689); - patientSet.add(200972); - - Collection filteredVariantSet = bucketIndexBySampleChr4.filterVariantSetForPatientSet(variantSet, patientSet); - - assertTrue("Empty Variant Set should remain empty", filteredVariantSet.isEmpty()); - } - - @Test - public void test_filterVariantSetForPatientSet_VariantsWithoutPatientsLastBucket() throws IOException { - System.out.println("test_filterVariantSetForPatientSet_VariantsWithoutPatientsLastBucket"); - - variantSet.add(spec1); - variantSet.add(spec2); - variantSet.add(spec3); - variantSet.add(spec4); - variantSet.add(spec5); - - patientSet.add(200706); - patientSet.add(200709); - - Collection filteredVariantSet = bucketIndexBySampleChr4.filterVariantSetForPatientSet(variantSet, patientSet); - - assertTrue("Patients should not match any variants in the list", filteredVariantSet.isEmpty()); - } - - @Test - public void test_filterVariantSetForPatientSet_PatientsWithNoVariantsFirstBucket() throws IOException { - System.out.println("test_filterVariantSetForPatientSet_PatientsWithNoVariantsFirstBucket"); - - variantSet.add(spec6); - variantSet.add(spec6b); - - patientSet.add(197506); - patientSet.add(197508); - patientSet.add(197509); - - Collection filteredVariantSet = bucketIndexBySampleChr4.filterVariantSetForPatientSet(variantSet, patientSet); - - assertTrue("Patients should not match any variants in the list", filteredVariantSet.isEmpty()); - } - - @Test - public void filterVariantSetForPatientSet_variantsDoNotExist_returnNoVariants() throws IOException { - System.out.println("test_filterVariantSetForPatientSet_PatientsWithNoVariantsFirstBucket"); - - variantSet.add(spec6); - variantSet.add(spec6b); - - patientSet.add(197506); - patientSet.add(197508); - patientSet.add(197509); - - Collection filteredVariantSet = bucketIndexBySampleChr4.filterVariantSetForPatientSet(variantSet, patientSet); - - assertTrue("Patients should not match any variants in the list", filteredVariantSet.isEmpty()); - } - - @Test - public void test_filterVariantSetForPatientSet_allValidFirstBucket() throws IOException { - System.out.println("test_filterVariantSetForPatientSet_allValidFirstBucket"); - - variantSet.add(spec6); - variantSet.add(spec6b); - - patientSet.add(200392); - patientSet.add(200689); - patientSet.add(200972); - - Collection filteredVariantSet = bucketIndexBySampleChr21.filterVariantSetForPatientSet(variantSet, patientSet); - - assertEquals("No variants should be filtered out", 2, filteredVariantSet.size()); - } - @Test - public void test_filterVariantSetForPatientSet_allValidFirstBucketWithNoCall() throws IOException { - System.out.println("test_filterVariantSetForPatientSet_allValidFirstBucket"); - - variantSet.add("chr20,5032061,A,G,ABC1,missense_variant"); - variantSet.add("chr20,5032061,A,G,DEF1,synonymous_variant"); - - patientSet.add(200392); - patientSet.add(200689); - patientSet.add(200972); - - Collection filteredVariantSet = bucketIndexBySampleChr20.filterVariantSetForPatientSet(variantSet, patientSet); - - assertEquals("No variants should be filtered out", 2, filteredVariantSet.size()); - } - - @Test - public void test_filterVariantSetForPatientSet_allValidDifferentPatients() throws IOException { - System.out.println("test_filterVariantSetForPatientSet_allValidDifferentPatients"); - - variantSet.add(spec1); - variantSet.add(spec4); - variantSet.add(spec5); - variantSet.add(spec7); - - patientSet.add(200194); - patientSet.add(200450); - patientSet.add(200710); - patientSet.add(198206); - - Collection filteredVariantSet = bucketIndexBySampleChr4.filterVariantSetForPatientSet(variantSet, patientSet); - filteredVariantSet.addAll(bucketIndexBySampleChr21.filterVariantSetForPatientSet(variantSet, patientSet)); - - assertEquals("No variants should be filtered out", (long)4, (long)filteredVariantSet.size()); - } - - @Test - @Disabled - public void test_filterVariantSetForPatientSet_someValidDifferentPatients() throws IOException { - System.out.println("test_filterVariantSetForPatientSet_allValidDifferentPatients"); - - //specs 1-5 are in the last bucket - variantSet.add(spec1); - variantSet.add(spec4); - variantSet.add(spec5); - variantSet.add(spec8); - variantSet.add(spec9); //none - - patientSet.add(3); - patientSet.add(9); - patientSet.add(10); - - Collection filteredVariantSet = bucketIndexBySampleChr4.filterVariantSetForPatientSet(variantSet, patientSet); - filteredVariantSet.addAll(bucketIndexBySampleChr21.filterVariantSetForPatientSet(variantSet, patientSet)); - - assertEquals("One variant should be filtered out", (long)4, (long)filteredVariantSet.size()); - assertFalse("Spec 9 should have been filtered out", filteredVariantSet.contains(spec9)); - } - + private static final String STORAGE_DIR = "./target/all/"; + + private static BucketIndexBySample bucketIndexBySampleChr4; + private static BucketIndexBySample bucketIndexBySampleChr20; + private static BucketIndexBySample bucketIndexBySampleChr21; + + // Some known variant specs from the input file. + // Some known variant specs from the input file. These have been designed for testing partially overlapping specs + private static final String spec1 = "chr4,9856624,CAAAAA,C,TVP23A,splice_acceptor_variant"; + private static final String spec2 = "chr4,9856624,CAAA,C,TVP23A,splice_acceptor_variant"; + private static final String spec3 = "chr4,9856624,CA,C,TVP23A,splice_acceptor_variant"; + private static final String spec4 = "chr4,9856624,C,CA,TVP23A,splice_acceptor_variant"; + private static final String spec5 = "chr4,9856624,CAAAAA,CA,TVP23A,splice_acceptor_variant"; + + private static final String spec6 = "chr21,5032061,A,G,LOC102723996,missense_variant"; + private static final String spec6b = "chr21,5032061,A,G,ABCDEF123456,synonymous_variant"; + private static final String spec7 = "chr21,5033914,A,G,LOC102723996,missense_variant"; + private static final String spec8 = "chr21,5033988,C,G,LOC102723996,synonymous_variant"; + private static final String spec9 = "chr21,5034028,C,T,LOC102723996,missense_variant"; + + + // these parameters to the BucketIndexBySample methods are configured by each test + Set variantSet; + List patientSet; + + @BeforeAll + public static void initializeBinfile() throws Exception { + BuildIntegrationTestEnvironment instance = BuildIntegrationTestEnvironment.INSTANCE; + VariantStore variantStoreChr4 = VariantStore.readInstance(STORAGE_DIR + "chr4/"); + bucketIndexBySampleChr4 = new BucketIndexBySample(variantStoreChr4, STORAGE_DIR + "chr4/"); + + VariantStore variantStoreChr20 = VariantStore.readInstance(STORAGE_DIR + "chr20/"); + bucketIndexBySampleChr20 = new BucketIndexBySample(variantStoreChr20, STORAGE_DIR + "chr20/"); + + VariantStore variantStoreChr21 = VariantStore.readInstance(STORAGE_DIR + "chr21/"); + bucketIndexBySampleChr21 = new BucketIndexBySample(variantStoreChr21, STORAGE_DIR + "chr21/"); + } + + @BeforeEach + public void setUpTest() { + // start with fresh, empty collections + variantSet = new HashSet<>(); + patientSet = new ArrayList<>(); + } + + @Test + public void test_filterVariantSetForPatientSet_noPatients() throws IOException { + variantSet.add(spec1); + variantSet.add(spec2); + variantSet.add(spec3); + + Collection filteredVariantSet = bucketIndexBySampleChr4.filterVariantSetForPatientSet(variantSet, patientSet); + + assertTrue("Empty Patient List should filter out all variants", filteredVariantSet.isEmpty()); + } + + @Test + public void test_filterVariantSetForPatientSet_noVariants() throws IOException { + patientSet.add(200392); + patientSet.add(200689); + patientSet.add(200972); + + Collection filteredVariantSet = bucketIndexBySampleChr4.filterVariantSetForPatientSet(variantSet, patientSet); + + assertTrue("Empty Variant Set should remain empty", filteredVariantSet.isEmpty()); + } + + @Test + public void test_filterVariantSetForPatientSet_VariantsWithoutPatientsLastBucket() throws IOException { + System.out.println("test_filterVariantSetForPatientSet_VariantsWithoutPatientsLastBucket"); + + variantSet.add(spec1); + variantSet.add(spec2); + variantSet.add(spec3); + variantSet.add(spec4); + variantSet.add(spec5); + + patientSet.add(200706); + patientSet.add(200709); + + Collection filteredVariantSet = bucketIndexBySampleChr4.filterVariantSetForPatientSet(variantSet, patientSet); + + assertTrue("Patients should not match any variants in the list", filteredVariantSet.isEmpty()); + } + + @Test + public void test_filterVariantSetForPatientSet_PatientsWithNoVariantsFirstBucket() throws IOException { + System.out.println("test_filterVariantSetForPatientSet_PatientsWithNoVariantsFirstBucket"); + + variantSet.add(spec6); + variantSet.add(spec6b); + + patientSet.add(197506); + patientSet.add(197508); + patientSet.add(197509); + + Collection filteredVariantSet = bucketIndexBySampleChr4.filterVariantSetForPatientSet(variantSet, patientSet); + + assertTrue("Patients should not match any variants in the list", filteredVariantSet.isEmpty()); + } + + @Test + public void filterVariantSetForPatientSet_variantsDoNotExist_returnNoVariants() throws IOException { + System.out.println("test_filterVariantSetForPatientSet_PatientsWithNoVariantsFirstBucket"); + + variantSet.add(spec6); + variantSet.add(spec6b); + + patientSet.add(197506); + patientSet.add(197508); + patientSet.add(197509); + + Collection filteredVariantSet = bucketIndexBySampleChr4.filterVariantSetForPatientSet(variantSet, patientSet); + + assertTrue("Patients should not match any variants in the list", filteredVariantSet.isEmpty()); + } + + @Test + public void test_filterVariantSetForPatientSet_allValidFirstBucket() throws IOException { + System.out.println("test_filterVariantSetForPatientSet_allValidFirstBucket"); + + variantSet.add(spec6); + variantSet.add(spec6b); + + patientSet.add(200392); + patientSet.add(200689); + patientSet.add(200972); + + Collection filteredVariantSet = bucketIndexBySampleChr21.filterVariantSetForPatientSet(variantSet, patientSet); + + assertEquals("No variants should be filtered out", 2, filteredVariantSet.size()); + } + + @Test + public void test_filterVariantSetForPatientSet_allValidFirstBucketWithNoCall() throws IOException { + System.out.println("test_filterVariantSetForPatientSet_allValidFirstBucket"); + + variantSet.add("chr20,5032061,A,G,ABC1,missense_variant"); + variantSet.add("chr20,5032061,A,G,DEF1,synonymous_variant"); + + patientSet.add(200392); + patientSet.add(200689); + patientSet.add(200972); + + Collection filteredVariantSet = bucketIndexBySampleChr20.filterVariantSetForPatientSet(variantSet, patientSet); + + assertEquals("No variants should be filtered out", 2, filteredVariantSet.size()); + } + + @Test + public void test_filterVariantSetForPatientSet_allValidDifferentPatients() throws IOException { + System.out.println("test_filterVariantSetForPatientSet_allValidDifferentPatients"); + + variantSet.add(spec1); + variantSet.add(spec4); + variantSet.add(spec5); + variantSet.add(spec7); + + patientSet.add(200194); + patientSet.add(200450); + patientSet.add(200710); + patientSet.add(198206); + + Collection filteredVariantSet = bucketIndexBySampleChr4.filterVariantSetForPatientSet(variantSet, patientSet); + filteredVariantSet.addAll(bucketIndexBySampleChr21.filterVariantSetForPatientSet(variantSet, patientSet)); + + assertEquals("No variants should be filtered out", (long) 4, (long) filteredVariantSet.size()); + } + + @Test + @Disabled + public void test_filterVariantSetForPatientSet_someValidDifferentPatients() throws IOException { + System.out.println("test_filterVariantSetForPatientSet_allValidDifferentPatients"); + + // specs 1-5 are in the last bucket + variantSet.add(spec1); + variantSet.add(spec4); + variantSet.add(spec5); + variantSet.add(spec8); + variantSet.add(spec9); // none + + patientSet.add(3); + patientSet.add(9); + patientSet.add(10); + + Collection filteredVariantSet = bucketIndexBySampleChr4.filterVariantSetForPatientSet(variantSet, patientSet); + filteredVariantSet.addAll(bucketIndexBySampleChr21.filterVariantSetForPatientSet(variantSet, patientSet)); + + assertEquals("One variant should be filtered out", (long) 4, (long) filteredVariantSet.size()); + assertFalse("Spec 9 should have been filtered out", filteredVariantSet.contains(spec9)); + } + } diff --git a/services/pic-sure-hpds/service/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/test/CountProcessorIntegrationTest.java b/services/pic-sure-hpds/service/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/test/CountProcessorIntegrationTest.java index bece9cecd..2f26218a9 100644 --- a/services/pic-sure-hpds/service/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/test/CountProcessorIntegrationTest.java +++ b/services/pic-sure-hpds/service/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/test/CountProcessorIntegrationTest.java @@ -36,10 +36,12 @@ public static void beforeAll() { public void runCategoryCrossCounts_twoFilters() { Query query = new Query(); query.setCrossCountFields(List.of()); - query.setCategoryFilters(Map.of( + query.setCategoryFilters( + Map.of( "\\open_access-1000Genomes\\data\\SEX\\", new String[] {"male", "female"}, "\\open_access-1000Genomes\\data\\POPULATION NAME\\", new String[] {"Finnish"} - )); + ) + ); query.setExpectedResultType(ResultType.CROSS_COUNT); Map> crossCounts = countProcessor.runCategoryCrossCounts(query); @@ -52,10 +54,9 @@ public void runCategoryCrossCounts_twoFilters() { @Test public void runCategoryCrossCounts_snpFilter() { Query query = new Query(); - query.setCategoryFilters(Map.of( - "chr21,5032061,A,G", new String[]{"0/1", "1/1"}, - "\\open_access-1000Genomes\\data\\SEX\\", new String[] {"male"} - )); + query.setCategoryFilters( + Map.of("chr21,5032061,A,G", new String[] {"0/1", "1/1"}, "\\open_access-1000Genomes\\data\\SEX\\", new String[] {"male"}) + ); query.setExpectedResultType(ResultType.CROSS_COUNT); Map> crossCounts = countProcessor.runCategoryCrossCounts(query); diff --git a/services/pic-sure-hpds/service/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/test/CountV3ProcessorIntegrationTest.java b/services/pic-sure-hpds/service/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/test/CountV3ProcessorIntegrationTest.java index 28c1b7bba..ac2ba4df8 100644 --- a/services/pic-sure-hpds/service/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/test/CountV3ProcessorIntegrationTest.java +++ b/services/pic-sure-hpds/service/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/test/CountV3ProcessorIntegrationTest.java @@ -122,9 +122,9 @@ public void runCategoryCrossCounts_twoFilters() { } /** - * With an OR clause the cohort is broader than any single filter's values, so a cross count reports each concept's full - * distribution across that cohort: SEX shows female (present via the OR'd POPULATION branch) alongside the called-out male, and - * POPULATION shows the populations of the OR'd males alongside the called-out Finnish. + * With an OR clause the cohort is broader than any single filter's values, so a cross count reports each concept's full distribution + * across that cohort: SEX shows female (present via the OR'd POPULATION branch) alongside the called-out male, and POPULATION shows the + * populations of the OR'd males alongside the called-out Finnish. */ @Test public void runCategoryCrossCounts_twoFiltersOr_reportsFullCohortDistribution() { diff --git a/services/pic-sure-hpds/service/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/test/VariantMetadataIndexTest.java b/services/pic-sure-hpds/service/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/test/VariantMetadataIndexTest.java index 7a53ac60e..3284ff1cf 100644 --- a/services/pic-sure-hpds/service/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/test/VariantMetadataIndexTest.java +++ b/services/pic-sure-hpds/service/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/test/VariantMetadataIndexTest.java @@ -20,159 +20,170 @@ public class VariantMetadataIndexTest { - /** - * The metadataIndex is non-mutable (or should be) so we only need one object to test - */ - private static VariantMetadataIndex variantMetadataIndexChr4; - private static VariantMetadataIndex variantMetadataIndexChr21; - public static String chr4BinFile = "target/all/chr4/VariantMetadata.javabin"; - public static String chr21BinFile = "target/all/chr21/VariantMetadata.javabin"; - VariantBucketHolder bucketCache = new VariantBucketHolder(); - - //Some known variant specs from the input file. These have been designed for testing partially overlapping specs - private static final String spec1 = "chr4,9856624,CAAAAA,C,TVP23A,splice_acceptor_variant"; private static final String spec1Info = "Gene_with_variant=TVP23A;Variant_consequence_calculated=splice_acceptor_variant;AC=401;AF=8.00719e-02;NS=2504;AN=5008;EAS_AF=3.37000e-02;EUR_AF=4.97000e-02;AFR_AF=1.64100e-01;AMR_AF=3.75000e-02;SAS_AF=7.57000e-02;DP=18352;AA=G|||;VT=SNP"; - private static final String spec2 = "chr4,9856624,CAAA,C,TVP23A,splice_acceptor_variant"; private static final String spec2Info = "Gene_with_variant=TVP23A;Variant_consequence_calculated=splice_acceptor_variant;AC=62;AF=1.23802e-02;NS=2504;AN=5008;EAS_AF=0.00000e+00;EUR_AF=1.00000e-03;AFR_AF=4.54000e-02;AMR_AF=1.40000e-03;SAS_AF=0.00000e+00;DP=18328;AA=T|||;VT=SNP"; - private static final String spec3 = "chr4,9856624,CA,C,TVP23A,splice_acceptor_variant"; private static final String spec3Info = "Gene_with_variant=TVP23A;Variant_consequence_calculated=splice_acceptor_variant;AC=8;AF=1.59744e-03;NS=2504;AN=5008;EAS_AF=0.00000e+00;EUR_AF=0.00000e+00;AFR_AF=6.10000e-03;AMR_AF=0.00000e+00;SAS_AF=0.00000e+00;DP=18519;AA=T|||;VT=SNP"; - private static final String spec4 = "chr4,9856624,C,CA,TVP23A,splice_acceptor_variant"; private static final String spec4Info = "Gene_with_variant=TVP23A;Variant_consequence_calculated=splice_acceptor_variant;AC=75;AF=1.49760e-02;NS=2504;AN=5008;EAS_AF=3.27000e-02;EUR_AF=2.49000e-02;AFR_AF=6.80000e-03;AMR_AF=4.30000e-03;SAS_AF=5.10000e-03;DP=18008;AA=A|||;VT=SNP"; - private static final String spec5 = "chr4,9856624,CAAAAA,CA,TVP23A,splice_acceptor_variant"; private static final String spec5Info = "Gene_with_variant=TVP23A;Variant_consequence_calculated=splice_acceptor_variant;AC=3033;AF=6.05631e-01;NS=2504;AN=5008;EAS_AF=5.23800e-01;EUR_AF=7.54500e-01;AFR_AF=4.28900e-01;AMR_AF=7.82400e-01;SAS_AF=6.50300e-01;DP=20851;VT=INDEL"; - - - @BeforeAll - public static void initializeBinfiles() throws Exception { - BuildIntegrationTestEnvironment instance = BuildIntegrationTestEnvironment.INSTANCE; - if(new File(chr4BinFile).exists()) { - try(ObjectInputStream in = new ObjectInputStream(new GZIPInputStream(new FileInputStream(chr4BinFile)))){ - variantMetadataIndexChr4 = (VariantMetadataIndex) in.readObject(); - }catch(Exception e) { - e.printStackTrace(); - } - } - if(new File(chr21BinFile).exists()) { - try(ObjectInputStream in = new ObjectInputStream(new GZIPInputStream(new FileInputStream(chr21BinFile)))){ - variantMetadataIndexChr21 = (VariantMetadataIndex) in.readObject(); - }catch(Exception e) { - e.printStackTrace(); - } - } - } - - @Test - public void findByMultipleVariantSpec_invalidSpec() { - List variants = List.of("chr21,5032061,A,G,NOTAGENE,missense_variant"); - Map> expectedResult = Map.of(); - Map> data = variantMetadataIndexChr21.findByMultipleVariantSpec(variants); - - assertEquals("Wrong number of records in response.", 1, data.size()); - assertEquals("The expected values were not found.", Set.of(), data.get("chr21,5032061,A,G,NOTAGENE,missense_variant")); - } - @Test - public void findByMultipleVariantSpec_validSpec() { - List variants = List.of("chr21,5032061,A,G,LOC102723996,missense_variant"); - Map> expectedResult = Map.of( - "chr21,5032061,A,G,LOC102723996,missense_variant" - , Set.of("Gene_with_variant=LOC102723996;Variant_severity=MODERATE;Variant_consequence_calculated=missense_variant;Variant_class=SNV;Variant_frequency_in_gnomAD=0.0001346;Variant_frequency_as_text=Rare")); - Map> data = variantMetadataIndexChr21.findByMultipleVariantSpec(variants); - - assertEquals("Wrong number of records in response.", data.size(), 1); - variants.stream().forEach(variant->{ - assertEquals("The expected values were not found.", expectedResult.get(variant), data.get(variant)); - }); - } - @Test - public void findByMultipleVariantSpec_validSpecs() { - List variants = List.of("chr21,5032061,A,G,LOC102723996,missense_variant", "chr21,5033914,A,G,LOC102723996,missense_variant"); - Map> expectedResult = Map.of( - "chr21,5032061,A,G,LOC102723996,missense_variant" - , Set.of("Gene_with_variant=LOC102723996;Variant_severity=MODERATE;Variant_consequence_calculated=missense_variant;Variant_class=SNV;Variant_frequency_in_gnomAD=0.0001346;Variant_frequency_as_text=Rare") - ,"chr21,5033914,A,G,LOC102723996,missense_variant" - , Set.of("Gene_with_variant=LOC102723996;Variant_severity=MODERATE;Variant_consequence_calculated=missense_variant;Variant_class=SNV;Variant_frequency_in_gnomAD=0.0009728;Variant_frequency_as_text=Rare")); - Map> data = variantMetadataIndexChr21.findByMultipleVariantSpec(variants); - - assertEquals("Wrong number of records in response.", data.size(), 2); - variants.stream().forEach(variant->{ - assertEquals("The expected values were not found.", expectedResult.get(variant), data.get(variant)); - }); - } - - @Test - public void testMultipleVariantSpecSamePOS() { - - List variants = List.of(spec1, spec4); - Map> expectedResult = Map.of( - spec1, Set.of(spec1Info), - spec4, Set.of(spec4Info)); - Map> data = variantMetadataIndexChr4.findByMultipleVariantSpec(variants); - - assertEquals("Wrong number of records in response.", data.size(), 2); - variants.stream().forEach(variant->{ - assertEquals("The expected values were not found.", expectedResult.get(variant), data.get(variant)); - }); - } - - @Test - public void testMultipleVariantSpecSamePOSAndREF() { - List variants = List.of(spec1, spec5); - Map> expectedResult = Map.of( - spec1, Set.of(spec1Info), - spec5, Set.of(spec5Info)); - Map> data = variantMetadataIndexChr4.findByMultipleVariantSpec(variants); - - assertEquals("Wrong number of records in response.", data.size(), 2); - variants.stream().forEach(variant->{ - assertEquals("The expected values were not found.", expectedResult.get(variant), data.get(variant)); - }); - } - - @Test - public void testMultipleVariantSpecSamePOSAndALT() { - List variants = List.of(spec1, spec2); - Map> expectedResult = Map.of( - spec1, Set.of(spec1Info), - spec2, Set.of(spec2Info)); - Map> data = variantMetadataIndexChr4.findByMultipleVariantSpec(variants); - - assertEquals("Wrong number of records in response.", data.size(), 2); - variants.stream().forEach(variant->{ - assertEquals("The expected values were not found.", expectedResult.get(variant), data.get(variant)); - }); - } - - /** - * The google API that we use throws an IllegalStateException on duplicate entries - */ - @Test - public void testMultipleVariantSpecSameSpec() { - assertThrows(IllegalStateException.class, () -> { - List variants = List.of(spec1, spec1); - Map> expectedResult = Map.of( - spec1, Set.of(spec1Info)); - Map> data = variantMetadataIndexChr4.findByMultipleVariantSpec(variants); - - assertEquals("Wrong number of records in response.", data.size(), 1); - variants.stream().forEach(variant->{ - assertEquals("The expected values were not found.", expectedResult.get(variant), data.get(variant)); - }); - }); - } - - @Test - public void testVariantSpecMapSorting() { - Map specMap = Map.of( - spec1, new String[]{spec1Info}, - spec2, new String[]{spec2Info}); - - TreeMap metadataSorted = new TreeMap<>((o1, o2) -> { - return new VariantSpec(o1).compareTo(new VariantSpec(o2)); - }); - metadataSorted.putAll(specMap); - - assertEquals("Wrong number of records in response.", metadataSorted.size(), 2); - assertNotNull("spec1 value not present in the sorted map", metadataSorted.get(spec1)); - assertEquals("Incorrect spec1 value in the sorted map", spec1Info, metadataSorted.get(spec1)[0]); - assertNotNull("spec2 value not present in the sorted map", metadataSorted.get(spec2)); - assertEquals("Incorrect spec2 value in the sorted map", spec2Info, metadataSorted.get(spec2)[0]); - } - - + /** + * The metadataIndex is non-mutable (or should be) so we only need one object to test + */ + private static VariantMetadataIndex variantMetadataIndexChr4; + private static VariantMetadataIndex variantMetadataIndexChr21; + public static String chr4BinFile = "target/all/chr4/VariantMetadata.javabin"; + public static String chr21BinFile = "target/all/chr21/VariantMetadata.javabin"; + VariantBucketHolder bucketCache = new VariantBucketHolder(); + + // Some known variant specs from the input file. These have been designed for testing partially overlapping specs + private static final String spec1 = "chr4,9856624,CAAAAA,C,TVP23A,splice_acceptor_variant"; + private static final String spec1Info = + "Gene_with_variant=TVP23A;Variant_consequence_calculated=splice_acceptor_variant;AC=401;AF=8.00719e-02;NS=2504;AN=5008;EAS_AF=3.37000e-02;EUR_AF=4.97000e-02;AFR_AF=1.64100e-01;AMR_AF=3.75000e-02;SAS_AF=7.57000e-02;DP=18352;AA=G|||;VT=SNP"; + private static final String spec2 = "chr4,9856624,CAAA,C,TVP23A,splice_acceptor_variant"; + private static final String spec2Info = + "Gene_with_variant=TVP23A;Variant_consequence_calculated=splice_acceptor_variant;AC=62;AF=1.23802e-02;NS=2504;AN=5008;EAS_AF=0.00000e+00;EUR_AF=1.00000e-03;AFR_AF=4.54000e-02;AMR_AF=1.40000e-03;SAS_AF=0.00000e+00;DP=18328;AA=T|||;VT=SNP"; + private static final String spec3 = "chr4,9856624,CA,C,TVP23A,splice_acceptor_variant"; + private static final String spec3Info = + "Gene_with_variant=TVP23A;Variant_consequence_calculated=splice_acceptor_variant;AC=8;AF=1.59744e-03;NS=2504;AN=5008;EAS_AF=0.00000e+00;EUR_AF=0.00000e+00;AFR_AF=6.10000e-03;AMR_AF=0.00000e+00;SAS_AF=0.00000e+00;DP=18519;AA=T|||;VT=SNP"; + private static final String spec4 = "chr4,9856624,C,CA,TVP23A,splice_acceptor_variant"; + private static final String spec4Info = + "Gene_with_variant=TVP23A;Variant_consequence_calculated=splice_acceptor_variant;AC=75;AF=1.49760e-02;NS=2504;AN=5008;EAS_AF=3.27000e-02;EUR_AF=2.49000e-02;AFR_AF=6.80000e-03;AMR_AF=4.30000e-03;SAS_AF=5.10000e-03;DP=18008;AA=A|||;VT=SNP"; + private static final String spec5 = "chr4,9856624,CAAAAA,CA,TVP23A,splice_acceptor_variant"; + private static final String spec5Info = + "Gene_with_variant=TVP23A;Variant_consequence_calculated=splice_acceptor_variant;AC=3033;AF=6.05631e-01;NS=2504;AN=5008;EAS_AF=5.23800e-01;EUR_AF=7.54500e-01;AFR_AF=4.28900e-01;AMR_AF=7.82400e-01;SAS_AF=6.50300e-01;DP=20851;VT=INDEL"; + + + @BeforeAll + public static void initializeBinfiles() throws Exception { + BuildIntegrationTestEnvironment instance = BuildIntegrationTestEnvironment.INSTANCE; + if (new File(chr4BinFile).exists()) { + try (ObjectInputStream in = new ObjectInputStream(new GZIPInputStream(new FileInputStream(chr4BinFile)))) { + variantMetadataIndexChr4 = (VariantMetadataIndex) in.readObject(); + } catch (Exception e) { + e.printStackTrace(); + } + } + if (new File(chr21BinFile).exists()) { + try (ObjectInputStream in = new ObjectInputStream(new GZIPInputStream(new FileInputStream(chr21BinFile)))) { + variantMetadataIndexChr21 = (VariantMetadataIndex) in.readObject(); + } catch (Exception e) { + e.printStackTrace(); + } + } + } + + @Test + public void findByMultipleVariantSpec_invalidSpec() { + List variants = List.of("chr21,5032061,A,G,NOTAGENE,missense_variant"); + Map> expectedResult = Map.of(); + Map> data = variantMetadataIndexChr21.findByMultipleVariantSpec(variants); + + assertEquals("Wrong number of records in response.", 1, data.size()); + assertEquals("The expected values were not found.", Set.of(), data.get("chr21,5032061,A,G,NOTAGENE,missense_variant")); + } + + @Test + public void findByMultipleVariantSpec_validSpec() { + List variants = List.of("chr21,5032061,A,G,LOC102723996,missense_variant"); + Map> expectedResult = Map.of( + "chr21,5032061,A,G,LOC102723996,missense_variant", + Set.of( + "Gene_with_variant=LOC102723996;Variant_severity=MODERATE;Variant_consequence_calculated=missense_variant;Variant_class=SNV;Variant_frequency_in_gnomAD=0.0001346;Variant_frequency_as_text=Rare" + ) + ); + Map> data = variantMetadataIndexChr21.findByMultipleVariantSpec(variants); + + assertEquals("Wrong number of records in response.", data.size(), 1); + variants.stream().forEach(variant -> { + assertEquals("The expected values were not found.", expectedResult.get(variant), data.get(variant)); + }); + } + + @Test + public void findByMultipleVariantSpec_validSpecs() { + List variants = + List.of("chr21,5032061,A,G,LOC102723996,missense_variant", "chr21,5033914,A,G,LOC102723996,missense_variant"); + Map> expectedResult = Map.of( + "chr21,5032061,A,G,LOC102723996,missense_variant", + Set.of( + "Gene_with_variant=LOC102723996;Variant_severity=MODERATE;Variant_consequence_calculated=missense_variant;Variant_class=SNV;Variant_frequency_in_gnomAD=0.0001346;Variant_frequency_as_text=Rare" + ), "chr21,5033914,A,G,LOC102723996,missense_variant", + Set.of( + "Gene_with_variant=LOC102723996;Variant_severity=MODERATE;Variant_consequence_calculated=missense_variant;Variant_class=SNV;Variant_frequency_in_gnomAD=0.0009728;Variant_frequency_as_text=Rare" + ) + ); + Map> data = variantMetadataIndexChr21.findByMultipleVariantSpec(variants); + + assertEquals("Wrong number of records in response.", data.size(), 2); + variants.stream().forEach(variant -> { + assertEquals("The expected values were not found.", expectedResult.get(variant), data.get(variant)); + }); + } + + @Test + public void testMultipleVariantSpecSamePOS() { + + List variants = List.of(spec1, spec4); + Map> expectedResult = Map.of(spec1, Set.of(spec1Info), spec4, Set.of(spec4Info)); + Map> data = variantMetadataIndexChr4.findByMultipleVariantSpec(variants); + + assertEquals("Wrong number of records in response.", data.size(), 2); + variants.stream().forEach(variant -> { + assertEquals("The expected values were not found.", expectedResult.get(variant), data.get(variant)); + }); + } + + @Test + public void testMultipleVariantSpecSamePOSAndREF() { + List variants = List.of(spec1, spec5); + Map> expectedResult = Map.of(spec1, Set.of(spec1Info), spec5, Set.of(spec5Info)); + Map> data = variantMetadataIndexChr4.findByMultipleVariantSpec(variants); + + assertEquals("Wrong number of records in response.", data.size(), 2); + variants.stream().forEach(variant -> { + assertEquals("The expected values were not found.", expectedResult.get(variant), data.get(variant)); + }); + } + + @Test + public void testMultipleVariantSpecSamePOSAndALT() { + List variants = List.of(spec1, spec2); + Map> expectedResult = Map.of(spec1, Set.of(spec1Info), spec2, Set.of(spec2Info)); + Map> data = variantMetadataIndexChr4.findByMultipleVariantSpec(variants); + + assertEquals("Wrong number of records in response.", data.size(), 2); + variants.stream().forEach(variant -> { + assertEquals("The expected values were not found.", expectedResult.get(variant), data.get(variant)); + }); + } + + /** + * The google API that we use throws an IllegalStateException on duplicate entries + */ + @Test + public void testMultipleVariantSpecSameSpec() { + assertThrows(IllegalStateException.class, () -> { + List variants = List.of(spec1, spec1); + Map> expectedResult = Map.of(spec1, Set.of(spec1Info)); + Map> data = variantMetadataIndexChr4.findByMultipleVariantSpec(variants); + + assertEquals("Wrong number of records in response.", data.size(), 1); + variants.stream().forEach(variant -> { + assertEquals("The expected values were not found.", expectedResult.get(variant), data.get(variant)); + }); + }); + } + + @Test + public void testVariantSpecMapSorting() { + Map specMap = Map.of(spec1, new String[] {spec1Info}, spec2, new String[] {spec2Info}); + + TreeMap metadataSorted = new TreeMap<>((o1, o2) -> { + return new VariantSpec(o1).compareTo(new VariantSpec(o2)); + }); + metadataSorted.putAll(specMap); + + assertEquals("Wrong number of records in response.", metadataSorted.size(), 2); + assertNotNull("spec1 value not present in the sorted map", metadataSorted.get(spec1)); + assertEquals("Incorrect spec1 value in the sorted map", spec1Info, metadataSorted.get(spec1)[0]); + assertNotNull("spec2 value not present in the sorted map", metadataSorted.get(spec2)); + assertEquals("Incorrect spec2 value in the sorted map", spec2Info, metadataSorted.get(spec2)[0]); + } + + } diff --git a/services/pic-sure-hpds/war/pom.xml b/services/pic-sure-hpds/war/pom.xml index ff4f44fdb..7d58bc818 100644 --- a/services/pic-sure-hpds/war/pom.xml +++ b/services/pic-sure-hpds/war/pom.xml @@ -6,7 +6,7 @@ pic-sure-hpds edu.harvard.hms.dbmi.avillach.hpds - 3.0.0-SNAPSHOT + ${revision} hpds-war war