Specter/submitSpecter.sh at master · mtrnka/Specter · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
#!/bin/bash
blib="gs://specter-dia/blibs/PC3_insilicoLib_A.blib"

while IFS= read -r line;
#for i in $( gsutil ls gs://specter-dia/data | head -n 35)
do
  if [[ $line == *.mzML ]]
    then
      gcloud dataproc jobs submit pyspark \
      /Users/mtrnka/Projects-Mine/SpectraNet/specter/Specter/Specter_Spark.py \
      --cluster $1 \
      --bucket specter-dia \
      --files $line,$blib \
      --properties=\
'spark.executor.memory=27G',\
'spark.executor.memoryOverhead=2G',\
'spark.executor.cores=4',\
'spark.driver.memory=27G',\
'spark.driver.cores=4',\
'spark.default.parallelism=120'\
      --py-files /Users/mtrnka/Projects-Mine/SpectraNet/specter/Specter/sparse_nnls.py \
      --project specter1 \
      -- ${line#gs://specter-dia/data/} \
         ${blib#gs://specter-dia/blibs/} \
         22000 end 5000 orbitrap 10 False

      gcloud dataproc jobs submit pyspark \
      /Users/mtrnka/Projects-Mine/SpectraNet/specter/Specter/Specter_Spark.py \
      --cluster $1 \
      --bucket specter-dia \
      --files $line,$blib \
      --properties=\
'spark.executor.memory=27G',\
'spark.executor.memoryOverhead=2G',\
'spark.executor.cores=4',\
'spark.driver.memory=27G',\
'spark.driver.cores=8',\
'spark.default.parallelism=120'\
      --py-files /Users/mtrnka/Projects-Mine/SpectraNet/specter/Specter/sparse_nnls.py \
      --project specter1 \
      -- ${line#gs://specter-dia/data/} \
         ${blib#gs://specter-dia/blibs/} \
         22001 start 5000 orbitrap 10 True
  fi
done < "$2"