diff --git a/assets/output_contract.tsv b/assets/output_contract.tsv index 5477c732..7b6ffbd1 100644 --- a/assets/output_contract.tsv +++ b/assets/output_contract.tsv @@ -52,52 +52,51 @@ file {family}/{sample}_dnapainter.csv 38 FAMILY_REPORT nextflow csv experimental file {family}/{sample}_inheritance.tsv.gz 38 FAMILY_REPORT nextflow tsv experimental - - file {family}/{sample}_segments.svg 38 FAMILY_REPORT nextflow svg experimental - - file {family}/{sample}_segments.tsv 38 FAMILY_REPORT nextflow tsv experimental - - -file {sample}/aligned/{sample}_read_count.tsv 02 ALIGN_MINIMAP2 nextflow tsv stable - - file {sample}/aligned/{sample}_sorted.bam 02 ALIGN_MINIMAP2 both bam stable bash - file {sample}/aligned/{sample}_sorted.bam.bai 02 ALIGN_MINIMAP2 both index stable bash - -file {sample}/aligned/{sample}_sorted.cram 34 CRAM_ARCHIVE nextflow cram experimental - - -file {sample}/aligned/{sample}_sorted.cram.crai 34 CRAM_ARCHIVE nextflow index experimental - - -file {sample}/aligned/{sample}_sorted.cram.flagstat 34 CRAM_ARCHIVE nextflow bin experimental - - -file {sample}/ancestry/{sample}_ancestry.tsv 25 PRS_SUMMARY nextflow tsv stable - - -file {sample}/ancestry_fine/{sample}_ancestry_fine.tsv 26b ANCESTRY_FINE_REPORT nextflow tsv experimental - - +file {sample}/aligned/{sample}_sorted.cram 34 CRAM_ARCHIVE both cram experimental - - +file {sample}/aligned/{sample}_sorted.cram.crai 34 CRAM_ARCHIVE both index experimental - - +file {sample}/aligned/{sample}_sorted.cram.flagstat 34 CRAM_ARCHIVE both bin experimental - - +file {sample}/ancestry/{sample}_ancestry.tsv 25 PRS_SUMMARY both tsv stable - - +file {sample}/ancestry_fine/{sample}_ancestry_fine.tsv 26b ANCESTRY_FINE_REPORT both tsv experimental - - file {sample}/ancestry_fine/{sample}_ancestry_fine.txt 26b ANCESTRY_FINE_REPORT nextflow text experimental - - -file {sample}/ancestry_fine/{sample}_painting.bed 26b ANCESTRY_FINE_REPORT nextflow bed experimental - - -file {sample}/ancestry_fine/{sample}_painting.png 26b ANCESTRY_FINE_REPORT nextflow png experimental - - -file {sample}/ancient/{sample}_ancient.tsv 44 ANCIENT_REPORT nextflow tsv experimental - - -file {sample}/ancient/{sample}_ancient.txt 44 ANCIENT_REPORT nextflow text experimental - - -file {sample}/ancient/{sample}_ancient_info.tsv 44 ANCIENT_REPORT nextflow tsv experimental - - -file {sample}/ancient/{sample}_ancient_pca.svg 44 ANCIENT_REPORT nextflow svg experimental - - -file {sample}/annotsv/{sample}_annotsv_input.vcf 05 ANNOTSV nextflow vcf stable - - -file {sample}/annotsv/{sample}_sv_annotated.tsv 05 ANNOTSV nextflow tsv stable - - -file {sample}/annotsv/{sample}_sv_class4_5.tsv 05 ANNOTSV nextflow tsv stable - - -file {sample}/annotsv/{sample}_sv_class4_5_checked.tsv 05 ANNOTSV nextflow tsv stable - - -file {sample}/archaic/{sample}_archaic.tsv 43 ARCHAIC_REPORT nextflow tsv experimental - - -file {sample}/archaic/{sample}_archaic.txt 43 ARCHAIC_REPORT nextflow text experimental - - -file {sample}/archaic/{sample}_archaic_segments.tsv 43 ARCHAIC_REPORT nextflow tsv experimental - - -file {sample}/bloodtype/{sample}_bloodtype.tsv 41 BLOODTYPE nextflow tsv experimental - - -file {sample}/bloodtype/{sample}_bloodtype_depth.tsv 41 BLOODTYPE_DEPTH nextflow tsv experimental - - -file {sample}/bloodtype/{sample}_bloodtype_evidence.tsv 41 BLOODTYPE nextflow tsv experimental - - -file {sample}/chip/{sample}_chip.tsv 49 CHIP_REPORT nextflow tsv experimental - - -file {sample}/chip/{sample}_chip_depth.tsv 49 CHIP_REPORT nextflow tsv experimental - - -file {sample}/chip/{sample}_chip_filtered.vcf.gz 49 CHIP_MUTECT2 nextflow vcf experimental - - +file {sample}/ancestry_fine/{sample}_painting.bed 26b ANCESTRY_FINE_REPORT both bed experimental - - +file {sample}/ancestry_fine/{sample}_painting.png 26b ANCESTRY_FINE_REPORT both png experimental - - +file {sample}/ancient/{sample}_ancient.tsv 44 ANCIENT_REPORT both tsv experimental - - +file {sample}/ancient/{sample}_ancient.txt 44 ANCIENT_REPORT both text experimental - - +file {sample}/ancient/{sample}_ancient_info.tsv 44 ANCIENT_REPORT both tsv experimental - - +file {sample}/ancient/{sample}_ancient_pca.svg 44 ANCIENT_REPORT both svg experimental - - +file {sample}/annotsv/{sample}_annotsv_input.vcf 05 ANNOTSV both vcf stable - - +file {sample}/annotsv/{sample}_sv_annotated.tsv 05 ANNOTSV both tsv stable - - +file {sample}/annotsv/{sample}_sv_class4_5.tsv 05 ANNOTSV both tsv stable - - +file {sample}/annotsv/{sample}_sv_class4_5_checked.tsv 05 ANNOTSV both tsv stable - - +file {sample}/archaic/{sample}_archaic.tsv 43 ARCHAIC_REPORT both tsv experimental - - +file {sample}/archaic/{sample}_archaic.txt 43 ARCHAIC_REPORT both text experimental - - +file {sample}/archaic/{sample}_archaic_segments.tsv 43 ARCHAIC_REPORT both tsv experimental - - +file {sample}/bloodtype/{sample}_bloodtype.tsv 41 BLOODTYPE both tsv experimental - - +file {sample}/bloodtype/{sample}_bloodtype_depth.tsv 41 BLOODTYPE_DEPTH both tsv experimental - - +file {sample}/bloodtype/{sample}_bloodtype_evidence.tsv 41 BLOODTYPE both tsv experimental - - +file {sample}/chip/{sample}_chip.tsv 49 CHIP_REPORT both tsv experimental - - +file {sample}/chip/{sample}_chip_depth.tsv 49 CHIP_REPORT both tsv experimental - - +file {sample}/chip/{sample}_chip_filtered.vcf.gz 49 CHIP_MUTECT2 both vcf experimental - - file {sample}/clinical/{sample}_clinical.vcf.gz 23 CLINICAL_FILTER both vcf stable bash - file {sample}/clinical/{sample}_clinical.vcf.gz.tbi 23 CLINICAL_FILTER nextflow index stable - - -file {sample}/clinical/{sample}_clinical_summary.tsv 23 CLINICAL_FILTER nextflow tsv stable - - -file {sample}/clinvar/{sample}_clinvar_cdna_check.tsv 06 CLINVAR_CDNA_CHECK nextflow tsv stable - - -file {sample}/clinvar/{sample}_clinvar_cdna_scan.tsv 06 CLINVAR_CDNA_CHECK nextflow tsv stable - - -file {sample}/clinvar/{sample}_clinvar_deep_intronic.tsv 06 CLINVAR_SCREEN nextflow tsv stable - - +file {sample}/clinical/{sample}_clinical_summary.tsv 23 CLINICAL_FILTER both tsv stable - - +file {sample}/clinvar/{sample}_clinvar_cdna_check.tsv 06 CLINVAR_CDNA_CHECK both tsv stable - - +file {sample}/clinvar/{sample}_clinvar_cdna_scan.tsv 06 CLINVAR_CDNA_CHECK both tsv stable - - +file {sample}/clinvar/{sample}_clinvar_deep_intronic.tsv 06 CLINVAR_SCREEN both tsv stable - - file {sample}/clinvar/{sample}_clinvar_hits.tsv 06 CLINVAR_SCREEN both tsv stable both chrom,pos,ref,alt,genotype,clinvar_id,geneinfo,clnsig,clnrevstat file {sample}/clinvar/{sample}_clinvar_hits.vcf 06 CLINVAR_SCREEN both vcf stable both INFO/GENEINFO,INFO/CLNSIG,INFO/CLNREVSTAT file {sample}/clinvar/{sample}_pass.vcf.gz 06 CLINVAR_SCREEN both vcf stable both FORMAT/GT file {sample}/clinvar/{sample}_pass.vcf.gz.tbi 06 CLINVAR_SCREEN both index stable both - file {sample}/cnvpytor/{sample}_cnvs.raw.vcf 18 CNVPYTOR nextflow vcf internal - - -file {sample}/cnvpytor/{sample}_cnvs.txt 18 CNVPYTOR nextflow text stable - - -file {sample}/cnvpytor/{sample}_cnvs.vcf.gz 18 CNVPYTOR_VCF nextflow vcf stable - - +file {sample}/cnvpytor/{sample}_cnvs.txt 18 CNVPYTOR both text stable - - +file {sample}/cnvpytor/{sample}_cnvs.vcf.gz 18 CNVPYTOR_VCF both vcf stable - - file {sample}/cnvpytor/{sample}_cnvs.vcf.gz.tbi 18 CNVPYTOR_VCF nextflow index stable - - -file {sample}/consumer_export/{sample}_23andme_v5.zip 40 CONSUMER_EXPORT nextflow zip experimental - - -file {sample}/consumer_export/{sample}_ancestrydna_layout.zip 40 CONSUMER_EXPORT nextflow zip experimental - - -file {sample}/consumer_export/{sample}_consumer_export.tsv 40 CONSUMER_EXPORT nextflow tsv experimental - - -file {sample}/couple/{sample}_couple_carrier.tsv 39 COUPLE_CARRIER nextflow tsv experimental - - +file {sample}/consumer_export/{sample}_23andme_v5.zip 40 CONSUMER_EXPORT both zip experimental - - +file {sample}/consumer_export/{sample}_ancestrydna_layout.zip 40 CONSUMER_EXPORT both zip experimental - - +file {sample}/consumer_export/{sample}_consumer_export.tsv 40 CONSUMER_EXPORT both tsv experimental - - +file {sample}/couple/{sample}_couple_carrier.tsv 39 COUPLE_CARRIER both tsv experimental - - file {sample}/coverage/{sample}.mosdepth.global.dist.txt 16b MOSDEPTH nextflow text stable - - file {sample}/coverage/{sample}.mosdepth.summary.txt 16b MOSDEPTH nextflow tsv stable nextflow chrom,length,bases,mean,min,max file {sample}/coverage/{sample}.regions.bed.gz 16b MOSDEPTH nextflow bed stable - - @@ -105,122 +104,123 @@ file {sample}/coverage/{sample}.thresholds.bed.gz 16b MOSDEPTH nextflow bed stab file {sample}/coverage/{sample}_callability.bed.gz 16b MOSDEPTH nextflow bed stable - - file {sample}/cpic/{sample}_cpic_recommendations.txt 27 CPIC_LOOKUP both text stable both - file {sample}/cpic/{sample}_phenotypes.tsv 27 CPIC_LOOKUP both tsv stable both - -file {sample}/cpsr/{sample}.cpsr.grch38.classification.tsv.gz 17 CPSR nextflow tsv stable - - -file {sample}/cpsr/{sample}.cpsr.grch38.html 17 CPSR nextflow html stable - - +file {sample}/cpsr/{sample}.cpsr.grch38.classification.tsv.gz 17 CPSR both tsv stable - - +file {sample}/cpsr/{sample}.cpsr.grch38.html 17 CPSR both html stable - - file {sample}/cpsr/{sample}_cpsr_cdna_check.tsv 17 CLINVAR_CDNA_CHECK both tsv stable - - -file {sample}/cyrius/{sample}_cyp2d6.tsv 21 CYRIUS nextflow tsv stable - - -file {sample}/cyrius/{sample}_cyp2d6_depth_check.tsv 21 CYRIUS nextflow tsv stable - - -file {sample}/delly/{sample}_sv.vcf.gz 19 DELLY_BCF2VCF nextflow vcf stable nextflow INFO/SVTYPE +file {sample}/cyrius/{sample}_cyp2d6.tsv 21 CYRIUS both tsv stable - - +file {sample}/cyrius/{sample}_cyp2d6_depth_check.tsv 21 CYRIUS both tsv stable - - +file {sample}/delly/{sample}_sv.vcf.gz 19 DELLY_BCF2VCF both vcf stable nextflow INFO/SVTYPE file {sample}/delly/{sample}_sv.vcf.gz.tbi 19 DELLY_BCF2VCF nextflow index stable nextflow - -file {sample}/exomiser/{sample}_exomiser.txt 48 EXOMISER_REPORT nextflow text experimental - - -file {sample}/exomiser/{sample}_exomiser.variants.tsv 48 EXOMISER nextflow tsv experimental - - -file {sample}/exomiser/{sample}_exomiser_terms.tsv 48 EXOMISER_REPORT nextflow tsv experimental - - -file {sample}/exomiser/{sample}_exomiser_top.tsv 48 EXOMISER_REPORT nextflow tsv experimental - - +file {sample}/exomiser/{sample}_exomiser.txt 48 EXOMISER_REPORT both text experimental - - +file {sample}/exomiser/{sample}_exomiser.variants.tsv 48 EXOMISER both tsv experimental - - +file {sample}/exomiser/{sample}_exomiser_terms.tsv 48 EXOMISER_REPORT both tsv experimental - - +file {sample}/exomiser/{sample}_exomiser_top.tsv 48 EXOMISER_REPORT both tsv experimental - - file {sample}/expansion_hunter/{sample}.json 09 EXPANSION_HUNTER nextflow json stable - - file {sample}/expansion_hunter/{sample}.vcf 09 EXPANSION_HUNTER nextflow vcf stable - - -file {sample}/expansion_hunter/{sample}_eh_stranger.vcf 09b STRANGER nextflow vcf stable - - +file {sample}/expansion_hunter/{sample}_eh_stranger.vcf 09b STRANGER both vcf stable - - file {sample}/expansion_hunter/{sample}_realigned.bam 09 EXPANSION_HUNTER nextflow bam stable - - -file {sample}/expansion_hunter/{sample}_rfc1_motif.tsv 09b STRANGER_RFC1 nextflow tsv stable - - -file {sample}/expansion_hunter/{sample}_str_motifs.tsv 09b STRANGER_RFC1 nextflow tsv stable - - +file {sample}/expansion_hunter/{sample}_rfc1_motif.tsv 09b STRANGER_RFC1 both tsv stable - - +file {sample}/expansion_hunter/{sample}_str_motifs.tsv 09b STRANGER_RFC1 both tsv stable - - file {sample}/fastq_trimmed/{sample}_R1.fastq.gz 01b FASTP nextflow bin internal - - file {sample}/fastq_trimmed/{sample}_R2.fastq.gz 01b FASTP nextflow bin internal - - -file {sample}/fastq_trimmed/{sample}_fastp.html 01b FASTP nextflow html stable - - -file {sample}/fastq_trimmed/{sample}_fastp.json 01b FASTP nextflow json stable - - +file {sample}/fastq_trimmed/{sample}_fastp.html 01b FASTP both html stable - - +file {sample}/fastq_trimmed/{sample}_fastp.json 01b FASTP both json stable - - file {sample}/hla/{sample}_hla_genotype.tsv 08 HLA_TYPING nextflow tsv stable - - -file {sample}/hla_optitype/{sample}_hla_concordance.tsv 08 HLA_QC nextflow tsv experimental - - -file {sample}/hla_optitype/{sample}_optitype_result.tsv 08 HLA_QC nextflow tsv experimental - - +file {sample}/hla_optitype/{sample}_hla_concordance.tsv 08 HLA_QC both tsv experimental - - +file {sample}/hla_optitype/{sample}_optitype_result.tsv 08 HLA_QC both tsv experimental - - file {sample}/hla_t1k/{sample}_hla_genotype.tsv 08 - bash tsv stable bash - -file {sample}/indexcov 16 INDEXCOV nextflow dir stable - - +file {sample}/indexcov 16 INDEXCOV both dir stable - - file {sample}/indexcov/indexcov-indexcov.ped 16 - bash tsv stable bash - file {sample}/kir/database_release.txt 08 KIR_TYPING nextflow text experimental - - -file {sample}/kir/{sample}_kir_genotype.tsv 08 KIR_TYPING nextflow tsv experimental - - +file {sample}/kir/{sample}_kir_genotype.tsv 08 KIR_TYPING both tsv experimental - - file {sample}/logs/run_status.tsv - - nextflow tsv stable - - -file {sample}/lpa/{sample}_lpa_kiv2.tsv 46 LPA nextflow tsv experimental - - -file {sample}/lpa/{sample}_lpa_kiv2_depth.tsv 46 LPA_DEPTH nextflow tsv experimental - - -file {sample}/lpa/{sample}_lpa_kiv2_depth_q1.tsv 46 LPA_DEPTH nextflow tsv experimental - - -file {sample}/manta/results/variants/candidateSV.vcf.gz 04 MANTA nextflow vcf stable - - -file {sample}/manta/results/variants/diploidSV.raw.vcf.gz 04 MANTA nextflow vcf stable - - -file {sample}/manta/results/variants/diploidSV.raw.vcf.gz.tbi 04 MANTA nextflow index stable - - -file {sample}/manta/results/variants/diploidSV.vcf.gz 04 MANTA nextflow vcf stable - - -file {sample}/manta/results/variants/diploidSV.vcf.gz.tbi 04 MANTA nextflow index stable - - -file {sample}/mei/{sample}_megane.log 50 MEI nextflow text experimental - - -file {sample}/mei/{sample}_megane.vcf 50 MEI nextflow vcf experimental - - -file {sample}/mei/{sample}_mei.tsv 50 MEI_REPORT nextflow tsv experimental - - -file {sample}/mei/{sample}_mei_annotsv.tsv 50 MEI_REPORT nextflow tsv experimental - - -file {sample}/mei/{sample}_mei_depth.tsv 50 MEI_DEPTH nextflow tsv experimental - - +file {sample}/lpa/{sample}_lpa_kiv2.tsv 46 LPA both tsv experimental - - +file {sample}/lpa/{sample}_lpa_kiv2_depth.tsv 46 LPA_DEPTH both tsv experimental - - +file {sample}/lpa/{sample}_lpa_kiv2_depth_q1.tsv 46 LPA_DEPTH both tsv experimental - - +file {sample}/manta/results/variants/candidateSV.vcf.gz 04 MANTA both vcf stable - - +file {sample}/manta/results/variants/diploidSV.raw.vcf.gz 04 MANTA both vcf stable - - +file {sample}/manta/results/variants/diploidSV.raw.vcf.gz.tbi 04 MANTA both index stable - - +file {sample}/manta/results/variants/diploidSV.vcf.gz 04 MANTA both vcf stable - - +file {sample}/manta/results/variants/diploidSV.vcf.gz.tbi 04 MANTA both index stable - - +file {sample}/mei/{sample}_megane.log 50 MEI both text experimental - - +file {sample}/mei/{sample}_megane.vcf 50 MEI both vcf experimental - - +file {sample}/mei/{sample}_mei.tsv 50 MEI_REPORT both tsv experimental - - +file {sample}/mei/{sample}_mei_annotsv.tsv 50 MEI_REPORT both tsv experimental - - +file {sample}/mei/{sample}_mei_depth.tsv 50 MEI_DEPTH both tsv experimental - - file {sample}/mei/{sample}_mei_mainchr.txt 50 MEI_DEPTH nextflow text internal - - -file {sample}/mei/{sample}_mei_sites.tsv 50 MEI_REPORT nextflow tsv experimental - - -file {sample}/mito/{sample}_chrM.fasta 12 MITO_EXTRACT_CHRM nextflow fasta stable - - +file {sample}/mei/{sample}_mei_sites.tsv 50 MEI_REPORT both tsv experimental - - +file {sample}/mito/{sample}_chrM.fasta 12 MITO_EXTRACT_CHRM both fasta stable - - file {sample}/mito/{sample}_chrM.vcf.gz 12 MITO_EXTRACT_CHRM nextflow vcf internal - - file {sample}/mito/{sample}_chrM.vcf.gz.tbi 12 MITO_EXTRACT_CHRM nextflow index internal - - -file {sample}/mito/{sample}_chrM_annotated.tsv 20 MITO_VARIANTS_ANNOTATE nextflow tsv stable - - -file {sample}/mito/{sample}_chrM_deletions.tsv 20 MITO_DELETIONS nextflow tsv stable - - +file {sample}/mito/{sample}_chrM_annotated.tsv 20 MITO_VARIANTS_ANNOTATE both tsv stable - - +file {sample}/mito/{sample}_chrM_deletions.tsv 20 MITO_DELETIONS both tsv stable - - file {sample}/mito/{sample}_chrM_filtered.vcf.gz 20 MITO_VARIANTS both vcf stable bash - file {sample}/mito/{sample}_chrM_filtered.vcf.gz.tbi 20 MITO_VARIANTS nextflow index stable - - file {sample}/mito/{sample}_chrM_mutect2.vcf.gz.stats 20 MITO_VARIANTS nextflow bin stable - - -file {sample}/mito/{sample}_haplocheck.txt 12 HAPLOCHECK nextflow text stable - - +file {sample}/mito/{sample}_haplocheck.txt 12 HAPLOCHECK both text stable - - file {sample}/mito/{sample}_haplogroup.txt 12 MITO_HAPLOGROUP both text stable bash - -file {sample}/mito/{sample}_mtdna_copies.tsv 20 MITO_DELETIONS nextflow tsv stable - - -file {sample}/mosaic/{sample}_mocha.log 51 MOCHA_CALL nextflow text experimental - - -file {sample}/mosaic/{sample}_mocha.tsv 51 MOCHA_REPORT nextflow tsv experimental - - -file {sample}/mosaic/{sample}_mocha_calls_raw.tsv 51 MOCHA_CALL nextflow tsv experimental - - -file {sample}/mosaic/{sample}_mocha_sites.tsv 51 MOCHA_CALL nextflow tsv experimental - - -file {sample}/mosaic/{sample}_mocha_stats.tsv 51 MOCHA_CALL nextflow tsv experimental - - -file {sample}/mosaic/{sample}_mocha_summary.tsv 51 MOCHA_REPORT nextflow tsv experimental - - +file {sample}/mito/{sample}_mtdna_copies.tsv 20 MITO_DELETIONS both tsv stable - - +file {sample}/mosaic/{sample}_mocha.log 51 MOCHA_CALL both text experimental - - +file {sample}/mosaic/{sample}_mocha.tsv 51 MOCHA_REPORT both tsv experimental - - +file {sample}/mosaic/{sample}_mocha_calls_raw.tsv 51 MOCHA_CALL both tsv experimental - - +file {sample}/mosaic/{sample}_mocha_sites.tsv 51 MOCHA_CALL both tsv experimental - - +file {sample}/mosaic/{sample}_mocha_stats.tsv 51 MOCHA_CALL both tsv experimental - - +file {sample}/mosaic/{sample}_mocha_summary.tsv 51 MOCHA_REPORT both tsv experimental - - file {sample}/locityper/{sample}_locityper.log 53 LOCITYPER both text experimental - - file {sample}/locityper/{sample}_locityper.tsv 53 LOCITYPER both tsv experimental - - file {sample}/locityper/work 53 LOCITYPER both dir experimental - - file {sample}/mosdepth/{sample}.mosdepth.summary.txt 16b - bash tsv stable bash chrom,length,bases,mean,min,max -file {sample}/paralogs/{sample}_*_copy_number.tsv 35 PARASCOPY nextflow tsv experimental - - -file {sample}/paralogs/{sample}_cn_dosage.tsv 35 PARASCOPY nextflow tsv experimental - - -file {sample}/paralogs/{sample}_cn_intervals.tsv 35 PARASCOPY nextflow tsv experimental - - +file {sample}/paralogs/{sample}_*_copy_number.tsv 35 PARASCOPY both tsv experimental - - +file {sample}/paralogs/{sample}_cn_dosage.tsv 35 PARASCOPY both tsv experimental - - +file {sample}/paralogs/{sample}_cn_intervals.tsv 35 PARASCOPY both tsv experimental - - file {sample}/paralogs/{sample}_cyp21a2_psv.tsv 35 PARASCOPY both tsv experimental - - -file {sample}/paralogs/{sample}_hba_copy_number.tsv 35 PARASCOPY nextflow tsv experimental - - -file {sample}/paralogs/{sample}_paraphase 35b PARAPHASE nextflow dir experimental - - -file {sample}/paralogs/{sample}_paraphase_copy_number.tsv 35b PARAPHASE_REPORT nextflow tsv experimental - - -file {sample}/paralogs/{sample}_paraphase_variants.tsv 35b PARAPHASE_REPORT nextflow tsv experimental - - -file {sample}/paralogs/{sample}_parascopy 35 PARASCOPY nextflow dir experimental - - -file {sample}/paralogs/{sample}_smn_copy_number.tsv 35 PARASCOPY nextflow tsv experimental - - -file {sample}/pgx_consensus/{sample}_outside_calls.tsv 36 PGX_CONSENSUS nextflow tsv stable - - -file {sample}/pgx_consensus/{sample}_pgx_consensus.tsv 36 PGX_CONSENSUS nextflow tsv stable - - +file {sample}/paralogs/{sample}_hba_copy_number.tsv 35 PARASCOPY both tsv experimental - - +file {sample}/paralogs/{sample}_paraphase 35b PARAPHASE both dir experimental - - +file {sample}/paralogs/{sample}_paraphase_copy_number.tsv 35b PARAPHASE_REPORT both tsv experimental - - +file {sample}/paralogs/{sample}_paraphase_variants.tsv 35b PARAPHASE_REPORT both tsv experimental - - +file {sample}/paralogs/{sample}_parascopy 35 PARASCOPY both dir experimental - - +file {sample}/paralogs/{sample}_smn_copy_number.tsv 35 PARASCOPY both tsv experimental - - +file {sample}/pgx_consensus/{sample}_outside_calls.tsv 36 PGX_CONSENSUS both tsv stable - - +file {sample}/pgx_consensus/{sample}_pgx_consensus.tsv 36 PGX_CONSENSUS both tsv stable - - file {sample}/pharmcat/{sample}.match.json 07 PHARMCAT nextflow json stable - - file {sample}/pharmcat/{sample}.phenotype.json 07 PHARMCAT nextflow json stable - - file {sample}/pharmcat/{sample}.report.html 07 PHARMCAT nextflow html stable - - file {sample}/pharmcat/{sample}.report.json 07 PHARMCAT nextflow json stable nextflow - -file {sample}/prs/pgsc_calc 25 PRS nextflow dir stable - - +file {sample}/prs/pgsc_calc 25 PRS both dir stable - - file {sample}/prs/{sample}_prs_summary.*.tsv 25 PRS_SUMMARY nextflow tsv stable - - -file {sample}/prs/{sample}_prs_summary.tsv 25 PRS_SUMMARY nextflow tsv stable - - -file {sample}/pypgx/{sample}_cyp2d6_depth_check.tsv 32 PYPGX nextflow tsv stable - - -file {sample}/pypgx/{sample}_pharmcat_comparison.tsv 27 CPIC_LOOKUP nextflow tsv stable - - +file {sample}/prs/{sample}_prs_summary.tsv 25 PRS_SUMMARY both tsv stable - - +file {sample}/pypgx/{sample}_cyp2d6_depth_check.tsv 32 PYPGX both tsv stable - - +file {sample}/pypgx/{sample}_pharmcat_comparison.tsv 27 CPIC_LOOKUP both tsv stable - - file {sample}/pypgx/{sample}_pypgx_results 32 PYPGX nextflow dir stable - - file {sample}/pypgx/{sample}_pypgx_summary.tsv 32 PYPGX both tsv stable bash - file {sample}/qc/somalier/{sample}.somalier 33 SOMALIER nextflow bin experimental - - file {sample}/qc/somalier/{sample}.somalier_id 33 SOMALIER nextflow bin internal - - file {sample}/qc/verifybamid2/{sample}.Ancestry 33 VERIFYBAMID2 nextflow bin experimental - - -file {sample}/qc/verifybamid2/{sample}.log 33 VERIFYBAMID2 nextflow text experimental - - +file {sample}/qc/verifybamid2/{sample}.log 33 VERIFYBAMID2 both text experimental - - file {sample}/qc/verifybamid2/{sample}.marker_check 33 VERIFYBAMID2 nextflow bin internal - - -file {sample}/qc/verifybamid2/{sample}.selfSM 33 VERIFYBAMID2 nextflow bin experimental - - -file {sample}/qc/{sample}_sample_qc.tsv 33 SAMPLE_QC nextflow tsv experimental - - +file {sample}/qc/verifybamid2/{sample}.selfSM 33 VERIFYBAMID2 both bin experimental - - +file {sample}/qc/{sample}_read_count.tsv 02 ALIGN_MINIMAP2 both tsv stable - - +file {sample}/qc/{sample}_sample_qc.tsv 33 SAMPLE_QC both tsv experimental - - file {sample}/roh/{sample}_roh.txt 11 ROH nextflow text stable nextflow - file {sample}/roh/{sample}_roh_summary.txt 11 ROH nextflow tsv stable nextflow chrom,start,end,length_bp,length_mb file {sample}/run_manifest.tsv - - nextflow tsv stable - - file {sample}/slivar/{sample}.ped 31 SLIVAR_PRIORITIZE nextflow tsv internal - - -file {sample}/slivar/{sample}_compound_het_pairs.tsv 31 COMPHET_PHASE nextflow tsv stable - - +file {sample}/slivar/{sample}_compound_het_pairs.tsv 31 COMPHET_PHASE both tsv stable - - file {sample}/slivar/{sample}_compound_hets.vcf.gz 31 SLIVAR both vcf stable bash - file {sample}/slivar/{sample}_prioritized.vcf.gz 31 SLIVAR_PRIORITIZE both vcf stable bash - file {sample}/slivar/{sample}_prioritized.vcf.gz.tbi 31 SLIVAR_PRIORITIZE nextflow index stable - - -file {sample}/slivar/{sample}_slivar_summary.tsv 31 SLIVAR_PRIORITIZE nextflow tsv stable - - +file {sample}/slivar/{sample}_slivar_summary.tsv 31 SLIVAR_PRIORITIZE both tsv stable - - file {sample}/summary.json 24 HTML_REPORT both json stable bash schema_version,sample,generated_utc,run,databases,manifest,sections,cyp2d6,not_run,not_assessed file {sample}/sv_duphold/{sample}_sv_duphold.vcf 15 DUPHOLD nextflow vcf stable - - file {sample}/sv_filtered/{sample}_sv_filtered.log 15 DUPHOLD_FILTER nextflow text stable - - file {sample}/sv_filtered/{sample}_sv_filtered.vcf.gz 15 DUPHOLD_FILTER nextflow vcf stable - - file {sample}/sv_filtered/{sample}_sv_filtered.vcf.gz.tbi 15 DUPHOLD_FILTER nextflow index stable - - -file {sample}/sv_merged/{sample}_sv_consensus.vcf.gz 22 SURVIVOR_SORT nextflow vcf stable - - -file {sample}/sv_merged/{sample}_sv_consensus.vcf.gz.tbi 22 SURVIVOR_SORT nextflow index stable - - -file {sample}/telomere/{sample} 10 TELOMERE_HUNTER nextflow dir stable - - -file {sample}/traits/{sample}_genotype_rules.tsv 42 TRAITS nextflow tsv stable - - -file {sample}/traits/{sample}_traits.tsv 42 TRAITS nextflow tsv stable - - -file {sample}/traits/{sample}_traits_genotypes.tsv 42 TRAITS_SITES nextflow tsv stable - - +file {sample}/sv_merged/{sample}_sv_consensus.vcf.gz 22 SURVIVOR_SORT both vcf stable - - +file {sample}/sv_merged/{sample}_sv_consensus.vcf.gz.tbi 22 SURVIVOR_SORT both index stable - - +file {sample}/telomere/{sample} 10 TELOMERE_HUNTER both dir stable - - +file {sample}/traits/{sample}_genotype_rules.tsv 42 TRAITS both tsv stable - - +file {sample}/traits/{sample}_traits.tsv 42 TRAITS both tsv stable - - +file {sample}/traits/{sample}_traits_genotypes.tsv 42 TRAITS_SITES both tsv stable - - file {sample}/vcf/{sample}.g.vcf.gz 03 DEEPVARIANT both vcf stable - - file {sample}/vcf/{sample}.g.vcf.gz.tbi 03 DEEPVARIANT both index stable - - file {sample}/vcf/{sample}.report.html 07 - bash html stable bash - @@ -231,36 +231,36 @@ file {sample}/vcf/{sample}_roh.txt 11 - bash text stable bash - file {sample}/vcf/{sample}_roh_summary.txt 11 - bash tsv stable bash chrom,start,end,length_bp,length_mb file {sample}/vcf_gatk/{sample}.vcf.gz 03a - bash vcf stable bash FORMAT/GT file {sample}/vep/{sample}_annotated.vcf.gz 30 VCFANNO_INDEX both vcf stable both INFO/REVEL -file {sample}/vep/{sample}_annotated.vcf.gz.tbi 30 VCFANNO_INDEX nextflow index stable nextflow - +file {sample}/vep/{sample}_annotated.vcf.gz.tbi 30 VCFANNO_INDEX both index stable nextflow - file {sample}/vep/{sample}_vep.vcf.gz 13 VEP both vcf stable - - -file {sample}/vep/{sample}_vep.vcf.gz.tbi 13 VEP nextflow index stable - - +file {sample}/vep/{sample}_vep.vcf.gz.tbi 13 VEP both index stable - - file {sample}/vep/{sample}_vep.vcf_summary.html 13 VEP nextflow html stable - - -file {sample}/viral/{sample}_human_coverage.tsv 47 VIRAL_READS nextflow tsv experimental - - -file {sample}/viral/{sample}_viral.tsv 47 VIRAL_REPORT nextflow tsv experimental - - -file {sample}/viral/{sample}_viral_coverage.tsv 47 VIRAL_READS nextflow tsv experimental - - -file {sample}/viral/{sample}_viral_depth.tsv 47 VIRAL_READS nextflow tsv experimental - - -file {sample}/viral/{sample}_viral_reads.bam 47 VIRAL_READS nextflow bam experimental - - -file {sample}/viral/{sample}_viral_reads.bam.bai 47 VIRAL_READS nextflow index experimental - - -file {sample}/vntyper/run 52 VNTYPER nextflow dir experimental - - -file {sample}/vntyper/{sample}_vntyper.log 52 VNTYPER nextflow text experimental - - -file {sample}/vntyper/{sample}_vntyper.tsv 52 VNTYPER nextflow tsv experimental - - -file {sample}/y_haplogroup/{sample}_chrY.bam 37 Y_PILEUP nextflow bam experimental - - -file {sample}/y_haplogroup/{sample}_chrY.bam.bai 37 Y_PILEUP nextflow index experimental - - +file {sample}/viral/{sample}_human_coverage.tsv 47 VIRAL_READS both tsv experimental - - +file {sample}/viral/{sample}_viral.tsv 47 VIRAL_REPORT both tsv experimental - - +file {sample}/viral/{sample}_viral_coverage.tsv 47 VIRAL_READS both tsv experimental - - +file {sample}/viral/{sample}_viral_depth.tsv 47 VIRAL_READS nextflow tsv internal - - +file {sample}/viral/{sample}_viral_reads.bam 47 VIRAL_READS both bam experimental - - +file {sample}/viral/{sample}_viral_reads.bam.bai 47 VIRAL_READS both index experimental - - +file {sample}/vntyper/run 52 VNTYPER both dir experimental - - +file {sample}/vntyper/{sample}_vntyper.log 52 VNTYPER both text experimental - - +file {sample}/vntyper/{sample}_vntyper.tsv 52 VNTYPER both tsv experimental - - +file {sample}/y_haplogroup/{sample}_chrY.bam 37 Y_PILEUP both bam experimental - - +file {sample}/y_haplogroup/{sample}_chrY.bam.bai 37 Y_PILEUP both index experimental - - file {sample}/y_haplogroup/{sample}_idxstats.txt 37 Y_PILEUP nextflow text internal - - -file {sample}/y_haplogroup/{sample}_y_haplogroup.txt 37 Y_HAPLOGROUP nextflow text experimental - - +file {sample}/y_haplogroup/{sample}_y_haplogroup.txt 37 Y_HAPLOGROUP both text experimental - - file {sample}/y_haplogroup/{sample}_y_pileup.txt 37 Y_PILEUP nextflow text internal - - -file {sample}/y_haplogroup/{sample}_y_private_snps.tsv 37 Y_PRIVATE nextflow tsv experimental - - -file {sample}/y_haplogroup/{sample}_y_tree.txt 37 Y_HAPLOGROUP nextflow text experimental - - -file {sample}/y_haplogroup/{sample}_y_tree_path.tsv 37 Y_HAPLOGROUP nextflow tsv experimental - - +file {sample}/y_haplogroup/{sample}_y_private_snps.tsv 37 Y_PRIVATE both tsv experimental - - +file {sample}/y_haplogroup/{sample}_y_tree.txt 37 Y_HAPLOGROUP both text experimental - - +file {sample}/y_haplogroup/{sample}_y_tree_path.tsv 37 Y_HAPLOGROUP both tsv experimental - - file {sample}/y_haplogroup/{sample}_y_tree_pileup.txt 37 Y_PILEUP nextflow text internal - - -file {sample}/y_str/{sample}_y_str.csv 45 Y_STR nextflow csv experimental - - -file {sample}/y_str/{sample}_y_str.log 45 Y_STR nextflow text experimental - - -file {sample}/y_str/{sample}_y_str.tsv 45 Y_STR nextflow tsv experimental - - -file {sample}/y_str/{sample}_y_str.vcf.gz 45 Y_STR nextflow vcf experimental - - -file {sample}/{sample}_charts.json 24 HTML_REPORT nextflow json stable - - +file {sample}/y_str/{sample}_y_str.csv 45 Y_STR both csv experimental - - +file {sample}/y_str/{sample}_y_str.log 45 Y_STR both text experimental - - +file {sample}/y_str/{sample}_y_str.tsv 45 Y_STR both tsv experimental - - +file {sample}/y_str/{sample}_y_str.vcf.gz 45 Y_STR both vcf experimental - - +file {sample}/{sample}_charts.json 24 HTML_REPORT both json stable - - file {sample}/{sample}_report.html 24 HTML_REPORT both html stable bash - file {sample}/{sample}_report.txt 24 - both text stable bash - -file {sample}/{sample}_search_index.json 24 HTML_REPORT nextflow json stable - - +file {sample}/{sample}_search_index.json 24 HTML_REPORT both json stable - - file {sample}/{sample}_sex_check.tsv 16 INDEXCOV nextflow tsv stable - - param input - - - - stable - - param outdir - - - - stable - - @@ -311,6 +311,7 @@ param parascopy_data - - - - experimental - - param yleaf_data - - - - experimental - - param hpo - - - - experimental - - param exomiser_data - - - - experimental - - +param pedigree - - - - experimental - - param family_name - - - - experimental - - param exomiser_proband - - - - experimental - - param exomiser_assembly - - - - experimental - - diff --git a/docs/output-contract.md b/docs/output-contract.md index 0d49b17c..bfab4e87 100644 --- a/docs/output-contract.md +++ b/docs/output-contract.md @@ -10,7 +10,7 @@ The full list is one file, [`assets/output_contract.tsv`](https://github.com/Gei - **`--tools` names:** every name `main.nf` accepts (`pharmcat`, `clinvar`, `vcfanno`, ...). - **`run-all.sh`:** its two arguments (` `) and every switch in its `SWITCHES` list (`TOOLS`, `THREADS`, `SKIP_TRIM`, `INTERVALS`, `REF_FASTA`, ...). - **Result files:** every file the Nextflow pipeline publishes, which is also what `run-all.sh` writes, at its path under `GENOME_DIR//` (or `--outdir`). For each one the contract gives its step, its format and the fields a reader can count on: the columns of a table, the INFO and FORMAT fields of a VCF header, the top-level keys of a JSON file (`summary.json`'s keys are the ones its schema, `tests/schema/summary.schema.json`, requires). -- **Single-step scripts:** the files listed with leg `bash` or `both`. Some single steps write to other folders than the pipeline does (step 7 writes PharmCAT's reports to `vcf/`, the pipeline to `pharmcat/`; step 16b writes to `mosdepth/`, the pipeline to `coverage/`). Both names are kept. A single-step output that is not in the file is not part of the promise yet. +- **Single-step scripts:** the files listed with leg `bash` or `both`. A row is `both` when the step's script (`scripts/-*.sh`) writes the same path under `GENOME_DIR`. Some single steps write to other folders than the pipeline does (step 7 writes PharmCAT's reports to `vcf/`, the pipeline to `pharmcat/`; step 16b writes to `mosdepth/`, the pipeline to `coverage/`). Both names are kept. A single-step output that is not in the file is not part of the promise yet. ## What counts as a break @@ -40,7 +40,7 @@ The engines that run by default, and every step that existed before 1.0, are `st Two checks read the contract, both with [`scripts/ci/check-output-contract.py`](https://github.com/GeiserX/OwnGenome/blob/main/scripts/ci/check-output-contract.py): -- **On every pull request, with no run** (`tests/test_output_contract.py`, in the `unit-tests` job of the Guard workflow): every param of `nextflow_schema.json`, every `--tools` name of `main.nf` and every switch of `run-all.sh` has a row, and every row still names one. Every file row of a Nextflow process still matches a file that process writes and publishes, and every file a module publishes has a row. A removed or renamed stable item fails as `BREAKING`. A new one with no row fails too, with the message to add it, so the contract is always the full list. The test also plants each kind of break on a copy of the code (a renamed output, a removed param, a removed switch, a new tool and a new output with no row) and requires each to fail. +- **On every pull request, with no run** (`tests/test_output_contract.py`, in the `unit-tests` job of the Guard workflow): every param of `nextflow_schema.json`, every `--tools` name of `main.nf` and every switch of `run-all.sh` has a row, and every row still names one. Every file row of a Nextflow process still matches a file that process writes and publishes, in the folder of a `publishDir` that keeps it (a `saveAs` that returns `null` for the file does not count), and every file a module publishes has a row. A published row with leg `nextflow` fails when its step's script writes the same path too, since its leg is then `both`; the check reads the script with its shell variables filled in, so a path it cannot follow is not flagged, and a `both` row it does not find is not refused. Every default in `nextflow_schema.json` must equal the value in `nextflow.config`. A removed or renamed stable item fails as `BREAKING`. A new one with no row fails too, with the message to add it, so the contract is always the full list. The test also plants each kind of break on a copy of the code (a renamed output, a removed param, a removed switch, a new tool and a new output with no row, a schema default that differs from `nextflow.config`, a row in the folder a `saveAs` drops the file from, and a `nextflow` row a step script also writes) and requires each to fail. - **On the HG002 end-to-end run** (`tests/e2e/output-contract-1-hg002.sh`): every stable file whose `e2e` column names a leg must be in the results that leg wrote, under its contracted name, with its required fields. The bash leg is the single-step scripts the base cases run; the Nextflow leg is case `60-nextflow`. Rows whose step the fixture run does not reach have `e2e` set to `-` and are held by the static check only. The two files `run-all.sh` writes itself, `run_manifest.tsv` and `logs/run_status.tsv`, are listed but no check holds them yet: neither the fixture run nor a module writes them. The same case copies the Nextflow results, drops the `clnsig` column of the ClinVar hits table and renames the ROH summary, and requires the check to fail on both. It also prints every file each leg wrote with its fields, which is what to read when the contract is written again. ## Changing the contract diff --git a/main.nf b/main.nf index 7160fb9d..fb57d54e 100644 --- a/main.nf +++ b/main.nf @@ -61,13 +61,12 @@ workflow { // ─── Validate inputs ──────────────────────────────────────────────── // Every parameter must be one nextflow_schema.json declares: Nextflow // takes any --name as a new parameter, so a misspelt option (--outdirr, - // --max-cpus) would be ignored with no word. --pedigree is read with - // containsKey and has no default; config_profile_* are the test - // profiles' labels. + // --max-cpus) would be ignored with no word. config_profile_* are the + // test profiles' labels. def schema_params = new groovy.json.JsonSlurper() .parseText(file("${projectDir}/nextflow_schema.json").text)['definitions'] .values().collectMany { group -> (group['properties'] ?: [:]).keySet() as List } - def known_params = schema_params + ['pedigree', 'config_profile_name', 'config_profile_description'] + def known_params = schema_params + ['config_profile_name', 'config_profile_description'] def unknown_params = params.keySet().findAll { k -> !known_params.contains(k) }.sort() if (unknown_params) { error "unknown parameter${unknown_params.size() > 1 ? 's' : ''} " + @@ -332,9 +331,8 @@ workflow { // leading family column: one family runs, the one --family_name names // (needed when the file holds more than one). Here only the family and // sample columns are read; FAMILY_PREPARE (bin/family_report.py prepare) - // checks the rest before the joint call starts. Read with containsKey, - // so a run without the option is not told about an undefined parameter. - def pedigree = params.containsKey('pedigree') ? params.pedigree : null + // checks the rest before the joint call starts. + def pedigree = params.pedigree // The family's folder under --outdir: --family_name, else 'family' def family_dir = params.family_name ?: 'family' def family_ids = [] diff --git a/nextflow.config b/nextflow.config index 176b8db8..5b95c3bf 100644 --- a/nextflow.config +++ b/nextflow.config @@ -121,6 +121,7 @@ params { viral_ref = null // viral: the viral genomes of step 47 (setup.sh --viral-data: reference/viral-/viral_genomes.fa) hpo = null // exomiser: the symptoms as HPO ids, comma separated (HP:0001250,HP:0001263); setting it runs exomiser (step 48) exomiser_data = null // exomiser: Exomiser's data folder (setup.sh --exomiser-data: reference/exomiser-) + pedigree = null // opt-in: a pedigree or families file (assets/pedigree.example.tsv, assets/families.example.tsv); runs the family workflow (steps 38, 39) family_name = null // --pedigree: the family to run of a families file (assets/families.example.tsv) that holds more than one; also its folder under --outdir (default family) exomiser_proband = null // exomiser: the sample the --hpo terms describe; needed when the samplesheet has more than one exomiser_assembly = 'hg38' // exomiser: the assembly of its data; hg19 only for Exomiser's cut-down test release (CI) diff --git a/nextflow_schema.json b/nextflow_schema.json index 6795fd58..91e62b6f 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -105,7 +105,7 @@ "tools": { "type": "string", "description": "Comma-separated list of tools to run; an unknown name stops the run. DB-heavy tools (vep, slivar, clinical_filter, cpsr, clinvar, expansion_hunter, sample_qc) and extras (stranger, ancestry (needs --ancestry_ref and prs), ancestry_fine (needs --ancestry_fine_panel), ancient (needs --ancient_panel; samples with a gVCF), pypgx, SV callers, cram_archive, cyrius, parascopy, paraphase (PacBio HiFi BAMs only), y_haplogroup (male samples), y_str (Y-STR repeat counts, male samples), consumer_export (samples with a gVCF), bloodtype (blood group antigens; RhD needs the BAM), lpa (the LPA KIV-2 repeat count; needs the BAM), vntyper (MUC1 VNTR frameshifts behind ADTKD-MUC1; needs the BAM), viral (viral DNA in the unmapped reads; needs --viral_ref and a BAM), exomiser (runs when --hpo is set), chip (clonal haematopoiesis from the BAM; needs --chip_gff), archaic (needs --archaic_data), mei (Alu, L1 and SVA insertions with MEGAnE; needs --megane_kmers and a BAM), mocha (mosaic chromosomal alterations; needs --mocha_data), locityper (a second opinion at HLA class II, KIR, CYP2D6 and FCGR; samples with a BAM; needs --locityper_data)) are opt-in. prs and vcfanno are skipped with a warning until --pgs_scoring or a score file is set. With pharmcat, PharmCAT also gets the HLA types of hla_typing and a CYP2D6 call pypgx and cyrius agree on (PGX_CONSENSUS).", - "default": "pharmcat,cpic,vcfanno,roh,prs,mito_haplogroup,telomere_hunter,mosdepth,mito_variants,html_report,multiqc", + "default": "pharmcat,cpic,vcfanno,roh,prs,mito_haplogroup,telomere_hunter,mosdepth,mito_variants,traits,html_report,multiqc", "fa_icon": "fas fa-toolbox" } } @@ -344,6 +344,12 @@ "description": "Exomiser's data folder, which `scripts/setup.sh --exomiser-data` installs as reference/exomiser- (its _hg38 and _phenotype folders). Required with --hpo.", "fa_icon": "fas fa-folder" }, + "pedigree": { + "type": "string", + "format": "file-path", + "description": "Opt-in: run the family workflow (steps 38 and 39) on the samples of the samplesheet this file names: a pedigree (assets/pedigree.example.tsv), or a families file with a leading family column (assets/families.example.tsv). A families file that holds more than one family needs --family_name, and only that family runs. The family that runs needs two or more of those samples, each with a gVCF, and --somalier_sites.", + "fa_icon": "fas fa-sitemap" + }, "family_name": { "type": "string", "pattern": "^[a-zA-Z0-9._][a-zA-Z0-9._-]*$", diff --git a/scripts/ci/check-output-contract.py b/scripts/ci/check-output-contract.py index e397cad5..d0dd192e 100755 --- a/scripts/ci/check-output-contract.py +++ b/scripts/ci/check-output-contract.py @@ -8,10 +8,14 @@ run: every param of nextflow_schema.json, every --tools name of main.nf (known_tools) and every run-all.sh switch (its SWITCHES list) has a row, and every row still names one; every file row of a process names a file - that process still writes and publishes; and every file a module - publishes has a row. A stable row that no longer matches is a break. A - new param, tool, switch or published file with no row fails too, so the - contract stays the full list. + that process still writes and publishes, in the folder of a publishDir + that keeps it; every file a module publishes has a row; a nextflow row + (not internal) whose step script writes the same path is refused (its + legs are both); + and every default of nextflow_schema.json equals nextflow.config's. A + stable row that no longer matches is a break. A new param, tool, switch + or published file with no row fails too, so the contract stays the full + list. --outputs DIR --leg bash|nextflow --sample S [--observe] The files a run wrote. DIR is the folder that holds / (GENOME_DIR @@ -109,8 +113,49 @@ def concrete(pattern, sample="SAMPLE"): return p.strip("/") +SAVE_AS = re.compile(r"^\s*(\w+)\s*->\s*\1\s*(?:\.(endsWith|startsWith)\(\s*'([^']*)'\s*\)|==\s*'([^']*)')" + r"\s*\?\s*([^?:]+?)\s*:\s*([^?:]+?)\s*$", re.S) + + +def closure_body(stmt): + """The body of a publishDir's saveAs closure, up to its own closing brace (an option + after it is not part of it), or None when the statement has no saveAs.""" + m = re.search(r"saveAs:\s*\{", stmt) + if not m: + return None + depth, i = 1, m.end() + while i < len(stmt): + depth += {"{": 1, "}": -1}.get(stmt[i], 0) + if depth == 0: + return stmt[m.end():i] + i += 1 + return stmt[m.end():] + + +def save_as(closure, name): + """What a publishDir's saveAs closure does to a task file name (an output glob): + 'keep' (published under its own name), 'drop' (returns null) or 'rename'. + A closure this reader does not follow counts as 'rename', which the static + check matches by basename only.""" + if closure is None: + return "keep" + m = SAVE_AS.match(closure) + if not m: + return "rename" + var, method, arg, eq, yes, no = m.groups() + if method == "endsWith": + cond = name.endswith(arg) + elif method == "startsWith": + cond = name.startswith(arg) + else: + cond = name == eq + expr = (yes if cond else no).strip() + return "drop" if expr == "null" else "keep" if expr == var else "rename" + + def module_processes(repo): - """{process: {'publish': [(dir, [patterns] or None, has_saveas)], 'outputs': [glob]}} of modules/local.""" + """{process: {'publish': [(dir, [patterns] or None, saveAs closure body or None)], 'outputs': [glob]}} + of modules/local.""" procs = {} for f in sorted(glob.glob(os.path.join(repo, "modules", "local", "*", "main.nf"))): src = open(f).read() @@ -134,7 +179,7 @@ def module_processes(repo): pat = re.search(r"pattern:\s*[\"']([^\"']+)[\"']", stmt) pubs.append((concrete(d.group(1)) if d else "", braces(pat.group(1)) if pat else None, - "saveAs" in stmt)) + closure_body(stmt))) out = re.search(r"^\s*output:\s*$(.*?)(?:^\s*(?:when|topic):|\Z)", head, re.M | re.S) outputs = [] if out: @@ -147,15 +192,19 @@ def module_processes(repo): def published(proc): - """(contracted-path glob, basename-only) pairs a process publishes; basename-only when a saveAs renames.""" + """(output glob, contracted-path glob, basename-only) of each file a process publishes. A publishDir + whose saveAs drops the file does not count; basename-only when its saveAs renames the file.""" items = [] - for d, pats, saveas in proc["publish"]: + for d, pats, closure in proc["publish"]: for g in proc["outputs"]: c = concrete(g) if pats is not None and not any(fnmatch.fnmatch(c.replace("*", "X"), p) or fnmatch.fnmatch(c, p) for p in pats): continue - items.append((g, (d + "/" + c) if d else c, saveas)) + action = save_as(closure, c) + if action == "drop": + continue + items.append((g, (d + "/" + c) if d else c, action == "rename")) return items @@ -165,6 +214,82 @@ def schema_params(repo): return {p for group in defs.values() for p in group.get("properties", {})} +def schema_defaults(repo): + s = json.load(open(os.path.join(repo, "nextflow_schema.json"))) + defs = s.get("definitions") or s.get("$defs") or {} + return {p: v["default"] for group in defs.values() for p, v in group.get("properties", {}).items() + if "default" in v} + + +def config_params(repo): + """{name: value} of nextflow.config's params block, for the plain literals (strings, numbers, + true, false, null); a value this reader cannot parse is left out.""" + src = open(os.path.join(repo, "nextflow.config")).read() + m = re.search(r"^params\s*\{(.*?)^\}", src, re.M | re.S) + out = {} + if not m: + return out + lit = {"true": True, "false": False, "null": None} + for name, val in re.findall(r"^\s*(\w+)\s*=\s*('[^']*'|\"[^\"]*\"|[-\w.]+)", m.group(1), re.M): + if val[0] in "'\"": + out[name] = val[1:-1] + elif val in lit: + out[name] = lit[val] + else: + try: + out[name] = int(val) + except ValueError: + try: + out[name] = float(val) + except ValueError: + pass + return out + + +VAR = re.compile(r"\$\{(\w+)\}|\$(\w+)") +ASSIGN = re.compile(r"^\s*(?:local\s+|export\s+|readonly\s+)?([A-Za-z_]\w*)=(\"[^\"\n]*\"|'[^'\n]*'|[^\s;#()]+)", re.M) + + +def bash_paths(src): + """The lines of a step script with its shell variables expanded: SAMPLE as {sample}, + GENOME_DIR as G, every other variable as each value it is assigned (a few at most).""" + assign = {} + for v, val in ASSIGN.findall(src): + assign.setdefault(v, []).append(val.strip("\"'")) + fixed = {"SAMPLE": "{sample}", "GENOME_DIR": "G", "1": "{sample}"} + + # depth counts every variable filled in on the line, not the nesting: a line with + # ${WORK}, ${P} and P's own chain of OUTDIR, S, GENOME_DIR and SAMPLE needs seven. + # The search stops at 64 expansions of a line, so a deep line stays cheap. + def expand(s, depth=0): + m = VAR.search(s) + if not m or depth > 24: + return [s] + name = m.group(1) or m.group(2) + vals = [fixed[name]] if name in fixed else assign.get(name) or ["\0"] + out = [] + for val in vals[:4]: + out.extend(expand(s[:m.start()] + val + s[m.end():], depth + 1)) + if len(out) >= 64: + break + return out[:64] + + return {e for line in src.split("\n") if not line.lstrip().startswith("#") for e in expand(line)} + + +def bash_writes(repo, step, name, cache): + """The step script that names this per-sample path (scripts/-*.sh), or None.""" + if not name.startswith("{sample}/") or "*" in name or step == "-": + return None + if step not in cache: + files = sorted(glob.glob(os.path.join(repo, "scripts", f"{step}-*.sh"))) + cache[step] = (files, "\n".join(bash_paths("".join(open(f).read() for f in files))) if files else "") + files, text = cache[step] + # the whole path: a longer name that starts with it (x.tsv.gz for x.tsv) is not it + hit = files and re.search(re.escape("G/" + name) + r"(?![\w.-])", text) + return os.path.relpath(files[0], repo) if hit else None + + def known_tools(repo): src = open(os.path.join(repo, "main.nf")).read() m = re.search(r"def known_tools = \[(.*?)\]", src, re.S) @@ -193,6 +318,14 @@ def check_static(repo, rows): for name in sorted(have[kind] - set(listed)): problems.append(f"{kind} {name} of {where[kind]} has no row in the contract: add one") + defaults, config = schema_defaults(repo), config_params(repo) + if not config: + problems.append("found no params in nextflow.config: the reader no longer matches the code") + for name, val in sorted(defaults.items()): + if name in config and config[name] != val: + problems.append(f"param {name}: nextflow_schema.json's default {val!r} is not nextflow.config's " + f"{config[name]!r}") + procs = module_processes(repo) if not procs: problems.append("found no processes under modules/local") @@ -214,6 +347,13 @@ def check_static(repo, rows): what = "BREAKING: the stable file" if r["stability"] == "stable" else "the file" problems.append(f"{what} {r['name']} (contract line {r['line']}) is not what {r['process']} " f"publishes ({p['file']}): renamed, moved or no longer written") + cache = {} + for r in rows: + if r["kind"] == "file" and r["legs"] == "nextflow" and r["stability"] != "internal": + script = bash_writes(repo, r["step"], r["name"], cache) + if script: + problems.append(f"{r['name']} (contract line {r['line']}) has legs nextflow, but {script} " + "writes it under the same name too: its legs are both") for name, p in sorted(procs.items()): rows_p = [concrete(r["name"]) for r in file_rows if r["process"] == name] if not p["publish"]: diff --git a/tests/test_output_contract.py b/tests/test_output_contract.py index 38c6de8a..8e896754 100644 --- a/tests/test_output_contract.py +++ b/tests/test_output_contract.py @@ -5,9 +5,14 @@ run-all.sh switch and published Nextflow file has its row, and every row still matches the code; 2. static controls, each on a copy of the code: a renamed published file, a - removed param, a removed switch, a new --tools name with no row and a new - published file with no row are each refused, and the stable ones are - named BREAKING; + removed param, a removed switch, a new --tools name with no row, a new + published file with no row and a schema default that is not + nextflow.config's are each refused, and the stable ones are named + BREAKING; so are, on a copy of the contract, a file row in the folder of + a publishDir whose saveAs drops it and a nextflow row whose step script + writes the same path (also on a line of seven shell variables); + --pedigree is a declared param; a saveAs closure is read up to its own + brace and a script path counts only as a whole path; 3. the run check on a planted results folder made from the contract's own e2e rows passes for both legs; then a dropped column, a dropped VCF field, a dropped JSON key and a renamed file are each refused, naming the @@ -63,11 +68,30 @@ def code_copy(tmp): dst = os.path.join(tmp, "repo") shutil.copytree(os.path.join(REPO, "modules"), os.path.join(dst, "modules")) os.makedirs(os.path.join(dst, "scripts")) - for f in ("main.nf", "nextflow_schema.json", os.path.join("scripts", "run-all.sh")): + for f in ("main.nf", "nextflow_schema.json", "nextflow.config"): shutil.copy(os.path.join(REPO, f), os.path.join(dst, f)) + for f in os.listdir(os.path.join(REPO, "scripts")): + if f.endswith(".sh"): + shutil.copy(os.path.join(REPO, "scripts", f), os.path.join(dst, "scripts", f)) return dst +def contract_copy(tmp, name, change): + """The contract with one row changed: change(cells) edits the cells of the file row called name.""" + path = os.path.join(tmp, "contract.tsv") + hit = False + with open(CONTRACT) as f, open(path, "w") as g: + for line in f: + c = line.rstrip("\n").split("\t") + if c[0] == "file" and c[1] == name: + change(c) + line = "\t".join(c) + "\n" + hit = True + g.write(line) + assert hit, f"no file row {name}" + return path + + def edit(path, old, new, count=1): src = open(path).read() assert old in src, f"{old!r} not in {path}" @@ -96,6 +120,9 @@ def static_controls(): lambda d: edit(os.path.join(d, "modules", "local", "roh", "main.nf"), 'path("*_roh.txt"),', 'path("*_roh.txt"),\n tuple val(meta), path("*_roh_extra.tsv"), emit: extra\n'), [r"ROH .* writes \*_roh_extra\.tsv and has no contract row"]), + ("a schema default that is not nextflow.config's (traits left out of --tools)", + lambda d: edit(os.path.join(d, "nextflow_schema.json"), "mito_variants,traits,html_report", "mito_variants,html_report"), + [r"param tools: nextflow_schema\.json's default .* is not nextflow\.config's"]), ] for desc, change, want in cases: tmp = tempfile.mkdtemp() @@ -109,6 +136,39 @@ def static_controls(): shutil.rmtree(tmp) + def to_aligned(c): + c[1] = "{sample}/aligned/{sample}_read_count.tsv" + + def to_nextflow(c): + c[4] = "nextflow" + + contract_cases = [ + ("the read count in aligned/, whose publishDir drops it (it is published to qc/)", + "{sample}/qc/{sample}_read_count.tsv", to_aligned, + [r"BREAKING: the stable file \{sample\}/aligned/\{sample\}_read_count\.tsv .* is not what ALIGN_MINIMAP2 publishes"]), + ("legs nextflow on a file step 49 writes under the same name", + "{sample}/chip/{sample}_chip.tsv", to_nextflow, + [r"\{sample\}/chip/\{sample\}_chip\.tsv .* has legs nextflow, but scripts/49-chip\.sh writes it"]), + ("legs nextflow on a file step 50 writes on a line of seven shell variables", + "{sample}/mei/{sample}_megane.log", to_nextflow, + [r"\{sample\}/mei/\{sample\}_megane\.log .* has legs nextflow, but scripts/50-mei\.sh writes it"]), + ] + for desc, name, change, want in contract_cases: + tmp = tempfile.mkdtemp() + try: + rc, out = run("--static", "--contract", contract_copy(tmp, name, change)) + missing = [w for w in want if not re.search(w, out)] + check(f"static control refused: {desc}", rc == 1 and not missing, f"rc={rc}, missing {missing}: {out[-1500:]}") + finally: + shutil.rmtree(tmp) + + config = open(os.path.join(REPO, "nextflow.config")).read() + schema = json.load(open(os.path.join(REPO, "nextflow_schema.json"))) + in_schema = any("pedigree" in g.get("properties", {}) for g in schema["definitions"].values()) + check("--pedigree is declared in nextflow.config and nextflow_schema.json", + re.search(r"^\s*pedigree\s*=\s*null\b", config, re.M) is not None and in_schema) + + def _drop_param(d, name): path = os.path.join(d, "nextflow_schema.json") s = json.load(open(path)) @@ -210,6 +270,29 @@ def run_controls(): shutil.rmtree(tmp) +def reader_controls(): + """The two readers the static check is built on, on inputs the code does not have yet.""" + import importlib.util + spec = importlib.util.spec_from_file_location("contract_check", CHECK) + m = importlib.util.module_from_spec(spec) + spec.loader.exec_module(m) + stmt = "publishDir \"x\", saveAs: { fn -> fn.endsWith('.log') ? null : fn }, enabled: { true }" + body = m.closure_body(stmt) + check("a saveAs closure ends at its own brace (an option after it is not part of it)", + body is not None and "enabled" not in body and m.save_as(body, "a.log") == "drop", repr(body)) + tmp = tempfile.mkdtemp() + try: + os.makedirs(os.path.join(tmp, "scripts")) + open(os.path.join(tmp, "scripts", "11-roh.sh"), "w").write( + 'S="${GENOME_DIR}/${SAMPLE}"\nbgzip -c x > "${S}/roh/${SAMPLE}_roh.txt.gz"\n') + short = m.bash_writes(tmp, "11", "{sample}/roh/{sample}_roh.txt", {}) + full = m.bash_writes(tmp, "11", "{sample}/roh/{sample}_roh.txt.gz", {}) + check("a step script writing x.txt.gz does not count as writing x.txt", + short is None and full == "scripts/11-roh.sh", f"{short!r} {full!r}") + finally: + shutil.rmtree(tmp) + + def malformed(): tmp = tempfile.mkdtemp() try: @@ -228,6 +311,7 @@ def malformed(): static_controls() +reader_controls() run_controls() malformed() print(f"{FAILS} failed")