bitbucket-pipelines.yml

#  Template python-build

#  This template allows you to validate your python code.
#  The workflow allows running tests and code linting on the default branch.
definitions:
  steps:
    - step: &z-run-any-anchor
        name: Run R or Stata code
        # commented out to speed up processing. Uncomment for debugging
        #artifacts:
        # - 1**/**
        script:
          - . ./tools/parse_yaml.sh
          - eval $(parse_yaml config.yml)
          - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
          - if [ -f tools/download_openicpsr-private.py ]; then python3 tools/download_openicpsr-private.py $openICPSRID; fi
          - chmod a+rx ./automations/*.sh
          - chmod a+rx ./*.sh
          - ./automations/00_unpack_zip.sh  $openICPSRID
          - ./automations/01_check_file_sizes.sh $openICPSRID
          - ./tools/linux-system-info.sh
          - ./tools/update_config.sh
          - ./run.sh
          - project="${openICPSRID:-$openicpsr}"
          - project="${project:-$dataverse}"
          - project="${project:-$zenodo}"
          - project="${project:-$osf}"
          - ./automations/20_commit_code.sh $project run
          - ./automations/20_commit_code.sh config.yml
          - git status
          - git push
          - ./automations/70_publish_comment.sh z-run-stata completed

pipelines:
  custom:
    1-populate-from-icpsr: #name of this pipeline
      - variables:          #list variable names under here
          # These do not need to have a value, if "config.yml" is filled out.
          - name: openICPSRID   
          - name: jiraticket
          - name: ZenodoID
          # Accepts: numeric ID, full URL, DOI, or community request URL.
          # Leave blank if jiraticket is set (orchestrator will query Jira).
          #- name: DataverseID
          #- name: OSFID
          - name: ProcessStata
            default: "yes"
            allowed-values:
              - "no"
              - "yes"
          - name: ProcessR
            default: "yes"
            allowed-values:
              - "no"
              - "yes"
          - name: ProcessPython
            default: "yes"
            allowed-values:
              - "no"
              - "yes"
          - name: ProcessJulia
            default: "no"
            allowed-values:
              - "no"
              - "yes"
          - name: ProcessPii
            default: "yes"
            allowed-values:
              - "no"
              - "yes"
          - name: SkipProcessing
            default: "no"
            allowed-values:
              - "no"
              - "yes"
      - step:
          name: Download
          image: python:3.12
          artifacts:
            - generated/**
            - cache/**
          caches:
            - pip
          script:
            - apt-get update && apt-get install -y zip
            - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
            - chmod a+rx ./automations/*.sh
            - . ./tools/parse_yaml.sh
            - eval $(parse_yaml config.yml)
            - if [ -z $openICPSRID ]; then openICPSRID=$openicpsr; fi
            - if [ -z $ZenodoID ]; then ZenodoID=$zenodo; fi
            - projectID="${openICPSRID}"
            - if [ -z "$jiraticket" ] && [ -n "${openICPSRID:-}" ]; then jiraticket=$(python3 tools/jira_find_task_by_icpsr.py "$openICPSRID" 2>/dev/null || true); else echo "Jira ticket not set"; fi
            - export jiraticket
            - echo "Using Jira case $jiraticket"
            - ./tools/update_config.sh
            - ./automations/70_publish_comment.sh 1-populate-from-icpsr started
            - if [ -d "${projectID:-__none__}" ]; then \rm -rf $projectID; fi
            - if [ ! -z $openICPSRID ]; then python3 tools/download_openicpsr-private.py $openICPSRID; fi
            - if [ -z "${openICPSRID:-}" ] && { [ ! -z "$ZenodoID" ] || [ ! -z "$jiraticket" ]; }; then zenodo_dir=$(python3 tools/download_zenodo.py ${ZenodoID:+--zenodo-id "$ZenodoID"} ${jiraticket:+--jira-ticket "$jiraticket"} --print-id 2>&1 | tail -1); fi
            - if [ -z "$projectID" ] && [ ! -z "${zenodo_dir:-}" ]; then projectID="$zenodo_dir"; fi
            - ./automations/00_unpack_zip.sh  $projectID
            - mkdir cache
            - if [ ! -z $openICPSRID ]; then mv *.zip cache/; fi
            - if [ ! -z "${zenodo_dir:-}" ]; then zip -rp cache/${projectID}.zip $projectID/* ; fi
            - ./automations/00_prepare_aux.sh
            - ./automations/01_check_file_sizes.sh $projectID
            - ./automations/02_list_data_files.sh $projectID
            - ./automations/03_list_program_files.sh $projectID
            - ./automations/04_create_manifest.sh $projectID
            - ./automations/05_check_zip_files.sh $projectID
            - ./automations/06_check_duplicates_and_zero_bytes.sh $projectID
            - ./automations/07_check_file_paths.sh $projectID
            - ./automations/22_compare_manifests.sh $projectID
            - ls -lR
      - step:
          name: Check downloads
          script:
            # check if there are files called $projectID.zip in cache
            - . ./tools/parse_yaml.sh
            - eval $(parse_yaml config.yml)
            - if [ -z $openICPSRID ]; then openICPSRID=$openicpsr; fi
            - if [ -z $ZenodoID ]; then ZenodoID=$zenodo; fi
            - projectID="${openICPSRID}"
            - if [ ! -z "$ZenodoID" ] && echo "$ZenodoID" | grep -q '/'; then ZenodoID=$(echo "$ZenodoID" | python3 -c "import sys,re; s=sys.stdin.read().strip().rstrip('/'); m=re.search(r'zenodo\.org/(?:records?|deposit)/(\d+)', s) or re.search(r'zenodo\.(\d+)', s); print(m.group(1) if m else s)"); fi
            - projectID="${projectID:-zenodo-$ZenodoID}"
            - if [ -f cache/$projectID.zip ]; then echo "✅ Found $projectID.zip in cache"; else echo "⚠️  Did not find $projectID.zip in cache. 🛑 You may need to use the BIG pipeline!!! "; exit 2; fi
      - parallel: # we will run these in parallel
        - step:
            name: Run Stata parser 
            image:
              name: larsvilhuber/bitbucket-stata:latest
              username: $W_DOCKER_USERNAME
              password: $W_DOCKER_PAT
            script:
              - . ./tools/parse_yaml.sh
              - eval $(parse_yaml config.yml)
              - if [ -z $openICPSRID ]; then openICPSRID=$openicpsr; fi
              - if [ -z $ZenodoID ]; then ZenodoID=$zenodo; fi
              - projectID="${openICPSRID}"
              - if [ ! -z "$ZenodoID" ] && echo "$ZenodoID" | grep -q '/'; then ZenodoID=$(echo "$ZenodoID" | python3 -c "import sys,re; s=sys.stdin.read().strip().rstrip('/'); m=re.search(r'zenodo\.org/(?:records?|deposit)/(\d+)', s) or re.search(r'zenodo\.(\d+)', s); print(m.group(1) if m else s)"); fi
              - projectID="${projectID:-zenodo-$ZenodoID}"
              - chmod a+rx ./automations/*.sh
              - ./automations/00_preliminaries.sh $ProcessStata $projectID
              - ./automations/00_unpack_zip.sh  $projectID
              - ./automations/10_run_stata_scanner.sh $projectID
            artifacts:
              - generated/**
        - step:
            name: Run Stata PII scanner
            image:
              name: larsvilhuber/bitbucket-stata:latest
              username: $W_DOCKER_USERNAME
              password: $W_DOCKER_PAT
            script:
              - . ./tools/parse_yaml.sh
              - eval $(parse_yaml config.yml)
              - if [ -z $openICPSRID ]; then openICPSRID=$openicpsr; fi
              - if [ -z $ZenodoID ]; then ZenodoID=$zenodo; fi
              - projectID="${openICPSRID}"
              - if [ ! -z "$ZenodoID" ] && echo "$ZenodoID" | grep -q '/'; then ZenodoID=$(echo "$ZenodoID" | python3 -c "import sys,re; s=sys.stdin.read().strip().rstrip('/'); m=re.search(r'zenodo\.org/(?:records?|deposit)/(\d+)', s) or re.search(r'zenodo\.(\d+)', s); print(m.group(1) if m else s)"); fi
              - projectID="${projectID:-zenodo-$ZenodoID}"
              - chmod a+rx ./automations/*.sh
              - ./automations/00_preliminaries.sh $ProcessPii $projectID
              - ./automations/00_unpack_zip.sh  $projectID
              - ./automations/16_run_pii_scanner.sh $projectID
            artifacts:
              - generated/**
        - step:
            name: Run R parser
            image: aeadataeditor/verification-r:latest
            script:
              - . ./tools/parse_yaml.sh
              - eval $(parse_yaml config.yml)
              - if [ -z $openICPSRID ]; then openICPSRID=$openicpsr; fi
              - if [ -z $ZenodoID ]; then ZenodoID=$zenodo; fi
              - projectID="${openICPSRID}"
              - if [ ! -z "$ZenodoID" ] && echo "$ZenodoID" | grep -q '/'; then ZenodoID=$(echo "$ZenodoID" | python3 -c "import sys,re; s=sys.stdin.read().strip().rstrip('/'); m=re.search(r'zenodo\.org/(?:records?|deposit)/(\d+)', s) or re.search(r'zenodo\.(\d+)', s); print(m.group(1) if m else s)"); fi
              - projectID="${projectID:-zenodo-$ZenodoID}"
              - chmod a+rx ./automations/*.sh
              - ./automations/00_preliminaries.sh $ProcessR $projectID
              - ./automations/00_unpack_zip.sh  $projectID
              - ./automations/14_run_r_scanner.sh $projectID
        - step:
            name: Run Python parser
            image: python:3.12
            script:
              - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
              - . ./tools/parse_yaml.sh
              - eval $(parse_yaml config.yml)
              - if [ -z $openICPSRID ]; then openICPSRID=$openicpsr; fi
              - if [ -z $ZenodoID ]; then ZenodoID=$zenodo; fi
              - projectID="${openICPSRID}"
              - if [ ! -z "$ZenodoID" ] && echo "$ZenodoID" | grep -q '/'; then ZenodoID=$(echo "$ZenodoID" | python3 -c "import sys,re; s=sys.stdin.read().strip().rstrip('/'); m=re.search(r'zenodo\.org/(?:records?|deposit)/(\d+)', s) or re.search(r'zenodo\.(\d+)', s); print(m.group(1) if m else s)"); fi
              - projectID="${projectID:-zenodo-$ZenodoID}"
              - chmod a+rx ./automations/*.sh
              - ./automations/00_preliminaries.sh $ProcessPython $projectID
              - ./automations/00_unpack_zip.sh  $projectID
              - ./automations/15_run_python_scanner.sh $projectID
            artifacts:
              - generated/**
        - step:
            name: Run Julia parser
            image: julia:latest
            script:
              - . ./tools/parse_yaml.sh
              - eval $(parse_yaml config.yml)
              - if [ -z $openICPSRID ]; then openICPSRID=$openicpsr; fi
              - if [ -z $ZenodoID ]; then ZenodoID=$zenodo; fi
              - projectID="${openICPSRID}"
              - if [ ! -z "$ZenodoID" ] && echo "$ZenodoID" | grep -q '/'; then ZenodoID=$(echo "$ZenodoID" | python3 -c "import sys,re; s=sys.stdin.read().strip().rstrip('/'); m=re.search(r'zenodo\.org/(?:records?|deposit)/(\d+)', s) or re.search(r'zenodo\.(\d+)', s); print(m.group(1) if m else s)"); fi
              - projectID="${projectID:-zenodo-$ZenodoID}"
              - chmod a+rx ./automations/*.sh
              - ./automations/00_preliminaries.sh $ProcessJulia $projectID
              - ./automations/00_unpack_zip.sh  $projectID
              - ./automations/17_run_julia_scanner.sh $projectID
            artifacts:
              - generated/**
        - step:
            name: Count lines and comments
            image:
              name: aldanial/cloc
              username: $W_DOCKER_USERNAME
              password: $W_DOCKER_PAT
            script:
              - . ./tools/parse_yaml.sh
              - eval $(parse_yaml config.yml)
              - if [ -z $openICPSRID ]; then openICPSRID=$openicpsr; fi
              - if [ -z $ZenodoID ]; then ZenodoID=$zenodo; fi
              - projectID="${openICPSRID}"
              - if [ ! -z "$ZenodoID" ] && echo "$ZenodoID" | grep -q '/'; then ZenodoID=$(echo "$ZenodoID" | python3 -c "import sys,re; s=sys.stdin.read().strip().rstrip('/'); m=re.search(r'zenodo\.org/(?:records?|deposit)/(\d+)', s) or re.search(r'zenodo\.(\d+)', s); print(m.group(1) if m else s)"); fi
              - projectID="${projectID:-zenodo-$ZenodoID}"
              - chmod a+rx ./automations/*.sh
              - ./automations/00_preliminaries.sh yes $projectID
              - ./automations/00_unpack_zip.sh  $projectID
              - ./automations/08_count_lines.sh $projectID
            artifacts:
              - generated/**
      ## collect all information
      - step:
          image: python:3.12
          name: Add info to REPLICATION.md
          script:
            - . ./tools/parse_yaml.sh
            - eval $(parse_yaml config.yml)
            - chmod a+rx ./automations/*.sh
            - ./automations/24_amend_report.sh
          artifacts:
            - generated/**
      - step: 
          image: python:3.12
          # This will re-extract the downloaded
          name: Commit everything back
          script:
            - . ./tools/parse_yaml.sh
            - eval $(parse_yaml config.yml)
            - if [ -z $openICPSRID ]; then openICPSRID=$openicpsr; else echo "openICPSRID not set"; fi
            - if [ -z $ZenodoID ]; then ZenodoID=$zenodo; else echo "ZenodoID not set"; fi
            - projectID="${openICPSRID}"
            - if [ ! -z "$ZenodoID" ] && echo "$ZenodoID" | grep -q '/'; then ZenodoID=$(echo "$ZenodoID" | python3 -c "import sys,re; s=sys.stdin.read().strip().rstrip('/'); m=re.search(r'zenodo\.org/(?:records?|deposit)/(\d+)', s) or re.search(r'zenodo\.(\d+)', s); print(m.group(1) if m else s)"); fi
            - projectID="${projectID:-zenodo-$ZenodoID}"
            - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
            - if [ -z "$jiraticket" ] && [ -n "${openICPSRID:-}" ]; then jiraticket=$(python3 tools/jira_find_task_by_icpsr.py "$openICPSRID" 2>/dev/null || true); else echo "Jira ticket not set"; fi
            - echo "Using Jira case $jiraticket"
            - export jiraticket
            - ./tools/update_config.sh
            - if [ -d cache ]; then ls -lR cache/*; fi
            - ls -l generated/*
            # repeat this part
            - if [ -d $projectID ]; then \rm -rf $projectID; fi
            # unpack it all
            - if [ -f cache/$projectID.zip ]; then mv cache/$projectID.zip .; fi
            - chmod a+rx ./automations/*.sh
            - ./automations/00_unpack_zip.sh  $projectID
            - ./automations/20_commit_code.sh $projectID
            - ./automations/21_cleanup.sh     $projectID
            - ./automations/25_replace_report.sh
            - ./automations/30_cleanup_aux.sh
            - ./automations/20_commit_code.sh config.yml notag
            - git status
            - git push && git push --tags
            - ./automations/70_publish_comment.sh 1-populate-from-icpsr completed
    2-merge-report: #name of this pipeline
      - variables:          #list variable names under here
          - name: jiraticket
      - step:
          name: Combine Part A and B of the report
          image: python:3.12
          caches:
            - pip
          script:
            - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
            - chmod a+rx automations/*.sh
            - ./automations/50_merge-parts.sh
            - git push
            - . ./tools/parse_yaml.sh
            - eval $(parse_yaml config.yml)       
            - if [ -z $openICPSRID ]; then openICPSRID=$openicpsr; else echo "openICPSRID not set"; fi
            - ./automations/70_publish_comment.sh 2-merge-report completed
    3-split-report: #name of this pipeline
      - variables:          #list variable names under here
          - name: jiraticket
      - step:
          name: Split REPLICATION.md into Part A and B of the report
          image: python:3.12
          caches:
            - pip
          script:
            - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
            - chmod a+rx automations/*.sh
            - ./automations/51_split-parts.sh
            - git push
            - . ./tools/parse_yaml.sh
            - eval $(parse_yaml config.yml)
            - if [ -z $openICPSRID ]; then openICPSRID=$openicpsr; else echo "openICPSRID not set"; fi
            - ./automations/70_publish_comment.sh 3-split-report completed
    4-refresh-tools: #name of this pipeline
      - step:
          name: Refresh the tools in this repository
          image: python:3.12
          caches:
            - pip
          script:
            - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
            - wget https://raw.githubusercontent.com/AEADataEditor/replication-template/master/tools/update_tools.sh
            - chmod a+rx update_tools.sh
            - ./update_tools.sh
            - git push
            - ./automations/70_publish_comment.sh 4-refresh-tools completed
    5-rename-directory: #name of this pipeline
      - variables:          #list variable names under here
          - name: oldName
          - name: newName
          - name: jiraticket
      - step:
          name: Rename a deposit directory
          image: python:3.12
          caches:
            - pip
          script:
            - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
            - . ./tools/parse_yaml.sh
            - eval $(parse_yaml config.yml)
            - if [ -z $openICPSRID ]; then openICPSRID=$openicpsr; else echo "openICPSRID not set"; fi            
            - git mv $oldName $newName
            - git commit -m "[skip ci] Rename $oldName to $newName"
            - export openICPSRID=$newName
            - ./tools/update_config.sh
            - ./automations/20_commit_code.sh config.yml notag
            - git push
            - ./automations/70_publish_comment.sh 5-rename-directory completed
    6-convert-eps-pdf:  # this can be run to convert EPS and PDF files to PNG
      - variables:          #list variable names under here
          # These do not need to have a value, if "config.yml" is filled out.
          - name: path
          - name: jiraticket
          - name: ProcessEPS
            default: "yes"
            allowed-values:
              - "no"
              - "yes"
          - name: ProcessPDF
            default: "no"
            allowed-values:
              - "no"
              - "yes"
          - name: DockerImg
            default: "dpokidov/imagemagick"
      - step:
          name: Convert EPS and PDF to PNG
          image: python:3.12
          caches:
            - pip
          services:
            - docker
          script:
            - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
            - . ./tools/parse_yaml.sh
            - eval $(parse_yaml config.yml)
            - if [ -z $openICPSRID ]; then openICPSRID=$openicpsr; else echo "openICPSRID not set"; fi
            - chmod a+rx ./automations/*.sh
            - ./automations/70_publish_comment.sh 6-convert-eps-pdf started
            - docker run --rm -v $(pwd):/imgs --entrypoint=/bin/bash $DockerImg ./automations/52_convert_eps_pdf.sh -p "$path" -e $ProcessEPS -d $ProcessPDF
            - git add "$path"
            - git commit -m "[skip ci] Converted EPS and PDF to PNG"
            - git push
            - ./automations/70_publish_comment.sh 6-convert-eps-pdf completed
    7-download-box-manifest: #name of this pipeline
      - variables:          #list variable names under here
          - name: repository_name
      - step:
          image: python:3.12
          name: Download Box and create manifests
          caches:
            - pip
          script:
            - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
            - chmod a+rx ./automations/*.sh
            - . ./tools/parse_yaml.sh
            - eval $(parse_yaml config.yml)
            - if [ -z $openICPSRID ]; then openICPSRID=$openicpsr; else echo "openICPSRID not set"; fi
            - chmod a+rx ./automations/*.sh
            - ./automations/70_publish_comment.sh 7-download-box-manifest started
            - python3 ./tools/download_box_private.py $repository_name
            - ./automations/00_unpack_zip.sh  restricted
            - ./automations/04_create_manifest.sh restricted restricted
            - git add -f generated/*
            - git commit -m "[skip ci] Downloaded Box files and created manifests"
            - git push
            - ./automations/70_publish_comment.sh 7-download-box-manifest completed
    8-download-sivacor: #name of this pipeline
      - variables:          #list variable names under here
          - name: jiraticket
            default: ""
      - step:
          image: python:3.12
          name: Download SIVACOR artifacts
          caches:
            - pip
          script:
            - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
            - if [ -f requirements-sivacor.txt ]; then pip install -r requirements-sivacor.txt; fi
            - . ./tools/parse_yaml.sh
            - eval $(parse_yaml config.yml)
            - if [ -z $openICPSRID ]; then openICPSRID=$openicpsr; else echo "openICPSRID not set"; fi
            - if [ -z "$jiraticket" ] && [ -n "${openICPSRID:-}" ]; then jiraticket=$(python3 tools/jira_find_task_by_icpsr.py "$openICPSRID" 2>/dev/null || true); else echo "Jira ticket not set"; fi
            - if [ -z "$jiraticket" ]; then jiraticket=$(basename $(pwd)); fi  # Determine Jira case from variable, directory name, or config.yml
            - 'if [ -z "$jiraticket" ]; then echo "Error: Could not determine Jira case"; exit 1; fi'
            - 'echo "Using Jira case: $jiraticket"'
            - ./automations/70_publish_comment.sh 8-download-sivacor started
            - python3 tools/download_sivacor.py $jiraticket  # Run the SIVACOR download script
            - git push -u origin HEAD  # Push the created branch (do not merge to master)
            - ./automations/70_publish_comment.sh 8-download-sivacor completed
    9-find-cran-date: #name of this pipeline
      - variables:
          - name: filename
            default: "renv.lock"
          - name: jiraticket
      - step:
          name: Find CRAN snapshot date from package file
          image: python:3.12
          artifacts:
            - generated/**
          caches:
            - pip
          script:
            - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
            - . ./tools/parse_yaml.sh
            - eval $(parse_yaml config.yml)
            - if [ -z $openICPSRID ]; then openICPSRID=$openicpsr; else echo "openICPSRID not set"; fi
            - export jiraticket
            - found_file=$(find . -name "$filename" -not -path "./.git/*" | head -1)
            - if [ -z "$found_file" ]; then echo "File '$filename' not found in repository"; exit 1; fi
            - echo "Found $found_file"
            - ./automations/70_publish_comment.sh 9-find-cran-date started
            - python3 tools/find_cran_date.py "$found_file" --output generated/notes-for-r.md
            - git add generated/notes-for-r.md
            - git diff --cached --quiet || git commit -m "[skip ci] Added R environment notes"
            - git push
            - ./automations/70_publish_comment.sh 9-find-cran-date completed
    w-big-populate-from-icpsr: #name of this pipeline
      - variables:          #list variable names under here
          # These do not need to have a value, if "config.yml" is filled out.
          - name: openICPSRID
          - name: ZenodoID
          # Accepts: numeric ID, full URL, DOI, or community request URL.
          # Leave blank if jiraticket is set (orchestrator will query Jira).
          - name: jiraticket
      - step:
          name: Download and commit
          size: 2x
          image: python:3.12
          caches:
            - pip
          script:
            - apt-get update && apt-get install -y cloc
            - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
            - . ./tools/parse_yaml.sh
            - eval $(parse_yaml config.yml)
            - if [ -z $openICPSRID ]; then openICPSRID=$openicpsr; fi
            - if [ -z $ZenodoID ]; then ZenodoID=$zenodo; fi
            - projectID="${openICPSRID}"
            - if [ -z "$jiraticket" ] && [ -n "${openICPSRID:-}" ]; then jiraticket=$(python3 tools/jira_find_task_by_icpsr.py "$openICPSRID" 2>/dev/null || true); fi
            - export jiraticket
            - ./tools/update_config.sh
            - ./automations/70_publish_comment.sh w-big-populate-from-icpsr started
            - if [ -d "${projectID:-__none__}" ]; then \rm -rf $projectID; fi
            - if [ ! -z $openICPSRID ]; then python3 tools/download_openicpsr-private.py $openICPSRID; fi
            - if [ -z "${openICPSRID:-}" ] && { [ ! -z "$ZenodoID" ] || [ ! -z "$jiraticket" ]; }; then zenodo_dir=$(python3 tools/download_zenodo.py ${ZenodoID:+--zenodo-id "$ZenodoID"} ${jiraticket:+--jira-ticket "$jiraticket"} --print-id 2>&1 | tail -1); fi
            - if [ -z "$projectID" ] && [ ! -z "${zenodo_dir:-}" ]; then projectID="$zenodo_dir"; fi
            - chmod a+rx ./automations/*.sh
            - ./automations/00_prepare_aux.sh
            - ./automations/00_unpack_zip.sh  $projectID
            - ./automations/01_check_file_sizes.sh $projectID
            - ./automations/02_list_data_files.sh $projectID
            - ./automations/03_list_program_files.sh $projectID
            - ./automations/04_create_manifest.sh $projectID
            - ./automations/05_check_zip_files.sh $projectID
            - ./automations/06_check_duplicates_and_zero_bytes.sh $projectID
            - ./automations/07_check_file_paths.sh $projectID
            - ./automations/08_count_lines.sh $projectID
            - ./automations/22_compare_manifests.sh $projectID
            - ./automations/24_amend_report.sh
            - ./automations/20_commit_code.sh $projectID
            - ./automations/21_cleanup.sh     $projectID
            - ./automations/25_replace_report.sh
            - ./automations/30_cleanup_aux.sh
            - ./automations/20_commit_code.sh config.yml notag
            - git status
            - git push
            - git push --tags
            - ./automations/70_publish_comment.sh w-big-populate-from-icpsr completed

    x-run-python:
      - variables: 
          - name: Script
            default: "run-python.sh"
      - step: 
          name: Run Python code
          image: python:3.11
          script:
            - if [ -f "$Script" ]; then chmod a+rx $Script; ./$Script; fi
            - if [ -f "post-run.sh" ]; then chmod a+rx post-run.sh; ./post-run.sh; fi
            - git push
    z-run-stata:  # name of this pipeline
      - variables:          #list variable names under here
          # These do not need to have a value, if "config.yml" is filled out.
          - name: openICPSRID   
          #- name: ZenodoID
          #- name: DataverseID
          #- name: OSFID
          - name: MainFile
            default: "main.do"
          - name: RunCommand
            default: "run.sh"
      - step: 
            <<: *z-run-any-anchor
            name: Run Stata code
            size: 2x
            image:
              name: larsvilhuber/bitbucket-stata:latest
              username: $W_DOCKER_USERNAME
              password: $W_DOCKER_PAT

    z-run-any-big: # name of this pipeline
      - variables:          #list variable names under here
          # These do not need to have a value, if "config.yml" is filled out.
          - name: openICPSRID   
          #- name: ZenodoID
          #- name: DataverseID
          #- name: OSFID
          - name: MainFile
          - name: RunCommand
            default: "run.sh"
      - step:
            name: Run R or Stata code
            size: 8x
            image:
              name: larsvilhuber/bitbucket-stata:latest
              username: $W_DOCKER_USERNAME
              password: $W_DOCKER_PAT
            # commented out to speed up processing. Uncomment for debugging
            #artifacts:
            # - 1**/**
            script:
              - set -v
              - . ./tools/parse_yaml.sh
              - eval $(parse_yaml config.yml)
              - if [ -z $openICPSRID ]; then openICPSRID=$openicpsr; fi
              - if [ -z $ZenodoID ]; then ZenodoID=$zenodo; fi
              - projectID="${openICPSRID}"
              - projectID="${projectID:-zenodo-$ZenodoID}"
              - if [ -f tools/download_openicpsr-private.py ]; then python3 tools/download_openicpsr-private.py $projectID; fi
              - chmod a+rx ./automations/*.sh
              - chmod a+rx ./*.sh
              - ./automations/00_unpack_zip.sh  $projectID
              - ./tools/linux-system-info.sh
              - ./tools/update_config.sh
              - ./run.sh
              - ./automations/20_commit_code.sh $projectID run
              - ./automations/20_commit_code.sh config.yml notag
              - git status
              - git push
              - ./automations/70_publish_comment.sh z-run-any-big completed

  # default:
  #     - step:
  #         name: Check commit message and run pipeline
  #         script:
  #           - |
  #             BITBUCKET_COMMIT_MESSAGE=`git log --format=%B -n 1 $BITBUCKET_COMMIT`
  #             if echo "$BITBUCKET_COMMIT_MESSAGE" | grep -qiE "run-stata|run-r"; then
  #               echo "Commit message contains 'run-*'. Running z-run- steps."
  #               exit 0  # This will allow the pipeline to continue to the next step
  #             else
  #               echo "Commit message does not contain 'run-stata|run-r'. Skipping z-run- steps."
  #               exit 1  # This will stop the pipeline here if the condition is not met
  #             fi
  #     - parallel:
  #       - step: 
  #             <<: *z-run-any-anchor
  #             name: Run Stata code
  #             size: 2x
  #             image:
  #               name: larsvilhuber/bitbucket-stata:latest
  #               username: $W_DOCKER_USERNAME
  #               password: $W_DOCKER_PAT
  #       - step:
  #             <<: *z-run-any-anchor
  #             name: Run R code
  #             size: 2x
  #             image:
  #               name: aeadataeditor/verification-r:latest