pyntxos/.github/workflows/scrape_update.yml at master · python-sprints/pyntxos · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
# This workflow will install Python dependencies, run tests and lint with a single version of Python
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions

name: Scrape + Update

on:
  push:
    branches: [ master ]

jobs:
  build:

    runs-on: ubuntu-latest

    strategy:
      matrix:
        python-version: [3.8]
        target_repository: [python-sprints/pyntxos-dashboard]
        target_repository_dir: [pyntxos-dashboard]
        add_folders_files: ["./data ./utils requirements.txt dashboard.ipynb"]

    steps:
    - uses: actions/checkout@v2

    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v2
      with:
        python-version: ${{ matrix.python-version }}

    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        python -m pip install -r scrape_requirements.txt

    - name: Scrape data
      run: |
        python -c "import utils.scrape; import utils.data; utils.data.dump(utils.scrape.scrape(api_key='${{ secrets.OPEN_MAP_QUEST_API_KEY }}'))"

    - name: Checkout dashboard repository
      uses: actions/checkout@v2
      with:
        repository: ${{ matrix.target_repository }}
        path: ${{ matrix.target_repository_dir }}
        fetch-depth: '0'
        ref: 'master'

    - name: Move files
      run: |
        cd ${{ matrix.target_repository_dir }}
        rm -rf ${{ matrix.add_folders_files }}
        cd ..
        mv ${{ matrix.add_folders_files }} ${{ matrix.target_repository_dir }}

    # To make this work you need the following:
    # 1) Run "ssh-keygen -t rsa -b 4096" and store into a temporary key file
    # 2) Copy the public key and store into the target (pages) repository "Settings" -> "Deploy Keys"
    # 3) Copy the private key and store into secrets of this repository as "SSH_PRIVATE_KEY"
    # 4) Add " " (blank space) into secrets of this repository as "KNOWN_HOSTS"
    - name: Install SSH key
      uses: shimataro/ssh-key-action@v2
      with:
         key: ${{ secrets.SSH_PRIVATE_KEY }}
         name: id_rsa # optional
         known_hosts: ${{ secrets.KNOWN_HOSTS }}

    - name: Update and push the pages repository
      run: |
        cd ${{ matrix.target_repository_dir }}
        git remote add github_actions_ssh git@github.com:${{ matrix.target_repository }}.git
        git config user.email $GITHUB_ACTOR
        git config user.name $GITHUB_ACTOR
        git add ${{ matrix.add_folders_files }}
        git diff --quiet && git diff --staged --quiet || git commit -m "update ${{ matrix.target_repository }} ${{ matrix.add_folders_files }}"
        git push -u github_actions_ssh master