xqtl-data/release.sos at main · jaempawi/xqtl-data · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
#!/usr/bin/env sos-runner
#vim: set filetype=python: set expandtab : ts=4:
#fileformat=SOS1.0

# This script recursively compiles all `*.md` files in the root folder via hugo
# and pushes the website to `gh-pages` branch

[global]
parameter: data_dir = 'descriptor'

[theme: provides = 'website/themes/hugo-book/README.md']
parameter: version = 'master'
download: dest_dir = 'website/themes', decompress=True, expand = '${ }'
  https://github.com/alex-shpak/hugo-book/archive/refs/heads/${version}.zip
bash: expand = '${ }'
  rm -rf website/themes/hugo-book && mv website/themes/hugo-book-${version} website/themes/hugo-book
  rm -f website/themes/${version}.zip

[default_1]
import os, glob, re
from collections import OrderedDict
def get_meta(files):
    data = {}
    for f in files:
        if os.path.basename(f) == 'README.md':
           continue
        lines = [x.strip() for x in open(f).readlines() if x.strip()]
        if len(lines) == 0:
           continue
        data[f] = {}
        if not lines[0].startswith('#') or lines[0].startswith('##'):
            raise ValueError(f'The first line of document "{f}" must start with "#" followed by dataset name')
        data[f]['content'] = lines
        data[f]['name'] = lines[0][1:].strip()
        data[f]['description'] = (lines[1].strip('.') + '.') if not lines[1].startswith('#') else ''
        try:
          data[f]['short_name'] = lines[[x.lower() for x in lines].index('## project abbreviation') + 1]
        except:
          pass
    return OrderedDict(sorted(data.items(), key = lambda i: i[1]['name'].lower()))

meta = get_meta(glob.glob(f'{data_dir}/**/*.md'))
categories = []
contents = OrderedDict()
toc = OrderedDict()
idx = 1
for k in meta:
    category = '1' if re.match(r"[-+]?\d+$", meta[k]['name'][0]) is not None else meta[k]['name'][0].upper()
    if category not in categories:
       idx = 1
       categories.append(category)
       toc[category] = '\n## {}\n\n----\n\n'.format(category if category != '1' else '\\#')
    if category not in contents:
       contents[category] = OrderedDict()
    desc_text = f"{' (%s)' % meta[k]['short_name'] if 'short_name' in meta[k] else ''}" + ( ': ' + meta[k]['description'] ) if meta[k]['description'] else ''
    html_name = "".join(x for x in meta[k]['name'].replace(' ', '-') if x.isalnum() or x in ['-', '_', '.']).lower()
    toc[category] += '{}. [{}]({}){}\n'.format(idx, meta[k]['name'], html_name, desc_text)
    contents[category][(meta[k]['content'][0].strip("#").strip(), html_name)] = '\n'.join(meta[k]['content']).strip()
    idx += 1

idx = 4
for k in contents:
    os.makedirs(f'website/content/{k.lower()}', exist_ok=True)
    with open(f'website/content/{k.lower()}/_index.md', 'w') as f:
         f.write('+++\nweight = {}\ntitle = "{}"\n+++\n\n'.format(idx, k if k != '1' else '#'))
         f.write(toc[k])
    idx += 1
    for num, kk in enumerate(contents[k]):
        with open(f'website/content/{k.lower()}/{kk[1]}.md', 'w') as f:
             f.write('---\ntitle: {1}\ntoc: true\nweight: {0}\n---\n\n'.format(num + 1, kk[0]))
             f.write(contents[k][kk])

menu = '+++\nheadless = true\n+++\n\n- Alphabetic List of Data Available\n'
for k in contents:
    menu += '  - [%s]({{< relref "/%s" >}})\n' % (k.upper(), k.lower())

os.makedirs(f'website/content/menu', exist_ok=True)
with open(f'website/content/menu/index.md', 'w') as f:
    f.write(menu)

# Update gh-pages
[default_2]
depends: 'website/themes/hugo-book/README.md', executable('ghp-import')
bash:
  set -e
  python3 toc.py descriptor/{gwas,omics,qtl,study_info,reference_data}/*.md -o README.md -t "FunGen-xQTL Data Catalog" -b "https://github.com/cumc/xqtl-analysis/tree/main/" -c "Lead analysts: "
  python3 toc.py descriptor/{gwas,omics,qtl,study_info,reference_data}/*.md -o website/content/_index.md -t "FunGen-xQTL Data Catalog" -b "https://github.com/cumc/xqtl-analysis/tree/main/" -c "Lead analysts: " -p -d "This documentation is still work in progress. We expect to complete it by summer 2024 if not sooner."
  cd website
  hugo
  ghp-import -n -p -f public

[clean]
bash: workdir='website'
  rm -rf themes resources content