Skip to content

Commit df15c1c

Browse files
byobyo
committed
Merge branch 'scripts-in-configmap'
2 parents f195eb2 + 4a26d43 commit df15c1c

5 files changed

Lines changed: 186 additions & 147 deletions

File tree

‎helm/osm-machinery/scripts/tile-server-entrypoint.sh‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
#!/bin/sh
2+
3+
set -eu
4+
5+
echo "Creating style data"
6+
cp -r /home/renderer/src/openstreetmap-carto-backup/* /data/style/
7+
cd /data/style/
8+
sed -i "/dbname:/a\\
9+
host: \"$PGHOST\"\\
10+
user: \"$PGUSER\"\\
11+
password: \"$PGPASSWORD\"
12+
" /data/style/project.mml
13+
14+
carto ${NAME_MML:-project.mml} > mapnik.xml
15+
16+
echo "INFO: Starting tile server"
17+
18+
service apache2 restart
19+
20+
mkdir /run/renderd || true
21+
chown renderer /run/renderd
22+
23+
sudo -u renderer renderd -f -c /etc/renderd.conf

‎helm/osm-machinery/scripts/tile-server-sync-cronjob-entrypoint.sh‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
{{- /* Create a filtered list of regions to support */}}
2+
{{- $regions := list }}
3+
{{- range $.Values.regions }}
4+
{{- if and .enabled ( not ( .skipDownload | default false ) ) }}
5+
{{- $regions = append $regions ( dict
6+
"enabled" .enabled
7+
"name" .name
8+
"pbf" .pbf
9+
"poly" .poly
10+
) }}
11+
{{- end }}
12+
{{- end }}
13+
{{- $regionsChecksum := toJson $regions | sha256sum }}
14+
#!/bin/sh
15+
16+
set -eu
17+
18+
mkdir -p /data/download/
19+
mkdir -p /data/database/
20+
mkdir -p /data/tiles/
21+
mkdir -p /data/style/
22+
23+
# Cleanup any old download files
24+
[ "$( ls -A /data/download/ )" ] && rm -rf /data/download/*
25+
26+
# Small fix to the update script to use pgdb password
27+
cp $(which openstreetmap-tiles-update-expire.sh ) /tmp/otue.sh
28+
sed -i 's/^TRIM_OPTIONS=.*$/TRIM_OPTIONS="-d $DBNAME --password"/g' /tmp/otue.sh
29+
30+
if [ z"$( cat /data/database/planet-import-complete 2>/dev/null || true )" = z"{{ $regionsChecksum }}" ]; then
31+
# The data has been imported, check if there's any diff and apply it
32+
echo "INFO: Synchronizing osm changes"
33+
34+
touch /var/log/tiles/run.log; tail -f /var/log/tiles/run.log >> /proc/1/fd/1 &
35+
touch /var/log/tiles/osmosis.log; tail -f /var/log/tiles/osmosis.log >> /proc/1/fd/1 &
36+
touch /var/log/tiles/expiry.log; tail -f /var/log/tiles/expiry.log >> /proc/1/fd/1 &
37+
touch /var/log/tiles/osm2pgsql.log; tail -f /var/log/tiles/osm2pgsql.log >> /proc/1/fd/1 &
38+
39+
sh /tmp/otue.sh
40+
41+
exit 0
42+
fi
43+
44+
# The data has not yet been imported or configuration of regions has changed,
45+
# start from scratch by importing the latest snapshot of the data
46+
47+
echo "INFO: Doing initial data import"
48+
49+
rm /data/database/planet-import-complete 2>/dev/null || true
50+
51+
[ "$( ls -A /data/database/ )" ] && rm -rf /data/database/*
52+
[ "$( ls -A /data/tiles/ )" ] && rm -rf /data/tiles/*
53+
[ "$( ls -A /data/style/ )" ] && rm -rf /data/style/*
54+
55+
echo "INFO: Creating style data"
56+
cp -r /home/renderer/src/openstreetmap-carto-backup/* /data/style/
57+
cd /data/style/
58+
carto ${NAME_MML:-project.mml} > mapnik.xml
59+
60+
{{- range $index, $region := $regions }}
61+
62+
echo "INFO: Preparing data for region {{ $region.name }}"
63+
64+
{{- $suffix := "" }}
65+
{{- if ne $index 0 }}{{ $suffix = printf "_%d" $index }}{{ end }}
66+
67+
echo "INFO: Download PBF file for region {{ $region.name }}"
68+
wget ${WGET_ARGS:-} {{ $region.pbf | quote }} -O /data/download/region{{ $suffix }}.osm.pbf
69+
70+
echo "INFO: Download POLY file for region {{ $region.name }}"
71+
wget ${WGET_ARGS:-} {{ $region.poly | quote }} -O /data/database/region{{ $suffix }}.poly
72+
73+
echo "INFO: Determining replication timestamp for region {{ $region.name }}"
74+
osmium fileinfo -g header.option.osmosis_replication_timestamp /data/download/region{{ $suffix }}.osm.pbf >> /data/download/replication_timestamps.txt
75+
76+
{{- if ne $index 0 }}
77+
78+
echo "INFO: Merging in new pbf file from region {{ $region.name }}"
79+
osmium merge /data/download/region.osm.pbf /data/download/region{{ $suffix }}.osm.pbf -o /data/download/region-merged.osm.pbf
80+
mv -f /data/download/region-merged.osm.pbf /data/download/region.osm.pbf
81+
rm -f /data/download/region{{ $suffix }}.osm.pbf
82+
83+
echo "INFO: Merging in new poly file from region {{ $region.name }}"
84+
( echo "" && cat /data/database/region{{ $suffix }}.poly ) >> /data/database/region.poly
85+
rm -f /data/database/region{{ $suffix }}.poly
86+
87+
{{- end }}
88+
{{- end }}
89+
90+
echo "INFO: Finished processing regions"
91+
92+
REPLICATION_TIMESTAMP="$(cat /data/download/replication_timestamps.txt | sort | head -n 1)"
93+
echo "INFO: Oldest replication timestamp: $REPLICATION_TIMESTAMP"
94+
95+
# initial setup of osmosis workspace (for consecutive updates)
96+
/tmp/otue.sh $REPLICATION_TIMESTAMP || true
97+
98+
# Import data
99+
osm2pgsql -d gis --create --slim -G --hstore \
100+
--tag-transform-script /data/style/${NAME_LUA:-openstreetmap-carto.lua} \
101+
--number-processes ${THREADS:-4} \
102+
-S /data/style/${NAME_STYLE:-openstreetmap-carto.style} \
103+
/data/download/region.osm.pbf \
104+
${OSM2PGSQL_EXTRA_ARGS:-} \
105+
;
106+
107+
rm -rf /data/download/*
108+
109+
# Create indexes
110+
if [ -f "/data/style/${NAME_SQL:-indexes.sql}" ]; then
111+
psql -d gis -f "/data/style/${NAME_SQL:-indexes.sql}"
112+
fi
113+
114+
# Import external data
115+
python3 /data/style/scripts/get-external-data.py -c /data/style/external-data.yml -D /data/style/data
116+
117+
# Mark the import as done
118+
echo -n "{{ $regionsChecksum }}" > /data/database/planet-import-complete
119+
120+
echo "INFO: Initialization complete!"

‎helm/osm-machinery/templates/tile-server-deployment.yaml‎

Lines changed: 17 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
{{- if $.Values.tileServer.webServer.enabled }}
2+
{{- $scripts := list "tile-server-entrypoint.sh" -}}
23
---
34
apiVersion: apps/v1
45
kind: Deployment
@@ -13,6 +14,10 @@ spec:
1314
metadata:
1415
labels:
1516
app: {{ print $.Release.Name "-tile-server" | quote }}
17+
annotations:
18+
{{- range $s := $scripts }}
19+
{{ print "checksum-scripts-" $s | quote }}: {{ tpl ($.Files.Get (print "scripts/" $s)) $ | sha256sum }}
20+
{{- end }}
1621
spec:
1722
containers:
1823
- name: tile-server
@@ -51,31 +56,9 @@ spec:
5156
- name: tmp
5257
mountPath: /var/lock
5358
subPath: var/lock
54-
command:
55-
- sh
56-
- -c
57-
- |
58-
set -eu
59-
60-
echo "Creating style data"
61-
cp -r /home/renderer/src/openstreetmap-carto-backup/* /data/style/
62-
cd /data/style/
63-
sed -i "/dbname:/a\\
64-
host: \"$PGHOST\"\\
65-
user: \"$PGUSER\"\\
66-
password: \"$PGPASSWORD\"
67-
" /data/style/project.mml
68-
69-
carto ${NAME_MML:-project.mml} > mapnik.xml
70-
71-
echo "INFO: Starting tile server"
72-
73-
service apache2 restart
74-
75-
mkdir /run/renderd || true
76-
chown renderer /run/renderd
77-
78-
sudo -u renderer renderd -f -c /etc/renderd.conf
59+
- name: scripts-volume
60+
mountPath: /app/scripts
61+
command: [ "sh", "/app/scripts/tile-server-entrypoint.sh" ]
7962

8063
volumes:
8164
- name: tmp
@@ -84,4 +67,13 @@ spec:
8467
- name: state
8568
persistentVolumeClaim:
8669
claimName: {{ print $.Release.Name "-tile-server-data" | quote }}
70+
71+
- name: scripts-volume
72+
configMap:
73+
name: {{ print $.Release.Name "-scripts" | quote }}
74+
items:
75+
{{- range $s := $scripts }}
76+
- key: {{ $s | quote }}
77+
path: {{ $s | quote }}
78+
{{- end }}
8779
{{- end }}

‎helm/osm-machinery/templates/tile-server-scripts-configmap.yaml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
apiVersion: v1
2+
kind: ConfigMap
3+
metadata:
4+
name: {{ print $.Release.Name "-scripts" | quote }}
5+
data:
6+
{{- range $path, $_ := .Files.Glob "scripts/**" }}
7+
{{ base $path }}: |
8+
{{- tpl ($.Files.Get $path) $ | nindent 4 }}
9+
{{- end }}

‎helm/osm-machinery/templates/tile-server-sync-cronjob.yaml‎

Lines changed: 17 additions & 122 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,5 @@
11
{{- if $.Values.tileServer.syncJob.enabled -}}
2-
{{- /* Create a filtered list of regions to support */}}
3-
{{- $regions := list }}
4-
{{- range $.Values.regions }}
5-
{{- if and .enabled ( not ( .skipDownload | default false ) ) }}
6-
{{- $regions = append $regions ( dict
7-
"enabled" .enabled
8-
"name" .name
9-
"pbf" .pbf
10-
"poly" .poly
11-
) }}
12-
{{- end }}
13-
{{- end }}
14-
{{- $regionsChecksum := toJson $regions | sha256sum }}
2+
{{- $scripts := list "tile-server-sync-cronjob-entrypoint.sh" -}}
153
---
164
apiVersion: batch/v1
175
kind: CronJob
@@ -34,6 +22,10 @@ spec:
3422
metadata:
3523
labels:
3624
app: {{ print $.Release.Name "-data-sync" | quote }}
25+
annotations:
26+
{{- range $s := $scripts }}
27+
{{ print "checksum-scripts-" $s | quote }}: {{ tpl ($.Files.Get (print "scripts/" $s)) $ | sha256sum }}
28+
{{- end }}
3729
spec:
3830
securityContext: {{- $.Values.tileServer.securityContext | toYaml | nindent 12 }}
3931
containers:
@@ -61,115 +53,9 @@ spec:
6153
- name: tmp
6254
mountPath: /var/run
6355
subPath: var/run
64-
command:
65-
- sh
66-
- -c
67-
- |
68-
set -eu
69-
70-
mkdir -p /data/download/
71-
mkdir -p /data/database/
72-
mkdir -p /data/tiles/
73-
mkdir -p /data/style/
74-
75-
# Cleanup any old download files
76-
[ "$( ls -A /data/download/ )" ] && rm -rf /data/download/*
77-
78-
# Small fix to the update script to use pgdb password
79-
cp $(which openstreetmap-tiles-update-expire.sh ) /tmp/otue.sh
80-
sed -i 's/^TRIM_OPTIONS=.*$/TRIM_OPTIONS="-d $DBNAME --password"/g' /tmp/otue.sh
81-
82-
if [ z"$( cat /data/database/planet-import-complete 2>/dev/null || true )" = z"{{ $regionsChecksum }}" ]; then
83-
# The data has been imported, check if there's any diff and apply it
84-
echo "INFO: Synchronizing osm changes"
85-
86-
touch /var/log/tiles/run.log; tail -f /var/log/tiles/run.log >> /proc/1/fd/1 &
87-
touch /var/log/tiles/osmosis.log; tail -f /var/log/tiles/osmosis.log >> /proc/1/fd/1 &
88-
touch /var/log/tiles/expiry.log; tail -f /var/log/tiles/expiry.log >> /proc/1/fd/1 &
89-
touch /var/log/tiles/osm2pgsql.log; tail -f /var/log/tiles/osm2pgsql.log >> /proc/1/fd/1 &
90-
91-
sh /tmp/otue.sh
92-
93-
exit 0
94-
fi
95-
96-
# The data has not yet been imported or configuration of regions has changed,
97-
# start from scratch by importing the latest snapshot of the data
98-
99-
echo "INFO: Doing initial data import"
100-
101-
rm /data/database/planet-import-complete 2>/dev/null || true
102-
103-
[ "$( ls -A /data/database/ )" ] && rm -rf /data/database/*
104-
[ "$( ls -A /data/tiles/ )" ] && rm -rf /data/tiles/*
105-
[ "$( ls -A /data/style/ )" ] && rm -rf /data/style/*
106-
107-
echo "INFO: Creating style data"
108-
cp -r /home/renderer/src/openstreetmap-carto-backup/* /data/style/
109-
cd /data/style/
110-
carto ${NAME_MML:-project.mml} > mapnik.xml
111-
112-
{{- range $index, $region := $regions }}
113-
114-
echo "INFO: Preparing data for region {{ $region.name }}"
115-
116-
{{- $suffix := "" }}
117-
{{- if ne $index 0 }}{{ $suffix = printf "_%d" $index }}{{ end }}
118-
119-
echo "INFO: Download PBF file for region {{ $region.name }}"
120-
wget ${WGET_ARGS:-} {{ $region.pbf | quote }} -O /data/download/region{{ $suffix }}.osm.pbf
121-
122-
echo "INFO: Download POLY file for region {{ $region.name }}"
123-
wget ${WGET_ARGS:-} {{ $region.poly | quote }} -O /data/database/region{{ $suffix }}.poly
124-
125-
echo "INFO: Determining replication timestamp for region {{ $region.name }}"
126-
osmium fileinfo -g header.option.osmosis_replication_timestamp /data/download/region{{ $suffix }}.osm.pbf >> /data/download/replication_timestamps.txt
127-
128-
{{- if ne $index 0 }}
129-
130-
echo "INFO: Merging in new pbf file from region {{ $region.name }}"
131-
osmium merge /data/download/region.osm.pbf /data/download/region{{ $suffix }}.osm.pbf -o /data/download/region-merged.osm.pbf
132-
mv -f /data/download/region-merged.osm.pbf /data/download/region.osm.pbf
133-
rm -f /data/download/region{{ $suffix }}.osm.pbf
134-
135-
echo "INFO: Merging in new poly file from region {{ $region.name }}"
136-
( echo "" && cat /data/database/region{{ $suffix }}.poly ) >> /data/database/region.poly
137-
rm -f /data/database/region{{ $suffix }}.poly
138-
139-
{{- end }}
140-
{{- end }}
141-
142-
echo "INFO: Finished processing regions"
143-
144-
REPLICATION_TIMESTAMP="$(cat /data/download/replication_timestamps.txt | sort | head -n 1)"
145-
echo "INFO: Oldest replication timestamp: $REPLICATION_TIMESTAMP"
146-
147-
# initial setup of osmosis workspace (for consecutive updates)
148-
/tmp/otue.sh $REPLICATION_TIMESTAMP || true
149-
150-
# Import data
151-
osm2pgsql -d gis --create --slim -G --hstore \
152-
--tag-transform-script /data/style/${NAME_LUA:-openstreetmap-carto.lua} \
153-
--number-processes ${THREADS:-4} \
154-
-S /data/style/${NAME_STYLE:-openstreetmap-carto.style} \
155-
/data/download/region.osm.pbf \
156-
${OSM2PGSQL_EXTRA_ARGS:-} \
157-
;
158-
159-
rm -rf /data/download/*
160-
161-
# Create indexes
162-
if [ -f "/data/style/${NAME_SQL:-indexes.sql}" ]; then
163-
psql -d gis -f "/data/style/${NAME_SQL:-indexes.sql}"
164-
fi
165-
166-
# Import external data
167-
python3 /data/style/scripts/get-external-data.py -c /data/style/external-data.yml -D /data/style/data
168-
169-
# Mark the import as done
170-
echo -n "{{ $regionsChecksum }}" > /data/database/planet-import-complete
171-
172-
echo "INFO: Initialization complete!"
56+
- name: scripts-volume
57+
mountPath: /app/scripts
58+
command: [ "sh", "/app/scripts/tile-server-sync-cronjob-entrypoint.sh" ]
17359

17460
volumes:
17561
- name: tmp
@@ -179,6 +65,15 @@ spec:
17965
persistentVolumeClaim:
18066
claimName: {{ print $.Release.Name "-tile-server-data" | quote }}
18167

68+
- name: scripts-volume
69+
configMap:
70+
name: {{ print $.Release.Name "-scripts" | quote }}
71+
items:
72+
{{- range $s := $scripts }}
73+
- key: {{ $s | quote }}
74+
path: {{ $s | quote }}
75+
{{- end }}
76+
18277
# Do not restart failed jobs - we will reschedule quickly anyway
18378
restartPolicy: Never
18479
{{- end -}}

0 commit comments

Comments
 (0)