Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 6 additions & 8 deletions .github/workflows/build-manifests.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
name: Fetch releases from S3
name: Build and publish releases artifacts

on:
push:
Expand Down Expand Up @@ -33,30 +33,28 @@ jobs:
with:
persist-credentials: false

- name: Set up Python 3.11
- name: Set up Python 3.12
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
with:
python-version: "3.11"
python-version: "3.12"

- name: Install dependencies
run: |
python -m pip install --upgrade pip
cd utils
pip install -r requirements.txt

- name: Build releases.json and latest.dbb
- name: Build releases.json and latest.ddb
run: |
cd utils
python3 fetch-releases-from-s3.py
python3 simple-registry-manifest.py
python3 fetch_releases_from_stac.py

- name: Copy output to publish directory
run: |
mkdir publish
cp 404.html publish/
cp utils/releases.json publish/
cp utils/registry-manifest.json publish/
cp utils/latest.ddb publish/
cp utils/latest.ddb publish/latest.dbb
cp overture_releases.yaml publish/

- name: Setup Pages
Expand Down
39 changes: 39 additions & 0 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
name: Test utils

on:
push:
branches: main
pull_request:

permissions:
contents: read

concurrency:
group: "test-${{ github.ref }}"
cancel-in-progress: true

jobs:
test:
name: Run unit tests
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
persist-credentials: false

- name: Set up Python 3.12
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
with:
python-version: "3.12"

- name: Install dependencies
run: |
python -m pip install --upgrade pip
cd utils
pip install -r requirements-test.txt

- name: Run tests
run: |
cd utils
python -m pytest tests/ -v
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@

__pycache__/
*.pyc
99 changes: 99 additions & 0 deletions 404.html
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta http-equiv="refresh" content="5;url=https://github.com/OvertureMaps/data#release-discovery">
<title>Moved | Overture Maps Data</title>
<link rel="preconnect" href="https://fonts.googleapis.com">
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
<link href="https://fonts.googleapis.com/css2?family=Montserrat:wght@300;400;700&display=swap" rel="stylesheet">
<style>
:root {
--omf-navy: #2C2E7F;
--omf-teal: #0EC1BD;
--omf-blue: #4051CC;
--omf-cyan: #4EDAD8;
--omf-dark: #001A39;
--omf-white: #FFFFFF;
}

* { box-sizing: border-box; margin: 0; padding: 0; }

body {
font-family: 'Montserrat', Arial, sans-serif;
background: var(--omf-dark);
color: var(--omf-white);
min-height: 100vh;
display: flex;
flex-direction: column;
align-items: center;
justify-content: center;
padding: 2rem;
text-align: center;
}

h1 {
font-weight: 700;
font-size: 0.8rem;
letter-spacing: 0.12em;
text-transform: uppercase;
color: var(--omf-cyan);
margin-bottom: 1.25rem;
}

p {
font-weight: 300;
font-size: 0.95rem;
line-height: 1.7;
max-width: 480px;
color: var(--omf-white);
margin-bottom: 0.75rem;
}

a {
color: var(--omf-teal);
text-decoration: none;
font-weight: 400;
}

a:hover { text-decoration: underline; color: var(--omf-cyan); }

.divider {
width: 40px;
height: 2px;
background: var(--omf-teal);
margin: 0 auto 1.5rem;
}

footer {
position: fixed;
bottom: 1.25rem;
font-size: 0.7rem;
font-weight: 300;
color: var(--omf-cyan);
opacity: 0.6;
}

footer a { color: inherit; }
</style>
</head>
<body>

<h1>This resource has moved</h1>
<div class="divider"></div>

<p>
The file you requested is no longer published here. Release discovery is
now handled by the
<a href="https://stac.overturemaps.org/catalog.json">Overture STAC catalog</a>
(SpatioTemporal Asset Catalog).
</p>
<p>
See the <a href="https://github.com/OvertureMaps/data#release-discovery">data repo README</a>
for details. Redirecting in 5 seconds&hellip;
</p>

<footer><a href="https://overturemaps.org">overturemaps.org</a></footer>

</body>
</html>
6 changes: 6 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -49,3 +49,9 @@ for each theme with the following enhancements:
Data Release Feedback
---
We are very interested in feedback on the Overture data. Please use the [Discussion](https://github.com/OvertureMaps/data/discussions) section of this repo to comment. Tagging it with the release and relevant theme name (Places, Transportation) will help direct your ideas. Please include as much detail as possible. The associated Task Force will carefully review each submission and offer feedback where required.

Release Discovery
---
Use the [Overture STAC catalog](https://stac.overturemaps.org/catalog.json) (SpatioTemporal Asset Catalog) for authoritative release discovery.

> **Deprecated:** `overture_releases.yaml`, `releases.json`, and `registry-manifest.json` (previously at `labs.overturemaps.org/data/`) are no longer maintained and will be removed in a future release. Use the STAC catalog instead.
3 changes: 3 additions & 0 deletions overture_releases.yaml
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
# DEPRECATED: This file is no longer maintained and will be removed in a future release.
# Use the Overture STAC catalog for authoritative release discovery:
# https://stac.overturemaps.org/catalog.json
- schema: "1.16.0"
release: "2026-04-15.0"
- schema: "1.16.0"
Expand Down
89 changes: 0 additions & 89 deletions utils/fetch-releases-from-s3.py

This file was deleted.

87 changes: 87 additions & 0 deletions utils/fetch_releases_from_stac.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
import json
import urllib.request
from urllib.parse import urlparse

import duckdb

STAC_CATALOG = "https://stac.overturemaps.org/catalog.json"
S3_BASE = "s3://overturemaps-us-west-2/release"
_USER_AGENT = "overturemaps-data/1.0"

VIEWS = [
("address", "addresses", "address"),
("bathymetry", "base", "bathymetry"),
("building", "buildings", "building"),
("building_part", "buildings", "building_part"),
("connector", "transportation", "connector"),
("division", "divisions", "division"),
("division_area", "divisions", "division_area"),
("division_boundary", "divisions", "division_boundary"),
("infrastructure", "base", "infrastructure"),
("land", "base", "land"),
("land_cover", "base", "land_cover"),
("land_use", "base", "land_use"),
("place", "places", "place"),
("segment", "transportation", "segment"),
("water", "base", "water"),
]


def fetch_catalog(url: str, timeout: int = 30) -> dict:
req = urllib.request.Request(url, headers={"User-Agent": _USER_AGENT})
with urllib.request.urlopen(req, timeout=timeout) as response:
return json.loads(response.read())


def _release_id_from_href(href: str) -> str:
parts = [p for p in urlparse(href).path.split("/") if p and p != "."]
return parts[0]


def parse_releases(catalog: dict) -> dict:
latest = catalog["latest"]
releases = sorted(
[
_release_id_from_href(link["href"])
for link in catalog["links"]
if link["rel"] == "child"
],
reverse=True,
)
return {"latest": latest, "releases": releases}


def build_views_sql(latest: str, s3_base: str = S3_BASE) -> str:
stmts = ["INSTALL spatial;", "LOAD spatial;"]
for view_name, theme, type_ in VIEWS:
path = f"{s3_base}/{latest}/theme={theme}/type={type_}/*.parquet"
stmts.append(
f"CREATE OR REPLACE VIEW {view_name} AS (\n"
f" SELECT * FROM read_parquet('{path}')\n);"
)
return "\n\n".join(stmts)


def create_duckdb_views(db_path: str, latest: str, s3_base: str = S3_BASE) -> None:
conn = duckdb.connect(db_path)
try:
conn.sql(build_views_sql(latest, s3_base))
finally:
conn.close()


def main():
catalog = fetch_catalog(STAC_CATALOG)
output = parse_releases(catalog)

for release in output["releases"]:
print(f" - {release}")

with open("releases.json", "w") as f:
f.write(json.dumps(output, indent=4))

create_duckdb_views("latest.ddb", output["latest"])


if __name__ == "__main__":
main()
2 changes: 2 additions & 0 deletions utils/requirements-test.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
-r requirements.txt
pytest>=8.0.0
2 changes: 0 additions & 2 deletions utils/requirements.txt
Comment thread
lowlydba marked this conversation as resolved.
Original file line number Diff line number Diff line change
@@ -1,3 +1 @@
obstore>=0.7.0
duckdb==1.3.2
pyarrow>=20.0.0
Loading
Loading