diff --git a/.gitignore b/.gitignore index 12d6b139..0ae5d8eb 100644 --- a/.gitignore +++ b/.gitignore @@ -121,3 +121,13 @@ infrastructure/aws/browser_config.js # imported docs docs/src/deployment/kubernetes/ eoapi-k8s/ + +# Maxar notebook downloads +demo/Maxar/collections.json +demo/Maxar/collections.json.* +demo/Maxar/items.json +demo/Maxar/items.json.zip + +# OAM notebook generated STAC files +demo/oam/oam_collection.json +demo/oam/oam_items.njson diff --git a/demo/Maxar/eoAPI_Maxar_demo.ipynb b/demo/Maxar/eoAPI_Maxar_demo.ipynb index 0d103c0b..8c375ef1 100644 --- a/demo/Maxar/eoAPI_Maxar_demo.ipynb +++ b/demo/Maxar/eoAPI_Maxar_demo.ipynb @@ -68,7 +68,8 @@ }, "outputs": [], "source": [ - "!python -m pip install \"pypgstac[psycopg]==0.9.2\"" + "# Dependencies are installed in the demo Docker image.\n", + "# Rebuild with `docker compose build demo-runner` after changing demo/requirements.txt.\n" ] }, { @@ -79,7 +80,7 @@ "outputs": [], "source": [ "# Download the collections file\n", - "!wget https://github.com/vincentsarago/MAXAR_opendata_to_pgstac/raw/main/Maxar/collections.json" + "!wget -q -O collections.json https://github.com/vincentsarago/MAXAR_opendata_to_pgstac/raw/main/Maxar/collections.json" ] }, { @@ -90,7 +91,7 @@ "outputs": [], "source": [ "# Download the items file\n", - "! wget https://github.com/vincentsarago/MAXAR_opendata_to_pgstac/raw/main/Maxar/items.json.zip && unzip items.json.zip && rm -rf items.json.zip" + "!wget -q -O items.json.zip https://github.com/vincentsarago/MAXAR_opendata_to_pgstac/raw/main/Maxar/items.json.zip && unzip -o items.json.zip && rm -f items.json.zip" ] }, { @@ -103,18 +104,18 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "id": "af16cb95", "metadata": {}, "outputs": [], "source": [ "# Ingest the collections\n", - "!pypgstac load collections collections.json --dsn postgresql://username:password@0.0.0.0:5439/postgis --method insert_ignore" + "!pypgstac load collections collections.json --dsn postgresql://username:password@database:5432/postgis --method insert_ignore" ] }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "id": "f7ded3bc", "metadata": { "scrolled": true @@ -122,7 +123,7 @@ "outputs": [], "source": [ "# Ingest the items\n", - "!pypgstac load items items.json --dsn postgresql://username:password@0.0.0.0:5439/postgis --method insert_ignore" + "!pypgstac load items items.json --dsn postgresql://username:password@database:5432/postgis --method insert_ignore" ] }, { @@ -139,7 +140,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "id": "dc3fa136", "metadata": {}, "outputs": [], @@ -148,7 +149,7 @@ "from psycopg import sql\n", "\n", "with psycopg.connect(\n", - " \"postgresql://username:password@0.0.0.0:5439/postgis\", \n", + " \"postgresql://username:password@database:5432/postgis\", \n", " autocommit=True,\n", " options=\"-c search_path=pgstac,public -c application_name=pgstac\",\n", ") as conn: \n", @@ -183,7 +184,7 @@ }, "outputs": [], "source": [ - "!python -m pip install httpx ipyleaflet" + "# httpx and ipyleaflet are installed in the demo Docker image.\n" ] }, { @@ -196,11 +197,12 @@ "from datetime import datetime\n", "\n", "import json\n", + "import os\n", "import httpx\n", "\n", "import ipyleaflet\n", "\n", - "stac_endpoint = \"http://127.0.0.1:8081\"" + "stac_endpoint = os.environ.get(\"STAC_API_URL\", \"http://127.0.0.1:8081\")\n" ] }, { @@ -339,9 +341,12 @@ "source": [ "items = httpx.get(f\"{stac_endpoint}/collections/{collection_id}/items\").json()\n", "\n", - "\n", - "print(f\"Nb Items in Db: {items['context']['matched']}\") # This is only available if CONTEXT=ON\n", - "print(f\"Returned {len(items['features'])} Items\")" + "matched = items.get(\"context\", {}).get(\"matched\")\n", + "if matched is None:\n", + " print(\"Nb Items in Db: unavailable; response has no STAC context\")\n", + "else:\n", + " print(f\"Nb Items in Db: {matched}\")\n", + "print(f\"Returned {len(items.get('features', []))} Items\")" ] }, { @@ -359,20 +364,27 @@ "metadata": {}, "outputs": [], "source": [ + "max_items = int(os.environ.get(\"MAXAR_MAX_ITEMS\", \"250\"))\n", + "page_size = min(max_items, 100)\n", "kahramanmaras_items = []\n", "\n", "url = f\"{stac_endpoint}/collections/{collection_id}/items\"\n", - "while True:\n", - " items = httpx.get(url, params={\"limit\": 100}).json()\n", - " \n", - " kahramanmaras_items.extend(items[\"features\"])\n", - " next_link = list(filter(lambda link: link[\"rel\"] == \"next\", items[\"links\"]))\n", + "while len(kahramanmaras_items) < max_items:\n", + " remaining = max_items - len(kahramanmaras_items)\n", + " items = httpx.get(url, params={\"limit\": min(page_size, remaining)}, timeout=30).json()\n", + " features = items.get(\"features\", [])\n", + " if not features:\n", + " break\n", + "\n", + " kahramanmaras_items.extend(features)\n", + " next_link = list(filter(lambda link: link[\"rel\"] == \"next\", items.get(\"links\", [])))\n", " if next_link:\n", " url = next_link[0][\"href\"]\n", " else:\n", " break\n", "\n", - "print(f\"Nb Items: {len(kahramanmaras_items)}\")" + "print(f\"Nb Items loaded for this demo run: {len(kahramanmaras_items)}\")\n", + "print(f\"MAXAR_MAX_ITEMS={max_items}\")" ] }, { @@ -424,7 +436,7 @@ "outputs": [], "source": [ "item = kahramanmaras_items[0]\n", - "print(\"Item example:\")\n", + "print(\"Initial item example:\")\n", "print(json.dumps(item, indent=4))" ] }, @@ -528,7 +540,11 @@ " )\n", ").json()\n", "\n", - "print(f\"Nb Items in Db: {pre_items_api['context']['matched']}\") # This is only available if CONTEXT=ON" + "matched = pre_items_api.get(\"context\", {}).get(\"matched\")\n", + "if matched is None:\n", + " print(\"Nb Items in Db: unavailable; response has no STAC context\")\n", + "else:\n", + " print(f\"Nb Items in Db: {matched}\")" ] }, { @@ -575,7 +591,33 @@ "metadata": {}, "outputs": [], "source": [ - "raster_endpoint = \"http://127.0.0.1:8082\"" + "raster_endpoint = os.environ.get(\"TITILER_URL\", \"http://127.0.0.1:8082\")\n", + "raster_public_endpoint = os.environ.get(\"TITILER_PUBLIC_URL\", \"http://127.0.0.1:8082\")\n", + "\n", + "def public_tilejson(tilejson):\n", + " if \"tiles\" not in tilejson:\n", + " raise KeyError(f\"TileJSON response has no tiles: {tilejson}\")\n", + " tilejson[\"tiles\"] = [tile.replace(raster_endpoint, raster_public_endpoint) for tile in tilejson[\"tiles\"]]\n", + " return tilejson\n", + "\n", + "def tilejson_urls(base_url):\n", + " # titiler-pgstac versions differ: newer OGC tile routes include WebMercatorQuad.\n", + " return (\n", + " f\"{base_url}/WebMercatorQuad/tilejson.json\",\n", + " f\"{base_url}/tilejson.json\",\n", + " )\n", + "\n", + "def get_tilejson(base_url, params):\n", + " errors = []\n", + " for url in tilejson_urls(base_url):\n", + " resp = httpx.get(url, params=params, timeout=60)\n", + " if resp.status_code == 200:\n", + " tilejson = resp.json()\n", + " if \"tiles\" not in tilejson:\n", + " raise RuntimeError(f\"TileJSON response has no tiles from {url}: {tilejson}\")\n", + " return public_tilejson(tilejson)\n", + " errors.append(f\"{url} -> HTTP {resp.status_code}: {resp.text[:200]}\")\n", + " raise RuntimeError(\"TileJSON request failed for all known routes: \" + \" | \".join(errors))\n" ] }, { @@ -585,18 +627,74 @@ "metadata": {}, "outputs": [], "source": [ - "# fetching Raster information for all the `raster` assets\n", - "item_id = item[\"id\"]\n", + "# Find the first sampled item that TiTiler can inspect and tile.\n", + "info = None\n", + "info_resp = None\n", + "asset_name = None\n", + "selected_tilejson = None\n", + "\n", + "for candidate in kahramanmaras_items:\n", + " candidate_id = candidate[\"id\"]\n", + " print(f\"Checking raster info for Item {candidate_id}\")\n", + " try:\n", + " candidate_resp = httpx.get(\n", + " f\"{raster_endpoint}/collections/{collection_id}/items/{candidate_id}/info\",\n", + " timeout=60,\n", + " )\n", + " except httpx.HTTPError as exc:\n", + " print(f\"Skipping {candidate_id}: {exc}\")\n", + " continue\n", + "\n", + " if candidate_resp.status_code != 200:\n", + " print(f\"Skipping {candidate_id}: HTTP {candidate_resp.status_code} {candidate_resp.text[:200]}\")\n", + " continue\n", + "\n", + " candidate_info = candidate_resp.json()\n", + " if not candidate_info:\n", + " print(f\"Skipping {candidate_id}: no raster assets returned\")\n", + " continue\n", + "\n", + " raster_asset_names = [name for name, asset in candidate_info.items() if \"bounds\" in asset or \"minzoom\" in asset or \"maxzoom\" in asset]\n", + " if \"visual\" in raster_asset_names:\n", + " raster_asset_names.remove(\"visual\")\n", + " raster_asset_names.insert(0, \"visual\")\n", + "\n", + " for candidate_asset_name in raster_asset_names:\n", + " try:\n", + " candidate_tilejson = get_tilejson(\n", + " f\"{raster_endpoint}/collections/{collection_id}/items/{candidate_id}\",\n", + " params=(\n", + " (\"assets\", candidate_asset_name),\n", + " (\"minzoom\", 12),\n", + " (\"maxzoom\", 19),\n", + " ),\n", + " )\n", + " except Exception as exc:\n", + " print(f\"Skipping {candidate_id}/{candidate_asset_name}: {exc}\")\n", + " continue\n", + "\n", + " item = candidate\n", + " item_id = candidate_id\n", + " info = candidate_info\n", + " info_resp = candidate_resp\n", + " asset_name = candidate_asset_name\n", + " selected_tilejson = candidate_tilejson\n", + " break\n", "\n", - "print(f\"Fetching Raster info for Item {item_id}\")\n", - "info = httpx.get(f\"{raster_endpoint}/collections/{collection_id}/items/{item_id}/info\").json()\n", + " if selected_tilejson is not None:\n", + " break\n", + "\n", + "if info is None:\n", + " raise RuntimeError(\"No sampled Maxar item returned usable raster tilejson. Increase MAXAR_MAX_ITEMS or inspect TiTiler logs.\")\n", "\n", + "print(f\"Using Item: {item_id}\")\n", "print(\"Returned metadata for Assets:\", list(info.keys()))\n", "print()\n", - "print(json.dumps(info[\"visual\"], indent=4))\n", + "print(f\"Using asset: {asset_name}\")\n", + "print(json.dumps(info[asset_name], indent=4))\n", "print()\n", "for name, asset in info.items():\n", - " print(name, asset[\"minzoom\"], asset[\"maxzoom\"])" + " print(name, asset.get(\"minzoom\", \"?\"), asset.get(\"maxzoom\", \"?\"))" ] }, { @@ -623,15 +721,8 @@ "metadata": {}, "outputs": [], "source": [ - "# `visual` Asset\n", - "tilejson = httpx.get(\n", - " f\"{raster_endpoint}/collections/{collection_id}/items/{item_id}/tilejson.json\",\n", - " params = (\n", - " (\"assets\", \"visual\"), # THIS PARAMETER IS MANDATORY\n", - " (\"minzoom\", 12), # By default the tiler will use 0\n", - " (\"maxzoom\", 19), # By default the tiler will use 24\n", - " )\n", - ").json()\n", + "# Selected raster asset\n", + "tilejson = selected_tilejson\n", "print(tilejson)\n", "\n", "bounds = tilejson[\"bounds\"]\n", @@ -849,16 +940,15 @@ "source": [ "search_id = pre_mosaic[\"id\"]\n", "\n", - "tilejson_pre = httpx.get(\n", - " f\"{raster_endpoint}/searches/{search_id}/tilejson.json\",\n", - " params = (\n", - " (\"assets\", \"visual\"), # THIS IS MANDATORY\n", + "tilejson_pre = get_tilejson(\n", + " f\"{raster_endpoint}/searches/{search_id}\",\n", + " params=(\n", + " (\"assets\", asset_name), # THIS IS MANDATORY\n", " (\"minzoom\", 12),\n", " (\"maxzoom\", 19), \n", - " )\n", - ").json()\n", + " ),\n", + ")\n", "print(tilejson_pre)\n", - "\n", "bounds = tilejson_pre[\"bounds\"]\n", "m = ipyleaflet.leaflet.Map(\n", " center=((bounds[1] + bounds[3]) / 2,(bounds[0] + bounds[2]) / 2),\n", @@ -898,16 +988,15 @@ "source": [ "search_id = post_mosaic[\"id\"]\n", "\n", - "tilejson_post = httpx.get(\n", - " f\"{raster_endpoint}/searches/{search_id}/tilejson.json\",\n", - " params = (\n", - " (\"assets\", \"visual\"), # THIS IS MANDATORY\n", + "tilejson_post = get_tilejson(\n", + " f\"{raster_endpoint}/searches/{search_id}\",\n", + " params=(\n", + " (\"assets\", asset_name), # THIS IS MANDATORY\n", " (\"minzoom\", 12),\n", " (\"maxzoom\", 19), \n", - " )\n", - ").json()\n", + " ),\n", + ")\n", "print(tilejson_post)\n", - "\n", "bounds = tilejson_post[\"bounds\"]\n", "m = ipyleaflet.leaflet.Map(\n", " center=((bounds[1] + bounds[3]) / 2,(bounds[0] + bounds[2]) / 2),\n", @@ -1005,6 +1094,28 @@ "\n", "Thank you for taking the time to go through this notebook." ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "364ca12f-993a-4657-961a-e6835caf677d", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a6fc5746-03ff-4ba0-8cc6-76ff77a99d69", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [], + "source": [] } ], "metadata": { @@ -1023,7 +1134,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.2" + "version": "3.11.15" } }, "nbformat": 4, diff --git a/demo/README.md b/demo/README.md new file mode 100644 index 00000000..12b0d7c5 --- /dev/null +++ b/demo/README.md @@ -0,0 +1,63 @@ +# Demo Runner + +The demos can be run from a Docker Compose service instead of from your host +Python environment. The container includes Jupyter, `pypgstac`, `rio-stac`, +AWS/S3 clients, and the Python libraries used by the notebooks. + +Start the main stack and the demo notebook server: + +```bash +docker compose --profile demo up -d --build demo-runner +``` + +Open Jupyter at: + +```text +http://127.0.0.1:8888 +``` + +The repository's `demo/` directory is bind-mounted at `/workspace/demo` in the +demo containers, so notebook edits and generated demo files are visible without +rebuilding the image. + +Load demo data into the local pgSTAC database with the named Compose targets: + +```bash +docker compose run --rm demo-noaa +docker compose run --rm demo-facebook +docker compose run --rm demo-cmip6 +docker compose run --rm demo-oam +``` + +Load every available demo dataset: + +```bash +docker compose run --rm demo-all +``` + +You can also call the loader directly from the notebook image: + +```bash +docker compose run --rm demo-runner load-demos noaa facebook +``` + +`all` loads the static NOAA and Facebook data and also loads generated CMIP6 +or OAM data if the corresponding item files already exist. Generate those files +from their notebooks first, or use the existing checked-out generated files when +available. + +Inside the demo container, use Compose service URLs: + +```text +DATABASE_URL=postgresql://username:password@database:5432/postgis +STAC_API_URL=http://stac-fastapi:8080 +TITILER_URL=http://titiler-pgstac +``` + +From your host browser, keep using the published ports: + +```text +STAC API: http://127.0.0.1:8081 +TiTiler: http://127.0.0.1:8082 +Browser: http://127.0.0.1:8085 +``` diff --git a/demo/cmip6/generate_cmip6_items.ipynb b/demo/cmip6/generate_cmip6_items.ipynb index 4b091898..c0380ee8 100644 --- a/demo/cmip6/generate_cmip6_items.ipynb +++ b/demo/cmip6/generate_cmip6_items.ipynb @@ -14,25 +14,30 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "6f788363", "metadata": {}, "outputs": [], "source": [ "import boto3\n", + "import os\n", "import fsspec\n", "import json\n", "from pystac import Catalog, Collection, Item, Asset, MediaType\n", "from datetime import datetime\n", "import rio_stac\n", + "import rasterio\n", "from pprint import pprint\n", "import concurrent.futures\n", - "import threading" + "import threading\n", + "# Use unsigned requests for public S3 buckets when rasterio/GDAL opens s3:// URLs.\n", + "os.environ.setdefault(\"AWS_NO_SIGN_REQUEST\", \"YES\")\n", + "os.environ.setdefault(\"CPL_VSIL_CURL_ALLOWED_EXTENSIONS\", \".tif,.tiff,.vrt\")\n" ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "8e4cfbb8", "metadata": { "tags": [ @@ -56,7 +61,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "cdaccd78", "metadata": {}, "outputs": [], @@ -67,7 +72,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "a7caab29", "metadata": {}, "outputs": [], @@ -77,18 +82,10 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "4936f757", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "23725 discovered from s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/\n" - ] - } - ], + "outputs": [], "source": [ "file_paths = fs_read.glob(f\"{s3_path}*\")\n", "print(f\"{len(file_paths)} discovered from {s3_path}\")" @@ -106,7 +103,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "999b0670", "metadata": {}, "outputs": [], @@ -125,18 +122,10 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "id": "6af269dc", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Subseted data to files for 1950 and 1951. 730 files to process.\n" - ] - } - ], + "outputs": [], "source": [ "if len(subset_files) == 0:\n", " raise Exception(f\"No files to process. Do COGs for the {model} model exist?\")\n", @@ -144,6 +133,23 @@ " print(f\"Subseted data to files for 1950 and 1951. {len(subset_files)} files to process.\")" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "958cc7d5-4849-44b7-80c2-ff3ed71bea2e", + "metadata": {}, + "outputs": [], + "source": [ + "sizes = [fs_read.info(path)[\"size\"] for path in subset_files]\n", + "total = sum(sizes)\n", + "\n", + "print(f\"{len(sizes)} files\")\n", + "print(f\"Total: {total / 1024**3:.2f} GiB\")\n", + "print(f\"Average: {total / len(sizes) / 1024**2:.2f} MiB\")\n", + "print(f\"Min: {min(sizes) / 1024**2:.2f} MiB\")\n", + "print(f\"Max: {max(sizes) / 1024**2:.2f} MiB\")" + ] + }, { "cell_type": "markdown", "id": "ea59aceb-b80a-4166-a684-74de4230ac4a", @@ -156,7 +162,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "id": "57dc1b5f", "metadata": {}, "outputs": [], @@ -169,7 +175,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "id": "45771e88", "metadata": { "lines_to_next_cell": 1 @@ -183,7 +189,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "id": "ececf9d5", "metadata": {}, "outputs": [], @@ -195,17 +201,19 @@ " day = day.replace('.tif', '')\n", " datetime_ = datetime.strptime(f'{year}{month}{day}', '%Y%m%d') \n", " # Create a new Item\n", - " item = rio_stac.create_stac_item(\n", - " id=filename,\n", - " source=s3_file,\n", - " collection=collection.id,\n", - " input_datetime=datetime_,\n", - " with_proj=True,\n", - " with_raster=True,\n", - " asset_name=\"data\",\n", - " asset_roles=[\"data\"],\n", - " asset_media_type=\"image/tiff; application=geotiff; profile=cloud-optimized\"\n", - " )\n", + " with rasterio.Env(AWS_NO_SIGN_REQUEST=\"YES\"):\n", + "\n", + " item = rio_stac.create_stac_item(\n", + " id=filename,\n", + " source=s3_file,\n", + " collection=collection.id,\n", + " input_datetime=datetime_,\n", + " with_proj=True,\n", + " with_raster=True,\n", + " asset_name=\"data\",\n", + " asset_roles=[\"data\"],\n", + " asset_media_type=\"image/tiff; application=geotiff; profile=cloud-optimized\"\n", + " )\n", " tiling_asset = Asset(\n", " href=s3_file,\n", " roles=['virtual', 'tiling'],\n", @@ -231,19 +239,10 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "id": "645d3ccb", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_01_01.tif\n", - "Processing s3://nex-gddp-cmip6-cog/daily/GISS-E2-1-G/historical/r1i1p1f2/tas/tas_day_GISS-E2-1-G_historical_r1i1p1f2_gn_1950_01_02.tif\n" - ] - } - ], + "outputs": [], "source": [ "lock = threading.Lock()\n", "file = open(stac_items_file, 'a')\n", @@ -263,23 +262,21 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "id": "6965e650-f89a-4c7d-9f41-11774a905b81", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "postgresql://postgres:password@localhost:5432/postgres\n", - "Inserting collection from CMIP6_daily_GISS-E2-1-G_tas_collection.json\n", - "Inserting items from CMIP6_daily_GISS-E2-1-G_tas_stac_items.ndjson\n" - ] - } - ], + "outputs": [], "source": [ "!./seed-db.sh {model} {variable}" ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "36ec6a31-c5b6-4d4a-8dbf-d314a5132e37", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { @@ -303,7 +300,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.12" + "version": "3.11.15" } }, "nbformat": 4, diff --git a/demo/cmip6/seed-db.sh b/demo/cmip6/seed-db.sh index 5f6166d7..bdc2267f 100755 --- a/demo/cmip6/seed-db.sh +++ b/demo/cmip6/seed-db.sh @@ -6,10 +6,10 @@ collection_json_file="CMIP6_daily_${model}_${variable}_collection.json" items_json_file="CMIP6_daily_${model}_${variable}_stac_items.ndjson" if [ -z "$DATABASE_URL" ]; then - username=postgres + username=username password=password - host=localhost - dbname=postgres + host=database + dbname=postgis port=5432 DATABASE_URL="postgresql://$username:$password@$host:$port/$dbname" fi diff --git a/demo/facebook/README.md b/demo/facebook/README.md index d18fb467..aa5e46ed 100644 --- a/demo/facebook/README.md +++ b/demo/facebook/README.md @@ -39,7 +39,7 @@ Note: We also recommend to use simpler item ID than the basename. "interval": [ [ "2016-01-01T00:00:00Z", - "null" + null ] ] } diff --git a/demo/facebook/demo.ipynb b/demo/facebook/demo.ipynb index 569c6477..a563f00f 100644 --- a/demo/facebook/demo.ipynb +++ b/demo/facebook/demo.ipynb @@ -5,10 +5,12 @@ "execution_count": null, "source": [ "import json\n", + "import os\n", "import requests\n", "from folium import Map, TileLayer, GeoJson\n", "\n", - "endpoint = \"\"" + "endpoint = os.environ.get(\"TITILER_URL\", \"http://127.0.0.1:8082\")\n", + "public_endpoint = os.environ.get(\"TITILER_PUBLIC_URL\", \"http://127.0.0.1:8082\")" ], "outputs": [], "metadata": {} @@ -29,11 +31,13 @@ " \"collections\": [\"facebook-population-density\"],\n", "}\n", "\n", - "response = requests.post(\n", - " f\"{endpoint}/mosaic/register\",\n", + "register_resp = requests.post(\n", + " f\"{endpoint}/searches/register\",\n", " json=body,\n", - ").json()\n", - "print(r)" + ")\n", + "register_resp.raise_for_status()\n", + "response = register_resp.json()\n", + "print(response)" ], "outputs": [], "metadata": {} @@ -48,8 +52,15 @@ ")\n", "\n", "# Fetch Tilejson (we HAVE TO add the asset name)\n", - "tj_resp = requests.get(\n", - " response['url'],\n", + "search_id = response.get(\"id\")\n", + "tilejson_url = response.get(\"url\")\n", + "if tilejson_url is None:\n", + " if search_id is None:\n", + " raise KeyError(f\"Search registration response has no id or url: {response}\")\n", + " tilejson_url = f\"{endpoint}/searches/{search_id}/WebMercatorQuad/tilejson.json\"\n", + "\n", + "tilejson_resp = requests.get(\n", + " tilejson_url,\n", " params={\n", " # Info to add to the tilejson response\n", " \"minzoom\": 4,\n", @@ -59,7 +70,22 @@ " \"rescale\": \"0,100\",\n", " \"colormap_name\": \"viridis\",\n", " }\n", - ").json()\n", + ")\n", + "if tilejson_resp.status_code == 404 and search_id is not None:\n", + " tilejson_url = f\"{endpoint}/searches/{search_id}/tilejson.json\"\n", + " tilejson_resp = requests.get(\n", + " tilejson_url,\n", + " params={\n", + " \"minzoom\": 4,\n", + " \"maxzoom\": 12,\n", + " \"assets\": \"cog\",\n", + " \"rescale\": \"0,100\",\n", + " \"colormap_name\": \"viridis\",\n", + " }\n", + " )\n", + "tilejson_resp.raise_for_status()\n", + "tj_resp = tilejson_resp.json()\n", + "tj_resp[\"tiles\"] = [tile.replace(endpoint, public_endpoint) for tile in tj_resp[\"tiles\"]]\n", "print(tj_resp)\n", "\n", "aod_layer = TileLayer(\n", @@ -106,4 +132,4 @@ }, "nbformat": 4, "nbformat_minor": 4 -} \ No newline at end of file +} diff --git a/demo/facebook/facebook_collection.json b/demo/facebook/facebook_collection.json index 7d758a6a..6ae74375 100644 --- a/demo/facebook/facebook_collection.json +++ b/demo/facebook/facebook_collection.json @@ -1 +1 @@ -{"id":"facebook-population-density","title":"High Resolution Population Density Maps","description":"Population data for a selection of countries, allocated to 1 arcsecond blocks and provided in a combination of CSV and Cloud-optimized GeoTIFF files. This refines CIESIN’s Gridded Population of the World using machine learning models on high-resolution worldwide Digital Globe satellite imagery. CIESIN population counts aggregated from worldwide census data are allocated to blocks where imagery appears to contain buildings.","stac_version":"1.0.0","license":"public-domain","links":[],"extent":{"spatial":{"bbox":[[-180,-90,180,90]]},"temporal":{"interval":[["2016-01-01T00:00:00Z","null"]]}}} +{"id":"facebook-population-density","title":"High Resolution Population Density Maps","description":"Population data for a selection of countries, allocated to 1 arcsecond blocks and provided in a combination of CSV and Cloud-optimized GeoTIFF files. This refines CIESIN’s Gridded Population of the World using machine learning models on high-resolution worldwide Digital Globe satellite imagery. CIESIN population counts aggregated from worldwide census data are allocated to blocks where imagery appears to contain buildings.","stac_version":"1.0.0","license":"public-domain","links":[],"extent":{"spatial":{"bbox":[[-180,-90,180,90]]},"temporal":{"interval":[["2016-01-01T00:00:00Z",null]]}}} diff --git a/demo/load-demos.sh b/demo/load-demos.sh new file mode 100755 index 00000000..3a4ad5e2 --- /dev/null +++ b/demo/load-demos.sh @@ -0,0 +1,154 @@ +#!/bin/sh +set -eu + +DEMO_ROOT="${DEMO_ROOT:-/workspace/demo}" +DATABASE_URL="${DATABASE_URL:-postgresql://username:password@database:5432/postgis}" + +load_pair() { + name="$1" + collection_file="$2" + items_file="$3" + + if [ ! -f "$collection_file" ]; then + echo "Skipping $name: missing collection file $collection_file" + return 0 + fi + + if [ ! -f "$items_file" ]; then + echo "Skipping $name: missing items file $items_file" + return 0 + fi + + echo "Loading $name collection from $collection_file" + pypgstac load collections "$collection_file" --dsn "$DATABASE_URL" --method insert_ignore + + echo "Loading $name items from $items_file" + pypgstac load items "$items_file" --dsn "$DATABASE_URL" --method insert_ignore +} + +load_noaa() { + load_pair \ + "noaa" \ + "$DEMO_ROOT/noaa/noaa-emergency-response.json" \ + "$DEMO_ROOT/noaa/noaa-eri-nashville2020.json" +} + +load_facebook() { + load_pair \ + "facebook" \ + "$DEMO_ROOT/facebook/facebook_collection.json" \ + "$DEMO_ROOT/facebook/facebook_items.json" +} + +load_cmip6() { + model="${CMIP6_MODEL:-GISS-E2-1-G}" + variable="${CMIP6_VARIABLE:-tas}" + prefix="$DEMO_ROOT/cmip6/CMIP6_daily_${model}_${variable}" + + load_pair \ + "cmip6" \ + "${prefix}_collection.json" \ + "${prefix}_stac_items.ndjson" +} + +normalize_ndjson() { + input_file="$1" + output_file="$2" + + python - "$input_file" "$output_file" <<'PY_NDJSON' +import json +import re +import sys +from pathlib import Path + +import orjson + +input_path = Path(sys.argv[1]) +output_path = Path(sys.argv[2]) +invalid_escape = re.compile(r'\\(?!["\\/bfnrtu])') + + +def loads_line(line, lineno): + raw = line.encode("utf-8") + try: + return orjson.loads(raw) + except orjson.JSONDecodeError: + pass + + try: + return json.loads(line) + except json.JSONDecodeError: + repaired = invalid_escape.sub(r'\\\\', line) + try: + return json.loads(repaired) + except json.JSONDecodeError as exc: + raise SystemExit(f"Invalid JSON in {input_path} at line {lineno}: {exc}") from exc + + +items = [] +with input_path.open("r", encoding="utf-8") as src: + for lineno, line in enumerate(src, 1): + if not line.strip(): + continue + item = loads_line(line, lineno) + if "collection" not in item: + raise SystemExit(f"Normalized item from {input_path} at line {lineno} has no collection field") + items.append(item) + +output_path.write_bytes(orjson.dumps(items)) +parsed = orjson.loads(output_path.read_bytes()) +if not isinstance(parsed, list): + raise SystemExit(f"Normalized OAM output should be a JSON array, got {type(parsed).__name__}") +if parsed and "collection" not in parsed[0]: + raise SystemExit("First normalized OAM item has no collection field") +PY_NDJSON +} +load_oam() { + collection_file="$DEMO_ROOT/oam/oam_collection.json" + items_file="$DEMO_ROOT/oam/oam_items.njson" + normalized_items_file="/tmp/oam_items.normalized.json" + + if [ ! -f "$collection_file" ]; then + echo "Skipping oam: missing collection file $collection_file" + return 0 + fi + + if [ ! -f "$items_file" ]; then + echo "Skipping oam: missing items file $items_file" + return 0 + fi + + normalize_ndjson "$items_file" "$normalized_items_file" + load_pair "oam" "$collection_file" "$normalized_items_file" +} + +if [ "$#" -eq 0 ]; then + set -- all +fi + +for demo_name in "$@"; do + case "$demo_name" in + all) + load_noaa + load_facebook + load_cmip6 + load_oam + ;; + noaa) + load_noaa + ;; + facebook) + load_facebook + ;; + cmip6) + load_cmip6 + ;; + oam) + load_oam + ;; + *) + echo "Unknown demo '$demo_name'. Expected one of: all, noaa, facebook, cmip6, oam" >&2 + exit 2 + ;; + esac +done diff --git a/demo/noaa/README.md b/demo/noaa/README.md index 9b746bbd..10e036e2 100644 --- a/demo/noaa/README.md +++ b/demo/noaa/README.md @@ -34,7 +34,7 @@ $ aws s3 ls noaa-eri-pds/2020_Nashville_Tornado/20200307a_RGB/ \ "interval": [ [ "2005-01-01T00:00:00Z", - "null" + null ] ] } diff --git a/demo/noaa/demo.ipynb b/demo/noaa/demo.ipynb index bd792815..aa8d81fd 100644 --- a/demo/noaa/demo.ipynb +++ b/demo/noaa/demo.ipynb @@ -3,19 +3,23 @@ { "cell_type": "code", "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "import json\n", + "import os\n", "import requests\n", "from folium import Map, TileLayer, GeoJson\n", "\n", - "endpoint = \"\"" - ], - "outputs": [], - "metadata": {} + "endpoint = os.environ.get(\"TITILER_URL\", \"http://127.0.0.1:8082\")\n", + "public_endpoint = os.environ.get(\"TITILER_PUBLIC_URL\", \"http://127.0.0.1:8082\")" + ] }, { "cell_type": "code", "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "geojson = {\"type\": \"Feature\", \"geometry\": {\"coordinates\": [[[-87.0251, 36.2251], [-87.0251, 36.0999], [-85.4249, 36.0999], [-85.4249, 36.2251], [-87.0251, 36.2251]]], \"type\": \"Polygon\"}}\n", "bounds = (-87.0251, 36.0999, -85.4249, 36.2251)\n", @@ -34,20 +38,20 @@ ")\n", "geo_json.add_to(m)\n", "m" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "markdown", + "metadata": {}, "source": [ "### Create Mosaic for the whole collection" - ], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "# Register Search Query\n", "body = {\n", @@ -56,18 +60,20 @@ " \"bbox\": bounds,\n", "}\n", "\n", - "response = requests.post(\n", - " f\"{endpoint}/mosaic/register\",\n", + "register_resp = requests.post(\n", + " f\"{endpoint}/searches/register\",\n", " json=body,\n", - ").json()\n", + ")\n", + "register_resp.raise_for_status()\n", + "response = register_resp.json()\n", "print(response)" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "code", "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "m = Map(\n", " location=((bounds[1] + bounds[3]) / 2,(bounds[0] + bounds[2]) / 2),\n", @@ -75,8 +81,15 @@ ")\n", "\n", "# Fetch Tilejson (we HAVE TO add the asset name)\n", - "tj_resp = requests.get(\n", - " response['url'],\n", + "search_id = response.get(\"id\")\n", + "tilejson_url = response.get(\"url\")\n", + "if tilejson_url is None:\n", + " if search_id is None:\n", + " raise KeyError(f\"Search registration response has no id or url: {response}\")\n", + " tilejson_url = f\"{endpoint}/searches/{search_id}/WebMercatorQuad/tilejson.json\"\n", + "\n", + "tilejson_resp = requests.get(\n", + " tilejson_url,\n", " params={\n", " # Info to add to the tilejson response\n", " \"minzoom\": 13,\n", @@ -84,7 +97,20 @@ " # query parameter to add to the tile URL\n", " \"assets\": \"cog\",\n", " }\n", - ").json()\n", + ")\n", + "if tilejson_resp.status_code == 404 and search_id is not None:\n", + " tilejson_url = f\"{endpoint}/searches/{search_id}/tilejson.json\"\n", + " tilejson_resp = requests.get(\n", + " tilejson_url,\n", + " params={\n", + " \"minzoom\": 13,\n", + " \"maxzoom\": 20,\n", + " \"assets\": \"cog\",\n", + " }\n", + " )\n", + "tilejson_resp.raise_for_status()\n", + "tj_resp = tilejson_resp.json()\n", + "tj_resp[\"tiles\"] = [tile.replace(endpoint, public_endpoint) for tile in tj_resp[\"tiles\"]]\n", "print(tj_resp)\n", "\n", "geo_json = GeoJson(\n", @@ -104,16 +130,21 @@ ")\n", "aod_layer.add_to(m)\n", "m" - ], + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, "outputs": [], - "metadata": {} + "source": [] }, { "cell_type": "code", "execution_count": null, - "source": [], + "metadata": {}, "outputs": [], - "metadata": {} + "source": [] } ], "metadata": { @@ -121,8 +152,8 @@ "hash": "e0a12c78cd70db9ff05ed68287a27ffcdd32788e19bdb884235a47fc6f52d8ad" }, "kernelspec": { - "name": "python3", - "display_name": "Python 3.8.2 64-bit ('py38': venv)" + "display_name": "Python 3.8.2 64-bit ('py38': venv)", + "name": "python3" }, "language_info": { "codemirror_mode": { @@ -139,4 +170,4 @@ }, "nbformat": 4, "nbformat_minor": 4 -} \ No newline at end of file +} diff --git a/demo/noaa/noaa-emergency-response.json b/demo/noaa/noaa-emergency-response.json index 5c7c1b87..41ed1776 100644 --- a/demo/noaa/noaa-emergency-response.json +++ b/demo/noaa/noaa-emergency-response.json @@ -1 +1 @@ -{"id":"noaa-emergency-response", "title": "NOAA Emergency Response Imagery", "description":"NOAA Emergency Response Imagery hosted on AWS Public Dataset.","stac_version":"1.0.0","license":"public-domain","links":[],"extent":{"spatial":{"bbox":[[-180,-90,180,90]]},"temporal":{"interval":[["2005-01-01T00:00:00Z","null"]]}}} +{"id":"noaa-emergency-response", "title": "NOAA Emergency Response Imagery", "description":"NOAA Emergency Response Imagery hosted on AWS Public Dataset.","stac_version":"1.0.0","license":"public-domain","links":[],"extent":{"spatial":{"bbox":[[-180,-90,180,90]]},"temporal":{"interval":[["2005-01-01T00:00:00Z",null]]}}} diff --git a/demo/oam/OpenAerialMap_demo.ipynb b/demo/oam/OpenAerialMap_demo.ipynb index f03648b4..09ebbcfe 100644 --- a/demo/oam/OpenAerialMap_demo.ipynb +++ b/demo/oam/OpenAerialMap_demo.ipynb @@ -42,17 +42,9 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'provided_by': 'OpenAerialMap', 'license': 'CC-BY 4.0', 'website': 'http://beta.openaerialmap.org', 'page': 1, 'limit': 1, 'found': 16194}\n" - ] - } - ], + "outputs": [], "source": [ "import httpx\n", "\n", @@ -79,7 +71,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -96,7 +88,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -169,7 +161,7 @@ " fout.write(\n", " json.dumps(\n", " item.to_dict(), ensure_ascii=False\n", - " ).encode(\"ascii\", \"ignore\").decode(\"utf-8\").replace('\\\\\"', \"\") + \"\\n\"\n", + " ).encode(\"ascii\", \"ignore\").decode(\"utf-8\") + \"\\n\"\n", " )\n", "\n", "init_data = min(\n", @@ -229,15 +221,54 @@ ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "metadata": {}, + "outputs": [], "source": [ - "```bash\n", - "pypgstac load collections oam_collection.json --dsn postgresql://{db-user}:{db-password}@{db-host}:{db-port}/{db-name} --method insert\n", + "import json\n", + "import os\n", + "import re\n", + "from pathlib import Path\n", + "\n", + "import orjson\n", + "\n", + "database_url = os.environ.get(\"DATABASE_URL\", \"postgresql://username:password@database:5432/postgis\")\n", + "items_file = Path(\"oam_items.njson\")\n", + "normalized_items_file = Path(\"/tmp/oam_items.normalized.json\")\n", + "invalid_escape = re.compile(r'\\\\(?![\"\\\\/bfnrtu])')\n", + "\n", + "def loads_line(line, lineno):\n", + " raw = line.encode(\"utf-8\")\n", + " try:\n", + " return orjson.loads(raw)\n", + " except orjson.JSONDecodeError:\n", + " pass\n", + "\n", + " try:\n", + " return json.loads(line)\n", + " except json.JSONDecodeError:\n", + " repaired = invalid_escape.sub(r'\\\\\\\\', line)\n", + " try:\n", + " return json.loads(repaired)\n", + " except json.JSONDecodeError as exc:\n", + " raise ValueError(f\"Invalid JSON in {items_file} at line {lineno}: {exc}\") from exc\n", + "\n", + "items = []\n", + "with items_file.open(\"r\", encoding=\"utf-8\") as src:\n", + " for lineno, line in enumerate(src, 1):\n", + " if not line.strip():\n", + " continue\n", + " item = loads_line(line, lineno)\n", + " if \"collection\" not in item:\n", + " raise ValueError(f\"Normalized item from {items_file} at line {lineno} has no collection field\")\n", + " items.append(item)\n", "\n", - "# NOTE: we need to set `--method ignore` because some items are duplicated in the OAM database\n", - "pypgstac load items oam_items.njson --dsn postgresql://{db-user}:{db-password}@{db-host}:{db-port}/{db-name} --method ignore\n", - "```" + "normalized_items_file.write_bytes(orjson.dumps(items))\n", + "\n", + "!pypgstac load collections oam_collection.json --dsn {database_url} --method insert_ignore\n", + "# OAM has duplicate item records, so use insert_ignore for repeatable local loads.\n", + "!pypgstac load items {normalized_items_file} --dsn {database_url} --method insert_ignore" ] }, { @@ -249,13 +280,14 @@ }, { "cell_type": "code", - "execution_count": 43, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ + "import os\n", "import httpx\n", "\n", - "stac_endpoint = \"https://stac.eoapi.dev\"" + "stac_endpoint = os.environ.get(\"STAC_API_URL\", \"http://127.0.0.1:8081\")" ] }, { @@ -269,19 +301,11 @@ }, { "cell_type": "code", - "execution_count": 63, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'limit': 100, 'matched': 6, 'returned': 6}\n" - ] - } - ], + "outputs": [], "source": [ - "# use /search endpoint with some `filter` parameter\n", + "# use /search endpoint with some filter parameter\n", "response = httpx.get(\n", " f\"{stac_endpoint}/search\",\n", " params={\n", @@ -290,118 +314,16 @@ " \"limit\": 100,\n", " },\n", ")\n", - "print(response.json()[\"context\"])\n", - "\n", - "feature_collection = response.json()" + "response.raise_for_status()\n", + "feature_collection = response.json()\n", + "print(feature_collection.get(\"context\", {\"returned\": len(feature_collection.get(\"features\", []))}))" ] }, { "cell_type": "code", - "execution_count": 64, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
Make this Notebook Trusted to load map: File -> Trust Notebook
" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 64, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Put the result of the search request (FeatureCollection) on a Map\n", "\n", @@ -425,103 +347,23 @@ }, { "cell_type": "code", - "execution_count": 66, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'id': '66e00568cd0baa0001b6206a', 'bbox': [1.23557, 44.917514, 1.239309, 44.919786], 'type': 'Feature', 'links': [{'rel': 'collection', 'type': 'application/json', 'href': 'https://stac.eoapi.dev/collections/openaerialmap'}, {'rel': 'parent', 'type': 'application/json', 'href': 'https://stac.eoapi.dev/collections/openaerialmap'}, {'rel': 'root', 'type': 'application/json', 'href': 'https://stac.eoapi.dev/'}, {'rel': 'self', 'type': 'application/geo+json', 'href': 'https://stac.eoapi.dev/collections/openaerialmap/items/66e00568cd0baa0001b6206a'}], 'assets': {'image': {'href': 'https://oin-hotosm.s3.us-east-1.amazonaws.com/66e0032ecd0baa0001b62068/0/66e0032ecd0baa0001b62069.tif', 'type': 'image/tiff; application=geotiff; profile=cloud-optimized', 'roles': ['data'], 'alternate': {'s3': {'href': 's3://oin-hotosm/66e0032ecd0baa0001b62068/0/66e0032ecd0baa0001b62069.tif'}}, 'alternate:name': 'S3'}}, 'geometry': {'type': 'MultiPolygon', 'coordinates': [[[[1.23557, 44.919728], [1.23564, 44.917514], [1.239309, 44.917573], [1.239239, 44.919786], [1.23557, 44.919728]]]]}, 'collection': 'openaerialmap', 'properties': {'title': '24ATLANG_2024', 'platform': 'uav', 'provider': 'CEN NA - MD', 'file:size': 29220029, 'end_datetime': '2024-09-10T08:25:39.686000Z', 'start_datetime': '2024-09-09T22:00:00Z'}, 'stac_version': '1.0.0', 'stac_extensions': ['https://stac-extensions.github.io/file/v2.1.0/schema.json', 'https://stac-extensions.github.io/alternate-assets/v1.2.0/schema.json']}\n", - "Available Assets: ['image']\n", - "Fetching Raster info for Item 66e00568cd0baa0001b6206a\n", - "Returned metadata for Assets: ['image']\n", - "\n", - "{\n", - " \"bounds\": [\n", - " 1.2355701143428997,\n", - " 44.917514394030164,\n", - " 1.239308582430391,\n", - " 44.919786336696625\n", - " ],\n", - " \"minzoom\": 16,\n", - " \"maxzoom\": 22,\n", - " \"band_metadata\": [\n", - " [\n", - " \"b1\",\n", - " {\n", - " \"STATISTICS_MAXIMUM\": \"255\",\n", - " \"STATISTICS_MEAN\": \"46.780351214711\",\n", - " \"STATISTICS_MINIMUM\": \"0\",\n", - " \"STATISTICS_STDDEV\": \"61.499979783294\",\n", - " \"STATISTICS_VALID_PERCENT\": \"100\"\n", - " }\n", - " ],\n", - " [\n", - " \"b2\",\n", - " {\n", - " \"STATISTICS_MAXIMUM\": \"255\",\n", - " \"STATISTICS_MEAN\": \"53.035191642437\",\n", - " \"STATISTICS_MINIMUM\": \"0\",\n", - " \"STATISTICS_STDDEV\": \"67.218967788236\",\n", - " \"STATISTICS_VALID_PERCENT\": \"100\"\n", - " }\n", - " ],\n", - " [\n", - " \"b3\",\n", - " {\n", - " \"STATISTICS_MAXIMUM\": \"255\",\n", - " \"STATISTICS_MEAN\": \"33.863411341068\",\n", - " \"STATISTICS_MINIMUM\": \"0\",\n", - " \"STATISTICS_STDDEV\": \"47.91602799174\",\n", - " \"STATISTICS_VALID_PERCENT\": \"100\"\n", - " }\n", - " ]\n", - " ],\n", - " \"band_descriptions\": [\n", - " [\n", - " \"b1\",\n", - " \"red\"\n", - " ],\n", - " [\n", - " \"b2\",\n", - " \"green\"\n", - " ],\n", - " [\n", - " \"b3\",\n", - " \"blue\"\n", - " ]\n", - " ],\n", - " \"dtype\": \"uint8\",\n", - " \"nodata_type\": \"Mask\",\n", - " \"colorinterp\": [\n", - " \"red\",\n", - " \"green\",\n", - " \"blue\"\n", - " ],\n", - " \"driver\": \"GTiff\",\n", - " \"count\": 3,\n", - " \"width\": 14486,\n", - " \"height\": 12302,\n", - " \"overviews\": [\n", - " 2,\n", - " 4,\n", - " 8,\n", - " 16,\n", - " 32\n", - " ]\n", - "}\n", - "\n", - "Min/Max zoom for Asset `Image` are 16 22\n" - ] - } - ], + "outputs": [], "source": [ - "# Let's visualize one Item using the raster API\n", - "import httpx\n", + "# Let us visualize one Item using the raster API\n", "import json\n", + "import os\n", + "import httpx\n", + "\n", + "raster_endpoint = os.environ.get(\"TITILER_URL\", \"http://127.0.0.1:8082\")\n", + "raster_public_endpoint = os.environ.get(\"TITILER_PUBLIC_URL\", \"http://127.0.0.1:8082\")\n", "\n", - "raster_endpoint = \"https://raster.eoapi.dev\"\n", + "def public_tilejson(tilejson):\n", + " if \"tiles\" not in tilejson:\n", + " raise RuntimeError(f\"TileJSON response has no tiles: {tilejson}\")\n", + " tilejson[\"tiles\"] = [tile.replace(raster_endpoint, raster_public_endpoint) for tile in tilejson[\"tiles\"]]\n", + " return tilejson\n", "\n", "feature = feature_collection[\"features\"][0]\n", "print(feature)\n", @@ -530,131 +372,49 @@ "collection_id = \"openaerialmap\"\n", "\n", "# Check what assets are available\n", - "resp = httpx.get(f\"{raster_endpoint}/collections/{collection_id}/items/{item_id}/assets\").json()\n", - "print(\"Available Assets: \", resp)\n", + "assets_resp = httpx.get(f\"{raster_endpoint}/collections/{collection_id}/items/{item_id}/assets\")\n", + "assets_resp.raise_for_status()\n", + "print(\"Available Assets: \", assets_resp.json())\n", "\n", - "# Fectch `Image` asset info\n", + "# Fetch image asset info\n", "print(f\"Fetching Raster info for Item {item_id}\")\n", - "info = httpx.get(f\"{raster_endpoint}/collections/{collection_id}/items/{item_id}/info\", params={\"assets\": \"image\"}).json()\n", + "info_resp = httpx.get(f\"{raster_endpoint}/collections/{collection_id}/items/{item_id}/info\", params={\"assets\": \"image\"})\n", + "info_resp.raise_for_status()\n", + "info = info_resp.json()\n", "\n", "print(\"Returned metadata for Assets:\", list(info.keys()))\n", "print()\n", "print(json.dumps(info[\"image\"], indent=4))\n", "print()\n", - "\n", - "print(\"Min/Max zoom for Asset `Image` are\", info[\"image\"][\"minzoom\"], info[\"image\"][\"maxzoom\"])" + "print(\"Min/Max zoom for Asset image are\", info[\"image\"].get(\"minzoom\", \"?\"), info[\"image\"].get(\"maxzoom\", \"?\"))" ] }, { "cell_type": "code", - "execution_count": 67, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'tilejson': '2.2.0', 'version': '1.0.0', 'scheme': 'xyz', 'tiles': ['https://raster.eoapi.dev/collections/openaerialmap/items/66e00568cd0baa0001b6206a/tiles/WebMercatorQuad/{z}/{x}/{y}@1x?assets=image'], 'minzoom': 16, 'maxzoom': 22, 'bounds': [1.23557, 44.917514, 1.239309, 44.919786], 'center': [1.2374395, 44.91865, 16]}\n" - ] - }, - { - "data": { - "text/html": [ - "
Make this Notebook Trusted to load map: File -> Trust Notebook
" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 67, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "\n", - "resp = httpx.get(\n", - " f\"{raster_endpoint}/collections/{collection_id}/items/{item_id}/WebMercatorQuad/tilejson.json\", params={\"assets\": \"image\", \"minzoom\": 16, \"maxzoom\": 22}\n", - ").json()\n", + "tilejson_resp = httpx.get(\n", + " f\"{raster_endpoint}/collections/{collection_id}/items/{item_id}/WebMercatorQuad/tilejson.json\",\n", + " params={\"assets\": \"image\", \"minzoom\": 16, \"maxzoom\": 22},\n", + ")\n", + "tilejson_resp.raise_for_status()\n", + "resp = public_tilejson(tilejson_resp.json())\n", "print(resp)\n", "\n", - "lat = (info[\"image\"][\"bounds\"][3] + info[\"image\"][\"bounds\"][1]) / 2.0\n", - "lon = (info[\"image\"][\"bounds\"][0] + info[\"image\"][\"bounds\"][2]) / 2.0\n", - "m = Map(tiles=\"OpenStreetMap\", location=(lat, lon), zoom_start=116)\n", + "# info[\"image\"][\"bounds\"] may be in the asset CRS, e.g. EPSG:2154.\n", + "# TileJSON bounds and STAC item bbox are lon/lat, which Folium expects.\n", + "lonlat_bounds = resp.get(\"bounds\", feature[\"bbox\"])\n", + "lat = (lonlat_bounds[3] + lonlat_bounds[1]) / 2.0\n", + "lon = (lonlat_bounds[0] + lonlat_bounds[2]) / 2.0\n", + "m = Map(tiles=\"OpenStreetMap\", location=(lat, lon), zoom_start=16)\n", "\n", "aod_layer = TileLayer(\n", " tiles=resp[\"tiles\"][0],\n", " attr=f\"Item {item_id}\",\n", - " min_zoom=16,\n", - " max_zoom=22,\n", + " min_zoom=resp.get(\"minzoom\", 16),\n", + " max_zoom=resp.get(\"maxzoom\", 22),\n", " max_native_zoom=17,\n", ")\n", "aod_layer.add_to(m)\n", @@ -670,51 +430,9 @@ }, { "cell_type": "code", - "execution_count": 96, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{\n", - " \"id\": \"2fb7cb756f682dbe56754d6f9d7cd50f\",\n", - " \"links\": [\n", - " {\n", - " \"rel\": \"metadata\",\n", - " \"title\": \"Mosaic metadata\",\n", - " \"type\": \"application/json\",\n", - " \"href\": \"https://raster.eoapi.dev/searches/2fb7cb756f682dbe56754d6f9d7cd50f/info\"\n", - " },\n", - " {\n", - " \"rel\": \"tilejson\",\n", - " \"title\": \"Link for TileJSON\",\n", - " \"type\": \"application/json\",\n", - " \"href\": \"https://raster.eoapi.dev/searches/2fb7cb756f682dbe56754d6f9d7cd50f/tilejson.json\"\n", - " },\n", - " {\n", - " \"rel\": \"map\",\n", - " \"title\": \"Link for Map viewer\",\n", - " \"type\": \"application/json\",\n", - " \"href\": \"https://raster.eoapi.dev/searches/2fb7cb756f682dbe56754d6f9d7cd50f/map\"\n", - " },\n", - " {\n", - " \"rel\": \"wmts\",\n", - " \"title\": \"Link for WMTS\",\n", - " \"type\": \"application/json\",\n", - " \"href\": \"https://raster.eoapi.dev/searches/2fb7cb756f682dbe56754d6f9d7cd50f/WMTSCapabilities.xml\"\n", - " },\n", - " {\n", - " \"rel\": \"tilejson\",\n", - " \"title\": \"TileJSON link for `image` layer.\",\n", - " \"type\": \"application/json\",\n", - " \"href\": \"https://raster.eoapi.dev/searches/2fb7cb756f682dbe56754d6f9d7cd50f/tilejson.json?assets=image\"\n", - " }\n", - " ]\n", - "}\n" - ] - } - ], + "outputs": [], "source": [ "response = httpx.post(\n", " f\"{raster_endpoint}/searches/register\",\n", @@ -755,135 +473,24 @@ }, { "cell_type": "code", - "execution_count": 98, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'tilejson': '2.2.0', 'name': 'CEN NA - MD', 'version': '1.0.0', 'scheme': 'xyz', 'tiles': ['https://raster.eoapi.dev/searches/2fb7cb756f682dbe56754d6f9d7cd50f/tiles/WebMercatorQuad/{z}/{x}/{y}?assets=image'], 'minzoom': 13, 'maxzoom': 22, 'bounds': [0.993319, 44.917514, 1.239309, 45.216803], 'center': [1.116314, 45.0671585, 13]}\n" - ] - }, - { - "data": { - "text/html": [ - "
Make this Notebook Trusted to load map: File -> Trust Notebook
" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 98, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "search_id = response[\"id\"]\n", "\n", - "resp = httpx.get(\n", - " f\"{raster_endpoint}/searches/{search_id}/WebMercatorQuad/tilejson.json\", params={\"assets\": \"image\"}\n", - ").json()\n", + "tilejson_resp = httpx.get(\n", + " f\"{raster_endpoint}/searches/{search_id}/WebMercatorQuad/tilejson.json\",\n", + " params={\"assets\": \"image\"},\n", + ")\n", + "tilejson_resp.raise_for_status()\n", + "resp = public_tilejson(tilejson_resp.json())\n", "print(resp)\n", "\n", - "lat = (resp[\"bounds\"][3] + resp[\"bounds\"][1]) / 2.0\n", - "lon = (resp[\"bounds\"][0] + resp[\"bounds\"][2]) / 2.0\n", - "mosaic = Map(tiles=\"OpenStreetMap\", location=(lat, lon), zoom_start=13)\n", + "west, south, east, north = resp[\"bounds\"]\n", + "lat = (north + south) / 2.0\n", + "lon = (west + east) / 2.0\n", + "mosaic = Map(tiles=\"OpenStreetMap\", location=(lat, lon), zoom_start=10)\n", "\n", "aod_layer = TileLayer(\n", " tiles=resp[\"tiles\"][0],\n", @@ -894,9 +501,25 @@ ")\n", "aod_layer.add_to(mosaic)\n", "geojson.add_to(mosaic)\n", + "\n", + "# Fit the map to the full mosaic/search extent, not just one image footprint.\n", + "mosaic.fit_bounds([[south, west], [north, east]])\n", "mosaic" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, { "cell_type": "code", "execution_count": null, @@ -907,7 +530,7 @@ ], "metadata": { "kernelspec": { - "display_name": "py39", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -921,9 +544,9 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.19" + "version": "3.11.15" } }, "nbformat": 4, - "nbformat_minor": 2 + "nbformat_minor": 4 } diff --git a/demo/requirements.txt b/demo/requirements.txt new file mode 100644 index 00000000..7e9e5ef2 --- /dev/null +++ b/demo/requirements.txt @@ -0,0 +1,15 @@ +awscli +boto3 +fsspec[s3] +folium +httpx +ipykernel +jupyterlab +pypgstac==0.9.6 +psycopg[binary] +psycopg-pool +pystac +requests +rio-stac + +ipyleaflet diff --git a/docker-compose.yml b/docker-compose.yml index 435632a9..07229d65 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,3 +1,30 @@ +x-demo-base: &demo-base + image: eoapi-demo:latest + build: + context: . + dockerfile: dockerfiles/Dockerfile.demo + profiles: + - demo + environment: + - DATABASE_URL=postgresql://username:password@database:5432/postgis + - STAC_API_URL=http://stac-fastapi:8080 + - TITILER_URL=http://titiler-pgstac + - TITILER_PUBLIC_URL=http://127.0.0.1:8082 + - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-} + - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-} + - AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION:-us-east-1} + - AWS_NO_SIGN_REQUEST=YES + - CPL_VSIL_CURL_ALLOWED_EXTENSIONS=.tif,.tiff,.vrt + volumes: + - ./demo:/workspace/demo:z + depends_on: + stac-fastapi: + condition: service_healthy + titiler-pgstac: + condition: service_healthy + database: + condition: service_healthy + services: # change to official image when available https://github.com/radiantearth/stac-browser/pull/386 @@ -5,17 +32,41 @@ services: build: context: dockerfiles dockerfile: Dockerfile.browser + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:8085 || exit 1"] + interval: 5s + timeout: 3s + retries: 20 ports: - - "${MY_DOCKER_IP:-127.0.0.1}:8085:8085" + - name: stac-browser + target: 8085 + published: 8085 + protocol: tcp + mode: host depends_on: - - stac-fastapi - - titiler-pgstac - - database + stac-fastapi: + condition: service_healthy + titiler-pgstac: + condition: service_healthy + database: + condition: service_healthy stac-fastapi: image: ghcr.io/stac-utils/stac-fastapi-pgstac:5.0.2 + healthcheck: + test: [ + "CMD-SHELL", + "python -c \"import urllib.request; urllib.request.urlopen('http://localhost:8080/_mgmt/ping', timeout=3).read()\"" + ] + interval: 5s + timeout: 3s + retries: 20 ports: - - "${MY_DOCKER_IP:-127.0.0.1}:8081:8081" + - name: stac-fastapi + target: 8080 + published: 8081 + protocol: tcp + mode: host environment: # Postgres connection - POSTGRES_USER=username @@ -27,19 +78,25 @@ services: - DB_MIN_CONN_SIZE=1 - DB_MAX_CONN_SIZE=1 depends_on: - - database - command: - bash -c "bash /tmp/scripts/wait-for-it.sh -t 120 -h database -p 5432 && uvicorn stac_fastapi.pgstac.app:app --host 0.0.0.0 --port 8081" - volumes: - - ./dockerfiles/scripts:/tmp/scripts + database: + condition: service_healthy titiler-pgstac: # At the time of writing, rasterio and psycopg wheels are not available for arm64 arch # so we force the image to be built with linux/amd64 platform: linux/amd64 image: ghcr.io/stac-utils/titiler-pgstac:1.7.2 + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://localhost:80 || exit 1"] + interval: 5s + timeout: 3s + retries: 20 ports: - - "${MY_DOCKER_IP:-127.0.0.1}:8082:8082" + - name: titiler-pgstac + target: 80 + published: 8082 + protocol: tcp + mode: host environment: # Postgres connection - POSTGRES_USER=username @@ -64,19 +121,26 @@ services: # TiTiler Config - MOSAIC_CONCURRENCY=1 # AWS S3 endpoint config - - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} - - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} + - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-} + - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-} + - AWS_NO_SIGN_REQUEST=YES depends_on: - - database - command: - bash -c "bash /tmp/scripts/wait-for-it.sh -t 120 -h database -p 5432 && uvicorn titiler.pgstac.main:app --host 0.0.0.0 --port 8082" - volumes: - - ./dockerfiles/scripts:/tmp/scripts + database: + condition: service_healthy tipg: image: ghcr.io/developmentseed/tipg:1.0.1 + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://localhost:80/health || exit 1"] + interval: 5s + timeout: 3s + retries: 20 ports: - - "${MY_DOCKER_IP:-127.0.0.1}:8083:8083" + - name: tipg + target: 80 + published: 8083 + protocol: tcp + mode: host environment: # Postgres connection - POSTGRES_USER=username @@ -86,15 +150,46 @@ services: - POSTGRES_PORT=5432 - DB_MIN_CONN_SIZE=1 - DB_MAX_CONN_SIZE=10 - command: - bash -c "bash /tmp/scripts/wait-for-it.sh -t 120 -h database -p 5432 && uvicorn tipg.main:app --host 0.0.0.0 --port 8083" depends_on: - - database - volumes: - - ./dockerfiles/scripts:/tmp/scripts + database: + condition: service_healthy + + demo-runner: + <<: *demo-base + ports: + - name: jupyter + target: 8888 + published: 8888 + protocol: tcp + mode: host + + demo-noaa: + <<: *demo-base + command: ["load-demos", "noaa"] + + demo-facebook: + <<: *demo-base + command: ["load-demos", "facebook"] + + demo-cmip6: + <<: *demo-base + command: ["load-demos", "cmip6"] + + demo-oam: + <<: *demo-base + command: ["load-demos", "oam"] + + demo-all: + <<: *demo-base + command: ["load-demos", "all"] database: image: ghcr.io/stac-utils/pgstac:v0.9.6 + healthcheck: + test: ["CMD-SHELL", "pg_isready -U username -d postgis"] + interval: 5s + timeout: 3s + retries: 20 environment: - POSTGRES_USER=username - POSTGRES_PASSWORD=password @@ -103,10 +198,17 @@ services: - PGPASSWORD=password - PGDATABASE=postgis ports: - - "${MY_DOCKER_IP:-127.0.0.1}:5439:5432" + - name: database + target: 5432 + published: 5439 + protocol: tcp + mode: host command: postgres -N 500 volumes: - - ./.pgdata:/var/lib/postgresql/data + - pgdata:/var/lib/postgresql/data + +volumes: + pgdata: networks: default: diff --git a/dockerfiles/Dockerfile.browser b/dockerfiles/Dockerfile.browser index 98480250..226ade21 100644 --- a/dockerfiles/Dockerfile.browser +++ b/dockerfiles/Dockerfile.browser @@ -6,7 +6,7 @@ ARG DYNAMIC_CONFIG=true WORKDIR /app RUN apk add --no-cache git -RUN git clone https://github.com/radiantearth/stac-browser.git . +RUN git clone --depth 1 --branch v3.3.5 https://github.com/radiantearth/stac-browser.git . # remove the default config.js RUN rm config.js RUN npm install diff --git a/dockerfiles/Dockerfile.demo b/dockerfiles/Dockerfile.demo new file mode 100644 index 00000000..d62640ea --- /dev/null +++ b/dockerfiles/Dockerfile.demo @@ -0,0 +1,27 @@ +FROM python:3.11-slim + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 + +WORKDIR /workspace + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + ca-certificates \ + git \ + postgresql-client \ + unzip \ + wget \ + && rm -rf /var/lib/apt/lists/* + +COPY demo/requirements.txt /tmp/demo-requirements.txt +RUN pip install -r /tmp/demo-requirements.txt + +COPY demo /workspace/demo +COPY demo/load-demos.sh /usr/local/bin/load-demos +RUN chmod +x /usr/local/bin/load-demos /workspace/demo/load-demos.sh + +EXPOSE 8888 + +CMD ["jupyter", "lab", "--ip=0.0.0.0", "--port=8888", "--no-browser", "--allow-root", "--NotebookApp.token=", "--NotebookApp.password="] diff --git a/dockerfiles/browser_config.js b/dockerfiles/browser_config.js index 89844ca8..497a82d7 100644 --- a/dockerfiles/browser_config.js +++ b/dockerfiles/browser_config.js @@ -1,5 +1,5 @@ module.exports = { - catalogUrl: "http://0.0.0.0:8081", + catalogUrl: "http://127.0.0.1:8081", catalogTitle: "eoAPI STAC Browser", allowExternalAccess: true, // Must be true if catalogUrl is not given allowedDomains: [], @@ -18,7 +18,7 @@ module.exports = { apiCatalogPriority: null, useTileLayerAsFallback: true, displayGeoTiffByDefault: false, - buildTileUrlTemplate: ({href, asset}) => "http://0.0.0.0:8082/cog/tiles/{z}/{x}/{y}@2x?url=" + encodeURIComponent(asset.href.startsWith("/vsi") ? asset.href : href), + buildTileUrlTemplate: ({href, asset}) => "http://127.0.0.1:8082/cog/tiles/{z}/{x}/{y}@2x?url=" + encodeURIComponent(asset.href.startsWith("/vsi") ? asset.href : href), stacProxyUrl: null, pathPrefix: "/", historyMode: "history", diff --git a/dockerfiles/scripts/wait-for-it.sh b/dockerfiles/scripts/wait-for-it.sh deleted file mode 100755 index 4634289d..00000000 --- a/dockerfiles/scripts/wait-for-it.sh +++ /dev/null @@ -1,186 +0,0 @@ -#!/usr/bin/env bash -# Use this script to test if a given TCP host/port are available - -###################################################### -# Copied from https://github.com/vishnubob/wait-for-it -###################################################### - -WAITFORIT_cmdname=${0##*/} - -echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } - -usage() -{ - cat << USAGE >&2 -Usage: - $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args] - -h HOST | --host=HOST Host or IP under test - -p PORT | --port=PORT TCP port under test - Alternatively, you specify the host and port as host:port - -s | --strict Only execute subcommand if the test succeeds - -q | --quiet Don't output any status messages - -t TIMEOUT | --timeout=TIMEOUT - Timeout in seconds, zero for no timeout - -- COMMAND ARGS Execute command with args after the test finishes -USAGE - exit 1 -} - -wait_for() -{ - if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then - echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" - else - echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout" - fi - WAITFORIT_start_ts=$(date +%s) - while : - do - if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then - nc -z $WAITFORIT_HOST $WAITFORIT_PORT - WAITFORIT_result=$? - else - (echo -n > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1 - WAITFORIT_result=$? - fi - if [[ $WAITFORIT_result -eq 0 ]]; then - WAITFORIT_end_ts=$(date +%s) - echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds" - break - fi - sleep 1 - done - return $WAITFORIT_result -} - -wait_for_wrapper() -{ - # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692 - if [[ $WAITFORIT_QUIET -eq 1 ]]; then - timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & - else - timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & - fi - WAITFORIT_PID=$! - trap "kill -INT -$WAITFORIT_PID" INT - wait $WAITFORIT_PID - WAITFORIT_RESULT=$? - if [[ $WAITFORIT_RESULT -ne 0 ]]; then - echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" - fi - return $WAITFORIT_RESULT -} - -# process arguments -while [[ $# -gt 0 ]] -do - case "$1" in - *:* ) - WAITFORIT_hostport=(${1//:/ }) - WAITFORIT_HOST=${WAITFORIT_hostport[0]} - WAITFORIT_PORT=${WAITFORIT_hostport[1]} - shift 1 - ;; - --child) - WAITFORIT_CHILD=1 - shift 1 - ;; - -q | --quiet) - WAITFORIT_QUIET=1 - shift 1 - ;; - -s | --strict) - WAITFORIT_STRICT=1 - shift 1 - ;; - -h) - WAITFORIT_HOST="$2" - if [[ $WAITFORIT_HOST == "" ]]; then break; fi - shift 2 - ;; - --host=*) - WAITFORIT_HOST="${1#*=}" - shift 1 - ;; - -p) - WAITFORIT_PORT="$2" - if [[ $WAITFORIT_PORT == "" ]]; then break; fi - shift 2 - ;; - --port=*) - WAITFORIT_PORT="${1#*=}" - shift 1 - ;; - -t) - WAITFORIT_TIMEOUT="$2" - if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi - shift 2 - ;; - --timeout=*) - WAITFORIT_TIMEOUT="${1#*=}" - shift 1 - ;; - --) - shift - WAITFORIT_CLI=("$@") - break - ;; - --help) - usage - ;; - *) - echoerr "Unknown argument: $1" - usage - ;; - esac -done - -if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then - echoerr "Error: you need to provide a host and port to test." - usage -fi - -WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15} -WAITFORIT_STRICT=${WAITFORIT_STRICT:-0} -WAITFORIT_CHILD=${WAITFORIT_CHILD:-0} -WAITFORIT_QUIET=${WAITFORIT_QUIET:-0} - -# Check to see if timeout is from busybox? -WAITFORIT_TIMEOUT_PATH=$(type -p timeout) -WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH) - -WAITFORIT_BUSYTIMEFLAG="" -if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then - WAITFORIT_ISBUSY=1 - # Check if busybox timeout uses -t flag - # (recent Alpine versions don't support -t anymore) - if timeout &>/dev/stdout | grep -q -e '-t '; then - WAITFORIT_BUSYTIMEFLAG="-t" - fi -else - WAITFORIT_ISBUSY=0 -fi - -if [[ $WAITFORIT_CHILD -gt 0 ]]; then - wait_for - WAITFORIT_RESULT=$? - exit $WAITFORIT_RESULT -else - if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then - wait_for_wrapper - WAITFORIT_RESULT=$? - else - wait_for - WAITFORIT_RESULT=$? - fi -fi - -if [[ $WAITFORIT_CLI != "" ]]; then - if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then - echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess" - exit $WAITFORIT_RESULT - fi - exec "${WAITFORIT_CLI[@]}" -else - exit $WAITFORIT_RESULT -fi