diff --git a/examples/elevation.tif b/examples/elevation.tif new file mode 100644 index 0000000..f8534f0 Binary files /dev/null and b/examples/elevation.tif differ diff --git a/examples/world-cities-tiling-example.ipynb b/examples/world-cities-tiling-example.ipynb index 971c68e..d231731 100644 --- a/examples/world-cities-tiling-example.ipynb +++ b/examples/world-cities-tiling-example.ipynb @@ -10,10 +10,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "6739e809", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found existing software environment build, returning\n" + ] + } + ], "source": [ "import coiled\n", "\n", @@ -23,7 +31,7 @@ " 'channels': ['conda-forge', 'defaults'],\n", " 'dependencies': [\n", " 'python=3.9',\n", - " 'mapshader',\n", + " 'mapshader=0.1.2',\n", " 'dask=2022.04.2',\n", " 'distributed=2022.4.2',\n", " 'cloudpickle=2.0.0',\n", @@ -152,7 +160,7 @@ " source_obj['transforms'] = transforms\n", " source_obj['service_types'] = ['tile', 'wms', 'image', 'geojson']\n", " source_obj['tiling'] = dict(min_zoom=1,\n", - " max_zoom=2,\n", + " max_zoom=5,\n", " xmin_field='buffer_0_4326_xmin',\n", " xmax_field='buffer_0_4326_xmax',\n", " ymin_field='buffer_0_4326_ymin',\n", @@ -184,7 +192,7 @@ "source": [ "from mapshader.tile_utils import list_tiles\n", "\n", - "tiles_ddf = list_tiles(cities_source, npartitions=2)\n", + "tiles_ddf = list_tiles(cities_source, npartitions=10)\n", "tiles_ddf.persist()" ] }, @@ -248,6 +256,14 @@ "m\n", "display(m)" ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f5aa09a8", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { diff --git a/examples/world-countries-tiling-example.ipynb b/examples/world-countries-tiling-example.ipynb index 58b79c1..434c3c9 100644 --- a/examples/world-countries-tiling-example.ipynb +++ b/examples/world-countries-tiling-example.ipynb @@ -23,7 +23,7 @@ " 'channels': ['conda-forge', 'defaults'],\n", " 'dependencies': [\n", " 'python=3.9',\n", - " 'mapshader',\n", + " 'mapshader=0.1.2',\n", " 'dask=2022.04.2',\n", " 'distributed=2022.4.2',\n", " 'cloudpickle=2.0.0',\n", @@ -53,7 +53,8 @@ ] }, "id": "c96ffb3b-165f-48cd-ba82-9bc02ae4f74e", - "outputId": "02d39e29-ee35-443c-fd41-137354731ad8" + "outputId": "02d39e29-ee35-443c-fd41-137354731ad8", + "scrolled": true }, "outputs": [], "source": [ @@ -156,7 +157,7 @@ " source_obj['transforms'] = transforms\n", " source_obj['service_types'] = ['tile']\n", " source_obj['tiling'] = dict(min_zoom=1,\n", - " max_zoom=2,\n", + " max_zoom=5,\n", " xmin_field='buffer_0_4326_xmin',\n", " xmax_field='buffer_0_4326_xmax',\n", " ymin_field='buffer_0_4326_ymin',\n", @@ -188,7 +189,7 @@ "source": [ "from mapshader.tile_utils import list_tiles\n", "\n", - "tiles_ddf = list_tiles(countries_source, npartitions=2)\n", + "tiles_ddf = list_tiles(countries_source, npartitions=100)\n", "tiles_ddf.persist()" ] }, @@ -252,6 +253,14 @@ "m\n", "display(m)" ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f451b0c0", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { diff --git a/examples/world-elevation-tiling-example.ipynb b/examples/world-elevation-tiling-example.ipynb new file mode 100644 index 0000000..facdd8d --- /dev/null +++ b/examples/world-elevation-tiling-example.ipynb @@ -0,0 +1,288 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "1c5181bc", + "metadata": {}, + "source": [ + "### Setup a Conda Environment on Coiled" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6739e809", + "metadata": {}, + "outputs": [], + "source": [ + "import coiled\n", + "\n", + "coiled.create_software_environment(\n", + " name='mapshader-2',\n", + " conda={\n", + " 'channels': ['conda-forge', 'defaults'],\n", + " 'dependencies': [\n", + " 'python=3.8',\n", + " 'mapshader=0.1.2',\n", + " 'dask=2022.04.2',\n", + " 'distributed=2022.4.2',\n", + " 'cloudpickle=2.0.0',\n", + " 'spatialpandas',\n", + " 'boto3',\n", + " ],\n", + " },\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "ca6c7296", + "metadata": {}, + "source": [ + "### Create Dask Cluster on Coiled " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c96ffb3b-165f-48cd-ba82-9bc02ae4f74e", + "metadata": { + "colab": { + "referenced_widgets": [ + "31f8880b3f93490f8db2674ed60bdb06" + ] + }, + "id": "c96ffb3b-165f-48cd-ba82-9bc02ae4f74e", + "outputId": "02d39e29-ee35-443c-fd41-137354731ad8" + }, + "outputs": [], + "source": [ + "from coiled.v2 import Cluster\n", + "cluster = Cluster(name='mapshader-tiler',\n", + " n_workers=2,\n", + " worker_cpu=2,\n", + " worker_options={\"nthreads\": 1},\n", + " scheduler_memory=\"8 GiB\",\n", + " software='mapshader-2')\n", + "\n", + "from dask.distributed import Client\n", + "client = Client(cluster)\n", + "print('Dashboard:', client.dashboard_link)" + ] + }, + { + "cell_type": "markdown", + "id": "1f2597cf", + "metadata": {}, + "source": [ + "### Clear cluster memory if necessary" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2f5ccd74", + "metadata": {}, + "outputs": [], + "source": [ + "client.restart()" + ] + }, + { + "cell_type": "markdown", + "id": "c0332ddc", + "metadata": {}, + "source": [ + "## Tile World Elevation" + ] + }, + { + "cell_type": "markdown", + "id": "7ed76e59", + "metadata": {}, + "source": [ + "### Setup Mapshader Source" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "50ac53ad", + "metadata": {}, + "outputs": [], + "source": [ + "import mapshader\n", + "import importlib\n", + "importlib.reload(mapshader)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "64daee8f", + "metadata": {}, + "outputs": [], + "source": [ + "from os import path\n", + "\n", + "import geopandas as gpd\n", + "import mapshader\n", + "import spatialpandas\n", + "\n", + "from mapshader.sources import RasterSource\n", + "\n", + "\n", + "def world_elevation_source():\n", + " # file data path\n", + " elevation_path = 's3://mapshader-testing/elevation.tif'\n", + " # construct transforms\n", + " squeeze_transform = dict(name='squeeze', args=dict(dim='band'))\n", + " cast_transform = dict(name='cast', args=dict(dtype='float64'))\n", + " orient_transform = dict(name='orient_array')\n", + " flip_transform = dict(name='flip_coords', args=dict(dim='y'))\n", + " reproject_transform = dict(name='reproject_raster', args=dict(epsg=3857))\n", + " transforms = [\n", + " squeeze_transform,\n", + " cast_transform,\n", + " orient_transform,\n", + " flip_transform,\n", + " reproject_transform,\n", + " ] \n", + " # construct value obj\n", + " source_obj = dict()\n", + " source_obj['name'] = 'Elevation'\n", + " source_obj['key'] = 'elevation'\n", + " source_obj['text'] = 'Elevation'\n", + " source_obj['description'] = 'Global Elevation Dataset'\n", + " source_obj['geometry_type'] = 'raster'\n", + " source_obj['shade_how'] = 'linear'\n", + " source_obj['cmap'] = ['white', 'black']\n", + " source_obj['span'] = 'min/max'\n", + " source_obj['raster_padding'] = 0\n", + " source_obj['raster_interpolate'] = 'linear'\n", + " source_obj['xfield'] = 'geometry'\n", + " source_obj['yfield'] = 'geometry'\n", + " source_obj['filepath'] = elevation_path\n", + " source_obj['transforms'] = transforms\n", + " source_obj['service_types'] = ['tile', 'wms', 'image', 'geojson']\n", + " source_obj['tiling'] = dict(min_zoom=1,\n", + " max_zoom=2,\n", + " )\n", + " return source_obj\n", + "\n", + "\n", + "elevation_source = RasterSource.from_obj(world_elevation_source())\n", + "elevation_source.load()\n", + "elevation_source.data" + ] + }, + { + "cell_type": "markdown", + "id": "6e64324b", + "metadata": {}, + "source": [ + "### Create a Dask DataFrame of Tile to Process based on Map Source feature extents" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0e03876e", + "metadata": {}, + "outputs": [], + "source": [ + "from mapshader.tile_utils import list_tiles\n", + "\n", + "tiles_ddf = list_tiles(elevation_source, npartitions=2)\n", + "tiles_ddf.persist()\n", + "tiles_ddf" + ] + }, + { + "cell_type": "markdown", + "id": "cfa0a560", + "metadata": {}, + "source": [ + "### Generate tile images and save to an S3 bucket" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "83e3b0af", + "metadata": {}, + "outputs": [], + "source": [ + "from mapshader.tile_utils import save_tiles_to_outpath\n", + "\n", + "save_tiles_to_outpath(elevation_source, tiles_ddf, outpath='s3://mapshader-tiling-test-999/elevation/')" + ] + }, + { + "cell_type": "markdown", + "id": "32088d09", + "metadata": {}, + "source": [ + "### View tiles on OSM basemap" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5f3160b6", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "from ipyleaflet import Map, TileLayer, basemaps, basemap_to_tiles\n", + "\n", + "tiles_url = 'https://mapshader-tiling-test-999.s3.amazonaws.com/elevation/{z}/{x}/{y}.png'\n", + "tile_layer=TileLayer(url=tiles_url)\n", + "\n", + "from ipyleaflet import Map, basemaps, basemap_to_tiles\n", + "\n", + "m = Map(\n", + " basemap=basemap_to_tiles(basemaps.OpenStreetMap.Mapnik),\n", + " center=(48.204793, 350.121558),\n", + " zoom=3\n", + " )\n", + "m\n", + "\n", + "m = Map(\n", + " basemap=basemap_to_tiles(basemaps.OpenStreetMap.Mapnik),\n", + " zoom=2,\n", + " scroll_wheel_zoom=True)\n", + "\n", + "m.add_layer(tile_layer)\n", + "m\n", + "display(m)" + ] + } + ], + "metadata": { + "colab": { + "name": "CoiledDemo.ipynb", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}