Skip to content

Commit

Permalink
Merge pull request #9 from ClimateImpactLab/feature/update-results
Browse files Browse the repository at this point in the history 
update to SLIIDERS v1.2
  • Loading branch information
@bolliger32
bolliger32 authored Apr 3, 2024
2 parents c53399d + b07a279 commit 8265330
Show file tree
Hide file tree
Showing 13 changed files with 499 additions and 413 deletions.
5 changes: 3 additions & 2 deletions HISTORY.rst
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,12 @@
History
=======

v1.2 (unreleased)
-------------------
v1.2.0
------
* Point `data-acquisition.ipynb` to updated Zenodo deposit that fixes the dtype of `subsets` variable in `diaz2016_inputs_raw.zarr.zip` to be bool rather than int8
* Variable name bugfix in `data-acquisition.ipynb`
* Add netcdf versions of SLIIDERS and the pyCIAM results to `upload-zenodo.ipynb`
* Update results in Zenodo record to use SLIIDERS v1.2

v1.1.2
------
Expand Down
136 changes: 64 additions & 72 deletions notebooks/models/create-surge-lookup-tables.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,23 @@
{
"cell_type": "code",
"execution_count": 1,
"id": "6b183aa3-366a-4f5f-bb95-5ef4bf48d438",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"%load_ext autoreload\n",
"%autoreload 2"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "cd4b2928-65a0-4733-9c71-fbefa85590be",
"metadata": {},
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"import sys\n",
Expand All @@ -41,16 +56,26 @@
},
{
"cell_type": "code",
"execution_count": 2,
"execution_count": 3,
"id": "afd03880-8e46-4d81-867d-bfb6505ea788",
"metadata": {},
"metadata": {
"tags": []
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/srv/conda/envs/notebook/lib/python3.9/site-packages/dask_gateway/client.py:21: FutureWarning: format_bytes is deprecated and will be removed in a future release. Please use dask.utils.format_bytes instead.\n",
" from distributed.utils import LoopRunner, format_bytes\n"
"/srv/conda/envs/notebook/lib/python3.10/site-packages/google/cloud/storage/transfer_manager.py:30: UserWarning: The module `transfer_manager` is a preview feature. Functionality and API may change. This warning will be removed in a future release.\n",
" warnings.warn(\n",
"/home/jovyan/git-repos/pyCIAM-public/notebooks/models/../shared.py:3: UserWarning: Shapely 2.0 is installed, but because PyGEOS is also installed, GeoPandas will still use PyGEOS by default for now. To force to use and test Shapely 2.0, you have to set the environment variable USE_PYGEOS=0. You can do this before starting the Python process, or in your code before importing geopandas:\n",
"\n",
"import os\n",
"os.environ['USE_PYGEOS'] = '0'\n",
"import geopandas\n",
"\n",
"In a future release, GeoPandas will switch to using Shapely by default. If you are using PyGEOS directly (calling PyGEOS functions on geometries from GeoPandas), this will then stop working and you are encouraged to migrate from PyGEOS to Shapely 2.0 (https://shapely.readthedocs.io/en/latest/migration_pygeos.html).\n",
" import geopandas as gpd\n"
]
}
],
Expand All @@ -75,7 +100,7 @@
},
{
"cell_type": "code",
"execution_count": 3,
"execution_count": 4,
"id": "47979c48-7fff-49b4-b445-ea6bf746fa37",
"metadata": {
"tags": [
Expand All @@ -86,18 +111,20 @@
"source": [
"# When running on larger/scalable dask cluster, may wish to specify number of workers\n",
"# Default is LocalCluster which will use the number of CPUs available on local machine\n",
"# N_WORKERS_MIN = 7\n",
"# N_WORKERS_MAX = 700\n",
"N_WORKERS_MIN = 7\n",
"N_WORKERS_MAX = 700\n",
"SEG_CHUNKSIZE = 5\n",
"\n",
"PARAMS = pd.read_json(PATH_PARAMS)[\"values\"]"
]
},
{
"cell_type": "code",
"execution_count": 4,
"execution_count": 5,
"id": "6104bced",
"metadata": {},
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"DMF_I = getattr(damage_funcs, PARAMS.dmf + \"_i\")\n",
Expand All @@ -106,70 +133,31 @@
},
{
"cell_type": "code",
"execution_count": 5,
"execution_count": 6,
"id": "dbb04b4a-14b8-4403-ad33-88bfe71bd8fc",
"metadata": {},
"metadata": {
"tags": []
},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
" <div style=\"width: 24px; height: 24px; background-color: #e1e1e1; border: 3px solid #9D9D9D; border-radius: 5px; position: absolute;\"> </div>\n",
" <div style=\"margin-left: 48px;\">\n",
" <h3 style=\"margin-bottom: 0px;\">Client</h3>\n",
" <p style=\"color: #9D9D9D; margin-bottom: 0px;\">Client-f992a2c6-c6d1-11ed-862e-665816eadc72</p>\n",
" <table style=\"width: 100%; text-align: left;\">\n",
"\n",
" <tr>\n",
" \n",
" <td style=\"text-align: left;\"><strong>Connection method:</strong> Cluster object</td>\n",
" <td style=\"text-align: left;\"><strong>Cluster type:</strong> dask_gateway.GatewayCluster</td>\n",
" \n",
" </tr>\n",
"\n",
" \n",
" <tr>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Dashboard: </strong> <a href=\"/services/dask-gateway/clusters/daskhub-dev.e0f56d1ca0ae4a42b9352ad6b7204454/status\" target=\"_blank\">/services/dask-gateway/clusters/daskhub-dev.e0f56d1ca0ae4a42b9352ad6b7204454/status</a>\n",
" </td>\n",
" <td style=\"text-align: left;\"></td>\n",
" </tr>\n",
" \n",
"\n",
" </table>\n",
"\n",
" \n",
" <details>\n",
" <summary style=\"margin-bottom: 20px;\"><h3 style=\"display: inline;\">Cluster Info</h3></summary>\n",
" <div style='background-color: #f2f2f2; display: inline-block; padding: 10px; border: 1px solid #999999;'>\n",
" <h3>GatewayCluster</h3>\n",
" <ul>\n",
" <li><b>Name: </b>daskhub-dev.e0f56d1ca0ae4a42b9352ad6b7204454\n",
" <li><b>Dashboard: </b><a href='/services/dask-gateway/clusters/daskhub-dev.e0f56d1ca0ae4a42b9352ad6b7204454/status' target='_blank'>/services/dask-gateway/clusters/daskhub-dev.e0f56d1ca0ae4a42b9352ad6b7204454/status</a>\n",
" </ul>\n",
"</div>\n",
"\n",
" </details>\n",
" \n",
"\n",
" </div>\n",
"</div>"
],
"application/vnd.jupyter.widget-view+json": {
"model_id": "fcadb94377b4430d9c2964ac9c808c2a",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"<Client: 'tls://10.4.102.2:8786' processes=0 threads=0, memory=0 B>"
"VBox(children=(HTML(value='<h2>GatewayCluster</h2>'), HBox(children=(HTML(value='\\n<div>\\n<style scoped>\\n …"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
"output_type": "display_data"
}
],
"source": [
"client = start_dask_cluster(\n",
" # n_workers_min=N_WORKERS_MIN,\n",
" # n_workers_max=N_WORKERS_MAX,\n",
")\n",
"client"
"client, cluster = start_dask_cluster()\n",
"cluster.adapt(minimum=N_WORKERS_MIN, maximum=N_WORKERS_MAX)\n",
"cluster"
]
},
{
Expand All @@ -182,7 +170,7 @@
},
{
"cell_type": "code",
"execution_count": 8,
"execution_count": 7,
"id": "1e823775-0d12-4e52-93d1-7e3586913e65",
"metadata": {
"tags": []
Expand All @@ -205,7 +193,7 @@
" start_year=PARAMS.model_start,\n",
" slr_0_years=PARAMS.slr_0_year,\n",
" client=client,\n",
" # client_kwargs={\"batch_size\": N_WORKERS_MAX},\n",
" client_kwargs={\"batch_size\": N_WORKERS_MAX},\n",
" force_overwrite=True,\n",
" seg_chunksize=SEG_CHUNKSIZE,\n",
" mc_dim=\"quantile\",\n",
Expand All @@ -223,9 +211,11 @@
},
{
"cell_type": "code",
"execution_count": 9,
"execution_count": 11,
"id": "f941a509-7035-4105-8b79-88a97e759737",
"metadata": {},
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# ensure completion and close cluster\n",
Expand All @@ -237,23 +227,25 @@
},
{
"cell_type": "code",
"execution_count": 10,
"execution_count": 12,
"id": "15563792-1ba0-435a-9981-e2b15eaa8dcd",
"metadata": {},
"metadata": {
"tags": []
},
"outputs": [
{
"data": {
"text/plain": [
"(None, None)"
]
},
"execution_count": 10,
"execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"client.cluster.close(), client.close()"
"cluster.close(), client.close()"
]
}
],
Expand All @@ -273,7 +265,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.10"
"version": "3.10.10"
},
"widgets": {
"application/vnd.jupyter.widget-state+json": {
Expand Down
Loading

0 comments on commit 8265330

Please sign in to comment.