diff --git a/notebooks/Dataset_generation/1_GCM_to_zarr.ipynb b/notebooks/Dataset_generation/1_GCM_to_zarr.ipynb index 248a971..179bd57 100644 --- a/notebooks/Dataset_generation/1_GCM_to_zarr.ipynb +++ b/notebooks/Dataset_generation/1_GCM_to_zarr.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": 5, "id": "7920aaa8-8228-432a-a322-365b46224cee", "metadata": {}, "outputs": [], @@ -12,19 +12,52 @@ }, { "cell_type": "code", - "execution_count": 2, - "id": "f5d732f9-a766-43d1-9bd4-578f6b46dc7f", + "execution_count": null, + "id": "6982dbdb", "metadata": {}, "outputs": [], "source": [ - "source_id = 'MIROC6'\n", - "institution_id = 'MIROC'\n", - "grid_label = 'gn'" + "# updates for later\n", + "# - add original_times to the clean_picontrol dataset\n", + "# - grab years from picontrol that are further into the time series\n" + ] + }, + { + "cell_type": "markdown", + "id": "6babae9e-4252-4297-b65a-57b070acc539", + "metadata": {}, + "source": [ + "\n", + "### breaking changes in `xclim` re: `convert_calendar`\n", + "\n", + "https://xclim.readthedocs.io/en/stable/changelog.html#v0-50-0-2024-06-17\n", + "\n", + "Calendar utilities that have an equivalent in xarray have been deprecated and will be removed in xclim v0.51.0. (GH/1010, PR/1761). This concerns the following members of `xclim.core.calendar`:\n", + " \n", + "`convert_calendar` Use `Dataset.convert_calendar`, `DataArray.convert_calendar` or `xr.coding.calendar_ops.convert_calendar` instead.\n", + " \n", + "- If your code passes target as an array, first convert the source to the target’s calendar and then reindex the result to target.\n", + " \n", + "- If you were using the `doy=True` option, replace it with `xc.core.calendar.convert_doy(source, target_cal).convert_calendar(target_cal)`.\n", + " \n", + "- `\"default\"` is no longer a valid calendar name for any xclim functions and will not be returned by `get_calendar`. Xarray has a `use_cftime` argument, xclim exposes it when the distinction is needed.\n", + " \n", + "\n", + "\n", + "`date_range` : Use `xarray.date_range` instead.\n", + " \n", + "`date_range_like`: Use `xarray.date_range_like` instead.\n", + " \n", + "`interp_calendar` : Use `Dataset.interp_calendar` or `xarray.coding.calendar_ops.interp_calendar` instead.\n", + " \n", + "`days_in_year` : Use `xarray.coding.calendar_ops._days_in_year` instead.\n", + " \n", + "`datetime_to_decimal_year` : Use `xarray.coding.calendar_ops._datetime_to_decimal_year` instead." ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 1, "id": "2bbc3871-f267-4406-9bf6-63349300c985", "metadata": {}, "outputs": [], @@ -32,778 +65,104 @@ "import os\n", "import secrets\n", "import dask.distributed as dd\n", - "from dask_gateway import Gateway\n", + "from dask_gateway import GatewayCluster\n", "import numpy as np\n", "import xarray as xr\n", "from xclim import sdba\n", - "from xclim.core.calendar import convert_calendar\n", + "# from xclim.core.calendar import convert_calendar\n", + "# from xr.Dataset import convert_calendar\n", "import xesmf as xe\n", "import pandas as pd\n", "import cftime\n", - "from xarray.backends.api import to_zarr" + "from xarray.backends.api import to_zarr\n", + "from datetime import datetime\n", + "import uuid\n", + "import importlib\n", + "\n", + "\n" ] }, { "cell_type": "code", - "execution_count": 4, - "id": "b8f47d27-0609-4acc-86a0-bdb16eba9df7", + "execution_count": 24, + "id": "104a4374-b6ea-4d47-b99d-785b6abf6881", "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "gs://impactlab-data-scratch/xq3rogfmpG12ATOWhGCYPNdSspH_vxlj/clean/hist/MIROC6/tas_r1.zarr\n", - "gs://impactlab-data-scratch/xq3rogfmpG12ATOWhGCYPNdSspH_vxlj/clean/histnat/MIROC6/tas_r2.zarr\n", - "gs://impactlab-data-scratch/xq3rogfmpG12ATOWhGCYPNdSspH_vxlj/clean/ERA5/MIROC6/tas.zarr\n" - ] + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ - "# SCRATCH_URL = f\"{os.environ['CIL_SCRATCH_PREFIX']}/{secrets.token_urlsafe(32 * 3//4)}\" #Use only if new scratch space is needed\n", - "SCRATCH_URL = \"gs://impactlab-data-scratch/xq3rogfmpG12ATOWhGCYPNdSspH_vxlj\"\n", - "\n", - "\n", - "if source_id == \"CanESM5\":\n", - "\n", - " IN_HIST_RAW_r1 = [ \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CCCma/tas_day_CanESM5_historical_r1i1p1f1_gn_18500101-20141231.nc\",\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CCCma/tas_day_CanESM5_ssp245_r1i1p1f1_gn_20150101-21001231.nc\"\n", - " ]\n", - " \n", - " IN_HIST_RAW_r2 = [ \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CCCma/tas_day_CanESM5_historical_r2i1p1f1_gn_18500101-20141231.nc\",\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CCCma/tas_day_CanESM5_ssp245_r2i1p1f1_gn_20150101-21001231.nc\"\n", - " ]\n", - "\n", - " IN_HIST_RAW_r3 = [ \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CCCma/tas_day_CanESM5_historical_r3i1p1f1_gn_18500101-20141231.nc\",\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CCCma/tas_day_CanESM5_ssp245_r3i1p1f1_gn_20150101-21001231.nc\"\n", - " ]\n", - "\n", - " IN_HISTNAT_RAW_r1 = [ \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CCCma/tas_day_CanESM5_hist-nat_r1i1p1f1_gn_18500101-20201231.nc\" ]\n", - " IN_HISTNAT_RAW_r2 = [ \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CCCma/tas_day_CanESM5_hist-nat_r2i1p1f1_gn_18500101-20201231.nc\" ]\n", - " IN_HISTNAT_RAW_r3 = [ \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CCCma/tas_day_CanESM5_hist-nat_r3i1p1f1_gn_18500101-20201231.nc\" ]\n", - "\n", - " IN_PICONTROL_RAW = [ \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CCCma/tas_day_CanESM5_piControl_r1i1p1f1_gn_60010101-62001231.nc\" ]\n", - " \n", - " PiControl_year_start = \"6001\"\n", - " \n", - "elif source_id == \"FGOALS-g3\":\n", - "\n", - " IN_HIST_RAW_r1 = [\n", - " \n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r1i1p1f1_gn_19690101-19691231.nc',\n", - " \n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r1i1p1f1_gn_19700101-19701231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r1i1p1f1_gn_19710101-19711231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r1i1p1f1_gn_19720101-19721231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r1i1p1f1_gn_19730101-19731231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r1i1p1f1_gn_19740101-19741231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r1i1p1f1_gn_19750101-19751231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r1i1p1f1_gn_19760101-19761231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r1i1p1f1_gn_19770101-19771231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r1i1p1f1_gn_19780101-19781231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r1i1p1f1_gn_19790101-19791231.nc',\n", - "\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r1i1p1f1_gn_19800101-19801231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r1i1p1f1_gn_19810101-19811231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r1i1p1f1_gn_19820101-19821231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r1i1p1f1_gn_19830101-19831231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r1i1p1f1_gn_19840101-19841231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r1i1p1f1_gn_19850101-19851231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r1i1p1f1_gn_19860101-19861231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r1i1p1f1_gn_19870101-19871231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r1i1p1f1_gn_19880101-19881231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r1i1p1f1_gn_19890101-19891231.nc',\n", - "\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r1i1p1f1_gn_19900101-19901231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r1i1p1f1_gn_19910101-19911231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r1i1p1f1_gn_19920101-19921231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r1i1p1f1_gn_19930101-19931231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r1i1p1f1_gn_19940101-19941231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r1i1p1f1_gn_19950101-19951231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r1i1p1f1_gn_19960101-19961231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r1i1p1f1_gn_19970101-19971231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r1i1p1f1_gn_19980101-19981231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r1i1p1f1_gn_19990101-19991231.nc',\n", - "\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r1i1p1f1_gn_20000101-20001231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r1i1p1f1_gn_20010101-20011231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r1i1p1f1_gn_20020101-20021231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r1i1p1f1_gn_20030101-20031231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r1i1p1f1_gn_20040101-20041231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r1i1p1f1_gn_20050101-20051231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r1i1p1f1_gn_20060101-20061231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r1i1p1f1_gn_20070101-20071231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r1i1p1f1_gn_20080101-20081231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r1i1p1f1_gn_20090101-20091231.nc',\n", - "\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r1i1p1f1_gn_20100101-20101231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r1i1p1f1_gn_20110101-20111231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r1i1p1f1_gn_20120101-20121231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r1i1p1f1_gn_20130101-20131231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r1i1p1f1_gn_20140101-20141231.nc',\n", - " \n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_ssp245_r2i1p1f1_gn_20150101-20151231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_ssp245_r2i1p1f1_gn_20160101-20161231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_ssp245_r2i1p1f1_gn_20170101-20171231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_ssp245_r2i1p1f1_gn_20180101-20181231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_ssp245_r2i1p1f1_gn_20190101-20191231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_ssp245_r2i1p1f1_gn_20200101-20201231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_ssp245_r2i1p1f1_gn_20210101-20211231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_ssp245_r2i1p1f1_gn_20220101-20221231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_ssp245_r2i1p1f1_gn_20230101-20231231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_ssp245_r2i1p1f1_gn_20240101-20241231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_ssp245_r2i1p1f1_gn_20250101-20251231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_ssp245_r2i1p1f1_gn_20260101-20261231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_ssp245_r2i1p1f1_gn_20270101-20271231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_ssp245_r2i1p1f1_gn_20280101-20281231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_ssp245_r2i1p1f1_gn_20290101-20291231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_ssp245_r2i1p1f1_gn_20300101-20301231.nc',\n", - " ]\n", - " \n", - " IN_HIST_RAW_r2 = [\n", - " \n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r4i1p1f1_gn_19690101-19691231.nc',\n", - " \n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r4i1p1f1_gn_19700101-19701231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r4i1p1f1_gn_19710101-19711231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r4i1p1f1_gn_19720101-19721231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r4i1p1f1_gn_19730101-19731231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r4i1p1f1_gn_19740101-19741231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r4i1p1f1_gn_19750101-19751231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r4i1p1f1_gn_19760101-19761231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r4i1p1f1_gn_19770101-19771231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r4i1p1f1_gn_19780101-19781231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r4i1p1f1_gn_19790101-19791231.nc',\n", - "\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r4i1p1f1_gn_19800101-19801231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r4i1p1f1_gn_19810101-19811231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r4i1p1f1_gn_19820101-19821231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r4i1p1f1_gn_19830101-19831231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r4i1p1f1_gn_19840101-19841231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r4i1p1f1_gn_19850101-19851231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r4i1p1f1_gn_19860101-19861231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r4i1p1f1_gn_19870101-19871231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r4i1p1f1_gn_19880101-19881231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r4i1p1f1_gn_19890101-19891231.nc',\n", - "\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r4i1p1f1_gn_19900101-19901231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r4i1p1f1_gn_19910101-19911231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r4i1p1f1_gn_19920101-19921231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r4i1p1f1_gn_19930101-19931231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r4i1p1f1_gn_19940101-19941231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r4i1p1f1_gn_19950101-19951231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r4i1p1f1_gn_19960101-19961231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r4i1p1f1_gn_19970101-19971231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r4i1p1f1_gn_19980101-19981231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r4i1p1f1_gn_19990101-19991231.nc',\n", - "\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r4i1p1f1_gn_20000101-20001231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r4i1p1f1_gn_20010101-20011231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r4i1p1f1_gn_20020101-20021231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r4i1p1f1_gn_20030101-20031231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r4i1p1f1_gn_20040101-20041231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r4i1p1f1_gn_20050101-20051231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r4i1p1f1_gn_20060101-20061231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r4i1p1f1_gn_20070101-20071231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r4i1p1f1_gn_20080101-20081231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r4i1p1f1_gn_20090101-20091231.nc',\n", - "\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r4i1p1f1_gn_20100101-20101231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r4i1p1f1_gn_20110101-20111231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r4i1p1f1_gn_20120101-20121231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r4i1p1f1_gn_20130101-20131231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r4i1p1f1_gn_20140101-20141231.nc',\n", - " \n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_ssp245_r4i1p1f1_gn_20150101-20151231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_ssp245_r4i1p1f1_gn_20160101-20161231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_ssp245_r4i1p1f1_gn_20170101-20171231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_ssp245_r4i1p1f1_gn_20180101-20181231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_ssp245_r4i1p1f1_gn_20190101-20191231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_ssp245_r4i1p1f1_gn_20200101-20201231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_ssp245_r4i1p1f1_gn_20210101-20211231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_ssp245_r4i1p1f1_gn_20220101-20221231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_ssp245_r4i1p1f1_gn_20230101-20231231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_ssp245_r4i1p1f1_gn_20240101-20241231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_ssp245_r4i1p1f1_gn_20250101-20251231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_ssp245_r4i1p1f1_gn_20260101-20261231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_ssp245_r4i1p1f1_gn_20270101-20271231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_ssp245_r4i1p1f1_gn_20280101-20281231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_ssp245_r4i1p1f1_gn_20290101-20291231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_ssp245_r4i1p1f1_gn_20300101-20301231.nc',\n", - " ]\n", - "\n", - " IN_HIST_RAW_r3 = [\n", - " \n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r3i1p1f1_gn_19690101-19691231.nc',\n", - " \n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r3i1p1f1_gn_19700101-19701231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r3i1p1f1_gn_19710101-19711231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r3i1p1f1_gn_19720101-19721231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r3i1p1f1_gn_19730101-19731231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r3i1p1f1_gn_19740101-19741231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r3i1p1f1_gn_19750101-19751231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r3i1p1f1_gn_19760101-19761231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r3i1p1f1_gn_19770101-19771231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r3i1p1f1_gn_19780101-19781231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r3i1p1f1_gn_19790101-19791231.nc',\n", - "\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r3i1p1f1_gn_19800101-19801231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r3i1p1f1_gn_19810101-19811231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r3i1p1f1_gn_19820101-19821231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r3i1p1f1_gn_19830101-19831231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r3i1p1f1_gn_19840101-19841231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r3i1p1f1_gn_19850101-19851231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r3i1p1f1_gn_19860101-19861231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r3i1p1f1_gn_19870101-19871231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r3i1p1f1_gn_19880101-19881231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r3i1p1f1_gn_19890101-19891231.nc',\n", - "\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r3i1p1f1_gn_19900101-19901231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r3i1p1f1_gn_19910101-19911231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r3i1p1f1_gn_19920101-19921231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r3i1p1f1_gn_19930101-19931231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r3i1p1f1_gn_19940101-19941231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r3i1p1f1_gn_19950101-19951231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r3i1p1f1_gn_19960101-19961231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r3i1p1f1_gn_19970101-19971231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r3i1p1f1_gn_19980101-19981231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r3i1p1f1_gn_19990101-19991231.nc',\n", - "\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r3i1p1f1_gn_20000101-20001231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r3i1p1f1_gn_20010101-20011231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r3i1p1f1_gn_20020101-20021231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r3i1p1f1_gn_20030101-20031231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r3i1p1f1_gn_20040101-20041231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r3i1p1f1_gn_20050101-20051231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r3i1p1f1_gn_20060101-20061231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r3i1p1f1_gn_20070101-20071231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r3i1p1f1_gn_20080101-20081231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r3i1p1f1_gn_20090101-20091231.nc',\n", - "\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r3i1p1f1_gn_20100101-20101231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r3i1p1f1_gn_20110101-20111231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r3i1p1f1_gn_20120101-20121231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r3i1p1f1_gn_20130101-20131231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r3i1p1f1_gn_20140101-20141231.nc',\n", - " \n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_ssp245_r3i1p1f1_gn_20150101-20151231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_ssp245_r3i1p1f1_gn_20160101-20161231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_ssp245_r3i1p1f1_gn_20170101-20171231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_ssp245_r3i1p1f1_gn_20180101-20181231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_ssp245_r3i1p1f1_gn_20190101-20191231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_ssp245_r3i1p1f1_gn_20200101-20201231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_ssp245_r3i1p1f1_gn_20210101-20211231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_ssp245_r3i1p1f1_gn_20220101-20221231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_ssp245_r3i1p1f1_gn_20230101-20231231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_ssp245_r3i1p1f1_gn_20240101-20241231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_ssp245_r3i1p1f1_gn_20250101-20251231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_ssp245_r3i1p1f1_gn_20260101-20261231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_ssp245_r3i1p1f1_gn_20270101-20271231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_ssp245_r3i1p1f1_gn_20280101-20281231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_ssp245_r3i1p1f1_gn_20290101-20291231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_ssp245_r3i1p1f1_gn_20300101-20301231.nc',\n", - " ]\n", - "\n", - " \n", - " \n", - " IN_HISTNAT_RAW_r1 = [\n", - " \n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r1i1p1f1_gn_19690101-19691231.nc',\n", - " \n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r1i1p1f1_gn_19700101-19701231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r1i1p1f1_gn_19710101-19711231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r1i1p1f1_gn_19720101-19721231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r1i1p1f1_gn_19730101-19731231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r1i1p1f1_gn_19740101-19741231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r1i1p1f1_gn_19750101-19751231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r1i1p1f1_gn_19760101-19761231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r1i1p1f1_gn_19770101-19771231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r1i1p1f1_gn_19780101-19781231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r1i1p1f1_gn_19790101-19791231.nc',\n", - "\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r1i1p1f1_gn_19800101-19801231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r1i1p1f1_gn_19810101-19811231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r1i1p1f1_gn_19820101-19821231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r1i1p1f1_gn_19830101-19831231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r1i1p1f1_gn_19840101-19841231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r1i1p1f1_gn_19850101-19851231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r1i1p1f1_gn_19860101-19861231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r1i1p1f1_gn_19870101-19871231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r1i1p1f1_gn_19880101-19881231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r1i1p1f1_gn_19890101-19891231.nc',\n", - "\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r1i1p1f1_gn_19900101-19901231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r1i1p1f1_gn_19910101-19911231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r1i1p1f1_gn_19920101-19921231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r1i1p1f1_gn_19930101-19931231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r1i1p1f1_gn_19940101-19941231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r1i1p1f1_gn_19950101-19951231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r1i1p1f1_gn_19960101-19961231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r1i1p1f1_gn_19970101-19971231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r1i1p1f1_gn_19980101-19981231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r1i1p1f1_gn_19990101-19991231.nc',\n", - "\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r1i1p1f1_gn_20000101-20001231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r1i1p1f1_gn_20010101-20011231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r1i1p1f1_gn_20020101-20021231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r1i1p1f1_gn_20030101-20031231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r1i1p1f1_gn_20040101-20041231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r1i1p1f1_gn_20050101-20051231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r1i1p1f1_gn_20060101-20061231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r1i1p1f1_gn_20070101-20071231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r1i1p1f1_gn_20080101-20081231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r1i1p1f1_gn_20090101-20091231.nc',\n", - "\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r1i1p1f1_gn_20100101-20101231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r1i1p1f1_gn_20110101-20111231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r1i1p1f1_gn_20120101-20121231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r1i1p1f1_gn_20130101-20131231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r1i1p1f1_gn_20140101-20141231.nc',\n", - " \n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r1i1p1f1_gn_20150101-20151231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r1i1p1f1_gn_20160101-20161231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r1i1p1f1_gn_20170101-20171231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r1i1p1f1_gn_20180101-20181231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r1i1p1f1_gn_20190101-20191231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r1i1p1f1_gn_20200101-20201231.nc',\n", - " ]\n", - "\n", - " \n", - " IN_HISTNAT_RAW_r2 = [\n", - " \n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r2i1p1f1_gn_19690101-19691231.nc',\n", - " \n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r2i1p1f1_gn_19700101-19701231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r2i1p1f1_gn_19710101-19711231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r2i1p1f1_gn_19720101-19721231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r2i1p1f1_gn_19730101-19731231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r2i1p1f1_gn_19740101-19741231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r2i1p1f1_gn_19750101-19751231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r2i1p1f1_gn_19760101-19761231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r2i1p1f1_gn_19770101-19771231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r2i1p1f1_gn_19780101-19781231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r2i1p1f1_gn_19790101-19791231.nc',\n", - "\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r2i1p1f1_gn_19800101-19801231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r2i1p1f1_gn_19810101-19811231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r2i1p1f1_gn_19820101-19821231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r2i1p1f1_gn_19830101-19831231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r2i1p1f1_gn_19840101-19841231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r2i1p1f1_gn_19850101-19851231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r2i1p1f1_gn_19860101-19861231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r2i1p1f1_gn_19870101-19871231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r2i1p1f1_gn_19880101-19881231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r2i1p1f1_gn_19890101-19891231.nc',\n", - "\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r2i1p1f1_gn_19900101-19901231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r2i1p1f1_gn_19910101-19911231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r2i1p1f1_gn_19920101-19921231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r2i1p1f1_gn_19930101-19931231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r2i1p1f1_gn_19940101-19941231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r2i1p1f1_gn_19950101-19951231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r2i1p1f1_gn_19960101-19961231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r2i1p1f1_gn_19970101-19971231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r2i1p1f1_gn_19980101-19981231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r2i1p1f1_gn_19990101-19991231.nc',\n", - "\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r2i1p1f1_gn_20000101-20001231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r2i1p1f1_gn_20010101-20011231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r2i1p1f1_gn_20020101-20021231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r2i1p1f1_gn_20030101-20031231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r2i1p1f1_gn_20040101-20041231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r2i1p1f1_gn_20050101-20051231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r2i1p1f1_gn_20060101-20061231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r2i1p1f1_gn_20070101-20071231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r2i1p1f1_gn_20080101-20081231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r2i1p1f1_gn_20090101-20091231.nc',\n", - "\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r2i1p1f1_gn_20100101-20101231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r2i1p1f1_gn_20110101-20111231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r2i1p1f1_gn_20120101-20121231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r2i1p1f1_gn_20130101-20131231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r2i1p1f1_gn_20140101-20141231.nc',\n", - " \n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r2i1p1f1_gn_20150101-20151231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r2i1p1f1_gn_20160101-20161231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r2i1p1f1_gn_20170101-20171231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r2i1p1f1_gn_20180101-20181231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r2i1p1f1_gn_20190101-20191231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r2i1p1f1_gn_20200101-20201231.nc',\n", - " ]\n", - "\n", - "\n", - "\n", - " IN_HISTNAT_RAW_r3 = [\n", - " \n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r3i1p1f1_gn_19690101-19691231.nc',\n", - " \n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r3i1p1f1_gn_19700101-19701231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r3i1p1f1_gn_19710101-19711231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r3i1p1f1_gn_19720101-19721231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r3i1p1f1_gn_19730101-19731231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r3i1p1f1_gn_19740101-19741231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r3i1p1f1_gn_19750101-19751231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r3i1p1f1_gn_19760101-19761231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r3i1p1f1_gn_19770101-19771231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r3i1p1f1_gn_19780101-19781231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r3i1p1f1_gn_19790101-19791231.nc',\n", - "\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r3i1p1f1_gn_19800101-19801231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r3i1p1f1_gn_19810101-19811231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r3i1p1f1_gn_19820101-19821231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r3i1p1f1_gn_19830101-19831231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r3i1p1f1_gn_19840101-19841231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r3i1p1f1_gn_19850101-19851231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r3i1p1f1_gn_19860101-19861231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r3i1p1f1_gn_19870101-19871231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r3i1p1f1_gn_19880101-19881231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r3i1p1f1_gn_19890101-19891231.nc',\n", - "\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r3i1p1f1_gn_19900101-19901231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r3i1p1f1_gn_19910101-19911231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r3i1p1f1_gn_19920101-19921231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r3i1p1f1_gn_19930101-19931231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r3i1p1f1_gn_19940101-19941231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r3i1p1f1_gn_19950101-19951231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r3i1p1f1_gn_19960101-19961231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r3i1p1f1_gn_19970101-19971231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r3i1p1f1_gn_19980101-19981231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r3i1p1f1_gn_19990101-19991231.nc',\n", - "\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r3i1p1f1_gn_20000101-20001231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r3i1p1f1_gn_20010101-20011231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r3i1p1f1_gn_20020101-20021231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r3i1p1f1_gn_20030101-20031231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r3i1p1f1_gn_20040101-20041231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r3i1p1f1_gn_20050101-20051231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r3i1p1f1_gn_20060101-20061231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r3i1p1f1_gn_20070101-20071231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r3i1p1f1_gn_20080101-20081231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r3i1p1f1_gn_20090101-20091231.nc',\n", - "\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r3i1p1f1_gn_20100101-20101231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r3i1p1f1_gn_20110101-20111231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r3i1p1f1_gn_20120101-20121231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r3i1p1f1_gn_20130101-20131231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r3i1p1f1_gn_20140101-20141231.nc',\n", - " \n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r3i1p1f1_gn_20150101-20151231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r3i1p1f1_gn_20160101-20161231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r3i1p1f1_gn_20170101-20171231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r3i1p1f1_gn_20180101-20181231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r3i1p1f1_gn_20190101-20191231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r3i1p1f1_gn_20200101-20201231.nc',\n", - " ]\n", - "\n", - "\n", - " IN_PICONTROL_RAW = [ \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-f3-L_piControl_r1i1p1f1_gr_11500101-11591231.nc\" ]\n", - " \n", - " PiControl_year_start = \"1150\" \n", - " \n", - "elif source_id == \"MIROC6\":\n", - "\n", - " IN_HIST_RAW_r1 = [\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_historical_r1i1p1f1_gn_19600101-19691231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_historical_r1i1p1f1_gn_19700101-19791231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_historical_r1i1p1f1_gn_19800101-19891231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_historical_r1i1p1f1_gn_19900101-19991231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_historical_r1i1p1f1_gn_20000101-20091231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_historical_r1i1p1f1_gn_20100101-20141231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_ssp245_r1i1p1f1_gn_20150101-20241231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_ssp245_r1i1p1f1_gn_20250101-20341231.nc',\n", - " ]\n", - "\n", - " IN_HIST_RAW_r2 = [\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_historical_r2i1p1f1_gn_19600101-19691231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_historical_r2i1p1f1_gn_19700101-19791231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_historical_r2i1p1f1_gn_19800101-19891231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_historical_r2i1p1f1_gn_19900101-19991231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_historical_r2i1p1f1_gn_20000101-20091231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_historical_r2i1p1f1_gn_20100101-20141231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_ssp245_r2i1p1f1_gn_20150101-20241231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_ssp245_r2i1p1f1_gn_20250101-20341231.nc',\n", - " ]\n", - " \n", - " IN_HIST_RAW_r3 = [\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_historical_r3i1p1f1_gn_19600101-19691231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_historical_r3i1p1f1_gn_19700101-19791231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_historical_r3i1p1f1_gn_19800101-19891231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_historical_r3i1p1f1_gn_19900101-19991231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_historical_r3i1p1f1_gn_20000101-20091231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_historical_r3i1p1f1_gn_20100101-20141231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_ssp245_r3i1p1f1_gn_20150101-20241231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_ssp245_r3i1p1f1_gn_20250101-20341231.nc',\n", - " ]\n", - " \n", - " IN_HISTNAT_RAW_r1 = [\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_hist-nat_r1i1p1f1_gn_19600101-19691231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_hist-nat_r1i1p1f1_gn_19700101-19791231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_hist-nat_r1i1p1f1_gn_19800101-19891231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_hist-nat_r1i1p1f1_gn_19900101-19991231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_hist-nat_r1i1p1f1_gn_20000101-20091231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_hist-nat_r1i1p1f1_gn_20100101-20191231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_hist-nat_r1i1p1f1_gn_20200101-20201231.nc'\n", - " ]\n", - " \n", - " IN_HISTNAT_RAW_r2 = [\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_hist-nat_r2i1p1f1_gn_19600101-19691231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_hist-nat_r2i1p1f1_gn_19700101-19791231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_hist-nat_r2i1p1f1_gn_19800101-19891231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_hist-nat_r2i1p1f1_gn_19900101-19991231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_hist-nat_r2i1p1f1_gn_20000101-20091231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_hist-nat_r2i1p1f1_gn_20100101-20191231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_hist-nat_r2i1p1f1_gn_20200101-20201231.nc'\n", - " ]\n", - "\n", - "\n", - " IN_HISTNAT_RAW_r3 = [\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_hist-nat_r3i1p1f1_gn_19600101-19691231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_hist-nat_r3i1p1f1_gn_19700101-19791231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_hist-nat_r3i1p1f1_gn_19800101-19891231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_hist-nat_r3i1p1f1_gn_19900101-19991231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_hist-nat_r3i1p1f1_gn_20000101-20091231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_hist-nat_r3i1p1f1_gn_20100101-20191231.nc',\n", - " '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_hist-nat_r3i1p1f1_gn_20200101-20201231.nc'\n", - " ]\n", - "\n", - " IN_PICONTROL_RAW = [ \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_piControl_r1i1p1f1_gn_36700101-36791231.nc\" ,\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_piControl_r1i1p1f1_gn_36800101-36891231.nc\",\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_piControl_r1i1p1f1_gn_36900101-36991231.nc\"]\n", - " \n", - " \n", - " PiControl_year_start = \"3670\"\n", - " \n", - "elif source_id == \"IPSL-CM6A-LR\":\n", - " IN_HIST_RAW_r1 = [\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/IPSL/tas_day_IPSL-CM6A-LR_historical_r1i1p1f1_gr_18500101-20141231.nc\",\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/IPSL/tas_day_IPSL-CM6A-LR_ssp245_r1i1p1f1_gr_20150101-21001231.nc\",\n", - " ]\n", - " IN_HIST_RAW_r2 = [\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw//tas/IPSL/tas_day_IPSL-CM6A-LR_historical_r2i1p1f1_gr_18500101-20141231.nc\",\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/IPSL/tas_day_IPSL-CM6A-LR_ssp245_r2i1p1f1_gr_20150101-21001231.nc\",\n", - " ]\n", - " IN_HIST_RAW_r3 = [\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/IPSL/tas_day_IPSL-CM6A-LR_historical_r3i1p1f1_gr_18500101-20141231.nc\",\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/IPSL/tas_day_IPSL-CM6A-LR_ssp245_r4i1p1f1_gr_20150101-21001231.nc\",\n", - " ]\n", - " \n", - " \n", - " IN_HISTNAT_RAW_r1 = [\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/IPSL/tas_day_IPSL-CM6A-LR_hist-nat_r1i1p1f1_gr_18500101-20201231.nc\",\n", - " ]\n", - " IN_HISTNAT_RAW_r2 = [\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw//tas/IPSL/tas_day_IPSL-CM6A-LR_hist-nat_r2i1p1f1_gr_18500101-20201231.nc\",\n", - " ]\n", - " IN_HISTNAT_RAW_r3 = [\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/IPSL/tas_day_IPSL-CM6A-LR_hist-nat_r6i1p1f1_gr_18500101-20201231.nc\",\n", - " ]\n", - " \n", - " IN_PICONTROL_RAW = [ \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/IPSL/tas_day_IPSL-CM6A-LR_piControl_r1i2p1f1_gr_18500101-20981231.nc\" ]\n", - " \n", - " PiControl_year_start = \"1850\"\n", - " \n", - "elif source_id == \"MRI-ESM2-0\":\n", - " IN_HIST_RAW_r1 = [\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_{}_historical_r1i1p1f1_{}_19500101-19991231.nc\".format(institution_id,source_id,grid_label),\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_{}_historical_r1i1p1f1_{}_20000101-20141231.nc\".format(institution_id,source_id,grid_label),\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_{}_ssp245_r1i1p1f1_{}_20150101-20641231.nc\".format(institution_id,source_id,grid_label),\n", - " ]\n", - " IN_HIST_RAW_r2 = [\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_{}_historical_r2i1p1f1_{}_19500101-19991231.nc\".format(institution_id,source_id,grid_label),\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_{}_historical_r2i1p1f1_{}_20000101-20141231.nc\".format(institution_id,source_id,grid_label),\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_{}_ssp245_r2i1p1f1_{}_20150101-20301231.nc\".format(institution_id,source_id,grid_label),\n", - " ]\n", - " IN_HIST_RAW_r3 = [\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_{}_historical_r3i1p1f1_{}_19500101-19991231.nc\".format(institution_id,source_id,grid_label),\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_{}_historical_r3i1p1f1_{}_20000101-20141231.nc\".format(institution_id,source_id,grid_label),\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_{}_ssp245_r3i1p1f1_{}_20150101-20301231.nc\".format(institution_id,source_id,grid_label),\n", - " ]\n", - " \n", - " IN_HISTNAT_RAW_r1 = [\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_{}_hist-nat_r1i1p1f1_{}_19500101-19991231.nc\".format(institution_id,source_id,grid_label),\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_{}_hist-nat_r1i1p1f1_{}_20000101-20141231.nc\".format(institution_id,source_id,grid_label),\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_{}_hist-nat_r1i1p1f1_{}_20150101-20201231.nc\".format(institution_id,source_id,grid_label),\n", - " ]\n", - " IN_HISTNAT_RAW_r2 = [\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw//tas/{}/tas_day_{}_hist-nat_r2i1p1f1_{}_19500101-19991231.nc\".format(institution_id,source_id,grid_label),\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_{}_hist-nat_r2i1p1f1_{}_20000101-20201231.nc\".format(institution_id,source_id,grid_label),\n", - " ]\n", - " IN_HISTNAT_RAW_r3 = [\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_{}_hist-nat_r3i1p1f1_{}_19500101-19991231.nc\".format(institution_id,source_id,grid_label),\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_{}_hist-nat_r3i1p1f1_{}_20000101-20141231.nc\".format(institution_id,source_id,grid_label),\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_{}_hist-nat_r3i1p1f1_{}_20150101-20201231.nc\".format(institution_id,source_id,grid_label),\n", - " ]\n", - " \n", - " IN_PICONTROL_RAW = [ \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MRI/tas_day_MRI-ESM2-0_piControl_r1i1p1f1_gn_20000101-20491231.nc\" ]\n", - " \n", - " PiControl_year_start = \"2000\"\n", - " \n", - "elif source_id == \"ACCESS-CM2\":\n", - " IN_HIST_RAW_r1 = [\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_{}_historical_r1i1p1f1_{}_19500101-19991231.nc\".format(institution_id,source_id,grid_label),\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_{}_historical_r1i1p1f1_{}_20000101-20141231.nc\".format(institution_id,source_id,grid_label),\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_{}_ssp245_r1i1p1f1_{}_20150101-20641231.nc\".format(institution_id,source_id,grid_label),\n", - " ]\n", - " IN_HIST_RAW_r2 = [\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_{}_historical_r2i1p1f1_{}_19500101-19991231.nc\".format(institution_id,source_id,grid_label),\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_{}_historical_r2i1p1f1_{}_20000101-20141231.nc\".format(institution_id,source_id,grid_label),\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_{}_ssp245_r2i1p1f1_{}_20150101-20641231.nc\".format(institution_id,source_id,grid_label),\n", - " ]\n", - " IN_HIST_RAW_r3 = [\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_{}_historical_r3i1p1f1_{}_19500101-19991231.nc\".format(institution_id,source_id,grid_label),\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_{}_historical_r3i1p1f1_{}_20000101-20141231.nc\".format(institution_id,source_id,grid_label),\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_{}_ssp245_r3i1p1f1_{}_20150101-20641231.nc\".format(institution_id,source_id,grid_label),\n", - " ]\n", - " \n", - " IN_HISTNAT_RAW_r1 = [\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_{}_hist-nat_r1i1p1f1_{}_19500101-19991231.nc\".format(institution_id,source_id,grid_label),\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_{}_hist-nat_r1i1p1f1_{}_20000101-20201231.nc\".format(institution_id,source_id,grid_label),\n", - " ]\n", - " IN_HISTNAT_RAW_r2 = [\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw//tas/{}/tas_day_{}_hist-nat_r2i1p1f1_{}_19500101-19991231.nc\".format(institution_id,source_id,grid_label),\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_{}_hist-nat_r2i1p1f1_{}_20000101-20201231.nc\".format(institution_id,source_id,grid_label),\n", - " ]\n", - " IN_HISTNAT_RAW_r3 = [\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_{}_hist-nat_r3i1p1f1_{}_19500101-19991231.nc\".format(institution_id,source_id,grid_label),\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_{}_hist-nat_r3i1p1f1_{}_20000101-20201231.nc\".format(institution_id,source_id,grid_label),\n", - " ]\n", - " \n", - " IN_PICONTROL_RAW = [ \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CSIRO-ARCCSS/tas_day_ACCESS-CM2_piControl_r1i1p1f1_gn_14000101-14491231.nc\" ]\n", - " \n", - " PiControl_year_start = \"1400\"\n", - " \n", - "elif source_id == \"ACCESS-ESM1-5\":\n", - "\n", - " IN_HIST_RAW_r1 = [\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_{}_historical_r1i1p1f1_{}_19500101-19991231.nc\".format(institution_id,source_id,grid_label),\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_{}_historical_r1i1p1f1_{}_20000101-20141231.nc\".format(institution_id,source_id,grid_label),\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_{}_ssp245_r1i1p1f1_{}_20150101-20641231.nc\".format(institution_id,source_id,grid_label),\n", - " ]\n", - " IN_HIST_RAW_r2 = [\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_{}_historical_r2i1p1f1_{}_19500101-19991231.nc\".format(institution_id,source_id,grid_label),\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_{}_historical_r2i1p1f1_{}_20000101-20141231.nc\".format(institution_id,source_id,grid_label),\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_{}_ssp245_r2i1p1f1_{}_20150101-20641231.nc\".format(institution_id,source_id,grid_label),\n", - " ]\n", - " IN_HIST_RAW_r3 = [\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_{}_historical_r3i1p1f1_{}_19500101-19991231.nc\".format(institution_id,source_id,grid_label),\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_{}_historical_r3i1p1f1_{}_20000101-20141231.nc\".format(institution_id,source_id,grid_label),\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_{}_ssp245_r3i1p1f1_{}_20150101-20641231.nc\".format(institution_id,source_id,grid_label),\n", - " ]\n", - " \n", - " IN_HISTNAT_RAW_r1 = [\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_{}_hist-nat_r1i1p1f1_{}_19500101-19991231.nc\".format(institution_id,source_id,grid_label),\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_{}_hist-nat_r1i1p1f1_{}_20000101-20201231.nc\".format(institution_id,source_id,grid_label),\n", - " ]\n", - " IN_HISTNAT_RAW_r2 = [\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw//tas/{}/tas_day_{}_hist-nat_r2i1p1f1_{}_19500101-19991231.nc\".format(institution_id,source_id,grid_label),\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_{}_hist-nat_r2i1p1f1_{}_20000101-20201231.nc\".format(institution_id,source_id,grid_label),\n", - " ]\n", - " IN_HISTNAT_RAW_r3 = [\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_{}_hist-nat_r3i1p1f1_{}_19500101-19991231.nc\".format(institution_id,source_id,grid_label),\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_{}_hist-nat_r3i1p1f1_{}_20000101-20201231.nc\".format(institution_id,source_id,grid_label),\n", - " ]\n", - " \n", - " IN_PICONTROL_RAW = [ \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CSIRO/tas_day_ACCESS-ESM1-5_piControl_r1i1p1f1_gn_10510101-11001231.nc\" ]\n", - " \n", - " PiControl_year_start = \"1051\"\n", - " \n", - "elif source_id == \"NorESM2-LM\":\n", - " IN_HISTNAT_RAW_r1 = [\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_NorESM2-LM_hist-nat_r1i1p1f1_gn_19700101-19791231.nc\".format(institution_id),\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_NorESM2-LM_hist-nat_r1i1p1f1_gn_19800101-19891231.nc\".format(institution_id),\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_NorESM2-LM_hist-nat_r1i1p1f1_gn_19900101-19991231.nc\".format(institution_id),\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_NorESM2-LM_hist-nat_r1i1p1f1_gn_20000101-20091231.nc\".format(institution_id), \n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_NorESM2-LM_hist-nat_r1i1p1f1_gn_20100101-20141231.nc\".format(institution_id),\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_NorESM2-LM_hist-nat_r1i1p1f1_gn_20150101-20201231.nc\".format(institution_id),\n", - " ]\n", - "\n", - " IN_HISTNAT_RAW_r2 = [\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_NorESM2-LM_hist-nat_r2i1p1f1_gn_19700101-19791231.nc\".format(institution_id),\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_NorESM2-LM_hist-nat_r2i1p1f1_gn_19800101-19891231.nc\".format(institution_id),\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_NorESM2-LM_hist-nat_r2i1p1f1_gn_19900101-19991231.nc\".format(institution_id),\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_NorESM2-LM_hist-nat_r2i1p1f1_gn_20000101-20091231.nc\".format(institution_id), \n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_NorESM2-LM_hist-nat_r2i1p1f1_gn_20100101-20141231.nc\".format(institution_id),\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_NorESM2-LM_hist-nat_r2i1p1f1_gn_20150101-20201231.nc\".format(institution_id),\n", - " ]\n", - "\n", - " IN_HISTNAT_RAW_r3 = [\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_NorESM2-LM_hist-nat_r3i1p1f1_gn_19700101-19791231.nc\".format(institution_id),\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_NorESM2-LM_hist-nat_r3i1p1f1_gn_19800101-19891231.nc\".format(institution_id),\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_NorESM2-LM_hist-nat_r3i1p1f1_gn_19900101-19991231.nc\".format(institution_id),\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_NorESM2-LM_hist-nat_r3i1p1f1_gn_20000101-20091231.nc\".format(institution_id), \n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_NorESM2-LM_hist-nat_r3i1p1f1_gn_20100101-20141231.nc\".format(institution_id),\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_NorESM2-LM_hist-nat_r3i1p1f1_gn_20150101-20201231.nc\".format(institution_id),\n", - " ]\n", - "\n", - "\n", - " IN_HIST_RAW_r1 = [\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_NorESM2-LM_historical_r1i1p1f1_gn_19700101-19791231.nc\".format(institution_id),\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_NorESM2-LM_historical_r1i1p1f1_gn_19800101-19891231.nc\".format(institution_id),\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_NorESM2-LM_historical_r1i1p1f1_gn_19900101-19991231.nc\".format(institution_id),\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_NorESM2-LM_historical_r1i1p1f1_gn_20000101-20091231.nc\".format(institution_id), \n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_NorESM2-LM_historical_r1i1p1f1_gn_20100101-20141231.nc\".format(institution_id),\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_NorESM2-LM_ssp245_r1i1p1f1_gn_20150101-20201231.nc\".format(institution_id),\n", - " ]\n", - "\n", - " IN_HIST_RAW_r2 = [\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_NorESM2-LM_historical_r2i1p1f1_gn_19700101-19791231.nc\".format(institution_id),\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_NorESM2-LM_historical_r2i1p1f1_gn_19800101-19891231.nc\".format(institution_id),\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_NorESM2-LM_historical_r2i1p1f1_gn_19900101-19991231.nc\".format(institution_id),\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_NorESM2-LM_historical_r2i1p1f1_gn_20000101-20091231.nc\".format(institution_id), \n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_NorESM2-LM_historical_r2i1p1f1_gn_20100101-20141231.nc\".format(institution_id),\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_NorESM2-LM_ssp245_r2i1p1f1_gn_20150101-20201231.nc\".format(institution_id),\n", - " ]\n", - "\n", - " IN_HIST_RAW_r3 = [\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_NorESM2-LM_historical_r3i1p1f1_gn_19700101-19791231.nc\".format(institution_id),\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_NorESM2-LM_historical_r3i1p1f1_gn_19800101-19891231.nc\".format(institution_id),\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_NorESM2-LM_historical_r3i1p1f1_gn_19900101-19991231.nc\".format(institution_id),\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_NorESM2-LM_historical_r3i1p1f1_gn_20000101-20091231.nc\".format(institution_id), \n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_NorESM2-LM_historical_r3i1p1f1_gn_20100101-20141231.nc\".format(institution_id),\n", - " \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/{}/tas_day_NorESM2-LM_ssp245_r3i1p1f1_gn_20150101-20201231.nc\".format(institution_id),\n", - " ]\n", - " \n", - " IN_PICONTROL_RAW_1 = [ \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/NCC/tas_day_NorESM2-LM_piControl_r1i1p1f1_gn_20710101-20801231.nc\"]\n", - " IN_PICONTROL_RAW_2 = [ \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/NCC/tas_day_NorESM2-LM_piControl_r1i1p1f1_gn_20810101-20901231.nc\"]\n", - " IN_PICONTROL_RAW_3 = [ \"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/NCC/tas_day_NorESM2-LM_piControl_r1i1p1f1_gn_20910101-21001231.nc\"]\n", - " \n", - " PiControl_year_start = \"2071\"\n", - "\n", - "IN_ERA5_RAW = \"/gcs/impactlab-data/climate/source_data/ERA-5/tas/daily/netcdf/v1.1/tas_daily_*.nc\"\n", - "\n", - "OUT_HIST_r1_URL = f\"{SCRATCH_URL}/clean/hist/{source_id}/tas_r1.zarr\"\n", - "OUT_HISTNAT_r1_URL = f\"{SCRATCH_URL}/clean/histnat/{source_id}/tas_r1.zarr\"\n", - "OUT_HIST_r2_URL = f\"{SCRATCH_URL}/clean/hist/{source_id}/tas_r2.zarr\"\n", - "OUT_HISTNAT_r2_URL = f\"{SCRATCH_URL}/clean/histnat/{source_id}/tas_r2.zarr\"\n", - "OUT_HIST_r3_URL = f\"{SCRATCH_URL}/clean/hist/{source_id}/tas_r3.zarr\"\n", - "OUT_HISTNAT_r3_URL = f\"{SCRATCH_URL}/clean/histnat/{source_id}/tas_r3.zarr\"\n", - "\n", - "OUT_ERA5_URL = f\"{SCRATCH_URL}/clean/ERA5/{source_id}/tas.zarr\"\n", - "\n", - "\n", - "print(OUT_HIST_r1_URL)\n", - "print(OUT_HISTNAT_r2_URL)\n", - "print(OUT_ERA5_URL)" + "importlib.reload(esm)\n" ] }, { "cell_type": "code", - "execution_count": 5, - "id": "0ed30c6f", + "execution_count": 25, + "id": "61f6cb86-e355-406f-9ad8-18e92bdf7e79", "metadata": {}, "outputs": [], "source": [ - "gateway = Gateway()" + "import sys\n", + "sys.path.append('../../src') # Add src directory to Python path\n", + "import ESM_info as esm" ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 3, + "id": "5222301c-7492-43eb-858b-fe314046b167", + "metadata": {}, + "outputs": [], + "source": [ + "uid = str(uuid.uuid4())\n", + "now = datetime.now()\n", + "\n", + "JUPYTER_IMAGE = os.environ['JUPYTER_IMAGE']\n", + "START_TIME = now.strftime(\"%Y-%m-%d %H:%M:%S\")" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "c6b44581-6686-4fbb-b4a6-8528a6ec7170", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + " JUPYTER_IMAGE='pangeo/pangeo-notebook:2024.11.11@sha256:16ac7877726fc5ac25070f863a901945ae678e11a299585f3c404cb3d3a34a24'\n", + " START_TIME='2025-08-05 16:21:08'\n", + " \n" + ] + } + ], + "source": [ + "print(\n", + " f\"\"\"\n", + " {JUPYTER_IMAGE=}\n", + " {START_TIME=}\n", + " \"\"\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 5, "id": "b0774350-02a7-418c-a2b7-8d499348a1fb", "metadata": {}, "outputs": [], "source": [ - "ERA5_out = xr.Dataset({'lat': (['lat'], np.arange(-90, 90, 0.25)),\n", + "ERA5_out_grid = xr.Dataset({'lat': (['lat'], np.arange(-90, 90, 0.25)),\n", " 'lon': (['lon'], np.arange(-0, 360, 0.25))})" ] }, @@ -815,6 +174,28 @@ "## Cleaning and standardizing raw data" ] }, + { + "cell_type": "code", + "execution_count": 6, + "id": "f5d732f9-a766-43d1-9bd4-578f6b46dc7f", + "metadata": {}, + "outputs": [], + "source": [ + "# source_id = 'MIROC6'\n", + "# institution_id = 'MIROC'\n", + "# grid_label = 'gn'\n", + "var = \"tas\"\n", + "\n", + "gcms = [\n", + " # \"CanESM5\",\n", + " \"MIROC6\",\n", + " \"MRI-ESM2-0\",\n", + " \"NorESM2-LM\",\n", + " \"ACCESS-CM2\",\n", + " \"ACCESS-ESM1-5\"\n", + "]" + ] + }, { "cell_type": "code", "execution_count": 7, @@ -822,8 +203,15 @@ "metadata": {}, "outputs": [], "source": [ - "def standardize(ds_in):\n", - " ds = ds_in.sel(time=slice(\"1979\", \"2020\"))\n", + "def standardize(ds_in, endyear=2020):\n", + " # select a time period starting in 1979 and ending in `endyear`\n", + " # convert coordinate names to standardized names\n", + " # Convert to a noleap calendar\n", + " # TODO: remove leap days here?\n", + " # Chunk to keep all of time together\n", + "\n", + " \n", + " ds = ds_in.sel(time=slice(\"1979\", str(endyear)))\n", " ds = ds.squeeze(drop=True)\n", "\n", " if \"longitude\" in ds.variables:\n", @@ -831,418 +219,1835 @@ " if \"latitude\" in ds.variables:\n", " ds = ds.rename({\"latitude\": \"lat\"})\n", " \n", - " ds = convert_calendar(ds, target=\"noleap\")\n", + " ds = xr.Dataset.convert_calendar(ds, calendar=\"noleap\")\n", " ds = ds.chunk({\"time\": -1, \"lat\": 2, \"lon\": -1})\n", "\n", - " return ds" + " return ds\n", + "\n", + "def noleap_date_range(start=\"2021-01-01\", periods=3650, freq=\"D\"):\n", + "\n", + " # create a 10-year daily date range, starting with `start`\n", + " return xr.date_range(\n", + " start=start,\n", + " periods=periods,\n", + " freq=freq,\n", + " calendar=\"noleap\",\n", + " use_cftime=True\n", + " )" ] }, { "cell_type": "code", "execution_count": 8, - "id": "aadd0b96-62d4-48b5-b5b3-8168f03ea9d9", + "id": "b6dea146", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "gs://impactlab-data-scratch/kemccusker/c5a7031e-064a-4353-803d-aca618565096/clean_2025-08-01\n" + ] + } + ], "source": [ - "def standardize_2030(ds_in):\n", - " ds = ds_in.sel(time=slice(\"1979\", \"2030\"))\n", - " ds = ds.squeeze(drop=True)\n", - "\n", - " if \"longitude\" in ds.variables:\n", - " ds = ds.rename({\"longitude\": \"lon\"})\n", - " if \"latitude\" in ds.variables:\n", - " ds = ds.rename({\"latitude\": \"lat\"})\n", - " \n", - " ds = convert_calendar(ds, target=\"noleap\")\n", - " ds = ds.chunk({\"time\": -1, \"lat\": 2, \"lon\": -1})\n", + "# tmp_uri = f\"{os.environ['CIL_SCRATCH_PREFIX']}/{os.environ['JUPYTERHUB_USER']}/{uid}\"\n", + "# OUT_BASE_URL = f\"{tmp_uri}/clean_{now.strftime(\"%Y-%m-%d\")}\"\n", "\n", - " return ds" + "OUT_BASE_URL = \"gs://impactlab-data-scratch/kemccusker/c5a7031e-064a-4353-803d-aca618565096/clean_2025-08-01\"\n", + "print(OUT_BASE_URL)\n", + "\n" ] }, { "cell_type": "code", - "execution_count": 9, - "id": "3b6be09c-63e3-4079-abcb-377444bbeaac", + "execution_count": null, + "id": "67b2a507-5a9e-40fd-92f0-dcd866f5764c", "metadata": {}, "outputs": [], - "source": [ - "GCM_file_paths = [\n", - " (IN_HIST_RAW_r1, OUT_HIST_r1_URL),\n", - " (IN_HIST_RAW_r2, OUT_HIST_r2_URL),\n", - " (IN_HIST_RAW_r3, OUT_HIST_r3_URL),\n", - " (IN_HISTNAT_RAW_r1, OUT_HISTNAT_r1_URL),\n", - " (IN_HISTNAT_RAW_r2, OUT_HISTNAT_r2_URL),\n", - " (IN_HISTNAT_RAW_r3, OUT_HISTNAT_r3_URL),\n", - " (IN_ERA5_RAW, OUT_ERA5_URL),\n", - "]" - ] + "source": [] }, { "cell_type": "code", - "execution_count": 10, - "id": "67d45c26-2be7-4991-ae3a-e6f833e6ed04", - "metadata": { - "tags": [] - }, + "execution_count": 20, + "id": "78312c6e-f413-482b-90d6-b0726bdd0b7e", + "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "https://notebooks.cilresearch.org/services/dask-gateway/clusters/jhub.91e1bc6f8c6840b79d9f7aca32071745/status\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "
\n", - "
\n", - "

Client

\n", - "

Client-b90cf8e4-42ed-11ee-a28a-fa58f7945e32

\n", - " \n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "\n", - "
Connection method: Cluster objectCluster type: dask_gateway.GatewayCluster
\n", - " Dashboard: https://notebooks.cilresearch.org/services/dask-gateway/clusters/jhub.91e1bc6f8c6840b79d9f7aca32071745/status\n", - "
\n", - "\n", - " \n", - " \n", - " \n", - "\n", - " \n", - "
\n", - "

Cluster Info

\n", - "
\n", - "

GatewayCluster

\n", - " \n", - "
\n", - "\n", - "
\n", - " \n", - "\n", - "
\n", - "
" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Saved zarr file: gs://impactlab-data-scratch/xq3rogfmpG12ATOWhGCYPNdSspH_vxlj/clean/hist/MIROC6/tas_r1.zarr\n", - "Saved zarr file: gs://impactlab-data-scratch/xq3rogfmpG12ATOWhGCYPNdSspH_vxlj/clean/hist/MIROC6/tas_r2.zarr\n", - "Saved zarr file: gs://impactlab-data-scratch/xq3rogfmpG12ATOWhGCYPNdSspH_vxlj/clean/hist/MIROC6/tas_r3.zarr\n" + "https://notebooks.cilresearch.org/services/dask-gateway/clusters/jhub.0d0c44f0faa64553916d2d349335cf50/status\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "/srv/conda/envs/notebook/lib/python3.10/site-packages/xarray/coding/times.py:710: SerializationWarning: Unable to decode time axis into full numpy.datetime64 objects, continuing using cftime.datetime objects instead, reason: dates out of range\n", - " dtype = _decode_cf_datetime_dtype(data, units, calendar, self.use_cftime)\n", - "/srv/conda/envs/notebook/lib/python3.10/site-packages/xarray/core/indexing.py:524: SerializationWarning: Unable to decode time axis into full numpy.datetime64 objects, continuing using cftime.datetime objects instead, reason: dates out of range\n", - " return np.asarray(array[self.key], dtype=None)\n", - "/srv/conda/envs/notebook/lib/python3.10/site-packages/xarray/coding/times.py:710: SerializationWarning: Unable to decode time axis into full numpy.datetime64 objects, continuing using cftime.datetime objects instead, reason: dates out of range\n", - " dtype = _decode_cf_datetime_dtype(data, units, calendar, self.use_cftime)\n", - "/srv/conda/envs/notebook/lib/python3.10/site-packages/xarray/core/indexing.py:524: SerializationWarning: Unable to decode time axis into full numpy.datetime64 objects, continuing using cftime.datetime objects instead, reason: dates out of range\n", - " return np.asarray(array[self.key], dtype=None)\n", - "/srv/conda/envs/notebook/lib/python3.10/site-packages/xarray/coding/times.py:710: SerializationWarning: Unable to decode time axis into full numpy.datetime64 objects, continuing using cftime.datetime objects instead, reason: dates out of range\n", - " dtype = _decode_cf_datetime_dtype(data, units, calendar, self.use_cftime)\n", - "/srv/conda/envs/notebook/lib/python3.10/site-packages/xarray/core/indexing.py:524: SerializationWarning: Unable to decode time axis into full numpy.datetime64 objects, continuing using cftime.datetime objects instead, reason: dates out of range\n", - " return np.asarray(array[self.key], dtype=None)\n" + "2025-08-05 20:17:14,762 - distributed.client - ERROR - Failed to reconnect to scheduler after 30.00 seconds, closing client\n" ] - }, + } + ], + "source": [ + "cluster = GatewayCluster(worker_image=JUPYTER_IMAGE, scheduler_image=JUPYTER_IMAGE)\n", + "client = cluster.get_client()\n", + "print(client.dashboard_link)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "b42439d9-af30-4222-86fa-7f12994238b9", + "metadata": {}, + "outputs": [], + "source": [ + "cluster.scale(50)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "67d45c26-2be7-4991-ae3a-e6f833e6ed04", + "metadata": { + "tags": [] + }, + "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Saved zarr file: gs://impactlab-data-scratch/xq3rogfmpG12ATOWhGCYPNdSspH_vxlj/clean/histnat/MIROC6/tas_r1.zarr\n" + "MIROC6\n", + "r1, hist\n", + "opening ['/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_historical_r1i1p1f1_gn_19600101-19691231.nc', '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_historical_r1i1p1f1_gn_19700101-19791231.nc', '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_historical_r1i1p1f1_gn_19800101-19891231.nc', '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_historical_r1i1p1f1_gn_19900101-19991231.nc', '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_historical_r1i1p1f1_gn_20000101-20091231.nc', '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_historical_r1i1p1f1_gn_20100101-20141231.nc', '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_ssp245_r1i1p1f1_gn_20150101-20241231.nc', '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_ssp245_r1i1p1f1_gn_20250101-20341231.nc']\n", + "clean_gcm Size: 3GB\n", + "Dimensions: (time: 18980, bnds: 2, lat: 128, lon: 256)\n", + "Coordinates:\n", + " * time (time) object 152kB 1979-01-01 12:00:00 ... 2030-12-31 12:00:00\n", + " * lat (lat) float64 1kB -88.93 -87.54 -86.14 ... 86.14 87.54 88.93\n", + " * lon (lon) float64 2kB 0.0 1.406 2.812 4.219 ... 355.8 357.2 358.6\n", + " height float64 8B 2.0\n", + "Dimensions without coordinates: bnds\n", + "Data variables:\n", + " time_bnds (time, bnds) datetime64[ns] 304kB dask.array\n", + " lat_bnds (time, lat, bnds) float64 39MB dask.array\n", + " lon_bnds (time, lon, bnds) float64 78MB dask.array\n", + " tas (time, lat, lon) float32 2GB dask.array\n", + "Attributes: (12/44)\n", + " Conventions: CF-1.7 CMIP-6.2\n", + " activity_id: CMIP\n", + " branch_method: standard\n", + " branch_time_in_child: 0.0\n", + " branch_time_in_parent: 0.0\n", + " creation_date: 2019-08-30T07:59:51Z\n", + " ... ...\n", + " title: MIROC6 output prepared for CMIP6\n", + " variable_id: tas\n", + " variant_label: r1i1p1f1\n", + " license: CMIP6 model data produced by MIROC is licensed un...\n", + " cmor_version: 3.4.0\n", + " tracking_id: hdl:21.14100/2863a95a-869d-4c80-9611-d73843d719a0\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "/srv/conda/envs/notebook/lib/python3.10/site-packages/xarray/coding/times.py:710: SerializationWarning: Unable to decode time axis into full numpy.datetime64 objects, continuing using cftime.datetime objects instead, reason: dates out of range\n", - " dtype = _decode_cf_datetime_dtype(data, units, calendar, self.use_cftime)\n", - "/srv/conda/envs/notebook/lib/python3.10/site-packages/xarray/core/indexing.py:524: SerializationWarning: Unable to decode time axis into full numpy.datetime64 objects, continuing using cftime.datetime objects instead, reason: dates out of range\n", - " return np.asarray(array[self.key], dtype=None)\n", - "/srv/conda/envs/notebook/lib/python3.10/site-packages/xarray/coding/times.py:710: SerializationWarning: Unable to decode time axis into full numpy.datetime64 objects, continuing using cftime.datetime objects instead, reason: dates out of range\n", - " dtype = _decode_cf_datetime_dtype(data, units, calendar, self.use_cftime)\n", - "/srv/conda/envs/notebook/lib/python3.10/site-packages/xarray/core/indexing.py:524: SerializationWarning: Unable to decode time axis into full numpy.datetime64 objects, continuing using cftime.datetime objects instead, reason: dates out of range\n", - " return np.asarray(array[self.key], dtype=None)\n", - "/srv/conda/envs/notebook/lib/python3.10/site-packages/xarray/coding/times.py:710: SerializationWarning: Unable to decode time axis into full numpy.datetime64 objects, continuing using cftime.datetime objects instead, reason: dates out of range\n", - " dtype = _decode_cf_datetime_dtype(data, units, calendar, self.use_cftime)\n", - "/srv/conda/envs/notebook/lib/python3.10/site-packages/xarray/core/indexing.py:524: SerializationWarning: Unable to decode time axis into full numpy.datetime64 objects, continuing using cftime.datetime objects instead, reason: dates out of range\n", - " return np.asarray(array[self.key], dtype=None)\n" + "/srv/conda/envs/notebook/lib/python3.12/site-packages/distributed/client.py:3371: UserWarning: Sending large graph of size 14.08 MiB.\n", + "This may cause some slowdown.\n", + "Consider loading the data with Dask directly\n", + " or using futures or delayed objects to embed the data into the graph without repetition.\n", + "See also https://docs.dask.org/en/stable/best-practices.html#load-data-with-dask for more information.\n", + " warnings.warn(\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Saved zarr file: gs://impactlab-data-scratch/xq3rogfmpG12ATOWhGCYPNdSspH_vxlj/clean/histnat/MIROC6/tas_r2.zarr\n" + "Saved zarr file: gs://impactlab-data-scratch/kemccusker/c5a7031e-064a-4353-803d-aca618565096/clean_2025-08-01/hist/MIROC6/tas_r1.zarr\n", + "r1, histnat\n", + "opening ['/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_hist-nat_r1i1p1f1_gn_19600101-19691231.nc', '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_hist-nat_r1i1p1f1_gn_19700101-19791231.nc', '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_hist-nat_r1i1p1f1_gn_19800101-19891231.nc', '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_hist-nat_r1i1p1f1_gn_19900101-19991231.nc', '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_hist-nat_r1i1p1f1_gn_20000101-20091231.nc', '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_hist-nat_r1i1p1f1_gn_20100101-20191231.nc', '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_hist-nat_r1i1p1f1_gn_20200101-20201231.nc']\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "/srv/conda/envs/notebook/lib/python3.10/site-packages/xarray/coding/times.py:710: SerializationWarning: Unable to decode time axis into full numpy.datetime64 objects, continuing using cftime.datetime objects instead, reason: dates out of range\n", + "/srv/conda/envs/notebook/lib/python3.12/site-packages/xarray/coding/times.py:1001: SerializationWarning: Unable to decode time axis into full numpy.datetime64 objects, continuing using cftime.datetime objects instead, reason: dates out of range\n", " dtype = _decode_cf_datetime_dtype(data, units, calendar, self.use_cftime)\n", - "/srv/conda/envs/notebook/lib/python3.10/site-packages/xarray/core/indexing.py:524: SerializationWarning: Unable to decode time axis into full numpy.datetime64 objects, continuing using cftime.datetime objects instead, reason: dates out of range\n", - " return np.asarray(array[self.key], dtype=None)\n", - "/srv/conda/envs/notebook/lib/python3.10/site-packages/xarray/coding/times.py:710: SerializationWarning: Unable to decode time axis into full numpy.datetime64 objects, continuing using cftime.datetime objects instead, reason: dates out of range\n", + "/srv/conda/envs/notebook/lib/python3.12/site-packages/xarray/core/indexing.py:514: SerializationWarning: Unable to decode time axis into full numpy.datetime64 objects, continuing using cftime.datetime objects instead, reason: dates out of range\n", + " return np.asarray(self.get_duck_array(), dtype=dtype, copy=copy)\n", + "/srv/conda/envs/notebook/lib/python3.12/site-packages/xarray/coding/times.py:1001: SerializationWarning: Unable to decode time axis into full numpy.datetime64 objects, continuing using cftime.datetime objects instead, reason: dates out of range\n", " dtype = _decode_cf_datetime_dtype(data, units, calendar, self.use_cftime)\n", - "/srv/conda/envs/notebook/lib/python3.10/site-packages/xarray/core/indexing.py:524: SerializationWarning: Unable to decode time axis into full numpy.datetime64 objects, continuing using cftime.datetime objects instead, reason: dates out of range\n", - " return np.asarray(array[self.key], dtype=None)\n", - "/srv/conda/envs/notebook/lib/python3.10/site-packages/xarray/coding/times.py:710: SerializationWarning: Unable to decode time axis into full numpy.datetime64 objects, continuing using cftime.datetime objects instead, reason: dates out of range\n", + "/srv/conda/envs/notebook/lib/python3.12/site-packages/xarray/core/indexing.py:514: SerializationWarning: Unable to decode time axis into full numpy.datetime64 objects, continuing using cftime.datetime objects instead, reason: dates out of range\n", + " return np.asarray(self.get_duck_array(), dtype=dtype, copy=copy)\n", + "/srv/conda/envs/notebook/lib/python3.12/site-packages/xarray/coding/times.py:1001: SerializationWarning: Unable to decode time axis into full numpy.datetime64 objects, continuing using cftime.datetime objects instead, reason: dates out of range\n", " dtype = _decode_cf_datetime_dtype(data, units, calendar, self.use_cftime)\n", - "/srv/conda/envs/notebook/lib/python3.10/site-packages/xarray/core/indexing.py:524: SerializationWarning: Unable to decode time axis into full numpy.datetime64 objects, continuing using cftime.datetime objects instead, reason: dates out of range\n", - " return np.asarray(array[self.key], dtype=None)\n" + "/srv/conda/envs/notebook/lib/python3.12/site-packages/xarray/core/indexing.py:514: SerializationWarning: Unable to decode time axis into full numpy.datetime64 objects, continuing using cftime.datetime objects instead, reason: dates out of range\n", + " return np.asarray(self.get_duck_array(), dtype=dtype, copy=copy)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Saved zarr file: gs://impactlab-data-scratch/xq3rogfmpG12ATOWhGCYPNdSspH_vxlj/clean/histnat/MIROC6/tas_r3.zarr\n" + "slicing years 3670 3679\n", + "convert picontrol calendar and chunk\n", + "remove leap days from data\n", + "set new date range\n", + "concat hist-nat and picontrol\n", + "clean_gcm Size: 3GB\n", + "Dimensions: (time: 18980, bnds: 2, lat: 128, lon: 256)\n", + "Coordinates:\n", + " * time (time) datetime64[ns] 152kB 1979-01-01T12:00:00 ... 2030-12-31\n", + " * lat (lat) float64 1kB -88.93 -87.54 -86.14 ... 86.14 87.54 88.93\n", + " * lon (lon) float64 2kB 0.0 1.406 2.812 4.219 ... 355.8 357.2 358.6\n", + " height float64 8B 2.0\n", + "Dimensions without coordinates: bnds\n", + "Data variables:\n", + " time_bnds (time, bnds) datetime64[ns] 304kB dask.array\n", + " lat_bnds (time, lat, bnds) float64 39MB dask.array\n", + " lon_bnds (time, lon, bnds) float64 78MB dask.array\n", + " tas (time, lat, lon) float32 2GB dask.array\n", + "Attributes: (12/44)\n", + " Conventions: CF-1.7 CMIP-6.2\n", + " activity_id: DAMIP\n", + " branch_method: standard\n", + " branch_time_in_child: 0.0\n", + " branch_time_in_parent: 0.0\n", + " creation_date: 2020-01-21T05:05:38Z\n", + " ... ...\n", + " title: MIROC6 output prepared for CMIP6\n", + " variable_id: tas\n", + " variant_label: r1i1p1f1\n", + " license: CMIP6 model data produced by MIROC is licensed un...\n", + " cmor_version: 3.5.0\n", + " tracking_id: hdl:21.14100/107c096e-3f2a-4391-bde1-a296379c093e\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "/srv/conda/envs/notebook/lib/python3.10/site-packages/xarray/core/indexing.py:1372: PerformanceWarning: Slicing is producing a large chunk. To accept the large\n", - "chunk and silence this warning, set the option\n", - " >>> with dask.config.set(**{'array.slicing.split_large_chunks': False}):\n", - " ... array[indexer]\n", - "\n", - "To avoid creating the large chunks, set the option\n", - " >>> with dask.config.set(**{'array.slicing.split_large_chunks': True}):\n", - " ... array[indexer]\n", - " return self.array[key]\n" + "/srv/conda/envs/notebook/lib/python3.12/site-packages/distributed/client.py:3371: UserWarning: Sending large graph of size 14.50 MiB.\n", + "This may cause some slowdown.\n", + "Consider loading the data with Dask directly\n", + " or using futures or delayed objects to embed the data into the graph without repetition.\n", + "See also https://docs.dask.org/en/stable/best-practices.html#load-data-with-dask for more information.\n", + " warnings.warn(\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Saved zarr file: gs://impactlab-data-scratch/xq3rogfmpG12ATOWhGCYPNdSspH_vxlj/clean/ERA5/MIROC6/tas.zarr\n" + "Saved zarr file: gs://impactlab-data-scratch/kemccusker/c5a7031e-064a-4353-803d-aca618565096/clean_2025-08-01/histnat/MIROC6/tas_r1.zarr\n", + "r2, hist\n", + "opening ['/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_historical_r2i1p1f1_gn_19600101-19691231.nc', '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_historical_r2i1p1f1_gn_19700101-19791231.nc', '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_historical_r2i1p1f1_gn_19800101-19891231.nc', '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_historical_r2i1p1f1_gn_19900101-19991231.nc', '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_historical_r2i1p1f1_gn_20000101-20091231.nc', '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_historical_r2i1p1f1_gn_20100101-20191231.nc', '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_historical_r2i1p1f1_gn_20100101-20141231.nc', '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_ssp245_r2i1p1f1_gn_20150101-20241231.nc', '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_ssp245_r2i1p1f1_gn_20250101-20341231.nc']\n" ] - } + }, + { + "ename": "FileNotFoundError", + "evalue": "[Errno 2] No such file or directory: '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_historical_r2i1p1f1_gn_20100101-20191231.nc'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mKeyError\u001b[0m Traceback (most recent call last)", + "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.12/site-packages/xarray/backends/file_manager.py:211\u001b[0m, in \u001b[0;36mCachingFileManager._acquire_with_cache_info\u001b[0;34m(self, needs_lock)\u001b[0m\n\u001b[1;32m 210\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 211\u001b[0m file \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_cache\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_key\u001b[49m\u001b[43m]\u001b[49m\n\u001b[1;32m 212\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mKeyError\u001b[39;00m:\n", + "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.12/site-packages/xarray/backends/lru_cache.py:56\u001b[0m, in \u001b[0;36mLRUCache.__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m 55\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_lock:\n\u001b[0;32m---> 56\u001b[0m value \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_cache\u001b[49m\u001b[43m[\u001b[49m\u001b[43mkey\u001b[49m\u001b[43m]\u001b[49m\n\u001b[1;32m 57\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_cache\u001b[38;5;241m.\u001b[39mmove_to_end(key)\n", + "\u001b[0;31mKeyError\u001b[0m: [, ('/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_historical_r2i1p1f1_gn_20100101-20191231.nc',), 'r', (('clobber', True), ('diskless', False), ('format', 'NETCDF4'), ('persist', False)), 'd50f76d9-7929-4939-a63e-0cf2b5bdba3d']", + "\nDuring handling of the above exception, another exception occurred:\n", + "\u001b[0;31mFileNotFoundError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[26], line 46\u001b[0m\n\u001b[1;32m 44\u001b[0m fn \u001b[38;5;241m=\u001b[39m esm\u001b[38;5;241m.\u001b[39mRAWDATAPATHS[source_id][scen][\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mr\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m+\u001b[39m \u001b[38;5;28mstr\u001b[39m(ensmem)]\n\u001b[1;32m 45\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mopening\u001b[39m\u001b[38;5;124m\"\u001b[39m, fn)\n\u001b[0;32m---> 46\u001b[0m raw_gcm \u001b[38;5;241m=\u001b[39m \u001b[43mxr\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen_mfdataset\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfn\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mconcat_dim\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mtime\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcombine\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mnested\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mengine\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mnetcdf4\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m 48\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m scen \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhistnat\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[1;32m 49\u001b[0m clean_gcm \u001b[38;5;241m=\u001b[39m standardize(raw_gcm, endyear\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m2020\u001b[39m)\n", + "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.12/site-packages/xarray/backends/api.py:1578\u001b[0m, in \u001b[0;36mopen_mfdataset\u001b[0;34m(paths, chunks, concat_dim, compat, preprocess, engine, data_vars, coords, combine, parallel, join, attrs_file, combine_attrs, **kwargs)\u001b[0m\n\u001b[1;32m 1575\u001b[0m open_ \u001b[38;5;241m=\u001b[39m open_dataset\n\u001b[1;32m 1576\u001b[0m getattr_ \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mgetattr\u001b[39m\n\u001b[0;32m-> 1578\u001b[0m datasets \u001b[38;5;241m=\u001b[39m [\u001b[43mopen_\u001b[49m\u001b[43m(\u001b[49m\u001b[43mp\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mopen_kwargs\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mfor\u001b[39;00m p \u001b[38;5;129;01min\u001b[39;00m paths]\n\u001b[1;32m 1579\u001b[0m closers \u001b[38;5;241m=\u001b[39m [getattr_(ds, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m_close\u001b[39m\u001b[38;5;124m\"\u001b[39m) \u001b[38;5;28;01mfor\u001b[39;00m ds \u001b[38;5;129;01min\u001b[39;00m datasets]\n\u001b[1;32m 1580\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m preprocess \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n", + "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.12/site-packages/xarray/backends/api.py:671\u001b[0m, in \u001b[0;36mopen_dataset\u001b[0;34m(filename_or_obj, engine, chunks, cache, decode_cf, mask_and_scale, decode_times, decode_timedelta, use_cftime, concat_characters, decode_coords, drop_variables, inline_array, chunked_array_type, from_array_kwargs, backend_kwargs, **kwargs)\u001b[0m\n\u001b[1;32m 659\u001b[0m decoders \u001b[38;5;241m=\u001b[39m _resolve_decoders_kwargs(\n\u001b[1;32m 660\u001b[0m decode_cf,\n\u001b[1;32m 661\u001b[0m open_backend_dataset_parameters\u001b[38;5;241m=\u001b[39mbackend\u001b[38;5;241m.\u001b[39mopen_dataset_parameters,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 667\u001b[0m decode_coords\u001b[38;5;241m=\u001b[39mdecode_coords,\n\u001b[1;32m 668\u001b[0m )\n\u001b[1;32m 670\u001b[0m overwrite_encoded_chunks \u001b[38;5;241m=\u001b[39m kwargs\u001b[38;5;241m.\u001b[39mpop(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moverwrite_encoded_chunks\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28;01mNone\u001b[39;00m)\n\u001b[0;32m--> 671\u001b[0m backend_ds \u001b[38;5;241m=\u001b[39m \u001b[43mbackend\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen_dataset\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 672\u001b[0m \u001b[43m \u001b[49m\u001b[43mfilename_or_obj\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 673\u001b[0m \u001b[43m \u001b[49m\u001b[43mdrop_variables\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdrop_variables\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 674\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mdecoders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 675\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 676\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 677\u001b[0m ds \u001b[38;5;241m=\u001b[39m _dataset_from_backend_dataset(\n\u001b[1;32m 678\u001b[0m backend_ds,\n\u001b[1;32m 679\u001b[0m filename_or_obj,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 689\u001b[0m \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs,\n\u001b[1;32m 690\u001b[0m )\n\u001b[1;32m 691\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m ds\n", + "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.12/site-packages/xarray/backends/netCDF4_.py:666\u001b[0m, in \u001b[0;36mNetCDF4BackendEntrypoint.open_dataset\u001b[0;34m(self, filename_or_obj, mask_and_scale, decode_times, concat_characters, decode_coords, drop_variables, use_cftime, decode_timedelta, group, mode, format, clobber, diskless, persist, auto_complex, lock, autoclose)\u001b[0m\n\u001b[1;32m 644\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mopen_dataset\u001b[39m( \u001b[38;5;66;03m# type: ignore[override] # allow LSP violation, not supporting **kwargs\u001b[39;00m\n\u001b[1;32m 645\u001b[0m \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m 646\u001b[0m filename_or_obj: \u001b[38;5;28mstr\u001b[39m \u001b[38;5;241m|\u001b[39m os\u001b[38;5;241m.\u001b[39mPathLike[Any] \u001b[38;5;241m|\u001b[39m BufferedIOBase \u001b[38;5;241m|\u001b[39m AbstractDataStore,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 663\u001b[0m autoclose\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m,\n\u001b[1;32m 664\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Dataset:\n\u001b[1;32m 665\u001b[0m filename_or_obj \u001b[38;5;241m=\u001b[39m _normalize_path(filename_or_obj)\n\u001b[0;32m--> 666\u001b[0m store \u001b[38;5;241m=\u001b[39m \u001b[43mNetCDF4DataStore\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 667\u001b[0m \u001b[43m \u001b[49m\u001b[43mfilename_or_obj\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 668\u001b[0m \u001b[43m \u001b[49m\u001b[43mmode\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmode\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 669\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mformat\u001b[39;49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mformat\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 670\u001b[0m \u001b[43m \u001b[49m\u001b[43mgroup\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgroup\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 671\u001b[0m \u001b[43m \u001b[49m\u001b[43mclobber\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mclobber\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 672\u001b[0m \u001b[43m \u001b[49m\u001b[43mdiskless\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdiskless\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 673\u001b[0m \u001b[43m \u001b[49m\u001b[43mpersist\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpersist\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 674\u001b[0m \u001b[43m \u001b[49m\u001b[43mauto_complex\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mauto_complex\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 675\u001b[0m \u001b[43m \u001b[49m\u001b[43mlock\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlock\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 676\u001b[0m \u001b[43m \u001b[49m\u001b[43mautoclose\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mautoclose\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 677\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 679\u001b[0m store_entrypoint \u001b[38;5;241m=\u001b[39m StoreBackendEntrypoint()\n\u001b[1;32m 680\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m close_on_error(store):\n", + "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.12/site-packages/xarray/backends/netCDF4_.py:453\u001b[0m, in \u001b[0;36mNetCDF4DataStore.open\u001b[0;34m(cls, filename, mode, format, group, clobber, diskless, persist, auto_complex, lock, lock_maker, autoclose)\u001b[0m\n\u001b[1;32m 449\u001b[0m kwargs[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mauto_complex\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m auto_complex\n\u001b[1;32m 450\u001b[0m manager \u001b[38;5;241m=\u001b[39m CachingFileManager(\n\u001b[1;32m 451\u001b[0m netCDF4\u001b[38;5;241m.\u001b[39mDataset, filename, mode\u001b[38;5;241m=\u001b[39mmode, kwargs\u001b[38;5;241m=\u001b[39mkwargs\n\u001b[1;32m 452\u001b[0m )\n\u001b[0;32m--> 453\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mcls\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mmanager\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mgroup\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgroup\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmode\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmode\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlock\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlock\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mautoclose\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mautoclose\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.12/site-packages/xarray/backends/netCDF4_.py:394\u001b[0m, in \u001b[0;36mNetCDF4DataStore.__init__\u001b[0;34m(self, manager, group, mode, lock, autoclose)\u001b[0m\n\u001b[1;32m 392\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_group \u001b[38;5;241m=\u001b[39m group\n\u001b[1;32m 393\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_mode \u001b[38;5;241m=\u001b[39m mode\n\u001b[0;32m--> 394\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mformat \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mds\u001b[49m\u001b[38;5;241m.\u001b[39mdata_model\n\u001b[1;32m 395\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_filename \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mds\u001b[38;5;241m.\u001b[39mfilepath()\n\u001b[1;32m 396\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mis_remote \u001b[38;5;241m=\u001b[39m is_remote_uri(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_filename)\n", + "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.12/site-packages/xarray/backends/netCDF4_.py:462\u001b[0m, in \u001b[0;36mNetCDF4DataStore.ds\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 460\u001b[0m \u001b[38;5;129m@property\u001b[39m\n\u001b[1;32m 461\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mds\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[0;32m--> 462\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_acquire\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.12/site-packages/xarray/backends/netCDF4_.py:456\u001b[0m, in \u001b[0;36mNetCDF4DataStore._acquire\u001b[0;34m(self, needs_lock)\u001b[0m\n\u001b[1;32m 455\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_acquire\u001b[39m(\u001b[38;5;28mself\u001b[39m, needs_lock\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m):\n\u001b[0;32m--> 456\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_manager\u001b[38;5;241m.\u001b[39macquire_context(needs_lock) \u001b[38;5;28;01mas\u001b[39;00m root:\n\u001b[1;32m 457\u001b[0m ds \u001b[38;5;241m=\u001b[39m _nc4_require_group(root, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_group, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_mode)\n\u001b[1;32m 458\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m ds\n", + "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.12/contextlib.py:137\u001b[0m, in \u001b[0;36m_GeneratorContextManager.__enter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 135\u001b[0m \u001b[38;5;28;01mdel\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39margs, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mkwds, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfunc\n\u001b[1;32m 136\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 137\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mnext\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgen\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 138\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mStopIteration\u001b[39;00m:\n\u001b[1;32m 139\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mgenerator didn\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mt yield\u001b[39m\u001b[38;5;124m\"\u001b[39m) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n", + "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.12/site-packages/xarray/backends/file_manager.py:199\u001b[0m, in \u001b[0;36mCachingFileManager.acquire_context\u001b[0;34m(self, needs_lock)\u001b[0m\n\u001b[1;32m 196\u001b[0m \u001b[38;5;129m@contextlib\u001b[39m\u001b[38;5;241m.\u001b[39mcontextmanager\n\u001b[1;32m 197\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21macquire_context\u001b[39m(\u001b[38;5;28mself\u001b[39m, needs_lock\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m):\n\u001b[1;32m 198\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"Context manager for acquiring a file.\"\"\"\u001b[39;00m\n\u001b[0;32m--> 199\u001b[0m file, cached \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_acquire_with_cache_info\u001b[49m\u001b[43m(\u001b[49m\u001b[43mneeds_lock\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 200\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 201\u001b[0m \u001b[38;5;28;01myield\u001b[39;00m file\n", + "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.12/site-packages/xarray/backends/file_manager.py:217\u001b[0m, in \u001b[0;36mCachingFileManager._acquire_with_cache_info\u001b[0;34m(self, needs_lock)\u001b[0m\n\u001b[1;32m 215\u001b[0m kwargs \u001b[38;5;241m=\u001b[39m kwargs\u001b[38;5;241m.\u001b[39mcopy()\n\u001b[1;32m 216\u001b[0m kwargs[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmode\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_mode\n\u001b[0;32m--> 217\u001b[0m file \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_opener\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_args\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 218\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_mode \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mw\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[1;32m 219\u001b[0m \u001b[38;5;66;03m# ensure file doesn't get overridden when opened again\u001b[39;00m\n\u001b[1;32m 220\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_mode \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124ma\u001b[39m\u001b[38;5;124m\"\u001b[39m\n", + "File \u001b[0;32msrc/netCDF4/_netCDF4.pyx:2470\u001b[0m, in \u001b[0;36mnetCDF4._netCDF4.Dataset.__init__\u001b[0;34m()\u001b[0m\n", + "File \u001b[0;32msrc/netCDF4/_netCDF4.pyx:2107\u001b[0m, in \u001b[0;36mnetCDF4._netCDF4._ensure_nc_success\u001b[0;34m()\u001b[0m\n", + "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: '/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_historical_r2i1p1f1_gn_20100101-20191231.nc'" + ] + } ], "source": [ - "gateway = Gateway()\n", - "img = os.environ[\"JUPYTER_IMAGE\"]\n", - "with gateway.new_cluster(worker_image=img, scheduler_image=img) as cluster:\n", - " print(cluster.dashboard_link)\n", - " client = cluster.get_client()\n", - " display(client)\n", - "\n", - " cluster.scale(100)\n", - " client.register_worker_plugin(dd.PipInstall(packages=[\"xclim==0.39.0\"]))\n", - " \n", + "# gateway = Gateway()\n", + "# img = os.environ[\"JUPYTER_IMAGE\"]\n", + "# with gateway.new_cluster(worker_image=img, scheduler_image=img) as cluster:\n", + "# print(cluster.dashboard_link)\n", + "# client = cluster.get_client()\n", + "# display(client)\n", + "\n", + "# cluster.scale(100)\n", + "# client.register_worker_plugin(dd.PipInstall(packages=[\"xclim==0.39.0\"]))\n", " \n", - " for in_file, out_url in GCM_file_paths:\n", - " raw_gcm = xr.open_mfdataset(in_file, concat_dim=\"time\", combine=\"nested\", engine='netcdf4')\n", - "\n", - " if in_file == IN_HISTNAT_RAW_r1:\n", - " if source_id != \"NorESM2-LM\":\n", - " picontrol = xr.open_mfdataset(IN_PICONTROL_RAW, concat_dim=\"time\", combine=\"nested\", engine='netcdf4')\n", - " original_times = picontrol.time.values\n", - " new_picontrol_start_year = 2021\n", - " new_picontrol_end_year = 2030\n", - " \n", - " if source_id == \"NorESM2-LM\":\n", - " picontrol = xr.open_mfdataset(IN_PICONTROL_RAW_1)\n", - " new_picontrol_start_year = 2021 \n", - " new_picontrol_end_year = 2030 \n", - " picontrol_subset = picontrol \n", - " original_times = picontrol_subset.time.values\n", - " new_times = np.array([cftime.DatetimeNoLeap(new_picontrol_start_year + (time.year - original_times[0].year), time.month, time.day, time.hour, time.minute, time.second, time.microsecond)\n", - " for time in original_times if not (time.month == 2 and time.day == 29)])\n", - " picontrol_subset['time'] = picontrol_subset['time'].astype(\"datetime64[ns]\")\n", - " picontrol_subset = convert_calendar(picontrol_subset, target=\"noleap\")\n", - " clean_picontrol = picontrol_subset.assign_coords(time=new_times) \n", - " clean_picontrol['time_bnds'] = clean_picontrol['time_bnds'].astype('datetime64[ns]')\n", - " clean_histnat = standardize(raw_gcm)\n", - " clean_gcm = xr.concat([clean_histnat, clean_picontrol], dim=\"time\")\n", - " \n", - " \n", - " elif source_id == \"ACCESS-ESM5-1\":\n", - " picontrol_subset = picontrol.sel(time=slice(str(int(PiControl_year_start)), str(int(PiControl_year_start) + 9)))\n", - " picontrol_subset = convert_calendar(picontrol_subset, target=\"noleap\")\n", - " picontrol_subset = picontrol_subset.chunk({\"time\": -1, \"lat\": 2, \"lon\": -1})\n", - " picontrol_subset = picontrol_subset.sel(time=~((picontrol_subset['time.month'] == 2) & (picontrol_subset['time.day'] == 29))).dropna('time')\n", - " original_times = picontrol_subset.time.values\n", - " new_times = np.array([cftime.DatetimeNoLeap(new_picontrol_start_year + (time.year - original_times[0].year), time.month, time.day, time.hour, time.minute, time.second, time.microsecond) for time in original_times])\n", - " picontrol_subset['time'] = picontrol_subset['time'].astype(\"datetime64[ns]\")\n", - " picontrol_subset = convert_calendar(picontrol_subset, target=\"noleap\")\n", - " clean_picontrol = picontrol_subset.assign_coords(time=new_times) \n", - " clean_picontrol['time_bnds'] = clean_picontrol['time_bnds'].astype('datetime64[ns]')\n", - " clean_histnat = standardize(raw_gcm)\n", - " clean_gcm = xr.concat([clean_histnat, clean_picontrol], dim=\"time\")\n", - "\n", - " \n", - " else:\n", - " \n", - " picontrol_subset = picontrol.sel(time=slice(PiControl_year_start, str(int(PiControl_year_start) + 9)))\n", - " picontrol_subset = convert_calendar(picontrol_subset, target=\"noleap\")\n", - " picontrol_subset = picontrol_subset.chunk({\"time\": -1, \"lat\": 2, \"lon\": -1})\n", - " picontrol_subset = picontrol_subset.sel(time=~((picontrol_subset['time.month'] == 2) & (picontrol_subset['time.day'] == 29))).dropna('time')\n", - " original_times = picontrol_subset.time.values\n", - " new_times = np.array([cftime.DatetimeNoLeap(new_picontrol_start_year + (time.year - original_times[0].year), time.month, time.day, time.hour, time.minute, time.second, time.microsecond) for time in original_times])\n", - " picontrol_subset['time'] = picontrol_subset['time'].astype(\"datetime64[ns]\")\n", - " clean_picontrol = picontrol_subset.assign_coords(time=new_times) \n", - " clean_picontrol['time_bnds'] = clean_picontrol['time_bnds'].astype('datetime64[ns]')\n", - " clean_histnat = standardize(raw_gcm)\n", - " clean_gcm = xr.concat([clean_histnat, clean_picontrol], dim=\"time\")\n", + " # new algorithm. The goal is to clean histnat and append 10 years of picontrol to the end of it.\n", + " # For each ensemble member, we need a different 10 years of picontrol.\n", + " # for each gcm:\n", + " # for each ensemble member (r) of histnat:\n", + " # read in histnat files (all years)\n", + " # standardize histnat (\"raw_gcm\")\n", + " # read in picontrol files\n", + " # if source_id == \"NorESM2-LM\", read in the first 10-yr picontrol file (do this incrementally by r)\n", + " # else, read in all picontrol files and subset incrememtal 10-year periods by r\n", + " # update the time dim to match the years we want, 2021-2030\n", + " # concatenate histnat and picontrol (\"clean_gcm\")\n", + " # save clean_gcm to zarr\n", + "\n", + "if 1:\n", + " if 1:\n", + " # Is this capturing all of the variations across the GCMS and scenarios? Need to test.\n", + " new_picontrol_start_year = 2021\n", + " new_picontrol_end_year = 2030\n", + "\n", + " for gcm in gcms:\n", + " print(gcm)\n", + " \n", + " source_id = esm.GCMS[gcm][\"source_id\"]\n", + " institution_id = esm.GCMS[gcm][\"institution_id\"]\n", + " grid_label = esm.GCMS[gcm][\"grid_label\"]\n", "\n", - " \n", - " elif in_file == IN_HISTNAT_RAW_r2:\n", - " if source_id != \"NorESM2-LM\":\n", - " picontrol = xr.open_mfdataset(IN_PICONTROL_RAW, concat_dim=\"time\", combine=\"nested\", engine='netcdf4')\n", - " original_times = picontrol.time.values\n", - " new_picontrol_start_year = 2021\n", - " new_picontrol_end_year = 2030\n", - " \n", - " if source_id == \"NorESM2-LM\":\n", - " picontrol = xr.open_mfdataset(IN_PICONTROL_RAW_2)\n", - " new_picontrol_start_year = 2021 \n", - " new_picontrol_end_year = 2030 \n", - " picontrol_subset = picontrol \n", - " original_times = picontrol_subset.time.values\n", - " new_times = np.array([cftime.DatetimeNoLeap(new_picontrol_start_year + (time.year - original_times[0].year), time.month, time.day, time.hour, time.minute, time.second, time.microsecond)\n", - " for time in original_times if not (time.month == 2 and time.day == 29)])\n", - " picontrol_subset['time'] = picontrol_subset['time'].astype(\"datetime64[ns]\")\n", - " picontrol_subset = convert_calendar(picontrol_subset, target=\"noleap\")\n", - " clean_picontrol = picontrol_subset.assign_coords(time=new_times) \n", - " clean_picontrol['time_bnds'] = clean_picontrol['time_bnds'].astype('datetime64[ns]')\n", - " clean_histnat = standardize(raw_gcm)\n", - " clean_gcm = xr.concat([clean_histnat, clean_picontrol], dim=\"time\")\n", - " \n", - " \n", - " elif source_id == \"ACCESS-ESM5-1\":\n", - " picontrol_subset = picontrol.sel(time=slice(str(int(PiControl_year_start) + 10), str(int(PiControl_year_start) + 19)))\n", - " picontrol_subset = convert_calendar(picontrol_subset, target=\"noleap\")\n", - " picontrol_subset = picontrol_subset.chunk({\"time\": -1, \"lat\": 2, \"lon\": -1})\n", - " picontrol_subset = picontrol_subset.sel(time=~((picontrol_subset['time.month'] == 2) & (picontrol_subset['time.day'] == 29))).dropna('time')\n", - " original_times = picontrol_subset.time.values\n", - " new_times = np.array([cftime.DatetimeNoLeap(new_picontrol_start_year + (time.year - original_times[0].year), time.month, time.day, time.hour, time.minute, time.second, time.microsecond) for time in original_times])\n", - " picontrol_subset['time'] = picontrol_subset['time'].astype(\"datetime64[ns]\")\n", - " picontrol_subset = convert_calendar(picontrol_subset, target=\"noleap\")\n", - " clean_picontrol = picontrol_subset.assign_coords(time=new_times) \n", - " clean_picontrol['time_bnds'] = clean_picontrol['time_bnds'].astype('datetime64[ns]')\n", - " clean_histnat = standardize(raw_gcm)\n", - " clean_gcm = xr.concat([clean_histnat, clean_picontrol], dim=\"time\")\n", - " \n", - " \n", - " else:\n", - " \n", - " picontrol_subset = picontrol.sel(time=slice(str(int(PiControl_year_start) + 10), str(int(PiControl_year_start) + 19)))\n", - " picontrol_subset = convert_calendar(picontrol_subset, target=\"noleap\")\n", - " picontrol_subset = picontrol_subset.chunk({\"time\": -1, \"lat\": 2, \"lon\": -1})\n", - " picontrol_subset = picontrol_subset.sel(time=~((picontrol_subset['time.month'] == 2) & (picontrol_subset['time.day'] == 29))).dropna('time')\n", - " original_times = picontrol_subset.time.values\n", - " new_times = np.array([cftime.DatetimeNoLeap(new_picontrol_start_year + (time.year - original_times[0].year), time.month, time.day, time.hour, time.minute, time.second, time.microsecond)\n", - " for time in original_times if not (time.month == 2 and time.day == 29)])\n", - " picontrol_subset['time'] = picontrol_subset['time'].astype(\"datetime64[ns]\")\n", - " clean_picontrol = picontrol_subset.assign_coords(time=new_times)\n", - " clean_picontrol['time_bnds'] = clean_picontrol['time_bnds'].astype('datetime64[ns]')\n", - " clean_histnat = standardize(raw_gcm)\n", - " clean_gcm = xr.concat([clean_histnat, clean_picontrol], dim=\"time\")\n", - " \n", - " \n", - " elif in_file == IN_HISTNAT_RAW_r3:\n", - " if source_id != \"NorESM2-LM\":\n", - " picontrol = xr.open_mfdataset(IN_PICONTROL_RAW, concat_dim=\"time\", combine=\"nested\", engine='netcdf4')\n", - " original_times = picontrol.time.values\n", - " new_picontrol_start_year = 2021\n", - " new_picontrol_end_year = 2030\n", + " for ensmem in range(1, 3): # read in histnat files\n", " \n", - " if source_id == \"NorESM2-LM\":\n", - " picontrol = xr.open_mfdataset(IN_PICONTROL_RAW_3)\n", - " new_picontrol_start_year = 2021 \n", - " new_picontrol_end_year = 2030 \n", - " picontrol_subset = picontrol \n", - " original_times = picontrol_subset.time.values\n", - " new_times = np.array([cftime.DatetimeNoLeap(new_picontrol_start_year + (time.year - original_times[0].year), time.month, time.day, time.hour, time.minute, time.second, time.microsecond)\n", - " for time in original_times if not (time.month == 2 and time.day == 29)])\n", - " picontrol_subset['time'] = picontrol_subset['time'].astype(\"datetime64[ns]\")\n", - " picontrol_subset = convert_calendar(picontrol_subset, target=\"noleap\")\n", - " clean_picontrol = picontrol_subset.assign_coords(time=new_times) \n", - " clean_picontrol['time_bnds'] = clean_picontrol['time_bnds'].astype('datetime64[ns]')\n", - " clean_histnat = standardize(raw_gcm)\n", - " clean_gcm = xr.concat([clean_histnat, clean_picontrol], dim=\"time\")\n", + " for scen in [\"hist\", \"histnat\"]:\n", + " print(f\"r{ensmem}, {scen}\")\n", " \n", + " out_url = f\"{OUT_BASE_URL}/{scen}/{source_id}/{var}_r{ensmem}.zarr\"\n", " \n", - " elif source_id == \"ACCESS-ESM5-1\":\n", - " picontrol_subset = picontrol.sel(time=slice(str(int(PiControl_year_start) + 20), str(int(PiControl_year_start) + 29)))\n", - " picontrol_subset = convert_calendar(picontrol_subset, target=\"noleap\")\n", - " picontrol_subset = picontrol_subset.chunk({\"time\": -1, \"lat\": 2, \"lon\": -1})\n", - " picontrol_subset = picontrol_subset.sel(time=~((picontrol_subset['time.month'] == 2) & (picontrol_subset['time.day'] == 29))).dropna('time')\n", - " original_times = picontrol_subset.time.values\n", - " new_times = np.array([cftime.DatetimeNoLeap(new_picontrol_start_year + (time.year - original_times[0].year), time.month, time.day, time.hour, time.minute, time.second, time.microsecond) for time in original_times])\n", - " picontrol_subset['time'] = picontrol_subset['time'].astype(\"datetime64[ns]\")\n", - " picontrol_subset = convert_calendar(picontrol_subset, target=\"noleap\")\n", - " clean_picontrol = picontrol_subset.assign_coords(time=new_times) \n", - " clean_picontrol['time_bnds'] = clean_picontrol['time_bnds'].astype('datetime64[ns]')\n", - " clean_histnat = standardize(raw_gcm)\n", - " clean_gcm = xr.concat([clean_histnat, clean_picontrol], dim=\"time\")\n", - " \n", - " \n", - " else:\n", - " \n", - " picontrol_subset = picontrol.sel(time=slice(str(int(PiControl_year_start) + 20), str(int(PiControl_year_start) + 29)))\n", - " picontrol_subset = convert_calendar(picontrol_subset, target=\"noleap\")\n", - " picontrol_subset = picontrol_subset.chunk({\"time\": -1, \"lat\": 2, \"lon\": -1})\n", - " picontrol_subset = picontrol_subset.sel(time=~((picontrol_subset['time.month'] == 2) & (picontrol_subset['time.day'] == 29))).dropna('time')\n", - " original_times = picontrol_subset.time.values\n", - " new_times = np.array([cftime.DatetimeNoLeap(new_picontrol_start_year + (time.year - original_times[0].year), time.month, time.day, time.hour, time.minute, time.second, time.microsecond) for time in original_times])\n", - " picontrol_subset['time'] = picontrol_subset['time'].astype(\"datetime64[ns]\")\n", - " clean_picontrol = picontrol_subset.assign_coords(time=new_times)\n", - " clean_picontrol['time_bnds'] = clean_picontrol['time_bnds'].astype('datetime64[ns]')\n", - " clean_histnat = standardize(raw_gcm)\n", - " clean_gcm = xr.concat([clean_histnat, clean_picontrol], dim=\"time\") \n", - " \n", - " \n", - " elif in_file == IN_ERA5_RAW:\n", - " regridder = xe.Regridder(raw_gcm, ERA5_out, 'bilinear')\n", - " raw_gcm = regridder(raw_gcm)\n", - " raw_gcm[\"tas\"].attrs[\"units\"] = \"K\"\n", - " clean_gcm = standardize(raw_gcm)\n", - " \n", - " else:\n", - " clean_gcm = standardize_2030(raw_gcm)\n", - " \n", - " clean_gcm.to_zarr(out_url, mode=\"w\", consolidated=True)\n", - " print(f\"Saved zarr file: {out_url}\")" + " fn = esm.RAWDATAPATHS[source_id][scen][\"r\" + str(ensmem)]\n", + " print(\"opening\", fn)\n", + " raw_gcm = xr.open_mfdataset(fn, concat_dim=\"time\", combine=\"nested\", engine='netcdf4')\n", + "\n", + " if scen == \"histnat\":\n", + " clean_gcm = standardize(raw_gcm, endyear=2020)\n", + "\n", + " # read in picontrol files to concat with histnat to 2030\n", + " # TODO: get years from picontrol that are further into the time series\n", + " if source_id == \"NorESM2-LM\":\n", + " # if NorESM2-LM, open one of the (10-year) picontrol files depending on which ensemble member we're on\n", + " picontrol = xr.open_mfdataset(esm.RAWDATAPATHS[source_id][\"picontrol\"][\"r1\"][ensmem-1])\n", + " else:\n", + " # for all other gcms, open all picontrol files and select a 10-year period depending on which ensemble member we're on\n", + " picontrol = xr.open_mfdataset(esm.RAWDATAPATHS[source_id][\"picontrol\"][\"r1\"], concat_dim=\"time\", combine=\"nested\", engine='netcdf4')\n", + " print(\"slicing years\", esm.RAWDATAPATHS[source_id][\"picontrol\"][\"startyear\"] + (ensmem-1)*10, \n", + " esm.RAWDATAPATHS[source_id][\"picontrol\"][\"startyear\"] + (ensmem-1)*10 + 9)\n", + " picontrol = picontrol.sel(time=slice(str(esm.RAWDATAPATHS[source_id][\"picontrol\"][\"startyear\"] + (ensmem-1)*10), \n", + " str(esm.RAWDATAPATHS[source_id][\"picontrol\"][\"startyear\"] + (ensmem-1)*10 + 9)))\n", + "\n", + " print(\"convert picontrol calendar and chunk\")\n", + " picontrol = xr.Dataset.convert_calendar(picontrol, calendar=\"noleap\")\n", + " picontrol = picontrol.chunk({\"time\": -1, \"lat\": 2, \"lon\": -1})\n", + " \n", + " print(\"remove leap days from data\")\n", + " # why do we need to remove leap days if cal is set to noleap? oh, b/c the data is set to nan I think.\n", + " picontrol = picontrol.sel(time=~((picontrol['time.month'] == 2) & (picontrol['time.day'] == 29))).dropna('time')\n", + "\n", + " original_times = picontrol.time.values\n", + " # new_times = np.array([cftime.DatetimeNoLeap(new_picontrol_start_year + (time.year - original_times[0].year), time.month, time.day, time.hour, time.minute, time.second, time.microsecond) for time in original_times])\n", + " # picontrol['time'] = picontrol['time'].astype(\"datetime64[ns]\")\n", + " # clean_picontrol = picontrol.assign_coords(time=new_times) \n", + " # clean_picontrol['time'] = clean_picontrol['time'].astype(\"datetime64[ns]\")\n", + " # clean_picontrol['time_bnds'] = clean_picontrol['time_bnds'].astype('datetime64[ns]')\n", + "\n", + " print(\"set new date range\")\n", + " new_times = noleap_date_range()\n", + " clean_picontrol = picontrol.assign_coords(time=new_times) \n", + "\n", + " print(\"concat hist-nat and picontrol\")\n", + " # TODO after testing this works, add \"original_times\" to the clean_picontrol dataset\n", + " clean_gcm = xr.concat([clean_gcm, clean_picontrol], dim=\"time\")\n", + "\n", + " # should we convert calendar again to be sure?\n", + " # clean_gcm = xr.Dataset.convert_calendar(clean_gcm, calendar=\"noleap\")\n", + "\n", + " clean_gcm['time'] = clean_gcm['time'].astype(\"datetime64[ns]\")\n", + " clean_gcm['time_bnds'] = clean_gcm['time_bnds'].astype('datetime64[ns]')\n", + " else:\n", + " clean_gcm = standardize(raw_gcm, endyear=2030)\n", + "\n", + " print(\"clean_gcm\", clean_gcm)\n", + " clean_gcm.to_zarr(out_url, mode=\"w\", consolidated=True)\n", + " print(f\"Saved zarr file: {out_url}\")\n", + "\n", + " # handle ERA5\n", + " fn = esm.RAWDATAPATHS[\"ERA5\"][var]\n", + " out_url = f\"{OUT_BASE_URL}/ERA5/{var}.zarr\"\n", + " raw_gcm = xr.open_mfdataset(fn, concat_dim=\"time\", combine=\"nested\", engine='netcdf4')\n", + " regridder = xe.Regridder(raw_gcm, ERA5_out_grid, 'bilinear')\n", + " raw_gcm = regridder(raw_gcm)\n", + " raw_gcm[\"tas\"].attrs[\"units\"] = \"K\"\n", + " clean_gcm = standardize(raw_gcm)\n", + " clean_gcm.to_zarr(out_url, mode=\"w\", consolidated=True)\n", + " print(f\"Saved zarr file: {out_url}\")\n", + " \n" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "384b19b8-8bbb-463b-b3fa-db13cf09368a", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[cftime.DatetimeNoLeap(3670, 1, 1, 12, 0, 0, 0, has_year_zero=True)\n", + " cftime.DatetimeNoLeap(3670, 1, 2, 12, 0, 0, 0, has_year_zero=True)\n", + " cftime.DatetimeNoLeap(3670, 1, 3, 12, 0, 0, 0, has_year_zero=True) ...\n", + " cftime.DatetimeNoLeap(3679, 12, 29, 12, 0, 0, 0, has_year_zero=True)\n", + " cftime.DatetimeNoLeap(3679, 12, 30, 12, 0, 0, 0, has_year_zero=True)\n", + " cftime.DatetimeNoLeap(3679, 12, 31, 12, 0, 0, 0, has_year_zero=True)]\n" + ] + } + ], + "source": [ + "print(original_times)" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "5fbfbdd2-0eac-4fc8-8589-42468a8b9326", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CFTimeIndex([2021-01-01 00:00:00, 2021-01-02 00:00:00, 2021-01-03 00:00:00,\n", + " 2021-01-04 00:00:00, 2021-01-05 00:00:00, 2021-01-06 00:00:00,\n", + " 2021-01-07 00:00:00, 2021-01-08 00:00:00, 2021-01-09 00:00:00,\n", + " 2021-01-10 00:00:00,\n", + " ...\n", + " 2030-12-22 00:00:00, 2030-12-23 00:00:00, 2030-12-24 00:00:00,\n", + " 2030-12-25 00:00:00, 2030-12-26 00:00:00, 2030-12-27 00:00:00,\n", + " 2030-12-28 00:00:00, 2030-12-29 00:00:00, 2030-12-30 00:00:00,\n", + " 2030-12-31 00:00:00],\n", + " dtype='object', length=3650, calendar='noleap', freq='D')\n" + ] + } + ], + "source": [ + "print(new_times)" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "e7924a3f-71b4-4a25-a430-d3dbbe0e1249", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
<xarray.DataArray 'time' (time: 3650)> Size: 29kB\n",
+       "array([cftime.DatetimeNoLeap(3670, 1, 1, 12, 0, 0, 0, has_year_zero=True),\n",
+       "       cftime.DatetimeNoLeap(3670, 1, 2, 12, 0, 0, 0, has_year_zero=True),\n",
+       "       cftime.DatetimeNoLeap(3670, 1, 3, 12, 0, 0, 0, has_year_zero=True), ...,\n",
+       "       cftime.DatetimeNoLeap(3679, 12, 29, 12, 0, 0, 0, has_year_zero=True),\n",
+       "       cftime.DatetimeNoLeap(3679, 12, 30, 12, 0, 0, 0, has_year_zero=True),\n",
+       "       cftime.DatetimeNoLeap(3679, 12, 31, 12, 0, 0, 0, has_year_zero=True)],\n",
+       "      dtype=object)\n",
+       "Coordinates:\n",
+       "  * time     (time) object 29kB 3670-01-01 12:00:00 ... 3679-12-31 12:00:00\n",
+       "    height   float64 8B 2.0\n",
+       "Attributes:\n",
+       "    bounds:         time_bnds\n",
+       "    axis:           T\n",
+       "    long_name:      time\n",
+       "    standard_name:  time
" + ], + "text/plain": [ + " Size: 29kB\n", + "array([cftime.DatetimeNoLeap(3670, 1, 1, 12, 0, 0, 0, has_year_zero=True),\n", + " cftime.DatetimeNoLeap(3670, 1, 2, 12, 0, 0, 0, has_year_zero=True),\n", + " cftime.DatetimeNoLeap(3670, 1, 3, 12, 0, 0, 0, has_year_zero=True), ...,\n", + " cftime.DatetimeNoLeap(3679, 12, 29, 12, 0, 0, 0, has_year_zero=True),\n", + " cftime.DatetimeNoLeap(3679, 12, 30, 12, 0, 0, 0, has_year_zero=True),\n", + " cftime.DatetimeNoLeap(3679, 12, 31, 12, 0, 0, 0, has_year_zero=True)],\n", + " dtype=object)\n", + "Coordinates:\n", + " * time (time) object 29kB 3670-01-01 12:00:00 ... 3679-12-31 12:00:00\n", + " height float64 8B 2.0\n", + "Attributes:\n", + " bounds: time_bnds\n", + " axis: T\n", + " long_name: time\n", + " standard_name: time" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "picontrol[\"time\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "abc43238-cd20-4654-823e-9a78865c10a4", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
<xarray.DataArray 'time' (time: 3650)> Size: 29kB\n",
+       "array([cftime.DatetimeNoLeap(2021, 1, 1, 0, 0, 0, 0, has_year_zero=True),\n",
+       "       cftime.DatetimeNoLeap(2021, 1, 2, 0, 0, 0, 0, has_year_zero=True),\n",
+       "       cftime.DatetimeNoLeap(2021, 1, 3, 0, 0, 0, 0, has_year_zero=True), ...,\n",
+       "       cftime.DatetimeNoLeap(2030, 12, 29, 0, 0, 0, 0, has_year_zero=True),\n",
+       "       cftime.DatetimeNoLeap(2030, 12, 30, 0, 0, 0, 0, has_year_zero=True),\n",
+       "       cftime.DatetimeNoLeap(2030, 12, 31, 0, 0, 0, 0, has_year_zero=True)],\n",
+       "      dtype=object)\n",
+       "Coordinates:\n",
+       "    height   float64 8B 2.0\n",
+       "  * time     (time) object 29kB 2021-01-01 00:00:00 ... 2030-12-31 00:00:00
" + ], + "text/plain": [ + " Size: 29kB\n", + "array([cftime.DatetimeNoLeap(2021, 1, 1, 0, 0, 0, 0, has_year_zero=True),\n", + " cftime.DatetimeNoLeap(2021, 1, 2, 0, 0, 0, 0, has_year_zero=True),\n", + " cftime.DatetimeNoLeap(2021, 1, 3, 0, 0, 0, 0, has_year_zero=True), ...,\n", + " cftime.DatetimeNoLeap(2030, 12, 29, 0, 0, 0, 0, has_year_zero=True),\n", + " cftime.DatetimeNoLeap(2030, 12, 30, 0, 0, 0, 0, has_year_zero=True),\n", + " cftime.DatetimeNoLeap(2030, 12, 31, 0, 0, 0, 0, has_year_zero=True)],\n", + " dtype=object)\n", + "Coordinates:\n", + " height float64 8B 2.0\n", + " * time (time) object 29kB 2021-01-01 00:00:00 ... 2030-12-31 00:00:00" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "clean_picontrol[\"time\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "7314902c-e02e-4c55-b727-6c6d1280046c", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
<xarray.DataArray 'time' (time: 18615)> Size: 149kB\n",
+       "array([cftime.DatetimeNoLeap(1979, 1, 1, 12, 0, 0, 0, has_year_zero=True),\n",
+       "       cftime.DatetimeNoLeap(1979, 1, 2, 12, 0, 0, 0, has_year_zero=True),\n",
+       "       cftime.DatetimeNoLeap(1979, 1, 3, 12, 0, 0, 0, has_year_zero=True), ...,\n",
+       "       cftime.DatetimeNoLeap(2030, 12, 29, 0, 0, 0, 0, has_year_zero=True),\n",
+       "       cftime.DatetimeNoLeap(2030, 12, 30, 0, 0, 0, 0, has_year_zero=True),\n",
+       "       cftime.DatetimeNoLeap(2030, 12, 31, 0, 0, 0, 0, has_year_zero=True)],\n",
+       "      dtype=object)\n",
+       "Coordinates:\n",
+       "  * time     (time) object 149kB 1979-01-01 12:00:00 ... 2030-12-31 00:00:00\n",
+       "    height   float64 8B 2.0\n",
+       "Attributes:\n",
+       "    bounds:         time_bnds\n",
+       "    axis:           T\n",
+       "    long_name:      time\n",
+       "    standard_name:  time
" + ], + "text/plain": [ + " Size: 149kB\n", + "array([cftime.DatetimeNoLeap(1979, 1, 1, 12, 0, 0, 0, has_year_zero=True),\n", + " cftime.DatetimeNoLeap(1979, 1, 2, 12, 0, 0, 0, has_year_zero=True),\n", + " cftime.DatetimeNoLeap(1979, 1, 3, 12, 0, 0, 0, has_year_zero=True), ...,\n", + " cftime.DatetimeNoLeap(2030, 12, 29, 0, 0, 0, 0, has_year_zero=True),\n", + " cftime.DatetimeNoLeap(2030, 12, 30, 0, 0, 0, 0, has_year_zero=True),\n", + " cftime.DatetimeNoLeap(2030, 12, 31, 0, 0, 0, 0, has_year_zero=True)],\n", + " dtype=object)\n", + "Coordinates:\n", + " * time (time) object 149kB 1979-01-01 12:00:00 ... 2030-12-31 00:00:00\n", + " height float64 8B 2.0\n", + "Attributes:\n", + " bounds: time_bnds\n", + " axis: T\n", + " long_name: time\n", + " standard_name: time" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "clean_gcm[\"time\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "9a58c8ee-d6f1-438a-b573-6ab8db2c537d", + "metadata": {}, + "outputs": [], + "source": [ + "# new_time_coord = pd.date_range(\"2021-01-01\", periods=3652, freq=\"D\")\n", + "# new_time_coord" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "1b9c2718-8f9a-4a61-b24b-06b96a27876a", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Data variables:\n", + " time_bnds (time, bnds) object 304kB dask.array\n", + " lat_bnds (time, lat, bnds) float64 39MB dask.array\n", + " lon_bnds (time, lon, bnds) float64 78MB dask.array\n", + " tas (time, lat, lon) float32 2GB dask.array" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "clean_gcm.data_vars" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "16cd6d12-d4d1-428b-b949-c17fb0b583b5", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Coordinates:\n", + " * time (time) object 152kB 1979-01-01 12:00:00 ... 2030-12-31 00:00:00\n", + " * lat (lat) float64 1kB -88.93 -87.54 -86.14 -84.74 ... 86.14 87.54 88.93\n", + " * lon (lon) float64 2kB 0.0 1.406 2.812 4.219 ... 354.4 355.8 357.2 358.6\n", + " height float64 8B 2.0" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "clean_gcm.coords" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "291731a1-b806-4676-9cc7-03df47998ad4", + "metadata": {}, + "outputs": [ + { + "ename": "GatewayClusterError", + "evalue": "cluster jhub.27d845818da14ccbae3eeb66231cd791 is not running", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mGatewayClusterError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[26], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mcluster\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mscale\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m 2\u001b[0m cluster\u001b[38;5;241m.\u001b[39mshutdown() \n", + "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.12/site-packages/dask_gateway/client.py:1099\u001b[0m, in \u001b[0;36mGatewayCluster.scale\u001b[0;34m(self, n, **kwargs)\u001b[0m\n\u001b[1;32m 1091\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mscale\u001b[39m(\u001b[38;5;28mself\u001b[39m, n, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m 1092\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"Scale the cluster to ``n`` workers.\u001b[39;00m\n\u001b[1;32m 1093\u001b[0m \n\u001b[1;32m 1094\u001b[0m \u001b[38;5;124;03m Parameters\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1097\u001b[0m \u001b[38;5;124;03m The number of workers to scale to.\u001b[39;00m\n\u001b[1;32m 1098\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[0;32m-> 1099\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgateway\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mscale_cluster\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mname\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mn\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.12/site-packages/dask_gateway/client.py:687\u001b[0m, in \u001b[0;36mGateway.scale_cluster\u001b[0;34m(self, cluster_name, n, **kwargs)\u001b[0m\n\u001b[1;32m 677\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mscale_cluster\u001b[39m(\u001b[38;5;28mself\u001b[39m, cluster_name, n, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m 678\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"Scale a cluster to n workers.\u001b[39;00m\n\u001b[1;32m 679\u001b[0m \n\u001b[1;32m 680\u001b[0m \u001b[38;5;124;03m Parameters\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 685\u001b[0m \u001b[38;5;124;03m The number of workers to scale to.\u001b[39;00m\n\u001b[1;32m 686\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[0;32m--> 687\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msync\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_scale_cluster\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcluster_name\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mn\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.12/site-packages/dask_gateway/client.py:344\u001b[0m, in \u001b[0;36mGateway.sync\u001b[0;34m(self, func, *args, **kwargs)\u001b[0m\n\u001b[1;32m 340\u001b[0m future \u001b[38;5;241m=\u001b[39m asyncio\u001b[38;5;241m.\u001b[39mrun_coroutine_threadsafe(\n\u001b[1;32m 341\u001b[0m func(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs), \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mloop\u001b[38;5;241m.\u001b[39masyncio_loop\n\u001b[1;32m 342\u001b[0m )\n\u001b[1;32m 343\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 344\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfuture\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mresult\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 345\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mBaseException\u001b[39;00m:\n\u001b[1;32m 346\u001b[0m future\u001b[38;5;241m.\u001b[39mcancel()\n", + "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.12/concurrent/futures/_base.py:456\u001b[0m, in \u001b[0;36mFuture.result\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m 454\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m CancelledError()\n\u001b[1;32m 455\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_state \u001b[38;5;241m==\u001b[39m FINISHED:\n\u001b[0;32m--> 456\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m__get_result\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 457\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 458\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mTimeoutError\u001b[39;00m()\n", + "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.12/concurrent/futures/_base.py:401\u001b[0m, in \u001b[0;36mFuture.__get_result\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 399\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_exception:\n\u001b[1;32m 400\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 401\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_exception\n\u001b[1;32m 402\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[1;32m 403\u001b[0m \u001b[38;5;66;03m# Break a reference cycle with the exception in self._exception\u001b[39;00m\n\u001b[1;32m 404\u001b[0m \u001b[38;5;28mself\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n", + "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.12/site-packages/dask_gateway/client.py:669\u001b[0m, in \u001b[0;36mGateway._scale_cluster\u001b[0;34m(self, cluster_name, n)\u001b[0m\n\u001b[1;32m 667\u001b[0m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_scale_cluster\u001b[39m(\u001b[38;5;28mself\u001b[39m, cluster_name, n):\n\u001b[1;32m 668\u001b[0m url \u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39maddress\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m/api/v1/clusters/\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mcluster_name\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m/scale\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m--> 669\u001b[0m resp \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_request(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mPOST\u001b[39m\u001b[38;5;124m\"\u001b[39m, url, json\u001b[38;5;241m=\u001b[39m{\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcount\u001b[39m\u001b[38;5;124m\"\u001b[39m: n})\n\u001b[1;32m 670\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 671\u001b[0m msg \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m resp\u001b[38;5;241m.\u001b[39mjson()\n", + "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.12/site-packages/dask_gateway/client.py:416\u001b[0m, in \u001b[0;36mGateway._request\u001b[0;34m(self, method, url, json)\u001b[0m\n\u001b[1;32m 414\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(msg)\n\u001b[1;32m 415\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m resp\u001b[38;5;241m.\u001b[39mstatus \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m409\u001b[39m:\n\u001b[0;32m--> 416\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m GatewayClusterError(msg)\n\u001b[1;32m 417\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m resp\u001b[38;5;241m.\u001b[39mstatus \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m500\u001b[39m:\n\u001b[1;32m 418\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m GatewayServerError(msg)\n", + "\u001b[0;31mGatewayClusterError\u001b[0m: cluster jhub.27d845818da14ccbae3eeb66231cd791 is not running" + ] + } + ], + "source": [ + "cluster.scale(0)\n", + "cluster.shutdown() " ] }, { @@ -1252,6 +2057,30 @@ "metadata": {}, "outputs": [], "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "51d2f3d4-2958-4319-9104-c7873f693929", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dbe7fb3e-8fc9-4480-9d13-c29488bd795f", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d3c03814-9d5b-4a7b-ba74-78d3bcee17e4", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { @@ -1270,7 +2099,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.12" + "version": "3.12.12" }, "vscode": { "interpreter": { diff --git a/src/ESM_info.py b/src/ESM_info.py new file mode 100644 index 0000000..a47a02e --- /dev/null +++ b/src/ESM_info.py @@ -0,0 +1,285 @@ +# This is meant to hold all the boring ESM identifying information, to streamline the notebooks a bit. +# In particular, use these objects in 1_GCM_to_zarr.ipynb +# THIS SCRIPT IS NOT COMPLETE AND IS WIP. +# TODO: make sure filepaths are consistent with usage in 1_GCM_to_zarr.ipynb in terms of formatting scenarios and gcms etc. + +# # SCRATCH_URL = f"{os.environ['CIL_SCRATCH_PREFIX']}/{secrets.token_urlsafe(32 * 3//4)}" #Use only if new scratch space is needed +# SCRATCH_URL = "gs://impactlab-data-scratch/xq3rogfmpG12ATOWhGCYPNdSspH_vxlj" + + +GCMS = {"CanESM5": {"source_id":"CanESM5", + "institution_id":"CCCma", + "grid_label":"gr"}, + # "FGOALS-g3": {"source_id":"FGOALS-g3", + # "institution_id":"CAS", + # "grid_label":"gr"}, + "MIROC6": {"source_id":"MIROC6", + "institution_id":"MIROC", + "grid_label":"gn"}, + # "IPSL-CM6A-LR": {"source_id":"IPSL-CM6A-LR", + # "institution_id":"IPSL", + # "grid_label":"gr"}, + "MRI-ESM2-0": {"source_id":"MRI-ESM2-0", + "institution_id":"MRI", + "grid_label":"gr"}, + "NorESM2-LM": {"source_id":"NorESM2-LM", + "institution_id":"NCC", + "grid_label":"gn"}, + "ACCESS-CM2": {"source_id":"ACCESS-CM2", + "institution_id":"CSIRO-ARCCSS", + "grid_label":"gn"}, + "ACCESS-ESM1-5": {"source_id":"ACCESS-ESM1-5", + "institution_id":"CSIRO", + "grid_label":"gn"}, + } + + +# nested dictionaries for GCM -> scenario -> ensemble number +RAWDATAPATHS = {} + +# Add CanESM5 +RAWDATAPATHS.update({"CanESM5": { + "hist": { + "r1": [ "/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CCCma/tas_day_CanESM5_historical_r1i1p1f1_gn_18500101-20141231.nc", + "/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CCCma/tas_day_CanESM5_ssp245_r1i1p1f1_gn_20150101-21001231.nc" + ], + "r2": [ "/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CCCma/tas_day_CanESM5_historical_r2i1p1f1_gn_18500101-20141231.nc", + "/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CCCma/tas_day_CanESM5_ssp245_r2i1p1f1_gn_20150101-21001231.nc" + ], + "r3": [ "/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CCCma/tas_day_CanESM5_historical_r3i1p1f1_gn_18500101-20141231.nc", + "/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CCCma/tas_day_CanESM5_ssp245_r3i1p1f1_gn_20150101-21001231.nc" + ] + }, + "histnat": { + "r1": [ "/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CCCma/tas_day_CanESM5_hist-nat_r1i1p1f1_gn_18500101-20201231.nc" ], + "r2": [ "/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CCCma/tas_day_CanESM5_hist-nat_r2i1p1f1_gn_18500101-20201231.nc" ], + "r3": [ "/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CCCma/tas_day_CanESM5_hist-nat_r3i1p1f1_gn_18500101-20201231.nc" ] + }, + "picontrol": { + "r1": [ "/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CCCma/tas_day_CanESM5_piControl_r1i1p1f1_gn_60010101-62001231.nc" ], + "startyear": 6001 + } +}}) + +# Add FGOALS-g3 +RAWDATAPATHS.update({"FGOALS-g3": { + "hist": { + "r1": [f"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r1i1p1f1_gn_{yr}0101-{yr}1231.nc" for yr in range(1969, 2015) ] +\ + [f"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_ssp245_r1i1p1f1_gn_{yr}0101-{yr}1231.nc" for yr in range(2015, 2031)], + "r2": [f"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r2i1p1f1_gn_{yr}0101-{yr}1231.nc" for yr in range(1969, 2015) ] +\ + [f"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_ssp245_r2i1p1f1_gn_{yr}0101-{yr}1231.nc" for yr in range(2015, 2031)], + "r3": [f"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_historical_r3i1p1f1_gn_{yr}0101-{yr}1231.nc" for yr in range(1969, 2015) ] +\ + [f"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_ssp245_r3i1p1f1_gn_{yr}0101-{yr}1231.nc" for yr in range(2015, 2031)], + }, + "histnat": { + "r1": [f"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r1i1p1f1_gn_{yr}0101-{yr}1231.nc" for yr in range(1969, 2021)], + "r2": [f"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r2i1p1f1_gn_{yr}0101-{yr}1231.nc" for yr in range(1969, 2021)], + "r3": [f"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-g3_hist-nat_r3i1p1f1_gn_{yr}0101-{yr}1231.nc" for yr in range(1969, 2021)], + }, + + "picontrol": { + "r1": [ "/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CAS/tas_day_FGOALS-f3-L_piControl_r1i1p1f1_gr_11500101-11591231.nc" ], + "startyear": 1150 + } +}})# end FGOALS-g3 + +# Add MIROC6 +RAWDATAPATHS.update({"MIROC6": { + "hist": { + "r1":[f"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_historical_r1i1p1f1_gn_{dec}0101-{dec+9}1231.nc" for dec in range(1960,2010,10)] +\ + [f"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_historical_r1i1p1f1_gn_20100101-20141231.nc"] +\ + [f"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_ssp245_r1i1p1f1_gn_{dec}0101-{dec+9}1231.nc" for dec in range(2015,2035,10)], + "r2":[f"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_historical_r2i1p1f1_gn_{dec}0101-{dec+9}1231.nc" for dec in range(1960,2011,10)] +\ + [f"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_historical_r2i1p1f1_gn_20100101-20141231.nc"] +\ + [f"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_ssp245_r2i1p1f1_gn_{dec}0101-{dec+9}1231.nc" for dec in range(2015,2035,10)], + "r3":[f"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_historical_r3i1p1f1_gn_{dec}0101-{dec+9}1231.nc" for dec in range(1960,2011,10)] +\ + [f"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_historical_r3i1p1f1_gn_20100101-20141231.nc"] +\ + [f"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_ssp245_r3i1p1f1_gn_{dec}0101-{dec+9}1231.nc" for dec in range(2015,2035,10)], + }, + "histnat": { + "r1": [f"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_hist-nat_r1i1p1f1_gn_{dec}0101-{dec+9}1231.nc" for dec in range(1960,2020,10)] +\ + [f"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_hist-nat_r1i1p1f1_gn_20200101-20201231.nc"], + "r2": [f"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_hist-nat_r2i1p1f1_gn_{dec}0101-{dec+9}1231.nc" for dec in range(1960,2020,10)] +\ + [f"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_hist-nat_r2i1p1f1_gn_20200101-20201231.nc"], + "r3": [f"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_hist-nat_r3i1p1f1_gn_{dec}0101-{dec+9}1231.nc" for dec in range(1960,2020,10)] +\ + [f"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_hist-nat_r3i1p1f1_gn_20200101-20201231.nc"], + }, + "picontrol": { + "r1": ["/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_piControl_r1i1p1f1_gn_36700101-36791231.nc", + "/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_piControl_r1i1p1f1_gn_36800101-36891231.nc", + "/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MIROC/tas_day_MIROC6_piControl_r1i1p1f1_gn_36900101-36991231.nc"], + "startyear": 3670 + } +}}) + +# Add IPSL-CM6A-LR +RAWDATAPATHS.update({"IPSL-CM6A-LR": { + "hist": { + "r1": [ + "/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/IPSL/tas_day_IPSL-CM6A-LR_historical_r1i1p1f1_gr_18500101-20141231.nc", + "/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/IPSL/tas_day_IPSL-CM6A-LR_ssp245_r1i1p1f1_gr_20150101-21001231.nc", + ], + "r2": [ + "/gcs/impactlab-data/climate/source_data/CMIP6/raw//tas/IPSL/tas_day_IPSL-CM6A-LR_historical_r2i1p1f1_gr_18500101-20141231.nc", + "/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/IPSL/tas_day_IPSL-CM6A-LR_ssp245_r2i1p1f1_gr_20150101-21001231.nc", + ], + "r3": [ + "/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/IPSL/tas_day_IPSL-CM6A-LR_historical_r3i1p1f1_gr_18500101-20141231.nc", + "/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/IPSL/tas_day_IPSL-CM6A-LR_ssp245_r4i1p1f1_gr_20150101-21001231.nc", + ] + }, + "histnat": { + "r1": [ + "/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/IPSL/tas_day_IPSL-CM6A-LR_hist-nat_r1i1p1f1_gr_18500101-20201231.nc", + ], + "r2": [ + "/gcs/impactlab-data/climate/source_data/CMIP6/raw//tas/IPSL/tas_day_IPSL-CM6A-LR_hist-nat_r2i1p1f1_gr_18500101-20201231.nc", + ], + "r3": [ + "/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/IPSL/tas_day_IPSL-CM6A-LR_hist-nat_r6i1p1f1_gr_18500101-20201231.nc", + ], + }, + "picontrol": { + "r1": [ "/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/IPSL/tas_day_IPSL-CM6A-LR_piControl_r1i2p1f1_gr_18500101-20981231.nc" ], + "startyear": 1850, + } +}}) + +# Add MRI-ESM2-0 +RAWDATAPATHS.update({"MRI-ESM2-0": { + "hist": { + "r1": [ + "/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MRI/tas_day_MRI-ESM2-0_historical_r1i1p1f1_gr_19500101-19991231.nc", + "/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MRI/tas_day_MRI-ESM2-0_historical_r1i1p1f1_gr_20000101-20141231.nc", + "/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MRI/tas_day_MRI-ESM2-0_ssp245_r1i1p1f1_gr_20150101-20641231.nc", + ], + "r2": [ + "/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MRI/tas_day_MRI-ESM2-0_historical_r2i1p1f1_gr_19500101-19991231.nc", + "/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MRI/tas_day_MRI-ESM2-0_historical_r2i1p1f1_gr_20000101-20141231.nc", + "/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MRI/tas_day_MRI-ESM2-0_ssp245_r2i1p1f1_gr_20150101-20301231.nc", + ], + "r3": [ + "/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MRI/tas_day_MRI-ESM2-0_historical_r3i1p1f1_gr_19500101-19991231.nc", + "/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MRI/tas_day_MRI-ESM2-0_historical_r3i1p1f1_gr_20000101-20141231.nc", + "/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MRI/tas_day_MRI-ESM2-0_ssp245_r3i1p1f1_gr_20150101-20301231.nc", + ], + }, + "histnat": { + "r1": [ + "/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MRI/tas_day_MRI-ESM2-0_hist-nat_r1i1p1f1_gr_19500101-19991231.nc", + "/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MRI/tas_day_MRI-ESM2-0_hist-nat_r1i1p1f1_gr_20000101-20141231.nc", + "/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MRI/tas_day_MRI-ESM2-0_hist-nat_r1i1p1f1_gr_20150101-20201231.nc" + ], + "r2": [ + "/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MRI/tas_day_MRI-ESM2-0_hist-nat_r2i1p1f1_gr_19500101-19991231.nc", + "/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MRI/tas_day_MRI-ESM2-0_hist-nat_r2i1p1f1_gr_20000101-20201231.nc", + ], + "r3": [ + "/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MRI/tas_day_MRI-ESM2-0_hist-nat_r3i1p1f1_gr_19500101-19991231.nc", + "/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MRI/tas_day_MRI-ESM2-0_hist-nat_r3i1p1f1_gr_20000101-20141231.nc", + "/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MRI/tas_day_MRI-ESM2-0_hist-nat_r3i1p1f1_gr_20150101-20201231.nc" + ], + }, + "picontrol": { + "r1": [ "/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/MRI/tas_day_MRI-ESM2-0_piControl_r1i1p1f1_gn_20000101-20491231.nc" ], + "startyear": 2000 + } +}}) + +# Add ACCESS-CM2 +timepershist = ["19500101-19991231", + "20000101-20141231", + "20150101-20641231"] +timepershistnat = ["19500101-19991231", + "20000101-20201231",] + +RAWDATAPATHS.update( + {"ACCESS-CM2": + {"hist": + {"r1": [f"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CSIRO-ARCCSS/tas_day_ACCESS-CM2_historical_r1i1p1f1_gn_{tp}.nc" for tp in timepershist], + "r2": [f"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CSIRO-ARCCSS/tas_day_ACCESS-CM2_historical_r2i1p1f1_gn_{tp}.nc" for tp in timepershist], + "r3": [f"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CSIRO-ARCCSS/tas_day_ACCESS-CM2_historical_r3i1p1f1_gn_{tp}.nc" for tp in timepershist], + }, + "histnat": + {"r1": [f"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CSIRO-ARCCSS/tas_day_ACCESS-CM2_hist-nat_r1i1p1f1_gn_{tp}.nc" for tp in timepershist], + "r2": [f"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CSIRO-ARCCSS/tas_day_ACCESS-CM2_hist-nat_r2i1p1f1_gn_{tp}.nc" for tp in timepershist], + "r3": [f"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CSIRO-ARCCSS/tas_day_ACCESS-CM2_hist-nat_r3i1p1f1_gn_{tp}.nc" for tp in timepershist], + }, + "picontrol": + {"r1": [ + "/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CSIRO-ARCCSS/tas_day_ACCESS-CM2_piControl_r1i1p1f1_gn_14000101-14491231.nc" + ], + "startyear": 1400 + } + } + } +) + +# Add ACCESS-ESM1-5 +timepershist = ["19500101-19991231", + "20000101-20141231", + "20150101-20641231",] +timepershistnat = ["19500101-19991231", + "20000101-20201231",] + +RAWDATAPATHS.update({"ACCESS-ESM1-5": { + "hist": { + "r1": [f"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CSIRO/tas_day_ACCESS-ESM1-5_historical_r1i1p1f1_gn_{tp}.nc" for tp in timepershist], + "r2": [f"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CSIRO/tas_day_ACCESS-ESM1-5_historical_r2i1p1f1_gn_{tp}.nc" for tp in timepershist], + "r3": [f"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CSIRO/tas_day_ACCESS-ESM1-5_historical_r3i1p1f1_gn_{tp}.nc" for tp in timepershist], + }, + "histnat": { + "r1": [f"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CSIRO/tas_day_ACCESS-ESM1-5_hist-nat_r1i1p1f1_gn_{tp}.nc" for tp in timepershistnat], + "r2": [f"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CSIRO/tas_day_ACCESS-ESM1-5_hist-nat_r2i1p1f1_gn_{tp}.nc" for tp in timepershistnat], + "r3": [f"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CSIRO/tas_day_ACCESS-ESM1-5_hist-nat_r3i1p1f1_gn_{tp}.nc" for tp in timepershistnat], + }, + "picontrol": { + "r1": [ "/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/CSIRO/tas_day_ACCESS-ESM1-5_piControl_r1i1p1f1_gn_10510101-11001231.nc" ], + "startyear": 1051 + } +}}) + +# Add NorESM2-LM +timepers = ["19700101-19791231", "19800101-19891231", + "19900101-19991231", "20000101-20091231", + "20100101-20141231", "20150101-20201231"] + +RAWDATAPATHS.update({"NorESM2-LM": { + "histnat": { + "r1": [ f"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/NCC/tas_day_NorESM2-LM_hist-nat_r1i1p1f1_gn_{tp}.nc" for tp in timepers], + "r2": [ f"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/NCC/tas_day_NorESM2-LM_hist-nat_r2i1p1f1_gn_{tp}.nc" for tp in timepers], + "r3": [ f"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/NCC/tas_day_NorESM2-LM_hist-nat_r3i1p1f1_gn_{tp}.nc" for tp in timepers], + }, + "hist": { + "r1": [ f"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/NCC/tas_day_NorESM2-LM_historical_r1i1p1f1_gn_{tp}.nc" for tp in timepers], + "r2": [ f"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/NCC/tas_day_NorESM2-LM_historical_r2i1p1f1_gn_{tp}.nc" for tp in timepers], + "r3": [ f"/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/NCC/tas_day_NorESM2-LM_historical_r3i1p1f1_gn_{tp}.nc" for tp in timepers], + }, + "picontrol": { + "r1": [ "/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/NCC/tas_day_NorESM2-LM_piControl_r1i1p1f1_gn_20710101-20801231.nc", + "/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/NCC/tas_day_NorESM2-LM_piControl_r1i1p1f1_gn_20810101-20901231.nc", + "/gcs/impactlab-data/climate/source_data/CMIP6/raw/tas/NCC/tas_day_NorESM2-LM_piControl_r1i1p1f1_gn_20910101-21001231.nc" ], + "startyear": 2071 + } +}}) + +RAWDATAPATHS.update({"ERA5": + {"tas": + ["/gcs/impactlab-data/climate/source_data/ERA-5/tas/daily/netcdf/v1.1/tas_daily_{yr}.nc" for yr in range(1950, 2021)]}}) + +# # ERA5 data path +# IN_ERA5_RAW = "/gcs/impactlab-data/climate/source_data/ERA-5/tas/daily/netcdf/v1.1/tas_daily_*.nc" + +# # Generate output URLs using the new nested dictionary structure +# OUT_HIST_r1_URL = f"{SCRATCH_URL}/clean/hist/{source_id}/tas_r1.zarr" +# OUT_HISTNAT_r1_URL = f"{SCRATCH_URL}/clean/histnat/{source_id}/tas_r1.zarr" +# OUT_HIST_r2_URL = f"{SCRATCH_URL}/clean/hist/{source_id}/tas_r2.zarr" +# OUT_HISTNAT_r2_URL = f"{SCRATCH_URL}/clean/histnat/{source_id}/tas_r2.zarr" +# OUT_HIST_r3_URL = f"{SCRATCH_URL}/clean/hist/{source_id}/tas_r3.zarr" +# OUT_HISTNAT_r3_URL = f"{SCRATCH_URL}/clean/histnat/{source_id}/tas_r3.zarr" + +# OUT_ERA5_URL = f"{SCRATCH_URL}/clean/ERA5/{source_id}/tas.zarr" + +# print(OUT_HIST_r1_URL) +# print(OUT_HISTNAT_r2_URL) +# print(OUT_ERA5_URL) \ No newline at end of file