{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Using pydap and pandas to read EBAS data\n",
"\n",
"See more at http://ebas.nilu.no/\n",
" \n",
"The EBAS database collects observational data on atmospheric chemical composition and physical properties from a variety of national and international research projects and monitoring programs, such as ACTRIS, AMAP, EMEP, GAW and HELCOM, as well as for the Norwegian monitoring programs funded by the Norwegian Environment Agency, the Ministry of Climate and Environment and NILU – Norwegian Institute for Air Research."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Import Python packages"
]
},
{
"cell_type": "code",
"execution_count": 140,
"metadata": {},
"outputs": [],
"source": [
"from pydap.client import open_dods, open_url\n",
"from netCDF4 import num2date\n",
"import pandas as pd\n",
"import cftime\n",
"import matplotlib.pyplot as plt"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Get data directly from EBAS database\n",
"\n",
"\n",
"- syntax: `ST_STATION_CODE.FT_TYPE.RE_REGIME_CODE.MA_MATRIX_NAME.CO_COMP_NAME.DS_RESCODE.FI_REF.ME_REF.DL_DATA_LEVEL.dods`\n",
"- or: `#station.instrument_type.IMG.matrix.component.resolution.instrument_reference.datalevel.dods`\n",
"- if no level, then `..dods`\n",
"- if doesn't work, download one file to check what are the `FI_REF` and `ME_REF`.\n",
"- other example: `http://dev-ebas-pydap.nilu.no/NO0042G.Hg_mon.IMG.air.mercury.1h.NO01L_tekran_42_dup.NO01L_afs..dods`"
]
},
{
"cell_type": "code",
"execution_count": 31,
"metadata": {},
"outputs": [],
"source": [
"# get directly from EBAS\n",
"ds = open_dods(\n",
"'http://dev-ebas-pydap.nilu.no/' \n",
"'NO0042G.dmps.IMG.aerosol.particle_number_size_distribution'\n",
"'.1h.NO01L_NILU_DMPSmodel2_ZEP.NO01L_dmps_DMPS_ZEP01.2.dods')"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Format data into pandas dataframe"
]
},
{
"cell_type": "code",
"execution_count": 122,
"metadata": {},
"outputs": [],
"source": [
"#get the actual data\n",
"dmps_data = ds['particle_number_size_distribution_amean']\n",
"\n",
"# get normalised size distribution in dNdlogDp\n",
"dNdlogDp = dmps_data.particle_number_size_distribution_amean.data\n",
"\n",
"# get time in datatime format using function from netCDF4 package\n",
"tim_dmps = num2date(dmps_data.time.data,units='days since 1900-01-01 00:00:00',\n",
"calendar ='gregorian')\n",
"\n",
"# get diameter vector\n",
"dp_NILU = dmps_data.D.data\n",
"\n",
"# make DataFrame to simplify the handling of data\n",
"df_NILU = pd.DataFrame(dNdlogDp.byteswap().newbyteorder(), index=dp_NILU, columns=tim_dmps)"
]
},
{
"cell_type": "code",
"execution_count": 123,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
"\n",
"
\n",
" \n",
" \n",
" 2016-05-06 05:30:00 \n",
" 2016-05-06 06:30:00 \n",
" 2016-05-06 07:30:00 \n",
" 2016-05-06 08:30:00 \n",
" 2016-05-06 09:30:00 \n",
" 2016-05-06 10:30:00 \n",
" 2016-05-06 11:30:00 \n",
" 2016-05-06 12:30:00 \n",
" 2016-05-06 13:30:00 \n",
" 2016-05-06 14:30:00 \n",
" ... \n",
" 2017-12-31 14:30:00 \n",
" 2017-12-31 15:30:00 \n",
" 2017-12-31 16:30:00 \n",
" 2017-12-31 17:30:00 \n",
" 2017-12-31 18:30:00 \n",
" 2017-12-31 19:30:00 \n",
" 2017-12-31 20:30:00 \n",
" 2017-12-31 21:30:00 \n",
" 2017-12-31 22:30:00 \n",
" 2017-12-31 23:30:00 \n",
" \n",
" \n",
" \n",
" \n",
" 10.0 \n",
" 20.16 \n",
" 27.86 \n",
" 33.89 \n",
" 38.96 \n",
" 32.82 \n",
" 44.25 \n",
" 295.19 \n",
" 697.74 \n",
" 840.26 \n",
" 946.78 \n",
" ... \n",
" 8.52 \n",
" 8.29 \n",
" 13.86 \n",
" 8.18 \n",
" 9.96 \n",
" 8.41 \n",
" 6.92 \n",
" 9.81 \n",
" 10.94 \n",
" 6.79 \n",
" \n",
" \n",
" 12.0 \n",
" 41.16 \n",
" 37.93 \n",
" 46.07 \n",
" 53.32 \n",
" 50.76 \n",
" 66.31 \n",
" 169.43 \n",
" 543.82 \n",
" 1132.41 \n",
" 1570.59 \n",
" ... \n",
" 8.50 \n",
" 9.41 \n",
" 7.67 \n",
" 7.36 \n",
" 6.70 \n",
" 7.47 \n",
" 7.53 \n",
" 7.19 \n",
" 6.92 \n",
" 7.05 \n",
" \n",
" \n",
" 14.0 \n",
" 59.79 \n",
" 48.38 \n",
" 57.63 \n",
" 64.44 \n",
" 68.81 \n",
" 92.40 \n",
" 100.44 \n",
" 330.05 \n",
" 889.54 \n",
" 1576.16 \n",
" ... \n",
" 10.86 \n",
" 11.29 \n",
" 5.13 \n",
" 8.20 \n",
" 5.69 \n",
" 6.98 \n",
" 8.08 \n",
" 5.97 \n",
" 4.80 \n",
" 10.96 \n",
" \n",
" \n",
" 17.0 \n",
" 52.01 \n",
" 45.14 \n",
" 56.40 \n",
" 62.01 \n",
" 73.59 \n",
" 102.47 \n",
" 79.43 \n",
" 117.88 \n",
" 266.77 \n",
" 772.37 \n",
" ... \n",
" 15.88 \n",
" 13.76 \n",
" 8.77 \n",
" 9.54 \n",
" 8.48 \n",
" 7.13 \n",
" 7.99 \n",
" 6.12 \n",
" 5.17 \n",
" 12.47 \n",
" \n",
" \n",
" 21.0 \n",
" 21.98 \n",
" 28.03 \n",
" 37.41 \n",
" 44.52 \n",
" 54.98 \n",
" 73.01 \n",
" 87.58 \n",
" 115.21 \n",
" 152.36 \n",
" 243.23 \n",
" ... \n",
" 23.46 \n",
" 17.98 \n",
" 23.14 \n",
" 11.82 \n",
" 15.61 \n",
" 9.80 \n",
" 8.35 \n",
" 9.21 \n",
" 8.28 \n",
" 7.09 \n",
" \n",
" \n",
"
\n",
"
5 rows × 14515 columns
\n",
"
"
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
}
],
"source": [
"fig = plt.figure(1, figsize=[15,5])\n",
"ax = df_NILU[tsel].plot(lw=2, colormap='coolwarm', marker='.', markersize=10, title='Aerosol particle number size distribution', fontsize=15)\n",
"ax.set_xlabel(\"particle size\", fontsize=15)\n",
"ax.set_ylabel(\"Number of particles\", fontsize=15)\n",
"ax.title.set_size(20)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Save into a local csv file"
]
},
{
"cell_type": "code",
"execution_count": 173,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"size_dist06May2016_0530.csv\n"
]
}
],
"source": [
"filename = 'size_dist' + tsel.strftime('%d%B%Y_%H%M') + '.csv'\n",
"print(filename)"
]
},
{
"cell_type": "code",
"execution_count": 169,
"metadata": {},
"outputs": [],
"source": [
"df_NILU[tsel].to_csv(filename, sep='\\t', index=True, header=True)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Save into Galaxy history\n",
"- You can also use [bioblend](https://bioblend.readthedocs.io/en/v0.5.3/index.html) python package to directly interact with your galaxy history"
]
},
{
"cell_type": "code",
"execution_count": 174,
"metadata": {},
"outputs": [],
"source": [
"!put -p size_dist06May2016_0530.csv -t tabular"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Save to NIRD via s3fs\n",
"- Make sure you have your credential in `$HOME/.aws/credentials`"
]
},
{
"cell_type": "code",
"execution_count": 178,
"metadata": {},
"outputs": [],
"source": [
"import s3fs"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Set the path on NIRD (and add \"s3:/\" in front of it)"
]
},
{
"cell_type": "code",
"execution_count": 184,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"s3://work/size_dist06May2016_0530.csv\n"
]
}
],
"source": [
"s3_path = \"s3://work/\" + filename\n",
"print(s3_path)"
]
},
{
"cell_type": "code",
"execution_count": 182,
"metadata": {},
"outputs": [],
"source": [
"fsg = s3fs.S3FileSystem(anon=False,\n",
" client_kwargs={\n",
" 'endpoint_url': 'https://forces2021.uiogeo-apps.sigma2.no/'\n",
" })"
]
},
{
"cell_type": "code",
"execution_count": 189,
"metadata": {},
"outputs": [],
"source": [
"bytes_to_write = df_NILU[tsel].to_csv(None, sep='\\t', index=True, header=True).encode()\n",
"\n",
"with fsg.open(s3_path, 'wb') as f:\n",
" f.write(bytes_to_write)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Check your file"
]
},
{
"cell_type": "code",
"execution_count": 190,
"metadata": {},
"outputs": [],
"source": [
"dfo = pd.read_csv(fsg.open(s3_path), sep='\\t', index_col=0)"
]
},
{
"cell_type": "code",
"execution_count": 193,
"metadata": {
"tags": []
},
"outputs": [
{
"data": {
"text/html": [
"\n",
"\n",
"
\n",
" \n",
" \n",
" 2016-05-06 05:30:00 \n",
" \n",
" \n",
" \n",
" \n",
" 10.0 \n",
" 20.16 \n",
" \n",
" \n",
" 12.0 \n",
" 41.16 \n",
" \n",
" \n",
" 14.0 \n",
" 59.79 \n",
" \n",
" \n",
" 17.0 \n",
" 52.01 \n",
" \n",
" \n",
" 21.0 \n",
" 21.98 \n",
" \n",
" \n",
"
\n",
"
"
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
}
],
"source": [
"ax = dfo.plot(lw=2, color='red', marker='.', markersize=10, title='Aerosol particle number size distribution', fontsize=15)\n",
"ax.set_xlabel(\"particle size\", fontsize=15)\n",
"ax.set_ylabel(\"Number of particles\", fontsize=15)\n",
"ax.title.set_size(14)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.8"
}
},
"nbformat": 4,
"nbformat_minor": 4
}