ENH: Added tutorial on writing files.

pysatTutorials/Tutorial-Creating_netCDF4_Files.ipynb

@@ -0,0 +1,266 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "575e87c5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import datetime as dt\n",
+    "import os\n",
+    "\n",
+    "import pysat\n",
+    "import netCDF4"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "009bad8b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Check for pysat data directory\n",
+    "if len(pysat.params['data_dirs']) == 0:\n",
+    "    print('Assigning demo directory.')\n",
+    "    pysat.params['data_dirs'] = './pysatDemo'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "22d0c4ee",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "filename = 'demo_test_file_{year:04d}{day:03d}.nc'\n",
+    "date = dt.datetime(2009, 1, 1)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3ab3a520",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Instrument with variety of 1D variables\n",
+    "inst = pysat.Instrument('pysat', 'testing')\n",
+    "\n",
+    "# Instrument with a variety of 1D and 2D variables\n",
+    "# inst = pysat.Instrument('pysat', 'testing2D')\n",
+    "\n",
+    "# Instrument with xarray data, mixed data dimensipnality.\n",
+    "# inst = pysat.Instrument('pysat', 'testmodel')\n",
+    "\n",
+    "# Instrument with xarray data, mixed data dimensipnality.\n",
+    "# inst = pysat.Instrument('pysat', 'testing2D_xarray')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "528d27f8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Load data\n",
+    "inst.load(date=date, use_header=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "73e28290",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Options when writing files.\n",
+    "\n",
+    "# Translate metadata labels to something new.\n",
+    "\n",
+    "# Metadata labels in the file may be different than used by Instrument object.\n",
+    "# Default behavior\n",
+    "meta_translation = None\n",
+    "inv_translation = None\n",
+    "export_nan = None\n",
+    "\n",
+    "# # Map existing labels to multiple labels in the file\n",
+    "\n",
+    "# Made up translation #1\n",
+    "# meta_translation = {inst.meta.labels.units: ['funny_units', 'serious_units'],\n",
+    "#                    inst.meta.labels.fill_val: ['funny_fill', 'fill_serious']}\n",
+    "# inv_translation = {'funny_units': inst.meta.labels.units,\n",
+    "#                    'serious_units': inst.meta.labels.units,\n",
+    "#                    'funny_fill': inst.meta.labels.fill_val,\n",
+    "#                    'fill_serious': inst.meta.labels.fill_val}\n",
+    "\n",
+    "\n",
+    "# Arbitrary processing of metadata is also supported when writing/loading files.\n",
+    "# See pysat documentation for more on the `meta_processor` keyword.\n",
+    "\n",
+    "\n",
+    "# Add additional metadata\n",
+    "\n",
+    "# `new_label` will only appear in the file for 'mlt' since values for other variables are NaN. To include metadata\n",
+    "# with NaN values, use the `export_nan` keyword. It will, by default, include fill, and the min and max values.\n",
+    "# Note that adding a new metadata type to meta will also add it to `meta.labels`.\n",
+    "\n",
+    "# drop_label = 'new_label'\n",
+    "# inst.meta['mlt'] = {drop_label: 1.}\n",
+    "# inst.meta.data\n",
+    "\n",
+    "# export_nan = [inst.meta.labels.fill_val, inst.meta.labels.max_val,\n",
+    "#               inst.meta.labels.min_val, inst.meta.labels.drop_label]\n",
+    "\n",
+    "\n",
+    "# By setting the `drop_meta_labels` keyword, users can prevent metadata information from being loaded.\n",
+    "\n",
+    "# Keep all metadata\n",
+    "drop_labels = []\n",
+    "\n",
+    "# Drop newly added label\n",
+    "# drop_labels = ['new_label']"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b14524d2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Add custom information to `inst.meta.header` which is written to file.\n",
+    "inst.meta.header.demo_thang_pysat_style = 'Yes'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "13bbd531",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Write file, but first, format filename.\n",
+    "form_filename = os.path.join(inst.files.data_path, filename.format(year=inst.yr, day=inst.doy))\n",
+    "\n",
+    "# Write file using `pysat.utils.io`.\n",
+    "pysat.utils.io.inst_to_netcdf(inst, form_filename, meta_translation=meta_translation, export_nan=export_nan)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "405c4d02",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# List global file attributes\n",
+    "with netCDF4.Dataset(form_filename) as data:\n",
+    "    print('Global File Attributes\\n')\n",
+    "    for attr in data.ncattrs():\n",
+    "        print('\\n', attr, ': ', data.getncattr(attr))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6575b4cd",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# List variable metadata\n",
+    "with netCDF4.Dataset(form_filename) as data:\n",
+    "    print('File Variable Attributes\\n')\n",
+    "    for var in data.variables.keys():\n",
+    "        print('File Variable: ', var)\n",
+    "        print(''.join(['----------------','-'*len(var)]))\n",
+    "        \n",
+    "        for nc_key in data.variables[var].ncattrs():\n",
+    "            print(nc_key, ': ', data.variables[var].getncattr(nc_key))\n",
+    "            \n",
+    "        print('\\n')\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a4ee3cba",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Load data and and meta using `pysat.utils`\n",
+    "data, meta = pysat.utils.io.load_netcdf(form_filename, pandas_format=inst.pandas_format, \n",
+    "                                        meta_translation=inv_translation,\n",
+    "                                        drop_meta_labels=drop_labels)\n",
+    "\n",
+    "# Print loaded metadata information to Jupyter.\n",
+    "meta.data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e4d8b292",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Print loaded data to Jupyter.\n",
+    "data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3d93a458",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Load data using pysat.Instrument. Presumes that current `inst` created NetCDF file.\n",
+    "# First, define general pysat.Instrument.\n",
+    "load_inst = pysat.Instrument('pysat', 'netcdf', pandas_format=inst.pandas_format, \n",
+    "                             update_files=True, file_format=filename, data_dir=inst.files.data_path,\n",
+    "                             meta_translation=inv_translation, drop_meta_labels=drop_labels)\n",
+    "\n",
+    "# Load data.\n",
+    "load_inst.load(2009, 1, use_header=True)\n",
+    "\n",
+    "# Print to Jupyter.\n",
+    "load_inst.data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "18edf55b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Print metadata to Jupyter.\n",
+    "load_inst.meta.data"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}