variable descriptions added to h5 meta

nsrdb/__init__.py

@@ -4,13 +4,16 @@
 
 import os
 
+import pandas as pd
+
 from ._version import __version__
 
 NSRDBDIR = os.path.dirname(os.path.realpath(__file__))
 CONFIGDIR = os.path.join(NSRDBDIR, 'config')
 DATADIR = os.path.join(NSRDBDIR, 'data_model', 'data')
 DEFAULT_VAR_META = os.path.join(CONFIGDIR, 'nsrdb_vars.csv')
 TESTDATADIR = os.path.join(os.path.dirname(NSRDBDIR), 'tests', 'data')
+VAR_DESCRIPTIONS = pd.read_csv(os.path.join(CONFIGDIR, 'var_descriptions.csv'))
 
 # This needs to go last because it depends on the global dirs above
 from nsrdb.nsrdb import NSRDB  # noqa: E402

nsrdb/cli.py

@@ -755,7 +755,6 @@ def tmy(ctx, config, verbose=False, pipeline_step=None, collect=False):
         }
     """  # noqa : D301
 
-    init_logger('nsrdb.tmy', log_level=ctx.obj['LOG_LEVEL'])
     mod_name = ModuleName.COLLECT_TMY if collect else ModuleName.TMY
     config = BaseCLI.from_config_preflight(
         ctx=ctx,

nsrdb/config/var_descriptions.csv

@@ -0,0 +1,37 @@
+var,description
+air_temperature,"Air temperature at 2 meters above the surface."
+alpha,"Alpha parameter specific to the dataset, representing a coefficient or scaling factor."
+aod,"Aerosol Optical Depth (AOD) measures the attenuation of solar radiation due to aerosol particles in the atmosphere, providing insight into air quality and visibility."
+asymmetry,"Asymmetry parameter specific to the dataset, often related to the directional properties of scattered radiation or aerosol properties."
+cld_opd_dcomp,"Cloud optical depth decomposed specific to the dataset, indicating the optical thickness of clouds and how it varies across different components."
+cld_press_acha,"Cloud pressure altitude specific to the dataset, representing the pressure level at which clouds are located."
+cloud_press_acha,"Cloud pressure altitude specific to the dataset, representing the pressure level at which clouds are located."
+cld_reff_dcomp,"Cloud effective radius decomposed specific to the dataset, describing the average size of cloud droplets or particles and its decomposition."
+clearsky_dhi,"Clear sky diffuse horizontal irradiance is the amount of solar radiation received per unit area on a horizontal surface under clear sky conditions, due to scattering from the atmosphere."
+clearsky_dni,"Clear sky direct normal irradiance is the amount of solar radiation received per unit area on a surface perpendicular to the sun's rays, under clear sky conditions."
+clearsky_ghi,"Clear sky global horizontal irradiance is the total amount of solar radiation received per unit area on a horizontal surface under clear sky conditions."
+cloud_type,"Type of cloud as classified by the dataset, indicating the cloud's physical structure or formation."
+dew_point,"Dew point temperature is the temperature to which air must be cooled at constant pressure and constant water vapor content to reach saturation, often leading to condensation."
+dhi,"Diffuse horizontal irradiance is the amount of solar radiation received per unit area on a horizontal surface, due to diffuse sky radiation."
+dni,"Direct normal irradiance is the amount of solar radiation received per unit area on a surface perpendicular to the sun's rays."
+ghi,"Global horizontal irradiance is the total amount of solar radiation received per unit area on a horizontal surface."
+fill_flag,"Fill flag indicating data quality or presence, often used to identify missing or interpolated data points."
+cloud_fill_flag,"Cloud fill flag indicating data quality or presence, specific to cloud-related data points."
+ozone,"Ozone concentration in the atmosphere, typically measured in Dobson units (DU) or parts per billion (ppb)."
+relative_humidity,"Relative humidity is the ratio of the amount of water vapor present in the air to the maximum amount that the air could hold at the given temperature, expressed as a percentage."
+solar_zenith_angle,"Solar zenith angle is the angle between the zenith (straight overhead point) and the position of the sun, measured from the observer's point on the Earth's surface."
+specific_humidity,"Specific humidity is the mass of water vapor present per unit mass of air, typically measured in grams of water vapor per kilogram of dry air."
+ssa,"Single Scattering Albedo (SSA) measures the ratio of the scattering coefficient to the extinction coefficient, providing information about the scattering properties of aerosols or particles."
+surface_albedo,"Surface albedo is the fraction of solar energy reflected by the Earth's surface, often varying based on surface type (e.g., snow-covered vs. vegetated)."
+surface_pressure,"Surface atmospheric pressure is the force per unit area exerted by the weight of the atmosphere above a specific location on the Earth's surface."
+total_precipitable_water,"Total precipitable water is the depth of liquid water that would result from precipitating all the water vapor in a vertical column above a given location, often measured in millimeters or centimeters."
+wind_direction,"Direction from which the wind is blowing, at 2 meters above the surface, measured in degrees clockwise from true north."
+wind_speed,"Wind speed at 2 meters above the surface."
+cloud_fraction,"Cloud fraction is the fraction of the sky covered by clouds, expressed as a percentage."
+cloud_probability,"Cloud probability is the likelihood of cloud presence at a specific location or time, often based on statistical or observational data."
+temp_3_75um_nom,"Nominal temperature at 3.75 micrometers wavelength, specific to the dataset."
+temp_11_0um_nom,"Nominal temperature at 11.0 micrometers wavelength, specific to the dataset."
+temp_11_0um_nom_stddev_3x3,"Standard deviation of nominal temperature at 11.0 micrometers wavelength, computed over a 3x3 grid or area."
+refl_0_65um_nom,"Nominal reflectance at 0.65 micrometers wavelength, specific to the dataset."
+refl_0_65um_nom_stddev_3x3,"Standard deviation of nominal reflectance at 0.65 micrometers wavelength, computed over a 3x3 grid or area."
+refl_3_75um_nom,"Nominal reflectance at 3.75 micrometers wavelength, specific to the dataset."

nsrdb/data_model/base_handler.py

@@ -8,7 +8,7 @@
 import numpy as np
 import pandas as pd
 
-from nsrdb import DATADIR, DEFAULT_VAR_META
+from nsrdb import DATADIR, DEFAULT_VAR_META, VAR_DESCRIPTIONS
 from nsrdb.file_handlers.file_system import NSRDBFileSystem as NSRDBfs
 
 logger = logging.getLogger(__name__)
@@ -78,8 +78,7 @@ def _parse_var_meta(inp):
 
         if var_meta is None:
             raise TypeError(
-                'Could not parse meta data for NSRDB variables '
-                'from: {}'.format(inp)
+                f'Could not parse meta data for NSRDB variables from: {inp}'
             )
 
         var_meta['var'] = var_meta['var'].str.strip(' ')
@@ -97,6 +96,7 @@ def attrs(self):
         """
 
         attrs = {
+            'description': self.description,
             'units': self.units,
             'scale_factor': self.scale_factor,
             'physical_min': self.physical_min,
@@ -245,6 +245,23 @@ def dtype(self):
         """
         return str(self.var_meta.loc[self.mask, 'final_dtype'].values[0])
 
+    @property
+    def description(self):
+        """Long variable description.
+
+        Returns
+        -------
+        description : str
+            Description of the variable to provide more info than the sometimes
+            opaque dset names.
+        """
+
+        return str(
+            VAR_DESCRIPTIONS.loc[
+                VAR_DESCRIPTIONS['var'] == self._name, 'description'
+            ].values[0]
+        )
+
     @property
     def units(self):
         """Get the units attribute.

nsrdb/gap_fill/irradiance_fill.py

@@ -22,7 +22,7 @@ def missing_cld_props(cloud_type, cld_opd_dcomp, cld_reff_dcomp):
         Array of cloud optical depths. Expected range is 0 - 160 with
         missing values <= 0.
     cld_reff_dcomp : np.ndarray
-        Array of cloud effective partical radii. Expected range is 0 - 160
+        Array of cloud effective particle radii. Expected range is 0 - 160
         with missing values <= 0.
 
     Returns

nsrdb/gap_fill/mlclouds_fill.py

@@ -26,7 +26,7 @@
 
 class MLCloudsFill:
     """
-    Use the MLClouds algorith with phygnn model to fill missing cloud data
+    Use the MLClouds algorithm with phygnn model to fill missing cloud data
     """
 
     DEFAULT_MODEL = MODEL_FPATH

tests/gap_fill/test_mlclouds_collect.py

@@ -1,4 +1,3 @@
-# pylint: skip-file
 """
 PyTest file for all sky daily processing after mlclouds daily gap fill