2018 auto config write

NREL · Jul 6, 2024 · c152a2f · c152a2f
1 parent 73978d8
commit c152a2f
Show file tree

Hide file tree

Showing 3 changed files with 61 additions and 55 deletions.
diff --git a/nsrdb/aggregation/aggregation.py b/nsrdb/aggregation/aggregation.py
@@ -78,26 +78,26 @@
 
 NSRDB_2018 = {
     'east': {
-        'data_sub_dir': 'east',
-        'tree_file': 'kdtree_nsrdb_meta_2km_east.pkl',
-        'meta_file': 'nsrdb_meta_2km_east.csv',
+        'data_sub_dir': 'nsrdb_full_east_2018_2km_10min/final/',
+        'tree_file': 'kdtree_nsrdb_meta_2km_full.pkl',
+        'meta_file': 'nsrdb_meta_2km_full.csv',
         'spatial': '2km',
-        'temporal': '15min',
-    },
-    'west': {
-        'data_sub_dir': 'west',
-        'tree_file': 'kdtree_west_psm_extent.pkl',
-        'meta_file': 'west_psm_extent.csv',
-        'spatial': '4km',
-        'temporal': '30min',
+        'temporal': '10min',
     },
     'conus': {
-        'data_sub_dir': 'conus',
+        'data_sub_dir': 'nsrdb_conus_east_2018_2km_5min/final/',
         'tree_file': 'kdtree_nsrdb_meta_2km_conus.pkl',
         'meta_file': 'nsrdb_meta_2km_conus.csv',
         'spatial': '2km',
         'temporal': '5min',
     },
+    'west': {
+        'data_sub_dir': 'nsrdb_west_full_2018_4km_30min/final/',
+        'tree_file': 'kdtree_nsrdb_meta_4km.pkl',
+        'meta_file': 'nsrdb_meta_4km.csv',
+        'spatial': '4km',
+        'temporal': '30min',
+    },
     'final': {
         'data_sub_dir': 'nsrdb_4km_30min',
         'fout': 'nsrdb_2018.h5',

diff --git a/nsrdb/create_configs.py b/nsrdb/create_configs.py
@@ -8,6 +8,7 @@
 import pprint
 
 from nsrdb import CONFIGDIR
+from nsrdb.aggregation.aggregation import NSRDB_2018
 from nsrdb.utilities.file_utils import (
     str_replace_dict,
 )
@@ -63,23 +64,13 @@ def main_all(cls, kwargs):
             cls.main(kwargs)
         elif kwargs['year'] == 2018:
             kwargs.update(
-                {
-                    'spatial': '2km',
-                    'extent': 'full',
-                    'freq': '10min',
-                    'satellite': 'east',
-                }
+                {'extent': 'full', 'satellite': 'east', **NSRDB_2018['east']}
             )
             cls.main(kwargs)
-            kwargs.update({'extent': 'conus', 'freq': '5min'})
+            kwargs.update({'extent': 'conus', **NSRDB_2018['conus']})
             cls.main(kwargs)
             kwargs.update(
-                {
-                    'spatial': '4km',
-                    'extent': 'full',
-                    'freq': '30min',
-                    'satellite': 'west',
-                }
+                {'extent': 'full', 'satellite': 'west', **NSRDB_2018['west']}
             )
             cls.main(kwargs)
         else:
@@ -183,10 +174,10 @@ def main(cls, kwargs):
         user_input['extent_tag'] = extent_tag_map[user_input['extent']]
         lon_seam = lon_seam_map[user_input['extent']]
 
-        if user_input['meta_file'] is None:
+        if user_input['meta_file'] is None and user_input['year'] != 2018:
             meta_file = f'nsrdb_meta_{user_input["spatial"]}'
 
-            if user_input['year'] > 2017:
+            if user_input['year'] > 2018:
                 meta_file += f'_{user_input["extent"]}'
 
             meta_file += f'_{user_input["satellite"]}_{lon_seam}.csv'
@@ -228,7 +219,7 @@ def post(cls, kwargs):
         pipeline_config = {'pipeline': []}
         out_dir = kwargs.get('out_dir', './')
 
-        if kwargs['year'] > 2017:
+        if kwargs['year'] > 2018:
             kwargs.update({'extent': 'conus'})
             config = cls._blend(kwargs)
             cls._write_config(
@@ -264,6 +255,14 @@ def post(cls, kwargs):
             pipeline_config['pipeline'].append(
                 {'collect-aggregate': './config_collect_aggregate.json'}
             )
+        elif kwargs['year'] == 2018:
+            config = cls._aggregate(kwargs)
+            cls._write_config(
+                config, 'config_aggregate.json', module_name='aggregate'
+            )
+            pipeline_config['pipeline'].append(
+                {'aggregate': './config_aggregate.json'}
+            )
         else:
             config = cls._blend(kwargs)
             cls._write_config(
@@ -333,33 +332,41 @@ def _aggregate(cls, kwargs):
 
         source_priority = user_input['source_priority']
 
-        NSRDB = {
-            'full_disk': {
-                'data_sub_dir': full_sub_dir,
-                'tree_file': full_tree_file,
-                'meta_file': full_meta_file,
-                'spatial': f'{user_input["full_spatial"]}',
-                'temporal': f'{user_input["full_freq"]}',
-            },
-            'conus': {
-                'data_sub_dir': conus_sub_dir,
-                'tree_file': conus_tree_file,
-                'meta_file': conus_meta_file,
-                'spatial': f'{user_input["conus_spatial"]}',
-                'temporal': f'{user_input["conus_freq"]}',
-            },
-            'final': {
-                'data_sub_dir': final_sub_dir,
-                'fout': 'nsrdb.h5',
-                'tree_file': tree_file.format(res=user_input['final_spatial']),
-                'meta_file': meta_file.format(res=user_input['final_spatial']),
-                'spatial': f'{user_input["final_spatial"]}',
-                'temporal': f'{user_input["final_freq"]}',
-                'source_priority': source_priority,
-            },
-        }
+        if user_input['year'] == 2018:
+            data = NSRDB_2018
+
+        else:
+            data = {
+                'full_disk': {
+                    'data_sub_dir': full_sub_dir,
+                    'tree_file': full_tree_file,
+                    'meta_file': full_meta_file,
+                    'spatial': f'{user_input["full_spatial"]}',
+                    'temporal': f'{user_input["full_freq"]}',
+                },
+                'conus': {
+                    'data_sub_dir': conus_sub_dir,
+                    'tree_file': conus_tree_file,
+                    'meta_file': conus_meta_file,
+                    'spatial': f'{user_input["conus_spatial"]}',
+                    'temporal': f'{user_input["conus_freq"]}',
+                },
+                'final': {
+                    'data_sub_dir': final_sub_dir,
+                    'fout': f'nsrdb_{user_input["year"]}.h5',
+                    'tree_file': tree_file.format(
+                        res=user_input['final_spatial']
+                    ),
+                    'meta_file': meta_file.format(
+                        res=user_input['final_spatial']
+                    ),
+                    'spatial': f'{user_input["final_spatial"]}',
+                    'temporal': f'{user_input["final_freq"]}',
+                    'source_priority': source_priority,
+                },
+            }
 
-        user_input['data'] = NSRDB
+        user_input['data'] = data
         run_name = user_input.get('run_name', None)
         user_input['run_name'] = (
             run_name

diff --git a/nsrdb/utilities/file_utils.py b/nsrdb/utilities/file_utils.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 """NSRDB data mover and manipulation utilities.
 
 @author: gbuster