core: move to get_config_dir/get_config_file

lib/backup.py

@@ -105,6 +105,9 @@ def create_backup(conf_base_dir, base_dir, filename_with_timestamp=False, before
         backup_filename += '_' + get_backupdate() + '_' + get_backuptime()
     backup_filename += '.zip'
 
+    # let backup.py create file/directory names the "old way", because this can
+    # be invoked by command line without a working SmartHome instance and so
+    # needs to create the paths for itself. Hope this always works... :)
     etc_dir = os.path.join(conf_base_dir, 'etc')
 
     if config_etc:

lib/item/structs.py

@@ -26,7 +26,7 @@
 
 import lib.shyaml as shyaml
 from lib.config import sanitize_items
-from lib.constants import (YAML_FILE, BASE_STRUCT, DIR_STRUCTS)
+from lib.constants import (YAML_FILE, BASE_STRUCT, DIR_STRUCTS, DIR_ETC)
 
 logger = logging.getLogger(__name__)
 
@@ -49,8 +49,8 @@ def __init__(self, smarthome):
         self.logger = logging.getLogger(__name__)
         self._sh = smarthome
         self.save_joined_structs = False
-        self.etc_dir = self._sh.get_etcdir()
-        self.structs_dir = self._sh.get_structsdir()
+        self.etc_dir = self._sh.get_config_dir(DIR_ETC)
+        self.structs_dir = self._sh.get_config_dir(DIR_STRUCTS)
 
 
     def return_struct_definitions(self, all=True):
@@ -86,13 +86,13 @@ def load_definitions_from_etc(self):
         - structs are read in from ../etc/struct.yaml by this procedure
         - further structs are read in from ../etc/struct_<prefix>.yaml by this procedure
         """
-        self.load_struct_definitions_from_file(self.etc_dir, BASE_STRUCT + YAML_FILE, '')
+        self.load_struct_definitions_from_file(self._sh.get_config_dir(DIR_ETC), BASE_STRUCT + YAML_FILE, '')
 
         # look for further struct files
         file_list = os.listdir(self.etc_dir)
         for filename in file_list:
             if filename.startswith(BASE_STRUCT + '_') and filename.endswith(YAML_FILE):
-                key_prefix = 'my.' + filename[7:-5]
+                key_prefix = 'my.' + filename[len(BASE_STRUCT):-len(YAML_FILE)]
                 self.load_struct_definitions_from_file(self.etc_dir, filename, key_prefix)
         return
 
@@ -119,7 +119,7 @@ def migrate_to_structs_dir(self):
         fl = os.listdir(self.etc_dir)
         for filename in fl:
             if filename.startswith(BASE_STRUCT + '_') and filename.endswith(YAML_FILE):
-                dest_fn = filename[7:]
+                dest_fn = filename[len(BASE_STRUCT):]
                 self.migrate_one_file(filename, dest_fn)
 
         filename = BASE_STRUCT + YAML_FILE
@@ -136,7 +136,7 @@ def load_definitions_from_structs(self):
 
         - structs are read in from ../structs/<name>.yaml by this procedure
         """
-        self.load_struct_definitions_from_file(self.etc_dir, BASE_STRUCT + YAML_FILE, '')
+        self.load_struct_definitions_from_file(self._sh.get_config_dir(DIR_ETC), BASE_STRUCT + YAML_FILE, '')
         if os.path.isdir(self.structs_dir):
             # look for struct files
             file_list = os.listdir(self.structs_dir)
@@ -145,7 +145,7 @@ def load_definitions_from_structs(self):
                     if filename == 'global_structs.yaml':
                         key_prefix = ''
                     else:
-                        key_prefix = 'my.' + filename[:-5]
+                        key_prefix = 'my.' + filename[:-len(YAML_FILE)]
                     self.load_struct_definitions_from_file(self.structs_dir, filename, key_prefix)
         else:
             self.logger.notice("../" + DIR_STRUCTS + " does not exist")

lib/log.py

@@ -34,7 +34,7 @@
 import collections
 
 import lib.shyaml as shyaml
-from lib.constants import YAML_FILE, DEFAULT_FILE, BASE_LOG
+from lib.constants import YAML_FILE, DEFAULT_FILE, BASE_LOG, DIR_ETC, DIR_VAR
 
 logs_instance = None
 
@@ -68,13 +68,15 @@ def __init__(self, sh):
         return
 
 
-    def configure_logging(self, config_filename=BASE_LOG + YAML_FILE):
+    def configure_logging(self, config_filename=''):
 
+        if not config_filename:
+            config_filename = self._sh.get_config_file(BASE_LOG)
         config_dict = self.load_logging_config(config_filename, ignore_notfound=True)
 
         if config_dict == None:
             print()
-            print(f"ERROR: Invalid logging configuration in file '{os.path.join(self._sh.get_etcdir(), config_filename)}'")
+            print(f"ERROR: Invalid logging configuration in file '{config_filename}'")
             print()
             exit(1)
 
@@ -141,7 +143,7 @@ def configure_logging(self, config_filename=BASE_LOG + YAML_FILE):
             logging.config.dictConfig(config_dict)
         except Exception as e:
             print()
-            print(f"ERROR: dictConfig: Invalid logging configuration in file '{os.path.join(self._sh.get_etcdir(), config_filename)}'")
+            print(f"ERROR: dictConfig: Invalid logging configuration in file '{config_filename}'")
             print(f"       Exception: {e}")
             print()
             return e
@@ -238,13 +240,16 @@ def return_logs(self):
 
     # ---------------------------------------------------------------------------
 
-    def load_logging_config(self, filename=BASE_LOG, ignore_notfound=False):
+    def load_logging_config(self, filename='', ignore_notfound=False):
         """
         Load config from logging.yaml to a dict
 
         If logging.yaml does not contain a 'shng_version' key, a backup is created
         """
-        conf_filename = os.path.join(self._sh.get_etcdir(), filename)
+        if not filename:
+            conf_filename = self._sh.get_config_file(BASE_LOG)
+        else:
+            conf_filename = os.path.join(self._sh.get_config_dir(DIR_ETC), filename)
         if not conf_filename.endswith(YAML_FILE) and not conf_filename.endswith(DEFAULT_FILE):
             conf_filename += YAML_FILE
         result = shyaml.yaml_load(conf_filename, ignore_notfound)
@@ -258,7 +263,7 @@ def save_logging_config(self, logging_config, create_backup=False):
         """
         if logging_config is not None:
             logging_config['shng_version'] = self._sh.version.split('-')[0][1:]
-            conf_filename = os.path.join(self._sh.get_etcdir(), BASE_LOG)
+            conf_filename = self._sh.get_config_file(BASE_LOG)
             shyaml.yaml_save_roundtrip(conf_filename, logging_config, create_backup=create_backup)
         return
 
@@ -270,7 +275,7 @@ def load_logging_config_for_edit(self):
         If logging.yaml does not contain a 'shng_version' key, a backup is created
         """
         #self.etc_dir = self._sh.get_etcdir()
-        conf_filename = os.path.join(self._sh.get_etcdir(), BASE_LOG)
+        conf_filename = self._sh.get_config_file(BASE_LOG)
         logging_config = shyaml.yaml_load_roundtrip(conf_filename)
         self.logger.info("load_logging_config_for_edit: shng_version={}".format(logging_config.get('shng_version', None)))
 
@@ -788,7 +793,7 @@ def __init__(self, logname='undefined', maxlen=35, level=logging.NOTSET,
         self._cache = cache
         self._maxlen = maxlen
         # save cache files in var/log/cache directory
-        cache_directory = os.path.join(logs_instance._sh.get_vardir(), 'log'+os.path.sep, 'cache'+os.path.sep)
+        cache_directory = os.path.join(logs_instance._sh.get_config_dir(DIR_VAR), 'log'+os.path.sep, 'cache'+os.path.sep)
         if cache is True:
             if not os.path.isdir(cache_directory):
                 os.makedirs(cache_directory)

lib/module.py

@@ -48,7 +48,7 @@
 
 import lib.config
 import lib.translation as translation
-from lib.constants import (KEY_CLASS_NAME, KEY_CLASS_PATH, KEY_INSTANCE, CONF_FILE, DIR_MODULES)
+from lib.constants import (KEY_CLASS_NAME, KEY_CLASS_PATH, KEY_INSTANCE, DIR_MODULES)
 from lib.utils import Utils
 from lib.metadata import Metadata
 
@@ -127,7 +127,7 @@ def _get_modulename_and_metadata(self, module, mod_conf):
         module_name = mod_conf.get('module_name','').lower()
         meta = None
         if module_name != '':
-            module_dir = os.path.join(self._basedir, DIR_MODULES, module_name)
+            module_dir = os.path.join(self._sh.get_config_dir(DIR_MODULES), module_name)
             if os.path.isdir(module_dir):
                 meta = Metadata(self._sh, module_name, 'module')
             else:

lib/scene.py

@@ -36,7 +36,7 @@
 from lib.utils import Utils
 from lib.shtime import Shtime
 import lib.shyaml as yaml
-from lib.constants import YAML_FILE
+from lib.constants import (YAML_FILE, DIR_SCENES)
 
 logger = logging.getLogger(__name__)
 
@@ -82,7 +82,7 @@ def _load_scenes(self):
         """
         self._scenes = {}
         self._learned_values = {}
-        self._scenes_dir = self._sh._scenes_dir
+        self._scenes_dir = self._sh.get_config_dir(DIR_SCENES)
         if not os.path.isdir(self._scenes_dir):
             logger.warning(translate("Directory '{scenes_dir}' not found. Ignoring scenes.", {'scenes_dir': self._scenes_dir}))
             return

lib/shtime.py

@@ -1140,9 +1140,7 @@ def _initialize_holidays(self):
         """
 
         if self.holidays is None:
-            self._etc_dir = self._sh._etc_dir
-            conf_filename = os.path.join(self._sh._etc_dir, BASE_HOLIDAY + YAML_FILE)
-            self.config = shyaml.yaml_load(conf_filename)
+            self.config = shyaml.yaml_load(self._sh.get_config_file(BASE_HOLIDAY))
             location = self.config.get('location', None)
 
             # prepopulate holidays for following years

lib/smarthome.py

@@ -176,6 +176,8 @@ def initialize_dir_vars(self):
         self._items_dir = os.path.join(self._conf_dir, DIR_ITEMS + os.path.sep)
         self._structs_dir = os.path.join(self._conf_dir, DIR_STRUCTS)
         self._logic_dir = os.path.join(self._conf_dir, DIR_LOGICS + os.path.sep)
+        # TODO: remove self._logic_dir later for uniformness (dirs with plural naming)
+        self._logics_dir = self._logic_dir
         self._functions_dir = os.path.join(self._conf_dir, DIR_UF + os.path.sep)
         self._scenes_dir = os.path.join(self._conf_dir, DIR_SCENES + os.path.sep)
 

lib/userfunctions.py

@@ -93,7 +93,7 @@ def init_lib(shng_base_dir=None, sh=None):
         base_dir = os.getcwd()
 
     if _sh:
-        _func_dir = _sh.get_functionsdir()
+        _func_dir = _sh.get_config_dir(DIR_UF)
     else:
         _func_dir = os.path.join(base_dir, _uf_subdir)
 

modules/admin/api_auth.py

@@ -31,6 +31,7 @@
 
 # from lib.item import Items
 from lib.utils import Utils
+from lib.constants import (DIR_ETC, DIR_MODULES)
 
 from .rest import RESTResource
 
@@ -44,8 +45,8 @@ def __init__(self, module):
         self.base_dir = self._sh.get_basedir()
         self.logger = logging.getLogger(__name__.split('.')[0] + '.' + __name__.split('.')[1] + '.' + __name__.split('.')[2][4:])
 
-        self.etc_dir = self._sh._etc_dir
-        self.modules_dir = os.path.join(self.base_dir, 'modules')
+        self.etc_dir = self._sh.get_config_dir(DIR_ETC)
+        self.modules_dir = self._sh.get_config_dir(DIR_MODULES)
 
         if self.module.rest_dispatch_force_exception:
             self.logger.notice(f"REST_dispatch_execute warnlevel is set to EXCEPTION")

modules/admin/api_config.py

@@ -27,6 +27,7 @@
 
 from lib.module import Modules
 import lib.shyaml as shyaml
+from lib.constants import (DIR_ETC, DIR_MODULES, BASE_SH, BASE_HOLIDAY, BASE_MODULE)
 
 from .rest import RESTResource
 
@@ -42,8 +43,8 @@ def __init__(self, module):
         self.base_dir = self._sh.get_basedir()
         self.logger = logging.getLogger(__name__.split('.')[0] + '.' + __name__.split('.')[1] + '.' + __name__.split('.')[2][4:])
 
-        self.etc_dir = self._sh.get_etcdir()
-        self.modules_dir = os.path.join(self.base_dir, 'modules')
+        self.etc_dir = self._sh.get_config_dir(DIR_ETC)
+        self.modules_dir = self._sh.get_config_dir(DIR_MODULES)
 
         self.core_conf = shyaml.yaml_load(os.path.join(self.modules_dir, 'core', 'module.yaml'))
         self.http_conf = shyaml.yaml_load(os.path.join(self.modules_dir, 'http', 'module.yaml'))
@@ -72,7 +73,7 @@ def update_configdict(self, config_dict, data, section='unknown'):
     #  Read holidays
     #
     def read_holidays(self):
-        self.holidays_confdata = shyaml.yaml_load(os.path.join(self.etc_dir, 'holidays.yaml'))
+        self.holidays_confdata = shyaml.yaml_load(self._sh.get_config_file(BASE_HOLIDAYS))
         if self.holidays_confdata.get('location', None) is not None:
             self.core_confdata['holidays_country'] = self.holidays_confdata['location'].get('country', '')
             self.core_confdata['holidays_province'] = self.holidays_confdata['location'].get('province', '')
@@ -95,7 +96,7 @@ def read_holidays(self):
     #  Update holidays
     #
     def update_holidays(self, data):
-        filename = os.path.join(self.etc_dir, 'holidays.yaml')
+        filename = self._sh.get_config_file(BASE_HOLIDAYS)
         self.holidays_confdata = shyaml.yaml_load_roundtrip(filename)
         self.logger.info("update_holidays: self.holidays_confdata = '{}'".format(self.holidays_confdata))
         self.logger.info("update_holidays: data['common']['data'] = '{}'".format(data['common']['data']))
@@ -152,10 +153,10 @@ def read(self, id=None):
         """
         self.logger.info("ConfigController.read(): config = {}".format(id))
 
-        self.core_confdata = shyaml.yaml_load(os.path.join(self.etc_dir, 'smarthome.yaml'))
+        self.core_confdata = shyaml.yaml_load(self._sh.get_config_file(BASE_SH))
         self.read_holidays()
 
-        self.module_confdata = shyaml.yaml_load(os.path.join(self.etc_dir, 'module.yaml'))
+        self.module_confdata = shyaml.yaml_load(self._sh.get_config_file(BASE_MODULE))
 
         result = {}
         if (not id) or id == 'core':
@@ -239,12 +240,12 @@ def update(self, id=None):
             self.update_holidays(data)
 
             # update etc/smarthome.yaml with data from admin frontend
-            self.core_confdata = shyaml.yaml_load_roundtrip(os.path.join(self.etc_dir, 'smarthome.yaml'))
+            self.core_confdata = shyaml.yaml_load_roundtrip(self._sh.get_config_file(BASE_SH))
             self.update_configdict(self.core_confdata, data, 'common')
-            shyaml.yaml_save_roundtrip(os.path.join(self.etc_dir, 'smarthome.yaml'), self.core_confdata, create_backup=True)
+            shyaml.yaml_save_roundtrip(self._sh.get_config_file(BASE_SH), self.core_confdata, create_backup=True)
 
             # update etc/module.yaml with data from admin frontend
-            self.module_confdata = shyaml.yaml_load_roundtrip(os.path.join(self.etc_dir, 'module.yaml'))
+            self.module_confdata = shyaml.yaml_load_roundtrip(self._sh.get_config_file(BASE_MODULE))
             self.update_configdict(self.module_confdata['http'], data, 'http')
             self.mod_http = Modules.get_instance().get_module('http')
             hashed_password = data.get('http', {}).get('data', {}).get('hashed_password', '')
@@ -276,7 +277,7 @@ def update(self, id=None):
                 self.module_confdata['mqtt'].pop('enabled', None)
             self.logger.info("Update: ['mqtt'] = {}".format(self.module_confdata['mqtt']))
             self.logger.info("Update: - enabled = {}".format(self.module_confdata['mqtt'].get('enabled', None)))
-            shyaml.yaml_save_roundtrip(os.path.join(self.etc_dir, 'module.yaml'), self.module_confdata, create_backup=True)
+            shyaml.yaml_save_roundtrip(self._sh.get_config_file(BASE_MODULE), self.module_confdata, create_backup=True)
 
             result = {"result": "ok"}
             return json.dumps(result)

modules/admin/api_files.py

@@ -35,6 +35,7 @@
 from lib.item_conversion import convert_yaml as convert_yaml
 from lib.item_conversion import parse_for_convert as parse_for_convert
 from lib.shtime import Shtime
+from lib.constants import (DIR_ETC, DIR_ITEMS, DIR_UF, DIR_SCENES, DIR_LOGICS, DIR_TPL, DIR_MODULES, BASE_LOG, BASE_STRUCT)
 
 
 # ======================================================================
@@ -51,16 +52,14 @@ def __init__(self, module):
         self.base_dir = self._sh.get_basedir()
         self.logger = logging.getLogger(__name__.split('.')[0] + '.' + __name__.split('.')[1] + '.' + __name__.split('.')[2][4:])
 
-        self.etc_dir = self._sh.get_etcdir()
-        self.items_dir = self._sh._items_dir
-        self.functions_dir = self._sh.get_functionsdir()
-        self.scenes_dir = self._sh._scenes_dir
-        self.logics_dir = self._sh.get_logicsdir()
-        self.template_dir = self._sh._template_dir
-        self.extern_conf_dir = self._sh._extern_conf_dir
-        self.modules_dir = os.path.join(self.base_dir, 'modules')
-        return
-
+        self.etc_dir = self._sh.get_config_dir(DIR_ETC)
+        self.items_dir = self._sh.get_config_dir(DIR_ITEMS)
+        self.functions_dir = self._sh.get_config_dir(DIR_UF)
+        self.scenes_dir = self._sh.get_config_dir(DIR_SCENES)
+        self.logics_dir = self._sh.get_config_dir(DIR_LOGICS)
+        self.template_dir = self._sh.get_config_dir(DIR_TPL)
+        self.extern_conf_dir = self._sh.get_confdir()
+        self.modules_dir = self._sh.get_config_dir(DIR_MODULES)
 
     def get_body(self, text=False, binary=False):
         """
@@ -113,7 +112,7 @@ def get_body(self, text=False, binary=False):
     def get_logging_config(self):
 
         self.logger.info("FilesController.get_logging_config()")
-        filename = os.path.join(self.etc_dir, 'logging.yaml')
+        filename = self._sh.get_config_file(BASE_LOG)
         read_data = None
         with open(filename, encoding='UTF-8') as f:
             read_data = f.read()
@@ -135,7 +134,7 @@ def save_logging_config(self):
         self.logger.debug("FilesController.save_logging_config(): '{}'".format(params))
 
 
-        filename = os.path.join(self.etc_dir, 'logging.yaml')
+        filename = self._sh.get_config_file(BASE_LOG)
         read_data = None
         with open(filename, 'w', encoding='UTF-8') as f:
             f.write(params)
@@ -150,7 +149,7 @@ def save_logging_config(self):
     def get_struct_config(self):
 
         self.logger.info("FilesController.get_struct_config()")
-        filename = os.path.join(self.etc_dir, 'struct.yaml')
+        filename = self._sh.get_config_file(BASE_STRUCT)
         if not(os.path.isfile(filename)):
             open(filename, 'a', encoding='UTF-8').close()
             self.logger.info("FilesController.get_struct_config(): created empty file {}".format(filename))
@@ -176,7 +175,7 @@ def save_struct_config(self):
         self.logger.debug("FilesController.save_struct_config(): '{}'".format(params))
 
 
-        filename = os.path.join(self.etc_dir, 'struct.yaml')
+        filename = self._sh.get_config_file(BASE_STRUCT)
         read_data = None
         with open(filename, 'w', encoding='UTF-8') as f:
             f.write(params)