From 12b4ff6615271bfe17c21aef10a51b3818ec52a1 Mon Sep 17 00:00:00 2001 From: Ari Sosnovsky Date: Wed, 10 Nov 2021 12:46:30 -0500 Subject: [PATCH] added config loader --- api/.vscode/launch.json | 21 +++ api/src/config/HassSafeConstructor.py | 198 +++++++++++++++++++++++++ api/src/config/__init__.py | 8 +- api/src/config/configuration_parser.py | 105 ------------- api/src/config/globals.py | 6 - api/src/env.py | 1 + api/src/logger.py | 16 ++ api/tests/unittests/test_config.py | 18 +++ api/tests/unittests/test_db.py | 4 +- api/tests/utils.py | 5 + 10 files changed, 263 insertions(+), 119 deletions(-) create mode 100644 api/.vscode/launch.json create mode 100644 api/src/config/HassSafeConstructor.py delete mode 100644 api/src/config/configuration_parser.py delete mode 100644 api/src/config/globals.py create mode 100644 api/src/logger.py create mode 100644 api/tests/unittests/test_config.py create mode 100644 api/tests/utils.py diff --git a/api/.vscode/launch.json b/api/.vscode/launch.json new file mode 100644 index 0000000..b44a6aa --- /dev/null +++ b/api/.vscode/launch.json @@ -0,0 +1,21 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Python: Current File", + "type": "python", + "request": "launch", + "program": "${file}", + "console": "integratedTerminal" + } + // ,{ + // "name": "Debug Unit Test", + // "type": "python", + // "request": "test", + // "justMyCode": false + // } + ] +} \ No newline at end of file diff --git a/api/src/config/HassSafeConstructor.py b/api/src/config/HassSafeConstructor.py new file mode 100644 index 0000000..a4e47d5 --- /dev/null +++ b/api/src/config/HassSafeConstructor.py @@ -0,0 +1,198 @@ +from pathlib import Path +from typing import Literal, NamedTuple, Optional, Union +from ruamel.yaml import YAML +from ruamel.yaml.constructor import SafeConstructor +from ruamel.yaml.nodes import Node +from src.logger import logger + + +class LoadedYaml(NamedTuple): + secrets: dict + config: dict + + +class IncludedYaml(NamedTuple): + original_path: Path + data: dict + + +class SecretValue(NamedTuple): + name: str + value: str + + +def load_hass_config(file_path: Path) -> LoadedYaml: + HAY = HassYaml() + data = HAY.load(file_path / "configuration.yaml") + return LoadedYaml(HAY.secrets, data) + + +class HassYaml: + """Load a YAML file based on the standard defined by HASS + + Args: + file_path (Path): path to yaml file + + Returns: + dict + """ + + def __init__(self) -> None: + self.secrets: dict = {} + self.current_file_path: Optional[str] = None + self.yaml_loader = YAML(typ="safe") + # Compat with HASS + self.yaml_loader.allow_duplicate_keys = True + + # Stub HASS constructors + class HassSafeConstructor(SafeConstructor): + """Hass specific SafeConstructor.""" + + def _include_yaml(_loader, node: Node): + """Load another YAML file and embeds it using the !include tag. + + Example: + device_tracker: !include device_tracker.yaml + """ + root_path = ( + self.current_file_path.parent + if self.current_file_path is not None + else Path("/") + ) + file_path = _upward_recursive_search(root_path, node.value) + if file_path is None: + logger.warning(f"Could not find `!include {node.value}`!") + return None + else: + logger.info(f"Found `!include {node.value}` in {file_path.absolute()}!") + + return IncludedYaml(file_path, self.__internal_constructor_load(file_path)) + + def _secret_yaml(loader, node: Node): + """Load secrets and embed it into the configuration YAML.""" + secret = self.secrets.get(node.value, None) + + if secret is None: + logger.warning( + f"The secret {node.value} failed to load, please check your `secret.yaml` file." + ) + return SecretValue(node.value, secret) + + HassSafeConstructor.add_constructor("!include", _include_yaml) + HassSafeConstructor.add_constructor("!env_var", _stub_tag) + HassSafeConstructor.add_constructor("!secret", _secret_yaml) + HassSafeConstructor.add_constructor("!include_dir_list", _stub_tag) + HassSafeConstructor.add_constructor("!include_dir_merge_list", _stub_tag) + HassSafeConstructor.add_constructor("!include_dir_named", _stub_tag) + HassSafeConstructor.add_constructor("!include_dir_merge_named", _stub_tag) + + self.yaml_loader.Constructor = HassSafeConstructor + + def load(self, file_path: Path) -> dict: + """Loads a given yaml file into a dictionary + + Args: + file_path (Path): path to yaml file + + Returns: + dict + """ + self.secrets = self.__load_secrets(file_path.parent) + return self.__load(file_path) + + def __internal_constructor_load(self, file_path: Path): + """This is similar to `self.load` function but it is only called internally by the yaml parsers + as it also manages the references to which file we are currently processing + + Args: + file_path (Path): path to yaml file + + Returns: + dict + """ + last_file_path = self.current_file_path + loaded = self.__load(file_path) + self.current_file_path = last_file_path + return loaded + + def __load(self, file_path: Path) -> dict: + """Generic load yaml function. + This function is called by either `self.load` or `self.__internal_constructor_load` + + Args: + file_path (Path): path to yaml file + + Returns: + dict + """ + self.current_file_path = file_path + with file_path.open(encoding="utf-8") as conf_file: + # If configuration file is empty YAML returns None + # We convert that to an empty dict + return self.yaml_loader.load(conf_file) or {} + + def __load_secrets(self, root_path: Path) -> dict: + """Preloads all of the secrets into memory + + Args: + root_path (Path): root path where we search for the serects.yaml file + + Returns: + dict: all secrets in dict format + """ + file_path = _upward_recursive_search(root_path, "secrets.yaml") + if file_path is None: + logger.warning(f"Secrets file could not be found!") + return {} + else: + logger.info(f"Found secrets file in `{file_path.absolute()}`!") + + with file_path.open() as secret_file: + return YAML(typ="safe").load(secret_file) + + +def _upward_recursive_search(root_dir: Path, file_name: str) -> Optional[Path]: + """Recursively search for a file or return None + i.e. + assume we are in local deployment where our /config is sitting in /home/awesomeuser/ha/config + you specified somewhere in the yaml `device_tracker: !include device_tracker.yaml` + then + root_dir = Path(/home/awesomeuser/ha/config) + file_name = device_tracker.yaml + we then first search /home/awesomeuser/ha/config + if we find nothin go up to /home/awesomeuser/ha + if we find nothin go up to /home/awesomeuser + if we find nothin go up to /home + if we find nothin go up to / + if we still find nothing return None + if we found something at any stage, we will return it's direct path + + Args: + root_dir (Path) + file_name (str) + + Returns: + Optional[Path] + """ + dir_path = root_dir + file_path = dir_path / file_name + for _ in root_dir.parts: + if file_path.exists(): + return file_path + dir_path = root_dir + file_path = dir_path / file_name + return None + + +def _stub_tag(constructor, node: Node): + """Stub a constructor with a dictionary.""" + seen = getattr(constructor, "_stub_seen", None) + + if seen is None: + seen = constructor._stub_seen = set() + + if node.tag not in seen: + logger.warning(f"YAML tag {node.tag} is not supported") + seen.add(node.tag) + + return {} diff --git a/api/src/config/__init__.py b/api/src/config/__init__.py index 614a88f..108a215 100644 --- a/api/src/config/__init__.py +++ b/api/src/config/__init__.py @@ -1,7 +1,7 @@ import os +from decouple import config from pathlib import Path -from .globals import CONFIGURATION_PATH -from .configuration_parser import load_hass_config +from .HassSafeConstructor import load_hass_config def find_hass_config() -> Path: @@ -11,7 +11,7 @@ def find_hass_config() -> Path: """Put together the default configuration directory based on the OS.""" data_dir = Path( - os.getenv("APPDATA") if os.name == "nt" else os.path.expanduser("~") + config("APPDATA", None) if os.name == "nt" else os.path.expanduser("~") ) config_dir = data_dir / ".homeassistant" @@ -26,8 +26,6 @@ def find_hass_config() -> Path: def db_url_from_hass_config(path: Path): """Find the recorder database url from a HASS config dir.""" - global CONFIGURATION_PATH - CONFIGURATION_PATH = path.resolve() config = load_hass_config(path) recorder = config.get("recorder") diff --git a/api/src/config/configuration_parser.py b/api/src/config/configuration_parser.py deleted file mode 100644 index 074e3a6..0000000 --- a/api/src/config/configuration_parser.py +++ /dev/null @@ -1,105 +0,0 @@ -import os - -from pathlib import Path -from ruamel.yaml import YAML -from ruamel.yaml.constructor import SafeConstructor -from .globals import CONFIGURATION_PATH - - -class HassSafeConstructor(SafeConstructor): - """Hass specific SafeConstructor.""" - - -def _secret_yaml(loader, node): - """Load secrets and embed it into the configuration YAML.""" - dirpath = Path(os.path.dirname(loader.name)) - - # Recursively search for the secrets.yaml file - # this might be needed when a yaml file is included like - # `!included folder/file.yaml`. Same rules as - # https://www.home-assistant.io/docs/configuration/secrets/#debugging-secrets - fname = dirpath / "secrets.yaml" - for _ in range(len(dirpath.parts)): - if fname.exists() or ( - CONFIGURATION_PATH is not None - and CONFIGURATION_PATH.exists() - and dirpath.samefile(CONFIGURATION_PATH) - ): - break - dirpath = dirpath.parent - fname = dirpath / "secrets.yaml" - - try: - with fname.open() as secret_file: - secrets = YAML(typ="safe").load(secret_file) - except FileNotFoundError: - raise ValueError("Secrets file {} not found".format(fname)) from None - - try: - return secrets[node.value] - except KeyError: - raise ValueError("Secret {} not found".format(node.value)) from None - - -def _include_yaml(loader, node): - """Load another YAML file and embeds it using the !include tag. - - Example: - device_tracker: !include device_tracker.yaml - """ - dirpath = Path(os.path.dirname(loader.name)) - fname = dirpath / node.value - for _ in range(len(dirpath.parts)): - if fname.exists() or ( - CONFIGURATION_PATH is not None - and CONFIGURATION_PATH.exists() - and dirpath.samefile(CONFIGURATION_PATH) - ): - break - dirpath = dirpath.parent - fname = dirpath / node.value - - return load_yaml(os.path.join(dirpath, node.value)) - - -def _stub_tag(constructor, node): - """Stub a constructor with a dictionary.""" - seen = getattr(constructor, "_stub_seen", None) - - if seen is None: - seen = constructor._stub_seen = set() - - if node.tag not in seen: - print("YAML tag {} is not supported".format(node.tag)) - seen.add(node.tag) - - return {} - - -HassSafeConstructor.add_constructor("!include", _include_yaml) -HassSafeConstructor.add_constructor("!env_var", _stub_tag) -HassSafeConstructor.add_constructor("!secret", _secret_yaml) -HassSafeConstructor.add_constructor("!include_dir_list", _stub_tag) -HassSafeConstructor.add_constructor("!include_dir_merge_list", _stub_tag) -HassSafeConstructor.add_constructor("!include_dir_named", _stub_tag) -HassSafeConstructor.add_constructor("!include_dir_merge_named", _stub_tag) - - -def load_hass_config(dir_path: Path): - """Load the HASS config.""" - return load_yaml(dir_path / "configuration.yaml") - - -def load_yaml(file_path: Path): - """Load a YAML file.""" - yaml = YAML(typ="safe") - # Compat with HASS - yaml.allow_duplicate_keys = True - # Stub HASS constructors - HassSafeConstructor.name = str(file_path.absolute()) - yaml.Constructor = HassSafeConstructor - - with file_path.open(encoding="utf-8") as conf_file: - # If configuration file is empty YAML returns None - # We convert that to an empty dict - return yaml.load(conf_file) or {} diff --git a/api/src/config/globals.py b/api/src/config/globals.py deleted file mode 100644 index c820f2e..0000000 --- a/api/src/config/globals.py +++ /dev/null @@ -1,6 +0,0 @@ -from pathlib import Path -from typing import Optional - - -# TODO: get rid of this -CONFIGURATION_PATH: Optional[Path] = None diff --git a/api/src/env.py b/api/src/env.py index 496fcdc..a8f5dc7 100644 --- a/api/src/env.py +++ b/api/src/env.py @@ -4,3 +4,4 @@ from pathlib import Path ROOT_FOLDER = Path(__file__).parent.parent BUILD_VERSION = config("BUILD_VERSION", "0.0.0") +LOG_LEVEL = config("LOG_LEVEL", "INFO") diff --git a/api/src/logger.py b/api/src/logger.py new file mode 100644 index 0000000..a5af2e1 --- /dev/null +++ b/api/src/logger.py @@ -0,0 +1,16 @@ +import logging + +from src.env import LOG_LEVEL + +if LOG_LEVEL == "DEBUG": + logging.basicConfig(level=logging.DEBUG) +elif LOG_LEVEL == "INFO": + logging.basicConfig(level=logging.INFO) +elif LOG_LEVEL == "WARN": + logging.basicConfig(level=logging.WARN) +elif LOG_LEVEL == "ERROR": + logging.basicConfig(level=logging.ERROR) +else: + logging.basicConfig(level=logging.NOTSET) + +logger = logging.getLogger() diff --git a/api/tests/unittests/test_config.py b/api/tests/unittests/test_config.py new file mode 100644 index 0000000..de59344 --- /dev/null +++ b/api/tests/unittests/test_config.py @@ -0,0 +1,18 @@ +from unittest import TestCase + +from src.config.HassSafeConstructor import SecretValue, load_hass_config, IncludedYaml +from tests.utils import SAMPLE_HA_PATH + + +class config_finder_tests(TestCase): + def test_find_sample_config(self): + secrets, config = load_hass_config(SAMPLE_HA_PATH) + self.assertIsInstance(config["template"], IncludedYaml) + self.assertIsInstance(config["binary_sensor"], IncludedYaml) + self.assertIsInstance(config["group"], IncludedYaml) + self.assertIsInstance(config["switch"], IncludedYaml) + self.assertEqual(secrets["myamazingpassword"], "holymoly") + self.assertEqual( + config["automation"].data[32]["trigger"][0]["device_id"], + SecretValue("baby_btn_device_id", "347492ffc6b909a55ebe08745fca1bf6"), + ) diff --git a/api/tests/unittests/test_db.py b/api/tests/unittests/test_db.py index 20db840..959a22c 100644 --- a/api/tests/unittests/test_db.py +++ b/api/tests/unittests/test_db.py @@ -1,10 +1,8 @@ -from pathlib import Path from unittest import TestCase from src.db.connection import HassDatabase from src.db import entities +from tests.utils import SAMPLE_DB_PATH -THIS_FOLDER = Path(__file__).parent -SAMPLE_DB_PATH = THIS_FOLDER.parent / "samples" / "home-assistant_v2.db" db = HassDatabase(f"sqlite:///{SAMPLE_DB_PATH.absolute()}") diff --git a/api/tests/utils.py b/api/tests/utils.py new file mode 100644 index 0000000..6b41354 --- /dev/null +++ b/api/tests/utils.py @@ -0,0 +1,5 @@ +from pathlib import Path + +THIS_FOLDER = Path(__file__).parent +SAMPLE_HA_PATH = THIS_FOLDER / "samples" +SAMPLE_DB_PATH = SAMPLE_HA_PATH / "home-assistant_v2.db"