From 08af01dd11066d73e312d60091cf2ccb3a46c157 Mon Sep 17 00:00:00 2001 From: Ari Date: Sun, 14 Aug 2022 15:31:42 -0400 Subject: [PATCH] config watcher --- api/src/automations_v2/automation_watcher.py | 110 ------------------ api/src/automations_v2/db.py | 4 + api/src/automations_v2/loader.py | 2 +- api/src/automations_v2/watchers/__init__.py | 0 api/src/automations_v2/watchers/automation.py | 56 +++++++++ .../automations_v2/watchers/base_handler.py | 57 +++++++++ .../automations_v2/watchers/base_watcher.py | 83 +++++++++++++ api/src/automations_v2/watchers/config.py | 61 ++++++++++ .../unittests/automations_v2/test_loader.py | 6 +- ...watcher.py => test_watchers_automation.py} | 16 +-- .../automations_v2/test_watchers_config.py | 37 ++++++ api/tests/unittests/automations_v2/utils.py | 8 +- 12 files changed, 316 insertions(+), 124 deletions(-) delete mode 100644 api/src/automations_v2/automation_watcher.py create mode 100644 api/src/automations_v2/watchers/__init__.py create mode 100644 api/src/automations_v2/watchers/automation.py create mode 100644 api/src/automations_v2/watchers/base_handler.py create mode 100644 api/src/automations_v2/watchers/base_watcher.py create mode 100644 api/src/automations_v2/watchers/config.py rename api/tests/unittests/automations_v2/{test_automation_watcher.py => test_watchers_automation.py} (84%) create mode 100644 api/tests/unittests/automations_v2/test_watchers_config.py diff --git a/api/src/automations_v2/automation_watcher.py b/api/src/automations_v2/automation_watcher.py deleted file mode 100644 index 82a9b85..0000000 --- a/api/src/automations_v2/automation_watcher.py +++ /dev/null @@ -1,110 +0,0 @@ -import asyncio -from threading import Thread, Event -from pathlib import Path -from time import sleep -from typing import List, Optional - -from watchdog.events import FileSystemEvent -from watchdog.observers import Observer -from watchdog.events import FileSystemEventHandler - -from src.logger import get_logger -from ..utils import throttle -from .db import AutomationDBConnection, DBError -from .loader import load_automation_path -from .types import ExtenededAutomationData -from .errors import FailedDeletion - -logger = get_logger(__file__) - - -class AutomationFileEventHandler(FileSystemEventHandler): - def __init__(self, automation_file: Path, automation_db_path: Path, file_loaded: Event) -> None: - super().__init__() - self.__automation_file = automation_file.absolute() - self.__automation_db = AutomationDBConnection(automation_db_path) - self.file_loaded = file_loaded - self.__loaded_automations: Optional[List[ExtenededAutomationData]] = None - asyncio.run(self.reload_file()) - - @throttle(seconds=60) - def reload_file(self): - try: - automations = list(load_automation_path(self.__automation_file)) - self.__automation_db.upsert_automations(automations) - self.__loaded_automations = automations - except Exception as err: - logger.error(err) - if isinstance(err, DBError): - logger.warning(f"Failed to reload {self.__automation_file} due to {err}") - else: - logger.warning(f"Failed to load {self.__automation_file} due to {err}") - if self.__loaded_automations is not None: - try: - self.__automation_db.delete_automations(self.__loaded_automations) - except DBError as err: - raise FailedDeletion( - f"failed to flush automations from database for the invalid file {self.__automation_file} due to {err}" - ) from err - self.file_loaded.set() - - def on_modified(self, event: FileSystemEvent): - logger.info(f"on_modified: {type(event)} {event.event_type} path : {event.src_path}") - if Path(event.src_path).absolute() == self.__automation_file: - asyncio.run(self.reload_file()) - - def on_created(self, event: FileSystemEvent): - logger.info(f"on_created: {type(event)} {event.event_type} path : {event.src_path}") - if Path(event.src_path).absolute() == self.__automation_file: - asyncio.run(self.reload_file()) - - def on_deleted(self, event: FileSystemEvent): - logger.info(f"on_deleted: {type(event)} {event.event_type} path : {event.src_path}") - if Path(event.src_path).absolute() == self.__automation_file: - asyncio.run(self.reload_file()) - - -class AutomationFileWatcher(Thread): - def __init__(self, automation_file: Path, automation_db_path: Path): - self.__automation_file = automation_file - self.__automation_db_path = automation_db_path - self.__stop = False - self.__file_loaded = Event() - Thread.__init__(self) - - def run(self) -> None: - self.__stop = False - logger.info(f"Watching {self.__automation_file} for changes...") - obs = Observer() - handler = AutomationFileEventHandler( - self.__automation_file, - self.__automation_db_path, - self.__file_loaded, - ) - obs.schedule(handler, self.__automation_file) - obs.start() - - while not self.__stop: - sleep(1) - - obs.stop() - logger.warning(f"Watcher for {self.__automation_file} is stopped!") - - def stop(self): - if not self.__stop: - logger.warning(f"Watcher for {self.__automation_file} is set to stop...") - self.__stop = True - - def __del__(self): - logger.warning(f"Deleting watcher for {self.__automation_file}...") - self.join() - - def join(self, timeout: float = 10) -> None: - logger.warning(f"Closing thread for watcher for {self.__automation_file}...") - self.stop() - return super().join(timeout) - - def wait_until_next_reload(self, reset_after: bool = False, *, timeout: int = 5): - self.__file_loaded.wait(timeout) - if reset_after: - self.__file_loaded.clear() diff --git a/api/src/automations_v2/db.py b/api/src/automations_v2/db.py index e065667..f7edc27 100644 --- a/api/src/automations_v2/db.py +++ b/api/src/automations_v2/db.py @@ -33,6 +33,10 @@ class AutomationDBConnection: def __init__(self, db_file: Path) -> None: self.db_file = db_file + def reset(self): + self.db_file.unlink(missing_ok=True) + self.create_db() + @contextmanager def get_cur(self) -> Iterator[sqlite3.Cursor]: conn = sqlite3.connect(self.db_file) diff --git a/api/src/automations_v2/loader.py b/api/src/automations_v2/loader.py index bd82722..a5b2bfe 100644 --- a/api/src/automations_v2/loader.py +++ b/api/src/automations_v2/loader.py @@ -29,7 +29,7 @@ def extract_automation_paths( elif isinstance(value, IncludedYaml): yield hass_config.root_path / value.path_str elif isinstance(value, IncludedYamlDir): - yield from extract_files(hass_config.root_path / value.name) + yield hass_config.root_path / value.name else: logger.warning(f"found an invalid type in {key}!") if not found: diff --git a/api/src/automations_v2/watchers/__init__.py b/api/src/automations_v2/watchers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/api/src/automations_v2/watchers/automation.py b/api/src/automations_v2/watchers/automation.py new file mode 100644 index 0000000..a90b3ec --- /dev/null +++ b/api/src/automations_v2/watchers/automation.py @@ -0,0 +1,56 @@ +from pathlib import Path +from typing import List, Optional + +from src.logger import get_logger + +from ..db import AutomationDBConnection, DBError +from ..loader import load_automation_path +from ..types import ExtenededAutomationData +from ..errors import FailedDeletion +from .base_handler import BaseFileEventHandler +from .base_watcher import BaseWatcher + +logger = get_logger(__file__) + + +class AutomationEventHandler(BaseFileEventHandler): + def __init__(self, automation_file: Path, automation_db_path: Path, **kwrgs) -> None: + self.__automation_file = automation_file.absolute() + self.__automation_db = AutomationDBConnection(automation_db_path) + self.__loaded_automations: Optional[List[ExtenededAutomationData]] = None + super().__init__(**kwrgs) + + def should_reload(self, p: Path) -> bool: + return p.absolute() == self.__automation_file + + def reload_file(self): + automations = list(load_automation_path(self.__automation_file)) + self.__automation_db.upsert_automations(automations) + self.__loaded_automations = automations + + def handle_failure(self, error: Exception): + logger.error(error) + if isinstance(error, DBError): + logger.warning(f"Failed to reload {self.__automation_file} due to {error}") + else: + logger.warning(f"Failed to load {self.__automation_file} due to {error}") + if self.__loaded_automations is not None: + try: + self.__automation_db.delete_automations(self.__loaded_automations) + except DBError as err: + raise FailedDeletion( + f"failed to flush automations from database for the invalid file {self.__automation_file} due to {err}" + ) from err + + +class AutomationFileWatcher(BaseWatcher): + def __init__(self, automation_file: Path, automation_db_path: Path): + self.__automation_db_path = automation_db_path + BaseWatcher.__init__(self, file_path=automation_file) + + def get_handler(self, **kwrgs) -> BaseFileEventHandler: + return AutomationEventHandler( + automation_file=self.file_path, + automation_db_path=self.__automation_db_path, + **kwrgs, + ) diff --git a/api/src/automations_v2/watchers/base_handler.py b/api/src/automations_v2/watchers/base_handler.py new file mode 100644 index 0000000..7d939e2 --- /dev/null +++ b/api/src/automations_v2/watchers/base_handler.py @@ -0,0 +1,57 @@ +import abc +import asyncio +import nest_asyncio + +from threading import Event +from pathlib import Path + +from watchdog.events import FileSystemEvent +from watchdog.events import FileSystemEventHandler + +from src.logger import get_logger +from src.utils import throttle + +nest_asyncio.apply() +logger = get_logger(__file__) + + +class BaseFileEventHandler(FileSystemEventHandler, abc.ABC): + def __init__(self, event_loaded: Event, event_errored: Event) -> None: + self.event_loaded = event_loaded + self.event_errored = event_errored + super().__init__() + asyncio.run(self._reload_file()) + + def should_reload(self, p: Path) -> bool: + raise NotImplementedError + + def reload_file(self): + raise NotImplementedError + + def handle_failure(self, error: Exception): + logger.error(error) + raise error from error + + @throttle(seconds=60) + def _reload_file(self): + try: + self.reload_file() + except Exception as err: + self.event_errored.set() + self.handle_failure(err) + self.event_loaded.set() + + def on_modified(self, event: FileSystemEvent): + logger.info(f"on_modified: {event.event_type} path : {event.src_path}") + if self.should_reload(Path(event.src_path)): + asyncio.run(self._reload_file()) + + def on_created(self, event: FileSystemEvent): + logger.info(f"on_created: {event.event_type} path : {event.src_path}") + if self.should_reload(Path(event.src_path)): + asyncio.run(self._reload_file()) + + def on_deleted(self, event: FileSystemEvent): + logger.info(f"on_deleted: {event.event_type} path : {event.src_path}") + if self.should_reload(Path(event.src_path)): + asyncio.run(self._reload_file()) diff --git a/api/src/automations_v2/watchers/base_watcher.py b/api/src/automations_v2/watchers/base_watcher.py new file mode 100644 index 0000000..7b44912 --- /dev/null +++ b/api/src/automations_v2/watchers/base_watcher.py @@ -0,0 +1,83 @@ +import abc +from pathlib import Path +from threading import Event +from time import sleep +from typing import Generic, Literal, Optional, TypeVar + +from watchdog.observers import Observer + +from src.logger import get_logger + +from .base_handler import BaseFileEventHandler + +logger = get_logger(__file__) + +BaseFileEventHandlerT = TypeVar("BaseFileEventHandlerT", bound=BaseFileEventHandler) + + +class BaseWatcher(Observer, abc.ABC, Generic[BaseFileEventHandlerT]): + def __init__(self, file_path: Path): + self.file_path = file_path + self.__event_file_loaded = Event() + self.__event_errored = Event() + Observer.__init__(self) + self.handler: Optional[BaseFileEventHandlerT] = None + + def start(self): + self.handler = self.get_handler( + event_errored=self.__event_errored, + event_loaded=self.__event_file_loaded, + ) + self.schedule(self.handler, self.file_path) + return super().start() + + def get_handler(self, event_loaded: Event, event_errored: Event) -> BaseFileEventHandlerT: + raise NotImplementedError + + def _post_stop(self, handler: BaseFileEventHandlerT): + return + + def __del__(self): + logger.warning(f"Deleting watcher for {self.file_path}...") + self.join() + + def join(self, timeout: float = 30) -> None: + logger.warning(f"Closing thread for watcher for {self.file_path}...") + super().stop() + super().join(timeout) + if self.is_alive(): + raise AssertionError(f"TIMEOUT! Failed to terminate watcher for {self.file_path}") + logger.warning(f"Watcher for {self.file_path} is stopped!") + if self.handler is not None: + self._post_stop(self.handler) + + def wait_until_next_reload(self, reset_after: bool = False, *, timeout: int = 5): + self.__event_file_loaded.wait(timeout) + if reset_after: + self.__event_file_loaded.clear() + + def wait_until_next_error(self, reset_after: bool = False, *, timeout: int = 5): + self.__event_errored.wait(timeout) + if reset_after: + self.__event_errored.clear() + + def wait_until_next_event( + self, + reset_after: bool = False, + *, + timeout: int = 5, + ) -> Literal["error", "loaded"]: + out = "" + while True: + if self.__event_file_loaded.is_set(): + out = "loaded" + break + elif self.__event_errored.is_set(): + out = "error" + break + else: + sleep(timeout) + if reset_after: + self.__event_file_loaded.clear() + self.__event_errored.clear() + return out diff --git a/api/src/automations_v2/watchers/config.py b/api/src/automations_v2/watchers/config.py new file mode 100644 index 0000000..99523dc --- /dev/null +++ b/api/src/automations_v2/watchers/config.py @@ -0,0 +1,61 @@ +from pathlib import Path +from threading import Event, Thread +from time import sleep +from typing import List, Literal + +from watchdog.observers import Observer + +from src.automations_v2.loader import extract_automation_paths +from src.automations_v2.db import AutomationDBConnection +from src.hass_config.loader import HassConfig +from src.logger import get_logger + +from .automation import AutomationFileWatcher +from .base_handler import BaseFileEventHandler +from .base_watcher import BaseWatcher + +logger = get_logger(__file__) + + +class ConfigEventHandler(BaseFileEventHandler): + def __init__(self, config_path: Path, db_path: Path, **kwargs) -> None: + self.__db = AutomationDBConnection(db_path) + self.__config_path = config_path + self.__hass_config = HassConfig(self.__config_path) + self.__automation_watchers: List[AutomationFileWatcher] = [] + super().__init__(**kwargs) + + def should_reload(self, p: Path) -> bool: + return p.absolute() == self.__config_path + + def reload_file(self): + self.clear_automation_watchers() + self.__db.reset() + for p in extract_automation_paths(self.__hass_config): + watcher = AutomationFileWatcher(automation_file=p, automation_db_path=self.__db.db_file) + watcher.start() + self.__automation_watchers.append(watcher) + + def clear_automation_watchers(self): + for auto_watchers in self.__automation_watchers: + auto_watchers.join() + self.__automation_watchers = [] + + def __del__(self): + self.clear_automation_watchers() + + +class ConfigFileWatcher(BaseWatcher): + def __init__(self, config_path: Path, automation_db_path: Path): + self.__automation_db_path = automation_db_path + BaseWatcher.__init__(self, file_path=config_path) + + def get_handler(self, **kwrgs) -> ConfigEventHandler: + return ConfigEventHandler( + config_path=self.file_path, + db_path=self.__automation_db_path, + **kwrgs, + ) + + def _post_stop(self, handler: ConfigEventHandler): + handler.clear_automation_watchers() diff --git a/api/tests/unittests/automations_v2/test_loader.py b/api/tests/unittests/automations_v2/test_loader.py index 8f008d7..04dd54d 100644 --- a/api/tests/unittests/automations_v2/test_loader.py +++ b/api/tests/unittests/automations_v2/test_loader.py @@ -99,9 +99,9 @@ class loader_tests(TestCase): (HA_CONFIG_EXAMPLE, 1), (HA_CONFIG2_EXAMPLE, 1), (HA_CONFIG3_EXAMPLE, 1), - (HA_CONFIG4_EXAMPLE, 3), - (HA_CONFIG5_EXAMPLE, 7), - (HA_CONFIG6_EXAMPLE, 8), + (HA_CONFIG4_EXAMPLE, 1), + (HA_CONFIG5_EXAMPLE, 3), + (HA_CONFIG6_EXAMPLE, 4), ]: with self.subTest(ha_path=ha_path, expected=expected): hass_config = HassConfig(ha_path) diff --git a/api/tests/unittests/automations_v2/test_automation_watcher.py b/api/tests/unittests/automations_v2/test_watchers_automation.py similarity index 84% rename from api/tests/unittests/automations_v2/test_automation_watcher.py rename to api/tests/unittests/automations_v2/test_watchers_automation.py index e416ff7..28960da 100644 --- a/api/tests/unittests/automations_v2/test_automation_watcher.py +++ b/api/tests/unittests/automations_v2/test_watchers_automation.py @@ -1,9 +1,9 @@ -from src.automations_v2.automation_watcher import AutomationFileWatcher +from src.automations_v2.watchers.automation import AutomationFileWatcher from tests.utils import HA_CONFIG6_EXAMPLE, HA_CONFIG_EXAMPLE, create_copy_of_example_config from .utils import TestWithDB -class watcher_tests(TestWithDB): +class automation_watcher_tests(TestWithDB): def test_bad_file(self): example_folder = create_copy_of_example_config() watcher = AutomationFileWatcher(example_folder / "automations.yaml", self.db_file) @@ -12,7 +12,7 @@ class watcher_tests(TestWithDB): self.assertEqual(total, 0) watcher.start() - watcher.wait_until_next_reload(True) + self.assertEqual(watcher.wait_until_next_event(True), "loaded") self.assertEqual(self.db.count_automations(), 32) (example_folder / "automations.yaml").write_text("haha!") @@ -28,7 +28,7 @@ class watcher_tests(TestWithDB): self.assertEqual(total, 0) watcher.start() - watcher.wait_until_next_reload(True) + self.assertEqual(watcher.wait_until_next_event(True), "loaded") self.assertEqual(self.db.count_automations(), 0) watcher.join() @@ -36,7 +36,7 @@ class watcher_tests(TestWithDB): example_folder = create_copy_of_example_config(HA_CONFIG6_EXAMPLE) watcher = AutomationFileWatcher(example_folder / "automations", self.db_file) watcher.start() - watcher.wait_until_next_reload(True) + self.assertEqual(watcher.wait_until_next_event(True), "loaded") self.assertEqual(self.db.count_automations(), 12) watcher.join() @@ -44,7 +44,7 @@ class watcher_tests(TestWithDB): example_folder = create_copy_of_example_config() watcher = AutomationFileWatcher(example_folder / "automations.yaml", self.db_file) watcher.start() - watcher.wait_until_next_reload(True) + self.assertEqual(watcher.wait_until_next_event(True), "loaded") self.assertEqual(self.db.count_automations(), 32) (example_folder / "automations.yaml").write_text("") (example_folder / "automations.yaml").write_text( @@ -58,7 +58,7 @@ class watcher_tests(TestWithDB): (example_folder / "automations.yaml").write_text( (HA_CONFIG_EXAMPLE / "automations.yaml").read_text() ) - watcher.wait_until_next_reload(True) + self.assertEqual(watcher.wait_until_next_event(True), "loaded") self.assertEqual(self.db.count_automations(), 32) watcher.join() @@ -70,7 +70,7 @@ class watcher_tests(TestWithDB): # self.assertEqual(total, 0) # watcher.start() - # watcher.wait_until_next_reload(True) + # self.assertEqual(watcher.wait_until_next_event(True), 'loaded') # self.assertEqual(self.db.count_automations(), 32) # autos = self.db.list_automations(0, 32) diff --git a/api/tests/unittests/automations_v2/test_watchers_config.py b/api/tests/unittests/automations_v2/test_watchers_config.py new file mode 100644 index 0000000..1115c92 --- /dev/null +++ b/api/tests/unittests/automations_v2/test_watchers_config.py @@ -0,0 +1,37 @@ +from src.automations_v2.watchers.config import ConfigFileWatcher + +from .utils import TestWithDB, make_temp_db +from tests.utils import ( + HA_CONFIG_EXAMPLE, + HA_CONFIG2_EXAMPLE, + HA_CONFIG3_EXAMPLE, + HA_CONFIG4_EXAMPLE, + HA_CONFIG5_EXAMPLE, + HA_CONFIG6_EXAMPLE, + create_copy_of_example_config, +) + + +class config_watcher_tests(TestWithDB): + def test_simple_config_loading(self): + for p, count in [ + (HA_CONFIG_EXAMPLE, 32), + (HA_CONFIG2_EXAMPLE, 1), + (HA_CONFIG3_EXAMPLE, 1032), + (HA_CONFIG4_EXAMPLE, 3), + (HA_CONFIG5_EXAMPLE, 10), + (HA_CONFIG6_EXAMPLE, 12), + ]: + with self.subTest(config=p.name): + db = make_temp_db() + db.reset() + config_path = create_copy_of_example_config(p) + config_watcher = ConfigFileWatcher( + config_path=config_path, + automation_db_path=db.db_file, + ) + self.assertEqual(db.count_automations(), 0) + config_watcher.start() + self.assertEqual(config_watcher.wait_until_next_event(True), "loaded") + self.assertEqual(db.count_automations(), count) + config_watcher.join() diff --git a/api/tests/unittests/automations_v2/utils.py b/api/tests/unittests/automations_v2/utils.py index 9e0c6df..5e8e72b 100644 --- a/api/tests/unittests/automations_v2/utils.py +++ b/api/tests/unittests/automations_v2/utils.py @@ -5,11 +5,15 @@ from tempfile import mktemp from src.automations_v2.db import AutomationDBConnection +def make_temp_db(): + return AutomationDBConnection(Path(mktemp() + ".db")) + + class TestWithDB(TestCase): def setUp(self) -> None: - self.db_file = Path(mktemp() + ".db") - self.db = AutomationDBConnection(self.db_file) + self.db = make_temp_db() self.db.create_db() + self.db_file = self.db.db_file def tearDown(self) -> None: self.db_file.unlink()