diff --git a/api/src/automations_v2/db.py b/api/src/automations_v2/db.py index f7edc27..db93f7b 100644 --- a/api/src/automations_v2/db.py +++ b/api/src/automations_v2/db.py @@ -78,7 +78,7 @@ class AutomationDBConnection: "{automation.alias}", "{automation.description}", "{automation.mode}", - "{automation.source_file}", + "{automation.source_file.absolute()}", "{automation.source_file_type}", "{encode_auto(automation)}" ) @@ -86,7 +86,7 @@ class AutomationDBConnection: alias = "{automation.alias}", description = "{automation.description}", mode = "{automation.mode}", - source_file = "{automation.source_file}", + source_file = "{automation.source_file.absolute()}", source_file_type = "{automation.source_file_type}", rest = "{encode_auto(automation)}" @@ -99,6 +99,13 @@ class AutomationDBConnection: f"DELETE FROM {automations_tbl} WHERE id = ?", [(a.id,) for a in automations] ) + def delete_automations_in_source_file(self, source_file: Path): + with self.get_cur() as cur: + cur.execute( + f"DELETE FROM {automations_tbl} WHERE source_file = ?", + (str(source_file.absolute()),), + ) + def get_automation(self, automation_id: str) -> ExtenededAutomationData: with self.get_cur() as cur: cur.execute( diff --git a/api/src/automations_v2/manager.py b/api/src/automations_v2/manager.py index e69de29..de3e39d 100644 --- a/api/src/automations_v2/manager.py +++ b/api/src/automations_v2/manager.py @@ -0,0 +1,51 @@ +from pathlib import Path + +from src.hass_config.loader import HassConfig +from src.yaml_serializer import dump_yaml, load_yaml +from .types import ExtenededAutomationData +from .watchers.config import ConfigFileWatcher +from .db import AutomationDBConnection + + +class AutomationManager: + def __init__(self, hass_config: HassConfig, db_path: Path) -> None: + self.hass_config = hass_config + self.db = AutomationDBConnection(db_path) + self.config_watcher = ConfigFileWatcher( + self.hass_config.get_configuration_path(), + self.db.db_file, + ) + self.config_watcher.start() + + def __del__(self): + self.config_watcher.join() + + @property + def is_db_loaded(self): + return self.config_watcher.is_loaded + + def list(self, offset: int, limit: int): + return self.db.list_automations(offset=offset, limit=limit) + + def count(self): + return self.db.count_automations() + + def get(self, auto_id: str): + return self.db.get_automation(auto_id) + + def update(self, automation: ExtenededAutomationData): + if automation.source_file_type == "obj": + raw_auto = automation.to_primitive(include_tags=False) + automation.source_file.write_text(dump_yaml(raw_auto)) + else: + with automation.source_file.open("r") as fp: + objs = load_yaml(fp) + if not isinstance(objs, list): + raise AssertionError( + f"Attemption to save automation to {automation.source_file} but encountering an invalid list yaml file. Did you modify this file prior to saving?\n{automation}" + ) + for i, obj in enumerate(objs): + if obj["id"] == automation.id: + objs[i] = automation.to_primitive(include_tags=False) + break + automation.source_file.write_text(dump_yaml(objs)) diff --git a/api/src/automations_v2/types.py b/api/src/automations_v2/types.py index 41999b8..f681db2 100644 --- a/api/src/automations_v2/types.py +++ b/api/src/automations_v2/types.py @@ -1,3 +1,4 @@ +from pathlib import Path from typing import Dict, List, Literal, Optional, Union from pydantic import BaseModel @@ -19,7 +20,7 @@ class BaseAutomation(BaseModel): class ExtenededAutomationData(BaseAutomation): - source_file: str + source_file: Path source_file_type: Literal["list", "obj"] tags: Dict[str, str] = Field(default_factory=dict) diff --git a/api/src/automations_v2/watchers/automation.py b/api/src/automations_v2/watchers/automation.py index a90b3ec..d5dcc54 100644 --- a/api/src/automations_v2/watchers/automation.py +++ b/api/src/automations_v2/watchers/automation.py @@ -2,6 +2,7 @@ from pathlib import Path from typing import List, Optional from src.logger import get_logger +from ...utils import extract_files from ..db import AutomationDBConnection, DBError from ..loader import load_automation_path @@ -17,30 +18,37 @@ class AutomationEventHandler(BaseFileEventHandler): def __init__(self, automation_file: Path, automation_db_path: Path, **kwrgs) -> None: self.__automation_file = automation_file.absolute() self.__automation_db = AutomationDBConnection(automation_db_path) - self.__loaded_automations: Optional[List[ExtenededAutomationData]] = None super().__init__(**kwrgs) - def should_reload(self, p: Path) -> bool: - return p.absolute() == self.__automation_file + def reload_file(self, file_path: Optional[Path], deleted: bool): + if file_path is None: + file_path = self.__automation_file - def reload_file(self): - automations = list(load_automation_path(self.__automation_file)) - self.__automation_db.upsert_automations(automations) - self.__loaded_automations = automations + self.clear_automations(file_path) + + if not deleted: + automations = list(load_automation_path(file_path)) + self.__automation_db.upsert_automations(automations) + + def clear_automations(self, file_path: Path): + if file_path.is_dir(): + for f in extract_files(file_path): + self.__automation_db.delete_automations_in_source_file(f) + else: + self.__automation_db.delete_automations_in_source_file(file_path) def handle_failure(self, error: Exception): logger.error(error) if isinstance(error, DBError): - logger.warning(f"Failed to reload {self.__automation_file} due to {error}") + logger.warning(f"Failed to reload {self.__automation_file} due to {repr(error)}") else: - logger.warning(f"Failed to load {self.__automation_file} due to {error}") - if self.__loaded_automations is not None: - try: - self.__automation_db.delete_automations(self.__loaded_automations) - except DBError as err: - raise FailedDeletion( - f"failed to flush automations from database for the invalid file {self.__automation_file} due to {err}" - ) from err + logger.warning(f"Failed to load {self.__automation_file} due to {repr(error)}") + try: + self.clear_automations(self.__automation_file) + except DBError as err: + raise FailedDeletion( + f"failed to flush automations from database for the invalid file {self.__automation_file} due to {repr(err)}" + ) from err class AutomationFileWatcher(BaseWatcher): diff --git a/api/src/automations_v2/watchers/base_handler.py b/api/src/automations_v2/watchers/base_handler.py index 7d939e2..2e9f0fb 100644 --- a/api/src/automations_v2/watchers/base_handler.py +++ b/api/src/automations_v2/watchers/base_handler.py @@ -1,5 +1,6 @@ import abc import asyncio +from typing import Optional import nest_asyncio from threading import Event @@ -20,12 +21,9 @@ class BaseFileEventHandler(FileSystemEventHandler, abc.ABC): self.event_loaded = event_loaded self.event_errored = event_errored super().__init__() - asyncio.run(self._reload_file()) + asyncio.run(self._reload_file(None, False)) - def should_reload(self, p: Path) -> bool: - raise NotImplementedError - - def reload_file(self): + def reload_file(self, file_path: Optional[Path], deleted: bool): raise NotImplementedError def handle_failure(self, error: Exception): @@ -33,9 +31,9 @@ class BaseFileEventHandler(FileSystemEventHandler, abc.ABC): raise error from error @throttle(seconds=60) - def _reload_file(self): + def _reload_file(self, file_path: Optional[Path], deleted: bool): try: - self.reload_file() + self.reload_file(file_path, deleted) except Exception as err: self.event_errored.set() self.handle_failure(err) @@ -43,15 +41,12 @@ class BaseFileEventHandler(FileSystemEventHandler, abc.ABC): def on_modified(self, event: FileSystemEvent): logger.info(f"on_modified: {event.event_type} path : {event.src_path}") - if self.should_reload(Path(event.src_path)): - asyncio.run(self._reload_file()) + asyncio.run(self._reload_file(Path(event.src_path), False)) def on_created(self, event: FileSystemEvent): logger.info(f"on_created: {event.event_type} path : {event.src_path}") - if self.should_reload(Path(event.src_path)): - asyncio.run(self._reload_file()) + asyncio.run(self._reload_file(Path(event.src_path), False)) def on_deleted(self, event: FileSystemEvent): logger.info(f"on_deleted: {event.event_type} path : {event.src_path}") - if self.should_reload(Path(event.src_path)): - asyncio.run(self._reload_file()) + asyncio.run(self._reload_file(Path(event.src_path), True)) diff --git a/api/src/automations_v2/watchers/base_watcher.py b/api/src/automations_v2/watchers/base_watcher.py index 7b44912..96cdceb 100644 --- a/api/src/automations_v2/watchers/base_watcher.py +++ b/api/src/automations_v2/watchers/base_watcher.py @@ -22,13 +22,14 @@ class BaseWatcher(Observer, abc.ABC, Generic[BaseFileEventHandlerT]): self.__event_errored = Event() Observer.__init__(self) self.handler: Optional[BaseFileEventHandlerT] = None + self.name = f"{self.__class__.__name__}-{self.file_path}-{self.name}" def start(self): self.handler = self.get_handler( event_errored=self.__event_errored, event_loaded=self.__event_file_loaded, ) - self.schedule(self.handler, self.file_path) + self.schedule(self.handler, self.file_path, recursive=self.file_path.is_dir()) return super().start() def get_handler(self, event_loaded: Event, event_errored: Event) -> BaseFileEventHandlerT: @@ -51,6 +52,14 @@ class BaseWatcher(Observer, abc.ABC, Generic[BaseFileEventHandlerT]): if self.handler is not None: self._post_stop(self.handler) + @property + def is_loaded(self): + return self.__event_file_loaded.is_set() + + @property + def is_errored(self): + return self.__event_errored.is_set() + def wait_until_next_reload(self, reset_after: bool = False, *, timeout: int = 5): self.__event_file_loaded.wait(timeout) if reset_after: diff --git a/api/src/automations_v2/watchers/config.py b/api/src/automations_v2/watchers/config.py index 99523dc..0095426 100644 --- a/api/src/automations_v2/watchers/config.py +++ b/api/src/automations_v2/watchers/config.py @@ -21,16 +21,20 @@ class ConfigEventHandler(BaseFileEventHandler): def __init__(self, config_path: Path, db_path: Path, **kwargs) -> None: self.__db = AutomationDBConnection(db_path) self.__config_path = config_path - self.__hass_config = HassConfig(self.__config_path) + self.__hass_config = HassConfig(self.__config_path.parent) self.__automation_watchers: List[AutomationFileWatcher] = [] super().__init__(**kwargs) - def should_reload(self, p: Path) -> bool: - return p.absolute() == self.__config_path - - def reload_file(self): + def reload_file(self, file_path: Path, deleted: bool): + if file_path is None: + file_path = self.__config_path + if file_path.absolute() != self.__config_path: + return self.clear_automation_watchers() self.__db.reset() + if deleted: + logger.error("Could not detect your configuration.yaml file, was is deleted?") + return for p in extract_automation_paths(self.__hass_config): watcher = AutomationFileWatcher(automation_file=p, automation_db_path=self.__db.db_file) watcher.start() diff --git a/api/src/logger.py b/api/src/logger.py index 0efaf6f..945e152 100644 --- a/api/src/logger.py +++ b/api/src/logger.py @@ -2,17 +2,11 @@ import logging from src.env import LOG_LEVEL -if LOG_LEVEL == "DEBUG": - logging.basicConfig(level=logging.DEBUG) -elif LOG_LEVEL == "INFO": - logging.basicConfig(level=logging.INFO) -elif LOG_LEVEL == "WARN": - logging.basicConfig(level=logging.WARN) -elif LOG_LEVEL == "ERROR": - logging.basicConfig(level=logging.ERROR) -else: - logging.basicConfig(level=logging.NOTSET) - +logging.basicConfig( + format="%(asctime)s|%(levelname)-4s[%(pathname)s:%(lineno)d@%(funcName)s] %(message)s", + datefmt="%Y-%m-%d:%H:%M:%S", + level=logging._nameToLevel.get(LOG_LEVEL, logging.INFO), +) logger = logging.getLogger() diff --git a/api/src/utils.py b/api/src/utils.py index 60a226a..a21e94a 100644 --- a/api/src/utils.py +++ b/api/src/utils.py @@ -2,7 +2,7 @@ import asyncio from datetime import datetime, timedelta from functools import wraps from pathlib import Path -from typing import Any, Callable, Coroutine, Iterator, Optional, Tuple, TypeVar +from typing import Any, Callable, Coroutine, Iterator, Optional, TypeVar FnOutT = TypeVar("FnOutT") diff --git a/api/tests/samples/config-6/automations/include_dir_merge_list/sub2/sub3/list3.yaml b/api/tests/samples/config-6/automations/include_dir_merge_list/sub2/sub3/list3.yaml index ab55fe7..1e0e460 100644 --- a/api/tests/samples/config-6/automations/include_dir_merge_list/sub2/sub3/list3.yaml +++ b/api/tests/samples/config-6/automations/include_dir_merge_list/sub2/sub3/list3.yaml @@ -1,9 +1,13 @@ - id: "sublist3-mlist2" alias: Sublist 3 - Merged List 2 - description: "" + description: "That was correct!" trigger: [] condition: [] - action: [] + action: + - service: "switch.turn_off" + target: + entity_id: "switch.bathroom" + data: mode: single - id: "sublist3-mlist3" alias: Sublist 3 - Merged List 3 diff --git a/api/tests/unittests/automations_v2/test_db.py b/api/tests/unittests/automations_v2/test_db.py index 6c2a46b..f985e57 100644 --- a/api/tests/unittests/automations_v2/test_db.py +++ b/api/tests/unittests/automations_v2/test_db.py @@ -1,3 +1,4 @@ +from pathlib import Path from src.automations_v2.types import ExtenededAutomationData from .utils import TestWithDB @@ -19,13 +20,13 @@ class db_tests(TestWithDB): ExtenededAutomationData( id="test", alias="test", - source_file="automation.yaml", + source_file=Path("automation.yaml").absolute(), source_file_type="obj", ), ExtenededAutomationData( id="test2", alias="test", - source_file="automation.yaml", + source_file=Path("automation.yaml").absolute(), source_file_type="obj", action=[{"event": {"nice": "one"}, "event_type": "custom"}], ), @@ -42,13 +43,13 @@ class db_tests(TestWithDB): ExtenededAutomationData( id="test", alias="test", - source_file="automation.yaml", + source_file=Path("automation.yaml").absolute(), source_file_type="obj", ), ExtenededAutomationData( id="test", alias="test", - source_file="automation.yaml", + source_file=Path("automation.yaml").absolute(), source_file_type="obj", action=[{"event": {"nice": "one"}, "event_type": "custom"}], ), @@ -79,3 +80,27 @@ class db_tests(TestWithDB): self.assertEqual(self.db.count_automations(), 2) self.db.delete_automations(originals) self.assertEqual(self.db.count_automations(), 0) + + def test_delete_automations_by_source_type(self): + originals = [ + ExtenededAutomationData( + id="test", + alias="test", + source_file="automation2.yaml", + source_file_type="obj", + ), + ExtenededAutomationData( + id="test2", + alias="test", + source_file="automation.yaml", + source_file_type="obj", + action=[{"event": {"nice": "one"}, "event_type": "custom"}], + ), + ] + self.assertEqual(self.db.count_automations(), 0) + self.db.upsert_automations(originals) + self.assertEqual(self.db.count_automations(), 2) + self.db.delete_automations_in_source_file(Path("automation2.yaml")) + self.assertEqual(self.db.count_automations(), 1) + [auto] = self.db.list_automations(0, 10) + self.assertEqual(auto.id, "test2") diff --git a/api/tests/unittests/automations_v2/test_manager.py b/api/tests/unittests/automations_v2/test_manager.py new file mode 100644 index 0000000..b52f2b4 --- /dev/null +++ b/api/tests/unittests/automations_v2/test_manager.py @@ -0,0 +1,45 @@ +from time import sleep +from src.hass_config.loader import HassConfig +from src.automations_v2.manager import AutomationManager + +from tests.utils import HA_CONFIG6_EXAMPLE, create_copy_of_example_config +from .utils import TestWithDB + + +class manager_tests(TestWithDB): + def test_load_complex_read_modify_delete(self): + config_path = create_copy_of_example_config(HA_CONFIG6_EXAMPLE) + automation_manager = AutomationManager( + HassConfig(config_path), + self.db_file, + ) + self.assertEqual(automation_manager.config_watcher.wait_until_next_event(True), "loaded") + self.assertEqual(automation_manager.count(), 12) + automation = automation_manager.get("sublist3-mlist2") + self.assertEqual(automation.description, "That was correct!") + self.assertEqual( + automation.action, + [ + { + "service": "switch.turn_off", + "target": {"entity_id": "switch.bathroom"}, + "data": None, + } + ], + ) + automation.trigger.append( + {"platform": "zone", "entity_id": "person.thor", "zone": "zone.azguard"} + ) + automation_manager.update(automation) + sleep(2) + automation = automation_manager.get("sublist3-mlist2") + self.assertEqual( + automation.trigger, + [ + { + "platform": "zone", + "entity_id": "person.thor", + "zone": "zone.azguard", + } + ], + ) diff --git a/api/tests/unittests/automations_v2/test_watchers_automation.py b/api/tests/unittests/automations_v2/test_watchers_automation.py index 28960da..7f43d5d 100644 --- a/api/tests/unittests/automations_v2/test_watchers_automation.py +++ b/api/tests/unittests/automations_v2/test_watchers_automation.py @@ -1,12 +1,23 @@ +from pathlib import Path +from time import sleep from src.automations_v2.watchers.automation import AutomationFileWatcher from tests.utils import HA_CONFIG6_EXAMPLE, HA_CONFIG_EXAMPLE, create_copy_of_example_config from .utils import TestWithDB class automation_watcher_tests(TestWithDB): + def make_safe_watcher(self, automation_file: Path): + self.watcher = AutomationFileWatcher(automation_file, self.db_file) + return self.watcher + + def tearDown(self) -> None: + if hasattr(self, "watcher"): + getattr(self, "watcher").join() + super().tearDown() + def test_bad_file(self): example_folder = create_copy_of_example_config() - watcher = AutomationFileWatcher(example_folder / "automations.yaml", self.db_file) + watcher = self.make_safe_watcher(example_folder / "automations.yaml") total = self.db.count_automations() self.assertEqual(total, 0) @@ -23,7 +34,7 @@ class automation_watcher_tests(TestWithDB): def test_load_empty_file(self): example_folder = create_copy_of_example_config() (example_folder / "automations.yaml").write_text("") - watcher = AutomationFileWatcher(example_folder / "automations.yaml", self.db_file) + watcher = self.make_safe_watcher(example_folder / "automations.yaml") total = self.db.count_automations() self.assertEqual(total, 0) @@ -34,7 +45,7 @@ class automation_watcher_tests(TestWithDB): def test_watch_folder(self): example_folder = create_copy_of_example_config(HA_CONFIG6_EXAMPLE) - watcher = AutomationFileWatcher(example_folder / "automations", self.db_file) + watcher = self.make_safe_watcher(example_folder / "automations") watcher.start() self.assertEqual(watcher.wait_until_next_event(True), "loaded") self.assertEqual(self.db.count_automations(), 12) @@ -42,7 +53,7 @@ class automation_watcher_tests(TestWithDB): def test_rapid_changes(self): example_folder = create_copy_of_example_config() - watcher = AutomationFileWatcher(example_folder / "automations.yaml", self.db_file) + watcher = self.make_safe_watcher(example_folder / "automations.yaml") watcher.start() self.assertEqual(watcher.wait_until_next_event(True), "loaded") self.assertEqual(self.db.count_automations(), 32) @@ -58,24 +69,6 @@ class automation_watcher_tests(TestWithDB): (example_folder / "automations.yaml").write_text( (HA_CONFIG_EXAMPLE / "automations.yaml").read_text() ) - self.assertEqual(watcher.wait_until_next_event(True), "loaded") + sleep(60) self.assertEqual(self.db.count_automations(), 32) watcher.join() - - # def test_valid_changes(self): - # example_folder = create_copy_of_example_config() - # watcher = AutomationFileWatcher(example_folder / "automations.yaml", self.db_file) - - # total = self.db.count_automations() - # self.assertEqual(total, 0) - - # watcher.start() - # self.assertEqual(watcher.wait_until_next_event(True), 'loaded') - # self.assertEqual(self.db.count_automations(), 32) - - # autos = self.db.list_automations(0, 32) - - # (example_folder / "automations.yaml").write_text("haha!") - # watcher.wait_until_next_reload(True, timeout=60 * 10) - # self.assertEqual(self.db.count_automations(), 0) - # watcher.join() diff --git a/api/tests/unittests/automations_v2/test_watchers_config.py b/api/tests/unittests/automations_v2/test_watchers_config.py index 1115c92..4fc6126 100644 --- a/api/tests/unittests/automations_v2/test_watchers_config.py +++ b/api/tests/unittests/automations_v2/test_watchers_config.py @@ -1,4 +1,5 @@ from src.automations_v2.watchers.config import ConfigFileWatcher +from src.hass_config.loader import HassConfig from .utils import TestWithDB, make_temp_db from tests.utils import ( @@ -27,7 +28,7 @@ class config_watcher_tests(TestWithDB): db.reset() config_path = create_copy_of_example_config(p) config_watcher = ConfigFileWatcher( - config_path=config_path, + config_path=HassConfig(config_path).get_configuration_path(), automation_db_path=db.db_file, ) self.assertEqual(db.count_automations(), 0) diff --git a/api/tmp.db b/api/tmp.db new file mode 100644 index 0000000..b6bc858 Binary files /dev/null and b/api/tmp.db differ