config watcher

This commit is contained in:
ari committed 2022-08-14 15:31:42 -04:00
1 parent 586f52ba56
commit 08af01dd11
12 files changed
+316 -124

No files matched your search

@@ -1,110 +0,0 @@
import asyncio
from threading import Thread, Event
from pathlib import Path
from time import sleep
from typing import List, Optional
from watchdog.events import FileSystemEvent
from watchdog.observers import Observer
from watchdog.events import FileSystemEventHandler
from src.logger import get_logger
from ..utils import throttle
from .db import AutomationDBConnection, DBError
from .loader import load_automation_path
from .types import ExtenededAutomationData
from .errors import FailedDeletion
logger = get_logger(__file__)
class AutomationFileEventHandler(FileSystemEventHandler):
def __init__(self, automation_file: Path, automation_db_path: Path, file_loaded: Event) -> None:
super().__init__()
self.__automation_file = automation_file.absolute()
self.__automation_db = AutomationDBConnection(automation_db_path)
self.file_loaded = file_loaded
self.__loaded_automations: Optional[List[ExtenededAutomationData]] = None
asyncio.run(self.reload_file())
@throttle(seconds=60)
def reload_file(self):
try:
automations = list(load_automation_path(self.__automation_file))
self.__automation_db.upsert_automations(automations)
self.__loaded_automations = automations
except Exception as err:
logger.error(err)
if isinstance(err, DBError):
logger.warning(f"Failed to reload {self.__automation_file} due to {err}")
else:
logger.warning(f"Failed to load {self.__automation_file} due to {err}")
if self.__loaded_automations is not None:
try:
self.__automation_db.delete_automations(self.__loaded_automations)
except DBError as err:
raise FailedDeletion(
f"failed to flush automations from database for the invalid file {self.__automation_file} due to {err}"
) from err
self.file_loaded.set()
def on_modified(self, event: FileSystemEvent):
logger.info(f"on_modified: {type(event)} {event.event_type} path : {event.src_path}")
if Path(event.src_path).absolute() == self.__automation_file:
asyncio.run(self.reload_file())
def on_created(self, event: FileSystemEvent):
logger.info(f"on_created: {type(event)} {event.event_type} path : {event.src_path}")
if Path(event.src_path).absolute() == self.__automation_file:
asyncio.run(self.reload_file())
def on_deleted(self, event: FileSystemEvent):
logger.info(f"on_deleted: {type(event)} {event.event_type} path : {event.src_path}")
if Path(event.src_path).absolute() == self.__automation_file:
asyncio.run(self.reload_file())
class AutomationFileWatcher(Thread):
def __init__(self, automation_file: Path, automation_db_path: Path):
self.__automation_file = automation_file
self.__automation_db_path = automation_db_path
self.__stop = False
self.__file_loaded = Event()
Thread.__init__(self)
def run(self) -> None:
self.__stop = False
logger.info(f"Watching {self.__automation_file} for changes...")
obs = Observer()
handler = AutomationFileEventHandler(
self.__automation_file,
self.__automation_db_path,
self.__file_loaded,
)
obs.schedule(handler, self.__automation_file)
obs.start()
while not self.__stop:
sleep(1)
obs.stop()
logger.warning(f"Watcher for {self.__automation_file} is stopped!")
def stop(self):
if not self.__stop:
logger.warning(f"Watcher for {self.__automation_file} is set to stop...")
self.__stop = True
def __del__(self):
logger.warning(f"Deleting watcher for {self.__automation_file}...")
self.join()
def join(self, timeout: float = 10) -> None:
logger.warning(f"Closing thread for watcher for {self.__automation_file}...")
self.stop()
return super().join(timeout)
def wait_until_next_reload(self, reset_after: bool = False, *, timeout: int = 5):
self.__file_loaded.wait(timeout)
if reset_after:
self.__file_loaded.clear()
+4
View File
@@ -33,6 +33,10 @@ class AutomationDBConnection:
def __init__(self, db_file: Path) -> None:
self.db_file = db_file
def reset(self):
self.db_file.unlink(missing_ok=True)
self.create_db()
@contextmanager
def get_cur(self) -> Iterator[sqlite3.Cursor]:
conn = sqlite3.connect(self.db_file)
+1 -1
View File
@@ -29,7 +29,7 @@ def extract_automation_paths(
elif isinstance(value, IncludedYaml):
yield hass_config.root_path / value.path_str
elif isinstance(value, IncludedYamlDir):
yield from extract_files(hass_config.root_path / value.name)
yield hass_config.root_path / value.name
else:
logger.warning(f"found an invalid type in {key}!")
if not found:
Whitespace-only changes.
@@ -0,0 +1,56 @@
from pathlib import Path
from typing import List, Optional
from src.logger import get_logger
from ..db import AutomationDBConnection, DBError
from ..loader import load_automation_path
from ..types import ExtenededAutomationData
from ..errors import FailedDeletion
from .base_handler import BaseFileEventHandler
from .base_watcher import BaseWatcher
logger = get_logger(__file__)
class AutomationEventHandler(BaseFileEventHandler):
def __init__(self, automation_file: Path, automation_db_path: Path, **kwrgs) -> None:
self.__automation_file = automation_file.absolute()
self.__automation_db = AutomationDBConnection(automation_db_path)
self.__loaded_automations: Optional[List[ExtenededAutomationData]] = None
super().__init__(**kwrgs)
def should_reload(self, p: Path) -> bool:
return p.absolute() == self.__automation_file
def reload_file(self):
automations = list(load_automation_path(self.__automation_file))
self.__automation_db.upsert_automations(automations)
self.__loaded_automations = automations
def handle_failure(self, error: Exception):
logger.error(error)
if isinstance(error, DBError):
logger.warning(f"Failed to reload {self.__automation_file} due to {error}")
else:
logger.warning(f"Failed to load {self.__automation_file} due to {error}")
if self.__loaded_automations is not None:
try:
self.__automation_db.delete_automations(self.__loaded_automations)
except DBError as err:
raise FailedDeletion(
f"failed to flush automations from database for the invalid file {self.__automation_file} due to {err}"
) from err
class AutomationFileWatcher(BaseWatcher):
def __init__(self, automation_file: Path, automation_db_path: Path):
self.__automation_db_path = automation_db_path
BaseWatcher.__init__(self, file_path=automation_file)
def get_handler(self, **kwrgs) -> BaseFileEventHandler:
return AutomationEventHandler(
automation_file=self.file_path,
automation_db_path=self.__automation_db_path,
**kwrgs,
)
@@ -0,0 +1,57 @@
import abc
import asyncio
import nest_asyncio
from threading import Event
from pathlib import Path
from watchdog.events import FileSystemEvent
from watchdog.events import FileSystemEventHandler
from src.logger import get_logger
from src.utils import throttle
nest_asyncio.apply()
logger = get_logger(__file__)
class BaseFileEventHandler(FileSystemEventHandler, abc.ABC):
def __init__(self, event_loaded: Event, event_errored: Event) -> None:
self.event_loaded = event_loaded
self.event_errored = event_errored
super().__init__()
asyncio.run(self._reload_file())
def should_reload(self, p: Path) -> bool:
raise NotImplementedError
def reload_file(self):
raise NotImplementedError
def handle_failure(self, error: Exception):
logger.error(error)
raise error from error
@throttle(seconds=60)
def _reload_file(self):
try:
self.reload_file()
except Exception as err:
self.event_errored.set()
self.handle_failure(err)
self.event_loaded.set()
def on_modified(self, event: FileSystemEvent):
logger.info(f"on_modified: {event.event_type} path : {event.src_path}")
if self.should_reload(Path(event.src_path)):
asyncio.run(self._reload_file())
def on_created(self, event: FileSystemEvent):
logger.info(f"on_created: {event.event_type} path : {event.src_path}")
if self.should_reload(Path(event.src_path)):
asyncio.run(self._reload_file())
def on_deleted(self, event: FileSystemEvent):
logger.info(f"on_deleted: {event.event_type} path : {event.src_path}")
if self.should_reload(Path(event.src_path)):
asyncio.run(self._reload_file())
@@ -0,0 +1,83 @@
import abc
from pathlib import Path
from threading import Event
from time import sleep
from typing import Generic, Literal, Optional, TypeVar
from watchdog.observers import Observer
from src.logger import get_logger
from .base_handler import BaseFileEventHandler
logger = get_logger(__file__)
BaseFileEventHandlerT = TypeVar("BaseFileEventHandlerT", bound=BaseFileEventHandler)
class BaseWatcher(Observer, abc.ABC, Generic[BaseFileEventHandlerT]):
def __init__(self, file_path: Path):
self.file_path = file_path
self.__event_file_loaded = Event()
self.__event_errored = Event()
Observer.__init__(self)
self.handler: Optional[BaseFileEventHandlerT] = None
def start(self):
self.handler = self.get_handler(
event_errored=self.__event_errored,
event_loaded=self.__event_file_loaded,
)
self.schedule(self.handler, self.file_path)
return super().start()
def get_handler(self, event_loaded: Event, event_errored: Event) -> BaseFileEventHandlerT:
raise NotImplementedError
def _post_stop(self, handler: BaseFileEventHandlerT):
return
def __del__(self):
logger.warning(f"Deleting watcher for {self.file_path}...")
self.join()
def join(self, timeout: float = 30) -> None:
logger.warning(f"Closing thread for watcher for {self.file_path}...")
super().stop()
super().join(timeout)
if self.is_alive():
raise AssertionError(f"TIMEOUT! Failed to terminate watcher for {self.file_path}")
logger.warning(f"Watcher for {self.file_path} is stopped!")
if self.handler is not None:
self._post_stop(self.handler)
def wait_until_next_reload(self, reset_after: bool = False, *, timeout: int = 5):
self.__event_file_loaded.wait(timeout)
if reset_after:
self.__event_file_loaded.clear()
def wait_until_next_error(self, reset_after: bool = False, *, timeout: int = 5):
self.__event_errored.wait(timeout)
if reset_after:
self.__event_errored.clear()
def wait_until_next_event(
self,
reset_after: bool = False,
*,
timeout: int = 5,
) -> Literal["error", "loaded"]:
out = ""
while True:
if self.__event_file_loaded.is_set():
out = "loaded"
break
elif self.__event_errored.is_set():
out = "error"
break
else:
sleep(timeout)
if reset_after:
self.__event_file_loaded.clear()
self.__event_errored.clear()
return out
+61
View File
@@ -0,0 +1,61 @@
from pathlib import Path
from threading import Event, Thread
from time import sleep
from typing import List, Literal
from watchdog.observers import Observer
from src.automations_v2.loader import extract_automation_paths
from src.automations_v2.db import AutomationDBConnection
from src.hass_config.loader import HassConfig
from src.logger import get_logger
from .automation import AutomationFileWatcher
from .base_handler import BaseFileEventHandler
from .base_watcher import BaseWatcher
logger = get_logger(__file__)
class ConfigEventHandler(BaseFileEventHandler):
def __init__(self, config_path: Path, db_path: Path, **kwargs) -> None:
self.__db = AutomationDBConnection(db_path)
self.__config_path = config_path
self.__hass_config = HassConfig(self.__config_path)
self.__automation_watchers: List[AutomationFileWatcher] = []
super().__init__(**kwargs)
def should_reload(self, p: Path) -> bool:
return p.absolute() == self.__config_path
def reload_file(self):
self.clear_automation_watchers()
self.__db.reset()
for p in extract_automation_paths(self.__hass_config):
watcher = AutomationFileWatcher(automation_file=p, automation_db_path=self.__db.db_file)
watcher.start()
self.__automation_watchers.append(watcher)
def clear_automation_watchers(self):
for auto_watchers in self.__automation_watchers:
auto_watchers.join()
self.__automation_watchers = []
def __del__(self):
self.clear_automation_watchers()
class ConfigFileWatcher(BaseWatcher):
def __init__(self, config_path: Path, automation_db_path: Path):
self.__automation_db_path = automation_db_path
BaseWatcher.__init__(self, file_path=config_path)
def get_handler(self, **kwrgs) -> ConfigEventHandler:
return ConfigEventHandler(
config_path=self.file_path,
db_path=self.__automation_db_path,
**kwrgs,
)
def _post_stop(self, handler: ConfigEventHandler):
handler.clear_automation_watchers()
@@ -99,9 +99,9 @@ class loader_tests(TestCase):
(HA_CONFIG_EXAMPLE, 1),
(HA_CONFIG2_EXAMPLE, 1),
(HA_CONFIG3_EXAMPLE, 1),
(HA_CONFIG4_EXAMPLE, 3),
(HA_CONFIG5_EXAMPLE, 7),
(HA_CONFIG6_EXAMPLE, 8),
(HA_CONFIG4_EXAMPLE, 1),
(HA_CONFIG5_EXAMPLE, 3),
(HA_CONFIG6_EXAMPLE, 4),
]:
with self.subTest(ha_path=ha_path, expected=expected):
hass_config = HassConfig(ha_path)
@@ -1,9 +1,9 @@
from src.automations_v2.automation_watcher import AutomationFileWatcher
from src.automations_v2.watchers.automation import AutomationFileWatcher
from tests.utils import HA_CONFIG6_EXAMPLE, HA_CONFIG_EXAMPLE, create_copy_of_example_config
from .utils import TestWithDB
class watcher_tests(TestWithDB):
class automation_watcher_tests(TestWithDB):
def test_bad_file(self):
example_folder = create_copy_of_example_config()
watcher = AutomationFileWatcher(example_folder / "automations.yaml", self.db_file)
@@ -12,7 +12,7 @@ class watcher_tests(TestWithDB):
self.assertEqual(total, 0)
watcher.start()
watcher.wait_until_next_reload(True)
self.assertEqual(watcher.wait_until_next_event(True), "loaded")
self.assertEqual(self.db.count_automations(), 32)
(example_folder / "automations.yaml").write_text("haha!")
@@ -28,7 +28,7 @@ class watcher_tests(TestWithDB):
self.assertEqual(total, 0)
watcher.start()
watcher.wait_until_next_reload(True)
self.assertEqual(watcher.wait_until_next_event(True), "loaded")
self.assertEqual(self.db.count_automations(), 0)
watcher.join()
@@ -36,7 +36,7 @@ class watcher_tests(TestWithDB):
example_folder = create_copy_of_example_config(HA_CONFIG6_EXAMPLE)
watcher = AutomationFileWatcher(example_folder / "automations", self.db_file)
watcher.start()
watcher.wait_until_next_reload(True)
self.assertEqual(watcher.wait_until_next_event(True), "loaded")
self.assertEqual(self.db.count_automations(), 12)
watcher.join()
@@ -44,7 +44,7 @@ class watcher_tests(TestWithDB):
example_folder = create_copy_of_example_config()
watcher = AutomationFileWatcher(example_folder / "automations.yaml", self.db_file)
watcher.start()
watcher.wait_until_next_reload(True)
self.assertEqual(watcher.wait_until_next_event(True), "loaded")
self.assertEqual(self.db.count_automations(), 32)
(example_folder / "automations.yaml").write_text("")
(example_folder / "automations.yaml").write_text(
@@ -58,7 +58,7 @@ class watcher_tests(TestWithDB):
(example_folder / "automations.yaml").write_text(
(HA_CONFIG_EXAMPLE / "automations.yaml").read_text()
)
watcher.wait_until_next_reload(True)
self.assertEqual(watcher.wait_until_next_event(True), "loaded")
self.assertEqual(self.db.count_automations(), 32)
watcher.join()
@@ -70,7 +70,7 @@ class watcher_tests(TestWithDB):
# self.assertEqual(total, 0)
# watcher.start()
# watcher.wait_until_next_reload(True)
# self.assertEqual(watcher.wait_until_next_event(True), 'loaded')
# self.assertEqual(self.db.count_automations(), 32)
# autos = self.db.list_automations(0, 32)
@@ -0,0 +1,37 @@
from src.automations_v2.watchers.config import ConfigFileWatcher
from .utils import TestWithDB, make_temp_db
from tests.utils import (
HA_CONFIG_EXAMPLE,
HA_CONFIG2_EXAMPLE,
HA_CONFIG3_EXAMPLE,
HA_CONFIG4_EXAMPLE,
HA_CONFIG5_EXAMPLE,
HA_CONFIG6_EXAMPLE,
create_copy_of_example_config,
)
class config_watcher_tests(TestWithDB):
def test_simple_config_loading(self):
for p, count in [
(HA_CONFIG_EXAMPLE, 32),
(HA_CONFIG2_EXAMPLE, 1),
(HA_CONFIG3_EXAMPLE, 1032),
(HA_CONFIG4_EXAMPLE, 3),
(HA_CONFIG5_EXAMPLE, 10),
(HA_CONFIG6_EXAMPLE, 12),
]:
with self.subTest(config=p.name):
db = make_temp_db()
db.reset()
config_path = create_copy_of_example_config(p)
config_watcher = ConfigFileWatcher(
config_path=config_path,
automation_db_path=db.db_file,
)
self.assertEqual(db.count_automations(), 0)
config_watcher.start()
self.assertEqual(config_watcher.wait_until_next_event(True), "loaded")
self.assertEqual(db.count_automations(), count)
config_watcher.join()
+6 -2
View File
@@ -5,11 +5,15 @@ from tempfile import mktemp
from src.automations_v2.db import AutomationDBConnection
def make_temp_db():
return AutomationDBConnection(Path(mktemp() + ".db"))
class TestWithDB(TestCase):
def setUp(self) -> None:
self.db_file = Path(mktemp() + ".db")
self.db = AutomationDBConnection(self.db_file)
self.db = make_temp_db()
self.db.create_db()
self.db_file = self.db.db_file
def tearDown(self) -> None:
self.db_file.unlink()