using sync process

This commit is contained in:
ari committed 2022-08-15 15:43:18 -04:00
1 parent 7aadfbc175
commit b0d924f63a
17 files changed
+384 -496

No files matched your search

+23 -15
View File
@@ -7,24 +7,27 @@ from pathlib import Path
from typing import Iterable, Iterator, List, Optional
from src.logger import get_logger
from .types import ExtenededAutomationData
from .errors import DBDataError, DBError
from .types import ExtenededAutomation
from .errors import DBDataError, DBError, DBNoAutomationFound
logger = get_logger(__file__)
automations_tbl = "autos"
def encode_auto(auto: ExtenededAutomationData) -> str:
def encode_auto(auto: ExtenededAutomation) -> str:
return base64.b64encode(
json.dumps(auto.to_primitive(include_tags=True)).encode("utf-8")
).decode("utf-8")
def decode_auto(auto_str: str, source_file: str, source_file_type: str) -> ExtenededAutomationData:
return ExtenededAutomationData(
def decode_auto(
auto_str: str, source_file: str, source_file_type: str, configuration_key: str
) -> ExtenededAutomation:
return ExtenededAutomation(
source_file=source_file,
source_file_type=source_file_type,
configuration_key=configuration_key,
**json.loads(base64.b64decode(auto_str.encode("utf-8")).decode("utf-8")),
)
@@ -63,12 +66,13 @@ class AutomationDBConnection:
mode TEXT NOT NULL,
source_file TEXT NOT NULL,
source_file_type TEXT NOT NULL,
configuration_key TEXT NOT NULL,
rest BLOB NOT NULL
);
"""
)
def upsert_automations(self, automations: List[ExtenededAutomationData]):
def upsert_automations(self, automations: List[ExtenededAutomation]):
with self.get_cur() as cur:
for automation in automations:
cur.execute(
@@ -80,6 +84,7 @@ class AutomationDBConnection:
"{automation.mode}",
"{automation.source_file.absolute()}",
"{automation.source_file_type}",
"{automation.configuration_key}",
"{encode_auto(automation)}"
)
ON CONFLICT DO UPDATE SET
@@ -88,12 +93,13 @@ class AutomationDBConnection:
mode = "{automation.mode}",
source_file = "{automation.source_file.absolute()}",
source_file_type = "{automation.source_file_type}",
configuration_key = "{automation.configuration_key}",
rest = "{encode_auto(automation)}"
""",
)
def delete_automations(self, automations: List[ExtenededAutomationData]):
def delete_automations(self, automations: List[ExtenededAutomation]):
with self.get_cur() as cur:
cur.executemany(
f"DELETE FROM {automations_tbl} WHERE id = ?", [(a.id,) for a in automations]
@@ -106,21 +112,23 @@ class AutomationDBConnection:
(str(source_file.absolute()),),
)
def get_automation(self, automation_id: str) -> ExtenededAutomationData:
def get_automation(self, automation_id: str) -> ExtenededAutomation:
with self.get_cur() as cur:
cur.execute(
f'SELECT rest, source_file, source_file_type FROM {automations_tbl} WHERE id = "{automation_id}"'
f'SELECT rest, source_file, source_file_type, configuration_key FROM {automations_tbl} WHERE id = "{automation_id}"'
)
objs = cur.fetchone()
[rest, source_file, source_file_type] = objs
return decode_auto(rest, source_file, source_file_type)
if objs is None:
raise DBNoAutomationFound(automation_id)
[rest, source_file, source_file_type, configuration_key] = objs
return decode_auto(rest, source_file, source_file_type, configuration_key)
def list_automations(self, offset: int, limit: int) -> Iterable[ExtenededAutomationData]:
def list_automations(self, offset: int, limit: int) -> Iterable[ExtenededAutomation]:
with self.get_cur() as cur:
for [rest, source_file, source_file_type] in cur.execute(
f"SELECT rest, source_file, source_file_type FROM {automations_tbl} LIMIT {limit} OFFSET {offset}"
for [rest, source_file, source_file_type, configuration_key] in cur.execute(
f"SELECT rest, source_file, source_file_type, configuration_key FROM {automations_tbl} LIMIT {limit} OFFSET {offset}"
):
yield decode_auto(rest, source_file, source_file_type)
yield decode_auto(rest, source_file, source_file_type, configuration_key)
def count_automations(self) -> int:
with self.get_cur() as cur:
+8
View File
@@ -5,6 +5,10 @@ class AutomationLoaderException(Exception):
)
class AttemptingToOverwriteAnIncompatibleFileError(AutomationLoaderException):
pass
class InvalidAutomationFile(AutomationLoaderException):
pass
@@ -19,3 +23,7 @@ class DBError(Exception):
class DBDataError(DBError):
pass
class DBNoAutomationFound(DBError):
pass
+17 -11
View File
@@ -1,5 +1,5 @@
from pathlib import Path
from typing import Any, Iterator, Union
from typing import Any, Iterator, Tuple, Union
from ..hass_config.loader import HassConfig
from src.json_serializer import normalize_obj
@@ -9,14 +9,14 @@ from src.yaml_serializer.types import IncludedYaml, IncludedYamlDir
from src.utils import extract_files
from .errors import InvalidAutomationFile
from .types import ExtenededAutomationData
from .types import ExtenededAutomation
logger = get_logger(__file__)
def extract_automation_paths(
hass_config: HassConfig,
) -> Iterator[Path]:
) -> Iterator[Tuple[str, Path]]:
found = False
for key, value in hass_config.configurations.items():
if str(key).startswith("automation"):
@@ -27,23 +27,27 @@ def extract_automation_paths(
"No support for automations baked into configuration.yaml! Please extract these into a seperate file"
)
elif isinstance(value, IncludedYaml):
yield hass_config.root_path / value.path_str
yield key, hass_config.root_path / value.path_str
elif isinstance(value, IncludedYamlDir):
yield hass_config.root_path / value.name
yield key, hass_config.root_path / value.name
else:
logger.warning(f"found an invalid type in {key}!")
if not found:
yield hass_config.root_path / "automations.yaml"
yield "", hass_config.root_path / "automations.yaml"
def load_automation_path(automation_path: Path) -> Iterator[ExtenededAutomationData]:
def load_automation_path(
automation_path: Path,
configuration_key: str,
) -> Iterator[ExtenededAutomation]:
"""
Args:
automation_path (Path)
configuration_key (str)
Returns:
Iterator[ExtenededAutomationData]
Iterator[ExtenededAutomation]
"""
if not automation_path.exists():
@@ -51,7 +55,7 @@ def load_automation_path(automation_path: Path) -> Iterator[ExtenededAutomationD
return
if automation_path.is_dir():
for f in extract_files(automation_path):
yield from load_automation_path(f)
yield from load_automation_path(f, configuration_key)
else:
try:
with automation_path.open("r") as fp:
@@ -65,8 +69,9 @@ def load_automation_path(automation_path: Path) -> Iterator[ExtenededAutomationD
if isinstance(automations, dict):
try:
yield ExtenededAutomationData(
yield ExtenededAutomation(
**clean_automation(automations),
configuration_key=configuration_key,
source_file=str(automation_path),
source_file_type="obj",
)
@@ -82,8 +87,9 @@ def load_automation_path(automation_path: Path) -> Iterator[ExtenededAutomationD
return
for automation in automations:
try:
yield ExtenededAutomationData(
yield ExtenededAutomation(
**clean_automation(automation),
configuration_key=configuration_key,
source_file=str(automation_path),
source_file_type="list",
)
+71 -19
View File
@@ -1,28 +1,44 @@
from pathlib import Path
from tempfile import mktemp
from typing import Optional, Union
from ..errors import ErrorSet
from ..logger import get_logger
from .errors import AttemptingToOverwriteAnIncompatibleFileError, DBError, InvalidAutomationFile
from src.hass_config.loader import HassConfig
from src.yaml_serializer import dump_yaml, load_yaml
from .types import ExtenededAutomationData
from .watchers.config import ConfigFileWatcher
from .loader import extract_automation_paths, load_automation_path
from .types import ExtenededAutomation
from .db import AutomationDBConnection
logger = get_logger(__file__)
class AutomationManager:
def __init__(self, hass_config: HassConfig, db_path: Path) -> None:
def __init__(self, hass_config: HassConfig, db_path: Optional[Path] = None) -> None:
if db_path is None:
db_path = Path(mktemp())
self.hass_config = hass_config
self.db = AutomationDBConnection(db_path)
self.config_watcher = ConfigFileWatcher(
self.hass_config.get_configuration_path(),
self.db.db_file,
)
self.config_watcher.start()
def __del__(self):
self.config_watcher.join()
@property
def is_db_loaded(self):
return self.config_watcher.is_loaded
def reload(self):
self.db.reset()
errors = []
for config_key, p in extract_automation_paths(self.hass_config):
try:
automations = list(load_automation_path(p, config_key))
self.db.upsert_automations(automations)
except InvalidAutomationFile as e:
logger.warning(e)
errors.append(e)
except DBError as e:
logger.warning(e)
errors.append(e)
if len(errors) > 0:
raise ErrorSet(*errors)
def list(self, offset: int, limit: int):
return self.db.list_automations(offset=offset, limit=limit)
@@ -33,10 +49,46 @@ class AutomationManager:
def get(self, auto_id: str):
return self.db.get_automation(auto_id)
def update(self, automation: ExtenededAutomationData):
def upsert(self, automation: ExtenededAutomation):
if not automation.source_file.parent.exists():
automation.source_file.parent.mkdir(parents=True)
objs: Optional[Union[list, dict]] = None
if automation.source_file.exists():
with automation.source_file.open("r") as fp:
objs = load_yaml(fp)
if automation.source_file_type == "obj":
if isinstance(objs, list):
raise AttemptingToOverwriteAnIncompatibleFileError(
f"{automation.source_file} is a list, but expected a obj for {automation}"
)
raw_auto = automation.to_primitive(include_tags=False)
automation.source_file.write_text(dump_yaml(raw_auto))
else:
if isinstance(objs, dict):
raise AttemptingToOverwriteAnIncompatibleFileError(
f"{automation.source_file} is a list, but expected a obj for {automation}"
)
elif objs is None:
objs = []
if not isinstance(objs, list):
raise AssertionError(
f"Attemption to save automation to {automation.source_file} but encountering an invalid list yaml file. Did you modify this file prior to saving?\n{automation}"
)
found = False
for i, obj in enumerate(objs):
if obj["id"] == automation.id:
objs[i] = automation.to_primitive(include_tags=False)
found = True
break
if not found:
objs.append(automation.to_primitive(include_tags=False))
automation.source_file.write_text(dump_yaml(objs))
def delete(self, automation: ExtenededAutomation):
if automation.source_file_type == "obj":
automation.source_file.unlink(missing_ok=True)
else:
with automation.source_file.open("r") as fp:
objs = load_yaml(fp)
@@ -44,8 +96,8 @@ class AutomationManager:
raise AssertionError(
f"Attemption to save automation to {automation.source_file} but encountering an invalid list yaml file. Did you modify this file prior to saving?\n{automation}"
)
kept = []
for i, obj in enumerate(objs):
if obj["id"] == automation.id:
objs[i] = automation.to_primitive(include_tags=False)
break
automation.source_file.write_text(dump_yaml(objs))
if obj["id"] != automation.id:
kept.append(obj)
automation.source_file.write_text(dump_yaml(kept))
+3 -1
View File
@@ -19,15 +19,17 @@ class BaseAutomation(BaseModel):
return self.dict(exclude_unset=True, exclude_none=True)
class ExtenededAutomationData(BaseAutomation):
class ExtenededAutomation(BaseAutomation):
source_file: Path
source_file_type: Literal["list", "obj"]
configuration_key: str
tags: Dict[str, str] = Field(default_factory=dict)
def to_primitive(self, include_tags: bool = False):
out = super().to_primitive()
del out["source_file"]
del out["source_file_type"]
del out["configuration_key"]
if include_tags and out.get("tags", None) is not None:
del out["tags"]
return out
Whitespace-only changes.
@@ -1,64 +0,0 @@
from pathlib import Path
from typing import List, Optional
from src.logger import get_logger
from ...utils import extract_files
from ..db import AutomationDBConnection, DBError
from ..loader import load_automation_path
from ..types import ExtenededAutomationData
from ..errors import FailedDeletion
from .base_handler import BaseFileEventHandler
from .base_watcher import BaseWatcher
logger = get_logger(__file__)
class AutomationEventHandler(BaseFileEventHandler):
def __init__(self, automation_file: Path, automation_db_path: Path, **kwrgs) -> None:
self.__automation_file = automation_file.absolute()
self.__automation_db = AutomationDBConnection(automation_db_path)
super().__init__(**kwrgs)
def reload_file(self, file_path: Optional[Path], deleted: bool):
if file_path is None:
file_path = self.__automation_file
self.clear_automations(file_path)
if not deleted:
automations = list(load_automation_path(file_path))
self.__automation_db.upsert_automations(automations)
def clear_automations(self, file_path: Path):
if file_path.is_dir():
for f in extract_files(file_path):
self.__automation_db.delete_automations_in_source_file(f)
else:
self.__automation_db.delete_automations_in_source_file(file_path)
def handle_failure(self, error: Exception):
logger.error(error)
if isinstance(error, DBError):
logger.warning(f"Failed to reload {self.__automation_file} due to {repr(error)}")
else:
logger.warning(f"Failed to load {self.__automation_file} due to {repr(error)}")
try:
self.clear_automations(self.__automation_file)
except DBError as err:
raise FailedDeletion(
f"failed to flush automations from database for the invalid file {self.__automation_file} due to {repr(err)}"
) from err
class AutomationFileWatcher(BaseWatcher):
def __init__(self, automation_file: Path, automation_db_path: Path):
self.__automation_db_path = automation_db_path
BaseWatcher.__init__(self, file_path=automation_file)
def get_handler(self, **kwrgs) -> BaseFileEventHandler:
return AutomationEventHandler(
automation_file=self.file_path,
automation_db_path=self.__automation_db_path,
**kwrgs,
)
@@ -1,52 +0,0 @@
import abc
import asyncio
from typing import Optional
import nest_asyncio
from threading import Event
from pathlib import Path
from watchdog.events import FileSystemEvent
from watchdog.events import FileSystemEventHandler
from src.logger import get_logger
from src.utils import throttle
nest_asyncio.apply()
logger = get_logger(__file__)
class BaseFileEventHandler(FileSystemEventHandler, abc.ABC):
def __init__(self, event_loaded: Event, event_errored: Event) -> None:
self.event_loaded = event_loaded
self.event_errored = event_errored
super().__init__()
asyncio.run(self._reload_file(None, False))
def reload_file(self, file_path: Optional[Path], deleted: bool):
raise NotImplementedError
def handle_failure(self, error: Exception):
logger.error(error)
raise error from error
@throttle(seconds=60)
def _reload_file(self, file_path: Optional[Path], deleted: bool):
try:
self.reload_file(file_path, deleted)
except Exception as err:
self.event_errored.set()
self.handle_failure(err)
self.event_loaded.set()
def on_modified(self, event: FileSystemEvent):
logger.info(f"on_modified: {event.event_type} path : {event.src_path}")
asyncio.run(self._reload_file(Path(event.src_path), False))
def on_created(self, event: FileSystemEvent):
logger.info(f"on_created: {event.event_type} path : {event.src_path}")
asyncio.run(self._reload_file(Path(event.src_path), False))
def on_deleted(self, event: FileSystemEvent):
logger.info(f"on_deleted: {event.event_type} path : {event.src_path}")
asyncio.run(self._reload_file(Path(event.src_path), True))
@@ -1,92 +0,0 @@
import abc
from pathlib import Path
from threading import Event
from time import sleep
from typing import Generic, Literal, Optional, TypeVar
from watchdog.observers import Observer
from src.logger import get_logger
from .base_handler import BaseFileEventHandler
logger = get_logger(__file__)
BaseFileEventHandlerT = TypeVar("BaseFileEventHandlerT", bound=BaseFileEventHandler)
class BaseWatcher(Observer, abc.ABC, Generic[BaseFileEventHandlerT]):
def __init__(self, file_path: Path):
self.file_path = file_path
self.__event_file_loaded = Event()
self.__event_errored = Event()
Observer.__init__(self)
self.handler: Optional[BaseFileEventHandlerT] = None
self.name = f"{self.__class__.__name__}-{self.file_path}-{self.name}"
def start(self):
self.handler = self.get_handler(
event_errored=self.__event_errored,
event_loaded=self.__event_file_loaded,
)
self.schedule(self.handler, self.file_path, recursive=self.file_path.is_dir())
return super().start()
def get_handler(self, event_loaded: Event, event_errored: Event) -> BaseFileEventHandlerT:
raise NotImplementedError
def _post_stop(self, handler: BaseFileEventHandlerT):
return
def __del__(self):
logger.warning(f"Deleting watcher for {self.file_path}...")
self.join()
def join(self, timeout: float = 30) -> None:
logger.warning(f"Closing thread for watcher for {self.file_path}...")
super().stop()
super().join(timeout)
if self.is_alive():
raise AssertionError(f"TIMEOUT! Failed to terminate watcher for {self.file_path}")
logger.warning(f"Watcher for {self.file_path} is stopped!")
if self.handler is not None:
self._post_stop(self.handler)
@property
def is_loaded(self):
return self.__event_file_loaded.is_set()
@property
def is_errored(self):
return self.__event_errored.is_set()
def wait_until_next_reload(self, reset_after: bool = False, *, timeout: int = 5):
self.__event_file_loaded.wait(timeout)
if reset_after:
self.__event_file_loaded.clear()
def wait_until_next_error(self, reset_after: bool = False, *, timeout: int = 5):
self.__event_errored.wait(timeout)
if reset_after:
self.__event_errored.clear()
def wait_until_next_event(
self,
reset_after: bool = False,
*,
timeout: int = 5,
) -> Literal["error", "loaded"]:
out = ""
while True:
if self.__event_file_loaded.is_set():
out = "loaded"
break
elif self.__event_errored.is_set():
out = "error"
break
else:
sleep(timeout)
if reset_after:
self.__event_file_loaded.clear()
self.__event_errored.clear()
return out
-65
View File
@@ -1,65 +0,0 @@
from pathlib import Path
from threading import Event, Thread
from time import sleep
from typing import List, Literal
from watchdog.observers import Observer
from src.automations_v2.loader import extract_automation_paths
from src.automations_v2.db import AutomationDBConnection
from src.hass_config.loader import HassConfig
from src.logger import get_logger
from .automation import AutomationFileWatcher
from .base_handler import BaseFileEventHandler
from .base_watcher import BaseWatcher
logger = get_logger(__file__)
class ConfigEventHandler(BaseFileEventHandler):
def __init__(self, config_path: Path, db_path: Path, **kwargs) -> None:
self.__db = AutomationDBConnection(db_path)
self.__config_path = config_path
self.__hass_config = HassConfig(self.__config_path.parent)
self.__automation_watchers: List[AutomationFileWatcher] = []
super().__init__(**kwargs)
def reload_file(self, file_path: Path, deleted: bool):
if file_path is None:
file_path = self.__config_path
if file_path.absolute() != self.__config_path:
return
self.clear_automation_watchers()
self.__db.reset()
if deleted:
logger.error("Could not detect your configuration.yaml file, was is deleted?")
return
for p in extract_automation_paths(self.__hass_config):
watcher = AutomationFileWatcher(automation_file=p, automation_db_path=self.__db.db_file)
watcher.start()
self.__automation_watchers.append(watcher)
def clear_automation_watchers(self):
for auto_watchers in self.__automation_watchers:
auto_watchers.join()
self.__automation_watchers = []
def __del__(self):
self.clear_automation_watchers()
class ConfigFileWatcher(BaseWatcher):
def __init__(self, config_path: Path, automation_db_path: Path):
self.__automation_db_path = automation_db_path
BaseWatcher.__init__(self, file_path=config_path)
def get_handler(self, **kwrgs) -> ConfigEventHandler:
return ConfigEventHandler(
config_path=self.file_path,
db_path=self.__automation_db_path,
**kwrgs,
)
def _post_stop(self, handler: ConfigEventHandler):
handler.clear_automation_watchers()
+12
View File
@@ -0,0 +1,12 @@
class ErrorSet(Exception):
"""An exception collection exception"""
def __init__(self, *errors: Exception) -> None:
self.errors = errors
super().__init__(self.gen_message())
def gen_message(self):
if len(self.errors) > 1:
return "Encountered multiple errors:\n >" + "\n >".join(map(repr, self.errors))
else:
return str(self.errors[0].args)
+1 -38
View File
@@ -1,42 +1,5 @@
import asyncio
from datetime import datetime, timedelta
from functools import wraps
from pathlib import Path
from typing import Any, Callable, Coroutine, Iterator, Optional, TypeVar
FnOutT = TypeVar("FnOutT")
class throttle(object):
"""source: https://gist.github.com/ChrisTM/5834503
Decorator that prevents a function from being called more than once every
time period.
To create a function that cannot be called more than once a minute:
@throttle(minutes=1)
def my_fun():
pass
"""
def __init__(self, seconds=0, minutes=0, hours=0):
self.throttle_period = timedelta(seconds=seconds, minutes=minutes, hours=hours)
self.time_of_last_call = datetime.min
self.current_id: Optional[datetime] = None
def __call__(self, fn: Callable[..., FnOutT]) -> Callable[..., Coroutine[Any, Any, FnOutT]]:
@wraps(fn)
async def wrapper(*args, **kwargs):
self.current_id = now = datetime.now()
time_since_last_call = now - self.time_of_last_call
if time_since_last_call > self.throttle_period:
self.time_of_last_call = now
return fn(*args, **kwargs)
else:
asyncio.sleep(self.throttle_period.total_seconds())
if self.current_id == now:
return fn(*args, **kwargs)
return wrapper
from typing import Iterator
def extract_files(p: Path) -> Iterator[Path]: