v2 initial work

This commit is contained in:
ari committed 2022-08-14 00:24:44 -04:00
1 parent 2bc7e11bb1
commit 3a89fe5efc
36 files changed
+830 -4

No files matched your search

View File
Whitespace-only changes.
+116
View File
@@ -0,0 +1,116 @@
import json
import sqlite3
import base64
from contextlib import contextmanager
from pathlib import Path
from typing import Iterable, Iterator, List, Optional
from src.logger import get_logger
from .types import ExtenededAutomationData
from .errors import DBDataError, DBError
logger = get_logger(__file__)
automations_tbl = "autos"
def encode_auto(auto: ExtenededAutomationData) -> str:
return base64.b64encode(
json.dumps(auto.to_primitive(include_tags=True)).encode("utf-8")
).decode("utf-8")
def decode_auto(auto_str: str, source_file: str, source_file_type: str) -> ExtenededAutomationData:
return ExtenededAutomationData(
source_file=source_file,
source_file_type=source_file_type,
**json.loads(base64.b64decode(auto_str.encode("utf-8")).decode("utf-8")),
)
class AutomationDBConnection:
def __init__(self, db_file: Path) -> None:
self.db_file = db_file
@contextmanager
def get_cur(self) -> Iterator[sqlite3.Cursor]:
conn = sqlite3.connect(self.db_file)
cur = conn.cursor()
try:
yield cur
conn.commit()
except sqlite3.IntegrityError as err:
raise DBDataError(*err.args) from err
except sqlite3.DatabaseError as err:
raise DBError(*err.args) from err
finally:
cur.close()
conn.close()
def create_db(self):
with self.get_cur() as cur:
cur.execute(
f"""
CREATE TABLE IF NOT EXISTS {automations_tbl} (
id TEXT NOT NULL PRIMARY KEY,
alias TEXT,
description TEXT,
mode TEXT NOT NULL,
source_file TEXT NOT NULL,
source_file_type TEXT NOT NULL,
rest BLOB NOT NULL
);
"""
)
def upsert_automations(self, automations: List[ExtenededAutomationData]):
with self.get_cur() as cur:
for automation in automations:
cur.execute(
f"""
INSERT INTO {automations_tbl} VALUES (
"{automation.id}",
"{automation.alias}",
"{automation.description}",
"{automation.mode}",
"{automation.source_file}",
"{automation.source_file_type}",
"{encode_auto(automation)}"
)
ON CONFLICT DO UPDATE SET
alias = "{automation.alias}",
description = "{automation.description}",
mode = "{automation.mode}",
source_file = "{automation.source_file}",
source_file_type = "{automation.source_file_type}",
rest = "{encode_auto(automation)}"
""",
)
def delete_automations(self, automations: List[ExtenededAutomationData]):
with self.get_cur() as cur:
cur.executemany(
f"DELETE FROM {automations_tbl} WHERE id = ?", [(a.id,) for a in automations]
)
def get_automation(self, automation_id: str) -> ExtenededAutomationData:
with self.get_cur() as cur:
cur.execute(
f'SELECT rest, source_file, source_file_type FROM {automations_tbl} WHERE id = "{automation_id}"'
)
objs = cur.fetchone()
[rest, source_file, source_file_type] = objs
return decode_auto(rest, source_file, source_file_type)
def list_automations(self, offset: int, limit: int) -> Iterable[ExtenededAutomationData]:
with self.get_cur() as cur:
for [rest, source_file, source_file_type] in cur.execute(
f"SELECT rest, source_file, source_file_type FROM {automations_tbl} LIMIT {limit} OFFSET {offset}"
):
yield decode_auto(rest, source_file, source_file_type)
def count_automations(self) -> int:
with self.get_cur() as cur:
return cur.execute(f"SELECT count(id) FROM {automations_tbl}").fetchone()[0]
+21
View File
@@ -0,0 +1,21 @@
class AutomationLoaderException(Exception):
def __init__(self, *args: object, **kwargs) -> None:
super().__init__(
"".join(map(str, args)) + ",".join((f"{k}={v}" for k, v in kwargs.items())),
)
class InvalidAutomationFile(AutomationLoaderException):
pass
class FailedDeletion(AutomationLoaderException):
pass
class DBError(Exception):
pass
class DBDataError(DBError):
pass
+93
View File
@@ -0,0 +1,93 @@
from pathlib import Path
from typing import Any, Iterator, Union
from src.json_serializer import normalize_obj
from src.yaml_serializer import load_yaml
from .errors import InvalidAutomationFile
from .types import ExtenededAutomationData
# Single loader
def load_automation_path(automation_path: Path) -> Iterator[ExtenededAutomationData]:
"""
Args:
automation_path (Path)
Returns:
Iterator[ExtenededAutomationData]
"""
try:
with automation_path.open("r") as fp:
automations = load_yaml(fp)
except Exception as err:
raise InvalidAutomationFile(
when="loading yaml",
automation_path=automation_path,
error=err,
) from err
if isinstance(automations, dict):
try:
yield ExtenededAutomationData(
**clean_automation(automations),
source_file=str(automation_path),
source_file_type="obj",
)
except Exception as err:
raise InvalidAutomationFile(
when="reading file contents",
automation_path=automation_path,
error=err,
) from err
else:
for automation in automations:
try:
yield ExtenededAutomationData(
**clean_automation(automation),
source_file=str(automation_path),
source_file_type="list",
)
except Exception as err:
raise InvalidAutomationFile(
when="reading file contents",
automation_path=automation_path,
error=err,
automation=automation,
) from err
def clean_automation(auto_raw: dict):
return {
**auto_raw,
"condition": ifempty_or_null_then_list(load_conditions(auto_raw.get("condition", []))),
"action": ifempty_or_null_then_list(load_conditions(auto_raw.get("action", []))),
"trigger": ifempty_or_null_then_list(normalize_obj(auto_raw.get("trigger", []))),
}
def ifempty_or_null_then_list(obj: Any):
if not isinstance(obj, list):
return []
else:
return obj
def load_conditions(conditions: Union[dict, list, str]):
if isinstance(conditions, list):
return list(map(ensure_condition_dictionary, conditions))
elif not conditions:
return []
else:
return [ensure_condition_dictionary(conditions)]
def ensure_condition_dictionary(c: Union[str, dict]):
c = normalize_obj(c)
if isinstance(c, str):
return {"condition": "template", "value_template": c}
elif isinstance(c, dict):
return c
raise InvalidAutomationFile(f"Invalid type for condition object {c}")
+32
View File
@@ -0,0 +1,32 @@
from typing import Dict, List, Literal, Optional, Union
from pydantic import BaseModel
from pydantic.fields import Field
class BaseAutomation(BaseModel):
id: str
alias: Optional[str] = None
description: Optional[str] = None
trigger_variables: Optional[Dict[str, Union[str, int, bool, float]]] = None
mode: str = "single"
trigger: List[dict] = []
condition: List[dict] = []
action: List[dict] = []
def to_primitive(self):
return self.dict(exclude_unset=True, exclude_none=True)
class ExtenededAutomationData(BaseAutomation):
source_file: str
source_file_type: Literal["list", "obj"]
tags: Dict[str, str] = Field(default_factory=dict)
def to_primitive(self, include_tags: bool = False):
out = super().to_primitive()
del out["source_file"]
del out["source_file_type"]
if include_tags and out.get("tags", None) is not None:
del out["tags"]
return out
+107
View File
@@ -0,0 +1,107 @@
from threading import Thread, Condition, Event
from pathlib import Path
from time import sleep
from typing import List, Optional
from watchdog.events import FileSystemEvent
from watchdog.observers import Observer
from watchdog.events import FileSystemEventHandler
from src.logger import get_logger
from .db import AutomationDBConnection, DBError
from .loader import load_automation_path
from .types import ExtenededAutomationData
from .errors import FailedDeletion
logger = get_logger(__file__)
class AutomationFileEventHandler(FileSystemEventHandler):
def __init__(self, automation_file: Path, automation_db_path: Path, file_loaded: Event) -> None:
super().__init__()
self.__automation_file = automation_file.absolute()
self.__automation_db = AutomationDBConnection(automation_db_path)
self.file_loaded = file_loaded
self.__loaded_automations: Optional[List[ExtenededAutomationData]] = None
self.reload_file()
def reload_file(self):
try:
automations = list(load_automation_path(self.__automation_file))
self.__automation_db.upsert_automations(automations)
self.__loaded_automations = automations
except Exception as err:
logger.error(err)
if isinstance(err, DBError):
logger.warning(f"Failed to reload {self.__automation_file} due to {err}")
else:
logger.warning(f"Failed to load {self.__automation_file} due to {err}")
if self.__loaded_automations is not None:
try:
self.__automation_db.delete_automations(self.__loaded_automations)
except DBError as err:
raise FailedDeletion(
f"failed to flush automations from database for the invalid file {self.__automation_file} due to {err}"
) from err
self.file_loaded.set()
def on_modified(self, event: FileSystemEvent):
logger.info(f"on_modified: {type(event)} {event.event_type} path : {event.src_path}")
if Path(event.src_path).absolute() == self.__automation_file:
self.reload_file()
def on_created(self, event: FileSystemEvent):
logger.info(f"on_created: {type(event)} {event.event_type} path : {event.src_path}")
if Path(event.src_path).absolute() == self.__automation_file:
self.reload_file()
def on_deleted(self, event: FileSystemEvent):
logger.info(f"on_deleted: {type(event)} {event.event_type} path : {event.src_path}")
if Path(event.src_path).absolute() == self.__automation_file:
self.reload_file()
class AutomationFileWatcher(Thread):
def __init__(self, automation_file: Path, automation_db_path: Path):
self.__automation_file = automation_file
self.__automation_db_path = automation_db_path
self.__stop = False
self.__file_loaded = Event()
Thread.__init__(self)
def run(self) -> None:
self.__stop = False
logger.info(f"Watching {self.__automation_file} for changes...")
obs = Observer()
handler = AutomationFileEventHandler(
self.__automation_file,
self.__automation_db_path,
self.__file_loaded,
)
obs.schedule(handler, self.__automation_file)
obs.start()
while not self.__stop:
sleep(1)
obs.stop()
logger.warning(f"Watcher for {self.__automation_file} is stopped!")
def stop(self):
if not self.__stop:
logger.warning(f"Watcher for {self.__automation_file} is set to stop...")
self.__stop = True
def __del__(self):
logger.warning(f"Deleting watcher for {self.__automation_file}...")
self.join()
def join(self, timeout: float = 10) -> None:
logger.warning(f"Closing thread for watcher for {self.__automation_file}...")
self.stop()
return super().join(timeout)
def wait_until_next_reload(self, reset_after: bool = False, *, timeout: int = 5):
self.__file_loaded.wait(timeout)
if reset_after:
self.__file_loaded.clear()
+1 -1
View File
@@ -6,7 +6,7 @@ from websockets.client import WebSocketClientProtocol, connect
from websockets.exceptions import ConnectionClosedOK
from src.env import HASSIO_TOKEN, HASSIO_WS
from src.logger import get_logger, logger
from src.logger import get_logger
OnMessage = Callable[[str], Coroutine]
logger = get_logger(__name__)
+1 -1
View File
@@ -36,6 +36,6 @@ def load_yaml(yaml_stream: TextIO) -> Union[dict, list]:
yaml_stream (io.StringIO)
Returns:
List[dict, list]
Union[dict, list]
"""
return yaml.load(yaml_stream, Loader=YamlSafeLoader) # nosec
@@ -0,0 +1,14 @@
id: "1659114822642"
alias: On Home Down
description: Example
mode: single
trigger:
- platform: homeassistant
event: shutdown
condition: []
action:
- service: counter.increment
target:
entity_id:
- counter.up_times
data: {}
@@ -0,0 +1,13 @@
id: "1659114647066"
alias: On Home Up
description: Example
mode: single
trigger:
- platform: homeassistant
event: start
condition: []
action:
- service: counter.increment
data: {}
target:
entity_id: counter.up_times
@@ -0,0 +1,13 @@
id: "1659114647067"
alias: Notify Washer
description: Example
mode: single
trigger:
- platform: homeassistant
event: start
condition: []
action:
- service: counter.increment
data: {}
target:
entity_id: counter.up_times
@@ -0,0 +1,4 @@
# Loads default set of integrations. Do not remove.
default_config:
automation: !include_dir_list ./automations
@@ -0,0 +1,14 @@
id: "1659114822642"
alias: On Home Down
description: Example
mode: single
trigger:
- platform: homeassistant
event: shutdown
condition: []
action:
- service: counter.increment
target:
entity_id:
- counter.up_times
data: {}
@@ -0,0 +1,13 @@
id: "1659114647066"
alias: HA Up
description: Example
mode: single
trigger:
- platform: homeassistant
event: start
condition: []
action:
- service: counter.increment
data: {}
target:
entity_id: counter.up_times
@@ -0,0 +1,13 @@
id: "1659114647067"
alias: Notify Washer
description: Example
mode: single
trigger:
- platform: homeassistant
event: start
condition: []
action:
- service: counter.increment
data: {}
target:
entity_id: counter.up_times
@@ -0,0 +1,7 @@
- id: "mlist1"
alias: Merged List 1
description: ""
trigger: []
condition: []
action: []
mode: single
@@ -0,0 +1,14 @@
- id: "mlist2"
alias: Merged List 2
description: ""
trigger: []
condition: []
action: []
mode: single
- id: "mlist3"
alias: Merged List 3
description: ""
trigger: []
condition: []
action: []
mode: single
@@ -0,0 +1,14 @@
- id: "sublist3-mlist2"
alias: Sublist 3 - Merged List 2
description: ""
trigger: []
condition: []
action: []
mode: single
- id: "sublist3-mlist3"
alias: Sublist 3 - Merged List 3
description: ""
trigger: []
condition: []
action: []
mode: single
@@ -0,0 +1,23 @@
- id: '165911464702444'
alias: In File
description: Example
mode: single
trigger:
- platform: homeassistant
event: start
condition: []
action:
- service: counter.increment
data: {}
target:
entity_id: counter.up_times
- id: '1660439437914'
alias: NEW
description: ''
trigger:
- platform: sun
event: sunrise
offset: 0
condition: []
action: []
mode: single
@@ -0,0 +1,12 @@
# Loads default set of integrations. Do not remove.
default_config:
# Text to speech
tts:
- platform: google_translate
automation manual: !include_dir_list ./automations/include_dir_list
automation ui: !include automations/ui.yaml
automation cools: !include_dir_merge_list automations/include_dir_merge_list
script: !include scripts.yaml
scene: !include scenes.yaml
@@ -0,0 +1,7 @@
- id: "1660439557193"
alias: SHOULD NOT BE LOADED!
description: ""
trigger: []
condition: []
action: []
mode: single
@@ -0,0 +1,23 @@
- id: "base1"
alias: BASE
description: Example
mode: single
trigger:
- platform: homeassistant
event: start
condition: []
action:
- service: counter.increment
data: {}
target:
entity_id: counter.up_times
- id: "base2"
alias: base 2
description: ""
trigger:
- platform: sun
event: sunrise
offset: 0
condition: []
action: []
mode: single
@@ -0,0 +1,14 @@
id: "1659114822642"
alias: On Home Down
description: Example
mode: single
trigger:
- platform: homeassistant
event: shutdown
condition: []
action:
- service: counter.increment
target:
entity_id:
- counter.up_times
data: {}
@@ -0,0 +1,13 @@
id: "1659114647066"
alias: HA Up
description: Example
mode: single
trigger:
- platform: homeassistant
event: start
condition: []
action:
- service: counter.increment
data: {}
target:
entity_id: counter.up_times
@@ -0,0 +1,13 @@
id: "1659114647067"
alias: Notify Washer
description: Example
mode: single
trigger:
- platform: homeassistant
event: start
condition: []
action:
- service: counter.increment
data: {}
target:
entity_id: counter.up_times
@@ -0,0 +1,7 @@
- id: "mlist1"
alias: Merged List 1
description: ""
trigger: []
condition: []
action: []
mode: single
@@ -0,0 +1,14 @@
- id: "mlist2"
alias: Merged List 2
description: ""
trigger: []
condition: []
action: []
mode: single
- id: "mlist3"
alias: Merged List 3
description: ""
trigger: []
condition: []
action: []
mode: single
@@ -0,0 +1,14 @@
- id: "sublist3-mlist2"
alias: Sublist 3 - Merged List 2
description: ""
trigger: []
condition: []
action: []
mode: single
- id: "sublist3-mlist3"
alias: Sublist 3 - Merged List 3
description: ""
trigger: []
condition: []
action: []
mode: single
@@ -0,0 +1,23 @@
- id: "ui1"
alias: UI 1
description: Example
mode: single
trigger:
- platform: homeassistant
event: start
condition: []
action:
- service: counter.increment
data: {}
target:
entity_id: counter.up_times
- id: "ui2"
alias: UI 2
description: ""
trigger:
- platform: sun
event: sunrise
offset: 0
condition: []
action: []
mode: single
Whitespace-only changes.
@@ -0,0 +1,81 @@
from src.automations_v2.types import ExtenededAutomationData
from .utils import TestWithDB
class db_tests(TestWithDB):
def test_db_creation(self):
with self.db.get_cur() as cur:
cur.execute("SELECT name FROM sqlite_schema WHERE type ='table'")
tables = cur.fetchall()
self.assertListEqual(tables, [("autos",)])
self.db.create_db()
with self.db.get_cur() as cur:
cur.execute("SELECT name FROM sqlite_schema WHERE type ='table'")
tables = cur.fetchall()
self.assertListEqual(tables, [("autos",)])
def test_insert_automations(self):
originals = [
ExtenededAutomationData(
id="test",
alias="test",
source_file="automation.yaml",
source_file_type="obj",
),
ExtenededAutomationData(
id="test2",
alias="test",
source_file="automation.yaml",
source_file_type="obj",
action=[{"event": {"nice": "one"}, "event_type": "custom"}],
),
]
self.db.upsert_automations(originals)
for original in originals:
with self.subTest(original=original):
self.assertEqual(self.db.get_automation(original.id), original)
autos = list(self.db.list_automations(0, 10))
self.assertEqual(len(autos), 2)
def test_insert_repeat_automations(self):
originals = [
ExtenededAutomationData(
id="test",
alias="test",
source_file="automation.yaml",
source_file_type="obj",
),
ExtenededAutomationData(
id="test",
alias="test",
source_file="automation.yaml",
source_file_type="obj",
action=[{"event": {"nice": "one"}, "event_type": "custom"}],
),
]
self.db.upsert_automations(originals)
autos = list(self.db.list_automations(0, 10))
self.assertEqual(len(autos), 1)
self.assertTrue(autos[0] == originals[1])
def test_delete_automations(self):
originals = [
ExtenededAutomationData(
id="test",
alias="test",
source_file="automation.yaml",
source_file_type="obj",
),
ExtenededAutomationData(
id="test2",
alias="test",
source_file="automation.yaml",
source_file_type="obj",
action=[{"event": {"nice": "one"}, "event_type": "custom"}],
),
]
self.assertEqual(self.db.count_automations(), 0)
self.db.upsert_automations(originals)
self.assertEqual(self.db.count_automations(), 2)
self.db.delete_automations(originals)
self.assertEqual(self.db.count_automations(), 0)
Whitespace-only changes.
@@ -0,0 +1,50 @@
from src.automations_v2.watcher import AutomationFileWatcher
from tests.utils import create_copy_of_example_config
from .utils import TestWithDB
class watcher_tests(TestWithDB):
def test_bad_file(self):
example_folder = create_copy_of_example_config()
watcher = AutomationFileWatcher(example_folder / "automations.yaml", self.db_file)
total = self.db.count_automations()
self.assertEqual(total, 0)
watcher.start()
watcher.wait_until_next_reload(True)
self.assertEqual(self.db.count_automations(), 32)
(example_folder / "automations.yaml").write_text("haha!")
watcher.wait_until_next_reload(True, timeout=60 * 10)
self.assertEqual(self.db.count_automations(), 0)
watcher.join()
def test_load_empty_file(self):
example_folder = create_copy_of_example_config()
(example_folder / "automations.yaml").write_text("")
watcher = AutomationFileWatcher(example_folder / "automations.yaml", self.db_file)
total = self.db.count_automations()
self.assertEqual(total, 0)
watcher.start()
watcher.wait_until_next_reload(True)
self.assertEqual(self.db.count_automations(), 0)
# def test_valid_changes(self):
# example_folder = create_copy_of_example_config()
# watcher = AutomationFileWatcher(example_folder / "automations.yaml", self.db_file)
# total = self.db.count_automations()
# self.assertEqual(total, 0)
# watcher.start()
# watcher.wait_until_next_reload(True)
# self.assertEqual(self.db.count_automations(), 32)
# autos = self.db.list_automations(0, 32)
# (example_folder / "automations.yaml").write_text("haha!")
# watcher.wait_until_next_reload(True, timeout=60 * 10)
# self.assertEqual(self.db.count_automations(), 0)
# watcher.join()
@@ -0,0 +1,15 @@
from pathlib import Path
from unittest import TestCase
from tempfile import mktemp
from src.automations_v2.db import AutomationDBConnection
class TestWithDB(TestCase):
def setUp(self) -> None:
self.db_file = Path(mktemp() + ".db")
self.db = AutomationDBConnection(self.db_file)
self.db.create_db()
def tearDown(self) -> None:
self.db_file.unlink()
+30 -2
View File
@@ -7,7 +7,6 @@ from src.yaml_serializer.types import (
IncludedYamlDir,
SecretValue,
)
from tests.utils import SAMPLES_FOLDER
class dumping_yamls_tests(TestCase):
@@ -25,10 +24,39 @@ class dumping_yamls_tests(TestCase):
yaml = dump_yaml({"google_password": SecretValue("gpass", "supersec")})
self.assertEqual(yaml, "google_password: !secret 'gpass'\n")
def test_dumping_stub(self):
def test_dumping_include_dir_list(self):
yaml = dump_yaml({"sensor": IncludedYamlDir("include_dir_list", "sensors/*yaml")})
self.assertEqual(yaml, "sensor: !include_dir_list 'sensors/*yaml'\n")
def test_dumping_include_dir_merge_list(self):
yaml = dump_yaml({"sensor": IncludedYamlDir("include_dir_merge_list", "sensors/*yaml")})
self.assertEqual(yaml, "sensor: !include_dir_merge_list 'sensors/*yaml'\n")
def test_load_complex_automation_setup(self):
yaml_obj = load_yaml(
io.StringIO(
"""
automation manual: !include_dir_list ./automations/include_dir_list
automation ui: !include automations/ui.yaml
automation: !include automations/base.yaml
automation cools: !include_dir_merge_list automations/include_dir_merge_list
"""
)
)
self.assertDictEqual(
dict(yaml_obj),
{
"automation manual": IncludedYamlDir(
"include_dir_list", "./automations/include_dir_list"
),
"automation ui": IncludedYaml("automations/ui.yaml"),
"automation cools": IncludedYamlDir(
"include_dir_merge_list", "automations/include_dir_merge_list"
),
"automation": IncludedYaml("automations/base.yaml"),
},
)
def test_load_stub(self):
yaml_obj = load_yaml(
io.StringIO(
+1
View File
@@ -14,6 +14,7 @@ SAMPLES_FOLDER = THIS_FOLDER / "samples"
HA_CONFIG_EXAMPLE = SAMPLES_FOLDER / "config"
HA_CONFIG2_EXAMPLE = SAMPLES_FOLDER / "config-2"
HA_CONFIG3_EXAMPLE = SAMPLES_FOLDER / "config-3"
HA_CONFIG4_EXAMPLE = SAMPLES_FOLDER / "config-4"
def get_example_automation_loader(