refactor of how automations are sorted

This commit is contained in:
ari committed 2022-08-27 13:58:41 -04:00
1 parent 80bfbdab80
commit fbbc5af12d
15 files changed
+155 -93

No files matched your search

+3 -3
View File
@@ -1,4 +1,4 @@
from typing import Dict, Generic, List, TypeVar
from typing import Generic, TypeVar
from pydantic import BaseModel
from pydantic.fields import Field
@@ -15,9 +15,9 @@ class ListParams(BaseModel):
class ListData(GenericModel, Generic[DataT]):
params: ListParams
total_items: int = Field(alias="totalItems")
data: List[DataT]
data: list[DataT]
class UpdateTags(BaseModel):
automation_id: str = Field(description="Automation Id to update")
tags: Dict[str, str]
tags: dict[str, str]
+14 -6
View File
@@ -3,7 +3,7 @@ import json
import sqlite3
from contextlib import contextmanager
from pathlib import Path
from typing import Iterable, Iterator, List, Optional
from typing import Iterable, Iterator
from src.logger import get_logger
@@ -26,17 +26,25 @@ def encode_auto(auto: ExtenededAutomation) -> str:
).decode("utf-8")
def encode_obj(auto: dict | list) -> str:
return base64.b64encode(json.dumps(auto).encode("utf-8")).decode("utf-8")
def decode_auto(
auto_str: str, source_file: str, source_file_type: str, configuration_key: str
) -> ExtenededAutomation:
return ExtenededAutomation(
source_file=source_file,
source_file_type=source_file_type,
configuration_key=configuration_key,
**json.loads(base64.b64decode(auto_str.encode("utf-8")).decode("utf-8")),
configuration_key=decode_obj(configuration_key), # type: ignore
**decode_obj(auto_str),
)
def decode_obj(obj_str: str) -> dict | list:
return json.loads(base64.b64decode(obj_str.encode("utf-8")).decode("utf-8"))
class AutomationDBConnection:
def __init__(self, db_file: Path) -> None:
self.db_file = db_file
@@ -69,13 +77,13 @@ class AutomationDBConnection:
mode TEXT NOT NULL,
source_file TEXT NOT NULL,
source_file_type TEXT NOT NULL,
configuration_key TEXT NOT NULL,
configuration_key BLOB NOT NULL,
rest BLOB NOT NULL
);
"""
)
def insert_automations(self, automations: List[ExtenededAutomation]):
def insert_automations(self, automations: list[ExtenededAutomation]):
with self.get_cur() as cur:
for automation in automations:
try:
@@ -88,7 +96,7 @@ class AutomationDBConnection:
"{automation.mode}",
"{automation.source_file}",
"{automation.source_file_type}",
"{automation.configuration_key}",
"{encode_obj(automation.configuration_key)}",
"{encode_auto(automation)}"
);
""",
+27 -12
View File
@@ -15,16 +15,31 @@ from src.yaml_serializer import (
from ..hass_config.loader import HassConfig
from .errors import InvalidAutomationFile
from .tags import TagManager
from .types import ExtenededAutomation
from .types import AutomationPathIter, ConfigurationKey, ExtenededAutomation
logger = get_logger(__file__)
AUTOMATION_CONFIGURATION_KEY = "automation shortumation"
def load_and_iter_automations(
hass_config: HassConfig, tag_manager: TagManager
) -> Iterator[ExtenededAutomation]:
for config_key, p in extract_automation_paths(hass_config):
try:
yield from load_automation_path(
hass_config.root_path,
hass_config.root_path / p,
config_key,
tag_manager,
)
except InvalidAutomationFile as e:
logger.error(e)
def extract_automation_paths(
hass_config: HassConfig,
) -> Iterator[Tuple[str, str]]:
) -> AutomationPathIter:
found = False
for config_key, automation_path in chain(
@@ -34,23 +49,23 @@ def extract_automation_paths(
yield config_key, automation_path
if not found:
yield "", "automations.yaml"
yield [""], "automations.yaml"
def extract_automation_root_paths(
hass_config: HassConfig,
) -> Iterator[Tuple[str, str]]:
) -> AutomationPathIter:
for key, value in extract_automation_keys(hass_config):
logger.info(f"Loading automations from '{key}'")
if isinstance(value, IncludedYamlDir): # type: ignore
yield key, str(value.path.relative_to(hass_config.root_path))
yield [key], str(value.path.relative_to(hass_config.root_path))
elif isinstance(value, list):
logger.warning(f"Failed to load from '{key}' as no support for inline automations yet.")
else:
logger.warning(f"found an invalid type in {key}!")
def extract_package_automation_paths(hass_config: HassConfig) -> Iterator[Tuple[str, str]]:
def extract_package_automation_paths(hass_config: HassConfig) -> AutomationPathIter:
for package_name, package_data in hass_config.pacakges.items():
if not isinstance(package_data, dict):
logger.warning(f"The package {package_name} is incorrectly configured!")
@@ -58,7 +73,7 @@ def extract_package_automation_paths(hass_config: HassConfig) -> Iterator[Tuple[
if automations := package_data.get("automation", None):
logger.info(f"Loading automations from package '{package_name}'")
if isinstance(automations, IncludedYamlDir): # type: ignore
yield package_name, str(automations.path.relative_to(hass_config.root_path))
yield [package_name], str(automations.path.relative_to(hass_config.root_path))
elif isinstance(automations, list):
logger.warning(
f"Failed to load from '{package_name}' as no support for inline automations yet."
@@ -67,12 +82,12 @@ def extract_package_automation_paths(hass_config: HassConfig) -> Iterator[Tuple[
logger.warning(f"found an invalid type in {package_name}!")
def get_base_automation_key(hass_config: HassConfig):
def get_base_automation_key(hass_config: HassConfig) -> Tuple[ConfigurationKey, str]:
found_any = False
for key, value in extract_automation_keys(hass_config):
found_any = True
if isinstance(value, IncludedYaml):
return key, str(value.path.relative_to(hass_config.root_path))
return [key], str(value.path.relative_to(hass_config.root_path))
configurations = hass_config.configurations
if found_any:
out_path = hass_config.get_backup_automation_file_path()
@@ -84,7 +99,7 @@ def get_base_automation_key(hass_config: HassConfig):
out_path.touch(exist_ok=True)
with hass_config.get_configuration_path().open("w") as fp:
fp.write(dump_yaml(configurations, root_path=hass_config.root_path))
return AUTOMATION_CONFIGURATION_KEY, out_file
return [AUTOMATION_CONFIGURATION_KEY], out_file
def extract_automation_keys(
@@ -105,14 +120,14 @@ def extract_automation_keys(
def load_automation_path(
root_path: Path,
automation_path: Path,
configuration_key: str,
configuration_key: ConfigurationKey,
tag_manager: TagManager,
) -> Iterator[ExtenededAutomation]:
"""
Args:
automation_path (Path)
configuration_key (str)
configuration_key (list[str])
Returns:
Iterator[ExtenededAutomation]
+24 -19
View File
@@ -1,12 +1,12 @@
from pathlib import Path
from tempfile import mktemp
from typing import Dict, Optional, Union
from typing import Optional, Union
from src.errors import ErrorSet
from src.hass_config.loader import HassConfig
from src.logger import get_logger
from src.yaml_serializer import dump_yaml, load_yaml
from ..errors import ErrorSet
from ..logger import get_logger
from .db import AutomationDBConnection
from .errors import (
AttemptingToOverwriteAnIncompatibleFileError,
@@ -14,8 +14,8 @@ from .errors import (
InvalidAutomationFile,
)
from .loader import (
extract_automation_paths,
get_base_automation_key,
load_and_iter_automations,
load_automation_path,
)
from .tags import TagManager
@@ -42,27 +42,32 @@ class AutomationManager:
if self.tag_path.exists():
self.tag_manager = TagManager.load(self.tag_path)
def reload(self):
def reload(self, batch_size: int = 100):
self.db.reset()
self.reload_tags()
errors = []
for config_key, p in extract_automation_paths(self.hass_config):
auto_it = load_and_iter_automations(self.hass_config, self.tag_manager)
batch: list[ExtenededAutomation] = []
stop = False
while not stop:
try:
automations = list(
load_automation_path(
self.hass_config.root_path,
self.hass_config.root_path / p,
config_key,
self.tag_manager,
)
)
self.db.insert_automations(automations)
except InvalidAutomationFile as e:
logger.warning(e)
errors.append(e)
batch.append(next(auto_it))
if len(batch) >= batch_size:
self.db.insert_automations(batch)
batch = []
except StopIteration:
if stop:
break
stop = True
except DBError as e:
logger.warning(e)
errors.append(e)
try:
if len(batch) >= 0:
self.db.insert_automations(batch)
except DBError as e:
logger.warning(e)
errors.append(e)
if len(errors) > 0:
raise ErrorSet(*errors)
@@ -131,7 +136,7 @@ class AutomationManager:
automation_path.write_text(dump_yaml(objs, self.hass_config.root_path))
self.update_tags(automation.id, automation.tags)
def update_tags(self, automation_id: str, tags: Dict[str, str]):
def update_tags(self, automation_id: str, tags: dict[str, str]):
self.reload_tags()
self.tag_manager[automation_id] = tags
self.tag_manager.save(self.tag_path)
+1 -2
View File
@@ -1,10 +1,9 @@
from pathlib import Path
from typing import Dict
from src.yaml_serializer import dump_yaml, load_yaml
class TagManager(Dict[str, Dict[str, str]]):
class TagManager(dict[str, dict[str, str]]):
def save(self, path: Path):
if not path.parent.exists():
path.parent.mkdir(parents=True)
+11 -9
View File
@@ -1,27 +1,29 @@
import json
from pathlib import Path
from typing import Dict, List, Literal, Optional, Union
from typing import Iterator, Literal, Optional, Union
from pydantic import BaseModel
from pydantic.fields import Field
ConfigurationKey = list[str]
AutomationPathIter = Iterator[tuple[ConfigurationKey, str]]
class BaseAutomation(BaseModel):
id: str
alias: Optional[str] = None
description: Optional[str] = None
trigger_variables: Optional[Dict[str, Union[str, int, bool, float]]] = None
trigger_variables: Optional[dict[str, Union[str, int, bool, float]]] = None
mode: str = "single"
trigger: List[dict] = []
condition: List[dict] = []
action: List[dict] = []
trigger: list[dict] = []
condition: list[dict] = []
action: list[dict] = []
def to_primitive(self):
return self.dict(exclude_unset=True, exclude_none=True)
class Automation(BaseAutomation):
tags: Dict[str, str] = Field(default_factory=dict)
tags: dict[str, str] = Field(default_factory=dict)
def to_primitive(self, include_tags: bool = False):
out = super().to_primitive()
@@ -32,8 +34,8 @@ class Automation(BaseAutomation):
class ExtenededAutomation(Automation):
source_file: str
source_file_type: Literal["list", "obj"]
configuration_key: str
source_file_type: Literal["list", "obj", "inline"]
configuration_key: ConfigurationKey
def to_primitive(self, include_tags: bool = False):
out = super().to_primitive(include_tags=include_tags)
+1 -2
View File
@@ -1,11 +1,10 @@
import argparse
from pathlib import Path
from typing import Dict, List
import yaml
def convert_tags(automations: List[dict], tags: Dict[int, dict]) -> Dict[str, dict]:
def convert_tags(automations: list[dict], tags: dict[int, dict]) -> dict[str, dict]:
converted = {}
for tagsIdx, tagValues in tags.items():
if tagsIdx < len(automations):
+1 -1
View File
@@ -1,5 +1,5 @@
import json
from typing import Any, List
from typing import Any
def json_dumps(obj: Any) -> str:
+14 -1
View File
@@ -1,5 +1,5 @@
from pathlib import Path
from typing import Iterator
from typing import Iterator, TypeVar
def extract_files(p: Path) -> Iterator[Path]:
@@ -18,3 +18,16 @@ def extract_files(p: Path) -> Iterator[Path]:
yield from extract_files(c)
else:
yield p
T = TypeVar("T")
def batch(it: Iterator[T], batch_size: int) -> Iterator[list[T]]:
b = []
for n in it:
b.append(n)
if len(b) >= batch_size:
yield b
if len(b) > 0:
yield b
@@ -12,5 +12,20 @@ tts:
script: !include scripts.yaml
scene: !include scenes.yaml
automation:
- id: "inline-auto"
alias: inline-auto
description: Example package auto
mode: single
trigger:
- platform: homeassistant
event: start
condition: []
action:
- service: counter.increment
data: {}
target:
entity_id: counter.up_times
homeassistant:
packages: !include_dir_merge_named packages
+4 -4
View File
@@ -25,7 +25,7 @@ class db_tests(TestWithDB):
alias="test",
source_file="automation.yaml",
source_file_type="obj",
configuration_key="automation",
configuration_key=["automation"],
),
ExtenededAutomation(
id="test2",
@@ -33,7 +33,7 @@ class db_tests(TestWithDB):
source_file="automation.yaml",
source_file_type="obj",
action=[{"event": {"nice": "one"}, "event_type": "custom"}],
configuration_key="automation",
configuration_key=["automation"],
),
]
self.db.insert_automations(originals)
@@ -50,7 +50,7 @@ class db_tests(TestWithDB):
alias="test",
source_file="automation.yaml",
source_file_type="obj",
configuration_key="automation",
configuration_key=["automation"],
),
ExtenededAutomation(
id="test",
@@ -58,7 +58,7 @@ class db_tests(TestWithDB):
source_file="automation.yaml",
source_file_type="obj",
action=[{"event": {"nice": "one"}, "event_type": "custom"}],
configuration_key="automation",
configuration_key=["automation"],
),
]
with self.assertRaises(DBDataError) as err:
@@ -10,7 +10,6 @@ from src.automations.tags import TagManager
from src.automations.types import ExtenededAutomation
from src.hass_config.loader import HassConfig
from tests.utils import (
HA_CONFIG10_EXAMPLE,
HA_CONFIG2_EXAMPLE,
HA_CONFIG3_EXAMPLE,
HA_CONFIG4_EXAMPLE,
@@ -19,6 +18,7 @@ from tests.utils import (
HA_CONFIG7_EXAMPLE,
HA_CONFIG8_EXAMPLE,
HA_CONFIG9_EXAMPLE,
HA_CONFIG10_EXAMPLE,
HA_CONFIG_EXAMPLE,
)
@@ -29,7 +29,7 @@ class loader_tests(TestCase):
load_automation_path(
HA_CONFIG2_EXAMPLE,
HA_CONFIG2_EXAMPLE / "automations.yaml",
configuration_key="automation",
configuration_key=["automation"],
tag_manager=TagManager({"1652069225859": {"type": "routine"}}),
)
)
@@ -40,7 +40,7 @@ class loader_tests(TestCase):
id="1652069225859",
source_file="automations.yaml",
source_file_type="list",
configuration_key="automation",
configuration_key=["automation"],
alias="Climate - Pref temperature ",
description="",
mode="single",
@@ -73,7 +73,7 @@ class loader_tests(TestCase):
load_automation_path(
HA_CONFIG4_EXAMPLE,
HA_CONFIG4_EXAMPLE / "automations" / "notify_washer.yaml",
configuration_key="automation base",
configuration_key=["automation base"],
tag_manager=TagManager({"1659114647067": {"room": "laundry"}}),
)
)
@@ -84,7 +84,7 @@ class loader_tests(TestCase):
id="1659114647067",
source_file="automations/notify_washer.yaml",
source_file_type="obj",
configuration_key="automation base",
configuration_key=["automation base"],
alias="Notify Washer",
description="Example",
mode="single",
@@ -111,7 +111,7 @@ class loader_tests(TestCase):
load_automation_path(
HA_CONFIG9_EXAMPLE,
HA_CONFIG9_EXAMPLE / "automations.yaml",
configuration_key="automation base",
configuration_key=["automation base"],
tag_manager=TagManager({}),
)
)
@@ -125,7 +125,7 @@ class loader_tests(TestCase):
load_automation_path(
file_path.parent,
file_path,
configuration_key="automation ui",
configuration_key=["automation ui"],
tag_manager=TagManager(),
)
)
@@ -138,7 +138,7 @@ class loader_tests(TestCase):
load_automation_path(
file_path.parent,
file_path,
configuration_key="automation manual",
configuration_key=["automation manual"],
tag_manager=TagManager(),
)
)
+22 -20
View File
@@ -13,6 +13,7 @@ from src.hass_config.loader import HassConfig
from tests.utils import (
HA_CONFIG6_EXAMPLE,
HA_CONFIG8_EXAMPLE,
HA_CONFIG10_EXAMPLE,
HA_CONFIG_EXAMPLE,
create_copy_of_example_config,
)
@@ -46,7 +47,7 @@ class manager_tests(TestWithDB):
}
],
)
self.assertEqual(automation.configuration_key, "automation cools")
self.assertEqual(automation.configuration_key, ["automation cools"])
automation.trigger.append(
{"platform": "zone", "entity_id": "person.thor", "zone": "zone.azguard"}
)
@@ -76,19 +77,13 @@ class manager_tests(TestWithDB):
def test_bad_automations_file(self):
config_path, automation_manager = self.get_manager(HA_CONFIG_EXAMPLE)
(config_path / "automations.yaml").write_text("bob is an uncle")
with self.assertRaises(ErrorSet) as err:
automation_manager.reload()
self.assertEqual(len(err.exception.errors), 1)
self.assertIsInstance(err.exception.errors[0], InvalidAutomationFile)
automation_manager.reload()
self.assertEqual(automation_manager.count(), 0)
def test_one_bad_automations_file(self):
config_path, automation_manager = self.get_manager(HA_CONFIG6_EXAMPLE)
(config_path / "automations" / "ui.yaml").write_text("bob is an uncle")
with self.assertRaises(ErrorSet) as err:
automation_manager.reload()
self.assertEqual(len(err.exception.errors), 1)
self.assertIsInstance(err.exception.errors[0], InvalidAutomationFile)
automation_manager.reload()
self.assertEqual(automation_manager.count(), 10)
def test_delete_automation_in_single_file(self):
@@ -96,7 +91,7 @@ class manager_tests(TestWithDB):
self.assertTrue((config_path / "automations/include_dir_list/home/down.yaml").exists())
automation_manager.reload()
automation = automation_manager.get("1659114822642")
self.assertEqual(automation.configuration_key, "automation manual")
self.assertEqual(automation.configuration_key, ["automation manual"])
automation_manager.delete(automation)
automation_manager.reload()
with self.assertRaises(DBNoAutomationFound):
@@ -110,7 +105,7 @@ class manager_tests(TestWithDB):
)
automation_manager.reload()
automation = automation_manager.get("mlist2")
self.assertEqual(automation.configuration_key, "automation cools")
self.assertEqual(automation.configuration_key, ["automation cools"])
automation_manager.delete(automation)
automation_manager.reload()
with self.assertRaises(DBNoAutomationFound):
@@ -128,7 +123,7 @@ class manager_tests(TestWithDB):
automation_manager.update(
ExtenededAutomation(
id="newlymade",
configuration_key="automation manual",
configuration_key=["automation manual"],
source_file="automations/include_dir_list/home/left.yaml",
source_file_type="obj",
)
@@ -139,7 +134,7 @@ class manager_tests(TestWithDB):
automation_manager.get("newlymade"),
ExtenededAutomation(
id="newlymade",
configuration_key="automation manual",
configuration_key=["automation manual"],
source_file="automations/include_dir_list/home/left.yaml",
source_file_type="obj",
),
@@ -156,7 +151,7 @@ class manager_tests(TestWithDB):
automation_manager.update(
ExtenededAutomation(
id="newlymade",
configuration_key="automation cools",
configuration_key=["automation cools"],
source_file="automations/include_dir_merge_list/sub/list2.yaml",
source_file_type="list",
),
@@ -170,7 +165,7 @@ class manager_tests(TestWithDB):
automation_manager.get("newlymade"),
ExtenededAutomation(
id="newlymade",
configuration_key="automation cools",
configuration_key=["automation cools"],
source_file="automations/include_dir_merge_list/sub/list2.yaml",
source_file_type="list",
),
@@ -187,7 +182,7 @@ class manager_tests(TestWithDB):
automation_manager.update(
ExtenededAutomation(
id="newlymade",
configuration_key="automation cools",
configuration_key=["automation cools"],
source_file="automations/include_dir_merge_list/sub/list4.yaml",
source_file_type="list",
),
@@ -201,7 +196,7 @@ class manager_tests(TestWithDB):
automation_manager.get("newlymade"),
ExtenededAutomation(
id="newlymade",
configuration_key="automation cools",
configuration_key=["automation cools"],
source_file="automations/include_dir_merge_list/sub/list4.yaml",
source_file_type="list",
),
@@ -219,7 +214,7 @@ class manager_tests(TestWithDB):
automation_manager.update(
ExtenededAutomation(
id="newlymade",
configuration_key="automation cools",
configuration_key=["automation cools"],
source_file="automations/include_dir_merge_list/sub/list2.yaml",
source_file_type="obj",
),
@@ -245,7 +240,7 @@ class manager_tests(TestWithDB):
automation_manager.update(
ExtenededAutomation(
id="newlymade",
configuration_key="automation cools",
configuration_key=["automation cools"],
source_file="automations/include_dir_list/home/down.yaml",
source_file_type="list",
),
@@ -333,4 +328,11 @@ class manager_tests(TestWithDB):
_, automation_manager = self.get_manager(HA_CONFIG8_EXAMPLE)
automation_manager.reload()
auto = automation_manager.get("package-sub")
self.assertEqual(auto.configuration_key, "pacakage2")
self.assertEqual(auto.configuration_key, ["pacakage2"])
def test_load_inline_auto(self):
_, automation_manager = self.get_manager(HA_CONFIG10_EXAMPLE)
automation_manager.reload()
auto = automation_manager.get("inline-auto")
self.assertEqual(auto.source_file_type, ["inline"])
self.assertEqual(auto.configuration_key, ["automation"])
@@ -2,7 +2,11 @@ from unittest import TestCase
from src.automations.types import BaseAutomation
from src.hass_config.loader import HassConfig
from tests.utils import HA_CONFIG8_EXAMPLE, SAMPLES_FOLDER, create_dummy_config_folder
from tests.utils import (
HA_CONFIG8_EXAMPLE,
SAMPLES_FOLDER,
create_dummy_config_folder,
)
class loader_tests(TestCase):
+5 -5
View File
@@ -1,7 +1,7 @@
from pathlib import Path
from shutil import copytree
from tempfile import mkdtemp
from typing import List, Literal, Optional, Tuple
from typing import Literal, Optional, Tuple
from src.automations.manager import AutomationManager
from src.automations.types import BaseAutomation
@@ -49,7 +49,7 @@ def create_copy_of_example_config(config_to_copy: Path = HA_CONFIG_EXAMPLE) -> P
def get_dummy_automation_loader(
auto: List[BaseAutomation],
auto: list[BaseAutomation],
secrets: Optional[dict] = None,
other_config: Optional[dict] = None,
automation_in_conifguration_mode: Literal["include", "inline", "none"] = "include",
@@ -57,7 +57,7 @@ def get_dummy_automation_loader(
"""Creates a dummy /config structure for testing and return the loader and config
Args:
auto (List[BaseAutomation]): automations
auto (list[BaseAutomation]): automations
secrets (Optional[dict], optional): dictionary of secrets
other_config (Optional[dict], optional): some config stuff to place into configuration.yaml. Defaults to None.
automation_in_conifguration_mode (Literal['include', 'inline', 'none'], optional): whether to !include automation as a separete file or in the configuration yaml. Defaults to inline.
@@ -77,7 +77,7 @@ def get_dummy_automation_loader(
def create_dummy_config_folder(
auto: List[BaseAutomation],
auto: list[BaseAutomation],
secrets: Optional[dict] = None,
other_config: Optional[dict] = None,
automation_in_conifguration_mode: Literal["include", "inline", "none"] = "include",
@@ -85,7 +85,7 @@ def create_dummy_config_folder(
"""Creates a dummy /config structure for testing
Args:
auto (List[BaseAutomation]): automations
auto (list[BaseAutomation]): automations
secrets (Optional[dict], optional): dictionary of secrets
other_config (Optional[dict], optional): some config stuff to place into configuration.yaml. Defaults to None.
automation_in_conifguration_mode (Literal['include', 'inline', 'none'], optional): whether to !include automation as a separete file or in the configuration yaml. Defaults to inline.