From 298a233a5ffdbb41e3ca4bd15260b39bf82f9efc Mon Sep 17 00:00:00 2001 From: Ari Date: Fri, 20 May 2022 10:53:53 -0400 Subject: [PATCH] changed yaml loaders --- api/setup.py | 2 +- api/src/yaml_serializer/__init__.py | 15 +++++++-- api/src/yaml_serializer/base_yaml.py | 19 ----------- api/src/yaml_serializer/dumper.py | 16 ++++++++++ api/src/yaml_serializer/loader.py | 26 +++++++++++++++ api/tests/unittests/test_yaml_serializer.py | 35 ++++++++++++++++----- 6 files changed, 83 insertions(+), 30 deletions(-) delete mode 100644 api/src/yaml_serializer/base_yaml.py create mode 100644 api/src/yaml_serializer/dumper.py create mode 100644 api/src/yaml_serializer/loader.py diff --git a/api/setup.py b/api/setup.py index 6ddbe63..294fd37 100644 --- a/api/setup.py +++ b/api/setup.py @@ -4,7 +4,7 @@ import setuptools # type: ignore setuptools.setup( python_requires=">=3.9", install_requires=[ - "ruamel.yaml==0.17.17", + "PyYAML==6.0", "fastapi==0.70.0", "pydantic==1.8.2", "uvicorn[standard]==0.15.0", diff --git a/api/src/yaml_serializer/__init__.py b/api/src/yaml_serializer/__init__.py index e0eb247..3a83a35 100644 --- a/api/src/yaml_serializer/__init__.py +++ b/api/src/yaml_serializer/__init__.py @@ -1,7 +1,9 @@ +import yaml import io from typing import Any, TextIO, Union -from src.yaml_serializer.base_yaml import yaml +from src.yaml_serializer.loader import YamlSafeLoader +from src.yaml_serializer.dumper import YamlSafeDumper def dump_yaml(obj: Any) -> str: @@ -14,7 +16,14 @@ def dump_yaml(obj: Any) -> str: str """ stream = io.StringIO() - yaml.dump(obj, stream) + yaml.dump( + obj, + stream, + default_flow_style=False, + allow_unicode=True, + sort_keys=False, + Dumper=YamlSafeDumper, + ) stream.seek(0) return stream.read() @@ -28,4 +37,4 @@ def load_yaml(yaml_stream: TextIO) -> Union[dict, list]: Returns: List[dict, list] """ - return yaml.load(yaml_stream) + return yaml.load(yaml_stream, Loader=YamlSafeLoader) diff --git a/api/src/yaml_serializer/base_yaml.py b/api/src/yaml_serializer/base_yaml.py deleted file mode 100644 index db8d052..0000000 --- a/api/src/yaml_serializer/base_yaml.py +++ /dev/null @@ -1,19 +0,0 @@ -from ruamel.yaml import YAML - -from src.yaml_serializer.types import ( - IncludedYaml, - IncludedYamlDir, - SecretValue, -) - -yaml = YAML() -yaml.register_class(IncludedYaml) -yaml.register_class(SecretValue) -yaml.register_class(IncludedYamlDir) -yaml.constructor.add_constructor("!include", IncludedYaml.from_yaml) -yaml.constructor.add_constructor("!secret", SecretValue.from_yaml) -yaml.constructor.add_constructor("!env_var", IncludedYamlDir.from_yaml) -yaml.constructor.add_constructor("!include_dir_list", IncludedYamlDir.from_yaml) -yaml.constructor.add_constructor("!include_dir_merge_list", IncludedYamlDir.from_yaml) -yaml.constructor.add_constructor("!include_dir_named", IncludedYamlDir.from_yaml) -yaml.constructor.add_constructor("!include_dir_merge_named", IncludedYamlDir.from_yaml) diff --git a/api/src/yaml_serializer/dumper.py b/api/src/yaml_serializer/dumper.py new file mode 100644 index 0000000..706708a --- /dev/null +++ b/api/src/yaml_serializer/dumper.py @@ -0,0 +1,16 @@ +import yaml + +from src.yaml_serializer.types import ( + IncludedYaml, + IncludedYamlDir, + SecretValue, +) + + +class YamlSafeDumper(yaml.SafeDumper): + pass + + +YamlSafeDumper.add_representer(IncludedYaml, IncludedYaml.to_yaml) +YamlSafeDumper.add_representer(SecretValue, SecretValue.to_yaml) +YamlSafeDumper.add_representer(IncludedYamlDir, IncludedYamlDir.to_yaml) diff --git a/api/src/yaml_serializer/loader.py b/api/src/yaml_serializer/loader.py new file mode 100644 index 0000000..ea989c2 --- /dev/null +++ b/api/src/yaml_serializer/loader.py @@ -0,0 +1,26 @@ +import yaml + +from src.yaml_serializer.types import ( + IncludedYaml, + IncludedYamlDir, + SecretValue, +) + + +class YamlSafeLoader(yaml.SafeLoader): + pass + # def compose_node(self, parent: yaml.nodes.Node, index: int) -> yaml.nodes.Node: # type: ignore[override] + # """Annotate a node with the first line it was seen.""" + # last_line: int = self.line + # node: yaml.nodes.Node = super().compose_node(parent, index) # type: ignore[assignment] + # node.__line__ = last_line + 1 # type: ignore[attr-defined] + # return node + + +YamlSafeLoader.add_constructor("!include", IncludedYaml.from_yaml) +YamlSafeLoader.add_constructor("!secret", SecretValue.from_yaml) +YamlSafeLoader.add_constructor("!env_var", IncludedYamlDir.from_yaml) +YamlSafeLoader.add_constructor("!include_dir_list", IncludedYamlDir.from_yaml) +YamlSafeLoader.add_constructor("!include_dir_merge_list", IncludedYamlDir.from_yaml) +YamlSafeLoader.add_constructor("!include_dir_named", IncludedYamlDir.from_yaml) +YamlSafeLoader.add_constructor("!include_dir_merge_named", IncludedYamlDir.from_yaml) diff --git a/api/tests/unittests/test_yaml_serializer.py b/api/tests/unittests/test_yaml_serializer.py index e87a954..39791a6 100644 --- a/api/tests/unittests/test_yaml_serializer.py +++ b/api/tests/unittests/test_yaml_serializer.py @@ -19,21 +19,29 @@ class dumping_yamls_tests(TestCase): ) } ) - self.assertEqual(yaml, "automations: !include automations.yaml\n") + self.assertEqual(yaml, "automations: !include 'automations.yaml'\n") def test_dumping_secrets(self): yaml = dump_yaml({"google_password": SecretValue("gpass", "supersec")}) - self.assertEqual(yaml, "google_password: !secret gpass\n") + self.assertEqual(yaml, "google_password: !secret 'gpass'\n") def test_dumping_stub(self): yaml = dump_yaml({"sensor": IncludedYamlDir("include_dir_list", "sensors/*yaml")}) - self.assertEqual(yaml, "sensor: !include_dir_list sensors/*yaml\n") + self.assertEqual(yaml, "sensor: !include_dir_list 'sensors/*yaml'\n") def test_load_stub(self): yaml_obj = load_yaml( io.StringIO( """ name: cool + date: "10:00" + bools: + - "yes" + - "no" + - true + - false + bobIsNull: + bobIsNull2: null google_password: !secret gpass sensor: !include_dir_list sensors/*yaml automations: !include automations.yaml @@ -44,6 +52,10 @@ class dumping_yamls_tests(TestCase): dict(yaml_obj), { "name": "cool", + "date": "10:00", + "bools": ["yes", "no", True, False], + "bobIsNull": None, + "bobIsNull2": None, "google_password": SecretValue("gpass", "n/a"), "sensor": IncludedYamlDir("include_dir_list", "sensors/*yaml"), "automations": IncludedYaml("automations.yaml"), @@ -51,8 +63,17 @@ class dumping_yamls_tests(TestCase): ) self.assertEqual( dump_yaml(yaml_obj), - "name: cool\n" - "google_password: !secret gpass\n" - "sensor: !include_dir_list sensors/*yaml\n" - f"automations: !include automations.yaml\n", + """name: cool +date: '10:00' +bools: +- 'yes' +- 'no' +- true +- false +bobIsNull: null +bobIsNull2: null +google_password: !secret 'gpass' +sensor: !include_dir_list 'sensors/*yaml' +automations: !include 'automations.yaml' +""", )