rougail-user-data-yaml/tests/test_load.py

import os
from pytest import fixture  # , raises
from pathlib import Path
from rougail import Rougail
#########################
from rougail.user_data_yaml import RougailUserDataYaml as RougailUserData
from json import load, dump, loads, dumps
from ruamel.yaml import YAML
#########################

from rougail_tests.utils import get_structures_list, get_rougail_config, get_values_for_config, config_to_dict, root_test_dir

EXT = "yml"

excludes = []
excludes = [
            '40_0leadership_follower_default_submulti',  # submulti is not allowed
            '40_0leadership_follower_default_submulti_calculation',  # submulti is not allowed
            '40_6leadership_follower_multi',  # submulti is not allowed
            ]

test_ok = get_structures_list(excludes)
# test_ok = [Path('../rougail-tests/structures/00_6port')]


def idfn(fixture_value):
    return fixture_value.name


@fixture(scope="module", params=test_ok, ids=idfn)
def test_dir(request):
    return request.param


def _test_structural_files(test_dir, namespace, ext, *, level, need_exclude=False):
    rougailconfig = get_rougail_config(test_dir, namespace)
    if not rougailconfig:
        return
    ##rougailconfig['tiramisu_cache'] = 'p.py'
    ##################################
    rougailconfig['step.user_data'] = ['yaml']
    ##################################
    dir_name = 'test'
    if namespace:
        dir_name += '_namespace'
    elif (test_dir / 'force_namespace').is_file():
        return
    rougail = Rougail(rougailconfig)
    config = rougail.run()
    ##################################
    root_path = Path('tests') / 'results' / test_dir.name
    makedict = root_path / 'makedict' / f'{level}.json'
    filename = root_path / 'file'
    if need_exclude:
        filename = filename / f'{level}_exclude.{EXT}'
    else:
        filename = filename / f'{level}.{EXT}'
    populate(filename, makedict, rougailconfig, level, need_exclude)
    rougailconfig['yaml.filename'] = [str(filename)]
    ##################################
    # loads variables in the tiramisu config
    generated_user_data = RougailUserData(config, rougailconfig=rougailconfig).run()
    errors = rougail.user_datas(generated_user_data)
    #expected output
    expected_filename = Path('tests') / 'results' / test_dir.name /  'makedict' / f'{level}.json'
    with expected_filename.open() as json_file:
        expected = load(json_file)
    # here is the effective test
    errors_file = Path('tests') / 'results' / test_dir.name /  'errors' / f'{level}.json'
    if need_exclude:
        errors = loads(dumps(errors).replace('all_exclude.yml', 'all.yml'))
    if not errors_file.is_file():
        errors_file.parent.mkdir(parents=True, exist_ok=True)
        with open(errors_file, 'a') as json_file:
            dump(errors, json_file, indent=4)
    with open(errors_file) as json_file:
        expected_errors = load(json_file)
    # expected_errors = {
    #     'errors': [],
    #     'warnings': [],
    # }
    assert expected_errors == errors, errors_file
    #
    config.property.read_only()
    config_dict = dict(config_to_dict(config.value.get()))
    assert expected == config_dict, expected_filename


def populate(filename, makedict_file, rougailconfig, level, need_exclude):
    if filename.is_file() and makedict_file.is_file():
        return
    config = Rougail(rougailconfig).run()
    values = get_values_for_config(config, not need_exclude, level)
    if not filename.is_file():
        filename.parent.mkdir(parents=True, exist_ok=True)
        with filename.open('w') as fh:
            ##############################################
            dump(values, fh, indent=4)
            ##############################################
    if  not makedict_file.is_file():
        makedict_file.parent.mkdir(parents=True, exist_ok=True)
        config.property.read_only()
        config_dict = dict(config_to_dict(config.value.get()))
        with makedict_file.open('w') as fh:
            dump(config_dict, fh, indent=4)
            fh.write('\n')


def test_structural_files_all(test_dir):
    "tests the output"
    _test_structural_files(test_dir, True, EXT, level='all')


def test_structural_files_all_exclude(test_dir):
    "tests the output"
    _test_structural_files(test_dir, True, EXT, level='all', need_exclude=True)


def test_structural_files_mandatories(test_dir):
    "tests the output"
    _test_structural_files(test_dir, True, EXT, level='mandatories')


def test_errors():
    rougailconfig = get_rougail_config(Path('tests/errors/60_0family_dynamic_upper_char'), False)
    ##################################
    rougailconfig['step.user_data'] = ['yaml']
    rougailconfig['yaml.filename'] = ['tests/errors/60_0family_dynamic_upper_char/file/all_exclude.yml']
    ##################################
    rougail = Rougail(rougailconfig)
    config = rougail.run()
    generated_user_data = RougailUserData(config, rougailconfig=rougailconfig).run()
    errors = rougail.user_datas(generated_user_data)
    assert errors == {'errors': [], 'warnings': [
         'cannot load variable path "dynVal1.var", the identifier "Val1" is not '
         'valid in the YAML file '
         '"tests/errors/60_0family_dynamic_upper_char/file/all_exclude.yml"',
         'variable or family "dynVal1" does not exist, it will be ignored when '
         'loading from the YAML file '
         '"tests/errors/60_0family_dynamic_upper_char/file/all_exclude.yml"',
        ]}


def test_errors_2():
    rougailconfig = get_rougail_config(Path('tests/errors/60_0family_dynamic_upper_char2'), False)
    ##################################
    rougailconfig['step.user_data'] = ['yaml']
    rougailconfig['yaml.filename'] = ['tests/errors/60_0family_dynamic_upper_char2/file/all_exclude.yml']
    ##################################
    rougail = Rougail(rougailconfig)
    config = rougail.run()
    generated_user_data = RougailUserData(config, rougailconfig=rougailconfig).run()
    errors = rougail.user_datas(generated_user_data)
    assert errors == {'errors': [], 'warnings': [
         'cannot load variable path "dynVal1.var", the identifier "Val1" is not '
         'valid in the YAML file '
         '"tests/errors/60_0family_dynamic_upper_char2/file/all_exclude.yml"',
         'variable or family "dynVal1" does not exist, it will be ignored when '
         'loading from the YAML file '
         '"tests/errors/60_0family_dynamic_upper_char2/file/all_exclude.yml"',
        ]}
    ##################################


#######################################################################
error_yaml = list((Path(__file__).parent / 'errors' / 'yaml').glob("*.yaml"))
error_yaml.sort()


@fixture(scope="module", params=error_yaml, ids=idfn)
def test_file_error(request):
    return request.param


def test_structural_files_error(test_file_error):
    rougailconfig = get_rougail_config(test_file_error.parent.parent / 'structure')
    ##################################
    rougailconfig['step.user_data'] = ['yaml']
    rougailconfig['yaml.filename'] = [str(test_file_error)]
    rougail = Rougail(rougailconfig)
    config = rougail.run()
    ##################################
    # loads variables in the tiramisu config
    generated_user_data = RougailUserData(config, rougailconfig=rougailconfig).run()
    errors = rougail.user_datas(generated_user_data)
    errors_file = test_file_error.parent.parent / "results" / test_file_error.name
    if not errors_file.is_file():
        errors_file.parent.mkdir(parents=True, exist_ok=True)
        with open(errors_file, 'a') as json_file:
            dump(errors, json_file, indent=4)
    with open(errors_file) as json_file:
        expected_errors = load(json_file)
    assert expected_errors == errors, errors_file


def test_structural_files_directory():
    test_dir = Path(__file__).parent / 'directory'
    rougailconfig = get_rougail_config(test_dir / 'structure')
    ##################################
    rougailconfig['step.user_data'] = ['yaml']
    rougailconfig['yaml.filename'] = [str(test_dir / 'yaml')]
    rougail = Rougail(rougailconfig)
    config = rougail.run()
    ##################################
    # loads variables in the tiramisu config
    generated_user_data = RougailUserData(config, rougailconfig=rougailconfig).run()
    rougail.user_datas(generated_user_data)
    config.property.read_only()
    data = dict(config_to_dict(config.value.get()))
    data_file = test_dir / "result.txt"
    if not data_file.is_file():
        with open(data_file, 'a') as json_file:
            dump(data, json_file, indent=4)
    with open(data_file) as json_file:
        expected = load(json_file)
    assert expected == data, data_file


def test_secret_all():
    root_test_dir / '00_6secret'
    rougailconfig = get_rougail_config(root_test_dir / '00_6secret', namespace=False)
    rougailconfig['step.user_data'] = ['yaml']
    rougailconfig["yaml.file_with_secrets"] = "all"
    rougailconfig["yaml.filename"] = ["tests/secrets/secret.yml"]
    rougail = Rougail(rougailconfig)
    config = rougail.run()
    user_data = RougailUserData(config, rougailconfig=rougailconfig).run()
    errors = rougail.user_datas(user_data)
    assert errors == {'errors': [], 'warnings': []}
    config_dict = dict(config_to_dict(config.value.get()))
    assert config_dict == {'secret1': "my_password", 'secret2': 'value'}


def test_secret_none():
    root_test_dir / '00_6secret'
    rougailconfig = get_rougail_config(root_test_dir / '00_6secret', namespace=False)
    rougailconfig['step.user_data'] = ['yaml']
    rougailconfig["yaml.file_with_secrets"] = "none"
    rougailconfig["yaml.filename"] = ["tests/secrets/secret.yml"]
    rougail = Rougail(rougailconfig)
    config = rougail.run()
    user_data = RougailUserData(config, rougailconfig=rougailconfig).run()
    errors = rougail.user_datas(user_data)
    assert errors == {'errors': ['the variable "secret1" contains secrets and should not be defined in the YAML file "tests/secrets/secret.yml"'], 'warnings': []}
    config_dict = dict(config_to_dict(config.value.get()))
    assert config_dict == {'secret1': None, 'secret2': 'value'}


def test_secret_first():
    root_test_dir / '00_6secret'
    rougailconfig = get_rougail_config(root_test_dir / '00_6secret', namespace=False)
    rougailconfig['step.user_data'] = ['yaml']
    rougailconfig["yaml.file_with_secrets"] = "first"
    rougailconfig["yaml.filename"] = ["tests/secrets/secret.yml", "tests/secrets/no_secret.yml"]
    rougail = Rougail(rougailconfig)
    config = rougail.run()
    user_data = RougailUserData(config, rougailconfig=rougailconfig).run()
    errors = rougail.user_datas(user_data)
    assert errors == {'errors': [], 'warnings': []}
    config_dict = dict(config_to_dict(config.value.get()))
    assert config_dict == {'secret1': "my_password", 'secret2': 'value'}


def test_secret_first_error():
    root_test_dir / '00_6secret'
    rougailconfig = get_rougail_config(root_test_dir / '00_6secret', namespace=False)
    rougailconfig['step.user_data'] = ['yaml']
    rougailconfig["yaml.file_with_secrets"] = "first"
    rougailconfig["yaml.filename"] = ["tests/secrets/no_secret.yml", "tests/secrets/secret.yml"]
    rougail = Rougail(rougailconfig)
    config = rougail.run()
    user_data = RougailUserData(config, rougailconfig=rougailconfig).run()
    errors = rougail.user_datas(user_data)
    assert errors == {'errors': ['the variable "secret1" contains secrets and should not be defined in the YAML file "tests/secrets/secret.yml"'], 'warnings': []}
    config_dict = dict(config_to_dict(config.value.get()))
    assert config_dict == {'secret1': None, 'secret2': 'value'}


def test_secret_last():
    root_test_dir / '00_6secret'
    rougailconfig = get_rougail_config(root_test_dir / '00_6secret', namespace=False)
    rougailconfig['step.user_data'] = ['yaml']
    rougailconfig["yaml.file_with_secrets"] = "last"
    rougailconfig["yaml.filename"] = ["tests/secrets/no_secret.yml", "tests/secrets/secret.yml"]
    rougail = Rougail(rougailconfig)
    config = rougail.run()
    user_data = RougailUserData(config, rougailconfig=rougailconfig).run()
    errors = rougail.user_datas(user_data)
    assert errors == {'errors': [], 'warnings': []}
    config_dict = dict(config_to_dict(config.value.get()))
    assert config_dict == {'secret1': "my_password", 'secret2': 'value'}


def test_secret_last_error():
    root_test_dir / '00_6secret'
    rougailconfig = get_rougail_config(root_test_dir / '00_6secret', namespace=False)
    rougailconfig['step.user_data'] = ['yaml']
    rougailconfig["yaml.file_with_secrets"] = "last"
    rougailconfig["yaml.filename"] = ["tests/secrets/secret.yml", "tests/secrets/no_secret.yml"]
    rougail = Rougail(rougailconfig)
    config = rougail.run()
    user_data = RougailUserData(config, rougailconfig=rougailconfig).run()
    errors = rougail.user_datas(user_data)
    assert errors == {'errors': ['the variable "secret1" contains secrets and should not be defined in the YAML file "tests/secrets/secret.yml"'], 'warnings': []}
    config_dict = dict(config_to_dict(config.value.get()))
    assert config_dict == {'secret1': None, 'secret2': 'value'}