rougail/tests/test_1_flattener.py

from os import getcwd
from os.path import isfile, join, isdir
from pytest import fixture, raises
from os import listdir, environ, makedirs
from json import load, dumps, loads
from yaml import dump as ydump

environ['TIRAMISU_LOCALE'] = 'en'

from rougail import RougailConvert, RougailConfig
from rougail.error import DictConsistencyError



dico_dirs = 'tests/dictionaries'


test_ok = set()
test_raise = set()

for test in listdir(dico_dirs):
    if isdir(join(dico_dirs, test)):
        if isdir(join(dico_dirs, test, 'tiramisu')):
            test_ok.add(test)
        elif test != '__pycache__':
            test_raise.add(test)

excludes = set([])
#excludes = set(['45multi_family_order'])
test_ok -= excludes
test_raise -= excludes
#test_ok = ['45multi_family_order']
#test_ok = []
#test_raise = ['80auto_autofreeze']
#test_raise = []

ORI_DIR = getcwd()

debug = False
#debug = True

test_ok = list(test_ok)
test_raise = list(test_raise)
test_ok.sort()
test_raise.sort()


@fixture(scope="module", params=test_ok)
def test_dir(request):
    return request.param


@fixture(scope="module", params=test_raise)
def test_dir_error(request):
    return request.param


def xml_to_yaml(filename, destfilename):
    import xmltodict
    def reorganize_variable(value, level):
        move = None
        new_value = []
        for type_ in ['variable', 'family']:
            if type_ in value:
                subvalue = value[type_]
                if isinstance(subvalue, list):
                    for val in subvalue:
                        if 'variable' in val or 'family' in val:
                            move_, val = reorganize_variable(val, level + 1)
                            if move_:
                                move = move_
                        if filename == 'tests/dictionaries/45multi_family_order/xml/00-base.xml' and \
                                val['@name'] == 'variable4':
                            #hack
                            move = val
                        else:
                            new_value.append({type_: val})
                else:
                    if 'variable' in subvalue or 'family' in subvalue:
                        move_, subvalue = reorganize_variable(subvalue, level + 1)
                        if move_:
                            move = move_
                    new_value.append({type_: subvalue})
        if level == 0:
            return new_value
        if level == 1 and  move:
            new_value.append({'variable': move})
        dico = {k: v for k, v in value.items() if k not in ['variable', 'family']}
        dico['variables'] = new_value
        return move, dico

    def _xml_to_yaml(dico, root=False):
        for key, value in dict(dico).items():
            del dico[key]
            if root and key == 'variables':
                value = reorganize_variable(value, 0)
            if isinstance(value, dict):
                _xml_to_yaml(value)
#                dico[key] = value
            if isinstance(value, list):
                dico[key] = value
                for idx, val in enumerate(value):
                    if isinstance(val, dict):
                        _xml_to_yaml(val)
                    elif isinstance(val, (str, bool)):
                        dico[key][idx] = {'text': val}
            else:
                #if value == 'True':
                #    value = True
                #elif value == 'False':
                #    value = False
#                if key == 'family':
#                    pass
#                    variable = {'variables': value['variable']}
#                    del value['variable']
#                    dico[key] = value
#                    dico[key]['variables'] = variable
                if key.startswith('@'):
                    # remove @
                    dico[key[1:]] = value
                elif key.startswith('#'):
                    dico[key[1:]] = value
                else:
                    if key not in ['variable', 'family'] and not isinstance(value, list):
                        if not value:
                            if key == 'override':
                                dico[key] = value
                            elif key == 'value':
                                dico[key] = [{'text': value}]
                        elif isinstance(value, (str, bool)):
                            dico[key] = [{'text': value}]
                        else:
                            dico[key] = [value]
                    else:
                        dico[key] = value
    with open(filename, 'rb') as fh:
        dico = loads(dumps(xmltodict.parse(fh)['rougail']))  # , attr_prefix="")
    _xml_to_yaml(dico, root=True)
    with open(destfilename, 'w') as fh:
        ydump(dico, fh, sort_keys=False)


def build_yml(test_dir, dirname):
    makedirs(dirname)
    for filename in listdir(join(test_dir, 'xml')):
        if not isfile(join(test_dir, 'xml', filename)):
            continue
        xml_to_yaml(join(test_dir, 'xml', filename), join(dirname, filename[:-4] + '.yml'))
    if isdir(join(test_dir, 'xml', 'subfolder')):
        makedirs(join(dirname, 'subfolder'))
        for filename in listdir(join(test_dir, 'xml', 'subfolder')):
            xml_to_yaml(join(test_dir, 'xml', 'subfolder', filename), join(dirname, 'subfolder', filename[:-4] + '.yml'))
    if isdir(join(test_dir, 'xml', 'extra_dirs')):
        makedirs(join(dirname, 'extra_dirs'))
        for extra in listdir(join(test_dir, 'xml', 'extra_dirs')):
            subfolder = join(test_dir, 'xml', 'extra_dirs', extra)
            if not isdir(subfolder):
                continue
            makedirs(join(dirname, 'extra_dirs', extra))
            for filename in listdir(subfolder):
                xml_to_yaml(join(test_dir, 'xml', 'extra_dirs', extra, filename), join(dirname, 'extra_dirs', extra, filename[:-4] + '.yml'))


def load_rougail_object(test_dir, ext):
    if isfile(join(test_dir, f'no_{ext}')):
        return
    rougailconfig = RougailConfig.copy()
    rougailconfig['functions_file'] = join(dico_dirs, '../eosfunc/test.py')
    dirs = [join(test_dir, ext)]
    if ext == 'yml' and not isdir(dirs[0]):
        build_yml(test_dir, dirs[0])
    subfolder = join(test_dir, ext, 'subfolder')
    if isdir(subfolder):
        dirs.append(subfolder)
    rougailconfig['dictionaries_dir'] = dirs
    rougailconfig['extra_dictionaries'] = {}
    if isdir(join(test_dir, ext, 'extra_dirs')):
        extras = listdir(join(test_dir, ext, 'extra_dirs'))
        extras.sort()
        for extra in extras:
            subfolder = join(test_dir, ext, 'extra_dirs', extra)
            if isdir(subfolder):
                rougailconfig['extra_dictionaries'][extra] = [subfolder]
    return RougailConvert(rougailconfig)


def launch_flattener(eolobj, path_prefix=None):
    eolobj.load_dictionaries(path_prefix=path_prefix)


def save(test_dir, eolobj, multi=False):
    tiramisu_objects = eolobj.save(None)
    if 'children=[]' in tiramisu_objects.split('\n')[-2]:
        raise Exception('empty tiramisu object?')
    tiramisu_dir = join(test_dir, 'tiramisu')
    if isdir(tiramisu_dir):
        if not multi:
            filename = 'base.py'
        else:
            filename = 'multi.py'
        tiramisu_file = join(tiramisu_dir, filename)
        if not isfile(tiramisu_file) or debug:
            with open(tiramisu_file, 'w') as fh:
                fh.write(tiramisu_objects)
        with open(tiramisu_file, 'r') as fh:
            tiramisu_objects_ori = fh.read()
        assert tiramisu_objects == tiramisu_objects_ori


def test_dictionary(test_dir):
    for ext in ['xml', 'yml']:
        assert getcwd() == ORI_DIR
        test_dir_ = join(dico_dirs, test_dir)
        eolobj = load_rougail_object(test_dir_, ext)
        launch_flattener(eolobj)
        save(test_dir_, eolobj)
        assert getcwd() == ORI_DIR


def test_dictionary_multi(test_dir):
    assert getcwd() == ORI_DIR
    test_dir = join(dico_dirs, test_dir)
    eolobj = load_rougail_object(test_dir, 'xml')
    launch_flattener(eolobj, path_prefix='1')
    launch_flattener(eolobj, path_prefix='2')
    save(test_dir, eolobj, multi=True)
    assert getcwd() == ORI_DIR


def test_error_dictionary(test_dir_error):
    for ext in ['xml', 'yml']:
        assert getcwd() == ORI_DIR
        test_dir = join(dico_dirs, test_dir_error)
        errno = 0
        eolobj = load_rougail_object(test_dir, ext)
        if eolobj is None:
            continue
        for i in listdir(test_dir):
            if i.startswith('errno_'):
                if errno:
                    raise Exception('multiple errno')
                errno = int(i.split('_')[1])
        with raises(DictConsistencyError) as err:
            launch_flattener(eolobj)
            save(test_dir, eolobj)
        if err.value.errno != errno:
            print(f'expected errno: {errno}, errno: {err.value.errno}')
            launch_flattener(eolobj)
            save(test_dir, eolobj)
        assert getcwd() == ORI_DIR