rougail/tests/test_1_flattener.py

from os import getcwd
from os.path import isfile, join, isdir
from pytest import fixture, raises
from os import listdir, environ, makedirs
from json import load, dumps, loads
from yaml import dump as ydump

environ['TIRAMISU_LOCALE'] = 'en'

from rougail import RougailConvert, RougailConfig
from rougail.error import DictConsistencyError



dico_dirs = 'tests/dictionaries'


test_ok = set()
test_raise = set()

for test in listdir(dico_dirs):
    if isdir(join(dico_dirs, test)):
        if isdir(join(dico_dirs, test, 'tiramisu')):
            test_ok.add(test)
        elif test != '__pycache__':
            test_raise.add(test)

excludes = set([])
#excludes = set(['45multi_family_order'])
test_ok -= excludes
test_raise -= excludes
#test_ok = ['01base_multi_notuniq']
#test_ok = []
#test_raise = ['80auto_autofreeze']
#test_raise = []

ORI_DIR = getcwd()

debug = False
#debug = True

test_ok = list(test_ok)
test_raise = list(test_raise)
test_ok.sort()
test_raise.sort()

test_dirs = []
for ext in ['xml', 'yml']:
    for val in test_ok:
        test_dirs.append((ext, val))
@fixture(scope="module", params=test_dirs)
def test_dir(request):
    return request.param


test_err_dirs = []
for ext in ['xml', 'yml']:
    for val in test_raise:
        test_err_dirs.append((ext, val))
@fixture(scope="module", params=test_err_dirs)
def test_dir_error(request):
    return request.param


def xml_to_yaml(filename, destfilename):
    import xmltodict
    def reorganize_variable(value, level):
        move = None
        new_value = []
        for type_ in ['variable', 'family']:
            if type_ in value:
                subvalue = value[type_]
                if isinstance(subvalue, list):
                    values = []
                    for val in subvalue:
                        if 'variable' in val or 'family' in val:
                            move_, val = reorganize_variable(val, level + 1)
                            if move_:
                                move = move_
                        if filename == 'tests/dictionaries/45multi_family_order/xml/00-base.xml' and \
                                val['@name'] == 'variable4':
                            #hack
                            move = val
                        else:
                            values.append(val)
                    if values:
                        new_value.append({type_: values})
                else:
                    if 'variable' in subvalue or 'family' in subvalue:
                        move_, subvalue = reorganize_variable(subvalue, level + 1)
                        if move_:
                            move = move_
                    new_value.append({type_: [subvalue]})
        if level == 0:
            return new_value
        if level == 1 and  move:
            new_value.append({'variable': [move]})
        dico = {k: v for k, v in value.items() if k not in ['variable', 'family']}
        dico['variables'] = new_value
        return move, dico

    def _xml_to_yaml(dico, root=False):
        for key, value in dict(dico).items():
            del dico[key]
            if root and key == 'variables':
                value = reorganize_variable(value, 0)
            if isinstance(value, dict):
                _xml_to_yaml(value)
            if isinstance(value, list):
                dico[key] = value
                for idx, val in enumerate(value):
                    if isinstance(val, dict):
                        _xml_to_yaml(val)
                    elif isinstance(val, (str, bool)):
                        dico[key][idx] = {'text': val}
            else:
                if value == 'True':
                    value = True
                elif value == 'False':
                    value = False
                else:
                    converted = False
                    try:
                        # if it's a number
                        nvalue = int(value)
                        assert str(nvalue) == value
                        value = nvalue
                        converted = True
                    except:
                        pass
                    if not converted:
                        try:
                            # if it's a number
                            nvalue = float(value)
                            assert str(nvalue) == value
                            value = nvalue
                            converted = True
                        except:
                            pass
                if key.startswith('@'):
                    # remove @
                    dico[key[1:]] = value
                elif key.startswith('#'):
                    dico[key[1:]] = value
                else:
                    if key not in ['variable', 'family'] and not isinstance(value, list):
                        if not value:
                            if key == 'override':
                                dico[key] = value
                            elif key == 'value':
                                dico[key] = [{'text': value}]
                        elif isinstance(value, (str, bool, int, float)):
                            dico[key] = [{'text': value}]
                        else:
                            dico[key] = [value]
                    else:
                        dico[key] = value
    with open(filename, 'rb') as fh:
        dico = loads(dumps(xmltodict.parse(fh)['rougail']))  # , attr_prefix="")
    _xml_to_yaml(dico, root=True)
    with open(destfilename, 'w') as fh:
        ydump(dico, fh, sort_keys=False)


def build_yml(test_dir, dirname):
    makedirs(dirname)
    for filename in listdir(join(test_dir, 'xml')):
        if not isfile(join(test_dir, 'xml', filename)):
            continue
        xml_to_yaml(join(test_dir, 'xml', filename), join(dirname, filename[:-4] + '.yml'))
    if isdir(join(test_dir, 'xml', 'subfolder')):
        makedirs(join(dirname, 'subfolder'))
        for filename in listdir(join(test_dir, 'xml', 'subfolder')):
            xml_to_yaml(join(test_dir, 'xml', 'subfolder', filename), join(dirname, 'subfolder', filename[:-4] + '.yml'))
    if isdir(join(test_dir, 'xml', 'extra_dirs')):
        makedirs(join(dirname, 'extra_dirs'))
        for extra in listdir(join(test_dir, 'xml', 'extra_dirs')):
            subfolder = join(test_dir, 'xml', 'extra_dirs', extra)
            if not isdir(subfolder):
                continue
            makedirs(join(dirname, 'extra_dirs', extra))
            for filename in listdir(subfolder):
                xml_to_yaml(join(test_dir, 'xml', 'extra_dirs', extra, filename), join(dirname, 'extra_dirs', extra, filename[:-4] + '.yml'))


def load_rougail_object(test_dir, ext):
    if isfile(join(test_dir, f'no_{ext}')):
        return
    rougailconfig = RougailConfig.copy()
    rougailconfig['functions_file'] = join(dico_dirs, '../eosfunc/test.py')
    dirs = [join(test_dir, ext)]
    if ext == 'yml' and not isdir(dirs[0]):
        build_yml(test_dir, dirs[0])
    subfolder = join(test_dir, ext, 'subfolder')
    if isdir(subfolder):
        dirs.append(subfolder)
    rougailconfig['dictionaries_dir'] = dirs
    rougailconfig['extra_dictionaries'] = {}
    if isdir(join(test_dir, ext, 'extra_dirs')):
        extras = listdir(join(test_dir, ext, 'extra_dirs'))
        extras.sort()
        for extra in extras:
            subfolder = join(test_dir, ext, 'extra_dirs', extra)
            if isdir(subfolder):
                rougailconfig['extra_dictionaries'][extra] = [subfolder]
    return RougailConvert(rougailconfig)


def launch_flattener(eolobj, path_prefix=None):
    eolobj.load_dictionaries(path_prefix=path_prefix)


def save(test_dir, eolobj, multi=False):
    tiramisu_objects = eolobj.save(None)
    if 'children=[]' in tiramisu_objects.split('\n')[-2]:
        raise Exception('empty tiramisu object?')
    tiramisu_dir = join(test_dir, 'tiramisu')
    if isdir(tiramisu_dir):
        if not multi:
            filename = 'base.py'
        else:
            filename = 'multi.py'
        tiramisu_file = join(tiramisu_dir, filename)
        if not isfile(tiramisu_file) or debug:
            with open(tiramisu_file, 'w') as fh:
                fh.write(tiramisu_objects)
        with open(tiramisu_file, 'r') as fh:
            tiramisu_objects_ori = fh.read()
        assert tiramisu_objects == tiramisu_objects_ori


def test_dictionary(test_dir):
    ext, test_dir_ = test_dir
    assert getcwd() == ORI_DIR
    test_dir_ = join(dico_dirs, test_dir_)
    eolobj = load_rougail_object(test_dir_, ext)
    if not eolobj:
        return
    launch_flattener(eolobj)
    save(test_dir_, eolobj)
    assert getcwd() == ORI_DIR


def test_dictionary_multi(test_dir):
    ext, test_dir_ = test_dir
    assert getcwd() == ORI_DIR
    test_dir_ = join(dico_dirs, test_dir_)
    eolobj = load_rougail_object(test_dir_, ext)
    if not eolobj:
        return
    launch_flattener(eolobj, path_prefix='1')
    launch_flattener(eolobj, path_prefix='2')
    save(test_dir_, eolobj, multi=True)
    assert getcwd() == ORI_DIR


def test_error_dictionary(test_dir_error):
    ext, test_dir = test_dir_error
    assert getcwd() == ORI_DIR
    test_dir = join(dico_dirs, test_dir)
    errno = 0
    eolobj = load_rougail_object(test_dir, ext)
    if eolobj is None:
        return
    for i in listdir(test_dir):
        if i.startswith('errno_'):
            if errno:
                raise Exception('multiple errno')
            errno = int(i.split('_')[1])
    with raises(DictConsistencyError) as err:
        launch_flattener(eolobj)
        save(test_dir, eolobj)
    if err.value.errno != errno:
        print(f'expected errno: {errno}, errno: {err.value.errno}')
        launch_flattener(eolobj)
        save(test_dir, eolobj)
    assert getcwd() == ORI_DIR