from os import getcwd from os.path import isfile, join, isdir from pytest import fixture, raises from os import listdir, environ, makedirs from json import load, dumps, loads from yaml import dump as ydump environ['TIRAMISU_LOCALE'] = 'en' from rougail import RougailConvert, RougailConfig from rougail.error import DictConsistencyError dico_dirs = 'tests/dictionaries' test_ok = set() test_raise = set() for test in listdir(dico_dirs): if isdir(join(dico_dirs, test)): if isdir(join(dico_dirs, test, 'tiramisu')): test_ok.add(test) elif test != '__pycache__': test_raise.add(test) excludes = set([]) #excludes = set(['45multi_family_order']) test_ok -= excludes test_raise -= excludes #test_ok = ['01base_multi_notuniq'] #test_ok = [] #test_raise = ['80auto_autofreeze'] #test_raise = [] ORI_DIR = getcwd() debug = False #debug = True test_ok = list(test_ok) test_raise = list(test_raise) test_ok.sort() test_raise.sort() test_dirs = [] for ext in ['xml', 'yml']: for val in test_ok: test_dirs.append((ext, val)) @fixture(scope="module", params=test_dirs) def test_dir(request): return request.param test_err_dirs = [] for ext in ['xml', 'yml']: for val in test_raise: test_err_dirs.append((ext, val)) @fixture(scope="module", params=test_err_dirs) def test_dir_error(request): return request.param def xml_to_yaml(filename, destfilename): import xmltodict def reorganize_variable(value, level): move = None new_value = [] for type_ in ['variable', 'family']: if type_ in value: subvalue = value[type_] if isinstance(subvalue, list): values = [] for val in subvalue: if 'variable' in val or 'family' in val: move_, val = reorganize_variable(val, level + 1) if move_: move = move_ if filename == 'tests/dictionaries/45multi_family_order/xml/00-base.xml' and \ val['@name'] == 'variable4': #hack move = val else: values.append(val) if values: new_value.append({type_: values}) else: if 'variable' in subvalue or 'family' in subvalue: move_, subvalue = reorganize_variable(subvalue, level + 1) if move_: move = move_ new_value.append({type_: [subvalue]}) if level == 0: return new_value if level == 1 and move: new_value.append({'variable': [move]}) dico = {k: v for k, v in value.items() if k not in ['variable', 'family']} dico['variables'] = new_value return move, dico def _xml_to_yaml(dico, root=False): for key, value in dict(dico).items(): del dico[key] if root and key == 'variables': value = reorganize_variable(value, 0) if isinstance(value, dict): _xml_to_yaml(value) if isinstance(value, list): dico[key] = value for idx, val in enumerate(value): if isinstance(val, dict): _xml_to_yaml(val) elif isinstance(val, (str, bool)): dico[key][idx] = {'text': val} else: if value == 'True': value = True elif value == 'False': value = False else: converted = False try: # if it's a number nvalue = int(value) assert str(nvalue) == value value = nvalue converted = True except: pass if not converted: try: # if it's a number nvalue = float(value) assert str(nvalue) == value value = nvalue converted = True except: pass if key.startswith('@'): # remove @ dico[key[1:]] = value elif key.startswith('#'): dico[key[1:]] = value else: if key not in ['variable', 'family'] and not isinstance(value, list): if not value: if key == 'override': dico[key] = value elif key == 'value': dico[key] = [{'text': value}] elif isinstance(value, (str, bool, int, float)): dico[key] = [{'text': value}] else: dico[key] = [value] else: dico[key] = value with open(filename, 'rb') as fh: dico = loads(dumps(xmltodict.parse(fh)['rougail'])) # , attr_prefix="") _xml_to_yaml(dico, root=True) with open(destfilename, 'w') as fh: ydump(dico, fh, sort_keys=False) def build_yml(test_dir, dirname): makedirs(dirname) for filename in listdir(join(test_dir, 'xml')): if not isfile(join(test_dir, 'xml', filename)): continue xml_to_yaml(join(test_dir, 'xml', filename), join(dirname, filename[:-4] + '.yml')) if isdir(join(test_dir, 'xml', 'subfolder')): makedirs(join(dirname, 'subfolder')) for filename in listdir(join(test_dir, 'xml', 'subfolder')): xml_to_yaml(join(test_dir, 'xml', 'subfolder', filename), join(dirname, 'subfolder', filename[:-4] + '.yml')) if isdir(join(test_dir, 'xml', 'extra_dirs')): makedirs(join(dirname, 'extra_dirs')) for extra in listdir(join(test_dir, 'xml', 'extra_dirs')): subfolder = join(test_dir, 'xml', 'extra_dirs', extra) if not isdir(subfolder): continue makedirs(join(dirname, 'extra_dirs', extra)) for filename in listdir(subfolder): xml_to_yaml(join(test_dir, 'xml', 'extra_dirs', extra, filename), join(dirname, 'extra_dirs', extra, filename[:-4] + '.yml')) def load_rougail_object(test_dir, ext): if isfile(join(test_dir, f'no_{ext}')): return rougailconfig = RougailConfig.copy() rougailconfig['functions_file'] = join(dico_dirs, '../eosfunc/test.py') dirs = [join(test_dir, ext)] if ext == 'yml' and not isdir(dirs[0]): build_yml(test_dir, dirs[0]) subfolder = join(test_dir, ext, 'subfolder') if isdir(subfolder): dirs.append(subfolder) rougailconfig['dictionaries_dir'] = dirs rougailconfig['extra_dictionaries'] = {} if isdir(join(test_dir, ext, 'extra_dirs')): extras = listdir(join(test_dir, ext, 'extra_dirs')) extras.sort() for extra in extras: subfolder = join(test_dir, ext, 'extra_dirs', extra) if isdir(subfolder): rougailconfig['extra_dictionaries'][extra] = [subfolder] return RougailConvert(rougailconfig) def launch_flattener(eolobj, path_prefix=None): eolobj.load_dictionaries(path_prefix=path_prefix) def save(test_dir, eolobj, multi=False): tiramisu_objects = eolobj.save(None) if 'children=[]' in tiramisu_objects.split('\n')[-2]: raise Exception('empty tiramisu object?') tiramisu_dir = join(test_dir, 'tiramisu') if isdir(tiramisu_dir): if not multi: filename = 'base.py' else: filename = 'multi.py' tiramisu_file = join(tiramisu_dir, filename) if not isfile(tiramisu_file) or debug: with open(tiramisu_file, 'w') as fh: fh.write(tiramisu_objects) with open(tiramisu_file, 'r') as fh: tiramisu_objects_ori = fh.read() assert tiramisu_objects == tiramisu_objects_ori def test_dictionary(test_dir): ext, test_dir_ = test_dir assert getcwd() == ORI_DIR test_dir_ = join(dico_dirs, test_dir_) eolobj = load_rougail_object(test_dir_, ext) if not eolobj: return launch_flattener(eolobj) save(test_dir_, eolobj) assert getcwd() == ORI_DIR def test_dictionary_multi(test_dir): ext, test_dir_ = test_dir assert getcwd() == ORI_DIR test_dir_ = join(dico_dirs, test_dir_) eolobj = load_rougail_object(test_dir_, ext) if not eolobj: return launch_flattener(eolobj, path_prefix='1') launch_flattener(eolobj, path_prefix='2') save(test_dir_, eolobj, multi=True) assert getcwd() == ORI_DIR def test_error_dictionary(test_dir_error): ext, test_dir = test_dir_error assert getcwd() == ORI_DIR test_dir = join(dico_dirs, test_dir) errno = 0 eolobj = load_rougail_object(test_dir, ext) if eolobj is None: return for i in listdir(test_dir): if i.startswith('errno_'): if errno: raise Exception('multiple errno') errno = int(i.split('_')[1]) with raises(DictConsistencyError) as err: launch_flattener(eolobj) save(test_dir, eolobj) if err.value.errno != errno: print(f'expected errno: {errno}, errno: {err.value.errno}') launch_flattener(eolobj) save(test_dir, eolobj) assert getcwd() == ORI_DIR