load only folder now

This commit is contained in:
Emmanuel Garette 2020-12-24 16:02:20 +01:00
parent 02edd0eaf0
commit 9feb45e165
3 changed files with 18 additions and 50 deletions

View file

@ -19,6 +19,9 @@ The RougailObjSpace
The visit/annotation stage is a complex step that corresponds to the Rougail The visit/annotation stage is a complex step that corresponds to the Rougail
procedures. procedures.
""" """
from typing import List
from .i18n import _ from .i18n import _
from .xmlreflector import XMLReflector from .xmlreflector import XMLReflector
from .annotator import SpaceAnnotator from .annotator import SpaceAnnotator
@ -140,12 +143,13 @@ class RougailObjSpace:
return '"' + '", "'.join(xmlfiles[:-1]) + '"' + ' and ' + '"' + xmlfiles[-1] + '"' return '"' + '", "'.join(xmlfiles[:-1]) + '"' + ' and ' + '"' + xmlfiles[-1] + '"'
def create_or_populate_from_xml(self, def create_or_populate_from_xml(self,
namespace, namespace: str,
xmlfolders, xmlfolders: List[str],
): ) -> List[str]:
"""Parses a bunch of XML files and populates the RougailObjSpace """Parses a bunch of XML files and populates the RougailObjSpace
""" """
for xmlfile, document in self.xmlreflector.load_xml_from_folders(xmlfolders): for xmlfile in self.xmlreflector.load_xml_from_folders(xmlfolders):
document = self.xmlreflector.parse_xmlfile(xmlfile)
self.redefine_variables = [] self.redefine_variables = []
self.xml_parse_document(xmlfile, self.xml_parse_document(xmlfile,
document, document,

View file

@ -1,7 +1,7 @@
# coding: utf-8 # coding: utf-8
from os.path import join, isfile, basename, isdir from typing import List
from os.path import join, isfile
from os import listdir from os import listdir
#from io import BytesIO
from lxml.etree import DTD, parse, tostring, XMLSyntaxError from lxml.etree import DTD, parse, tostring, XMLSyntaxError
@ -42,51 +42,15 @@ class XMLReflector(object):
raise DictConsistencyError(_(f'"{xmlfile}" not a valid XML file: {self.dtd.error_log.filter_from_errors()[0]}'), 43) raise DictConsistencyError(_(f'"{xmlfile}" not a valid XML file: {self.dtd.error_log.filter_from_errors()[0]}'), 43)
return document.getroot() return document.getroot()
def load_xml_from_folders(self, xmlfolders): def load_xml_from_folders(self,
xmlfolders: List[str],
):
"""Loads all the XML files located in the xmlfolders' list """Loads all the XML files located in the xmlfolders' list
:param xmlfolders: list of full folder's name :param xmlfolders: list of full folder's name
""" """
documents = []
if not isinstance(xmlfolders, list):
xmlfolders = [xmlfolders]
for xmlfolder in xmlfolders: for xmlfolder in xmlfolders:
if isinstance(xmlfolder, list) or isinstance(xmlfolder, tuple): filenames = [join(xmlfolder, filename) for filename in listdir(xmlfolder) if filename.endswith('.xml')]
# directory group : collect files from each
# directory and sort them before loading
group_files = []
for idx, subdir in enumerate(xmlfolder):
if isdir(subdir):
for filename in listdir(subdir):
group_files.append((filename, idx, subdir))
else:
group_files.append(basename(subdir), idx, dirname(subdir))
def sort_group(file1, file2):
if file1[0] == file2[0]:
# sort by initial xmlfolder order if same name
return file1[1].__cmp__(file2[1])
# sort by filename
elif file1[0] > file2[0]:
return 1
else:
return -1
group_files.sort(sort_group)
filenames = [join(f[2], f[0]) for f in group_files]
elif isdir(xmlfolder):
filenames = []
for filename in listdir(xmlfolder):
filenames.append(join(xmlfolder, filename))
filenames.sort() filenames.sort()
else:
filenames = [xmlfolder]
for xmlfile in filenames: for xmlfile in filenames:
if xmlfile.endswith('.xml'): yield xmlfile
#xmlfile_path = join(xmlfolder, xmlfile)
documents.append((xmlfile, self.parse_xmlfile(xmlfile)))
return documents
def save_xmlfile(self, xmlfilename, xml): # pylint: disable=R0201
"""Write a bunch of XML on the disk
"""
with open(xmlfilename, 'w') as xmlfh:
xmlfh.write(tostring(xml, pretty_print=True, encoding="UTF-8", xml_declaration=True).decode('utf8'))

View file

@ -55,10 +55,10 @@ def test_dir_error(request):
def launch_flattener(test_dir, test_ok=False): def launch_flattener(test_dir, test_ok=False):
eolobj = objspace.RougailObjSpace(Config['dtdfilename']) eolobj = objspace.RougailObjSpace(Config['dtdfilename'])
dirs = test_dir dirs = [test_dir]
subfolder = join(test_dir, 'subfolder') subfolder = join(test_dir, 'subfolder')
if isdir(subfolder): if isdir(subfolder):
dirs = [dirs, subfolder] dirs.append(subfolder)
eolobj.create_or_populate_from_xml(Config['variable_namespace'], dirs) eolobj.create_or_populate_from_xml(Config['variable_namespace'], dirs)
subfolder = join(test_dir, 'extra_dirs', 'extra') subfolder = join(test_dir, 'extra_dirs', 'extra')
if isdir(subfolder): if isdir(subfolder):