From 30a605a81cf85c752fff66ad60acb2302ca09051 Mon Sep 17 00:00:00 2001 From: Emmanuel Garette Date: Sat, 1 Oct 2022 22:33:11 +0200 Subject: [PATCH] convert to ansible --- ansible/action_plugins/build_images.py | 48 ++++++ ansible/action_plugins/machinectl.py | 14 ++ ansible/action_plugins/rougail.py | 55 +++++++ ansible/file.txt | 1 + ansible/filter_plugins/fileslist.py | 114 ++++++++++++++ ansible/host.yml | 143 +++++++++++++++++ ansible/installations | 1 + ansible/inventory.json | 15 ++ ansible/inventory.py | 114 ++++++++++++++ ansible/library/machinectl.py | 205 +++++++++++++++++++++++++ ansible/machine.yml | 93 +++++++++++ ansible/machines.yml | 33 ++++ ansible/password | 1 + ansible/pki | 1 + ansible/playbook.txt | 23 +++ ansible/playbook.yml | 28 ++++ bootstrap.py | 15 +- sbin/build_image | 196 +++++++++++++++++++++++ sbin/make_changelog | 181 ++++++++++++++++++++++ sbin/make_volatile | 77 ++++++++++ sbin/update_images | 79 ++++++++++ src/risotto/image.py | 61 +++++--- src/risotto/machine.py | 98 +++++++----- 23 files changed, 1521 insertions(+), 75 deletions(-) create mode 100644 ansible/action_plugins/build_images.py create mode 100644 ansible/action_plugins/machinectl.py create mode 100644 ansible/action_plugins/rougail.py create mode 100644 ansible/file.txt create mode 100644 ansible/filter_plugins/fileslist.py create mode 100644 ansible/host.yml create mode 120000 ansible/installations create mode 100644 ansible/inventory.json create mode 100755 ansible/inventory.py create mode 100644 ansible/library/machinectl.py create mode 100644 ansible/machine.yml create mode 100644 ansible/machines.yml create mode 120000 ansible/password create mode 120000 ansible/pki create mode 100644 ansible/playbook.txt create mode 100644 ansible/playbook.yml create mode 100755 sbin/build_image create mode 100755 sbin/make_changelog create mode 100755 sbin/make_volatile create mode 100644 sbin/update_images diff --git a/ansible/action_plugins/build_images.py b/ansible/action_plugins/build_images.py new file mode 100644 index 0000000..81f263b --- /dev/null +++ b/ansible/action_plugins/build_images.py @@ -0,0 +1,48 @@ +#!/usr/bin/python3 + +from os import listdir, makedirs +from os.path import isfile, isdir, join, dirname +from shutil import copy2, copytree, rmtree + +from ansible.plugins.action import ActionBase +from risotto.utils import RISOTTO_CONFIG + + +class ActionModule(ActionBase): + def run(self, tmp=None, task_vars=None): + super(ActionModule, self).run(tmp, task_vars) + module_args = self._task.args.copy() + modules = module_args['modules'] + dataset_directory = RISOTTO_CONFIG['directories']['dataset'] + install_dir = join('/tmp/risotto/images') + if isdir(install_dir): + rmtree(install_dir) + for module_name, depends in modules.items(): + for depend in depends: + manual = join(dataset_directory, depend, 'manual', 'image') + if not isdir(manual): + continue + for filename in listdir(manual): + src_file = join(manual, filename) + dst_file = join(install_dir, module_name, filename) + if isdir(src_file): + if not isdir(dst_file): + makedirs(dst_file) + for subfilename in listdir(src_file): + if not isfile(dst_file): + src = join(src_file, subfilename) + dst = join(dst_file, subfilename) + if isfile(src): + copy2(src, dst) + else: + copytree(src, dst) + elif not isfile(dst_file): + dst = dirname(dst_file) + if not isdir(dst): + makedirs(dst) + if isfile(src_file): + copy2(src_file, dst_file) + else: + copytree(src_file, dst_file) + return dict(ansible_facts=dict({})) + diff --git a/ansible/action_plugins/machinectl.py b/ansible/action_plugins/machinectl.py new file mode 100644 index 0000000..c7ad2a9 --- /dev/null +++ b/ansible/action_plugins/machinectl.py @@ -0,0 +1,14 @@ +#!/usr/bin/python3 +from ansible.plugins.action import ActionBase + + +class ActionModule(ActionBase): + def run(self, tmp=None, task_vars=None): + super(ActionModule, self).run(tmp, task_vars) + module_args = self._task.args.copy() + module_return = self._execute_module(module_name='machinectl', + module_args=module_args, + task_vars=task_vars, tmp=tmp) + if module_return.get('failed'): + return module_return + return {'ansible_facts': {}, 'changed': module_return['changed']} diff --git a/ansible/action_plugins/rougail.py b/ansible/action_plugins/rougail.py new file mode 100644 index 0000000..cd6a1af --- /dev/null +++ b/ansible/action_plugins/rougail.py @@ -0,0 +1,55 @@ +#!/usr/bin/python3 +from ansible.plugins.action import ActionBase +from asyncio import run +from shutil import rmtree +from os.path import isdir, join +from os import makedirs + +from risotto.machine import templates, load, ROUGAIL_NAMESPACE +from risotto.utils import RISOTTO_CONFIG +from rougail.utils import normalize_family + + +TIRAMISU_CACHE = 'tiramisu_cache.py' +VALUES_CACHE = 'values_cache.py' +INSTALL_DIR = RISOTTO_CONFIG['directories']['dest'] + + +async def build_files(server_name, is_host): + module_infos, rougailconfig, config = await load(TIRAMISU_CACHE, + VALUES_CACHE, + ) + subconfig = config.option(normalize_family(server_name)) + module_name = await subconfig.option(await subconfig.information.get('provider:global:module_name')).value.get() + module_info = module_infos[module_name] + rougailconfig['tmp_dir'] = 'tmp' + rougailconfig['destinations_dir'] = INSTALL_DIR + rougailconfig['templates_dir'] = module_info['infos'].templates_dir + if is_host: + tmpfile = await subconfig.option(f'{ROUGAIL_NAMESPACE}.host_install_dir').value.get() + rougailconfig['tmpfile_dest_dir'] = f'{tmpfile}' + rougailconfig['default_systemd_directory'] = '/usr/local/lib/systemd' + else: + rougailconfig['tmpfile_dest_dir'] = '/usr/local/lib' + rougailconfig['default_systemd_directory'] = '/systemd' + if isdir(rougailconfig['destinations_dir']): + rmtree(rougailconfig['destinations_dir']) + if isdir(rougailconfig['tmp_dir']): + rmtree(rougailconfig['tmp_dir']) + makedirs(rougailconfig['tmp_dir']) + makedirs(rougailconfig['destinations_dir']) + await templates(server_name, + subconfig, + rougailconfig, + ) + + +class ActionModule(ActionBase): + def run(self, tmp=None, task_vars=None): + super(ActionModule, self).run(tmp, task_vars) + module_args = self._task.args.copy() + name = module_args['hostname'] + is_host = module_args['is_host'] + + run(build_files(name, is_host)) + return dict(ansible_facts=dict({})) diff --git a/ansible/file.txt b/ansible/file.txt new file mode 100644 index 0000000..8714907 --- /dev/null +++ b/ansible/file.txt @@ -0,0 +1 @@ +{'pouet': 'a'} diff --git a/ansible/filter_plugins/fileslist.py b/ansible/filter_plugins/fileslist.py new file mode 100644 index 0000000..d593b70 --- /dev/null +++ b/ansible/filter_plugins/fileslist.py @@ -0,0 +1,114 @@ +#!/usr/bin/python3 + + +from os.path import dirname + + +def _add(files, file_data, name, name_only, prefix): + if prefix is not None: + name = prefix + name + if name_only: + files.append(name) + else: + files.append({'name': name, + 'owner': file_data['owner'], + 'group': file_data['group'], + 'mode': file_data['mode'], + }) + + +def fileslist(data, is_host=False, name_only=False, prefix=None): + files = [] + if is_host: + base_systemd = '/usr/local/lib' + else: + base_systemd = '' + _add(files, + {'owner': 'root', 'group': 'root', 'mode': '0755'}, + f'/tmpfiles.d/0rougail.conf', + name_only, + prefix, + ) + for service, service_data in data.items(): + if service_data['activate'] and service_data['engine'] != 'none': + _add(files, + {'owner': 'root', 'group': 'root', 'mode': '0755'}, + base_systemd + '/systemd/system/' + service_data['doc'], + name_only, + prefix, + ) + if service_data['activate'] and 'overrides' in service_data: + for override_data in service_data['overrides'].values(): + _add(files, + {'owner': 'root', 'group': 'root', 'mode': '0755'}, + base_systemd + '/systemd/system/' + override_data['name'] + '.d/rougail.conf', + name_only, + prefix, + ) + if 'files' not in service_data: + continue + for file_data in service_data['files'].values(): + if not file_data['activate'] or file_data['included'] == 'content': + continue + if isinstance(file_data['name'], list): + for name in file_data['name']: + _add(files, file_data, name, name_only, prefix) + else: + _add(files, file_data, file_data['name'], name_only, prefix) + + return files + + +def directorieslist(data): + directories = {'/usr/local/lib/systemd/system/'} + for service, service_data in data.items(): + if 'files' not in service_data: + continue + for file_data in service_data['files'].values(): + if not file_data['activate']: + continue + if isinstance(file_data['name'], list): + for name in file_data['name']: + directories.add(dirname(name)) + else: + directories.add(dirname(file_data['name'])) + + return list(directories) + + +def machineslist(data, only=None, only_name=False): + srv = [] + if only is not None: + if only_name: + srv.append(only) + else: + srv.append({'name': only, + 'srv': data[only]['machine']['add_srv'], + } + ) + else: + for host, host_data in data.items(): + if '.' not in host or not isinstance(host_data, dict) or 'general' not in host_data or host_data['general']['module_name'] == 'host': + continue + if only_name: + srv.append(host) + else: + srv.append({'name': host, + 'srv': host_data['machine']['add_srv'], + } + ) + return srv + + +def modulename(data, servername): + return data[servername]['module_name'] + + +class FilterModule: + def filters(self): + return { + 'fileslist': fileslist, + 'directorieslist': directorieslist, + 'machineslist': machineslist, + 'modulename': modulename, + } diff --git a/ansible/host.yml b/ansible/host.yml new file mode 100644 index 0000000..72fb652 --- /dev/null +++ b/ansible/host.yml @@ -0,0 +1,143 @@ +--- +- name: "Populate service facts" + service_facts: + +- name: "Stop services" + when: item.value['manage'] and item.value['activate'] and item.value['doc'].endswith('.service') and not item.value['doc'].endswith('@.service') and item.value['engine'] != 'none' + ansible.builtin.service: + name: "{{ item.value['doc'] }}" + state: stopped + loop: "{{ vars[inventory_hostname]['services'] | dict2items }}" + loop_control: + label: "{{ item.value['doc'] }}" + +- name: "Packages installation" + apt: + pkg: "{{ vars[inventory_hostname]['general']['host_packages'] }}" + update_cache: yes + state: latest + +- name: "Build host files" + local_action: + module: rougail + hostname: "{{ inventory_hostname }}" + is_host: True + +- name: "Create host directories" + file: path={{ item }} state=directory mode=0755 + loop: "{{ vars[inventory_hostname]['services'] | directorieslist }}" + +- name: "Copy systemd-tmpfiles" + when: item.name.startswith('/usr/local/lib/risotto-tmpfiles.d') + ansible.builtin.copy: + src: installations/{{ item.name }} + dest: "{{ item.name }}" + owner: "{{ item.owner }}" + group: "{{ item.group }}" + mode: "{{ item.mode }}" + loop: "{{ vars[inventory_hostname]['services'] | fileslist(is_host=True) }}" + loop_control: + label: "{{ item.name}}" + +- name: "Execute systemd-tmpfiles" + when: item.name.startswith('/usr/local/lib/risotto-tmpfiles.d') + command: /usr/bin/systemd-tmpfiles --create --clean --remove {{ item.name }} + loop: "{{ vars[inventory_hostname]['services'] | fileslist(is_host=True) }}" + loop_control: + label: "{{ item.name}}" + +- name: "Copy host files" + when: not item.name.startswith('/usr/local/lib/tmpfiles.d') + ansible.builtin.copy: + src: installations/{{ item.name }} + dest: "{{ item.name }}" + owner: "{{ item.owner }}" + group: "{{ item.group }}" + mode: "{{ item.mode }}" + loop: "{{ vars[inventory_hostname]['services'] | fileslist(is_host=True) }}" + loop_control: + label: "{{ item.name}}" + +- name: "Reload systemd services configuration" + ansible.builtin.systemd: + daemon_reload: yes + +- name: "Enable services" + when: item.value['manage'] and item.value['activate'] and '@.service' not in item.value['doc'] + ansible.builtin.service: + name: "{{ item.value['doc'] }}" + enabled: yes + loop: "{{ vars[inventory_hostname]['services'] | dict2items }}" + loop_control: + label: "{{ item.value['doc'] }}" + +- name: "Disable services" + when: item.value['manage'] and not item.value['activate'] and not item.value['undisable'] and '@.service' not in item.value['doc'] + ansible.builtin.service: + name: "{{ item.value['doc'] }}" + enabled: yes + loop: "{{ vars[inventory_hostname]['services'] | dict2items }}" + loop_control: + label: "{{ item.value['doc'] }}" + +- name: "Start services" + when: item.value['manage'] and item.value['activate'] and item.value['doc'].endswith('.service') and not item.value['doc'].endswith('@.service') and item.value['engine'] != 'none' + ansible.builtin.service: + name: "{{ item.value['doc'] }}" + state: started + loop: "{{ vars[inventory_hostname]['services'] | dict2items }}" + loop_control: + label: "{{ item.value['doc'] }}" + +- name: "Restart services" + when: item.value['manage'] and item.value['activate'] and item.value['doc'].endswith('.service') and not item.value['doc'].endswith('@.service') and item.value['engine'] == 'none' + ansible.builtin.service: + name: "{{ item.value['doc'] }}" + state: restarted + loop: "{{ vars[inventory_hostname]['services'] | dict2items }}" + loop_control: + label: "{{ item.value['doc'] }}" + +- name: "Copy machines scripts" + ansible.builtin.copy: + src: "{{ item }}" + dest: "/usr/local/sbin" + owner: "root" + group: "root" + mode: "0755" + loop: "{{ lookup('fileglob', '../sbin/*', wantlist=True) | list }}" + +# Images informations +- name: "Remove images tar" + local_action: + module: file + path: /tmp/risotto/images.tar + state: absent + +- name: "Build images files" + local_action: + module: build_images + modules: "{{ vars['modules'] }}" + +- name: "Compress images files" + local_action: + module: archive + path: "/tmp/risotto/images/" + dest: /tmp/risotto/images.tar + format: tar + +- name: "Remove dest images files" + file: + path: /var/lib/risotto/images_files + state: absent + +- name: "Create images files" + file: + path: /var/lib/risotto/images_files + state: directory + mode: "0700" + +- name: "Copy images files" + unarchive: + src: "/tmp/risotto/images.tar" + dest: "/var/lib/risotto/images_files" diff --git a/ansible/installations b/ansible/installations new file mode 120000 index 0000000..00e5d0b --- /dev/null +++ b/ansible/installations @@ -0,0 +1 @@ +/home/gnunux/git/risotto/risotto/installations/ \ No newline at end of file diff --git a/ansible/inventory.json b/ansible/inventory.json new file mode 100644 index 0000000..7c658c9 --- /dev/null +++ b/ansible/inventory.json @@ -0,0 +1,15 @@ +{ + "_meta": { + "hostvars": {} + }, + "all": { + "children": [ + "ungrouped" + ] + }, + "ungrouped": { + "hosts": [ + "cloud.silique.fr" + ] + } +} diff --git a/ansible/inventory.py b/ansible/inventory.py new file mode 100755 index 0000000..588a6c5 --- /dev/null +++ b/ansible/inventory.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python + +''' +Example custom dynamic inventory script for Ansible, in Python. +''' + +from argparse import ArgumentParser +from json import dumps, JSONEncoder +from os import remove +from os.path import isfile +from asyncio import run + +from risotto.machine import load +from risotto.image import load_config +from risotto.utils import SERVERS +from tiramisu.error import PropertiesOptionError +from rougail.utils import normalize_family +from rougail import RougailSystemdTemplate +from rougail.template.base import RougailLeader, RougailExtra + +TIRAMISU_CACHE = 'tiramisu_cache.py' +VALUES_CACHE = 'values_cache.py' + + +class RougailEncoder(JSONEncoder): + def default(self, obj): + if isinstance(obj, RougailLeader): + return obj._follower + if isinstance(obj, RougailExtra): + return obj._suboption + if isinstance(obj, PropertiesOptionError): + return 'PropertiesOptionError' + return JSONEncoder.default(self, obj) + + +class RisottoInventory(object): + def __init__(self): + parser = ArgumentParser() + parser.add_argument('--list', action='store_true') + parser.add_argument('--host', action='store') + self.args = parser.parse_args() + + async def run(self): + if self.args.list: + if isfile(TIRAMISU_CACHE): + remove(TIRAMISU_CACHE) + if isfile(VALUES_CACHE): + remove(VALUES_CACHE) + return await self.do_inventory() + elif self.args.host: + return await self.get_vars(self.args.host) + raise Exception('pfff') + + async def do_inventory(self): + module_infos = load_config(True, + True, + True, + ) + servers = [] + for server_name, server in SERVERS.items(): + module_name = server['module'] + if module_name != 'host': + continue + servers.append(server_name) + return dumps({ + 'group': { + 'hosts': servers, + 'vars': { + # FIXME + 'ansible_ssh_host': '192.168.56.156', + 'ansible_ssh_user': 'root', + 'ansible_python_interpreter': '/usr/bin/python3' + } + } + }) + + async def get_vars(self, + host_name: str, + ) -> dict: + try: + module_infos, rougailconfig, config = await load(TIRAMISU_CACHE, + VALUES_CACHE, + ) + except Exception as err: + # import traceback + # traceback.print_exc() + print(err) + exit(1) + ret = {} + modules = set() + for server_name, server in SERVERS.items(): + if server['module'] == 'host' and server_name != host_name: + continue + modules.add(server['module']) + subconfig = config.option(normalize_family(server_name)) + engine = RougailSystemdTemplate(subconfig, rougailconfig) + await engine.load_variables() + if server['module'] != 'host' and engine.rougail_variables_dict['general']['host'] != host_name: + continue + ret[server_name] = engine.rougail_variables_dict + ret['modules'] = {module_name: module_info['infos'].depends for module_name, module_info in module_infos.items() if module_name in modules} + ret['configure_host'] = True + ret['only_machine'] = None + return dumps(ret, cls=RougailEncoder) + + +# Get the inventory. +async def main(): + inv = RisottoInventory() + values = await inv.run() + print(values) + + +run(main()) diff --git a/ansible/library/machinectl.py b/ansible/library/machinectl.py new file mode 100644 index 0000000..4bcea61 --- /dev/null +++ b/ansible/library/machinectl.py @@ -0,0 +1,205 @@ +#!/usr/bin/python3 + +from time import sleep +from os import fdopen +from dbus import SystemBus, Array +from dbus.exceptions import DBusException + +from ansible.module_utils.basic import AnsibleModule + + + +def stop(bus, machines): + changed = False + remote_object = bus.get_object('org.freedesktop.machine1', + '/org/freedesktop/machine1', + False, + ) + res = remote_object.ListMachines(dbus_interface='org.freedesktop.machine1.Manager') + started_machines = [str(r[0]) for r in res if str(r[0]) != '.host'] + for host in machines: + if host not in started_machines: + continue + changed = True + remote_object.TerminateMachine(host, dbus_interface='org.freedesktop.machine1.Manager') + idx = 0 + errors = [] + while True: + res = remote_object.ListMachines(dbus_interface='org.freedesktop.machine1.Manager') + started_machines = [str(r[0]) for r in res if str(r[0]) != '.host'] + for host in machines: + if host in started_machines: + break + else: + break + sleep(1) + idx += 1 + if idx == 120: + errors.append('Cannot not stopped: ' + ','.join(started_machines)) + break + return changed, errors + + +def start(bus, machines): + changed = False + remote_object = bus.get_object('org.freedesktop.machine1', + '/org/freedesktop/machine1', + False, + ) + res = remote_object.ListMachines(dbus_interface='org.freedesktop.machine1.Manager') + started_machines = [str(r[0]) for r in res if str(r[0]) != '.host'] + remote_object_system = bus.get_object('org.freedesktop.systemd1', + '/org/freedesktop/systemd1', + False, + ) + for host in machines: + if host in started_machines: + continue + changed = True + service = f'systemd-nspawn@{host}.service' + remote_object_system.StartUnit(service, 'fail', dbus_interface='org.freedesktop.systemd1.Manager') + errors = [] + idx = 0 + while True: + res = remote_object.ListMachines(dbus_interface='org.freedesktop.machine1.Manager') + started_machines = [str(r[0]) for r in res if str(r[0]) != '.host'] + for host in machines: + if host not in started_machines: + break + else: + break + sleep(1) + idx += 1 + if idx == 120: + hosts = set(machines) - set(started_machines) + errors.append('Cannot not start: ' + ','.join(hosts)) + break + if not errors: + idx = 0 + for host in machines: + cmd = ['/usr/bin/systemctl', 'is-system-running'] + error = False + while True: + try: + res = remote_object.OpenMachineShell(host, + '', + cmd[0], + Array(cmd, signature='s'), + Array(['TERM=dumb'], signature='s'), + dbus_interface='org.freedesktop.machine1.Manager', + ) + fd = res[0].take() + fh = fdopen(fd) + ret = [] + while True: + try: + ret.append(fh.readline().strip()) + except OSError as err: + if err.errno != 5: + raise err from err + break + if not ret: + errors.append(f'Cannot check {host} status') + error = True + break + if ret[0] in ['running', 'degraded']: + break + except DBusException: + pass + idx += 1 + sleep(1) + if idx == 120: + errors.append(f'Cannot not start {host} ({ret})') + break + if error: + continue + if ret[0] == 'running': + continue + cmd = ['/usr/bin/systemctl', '--state=failed', '--no-legend', '--no-page'] + res = remote_object.OpenMachineShell(host, + '', + cmd[0], + Array(cmd, signature='s'), + Array(['TERM=dumb'], signature='s'), + dbus_interface='org.freedesktop.machine1.Manager', + ) + fd = res[0].take() + fh = fdopen(fd) + ret = [] + idx2 = 0 + while True: + try: + ret.append(fh.readline().strip()) + except OSError as err: + if err.errno != 5: + raise err from err + break + idx2 += 1 + if idx2 == 120: + errors.append(f'Cannot not get status to {host}') + break + errors.append(f'{host}: ' + '\n'.join(ret)) + return changed, errors + +def run_module(): + # define available arguments/parameters a user can pass to the module + module_args = dict( + state=dict(type='str', required=True), + machines=dict(type='list', required=True), + ) + + # seed the result dict in the object + # we primarily care about changed and state + # changed is if this module effectively modified the target + # state will include any data that you want your module to pass back + # for consumption, for example, in a subsequent task + result = dict( + changed=False, + message='' + ) + + # the AnsibleModule object will be our abstraction working with Ansible + # this includes instantiation, a couple of common attr would be the + # args/params passed to the execution, as well as if the module + # supports check mode + module = AnsibleModule( + argument_spec=module_args, + supports_check_mode=True + ) + # if the user is working with this module in only check mode we do not + # want to make any changes to the environment, just return the current + # state with no modifications + if module.check_mode: + module.exit_json(**result) + + bus = SystemBus() + + # manipulate or modify the state as needed (this is going to be the + # part where your module will do what it needs to do) + machines = module.params['machines'] + if module.params['state'] == 'stopped': + result['changed'], errors = stop(bus, machines) + if errors: + errors = '\n\n'.join(errors) + module.fail_json(msg=f'Some machines are not stopping correctly {errors}', **result) + elif module.params['state'] == 'started': + result['changed'], errors = start(bus, machines) + if errors: + errors = '\n\n'.join(errors) + module.fail_json(msg=f'Some machines are not running correctly {errors}', **result) + else: + module.fail_json(msg=f"Unknown state: {module.params['state']}") + + + + # in the event of a successful module execution, you will want to + # simple AnsibleModule.exit_json(), passing the key/value results + module.exit_json(**result) + + +def main(): + run_module() + + +if __name__ == '__main__': + main() diff --git a/ansible/machine.yml b/ansible/machine.yml new file mode 100644 index 0000000..1c00f83 --- /dev/null +++ b/ansible/machine.yml @@ -0,0 +1,93 @@ +- name: "Create SRV directory for {{ item.name}}" + when: "item.srv" + file: path=/var/lib/risotto/srv/{{ item.name }} state=directory mode=0755 + +- name: "Create SystemD directory for {{ item.name }}" + file: path=/var/lib/risotto/journals/{{ item.name }} state=directory mode=0755 + +- name: "Build machine files for {{ item.name }}" + local_action: + module: rougail + hostname: "{{ item.name}}" + is_host: False + +- name: "Get local informations for {{ item.name }} configuration's file" + local_action: + module: stat + path: "installations{{ file.name }}" + checksum: sha256 + get_checksum: yes + loop: "{{ vars[item.name]['services'] | fileslist }}" + loop_control: + loop_var: file + label: "{{ file.name }}" + register: local_configuration + +- name: "Get remote informations for {{ item.name }} configuration's file" + stat: + path: "/var/lib/risotto/configurations/{{ item.name }}{{ file.name }}" + checksum: sha256 + get_checksum: yes + loop: "{{ vars[item.name]['services'] | fileslist }}" + loop_control: + loop_var: file + label: "{{ file.name }}" + register: remote_configuration + +- name: "Configuration's file is up to date in {{ item.name }}" + debug: + msg: "file is {{ 'out of date' if not file[1].stat.exists or file[0].stat.checksum != file[1].stat.checksum else 'up to date' }}" + changed_when: not file[1].stat.exists or file[0].stat.checksum != file[1].stat.checksum + loop: "{{ local_configuration.results | zip(remote_configuration.results) | list }}" + loop_control: + loop_var: file + label: "{{ file[0]['stat']['path'] }}" + ignore_errors: true + register: up_to_date_configuration + +- name: "Remove Compressed files for {{ item.name }}" + local_action: + module: file + path: /tmp/new_configurations/{{ item.name }} + state: absent + when: up_to_date_configuration.changed + +- name: "Compress files for {{ item.name }}" + local_action: + module: archive + path: "installations/" + dest: /tmp/new_configurations/{{ item.name }} + format: tar + when: up_to_date_configuration.changed + +- name: "Create system directory for {{ item.name }}" + file: + path: /var/lib/machines/{{ item.name }} + state: directory + register: system_directory_created + +- name: "Check image for {{ item.name }}" + stat: + path: "/var/lib/risotto/images/{{ vars | modulename(item.name) }}.tar" + register: register_name + when: system_directory_created.changed + +- name: "Build image for {{ item.name }}" + ansible.builtin.shell: "/usr/local/sbin/build_image {{ vars | modulename(item.name) }}" + when: system_directory_created.changed and not register_name.stat.exists + +- name: "Uncompress machine image for {{ item.name }}" + unarchive: + src: "/var/lib/risotto/images/{{ vars | modulename(item.name) }}.tar" + remote_src: true + dest: /var/lib/machines/{{ item.name }}/ + when: system_directory_created.changed + +- name: "SHA machine image for {{ item.name }}" + ansible.builtin.copy: + src: "/var/lib/risotto/images/{{ vars | modulename(item.name) }}.tar.sha" + remote_src: true + dest: "/var/lib/risotto/configurations/sha/{{ item.name }}.sha" + owner: "root" + group: "root" + when: system_directory_created.changed diff --git a/ansible/machines.yml b/ansible/machines.yml new file mode 100644 index 0000000..a190b08 --- /dev/null +++ b/ansible/machines.yml @@ -0,0 +1,33 @@ +- name: "Stop machines with new configuration" + machinectl: + state: stopped + machines: "{{ lookup('fileglob', '/tmp/new_configurations/*', wantlist=True) | map('basename') | list }}" + +- name: "Remove files directory" + file: + path: "/var/lib/risotto/configurations/{{ item }}" + state: absent + loop: "{{ lookup('fileglob', '/tmp/new_configurations/*', wantlist=True) | map('basename') | list }}" + +- name: "Create files directory" + file: + path: "/var/lib/risotto/configurations/{{ item }}" + state: directory + loop: "{{ lookup('fileglob', '/tmp/new_configurations/*', wantlist=True) | map('basename') | list }}" + +- name: "Copy configuration" + unarchive: + src: "{{ item }}" + dest: /var/lib/risotto/configurations/{{ item | basename }}/ + loop: "{{ lookup('fileglob', '/tmp/new_configurations/*', wantlist=True) }}" + +- name: "Start machines" + machinectl: + state: started + machines: "{{ vars | machineslist(only_name=True) }}" + +- name: "Remove compressed files directory" + local_action: + module: file + path: /tmp/new_configurations + state: absent diff --git a/ansible/password b/ansible/password new file mode 120000 index 0000000..308c981 --- /dev/null +++ b/ansible/password @@ -0,0 +1 @@ +../password/ \ No newline at end of file diff --git a/ansible/pki b/ansible/pki new file mode 120000 index 0000000..b424940 --- /dev/null +++ b/ansible/pki @@ -0,0 +1 @@ +../pki/ \ No newline at end of file diff --git a/ansible/playbook.txt b/ansible/playbook.txt new file mode 100644 index 0000000..5626aa0 --- /dev/null +++ b/ansible/playbook.txt @@ -0,0 +1,23 @@ + - name: installation dépendances + apt: + pkg: + - systemd-container + - dnf + - jq + - debootstrap + - htop + - gettext + - patch + - unzip + - mlocate + - xz-utils + - iptables + update_cache: yes + state: latest + +MARCHE + - name: installation dépendances + apt: + pkg: "{{ packages }}" + update_cache: yes + state: latest diff --git a/ansible/playbook.yml b/ansible/playbook.yml new file mode 100644 index 0000000..fb9169c --- /dev/null +++ b/ansible/playbook.yml @@ -0,0 +1,28 @@ +--- +#FIXME : si on redemarre a appel tmpfiles.d .... +- name: Risotto + hosts: cloud.silique.fr + tasks: + - name: "Configure the host" + include_tasks: host.yml + when: configure_host == true + + - name: "Remove compressed files directory" + local_action: + module: file + path: /tmp/new_configurations + state: absent + + - name: "Create compressed configuration files directory" + local_action: + module: file + path: /tmp/new_configurations + state: directory + mode: 0700 + + - name: "Prepare machine configuration" + include_tasks: machine.yml + loop: "{{ vars | machineslist(only=only_machine) }}" + + - name: "Install and apply configurations" + include_tasks: machines.yml diff --git a/bootstrap.py b/bootstrap.py index 02f6ea8..b31d6d3 100755 --- a/bootstrap.py +++ b/bootstrap.py @@ -19,24 +19,13 @@ CONFIG_ORI_DIR = 'ori' SRV_DEST_DIR = 'srv' -def tiramisu_display_name(kls, - dyn_name: 'Base'=None, - suffix: str=None, - ) -> str: - # FIXME - if dyn_name is not None: - name = kls.impl_getpath() + str(suffix) - else: - name = kls.impl_getpath() - return name - - async def main(): if isdir(INSTALL_DIR): rmtree(INSTALL_DIR) makedirs(INSTALL_DIR) try: - module_infos, rougailconfig, config = await load(display_name=tiramisu_display_name, + module_infos, rougailconfig, config = await load('a.py', + 'n.py', clean_directories=True, copy_manual_dir=True, copy_tests=True, diff --git a/sbin/build_image b/sbin/build_image new file mode 100755 index 0000000..d15c86b --- /dev/null +++ b/sbin/build_image @@ -0,0 +1,196 @@ +#!/bin/bash -e + +IMAGE_NAME=$1 + +if [ -z "$IMAGE_NAME" ]; then + echo "PAS DE NOM DE MODULE" + exit 1 +fi + +# root dir configuration +RISOTTO_DIR="/var/lib/risotto" +RISOTTO_IMAGE_DIR="$RISOTTO_DIR/images" +# image configuration +IMAGE_BASE_RISOTTO_BASE_DIR="$RISOTTO_IMAGE_DIR/image_bases" +IMAGE_NAME_RISOTTO_IMAGE_DIR="$RISOTTO_IMAGE_DIR/$IMAGE_NAME" +IMAGE_NAME_RISOTTO_IMAGE_NAME="$RISOTTO_IMAGE_DIR/$IMAGE_NAME".tar +IMAGE_DIR_RECIPIENT_IMAGE="/var/lib/risotto/images_files/$IMAGE_NAME" + + +#FIXME ou ? + +rm -rf "$IMAGE_NAME_RISOTTO_IMAGE_DIR" "$RISOTTO_IMAGE_DIR/tmp" +mkdir -p "$RISOTTO_IMAGE_DIR" +PKG="" +BASE_DIR="" +for script in $(ls "$IMAGE_DIR_RECIPIENT_IMAGE"/preinstall/*.sh 2> /dev/null); do + . "$script" +done + +if [ -z "$OS_NAME" ]; then + echo "NO OS NAME DEFINED" + exit 0 +fi +if [ -z "$RELEASEVER" ]; then + echo "NO RELEASEVER DEFINED" + exit 0 +fi +if [ -z "$INSTALL_TOOL" ]; then + echo "NO INSTALL TOOL DEFINED" + exit 0 +fi +BASE_NAME="$OS_NAME-$RELEASEVER" +BASE_DIR="$IMAGE_BASE_RISOTTO_BASE_DIR/$BASE_NAME" +BASE_TAR="$IMAGE_BASE_RISOTTO_BASE_DIR-$BASE_NAME".tar +BASE_PKGS_FILE="$IMAGE_BASE_RISOTTO_BASE_DIR-$BASE_NAME.pkgs" +BASE_LOCK="$IMAGE_BASE_RISOTTO_BASE_DIR-$BASE_NAME.build" + + +function dnf_opt_base() { + INSTALL_DIR=$1 + echo "--setopt=install_weak_deps=False --nodocs --noplugins --installroot=$INSTALL_DIR --releasever $RELEASEVER" +} + +function dnf_opt() { + INSTALL_DIR=$1 + INSTALL_PKG=$2 + OPT=$(dnf_opt_base "$INSTALL_DIR") + echo "$OPT install $INSTALL_PKG" +} +function new_package_base() { + if [ "$INSTALL_TOOL" = "dnf" ]; then + OPT=$(dnf_opt "$BASE_DIR" "$BASE_PKG") + dnf --assumeno $OPT | grep ^" " > "$BASE_PKGS_FILE".new + else + debootstrap --include="$BASE_PKG" --variant=minbase "$RELEASEVER" "$BASE_DIR" > /dev/null + chroot "$BASE_DIR" dpkg-query -f '${binary:Package} ${source:Version}\n' -W > "$BASE_PKGS_FILE".new + fi +} +function install_base() { + if [ "$INSTALL_TOOL" = "dnf" ]; then + OPT=$(dnf_opt "$BASE_DIR" "$BASE_PKG") + dnf --assumeyes $OPT + fi +} +function new_package() { + if [ "$INSTALL_TOOL" = "dnf" ]; then + OPT=$(dnf_opt_base "$IMAGE_NAME_RISOTTO_IMAGE_DIR") + dnf $OPT update + OPT=$(dnf_opt "$IMAGE_NAME_RISOTTO_IMAGE_DIR" "$PKG") + dnf --assumeno $OPT | grep ^" " > "$IMAGE_NAME_RISOTTO_IMAGE_DIR".pkgs.new + else + chroot "$IMAGE_NAME_RISOTTO_IMAGE_DIR" apt update > /dev/null 2>&1 + chroot "$IMAGE_NAME_RISOTTO_IMAGE_DIR" apt install --no-install-recommends --yes $PKG -s 2>/dev/null|grep ^"Inst " > "$IMAGE_NAME_RISOTTO_IMAGE_DIR".pkgs.new + fi +} +function install_pkg() { + if [ "$INSTALL_TOOL" = "dnf" ]; then + OPT=$(dnf_opt "$IMAGE_NAME_RISOTTO_IMAGE_DIR" "$PKG") + dnf --assumeyes $OPT + else + chroot "$IMAGE_NAME_RISOTTO_IMAGE_DIR" apt install --no-install-recommends --yes $PKG + fi +} + + +if [ ! -f "$BASE_LOCK" ] || [ ! -f "$BASE_TAR" ]; then + echo " - reinstallation de l'image de base" + rm -rf "$BASE_DIR" + new_package_base + diff -u "$BASE_PKGS_FILE" "$BASE_PKGS_FILE".new && NEW_BASE=false || NEW_BASE=true + if [ ! -f "$BASE_TAR" ] || [ "$NEW_BASE" = true ]; then + mkdir -p "$IMAGE_BASE_RISOTTO_BASE_DIR" + install_base + cd "$IMAGE_BASE_RISOTTO_BASE_DIR" + tar cf "$BASE_TAR" "$BASE_NAME" + cd - > /dev/null + if [ -f "$BASE_PKGS_FILE" ]; then + mv "$BASE_PKGS_FILE" "$BASE_PKGS_FILE".old + fi + mv "$BASE_PKGS_FILE".new "$BASE_PKGS_FILE" + rm -rf "$IMAGE_BASE_RISOTTO_BASE_DIR" + fi + rm -rf "$BASE_DIR" + touch "$BASE_LOCK" +fi + +tar xf "$BASE_TAR" +mv "$BASE_NAME" "$IMAGE_NAME_RISOTTO_IMAGE_DIR" +if [ -n "$COPR" ]; then + #FIXME signature... + mkdir -p "$REPO_DIR" + cd "$REPO_DIR" + wget -q "$COPR" + cd - > /dev/null +fi +if [ "$FUSION" = true ]; then + dnf -y install "https://download1.rpmfusion.org/free/fedora/rpmfusion-free-release-$RELEASEVER.noarch.rpm" --installroot="$IMAGE_NAME_RISOTTO_IMAGE_DIR" > /dev/null +fi + +# FIXME verifier s'il y a des modifs sur pre/post +if [ -f "$IMAGE_NAME_RISOTTO_IMAGE_DIR".base.pkgs ] && [ -f "$IMAGE_NAME_RISOTTO_IMAGE_DIR".pkgs ]; then + echo " - différence(s) avec les paquets de base" + diff -u "$IMAGE_NAME_RISOTTO_IMAGE_DIR".base.pkgs "$BASE_PKGS_FILE" && INSTALL=false || INSTALL=true +else + INSTALL=true +fi +new_package +if [ "$INSTALL" = false ]; then + echo " - différence(s) avec les paquets de l'image" + diff -u "$IMAGE_NAME_RISOTTO_IMAGE_DIR".pkgs "$IMAGE_NAME_RISOTTO_IMAGE_DIR".pkgs.new && INSTALL=false || INSTALL=true +fi +find "$IMAGE_DIR_RECIPIENT_IMAGE" -type f -exec md5sum '{}' \; > "$IMAGE_NAME_RISOTTO_IMAGE_DIR".md5sum.new +if [ "$INSTALL" = false ]; then + diff -u "$IMAGE_NAME_RISOTTO_IMAGE_DIR".md5sum "$IMAGE_NAME_RISOTTO_IMAGE_DIR".md5sum.new && INSTALL=false || INSTALL=true +fi +if [ "$INSTALL" = true ]; then + echo " - installation" + if [ -f "$IMAGE_NAME_RISOTTO_IMAGE_DIR"_"$RELEASEVER".version ]; then + VERSION=$(cat "$IMAGE_NAME_RISOTTO_IMAGE_DIR"_"$RELEASEVER".version) + else + VERSION=0 + fi + mkdir "$RISOTTO_IMAGE_DIR/tmp" + ORI_DIR=$PWD + cd "$RISOTTO_IMAGE_DIR/tmp" + if [ ! "$VERSION" = 0 ] && [ -f "$IMAGE_NAME_RISOTTO_IMAGE_NAME" ]; then + tar xf "$IMAGE_NAME_RISOTTO_IMAGE_NAME" +# if [ "$INSTALL_TOOL" = "apt" ]; then +# chown _apt "$IMAGE_NAME" +# fi +# else +# mkdir "$IMAGE_NAME" + fi + #cd "$IMAGE_NAME" + make_changelog "$IMAGE_NAME" "$VERSION" "$OS_NAME" "$RELEASEVER" > "$IMAGE_NAME_RISOTTO_IMAGE_DIR"_"$RELEASEVER"_"$VERSION"_changelog.md + cd $ORI_DIR + rm -rf "$RISOTTO_IMAGE_DIR/tmp" + install_pkg + sleep 2 + + for script in $(ls $IMAGE_DIR_RECIPIENT_IMAGE/postinstall/*.sh 2> /dev/null); do + . "$script" + done + + CONTAINER=$IMAGE_NAME make_volatile /etc + if [ ! "$?" = 0 ]; then + echo "make_volatile failed" + exit 1 + fi + cd "$RISOTTO_IMAGE_DIR/$IMAGE_NAME" + if [ -f "$IMAGE_NAME_RISOTTO_IMAGE_NAME" ]; then + mv -f "$IMAGE_NAME_RISOTTO_IMAGE_NAME" "$IMAGE_NAME_RISOTTO_IMAGE_NAME".old + fi + tar cf "$IMAGE_NAME_RISOTTO_IMAGE_NAME" . + sha256sum "$IMAGE_NAME_RISOTTO_IMAGE_NAME" > "$IMAGE_NAME_RISOTTO_IMAGE_NAME".sha + cd - > /dev/null + cp -f "$BASE_PKGS_FILE" "$IMAGE_NAME_RISOTTO_IMAGE_DIR".base.pkgs + mv -f "$IMAGE_NAME_RISOTTO_IMAGE_DIR".pkgs.new "$IMAGE_NAME_RISOTTO_IMAGE_DIR".pkgs + mv -f "$IMAGE_NAME_RISOTTO_IMAGE_DIR".md5sum.new "$IMAGE_NAME_RISOTTO_IMAGE_DIR".md5sum + VERSION=$((VERSION + 1)) + echo "$VERSION" > "$IMAGE_NAME_RISOTTO_IMAGE_DIR"_"$RELEASEVER".version +fi +rm -rf "$IMAGE_NAME_RISOTTO_IMAGE_DIR" + +echo " => OK" +exit 0 diff --git a/sbin/make_changelog b/sbin/make_changelog new file mode 100755 index 0000000..c2da6ea --- /dev/null +++ b/sbin/make_changelog @@ -0,0 +1,181 @@ +#!/usr/bin/env python3 + + +import logging +from dnf.conf import Conf +from dnf.cli.cli import BaseCli, Cli +from dnf.cli.output import Output +from dnf.cli.option_parser import OptionParser +from dnf.i18n import _, ucd +from datetime import datetime, timezone +from sys import argv +from os import getcwd, unlink +from os.path import isfile, join +from glob import glob +from subprocess import run + + +# List new or removed file +def read_dnf_pkg_file(os_name, filename1, filename2): + if os_name == 'debian': + idx_pkg = 0, 1 + idx_version = 1, 2 + header_idx = 0, 0 + else: + idx_pkg = 0, 0 + idx_version = 2, 2 + header_idx = 2, 2 + pass + pkgs = {} + for fidx, filename in enumerate((filename1, filename2)): + if not isfile(filename): + continue + with open(filename, 'r') as pkgs_fh: + for idx, pkg_line in enumerate(pkgs_fh.readlines()): + if idx < header_idx[fidx]: + # header + continue + sp_line = pkg_line.strip().split() + if len(sp_line) < idx_version[fidx] + 1: + continue + if sp_line[idx_pkg[fidx]] in pkgs: + raise Exception(f'package already set {sp_line[0]}?') + version = sp_line[idx_version[fidx]] + if os_name == 'debian' and version.startswith('('): + version = version[1:] + pkgs[sp_line[idx_pkg[fidx]]] = version + return pkgs + + +def list_packages(title, packages, packages_info): + print(f'# {title}\n') + if not packages: + print('*Aucun*') + packages = list(packages) + packages = sorted(packages) + for idx, pkg in enumerate(packages): + print(f' - {pkg} ({packages_info[pkg]})') + print() + + +# List updated packages +class CustomOutput(Output): + def listPkgs(self, *args, **kwargs): + # do not display list + pass + + +def format_changelog_markdown(changelog): + """Return changelog formatted as in spec file""" + text = '\n'.join([f' {line}' for line in changelog['text'].split('\n')]) + chlog_str = ' - %s %s\n\n%s\n' % ( + changelog['timestamp'].strftime("%a %b %d %X %Y"), + ucd(changelog['author']), + ucd(text)) + return chlog_str + + +def print_changelogs_markdown(packages): + # group packages by src.rpm to avoid showing duplicate changelogs + self = BASE + bysrpm = dict() + for p in packages: + # there are packages without source_name, use name then. + bysrpm.setdefault(p.source_name or p.name, []).append(p) + for source_name in sorted(bysrpm.keys()): + bin_packages = bysrpm[source_name] + print('- ' + _("Changelogs for {}").format(', '.join([str(pkg) for pkg in bin_packages]))) + print() + for chl in self.latest_changelogs(bin_packages[0]): + print(format_changelog_markdown(chl)) + + +def dnf_update(image_name): + conf = Conf() + # obsoletes are already listed + conf.obsoletes = False + with BaseCli(conf) as base: + global BASE + BASE = base + base.print_changelogs = print_changelogs_markdown + custom_output = CustomOutput(base.output.base, base.output.conf) + base.output = custom_output + cli = Cli(base) + image_dir = join(getcwd(), image_name) + cli.configure(['--setopt=install_weak_deps=False', '--nodocs', '--noplugins', '--installroot=' + image_dir, '--releasever', '35', 'check-update', '--changelog'], OptionParser()) + logger = logging.getLogger("dnf") + for h in logger.handlers: + logger.removeHandler(h) + logger.addHandler(logging.NullHandler()) + cli.run() + + +def main(os_name, image_name, old_version, releasever): + date = datetime.now(timezone.utc).isoformat() + if old_version == 0: + title = f"Création de l'image {image_name}" + subtitle = f"Les paquets de la première image {image_name} sur base Fedora {releasever}" + else: + title = f"Nouvelle version de l'image {image_name}" + subtitle = f"Différence des paquets de l'image {image_name} sur base Fedora {releasever} entre la version {old_version} et {old_version + 1}" + print(f"""+++ +title = "{title}" +description = "{subtitle}" +date = {date} +updated = {date} +draft = false +template = "blog/page.html" + +[taxonomies] +authors = ["Automate"] + +[extra] +lead = "{subtitle}." +type = "installe" ++++ +""") + new_dict = read_dnf_pkg_file(os_name, f'/var/lib/risotto/images/image_bases-{os_name}-{releasever}.pkgs', f'/var/lib/risotto/images/{image_name}.pkgs.new') + new_pkg = new_dict.keys() + old_file = f'/var/lib/risotto/images/{image_name}.pkgs' + if not old_version or not isfile(old_file): + list_packages('Liste des paquets', new_pkg, new_dict) + else: + ori_dict = read_dnf_pkg_file(os_name, f'/var/lib/risotto/images/{image_name}.base.pkgs', old_file) + ori_pkg = ori_dict.keys() + list_packages('Les paquets supprimés', ori_pkg - new_pkg, ori_dict) + list_packages('Les paquets ajoutés', new_pkg - ori_pkg, new_dict) + print('# Les paquets mises à jour\n') + if os_name == 'fedora': + dnf_update(image_name) + else: + for filename in glob('*.deb'): + unlink(filename) + for package in ori_pkg & new_dict: + if ori_dict[package] == new_dict[package]: + continue + info = run(['apt', 'download', package], capture_output=True) + if info.returncode: + raise Exception(f'cannot download {package}: {info}') + packages = list(glob('*.deb')) + packages.sort() + for package in packages: + info = run(['chroot', '.', 'apt-listchanges', '--which', 'both', '-f', 'text', package], capture_output=True) + if info.returncode: + raise Exception(f'cannot list changes for {package}: {info}') + header = True + for line in info.stdout.decode().split('\n'): + if not header: + print(line) + if line.startswith('-----------------------'): + header = False + print() + unlink(package) + + + +if __name__ == "__main__": + image_name = argv[1] + old_version = int(argv[2]) + os_name = argv[3] + releasever = argv[4] + main(os_name, image_name, old_version, releasever) diff --git a/sbin/make_volatile b/sbin/make_volatile new file mode 100755 index 0000000..2227692 --- /dev/null +++ b/sbin/make_volatile @@ -0,0 +1,77 @@ +#!/bin/bash -e +if [ -z $CONTAINER ]; then + echo "PAS DE CONTAINER" + exit 1 +fi +ROOT="/var/lib/risotto/images/$CONTAINER" +echo "$ROOT" +DESTDIR="$ROOT/usr/lib/tmpfiles.d" +CONF_DST="/usr/share/factory" +EXCLUDES="^($ROOT/etc/passwd|$ROOT/etc/group|$ROOT/etc/.updated|$ROOT/etc/.pwd.lock|$ROOT/etc/systemd/network/dhcp.network|$ROOT/etc/sudoers.d/qemubuild)$" +ONLY_COPY="^($ROOT/etc/localtime)$" +FORCE_LINKS="^($ROOT/etc/udev/hwdb.bin)$" + +function execute() { + chroot $ROOT $@ +} + +function file_dir_in_tmpfiles() { + letter=$1 + directory=$2 + local_directory=$(echo $directory|sed "s@^$ROOT@@g") + mode=$(execute "/usr/bin/stat" "--format" "%a" "$local_directory" | grep -o "[0-9.]\+") + user=$(execute "/usr/bin/stat" "--format" "%U" "$local_directory" | grep -o "[0-9a-zA-Z.-]\+") + group=$(execute "/usr/bin/stat" "--format" "%G" "$local_directory" | grep -o "[0-9a-zA-Z.-]\+") + echo "$letter $local_directory $mode $user $group - -" +} + +function calc_symlink_in_tmpfiles() { + dest_name=$1 + local_dest_name=$2 + src_file=$(readlink "$dest_name") + symlink_in_tmpfiles "$local_dest_name" "$src_file" +} + +function symlink_in_tmpfiles() { + dest_name=$1 + src_file=$2 + echo "L+ $dest_name - - - - $src_file" +} + +function main() { + dir_config_orig=$1 + name="${dir_config_orig//\//-}" + dir_config_orig=$ROOT$dir_config_orig + + mkdir -p "$DESTDIR" + mkdir -p "$ROOTCONF_DST$dir_config_orig" + systemd_conf="$DESTDIR/risotto$name.conf" + rm -f $systemd_conf + shopt -s globstar + for src_file in $dir_config_orig/**; do + local_src=$(echo $src_file|sed "s@$ROOT@@g") + dest_file="$ROOT$CONF_DST$local_src" + if [[ "$src_file" =~ $EXCLUDES ]]; then + echo "$src_file: exclude" >&2 + elif [[ -L "$src_file" ]]; then + calc_symlink_in_tmpfiles "$src_file" "$local_src" >> $systemd_conf + elif [[ "$src_file" =~ $FORCE_LINKS ]]; then + symlink_in_tmpfiles "$src_file" "$dest_file" >> $systemd_conf + elif [[ -d "$src_file" ]]; then + file_dir_in_tmpfiles 'd' "$src_file" >> $systemd_conf + [[ ! -d "$dest_file" ]] && mkdir -p "$dest_file" + #echo "$src_file: directory ok" + else + if [[ ! "$src_file" =~ $ONLY_COPY ]]; then + file_dir_in_tmpfiles "C" "$src_file" >> $systemd_conf + fi + [[ -e "$dest_file" ]] && rm -f "$dest_file" + # not a symlink... an hardlink + ln "$src_file" "$dest_file" + #echo "$src_file: file ok" + fi + done +} +main "$1" +echo "fin" +exit 0 diff --git a/sbin/update_images b/sbin/update_images new file mode 100644 index 0000000..c2e250c --- /dev/null +++ b/sbin/update_images @@ -0,0 +1,79 @@ +#!/bin/bash -e + +# root dir configuration +RISOTTO_DIR="/var/lib/risotto" +RISOTTO_IMAGE_DIR="$RISOTTO_DIR/images" +# image configuration +IMAGE_BASE_RISOTTO_BASE_DIR="$RISOTTO_IMAGE_DIR/image_bases" + +rm -f $IMAGE_BASE_RISOTTO_BASE_DIR*.build + +ls /var/lib/risotto/images_files/ | while read image; do + if [ -d /var/lib/risotto/images_files/"$image" ]; then + echo + echo "Install image $image" + /usr/local/sbin/build_image "$image" + fi +done +rm -f $IMAGE_BASE_RISOTTO_BASE_DIR*.build + +MACHINES="" +for nspawn in $(ls /etc/systemd/nspawn/*.nspawn); do + nspawn_file=$(basename $nspawn) + machine=${nspawn_file%.*} + MACHINES="$MACHINES$machine " + MACHINE_MACHINES_DIR="/var/lib/machines/$machine" + echo "Install machine $machine" + SHA_MACHINE="$RISOTTO_DIR/configurations/sha/$machine".sha + content=$(cat $SHA_MACHINE) + IMAGE_NAME_RISOTTO_IMAGE_NAME=${content##* } + diff -q "$IMAGE_NAME_RISOTTO_IMAGE_NAME".sha "$SHA_MACHINE" || ( + machinectl stop $machine + while true; do + machinectl status "$machine" > /dev/null 2>&1 || break + sleep 1 + done + rm -rf "$MACHINE_MACHINES_DIR" + mkdir "$MACHINE_MACHINES_DIR" + cd "$MACHINE_MACHINES_DIR" + tar xf "$IMAGE_NAME_RISOTTO_IMAGE_NAME" + cp -a "$IMAGE_NAME_RISOTTO_IMAGE_NAME".sha "$SHA_MACHINE" + ) +done +machinectl start $MACHINES +STARTED="" +DEGRADED="" +found=true +idx=0 +while [ $found = true ]; do + found=false + echo "tentative $idx" + for machine in $MACHINES; do + if ! echo $STARTED | grep -q " $machine "; then + status=$(machinectl -q shell $machine /usr/bin/systemctl is-system-running || true) + if echo "$status" | grep -q degraded; then + STARTED="$STARTED $machine " + DEGRADED="$DEGRADED $machine" + elif echo "$status" | grep -q running; then + STARTED="$STARTED $machine " + else + found=true + echo "status actuel de $machine : $status" + fi + fi + done + sleep 2 + idx=$((idx+1)) + if [ $idx = 60 ]; then + break + fi +done +retcode=0 +for machine in $DEGRADED; do + echo + echo "========= $machine" + machinectl -q shell $machine /usr/bin/systemctl --state=failed --no-legend --no-pager + retcode=1 +done + +exit $retcode diff --git a/src/risotto/image.py b/src/risotto/image.py index dd7d62c..fadd9d3 100644 --- a/src/risotto/image.py +++ b/src/risotto/image.py @@ -21,6 +21,7 @@ class ModuleCfg(): self.templates_dir = [] self.extra_dictionaries = {} self.servers = [] + self.depends = [] def __repr__(self): return str(vars(self)) @@ -128,7 +129,6 @@ def load_applicationservice_cfg(appname: str, def load_applicationservice(appname: str, - added: list, install_dir: str, cfg: ModuleCfg, applications: dict, @@ -150,7 +150,7 @@ def load_applicationservice(appname: str, copy_manual_dir, copy_tests, ) - added.append(appname) + cfg.depends.append(appname) with open(applicationservice_file) as yaml: app = yaml_load(yaml, Loader=SafeLoader) provider = app.get('provider') @@ -163,19 +163,27 @@ def load_applicationservice(appname: str, suppliers.setdefault(supplier, []) if appname not in suppliers[supplier]: suppliers[supplier].append(appname) + if 'distribution' in app: + distribution = app['distribution'] + else: + distribution = None for xml in app.get('depends', []): - if xml in added: + if xml in cfg.depends: continue - load_applicationservice(xml, - added, - install_dir, - cfg, - applications, - copy_manual_dir, - copy_tests, - providers, - suppliers, - ) + ret = load_applicationservice(xml, + install_dir, + cfg, + applications, + copy_manual_dir, + copy_tests, + providers, + suppliers, + ) + if ret: + if distribution: + raise Exception(f'duplicate distribution for {cfg.module_name} ({distribution} and {ret})') + distribution = ret + return distribution def load_image_informations(module_name: str, @@ -188,18 +196,23 @@ def load_image_informations(module_name: str, suppliers: dict, ) -> ModuleCfg: cfg = ModuleCfg(module_name) - added = [] + distribution = None for applicationservice in datas['applicationservices']: - load_applicationservice(applicationservice, - added, - install_dir, - cfg, - applications, - copy_manual_dir, - copy_tests, - providers, - suppliers, - ) + ret = load_applicationservice(applicationservice, + install_dir, + cfg, + applications, + copy_manual_dir, + copy_tests, + providers, + suppliers, + ) + if ret: + if distribution: + raise Exception(f'duplicate distribution for {cfg.module_name} ({distribution} and {ret})') + distribution = ret + if module_name != 'host' and not distribution: + raise Exception(f'cannot found any linux distribution for {module_name}') return cfg diff --git a/src/risotto/machine.py b/src/risotto/machine.py index 0b1110d..79ceebf 100644 --- a/src/risotto/machine.py +++ b/src/risotto/machine.py @@ -2,7 +2,8 @@ from .utils import SERVERS, SERVERS_JSON, MULTI_FUNCTIONS, load_domains from .image import load_config, valid_mandatories # , load_modules_rougail_config from rougail import RougailConfig, RougailConvert from .rougail.annotator import calc_providers, calc_providers_global, calc_providers_dynamic, calc_providers_dynamic_follower, calc_providers_follower -from os import makedirs +from os.path import isfile +from json import dump as json_dump, load as json_load # from tiramisu import Config from .utils import value_pprint @@ -10,6 +11,18 @@ from rougail.utils import normalize_family from rougail import RougailSystemdTemplate # # +def tiramisu_display_name(kls, + dyn_name: 'Base'=None, + suffix: str=None, + ) -> str: + # FIXME + if dyn_name is not None: + name = kls.impl_getpath() + str(suffix) + else: + name = kls.impl_getpath() + return name + + async def set_values(server_name, config, datas): if 'values' not in datas: return @@ -50,7 +63,6 @@ FUNCTIONS = {'get_ip_from_domain': get_ip_from_domain, async def templates(server_name, config, templates_informations, - srv=False, just_copy=False, ): engine = RougailSystemdTemplate(config, templates_informations) @@ -71,22 +83,20 @@ async def templates(server_name, await value_pprint(values, config) print(err) print(await config.option('general.nginx.nginx_default_http').value.get()) - exit(1) - #raise err from err + raise err from err if just_copy: for eng, old_engine in ori_engines.items(): engine.engines[eng] = old_engine - if srv: - makedirs(srv) - -async def load(display_name=None, +async def load(cache_file, + cache_values, clean_directories=False, copy_manual_dir=False, copy_tests=False, hide_secret=False, ): + display_name=tiramisu_display_name #load_zones() # # load images #FIXME useful @@ -109,29 +119,34 @@ async def load(display_name=None, module_info = module_infos[datas['module']] functions_files |= set(module_info['infos'].functions_file) cfg['functions_file'] = list(functions_files) - eolobj = RougailConvert(cfg) - cfg['risotto_globals'] = {} - for server_name, datas in SERVERS.items(): - module_info = module_infos[datas['module']] - cfg['dictionaries_dir'] = module_info['infos'].dictionaries_dir - cfg['extra_dictionaries'] = module_info['infos'].extra_dictionaries - informations = SERVERS_JSON['servers'][server_name].get('informations') - if informations: - cfg['risotto_globals'][server_name] = {'global:server_name': server_name, - 'global:zones_name': informations['zones_name'], - 'global:zones_list': list(range(len(informations['zones_name']))), - } - values = [] - for s_idx in cfg['risotto_globals'][server_name]['global:zones_list']: - if not s_idx: - values.append(server_name) - else: - values.append(informations['extra_domainnames'][s_idx - 1]) - cfg['risotto_globals'][server_name]['global:server_names'] = values - else: - cfg['risotto_globals'][server_name] = {'global:server_name': server_name} - eolobj.load_dictionaries(path_prefix=server_name) - tiram_obj = eolobj.save(None) + if not isfile(cache_file): + eolobj = RougailConvert(cfg) + cfg['risotto_globals'] = {} + for server_name, datas in SERVERS.items(): + module_info = module_infos[datas['module']] + cfg['dictionaries_dir'] = module_info['infos'].dictionaries_dir + cfg['extra_dictionaries'] = module_info['infos'].extra_dictionaries + informations = SERVERS_JSON['servers'][server_name].get('informations') + if informations: + cfg['risotto_globals'][server_name] = {'global:server_name': server_name, + 'global:zones_name': informations['zones_name'], + 'global:zones_list': list(range(len(informations['zones_name']))), + } + values = [] + for s_idx in cfg['risotto_globals'][server_name]['global:zones_list']: + if not s_idx: + values.append(server_name) + else: + values.append(informations['extra_domainnames'][s_idx - 1]) + cfg['risotto_globals'][server_name]['global:server_names'] = values + else: + cfg['risotto_globals'][server_name] = {'global:server_name': server_name} + cfg['risotto_globals'][server_name]['global:module_name'] = datas['module'] + eolobj.load_dictionaries(path_prefix=server_name) + tiram_obj = eolobj.save(cache_file) + else: + with open(cache_file) as fh: + tiram_obj = fh.read() optiondescription = FUNCTIONS.copy() try: exec(tiram_obj, None, optiondescription) @@ -141,11 +156,18 @@ async def load(display_name=None, config = await Config(optiondescription['option_0'], display_name=display_name, ) - await config.property.pop('validator') - await config.property.pop('cache') - for server_name, datas in SERVERS.items(): - await set_values(server_name, config, datas) - await config.property.read_only() - await config.property.add('cache') - await valid_mandatories(config) + if not isfile(cache_values): + await config.property.pop('validator') + await config.property.pop('cache') + for server_name, datas in SERVERS.items(): + await set_values(server_name, config, datas) + await config.property.read_only() + await config.property.add('cache') + await valid_mandatories(config) + with open(cache_values, 'w') as fh: + json_dump(await config.value.exportation(), fh) + else: + with open(cache_values, 'r') as fh: + await config.value.importation(json_load(fh)) + await config.property.read_only() return module_infos, cfg, config