gnunux/risotto
1
0
Fork 0
forked from stove/risotto
Code Issues Pull requests Projects Releases Wiki Activity

Compare commits

base: gnunux:54ff8f23ed774762de9d9686aa6eb05508ffe9e1
Branches Tags
gnunux:main
stove:main
..
compare: gnunux:7b74984fb7b3c0cca8c89563d9d04b79828b98f1
Branches Tags
gnunux:main
stove:main

No commits in common. "54ff8f23ed774762de9d9686aa6eb05508ffe9e1" and "7b74984fb7b3c0cca8c89563d9d04b79828b98f1" have entirely different histories.
54ff8f23ed ... 7b74984fb7

21 changed files with 534 additions and 872 deletions
Show stats Expand all files Collapse all files

120
ansible/action_plugins/rougail.py
View file

@ -1,16 +1,10 @@
#!/usr/bin/python3 #!/usr/bin/python3
from asyncio import run
from os import readlink, walk, chdir, getcwd, makedirs from os import readlink, walk, chdir, getcwd, makedirs
from os.path import join, islink, isdir from os.path import join, islink, isdir
from typing import Dict, Any
from shutil import rmtree, copy2
import tarfile
from ansible.module_utils._text import to_text
from ansible import constants
from rougail.template import base
from rougail.error import TemplateError
from risotto.machine import build_files, INSTALL_DIR, INSTALL_CONFIG_DIR, INSTALL_TMPL_DIR, INSTALL_IMAGES_DIR, INSTALL_TESTS_DIR from risotto.machine import build_files, INSTALL_DIR, INSTALL_CONFIG_DIR, INSTALL_TMPL_DIR, INSTALL_IMAGES_DIR, INSTALL_TESTS_DIR
from risotto.utils import custom_filters from shutil import rmtree
import tarfile
try: try:
@ -19,7 +13,6 @@ try:
class FakeModule(AnsibleModule): class FakeModule(AnsibleModule):
def __init__(self): def __init__(self):
pass pass
from ansible.plugins.action.template import ActionModule as TmplActionModule
except: except:
class ActionBase(): class ActionBase():
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
@ -71,24 +64,15 @@ class ActionModule(ActionBase):
only_machine = module_args.pop('only_machine') only_machine = module_args.pop('only_machine')
configure_host = module_args.pop('configure_host') configure_host = module_args.pop('configure_host')
copy_tests = module_args.pop('copy_tests') copy_tests = module_args.pop('copy_tests')
# define ansible engine
base.ENGINES['ansible'] = Tmpl(task_vars,
self._task,
self._connection,
self._play_context,
self._loader,
self._templar,
self._shared_loader_obj,
)
if 'copy_templates' in module_args: if 'copy_templates' in module_args:
copy_templates = module_args.pop('copy_templates') copy_templates = module_args.pop('copy_templates')
else: else:
copy_templates = False copy_templates = False
directories, certificates = build_files(hostname, directories, certificates = run(build_files(hostname,
only_machine, only_machine,
False, False,
copy_tests, copy_tests,
) ))
module_args['directories'] = list(directories.values()) module_args['directories'] = list(directories.values())
module_args['directories'].append('/var/lib/risotto/images_files') module_args['directories'].append('/var/lib/risotto/images_files')
remote = self._execute_module(module_name='compare', remote = self._execute_module(module_name='compare',
@ -102,14 +86,17 @@ class ActionModule(ActionBase):
msg = remote['msg'] msg = remote['msg']
raise Exception(f'error in remote action: {msg}') raise Exception(f'error in remote action: {msg}')
if copy_templates: if copy_templates:
build_files(hostname, run(build_files(hostname,
only_machine, only_machine,
True, True,
copy_tests, copy_tests,
) ))
machines_changed = [] machines_changed = []
tls_machine = None
for machine, directory in directories.items(): for machine, directory in directories.items():
if machine.startswith('tls.'):
tls_machine = machine
if directory not in remote['directories']: if directory not in remote['directories']:
machines_changed.append(machine) machines_changed.append(machine)
continue continue
@ -163,12 +150,6 @@ class ActionModule(ActionBase):
tar_filename = f'{ARCHIVES_DIR}/{INSTALL_IMAGES_DIR}.tar' tar_filename = f'{ARCHIVES_DIR}/{INSTALL_IMAGES_DIR}.tar'
with tarfile.open(tar_filename, 'w') as archive: with tarfile.open(tar_filename, 'w') as archive:
archive.add('.') archive.add('.')
self._execute_module(module_name='file',
module_args={'path': '/tmp/new_configurations',
'state': 'directory',
},
task_vars=task_vars,
)
self._transfer_file(tar_filename, tar_filename) self._transfer_file(tar_filename, tar_filename)
# tests # tests
self._execute_module(module_name='file', self._execute_module(module_name='file',
@ -209,72 +190,5 @@ class ActionModule(ActionBase):
changed=changed, changed=changed,
machines_changed=machines, machines_changed=machines,
host_changed=self._task.args['hostname'] in machines_changed, host_changed=self._task.args['hostname'] in machines_changed,
tls_machine=tls_machine,
) )
class FakeCopy:
def __init__(self, task):
self.task = task
def run(self, *args, **kwargs):
copy2(self.task.args['src'], self.task.args['dest'])
return {}
class FakeGet:
def __init__(self, klass):
self.klass = klass
def fake_get(self, action, *args, task, **kwargs):
if action == 'ansible.legacy.copy':
return FakeCopy(task)
return self.klass.ori_get(action, *args, task=task, **kwargs)
class Tmpl(TmplActionModule):
def __init__(self, task_vars, *args):
super().__init__(*args)
self.task_vars = task_vars
def _early_needs_tmp_path(self):
# do not create tmp remotely
return False
def process(self,
filename: str,
source: str,
true_destfilename: str,
destfilename: str,
destdir: str,
variable: Any,
index: int,
rougail_variables_dict: Dict,
eosfunc: Dict,
extra_variables: Any=None,
):
if source is not None: # pragma: no cover
raise TemplateError(_('source is not supported for ansible'))
task_vars = rougail_variables_dict | self.task_vars
if variable is not None:
task_vars['rougail_variable'] = variable
if index is not None:
task_vars['rougail_index'] = index
if extra_variables:
task_vars['extra_variables'] = extra_variables
task_vars['rougail_filename'] = true_destfilename
task_vars['rougail_destination_dir'] = destdir
self._task.args['src'] = filename
self._task.args['dest'] = destfilename
# add custom filter
custom_filters.update(eosfunc)
# do not copy file in host but stay it locally
self._shared_loader_obj.action_loader.ori_get = self._shared_loader_obj.action_loader.get
self._shared_loader_obj.action_loader.get = FakeGet(self._shared_loader_obj.action_loader).fake_get
# template
ret = self.run(task_vars=task_vars)
# restore get function
self._shared_loader_obj.action_loader.get = self._shared_loader_obj.action_loader.ori_get
# remove custom filter
custom_filters.clear()
if ret.get('failed'):
raise TemplateError(f'error while templating "{filename}": {ret["msg"]}')

8
ansible/filter_plugins/custom.py
View file

@ -1,8 +0,0 @@
from risotto.utils import custom_filters
class FilterModule:
"""This filter is used to load custom filter from dataset
"""
def filters(self):
return custom_filters

2
ansible/filter_plugins/fileslist.py
View file

@ -127,7 +127,7 @@ def machineslist(data, only=None, only_name=False):
def modulename(data, servername): def modulename(data, servername):
return data[servername]['general']['module_name'] return data[servername]['module_name']
class FilterModule: class FilterModule:

32
ansible/filter_plugins/normalize_family.py
View file

@ -1,32 +0,0 @@
#!/usr/bin/python3
"""
Silique (https://www.silique.fr)
Copyright (C) 2023
distribued with GPL-2 or later license
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
"""
from rougail.utils import normalize_family
class FilterModule:
def filters(self):
return {
'normalize_family': normalize_family,
}

11
ansible/filter_plugins/raise.py
View file

@ -1,11 +0,0 @@
from jinja2.exceptions import TemplateRuntimeError
def fraise(msg):
raise TemplateRuntimeError(msg)
class FilterModule:
def filters(self):
return {
'raise': fraise,
}

30
ansible/host.yml
View file

@ -2,14 +2,6 @@
- name: "Populate service facts" - name: "Populate service facts"
service_facts: service_facts:
- name: "Set timezone"
timezone:
name: Europe/Paris
- name: Set a hostname
ansible.builtin.hostname:
name: "{{ inventory_hostname }}"
- name: "Packages installation" - name: "Packages installation"
apt: apt:
pkg: "{{ vars[inventory_hostname]['general']['host_packages'] }}" pkg: "{{ vars[inventory_hostname]['general']['host_packages'] }}"
@ -71,25 +63,3 @@
path: /var/lib/risotto/tls path: /var/lib/risotto/tls
state: directory state: directory
mode: "755" mode: "755"
- name: "Add keyrings directory"
file:
path: /etc/apt/keyrings
state: directory
mode: "755"
- name: "Add vector signed repositories"
ansible.builtin.get_url:
url: https://repositories.timber.io/public/vector/gpg.3543DB2D0A2BC4B8.key
dest: /etc/apt/keyrings/vector.asc
- name: "Add vector repository"
ansible.builtin.apt_repository:
repo: "deb [signed-by=/etc/apt/keyrings/vector.asc] https://repositories.timber.io/public/vector/deb/debian {{ ansible_distribution_release }} main"
state: present
- name: "Install vector"
ansible.builtin.apt:
name: vector
update_cache: yes
state: present

65
ansible/inventory.py
View file

@ -8,8 +8,8 @@ from argparse import ArgumentParser
from json import load as json_load, dumps, JSONEncoder from json import load as json_load, dumps, JSONEncoder
from os import remove from os import remove
from os.path import isfile from os.path import isfile
from asyncio import run
from traceback import print_exc from traceback import print_exc
from sys import stderr, argv
from risotto.machine import load, TIRAMISU_CACHE, VALUES_CACHE, INFORMATIONS_CACHE, ROUGAIL_NAMESPACE, ROUGAIL_NAMESPACE_DESCRIPTION from risotto.machine import load, TIRAMISU_CACHE, VALUES_CACHE, INFORMATIONS_CACHE, ROUGAIL_NAMESPACE, ROUGAIL_NAMESPACE_DESCRIPTION
from tiramisu import Config from tiramisu import Config
@ -40,13 +40,12 @@ class RisottoInventory(object):
parser.add_argument('--host', action='store') parser.add_argument('--host', action='store')
parser.add_argument('--nocache', action='store_true') parser.add_argument('--nocache', action='store_true')
parser.add_argument('--debug', action='store_true') parser.add_argument('--debug', action='store_true')
parser.add_argument('--pretty_print', action='store_true')
self.args = parser.parse_args() self.args = parser.parse_args()
if self.args.debug: if self.args.debug:
global DEBUG global DEBUG
DEBUG = True DEBUG = True
def run(self): async def run(self):
if self.args.list and self.args.host: if self.args.list and self.args.host:
raise Exception('cannot have --list and --host together') raise Exception('cannot have --list and --host together')
if self.args.list or self.args.nocache: if self.args.list or self.args.nocache:
@ -56,20 +55,20 @@ class RisottoInventory(object):
remove(VALUES_CACHE) remove(VALUES_CACHE)
if isfile(INFORMATIONS_CACHE): if isfile(INFORMATIONS_CACHE):
remove(INFORMATIONS_CACHE) remove(INFORMATIONS_CACHE)
config = load(TIRAMISU_CACHE, config = await load(TIRAMISU_CACHE,
VALUES_CACHE, VALUES_CACHE,
INFORMATIONS_CACHE, INFORMATIONS_CACHE,
) )
if self.args.list: if self.args.list:
return self.do_inventory(config) return await self.do_inventory(config)
elif self.args.host: elif self.args.host:
return self.get_vars(config, self.args.host) return await self.get_vars(config, self.args.host)
raise Exception('pfff') raise Exception('pfff')
def do_inventory(self, async def do_inventory(self,
config: Config, config: Config,
) -> dict: ) -> dict:
servers = [subconfig.doc() for subconfig in config.option.list('optiondescription') if subconfig.information.get('module') == 'host'] servers = [await subconfig.option.doc() for subconfig in await config.option.list('optiondescription') if await subconfig.information.get('module') == 'host']
return dumps({ return dumps({
'group': { 'group': {
'hosts': servers, 'hosts': servers,
@ -82,54 +81,44 @@ class RisottoInventory(object):
} }
}) })
def get_vars(self, async def get_vars(self,
config: Config, config: Config,
host_name: str, host_name: str,
) -> dict: ) -> dict:
ret = {} ret = {}
rougailconfig = RougailConfig.copy() rougailconfig = RougailConfig.copy()
rougailconfig['variable_namespace'] = ROUGAIL_NAMESPACE rougailconfig['variable_namespace'] = ROUGAIL_NAMESPACE
rougailconfig['variable_namespace_description'] = ROUGAIL_NAMESPACE_DESCRIPTION rougailconfig['variable_namespace_description'] = ROUGAIL_NAMESPACE_DESCRIPTION
for subconfig in config.option.list('optiondescription'): for subconfig in await config.option.list('optiondescription'):
server_name = subconfig.description() server_name = await subconfig.option.description()
module_name = subconfig.option(subconfig.information.get('provider:global:module_name')).value.get() module_name = await subconfig.option(await subconfig.information.get('provider:global:module_name')).value.get()
if module_name == 'host' and server_name != host_name: if module_name == 'host' and server_name != host_name:
continue continue
engine = RougailSystemdTemplate(subconfig, rougailconfig) engine = RougailSystemdTemplate(subconfig, rougailconfig)
engine.load_variables(with_flatten=False) await engine.load_variables()
if module_name != 'host' and engine.rougail_variables_dict['general']['host'] != host_name: if module_name != 'host' and engine.rougail_variables_dict['general']['host'] != host_name:
continue continue
ret[server_name] = engine.rougail_variables_dict ret[server_name] = engine.rougail_variables_dict
ret['modules'] = config.information.get('modules') ret['modules'] = await config.information.get('modules')
ret['delete_old_image'] = False ret['delete_old_image'] = False
ret['configure_host'] = True ret['configure_host'] = True
ret['only_machine'] = None ret['only_machine'] = None
ret['copy_templates'] = False ret['copy_templates'] = False
ret['copy_tests'] = False ret['copy_tests'] = False
ret['host_install_dir'] = ret[host_name]['general']['host_install_dir'] ret['host_install_dir'] = ret[host_name].pop('host_install_dir')
return dumps(ret, cls=RougailEncoder) return dumps(ret, cls=RougailEncoder)
# Get the inventory. # Get the inventory.
def main(): async def main():
try: try:
inv = RisottoInventory() inv = RisottoInventory()
values = inv.run() values = await inv.run()
if inv.args.pretty_print: print(values)
from pprint import pprint
from json import loads
pprint(loads(values))
else:
print(values)
except Exception as err: except Exception as err:
if DEBUG: if DEBUG:
print_exc() print_exc()
print('---', file=stderr) print(err)
extra=''
else:
extra=f'\nmore informations with commandline "{" ".join(argv)} --debug"'
print(f'{err}{extra}', file=stderr)
exit(1)
main() run(main())

8
ansible/library/machinectl.py
View file

@ -154,7 +154,7 @@ def run_module():
module_args = dict( module_args = dict(
state=dict(type='str', required=True), state=dict(type='str', required=True),
machines=dict(type='list', required=True), machines=dict(type='list', required=True),
tls_machine=dict(type='str', required=False), tls_machine=dict(type='str', required=True),
) )
# seed the result dict in the object # seed the result dict in the object
@ -184,10 +184,10 @@ def run_module():
# manipulate or modify the state as needed (this is going to be the # manipulate or modify the state as needed (this is going to be the
# part where your module will do what it needs to do) # part where your module will do what it needs to do)
machines = module.params['machines'] machines = module.params['machines']
tls_machine = module.params.get('tls_machine') tls_machine = module.params['tls_machine']
if module.params['state'] == 'stopped': if module.params['state'] == 'stopped':
if tls_machine and tls_machine in machines: if tls_machine and tls_machine not in machines:
machines.remove(tls_machine) machines.append(tls_machine)
bus = SystemBus() bus = SystemBus()
result['changed'], errors = stop(bus, machines) result['changed'], errors = stop(bus, machines)
if errors: if errors:

13
ansible/machine.yml
View file

@ -1,2 +1,15 @@
- name: "Create SRV directory for {{ item.name}}"
file:
path: /var/lib/risotto/srv/{{ item.name }}
state: directory
mode: 0755
when: "item.srv"
- name: "Create journald directory for {{ item.name }}"
file:
path: /var/lib/risotto/journals/{{ item.name }}
state: directory
mode: 0755
- name: "Create informations for {{ item.name }}" - name: "Create informations for {{ item.name }}"
ansible.builtin.shell: "/usr/bin/echo {{ vars | modulename(item.name) }} > /var/lib/risotto/machines_informations/{{ item.name }}.image" ansible.builtin.shell: "/usr/bin/echo {{ vars | modulename(item.name) }} > /var/lib/risotto/machines_informations/{{ item.name }}.image"

34
ansible/machines.yml
View file

@ -1,46 +1,48 @@
- name: "Rebuild images" - name: "Rebuild images"
ansible.builtin.shell: "/usr/local/sbin/update_images {{ vars[vars['inventory_hostname']]['general']['tls_server'] }} do_not_start" ansible.builtin.shell: "/usr/local/sbin/update_images just_need_images"
register: ret register: ret
failed_when: ret.rc != 0 failed_when: ret.rc != 0
- name: "Stop machine TLS" - name: "Stop machine TLS"
machinectl: machinectl:
state: stopped state: stopped
machines: "{{ vars[vars['inventory_hostname']]['general']['tls_server'] }}" machines: "{{ build_host.tls_machine }}"
when: vars[vars['inventory_hostname']]['general']['tls_server'] in machines_changed tls_machine: "{{ build_host.tls_machine }}"
when: build_host.tls_machine in build_host.machines_changed
- name: "Remove TLS files directory" - name: "Remove TLS files directory"
file: file:
path: "/var/lib/risotto/configurations/{{ vars[vars['inventory_hostname']]['general']['tls_server'] }}" path: "/var/lib/risotto/configurations/{{ build_host.tls_machine }}"
state: absent state: absent
when: vars[vars['inventory_hostname']]['general']['tls_server'] in machines_changed when: build_host.tls_machine in build_host.machines_changed
- name: "Copy TLS configuration" - name: "Copy TLS configuration"
unarchive: unarchive:
src: /tmp/new_configurations/machines.tar src: /tmp/new_configurations/machines.tar
dest: "/var/lib/risotto/configurations/" dest: "/var/lib/risotto/configurations/"
include: "{{ vars[vars['inventory_hostname']]['general']['tls_server'] }}" include: "{{ build_host.tls_machine }}"
owner: root owner: root
group: root group: root
when: vars[vars['inventory_hostname']]['general']['tls_server'] in machines_changed when: build_host.tls_machine in build_host.machines_changed
- name: "Start machine TLS" - name: "Start machine TLS"
machinectl: machinectl:
state: started state: started
machines: "{{ vars[vars['inventory_hostname']]['general']['tls_server'] }}" machines: "{{ build_host.tls_machine }}"
when: vars[vars['inventory_hostname']]['general']['tls_server'] in machines_changed tls_machine: "{{ build_host.tls_machine }}"
when: build_host.tls_machine in build_host.machines_changed
- name: "Stop machines with new configuration {{ machines_changed }}" - name: "Stop machines with new configuration {{ build_host.machines_changed }}"
machinectl: machinectl:
state: stopped state: stopped
machines: "{{ machines_changed }}" machines: "{{ build_host.machines_changed }}"
tls_machine: "{{ vars[vars['inventory_hostname']]['general']['tls_server'] }}" tls_machine: "{{ build_host.tls_machine }}"
- name: "Remove files directory" - name: "Remove files directory"
file: file:
path: "/var/lib/risotto/configurations/{{ item }}" path: "/var/lib/risotto/configurations/{{ item }}"
state: absent state: absent
loop: "{{ machines_changed }}" loop: "{{ build_host.machines_changed }}"
- name: "Copy configuration" - name: "Copy configuration"
unarchive: unarchive:
@ -48,19 +50,19 @@
dest: /var/lib/risotto/configurations/ dest: /var/lib/risotto/configurations/
owner: root owner: root
group: root group: root
when: machines_changed when: build_host.machines_changed
- name: "Enable machines" - name: "Enable machines"
machinectl: machinectl:
state: enabled state: enabled
machines: "{{ vars | machineslist(only_name=True) }}" machines: "{{ vars | machineslist(only_name=True) }}"
tls_machine: "{{ vars[vars['inventory_hostname']]['general']['tls_server'] }}" tls_machine: "{{ build_host.tls_machine }}"
- name: "Start machines" - name: "Start machines"
machinectl: machinectl:
state: started state: started
machines: "{{ vars | machineslist(only_name=True) }}" machines: "{{ vars | machineslist(only_name=True) }}"
tls_machine: "{{ vars[vars['inventory_hostname']]['general']['tls_server'] }}" tls_machine: "{{ build_host.tls_machine }}"
- name: "Remove compressed files directory" - name: "Remove compressed files directory"
local_action: local_action:

4
ansible/playbook.yml
View file

@ -11,7 +11,7 @@
copy_templates: "{{ copy_templates }}" copy_templates: "{{ copy_templates }}"
register: build_host register: build_host
- name: "Change" - name: Print return information from the previous task
ansible.builtin.debug: ansible.builtin.debug:
var: build_host var: build_host
@ -31,5 +31,3 @@
# #
- name: "Install and apply configurations" - name: "Install and apply configurations"
include_tasks: machines.yml include_tasks: machines.yml
vars:
machines_changed: "{{ build_host.machines_changed }}"

18
ansible/sbin/backup_images
View file

@ -1,5 +1,6 @@
#!/bin/bash -ex #!/bin/bash -e
#START=$1
BACKUP_DIR="/root/backup" BACKUP_DIR="/root/backup"
MACHINES="" MACHINES=""
@ -14,14 +15,23 @@ done
cd /var/lib/risotto/srv/ cd /var/lib/risotto/srv/
mkdir -p "$BACKUP_DIR" mkdir -p "$BACKUP_DIR"
for machine in $MACHINES; do for machine in $MACHINES; do
# machinectl stop $machine || true
# while true; do
# machinectl status "$machine" > /dev/null 2>&1 || break
# sleep 1
# done
BACKUP_FILE="$BACKUP_DIR/backup_$machine.tar.bz2" BACKUP_FILE="$BACKUP_DIR/backup_$machine.tar.bz2"
rm -f "$BACKUP_FILE" rm -f "$BACKUP_FILE"
if [ -f "/var/lib/risotto/configurations/$machine/sbin/risotto_backup" ]; then if [ -f "/var/lib/risotto/configurations/$machine/sbin/risotto_backup" ]; then
machinectl -q shell $machine /usr/local/lib/sbin/risotto_backup machinectl -q shell $machine /usr/local/lib/sbin/risotto_backup
tar --ignore-failed-read -cJf $BACKUP_FILE $machine/backup tar -cJf $BACKUP_FILE $machine/backup
elif [ ! -f "/var/lib/risotto/configurations/$machine/no_risotto_backup" ]; then else
tar --ignore-failed-read -cJf $BACKUP_FILE $machine tar -cJf $BACKUP_FILE $machine
fi fi
done done
#if [ -z "$START" ]; then
# machinectl start $MACHINES
#fi
exit 0 exit 0

2
ansible/sbin/build_image
View file

@ -1,4 +1,4 @@
#!/bin/bash -ex #!/bin/bash -e
IMAGE_NAME=$1 IMAGE_NAME=$1

2
ansible/sbin/diagnose
View file

@ -38,7 +38,7 @@ for machine in $MACHINES; do
if ! echo "$STARTED" | grep -q " $machine "; then if ! echo "$STARTED" | grep -q " $machine "; then
echo echo
echo "========= $machine" echo "========= $machine"
machinectl -q shell $machine /usr/bin/systemctl is-system-running 2>/dev/null || systemctl status systemd-nspawn@$machine.service || true machinectl -q shell $machine /usr/bin/systemctl is-system-running 2>/dev/null || echo "not started"
fi fi
done done
echo $DEGRADED echo $DEGRADED

48
ansible/sbin/update_images
View file

@ -1,12 +1,5 @@
#!/bin/bash -e #!/bin/bash -e
TLS_SERVER=$1
if [ -z "$TLS_SERVER" ]; then
echo "$0 nom_tls_server"
exit 1
fi
DO_NOT_START=$2
REBOOT_EVERY_MONDAY=$3
# root dir configuration # root dir configuration
RISOTTO_DIR="/var/lib/risotto" RISOTTO_DIR="/var/lib/risotto"
RISOTTO_IMAGE_DIR="$RISOTTO_DIR/images" RISOTTO_IMAGE_DIR="$RISOTTO_DIR/images"
@ -23,65 +16,34 @@ ls /var/lib/risotto/images_files/ | while read image; do
if [ -d /var/lib/risotto/images_files/"$image" ]; then if [ -d /var/lib/risotto/images_files/"$image" ]; then
echo echo
echo "Install image $image" | tee -a /var/log/risotto/update_images.log echo "Install image $image" | tee -a /var/log/risotto/update_images.log
/usr/local/sbin/build_image "$image" | tee -a /var/log/risotto/update_images.log || (echo "PROBLEME" | tee -a /var/log/risotto/update_images.log; true) /usr/local/sbin/build_image "$image" "$1" | tee -a /var/log/risotto/update_images.log || (echo "PROBLEME" | tee -a /var/log/risotto/update_images.log; true)
fi fi
done done
idx=0
if [ -z "$DO_NOT_START" ]; then
machinectl reboot "$TLS_SERVER" || machinectl start "$TLS_SERVER"
while true; do
status=$(machinectl -q shell "$TLS_SERVER" /usr/bin/systemctl is-system-running 2>/dev/null || echo "not started")
if echo "$status" | grep -q degraded || echo "$status" | grep -q running; then
break
fi
idx=$((idx+1))
if [ $idx = 60 ]; then
echo "le serveur $TLS_SERVER n'a pas encore redémarré"
break
fi
sleep 2
done
fi
MACHINES="" MACHINES=""
for nspawn in $(ls /etc/systemd/nspawn/*.nspawn); do for nspawn in $(ls /etc/systemd/nspawn/*.nspawn); do
nspawn_file=$(basename "$nspawn") nspawn_file=$(basename $nspawn)
machine=${nspawn_file%.*} machine=${nspawn_file%.*}
MACHINES="$MACHINES$machine " MACHINES="$MACHINES$machine "
MACHINE_MACHINES_DIR="/var/lib/machines/$machine" MACHINE_MACHINES_DIR="/var/lib/machines/$machine"
IMAGE_NAME_RISOTTO_IMAGE_NAME="$(cat $RISOTTO_DIR/machines_informations/$machine.image)" IMAGE_NAME_RISOTTO_IMAGE_NAME="$(cat $RISOTTO_DIR/machines_informations/$machine.image)"
MACHINE_INFO="$RISOTTO_DIR/machines_informations/" MACHINE_INFO="$RISOTTO_DIR/machines_informations/"
VERSION_MACHINE="$MACHINE_INFO/$machine.version" VERSION_MACHINE="$MACHINE_INFO/$machine.version"
if [ -n "$REBOOT_EVERY_MONDAY" ] && [ "$(date +%u)" = 1 ]; then
# update TLS certificate every monday, so stop container
machinectl stop "$machine" 2> /dev/null || true
while true; do
machinectl status "$machine" > /dev/null 2>&1 || break
sleep 1
done
fi
if [ ! -d "$MACHINE_MACHINES_DIR" ]; then
rm -f "$VERSION_MACHINE"
fi
diff -q "$RISOTTO_IMAGE_DIR/$IMAGE_NAME_RISOTTO_IMAGE_NAME".version "$VERSION_MACHINE" &> /dev/null || ( diff -q "$RISOTTO_IMAGE_DIR/$IMAGE_NAME_RISOTTO_IMAGE_NAME".version "$VERSION_MACHINE" &> /dev/null || (
echo "Reinstall machine $machine" echo "Reinstall machine $machine"
machinectl stop "$machine" 2> /dev/null || true machinectl stop $machine || true
while true; do while true; do
machinectl status "$machine" > /dev/null 2>&1 || break machinectl status "$machine" > /dev/null 2>&1 || break
sleep 1 sleep 1
done done
rm -rf "$MACHINE_MACHINES_DIR" rm -rf "$MACHINE_MACHINES_DIR"
mkdir "$MACHINE_MACHINES_DIR" mkdir "$MACHINE_MACHINES_DIR"
cp -a --reflink=auto "$RISOTTO_IMAGE_DIR/$IMAGE_NAME_RISOTTO_IMAGE_NAME/"* "$MACHINE_MACHINES_DIR" cp -a --reflink=auto $RISOTTO_IMAGE_DIR/$IMAGE_NAME_RISOTTO_IMAGE_NAME/* $MACHINE_MACHINES_DIR
cp -a --reflink=auto "$RISOTTO_IMAGE_DIR/$IMAGE_NAME_RISOTTO_IMAGE_NAME".version "$VERSION_MACHINE" cp -a --reflink=auto "$RISOTTO_IMAGE_DIR/$IMAGE_NAME_RISOTTO_IMAGE_NAME".version "$VERSION_MACHINE"
) )
done done
if [ -z "$DO_NOT_START" ]; then if [ -z "$1" ]; then
echo "start $MACHINES"
machinectl start $MACHINES machinectl start $MACHINES
sleep 5
journalctl -n 100 --no-pager
diagnose diagnose
fi fi
exit 0 exit 0

103
sbin/risotto_display
View file

@ -1,5 +1,6 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
from asyncio import run
from tabulate import tabulate from tabulate import tabulate
from argparse import ArgumentParser from argparse import ArgumentParser
@ -17,19 +18,19 @@ def list_to_string(lst):
return lst return lst
def get_files_subelements(type_name, element, files_subelement, files_cols): async def get_files_subelements(type_name, element, files_subelement, files_cols):
data = {} data = {}
if not element.option('activate').value.get(): if not await element.option('activate').value.get():
return data return data
for subelement in files_subelement.values(): for subelement in files_subelement.values():
if subelement['type'] == 'subelement': if subelement['type'] == 'subelement':
try: try:
value = list_to_string(element.option(subelement['key']).value.get()) value = list_to_string(await element.option(subelement['key']).value.get())
# FIXME except AttributeError: # FIXME except AttributeError:
except Exception: except Exception:
value = '' value = ''
elif subelement['type'] == 'information': elif subelement['type'] == 'information':
value = element.information.get(subelement['key'], '') value = await element.information.get(subelement['key'], '')
elif subelement['type'] == 'none': elif subelement['type'] == 'none':
value = subelement['value'] value = subelement['value']
else: else:
@ -46,7 +47,7 @@ def get_files_subelements(type_name, element, files_subelement, files_cols):
return data return data
def services(config, values): async def services(config, values):
files_subelement = {'Source': {'key': 'source', 'type': 'information'}, files_subelement = {'Source': {'key': 'source', 'type': 'information'},
'Nom': {'key': 'name', 'type': 'subelement'}, 'Nom': {'key': 'name', 'type': 'subelement'},
'Variable': {'key': 'variable', 'type': 'subelement'}, 'Variable': {'key': 'variable', 'type': 'subelement'},
@ -56,26 +57,26 @@ def services(config, values):
'Moteur': {'key': 'engine', 'type': 'information'}, 'Moteur': {'key': 'engine', 'type': 'information'},
} }
disabled_services = [] disabled_services = []
for service in config.option.list(type="all"): for service in await config.option.list(type="all"):
doc = service.option.doc() doc = await service.option.doc()
files_lst = [] files_lst = []
files_cols = set() files_cols = set()
if not service.option('manage').value.get(): if not await service.option('manage').value.get():
doc += " - unmanaged" doc += " - unmanaged"
if not service.option('activate').value.get(): if not await service.option('activate').value.get():
disabled_services.append([doc]) disabled_services.append([doc])
else: else:
for type in service.list(type="all"): for type in await service.list(type="all"):
type_name = type.option.doc() type_name = await type.option.doc()
if type_name in ['files', 'overrides']: if type_name in ['files', 'overrides']:
for element in type.list(type="all"): for element in await type.list(type="all"):
data = get_files_subelements(type_name, element, files_subelement, files_cols) data = await get_files_subelements(type_name, element, files_subelement, files_cols)
if data: if data:
files_lst.append(data) files_lst.append(data)
elif type_name == 'manage': elif type_name == 'manage':
pass pass
elif type_name == 'activate': elif type_name == 'activate':
if not type.value.get(): if not await type.value.get():
doc += " - unactivated" doc += " - unactivated"
else: else:
print("FIXME " + type_name) print("FIXME " + type_name)
@ -88,19 +89,19 @@ def services(config, values):
values["Services désactivés"] = {'keys': ['Nom'], 'lst': disabled_services} values["Services désactivés"] = {'keys': ['Nom'], 'lst': disabled_services}
def table_leader(config, read_only): async def table_leader(config, read_only):
keys = ['Description'] keys = ['Description']
if read_only: if read_only:
keys.append('Cachée') keys.append('Cachée')
leadership_lst = config.list(type="all") leadership_lst = await config.list(type="all")
leader = leadership_lst.pop(0) leader = leadership_lst.pop(0)
leader_owner = leader.owner.get() leader_owner = await leader.owner.get()
follower_names = [follower.option.name() for follower in leadership_lst] follower_names = [await follower.option.name() for follower in leadership_lst]
doc = leader.option.doc() doc = await leader.option.doc()
properties = leader.property.get() properties = await leader.property.get()
if 'mandatory' in properties: if 'mandatory' in properties:
doc += '*' doc += '*'
name = leader.option.name() name = await leader.option.name()
lst = [[f'{doc} ({name})']] lst = [[f'{doc} ({name})']]
if read_only: if read_only:
if 'hidden' in properties: if 'hidden' in properties:
@ -108,7 +109,7 @@ def table_leader(config, read_only):
else: else:
hidden = '' hidden = ''
lst[0].append(hidden) lst[0].append(hidden)
for idx, leader_value in enumerate(leader.value.get()): for idx, leader_value in enumerate(await leader.value.get()):
keys.append(f'Valeur {idx}') keys.append(f'Valeur {idx}')
keys.append(f'Utilisateur {idx}') keys.append(f'Utilisateur {idx}')
lst[0].append(leader_value) lst[0].append(leader_value)
@ -116,11 +117,11 @@ def table_leader(config, read_only):
for follower_idx, follower_name in enumerate(follower_names): for follower_idx, follower_name in enumerate(follower_names):
follower_option = config.option(follower_name, idx) follower_option = config.option(follower_name, idx)
if idx == 0: if idx == 0:
doc = follower_option.option.doc() doc = await follower_option.option.doc()
properties = follower_option.property.get() properties = await follower_option.property.get()
if 'mandatory' in properties: if 'mandatory' in properties:
doc += '*' doc += '*'
name = follower_option.option.name() name = await follower_option.option.name()
lst.append([f'{doc} ({name})']) lst.append([f'{doc} ({name})'])
if read_only: if read_only:
if 'hidden' in properties: if 'hidden' in properties:
@ -129,48 +130,48 @@ def table_leader(config, read_only):
hidden = '' hidden = ''
lst[-1].append(hidden) lst[-1].append(hidden)
try: try:
lst[follower_idx + 1].append(list_to_string(follower_option.value.get())) lst[follower_idx + 1].append(list_to_string(await follower_option.value.get()))
lst[follower_idx + 1].append(follower_option.owner.get()) lst[follower_idx + 1].append(await follower_option.owner.get())
except PropertiesOptionError: except PropertiesOptionError:
pass pass
# leader = next leader_iter # leader = next leader_iter
# if master_values is None: # if master_values is None:
# master_values = subconfig.value.get() # master_values = await subconfig.value.get()
return {'keys': keys, 'lst': lst} return {'keys': keys, 'lst': lst}
def table(config, prefix_len, values, read_only): async def table(config, prefix_len, values, read_only):
lst = [] lst = []
for subconfig in config.option.list(type="all"): for subconfig in await config.option.list(type="all"):
# prefix = prefix_len * 2 * ' ' # prefix = prefix_len * 2 * ' '
# if subconfig.option.isoptiondescription(): # if await subconfig.option.isoptiondescription():
# prefix += '=>' # prefix += '=>'
# else: # else:
# prefix += '-' # prefix += '-'
# display_str = f'{prefix} {description}' # display_str = f'{prefix} {description}'
# if name != description: # if name != description:
# display_str = f'{display_str} ({name})' # display_str = f'{display_str} ({name})'
name = subconfig.option.name() name = await subconfig.option.name()
doc = subconfig.option.doc() doc = await subconfig.option.doc()
if prefix_len == 0 and ROUGAIL_NAMESPACE != name: if prefix_len == 0 and ROUGAIL_NAMESPACE != name:
doc = doc.capitalize() doc = doc.capitalize()
if prefix_len == 0 and name == 'services': if prefix_len == 0 and name == 'services':
values['Services'] = {} values['Services'] = {}
services(subconfig, values['Services']) await services(subconfig, values['Services'])
elif subconfig.option.isoptiondescription(): elif await subconfig.option.isoptiondescription():
od_name = f'{doc} ({(subconfig.option.path()).split(".", 1)[1]})' od_name = f'{doc} ({(await subconfig.option.path()).split(".", 1)[1]})'
values[od_name] = None values[od_name] = None
if subconfig.option.isleadership(): if await subconfig.option.isleadership():
values[od_name] = table_leader(subconfig, read_only) values[od_name] = await table_leader(subconfig, read_only)
else: else:
values[od_name] = table(subconfig, prefix_len + 1, values, read_only) values[od_name] = await table(subconfig, prefix_len + 1, values, read_only)
else: else:
value = list_to_string(subconfig.value.get()) value = list_to_string(await subconfig.value.get())
doc = subconfig.option.doc() doc = await subconfig.option.doc()
properties = subconfig.property.get() properties = await subconfig.property.get()
if 'mandatory' in properties: if 'mandatory' in properties:
doc += '*' doc += '*'
name = subconfig.option.name() name = await subconfig.option.name()
lst.append([f'{doc} ({name})', value]) lst.append([f'{doc} ({name})', value])
if read_only: if read_only:
if 'hidden' in properties: if 'hidden' in properties:
@ -178,7 +179,7 @@ def table(config, prefix_len, values, read_only):
else: else:
hidden = '' hidden = ''
lst[-1].append(hidden) lst[-1].append(hidden)
lst[-1].append(subconfig.owner.get()) lst[-1].append(await subconfig.owner.get())
keys = ['Description', 'Valeur'] keys = ['Description', 'Valeur']
if read_only: if read_only:
keys.append('Cachée') keys.append('Cachée')
@ -186,7 +187,7 @@ def table(config, prefix_len, values, read_only):
return {'keys': keys, 'lst': lst} return {'keys': keys, 'lst': lst}
def main(): async def main():
parser = ArgumentParser() parser = ArgumentParser()
parser.add_argument('server_name') parser.add_argument('server_name')
parser.add_argument('--read_only', action='store_true') parser.add_argument('--read_only', action='store_true')
@ -198,18 +199,18 @@ def main():
values = {} values = {}
server_name = args.server_name server_name = args.server_name
config = load(hide_secret=HIDE_SECRET, config = await load(hide_secret=HIDE_SECRET,
original_display_name=True, original_display_name=True,
valid_mandatories=args.read_only, valid_mandatories=args.read_only,
) )
if not args.read_only: if not args.read_only:
config.property.read_write() await config.property.read_write()
root_option = config.option(normalize_family(server_name)) root_option = config.option(normalize_family(server_name))
try: try:
root_option.option.get() await root_option.option.get()
except AttributeError: except AttributeError:
exit(f'Unable to find {server_name} configuration: {[o.option.description() for o in config.option.list(type="optiondescription")]}') exit(f'Unable to find {server_name} configuration: {[await o.option.description() for o in await config.option.list(type="optiondescription")]}')
table(root_option, 0, values, args.read_only) await table(root_option, 0, values, args.read_only)
for title, dico in values.items(): for title, dico in values.items():
if title == 'Services': if title == 'Services':
if not dico: if not dico:
@ -232,4 +233,4 @@ def main():
print(tabulate(dico['lst'], headers=dico['keys'], tablefmt="fancy_grid")) print(tabulate(dico['lst'], headers=dico['keys'], tablefmt="fancy_grid"))
main() run(main())

17
sbin/risotto_templates
View file

@ -1,12 +1,13 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
from asyncio import run
from argparse import ArgumentParser from argparse import ArgumentParser
from traceback import print_exc from traceback import print_exc
from risotto.machine import remove_cache, build_files, INSTALL_DIR from risotto.machine import remove_cache, build_files, INSTALL_DIR
def main(): async def main():
parser = ArgumentParser() parser = ArgumentParser()
parser.add_argument('server_name', nargs='?') parser.add_argument('server_name', nargs='?')
parser.add_argument('--nocache', action='store_true') parser.add_argument('--nocache', action='store_true')
@ -18,12 +19,12 @@ def main():
remove_cache() remove_cache()
try: try:
build_files(None, await build_files(None,
args.server_name, args.server_name,
False, False,
args.copy_tests, args.copy_tests,
template=args.template, template=args.template,
) )
except Exception as err: except Exception as err:
if args.debug: if args.debug:
print_exc() print_exc()
@ -31,4 +32,4 @@ def main():
print(f'templates generated in "{INSTALL_DIR}" directory') print(f'templates generated in "{INSTALL_DIR}" directory')
main() run(main())

68
src/risotto/image.py
View file

@ -19,8 +19,8 @@ class ModuleCfg():
self.depends = [] self.depends = []
self.manuals = [] self.manuals = []
self.tests = [] self.tests = []
#self.providers = [] self.providers = []
#self.suppliers = [] self.suppliers = []
def __repr__(self): def __repr__(self):
return str(vars(self)) return str(vars(self))
@ -131,10 +131,10 @@ class Modules:
if appname not in cfg.providers[provider]: if appname not in cfg.providers[provider]:
cfg.providers[provider].append(appname) cfg.providers[provider].append(appname)
supplier = app.get('supplier') supplier = app.get('supplier')
#if supplier: if supplier:
# self.suppliers.setdefault(supplier, []) self.suppliers.setdefault(supplier, [])
# if appname not in self.suppliers[supplier]: if appname not in self.suppliers[supplier]:
# self.suppliers[supplier].append(appname) self.suppliers[supplier].append(appname)
if 'distribution' in app and app['distribution']: if 'distribution' in app and app['distribution']:
distribution = appname distribution = appname
else: else:
@ -216,57 +216,33 @@ def applicationservice_copy(src_file: str,
copytree(src_file, dst_file) copytree(src_file, dst_file)
def valid_mandatories(config): async def valid_mandatories(config):
mandatories = config.value.mandatory() mandatories = await config.value.mandatory()
config.property.remove('mandatory') await config.property.pop('mandatory')
hidden = {} hidden = {}
variables = {} variables = {}
title = None title = None
if mandatories: if mandatories:
server_name = None server_name = None
for mandatory_option in mandatories: for mandatory in mandatories:
path_server_name, path = mandatory_option.path().split('.', 1) path_server_name, path = mandatory.split('.', 1)
var_server_name = config.option(path_server_name).description() var_server_name = await config.option(path_server_name).option.description()
if server_name != var_server_name: if server_name != var_server_name:
server_name = var_server_name server_name = var_server_name
title = f'=== Missing variables for {server_name} ===' title = f'=== Missing variables for {server_name} ==='
text = mandatory_option.doc() suboption = config.option(mandatory)
text = await suboption.option.doc()
msg = f' - {text} ({path})' msg = f' - {text} ({path})'
supplier = mandatory_option.information.get('supplier', None) supplier = await suboption.information.get('supplier', None)
if supplier: if supplier:
msg += f' you could add a service that provides "{supplier}"' msg += f' you could add a service that provides "{supplier}"'
if mandatory_option.isfollower(): try:
leader = mandatory_option.leader() await config.option(mandatory).value.get()
try: variables.setdefault(title, []).append(msg)
leader_value = leader.value.get() except PropertiesOptionError as err:
except PropertiesOptionError as err: if 'hidden' not in err.proptype:
if 'hidden' not in err.proptype: raise PropertiesOptionError(err)
raise err from err hidden.setdefault(title, []).append(msg)
hidden.setdefault(title, []).append(msg)
else:
config.property.add('mandatory')
for idx in range(mandatory_option.value.len()):
try:
config.option(mandatory_option.path(), idx).value.get()
except PropertiesOptionError as err:
path = leader.path()
spath = path.split('.', 1)[1]
submsg = f'{msg} at index {idx} (value of leader "{leader.doc()}" ({spath}) is "{leader_value[idx]}")'
if 'hidden' in err.proptype:
hidden.setdefault(title, []).append(submsg)
elif 'mandatory' in err.proptype:
variables.setdefault(title, []).append(submsg)
else:
raise err from err
config.property.remove('mandatory')
else:
try:
mandatory_option.value.get()
variables.setdefault(title, []).append(msg)
except PropertiesOptionError as err:
if 'hidden' not in err.proptype:
raise err from err
hidden.setdefault(title, []).append(msg)
if not variables: if not variables:
variables = hidden variables = hidden
return variables return variables

310
src/risotto/machine.py
View file

@ -5,11 +5,11 @@ from .rougail.annotator import calc_providers, calc_providers_global, calc_provi
from rougail import RougailConfig, RougailConvert from rougail import RougailConfig, RougailConvert
from os import remove, makedirs, listdir, chmod from os import remove, makedirs, listdir, chmod
from os.path import isfile, isdir, abspath, join, dirname from os.path import isfile, isdir, abspath, join, dirname
from pickle import dump as pickle_dump, load as pickle_load from json import dump as json_dump, load as json_load
from yaml import load as yaml_load, SafeLoader from yaml import load as yaml_load, SafeLoader
from ipaddress import IPv4Interface, ip_network from ipaddress import ip_network
# #
from tiramisu import Config, valid_network_netmask, valid_ip_netmask, valid_broadcast, valid_in_network, valid_not_equal, calc_value, calc_value_property_help from tiramisu import Config, valid_network_netmask, valid_ip_netmask, valid_broadcast, valid_in_network, valid_not_equal, calc_value
from rougail.utils import normalize_family from rougail.utils import normalize_family
from rougail import RougailSystemdTemplate from rougail import RougailSystemdTemplate
from shutil import copy2, copytree, rmtree from shutil import copy2, copytree, rmtree
@ -29,8 +29,8 @@ def tiramisu_display_name(kls,
CONFIG_FILE = 'servers.yml' CONFIG_FILE = 'servers.yml'
TIRAMISU_CACHE = 'tiramisu_cache.py' TIRAMISU_CACHE = 'tiramisu_cache.py'
VALUES_CACHE = 'values_cache.pickle' VALUES_CACHE = 'values_cache.json'
INFORMATIONS_CACHE = 'informations_cache.pickle' INFORMATIONS_CACHE = 'informations_cache.json'
INSTALL_DIR = RISOTTO_CONFIG['directories']['dest'] INSTALL_DIR = RISOTTO_CONFIG['directories']['dest']
INSTALL_CONFIG_DIR = 'configurations' INSTALL_CONFIG_DIR = 'configurations'
INSTALL_TMPL_DIR= 'templates' INSTALL_TMPL_DIR= 'templates'
@ -47,7 +47,6 @@ FUNCTIONS = {'calc_providers': calc_providers,
'valid_in_network': valid_in_network, 'valid_in_network': valid_in_network,
'valid_not_equal': valid_not_equal, 'valid_not_equal': valid_not_equal,
'calc_value': calc_value, 'calc_value': calc_value,
'calc_value_property_help': calc_value_property_help,
'normalize_family': normalize_family, 'normalize_family': normalize_family,
} }
@ -89,28 +88,28 @@ def remove_cache():
remove(INFORMATIONS_CACHE) remove(INFORMATIONS_CACHE)
def templates(server_name, async def templates(server_name,
config, config,
just_copy=False, just_copy=False,
copy_manuals=False, copy_manuals=False,
template=None, template=None,
extra_variables=None, extra_variables=None,
): ):
subconfig = config.option(normalize_family(server_name)) subconfig = config.option(normalize_family(server_name))
try: try:
subconfig.get() await subconfig.option.get()
except: except:
servers = [server.description() for server in config.list('optiondescription')] servers = [await server.option.description() for server in await config.option.list('optiondescription')]
raise Exception(f'cannot find server name "{server_name}": {servers}') raise Exception(f'cannot find server name "{server_name}": {servers}')
rougailconfig = RougailConfig.copy() rougailconfig = RougailConfig.copy()
rougailconfig['variable_namespace'] = ROUGAIL_NAMESPACE rougailconfig['variable_namespace'] = ROUGAIL_NAMESPACE
rougailconfig['variable_namespace_description'] = ROUGAIL_NAMESPACE_DESCRIPTION rougailconfig['variable_namespace_description'] = ROUGAIL_NAMESPACE_DESCRIPTION
rougailconfig['tmp_dir'] = 'tmp' rougailconfig['tmp_dir'] = 'tmp'
rougailconfig['templates_dir'] = subconfig.information.get('templates_dir') rougailconfig['templates_dir'] = await subconfig.information.get('templates_dir')
rougailconfig['patches_dir'] = subconfig.information.get('patches_dir') rougailconfig['patches_dir'] = await subconfig.information.get('patches_dir')
rougailconfig['functions_file'] = subconfig.information.get('functions_files') rougailconfig['functions_file'] = await subconfig.information.get('functions_files')
module = subconfig.information.get('module') module = await subconfig.information.get('module')
is_host = module == 'host' is_host = module == 'host'
if is_host: if is_host:
rougailconfig['systemd_tmpfile_delete_before_create'] = True rougailconfig['systemd_tmpfile_delete_before_create'] = True
@ -139,15 +138,15 @@ def templates(server_name,
engine.engines[eng] = engine.engines['none'] engine.engines[eng] = engine.engines['none']
try: try:
if not template: if not template:
engine.instance_files(extra_variables=extra_variables) await engine.instance_files(extra_variables=extra_variables)
else: else:
engine.instance_file(template, extra_variables=extra_variables) await engine.instance_file(template, extra_variables=extra_variables)
except Exception as err: except Exception as err:
print() print()
print(f'=== Configuration: {server_name} ===') print(f'=== Configuration: {server_name} ===')
try: try:
values = subconfig.value.dict() values = await subconfig.value.dict()
value_pprint(values, subconfig) await value_pprint(values, subconfig)
except: except:
pass pass
raise err from err raise err from err
@ -160,17 +159,17 @@ def templates(server_name,
if copy_manuals and not is_host: if copy_manuals and not is_host:
dest_dir = join(INSTALL_DIR, INSTALL_IMAGES_DIR, module) dest_dir = join(INSTALL_DIR, INSTALL_IMAGES_DIR, module)
if not isdir(dest_dir): if not isdir(dest_dir):
for manual in subconfig.information.get('manuals_dirs'): for manual in await subconfig.information.get('manuals_dirs'):
for filename in listdir(manual): for filename in listdir(manual):
src_file = join(manual, filename) src_file = join(manual, filename)
dst_file = join(dest_dir, filename) dst_file = join(dest_dir, filename)
copy(src_file, dst_file) copy(src_file, dst_file)
copy_tests = config.information.get('copy_tests') copy_tests = await config.information.get('copy_tests')
if copy_tests and not is_host: if copy_tests and not is_host:
dest_dir = join(INSTALL_DIR, INSTALL_TESTS_DIR, module) dest_dir = join(INSTALL_DIR, INSTALL_TESTS_DIR, module)
if not isdir(dest_dir): if not isdir(dest_dir):
for tests in subconfig.information.get('tests_dirs'): for tests in await subconfig.information.get('tests_dirs'):
for filename in listdir(tests): for filename in listdir(tests):
src_file = join(tests, filename) src_file = join(tests, filename)
dst_file = join(dest_dir, filename) dst_file = join(dest_dir, filename)
@ -179,6 +178,7 @@ def templates(server_name,
class Loader: class Loader:
def __init__(self, def __init__(self,
clean_directories,
hide_secret, hide_secret,
original_display_name, original_display_name,
valid_mandatories, valid_mandatories,
@ -188,6 +188,10 @@ class Loader:
self.original_display_name = original_display_name self.original_display_name = original_display_name
self.valid_mandatories = valid_mandatories self.valid_mandatories = valid_mandatories
self.config_file = config_file self.config_file = config_file
if clean_directories:
if isdir(INSTALL_DIR):
rmtree(INSTALL_DIR)
makedirs(INSTALL_DIR)
def load_tiramisu_file(self): def load_tiramisu_file(self):
"""Load config file (servers.yml) and build tiramisu file with dataset informations """Load config file (servers.yml) and build tiramisu file with dataset informations
@ -220,54 +224,28 @@ class Loader:
rougail = RougailConvert(cfg) rougail = RougailConvert(cfg)
for host_name, datas in self.servers_json['hosts'].items(): for host_name, datas in self.servers_json['hosts'].items():
for server_name, server_datas in datas['servers'].items(): for server_name, server_datas in datas['servers'].items():
if 'provider_zone' not in server_datas and 'zones_name' not in server_datas: for zone in server_datas['informations']['zones_name']:
raise Exception(f'cannot find "zones_name" attribute for server "{server_name}"')
if 'provider_zone' in server_datas:
zones_name.setdefault(server_datas['provider_zone'], []).append(server_name)
if 'zones_name' not in server_datas:
server_datas['zones_name'] = []
if server_datas['provider_zone'] in server_datas['zones_name']:
raise Exception(_('provider_zone "{server_datas["provider_zone"]}" must not be in "zones" "{server_datas["zones_name"]}"'))
# external zone is better in first place
if server_datas['zones_name'] and self.servers_json['zones']['external_zone'] == server_datas['zones_name'][0]:
server_datas['zones_name'].append(server_datas['provider_zone'])
else:
server_datas['zones_name'].insert(0, server_datas['provider_zone'])
# if server_datas['zones_name'] and server_datas['provider_zone'] == self.servers_json['zones']['external_zone']:
# server_datas['zones_name'].insert(0, server_datas['provider_zone'])
# else:
# server_datas['zones_name'].append(server_datas['provider_zone'])
for zone in server_datas['zones_name']:
zones_name.setdefault(zone, []).append(server_name) zones_name.setdefault(zone, []).append(server_name)
self.zones = {} self.zones = {}
zones_network = ip_network(self.servers_json['zones']['network']) zones_network = ip_network(self.servers_json['zones']['network'])
zone_start_ip = zones_network.network_address zone_start_ip = zones_network.network_address
domain_name = self.servers_json['zones']['prefix_domain_name'] domain_name = self.servers_json['zones']['prefix_domain_name']
for zone_name in zones_name: for idx, zone_name in enumerate(zones_name):
len_zone = len(zones_name[zone_name]) sub_network = ip_network(f'{zone_start_ip}/28')
for zone_cidr in [29, 28, 27, 26]: if not sub_network.subnet_of(zones_network):
try: raise Exception('not enough IP available')
sub_network = ip_network(f'{zone_start_ip}/{zone_cidr}') if sub_network.num_addresses < len(zones_name[zone_name]):
except ValueError: #FIXME should try to increase network!
# calc network address for this mask raise Exception(f'network too small for zone {zone_name}')
zone_start_ip = IPv4Interface(f'{zone_start_ip}/{zone_cidr}').network.broadcast_address + 1 if idx == 0:
sub_network = ip_network(f'{zone_start_ip}/{zone_cidr}')
if not sub_network.subnet_of(zones_network):
raise Exception('not enough IP available')
length = sub_network.num_addresses - 3 # network + broadcast + host
if length >= len_zone:
break
else:
raise Exception(f'network too small for zone "{zone_name}" ({sub_network.num_addresses - 2} < {len_zone})')
if self.servers_json['zones']['external_zone'] == zone_name:
zone_domaine_name = domain_name zone_domaine_name = domain_name
else: else:
zone_domaine_name = zone_name + '.' + domain_name zone_domaine_name = zone_name + '.' + domain_name
network = sub_network.network_address network = sub_network.network_address
self.zones[zone_name] = {'domain_name': zone_domaine_name, self.zones[zone_name] = {'domain_name': zone_domaine_name,
'network': str(sub_network), 'network': str(sub_network),
'host_ip': str(network + 1), 'host_ip': str(network + 1),
'length': length,
'start_ip': str(network + 2) 'start_ip': str(network + 2)
} }
zone_start_ip = str(sub_network.broadcast_address + 1) zone_start_ip = str(sub_network.broadcast_address + 1)
@ -288,13 +266,9 @@ class Loader:
# load host # load host
module_info = modules.get('host') module_info = modules.get('host')
tls_host_name = f'{server_name}.{self.zones[list(self.zones)[0]]["domain_name"]}'
cfg['risotto_globals'][host_name] = {'global:server_name': host_name, cfg['risotto_globals'][host_name] = {'global:server_name': host_name,
'global:server_names': [host_name for zone in self.zones],
'global:zones_name': list(self.zones),
'global:module_name': 'host', 'global:module_name': 'host',
'global:host_install_dir': abspath(INSTALL_DIR), 'global:host_install_dir': abspath(INSTALL_DIR),
'global:tls_server': tls_host_name,
} }
functions_files |= set(module_info.functions_file) functions_files |= set(module_info.functions_file)
self.load_dictionaries(cfg, self.load_dictionaries(cfg,
@ -306,7 +280,7 @@ class Loader:
modules_info = {} modules_info = {}
for server_name, server_datas in datas['servers'].items(): for server_name, server_datas in datas['servers'].items():
module_info = modules.get(server_datas['applicationservice']) module_info = modules.get(server_datas['applicationservice'])
zones_name = server_datas['zones_name'] zones_name = server_datas['informations']['zones_name']
values = [f'{server_name}.{self.zones[zone_name]["domain_name"]}' for zone_name in zones_name] values = [f'{server_name}.{self.zones[zone_name]["domain_name"]}' for zone_name in zones_name]
if server_datas['applicationservice'] == 'tls': if server_datas['applicationservice'] == 'tls':
true_host_name = f'{server_name}.{self.zones[list(self.zones)[0]]["domain_name"]}' true_host_name = f'{server_name}.{self.zones[list(self.zones)[0]]["domain_name"]}'
@ -318,10 +292,7 @@ class Loader:
'global:zones_name': zones_name, 'global:zones_name': zones_name,
'global:zones_list': list(range(len(zones_name))), 'global:zones_list': list(range(len(zones_name))),
'global:module_name': server_datas['applicationservice'], 'global:module_name': server_datas['applicationservice'],
'global:prefix_domain_name': self.servers_json['zones']['prefix_domain_name']
} }
if 'provider_zone' in server_datas:
cfg['risotto_globals'][true_host_name]['global:provider_zone'] = server_datas['provider_zone']
server_datas['server_name'] = true_host_name server_datas['server_name'] = true_host_name
functions_files |= set(module_info.functions_file) functions_files |= set(module_info.functions_file)
self.load_dictionaries(cfg, self.load_dictionaries(cfg,
@ -338,24 +309,26 @@ class Loader:
zones = set() zones = set()
dns_module_name = None dns_module_name = None
for host in self.servers_json['hosts'].values(): for host in self.servers_json['hosts'].values():
zones = [self.servers_json['zones']['external_zone'], None] zones = [None, None]
for server_name, datas in host['servers'].items(): for server_name, datas in host['servers'].items():
if not 'applicationservice' in datas:
raise Exception(f'cannot find applicationservice for "{server_name}"')
if datas['applicationservice'] == 'tls': if datas['applicationservice'] == 'tls':
raise Exception(f'forbidden module name "tls" for server "{server_name}"') raise Exception(f'forbidden module name "tls" for server {server_name}')
#FIXME use provider! #FIXME use provider!
if datas['applicationservice'] == 'nginx-reverse-proxy' and len(datas['zones_name']) > 0: if datas['applicationservice'] == 'nginx-reverse-proxy' and len(datas['informations']['zones_name']) > 0:
if dns_module_name: if dns_module_name:
break break
zones[1] = datas['provider_zone'] # always add tls machine in second zone of reverse proxy
zones[1] = datas['informations']['zones_name'][0]
if datas['applicationservice'] == 'unbound':
# always add tls machine in second zone of reverse proxy
zones[0] = datas['informations']['zones_name'][0]
if None in zones: if None in zones:
zones = [] zones = []
else: else:
if zones[0] == zones[1]: if zones[0] == zones[1]:
zones = [zones[0]] zones = [zones[0]]
host['servers']['tls'] = {'applicationservice': 'tls', host['servers']['tls'] = {'applicationservice': 'tls',
'zones_name': list(zones), 'informations': {'zones_name': list(zones)},
} }
def load_dictionaries(self, cfg, module_info, server_name, rougail): def load_dictionaries(self, cfg, module_info, server_name, rougail):
@ -371,7 +344,7 @@ class Loader:
self.manuals_dirs[server_name] = module_info.manuals self.manuals_dirs[server_name] = module_info.manuals
self.tests_dirs[server_name] = module_info.tests self.tests_dirs[server_name] = module_info.tests
def tiramisu_file_to_tiramisu(self): async def tiramisu_file_to_tiramisu(self):
# l # l
tiramisu_space = FUNCTIONS.copy() tiramisu_space = FUNCTIONS.copy()
try: try:
@ -383,78 +356,78 @@ class Loader:
display_name = None display_name = None
else: else:
display_name = tiramisu_display_name display_name = tiramisu_display_name
self.config = Config(tiramisu_space['option_0'], self.config = await Config(tiramisu_space['option_0'],
display_name=display_name, display_name=display_name,
) )
def load_values_and_informations(self): async def load_values_and_informations(self):
config = self.config config = self.config
config.property.read_write() await config.property.read_write()
config.property.remove('validator') await config.property.pop('validator')
config.property.remove('cache') await config.property.pop('cache')
load_zones(self.zones, self.servers_json['hosts']) load_zones(self.zones, self.servers_json['hosts'])
config.information.set('zones', self.zones) await config.information.set('zones', self.zones)
for host_name, hosts_datas in self.servers_json['hosts'].items(): for host_name, hosts_datas in self.servers_json['hosts'].items():
information = config.option(normalize_family(host_name)).information information = config.option(normalize_family(host_name)).information
information.set('module', 'host') await information.set('module', 'host')
information.set('templates_dir', self.templates_dir[host_name]) await information.set('templates_dir', self.templates_dir[host_name])
information.set('patches_dir', self.patches_dir[host_name]) await information.set('patches_dir', self.patches_dir[host_name])
information.set('functions_files', self.functions_files[host_name]) await information.set('functions_files', self.functions_files[host_name])
self.set_values(host_name, config, hosts_datas) await self.set_values(host_name, config, hosts_datas)
for datas in hosts_datas['servers'].values(): for datas in hosts_datas['servers'].values():
server_name = datas['server_name'] server_name = datas['server_name']
information = config.option(normalize_family(server_name)).information information = config.option(normalize_family(server_name)).information
information.set('module', datas['applicationservice']) await information.set('module', datas['applicationservice'])
information.set('templates_dir', self.templates_dir[server_name]) await information.set('templates_dir', self.templates_dir[server_name])
information.set('patches_dir', self.patches_dir[server_name]) await information.set('patches_dir', self.patches_dir[server_name])
information.set('functions_files', self.functions_files[server_name]) await information.set('functions_files', self.functions_files[server_name])
information.set('manuals_dirs', self.manuals_dirs[server_name]) await information.set('manuals_dirs', self.manuals_dirs[server_name])
information.set('tests_dirs', self.tests_dirs[server_name]) await information.set('tests_dirs', self.tests_dirs[server_name])
self.set_values(server_name, config, datas) await self.set_values(server_name, config, datas)
config.information.set('copy_tests', False) await config.information.set('copy_tests', False)
# FIXME only one host_name is supported # FIXME only one host_name is supported
config.information.set('modules', self.modules[host_name]) await config.information.set('modules', self.modules[host_name])
# config.information.set('modules', {module_name: module_info.depends for module_name, module_info in self.module_infos.items() if module_name in modules}) # await config.information.set('modules', {module_name: module_info.depends for module_name, module_info in self.module_infos.items() if module_name in modules})
config.property.add('cache') await config.property.add('cache')
if self.valid_mandatories: if self.valid_mandatories:
messages = valid_mandatories(config) messages = await valid_mandatories(config)
if messages: if messages:
msg = '' msg = ''
for title, variables in messages.items(): for title, variables in messages.items():
msg += '\n' + title + '\n' msg += '\n' + title + '\n'
msg += '\n'.join(variables) msg += '\n'.join(variables)
raise Exception(msg) raise Exception(msg)
config.property.read_only() await config.property.read_only()
with open(VALUES_CACHE, 'wb') as fh: with open(VALUES_CACHE, 'w') as fh:
pickle_dump(config.value.exportation(), fh) json_dump(await config.value.exportation(), fh)
with open(INFORMATIONS_CACHE, 'wb') as fh: with open(INFORMATIONS_CACHE, 'w') as fh:
pickle_dump(config.information.exportation(), fh) json_dump(await config.information.exportation(), fh)
def set_values(self, async def set_values(self,
server_name, server_name,
config, config,
datas, datas,
): ):
if 'values' not in datas: if 'values' not in datas:
return return
if not isinstance(datas['values'], dict): if not isinstance(datas['values'], dict):
raise Exception(f'Values of "{server_name}" are not a dict: {datas["values"]}') raise Exception(f'Values of "{server_name}" are not a dict: {datas["values"]}')
server_path = normalize_family(server_name) server_path = normalize_family(server_name)
config.owner.set(self.config_file) await config.owner.set(self.config_file)
for vpath, value in datas['values'].items(): for vpath, value in datas['values'].items():
path = f'{server_path}.{vpath}' path = f'{server_path}.{vpath}'
try: try:
if isinstance(value, dict): if isinstance(value, dict):
for idx, val in value.items(): for idx, val in value.items():
config.option(path, int(idx)).value.set(val) await config.option(path, int(idx)).value.set(val)
else: else:
config.option(path).value.set(value) await config.option(path).value.set(value)
except Exception as err: except Exception as err:
value_pprint(config.value.dict(), config) await value_pprint(await config.value.dict(), config)
error_msg = f'cannot configure variable {vpath} for server "{server_name}": {err}' error_msg = f'cannot configure variable {vpath} for server "{server_name}": {err}'
raise Exception(error_msg) from err raise Exception(error_msg) from err
config.owner.set('user') await config.owner.set('user')
class LoaderCache(Loader): class LoaderCache(Loader):
@ -462,60 +435,63 @@ class LoaderCache(Loader):
with open(TIRAMISU_CACHE) as fh: with open(TIRAMISU_CACHE) as fh:
self.tiram_obj = fh.read() self.tiram_obj = fh.read()
def load_values_and_informations(self): async def load_values_and_informations(self):
with open(VALUES_CACHE, 'rb') as fh: with open(VALUES_CACHE, 'r') as fh:
self.config.value.importation(pickle_load(fh)) await self.config.value.importation(json_load(fh))
with open(INFORMATIONS_CACHE, 'rb') as fh: with open(INFORMATIONS_CACHE, 'r') as fh:
self.config.information.importation(pickle_load(fh)) informations = json_load(fh)
# null is not a valid key in json => 'null'
informations[None] = informations.pop('null')
await self.config.information.importation(informations)
def load(hide_secret=False, async def load(clean_directories=False,
original_display_name: bool=False, hide_secret=False,
valid_mandatories: bool=True, original_display_name: bool=False,
copy_tests: bool=False, valid_mandatories: bool=True,
): copy_tests: bool=False,
):
if isfile(TIRAMISU_CACHE) and isfile(VALUES_CACHE) and isfile(INFORMATIONS_CACHE): if isfile(TIRAMISU_CACHE) and isfile(VALUES_CACHE) and isfile(INFORMATIONS_CACHE):
loader_obj = LoaderCache loader_obj = LoaderCache
else: else:
loader_obj = Loader loader_obj = Loader
loader = loader_obj(hide_secret, loader = loader_obj(clean_directories,
hide_secret,
original_display_name, original_display_name,
valid_mandatories, valid_mandatories,
) )
loader.load_tiramisu_file() loader.load_tiramisu_file()
loader.tiramisu_file_to_tiramisu() await loader.tiramisu_file_to_tiramisu()
loader.load_values_and_informations() await loader.load_values_and_informations()
config = loader.config config = loader.config
config.property.read_only() await config.property.read_only()
config.information.set('copy_tests', copy_tests) await config.information.set('copy_tests', copy_tests)
config.cache.reset() await config.cache.reset()
return config return config
def build_files(hostname: str, async def build_files(hostname: str,
only_machine: str, only_machine: str,
just_copy: bool, just_copy: bool,
copy_tests: bool, copy_tests: bool,
template: str=None, template: str=None,
) -> None: ) -> None:
if isdir(INSTALL_DIR):
rmtree(INSTALL_DIR)
makedirs(INSTALL_DIR)
with open(CONFIG_FILE, 'r') as server_fh: with open(CONFIG_FILE, 'r') as server_fh:
servers_json = yaml_load(server_fh, Loader=SafeLoader) servers_json = yaml_load(server_fh, Loader=SafeLoader)
config = load(copy_tests=copy_tests) config = await load(copy_tests=copy_tests)
machines = [subconfig.description() for subconfig in config.option.list(type='optiondescription')] machines = [await subconfig.option.description() for subconfig in await config.option.list(type='optiondescription')]
certificates = {'certificates': {}, certificates = {'certificates': {},
'configuration': servers_json['certificates'], 'configuration': servers_json['certificates'],
} }
# get certificates informations # get certificates informations
tls_machine = config.option(f'{normalize_family(hostname)}.general.tls_server').value.get() tls_machine = None
for machine in machines: for machine in machines:
if machine == tls_machine: if machine.startswith('tls.'):
tls_machine = machine
continue continue
if hostname is None: if hostname is None:
# FIXME multi host! # FIXME multi host!
hostname = config.option(normalize_family(machine)).option('general.host_name').value.get() hostname = await config.option(normalize_family(machine)).option('general.host_name').value.get()
if just_copy: if just_copy:
continue continue
is_host = machine == hostname is_host = machine == hostname
@ -524,24 +500,24 @@ def build_files(hostname: str,
machine_config = config.option(normalize_family(machine)) machine_config = config.option(normalize_family(machine))
certificate_names = [] certificate_names = []
private_names = [] private_names = []
for service in machine_config.option('services').list('optiondescription'): for service in await machine_config.option('services').option.list('optiondescription'):
if not service.option('activate').value.get(): if not await service.option('activate').value.get():
continue continue
# if service.option('manage').value.get(): # if await service.option('manage').value.get():
# certificate_type = 'server' # certificate_type = 'server'
# else: # else:
# certificate_type = 'client' # certificate_type = 'client'
tls_ca_directory = machine_config.option('general.tls_ca_directory').value.get() tls_ca_directory = await machine_config.option('general.tls_ca_directory').value.get()
tls_cert_directory = machine_config.option('general.tls_cert_directory').value.get() tls_cert_directory = await machine_config.option('general.tls_cert_directory').value.get()
tls_key_directory = machine_config.option('general.tls_key_directory').value.get() tls_key_directory = await machine_config.option('general.tls_key_directory').value.get()
try: try:
for certificate in service.option('certificates').list('all'): for certificate in await service.option('certificates').option.list('all'):
if not certificate.option('activate').value.get(): if not await certificate.option('activate').value.get():
continue continue
certificate_data = {key.rsplit('.', 1)[1]: value for key, value in certificate.value.dict().items()} certificate_data = await certificate.value.dict()
certificate_data['type'] = certificate.information.get('type') certificate_data['type'] = await certificate.information.get('type')
certificate_data['authority'] = join(tls_ca_directory, certificate.information.get('authority') + '.crt') certificate_data['authority'] = join(tls_ca_directory, await certificate.information.get('authority') + '.crt')
certificate_data['format'] = certificate.information.get('format') certificate_data['format'] = await certificate.information.get('format')
is_list_name = isinstance(certificate_data['name'], list) is_list_name = isinstance(certificate_data['name'], list)
is_list_domain = isinstance(certificate_data['domain'], list) is_list_domain = isinstance(certificate_data['domain'], list)
if is_list_name != is_list_domain: if is_list_name != is_list_domain:
@ -550,7 +526,7 @@ def build_files(hostname: str,
certificate_data['provider'] = 'autosigne' certificate_data['provider'] = 'autosigne'
if is_list_name: if is_list_name:
if len(certificate_data['name']) != len(certificate_data['domain']): if len(certificate_data['name']) != len(certificate_data['domain']):
raise Exception('certificate name and domain name must have same length') raise Exception('certificate name and domain name must have same lenght')
for idx, certificate_name in enumerate(certificate_data['name']): for idx, certificate_name in enumerate(certificate_data['name']):
cert_data = certificate_data.copy() cert_data = certificate_data.copy()
if certificate_data['format'] == 'cert_key': if certificate_data['format'] == 'cert_key':
@ -589,19 +565,19 @@ def build_files(hostname: str,
continue continue
if only_machine and only_machine != machine: if only_machine and only_machine != machine:
continue continue
templates(machine, await templates(machine,
config, config,
just_copy=just_copy, just_copy=just_copy,
copy_manuals=True, copy_manuals=True,
template=template, template=template,
extra_variables=certificates, extra_variables=certificates,
) )
is_host = machine == hostname is_host = machine == hostname
if is_host: if is_host:
directories[machine] = '/usr/local/lib' directories[machine] = '/usr/local/lib'
elif not just_copy: elif not just_copy:
machine_config = config.option(normalize_family(machine)) machine_config = config.option(normalize_family(machine))
directories[machine] = machine_config.option('general.config_dir').value.get() directories[machine] = await machine_config.option('general.config_dir').value.get()
if only_machine: if only_machine:
return directories return directories
if only_machine: if only_machine:

447
src/risotto/rougail/annotator.py
View file

@ -26,19 +26,19 @@ def _parse_kwargs(provider, dns, kwargs, index=None):
continue continue
elif data['dns'] not in dns: elif data['dns'] not in dns:
continue continue
# del data['dns'] del data['dns']
yield data yield data
@multi_function @multi_function
def calc_providers_global(provider, multi, unique, value, suffix=None): def calc_providers_global(provider, multi, value, suffix=None):
if suffix is not None: if suffix is not None:
return value[int(suffix)] return value[int(suffix)]
return value return value
@multi_function @multi_function
def calc_providers_follower(provider, multi, unique, dns, leader, index, **kwargs): def calc_providers_follower(provider, multi, dns, leader, index, **kwargs):
ret = [] ret = []
for data in _parse_kwargs(provider, dns, kwargs): for data in _parse_kwargs(provider, dns, kwargs):
if 'value' not in data: if 'value' not in data:
@ -64,7 +64,7 @@ def calc_providers_follower(provider, multi, unique, dns, leader, index, **kwarg
@multi_function @multi_function
def calc_providers_dynamic_follower(provider, multi, unique, dns, leader, index, suffix, **kwargs): def calc_providers_dynamic_follower(provider, multi, dns, leader, index, suffix, **kwargs):
ret = [] ret = []
for data in _parse_kwargs(provider, dns, kwargs): for data in _parse_kwargs(provider, dns, kwargs):
if 'value' not in data: if 'value' not in data:
@ -92,7 +92,7 @@ def calc_providers_dynamic_follower(provider, multi, unique, dns, leader, index,
@multi_function @multi_function
def calc_providers_dynamic(provider, multi, unique, dns, suffix, **kwargs): def calc_providers_dynamic(provider, multi, dns, suffix, **kwargs):
ret = [] ret = []
for data in _parse_kwargs(provider, dns, kwargs): for data in _parse_kwargs(provider, dns, kwargs):
if 'value' not in data: if 'value' not in data:
@ -101,7 +101,7 @@ def calc_providers_dynamic(provider, multi, unique, dns, suffix, **kwargs):
continue continue
if isinstance(data['value'], list): if isinstance(data['value'], list):
for v in data['value']: for v in data['value']:
if not unique or v not in ret: if v not in ret:
ret.append(v) ret.append(v)
elif data['value'] not in ret: elif data['value'] not in ret:
ret.append(data['value']) ret.append(data['value'])
@ -112,14 +112,10 @@ def calc_providers_dynamic(provider, multi, unique, dns, suffix, **kwargs):
@multi_function @multi_function
def calc_providers(provider, multi, unique, dns, **kwargs): def calc_providers(provider, multi, dns, suffix=None, **kwargs):
ret = [] ret = []
for data in _parse_kwargs(provider, dns, kwargs): for data in _parse_kwargs(provider, dns, kwargs):
if isinstance(data['value'], list): if isinstance(data['value'], list):
commun_dns = list(set(data['dns']) & set(dns))
if len(commun_dns) == 1:
ret.append(data['value'][data['dns'].index(commun_dns[0])])
continue
for v in data['value']: for v in data['value']:
if v in ret: if v in ret:
continue continue
@ -139,289 +135,216 @@ class Annotator(Walk):
objectspace: 'RougailObjSpace', objectspace: 'RougailObjSpace',
*args): *args):
self.objectspace = objectspace self.objectspace = objectspace
self.get_suppliers_providers() self.set_suppliers()
self.dispatch_provider_supplier_to_zones()
self.dispatch_provider_to_zones()
self.convert_providers() self.convert_providers()
self.convert_suppliers() self.convert_suppliers()
def get_suppliers_providers(self) -> None: def set_suppliers(self) -> dict:
""" get supplier informations """ get supplier informations
return something like: return something like:
{'Host': ['host1.example.net', 'host2.example.net']} {'Host': ['host1.example.net', 'host2.example.net']}
""" """
self.suppliers = {} self.suppliers = {}
self.providers = {}
for variable in self.get_variables(): for variable in self.get_variables():
if not hasattr(variable, 'supplier') and not hasattr(variable, 'provider'): if not hasattr(variable, 'supplier') or ':' in variable.supplier:
continue continue
nf_dns = variable.path.split('.', 1)[0] nf_dns = variable.path.split('.', 1)[0]
server_name = self.objectspace.space.variables[nf_dns].doc server_name = self.objectspace.space.variables[nf_dns].doc
# supplier self.suppliers.setdefault(variable.supplier, []).append({'option': variable,
if hasattr(variable, 'supplier') and ':' not in variable.supplier: 'dns': server_name,
'path_prefix': nf_dns,
'server_names': self.objectspace.rougailconfig['risotto_globals'][server_name]['global:server_names'],
'zone_names': self.objectspace.rougailconfig['risotto_globals'][server_name]['global:zones_name'],
'zones': set(self.objectspace.rougailconfig['risotto_globals'][server_name]['global:zones_name'])
})
if not hasattr(variable, 'information'):
variable.information = self.objectspace.information(variable.xmlfiles)
variable.information.supplier = variable.supplier
def convert_providers(self):
self.providers = {}
for variable in self.get_variables():
if not hasattr(variable, 'provider'):
continue
nf_dns = variable.path.split('.', 1)[0]
server_name = self.objectspace.space.variables[nf_dns].doc
provider_name = variable.provider
if self.objectspace.rougailconfig['risotto_globals'][server_name]['global:module_name'] == 'host':
server_names = [server_name]
else:
server_names = self.objectspace.rougailconfig['risotto_globals'][server_name]['global:server_names'] server_names = self.objectspace.rougailconfig['risotto_globals'][server_name]['global:server_names']
zones = self.objectspace.rougailconfig['risotto_globals'][server_name]['global:zones_name'] if provider_name != 'Host' and not provider_name.startswith('Host:') and not provider_name.startswith('global:'):
s_data = {'option': variable, p_data = {'option': variable,
'dns': server_name, 'dns': server_name,
'path_prefix': nf_dns, 'path_prefix': nf_dns,
'server_names': server_names, 'server_names': server_names,
'zones': zones, 'zone_names': self.objectspace.rougailconfig['risotto_globals'][server_name]['global:zones_name'],
'providers_zone': {}, 'zones': set(self.objectspace.rougailconfig['risotto_globals'][server_name]['global:zones_name']),
} }
if variable.supplier != 'Host' and not variable.supplier.startswith('Host:') and not variable.supplier.startswith('global:'): else:
if 'global:provider_zone' in self.objectspace.rougailconfig['risotto_globals'][server_name]: p_data = None
s_data['provider_zone'] = self.objectspace.rougailconfig['risotto_globals'][server_name]['global:provider_zone'] if ':' in provider_name:
self.suppliers.setdefault(variable.supplier, []).append(s_data) key_name, key_type = provider_name.rsplit(':', 1)
if not hasattr(variable, 'information'): is_provider = False
variable.information = self.objectspace.information(variable.xmlfiles) else:
variable.information.supplier = variable.supplier key_name = key_type = provider_name
# provider is_provider = True
if hasattr(variable, 'provider'): if provider_name != 'Host':
provider_name = variable.provider self.providers.setdefault(provider_name, []).append(p_data)
p_data = {'option': variable, if key_name != 'global' and key_name not in self.suppliers:
'dns': server_name, #warn(f'cannot find supplier "{key_name}" for "{server_name}"')
'provider_name': provider_name, continue
'path_prefix': nf_dns, # create a fill for this variable
'suppliers_zone': {}, fill = self.objectspace.fill(variable.xmlfiles)
} new_target = self.objectspace.target(variable.xmlfiles)
if variable.provider != 'Host' and not variable.provider.startswith('Host:') and not variable.provider.startswith('global:'): new_target.name = variable
if 'global:provider_zone' in self.objectspace.rougailconfig['risotto_globals'][server_name]: fill.target = [new_target]
p_data['provider_zone'] = self.objectspace.rougailconfig['risotto_globals'][server_name]['global:provider_zone'] if key_name == 'global':
if self.objectspace.rougailconfig['risotto_globals'][server_name]['global:module_name'] == 'host': fill.name = 'calc_providers_global'
server_names = [server_name] elif self.objectspace.paths.is_dynamic(variable):
else:
server_names = self.objectspace.rougailconfig['risotto_globals'][server_name]['global:server_names']
p_data['server_names'] = server_names
if self.objectspace.rougailconfig['risotto_globals'][server_name]['global:module_name'] != 'host':
p_data['zones'] = self.objectspace.rougailconfig['risotto_globals'][server_name]['global:zones_name']
if ':' in provider_name:
p_data['is_main_provider'] = False
provider = provider_name.rsplit(':', 1)[0]
else:
p_data['is_main_provider'] = True
provider = variable.provider
self.providers.setdefault(provider, []).append(p_data)
def dispatch_provider_supplier_to_zones(self):
"""calculate zone where provider and supplier communicate
"""
self.providers_zone = {}
for provider_name, p_datas in self.providers.items():
for p_data in p_datas:
if provider_name in ['global', 'Host'] or provider_name not in self.suppliers:
continue
if not 'provider_zone' in p_data:
provider_zone = None
else:
provider_zone = p_data['provider_zone']
for s_data in self.suppliers[provider_name]:
if not provider_zone:
if provider_name.endswith('Client'):
p_server = provider_name[0:-6]
if p_server not in self.providers:
continue
for p_data_t in self.providers[p_server]:
if p_data_t['dns'] == s_data['dns'] and 'provider_zone' in p_data_t:
zone = p_data_t['provider_zone']
break
else:
continue
else:
continue
else:
zone = provider_zone
if zone not in s_data['zones']:
continue
s_data['providers_zone'][provider_name] = zone
p_data['suppliers_zone'].setdefault(provider_name, {})[s_data['dns']] = zone
self.providers_zone.setdefault(zone, set()).add(provider_name)
def dispatch_provider_to_zones(self):
""" add information with provider zone domain name
"""
self.providers_zone = {}
for provider_name, p_datas in self.providers.items():
for p_data in p_datas:
if provider_name in ['global', 'Host'] or provider_name not in self.suppliers:
continue
if not 'provider_zone' in p_data:
continue
provider_zone = p_data['provider_zone']
family = self.objectspace.paths.get_variable(f"providers",
namespace=self.objectspace.rougailconfig['variable_namespace'],
force_path_prefix=p_data['path_prefix'],
)
if not hasattr(family, 'information'):
family.information = self.objectspace.information(family.xmlfiles)
name_in_zone = p_data['server_names'][p_data['zones'].index(provider_zone)]
setattr(family.information, provider_name, name_in_zone)
setattr(family.information, f'{provider_name}:zone', provider_zone)
def convert_providers(self):
for provider_name, providers_data in self.providers.items():
for provider_data in providers_data:
if provider_name != 'global' and provider_name not in self.suppliers:
continue
# create a fill for this variable
variable = provider_data['option']
fill = self.objectspace.fill(variable.xmlfiles)
new_target = self.objectspace.target(variable.xmlfiles)
new_target.name = variable
fill.target = [new_target]
if provider_name == 'global':
fill.name = 'calc_providers_global'
elif self.objectspace.paths.is_dynamic(variable):
if self.objectspace.paths.is_follower(variable):
fill.name = 'calc_providers_dynamic_follower'
else:
fill.name = 'calc_providers_dynamic'
elif self.objectspace.paths.is_follower(variable):
fill.name = 'calc_providers_follower'
else:
fill.name = 'calc_providers'
fill.namespace = variable.namespace
fill.index = 0
# first parameter: the provider name (something link Host:incoming_ports)
param = self.objectspace.param(variable.xmlfiles)
param.name = 'provider'
param.text = provider_data['provider_name']
fill.param = [param]
# second parameter: current variable is a multi variable?
param = self.objectspace.param(variable.xmlfiles)
param.name = 'multi'
param.text = variable.multi
param.type = 'boolean'
fill.param.append(param)
#
param = self.objectspace.param(variable.xmlfiles)
param.name = 'unique'
param.text = variable.unique != "False"
param.type = 'boolean'
fill.param.append(param)
#
if self.objectspace.paths.is_follower(variable): if self.objectspace.paths.is_follower(variable):
fill.name = 'calc_providers_dynamic_follower'
else:
fill.name = 'calc_providers_dynamic'
elif self.objectspace.paths.is_follower(variable):
fill.name = 'calc_providers_follower'
else:
fill.name = 'calc_providers'
fill.namespace = variable.namespace
fill.index = 0
# first parameter: the provider name (something link Host:incoming_ports)
param = self.objectspace.param(variable.xmlfiles)
param.name = 'provider'
param.text = provider_name
fill.param = [param]
# second parameter: current variable is a multi variable?
param = self.objectspace.param(variable.xmlfiles)
param.name = 'multi'
param.text = variable.multi
param.type = 'boolean'
fill.param.append(param)
if self.objectspace.paths.is_follower(variable):
param = self.objectspace.param(variable.xmlfiles)
param.name = 'leader'
param.text = self.objectspace.paths.get_leader(variable)
param.propertyerror = False
param.type = 'variable'
fill.param.append(param)
try:
leader_provider = self.objectspace.paths.get_leader(variable).provider
except:
leader_provider = None
#
param = self.objectspace.param(variable.xmlfiles)
param.name = 'index'
param.type = 'index'
fill.param.append(param)
if self.objectspace.paths.is_dynamic(variable):
# if dynamic: current suffix
# and add current DNS name, this is useful to known if supplier is link to this provider
param = self.objectspace.param(variable.xmlfiles)
param.name = 'suffix'
param.type = 'suffix'
fill.param.append(param)
if key_name != 'global':
param = self.objectspace.param(variable.xmlfiles)
param.name = 'dns'
param.text = server_names
fill.param.append(param)
if key_name == 'global':
param = self.objectspace.param(variable.xmlfiles)
param.name = 'value'
if provider_name in self.objectspace.rougailconfig['risotto_globals'][server_name]:
value = self.objectspace.rougailconfig['risotto_globals'][server_name][provider_name]
param.text = value
if isinstance(value, bool):
param.type = 'boolean'
else:
param.text = provider_name
param.type = 'information'
fill.param.append(param)
else:
# parse all supplier link to current provider
for idx, data in enumerate(self.suppliers[key_name]):
if p_data:
common_zones = data['zones'] & p_data['zones']
if not common_zones:
continue
for zidx, zone in enumerate(data['zone_names']):
if zone in common_zones:
break
dns = data['server_names'][zidx]
else:
dns = data['dns']
option = data['option']
# if not provider, get the true option that we want has value
if not is_provider:
path_prefix = data['path_prefix']
try:
supplier_option = self.objectspace.paths.get_supplier(f'supplier:{provider_name}', path_prefix)
except KeyError:
#warn(f'cannot find supplier "{provider_name}" for "{dns}"')
continue
# first of all, get the supplier name
param = self.objectspace.param(variable.xmlfiles) param = self.objectspace.param(variable.xmlfiles)
param.name = 'leader' param.name = f'dns_{idx}'
param.text = self.objectspace.paths.get_leader(variable) param.text = option
param.propertyerror = False param.propertyerror = False
param.type = 'variable' param.type = 'variable'
fill.param.append(param) fill.param.append(param)
try: if not is_provider and \
leader_provider = self.objectspace.paths.get_leader(variable).provider self.objectspace.paths.is_follower(variable):
except:
leader_provider = None
#
param = self.objectspace.param(variable.xmlfiles)
param.name = 'index'
param.type = 'index'
fill.param.append(param)
if self.objectspace.paths.is_dynamic(variable):
# if dynamic: current suffix
# and add current DNS name, this is useful to known if supplier is link to this provider
param = self.objectspace.param(variable.xmlfiles)
param.name = 'suffix'
param.type = 'suffix'
fill.param.append(param)
if provider_name != 'global':
param = self.objectspace.param(variable.xmlfiles)
param.name = 'dns'
param.text = provider_data['server_names']
fill.param.append(param)
if provider_name == 'global':
param = self.objectspace.param(variable.xmlfiles)
param.name = 'value'
if provider_data['provider_name'] in self.objectspace.rougailconfig['risotto_globals'][provider_data['dns']]:
value = self.objectspace.rougailconfig['risotto_globals'][provider_data['dns']][provider_data['provider_name']]
param.text = value
if isinstance(value, bool):
param.type = 'boolean'
else:
param.text = provider_data['provider_name']
param.type = 'information'
fill.param.append(param)
else:
# parse all supplier link to current provider
for idx, data in enumerate(self.suppliers[provider_name]):
if 'zones' in provider_data:
if provider_name not in data['providers_zone']:
continue
zone = data['providers_zone'][provider_name]
zidx = data['zones'].index(zone)
dns = data['server_names'][zidx]
else:
dns = data['dns']
option = data['option']
# if not provider, get the true option that we want have value
if not provider_data['is_main_provider']:
path_prefix = data['path_prefix']
try:
supplier_option = self.objectspace.paths.get_supplier(f'supplier:{provider_data["provider_name"]}', path_prefix)
except KeyError:
#warn(f'cannot find supplier "{provider_name}" for "{dns}"')
continue
# first of all, get the supplier name
param = self.objectspace.param(variable.xmlfiles) param = self.objectspace.param(variable.xmlfiles)
param.name = f'dns_{idx}' param.name = f'leader_{idx}'
param.text = option if fill.name == 'calc_providers_follower':
param.propertyerror = False param.text = dns
param.type = 'variable' else:
fill.param.append(param) if self.objectspace.paths.is_follower(supplier_option):
if not provider_data['is_main_provider'] and \ param.text = self.objectspace.paths.get_leader(supplier_option)
self.objectspace.paths.is_follower(variable):
param = self.objectspace.param(variable.xmlfiles)
param.name = f'leader_{idx}'
if fill.name == 'calc_providers_follower':
param.text = dns
else: else:
if self.objectspace.paths.is_follower(supplier_option): param.text = self.objectspace.paths.get_supplier(f'supplier:{leader_provider}', path_prefix)
param.text = self.objectspace.paths.get_leader(supplier_option)
else:
param.text = self.objectspace.paths.get_supplier(f'supplier:{leader_provider}', path_prefix)
param.propertyerror = False
param.type = 'variable'
fill.param.append(param)
# get the current DNS name for dynamic variable
if self.objectspace.paths.is_dynamic(variable):
param = self.objectspace.param(variable.xmlfiles)
param.name = f'dynamic_{idx}'
param.text = dns
fill.param.append(param)
# get the current value!
param = self.objectspace.param(variable.xmlfiles)
param.name = f'value_{idx}'
if provider_data['is_main_provider']:
param.text = dns
else:
param.text = supplier_option
param.propertyerror = False param.propertyerror = False
param.type = 'variable' param.type = 'variable'
fill.param.append(param) fill.param.append(param)
if not hasattr(self.objectspace.space.variables[provider_data['path_prefix']], 'constraints'): # get the current DNS name for dynamic variable
self.objectspace.space.variables[provider_data['path_prefix']].constraints = self.objectspace.constraints(None) if self.objectspace.paths.is_dynamic(variable):
if not hasattr(self.objectspace.space.variables[provider_data['path_prefix']].constraints, 'fill'): param = self.objectspace.param(variable.xmlfiles)
self.objectspace.space.variables[provider_data['path_prefix']].constraints.fill = [] param.name = f'dynamic_{idx}'
self.objectspace.space.variables[provider_data['path_prefix']].constraints.fill.append(fill) param.text = dns
fill.param.append(param)
# get the current value!
param = self.objectspace.param(variable.xmlfiles)
param.name = f'value_{idx}'
if is_provider:
param.text = dns
else:
param.text = supplier_option
param.propertyerror = False
param.type = 'variable'
fill.param.append(param)
if not hasattr(self.objectspace.space.variables[nf_dns], 'constraints'):
self.objectspace.space.variables[nf_dns].constraints = self.objectspace.constraints(None)
if not hasattr(self.objectspace.space.variables[nf_dns].constraints, 'fill'):
self.objectspace.space.variables[nf_dns].constraints.fill = []
self.objectspace.space.variables[nf_dns].constraints.fill.append(fill)
def convert_suppliers(self): def convert_suppliers(self):
for supplier_name, s_datas in self.suppliers.items(): for supplier, data in self.suppliers.items():
if supplier_name == 'Host': if supplier == 'Host':
continue continue
for s_data in s_datas: for s_dico in data:
if supplier_name not in self.providers: if supplier not in self.providers:
continue continue
for p_data in self.providers[supplier_name]: for p_dico in self.providers[supplier]:
if s_data['dns'] == p_data['dns']: common_zones = s_dico['zones'] & p_dico['zones']
# supplier and provider are in same machine if not common_zones:
continue continue
if supplier_name not in p_data['suppliers_zone'] or s_data['dns'] not in p_data['suppliers_zone'][supplier_name]: for idx, zone in enumerate(p_dico['zone_names']):
continue if zone in common_zones:
# get the DNS name in supplier zone break
zone = p_data['suppliers_zone'][supplier_name][s_data['dns']] dns = p_dico['server_names'][idx]
if zone not in p_data['zones']: s_dico['option'].value = dns
continue
zidx = p_data['zones'].index(zone)
dns = p_data['server_names'][zidx]
new_value = self.objectspace.value(None) new_value = self.objectspace.value(None)
new_value.name = dns new_value.name = dns
s_data['option'].value = [new_value] s_dico['option'].value = [new_value]
break break

64
src/risotto/utils.py
View file

@ -4,7 +4,6 @@ from typing import List
from ipaddress import ip_address from ipaddress import ip_address
from toml import load as toml_load from toml import load as toml_load
from json import load, dump from json import load, dump
from json.decoder import JSONDecodeError
from pprint import pprint from pprint import pprint
@ -16,10 +15,6 @@ HERE = environ['PWD']
IP_DIR = join(HERE, 'ip') IP_DIR = join(HERE, 'ip')
# custom filters from dataset
custom_filters = {}
config_file = environ.get('CONFIG_FILE', 'risotto.conf') config_file = environ.get('CONFIG_FILE', 'risotto.conf')
if isfile(config_file): if isfile(config_file):
with open(config_file, 'r') as fh: with open(config_file, 'r') as fh:
@ -40,10 +35,10 @@ def multi_function(function):
return function return function
def value_pprint(dico, config): async def value_pprint(dico, config):
pprint_dict = {} pprint_dict = {}
for path, value in dico.items(): for path, value in dico.items():
if config.option(path).type() == 'password' and value: if await config.option(path).option.type() == 'password' and value:
value = 'X' * len(value) value = 'X' * len(value)
pprint_dict[path] = value pprint_dict[path] = value
pprint(pprint_dict) pprint(pprint_dict)
@ -54,45 +49,28 @@ def load_zones(zones, hosts):
makedirs(IP_DIR) makedirs(IP_DIR)
json_file = join(IP_DIR, 'zones.json') json_file = join(IP_DIR, 'zones.json')
if isfile(json_file): if isfile(json_file):
try: with open(json_file, 'r') as fh:
with open(json_file, 'r') as fh: zones_ip = load(fh)
ori_zones_ip = load(fh)
except JSONDecodeError:
ori_zones_ip = {}
else: else:
ori_zones_ip = {} zones_ip = {}
new_zones_ip = {} for host_name, hosts in hosts.items():
# cache, machine should not change IP for server_name, server in hosts['servers'].items():
for host_name, dhosts in hosts.items(): server_zones = server['informations']['zones_name']
for server_name, server in dhosts['servers'].items():
server_zones = server['zones_name']
for idx, zone_name in enumerate(server_zones): for idx, zone_name in enumerate(server_zones):
zone = zones[zone_name] zone = zones[zone_name]
zone.setdefault('hosts', {}) zone.setdefault('hosts', {})
if zone_name not in new_zones_ip: # FIXME make a cache, machine should not change IP
new_zones_ip[zone_name] = {} if zone_name not in zones_ip:
if zone_name in ori_zones_ip and server_name in ori_zones_ip[zone_name]: zones_ip[zone_name] = {}
server_index = ori_zones_ip[zone_name][server_name] if server_name in zones_ip[zone_name]:
if server_index >= zone['length']: server_index = zones_ip[zone_name][server_name]
server_index = None elif not zones_ip[zone_name]:
elif server_index in new_zones_ip[zone_name].values(): server_index = 0
server_index = None
else: else:
server_index = None # it's the last ip + 1
new_zones_ip[zone_name][server_name] = server_index server_index = zones_ip[zone_name][list(zones_ip[zone_name].keys())[-1]] + 1
for zone_name, servers in new_zones_ip.items(): ip = str(ip_address(zone['start_ip']) + server_index)
for server_name, server_idx in servers.items(): zone['hosts'][server_name] = ip
if server_idx is not None: zones_ip[zone_name][server_name] = server_index
continue
for new_idx in range(zones[zone_name]['length']):
if new_idx not in new_zones_ip[zone_name].values():
new_zones_ip[zone_name][server_name] = new_idx
break
else:
raise Exception(f'cannot find free IP in zone "{zone_name}" for "{server_name}"')
for zone_name, servers in new_zones_ip.items():
start_ip = ip_address(zones[zone_name]['start_ip'])
for server_name, server_index in servers.items():
zones[zone_name]['hosts'][server_name] = str(start_ip + server_index)
with open(json_file, 'w') as fh: with open(json_file, 'w') as fh:
dump(new_zones_ip, fh) dump(zones_ip, fh)