better config change detection

ansible/filter_plugins/fileslist.py

@@ -30,21 +30,30 @@ def fileslist(data, is_host=False, name_only=False, prefix=None):
              prefix,
              )
     for service, service_data in data.items():
-        if service_data['activate'] and service_data['engine'] != 'none':
+        if not service_data['activate']:
-            _add(files,
+            if service_data['engine'] == 'none' and service_data['type'] == 'service' and not 'overrides' in service_data:
-                 {'owner': 'root', 'group': 'root', 'mode': '0755'},
-                 base_systemd + '/systemd/system/' + service_data['doc'],
-                 name_only,
-                 prefix,
-                 )
-        if service_data['activate'] and 'overrides' in service_data:
-            for override_data in service_data['overrides'].values():
                 _add(files,
                      {'owner': 'root', 'group': 'root', 'mode': '0755'},
-                     base_systemd + '/systemd/system/' + override_data['name'] + '.d/rougail.conf',
+                     base_systemd + '/systemd/system/' + service_data['doc'],
                      name_only,
                      prefix,
                      )
+        else:
+            if service_data['activate'] and service_data['engine'] != 'none':
+                _add(files,
+                     {'owner': 'root', 'group': 'root', 'mode': '0755'},
+                     base_systemd + '/systemd/system/' + service_data['doc'],
+                     name_only,
+                     prefix,
+                     )
+            if service_data['activate'] and 'overrides' in service_data:
+                for override_data in service_data['overrides'].values():
+                    _add(files,
+                         {'owner': 'root', 'group': 'root', 'mode': '0755'},
+                         base_systemd + '/systemd/system/' + override_data['name'] + '.d/rougail.conf',
+                         name_only,
+                         prefix,
+                         )
         if 'files' not in service_data:
             continue
         for file_data in service_data['files'].values():

ansible/inventory.py

@@ -99,6 +99,7 @@ class RisottoInventory(object):
                 continue
             ret[server_name] = engine.rougail_variables_dict
         ret['modules'] = {module_name: module_info['infos'].depends for module_name, module_info in module_infos.items() if module_name in modules}
+        ret['delete_old_image'] = False
         ret['configure_host'] = True
         ret['only_machine'] = None
         return dumps(ret, cls=RougailEncoder)

ansible/library/machinectl.py

@@ -4,6 +4,7 @@ from time import sleep
 from os import fdopen
 from dbus import SystemBus, Array
 from dbus.exceptions import DBusException
+from subprocess import run
 
 from ansible.module_utils.basic import AnsibleModule
 
@@ -141,6 +142,13 @@ def start(bus, machines):
             errors.append(f'{host}: ' + '\n'.join(ret))
     return changed, errors
 
+
+def enable(machines):
+    cmd = ['/usr/bin/machinectl', 'enable'] + machines
+    run(cmd)
+    return True
+
+
 def run_module():
     # define available arguments/parameters a user can pass to the module
     module_args = dict(
@@ -172,21 +180,23 @@ def run_module():
     if module.check_mode:
         module.exit_json(**result)
 
-    bus = SystemBus()
-
     # manipulate or modify the state as needed (this is going to be the
     # part where your module will do what it needs to do)
     machines = module.params['machines']
     if module.params['state'] == 'stopped':
+        bus = SystemBus()
         result['changed'], errors = stop(bus, machines)
         if errors:
             errors = '\n\n'.join(errors)
             module.fail_json(msg=f'Some machines are not stopping correctly {errors}', **result)
     elif module.params['state'] == 'started':
+        bus = SystemBus()
         result['changed'], errors = start(bus, machines)
         if errors:
             errors = '\n\n'.join(errors)
             module.fail_json(msg=f'Some machines are not running correctly {errors}', **result)
+    elif module.params['state'] == 'enabled':
+        result['changed'] = enable(machines)
     else:
         module.fail_json(msg=f"Unknown state: {module.params['state']}")
 

ansible/machine.yml

@@ -17,6 +17,7 @@
     path: "installations{{ file.name }}"
     checksum: sha256
     get_checksum: yes
+    follow: true
   loop: "{{ vars[item.name]['services'] | fileslist }}"
   loop_control:
     loop_var: file
@@ -28,6 +29,7 @@
     path: "/var/lib/risotto/configurations/{{ item.name }}{{ file.name }}"
     checksum: sha256
     get_checksum: yes
+    follow: true
   loop: "{{ vars[item.name]['services'] | fileslist }}"
   loop_control:
     loop_var: file
@@ -36,12 +38,12 @@
 
 - name: "Configuration's file is up to date in {{ item.name }}"
   debug:
-    msg: "file is {{ 'out of date' if not file[1].stat.exists or file[0].stat.checksum != file[1].stat.checksum else  'up to date' }}"
+    msg: "file is {{ 'out of date' if not file[1].stat.exists or (not 'checksum' in file[0].stat and 'checksum' in file[1].stat) or ('checksum' in file[0].stat and not 'checksum' in file[1].stat) or ('checksum' in file[0].stat and 'checksum' in file[1].stat and file[0].stat.checksum != file[1].stat.checksum) else 'up to date' }}"
-  changed_when: not file[1].stat.exists or file[0].stat.checksum != file[1].stat.checksum
+  changed_when: not file[1].stat.exists or (not 'checksum' in file[0].stat and 'checksum' in file[1].stat) or ('checksum' in file[0].stat and not 'checksum' in file[1].stat) or ('checksum' in file[0].stat and 'checksum' in file[1].stat and file[0].stat.checksum != file[1].stat.checksum)
   loop: "{{ local_configuration.results | zip(remote_configuration.results) | list }}"
   loop_control:
     loop_var: file
-    label: "{{ file[0]['stat']['path'] }}"
+    label: "{{ file[0]['file']['name'] }}"
   ignore_errors: true
   register: up_to_date_configuration
 
@@ -60,6 +62,24 @@
     format: tar
   when: up_to_date_configuration.changed
 
+- name: "Remove old image {{ vars | modulename(item.name) }}"
+  file:
+    path: "/var/lib/risotto/images/{{ vars | modulename(item.name) }}.tar"
+    state: absent
+  when: delete_old_image == true
+
+- name: "Stop machine {{ item.name }}"
+  machinectl:
+    state: stopped
+    machines: "{{ item.name }}"
+  when: delete_old_image == true
+
+- name: "Remove old machine {{ item.name }}"
+  file:
+    path: /var/lib/machines/{{ item.name }}
+    state: absent
+  when: delete_old_image == true
+
 - name: "Create system directory for {{ item.name }}"
   file:
     path: /var/lib/machines/{{ item.name }}
@@ -69,6 +89,7 @@
 - name: "Check image for {{ item.name }}"
   stat:
     path: "/var/lib/risotto/images/{{ vars | modulename(item.name) }}.tar"
+    follow: true
   register: register_name
   when: system_directory_created.changed
 

ansible/machines.yml

@@ -21,6 +21,11 @@
     dest: /var/lib/risotto/configurations/{{ item | basename }}/
   loop: "{{ lookup('fileglob', '/tmp/new_configurations/*', wantlist=True) }}"
 
+- name: "Enable machines"
+  machinectl:
+    state: enabled
+    machines: "{{ vars | machineslist(only_name=True) }}"
+
 - name: "Start machines"
   machinectl:
     state: started

sbin/build_image

@@ -131,6 +131,7 @@ fi
 if [ -f "$IMAGE_NAME_RISOTTO_IMAGE_DIR".base.pkgs ] && [ -f "$IMAGE_NAME_RISOTTO_IMAGE_DIR".pkgs ]; then
     echo "  - différence(s) avec les paquets de base"
     diff -u "$IMAGE_NAME_RISOTTO_IMAGE_DIR".base.pkgs "$BASE_PKGS_FILE" && INSTALL=false || INSTALL=true
+    [ ! -f "$IMAGE_NAME_RISOTTO_IMAGE_NAME" ] && INSTALL=true
 else
     INSTALL=true
 fi

sbin/diagnose

@@ -0,0 +1,52 @@
+#!/bin/bash -e
+
+MACHINES=""
+for nspawn in $(ls /etc/systemd/nspawn/*.nspawn); do
+    nspawn_file=$(basename $nspawn)
+    machine=${nspawn_file%.*}
+    MACHINES="$MACHINES$machine "
+done
+STARTED=""
+DEGRADED=""
+found=true
+idx=0
+while [ $found = true ]; do
+    found=false
+    echo "tentative $idx"
+    for machine in $MACHINES; do
+        if ! echo $STARTED | grep -q " $machine "; then
+            status=$(machinectl -q shell $machine /usr/bin/systemctl is-system-running 2>/dev/null || echo "not started")
+            if echo "$status" | grep -q degraded; then
+                STARTED="$STARTED $machine "
+                DEGRADED="$DEGRADED $machine"
+            elif echo "$status" | grep -q running; then
+                STARTED="$STARTED $machine "
+            else
+                found=true
+                echo "status actuel de $machine : $status"
+            fi
+        fi
+    done
+    sleep 2
+    idx=$((idx+1))
+    if [ $idx = 60 ]; then
+        break
+    fi
+done
+retcode=0
+for machine in $MACHINES; do
+    if ! echo "$STARTED" | grep -q " $machine "; then
+        echo
+        echo "========= $machine"
+        machinectl -q shell $machine /usr/bin/systemctl is-system-running 2>/dev/null || echo "not started"
+    fi
+done
+echo $DEGRADED
+for machine in $DEGRADED; do
+    echo
+    echo "========= $machine"
+    machinectl -q shell $machine /usr/bin/systemctl --state=failed --no-legend --no-pager
+    retcode=1
+done
+
+exit $retcode

sbin/update_images

@@ -15,7 +15,7 @@ ls /var/lib/risotto/images_files/ | while read image; do
         /usr/local/sbin/build_image "$image"
     fi
 done
-rm -f $IMAGE_BASE_RISOTTO_BASE_DIR*.build
+#rm -f $IMAGE_BASE_RISOTTO_BASE_DIR*.build
 
 MACHINES=""
 for nspawn in $(ls /etc/systemd/nspawn/*.nspawn); do
@@ -23,11 +23,11 @@ for nspawn in $(ls /etc/systemd/nspawn/*.nspawn); do
     machine=${nspawn_file%.*}
     MACHINES="$MACHINES$machine "
     MACHINE_MACHINES_DIR="/var/lib/machines/$machine"
-    echo "Install machine $machine"
     SHA_MACHINE="$RISOTTO_DIR/configurations/sha/$machine".sha
     content=$(cat $SHA_MACHINE)
     IMAGE_NAME_RISOTTO_IMAGE_NAME=${content##* }
-    diff -q "$IMAGE_NAME_RISOTTO_IMAGE_NAME".sha "$SHA_MACHINE" || (
+    diff -q "$IMAGE_NAME_RISOTTO_IMAGE_NAME".sha "$SHA_MACHINE" > /dev/null || (
+        echo "Reinstall machine $machine"
         machinectl stop $machine
         while true; do
             machinectl status "$machine" > /dev/null 2>&1 || break
@@ -41,39 +41,4 @@ for nspawn in $(ls /etc/systemd/nspawn/*.nspawn); do
     )
 done
 machinectl start $MACHINES
-STARTED=""
+diagnose
-DEGRADED=""
-found=true
-idx=0
-while [ $found = true ]; do
-    found=false
-    echo "tentative $idx"
-    for machine in $MACHINES; do
-        if ! echo $STARTED | grep -q " $machine "; then
-            status=$(machinectl -q shell $machine /usr/bin/systemctl is-system-running || true)
-            if echo "$status" | grep -q degraded; then
-                STARTED="$STARTED $machine "
-                DEGRADED="$DEGRADED $machine"
-            elif echo "$status" | grep -q running; then
-                STARTED="$STARTED $machine "
-            else
-                found=true
-                echo "status actuel de $machine : $status"
-            fi
-        fi
-    done
-    sleep 2
-    idx=$((idx+1))
-    if [ $idx = 60 ]; then
-        break
-    fi
-done
-retcode=0
-for machine in $DEGRADED; do
-    echo
-    echo "========= $machine"
-    machinectl -q shell $machine /usr/bin/systemctl --state=failed --no-legend --no-pager
-    retcode=1
-done
-
-exit $retcode

src/risotto/image.py

@@ -163,8 +163,8 @@ def load_applicationservice(appname: str,
         suppliers.setdefault(supplier, [])
         if appname not in suppliers[supplier]:
             suppliers[supplier].append(appname)
-    if 'distribution' in app:
+    if 'distribution' in app and app['distribution']:
-        distribution = app['distribution']
+        distribution = appname
     else:
         distribution = None
     for xml in app.get('depends', []):
@@ -181,7 +181,7 @@ def load_applicationservice(appname: str,
                                       )
         if ret:
             if distribution:
-                raise Exception(f'duplicate distribution for {cfg.module_name} ({distribution} and {ret})')
+                raise Exception(f'duplicate distribution for {cfg.module_name}: {distribution} and {ret} (dependencies: {cfg.depends}) ')
             distribution = ret
     return distribution
 
@@ -209,7 +209,7 @@ def load_image_informations(module_name: str,
                                       )
         if ret:
             if distribution:
-                raise Exception(f'duplicate distribution for {cfg.module_name} ({distribution} and {ret})')
+                raise Exception(f'duplicate distribution for {cfg.module_name}: {distribution} and {ret} (dependencies: {cfg.depends}) ')
             distribution = ret
     if module_name != 'host' and not distribution:
         raise Exception(f'cannot found any linux distribution for {module_name}')