first commit

funcs.py

@@ -0,0 +1,233 @@
+from tiramisu import valid_network_netmask, valid_ip_netmask, valid_broadcast, valid_in_network, valid_not_equal as valid_differ, valid_not_equal, calc_value
+from ipaddress import ip_address
+from os.path import dirname, abspath, join as _join, isdir as _isdir, isfile as _isfile
+from typing import List
+from json import load
+from secrets import token_urlsafe as _token_urlsafe
+
+from rougail.utils import normalize_family
+
+from utils import multi_function, CONFIGS
+from x509 import gen_cert as _x509_gen_cert, gen_ca as _x509_gen_ca, gen_pub as _x509_gen_pub, has_pub as _x509_has_pub
+# =============================================================
+# fork of risotto-setting/src/risotto_setting/config/config.py
+
+with open('servers.json', 'r') as server_fh:
+    ZONES_SERVER = load(server_fh)
+
+
+ZONES = None
+DOMAINS = None
+HERE = dirname(abspath(__file__))
+
+
+def load_zones():
+    global ZONES
+    if ZONES is not None:
+        return
+    ZONES = ZONES_SERVER['zones']
+    for server_name, server in ZONES_SERVER['servers'].items():
+        if 'informations' not in server:
+            continue
+        server_zones = server['informations']['zones_name']
+        server_extra_domainnames = server['informations'].get('extra_domainnames', [])
+        if len(server_zones) > 1 and len(server_zones) != len(server_extra_domainnames) + 1:
+            raise Exception(f'the server "{server_name}" has more that one zone, please set correct number of extra_domainnames ({len(server_zones) - 1} instead of {len(server_extra_domainnames)})')
+            
+        for idx, zone_name in enumerate(server_zones):
+            zone_domain_name = ZONES[zone_name]['domain_name']
+            if idx == 0:
+                zone_server_name = server_name
+            else:
+                zone_server_name = server_extra_domainnames[idx - 1]
+            server_domain_name = zone_server_name.split('.', 1)[1]
+            if zone_domain_name and zone_domain_name != server_domain_name:
+                raise Exception(f'wrong server_name "{zone_server_name}" in zone "{zone_name}" should ends with "{zone_domain_name}"')
+            ZONES[zone_name].setdefault('hosts', []).append(server_name)
+
+
+def load_domains():
+    load_zones()
+    global DOMAINS
+    if DOMAINS is not None:
+        return
+    DOMAINS = {}
+    for zone_name, zone in ZONES_SERVER['zones'].items():
+        if 'domain_name' in zone:
+            hosts = []
+            ips = []
+            for host in ZONES[zone_name].get('hosts', []):
+                hosts.append(host.split('.', 1)[0])
+                ips.append(get_ip(host, [zone_name], 0))
+            DOMAINS[zone['domain_name']] = (tuple(hosts), tuple(ips))
+
+
+def get_ip(server_name: str,
+           zones_name: List[str],
+           index: str,
+           ) -> str:
+    if server_name is None:
+        return
+    load_zones()
+    index = int(index)
+    zone_name = zones_name[index]
+    if zone_name not in ZONES:
+        raise ValueError(f"cannot set IP in unknown zone '{zone_name}'")
+    zone = ZONES[zone_name]
+    if server_name not in zone['hosts']:
+        raise ValueError(f"cannot set IP in unknown server '{server_name}'")
+    server_index = zone['hosts'].index(server_name)
+#    print(server_name, zones_name, index, str(ip_address(zone['start_ip']) + server_index))
+    return str(ip_address(zone['start_ip']) + server_index)
+
+
+@multi_function
+def get_chain(authority_cn,
+              authority_name,
+              ):
+    if not authority_name or authority_name is None:
+        if isinstance(authority_name, list):
+            return []
+        return
+    if not isinstance(authority_cn, list):
+        is_list = False
+        authority_cn = [authority_cn]
+    else:
+        is_list = True
+        authorities = []
+
+    for auth_cn in authority_cn:
+        ret = _x509_gen_ca(auth_cn,
+                           authority_name,
+                           HERE,
+                           )
+        if not is_list:
+            return ret
+        authorities.append(ret)
+    return authorities
+
+
+@multi_function
+def get_certificate(cn,
+                    authority_name,
+                    authority_cn=None,
+                    extra_domainnames=[],
+                    type='server',
+                    ):
+    if isinstance(cn, list) and extra_domainnames:
+        raise Exception('cn cannot be a list with extra_domainnames set')
+    if not cn or authority_name is None:
+        if isinstance(cn, list):
+            return []
+        return
+    return _x509_gen_cert(cn,
+                          extra_domainnames,
+                          authority_cn,
+                          authority_name,
+                          type,
+                          'crt',
+                          HERE,
+                          )
+
+
+@multi_function
+def get_private_key(cn,
+                    authority_name=None,
+                    authority_cn=None,
+                    type='server',
+                    ):
+    if not cn:
+        if isinstance(cn, list):
+            return []
+        return
+    if authority_name is None:
+        if _x509_has_pub(cn, HERE):
+            return _x509_gen_pub(cn,
+                                 'key',
+                                 HERE,
+                                 )
+        if isinstance(cn, list):
+            return []
+        return
+    return _x509_gen_cert(cn,
+                          [],
+                          authority_cn,
+                          authority_name,
+                          type,
+                          'key',
+                          HERE,
+                          )
+
+
+def get_public_key(cn):
+    if not cn:
+        return
+    return _x509_gen_pub(cn,
+                         'pub',
+                         HERE,
+                         )
+
+
+def zone_information(zone_name: str,
+                     type: str,
+                     multi: bool=False,
+                     index: int=None,
+                     ) -> str:
+    if not zone_name:
+        return
+    if type == 'gateway' and index != 0:
+        return
+    load_zones()
+    if zone_name not in ZONES:
+        raise ValueError(f"cannot get zone informations in unknown zone '{zone_name}'")
+    zone = ZONES[zone_name]
+    if type not in zone:
+        raise ValueError(f"unknown type '{type}' in zone '{zone_name}'")
+    value = zone[type]
+    if multi:
+        value = [value]
+    return value
+
+
+def get_internal_zones() -> List[str]:
+    load_domains()
+    return list(DOMAINS.keys())
+
+
+@multi_function
+def get_zones_info(type: str) -> str:
+    ret = []
+    for data in ZONES_SERVER['zones'].values():
+        ret.append(data[type])
+    return ret
+
+@multi_function
+def get_internal_zone_names() -> List[str]:
+    load_zones()
+    return list(ZONES.keys())
+
+
+def get_internal_zone_information(zone: str,
+                                  info: str,
+                                  ) -> str:
+    load_domains()
+    if info == 'cidr':
+        return ZONES[zone]['gateway'] + '/' + ZONES[zone]['network'].split('/')[-1]
+    return ZONES[zone][info]
+
+
+def get_internal_info_in_zone(zone: str,
+                              auto: bool,
+                              type: str,
+                              index: int=None,
+                              ) -> List[str]:
+    if not auto:
+        return
+    for domain_name, domain in DOMAINS.items():
+        if zone == domain_name:
+            if type == 'host':
+                return list(domain[0])
+            else:
+                return domain[1][index]
+
+# =============================================================

servers.json

@@ -0,0 +1,191 @@
+{"zones": {"external": {"network": "192.168.45.0/24",
+                      "gateway": "192.168.45.1",
+                      "start_ip": "192.168.45.10",
+                      "domain_name": "in.silique.fr"
+                    },
+           "list": {"network": "192.168.47.0/24",
+                    "gateway": "192.168.47.1",
+                    "start_ip": "192.168.47.10",
+                    "domain_name": "list.silique.fr"
+                    }
+         },
+ "modules": {"host": {"applicationservices": ["host-systemd-machined"]},
+             "unbound": {"applicationservices": ["unbound", "provider-systemd-machined"]},
+             "nsd": {"applicationservices": ["nsd", "provider-systemd-machined"]},
+             "revprox": {"applicationservices": ["nginx-reverse-proxy-server", "provider-systemd-machined"]},
+             "postgresql": {"applicationservices": ["postgresql-server", "provider-systemd-machined"]},
+             "redis": {"applicationservices": ["redis-server", "provider-systemd-machined"]},
+             "ldap": {"applicationservices": ["openldap-server", "provider-systemd-machined"]},
+             "lemonldap": {"applicationservices": ["lemonldap", "provider-systemd-machined"]},
+             "nextcloud": {"applicationservices": ["nextcloud", "provider-systemd-machined"]},
+             "mail": {"applicationservices": ["postfix-relay", "provider-systemd-machined"]},
+             "dovecot": {"applicationservices": ["dovecot", "provider-systemd-machined"]},
+             "mailman": {"applicationservices": ["mailman", "provider-systemd-machined"]},
+             "gitea": {"applicationservices": ["gitea", "provider-systemd-machined"]},
+             "roundcube": {"applicationservices": ["roundcube", "provider-systemd-machined"]},
+             "vaultwarden": {"applicationservices": ["vaultwarden", "provider-systemd-machined"]}
+             },
+ "servers": {"cloud": {"module": "host",
+                      "values": {"rougail.host_install_dir": "/root/installations",
+                                 "rougail.host_dhcp_interface": ["enp3s0"]
+                                 }
+                      },
+             "unbound.in.silique.fr": {"module": "unbound",
+                                       "informations": {"zones_name": ["external"]},
+                                       "values": {"rougail.host": "cloud",
+                                                  "rougail.dns_resolver.unbound_default_forwards": ["8.8.8.8"]
+                                                  }
+                                       },
+             "nsd.in.silique.fr": {"module": "nsd",
+                                   "informations": {"zones_name": ["external", "list"],
+                                                    "extra_domainnames": ["nsd.list.silique.fr"]
+                                                    },
+                                   "values": {"rougail.host": "cloud",
+                                              "rougail.dns_server.nsd_resolver": "unbound.in.silique.fr"
+                                              }
+                                    },
+              "revprox.in.silique.fr": {"module": "revprox",
+                                        "informations": {"zones_name": ["external"]},
+                                        "values": {"rougail.host": "cloud",
+                                        "rougail.dns.dns_client_address": "nsd.in.silique.fr",
+                                                   "rougail.nginx.nginx_default": "cloud.silique.fr"
+                                                   }
+                                        },
+              "mail.in.silique.fr": {"module": "mail",
+                                     "informations": {"zones_name": ["external", "list"],
+                                                      "extra_domainnames": ["mail.list.silique.fr"]
+                                                      },
+                                     "values": {"rougail.host": "cloud",
+                                                "rougail.dns.dns_client_address": "unbound.in.silique.fr",
+                                                "rougail.postfix.postfix_mail_hostname": "mail.silique.fr"
+                                                }
+                                     },
+              "dovecot.in.silique.fr": {"module": "dovecot",
+                                        "informations": {"zones_name": ["external"]
+                                                         },
+                                        "values": {"rougail.host": "cloud",
+                                                   "rougail.dns.dns_client_address": "nsd.in.silique.fr",
+                                                   "rougail.postfix.postfix_my_domains": ["cloud.silique.fr"],
+                                                   "rougail.smtp.smtp_relay_address": "mail.in.silique.fr",
+                                                   "rougail.annuaire.ldap_server_address": "ldap.in.silique.fr",
+                                                   "rougail.dovecot.revprox_server_domainname": "revprox.in.silique.fr",
+                                                   "rougail.oauth2_client.oauth2_client_server_domainname": "lemonldap.in.silique.fr"
+                                                   }
+                                         },
+              "redis-rc.in.silique.fr": {"module": "redis",
+                                         "informations": {"zones_name": ["external"]},
+                                         "values": {"rougail.host": "cloud",
+                                                    "rougail.dns.dns_client_address": "nsd.in.silique.fr"
+                                                    }
+                                         },
+              "redis-nc.in.silique.fr": {"module": "redis",
+                                         "informations": {"zones_name": ["external"]},
+                                         "values": {"rougail.host": "cloud",
+                                                    "rougail.dns.dns_client_address": "nsd.in.silique.fr"
+                                                    }
+                                         },
+              "redis-gi.in.silique.fr": {"module": "redis",
+                                         "informations": {"zones_name": ["external"]},
+                                         "values": {"rougail.host": "cloud",
+                                                    "rougail.dns.dns_client_address": "nsd.in.silique.fr"
+                                                    }
+                                         },
+              "ldap.in.silique.fr": {"module": "ldap",
+                                     "informations": {"zones_name": ["external"]},
+                                     "values": {"rougail.host": "cloud",
+                                                "rougail.dns.dns_client_address": "nsd.in.silique.fr",
+                                                "accounts.users.ldap_user_mail": ["gnunux@silique.fr", "bbohard@silique.fr", "ddtddt@silique.fr"],
+                                                "accounts.users.ldap_user_uid": {"0": "gnunux", "1": "bbohard", "2": "ddtddt"},
+                                                "accounts.users.ldap_user_sn": {"0": "Emmanuel", "1": "Benjamin", "2": "Damien"},
+                                                "accounts.users.ldap_user_gn": {"0": "Garette", "1": "Bohard", "2": "Thomas"}
+                                                }
+                                     },
+              "lemonldap.in.silique.fr": {"module": "lemonldap",
+                                          "informations": {"zones_name": ["external"]},
+                                          "values": {"rougail.host": "cloud",
+                                                     "rougail.dns.dns_client_address": "nsd.in.silique.fr",
+                                                     "rougail.annuaire.ldap_server_address": "ldap.in.silique.fr",
+                                                     "rougail.smtp.smtp_relay_address": "mail.in.silique.fr",
+                                                     "rougail.nginx.revprox_client_server_domainname": "revprox.in.silique.fr",
+                                                     "rougail.nginx.revprox_client_external_domainname": "auth.silique.fr",
+                                                     "rougail.lemonldap.lemon_domain": "cloud.silique.fr",
+                                                     "rougail.lemonldap.lemon_mail_admin": "gnunux@silique.fr"
+                                                     }
+                                          },
+              "nextcloud.in.silique.fr": {"module": "nextcloud",
+                                          "informations": {"zones_name": ["external"]},
+                                          "values": {"rougail.host": "cloud",
+                                                     "rougail.dns.dns_client_address": "nsd.in.silique.fr",
+                                                     "rougail.nextcloud.nextcloud_mail_admin": "gnunux@silique.fr",
+                                                     "rougail.postgresql.pg_client_server_domainname": "postgresql.in.silique.fr",
+                                                     "rougail.annuaire.ldap_server_address": "ldap.in.silique.fr",
+                                                     "rougail.redis.redis_client_server_domainname": "redis-nc.in.silique.fr",
+                                                     "rougail.smtp.smtp_relay_address": "mail.in.silique.fr",
+                                                     "rougail.nginx.revprox_client_server_domainname": "revprox.in.silique.fr",
+                                                     "rougail.nginx.revprox_client_external_domainname": "cloud.silique.fr",
+                                                     "rougail.oauth2_client.oauth2_client_server_domainname": "lemonldap.in.silique.fr"
+                                                     }
+                                          },
+              "roundcube.in.silique.fr": {"module": "roundcube",
+                                          "informations": {"zones_name": ["external"]},
+                                          "values": {"rougail.host": "cloud",
+                                                     "rougail.dns.dns_client_address": "nsd.in.silique.fr",
+                                                     "rougail.postgresql.pg_client_server_domainname": "postgresql.in.silique.fr",
+                                                     "rougail.annuaire.ldap_server_address": "ldap.in.silique.fr",
+                                                     "rougail.nginx.revprox_client_server_domainname": "revprox.in.silique.fr",
+                                                     "rougail.nginx.revprox_client_external_domainname": "cloud.silique.fr",
+                                                     "rougail.redis.redis_client_server_domainname": "redis-rc.in.silique.fr",
+                                                     "rougail.imap.imap_address": "dovecot.in.silique.fr",
+                                                     "rougail.oauth2_client.oauth2_client_server_domainname": "lemonldap.in.silique.fr"
+                                                     }
+                                          },
+              "postgresql.in.silique.fr": {"module": "postgresql",
+                                           "informations": {"zones_name": ["external", "list"],
+                                           "extra_domainnames": ["postgresql.list.silique.fr"]
+                                           },
+                                           "values": {"rougail.host": "cloud",
+                                                      "rougail.dns.dns_client_address": "nsd.in.silique.fr"
+                                                      }
+                                           },
+              "mailman.list.silique.fr": {"module": "mailman",
+                                          "informations": {"zones_name": ["list"]
+                                                           },
+                                          "values": {"rougail.host": "cloud",
+                                                     "rougail.dns.dns_client_address": "nsd.list.silique.fr",
+                                                     "rougail.smtp.smtp_relay_address": "mail.list.silique.fr",
+                                                     "rougail.postgresql.pg_client_server_domainname": "postgresql.list.silique.fr",
+                                                     "rougail.nginx.revprox_client_server_domainname": "revprox.in.silique.fr",
+                                                     "rougail.nginx.revprox_client_external_domainname": "cloud.silique.fr",
+                                                     "rougail.mailman.mailman_mail_owner": "admin@silique.fr",
+                                                     "rougail.mailman.mailman_domains": ["lists.silique.fr"],
+                                                     "rougail.oauth2_client.oauth2_client_server_domainname": "lemonldap.in.silique.fr",
+                                                     "mailman.list_lists_silique_fr.name_lists_silique_fr": ["list1", "list2"]
+                                                     }
+                                          },
+              "vaultwarden.in.silique.fr": {"module": "vaultwarden",
+                                            "informations": {"zones_name": ["external"]},
+                                            "values": {"rougail.host": "cloud",
+                                                       "rougail.dns.dns_client_address": "nsd.in.silique.fr",
+                                                       "rougail.vaultwarden.vaultwarden_admin_email": "gnunux@silique.fr",
+                                                       "rougail.postgresql.pg_client_server_domainname": "postgresql.in.silique.fr",
+                                                       "rougail.nginx.revprox_client_server_domainname": "revprox.in.silique.fr",
+                                                       "rougail.nginx.revprox_client_external_domainname": "cloud.silique.fr",
+                                                       "rougail.smtp.smtp_relay_address": "mail.in.silique.fr"
+                                                       }
+                                                 },
+              "gitea.in.silique.fr": {"module": "gitea",
+                                      "informations": {"zones_name": ["external"]
+                                                       },
+                                      "values": {"rougail.host": "cloud",
+                                                 "rougail.dns.dns_client_address": "nsd.in.silique.fr",
+                                                 "rougail.smtp.smtp_relay_address": "mail.in.silique.fr",
+                                                 "rougail.gitea.gitea_mail_sender": "gitea@silique.fr",
+                                                 "rougail.postgresql.pg_client_server_domainname": "postgresql.in.silique.fr",
+                                                 "rougail.nginx.revprox_client_server_domainname": "revprox.in.silique.fr",
+                                                 "rougail.redis.redis_client_server_domainname": "redis-gi.in.silique.fr",
+                                                 "rougail.oauth2_client.oauth2_client_server_domainname": "lemonldap.in.silique.fr",
+                                                 "rougail.nginx.revprox_client_external_domainname": "cloud.silique.fr"
+                                                 }
+                                      }
+             }
+}

src/utils.py

@@ -0,0 +1,10 @@
+MULTI_FUNCTIONS = []
+CONFIGS = {}
+
+
+def multi_function(function):
+    global MULTI_FUNCTIONS
+    name = function.__name__
+    if name not in MULTI_FUNCTIONS:
+        MULTI_FUNCTIONS.append(name)
+    return function

src/x509.py

@@ -0,0 +1,254 @@
+from OpenSSL.crypto import load_certificate, load_privatekey, dump_certificate, dump_privatekey, dump_publickey, PKey, X509, X509Extension, TYPE_RSA, FILETYPE_PEM
+from os import makedirs, symlink
+from os.path import join, isdir, isfile, exists
+#from shutil import rmtree
+from datetime import datetime
+
+
+PKI_DIR = 'pki/x509'
+#FIXME
+EMAIL = 'gnunux@gnunux.info'
+COUNTRY = 'FR'
+LOCALITY = 'Dijon'
+STATE = 'France'
+ORG_NAME = 'Cadoles'
+ORG_UNIT_NAME = 'CSS'
+
+
+def _gen_key_pair():
+    key = PKey()
+    key.generate_key(TYPE_RSA, 4096)
+    return key
+
+
+def _gen_cert(is_ca,
+              common_names,
+              serial_number,
+              validity_end_in_seconds,
+              key_file,
+              cert_file,
+              type=None,
+              ca_cert=None,
+              ca_key=None,
+              email_address=None,
+              country_name=None,
+              locality_name=None,
+              state_or_province_name=None,
+              organization_name=None,
+              organization_unit_name=None,
+              ):
+    #can look at generated file using openssl:
+    #openssl x509 -inform pem -in selfsigned.crt -noout -text
+    # create a key pair
+    if isfile(key_file):
+        with open(key_file) as fh:
+            filecontent = bytes(fh.read(), 'utf-8')
+            key = load_privatekey(FILETYPE_PEM, filecontent)
+    else:
+        key = _gen_key_pair()
+    cert = X509()
+    cert.set_version(2)
+    cert.get_subject().C = country_name
+    cert.get_subject().ST = state_or_province_name
+    cert.get_subject().L = locality_name
+    cert.get_subject().O = organization_name
+    cert.get_subject().OU = organization_unit_name
+    cert.get_subject().CN = common_names[0]
+    cert.get_subject().emailAddress = email_address
+    cert_ext = []
+    if not is_ca:
+        cert_ext.append(X509Extension(b'basicConstraints', False, b'CA:FALSE'))
+        cert_ext.append(X509Extension(b'keyUsage', True, b'digitalSignature, keyEncipherment'))
+        cert_ext.append(X509Extension(b'subjectAltName', False, ", ".join([f'DNS:{common_name}' for common_name in common_names]).encode('ascii')))
+        if type == 'server':
+            cert_ext.append(X509Extension(b'extendedKeyUsage', True, b'serverAuth'))
+        else:
+            cert_ext.append(X509Extension(b'extendedKeyUsage', True, b'clientAuth'))
+    else:
+        cert_ext.append(X509Extension(b'basicConstraints', False, b'CA:TRUE'))
+        cert_ext.append(X509Extension(b"keyUsage", True, b'keyCertSign, cRLSign'))
+        cert_ext.append(X509Extension(b'subjectAltName', False, f'email:{email_address}'.encode()))
+    cert_ext.append(X509Extension(b'subjectKeyIdentifier', False, b"hash", subject=cert))
+    cert.add_extensions(cert_ext)
+    cert.set_serial_number(serial_number)
+    cert.gmtime_adj_notBefore(0)
+    cert.gmtime_adj_notAfter(validity_end_in_seconds)
+    if is_ca:
+        ca_cert = cert
+        ca_key = key
+    else:
+        with open(ca_cert) as fh:
+            filecontent = bytes(fh.read(), 'utf-8')
+            ca_cert = load_certificate(FILETYPE_PEM, filecontent)
+        with open(ca_key) as fh:
+            filecontent = bytes(fh.read(), 'utf-8')
+            ca_key = load_privatekey(FILETYPE_PEM, filecontent)
+    cert.set_issuer(ca_cert.get_subject())
+    cert.add_extensions([X509Extension(b"authorityKeyIdentifier", False, b'keyid:always', issuer=ca_cert)])
+    cert.set_pubkey(key)
+    cert.sign(ca_key, "sha512")
+
+    with open(cert_file, "wt") as f:
+        f.write(dump_certificate(FILETYPE_PEM, cert).decode("utf-8"))
+    with open(key_file, "wt") as f:
+        f.write(dump_privatekey(FILETYPE_PEM, key).decode("utf-8"))
+
+
+def gen_ca(authority_dns,
+           authority_name,
+           base_dir,
+           ):
+    authority_cn = authority_name + '+' + authority_dns
+    week_number = datetime.now().isocalendar().week
+    root_dir_name = join(base_dir, PKI_DIR, authority_cn)
+    ca_dir_name = join(root_dir_name, 'ca')
+    sn_ca_name = join(ca_dir_name, 'serial_number')
+    key_ca_name = join(ca_dir_name, 'private.key')
+    cert_ca_name = join(ca_dir_name, f'certificate_{week_number}.crt')
+    if not isfile(cert_ca_name):
+        if not isdir(ca_dir_name):
+            # rmtree(ca_dir_name)
+            makedirs(ca_dir_name)
+        if isfile(sn_ca_name):
+            with open(sn_ca_name, 'r') as fh:
+                serial_number = int(fh.read().strip()) + 1
+        else:
+            serial_number = 0
+        _gen_cert(True,
+                  [authority_cn],
+                  serial_number,
+                  10*24*60*60,
+                  key_ca_name,
+                  cert_ca_name,
+                  email_address=EMAIL,
+                  country_name=COUNTRY,
+                  locality_name=LOCALITY,
+                  state_or_province_name=STATE,
+                  organization_name=ORG_NAME,
+                  organization_unit_name=ORG_UNIT_NAME,
+                  )
+        with open(sn_ca_name, 'w') as fh:
+            fh.write(str(serial_number))
+    with open(cert_ca_name, 'r') as fh:
+        return fh.read().strip()
+
+
+def gen_cert_iter(cn,
+                  extra_domainnames,
+                  authority_cn,
+                  authority_name,
+                  type,
+                  base_dir,
+                  dir_name,
+                  ):
+    week_number = datetime.now().isocalendar().week
+    root_dir_name = join(base_dir, PKI_DIR, authority_cn)
+    ca_dir_name = join(root_dir_name, 'ca')
+    key_ca_name = join(ca_dir_name, 'private.key')
+    cert_ca_name = join(ca_dir_name, f'certificate_{week_number}.crt')
+    sn_name = join(dir_name, f'serial_number')
+    key_name = join(dir_name, f'private.key')
+    cert_name = join(dir_name, f'certificate_{week_number}.crt')
+    if not isfile(cert_ca_name):
+        raise Exception(f'cannot find CA file "{cert_ca_name}"')
+    if not isfile(cert_name):
+        if not isdir(dir_name):
+            makedirs(dir_name)
+        if isfile(sn_name):
+            with open(sn_name, 'r') as fh:
+                serial_number = int(fh.read().strip()) + 1
+        else:
+            serial_number = 0
+        common_names = [cn]
+        common_names.extend(extra_domainnames)
+        _gen_cert(False,
+                  common_names,
+                  serial_number,
+                  10*24*60*60,
+                  key_name,
+                  cert_name,
+                  ca_cert=cert_ca_name,
+                  ca_key=key_ca_name,
+                  type=type,
+                  email_address=EMAIL,
+                  country_name=COUNTRY,
+                  locality_name=LOCALITY,
+                  state_or_province_name=STATE,
+                  organization_name=ORG_NAME,
+                  organization_unit_name=ORG_UNIT_NAME,
+                  )
+        with open(sn_name, 'w') as fh:
+            fh.write(str(serial_number))
+        for extra in extra_domainnames:
+            extra_dir_name = join(base_dir, PKI_DIR, authority_name + '+' + extra)
+            if not exists(extra_dir_name):
+                symlink(root_dir_name, extra_dir_name)
+    for extra in extra_domainnames:
+        extra_dir_name = join(base_dir, PKI_DIR, authority_name + '+' + extra)
+        if not exists(extra_dir_name):
+            raise Exception(f'file {extra_dir_name} not already exists that means subjectAltName is not set in certificat, please remove {cert_name}')
+    return cert_name
+
+
+def gen_cert(cn,
+             extra_domainnames,
+             authority_cn,
+             authority_name,
+             type,
+             file_type,
+             base_dir,
+             ):
+    if '.' in authority_name:
+        raise Exception(f'dot is not allowed in authority_name "{authority_name}"')
+    if type == 'server' and authority_cn is None:
+        authority_cn = cn
+    if authority_cn is None:
+        raise Exception(f'authority_cn is mandatory when authority type is client')
+    if extra_domainnames is None:
+        extra_domainnames = []
+    auth_cn = authority_name + '+' + authority_cn
+    dir_name = join(base_dir, PKI_DIR, auth_cn, 'certificats', cn, type)
+    if file_type == 'crt':
+        filename = gen_cert_iter(cn,
+                                 extra_domainnames,
+                                 auth_cn,
+                                 authority_name,
+                                 type,
+                                 base_dir,
+                                 dir_name,
+                                 )
+    else:
+        filename = join(dir_name, f'private.key')
+    with open(filename, 'r') as fh:
+        return fh.read().strip()
+
+
+def has_pub(cn,
+            base_dir,
+            ):
+    dir_name = join(base_dir, PKI_DIR, 'public', cn)
+    cert_name = join(dir_name, f'public.pub')
+    return isfile(cert_name)
+
+
+def gen_pub(cn,
+            file_type,
+            base_dir,
+            ):
+    dir_name = join(base_dir, PKI_DIR, 'public', cn)
+    key_name = join(dir_name, f'private.key')
+    if file_type == 'pub':
+        pub_name = join(dir_name, f'public.pub')
+        if not isfile(pub_name):
+            if not isdir(dir_name):
+                makedirs(dir_name)
+            key = _gen_key_pair()
+            with open(pub_name, "wt") as f:
+                f.write(dump_publickey(FILETYPE_PEM, key).decode("utf-8"))
+            with open(key_name, "wt") as f:
+                f.write(dump_privatekey(FILETYPE_PEM, key).decode("utf-8"))
+        filename = pub_name
+    else:
+        filename = key_name
+    with open(filename, 'r') as fh:
+        return fh.read().strip()

test.py

@@ -0,0 +1,472 @@
+#!/usr/bin/env python3
+
+from asyncio import run
+from os import listdir, link, makedirs
+from os.path import isdir, isfile, join
+from shutil import rmtree, copy2, copytree
+from json import load as json_load
+from yaml import load, SafeLoader
+from pprint import pprint
+from typing import Any
+from warnings import warn_explicit
+from copy import copy
+
+from tiramisu import Config
+from tiramisu.error import ValueWarning
+from rougail import RougailConfig, RougailConvert, RougailSystemdTemplate
+from rougail.utils import normalize_family
+#from rougail.error import TemplateError
+
+from utils import MULTI_FUNCTIONS, CONFIGS
+
+DATASET_DIRECTORY = '/home/gnunux/git/risotto_cadoles/risotto-dataset/seed'
+FUNCTIONS = 'funcs.py'
+CONFIG_DEST_DIR = 'configurations'
+SRV_DEST_DIR = 'srv'
+INSTALL_DIR = 'installations'
+#             "netbox.in.gnunux.info": {"applicationservices": ["netbox", "provider-systemd-machined"],
+#                                       "informations": {"zones_name": ["gnunux"]},
+#                                       "values": {"rougail.postgresql.pg_client_server_domainname": "postgresql.in.gnunux.info",
+#                                                  "rougail.redis.redis_client_server_domainname": "redis.in.gnunux.info",
+#                                                  "rougail.nginx.revprox_client_server_domainname": "revprox.in.gnunux.info",
+#                                                  "rougail.nginx.revprox_client_external_domainname": "in.gnunux.info"
+#                                                  }
+#                                       },
+
+
+
+
+with open('servers.json', 'r') as server_fh:
+    jsonfile = json_load(server_fh)
+    SERVERS = jsonfile['servers']
+    MODULES = jsonfile['modules']
+
+
+async def set_linked(linked_server: str,
+                     linked_provider: str,
+                     linked_value: str,
+                     linked_returns: str=None,
+                     dynamic: str=None,
+                     ):
+    if None in (linked_server, linked_provider, linked_value):
+        return
+    if linked_server not in CONFIGS:
+        warn_explicit(ValueWarning(f'cannot find linked server "{linked_server}"'),
+                      ValueWarning,
+                      __file__,
+                      0,
+                      )
+        return
+    config = CONFIGS[linked_server][0]
+    path = await config.information.get('provider:' + linked_provider, None)
+    if not path:
+        warn_explicit(ValueWarning(f'cannot find provider "{linked_provider}" in linked server "{linked_server}"'),
+                      ValueWarning,
+                      __file__,
+                      0,
+                      )
+        return
+    await config.property.read_write()
+    try:
+        option = config.forcepermissive.option(path)
+        if await option.option.ismulti():
+            values = await option.value.get()
+            if linked_value not in values:
+                values.append(linked_value)
+                await option.value.set(values)
+        else:
+            await option.value.set(linked_value)
+    except Exception as err:
+        await config.property.read_only()
+        raise err from err
+    await config.property.read_only()
+    if linked_returns is not None:
+        linked_variable = await config.information.get('provider:' + linked_returns, None)
+        if not linked_variable:
+            warn_explicit(ValueWarning(f'cannot find linked variable "{linked_returns}" in linked server "{linked_server}"'),
+                          ValueWarning,
+                          __file__,
+                          0,
+                          )
+            return
+    else:
+        linked_variable = None
+    if linked_variable is not None:
+        if dynamic:
+            linked_variable = linked_variable.replace('{suffix}', normalize_family(dynamic))
+        elif '{suffix}' in linked_variable:
+            idx = CONFIGS[linked_server][3]
+            linked_variable = linked_variable.replace('{suffix}', str(idx))
+        ret = await config.forcepermissive.option(linked_variable).value.get()
+    else:
+        ret = normalize_family(linked_value)
+    return ret
+
+
+async def get_linked_configuration(linked_server: str,
+                                   linked_provider: str,
+                                   dynamic: str=None,
+                                   ):
+    if linked_server not in CONFIGS:
+        warn_explicit(ValueWarning(f'cannot find linked server "{linked_server}"'),
+                      ValueWarning,
+                      __file__,
+                      1,
+                      )
+        return
+    config = CONFIGS[linked_server][0]
+    path = await config.information.get('provider:' + linked_provider, None)
+    if not path:
+        warn_explicit(ValueWarning(f'cannot find variable "{path}" in linked server "{linked_server}"'),
+                      ValueWarning,
+                      __file__,
+                      1,
+                      )
+        return
+    if dynamic:
+        path = path.replace('{suffix}', normalize_family(dynamic))
+    try:
+        return await config.forcepermissive.option(path).value.get()
+    except AttributeError as err:
+        warn_explicit(ValueWarning(f'cannot find get value of "{path}" in linked server "{linked_server}": {err}'),
+                      ValueWarning,
+                      __file__,
+                      1,
+                      )
+
+
+class Empty:
+    pass
+empty = Empty()
+
+
+async def set_linked_configuration(_linked_value: Any,
+                                   linked_server: str,
+                                   linked_provider: str,
+                                   linked_value: Any=empty,
+                                   dynamic: str=None,
+                                   leader_provider: str=None,
+                                   leader_value: Any=None,
+                                   ):
+    if linked_value is not empty:
+        _linked_value = linked_value
+    linked_value = _linked_value
+    if linked_server is None:
+        return
+    if linked_value is None or linked_server not in CONFIGS:
+        warn_explicit(ValueWarning(f'cannot find linked server "{linked_server}"'),
+                      ValueWarning,
+                      __file__,
+                      2,
+                      )
+        return
+    config = CONFIGS[linked_server][0]
+    path = await config.information.get('provider:' + linked_provider, None)
+    if not path:
+        warn_explicit(ValueWarning(f'cannot find variable "{path}" in linked server "{linked_server}"'),
+                      ValueWarning,
+                      __file__,
+                      2,
+                      )
+        return
+    if dynamic:
+        path = path.replace('{suffix}', normalize_family(dynamic))
+    await config.property.read_write()
+    try:
+        if leader_provider is not None:
+            leader_path = await config.information.get('provider:' + leader_provider, None)
+            if not leader_path:
+                await config.property.read_only()
+                warn_explicit(ValueWarning(f'cannot find leader variable "{path}" in linked server "{linked_server}"'),
+                              ValueWarning,
+                              __file__,
+                              2,
+                              )
+                return
+            if dynamic:
+                leader_path = leader_path.replace('{suffix}', normalize_family(dynamic))
+            values = await config.forcepermissive.option(leader_path).value.get()
+            if leader_value in values:
+                slave_idx = values.index(leader_value)
+                slave_option = config.forcepermissive.option(path, slave_idx)
+                if await slave_option.option.issubmulti():
+                    slave_values = await slave_option.value.get()
+                    if linked_value not in slave_values:
+                        slave_values.append(linked_value)
+                        await slave_option.value.set(slave_values)
+
+                else:
+                    await slave_option.value.set(linked_value)
+        else:
+            option = config.forcepermissive.option(path)
+            if await option.option.ismulti() and not isinstance(linked_value, list):
+                values = await option.value.get()
+                if linked_value not in values:
+                    values.append(linked_value)
+                    await option.value.set(values)
+            else:
+                await option.value.set(linked_value)
+    except AttributeError as err:
+        #raise ValueError(str(err)) from err
+        pass
+    except Exception as err:
+        await config.property.read_only()
+        raise err from err
+    await config.property.read_only()
+
+
+def tiramisu_display_name(kls,
+                          dyn_name: 'Base'=None,
+                          suffix: str=None,
+                          ) -> str:
+    if dyn_name is not None:
+        name = kls.impl_getpath() + suffix
+    else:
+        name = kls.impl_getpath()
+    return name
+
+
+def load_applications():
+    applications = {}
+    for distrib in listdir(DATASET_DIRECTORY):
+        distrib_dir = join(DATASET_DIRECTORY, distrib, 'applicationservice')
+        if not isdir(distrib_dir):
+            continue
+        for release in listdir(distrib_dir):
+            release_dir = join(distrib_dir, release)
+            if not isdir(release_dir):
+                continue
+            for applicationservice in listdir(release_dir):
+                applicationservice_dir = join(release_dir, applicationservice)
+                if not isdir(applicationservice_dir):
+                    continue
+                if applicationservice in applications:
+                    raise Exception(f'multi applicationservice: {applicationservice} ({applicationservice_dir} <=> {applications[applicationservice]})')
+                applications[applicationservice] = applicationservice_dir
+    return applications
+
+
+class ModuleCfg():
+    def __init__(self):
+        self.dictionaries_dir = []
+        self.modules = []
+        self.functions_file = [FUNCTIONS]
+        self.templates_dir = []
+        self.extra_dictionaries = {}
+        self.servers = []
+
+
+def build_module(module_name, datas, module_infos):
+    install_dir = join(INSTALL_DIR, module_name)
+    makedirs(install_dir)
+    applications = load_applications()
+    cfg = ModuleCfg()
+    module_infos[module_name] = cfg
+    def calc_depends(appname, added):
+        if appname in added:
+            return
+        as_dir = applications[appname]
+        cfg.modules.append(appname)
+        dictionaries_dir = join(as_dir, 'dictionaries')
+        if isdir(dictionaries_dir):
+            cfg.dictionaries_dir.append(dictionaries_dir)
+        funcs_dir = join(as_dir, 'funcs')
+        if isdir(funcs_dir):
+            for f in listdir(funcs_dir):
+                if f.startswith('__'):
+                    continue
+                cfg.functions_file.append(join(funcs_dir, f))
+        templates_dir = join(as_dir, 'templates')
+        if isdir(templates_dir):
+            cfg.templates_dir.append(templates_dir)
+        extras_dir = join(as_dir, 'extras')
+        if isdir(extras_dir):
+            for extra in listdir(extras_dir):
+                extra_dir = join(extras_dir, extra)
+                if isdir(extra_dir):
+                    cfg.extra_dictionaries.setdefault(extra, []).append(extra_dir)
+        for type in ['image', 'install']:
+            manual_dir = join(as_dir, 'manual', type)
+            if isdir(manual_dir):
+                for filename in listdir(manual_dir):
+                    src_file = join(manual_dir, filename)
+                    if type == 'image':
+                        dst_file = join(install_dir, filename)
+                        verify = False
+                    else:
+                        dst_file= join(INSTALL_DIR, filename)
+                        verify = True
+                    if isdir(src_file):
+                        if not isdir(dst_file):
+                            makedirs(dst_file)
+                        for subfilename in listdir(src_file):
+                            if not verify or not isfile(dst_file):
+                                src = join(src_file, subfilename)
+                                dst = join(dst_file, subfilename)
+                                if isfile(src):
+                                    copy2(src, dst)
+                                else:
+                                    copytree(src, dst)
+                    elif not verify or not isfile(dst_file):
+                        src = join(manual_dir, filename)
+                        dst = dst_file
+                        if isfile(src):
+                            copy2(src, dst)
+                        else:
+                            copytree(src, dst)
+        added.append(appname)
+        with open(join(as_dir, 'applicationservice.yml')) as yaml:
+            app = load(yaml, Loader=SafeLoader)
+
+        for xml in app.get('depends', []):
+            calc_depends(xml, added)
+    added = []
+    for applicationservice in datas['applicationservices']:
+        calc_depends(applicationservice, added)
+
+
+async def build(server_name, datas, module_infos):
+    if server_name in CONFIGS:
+        raise Exception(f'server "{server_name}" is duplicate')
+    cfg = RougailConfig.copy()
+    module_info = module_infos[datas['module']]
+    module_info.servers.append(server_name)
+    if datas['module'] == 'host':
+        cfg['tmpfile_dest_dir'] = datas['values']['rougail.host_install_dir'] + '/host/configurations/' + server_name
+    cfg['templates_dir'] = module_info.templates_dir
+    cfg['dictionaries_dir'] = module_info.dictionaries_dir
+    cfg['functions_file'] = module_info.functions_file
+    cfg['multi_functions'] = MULTI_FUNCTIONS 
+    cfg['extra_dictionaries'] = module_info.extra_dictionaries
+    cfg['extra_annotators'].append('risotto_setting.rougail')
+    optiondescription = {'set_linked': set_linked,
+                         'get_linked_configuration': get_linked_configuration,
+                         'set_linked_configuration': set_linked_configuration,
+                         }
+    cfg['internal_functions'] = list(optiondescription.keys())
+    try:
+        eolobj = RougailConvert(cfg)
+    except Exception as err:
+        print(f'Try to load {module_info.modules}')
+        raise err from err
+    xml = eolobj.save(None)
+    #print(xml)
+    #cfg['patches_dir'] = join(test_dir, 'patches')
+    cfg['tmp_dir'] = 'tmp'
+    cfg['destinations_dir'] = join(INSTALL_DIR, datas['module'], CONFIG_DEST_DIR, server_name)
+    if isdir('tmp'):
+        rmtree('tmp')
+    makedirs('tmp')
+    makedirs(cfg['destinations_dir'])
+    try:
+        exec(xml, None, optiondescription)
+    except Exception as err:
+        print(xml)
+        raise Exception(f'unknown error when load tiramisu object {err}') from err
+    config = await Config(optiondescription['option_0'], display_name=tiramisu_display_name)
+    await config.property.read_write()
+    try:
+        if await config.option('machine.add_srv').value.get():
+            srv = join(INSTALL_DIR, SRV_DEST_DIR, server_name)
+        else:
+            srv = None
+    except AttributeError:
+        srv = None
+    await config.property.read_write()
+    CONFIGS[server_name] = (config, cfg, srv, 0)
+
+
+async def value_pprint(dico, config):
+    pprint_dict = {}
+    for path, value in dico.items():
+        if await config.option(path).option.type() == 'password' and value:
+            value = 'X' * len(value)
+        pprint_dict[path] = value
+    pprint(pprint_dict)
+
+
+async def set_values(server_name, config, datas):
+    if 'informations' in datas:
+        for information, value in datas['informations'].items():
+            await config.information.set(information, value)
+        if 'extra_domainnames' in datas['informations']:
+            for idx, extra_domainname in enumerate(datas['informations']['extra_domainnames']):
+                if extra_domainname in CONFIGS:
+                    raise Exception(f'server "{server_name}" is duplicate')
+                value = list(CONFIGS[server_name])
+                value[3] = idx + 1
+                CONFIGS[extra_domainname] = tuple(value)
+    await config.information.set('server_name', server_name)
+    await config.property.read_write()
+    try:
+        if 'values' in datas:
+            for path, value in datas['values'].items():
+                if isinstance(value, dict):
+                    for idx, val in value.items():
+                        await config.option(path, int(idx)).value.set(val)
+                else:
+                    await config.option(path).value.set(value)
+    except Exception as err:
+        await value_pprint(await config.value.dict(), config)
+        error_msg = f'cannot configure server "{server_name}": {err}'
+        raise Exception(error_msg) from err
+    await config.property.read_only()
+    #await config.value.dict()
+
+
+async def valid_mandatories(server_name, config):
+    mandatories = await config.value.mandatory()
+    if mandatories:
+        print()
+        print(f'=== Configuration: {server_name} ===')
+        await config.property.pop('mandatory')
+        await value_pprint(await config.value.dict(), config)
+        raise Exception(f'server "{server_name}" has mandatories variables without values "{", ".join(mandatories)}"')
+
+
+async def templates(server_name, config, cfg, srv, int_idx):
+    values = await config.value.dict()
+    engine = RougailSystemdTemplate(config, cfg)
+#    if server_name == 'roundcube.in.gnunux.info':
+#        print()
+#        print(f'=== Configuration: {server_name} ===')
+#        pprint(values)
+    try:
+        await engine.instance_files()
+    except Exception as err:
+        print()
+        print(f'=== Configuration: {server_name} ===')
+        await value_pprint(values, config)
+        raise err from err
+    if srv:
+        makedirs(srv)
+
+
+async def main():
+    if isdir(INSTALL_DIR):
+        rmtree(INSTALL_DIR)
+    makedirs(INSTALL_DIR)
+    module_infos = {}
+    for module_name, datas in MODULES.items():
+        build_module(module_name, datas, module_infos)
+    for server_name, datas in SERVERS.items():
+        await build(server_name, datas, module_infos)
+    for module_name, cfg in module_infos.items():
+        with open(join(INSTALL_DIR, module_name, 'install_machines'), 'w') as fh:
+            for server_name in cfg.servers:
+                fh.write(f'./install_machine {module_name} {server_name}\n')
+    for server_name, datas in SERVERS.items():
+        await set_values(server_name, CONFIGS[server_name][0], datas)
+    for server_name in SERVERS:
+        config = CONFIGS[server_name][0]
+        await config.property.pop('mandatory')
+        await config.value.dict()
+        await config.property.add('mandatory')
+    for server_name in SERVERS:
+        await valid_mandatories(server_name, CONFIGS[server_name][0])
+#    print(await CONFIGS['revprox.in.gnunux.info'][0].option('nginx.reverse_proxy_for_netbox_in_gnunux_info.reverse_proxy_netbox_in_gnunux_info.revprox_url_netbox_in_gnunux_info', 0).value.get())
+    for server_name in SERVERS:
+        await templates(server_name, *CONFIGS[server_name])
+
+
+run(main())