From 946506f27c2a0779f9928adb0589fb05612c8f1a Mon Sep 17 00:00:00 2001 From: Emmanuel Garette Date: Tue, 8 Mar 2022 20:47:55 +0100 Subject: [PATCH] first commit --- funcs.py | 233 ++++++++++++++++++++++++ servers.json | 191 ++++++++++++++++++++ src/__init__.py | 0 src/utils.py | 10 + src/x509.py | 254 ++++++++++++++++++++++++++ test.py | 472 ++++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 1160 insertions(+) create mode 100644 funcs.py create mode 100644 servers.json create mode 100644 src/__init__.py create mode 100644 src/utils.py create mode 100644 src/x509.py create mode 100755 test.py diff --git a/funcs.py b/funcs.py new file mode 100644 index 0000000..ff41dfd --- /dev/null +++ b/funcs.py @@ -0,0 +1,233 @@ +from tiramisu import valid_network_netmask, valid_ip_netmask, valid_broadcast, valid_in_network, valid_not_equal as valid_differ, valid_not_equal, calc_value +from ipaddress import ip_address +from os.path import dirname, abspath, join as _join, isdir as _isdir, isfile as _isfile +from typing import List +from json import load +from secrets import token_urlsafe as _token_urlsafe + +from rougail.utils import normalize_family + +from utils import multi_function, CONFIGS +from x509 import gen_cert as _x509_gen_cert, gen_ca as _x509_gen_ca, gen_pub as _x509_gen_pub, has_pub as _x509_has_pub +# ============================================================= +# fork of risotto-setting/src/risotto_setting/config/config.py + +with open('servers.json', 'r') as server_fh: + ZONES_SERVER = load(server_fh) + + +ZONES = None +DOMAINS = None +HERE = dirname(abspath(__file__)) + + +def load_zones(): + global ZONES + if ZONES is not None: + return + ZONES = ZONES_SERVER['zones'] + for server_name, server in ZONES_SERVER['servers'].items(): + if 'informations' not in server: + continue + server_zones = server['informations']['zones_name'] + server_extra_domainnames = server['informations'].get('extra_domainnames', []) + if len(server_zones) > 1 and len(server_zones) != len(server_extra_domainnames) + 1: + raise Exception(f'the server "{server_name}" has more that one zone, please set correct number of extra_domainnames ({len(server_zones) - 1} instead of {len(server_extra_domainnames)})') + + for idx, zone_name in enumerate(server_zones): + zone_domain_name = ZONES[zone_name]['domain_name'] + if idx == 0: + zone_server_name = server_name + else: + zone_server_name = server_extra_domainnames[idx - 1] + server_domain_name = zone_server_name.split('.', 1)[1] + if zone_domain_name and zone_domain_name != server_domain_name: + raise Exception(f'wrong server_name "{zone_server_name}" in zone "{zone_name}" should ends with "{zone_domain_name}"') + ZONES[zone_name].setdefault('hosts', []).append(server_name) + + +def load_domains(): + load_zones() + global DOMAINS + if DOMAINS is not None: + return + DOMAINS = {} + for zone_name, zone in ZONES_SERVER['zones'].items(): + if 'domain_name' in zone: + hosts = [] + ips = [] + for host in ZONES[zone_name].get('hosts', []): + hosts.append(host.split('.', 1)[0]) + ips.append(get_ip(host, [zone_name], 0)) + DOMAINS[zone['domain_name']] = (tuple(hosts), tuple(ips)) + + +def get_ip(server_name: str, + zones_name: List[str], + index: str, + ) -> str: + if server_name is None: + return + load_zones() + index = int(index) + zone_name = zones_name[index] + if zone_name not in ZONES: + raise ValueError(f"cannot set IP in unknown zone '{zone_name}'") + zone = ZONES[zone_name] + if server_name not in zone['hosts']: + raise ValueError(f"cannot set IP in unknown server '{server_name}'") + server_index = zone['hosts'].index(server_name) +# print(server_name, zones_name, index, str(ip_address(zone['start_ip']) + server_index)) + return str(ip_address(zone['start_ip']) + server_index) + + +@multi_function +def get_chain(authority_cn, + authority_name, + ): + if not authority_name or authority_name is None: + if isinstance(authority_name, list): + return [] + return + if not isinstance(authority_cn, list): + is_list = False + authority_cn = [authority_cn] + else: + is_list = True + authorities = [] + + for auth_cn in authority_cn: + ret = _x509_gen_ca(auth_cn, + authority_name, + HERE, + ) + if not is_list: + return ret + authorities.append(ret) + return authorities + + +@multi_function +def get_certificate(cn, + authority_name, + authority_cn=None, + extra_domainnames=[], + type='server', + ): + if isinstance(cn, list) and extra_domainnames: + raise Exception('cn cannot be a list with extra_domainnames set') + if not cn or authority_name is None: + if isinstance(cn, list): + return [] + return + return _x509_gen_cert(cn, + extra_domainnames, + authority_cn, + authority_name, + type, + 'crt', + HERE, + ) + + +@multi_function +def get_private_key(cn, + authority_name=None, + authority_cn=None, + type='server', + ): + if not cn: + if isinstance(cn, list): + return [] + return + if authority_name is None: + if _x509_has_pub(cn, HERE): + return _x509_gen_pub(cn, + 'key', + HERE, + ) + if isinstance(cn, list): + return [] + return + return _x509_gen_cert(cn, + [], + authority_cn, + authority_name, + type, + 'key', + HERE, + ) + + +def get_public_key(cn): + if not cn: + return + return _x509_gen_pub(cn, + 'pub', + HERE, + ) + + +def zone_information(zone_name: str, + type: str, + multi: bool=False, + index: int=None, + ) -> str: + if not zone_name: + return + if type == 'gateway' and index != 0: + return + load_zones() + if zone_name not in ZONES: + raise ValueError(f"cannot get zone informations in unknown zone '{zone_name}'") + zone = ZONES[zone_name] + if type not in zone: + raise ValueError(f"unknown type '{type}' in zone '{zone_name}'") + value = zone[type] + if multi: + value = [value] + return value + + +def get_internal_zones() -> List[str]: + load_domains() + return list(DOMAINS.keys()) + + +@multi_function +def get_zones_info(type: str) -> str: + ret = [] + for data in ZONES_SERVER['zones'].values(): + ret.append(data[type]) + return ret + +@multi_function +def get_internal_zone_names() -> List[str]: + load_zones() + return list(ZONES.keys()) + + +def get_internal_zone_information(zone: str, + info: str, + ) -> str: + load_domains() + if info == 'cidr': + return ZONES[zone]['gateway'] + '/' + ZONES[zone]['network'].split('/')[-1] + return ZONES[zone][info] + + +def get_internal_info_in_zone(zone: str, + auto: bool, + type: str, + index: int=None, + ) -> List[str]: + if not auto: + return + for domain_name, domain in DOMAINS.items(): + if zone == domain_name: + if type == 'host': + return list(domain[0]) + else: + return domain[1][index] + +# ============================================================= diff --git a/servers.json b/servers.json new file mode 100644 index 0000000..9dbe0c5 --- /dev/null +++ b/servers.json @@ -0,0 +1,191 @@ +{"zones": {"external": {"network": "192.168.45.0/24", + "gateway": "192.168.45.1", + "start_ip": "192.168.45.10", + "domain_name": "in.silique.fr" + }, + "list": {"network": "192.168.47.0/24", + "gateway": "192.168.47.1", + "start_ip": "192.168.47.10", + "domain_name": "list.silique.fr" + } + }, + "modules": {"host": {"applicationservices": ["host-systemd-machined"]}, + "unbound": {"applicationservices": ["unbound", "provider-systemd-machined"]}, + "nsd": {"applicationservices": ["nsd", "provider-systemd-machined"]}, + "revprox": {"applicationservices": ["nginx-reverse-proxy-server", "provider-systemd-machined"]}, + "postgresql": {"applicationservices": ["postgresql-server", "provider-systemd-machined"]}, + "redis": {"applicationservices": ["redis-server", "provider-systemd-machined"]}, + "ldap": {"applicationservices": ["openldap-server", "provider-systemd-machined"]}, + "lemonldap": {"applicationservices": ["lemonldap", "provider-systemd-machined"]}, + "nextcloud": {"applicationservices": ["nextcloud", "provider-systemd-machined"]}, + "mail": {"applicationservices": ["postfix-relay", "provider-systemd-machined"]}, + "dovecot": {"applicationservices": ["dovecot", "provider-systemd-machined"]}, + "mailman": {"applicationservices": ["mailman", "provider-systemd-machined"]}, + "gitea": {"applicationservices": ["gitea", "provider-systemd-machined"]}, + "roundcube": {"applicationservices": ["roundcube", "provider-systemd-machined"]}, + "vaultwarden": {"applicationservices": ["vaultwarden", "provider-systemd-machined"]} + }, + "servers": {"cloud": {"module": "host", + "values": {"rougail.host_install_dir": "/root/installations", + "rougail.host_dhcp_interface": ["enp3s0"] + } + }, + "unbound.in.silique.fr": {"module": "unbound", + "informations": {"zones_name": ["external"]}, + "values": {"rougail.host": "cloud", + "rougail.dns_resolver.unbound_default_forwards": ["8.8.8.8"] + } + }, + "nsd.in.silique.fr": {"module": "nsd", + "informations": {"zones_name": ["external", "list"], + "extra_domainnames": ["nsd.list.silique.fr"] + }, + "values": {"rougail.host": "cloud", + "rougail.dns_server.nsd_resolver": "unbound.in.silique.fr" + } + }, + "revprox.in.silique.fr": {"module": "revprox", + "informations": {"zones_name": ["external"]}, + "values": {"rougail.host": "cloud", + "rougail.dns.dns_client_address": "nsd.in.silique.fr", + "rougail.nginx.nginx_default": "cloud.silique.fr" + } + }, + "mail.in.silique.fr": {"module": "mail", + "informations": {"zones_name": ["external", "list"], + "extra_domainnames": ["mail.list.silique.fr"] + }, + "values": {"rougail.host": "cloud", + "rougail.dns.dns_client_address": "unbound.in.silique.fr", + "rougail.postfix.postfix_mail_hostname": "mail.silique.fr" + } + }, + "dovecot.in.silique.fr": {"module": "dovecot", + "informations": {"zones_name": ["external"] + }, + "values": {"rougail.host": "cloud", + "rougail.dns.dns_client_address": "nsd.in.silique.fr", + "rougail.postfix.postfix_my_domains": ["cloud.silique.fr"], + "rougail.smtp.smtp_relay_address": "mail.in.silique.fr", + "rougail.annuaire.ldap_server_address": "ldap.in.silique.fr", + "rougail.dovecot.revprox_server_domainname": "revprox.in.silique.fr", + "rougail.oauth2_client.oauth2_client_server_domainname": "lemonldap.in.silique.fr" + } + }, + "redis-rc.in.silique.fr": {"module": "redis", + "informations": {"zones_name": ["external"]}, + "values": {"rougail.host": "cloud", + "rougail.dns.dns_client_address": "nsd.in.silique.fr" + } + }, + "redis-nc.in.silique.fr": {"module": "redis", + "informations": {"zones_name": ["external"]}, + "values": {"rougail.host": "cloud", + "rougail.dns.dns_client_address": "nsd.in.silique.fr" + } + }, + "redis-gi.in.silique.fr": {"module": "redis", + "informations": {"zones_name": ["external"]}, + "values": {"rougail.host": "cloud", + "rougail.dns.dns_client_address": "nsd.in.silique.fr" + } + }, + "ldap.in.silique.fr": {"module": "ldap", + "informations": {"zones_name": ["external"]}, + "values": {"rougail.host": "cloud", + "rougail.dns.dns_client_address": "nsd.in.silique.fr", + "accounts.users.ldap_user_mail": ["gnunux@silique.fr", "bbohard@silique.fr", "ddtddt@silique.fr"], + "accounts.users.ldap_user_uid": {"0": "gnunux", "1": "bbohard", "2": "ddtddt"}, + "accounts.users.ldap_user_sn": {"0": "Emmanuel", "1": "Benjamin", "2": "Damien"}, + "accounts.users.ldap_user_gn": {"0": "Garette", "1": "Bohard", "2": "Thomas"} + } + }, + "lemonldap.in.silique.fr": {"module": "lemonldap", + "informations": {"zones_name": ["external"]}, + "values": {"rougail.host": "cloud", + "rougail.dns.dns_client_address": "nsd.in.silique.fr", + "rougail.annuaire.ldap_server_address": "ldap.in.silique.fr", + "rougail.smtp.smtp_relay_address": "mail.in.silique.fr", + "rougail.nginx.revprox_client_server_domainname": "revprox.in.silique.fr", + "rougail.nginx.revprox_client_external_domainname": "auth.silique.fr", + "rougail.lemonldap.lemon_domain": "cloud.silique.fr", + "rougail.lemonldap.lemon_mail_admin": "gnunux@silique.fr" + } + }, + "nextcloud.in.silique.fr": {"module": "nextcloud", + "informations": {"zones_name": ["external"]}, + "values": {"rougail.host": "cloud", + "rougail.dns.dns_client_address": "nsd.in.silique.fr", + "rougail.nextcloud.nextcloud_mail_admin": "gnunux@silique.fr", + "rougail.postgresql.pg_client_server_domainname": "postgresql.in.silique.fr", + "rougail.annuaire.ldap_server_address": "ldap.in.silique.fr", + "rougail.redis.redis_client_server_domainname": "redis-nc.in.silique.fr", + "rougail.smtp.smtp_relay_address": "mail.in.silique.fr", + "rougail.nginx.revprox_client_server_domainname": "revprox.in.silique.fr", + "rougail.nginx.revprox_client_external_domainname": "cloud.silique.fr", + "rougail.oauth2_client.oauth2_client_server_domainname": "lemonldap.in.silique.fr" + } + }, + "roundcube.in.silique.fr": {"module": "roundcube", + "informations": {"zones_name": ["external"]}, + "values": {"rougail.host": "cloud", + "rougail.dns.dns_client_address": "nsd.in.silique.fr", + "rougail.postgresql.pg_client_server_domainname": "postgresql.in.silique.fr", + "rougail.annuaire.ldap_server_address": "ldap.in.silique.fr", + "rougail.nginx.revprox_client_server_domainname": "revprox.in.silique.fr", + "rougail.nginx.revprox_client_external_domainname": "cloud.silique.fr", + "rougail.redis.redis_client_server_domainname": "redis-rc.in.silique.fr", + "rougail.imap.imap_address": "dovecot.in.silique.fr", + "rougail.oauth2_client.oauth2_client_server_domainname": "lemonldap.in.silique.fr" + } + }, + "postgresql.in.silique.fr": {"module": "postgresql", + "informations": {"zones_name": ["external", "list"], + "extra_domainnames": ["postgresql.list.silique.fr"] + }, + "values": {"rougail.host": "cloud", + "rougail.dns.dns_client_address": "nsd.in.silique.fr" + } + }, + "mailman.list.silique.fr": {"module": "mailman", + "informations": {"zones_name": ["list"] + }, + "values": {"rougail.host": "cloud", + "rougail.dns.dns_client_address": "nsd.list.silique.fr", + "rougail.smtp.smtp_relay_address": "mail.list.silique.fr", + "rougail.postgresql.pg_client_server_domainname": "postgresql.list.silique.fr", + "rougail.nginx.revprox_client_server_domainname": "revprox.in.silique.fr", + "rougail.nginx.revprox_client_external_domainname": "cloud.silique.fr", + "rougail.mailman.mailman_mail_owner": "admin@silique.fr", + "rougail.mailman.mailman_domains": ["lists.silique.fr"], + "rougail.oauth2_client.oauth2_client_server_domainname": "lemonldap.in.silique.fr", + "mailman.list_lists_silique_fr.name_lists_silique_fr": ["list1", "list2"] + } + }, + "vaultwarden.in.silique.fr": {"module": "vaultwarden", + "informations": {"zones_name": ["external"]}, + "values": {"rougail.host": "cloud", + "rougail.dns.dns_client_address": "nsd.in.silique.fr", + "rougail.vaultwarden.vaultwarden_admin_email": "gnunux@silique.fr", + "rougail.postgresql.pg_client_server_domainname": "postgresql.in.silique.fr", + "rougail.nginx.revprox_client_server_domainname": "revprox.in.silique.fr", + "rougail.nginx.revprox_client_external_domainname": "cloud.silique.fr", + "rougail.smtp.smtp_relay_address": "mail.in.silique.fr" + } + }, + "gitea.in.silique.fr": {"module": "gitea", + "informations": {"zones_name": ["external"] + }, + "values": {"rougail.host": "cloud", + "rougail.dns.dns_client_address": "nsd.in.silique.fr", + "rougail.smtp.smtp_relay_address": "mail.in.silique.fr", + "rougail.gitea.gitea_mail_sender": "gitea@silique.fr", + "rougail.postgresql.pg_client_server_domainname": "postgresql.in.silique.fr", + "rougail.nginx.revprox_client_server_domainname": "revprox.in.silique.fr", + "rougail.redis.redis_client_server_domainname": "redis-gi.in.silique.fr", + "rougail.oauth2_client.oauth2_client_server_domainname": "lemonldap.in.silique.fr", + "rougail.nginx.revprox_client_external_domainname": "cloud.silique.fr" + } + } + } +} diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/utils.py b/src/utils.py new file mode 100644 index 0000000..a8e669c --- /dev/null +++ b/src/utils.py @@ -0,0 +1,10 @@ +MULTI_FUNCTIONS = [] +CONFIGS = {} + + +def multi_function(function): + global MULTI_FUNCTIONS + name = function.__name__ + if name not in MULTI_FUNCTIONS: + MULTI_FUNCTIONS.append(name) + return function diff --git a/src/x509.py b/src/x509.py new file mode 100644 index 0000000..dbaa046 --- /dev/null +++ b/src/x509.py @@ -0,0 +1,254 @@ +from OpenSSL.crypto import load_certificate, load_privatekey, dump_certificate, dump_privatekey, dump_publickey, PKey, X509, X509Extension, TYPE_RSA, FILETYPE_PEM +from os import makedirs, symlink +from os.path import join, isdir, isfile, exists +#from shutil import rmtree +from datetime import datetime + + +PKI_DIR = 'pki/x509' +#FIXME +EMAIL = 'gnunux@gnunux.info' +COUNTRY = 'FR' +LOCALITY = 'Dijon' +STATE = 'France' +ORG_NAME = 'Cadoles' +ORG_UNIT_NAME = 'CSS' + + +def _gen_key_pair(): + key = PKey() + key.generate_key(TYPE_RSA, 4096) + return key + + +def _gen_cert(is_ca, + common_names, + serial_number, + validity_end_in_seconds, + key_file, + cert_file, + type=None, + ca_cert=None, + ca_key=None, + email_address=None, + country_name=None, + locality_name=None, + state_or_province_name=None, + organization_name=None, + organization_unit_name=None, + ): + #can look at generated file using openssl: + #openssl x509 -inform pem -in selfsigned.crt -noout -text + # create a key pair + if isfile(key_file): + with open(key_file) as fh: + filecontent = bytes(fh.read(), 'utf-8') + key = load_privatekey(FILETYPE_PEM, filecontent) + else: + key = _gen_key_pair() + cert = X509() + cert.set_version(2) + cert.get_subject().C = country_name + cert.get_subject().ST = state_or_province_name + cert.get_subject().L = locality_name + cert.get_subject().O = organization_name + cert.get_subject().OU = organization_unit_name + cert.get_subject().CN = common_names[0] + cert.get_subject().emailAddress = email_address + cert_ext = [] + if not is_ca: + cert_ext.append(X509Extension(b'basicConstraints', False, b'CA:FALSE')) + cert_ext.append(X509Extension(b'keyUsage', True, b'digitalSignature, keyEncipherment')) + cert_ext.append(X509Extension(b'subjectAltName', False, ", ".join([f'DNS:{common_name}' for common_name in common_names]).encode('ascii'))) + if type == 'server': + cert_ext.append(X509Extension(b'extendedKeyUsage', True, b'serverAuth')) + else: + cert_ext.append(X509Extension(b'extendedKeyUsage', True, b'clientAuth')) + else: + cert_ext.append(X509Extension(b'basicConstraints', False, b'CA:TRUE')) + cert_ext.append(X509Extension(b"keyUsage", True, b'keyCertSign, cRLSign')) + cert_ext.append(X509Extension(b'subjectAltName', False, f'email:{email_address}'.encode())) + cert_ext.append(X509Extension(b'subjectKeyIdentifier', False, b"hash", subject=cert)) + cert.add_extensions(cert_ext) + cert.set_serial_number(serial_number) + cert.gmtime_adj_notBefore(0) + cert.gmtime_adj_notAfter(validity_end_in_seconds) + if is_ca: + ca_cert = cert + ca_key = key + else: + with open(ca_cert) as fh: + filecontent = bytes(fh.read(), 'utf-8') + ca_cert = load_certificate(FILETYPE_PEM, filecontent) + with open(ca_key) as fh: + filecontent = bytes(fh.read(), 'utf-8') + ca_key = load_privatekey(FILETYPE_PEM, filecontent) + cert.set_issuer(ca_cert.get_subject()) + cert.add_extensions([X509Extension(b"authorityKeyIdentifier", False, b'keyid:always', issuer=ca_cert)]) + cert.set_pubkey(key) + cert.sign(ca_key, "sha512") + + with open(cert_file, "wt") as f: + f.write(dump_certificate(FILETYPE_PEM, cert).decode("utf-8")) + with open(key_file, "wt") as f: + f.write(dump_privatekey(FILETYPE_PEM, key).decode("utf-8")) + + +def gen_ca(authority_dns, + authority_name, + base_dir, + ): + authority_cn = authority_name + '+' + authority_dns + week_number = datetime.now().isocalendar().week + root_dir_name = join(base_dir, PKI_DIR, authority_cn) + ca_dir_name = join(root_dir_name, 'ca') + sn_ca_name = join(ca_dir_name, 'serial_number') + key_ca_name = join(ca_dir_name, 'private.key') + cert_ca_name = join(ca_dir_name, f'certificate_{week_number}.crt') + if not isfile(cert_ca_name): + if not isdir(ca_dir_name): + # rmtree(ca_dir_name) + makedirs(ca_dir_name) + if isfile(sn_ca_name): + with open(sn_ca_name, 'r') as fh: + serial_number = int(fh.read().strip()) + 1 + else: + serial_number = 0 + _gen_cert(True, + [authority_cn], + serial_number, + 10*24*60*60, + key_ca_name, + cert_ca_name, + email_address=EMAIL, + country_name=COUNTRY, + locality_name=LOCALITY, + state_or_province_name=STATE, + organization_name=ORG_NAME, + organization_unit_name=ORG_UNIT_NAME, + ) + with open(sn_ca_name, 'w') as fh: + fh.write(str(serial_number)) + with open(cert_ca_name, 'r') as fh: + return fh.read().strip() + + +def gen_cert_iter(cn, + extra_domainnames, + authority_cn, + authority_name, + type, + base_dir, + dir_name, + ): + week_number = datetime.now().isocalendar().week + root_dir_name = join(base_dir, PKI_DIR, authority_cn) + ca_dir_name = join(root_dir_name, 'ca') + key_ca_name = join(ca_dir_name, 'private.key') + cert_ca_name = join(ca_dir_name, f'certificate_{week_number}.crt') + sn_name = join(dir_name, f'serial_number') + key_name = join(dir_name, f'private.key') + cert_name = join(dir_name, f'certificate_{week_number}.crt') + if not isfile(cert_ca_name): + raise Exception(f'cannot find CA file "{cert_ca_name}"') + if not isfile(cert_name): + if not isdir(dir_name): + makedirs(dir_name) + if isfile(sn_name): + with open(sn_name, 'r') as fh: + serial_number = int(fh.read().strip()) + 1 + else: + serial_number = 0 + common_names = [cn] + common_names.extend(extra_domainnames) + _gen_cert(False, + common_names, + serial_number, + 10*24*60*60, + key_name, + cert_name, + ca_cert=cert_ca_name, + ca_key=key_ca_name, + type=type, + email_address=EMAIL, + country_name=COUNTRY, + locality_name=LOCALITY, + state_or_province_name=STATE, + organization_name=ORG_NAME, + organization_unit_name=ORG_UNIT_NAME, + ) + with open(sn_name, 'w') as fh: + fh.write(str(serial_number)) + for extra in extra_domainnames: + extra_dir_name = join(base_dir, PKI_DIR, authority_name + '+' + extra) + if not exists(extra_dir_name): + symlink(root_dir_name, extra_dir_name) + for extra in extra_domainnames: + extra_dir_name = join(base_dir, PKI_DIR, authority_name + '+' + extra) + if not exists(extra_dir_name): + raise Exception(f'file {extra_dir_name} not already exists that means subjectAltName is not set in certificat, please remove {cert_name}') + return cert_name + + +def gen_cert(cn, + extra_domainnames, + authority_cn, + authority_name, + type, + file_type, + base_dir, + ): + if '.' in authority_name: + raise Exception(f'dot is not allowed in authority_name "{authority_name}"') + if type == 'server' and authority_cn is None: + authority_cn = cn + if authority_cn is None: + raise Exception(f'authority_cn is mandatory when authority type is client') + if extra_domainnames is None: + extra_domainnames = [] + auth_cn = authority_name + '+' + authority_cn + dir_name = join(base_dir, PKI_DIR, auth_cn, 'certificats', cn, type) + if file_type == 'crt': + filename = gen_cert_iter(cn, + extra_domainnames, + auth_cn, + authority_name, + type, + base_dir, + dir_name, + ) + else: + filename = join(dir_name, f'private.key') + with open(filename, 'r') as fh: + return fh.read().strip() + + +def has_pub(cn, + base_dir, + ): + dir_name = join(base_dir, PKI_DIR, 'public', cn) + cert_name = join(dir_name, f'public.pub') + return isfile(cert_name) + + +def gen_pub(cn, + file_type, + base_dir, + ): + dir_name = join(base_dir, PKI_DIR, 'public', cn) + key_name = join(dir_name, f'private.key') + if file_type == 'pub': + pub_name = join(dir_name, f'public.pub') + if not isfile(pub_name): + if not isdir(dir_name): + makedirs(dir_name) + key = _gen_key_pair() + with open(pub_name, "wt") as f: + f.write(dump_publickey(FILETYPE_PEM, key).decode("utf-8")) + with open(key_name, "wt") as f: + f.write(dump_privatekey(FILETYPE_PEM, key).decode("utf-8")) + filename = pub_name + else: + filename = key_name + with open(filename, 'r') as fh: + return fh.read().strip() diff --git a/test.py b/test.py new file mode 100755 index 0000000..aa532c2 --- /dev/null +++ b/test.py @@ -0,0 +1,472 @@ +#!/usr/bin/env python3 + +from asyncio import run +from os import listdir, link, makedirs +from os.path import isdir, isfile, join +from shutil import rmtree, copy2, copytree +from json import load as json_load +from yaml import load, SafeLoader +from pprint import pprint +from typing import Any +from warnings import warn_explicit +from copy import copy + +from tiramisu import Config +from tiramisu.error import ValueWarning +from rougail import RougailConfig, RougailConvert, RougailSystemdTemplate +from rougail.utils import normalize_family +#from rougail.error import TemplateError + +from utils import MULTI_FUNCTIONS, CONFIGS + +DATASET_DIRECTORY = '/home/gnunux/git/risotto_cadoles/risotto-dataset/seed' +FUNCTIONS = 'funcs.py' +CONFIG_DEST_DIR = 'configurations' +SRV_DEST_DIR = 'srv' +INSTALL_DIR = 'installations' +# "netbox.in.gnunux.info": {"applicationservices": ["netbox", "provider-systemd-machined"], +# "informations": {"zones_name": ["gnunux"]}, +# "values": {"rougail.postgresql.pg_client_server_domainname": "postgresql.in.gnunux.info", +# "rougail.redis.redis_client_server_domainname": "redis.in.gnunux.info", +# "rougail.nginx.revprox_client_server_domainname": "revprox.in.gnunux.info", +# "rougail.nginx.revprox_client_external_domainname": "in.gnunux.info" +# } +# }, + + + + +with open('servers.json', 'r') as server_fh: + jsonfile = json_load(server_fh) + SERVERS = jsonfile['servers'] + MODULES = jsonfile['modules'] + + +async def set_linked(linked_server: str, + linked_provider: str, + linked_value: str, + linked_returns: str=None, + dynamic: str=None, + ): + if None in (linked_server, linked_provider, linked_value): + return + if linked_server not in CONFIGS: + warn_explicit(ValueWarning(f'cannot find linked server "{linked_server}"'), + ValueWarning, + __file__, + 0, + ) + return + config = CONFIGS[linked_server][0] + path = await config.information.get('provider:' + linked_provider, None) + if not path: + warn_explicit(ValueWarning(f'cannot find provider "{linked_provider}" in linked server "{linked_server}"'), + ValueWarning, + __file__, + 0, + ) + return + await config.property.read_write() + try: + option = config.forcepermissive.option(path) + if await option.option.ismulti(): + values = await option.value.get() + if linked_value not in values: + values.append(linked_value) + await option.value.set(values) + else: + await option.value.set(linked_value) + except Exception as err: + await config.property.read_only() + raise err from err + await config.property.read_only() + if linked_returns is not None: + linked_variable = await config.information.get('provider:' + linked_returns, None) + if not linked_variable: + warn_explicit(ValueWarning(f'cannot find linked variable "{linked_returns}" in linked server "{linked_server}"'), + ValueWarning, + __file__, + 0, + ) + return + else: + linked_variable = None + if linked_variable is not None: + if dynamic: + linked_variable = linked_variable.replace('{suffix}', normalize_family(dynamic)) + elif '{suffix}' in linked_variable: + idx = CONFIGS[linked_server][3] + linked_variable = linked_variable.replace('{suffix}', str(idx)) + ret = await config.forcepermissive.option(linked_variable).value.get() + else: + ret = normalize_family(linked_value) + return ret + + +async def get_linked_configuration(linked_server: str, + linked_provider: str, + dynamic: str=None, + ): + if linked_server not in CONFIGS: + warn_explicit(ValueWarning(f'cannot find linked server "{linked_server}"'), + ValueWarning, + __file__, + 1, + ) + return + config = CONFIGS[linked_server][0] + path = await config.information.get('provider:' + linked_provider, None) + if not path: + warn_explicit(ValueWarning(f'cannot find variable "{path}" in linked server "{linked_server}"'), + ValueWarning, + __file__, + 1, + ) + return + if dynamic: + path = path.replace('{suffix}', normalize_family(dynamic)) + try: + return await config.forcepermissive.option(path).value.get() + except AttributeError as err: + warn_explicit(ValueWarning(f'cannot find get value of "{path}" in linked server "{linked_server}": {err}'), + ValueWarning, + __file__, + 1, + ) + + +class Empty: + pass +empty = Empty() + + +async def set_linked_configuration(_linked_value: Any, + linked_server: str, + linked_provider: str, + linked_value: Any=empty, + dynamic: str=None, + leader_provider: str=None, + leader_value: Any=None, + ): + if linked_value is not empty: + _linked_value = linked_value + linked_value = _linked_value + if linked_server is None: + return + if linked_value is None or linked_server not in CONFIGS: + warn_explicit(ValueWarning(f'cannot find linked server "{linked_server}"'), + ValueWarning, + __file__, + 2, + ) + return + config = CONFIGS[linked_server][0] + path = await config.information.get('provider:' + linked_provider, None) + if not path: + warn_explicit(ValueWarning(f'cannot find variable "{path}" in linked server "{linked_server}"'), + ValueWarning, + __file__, + 2, + ) + return + if dynamic: + path = path.replace('{suffix}', normalize_family(dynamic)) + await config.property.read_write() + try: + if leader_provider is not None: + leader_path = await config.information.get('provider:' + leader_provider, None) + if not leader_path: + await config.property.read_only() + warn_explicit(ValueWarning(f'cannot find leader variable "{path}" in linked server "{linked_server}"'), + ValueWarning, + __file__, + 2, + ) + return + if dynamic: + leader_path = leader_path.replace('{suffix}', normalize_family(dynamic)) + values = await config.forcepermissive.option(leader_path).value.get() + if leader_value in values: + slave_idx = values.index(leader_value) + slave_option = config.forcepermissive.option(path, slave_idx) + if await slave_option.option.issubmulti(): + slave_values = await slave_option.value.get() + if linked_value not in slave_values: + slave_values.append(linked_value) + await slave_option.value.set(slave_values) + + else: + await slave_option.value.set(linked_value) + else: + option = config.forcepermissive.option(path) + if await option.option.ismulti() and not isinstance(linked_value, list): + values = await option.value.get() + if linked_value not in values: + values.append(linked_value) + await option.value.set(values) + else: + await option.value.set(linked_value) + except AttributeError as err: + #raise ValueError(str(err)) from err + pass + except Exception as err: + await config.property.read_only() + raise err from err + await config.property.read_only() + + +def tiramisu_display_name(kls, + dyn_name: 'Base'=None, + suffix: str=None, + ) -> str: + if dyn_name is not None: + name = kls.impl_getpath() + suffix + else: + name = kls.impl_getpath() + return name + + +def load_applications(): + applications = {} + for distrib in listdir(DATASET_DIRECTORY): + distrib_dir = join(DATASET_DIRECTORY, distrib, 'applicationservice') + if not isdir(distrib_dir): + continue + for release in listdir(distrib_dir): + release_dir = join(distrib_dir, release) + if not isdir(release_dir): + continue + for applicationservice in listdir(release_dir): + applicationservice_dir = join(release_dir, applicationservice) + if not isdir(applicationservice_dir): + continue + if applicationservice in applications: + raise Exception(f'multi applicationservice: {applicationservice} ({applicationservice_dir} <=> {applications[applicationservice]})') + applications[applicationservice] = applicationservice_dir + return applications + + +class ModuleCfg(): + def __init__(self): + self.dictionaries_dir = [] + self.modules = [] + self.functions_file = [FUNCTIONS] + self.templates_dir = [] + self.extra_dictionaries = {} + self.servers = [] + + +def build_module(module_name, datas, module_infos): + install_dir = join(INSTALL_DIR, module_name) + makedirs(install_dir) + applications = load_applications() + cfg = ModuleCfg() + module_infos[module_name] = cfg + def calc_depends(appname, added): + if appname in added: + return + as_dir = applications[appname] + cfg.modules.append(appname) + dictionaries_dir = join(as_dir, 'dictionaries') + if isdir(dictionaries_dir): + cfg.dictionaries_dir.append(dictionaries_dir) + funcs_dir = join(as_dir, 'funcs') + if isdir(funcs_dir): + for f in listdir(funcs_dir): + if f.startswith('__'): + continue + cfg.functions_file.append(join(funcs_dir, f)) + templates_dir = join(as_dir, 'templates') + if isdir(templates_dir): + cfg.templates_dir.append(templates_dir) + extras_dir = join(as_dir, 'extras') + if isdir(extras_dir): + for extra in listdir(extras_dir): + extra_dir = join(extras_dir, extra) + if isdir(extra_dir): + cfg.extra_dictionaries.setdefault(extra, []).append(extra_dir) + for type in ['image', 'install']: + manual_dir = join(as_dir, 'manual', type) + if isdir(manual_dir): + for filename in listdir(manual_dir): + src_file = join(manual_dir, filename) + if type == 'image': + dst_file = join(install_dir, filename) + verify = False + else: + dst_file= join(INSTALL_DIR, filename) + verify = True + if isdir(src_file): + if not isdir(dst_file): + makedirs(dst_file) + for subfilename in listdir(src_file): + if not verify or not isfile(dst_file): + src = join(src_file, subfilename) + dst = join(dst_file, subfilename) + if isfile(src): + copy2(src, dst) + else: + copytree(src, dst) + elif not verify or not isfile(dst_file): + src = join(manual_dir, filename) + dst = dst_file + if isfile(src): + copy2(src, dst) + else: + copytree(src, dst) + added.append(appname) + with open(join(as_dir, 'applicationservice.yml')) as yaml: + app = load(yaml, Loader=SafeLoader) + + for xml in app.get('depends', []): + calc_depends(xml, added) + added = [] + for applicationservice in datas['applicationservices']: + calc_depends(applicationservice, added) + + +async def build(server_name, datas, module_infos): + if server_name in CONFIGS: + raise Exception(f'server "{server_name}" is duplicate') + cfg = RougailConfig.copy() + module_info = module_infos[datas['module']] + module_info.servers.append(server_name) + if datas['module'] == 'host': + cfg['tmpfile_dest_dir'] = datas['values']['rougail.host_install_dir'] + '/host/configurations/' + server_name + cfg['templates_dir'] = module_info.templates_dir + cfg['dictionaries_dir'] = module_info.dictionaries_dir + cfg['functions_file'] = module_info.functions_file + cfg['multi_functions'] = MULTI_FUNCTIONS + cfg['extra_dictionaries'] = module_info.extra_dictionaries + cfg['extra_annotators'].append('risotto_setting.rougail') + optiondescription = {'set_linked': set_linked, + 'get_linked_configuration': get_linked_configuration, + 'set_linked_configuration': set_linked_configuration, + } + cfg['internal_functions'] = list(optiondescription.keys()) + try: + eolobj = RougailConvert(cfg) + except Exception as err: + print(f'Try to load {module_info.modules}') + raise err from err + xml = eolobj.save(None) + #print(xml) + #cfg['patches_dir'] = join(test_dir, 'patches') + cfg['tmp_dir'] = 'tmp' + cfg['destinations_dir'] = join(INSTALL_DIR, datas['module'], CONFIG_DEST_DIR, server_name) + if isdir('tmp'): + rmtree('tmp') + makedirs('tmp') + makedirs(cfg['destinations_dir']) + try: + exec(xml, None, optiondescription) + except Exception as err: + print(xml) + raise Exception(f'unknown error when load tiramisu object {err}') from err + config = await Config(optiondescription['option_0'], display_name=tiramisu_display_name) + await config.property.read_write() + try: + if await config.option('machine.add_srv').value.get(): + srv = join(INSTALL_DIR, SRV_DEST_DIR, server_name) + else: + srv = None + except AttributeError: + srv = None + await config.property.read_write() + CONFIGS[server_name] = (config, cfg, srv, 0) + + +async def value_pprint(dico, config): + pprint_dict = {} + for path, value in dico.items(): + if await config.option(path).option.type() == 'password' and value: + value = 'X' * len(value) + pprint_dict[path] = value + pprint(pprint_dict) + + +async def set_values(server_name, config, datas): + if 'informations' in datas: + for information, value in datas['informations'].items(): + await config.information.set(information, value) + if 'extra_domainnames' in datas['informations']: + for idx, extra_domainname in enumerate(datas['informations']['extra_domainnames']): + if extra_domainname in CONFIGS: + raise Exception(f'server "{server_name}" is duplicate') + value = list(CONFIGS[server_name]) + value[3] = idx + 1 + CONFIGS[extra_domainname] = tuple(value) + await config.information.set('server_name', server_name) + await config.property.read_write() + try: + if 'values' in datas: + for path, value in datas['values'].items(): + if isinstance(value, dict): + for idx, val in value.items(): + await config.option(path, int(idx)).value.set(val) + else: + await config.option(path).value.set(value) + except Exception as err: + await value_pprint(await config.value.dict(), config) + error_msg = f'cannot configure server "{server_name}": {err}' + raise Exception(error_msg) from err + await config.property.read_only() + #await config.value.dict() + + +async def valid_mandatories(server_name, config): + mandatories = await config.value.mandatory() + if mandatories: + print() + print(f'=== Configuration: {server_name} ===') + await config.property.pop('mandatory') + await value_pprint(await config.value.dict(), config) + raise Exception(f'server "{server_name}" has mandatories variables without values "{", ".join(mandatories)}"') + + +async def templates(server_name, config, cfg, srv, int_idx): + values = await config.value.dict() + engine = RougailSystemdTemplate(config, cfg) +# if server_name == 'roundcube.in.gnunux.info': +# print() +# print(f'=== Configuration: {server_name} ===') +# pprint(values) + try: + await engine.instance_files() + except Exception as err: + print() + print(f'=== Configuration: {server_name} ===') + await value_pprint(values, config) + raise err from err + if srv: + makedirs(srv) + + +async def main(): + if isdir(INSTALL_DIR): + rmtree(INSTALL_DIR) + makedirs(INSTALL_DIR) + module_infos = {} + for module_name, datas in MODULES.items(): + build_module(module_name, datas, module_infos) + for server_name, datas in SERVERS.items(): + await build(server_name, datas, module_infos) + for module_name, cfg in module_infos.items(): + with open(join(INSTALL_DIR, module_name, 'install_machines'), 'w') as fh: + for server_name in cfg.servers: + fh.write(f'./install_machine {module_name} {server_name}\n') + for server_name, datas in SERVERS.items(): + await set_values(server_name, CONFIGS[server_name][0], datas) + for server_name in SERVERS: + config = CONFIGS[server_name][0] + await config.property.pop('mandatory') + await config.value.dict() + await config.property.add('mandatory') + for server_name in SERVERS: + await valid_mandatories(server_name, CONFIGS[server_name][0]) +# print(await CONFIGS['revprox.in.gnunux.info'][0].option('nginx.reverse_proxy_for_netbox_in_gnunux_info.reverse_proxy_netbox_in_gnunux_info.revprox_url_netbox_in_gnunux_info', 0).value.get()) + for server_name in SERVERS: + await templates(server_name, *CONFIGS[server_name]) + + +run(main())