gnunux/dataset
1
0
Fork 0
forked from stove/dataset
Code Issues Pull requests Projects Releases Wiki Activity

Merge branch 'develop'

Browse source
This commit is contained in:
Emmanuel Garette 2022-08-25 11:12:09 +02:00
commit b1e7064488
55 changed files with 702 additions and 180 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
seed/base-machine/manual/install/diff.py
View file

@ -1,12 +1,11 @@
#!/usr/bin/env python3
from os.path import join
from filecmp import dircmp
from difflib import unified_diff
from sys import stdout, argv
from os import walk
from os.path import join
from os.path import join, islink
from datetime import datetime, timezone
@ -87,6 +86,9 @@ for filename in old - new:
for filename in new - old:
if islink(join(NEW_DIR, filename)):
print(f'\n- lien {filename} ajouté\n')
else:
print(f'\n- fichier {filename} ajouté :\n')
with open(join(NEW_DIR, filename), 'r') as fh:
if WEBSITE:

36
seed/base-machine/manual/install/install_machines
View file

@ -23,5 +23,39 @@ for image in *; do
done
machinectl enable $MACHINES
machinectl start $MACHINES
STARTED=""
DEGRADED=""
found=true
idx=0
while [ $found = true ]; do
found=false
echo "tentative $idx"
for machine in $MACHINES; do
if ! echo $STARTED | grep -q " $machine "; then
status=$(machinectl -q shell $machine /usr/bin/systemctl is-system-running || true)
if echo "$status" | grep -q degraded; then
STARTED="$STARTED $machine "
DEGRADED="$DEGRADED $machine"
elif echo "$status" | grep -q running; then
STARTED="$STARTED $machine "
else
found=true
echo "status actuel de $machine : $status"
fi
fi
done
sleep 2
idx=$((idx+1))
if [ $idx = 60 ]; then
break
fi
done
retcode=0
for machine in $DEGRADED; do
echo
echo "========= $machine"
machinectl -q shell $machine /usr/bin/systemctl --state=failed --no-legend --no-pager
retcode=1
done
exit 0
exit $retcode

4
seed/base-machine/tests/execute.py
View file

@ -2,12 +2,14 @@ from os import fdopen
from dbus import SystemBus, Array
def run(host, cmd):
def run(host, cmd, user=None):
bus = SystemBus()
remote_object = bus.get_object('org.freedesktop.machine1',
'/org/freedesktop/machine1',
False,
)
if user is not None:
cmd = ['/bin/su', '-', user, '-s', '/bin/bash', '-c', ' '.join(cmd)]
res = remote_object.OpenMachineShell(host,
'',
cmd[0],

6
seed/dovecot/dictionaries/26_dovecot.xml
View file

@ -18,7 +18,7 @@
</service>
<service name='dovecot-init'>
<override/>
<file>/etc/nginx/conf.d/autoconfig.conf</file>
<file>/etc/nginx/default.d/autoconfig.conf</file>
</service>
<service name='nginx'>
<file source='config-v1.1.xml' file_type="variable" variable="mail_domains">well_known_filenames</file>
@ -90,8 +90,8 @@
<variable name="revprox_client_external_domainnames" redefine="True"/>
<variable name="revprox_client_web_address" redefine="True"/>
</family>
<variable name="nginx_default_https" redefine="True">
<value>False</value>
<variable name="nginx_root" redefine='True'>
<value>/var/www/html</value>
</variable>
</family>
</variables>

14
seed/dovecot/templates/autoconfig.conf
View file

@ -1,12 +1,2 @@
server {
listen 443 ssl;
server_name %%domain_name_eth0;
ssl_client_certificate %%revprox_ca_file;
ssl_certificate %%revprox_cert_file;
ssl_certificate_key %%revprox_key_file;
root /var/www/html/;
# To allow POST on static pages
error_page 405 =200 $uri;
}
# To allow POST on static pages
error_page 405 =200 $uri;

2
seed/dovecot/templates/dovecot-init.service
View file

@ -1,5 +1,5 @@
[Unit]
After=network.target
After=risotto.target
[Service]
ExecStart=

2
seed/gitea/templates/gitea.service
View file

@ -1,7 +1,7 @@
#ORIGIN https://raw.githubusercontent.com/go-gitea/gitea/main/contrib/systemd/gitea.service
[Unit]
Description=Gitea (Git with a cup of tea)
After=network.target postgresqlclient.service
After=risotto.target
[Service]
# Modify these two values and uncomment them if you have

2
seed/ldap-client/dictionaries/21_ldap-client.xml
View file

@ -2,7 +2,7 @@
<rougail version="0.10">
<services>
<service name="ldap_client" manage="False">
<service name="ldap-client" target="risotto" engine="creole">
<file source="ldap.conf" file_type="variable">ldap_client_file</file>
<file source="ca_LDAP.crt" file_type="variable">ldap_ca_file</file>
<file source="ldap_client.crt" file_type="variable">ldap_cert_file</file>

8
seed/ldap-client/templates/ldap-client.service Normal file
View file

@ -0,0 +1,8 @@
[Unit]
After=network-online.target
Before=risotto.target
[Service]
Type=oneshot
ExecStart=/usr/bin/timeout 90 bash -c 'while ! 3<> /dev/tcp/%%ldap_server_address/%%ldap_port; do sleep 1; done'

2
seed/lemonldap/dictionaries/70_lemonldap_ng.xml
View file

@ -25,10 +25,10 @@
</services>
<variables>
<family name="nginx">
<variable name="oauth2_client_external_domain" type="domainname" hidden="True" supplier="OAuth2Client:external_domain"/>
<variable name="nginx_default_https" redefine="True">
<value>False</value>
</variable>
<variable name="oauth2_client_external_domain" type="domainname" hidden="True" supplier="OAuth2Client:external_domain"/>
</family>
<family name="lemonldap" description="LemonLDAP" help="Configuration de la solution d'authentification unique LemonLDAP::NG">
<variable name="lemon_proc" type="number" description="Nombre de processus dédié à LemonLdap (équivalent au nombre de processeurs)" mandatory="True">

3
seed/lemonldap/templates/lemonldap-ng-fastcgi-server.service
View file

@ -1,7 +1,6 @@
[Unit]
After=nginx.service
After=risotto.target nginx.service
[Service]
ExecStartPre=/usr/bin/timeout 90 bash -c 'while ! 3<> /dev/tcp/%%ldap_server_address/%%ldap_port; do sleep 1; done'
ExecStartPost=-/usr/bin/timeout 10 bash -c 'while ! /usr/local/lib/sbin/interne_well_known.pl > /var/www/html/.well-known/openid-configuration/int; do sleep 1; done'
ExecStartPost=-/bin/bash -c '/usr/local/lib/sbin/interne_well_known.pl no > /var/www/html/.well-known/openid-configuration/ext'

1
seed/lemonldap/templates/wget.pl
View file

@ -1,7 +1,6 @@
%echo "#!/usr/bin/env perl"
use HTTP::Tiny;
use JSON qw(from_json to_json);
my $response = HTTP::Tiny->new->get('https://%%domain_name_eth0/.well-known/openid-configuration');

2
seed/mailman/applicationservice.yml
View file

@ -5,5 +5,5 @@ depends:
- postgresql-client
- relay-lmtp-client
- reverse-proxy-client
- nginx-common
- nginx-https
- oauth2-client

8
seed/mailman/dictionaries/31_mailman.xml
View file

@ -11,8 +11,9 @@
<service name="postorius" target="multi-user" engine="creole">
<file engine="none">/etc/postorius/gunicorn_config.py</file>
<file engine="none" source="sysuser-postorius.conf">/sysusers.d/0postorius.conf</file>
<file source="config-nginx.conf">/etc/nginx/conf.d/postorius.conf</file>
<file source="config-nginx.conf">/etc/nginx/default.d/postorius.conf</file>
<file source="postorius-settings.py">/etc/mailman3.d/postorius.py</file>
<file>/tests/mailman.yml</file>
</service>
<service name="postgresqlclient" target="multi-user" engine="creole">
<file owner="postorius" mode="400">/etc/pki/tls/private/postgresql_postorius.key</file>
@ -47,6 +48,11 @@
<variable name="oauth2_client_external" redefine="True" remove_fill="True"/>
</family>
</family>
<family name="nginx">
<variable name="nginx_root" redefine="True">
<value>/usr/share/webapps/postorius</value>
</variable>
</family>
<family name="postgresql">
<variable name="pg_client_key_owner" redefine="True">
<value>mailman</value>

2
seed/mailman/extras/mailman/20_mailman.xml
View file

@ -3,7 +3,7 @@
<variables>
<family name="list_" description="Listes du domaine " dynamic="mailman_domains">
<variable name="name_" description="Nom des listes" type="unix_user" multi="True" mandatory="True"/>
<variable name="names_" description="Address names" type="string" multi="True" mandatory="True" hidden="True"/>
<variable name="names_" description="Address names" type="string" mandatory="True" hidden="True"/>
</family>
<variable name="names_" description="Address names" type="string" multi="True" mandatory="True" hidden="True" supplier="LMTP:criteria"/>
</variables>

24
seed/mailman/funcs/mailman.py
View file

@ -1,20 +1,18 @@
from risotto.utils import multi_function as _multi_function
from itertools import chain
@_multi_function
def mailman_emails(lists, domain):
ret = []
for lst in lists:
for suffix in [None, 'bounces(\+.*)?', 'confirm(\+.*)?', 'join', 'leave', 'owner', 'request', 'subscribe', 'unsubscribe']:
if suffix:
lst_name = lst + '-' + suffix
else:
lst_name = lst
ret.append(lst_name + '@' + domain)
return ret
return '.*@' + domain
# ret = []
# for lst in lists:
# for suffix in [None, 'bounces(\+.*)?', 'confirm(\+.*)?', 'join', 'leave', 'owner', 'request', 'subscribe', 'unsubscribe']:
# if suffix:
# lst_name = lst + '-' + suffix
# else:
# lst_name = lst
# ret.append(lst_name + '@' + domain)
# return ret
@_multi_function
def mailman_concat(lists):
# list of lists to a single list
return list(chain(*lists))
return lists

35
seed/mailman/templates/config-nginx.conf
View file

@ -1,20 +1,10 @@
server {
listen 443 ssl;
server_name %%domain_name_eth0;
ssl_client_certificate %%revprox_ca_file;
ssl_certificate %%revprox_cert_file;
ssl_certificate_key %%revprox_key_file;
charset utf-8;
client_max_body_size 75M;
root /usr/share/webapps/postorius;
location /mailman/postorius_static {
charset utf-8;
client_max_body_size 75M;
location /mailman/postorius_static {
alias /usr/lib/python3.10/site-packages/postorius/static;
}
#FIXME user-profile seems to be in hyperkitty redirect in existing page
location /mailman/user-profile {
}
#FIXME user-profile seems to be in hyperkitty redirect in existing page
location /mailman/user-profile {
proxy_pass http://127.0.0.1:8002/postorius/users;
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
@ -23,9 +13,9 @@ server {
proxy_set_header X-Forwarded-Server $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
}
}
%for %%location in ['accounts', 'admin', 'postorius']
location /mailman/%%location {
location /mailman/%%location {
proxy_pass http://127.0.0.1:8002/%%location;
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
@ -34,9 +24,8 @@ server {
proxy_set_header X-Forwarded-Server $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
}
%end for
location /mailman {
rewrite ^(/mailman/.*)$ /mailman/postorius/ permanent;
}
}
%end for
location /mailman {
rewrite ^(/mailman/.*)$ /mailman/postorius/ permanent;
}

12
seed/mailman/templates/mailman.yml Normal file
View file

@ -0,0 +1,12 @@
%set %%username="rougail_test@silique.fr"
ip: %%ip_eth0
revprox_ip: %%revprox_client_server_ip
%set %%domain = %%revprox_client_external_domainnames[0]
domain_name: %%domain
base_url: https://%%domain%%domain.revprox_client_location
auth_url: %%oauth2_client_external[0]
auth_server: %%oauth2_server_domainname
username: %%username
password: %%get_password(server_name='test', username=%%username, description='test', type="cleartext", hide=%%hide_secret, temporary=True)
mailman_domain: %%mailman_domains[0]
internal_address: %%domain_name_eth0

2
seed/mailman/templates/mailman3.service
View file

@ -1,6 +1,6 @@
[Unit]
Description=Postorius WSGI Service
After=postgresqlclient.service
After=risotto.target
[Service]
%for %%domain in %%mailman_domains

2
seed/mailman/templates/postorius.service
View file

@ -1,6 +1,6 @@
[Unit]
Description=Postorius WSGI Service
After=network.target postgresqlclient.service
After=risotto.target
[Service]
Type=notify

396
seed/mailman/tests/test_mailman.py Normal file
View file

@ -0,0 +1,396 @@
from yaml import load, SafeLoader
from os import environ
from os.path import join, isdir
from revprox import Authentication
from execute import run
from re import search
from time import sleep
from imaplib2 import IMAP4_SSL
from smtplib import SMTP
from email import message_from_bytes
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
DATA = None
DATA_IMAP = None
IMAP = None
IMAP_FAMILY = None
def get_data():
global DATA
if not DATA:
conf_file = f'{environ["MACHINE_TEST_DIR"]}/mailman.yml'
with open(conf_file) as yaml:
DATA = load(yaml, Loader=SafeLoader)
return DATA
def get_imap_data():
global DATA_IMAP
if not DATA_IMAP:
machine_test_dir = environ['MACHINE_TEST_DIR']
print("FIXME")
machine_test_dir = machine_test_dir.replace('mailman', 'dovecot')
if not isdir(machine_test_dir):
print('!!! No local IMAP server found !!!')
return
conf_file = f'{machine_test_dir}/imap.yml'
with open(conf_file) as yaml:
DATA_IMAP = load(yaml, Loader=SafeLoader)
return DATA_IMAP
def get_authentication(data, family=False):
if family:
username = data['username_family']
password = data['password_family']
else:
username = data['username']
password = data['password']
return Authentication(data['auth_url'],
data['auth_server'],
data['revprox_ip'],
username,
password,
f"""<title>
Listes - {data["domain_name"]}
</title>""",
)
def check_mail(subject, family=False, reply=False):
global IMAP, IMAP_FAMILY
data = get_imap_data()
if data is None:
return
if family:
if IMAP_FAMILY is None:
IMAP_FAMILY = IMAP4_SSL(data['address'])
IMAP_FAMILY.LOGIN(data['username_family'], data['password_family'] + '2')
imap = IMAP_FAMILY
else:
if IMAP is None:
IMAP = IMAP4_SSL(data['address'])
IMAP.LOGIN(data['username'], data['password'])
imap = IMAP
imap.SELECT(readonly=False)
typ, req = imap.SEARCH(None, 'ALL')
assert typ == 'OK'
if not req[0].decode():
raise Exception('pas de mail')
num = req[0].decode().split()[-1]
field = imap.FETCH(num, '(RFC822)')
assert field[0] == 'OK'
msg = message_from_bytes(field[1][-2][-1])
#if msg.is_multipart():
# for part in msg.walk():
# # extract content type of email
# try:
# print(part.get_payload(decode=True).decode())
# except:
# pass
#else:
# print(msg.get_payload(decode=True).decode())
if subject is not None:
assert subject == msg['Subject']
if reply:
reply_message(msg)
ret = imap.store(num, '+FLAGS', '\\Deleted')
assert ret[0] == 'OK', f'error when deleting mail: {ret}'
imap.expunge()
# imap.CLOSE()
# imap.LOGOUT()
def reply_message(msg):
data = get_imap_data()
body = MIMEText('resend')
message = MIMEMultipart()
message["to"] = msg["From"]
message['from'] = msg["To"]
message['subject'] = f'Re: {msg["Subject"]}'
message.add_header('reply-to', msg["From"])
message.attach(body)
#
smtp = SMTP(data['address'], '587')
smtp.starttls()
smtp.login(data['username_family'], data['password_family'] + '2')
smtp.sendmail(msg["To"],
msg["From"],
message.as_string(),
)
smtp.quit()
def send_mail(subject, data, data_mm, family=False, add_mail=''):
list_name = 'test'
smtp = SMTP(data['address'], '587')
smtp.starttls()
if not family:
sender = data['username']
smtp.login(sender, data['password'])
else:
sender = data['username_family']
smtp.login(sender, data['password_family'] + '2')
list_addr = f'{list_name}{add_mail}@{data_mm["mailman_domain"]}'
msg = f"""From: {sender}\r\n\
To: {list_addr}\r\n\
Subject: {subject}\r\n\
\r\n\
MESSAGE"""
smtp.sendmail(sender,
list_addr,
msg,
)
smtp.quit()
def test_mailman():
data = get_data()
get_authentication(data)
def test_mailman_login():
data = get_data()
authentication = get_authentication(data)
content = authentication.get(data['auth_url'])
login = data['username'].split('@')[0]
assert f'Successfully signed in as {login}.' in content
def test_mailman_list():
data = get_data()
authentication = get_authentication(data)
list_name = 'test'
search_list = f'href="/mailman/postorius/lists/{list_name}.{data["mailman_domain"]}/"'
if 'FIRST_RUN' in environ:
content = authentication.get(data['auth_url'])
assert search_list not in content
result = run(data['internal_address'],
['/usr/bin/mailman3', 'create', '--language', 'fr', '-o', data['username'], '-N', '-d', f'{list_name}@{data["mailman_domain"]}'],
'mailman',
)
assert list(result) == [f'Liste de diffusion créée : {list_name}@{data["mailman_domain"]}']
content = authentication.get(data['auth_url'])
assert search_list in content
def test_mailman_inscription():
data = get_data()
authentication = get_authentication(data)
list_name = 'test'
search_inscription = f'<td>Adresse principale ({data["username"]})</td>'
url = join(data['base_url'], f'postorius/lists/{list_name}.{data["mailman_domain"]}/')
if 'FIRST_RUN' in environ:
content = authentication.get(url)
assert search_inscription not in content
pattern_csrf = r'name="csrfmiddlewaretoken" value="([a-zA-Z0-9\-\_=]+)"'
pattern_sub = r'<option value="([a-zA-Z0-9\-\_=]+)">Adresse principale'
csrf = search(pattern_csrf, content)[1]
subscriber = search(pattern_sub, content)[1]
headers = {'Referer': url}
authentication.post(url + 'subscribe', {'csrfmiddlewaretoken': csrf, 'subscriber': subscriber, 'delivery_mode': 'regular', 'display_name': ''}, headers=headers)
subject = f'=?utf-8?q?Bienvenue_sur_la_liste_de_diffusion_=22{list_name.capitalize()}=22?='
idx = 0
while True:
try:
check_mail(subject)
break
except:
idx += 1
if idx == 10:
raise Exception('mail not arrived')
sleep(1)
content = authentication.get(url)
assert search_inscription in content
def test_send_mail():
data = get_imap_data()
if data is None:
return
data_mm = get_data()
subject = 'TEST MAILMAN'
send_mail(subject, data, data_mm)
idx = 0
while True:
try:
check_mail("=?utf-8?b?W1Rlc3Rd?= TEST MAILMAN")
break
except:
idx += 1
if idx == 10:
raise Exception('mail not arrived')
sleep(1)
def test_send_wrong_mail():
data = get_imap_data()
if data is None:
return
data_mm = get_data()
url = f'{data_mm["base_url"]}postorius/lists/test.{data_mm["mailman_domain"]}/held_messages'
# no moderated mail
authentication = get_authentication(data_mm)
content = authentication.get(url)
subject = 'TEST MAILMAN'
assert subject not in content
# send mail
smtp = SMTP(data['address'], '587')
smtp.starttls()
smtp.login(data['username_family'], data['password_family'] + '2')
list_name = 'test'
list_addr = f'{list_name}@{data_mm["mailman_domain"]}'
msg = f"""From: {data["username_family"]}\r\n\
To: {list_addr}\r\n\
Subject: {subject}\r\n\
\r\n\
MESSAGE"""
smtp.sendmail(data['username_family'], list_addr, msg)
smtp.quit()
idx = 0
mailman_domain = data_mm['mailman_domain'].replace('.', '=2E')
while True:
try:
check_mail(f'=?utf-8?q?Votre_message_=C3=A0_test=40{mailman_domain}_attend_la_validation_d=27un_mod=C3=A9rateur?=', family=True)
break
except:
idx += 1
if idx == 10:
raise Exception('mail not arrived...')
sleep(1)
#
mail = data['username_family'].replace('@', '=40').replace('.', '=2E').replace('_', '=5F')
while True:
try:
check_mail(f'=?utf-8?q?Le_message_test=40{mailman_domain}_de_{mail}_n=C3=A9cessite_une_validation?=')
break
except:
idx += 1
if idx == 10:
raise Exception('mail 2 not arrived...')
sleep(1)
# reject mail
content = authentication.get(url)
assert subject in content
pattern_csrf = r'name="csrfmiddlewaretoken" value="([a-zA-Z0-9\-\_=]+)"'
csrf = search(pattern_csrf, content)[1]
pattern_msg_id = r'class="message-checkbox" name="choices" value="([a-zA-Z0-9\-\_=]+)"'
msg_id = search(pattern_msg_id, content)[1]
#
headers = {'Referer': url}
authentication.post(url, {'csrfmiddlewaretoken': csrf, 'reject': 'Rejeter', 'choices': msg_id}, headers=headers)
content = authentication.get(url)
assert subject not in content
while True:
try:
check_mail('=?utf-8?q?Requ=C3=AAte_pour_la_liste_de_diffusion_=22Test=22_rejet=C3=A9e?=', family=True)
break
except:
idx += 1
if idx == 10:
raise Exception('mail reject not arrived...')
sleep(1)
def test_mailman_create_delete():
data = get_data()
authentication = get_authentication(data)
list_name = 'test_tmp'
search_list = f'href="/mailman/postorius/lists/{list_name}.{data["mailman_domain"]}/"'
#
content = authentication.get(data['auth_url'])
assert search_list not in content
#
result = run(data['internal_address'],
['/usr/bin/mailman3', 'create', '--language', 'fr', '-o', data['username'], '-N', '-d', f'{list_name}@{data["mailman_domain"]}'],
'mailman',
)
assert list(result) == [f'Liste de diffusion créée : {list_name}@{data["mailman_domain"]}']
content = authentication.get(data['auth_url'])
assert search_list in content
#
result = run(data['internal_address'],
['/usr/bin/mailman3', 'remove', f'{list_name}@{data["mailman_domain"]}'],
'mailman',
)
assert list(result) == [f'Liste supprimée : {list_name}@{data["mailman_domain"]}']
content = authentication.get(data['auth_url'])
assert search_list not in content
def test_mailman_inscription_desinscription_mail():
list_name = 'test'
data = get_imap_data()
if data is None:
return
data_mm = get_data()
send_mail('subscribe', data, data_mm, family=True, add_mail='-join')
idx = 0
# inscription + validation
mailman_domain = data_mm['mailman_domain'].replace('.', '=2E')
while True:
try:
check_mail(f'=?utf-8?q?Votre_validation_est_n=C3=A9cessaire_pour_vous_abonner_de_la_liste_de_diffusion_test=40{mailman_domain}?=', family=True, reply=True)
break
except:
idx += 1
if idx == 10:
raise Exception('mail not arrived...')
sleep(1)
idx = 0
while True:
try:
check_mail(f'=?utf-8?q?Bienvenue_sur_la_liste_de_diffusion_=22{list_name.capitalize()}=22?=', family=True)
break
except:
idx += 1
if idx == 10:
raise Exception('confirmation not arrived...')
sleep(1)
# send mail and check
subject = 'TEST MAILMAN'
send_mail(subject, data, data_mm)
idx = 0
while True:
try:
check_mail("=?utf-8?b?W1Rlc3Rd?= TEST MAILMAN")
break
except:
idx += 1
if idx == 10:
raise Exception('mail not arrived')
sleep(1)
while True:
try:
check_mail("=?utf-8?b?W1Rlc3Rd?= TEST MAILMAN", family=True)
break
except:
idx += 1
if idx == 10:
raise Exception('mail not arrived')
sleep(1)
# unsubscribe
send_mail('unsubscribe', data, data_mm, family=True, add_mail="-leave")
idx = 0
while True:
try:
check_mail(f'=?utf-8?q?Votre_validation_est_n=C3=A9cessaire_pour_vous_d=C3=A9sabonner_de_la_liste_de_diffusion_test=40{mailman_domain}?=', family=True, reply=True)
break
except:
idx += 1
if idx == 10:
raise Exception('unsubscribe not arrived...')
sleep(1)
idx = 0
while True:
try:
check_mail(f'=?utf-8?q?Vous_avez_=C3=A9t=C3=A9_d=C3=A9sinscrits_de_la_liste_de_diffusion_Test?=', family=True)
break
except:
idx += 1
if idx == 10:
raise Exception('unsubscribe confirmation not arrived...')
sleep(1)

2
seed/mariadb-client/dictionaries/20_mariadb.xml
View file

@ -1,7 +1,7 @@
<?xml version='1.0' encoding='UTF-8'?>
<rougail version="0.10">
<services>
<service name="mariadbclient" target="multi-user" engine="creole"/>
<service name="mariadbclient" target="risotto" engine="creole"/>
</services>
<variables>
<family name="mariadb" description="MariaDB">

3
seed/mariadb-client/templates/mariadbclient.service
View file

@ -1,6 +1,7 @@
[Unit]
Description=Waiting for mariadb server
Before=network.target
After=network-online.target
Before=risotto.target
[Service]
Type=oneshot

2
seed/nextcloud/templates/nextcloud.service
View file

@ -1,6 +1,6 @@
[Unit]
Description=Nextcloud management
After=postgresqlclient.service
After=risotto.target
Before=apache.service php-fpm.service
[Service]

1
seed/nginx-common/dictionaries/20_nginx.xml → seed/nginx-common/dictionaries/21_nginx.xml
View file

@ -13,6 +13,7 @@
<file file_type="variable" source="ca_InternalReverseProxy.crt">revprox_ca_file</file>
<file filelist="nginx_default_https" mode="600">/etc/pki/tls/certs/nginx.crt</file>
<file filelist="nginx_default_https" mode="600">/etc/pki/tls/private/nginx.key</file>
<file>/tests/nginx-common.yml</file>
</service>
</services>
<variables>

13
seed/nginx-common/templates/nginx-common.yml Normal file
View file

@ -0,0 +1,13 @@
address: %%ip_eth0
nginx_default_http: %slurp
%if %%getVar('nginx_default_http', False) and not %%getVar('revprox_client_external_domainnames', None)
true
%else
false
%end if
nginx_default_https: %slurp
%if %%getVar('nginx_default_https', False) and not %%getVar('revprox_client_external_domainnames', None)
true
%else
false
%end if

18
seed/nginx-common/templates/nginx.conf
View file

@ -76,14 +76,24 @@ http {
%if %%nginx_default_https
server {
listen 443 ssl http2;
server_name %%domain_name_eth0;
%if %%getVar('revprox_client_external_domainnames', None)
%for %%domain in %%revprox_client_external_domainnames
server_name %%domain;
%end for
%else
server_name _;
%end if
root %%nginx_root;
# ssl_certificate "/etc/pki/nginx/server.crt";
# ssl_certificate_key "/etc/pki/nginx/private/server.key";
ssl_certificate /etc/pki/tls/certs/nginx.crt;
ssl_certificate_key /etc/pki/tls/private/nginx.key;
ssl_client_certificate /etc/pki/ca-trust/source/anchors/ca_InternalReverseProxy.crt;
%if %%getVar('revprox_client_external_domainnames', None)
ssl_client_certificate %%revprox_ca_file;
%else
ssl_client_certificate /etc/pki/ca-trust/source/anchors/ca_HTTP.crt;
%end if
ssl_session_cache shared:SSL:1m;
ssl_session_timeout 10m;
@ -105,5 +115,7 @@ http {
%else
include /etc/nginx/sites-enabled/*;
%end if
%if not %%getVar('revprox_client_external_domainnames', None)
include /etc/nginx/sites-enabled/*;
%end if
}

50
seed/nginx-common/tests/test_nginx_commmon.py Normal file
View file

@ -0,0 +1,50 @@
from yaml import load, SafeLoader
from os import environ
from pytest import raises
import warnings
import socket
from requests import get
from requests.exceptions import SSLError
def req(url, ip, verify=True):
# Monkey patch to force IPv4 resolution
old_getaddrinfo = socket.getaddrinfo
def new_getaddrinfo(*args, **kwargs):
ret = old_getaddrinfo(*args, **kwargs)
dns = list(ret[0])
dns[-1] = (ip, dns[-1][1])
return [dns]
socket.getaddrinfo = new_getaddrinfo
if not verify:
with warnings.catch_warnings():
warnings.simplefilter("ignore")
ret = get(url, verify=verify)
else:
ret = get(url, verify=verify)
ret_code = ret.status_code
content = ret.content
socket.getaddrinfo = old_getaddrinfo
return ret_code, content.decode()
def test_revprox():
conf_file = f'{environ["MACHINE_TEST_DIR"]}/nginx-common.yml'
with open(conf_file) as yaml:
data = load(yaml, Loader=SafeLoader)
# test unknown domain
url = 'google.fr'
protocols = []
if data['nginx_default_http']:
protocols.append('http')
if data['nginx_default_https']:
protocols.append('https')
# test certificate
with raises(SSLError):
# not certificat problem for https://{url}
req(f'https://{url}', data['address'])
for protocol in protocols:
ret_code, content = req(f'{protocol}://{url}', data['address'], verify=False)
assert ret_code == 200, f'{protocol}://{url} do not returns code 200 but {ret_code}'
assert "<title>Test Page for the HTTP Server on Fedora</title>" in content, f'{protocol}://{url} do not returns default fedora page'

21
seed/nginx-https/dictionaries/25_nginx.xml
View file

@ -1,22 +1,16 @@
<?xml version='1.0' encoding='UTF-8'?>
<rougail version="0.10">
<services>
<service name='nginx'>
<file filelist="nginx_default_risotto">/etc/nginx/default.d/risotto.conf</file>
</service>
</services>
<variables>
<family name="nginx">
<variable name="nginx_default_https" redefine="True">
<variable name="nginx_default_http" redefine="True" hidden="True">
<value>False</value>
</variable>
<variable name="nginx_default_https" redefine="True" hidden="True">
<value>True</value>
</variable>
<variable name="php_fpm_user" redefine="True" exists="True">
<value>nginx</value>
</variable>
<variable name="nginx_root_directory" type="filename"/>
<variable name="nginx_locations" type="filename" multi="True" mandatory="True">
<value>/</value>
</variable>
</family>
<family name="redis" description="Redis">
<variable name="redis_client_key_owner" redefine="True" exists="True">
@ -29,11 +23,4 @@
</variable>
</family>
</variables>
<constraints>
<condition name="disabled_if_in" source="nginx_root_directory">
<param type="nil"/>
<target type="filelist">nginx_default_risotto</target>
<target type="variable">nginx_locations</target>
</condition>
</constraints>
</rougail>

8
seed/nginx-reverse-proxy/dictionaries/20_nginx.xml Normal file
View file

@ -0,0 +1,8 @@
<?xml version='1.0' encoding='UTF-8'?>
<rougail version="0.10">
<services>
<service name='nginx'>
<file>/etc/pki/ca-trust/source/anchors/ca_HTTP.crt</file>
</service>
</services>
</rougail>

5
seed/nginx-reverse-proxy/dictionaries/25_nginx.xml
View file

@ -4,7 +4,7 @@
<service name='nginx'>
<override engine="creole"/>
<file source="nginx-options-rp.conf">/etc/nginx/conf.d/options-rp.conf</file>
<file source="revprox-nginx.conf">/etc/nginx/conf.d/risotto.conf</file>
<file source="revprox-nginx.conf">/etc/nginx/sites-enabled/risotto.conf</file>
<file source="certificate.crt" file_type="variable" mode="600" variable="nginx.revprox_domainnames">nginx.nginx_certificate_filename</file>
<file source="private.key" file_type="variable" mode="600" variable="nginx.revprox_domainnames">nginx.nginx_private_key_filename</file>
<file>/tests/reverse-proxy.yml</file>
@ -22,6 +22,9 @@
<variable name="nginx_default_http" redefine="True">
<value>True</value>
</variable>
<variable name="nginx_default_https" redefine="True">
<value>True</value>
</variable>
</family>
</variables>
</rougail>

3
seed/nginx-reverse-proxy/templates/ca_HTTP.crt Normal file
View file

@ -0,0 +1,3 @@
%for %%idx in %%range(%%len(%%zones_list))
%%get_chain(authority_cn=%%getVar('domain_name_eth' + %%str(%%idx)), authority_name="HTTP", hide=%%hide_secret)
%end for

2
seed/nginx-reverse-proxy/templates/nginx.crt Normal file
View file

@ -0,0 +1,2 @@
%%get_certificate(%%nginx_default, authority_cn=%%domain_name_eth0, authority_name='HTTP', type="server", hide=%%hide_secret)
%%get_chain(%%nginx_default, 'HTTP', hide=%%hide_secret)

1
seed/nginx-reverse-proxy/templates/nginx.key Normal file
View file

@ -0,0 +1 @@
%%get_private_key(%%nginx_default, authority_cn=%%domain_name_eth0, authority_name='HTTP', type='server', hide=%%hide_secret)

2
seed/nginx-reverse-proxy/templates/revprox-nginx.conf
View file

@ -41,7 +41,7 @@ server {
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header Destination $dest;
%end if
proxy_ssl_trusted_certificate /etc/pki/ca-trust/source/anchors/ca_InternalReverseProxy.crt;
proxy_ssl_trusted_certificate %%revprox_ca_file;
proxy_ssl_verify on;
proxy_ssl_verify_depth 2;
proxy_ssl_session_reuse on;

13
seed/nginx-reverse-proxy/tests/test_revprox.py
View file

@ -32,19 +32,8 @@ def test_revprox():
conf_file = f'{environ["MACHINE_TEST_DIR"]}/reverse-proxy.yml'
with open(conf_file) as yaml:
data = load(yaml, Loader=SafeLoader)
# test unknown domain
url = 'google.fr'
ret_code, content = req(f'https://{url}', data['address'], verify=False)
assert ret_code == 200, f'https://{url} do not returns code 200 but {ret_code}'
assert "<title>Test Page for the HTTP Server on Fedora</title>" in content, f'https://{url} returns default fedora page'
# test certificate
try:
req(f'https://{url}', data['address'])
raise Exception(f'not certificat problem for https://{url}')
except SSLError:
pass
# test known domains
for url in data['urls']:
ret_code, content = req(f'https://{url}', data['address'])
assert ret_code == 200, f'https://{url} do not returns code 200 but {ret_code}'
assert "<title>Test Page for the HTTP Server on Fedora</title>" not in content, f'https://{url} returns default fedora page'
assert "<title>Test Page for the HTTP Server on Fedora</title>" not in content, f'https://{url} do returns default fedora page'

2
seed/nsd/templates/nsd.service
View file

@ -1,2 +1,2 @@
[Unit]
After=network.target
After=risotto.target

3
seed/oauth2-client/dictionaries/30_oauth2_client.xml
View file

@ -1,5 +1,8 @@
<?xml version='1.0' encoding='UTF-8'?>
<rougail version="0.10">
<services>
<service name="oauth2-client" target="risotto" engine="creole"/>
</services>
<variables>
<family name="oauth2_client" description="OAuth2 client">
<variable name="oauth2_client_server_domainname" type="domainname" description="OAuth2 server domain name" mandatory='True' supplier="OAuth2"/>

1
seed/oauth2-client/manual/image/preinstall/oauth2-client.sh Normal file
View file

@ -0,0 +1 @@
PKG="$PKG curl"

7
seed/oauth2-client/templates/oauth2-client.service Normal file
View file

@ -0,0 +1,7 @@
[Unit]
After=network-online.target
Before=risotto.target
[Service]
Type=oneshot
ExecStart=/usr/bin/timeout 90 bash -c 'while ! [ "$(/usr/bin/curl --write-out '%{http_code}' --silent --output /dev/null https://%%oauth2_client_server_domainname/.well-known/openid-configuration)" = 200 ]; do sleep 1; done;'

2
seed/peertube/applicationservice.yml
View file

@ -7,5 +7,5 @@ depends:
- relay-mail-client
- reverse-proxy-client
- redis-client
- nginx-common
- nginx-https
- oauth2-client

6
seed/peertube/dictionaries/30_peertube.xml
View file

@ -6,7 +6,8 @@
<file engine="none" source="sysuser-peertube.conf">/sysusers.d/0peertube.conf</file>
<file engine="none" source="tmpfile-peertube.conf">/tmpfiles.d/0peertube.conf</file>
<file>/etc/peertube/production.yaml</file>
<file source="nginx.peertube.conf">/etc/nginx/conf.d/peertube.conf</file>
<file source="nginx.peertube.conf">/etc/nginx/default.d/peertube.conf</file>
<file source="nginx.peertube.conf.d.conf">/etc/nginx/conf.d/peertube.conf</file>
</service>
</services>
<variables>
@ -45,6 +46,9 @@
</family>
</family>
<family name="nginx">
<variable name="nginx_root" redefine='True'>
<value>/usr/share/peertube</value>
</variable>
<family name="revprox_client">
<variable name="revprox_client_location" redefine="True">
<value>/</value>

37
seed/peertube/templates/nginx.peertube.conf
View file

@ -16,15 +16,14 @@
# GNUNUX location / { return 301 https://$host$request_uri; }
# GNUNUX }
upstream %%domain_name_eth0 {
# GNUNUX upstream %%domain_name_eth0 {
# GNUNUX server ${PEERTUBE_HOST};
server localhost:9000;
}
# GNUNUX }
server {
listen 443 ssl http2;
listen [::]:443 ssl http2;
server_name %%domain_name_eth0;
# GNUNUX server {
# GNUNUX listen 443 ssl http2;
# GNUNUX listen [::]:443 ssl http2;
# GNUNUX server_name %%domain_name_eth0;
# GNUNUX access_log /var/log/nginx/peertube.access.log; # reduce I/0 with buffer=10m flush=5m
# GNUNUX error_log /var/log/nginx/peertube.error.log;
@ -35,11 +34,6 @@ server {
##
# GNUNUX ssl_certificate /etc/letsencrypt/live/${WEBSERVER_HOST}/fullchain.pem;
# GNUNUX ssl_certificate_key /etc/letsencrypt/live/${WEBSERVER_HOST}/privkey.pem;
#>GNUNUX
ssl_client_certificate %%revprox_ca_file;
ssl_certificate %%revprox_cert_file;
ssl_certificate_key %%revprox_key_file;
#<GNUNUX
# GNUNUX location ^~ '/.well-known/acme-challenge' {
# GNUNUX default_type "text/plain";
@ -51,14 +45,14 @@ server {
# based on Mozilla Guideline v5.6
##
ssl_protocols TLSv1.2 TLSv1.3;
ssl_prefer_server_ciphers on;
ssl_ciphers ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256; # add ECDHE-RSA-AES256-SHA if you want compatibility with Android 4
ssl_session_timeout 1d; # defaults to 5m
ssl_session_cache shared:SSL:10m; # estimated to 40k sessions
ssl_session_tickets off;
ssl_stapling on;
ssl_stapling_verify on;
# GNUNUX ssl_protocols TLSv1.2 TLSv1.3;
# GNUNUX ssl_prefer_server_ciphers on;
# GNUNUX ssl_ciphers ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256; # add ECDHE-RSA-AES256-SHA if you want compatibility with Android 4
# GNUNUX ssl_session_timeout 1d; # defaults to 5m
# GNUNUX ssl_session_cache shared:SSL:10m; # estimated to 40k sessions
# GNUNUX ssl_session_tickets off;
# GNUNUX ssl_stapling on;
# GNUNUX ssl_stapling_verify on;
# HSTS (https://hstspreload.org), requires to be copied in 'location' sections that have add_header directives
#add_header Strict-Transport-Security "max-age=63072000; includeSubDomains";
@ -145,7 +139,6 @@ server {
##
# GNUNUX root /var/www/peertube/storage;
root /usr/share/peertube;
# Enable compression for JS/CSS/HTML, for improved client load times.
# It might be nice to compress JSON/XML as returned by the API, but
@ -268,4 +261,4 @@ server {
try_files $uri @api;
}
}
# GNUNUX }

4
seed/peertube/templates/nginx.peertube.conf.d.conf Normal file
View file

@ -0,0 +1,4 @@
upstream %%domain_name_eth0 {
# GNUNUX server ${PEERTUBE_HOST};
server localhost:9000;
}

9
seed/piwigo/dictionaries/31_piwigo.xml
View file

@ -7,16 +7,13 @@
<file>/etc/piwigo/database.inc.php</file>
<file mode="755">/sbin/piwigo.sh</file>
<file engine="none">/etc/php-fpm.d/piwigo.conf</file>
<file source="piwigo.nginx.conf">/etc/nginx/default.d/piwigo.conf</file>
</service>
</services>
<variables>
<variable name="piwigo_admin_email" type="mail" description="Adresse courriel de l'administrateur Piwigo" mandatory="True"/>
<variable name="piwigo_admin_password" type="password" auto_save="False" hidden="True"/>
<family name="nginx">
<variable name="nginx_root_directory" mandatory="True" redefine="True">
<value>/usr/local/share/piwigo</value>
</variable>
</family>
<variable name="piwigo_locations" type="filename" multi="True" mandatory="True"/>
<variable name="piwigo_title" type="string" description="Titre de l'album" mandatory="True">
<value>Album photographique</value>
</variable>
@ -53,7 +50,7 @@
</fill>
<fill name="get_locations">
<param name="usernames" type="variable">piwigo_users</param>
<target>nginx_locations</target>
<target>piwigo_locations</target>
</fill>
</constraints>
</rougail>

16
seed/nginx-https/templates/risotto.conf → seed/piwigo/templates/piwigo.nginx.conf
View file

@ -1,3 +1,5 @@
# To allow POST on static pages
error_page 405 =200 $uri;
add_header X-Frame-Options "SAMEORIGIN";
add_header X-Content-Type-Options nosniff;
add_header X-XSS-Protection "1; mode=block";
@ -7,17 +9,14 @@ add_header X-Permitted-Cross-Domain-Policies none;
add_header Strict-Transport-Security 'max-age=31536000; includeSubDomains;';
add_header Referrer-Policy no-referrer always;
%for %%location in %%nginx_locations
%for %%location in %%piwigo_locations
location %%location {
%if %%location == '/'
%if %%location == '/'
root %slurp
%else
%else
alias %slurp
%end if
%%nginx_root_directory;
%if not %%getVar('php_fpm_installed', False)
index index.html;
%else
%end if
/usr/local/share/piwigo;
index index.php;
location ~ ^(?<script_name>.+?\.php)(?<path_info>/.*)?$ {
fastcgi_pass php-fpm;
@ -25,6 +24,5 @@ location %%location {
fastcgi_param SCRIPT_FILENAME $request_filename;
include fastcgi_params;
}
%end if
}
%end for

2
seed/piwigo/templates/piwigo.service
View file

@ -1,6 +1,6 @@
[Unit]
Description=Piwigo management
After=mariadbclient.service
After=risotto.target
Before=nginx.service php-fpm.service
[Service]

2
seed/pleroma/dictionaries/30_pleroma.xml
View file

@ -7,7 +7,7 @@
<file engine="none" source="tmpfile-peertube.conf">/tmpfiles.d/0peertube.conf</file>
<file>/etc/peertube/production.yaml</file>
<file engine="none">/etc/pam.d/login</file>
<file source="nginx.peertube.conf">/etc/nginx/conf.d/peertube.conf</file>
<file source="nginx.peertube.conf">/etc/nginx/sites-enabled/peertube.conf</file>
</service>
</services>
<variables>

2
seed/postgresql-client/dictionaries/23_postgresql.xml
View file

@ -1,7 +1,7 @@
<?xml version='1.0' encoding='UTF-8'?>
<rougail version="0.10">
<services>
<service name="postgresqlclient" target="multi-user" engine="creole">
<service name="postgresqlclient" target="risotto" engine="creole">
<file mode="400">/secrets/postgresql.pass</file>
<file>/etc/pki/ca-trust/source/anchors/ca_PostgreSQL.crt</file>
<file>/etc/pki/tls/certs/postgresql.crt</file>

3
seed/postgresql-client/templates/postgresqlclient.service
View file

@ -1,6 +1,7 @@
[Unit]
Description=Waiting for postgresql server
Before=network.target
After=network-online.target
Before=risotto.target
[Service]
Type=oneshot

1
seed/postgresql/DEBUG.md Normal file
View file

@ -0,0 +1 @@
pg_dumpall --clean > /srv/database.sql

8
seed/reverse-proxy-client/tests/revprox.py
View file

@ -33,7 +33,7 @@ class Authentication:
ret = req.get(url)
code = ret.status_code
content = ret.content
assert code == 200
assert code == 200, f"cannot access to lemonldap; {content}"
assert b'<title trspan="authPortal">Authentication portal</title>' in content, f'cannot find LemonLdap title: {content}'
def auth_lemonldap(self,
@ -62,7 +62,8 @@ class Authentication:
authorize_url = f'{portal_url}authorize'
ret = req.get(authorize_url)
assert ret.status_code == 200
assert title in ret.content.decode()
content = ret.content.decode()
assert title in content, f'cannot find {title} in {content}'
def get(self,
url,
@ -78,7 +79,8 @@ class Authentication:
def post(self,
url,
data,
headers=None,
):
with MookDns(self.ip):
ret = post(url, cookies=self.cookies, data=data)
ret = post(url, cookies=self.cookies, data=data, headers=headers)
assert ret.status_code == 200, f'return code is {ret.status_code}'

2
seed/roundcube/templates/roundcube.service
View file

@ -1,6 +1,6 @@
[Unit]
Description=Roundcube database init
After=postgresqlclient.service
After=risotto.target
Before=nginx.service php-fpm.service
[Service]

1
seed/systemd/dictionaries/15-systemd.xml
View file

@ -29,6 +29,7 @@
<file>/secrets/root.pwd</file>
<file engine="none">/tmpfiles.d/risotto-volatile.conf</file>
</service>
<service name="risotto" target="multi-user" type="target" engine="none"/>
</services>
<variables>
<variable name='root_password' type="password" description="Mot de passe de l'administrateur système root" auto_save='False' mandatory="True"/>

5
seed/systemd/templates/risotto.target Normal file
View file

@ -0,0 +1,5 @@
[Unit]
Description=Waiting for all dependencies
Before=multi-user.target
After=network-online.target
Wants=network-online.target