From 97e5b8e02ecb0719948bcfc605c3faf943fed8a6 Mon Sep 17 00:00:00 2001 From: Emmanuel Garette Date: Mon, 18 Jul 2022 17:06:12 +0200 Subject: [PATCH] upgrade postgresql to fedora 36 --- .../templates/postgresqlclient.service | 2 +- seed/postgresql/applicationservice.yml | 2 +- .../postgresql/dictionaries/22_postgresql.xml | 3 +- .../image/preinstall/postgresql_server.sh | 2 + seed/postgresql/templates/postgresql.conf | 126 ++++++++++-------- seed/postgresql/templates/postgresql.service | 35 ++++- seed/postgresql/templates/postgresql_init | 30 +++-- .../templates/sysuser-postgresql.conf | 2 +- .../templates/tmpfiles.postgresql.conf | 2 + seed/reverse-proxy-client/tests/revprox.py | 2 +- 10 files changed, 130 insertions(+), 76 deletions(-) create mode 100644 seed/postgresql/templates/tmpfiles.postgresql.conf diff --git a/seed/postgresql-client/templates/postgresqlclient.service b/seed/postgresql-client/templates/postgresqlclient.service index 0dc6a15..7addde1 100644 --- a/seed/postgresql-client/templates/postgresqlclient.service +++ b/seed/postgresql-client/templates/postgresqlclient.service @@ -5,5 +5,5 @@ Before=network.target [Service] Type=oneshot Environment=PGPASSFILE=/usr/local/lib/secrets/postgresql.pass -ExecStart=/usr/bin/timeout 90 bash -c 'while ! 3<> /dev/tcp/%%pg_client_server_domainname/5432; do sleep 1; done; echo "POSTGRESQL STARTED"' +ExecStart=/usr/bin/timeout 300 bash -c 'while ! 3<> /dev/tcp/%%pg_client_server_domainname/5432; do sleep 1; done; echo "POSTGRESQL STARTED"' ExecStart=/usr/bin/timeout 90 bash -c 'while ! /usr/bin/psql --set=sslmode=verify-full -h %%pg_client_server_domainname -U %%pg_client_username %%pg_client_database -c "\l"; do sleep 1; done; echo "POSTGRESQL READY"' diff --git a/seed/postgresql/applicationservice.yml b/seed/postgresql/applicationservice.yml index d205722..e445853 100644 --- a/seed/postgresql/applicationservice.yml +++ b/seed/postgresql/applicationservice.yml @@ -2,5 +2,5 @@ format: '0.1' description: Postgresql depends: - server - - base-fedora-35 + - base-fedora-36 provider: Postgresql diff --git a/seed/postgresql/dictionaries/22_postgresql.xml b/seed/postgresql/dictionaries/22_postgresql.xml index 40e9bf8..2fa2ef7 100644 --- a/seed/postgresql/dictionaries/22_postgresql.xml +++ b/seed/postgresql/dictionaries/22_postgresql.xml @@ -2,7 +2,7 @@ - + accounts.remote_.remote_ip_ /etc/postgresql/postgresql.conf /etc/postgresql/pg_hba.conf @@ -10,6 +10,7 @@ /etc/postgresql/pg_ident.conf /bin/postgresql_init /sysusers.d/0postgresql.conf + /tmpfiles.d/0postgresql.conf /etc/pki/ca-trust/source/anchors/ca_PostgreSQL.crt /etc/pki/tls/certs/postgresql.crt /etc/pki/tls/private/postgresql.key diff --git a/seed/postgresql/manual/image/preinstall/postgresql_server.sh b/seed/postgresql/manual/image/preinstall/postgresql_server.sh index 4bdeb8e..3f65348 100644 --- a/seed/postgresql/manual/image/preinstall/postgresql_server.sh +++ b/seed/postgresql/manual/image/preinstall/postgresql_server.sh @@ -1 +1,3 @@ PKG="$PKG postgresql-server postgresql-contrib" +# for postgresql-setup +PKG="$PKG util-linux postgresql-upgrade" diff --git a/seed/postgresql/templates/postgresql.conf b/seed/postgresql/templates/postgresql.conf index 8849422..44df34c 100644 --- a/seed/postgresql/templates/postgresql.conf +++ b/seed/postgresql/templates/postgresql.conf @@ -47,9 +47,6 @@ directiveStartToken = § #data_directory = 'ConfigDir' # use data in another directory # (change requires restart) -#>GNUNUX -data_directory = '/srv/postgresql' -#GNUNUX @@ -116,7 +113,7 @@ unix_socket_directories = '/var/run/postgresql' #>GNUNUX authentication_timeout = §§{pg_authentication_timeout}s #GNUNUX wal_buffers = §§pg_wal_buffers +#GNUNUX #max_wal_size = 1GB max_wal_size = §§{pg_max_wal_size}§§pg_max_wal_size_unit #GNUNUX +effective_cache_size = §§{pg_effective_cache_size}§§pg_effective_cache_size_unit +#GNUNUX #log_destination = 'stderr' # Valid values are combinations of # stderr, csvlog, syslog, and eventlog, # depending on platform. csvlog # requires logging_collector to be on. -log_destination = 'syslog' -#GNUNUX +log_destination = 'syslog' +# 0 logs only + # actions running at least this number + # of milliseconds. #log_checkpoints = off #log_connections = off #log_disconnections = off @@ -579,9 +589,11 @@ log_destination = 'syslog' # %h = remote host # %b = backend type # %p = process ID + # %P = process ID of parallel group leader # %t = timestamp without milliseconds # %m = timestamp with milliseconds # %n = timestamp with milliseconds (as a Unix epoch) + # %Q = query ID (0 if none or not computed) # %i = command tag # %e = SQL state # %c = session ID @@ -594,6 +606,8 @@ log_destination = 'syslog' # %% = '%' # e.g. '<%u%%%d> ' #log_lock_waits = off # log lock waits >= deadlock_timeout +#log_recovery_conflict_waits = off # log standby recovery conflict waits + # >= deadlock_timeout #log_parameter_max_length = -1 # when logging statements, limit logged # bind-parameter values to N bytes; # -1 means print in full, 0 disables @@ -608,6 +622,7 @@ log_destination = 'syslog' #FIXME en dure ? log_timezone = 'Europe/Paris' + #------------------------------------------------------------------------------ # PROCESS TITLE #------------------------------------------------------------------------------ @@ -624,19 +639,21 @@ log_timezone = 'Europe/Paris' # - Query and Index Statistics Collector - #track_activities = on +#track_activity_query_size = 1024 # (change requires restart) #track_counts = on #track_io_timing = off +#track_wal_io_timing = off #track_functions = none # none, pl, all -#track_activity_query_size = 1024 # (change requires restart) #stats_temp_directory = 'pg_stat_tmp' # - Monitoring - +#compute_query_id = auto +#log_statement_stats = off #log_parser_stats = off #log_planner_stats = off #log_executor_stats = off -#log_statement_stats = off #------------------------------------------------------------------------------ @@ -652,10 +669,6 @@ autovacuum = on autovacuum = off §end if # 0 logs only - # actions running at least this number - # of milliseconds. #autovacuum_max_workers = 3 # max number of autovacuum subprocesses # (change requires restart) #autovacuum_naptime = 1min # time between autovacuum runs @@ -701,10 +714,11 @@ autovacuum = off # error #search_path = '"$user", public' # schema names #row_security = on +#default_table_access_method = 'heap' #default_tablespace = '' # a tablespace name, '' uses the default +#default_toast_compression = 'pglz' # 'pglz' or 'lz4' #temp_tablespaces = '' # a list of tablespace names, '' uses # only default tablespace -#default_table_access_method = 'heap' #check_function_bodies = on #default_transaction_isolation = 'read committed' #default_transaction_read_only = off @@ -713,17 +727,16 @@ autovacuum = off #statement_timeout = 0 # in milliseconds, 0 is disabled #lock_timeout = 0 # in milliseconds, 0 is disabled #idle_in_transaction_session_timeout = 0 # in milliseconds, 0 is disabled -#vacuum_freeze_min_age = 50000000 +#idle_session_timeout = 0 # in milliseconds, 0 is disabled #vacuum_freeze_table_age = 150000000 -#vacuum_multixact_freeze_min_age = 5000000 +#vacuum_freeze_min_age = 50000000 +#vacuum_failsafe_age = 1600000000 #vacuum_multixact_freeze_table_age = 150000000 -#vacuum_cleanup_index_scale_factor = 0.1 # fraction of total number of tuples - # before index cleanup, 0 always performs - # index cleanup +#vacuum_multixact_freeze_min_age = 5000000 +#vacuum_multixact_failsafe_age = 1600000000 #bytea_output = 'hex' # hex, escape #xmlbinary = 'base64' #xmloption = 'content' -#gin_fuzzy_search_limit = 0 #gin_pending_list_limit = 4MB # - Locale and Formatting - @@ -757,14 +770,15 @@ default_text_search_config = 'pg_catalog.french' # - Shared Library Preloading - -#shared_preload_libraries = '' # (change requires restart) #local_preload_libraries = '' #session_preload_libraries = '' +#shared_preload_libraries = '' # (change requires restart) #jit_provider = 'llvmjit' # JIT library to use # - Other Defaults - #dynamic_library_path = '$libdir' +#gin_fuzzy_search_limit = 0 #------------------------------------------------------------------------------ @@ -792,7 +806,6 @@ default_text_search_config = 'pg_catalog.french' #backslash_quote = safe_encoding # on, off, or safe_encoding #escape_string_warning = on #lo_compat_privileges = off -#operator_precedence_warning = off #quote_all_identifiers = off #standard_conforming_strings = on #synchronize_seqscans = on @@ -811,6 +824,7 @@ default_text_search_config = 'pg_catalog.french' #data_sync_retry = off # retry or panic on failure to fsync # data? # (change requires restart) +#recovery_init_sync_method = fsync # fsync, syncfs (Linux 5.8+) #------------------------------------------------------------------------------ diff --git a/seed/postgresql/templates/postgresql.service b/seed/postgresql/templates/postgresql.service index 2e4e745..a65b4c5 100644 --- a/seed/postgresql/templates/postgresql.service +++ b/seed/postgresql/templates/postgresql.service @@ -1,11 +1,38 @@ [Service] -ExecStartPre= -ExecStartPre=+/usr/local/lib/bin/postgresql_init -ExecStartPre=/usr/libexec/postgresql-check-db-dir %N -Environment=PGDATA=/srv/postgresql +Environment=PGDATA=/srv/postgresql/postgresql Environment=PG_CONF=/etc/postgresql/postgresql.conf Environment=PG_HBA=/etc/postgresql/pg_hba.conf Environment=PG_IDENT=/etc/postgresql/pg_ident.conf +Environment=LC_ALL=fr_FR.UTF-8 +ExecStartPre= +ExecStartPre=+/usr/local/lib/bin/postgresql_init +# if upgrade needed, do it +ExecStartPre=/bin/bash -c '%slurp +/usr/libexec/postgresql-check-db-dir %N || (%slurp + echo "UPGRADE" &&%slurp +# directory creation must have 700 rights + umask 0077 &&%slurp +# pg_upgrade do not like ssl activation + /bin/grep -v "ssl " ${PG_CONF} > /tmp/postgresql.conf &&%slurp + mv -f /tmp/postgresql.conf ${PGDATA}/postgresql.conf &&%slurp +# pg_upgrade modify pg_hba.conf so copy it + /bin/rm ${PGDATA}/pg_hba.conf &&%slurp + /bin/cp -af ${PG_HBA} ${PGDATA} &&%slurp +# do upgrade + /usr/bin/postgresql-setup --upgrade &&%slurp +# re do link + ln -sf ${PG_HBA} ${PGDATA}/ &&%slurp + ln -sf ${PG_CONF} ${PGDATA}/ &&%slurp +# remove old cluster + /srv/postgresql/postgresql/delete_old_cluster.sh &&%slurp + rm -f /srv/postgresql/postgresql/delete_old_cluster.sh &&%slurp +# force index (see later) + touch /srv/postgresql/risotto_upgrade.lock%slurp +)' +# recheck db +ExecStartPre=/usr/libexec/postgresql-check-db-dir %N ExecStart= ExecStart=/usr/bin/postmaster -D ${PGDATA} -c config_file=${PG_CONF} -c hba_file=${PG_HBA} -c ident_file=${PG_IDENT} ExecStartPost=-/usr/bin/psql -f /etc/postgresql/postgresql.sql +# if lock do reindex +ExecStartPost=/bin/bash -c 'if [ -f /srv/postgresql/risotto_upgrade.lock ];then echo REINDEX; /usr/bin/reindexdb && rm -f /srv/postgresql/risotto_upgrade.lock; fi' diff --git a/seed/postgresql/templates/postgresql_init b/seed/postgresql/templates/postgresql_init index fc84c6c..fda9a32 100644 --- a/seed/postgresql/templates/postgresql_init +++ b/seed/postgresql/templates/postgresql_init @@ -1,14 +1,22 @@ #!/bin/bash -e -[ -d "/srv/postgresql" ] && exit 0 || true - -/bin/mkdir /srv/postgresql -/bin/chown postgres: /srv/postgresql -mkdir /var/lib/pgsql -/bin/chown postgres: /var/lib/pgsql -/usr/bin/postgresql-setup --initdb -/bin/rm /srv/postgresql/postgresql.conf -/bin/rm /srv/postgresql/pg_hba.conf -/bin/rm /srv/postgresql/pg_ident.conf - +if [ ! -d "/srv/postgresql" ]; then + /bin/mkdir -p /srv/postgresql/postgresql + /bin/chown -R postgres: /srv/postgresql + /usr/bin/postgresql-setup --initdb + #/bin/rm /srv/postgresql/postgresql.conf + #/bin/rm /srv/postgresql/pg_hba.conf + #/bin/rm /srv/postgresql/pg_ident.conf +elif [ ! -d "/srv/postgresql/postgresql" ]; then + # migrate /srv/postgresql to /srv/postgresql/postgresql + # needed for upgrade... + mkdir /srv/postgresql/postgresql + mv /srv/postgresql/* /srv/postgresql/postgresql || true + chown postgres: /srv/postgresql/postgresql + chmod 700 /srv/postgresql/postgresql +fi +# for postgresql-setup... +/bin/ln -sf /etc/postgresql/postgresql.conf /srv/postgresql/postgresql/postgresql.conf +/bin/ln -sf /etc/postgresql/pg_hba.conf /srv/postgresql/postgresql/pg_hba.conf +/bin/ln -sf /etc/postgresql/pg_ident.conf /srv/postgresql/postgresql/pg_ident.conf exit 0 diff --git a/seed/postgresql/templates/sysuser-postgresql.conf b/seed/postgresql/templates/sysuser-postgresql.conf index d07f84f..4458876 100644 --- a/seed/postgresql/templates/sysuser-postgresql.conf +++ b/seed/postgresql/templates/sysuser-postgresql.conf @@ -1,3 +1,3 @@ g postgres 26 - -u postgres 26:26 "PostgreSQL Server" /srv/postgresql /bin/bash +u postgres 26:26 "PostgreSQL Server" /srv/postgresql/postgresql /bin/bash diff --git a/seed/postgresql/templates/tmpfiles.postgresql.conf b/seed/postgresql/templates/tmpfiles.postgresql.conf new file mode 100644 index 0000000..29b3c42 --- /dev/null +++ b/seed/postgresql/templates/tmpfiles.postgresql.conf @@ -0,0 +1,2 @@ +# for postgresql-setup only... +d /var/lib/pgsql/ 0750 postgres postgres - diff --git a/seed/reverse-proxy-client/tests/revprox.py b/seed/reverse-proxy-client/tests/revprox.py index 9b85bf5..bb9ab17 100644 --- a/seed/reverse-proxy-client/tests/revprox.py +++ b/seed/reverse-proxy-client/tests/revprox.py @@ -34,7 +34,7 @@ class Authentication: code = ret.status_code content = ret.content assert code == 200 - assert b'Authentication portal' in content + assert b'Authentication portal' in content, f'cannot find LemonLdap title: {content}' def auth_lemonldap(self, req,