Обновление версии контейнеров до 15.4 и других последних минорных ве…

…рсий - Поправил документацию - Добавил новую папку /app_db_init_sql для запуска скриптов во вновь созданной БД с именем APP_DB - Добавил переменную APP_DB_PASSWORD. - Обновил минорные версии для pgbouncer, postgres, mamonsu - Исправление ложных срабатываний в контроле JOBs для mamonsu - Update init_db_3_cron.sql
sgrinko · Aug 22, 2023 · f6454fc · f6454fc
1 parent 8f2e196
commit f6454fc
Show file tree

Hide file tree

Showing 41 changed files with 209 additions and 70 deletions.
diff --git a/12/bin/hub_push.sh b/12/bin/hub_push.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 VERSION=12
-MINOR=15
+MINOR=16
 
 set -euo pipefail
 
@@ -72,6 +72,6 @@ if ! docker image ls | grep "grufos/mamonsu" ; then
     echo "    push ..."
     docker tag ${VERSION}_mamonsu:latest grufos/mamonsu:latest
 #    docker push grufos/mamonsu:latest
-    docker tag grufos/mamonsu:latest grufos/mamonsu:${VERSION}_3.5.3
-    docker push grufos/mamonsu:${VERSION}_3.5.3
+    docker tag grufos/mamonsu:latest grufos/mamonsu:${VERSION}_3.5.5
+    docker push grufos/mamonsu:${VERSION}_3.5.5
 fi
diff --git a/12/docker-mamonsu/Dockerfile b/12/docker-mamonsu/Dockerfile
@@ -9,7 +9,7 @@ ENV DEBIAN_RELEASE bullseye
 ENV PG_MAJOR 12
 ENV BACKUP_PATH /mnt/pgbak
 # version mamonsu
-ENV VERSION 3.5.3
+ENV VERSION 3.5.5
 
 # explicitly set user/group IDs
 RUN set -eux; \

diff --git a/12/docker-mamonsu/pg_jobs_check.py b/12/docker-mamonsu/pg_jobs_check.py
@@ -13,7 +13,7 @@ class PgJobsCheck(Plugin):
     # получаем список всех БД
     query_agent_discovery = "select datname from pg_catalog.pg_database where datistemplate = false"
     # контролируем ошибки для конкретной БД
-    query = "select count(*) from cron.get_job_run_details('{1}','{0}'::interval) where status <> 'succeeded'"
+    query = "select count(*) from cron.get_job_run_details('{1}','{0}'::interval) where status = 'failed'"
     query_table_exists = "select 1 from pg_class where relname='job_run_details' and relnamespace=(select oid from pg_namespace where nspname='cron')"
 
     AgentPluginType = 'pg'

diff --git a/12/docker-postgres/Dockerfile b/12/docker-postgres/Dockerfile
@@ -4,7 +4,7 @@
 # https://hub.docker.com/r/postgis/postgis
 #    https://github.com/postgis/docker-postgis
 #
-FROM postgres:12.15
+FROM postgres:12.16
 
 LABEL maintainer="Sergey Grinko <sergey.grinko@gmail.com>"
 
@@ -103,14 +103,15 @@ RUN localedef -i ru_RU -c -f UTF-8 -A /usr/share/locale/locale.alias ru_RU.UTF-8
       && cd / \
       && ln -s /usr/share/postgresql/$PG_MAJOR/tsearch_data /usr/share/postgresql/ \
       # ====== clean all unused package...
-      && apt-mark hold llvm-11-runtime \
       && apt-get purge -y gcc freetds-dev git make postgresql-server-dev-$PG_MAJOR libicu-dev *-dev* \
       && apt-get -f install \
       && apt-get -y autoremove \
       && apt-get -y clean \
       && apt-get -y autoclean \
       && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /var/cache/apt/* \
-      && mkdir -p /docker-entrypoint-initdb.d
+      && mkdir -p /docker-entrypoint-initdb.d \
+      && mkdir -p /app_db_init_sql \
+      && chmod 0777 /app_db_init_sql
 
 COPY ./locales.conf /etc/locales.conf
 # copy bash files

diff --git a/12/docker-postgres/initdb-extension.sh b/12/docker-postgres/initdb-extension.sh
@@ -61,7 +61,7 @@ cd /usr/local/bin/
 
 # Create the 'template_extension' template DB and application DB
 if [ "$APP_DB" != "" ]; then
-  psql -f pre.sql -v DEPLOY_PASSWORD="$DEPLOY_PASSWORD" -v PGBOUNCER_PASSWORD="$PGBOUNCER_PASSWORD" -v APP_DB="$APP_DB"
+  psql -f pre.sql -v DEPLOY_PASSWORD="$DEPLOY_PASSWORD" -v PGBOUNCER_PASSWORD="$PGBOUNCER_PASSWORD" -v APP_DB="$APP_DB" -v APP_DB_PASSWORD="$APP_DB_PASSWORD"
 else
   psql -f pre.sql -v DEPLOY_PASSWORD="$DEPLOY_PASSWORD" -v PGBOUNCER_PASSWORD="$PGBOUNCER_PASSWORD"
 fi
@@ -80,6 +80,13 @@ for DB in "$POSTGRES_DB" template_extension "$APP_DB" ; do
         if [ "$DB" != "template_extension" ] ; then
             psql --dbname="$DB" -f db_target.sql -v DEV_SCHEMA="$DEV_SCHEMA" -v email_server="$EMAIL_SERVER"
         fi
+        if [ "$DB" = "$APP_DB" ] ; then
+            echo " Иннициализируем БД всеми скриптами из каталога /app_db_init_sql ..."
+            for file_sql in $(ls /app_db_init_sql/*.sql) ; do
+                echo "Файл: $file_sql"
+                psql --dbname="$DB" -f "$file_sql"
+            done
+        fi
     fi
   fi
 done

diff --git a/12/docker-postgres/sql/init_db_3_cron.sql b/12/docker-postgres/sql/init_db_3_cron.sql
@@ -212,10 +212,10 @@ GRANT ALL ON SCHEMA cron TO postgres;
 -- в 1-ю неделю месяца замораживаем идентификаторы транзакций, в остальные недели только собираем статистику
 \if :{?IS_SETUPDB}
   \if :IS_SETUPDB
-    select cron.schedule('vacuum JOB freeze', '0 0 1-7 * 0', 'vacuum (freeze,analyze);'); -- 1-я неделя каждого месяца
-    select cron.schedule('vacuum JOB', '0 0 8-31 * 0', 'vacuum (analyze);');              -- 2-4 неделя каждого месяца
+    select cron.schedule('vacuum JOB freeze', '0 0 1-7 1,6 */7', 'vacuum (freeze,analyze);'); -- 1-я неделя каждого полугодия
+    select cron.schedule('vacuum JOB', '0 0 8-31 * */7', 'vacuum (analyze);');              -- 2-4 неделя каждого месяца
   \endif
 \else
-    select cron.schedule('vacuum JOB freeze', '0 0 1-7 * 0', 'vacuum (freeze,analyze);'); -- 1-я неделя каждого месяца
-    select cron.schedule('vacuum JOB', '0 0 8-31 * 0', 'vacuum (analyze);');              -- 2-4 неделя каждого месяца
+    select cron.schedule('vacuum JOB freeze', '0 0 1-7 1,6 */7', 'vacuum (freeze,analyze);'); -- 1-я неделя каждого полугодия
+    select cron.schedule('vacuum JOB', '0 0 8-31 * */7', 'vacuum (analyze);');              -- 2-4 неделя каждого месяца
 \endif
diff --git a/12/docker-postgres/sql/pre.sql b/12/docker-postgres/sql/pre.sql
@@ -15,6 +15,17 @@ select not exists(select true FROM pg_catalog.pg_database where datname='templat
   \if :is_check
     CREATE DATABASE :"APP_DB";
   \endif
+
+  -- роль для приложения
+  select not exists(select true FROM pg_catalog.pg_roles where rolname=:'APP_DB') as is_check
+  \gset
+  \if :is_check
+      CREATE ROLE :"APP_DB" ;
+      SET log_statement='none';
+      ALTER ROLE :"APP_DB" WITH NOSUPERUSER INHERIT NOCREATEROLE NOCREATEDB LOGIN NOREPLICATION NOBYPASSRLS PASSWORD :'APP_DB_PASSWORD'; 
+      SET log_statement='ddl';
+      ALTER DATABASE :"APP_DB" OWNER TO :"APP_DB";
+  \endif
 \endif
 
 -- create role deploy
@@ -109,3 +120,9 @@ GRANT CONNECT ON DATABASE postgres TO execution_group;
 GRANT CONNECT ON DATABASE postgres TO read_procedure_group;
 GRANT CONNECT ON DATABASE postgres TO monitoring_group;
 GRANT ALL PRIVILEGES ON TABLESPACE pg_global TO monitoring_group;
+
+-- на пока даём права как для ORM роли приложения
+\if :{?APP_DB}
+  GRANT write_group TO :"APP_DB" ;
+  GRANT execution_group TO :"APP_DB" ;
+\endif
diff --git a/12/postgres-service.yml b/12/postgres-service.yml
@@ -17,11 +17,13 @@ services:
       - "/var/log/postgresql1:/var/log/postgresql"
       - "/mnt/pgbak2/:/mnt/pgbak/"
       - "/usr/share/postgres/12_1/tsearch_data:/usr/share/postgresql/tsearch_data"
+      - "/var/lib/pgsql/12_1/app_db_init_sql:/app_db_init_sql"
     ports:
       - "5433:5432"
     environment:
       ENV_DB_VALUE: DEV
       APP_DB: "My_db"
+      APP_DB_PASSWORD: qweasdzxc
 #      POSTGRES_INITDB_ARGS: "--locale=ru_RU.UTF8 --data-checksums"
       POSTGRES_PASSWORD: qweasdzxc
       POSTGRES_HOST_AUTH_METHOD: trust

diff --git a/12/postgres-service_all.yml b/12/postgres-service_all.yml
@@ -17,11 +17,13 @@ services:
       - "/var/log/postgresql1:/var/log/postgresql"
       - "/mnt/pgbak2/:/mnt/pgbak/"
       - "/usr/share/postgres/12_1/tsearch_data:/usr/share/postgresql/tsearch_data"
+      - "/var/lib/pgsql/12_1/app_db_init_sql:/app_db_init_sql"
     ports:
       - "5433:5432"
     environment:
       ENV_DB_VALUE: DEV
       APP_DB: "My_db"
+      APP_DB_PASSWORD: qweasdzxc
 #      POSTGRES_INITDB_ARGS: "--locale=ru_RU.UTF8 --data-checksums"
       POSTGRES_PASSWORD: qweasdzxc
       POSTGRES_HOST_AUTH_METHOD: trust

diff --git a/12/postgres-service_pgb.yml b/12/postgres-service_pgb.yml
@@ -17,11 +17,13 @@ services:
       - "/var/log/postgresql1:/var/log/postgresql"
       - "/mnt/pgbak2/:/mnt/pgbak/"
       - "/usr/share/postgres/12_1/tsearch_data:/usr/share/postgresql/tsearch_data"
+      - "/var/lib/pgsql/12_1/app_db_init_sql:/app_db_init_sql"
     ports:
       - "5433:5432"
     environment:
       ENV_DB_VALUE: DEV
       APP_DB: "My_db"
+      APP_DB_PASSWORD: qweasdzxc
 #      POSTGRES_INITDB_ARGS: "--locale=ru_RU.UTF8 --data-checksums"
       POSTGRES_PASSWORD: qweasdzxc
       POSTGRES_HOST_AUTH_METHOD: trust

diff --git a/13/bin/hub_push.sh b/13/bin/hub_push.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 VERSION=13
-MINOR=11
+MINOR=12
 
 set -euo pipefail
 
@@ -72,6 +72,6 @@ if ! docker image ls | grep "grufos/mamonsu" ; then
     echo "    push ..."
     docker tag ${VERSION}_mamonsu:latest grufos/mamonsu:latest
 #    docker push grufos/mamonsu:latest
-    docker tag grufos/mamonsu:latest grufos/mamonsu:${VERSION}_3.5.3
-    docker push grufos/mamonsu:${VERSION}_3.5.3
+    docker tag grufos/mamonsu:latest grufos/mamonsu:${VERSION}_3.5.5
+    docker push grufos/mamonsu:${VERSION}_3.5.5
 fi
diff --git a/13/docker-mamonsu/Dockerfile b/13/docker-mamonsu/Dockerfile
@@ -9,7 +9,7 @@ ENV DEBIAN_RELEASE bullseye
 ENV PG_MAJOR 13
 ENV BACKUP_PATH /mnt/pgbak
 # version mamonsu
-ENV VERSION 3.5.3
+ENV VERSION 3.5.5
 
 # explicitly set user/group IDs
 RUN set -eux; \

diff --git a/13/docker-mamonsu/pg_jobs_check.py b/13/docker-mamonsu/pg_jobs_check.py
@@ -13,7 +13,7 @@ class PgJobsCheck(Plugin):
     # получаем список всех БД
     query_agent_discovery = "select datname from pg_catalog.pg_database where datistemplate = false"
     # контролируем ошибки для конкретной БД
-    query = "select count(*) from cron.get_job_run_details('{1}','{0}'::interval) where status <> 'succeeded'"
+    query = "select count(*) from cron.get_job_run_details('{1}','{0}'::interval) where status = 'failed'"
     query_table_exists = "select 1 from pg_class where relname='job_run_details' and relnamespace=(select oid from pg_namespace where nspname='cron')"
 
     AgentPluginType = 'pg'

diff --git a/13/docker-postgres/Dockerfile b/13/docker-postgres/Dockerfile
@@ -4,7 +4,7 @@
 # https://hub.docker.com/r/postgis/postgis
 #    https://github.com/postgis/docker-postgis
 #
-FROM postgres:13.11
+FROM postgres:13.12
 
 LABEL maintainer="Sergey Grinko <sergey.grinko@gmail.com>"
 
@@ -103,14 +103,15 @@ RUN localedef -i ru_RU -c -f UTF-8 -A /usr/share/locale/locale.alias ru_RU.UTF-8
       && cd / \
       && ln -s /usr/share/postgresql/$PG_MAJOR/tsearch_data /usr/share/postgresql/ \
       # ====== clean all unused package...
-      && apt-mark hold llvm-11-runtime \
       && apt-get purge -y gcc freetds-dev git make postgresql-server-dev-$PG_MAJOR libicu-dev *-dev* \
       && apt-get -f install \
       && apt-get -y autoremove \
       && apt-get -y clean \
       && apt-get -y autoclean \
       && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /var/cache/apt/* \
-      && mkdir -p /docker-entrypoint-initdb.d
+      && mkdir -p /docker-entrypoint-initdb.d \
+      && mkdir -p /app_db_init_sql \
+      && chmod 0777 /app_db_init_sql
 
 COPY ./locales.conf /etc/locales.conf
 # copy bash files

diff --git a/13/docker-postgres/initdb-extension.sh b/13/docker-postgres/initdb-extension.sh
@@ -61,7 +61,7 @@ cd /usr/local/bin/
 
 # Create the 'template_extension' template DB and application DB
 if [ "$APP_DB" != "" ]; then
-  psql -f pre.sql -v DEPLOY_PASSWORD="$DEPLOY_PASSWORD" -v PGBOUNCER_PASSWORD="$PGBOUNCER_PASSWORD" -v APP_DB="$APP_DB"
+  psql -f pre.sql -v DEPLOY_PASSWORD="$DEPLOY_PASSWORD" -v PGBOUNCER_PASSWORD="$PGBOUNCER_PASSWORD" -v APP_DB="$APP_DB" -v APP_DB_PASSWORD="$APP_DB_PASSWORD"
 else
   psql -f pre.sql -v DEPLOY_PASSWORD="$DEPLOY_PASSWORD" -v PGBOUNCER_PASSWORD="$PGBOUNCER_PASSWORD"
 fi
@@ -80,6 +80,13 @@ for DB in "$POSTGRES_DB" template_extension "$APP_DB" ; do
         if [ "$DB" != "template_extension" ] ; then
             psql --dbname="$DB" -f db_target.sql -v DEV_SCHEMA="$DEV_SCHEMA" -v email_server="$EMAIL_SERVER"
         fi
+        if [ "$DB" = "$APP_DB" ] ; then
+            echo " Иннициализируем БД всеми скриптами из каталога /app_db_init_sql ..."
+            for file_sql in $(ls /app_db_init_sql/*.sql) ; do
+                echo "Файл: $file_sql"
+                psql --dbname="$DB" -f "$file_sql"
+            done
+        fi
     fi
   fi
 done

diff --git a/13/docker-postgres/sql/init_db_3_cron.sql b/13/docker-postgres/sql/init_db_3_cron.sql
@@ -212,10 +212,10 @@ GRANT ALL ON SCHEMA cron TO postgres;
 -- в 1-ю неделю месяца замораживаем идентификаторы транзакций, в остальные недели только собираем статистику
 \if :{?IS_SETUPDB}
   \if :IS_SETUPDB
-    select cron.schedule('vacuum JOB freeze', '0 0 1-7 * 0', 'vacuum (freeze,analyze);'); -- 1-я неделя каждого месяца
-    select cron.schedule('vacuum JOB', '0 0 8-31 * 0', 'vacuum (analyze);');       -- 2-4 неделя каждого месяца
+    select cron.schedule('vacuum JOB freeze', '0 0 1-7 1,6 */7', 'vacuum (freeze,analyze);'); -- 1-я неделя каждого полугодия
+    select cron.schedule('vacuum JOB', '0 0 8-31 * */7', 'vacuum (analyze);');       -- 2-4 неделя каждого месяца
   \endif
 \else
-    select cron.schedule('vacuum JOB freeze', '0 0 1-7 * 0', 'vacuum (freeze,analyze);'); -- 1-я неделя каждого месяца
-    select cron.schedule('vacuum JOB', '0 0 8-31 * 0', 'vacuum (analyze);');       -- 2-4 неделя каждого месяца
+    select cron.schedule('vacuum JOB freeze', '0 0 1-7 1,6 */7', 'vacuum (freeze,analyze);'); -- 1-я неделя каждого полугодия
+    select cron.schedule('vacuum JOB', '0 0 8-31 * */7', 'vacuum (analyze);');       -- 2-4 неделя каждого месяца
 \endif
diff --git a/13/docker-postgres/sql/pre.sql b/13/docker-postgres/sql/pre.sql
@@ -15,6 +15,17 @@ select not exists(select true FROM pg_catalog.pg_database where datname='templat
   \if :is_check
     CREATE DATABASE :"APP_DB";
   \endif
+
+  -- роль для приложения
+  select not exists(select true FROM pg_catalog.pg_roles where rolname=:'APP_DB') as is_check
+  \gset
+  \if :is_check
+      CREATE ROLE :"APP_DB" ;
+      SET log_statement='none';
+      ALTER ROLE :"APP_DB" WITH NOSUPERUSER INHERIT NOCREATEROLE NOCREATEDB LOGIN NOREPLICATION NOBYPASSRLS PASSWORD :'APP_DB_PASSWORD'; 
+      SET log_statement='ddl';
+      ALTER DATABASE :"APP_DB" OWNER TO :"APP_DB";
+  \endif
 \endif
 
 -- create role deploy
@@ -109,3 +120,9 @@ GRANT CONNECT ON DATABASE postgres TO execution_group;
 GRANT CONNECT ON DATABASE postgres TO read_procedure_group;
 GRANT CONNECT ON DATABASE postgres TO monitoring_group;
 GRANT ALL PRIVILEGES ON TABLESPACE pg_global TO monitoring_group;
+
+-- на пока даём права как для ORM роли приложения
+\if :{?APP_DB}
+  GRANT write_group TO :"APP_DB" ;
+  GRANT execution_group TO :"APP_DB" ;
+\endif
diff --git a/13/postgres-service.yml b/13/postgres-service.yml
@@ -17,11 +17,13 @@ services:
       - "/var/log/postgresql1:/var/log/postgresql"
       - "/mnt/pgbak2/:/mnt/pgbak/"
       - "/usr/share/postgres/13_1/tsearch_data:/usr/share/postgresql/tsearch_data"
+      - "/var/lib/pgsql/13_1/app_db_init_sql:/app_db_init_sql"
     ports:
       - "5433:5432"
     environment:
       ENV_DB_VALUE: DEV
       APP_DB: "My_db"
+      APP_DB_PASSWORD: qweasdzxc
 #      POSTGRES_INITDB_ARGS: "--locale=ru_RU.UTF8 --data-checksums"
       POSTGRES_PASSWORD: qweasdzxc
       POSTGRES_HOST_AUTH_METHOD: trust

diff --git a/13/postgres-service_all.yml b/13/postgres-service_all.yml
@@ -17,11 +17,13 @@ services:
       - "/var/log/postgresql1:/var/log/postgresql"
       - "/mnt/pgbak2/:/mnt/pgbak/"
       - "/usr/share/postgres/13_1/tsearch_data:/usr/share/postgresql/tsearch_data"
+      - "/var/lib/pgsql/13_1/app_db_init_sql:/app_db_init_sql"
     ports:
       - "5433:5432"
     environment:
       ENV_DB_VALUE: DEV
       APP_DB: "My_db"
+      APP_DB_PASSWORD: qweasdzxc
 #      POSTGRES_INITDB_ARGS: "--locale=ru_RU.UTF8 --data-checksums"
       POSTGRES_PASSWORD: qweasdzxc
       POSTGRES_HOST_AUTH_METHOD: trust

diff --git a/13/postgres-service_pgb.yml b/13/postgres-service_pgb.yml
@@ -17,11 +17,13 @@ services:
       - "/var/log/postgresql1:/var/log/postgresql"
       - "/mnt/pgbak2/:/mnt/pgbak/"
       - "/usr/share/postgres/13_1/tsearch_data:/usr/share/postgresql/tsearch_data"
+      - "/var/lib/pgsql/13_1/app_db_init_sql:/app_db_init_sql"
     ports:
       - "5433:5432"
     environment:
       ENV_DB_VALUE: DEV
       APP_DB: "My_db"
+      APP_DB_PASSWORD: qweasdzxc
 #      POSTGRES_INITDB_ARGS: "--locale=ru_RU.UTF8 --data-checksums"
       POSTGRES_PASSWORD: qweasdzxc
       POSTGRES_HOST_AUTH_METHOD: trust

diff --git a/14/bin/hub_push.sh b/14/bin/hub_push.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 VERSION=14
-MINOR=8
+MINOR=9
 
 set -euo pipefail
 
@@ -72,6 +72,6 @@ if ! docker image ls | grep "grufos/mamonsu" ; then
     echo "    push ..."
     docker tag ${VERSION}_mamonsu:latest grufos/mamonsu:latest
 #    docker push grufos/mamonsu:latest
-    docker tag grufos/mamonsu:latest grufos/mamonsu:${VERSION}_3.5.3
-    docker push grufos/mamonsu:${VERSION}_3.5.3
+    docker tag grufos/mamonsu:latest grufos/mamonsu:${VERSION}_3.5.5
+    docker push grufos/mamonsu:${VERSION}_3.5.5
 fi
diff --git a/14/docker-mamonsu/Dockerfile b/14/docker-mamonsu/Dockerfile
@@ -9,7 +9,7 @@ ENV DEBIAN_RELEASE bullseye
 ENV PG_MAJOR 14
 ENV BACKUP_PATH /mnt/pgbak
 # version mamonsu
-ENV VERSION 3.5.3
+ENV VERSION 3.5.5
 
 # explicitly set user/group IDs
 RUN set -eux; \

diff --git a/14/docker-mamonsu/pg_jobs_check.py b/14/docker-mamonsu/pg_jobs_check.py
@@ -13,7 +13,7 @@ class PgJobsCheck(Plugin):
     # получаем список всех БД
     query_agent_discovery = "select datname from pg_catalog.pg_database where datistemplate = false"
     # контролируем ошибки для конкретной БД
-    query = "select count(*) from cron.get_job_run_details('{1}','{0}'::interval) where status <> 'succeeded'"
+    query = "select count(*) from cron.get_job_run_details('{1}','{0}'::interval) where status = 'failed'"
     query_table_exists = "select 1 from pg_class where relname='job_run_details' and relnamespace=(select oid from pg_namespace where nspname='cron')"
 
     AgentPluginType = 'pg'