From d21219fe3ec30b660cf62affef7800b2a83465c4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Witkowski?= <contact@lukaszwitkowski.com>
Date: Tue, 30 Jul 2024 11:39:41 +0200
Subject: [PATCH] Introduce Flyway to manage db schema (#15)

---
 .github/workflows/db-migration.yaml         | 24 ++++++++++++++++++++
 README.md                                   | 10 ++++-----
 backend/run_tests.sh                        | 16 -------------
 db/{ddl.sql => migrations/V1__baseline.sql} |  2 --
 db/run_migrations_test.sh                   | 25 +++++++++++++++++++++
 docker-compose.yaml                         | 11 ++++++---
 6 files changed, 62 insertions(+), 26 deletions(-)
 create mode 100644 .github/workflows/db-migration.yaml
 rename db/{ddl.sql => migrations/V1__baseline.sql} (94%)
 create mode 100755 db/run_migrations_test.sh

diff --git a/.github/workflows/db-migration.yaml b/.github/workflows/db-migration.yaml
new file mode 100644
index 0000000..99c1159
--- /dev/null
+++ b/.github/workflows/db-migration.yaml
@@ -0,0 +1,24 @@
+name: DB Migration
+
+on:
+  push:
+    branches: [ main ]
+    paths:
+      - db/**
+  pull_request:
+    branches: [ main ]
+    paths:
+      - db/**
+  workflow_dispatch:
+
+jobs:
+  db-schema-migration:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: 'Test flyway migration scripts'
+        working-directory: ./db
+        run: |
+          ./run_migrations_test.sh
\ No newline at end of file
diff --git a/README.md b/README.md
index e6b6901..d3349e4 100644
--- a/README.md
+++ b/README.md
@@ -13,7 +13,7 @@ This project aims at reviving www.aero-offers.com - invaluable source of price t
     - `job_fetch_offers` - scans few portals (e.g. soaring.de) and stores new offers in the database (not yet classified)
     - `job_reclassify_offers` - assigns manufacturer and model to new (not yet classified) offers stored in the database
     - `job_update_exchange_rates` - updates currency exchange rates based ok ECP api
-- `db` - PostgreSQL 15 database
+- `db` - PostgreSQL 15 database with DDL scripts managed by Flyway
 
 ### TODO
 - use Azure secrets for db credentials
@@ -35,14 +35,14 @@ This project aims at reviving www.aero-offers.com - invaluable source of price t
 - docker (compose)
 - npm
 
-Unzip database backup (optional)
+Start Postgres in docker (available for debugging via `localhost:25432`):
 ```bash
-cd db && unzip -qq prod_dump_2024_06_31.sql.zip
+docker-compose up ca-aerooffers-db flyway
 ```
 
-Start Postgres in docker (available for debugging via `localhost:25432`):
+Unzip database backup (optional) and load into DB
 ```bash
-docker-compose up ca-aerooffers-db
+cd db && unzip -qq prod_dump_2024_06_31.sql.zip
 ```
 
 Install python packages
diff --git a/backend/run_tests.sh b/backend/run_tests.sh
index 6ed2e15..bc16678 100755
--- a/backend/run_tests.sh
+++ b/backend/run_tests.sh
@@ -1,21 +1,5 @@
 export PYTHONPATH=$PYTHONPATH':./'
 
-export DB_PORT=35432
-
-echo "Starting Postgres docker container"
-
-docker run \
-    --name test-db \
-    -d \
-    -e 'POSTGRES_DB=aircraft_offers' \
-    -e 'POSTGRES_USER=aircraft_offers' \
-    -e 'POSTGRES_PASSWORD=aircraft_offers' \
-    -p ${DB_PORT}':5432' \
-    -v ${PWD}'/../db/ddl.sql:/docker-entrypoint-initdb.d/ddl.sql' \
-    postgres:15-alpine
-
-sleep 5
-
 python3 -m unittest -f
 
 if [[ $? -ne 0 ]]; then
diff --git a/db/ddl.sql b/db/migrations/V1__baseline.sql
similarity index 94%
rename from db/ddl.sql
rename to db/migrations/V1__baseline.sql
index 4347002..c8e10d3 100644
--- a/db/ddl.sql
+++ b/db/migrations/V1__baseline.sql
@@ -1,5 +1,3 @@
-SET client_encoding = 'UTF8';
-
 CREATE TABLE aircraft_offer (
     id SERIAL PRIMARY KEY,
     date DATE,
diff --git a/db/run_migrations_test.sh b/db/run_migrations_test.sh
new file mode 100755
index 0000000..3afa9ac
--- /dev/null
+++ b/db/run_migrations_test.sh
@@ -0,0 +1,25 @@
+echo "Starting Postgres docker container"
+
+docker network create test-network
+
+docker run --rm \
+    --name test-db \
+    -d \
+    --network test-network \
+    -e 'POSTGRES_DB=aircraft_offers' \
+    -e 'POSTGRES_USER=aircraft_offers' \
+    -e 'POSTGRES_PASSWORD=aircraft_offers' \
+    postgres:15-alpine
+
+sleep 2
+
+echo "Running migrations"
+
+docker run --rm \
+    --network test-network \
+    -v ${PWD}'/migrations/:/flyway/sql' \
+    flyway/flyway \
+    -url=jdbc:postgresql://test-db:5432/aircraft_offers -user=aircraft_offers -password=aircraft_offers migrate
+
+docker rm -f test-db
+docker network rm test-network
\ No newline at end of file
diff --git a/docker-compose.yaml b/docker-compose.yaml
index cc1dee4..88c2a60 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -18,16 +18,21 @@ services:
       - DB_PW=aircraft_offers
     depends_on:
       - ca-aerooffers-db
+      - flyway
 
   ca-aerooffers-db:
     image: postgres:15-alpine
-    restart: always
     environment:
       - POSTGRES_DB=aircraft_offers
       - POSTGRES_USER=aircraft_offers
       - POSTGRES_PASSWORD=aircraft_offers
     ports:
       - '25432:5432'
+
+  flyway:
+    image: flyway/flyway
+    command: -url=jdbc:postgresql://ca-aerooffers-db:5432/aircraft_offers -user=aircraft_offers -password=aircraft_offers migrate
     volumes:
-      - ./db/ddl.sql:/docker-entrypoint-initdb.d/ddl.sql
-      #- ./db/prod_dump_2024_06_31.sql:/docker-entrypoint-initdb.d/prod_dump_2024_06_31.sql
+      - ./db/migrations:/flyway/sql
+    depends_on:
+      - ca-aerooffers-db
\ No newline at end of file