Introduce Flyway to manage db schema (#15)

.github/workflows/db-migration.yaml

@@ -0,0 +1,24 @@
+name: DB Migration
+
+on:
+  push:
+    branches: [ main ]
+    paths:
+      - db/**
+  pull_request:
+    branches: [ main ]
+    paths:
+      - db/**
+  workflow_dispatch:
+
+jobs:
+  db-schema-migration:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: 'Test flyway migration scripts'
+        working-directory: ./db
+        run: |
+          ./run_migrations_test.sh

README.md

@@ -13,7 +13,7 @@ This project aims at reviving www.aero-offers.com - invaluable source of price t
     - `job_fetch_offers` - scans few portals (e.g. soaring.de) and stores new offers in the database (not yet classified)
     - `job_reclassify_offers` - assigns manufacturer and model to new (not yet classified) offers stored in the database
     - `job_update_exchange_rates` - updates currency exchange rates based ok ECP api
-- `db` - PostgreSQL 15 database
+- `db` - PostgreSQL 15 database with DDL scripts managed by Flyway
 
 ### TODO
 - use Azure secrets for db credentials
@@ -35,14 +35,14 @@ This project aims at reviving www.aero-offers.com - invaluable source of price t
 - docker (compose)
 - npm
 
-Unzip database backup (optional)
+Start Postgres in docker (available for debugging via `localhost:25432`):
 ```bash
-cd db && unzip -qq prod_dump_2024_06_31.sql.zip
+docker-compose up ca-aerooffers-db flyway
 ```
 
-Start Postgres in docker (available for debugging via `localhost:25432`):
+Unzip database backup (optional) and load into DB
 ```bash
-docker-compose up ca-aerooffers-db
+cd db && unzip -qq prod_dump_2024_06_31.sql.zip
 ```
 
 Install python packages

backend/run_tests.sh

@@ -1,21 +1,5 @@
 export PYTHONPATH=$PYTHONPATH':./'
 
-export DB_PORT=35432
-
-echo "Starting Postgres docker container"
-
-docker run \
-    --name test-db \
-    -d \
-    -e 'POSTGRES_DB=aircraft_offers' \
-    -e 'POSTGRES_USER=aircraft_offers' \
-    -e 'POSTGRES_PASSWORD=aircraft_offers' \
-    -p ${DB_PORT}':5432' \
-    -v ${PWD}'/../db/ddl.sql:/docker-entrypoint-initdb.d/ddl.sql' \
-    postgres:15-alpine
-
-sleep 5
-
 python3 -m unittest -f
 
 if [[ $? -ne 0 ]]; then

db/ddl.sql → db/migrations/V1__baseline.sql

@@ -1,5 +1,3 @@
-SET client_encoding = 'UTF8';
-
 CREATE TABLE aircraft_offer (
     id SERIAL PRIMARY KEY,
     date DATE,

db/run_migrations_test.sh

@@ -0,0 +1,25 @@
+echo "Starting Postgres docker container"
+
+docker network create test-network
+
+docker run --rm \
+    --name test-db \
+    -d \
+    --network test-network \
+    -e 'POSTGRES_DB=aircraft_offers' \
+    -e 'POSTGRES_USER=aircraft_offers' \
+    -e 'POSTGRES_PASSWORD=aircraft_offers' \
+    postgres:15-alpine
+
+sleep 2
+
+echo "Running migrations"
+
+docker run --rm \
+    --network test-network \
+    -v ${PWD}'/migrations/:/flyway/sql' \
+    flyway/flyway \
+    -url=jdbc:postgresql://test-db:5432/aircraft_offers -user=aircraft_offers -password=aircraft_offers migrate
+
+docker rm -f test-db
+docker network rm test-network

docker-compose.yaml

@@ -18,16 +18,21 @@ services:
       - DB_PW=aircraft_offers
     depends_on:
       - ca-aerooffers-db
+      - flyway
 
   ca-aerooffers-db:
     image: postgres:15-alpine
-    restart: always
     environment:
       - POSTGRES_DB=aircraft_offers
       - POSTGRES_USER=aircraft_offers
       - POSTGRES_PASSWORD=aircraft_offers
     ports:
       - '25432:5432'
+
+  flyway:
+    image: flyway/flyway
+    command: -url=jdbc:postgresql://ca-aerooffers-db:5432/aircraft_offers -user=aircraft_offers -password=aircraft_offers migrate
     volumes:
-      - ./db/ddl.sql:/docker-entrypoint-initdb.d/ddl.sql
-      #- ./db/prod_dump_2024_06_31.sql:/docker-entrypoint-initdb.d/prod_dump_2024_06_31.sql
+      - ./db/migrations:/flyway/sql
+    depends_on:
+      - ca-aerooffers-db