diff --git a/README.md b/README.md index 070671a..aadedcb 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,9 @@ This project aims at reviving www.aero-offers.com - invaluable source of price t - `job_update_exchange_rates` - updates currency exchange rates based ok ECP api - `db` - PostgreSQL 15 database with DDL scripts managed by Flyway +### Prod environment +Currently the project is being onboarded to Azure Cloud (still WIP). + ### TODO - [x] deploy working ui, api and db to Azure - [x] fix segelflug spider/crawler @@ -29,6 +32,7 @@ This project aims at reviving www.aero-offers.com - invaluable source of price t - [ ] fix aircraft type dropdown - [ ] fix & polish CSS in UI - [ ] update/simplify legal subpage +- [ ] use https://github.com/weglide/GliderList ### Running locally without Python nor NodeJS `docker compose up --build` - starts postgres, python backend and frontend apps (http://localhost:8080/) diff --git a/ansible/aero_offers.cfg b/ansible/aero_offers.cfg deleted file mode 100644 index e69de29..0000000 diff --git a/ansible/deploy.yml b/ansible/deploy.yml deleted file mode 100644 index 6fbf684..0000000 --- a/ansible/deploy.yml +++ /dev/null @@ -1,148 +0,0 @@ -- name: "Deploy Backend & Frontend" - hosts: all - vars: - code_dir: "/usr/share/AircraftOffers" - tasks: - - name: "Build Frontend for Production (locally)" - shell: | - cd ../frontend - NODE_ENV="production" - npm run build - delegate_to: localhost - - - name: "Stop uwsgi service (if existing)" - systemd: - state: stopped - name: uwsgi - become_user: root - become_method: sudo - become: yes - ignore_errors: true - - - name: "Stop nginx" - systemd: - name: nginx - state: stopped - become: true - become_method: sudo - become_user: root - - - name: "Remove existing Code directory" - file: - path: "{{ code_dir }}" - state: absent - become_user: root - become_method: sudo - become: yes - - - name: "Create directory for sourcecode" - file: - path: "{{ code_dir }}" - state: directory - owner: ec2-user - become_user: root - become_method: sudo - become: yes - - - name: "Synchronize Code (using rsync)" - synchronize: - src: ../../ - dest: "{{ code_dir }}" - - - name: "Install settings for production environment" - copy: - src: "{{ code_dir }}/settings_prod.py" - dest: "{{ code_dir }}/settings.py" - remote_src: yes - - - name: "Install python 3 packages" - pip: - requirements: "{{ code_dir }}/requirements.txt" - executable: pip3 - become_user: root - become_method: sudo - become: yes - - - name: "Ensure old crontab entries are removed" - ansible.builtin.cron: - name: "{{ item }}" - state: absent - with_items: ["run crawler", "fetch exchange rates"] - - - name: "Ensure Crawler & Classifier is run every day at 5:00 UTC" - cron: - name: "run crawler" - minute: "0" - hour: "5" - job: "cd {{ code_dir }} && python3 job_fetch_offers.py && python3 job_reclassify_offers.py" - - - name: "Ensure Exchange rates are fetched from ECB every day at 16:00 UTC (they are updated usually at 16:00 CET)" - cron: - name: "fetch exchange rates" - minute: "0" - hour: "16" - job: "cd {{ code_dir }} && python3 job_update_exchange_rates.py" - - - name: "Change user for frontend dist to nginx" - file: - path: "{{ code_dir }}/frontend/dist" - state: directory - recurse: yes - owner: nginx - group: nginx - mode: '770' - become_user: root - become_method: sudo - become: yes - - - name: "Make web directory readable for uwsgi user" - file: - path: "{{ code_dir }}/web" - state: directory - recurse: yes - owner: uwsgi - group: nginx - mode: '770' - become_user: root - become_method: sudo - become: yes - - - name: "Install uwsgi service to serve the API" - copy: - src: "{{ code_dir }}/ansible/uwsgi.service" - dest: /etc/systemd/system/ - remote_src: yes - become_user: root - become_method: sudo - become: yes - - - name: "Make sure the socket location directory is present (/run/aero-offers/)" - file: - path: /run/aero-offers - state: directory - owner: uwsgi - group: nginx - - - name: "Make sure web directory can be read by uwsgi" - ansible.builtin.file: - path: "{{ code_dir }}/web" - state: directory - recurse: yes - owner: uwsgi - group: nginx - - - name: "Start uwsgi service to serve API" - systemd: - state: started - name: uwsgi - become_user: root - become_method: sudo - become: yes - - - name: "Reload nginx" - systemd: - name: nginx - state: reloaded - become: true - become_method: sudo - become_user: root \ No newline at end of file diff --git a/ansible/hosts b/ansible/hosts deleted file mode 100644 index 6b66c19..0000000 --- a/ansible/hosts +++ /dev/null @@ -1,8 +0,0 @@ -[backend] -some_ec2_instance.compute.amazonaws.com ansible_user=ec2-user ansible_ssh_private_key_file=~/aircraft-offers-stockholm.pem - -[backend:vars] -code_dir = /usr/share/AircraftOffers - -[database] -some_ec2_instance.compute.amazonaws.com ansible_user=ec2-user ansible_ssh_private_key_file=~/aircraft-offers-stockholm.pem \ No newline at end of file diff --git a/ansible/roles/backend/tasks/main.yml b/ansible/roles/backend/tasks/main.yml deleted file mode 100644 index d7760ae..0000000 --- a/ansible/roles/backend/tasks/main.yml +++ /dev/null @@ -1,23 +0,0 @@ -- name: "python3, gcc, etc (needed for uwsgi)" - ansible.builtin.dnf: - name: - - "python3" - - "python3-devel" - - "python3-pip" - - "gcc" - - "gcc-c++" - - "make" - - "cronie" - state: "latest" - become_user: root - become_method: sudo - become: yes - -- name: "Create directory for sourcecode" - file: - path: "{{ code_dir }}" - state: directory - owner: ec2-user - become_user: root - become_method: sudo - become: yes diff --git a/ansible/roles/db/files/pg_hba.conf b/ansible/roles/db/files/pg_hba.conf deleted file mode 100644 index 9d9c6b6..0000000 --- a/ansible/roles/db/files/pg_hba.conf +++ /dev/null @@ -1,97 +0,0 @@ -# PostgreSQL Client Authentication Configuration File -# =================================================== -# -# Refer to the "Client Authentication" section in the PostgreSQL -# documentation for a complete description of this file. A short -# synopsis follows. -# -# This file controls: which hosts are allowed to connect, how clients -# are authenticated, which PostgreSQL user names they can use, which -# databases they can access. Records take one of these forms: -# -# local DATABASE USER METHOD [OPTIONS] -# host DATABASE USER ADDRESS METHOD [OPTIONS] -# hostssl DATABASE USER ADDRESS METHOD [OPTIONS] -# hostnossl DATABASE USER ADDRESS METHOD [OPTIONS] -# hostgssenc DATABASE USER ADDRESS METHOD [OPTIONS] -# hostnogssenc DATABASE USER ADDRESS METHOD [OPTIONS] -# -# (The uppercase items must be replaced by actual values.) -# -# The first field is the connection type: -# - "local" is a Unix-domain socket -# - "host" is a TCP/IP socket (encrypted or not) -# - "hostssl" is a TCP/IP socket that is SSL-encrypted -# - "hostnossl" is a TCP/IP socket that is not SSL-encrypted -# - "hostgssenc" is a TCP/IP socket that is GSSAPI-encrypted -# - "hostnogssenc" is a TCP/IP socket that is not GSSAPI-encrypted -# -# DATABASE can be "all", "sameuser", "samerole", "replication", a -# database name, or a comma-separated list thereof. The "all" -# keyword does not match "replication". Access to replication -# must be enabled in a separate record (see example below). -# -# USER can be "all", a user name, a group name prefixed with "+", or a -# comma-separated list thereof. In both the DATABASE and USER fields -# you can also write a file name prefixed with "@" to include names -# from a separate file. -# -# ADDRESS specifies the set of hosts the record matches. It can be a -# host name, or it is made up of an IP address and a CIDR mask that is -# an integer (between 0 and 32 (IPv4) or 128 (IPv6) inclusive) that -# specifies the number of significant bits in the mask. A host name -# that starts with a dot (.) matches a suffix of the actual host name. -# Alternatively, you can write an IP address and netmask in separate -# columns to specify the set of hosts. Instead of a CIDR-address, you -# can write "samehost" to match any of the server's own IP addresses, -# or "samenet" to match any address in any subnet that the server is -# directly connected to. -# -# METHOD can be "trust", "reject", "md5", "password", "scram-sha-256", -# "gss", "sspi", "ident", "peer", "pam", "ldap", "radius" or "cert". -# Note that "password" sends passwords in clear text; "md5" or -# "scram-sha-256" are preferred since they send encrypted passwords. -# -# OPTIONS are a set of options for the authentication in the format -# NAME=VALUE. The available options depend on the different -# authentication methods -- refer to the "Client Authentication" -# section in the documentation for a list of which options are -# available for which authentication methods. -# -# Database and user names containing spaces, commas, quotes and other -# special characters must be quoted. Quoting one of the keywords -# "all", "sameuser", "samerole" or "replication" makes the name lose -# its special character, and just match a database or username with -# that name. -# -# This file is read on server startup and when the server receives a -# SIGHUP signal. If you edit the file on a running system, you have to -# SIGHUP the server for the changes to take effect, run "pg_ctl reload", -# or execute "SELECT pg_reload_conf()". -# -# Put your actual configuration here -# ---------------------------------- -# -# If you want to allow non-local connections, you need to add more -# "host" records. In that case you will also need to make PostgreSQL -# listen on a non-local interface via the listen_addresses -# configuration parameter, or via the -i or -h command line switches. - - - -# TYPE DATABASE USER ADDRESS METHOD - -# "local" is for Unix domain socket connections only -local all all peer -# IPv4 local connections: -host all postgres 127.0.0.1/32 ident -# IPv6 local connections: -host all postgres ::1/128 ident -# aero-offers -host aircraft_offers aircraftoffers 127.0.0.1/32 password -host aircraft_offers aircraftoffers ::1/128 password -# Allow replication connections from localhost, by a user with the -# replication privilege. -#local replication all peer -#host replication all 127.0.0.1/32 ident -#host replication all ::1/128 ident \ No newline at end of file diff --git a/ansible/roles/db/tasks/main.yml b/ansible/roles/db/tasks/main.yml deleted file mode 100644 index 50da10d..0000000 --- a/ansible/roles/db/tasks/main.yml +++ /dev/null @@ -1,66 +0,0 @@ ---- -# This playbook will install PostgreSQL and create db user and give permissions. -- name: Install PostgreSQL package - ansible.builtin.dnf: - name: - - postgresql15 - - postgresql15-server - - python3-psycopg2 - state: latest - -- name: Find out if PostgreSQL is initialized - ansible.builtin.stat: - path: "/var/lib/pgsql/data/pg_hba.conf" - register: postgres_data - -- name: Initialize PostgreSQL - shell: "postgresql-setup initdb" - when: not postgres_data.stat.exists - -- name: Update pg_hba.conf - copy: - src: pg_hba.conf - dest: /var/lib/pgsql/data - owner: postgres - group: postgres - mode: 0600 - -- name: Start PostgreSQL Service - service: - name: postgresql - state: started - enabled: yes - -- name: create postgresql user - postgresql_user: - user: aircraftoffers - # put your database password here... - password: "redacted" - expires: infinity - become: true - become_user: postgres - -- name: create postgresql db - postgresql_db: - name: aircraft_offers - owner: aircraftoffers - state: present - become: true - become_user: postgres - -- name: GRANT ALL to aircraftoffers - community.postgresql.postgresql_privs: - type: database - db: aircraft_offers - state: present - privs: ALL - role: aircraftoffers - become: true - become_user: postgres - -- name: Reload PostgreSQL service - service: - name: postgresql - state: restarted - enabled: yes - become: true \ No newline at end of file diff --git a/ansible/roles/nginx/files/nginx.conf b/ansible/roles/nginx/files/nginx.conf deleted file mode 100644 index 6453597..0000000 --- a/ansible/roles/nginx/files/nginx.conf +++ /dev/null @@ -1,101 +0,0 @@ -# For more information on configuration, see: -# * Official English Documentation: http://nginx.org/en/docs/ -# * Official Russian Documentation: http://nginx.org/ru/docs/ - -user nginx; -worker_processes auto; -error_log /var/log/nginx/error.log; -pid /run/nginx.pid; - -# Load dynamic modules. See /usr/share/doc/nginx/README.dynamic. -include /usr/share/nginx/modules/*.conf; - -events { - worker_connections 1024; -} - -http { - log_format main '$remote_addr - $remote_user [$time_local] "$request" ' - '$status $body_bytes_sent "$http_referer" ' - '"$http_user_agent" "$http_x_forwarded_for"'; - - access_log /var/log/nginx/access.log main; - - sendfile on; - tcp_nopush on; - tcp_nodelay on; - keepalive_timeout 65; - types_hash_max_size 2048; - - include /etc/nginx/mime.types; - default_type application/octet-stream; - - # Load modular configuration files from the /etc/nginx/conf.d directory. - # See http://nginx.org/en/docs/ngx_core_module.html#include - # for more information. - include /etc/nginx/conf.d/*.conf; - - server { - listen 80 default_server; - listen [::]:80 default_server; - server_name _; - root /usr/share/AircraftOffers/frontend/dist; - - # Load configuration files for the default server block. - include /etc/nginx/default.d/*.conf; - - location / { - try_files $uri $uri/ @rewrites; - } - - location @rewrites { - rewrite ^(.+)$ /index.html last; - } - - location /api { - include uwsgi_params; - rewrite /api/(.+) /$1 break; # remove /api - uwsgi_pass unix:/run/aero-offers/api.sock; - } - - error_page 404 /404.html; - location = /40x.html { - } - - error_page 500 502 503 504 /50x.html; - location = /50x.html { - } - } - -# Settings for a TLS enabled server. -# -# server { -# listen 443 ssl http2 default_server; -# listen [::]:443 ssl http2 default_server; -# server_name _; -# root /usr/share/nginx/html; -# -# ssl_certificate "/etc/pki/nginx/server.crt"; -# ssl_certificate_key "/etc/pki/nginx/private/server.key"; -# ssl_session_cache shared:SSL:1m; -# ssl_session_timeout 10m; -# ssl_ciphers HIGH:!aNULL:!MD5; -# ssl_prefer_server_ciphers on; -# -# # Load configuration files for the default server block. -# include /etc/nginx/default.d/*.conf; -# -# location / { -# } -# -# error_page 404 /404.html; -# location = /40x.html { -# } -# -# error_page 500 502 503 504 /50x.html; -# location = /50x.html { -# } -# } - -} - diff --git a/ansible/roles/nginx/tasks/main.yml b/ansible/roles/nginx/tasks/main.yml deleted file mode 100644 index 4bb3d3a..0000000 --- a/ansible/roles/nginx/tasks/main.yml +++ /dev/null @@ -1,53 +0,0 @@ ---- -- name: "Install nginx" - ansible.builtin.dnf: - name: - - "nginx" - state: latest - become_user: root - become_method: sudo - become: yes - -- name: "Copy nginx config file" - copy: - src: nginx.conf - dest: /etc/nginx/nginx.conf - become_user: root - become_method: sudo - become: yes - -- name: "Create User uwsgi:nginx for running uwsgi" - user: - name: uwsgi - comment: technical user for uwsgi - group: nginx - become_user: root - become_method: sudo - become: yes - -- name: "Make sure also ec2-user is part of nginx group" - user: - name: ec2-user - group: nginx - append: yes - become_user: root - become_method: sudo - become: yes - -- name: "Create directory for unix file socket (uwsgi)" - file: - path: "/run/aero-offers" - owner: uwsgi - group: nginx - state: directory - become_user: root - become_method: sudo - become: yes - -- name: "Reload configuration" - systemd: - name: nginx - state: reloaded - become: true - become_method: sudo - become_user: root \ No newline at end of file diff --git a/ansible/setup_db.yml b/ansible/setup_db.yml deleted file mode 100644 index b67e75e..0000000 --- a/ansible/setup_db.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -- hosts: database - gather_facts: yes - become: yes - roles: - - db \ No newline at end of file diff --git a/ansible/setup_vm.yml b/ansible/setup_vm.yml deleted file mode 100644 index b664405..0000000 --- a/ansible/setup_vm.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- -- hosts: backend - gather_facts: yes - become: yes - roles: - - nginx - - backend \ No newline at end of file diff --git a/ansible/uwsgi.service b/ansible/uwsgi.service deleted file mode 100644 index e2bcba7..0000000 --- a/ansible/uwsgi.service +++ /dev/null @@ -1,30 +0,0 @@ -# /etc/systemd/system/pyramid.service - -[Unit] -Description=Aero-Offers API - -# Requirements -Requires=network.target - -# Dependency ordering -After=network.target - -[Service] -TimeoutStartSec=0 -RestartSec=10 -Restart=always - -# path to app -WorkingDirectory=/usr/share/AircraftOffers/web -# the user that you want to run app by -User=uwsgi - -KillSignal=SIGQUIT -Type=notify -NotifyAccess=all - -# Main process -ExecStart=/usr/local/bin/uwsgi --ini uwsgi.ini - -[Install] -WantedBy=multi-user.target \ No newline at end of file