From 9a8b5fa621203107b82ac85332f301f63a11a1b6 Mon Sep 17 00:00:00 2001 From: Marek Susicky Date: Sat, 16 Jan 2021 18:46:50 +0100 Subject: [PATCH 01/69] Init version of the app - not working --- README.md | 20 +++++++- __init__.py | 0 app.py | 49 +++++++++++++++++++ config.py | 13 +++++ create_db.sql | 93 ++++++++++++++++++++++++++++++++++++ models.py | 21 ++++++++ requirements.txt | 8 ++++ setup.py | 28 +++++++++++ static/css/styles.css | 8 ++++ static/js/functions.js | 0 templates/base.html | 60 +++++++++++++++++++++++ templates/index.html | 31 ++++++++++++ templates/ockovani_info.html | 55 +++++++++++++++++++++ views.py | 40 ++++++++++++++++ 14 files changed, 425 insertions(+), 1 deletion(-) create mode 100644 __init__.py create mode 100644 app.py create mode 100644 config.py create mode 100644 create_db.sql create mode 100644 models.py create mode 100644 requirements.txt create mode 100644 setup.py create mode 100644 static/css/styles.css create mode 100644 static/js/functions.js create mode 100644 templates/base.html create mode 100644 templates/index.html create mode 100644 templates/ockovani_info.html create mode 100644 views.py diff --git a/README.md b/README.md index f39d4943..dd4c21d6 100644 --- a/README.md +++ b/README.md @@ -1 +1,19 @@ -# ockovani-covid \ No newline at end of file +# Volná místa na COVID-19 očkování + +## Instalace + +### Virtual environment + +#### Create venv +python3 -m venv venv + +#### Activate venv +source venv/bin/activate + +#### Install requirements +pip install -r requirements.txt + +### +export FLASK_APP=app.py +export FLASK_ENV=development +flask run diff --git a/__init__.py b/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app.py b/app.py new file mode 100644 index 00000000..523f4b70 --- /dev/null +++ b/app.py @@ -0,0 +1,49 @@ +from flask import Flask, g +from models import db +from views import * +import logging +from logging.handlers import RotatingFileHandler + +app = Flask(__name__) +app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False +app.config['SQLALCHEMY_ECHO'] = True + +some_engine = create_engine('postgresql://ockovani:ockovani2021@localhost:5432/ockovani') +Session = sessionmaker(bind=some_engine) +app.session = Session() +app.logger.info('Connection set up') + +@app.before_request +def before_request(): + g.user = None + +def create_app(config): + """Construct the core application.""" + app.config.from_object(config) + db.init_app(app) + with app.app_context(): + a = 0 + app.logger.info(app.config['SQLALCHEMY_DATABASE_URI']) + app.session=db.session + return app + + +def register_blueprints(app): + app.register_blueprint(my_view) + + +if __name__ == '__main__': + app = create_app('config.myConfig') + handler = RotatingFileHandler('ockovani.log', maxBytes=10000, backupCount=1) + handler.setLevel(logging.DEBUG) + + # Logging of SQL queries + #logging.basicConfig() + #logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO) + + handler.setFormatter(logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')) + app.logger.addHandler(handler) + #app.logger.setLevel(logging.DEBUG) + register_blueprints(app) + + app.run(debug=True, threaded=True, port=5678, host='127.0.0.1') diff --git a/config.py b/config.py new file mode 100644 index 00000000..3a9f58f9 --- /dev/null +++ b/config.py @@ -0,0 +1,13 @@ +from flask import Flask, g + +class Config: + SQLALCHEMY_DATABASE_URI = 'sqlite:///database.db' + SEND_FILE_MAX_AGE_DEFAULT = 300 + +class TestConfig(Config): + SQLALCHEMY_DATABASE_URI = 'sqlite:///:memory:' + +class myConfig: + FLASK_APP = 'app.py' + FLASK_DEBUG = 1 + SQLALCHEMY_DATABASE_URI = 'postgresql://ockovani:foaeiwfjewfoij@localhost:5432/ockovani' diff --git a/create_db.sql b/create_db.sql new file mode 100644 index 00000000..3a0a72c7 --- /dev/null +++ b/create_db.sql @@ -0,0 +1,93 @@ +CREATE ROLE ockovani WITH + LOGIN + NOSUPERUSER + NOCREATEDB + NOCREATEROLE + INHERIT + NOREPLICATION + CONNECTION LIMIT -1 + PASSWORD 'xxxxxx'; + +CREATE DATABASE ockovani + WITH + OWNER = ockovani + ENCODING = 'UTF8' + CONNECTION LIMIT = -1; + +-- Table: public.dny + +-- DROP TABLE public.dny; + +CREATE TABLE public.dny +( + den_id serial NOT NULL, + datum date NOT NULL, + CONSTRAINT dny_pkey PRIMARY KEY (den_id) +) + +TABLESPACE pg_default; + +ALTER TABLE public.dny + OWNER to ockovani; + +COMMENT ON TABLE public.dny + IS 'Tabulka s jednotlivymi dny ny ockovani'; + +-- Table: public.import_log + +-- DROP TABLE public.import_log; + +CREATE TABLE public.import_log +( + import_id bigint NOT NULL DEFAULT nextval('import_log_import_id_seq'::regclass), + spusteni timestamp without time zone NOT NULL, + CONSTRAINT import_log_pkey PRIMARY KEY (import_id) +) + +TABLESPACE pg_default; + +ALTER TABLE public.import_log + OWNER to ockovani; + +-- Table: public.kapacita + +-- DROP TABLE public.kapacita; + +CREATE TABLE public.kapacita +( + misto_id bigint NOT NULL, + datum date NOT NULL, + raw_data json, + pocet_mist integer, + datum_ziskani timestamp without time zone, + import_id bigint NOT NULL, + CONSTRAINT kapacita_pkey PRIMARY KEY (misto_id, import_id, datum) +) PARTITION BY LIST (import_id); + +ALTER TABLE public.kapacita + OWNER to ockovani; + +COMMENT ON TABLE public.kapacita + IS 'Tabulka se zjistenymi kapacitami'; + +-- Table: public.ockovaci_misto + +-- DROP TABLE public.ockovaci_misto; + +CREATE TABLE public.ockovaci_misto +( + misto_id bigserial NOT NULL, + nazev character varying COLLATE pg_catalog."default" NOT NULL, + service_id integer NOT NULL, + operation_id integer NOT NULL, + place_id integer NOT NULL, + mesto character varying COLLATE pg_catalog."default", + CONSTRAINT ockovaci_misto_pkey PRIMARY KEY (misto_id) +) + +TABLESPACE pg_default; + +ALTER TABLE public.ockovaci_misto + OWNER to ockovani; + + \ No newline at end of file diff --git a/models.py b/models.py new file mode 100644 index 00000000..b5703295 --- /dev/null +++ b/models.py @@ -0,0 +1,21 @@ +from datetime import datetime +from flask_sqlalchemy import SQLAlchemy +from sqlalchemy import Column, Integer, Unicode, DateTime, PrimaryKeyConstraint +from sqlalchemy.orm import sessionmaker + +db = SQLAlchemy() + +class MyModel(db.Model): + id = Column(Integer, primary_key=True) + some_string = Column(Unicode) + some_integer = Column(Integer) + some_time = Column(DateTime, default=datetime.now) + + +class OckovaciMisto(db.Model): + misto_id = Column(Integer, primary_key=True) + nazev = Column(Unicode) + service_id = Column(Integer) + operation_id= Column(Integer) + place_id= Column(Integer) + mesto= Column(Unicode) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..c839f3e4 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,8 @@ +Flask>=1.1.2 +Flask-Restless>=0.16.0 +Flask-SQLAlchemy>=2.0 +SQLAlchemy>=1.0.2 +Werkzeug>=0.16.1 +psycopg2==2.7.7 +markupsafe +requests \ No newline at end of file diff --git a/setup.py b/setup.py new file mode 100644 index 00000000..437b5c89 --- /dev/null +++ b/setup.py @@ -0,0 +1,28 @@ +import pathlib +from setuptools import find_packages, setup + +# The directory containing this file +HERE = pathlib.Path(__file__).parent + +# The text of the README file +README = (HERE / "README.md").read_text() + +# This call to setup() does all the work +setup( + name='ockovani', + version='1.0.0', + description='Ockovani app', + long_description=README, + long_description_content_type="text/markdown", + url='https://www.datazeet.cz', + packages=find_packages(exclude=("tests",)), + install_requires=["Flask>=1.1.2","Flask-Restless>=0.16.0","Flask-SQLAlchemy>=2.0","SQLAlchemy>=1.0.2","Werkzeug>=0.16.1","psycopg2==2.7.7","markupsafe"], + author='msusicky', + author_email='marek@susicky.net', + include_package_data=True, + entry_points={ + "console_scripts": [ + "ockovani=app.__main__:main", + ] + }, +) \ No newline at end of file diff --git a/static/css/styles.css b/static/css/styles.css new file mode 100644 index 00000000..27d32813 --- /dev/null +++ b/static/css/styles.css @@ -0,0 +1,8 @@ +table { + border: 1px solid black; + margin-bottom: 2em; +} + +td { + border: 1px dotted black; +} \ No newline at end of file diff --git a/static/js/functions.js b/static/js/functions.js new file mode 100644 index 00000000..e69de29b diff --git a/templates/base.html b/templates/base.html new file mode 100644 index 00000000..869234de --- /dev/null +++ b/templates/base.html @@ -0,0 +1,60 @@ + + + + + + {# Add a banner if using old IE8 #} + + + + {% block title %}{% endblock %} + + {# All templates which inherit this one will have jQuery #} + + + + + + {% block head %}{% endblock %} + + +{% block navbar %} + +{% endblock %} + +{% block body %}{% endblock %} + + +{% block scripts %} +{% endblock %} +
+
+
16.1.2021
+ + diff --git a/templates/index.html b/templates/index.html new file mode 100644 index 00000000..eb840b6e --- /dev/null +++ b/templates/index.html @@ -0,0 +1,31 @@ +{% extends 'base.html' %} + +{% block title %} + Očkování +{% endblock %} + +{% block head %} + +{% endblock %} + +{% block body %} +

Aplikace s přehledem volných míst na COVID-19 očkování

+ + {% if login %} +

Logged as: {{ login }}

+ {% endif %} + Povídání o aplikaci a co umí. +{% endblock %} + +{% block scripts %} + +{% endblock %} + + +{# +Note: you can also put css/js in 'base.html' if you want it to apply to all templates +which extend base.html (like this one) +#} \ No newline at end of file diff --git a/templates/ockovani_info.html b/templates/ockovani_info.html new file mode 100644 index 00000000..d6848a65 --- /dev/null +++ b/templates/ockovani_info.html @@ -0,0 +1,55 @@ +{% extends 'base.html' %} + +{% block title %} + Očkování Flask App +{% endblock %} + +{% block head %} + +{% endblock %} + +{% block body %} +

Informace o volných kapacitách

+ + {% if volne_terminy %} + {{ volne_terminy }} + {% endif %} + + {% if data %} +
+ + + + + + + + {% for item in data %} + + + + + {% endfor %} +
Table header
{{ item.item }}
+
+ {% else %} + No Data! + + {% endif %} + +{% endblock %} + +{% block scripts %} + + +{% endblock %} + + +{# +Note: you can also put css/js in 'base.html' if you want it to apply to all templates +which extend base.html (like this one) +#} + diff --git a/views.py b/views.py new file mode 100644 index 00000000..45c32d7a --- /dev/null +++ b/views.py @@ -0,0 +1,40 @@ +from flask import Blueprint, render_template, g, session, redirect, url_for, request +from sqlalchemy import * +from sqlalchemy import text +from sqlalchemy.orm import sessionmaker +from markupsafe import escape +import requests + +from app import app +from models import OckovaciMisto + +my_view = Blueprint('my_view', __name__, template_folder="templates") + +@my_view.route('/') +def index(): + app.logger.info('index') + all_data = None + return render_template('index.html', data=all_data) + +#@my_view.route("/ockovaci_mista/") +#def ockovaci_mista(mesto): + """ + It lists only specified abstract roles. + :param ar_id: + :return: + """ +# nactene_informace = app.session.query(OckovaciMista).from_statement(text("SELECT ... FROM ... where ar_id=:mesto_param")).params(mesto_param=mesto).all() +# return render_template('abstract_komponent_roles.html', data=nactene_informace) + +@my_view.route("/info") +def info(): + ockovani_info=app.session.query(OckovaciMisto).from_statement(text("select * from ockovaci_misto")).all() + + #my small test of API + url = 'https://reservatic.com/public_services/411747/public_operations/5971/hours?date=2021-4-09' + data = '' + response = requests.get(url, data=data, headers={"Content-Type": "application/json"}) + #v response mam volne terminy :) + print(response) + + return render_template('ockovani_info.html', data=ockovani_info, volne_terminy=response.text) From fe0263067dc5f4dc0fb72d5c947c35763a3abe11 Mon Sep 17 00:00:00 2001 From: Marek Susicky Date: Sat, 16 Jan 2021 21:46:20 +0100 Subject: [PATCH 02/69] Minor changes --- create_db.sql | 2 +- models.py | 9 ++++++ templates/misto.html | 55 ++++++++++++++++++++++++++++++++++++ templates/ockovani_info.html | 12 ++++---- views.py | 27 ++++++++++-------- 5 files changed, 86 insertions(+), 19 deletions(-) create mode 100644 templates/misto.html diff --git a/create_db.sql b/create_db.sql index 3a0a72c7..9f04c868 100644 --- a/create_db.sql +++ b/create_db.sql @@ -62,7 +62,7 @@ CREATE TABLE public.kapacita datum_ziskani timestamp without time zone, import_id bigint NOT NULL, CONSTRAINT kapacita_pkey PRIMARY KEY (misto_id, import_id, datum) -) PARTITION BY LIST (import_id); +); ALTER TABLE public.kapacita OWNER to ockovani; diff --git a/models.py b/models.py index b5703295..9b5e8fcb 100644 --- a/models.py +++ b/models.py @@ -19,3 +19,12 @@ class OckovaciMisto(db.Model): operation_id= Column(Integer) place_id= Column(Integer) mesto= Column(Unicode) + +class VolnaMistaQuery(db.Model): + query = Column(Unicode, primary_key=True) + +class OckovaciKapacity(db.Model): + misto_id = Column(Integer, primary_key=True) + mesto = Column(Unicode) + datum = Column(DateTime) + kapacita= Column(Integer) diff --git a/templates/misto.html b/templates/misto.html new file mode 100644 index 00000000..06ac1fee --- /dev/null +++ b/templates/misto.html @@ -0,0 +1,55 @@ +{% extends 'base.html' %} + +{% block title %} + Očkování Flask App +{% endblock %} + +{% block head %} + +{% endblock %} + +{% block body %} +

Informace o volných kapacitách místa {{ misto }}

+ + {% if data %} +
+ + + + + + + + + + {% for item in data %} + + + + + + + {% endfor %} +
NázevMěstoDatumKapacita
{{ item.nazev }}{{ item.mesto }}{{ item.datum }}{{ item.kapacita }}
+
+ {% else %} + Žádná data! + + {% endif %} + +{% endblock %} + +{% block scripts %} + + +{% endblock %} + + +{# +Note: you can also put css/js in 'base.html' if you want it to apply to all templates +which extend base.html (like this one) +#} + diff --git a/templates/ockovani_info.html b/templates/ockovani_info.html index d6848a65..e494ade0 100644 --- a/templates/ockovani_info.html +++ b/templates/ockovani_info.html @@ -15,19 +15,19 @@

Informace o volných kapacitách

{{ volne_terminy }} {% endif %} - {% if data %} + {% if ockovaci_mista %}
- - + + - {% for item in data %} + {% for item in ockovaci_mista %} - - + + {% endfor %}
Table headerNázevMěsto
{{ item.item }}{{ item.nazev }}{{ item.mesto }}
diff --git a/views.py b/views.py index 45c32d7a..a8765cfd 100644 --- a/views.py +++ b/views.py @@ -6,7 +6,7 @@ import requests from app import app -from models import OckovaciMisto +from models import OckovaciMisto, VolnaMistaQuery, OckovaciKapacity my_view = Blueprint('my_view', __name__, template_folder="templates") @@ -16,19 +16,19 @@ def index(): all_data = None return render_template('index.html', data=all_data) -#@my_view.route("/ockovaci_mista/") -#def ockovaci_mista(mesto): - """ - It lists only specified abstract roles. - :param ar_id: - :return: - """ -# nactene_informace = app.session.query(OckovaciMista).from_statement(text("SELECT ... FROM ... where ar_id=:mesto_param")).params(mesto_param=mesto).all() -# return render_template('abstract_komponent_roles.html', data=nactene_informace) +@my_view.route("/mesto/") +def info_mesto(mesto): + nactene_informace = app.session.query(OckovaciKapacity).from_statement(text("SELECT m.mesto, m.nazev, k.* FROM ockovaci_misto m JOIN kapacita k ON (m.misto_id=k.misto_id) where m.mesto=:mesto_param")).params(mesto_param=mesto).all() + return render_template('mesto.html', data=nactene_informace, mesto=mesto) + +@my_view.route("/misto/") +def info_misto(misto): + nactene_informace = app.session.query(OckovaciKapacity).from_statement(text("SELECT m.mesto, m.nazev, k.* FROM ockovaci_misto m JOIN kapacita k ON (m.misto_id=k.misto_id) where m.nazev=:misto_param")).params(misto_param=misto).all() + return render_template('misto.html', data=nactene_informace, misto=misto) @my_view.route("/info") def info(): - ockovani_info=app.session.query(OckovaciMisto).from_statement(text("select * from ockovaci_misto")).all() + ockovani_info=app.session.query(OckovaciMisto).from_statement(text("select * from public.ockovaci_misto")).all() #my small test of API url = 'https://reservatic.com/public_services/411747/public_operations/5971/hours?date=2021-4-09' @@ -37,4 +37,7 @@ def info(): #v response mam volne terminy :) print(response) - return render_template('ockovani_info.html', data=ockovani_info, volne_terminy=response.text) + #ockovani_queries = app.session.query(VolnaMistaQuery).from_statement(text("select \'https://reservatic.com/public_services/\' | | service_id | | \'/public_operations/\' | | operation_id | | \'/hours?date=\' | | d.datum from ockovaci_misto om cross join dny d")).all() + #print(ockovani_queries) + + return render_template('ockovani_info.html', ockovaci_mista=ockovani_info, volne_terminy=response.text) From 8af251671ebb3604ad3d67e6c54a085bc0108a49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Stan=C4=9Bk?= Date: Sat, 16 Jan 2021 22:51:31 +0100 Subject: [PATCH 03/69] watchdog scrapper --- tools/.gitignore | 1 + tools/scrapper_watch_dog.py | 56 +++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) create mode 100644 tools/.gitignore create mode 100644 tools/scrapper_watch_dog.py diff --git a/tools/.gitignore b/tools/.gitignore new file mode 100644 index 00000000..51fd01e7 --- /dev/null +++ b/tools/.gitignore @@ -0,0 +1 @@ +out.csv \ No newline at end of file diff --git a/tools/scrapper_watch_dog.py b/tools/scrapper_watch_dog.py new file mode 100644 index 00000000..8fbe44ef --- /dev/null +++ b/tools/scrapper_watch_dog.py @@ -0,0 +1,56 @@ +import time +import requests +from bs4 import BeautifulSoup + + +def _login(email, password): + authentication_url = 'https://reservatic.com/cs/users/sign_in' + response = requests.get(authentication_url) + soup = BeautifulSoup(response.text) + token = soup.find('input', {'name': 'authenticity_token'}).get('value') + + data = { + 'authenticity_token': token, + 'user[email]': email, + 'user[password]': password, + 'commit': 'Přihlásit' + } + + session = requests.session() + session.post(authentication_url, data=data) + return session + + +def _parse(session, file, start: int, stop: int): + base_url = 'https://reservatic.com/watchdogs/new?locale=cs&operation_id=5867&place_id=2905&service_id=' + + for i in range(start, stop): + service_url = base_url + str(i) + soup = _open_url(session, service_url) + label = soup.find('label', {'for': 'watchdog_service_id'}) + if label: + firma = str(label.next_sibling.next_sibling.next_sibling).strip() + _write(file, firma, i) + + +def _open_url(session, url: str): + # workaround login + session = requests.session() + session.cookies.set('_reservatic_session2020_a', '') + # workaround login + + response = session.get(url) + soup = BeautifulSoup(response.text) + time.sleep(1) + return soup + + +def _write(file, firma, service_id): + file.write(firma + ';' + str(service_id) + '\n') + + +if __name__ == "__main__": + session = _login('', '') # not needed to fill, login doesn't work - use session from your web browser + file = open("out.csv", "a", encoding="utf-8") + _parse(session, file, 411400, 412100) + file.close() \ No newline at end of file From a91c88f0b59b7371843536a76a7bec597e47e0b8 Mon Sep 17 00:00:00 2001 From: vaclavpokorny Date: Sun, 17 Jan 2021 01:27:45 +0100 Subject: [PATCH 04/69] Restyle a bit and make working obtaining of data from database. --- models.py | 10 +++++--- templates/base.html | 6 ++--- templates/mesto.html | 57 ++++++++++++++++++++++++++++++++++++++++++++ templates/misto.html | 55 ++++++++++++++++++++++-------------------- views.py | 16 +++++++++++-- 5 files changed, 110 insertions(+), 34 deletions(-) create mode 100644 templates/mesto.html diff --git a/models.py b/models.py index 9b5e8fcb..44ac1c71 100644 --- a/models.py +++ b/models.py @@ -24,7 +24,11 @@ class VolnaMistaQuery(db.Model): query = Column(Unicode, primary_key=True) class OckovaciKapacity(db.Model): - misto_id = Column(Integer, primary_key=True) mesto = Column(Unicode) - datum = Column(DateTime) - kapacita= Column(Integer) + nazev = Column(Unicode) + datum = Column(DateTime, primary_key=True) + pocet_mist = Column(Integer) + misto_id = Column(Integer, primary_key=True) + + def __repr__(self): + return f"{self.mesto} - {self.nazev}:{self.misto_id} - {self.datum}: {self.pocet_mist}" diff --git a/templates/base.html b/templates/base.html index 869234de..457a629a 100644 --- a/templates/base.html +++ b/templates/base.html @@ -53,8 +53,8 @@ {% block scripts %} {% endblock %} -
-
-
16.1.2021
+
+ 16.1.2021 +
diff --git a/templates/mesto.html b/templates/mesto.html new file mode 100644 index 00000000..55b52fce --- /dev/null +++ b/templates/mesto.html @@ -0,0 +1,57 @@ +{% extends 'base.html' %} + +{% block title %} + Očkování Flask App +{% endblock %} + +{% block head %} + +{% endblock %} + +{% block body %} +
+
+

Informace o volných kapacitách místa {{ mesto }}

+ + {% if data %} + + + + + + + + + + + {% for item in data %} + + + + + + + {% endfor %} + +
NázevMěstoDatumKapacita
{{ item.nazev }}{{ item.mesto }}{{ item.datum }}{{ item.pocet_mist }}
+ {% else %} + Žádná data! + {% endif %} +
+
+{% endblock %} + +{% block scripts %} + + +{% endblock %} + + +{# +Note: you can also put css/js in 'base.html' if you want it to apply to all templates +which extend base.html (like this one) +#} + diff --git a/templates/misto.html b/templates/misto.html index 06ac1fee..4b292e92 100644 --- a/templates/misto.html +++ b/templates/misto.html @@ -9,33 +9,36 @@ {% endblock %} {% block body %} -

Informace o volných kapacitách místa {{ misto }}

- - {% if data %} -
- - - - - - - - - - {% for item in data %} - - - - - - - {% endfor %} -
NázevMěstoDatumKapacita
{{ item.nazev }}{{ item.mesto }}{{ item.datum }}{{ item.kapacita }}
+
+
+

Informace o volných kapacitách místa {{ misto }}

+ + {% if data %} + + + + + + + + + + + {% for item in data %} + + + + + + + {% endfor %} + +
NázevMěstoDatumKapacita
{{ item.nazev }}{{ item.mesto }}{{ item.datum }}{{ item.pocet_mist }}
+ {% else %} + Žádná data! + {% endif %}
- {% else %} - Žádná data! - - {% endif %} +
{% endblock %} diff --git a/views.py b/views.py index a8765cfd..f2e53864 100644 --- a/views.py +++ b/views.py @@ -18,12 +18,24 @@ def index(): @my_view.route("/mesto/") def info_mesto(mesto): - nactene_informace = app.session.query(OckovaciKapacity).from_statement(text("SELECT m.mesto, m.nazev, k.* FROM ockovaci_misto m JOIN kapacita k ON (m.misto_id=k.misto_id) where m.mesto=:mesto_param")).params(mesto_param=mesto).all() + nactene_informace = app.session.query(OckovaciKapacity).from_statement( + text( + "SELECT m.mesto, m.nazev, k.datum, k.pocet_mist, k.misto_id FROM ockovaci_misto m " + "JOIN kapacita k ON (m.misto_id=k.misto_id) " + "WHERE m.mesto=:mesto_param" + ) + ).params(mesto_param=mesto).all() return render_template('mesto.html', data=nactene_informace, mesto=mesto) @my_view.route("/misto/") def info_misto(misto): - nactene_informace = app.session.query(OckovaciKapacity).from_statement(text("SELECT m.mesto, m.nazev, k.* FROM ockovaci_misto m JOIN kapacita k ON (m.misto_id=k.misto_id) where m.nazev=:misto_param")).params(misto_param=misto).all() + nactene_informace = app.session.query(OckovaciKapacity).from_statement( + text( + "SELECT m.mesto, m.nazev, k.datum, k.pocet_mist, k.misto_id FROM ockovaci_misto m " + "JOIN kapacita k ON (m.misto_id=k.misto_id) " + "WHERE m.nazev=:misto_param" + ) + ).params(misto_param=misto).all() return render_template('misto.html', data=nactene_informace, misto=misto) @my_view.route("/info") From f20dfc29a68b186ca3e0a244994067c7d84832a2 Mon Sep 17 00:00:00 2001 From: vaclavpokorny Date: Sun, 17 Jan 2021 10:44:15 +0100 Subject: [PATCH 05/69] Unify application style. --- static/css/styles.css | 16 +++++++++++ templates/base.html | 8 +++--- templates/index.html | 14 +++++---- templates/mesto.html | 2 +- templates/misto.html | 2 +- templates/ockovani_info.html | 56 +++++++++++++++++++----------------- views.py | 11 +++---- 7 files changed, 66 insertions(+), 43 deletions(-) diff --git a/static/css/styles.css b/static/css/styles.css index 27d32813..07fb439a 100644 --- a/static/css/styles.css +++ b/static/css/styles.css @@ -5,4 +5,20 @@ table { td { border: 1px dotted black; +} + +.margin-under-navbar { + margin-top: 3rem; +} + +.jumbotron { + background-color: rgba(0,0,0, 0.05); +} + +.navbar-link { + color: ivory; +} + +.footer { + background-color: rgba(0,0,0, 0.07); } \ No newline at end of file diff --git a/templates/base.html b/templates/base.html index 457a629a..0ab2ca28 100644 --- a/templates/base.html +++ b/templates/base.html @@ -29,7 +29,7 @@ {% block navbar %}
diff --git a/templates/misto.html b/templates/misto.html index cdf87748..42bc0fc7 100644 --- a/templates/misto.html +++ b/templates/misto.html @@ -44,7 +44,7 @@

Očkovací místo: {{ misto.nazev }}

{% else %} {% endif %}
From 83dcb6ad46d4dce2f0ab06e181805578512c25fd Mon Sep 17 00:00:00 2001 From: Marek Susicky Date: Tue, 19 Jan 2021 21:03:02 +0100 Subject: [PATCH 34/69] Added disclaimer and fetching the last import_id --- data/migrations/202101191111.sql | 2 + templates/base.html | 67 ++++++++++++++++++++------------ tools/create_static_pages.sh | 2 + tools/run_fetcher.sh | 1 + views.py | 31 +++++++++++---- 5 files changed, 71 insertions(+), 32 deletions(-) create mode 100644 data/migrations/202101191111.sql create mode 100644 tools/create_static_pages.sh create mode 100644 tools/run_fetcher.sh diff --git a/data/migrations/202101191111.sql b/data/migrations/202101191111.sql new file mode 100644 index 00000000..da2e23a2 --- /dev/null +++ b/data/migrations/202101191111.sql @@ -0,0 +1,2 @@ +alter table public.kapacita alter column datum_ziskani set default now(); +ALTER TABLE import_log ADD COLUMN status character varying COLLATE pg_catalog."default" NOT NULL DEFAULT 'N/A'; diff --git a/templates/base.html b/templates/base.html index 949aba84..b4741db9 100644 --- a/templates/base.html +++ b/templates/base.html @@ -2,15 +2,16 @@ - + {# Add a banner if using old IE8 #} @@ -18,11 +19,21 @@ {# All templates which inherit this one will have jQuery #} - - - + + + @@ -32,21 +43,23 @@ {% block navbar %} {% endblock %} @@ -59,7 +72,11 @@ {% block scripts %} {% endblock %}
- Vznik 16. 01. 2021, poslední aktualizace {{ last_update }} + Vznik 16. 01. 2021, poslední aktualizace {{ last_update }}. Data na tomto webu jsou pouze orientační, bez + jakékoliv záruky a nelze je považovat za spolehlivé. Provozovatel se zříká zodpovědnosti za veškeré škody a újmy, + které by použitím těchto dat mohly vzniknout. Web slouží pouze k testování schopností programovacího jazyka Python a + podobnost s reálnými skutečnostmi je čistě náhodná. Pro informace o očkování COVID-19 navštivte web https://koronavirus.mzcr.cz/ockovani-proti-covid-19/
diff --git a/tools/create_static_pages.sh b/tools/create_static_pages.sh new file mode 100644 index 00000000..e82b1fe8 --- /dev/null +++ b/tools/create_static_pages.sh @@ -0,0 +1,2 @@ +wget http://127.0.0.1:5000 +wget http://127.0.0.1:5000/info \ No newline at end of file diff --git a/tools/run_fetcher.sh b/tools/run_fetcher.sh new file mode 100644 index 00000000..8d0d8531 --- /dev/null +++ b/tools/run_fetcher.sh @@ -0,0 +1 @@ +nohup python3 -u ./freespace_fetcher.py >> fetcher.log & \ No newline at end of file diff --git a/views.py b/views.py index 996cdd9f..e9633088 100644 --- a/views.py +++ b/views.py @@ -21,10 +21,11 @@ def info_mesto(mesto): text( "SELECT m.mesto, m.kraj, m.nazev, k.datum, k.pocet_mist, k.misto_id FROM ockovaci_misto m " "JOIN kapacita k ON (m.misto_id=k.misto_id) " - "WHERE m.mesto=:mesto_param " + "WHERE m.mesto=:mesto_param and k.import_id=(SELECT max(import_id) FROM import_log)" "ORDER BY k.datum, m.nazev" ) ).params(mesto_param=mesto).all() + # TODO casem zmenit to max_import_id return render_template('mesto.html', data=nactene_informace, mesto=mesto, last_update=last_update()) @@ -35,7 +36,7 @@ def info_kraj(kraj): text( "SELECT m.mesto, m.kraj, m.nazev, k.datum, k.pocet_mist, k.misto_id FROM ockovaci_misto m " "JOIN kapacita k ON (m.misto_id=k.misto_id) " - "WHERE m.kraj=:kraj_param " + "WHERE m.kraj=:kraj_param and k.import_id=(SELECT max(import_id) FROM import_log)" "ORDER BY k.datum, m.mesto, m.nazev" ) ).params(kraj_param=kraj).all() @@ -49,7 +50,7 @@ def info_misto(misto): text( "SELECT m.mesto, m.kraj, m.nazev, k.datum, k.pocet_mist, k.misto_id FROM ockovaci_misto m " "JOIN kapacita k ON (m.misto_id=k.misto_id) " - "WHERE m.misto_id=:misto_param " + "WHERE m.misto_id=:misto_param and k.import_id=(SELECT max(import_id) FROM import_log)" "ORDER BY k.datum" ) ).params(misto_param=misto).all() @@ -67,12 +68,14 @@ def info_misto(misto): def info(): ockovani_info = app.session.query(OckovaciMisto.misto_id, OckovaciMisto.nazev, OckovaciMisto.service_id, OckovaciMisto.operation_id, OckovaciMisto.place_id, OckovaciMisto.mesto, - OckovaciMisto.kraj, func.max(Kapacita.pocet_mist).label("pocet_mist"))\ - .outerjoin(Kapacita, Kapacita.misto_id == OckovaciMisto.misto_id)\ + OckovaciMisto.kraj, func.max(Kapacita.pocet_mist).label("pocet_mist")) \ + .outerjoin(Kapacita, Kapacita.misto_id == OckovaciMisto.misto_id) \ + .filter(Kapacita.import_id == last_update_import_id()) \ .group_by(OckovaciMisto.misto_id, OckovaciMisto.nazev, OckovaciMisto.service_id, OckovaciMisto.operation_id, - OckovaciMisto.place_id, OckovaciMisto.mesto, OckovaciMisto.kraj)\ - .order_by(OckovaciMisto.kraj, OckovaciMisto.mesto, OckovaciMisto.nazev)\ + OckovaciMisto.place_id, OckovaciMisto.mesto, OckovaciMisto.kraj) \ + .order_by(OckovaciMisto.kraj, OckovaciMisto.mesto, OckovaciMisto.nazev) \ .all() + return render_template('ockovani_info.html', ockovaci_mista=ockovani_info, last_update=last_update()) @@ -84,3 +87,17 @@ def last_update(): last_run_formatted = last_run.strftime('%d. %m. %Y %H:%M') return last_run_formatted + + +def last_update_import_id(): + """ + For better filtering. + @return: + """ + last_run = app.session.query(func.max(ImportLog.import_id)).first()[0] + if last_run is None: + max_import_id = 0 + else: + max_import_id = last_run + + return max_import_id From e05a0c6ad9d9d86e685b1ae0efa9a5c5b026a2c3 Mon Sep 17 00:00:00 2001 From: Marek Susicky Date: Tue, 19 Jan 2021 21:15:29 +0100 Subject: [PATCH 35/69] Fixed footer --- templates/base.html | 16 +++++++++++----- tools/create_static_pages.sh | 15 ++++++++++++++- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/templates/base.html b/templates/base.html index b4741db9..56282c69 100644 --- a/templates/base.html +++ b/templates/base.html @@ -34,6 +34,8 @@ + + @@ -72,11 +74,15 @@ {% block scripts %} {% endblock %}
- Vznik 16. 01. 2021, poslední aktualizace {{ last_update }}. Data na tomto webu jsou pouze orientační, bez - jakékoliv záruky a nelze je považovat za spolehlivé. Provozovatel se zříká zodpovědnosti za veškeré škody a újmy, - které by použitím těchto dat mohly vzniknout. Web slouží pouze k testování schopností programovacího jazyka Python a - podobnost s reálnými skutečnostmi je čistě náhodná. Pro informace o očkování COVID-19 navštivte web https://koronavirus.mzcr.cz/ockovani-proti-covid-19/ + Vznik 16. 01. 2021, poslední aktualizace {{ last_update }}. Zdroj mapy www.mylog.cz. +
+ Data na tomto webu jsou pouze orientační, bez + jakékoliv záruky a nelze je považovat za spolehlivé. Provozovatel se zříká zodpovědnosti za veškeré škody a + újmy, + které by použitím těchto dat mohly vzniknout. Web slouží pouze k testování schopností programovacího jazyka + Python a + podobnost s reálnými skutečnostmi je čistě náhodná. Pro informace o očkování COVID-19 navštivte web https://koronavirus.mzcr.cz/ockovani-proti-covid-19/
diff --git a/tools/create_static_pages.sh b/tools/create_static_pages.sh index e82b1fe8..9414b0be 100644 --- a/tools/create_static_pages.sh +++ b/tools/create_static_pages.sh @@ -1,2 +1,15 @@ wget http://127.0.0.1:5000 -wget http://127.0.0.1:5000/info \ No newline at end of file +wget http://127.0.0.1:5000/info +wget http://127.0.0.1:5000/kraj/Libereck%C3%BD +wget http://127.0.0.1:5000/kraj/Jiho%C4%8Desk%C3%BD +wget http://127.0.0.1:5000/kraj/Jihomoravsk%C3%BD +wget http://127.0.0.1:5000/kraj/Karlovarsk%C3%BD +wget http://127.0.0.1:5000/kraj/Kr%C3%A1lov%C3%A9hradeck%C3%BD +wget http://127.0.0.1:5000/kraj/Moravskoslezsk%C3%BD +wget http://127.0.0.1:5000/kraj/Olomouck%C3%BD +wget http://127.0.0.1:5000/kraj/Plze%C5%88sk%C3%BD +wget http://127.0.0.1:5000/kraj/Praha +wget http://127.0.0.1:5000/kraj/St%C5%99edo%C4%8Desk%C3%BD +wget http://127.0.0.1:5000/kraj/%C3%9Asteck%C3%BD +wget http://127.0.0.1:5000/kraj/Vyso%C4%8Dina +wget http://127.0.0.1:5000/kraj/Zl%C3%ADnsk%C3%BD \ No newline at end of file From c102fadaded8ec4578eb35527bac2906a20bd535 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Stan=C4=9Bk?= Date: Tue, 19 Jan 2021 21:30:15 +0100 Subject: [PATCH 36/69] last import id takes only finished --- views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/views.py b/views.py index e9633088..f1a3969f 100644 --- a/views.py +++ b/views.py @@ -80,7 +80,7 @@ def info(): def last_update(): - last_run = app.session.query(func.max(ImportLog.spusteni)).first()[0] + last_run = app.session.query(func.max(ImportLog.spusteni)).filter(ImportLog.status == 'FINISHED').first()[0] if last_run is None: last_run_formatted = 'nikdy' else: @@ -94,7 +94,7 @@ def last_update_import_id(): For better filtering. @return: """ - last_run = app.session.query(func.max(ImportLog.import_id)).first()[0] + last_run = app.session.query(func.max(ImportLog.import_id)).filter(ImportLog.status == 'FINISHED').first()[0] if last_run is None: max_import_id = 0 else: From addf0e707858867cb69a360696ccef40413058d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Stan=C4=9Bk?= Date: Tue, 19 Jan 2021 22:26:27 +0100 Subject: [PATCH 37/69] insert days + filtering in fetcher --- data/migrations/202101192133.sql | 45 ++++++++++++++++++++++++++++++++ freespace_fetcher.py | 3 ++- 2 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 data/migrations/202101192133.sql diff --git a/data/migrations/202101192133.sql b/data/migrations/202101192133.sql new file mode 100644 index 00000000..7ef22e88 --- /dev/null +++ b/data/migrations/202101192133.sql @@ -0,0 +1,45 @@ +INSERT INTO dny (den_id, datum) VALUES (31, '2021-02-16'); +INSERT INTO dny (den_id, datum) VALUES (32, '2021-02-17'); +INSERT INTO dny (den_id, datum) VALUES (33, '2021-02-18'); +INSERT INTO dny (den_id, datum) VALUES (34, '2021-02-19'); +INSERT INTO dny (den_id, datum) VALUES (35, '2021-02-20'); +INSERT INTO dny (den_id, datum) VALUES (36, '2021-02-21'); +INSERT INTO dny (den_id, datum) VALUES (37, '2021-02-22'); +INSERT INTO dny (den_id, datum) VALUES (38, '2021-02-23'); +INSERT INTO dny (den_id, datum) VALUES (39, '2021-02-24'); +INSERT INTO dny (den_id, datum) VALUES (40, '2021-02-25'); +INSERT INTO dny (den_id, datum) VALUES (41, '2021-02-26'); +INSERT INTO dny (den_id, datum) VALUES (42, '2021-02-27'); +INSERT INTO dny (den_id, datum) VALUES (43, '2021-02-28'); + +INSERT INTO dny (den_id, datum) VALUES (44, '2021-03-01'); +INSERT INTO dny (den_id, datum) VALUES (45, '2021-03-02'); +INSERT INTO dny (den_id, datum) VALUES (46, '2021-03-03'); +INSERT INTO dny (den_id, datum) VALUES (47, '2021-03-04'); +INSERT INTO dny (den_id, datum) VALUES (48, '2021-03-05'); +INSERT INTO dny (den_id, datum) VALUES (49, '2021-03-06'); +INSERT INTO dny (den_id, datum) VALUES (50, '2021-03-07'); +INSERT INTO dny (den_id, datum) VALUES (51, '2021-03-08'); +INSERT INTO dny (den_id, datum) VALUES (52, '2021-03-09'); +INSERT INTO dny (den_id, datum) VALUES (53, '2021-03-10'); +INSERT INTO dny (den_id, datum) VALUES (54, '2021-03-11'); +INSERT INTO dny (den_id, datum) VALUES (55, '2021-03-12'); +INSERT INTO dny (den_id, datum) VALUES (56, '2021-03-13'); +INSERT INTO dny (den_id, datum) VALUES (57, '2021-03-14'); +INSERT INTO dny (den_id, datum) VALUES (58, '2021-03-15'); +INSERT INTO dny (den_id, datum) VALUES (59, '2021-03-16'); +INSERT INTO dny (den_id, datum) VALUES (60, '2021-03-17'); +INSERT INTO dny (den_id, datum) VALUES (61, '2021-03-18'); +INSERT INTO dny (den_id, datum) VALUES (62, '2021-03-19'); +INSERT INTO dny (den_id, datum) VALUES (63, '2021-03-20'); +INSERT INTO dny (den_id, datum) VALUES (64, '2021-03-21'); +INSERT INTO dny (den_id, datum) VALUES (65, '2021-03-22'); +INSERT INTO dny (den_id, datum) VALUES (66, '2021-03-23'); +INSERT INTO dny (den_id, datum) VALUES (67, '2021-03-24'); +INSERT INTO dny (den_id, datum) VALUES (68, '2021-03-25'); +INSERT INTO dny (den_id, datum) VALUES (69, '2021-03-26'); +INSERT INTO dny (den_id, datum) VALUES (70, '2021-03-27'); +INSERT INTO dny (den_id, datum) VALUES (71, '2021-03-28'); +INSERT INTO dny (den_id, datum) VALUES (72, '2021-03-29'); +INSERT INTO dny (den_id, datum) VALUES (73, '2021-03-30'); +INSERT INTO dny (den_id, datum) VALUES (74, '2021-03-31'); diff --git a/freespace_fetcher.py b/freespace_fetcher.py index 1a6c6169..3553bfdf 100644 --- a/freespace_fetcher.py +++ b/freespace_fetcher.py @@ -88,7 +88,8 @@ def fetch_all(self): """ dny_all = self.session.query(Dny).from_statement( text( - "SELECT den_id, datum FROM dny" + "SELECT den_id, datum FROM dny ", + "WHERE datum BETWEEN NOW() AND NOW() + INTERVAL '30 days'" ) ).all() # print(dny_all) From 4b98435529296de23013e57741fe5da43c431040 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Stan=C4=9Bk?= Date: Tue, 19 Jan 2021 22:49:15 +0100 Subject: [PATCH 38/69] links fixed --- app.py | 2 +- static/css/styles.css | 16 ---------------- static/js/functions.js | 0 templates/base.html | 14 +++----------- templates/index.html | 2 +- templates/kraj.html | 2 +- templates/mesto.html | 2 +- templates/misto.html | 2 +- templates/ockovani_info.html | 2 +- 9 files changed, 9 insertions(+), 33 deletions(-) delete mode 100644 static/css/styles.css delete mode 100644 static/js/functions.js diff --git a/app.py b/app.py index d5a3a874..50c7cd69 100644 --- a/app.py +++ b/app.py @@ -29,7 +29,7 @@ def create_app(config): def register_blueprints(app): - app.register_blueprint(my_view) + app.register_blueprint(my_view, url_prefix='/ockovani-covid') if __name__ == '__main__': diff --git a/static/css/styles.css b/static/css/styles.css deleted file mode 100644 index ceaa4a5a..00000000 --- a/static/css/styles.css +++ /dev/null @@ -1,16 +0,0 @@ -table { - border: 1px solid black; - margin-bottom: 2em; -} - -td { - border: 1px dotted black; -} - -.jumbotron { - background-color: rgba(0,0,0, 0.05); -} - -.footer { - background-color: rgba(0,0,0, 0.07); -} \ No newline at end of file diff --git a/static/js/functions.js b/static/js/functions.js deleted file mode 100644 index e69de29b..00000000 diff --git a/templates/base.html b/templates/base.html index 56282c69..9dad31aa 100644 --- a/templates/base.html +++ b/templates/base.html @@ -30,22 +30,14 @@ crossorigin="anonymous"> - - {% block head %}{% endblock %} {% block navbar %} {% endblock %}
+ {% block body %}{% endblock %}
@@ -80,7 +92,8 @@ {% block scripts %} {% endblock %}
- Vznik 16. 01. 2021, poslední aktualizace {{ last_update }}. Zdroj mapy www.mylog.cz. + Vznik 16. 01. 2021, poslední aktualizace {{ last_update }}. Zdroj mapy www.mylog.cz.
Data na tomto webu jsou pouze orientační, bez jakékoliv záruky a nelze je považovat za spolehlivé. Provozovatel se zříká zodpovědnosti za veškeré škody a From 2fb7f0fde5a53bf28d6c53392e39945e637b1c15 Mon Sep 17 00:00:00 2001 From: Marek Susicky Date: Thu, 25 Feb 2021 12:25:14 +0100 Subject: [PATCH 68/69] Updated README --- README.md | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 1e875c67..688f73fd 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,20 @@ # Volná místa na COVID-19 očkování +Za systémem stojí Jan Staněk (http://jstanek.cz/), Marek Sušický (marek(at)susicky.net) a přátele, kteří poskytli cenné připomínky. -## Instalace +Během marné snahy zajistit očkovací místa pro příbuzné jsme si všimli toho, že neexistuje žádný přehled volných míst. Ještě v lednu jsme začali tvořit aplikaci, ale narazili na neexistenci dat. Pak došly vakcíny a nedávalo smysl systém spouštět. Nyní je situace taková, že mnoho lidí čeká na vakcinaci, ale pokud nejsou na tom správném místě, budou čekat dál. Na jiných místech už ale lidé 80+ "došli". S naší mapou se lidé mohou přeregistrovat, dostat vakcinu rychleji a zefektivnit celý proces očkování. Prosím kohokoliv, kdo může přispět ke zveřejnění oficiálních dat o volných kapacitách a distribuci vakcín, aby tak učinil. + +## Napsali o nás +https://denikn.cz/569269/kde-maji-volna-mista-programatori-po-nocich-vymysleli-aplikaci-ktera-muze-zkratit-cekani-na-vakcinu + +## Poznámky k fungování +Pro získávání dat využívá metody scrapingu. Vzhledem k tomu, že jsou data zveřejňována na githubu, **prosím scraper nespouštějte na vlastním prostředí a nezpůsobujte tak zbytečnou další zátěž rezervačnímu systému!** + +Aplikace se skládá z modulu fetcher, pak samotného webu a skriptu, který web stáhne a publikuje na github pages. Tento krok je realizován proto, že nechceme vystavovat veřejně aplikační server a chceme přenést zátěž na prostředky Githubu. Navíc jde o statické stránky, u kterých není problém obsloužit mnoho tisíc lidí současně. + + +# How to run it [ENG] + +## Installation ### Virtual environment From 64eb93b46b921bdb5bbbb9798fe1b9f64cb8c08e Mon Sep 17 00:00:00 2001 From: Marek Susicky Date: Thu, 25 Feb 2021 12:35:08 +0100 Subject: [PATCH 69/69] Updated README --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 688f73fd..8df2507a 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,10 @@ Aplikace se skládá z modulu fetcher, pak samotného webu a skriptu, který web ### ``` +python app.py +``` +or +``` export FLASK_APP=app.py export FLASK_ENV=development flask run