-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathcronjob.sh
34 lines (34 loc) · 2.59 KB
/
cronjob.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
datafolder="public/data"
datekawal=$(date +%Y-%m-%d_%H.%M)
filekawal="${datafolder}/kawal_${datekawal}"
scrapy crawl pilpreskawal -o "${filekawal}.xlsx"
datekpu=$(date +%Y-%m-%d_%H.%M)
filekpu="${datafolder}/kpu_${datekpu}"
scrapy crawl pilpres -o "${filekpu}.xlsx"
python excel_to_csv.py ${filekpu} ${filekawal}
psql -h 127.0.0.1 -w -U postgres -d pemilu -c "truncate pilpres_kawal"
psql -h 127.0.0.1 -w -U postgres -d pemilu -c "\copy pilpres_kawal FROM '${PWD}/${filekawal}.csv' delimiter '|' csv header QUOTE E'\b' NULL AS '';"
psql -h 127.0.0.1 -w -U postgres -d pemilu -c "truncate pilpres_kpu"
psql -h 127.0.0.1 -w -U postgres -d pemilu -c "\copy pilpres_kpu FROM '${PWD}/${filekpu}.csv' delimiter '|' csv header QUOTE E'\b' NULL AS '';"
psql -h 127.0.0.1 -w -U postgres -d pemilu -c "REFRESH MATERIALIZED VIEW m_kawal;"
psql -h 127.0.0.1 -w -U postgres -d pemilu -c "REFRESH MATERIALIZED VIEW m_kpu;"
psql -h 127.0.0.1 -w -U postgres -d pemilu -c "REFRESH MATERIALIZED VIEW m_unmatch_data;"
psql -h 127.0.0.1 -w -U postgres -d pemilu -c "REFRESH MATERIALIZED VIEW m_matched_data;"
psql -h 127.0.0.1 -w -U postgres -d pemilu -c "REFRESH MATERIALIZED VIEW m_top_presentase;"
psql -h 127.0.0.1 -w -U postgres -d pemilu -c "REFRESH MATERIALIZED VIEW m_low_presentase;"
psql -h 127.0.0.1 -w -U postgres -d pemilu -c "REFRESH MATERIALIZED VIEW m_tps_not_in_dpt;"
psql -h 127.0.0.1 -w -U postgres -d pemilu -c "\copy (SELECT json_agg(m_unmatch_data) FROM m_unmatch_data) To '${PWD}/${datafolder}/komparasi_${datekpu}.json'"
sed -i 's/\\n//g' "${PWD}/${datafolder}/komparasi_${datekpu}.json"
psql -h 127.0.0.1 -w -U postgres -d pemilu -c "\copy (SELECT json_agg(m_matched_data) FROM m_matched_data) To '${PWD}/${datafolder}/match_${datekpu}.json'"
sed -i 's/\\n//g' "${PWD}/${datafolder}/match_${datekpu}.json"
psql -h 127.0.0.1 -w -U postgres -d pemilu -c "\copy (SELECT json_agg(m_top_presentase) FROM m_top_presentase) To '${PWD}/${datafolder}/toplist_${datekpu}.json'"
sed -i 's/\\n//g' "${PWD}/${datafolder}/toplist_${datekpu}.json"
psql -h 127.0.0.1 -w -U postgres -d pemilu -c "\copy (SELECT json_agg(m_low_presentase) FROM m_low_presentase) To '${PWD}/${datafolder}/lowlist_${datekpu}.json'"
sed -i 's/\\n//g' "${PWD}/${datafolder}/lowlist_${datekpu}.json"
psql -h 127.0.0.1 -w -U postgres -d pemilu -c "\copy (SELECT json_agg(m_tps_not_in_dpt) FROM m_tps_not_in_dpt) To '${PWD}/${datafolder}/notlist_${datekpu}.json'"
sed -i 's/\\n//g' "${PWD}/${datafolder}/notlist_${datekpu}.json"
rm -rf -d public/data/*.csv
ls "${datafolder}" > "${datafolder}/file_list.txt"
git add "${datafolder}"
git commit -m "update data ${datekpu}"
git push origin