From ee7bd71b68ee592ad03bc2a225e9658be6859a7d Mon Sep 17 00:00:00 2001 From: "Sean P. Goggins" Date: Tue, 5 Sep 2023 09:06:13 -0500 Subject: [PATCH 1/2] version update in dev Signed-off-by: Sean P. Goggins --- README.md | 4 ++-- docker/backend/Dockerfile | 2 +- docker/database/Dockerfile | 2 +- metadata.py | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 83beb96ac..81b954cf4 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Augur NEW Release v0.53.0 +# Augur NEW Release v0.53.1 [![first-timers-only](https://img.shields.io/badge/first--timers--only-friendly-blue.svg?style=flat-square)](https://www.firsttimersonly.com/) We follow the [First Timers Only](https://www.firsttimersonly.com/) philosophy of tagging issues for first timers only, and walking one newcomer through the resolution process weekly. [You can find these issues tagged with "first timers only" on our issues list.](https://github.com/chaoss/augur/labels/first-timers-only). @@ -8,7 +8,7 @@ ### [If you want to jump right in, updated docker build/compose and bare metal installation instructions are available here](docs/new-install.md) -Augur is now releasing a dramatically improved new version to the main branch. It is also available here: https://github.com/chaoss/augur/releases/tag/v0.53.0 +Augur is now releasing a dramatically improved new version to the main branch. It is also available here: https://github.com/chaoss/augur/releases/tag/v0.53.1 - The `main` branch is a stable version of our new architecture, which features: - Dramatic improvement in the speed of large scale data collection (100,000+ repos). All data is obtained for 100k+ repos within 2 weeks. - A new job management architecture that uses Celery and Redis to manage queues, and enables users to run a Flower job monitoring dashboard diff --git a/docker/backend/Dockerfile b/docker/backend/Dockerfile index 3bcad3a6a..35d855d65 100644 --- a/docker/backend/Dockerfile +++ b/docker/backend/Dockerfile @@ -2,7 +2,7 @@ FROM python:3.8.11-slim-buster LABEL maintainer="outdoors@acm.org" -LABEL version="0.53.0" +LABEL version="0.53.1" ENV DEBIAN_FRONTEND=noninteractive diff --git a/docker/database/Dockerfile b/docker/database/Dockerfile index 203fcffca..5a9130729 100644 --- a/docker/database/Dockerfile +++ b/docker/database/Dockerfile @@ -2,7 +2,7 @@ FROM postgres:12 LABEL maintainer="outdoors@acm.org" -LABEL version="0.53.0" +LABEL version="0.53.1" ENV POSTGRES_DB "test" ENV POSTGRES_USER "augur" diff --git a/metadata.py b/metadata.py index ebca442cc..9b8a4ed3c 100644 --- a/metadata.py +++ b/metadata.py @@ -5,8 +5,8 @@ __short_description__ = "Python 3 package for free/libre and open-source software community metrics, models & data collection" -__version__ = "0.53.0" -__release__ = "v0.53.0 (Shamrock Shake)" +__version__ = "0.53.1" +__release__ = "v0.53.1 (Shamrock Shake)" __license__ = "MIT" __copyright__ = "University of Missouri, University of Nebraska-Omaha, CHAOSS, Brian Warner & Augurlabs 2023" From e51ded1901d8f26dbcf5abe762313817cd706bab Mon Sep 17 00:00:00 2001 From: "Sean P. Goggins" Date: Tue, 5 Sep 2023 11:40:40 -0500 Subject: [PATCH 2/2] documentation update Signed-off-by: Sean P. Goggins --- .gitignore | 3 +- docs/.DS_Store | Bin 6148 -> 0 bytes docs/dev-osx-install.md | 28 +- docs/source/.DS_Store | Bin 8196 -> 0 bytes docs/source/development-guide/.DS_Store | Bin 6148 -> 0 bytes docs/source/getting-started/.DS_Store | Bin 6148 -> 0 bytes .../getting-started/dev-osx-install.rst | 624 ++++++++++++++++++ .../getting-started}/new-install.rst | 67 +- docs/source/quick-start.rst | 327 +-------- scripts/.DS_Store | Bin 6148 -> 0 bytes tests/.DS_Store | Bin 6148 -> 0 bytes tests/test_workers/.DS_Store | Bin 6148 -> 0 bytes 12 files changed, 681 insertions(+), 368 deletions(-) delete mode 100644 docs/.DS_Store delete mode 100644 docs/source/.DS_Store delete mode 100644 docs/source/development-guide/.DS_Store delete mode 100644 docs/source/getting-started/.DS_Store create mode 100644 docs/source/getting-started/dev-osx-install.rst rename docs/{ => source/getting-started}/new-install.rst (90%) delete mode 100644 scripts/.DS_Store delete mode 100644 tests/.DS_Store delete mode 100644 tests/test_workers/.DS_Store diff --git a/.gitignore b/.gitignore index 2feaa9803..46e492cd4 100644 --- a/.gitignore +++ b/.gitignore @@ -4,13 +4,12 @@ env.txt docker_env.txt pyenv.txt augur_export_env.sh -.DS_Store +*DS_Store *.config.json !docker.config.json config.yml reports.yml - node_modules/ .idea/ logs/ diff --git a/docs/.DS_Store b/docs/.DS_Store deleted file mode 100644 index 46c49a1d71f1295f2d3ec73e84c5b0f457c8d3cb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHKOG*Pl5UtW61KDKhvagUEv_rcWasf#qx=5H{$i5feAq&qTcmOZr)>HWEqcX{W zyC6~p-LJZSGxK1&yNHO}-EvAaA)*FDkVP2~F%P=VJa`0Tt+AlH)%|p~=+>KoLVwXD z?|wmB>S#xgbnyPXe|kCeue(y_t8NZ^bMsn0-rKjUsL3ZXKAk1pvx5S_QiF5|Wb~vtu=c2f~&L zv{d#MgDoBQmJ>Puukp(aHu?RO zSOo*Yz&~SvM`c!8e3YN9N1rEWZNj+45Rtel3IuxZ5`cy5BbVwle-a&W*|8c*6|rkN PFfIa0NKC=N2^jbU+lMeR diff --git a/docs/dev-osx-install.md b/docs/dev-osx-install.md index 66f0c471e..5c8ab6f67 100644 --- a/docs/dev-osx-install.md +++ b/docs/dev-osx-install.md @@ -1,5 +1,7 @@ ## Augur Setup +**NOTE**: Currently, our machine learning dependencies allow Augur to only fully support python 3.8 to python 3.10. Python 3.11 will sometimes work, but often there are libraries at the operating system level that have not yet been updated to support machine learning libraries at python 3.11. + # OSX: Note: This has **MOSTLY** been tested on Apple Silicon with Python 3.11 at this time, however, one user has been successful with Intel based Apple computers. ## For OSX You Need to make sure to install XCode Command line tools: ```shell @@ -38,6 +40,19 @@ export PKG_CONFIG_PATH="/opt/homebrew/opt/openblas/lib/pkgconfig" ## Pre-Requisite Operating System Level Packages Here we ensure your system is up to date, install required python libraries, install postgresql, and install our queuing infrastrucutre, which is composed of redis-server and rabbitmq-server +### Updating your Path: Necessary for rabbitmq on OSX +#### for macOS Intel +`export PATH=$PATH:/usr/local/sbin` +#### for Apple Silicon +`export PATH=$PATH:/opt/homebrew/sbin` + +***These should be added to your .zshrc or other environment file loaded when you open a terminal*** + +#### for macOS Intel +`export PATH=$PATH:/usr/local/sbin:$PATH` +#### for Apple Silicon +`export PATH=$PATH:/opt/homebrew/sbin:$PATH` + ### Executable ```shell brew update ; @@ -77,19 +92,6 @@ rabbitmqctl set_user_tags augur augurTag administrator; rabbitmqctl set_permissions -p augur_vhost augur ".*" ".*" ".*"; ``` -### Updating your Path: Necessary for rabbitmq on OSX -#### for macOS Intel -`export PATH=$PATH:/usr/local/sbin` -#### for Apple Silicon -`export PATH=$PATH:/opt/homebrew/sbin` - -***These should be added to your .zshrc or other environment file loaded when you open a terminal*** - -#### for macOS Intel -`export PATH=$PATH:/usr/local/sbin:$PATH` -#### for Apple Silicon -`export PATH=$PATH:/opt/homebrew/sbin:$PATH` - - We need rabbitmq_management so we can purge our own queues with an API call - We need a user - We need a vhost diff --git a/docs/source/.DS_Store b/docs/source/.DS_Store deleted file mode 100644 index e0a2bd9df85cc344c1ae2c897f32315aa0f0585a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8196 zcmeHMyKWOf6g}famK72K5=De4wxUa6_<_Zk4xuP+k%(ij91AaN`GEwYSP9XjNEI9cUCiJG;{CFxySp^9RpB}X}1pS0;sWSymE=n zlA`BzEtHBIxj|I)2l_|`y?U#YrNe@?42S}vfG8jehytR(zn}oW*}Bw9zWa7oT2Vk0 z_%9XE?}vz0W9%@uXkHy`ObS5UF6k;B+wZqc}MO`AO^=c-(ap{zUSBW+HM z9p)CTJCt>Yayu)Rp(wLE{UT!y6)rf*@Ye9zmDvP5kZT^7sb_)-uRiaD9`zlw z-^Pe-f^NWU`~22w51;GpjfdaAHO#%cVjrLV`d&diJBK=+8kr|1hCN0xL_dh2GJd_x z#^-$h%S_+GT0YBP*#*2R=DDkqQwPW7mZI%*QrmCtl2b76^Et!E`S0zm;d3do@p-5n zK3DbwF5iEBr$`Y6E?$8RyTz`~|L>Kb|6e>2*+CQ#1^$`>YO~pDHn_dZtqVkSt{t%+ svg)Gka*M`=jY-FeCLJfv{xD=eqN(FS))i)Afg-^VYCEmAHI%=;&Qpx ziEHr8UA$ah?S#);--&AcAgX)dRu>`-EuiWMvuR;YeXshHwZqANx7PPT{(fL;^~Q+e zab>f7ygGWkpHfCwaYRk823p;7``r8wZ#P4(MzL=0Y5iVa+U2}iy^ek8HXT#T&rVNi z54VQ9k5LU1`#ZV?3_Q3ju788ReoRrRtacpK~ZzxJ&pfwIT3e&C~c1x*Y30)7C7788T8 QK;(~rra_c2@T&}b0%B~qYXATM diff --git a/docs/source/getting-started/.DS_Store b/docs/source/getting-started/.DS_Store deleted file mode 100644 index fe3bef70208633ef76165a1ed38b1663fb18af73..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHKOHRWu5FNJ_Dhi9P5KCWxMK6F%6}D`80ccW6rKG9&tyuUxu;3D$go7a7c&w0+ zc2OY^LNn3$IpZ1I^DJdzA~KWvd`vVVq9K&Aw~uCn@HlHndeOpzCd_e3E6VAZ3R-r& z)$kV?;IliVj4CSeG|z8EbDZQ7Cwhn84P9Y0qj|9yPqV6AOETvVyT+GQn$C)9hPdHL zKY4m|ucy1uk$-!R^>s53k1fz1vSkhy$oICDXY&5`b-bC!^GlsaYmTz|+~+o*);g5lGFCR_o41DJ!L&vyxqu@PNF zFNhI{kx-z7ns~)95)QkKelDUHlyEX(Wy}*-Ht~jH!s@Ww;7-N~YO4$=18oNS@;c)B zf42Sp-wx6@Wk4DDR}7dzI!z}?Dc06ba$IXI^a9GleqOK+K_`@A_;M*ef(C)zatG)l TdO=tq`Xdl%&_)^fRR%r)!tY`q diff --git a/docs/source/getting-started/dev-osx-install.rst b/docs/source/getting-started/dev-osx-install.rst new file mode 100644 index 000000000..edd643042 --- /dev/null +++ b/docs/source/getting-started/dev-osx-install.rst @@ -0,0 +1,624 @@ +Augur Setup +=========== + +**NOTE**: Currently, our machine learning dependencies allow Augur to +only fully support python 3.8 to python 3.10. Python 3.11 will sometimes +work, but often there are libraries at the operating system level that +have not yet been updated to support machine learning libraries at +python 3.11. + +OSX: Note: This has **MOSTLY** been tested on Apple Silicon with Python 3.11 at this time, however, one user has been successful with Intel based Apple computers. +================================================================================================================================================================== + +For OSX You Need to make sure to install XCode Command line tools: +------------------------------------------------------------------ + +.. code:: shell + + xcode-select --install + +*WARNING: rabbitmq, redis, and postgresql will, by default, set +themselves up to automatically start when your computer starts. This can +be a significant battery drain if you are on battery and not using +Augur. For those reasons, go into your system preferences, startup items +menu (wherever it is now, because Apple changes it more than Zoolander +changes outfits), and turn those “autostart” options off. :)* + +*NOTE: If you do not shutoff rabbitmq and redis at the command line +before shutting down, they will restart themselves anyway on restart. +``brew services stop rabbitmq ;brew services stop redis;``* + +You also need to install these libraries if you are using apple silicon as of June, 2023 +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +brew install gfortran; brew install llvm; echo ‘export +PATH=“/opt/homebrew/opt/llvm/bin:$PATH”’ >> ~/.zshrc; brew install +Pkg-config; brew install openblas; + +Add these lines to your .zshrc file: +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code:: shell + + export LDFLAGS="-L/opt/homebrew/opt/llvm/lib" + export CPPFLAGS="-I/opt/homebrew/opt/llvm/include" + export LDFLAGS="-L/opt/homebrew/opt/openblas/lib "$LDFLAGS + export CPPFLAGS="-I/opt/homebrew/opt/openblas/include "$CPPFLAGS + export PKG_CONFIG_PATH="/opt/homebrew/opt/openblas/lib/pkgconfig" + +Git Platform Requirements (Things to have setup prior to initiating installation.) +---------------------------------------------------------------------------------- + +1. Obtain a GitHub Access Token: https://github.com/settings/tokens +2. Obtain a GitLab Access Token: + https://gitlab.com/-/profile/personal_access_tokens + +Fork and Clone Augur +~~~~~~~~~~~~~~~~~~~~ + +1. Fork https://github.com/chaoss/augur +2. Clone your fork. We recommend creating a ``github`` directory in your + user’s base directory. + +Pre-Requisite Operating System Level Packages +--------------------------------------------- + +Here we ensure your system is up to date, install required python +libraries, install postgresql, and install our queuing infrastrucutre, +which is composed of redis-server and rabbitmq-server + +Updating your Path: Necessary for rabbitmq on OSX +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +for macOS Intel +^^^^^^^^^^^^^^^ + +``export PATH=$PATH:/usr/local/sbin`` #### for Apple Silicon +``export PATH=$PATH:/opt/homebrew/sbin`` + +**These should be added to your .zshrc or other environment file loaded +when you open a terminal** + +.. _for-macos-intel-1: + +for macOS Intel +^^^^^^^^^^^^^^^ + +``export PATH=$PATH:/usr/local/sbin:$PATH`` #### for Apple Silicon +``export PATH=$PATH:/opt/homebrew/sbin:$PATH`` + +Executable +~~~~~~~~~~ + +.. code:: shell + + brew update ; + brew upgrade ; + brew install rabbitmq ; + brew install redis ; + brew install postgresql@14 ; + brew install python3-yq ; + brew install python@3.11 ; + brew install postgresql@14 ; + brew install go ; #required: Go Needs to be version 1.19.x or higher. + brew install nginx ; # required for hosting + brew install geckodriver; + + # You will almost certainly need to reboot after this. + +RabbitMQ Configuration +~~~~~~~~~~~~~~~~~~~~~~ + +The default timeout for RabbitMQ needs to be set. + +.. code:: shell + + sudo vi /opt/homebrew/etc/rabbitmq/advanced.config + +Add this one line to that file (the period at the end matters): + +.. code:: shell + + [ {rabbit, [ {consumer_timeout, undefined} ]} ]. + +Rabbitmq Broker Configuration +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You have to setup a specific user, and broker host for your augur +instance. You can accomplish this by running the below commands: + +.. code:: shell + + rabbitmq-plugins enable rabbitmq_management; + rabbitmqctl add_user augur password123; + rabbitmqctl add_vhost augur_vhost; + rabbitmqctl set_user_tags augur augurTag administrator; + rabbitmqctl set_permissions -p augur_vhost augur ".*" ".*" ".*"; + +- We need rabbitmq_management so we can purge our own queues with an + API call +- We need a user +- We need a vhost +- We then set permissions + +NOTE: it is important to have a static hostname when using rabbitmq as +it uses hostname to communicate with nodes. + +If your setup of rabbitmq is successful your broker url should look like +this: + +**broker_url = ``amqp://augur:password123@localhost:5672/augur_vhost``** + +You will be asked for the broker URL on install of Augur. You can copy +and paste the line above +(``amqp://augur:password123@localhost:5672/augur_vhost``) if you created +the users and virtual hosts under “Broker Configuration”, above. + +Things to start before augur later +---------------------------------- + +.. code:: shell + + brew services start rabbitmq ; + brew services start redis; + brew services start postgresql@14; + +If Issues Starting rabbitmq +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If you get this error: + +:: + + brew services start rabbitmq + Bootstrap failed: 5: Input/output error + Try re-running the command as root for richer errors. + Error: Failure while executing; `/bin/launchctl bootstrap gui/501 /Users/sean/Library/LaunchAgents/homebrew.mxcl.rabbitmq.plist` exited with 5. + +Execute this command: + +:: + + launchctl unload -w /Users/sean/Library/LaunchAgents/homebrew.mxcl.rabbitmq.plist + +Replace the specific path with the one after +``/Users/sean/Library/LaunchAgents/`` in your error message. This was +tested on Apple Silicon. + +Git Configuration +----------------- + +There are some Git configuration parameters that help when you are +cloning repos over time, and a platform prompts you for credentials when +it finds a repo is deleted: + +.. code:: shell + + git config --global diff.renames true; + git config --global diff.renameLimit 200000; + git config --global credential.helper cache; + git config --global credential.helper 'cache --timeout=9999999999999'; + +Postgresql Configuration +------------------------ + +Create a PostgreSQL database for Augur to use + +.. code:: shell + + ... really varies depending how you installed postgres. TBD + +Then, from within the resulting postgresql shell: + +.. code:: sql + + CREATE DATABASE augur; + CREATE USER augur WITH ENCRYPTED PASSWORD 'password'; + GRANT ALL PRIVILEGES ON DATABASE augur TO augur; + GRANT ALL ON SCHEMA public TO augur; + +After that, return to your user by exiting ``psql`` + +:: + + postgres=# \quit + +Here we want to start an SSL connection to the ``augur`` database on +port 5432: + +.. code:: shell + + psql -h localhost -U postgres -p 5432 + +Now type ``exit`` to log off the postgres user, and ``exit`` a SECOND +time to log off the root user. + +.. code:: shell + + exit + exit + +RabbitMQ Developer Note: +~~~~~~~~~~~~~~~~~~~~~~~~ + +These are the queues we create: - celery (the main queue) - secondary - +scheduling + +The endpoints to hit to purge queues on exit are: + +:: + + curl -i -u augur:password123 -XDELETE http://localhost:15672/api/queues/augur_vhost/celery + + curl -i -u augur:password123 -XDELETE http://localhost:15672/api/queues/augur_vhost/secondary + + curl -i -u augur:password123 -XDELETE http://localhost:15672/api/queues/augur_vhost/scheduling + +We provide this functionality to limit, as far as possible, the need for +sudo privileges on the Augur operating system user. With sudo, you can +accomplish the same thing with (Given a vhost named AugurB [case +sensitive]): + +1. To list the queues + +:: + + rabbitmqctl list_queues -p AugurB name messages consumers + +2. To empty the queues, simply execute the command for your queues. + Below are the 3 queues that Augur creates for you: + +:: + + rabbitmqctl purge_queue celery -p augur_vhost + rabbitmqctl purge_queue secondary -p augur_vhost + rabbitmqctl purge_queue scheduling -p augur_vhost + +Where augur_vhost is the vhost. The management API at port 15672 will +only exist if you have already installed the rabbitmq_management plugin. + +**During Augur installation, you will be prompted for this broker_url** + +Installing and Configuring Augur! +--------------------------------- + +Create a Python Virtual Environment +``python3 -m venv ~/virtual-env-directory`` + +Activate your Python Virtual Environment +``source ~/virtual-env-directory/bin/activate`` + +From the root of the Augur Directory, type ``make install``. You will be +prompted to provide: + +- “User” is the PSQL database user, which is ``augur`` if you followed + instructions exactly +- “Password” is the above user’s password +- “Host” is the domain used with nginx, e.g. ``ai.chaoss.io`` +- “Port” is 5432 unless you reconfigured something +- “Database” is the name of the Augur database, which is ``augur`` if + you followed instructions exactly +- The GitHub token created earlier +- Then the username associated with it +- Then the same for GitLab +- and finally a directory to clone repositories to + +Post Installation of Augur +-------------------------- + +Redis Broker Configuration +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If applications other than Augur are running on the same server, and +using ``redis-server`` it is important to ensure that Augur and these +other applications (or additional instances of Augur) are using distinct +“cache_group”. You can change from the default value of zero by editing +the ``augur_operations.config`` table directly, looking for the “Redis” +section_name, and the “cache_group” setting_name. This SQL is also a +template: + +.. code:: sql + + UPDATE augur_operations.config + SET value = 2 + WHERE + section_name='Redis' + AND + setting_name='cache_group'; + +What does Redis Do? +^^^^^^^^^^^^^^^^^^^ + +Redis is used to make the state of data collection jobs visible on an +external dashboard, like Flower. Internally, Augur relies on Redis to +cache GitHub API Keys, and for OAuth Authentication. Redis is used to +maintain awareness of Augur’s internal state. + +What does RabbitMQ Do? +^^^^^^^^^^^^^^^^^^^^^^ + +Augur is a distributed system. Even on one server, there are many +collection processes happening simultaneously. Each job to collect data +is put on the RabbitMQ Queue by Augur’s “Main Brain”. Then independent +workers pop messages off the RabbitMQ Queue and go collect the data. +These tasks then become standalone processes that report their +completion or failure states back to the Redis server. + +**Edit** the ``/etc/redis/redis.conf`` file to ensure these parameters +are configured in this way: + +.. code:: shell + + supervised systemd + databases 900 + maxmemory-samples 10 + maxmemory 20GB + +**NOTE**: You may be able to have fewer databases and lower maxmemory +settings. This is a function of how many repositories you are collecting +data for at a given time. The more repositories you are managing data +for, the close to these settings you will need to be. + +**Consequences** : If the settings are too low for Redis, Augur’s +maintainer team has observed cases where collection appears to stall. +(TEAM: This is a working theory as of 3/10/2023 for Ubuntu 22.x, based +on EC2 experiments.) + +(OPTIONAL: NOT FOR DEV: Proxying Augur through Nginx) +----------------------------------------------------- + +Assumes nginx is installed. + +Then you create a file for the server you want Augur to run under in the +location of your ``sites-enabled`` directory for nginx. In this example, +Augur is running on port 5038: (the long timeouts on the settings page +is for when a user adds a large number of repos or orgs in a single +session to prevent timeouts from nginx) + +.. _for-macos-intel-2: + +For MacOS Intel: +~~~~~~~~~~~~~~~~ + +This gist explains where sites-enabled is: +``https://gist.github.com/jimothyGator/5436538`` + +Logs for nginx should go in this directory: + +``mkdir /Library/Logs/nginx`` + +``/Library/Logs/nginx`` + +For Apple Silicon: +~~~~~~~~~~~~~~~~~~ + +There is no ``sites-enabled`` directory. Server configurations go here: +``/opt/homebrew/etc/nginx/servers`` + +Logs for nginx should go in this directory: +``/opt/homebrew/var/log/nginx`` + +:: + + server { + server_name ai.chaoss.io; + + location /api/unstable/ { + proxy_pass http://ai.chaoss.io:5038; + proxy_set_header Host $host; + } + + location / { + proxy_pass http://127.0.0.1:5038; + } + + location /settings { + + proxy_read_timeout 800; + proxy_connect_timeout 800; + proxy_send_timeout 800; + } + + error_log /var/log/nginx/augurview.osshealth.error.log; + access_log /var/log/nginx/augurview.osshealth.access.log; + + } + +(OPTIONAL: NOT FOR DEV) Setting up SSL (https) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Install Certbot: **NOTE: certbot does not currently run on Apple +Silicon, as it is looking for information in MacOS Intel directories** + +:: + + brew update; + brew upgrade; + brew install certbot; + brew install openssl; + brew install brew install python-typing-extensions + +Generate a certificate for the specific domain for which you have a file +already in the sites-enabled directory for nginx (located at +``/etc/nginx/sites-enabled`` on Ubuntu): + +:: + + brew certbot -v --nginx -d ai.chaoss.io + +In the example file above. Your resulting nginx sites-enabled file will +look like this: + +:: + + server { + server_name ai.chaoss.io; + + location /api/unstable/ { + proxy_pass http://ai.chaoss.io:5038; + proxy_set_header Host $host; + } + + location / { + proxy_pass http://127.0.0.1:5038; + } + + location /settings { + + proxy_read_timeout 800; + proxy_connect_timeout 800; + proxy_send_timeout 800; + } + + error_log /var/log/nginx/augurview.osshealth.error.log; + access_log /var/log/nginx/augurview.osshealth.access.log; + + listen 443 ssl; # managed by Certbot + ssl_certificate /etc/letsencrypt/live/ai.chaoss.io/fullchain.pem; # managed by Certbot + ssl_certificate_key /etc/letsencrypt/live/ai.chaoss.io/privkey.pem; # managed by Certbot + include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot + + } + + server { + if ($host = ai.chaoss.io) { + return 301 https://$host$request_uri; + } # managed by Certbot + + + listen 80; + server_name ai.chaoss.io; + return 404; # managed by Certbot + + + } + +Possible EC2 Configuration Requirements +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +With virtualization there may be issues associated with redis-server +connections exceeding available memory. In these cases, the following +workarounds help to resolve issues. + +Specifically, you may find this error in your augur logs: + +.. code:: shell + + redis.exceptions.ConnectionError: Error 111 connecting to 127.0.0.1:6379. Connection refused. + +**INSTALL** ``sudo apt install libhugetlbfs-bin`` + +**COMMAND**: + +:: + + sudo hugeadm --thp-never && + sudo echo never > /sys/kernel/mm/transparent_hugepage/enabled + +.. code:: shell + + sudo vi /etc/rc.local + +**paste** into ``/etc/rc.local`` + +.. code:: shell + + if test -f /sys/kernel/mm/transparent_hugepage/enabled; then + echo never > /sys/kernel/mm/transparent_hugepage/enabled + fi + +**EDIT** : ``/etc/default/grub`` add the following line: + +.. code:: shell + + GRUB_DISABLE_OS_PROBER=true + +.. _postgresql-configuration-1: + +Postgresql Configuration +------------------------ + +Your postgresql instance should optimally allow 1,000 connections: + +.. code:: shell + + max_connections = 1000 # (change requires restart) + shared_buffers = 8GB # min 128kB + work_mem = 2GB # min 64kB + +Augur will generally hold up to 150 simultaneous connections while +collecting data. The 1,000 number is recommended to accommodate both +collection and analysis on the same database. Use of PGBouncer or other +utility may change these characteristics. + +Augur Commands +-------------- + +To access command line options, use ``augur --help``. To load repos from +GitHub organizations prior to collection, or in other ways, the direct +route is ``augur db --help``. + +Start a Flower Dashboard, which you can use to monitor progress, and +report any failed processes as issues on the Augur GitHub site. The +error rate for tasks is currently 0.04%, and most errors involve +unhandled platform API timeouts. We continue to identify and add fixes +to handle these errors through additional retries. Starting Flower: +``(nohup celery -A augur.tasks.init.celery_app.celery_app flower --port=8400 --max-tasks=1000000 &)`` +NOTE: You can use any open port on your server, and access the dashboard +in a browser with http://servername-or-ip:8400 in the example above +(assuming you have access to that port, and its open on your network.) + +If you’re using a virtual machine within Windows and you get an error +about missing AVX instructions, you should kill Hyper-V. Even if it +doesn’t *appear* to be active, it might still be affecting your VM. +Follow `these instructions `__ to +disable Hyper-V, and afterward AVX should pass to the VM. + +Starting your Augur Instance +---------------------------- + +Start Augur: ``(nohup augur backend start &)`` + +When data collection is complete you will see only a single task running +in your flower Dashboard. + +Accessing Repo Addition and Visualization Front End +--------------------------------------------------- + +Your Augur instance will now be available at +http://hostname.io:port_number + +For example: http://chaoss.tv:5038 + +Note: Augur will run on port 5000 by default (you probably need to +change that in augur_operations.config for OSX) + +Stopping your Augur Instance +---------------------------- + +You can stop augur with ``augur backend stop``, followed by +``augur backend kill``. We recommend waiting 5 minutes between commands +so Augur can shutdown more gently. There is no issue with data integrity +if you issue them seconds apart, its just that stopping is nicer than +killing. + +Docker +~~~~~~ + +1. Make sure docker, and docker-compose are both installed +2. Modify the ``environment.txt`` file in the root of the repository to + include your GitHub and GitLab API keys. +3. If you are already running postgresql on your server you have two + choices: + + - Change the port mappings in the ``docker-compose.yml`` file to + match ports for Postgresql not currently in use. + - Change to variables in ``environment.txt`` to include the correct + values for your local, non-docker-container database. + +4. ``sudo docker build -t augur-new -f docker/backend/Dockerfile .`` +5. ``sudo docker-compose --env-file ./environment.txt --file docker-compose.yml up`` + to run the database in a Docker Container or + ``sudo docker-compose --env-file ./environment.txt --file docker-compose.yml up`` + to connect to an already running database. diff --git a/docs/new-install.rst b/docs/source/getting-started/new-install.rst similarity index 90% rename from docs/new-install.rst rename to docs/source/getting-started/new-install.rst index 6b5a0ca9c..1455f6729 100644 --- a/docs/new-install.rst +++ b/docs/source/getting-started/new-install.rst @@ -1,5 +1,5 @@ Augur Setup ------------ +=========== Ubuntu 22.x =========== @@ -47,7 +47,7 @@ Executable sudo snap install go --classic && #required: Go Needs to be version 1.19.x or higher. Snap is the package manager that gets you to the right version. Classic enables it to actually be installed at the correct version. sudo apt install nginx && # required for hosting sudo add-apt-repository ppa:mozillateam/firefox-next && - sudo apt install firefox=111.0~b8+build1-0ubuntu0.22.04.1 && + sudo apt install firefox=115.0~b2+build1-0ubuntu0.22.04.1 && sudo apt install firefox-geckodriver # You will almost certainly need to reboot after this. @@ -100,9 +100,9 @@ Then, from within the resulting postgresql shell: CREATE USER augur WITH ENCRYPTED PASSWORD 'password'; GRANT ALL PRIVILEGES ON DATABASE augur TO augur; -**If you're using PostgreSQL 15 or later**, default database permissions will -prevent Augur's installer from configuring the database. Add one last line -after the above to fix this: +**If you’re using PostgreSQL 15 or later**, default database permissions +will prevent Augur’s installer from configuring the database. Add one +last line after the above to fix this: .. code:: sql @@ -114,14 +114,15 @@ After that, return to your user by exiting ``psql`` postgres=# \quit -Here we want to start an SSL connection to the ``augur`` database on port 5432: +Here we want to start an SSL connection to the ``augur`` database on +port 5432: .. code:: shell psql -h localhost -U postgres -p 5432 -Now type ``exit`` to log off the postgres user, and ``exit`` a SECOND time to -log off the root user. +Now type ``exit`` to log off the postgres user, and ``exit`` a SECOND +time to log off the root user. .. code:: shell @@ -136,7 +137,7 @@ instance. You can accomplish this by running the below commands: .. code:: shell - sudo rabbitmq-plugins enable rabbitmq_management + sudo rabbitmq-plugins enable rabbitmq_management && sudo rabbitmqctl add_user augur password123 && sudo rabbitmqctl add_vhost augur_vhost && sudo rabbitmqctl set_user_tags augur augurTag administrator && @@ -151,7 +152,7 @@ instance. You can accomplish this by running the below commands: NOTE: it is important to have a static hostname when using rabbitmq as it uses hostname to communicate with nodes. -RabbitMQ's server can then be started from systemd: +RabbitMQ’s server can then be started from systemd: .. code:: shell @@ -209,7 +210,7 @@ Proxying Augur through Nginx Assumes nginx is installed. Then you create a file for the server you want Augur to run under in the -location of your ``sites-enabled`` directory for nginx (In this example, +location of your ``sites-enabled`` directory for nginx. In this example, Augur is running on port 5038: (the long timeouts on the settings page is for when a user adds a large number of repos or orgs in a single session to prevent timeouts from nginx) @@ -247,10 +248,10 @@ Install Certbot: :: - sudo apt update - sudo apt upgrade - sudo apt install certbot - apt-get install python3-certbot-nginx + sudo apt update && + sudo apt upgrade && + sudo apt install certbot && + sudo apt-get install python3-certbot-nginx Generate a certificate for the specific domain for which you have a file already in the sites-enabled directory for nginx (located at @@ -320,17 +321,17 @@ Activate your Python Virtual Environment From the root of the Augur Directory, type ``make install``. You will be prompted to provide: -- "User" is the PSQL database user, which is ``augur`` if you followed - instructions exactly -- "Password" is the above user's password -- "Host" is the domain used with nginx, e.g. ``ai.chaoss.io`` -- "Port" is 5432 unless you reconfigured something -- "Database" is the name of the Augur database, which is ``augur`` if you - followed instructions exactly -- The GitHub token created earlier -- Then the username associated with it -- Then the same for GitLab -- and finally a directory to clone repositories to +- “User” is the PSQL database user, which is ``augur`` if you followed + instructions exactly +- “Password” is the above user’s password +- “Host” is the domain used with nginx, e.g. ``ai.chaoss.io`` +- “Port” is 5432 unless you reconfigured something +- “Database” is the name of the Augur database, which is ``augur`` if + you followed instructions exactly +- The GitHub token created earlier +- Then the username associated with it +- Then the same for GitLab +- and finally a directory to clone repositories to Post Installation of Augur -------------------------- @@ -412,8 +413,8 @@ Specifically, you may find this error in your augur logs: :: - hugeadm --thp-never` && - echo never > /sys/kernel/mm/transparent_hugepage/enabled + sudo hugeadm --thp-never && + sudo echo never > /sys/kernel/mm/transparent_hugepage/enabled .. code:: shell @@ -468,11 +469,11 @@ NOTE: You can use any open port on your server, and access the dashboard in a browser with http://servername-or-ip:8400 in the example above (assuming you have access to that port, and its open on your network.) -If you're using a virtual machine within Windows and you get an error about -missing AVX instructions, you should kill Hyper-V. Even if it doesn't *appear* -to be active, it might still be affecting your VM. Follow -`these instructions `_ to disable -Hyper-V, and afterward AVX should pass to the VM. +If you’re using a virtual machine within Windows and you get an error +about missing AVX instructions, you should kill Hyper-V. Even if it +doesn’t *appear* to be active, it might still be affecting your VM. +Follow `these instructions `__ to +disable Hyper-V, and afterward AVX should pass to the VM. Starting your Augur Instance ---------------------------- diff --git a/docs/source/quick-start.rst b/docs/source/quick-start.rst index 6ae6836cf..906f4beea 100644 --- a/docs/source/quick-start.rst +++ b/docs/source/quick-start.rst @@ -1,330 +1,17 @@ Quickstart =============== -Ubuntu 22.x -=========== +Select installation instructions from those most closely related to the operating system that you use below. Note that Augur's dependencies do not consistently support python 3.11 at this time. Python 3.8 - Python 3.10 have been tested on each platform. -We default to this version of Ubuntu for the moment because Augur does -not yet support python3.10, which is the default version of python3.x -distributed with Ubuntu 22.0x.x +.. toctree:: + :maxdepth: 2 -Git Platform Requirements (Things to have setup prior to initiating installation.) ----------------------------------------------------------------------------------- + getting-started/new-install + getting-started/dev-osx-install -1. Obtain a GitHub Access Token: https://github.com/settings/tokens -2. Obtain a GitLab Access Token: - https://gitlab.com/-/profile/personal_access_tokens -Fork and Clone Augur -~~~~~~~~~~~~~~~~~~~~ - -1. Fork https://github.com/chaoss/augur -2. Clone your fork. We recommend creating a ``github`` directory in your - user’s base directory. - -Pre-Requisite Operating System Level Packages ---------------------------------------------- - -Here we ensure your system is up to date, install required python -libraries, install postgresql, and install our queuing infrastrucutre, -which is composed of redis-server and rabbitmq-server - -Executable -~~~~~~~~~~ - -.. code:: shell - - sudo apt update && - sudo apt upgrade && - sudo apt install software-properties-common && - sudo apt install python3-dev && - sudo apt install python3.10-venv && - sudo apt install postgresql postgresql-contrib postgresql-client && - sudo apt install build-essential && - sudo apt install redis-server && # required - sudo apt install erlang && # required - sudo apt install rabbitmq-server && #required - sudo snap install go --classic && #required: Go Needs to be version 1.19.x or higher. Snap is the package manager that gets you to the right version. Classic enables it to actually be installed at the correct version. - sudo apt install nginx && # required for hosting - sudo add-apt-repository ppa:mozillateam/firefox-next && - sudo apt install firefox=111.0~b8+build1-0ubuntu0.22.04.1 && - sudo apt install firefox-geckodriver - - # You will almost certainly need to reboot after this. - -RabbitMQ Configuration -~~~~~~~~~~~~~~~~~~~~~~ - -The default timeout for RabbitMQ needs to be set on Ubuntu 22.x. - -.. code:: shell - - sudo vi /etc/rabbitmq/advanced.config - -Add this one line to that file (the period at the end matters): - -.. code:: shell - - [ {rabbit, [ {consumer_timeout, undefined} ]} ]. - -Git Configuration ------------------ - -There are some Git configuration parameters that help when you are -cloning repos over time, and a platform prompts you for credentials when -it finds a repo is deleted: - -.. code:: shell - - git config --global diff.renames true - git config --global diff.renameLimit 200000 - git config --global credential.helper cache - git config --global credential.helper 'cache --timeout=9999999999999' - -Postgresql Configuration ------------------------- - -Create a PostgreSQL database for Augur to use - -.. code:: shell - - sudo su - && - su - postgres && - psql - -Then, from within the resulting postgresql shell: - -.. code:: sql - - CREATE DATABASE augur; - CREATE USER augur WITH ENCRYPTED PASSWORD 'password'; - GRANT ALL PRIVILEGES ON DATABASE augur TO augur; - -Once you are successfully logged out, return to your user by exiting -``psql``, then typing ``exit`` to exit the postgres user, and ``exit`` a -SECOND time to exit the root user. - -:: - - postgres=# \quit - -.. code:: shell - - exit - exit - -Rabbitmq Broker Configuration ------------------------------ - -You have to setup a specific user, and broker host for your augur -instance. You can accomplish this by running the below commands: - -.. code:: shell - - sudo rabbitmq-plugins enable rabbitmq_management - sudo rabbitmqctl add_user augur password123 && - sudo rabbitmqctl add_vhost augur_vhost && - sudo rabbitmqctl set_user_tags augur augurTag administrator && - sudo rabbitmqctl set_permissions -p augur_vhost augur ".*" ".*" ".*" - -- We need rabbitmq_management so we can purge our own queues with an - API call -- We need a user -- We need a vhost -- We then set permissions - -NOTE: it is important to have a static hostname when using rabbitmq as -it uses hostname to communicate with nodes. - -If your setup of rabbitmq is successful your broker url should look like -this: - -**broker_url = ``amqp://augur:password123@localhost:5672/augur_vhost``** - -RabbitMQ Developer Note: -~~~~~~~~~~~~~~~~~~~~~~~~ - -These are the queues we create: - celery (the main queue) - secondary - -scheduling - -The endpoints to hit to purge queues on exit are: - -:: - - curl -i -u augur:password123 -XDELETE http://localhost:15672/api/queues/AugurB/celery - - curl -i -u augur:password123 -XDELETE http://localhost:15672/api/queues/AugurB/secondary - - curl -i -u augur:password123 -XDELETE http://localhost:15672/api/queues/AugurB/scheduling - -We provide this functionality to limit, as far as possible, the need for -sudo privileges on the Augur operating system user. With sudo, you can -accomplish the same thing with (Given a vhost named AugurB [case -sensitive]): - -1. To list the queues - -:: - - sudo rabbitmqctl list_queues -p AugurB name messages consumers - -2. To empty the queues, simply execute the command for your queues. - Below are the 3 queues that Augur creates for you: - -:: - - sudo rabbitmqctl purge_queue celery -p AugurB - sudo rabbitmqctl purge_queue secondary -p AugurB - sudo rabbitmqctl purge_queue scheduling -p AugurB - -Where AugurB is the vhost. The management API at port 15672 will only -exist if you have already installed the rabbitmq_management plugin. - -**During Augur installation, you will be prompted for this broker_url** - -Proxying Augur through Nginx ----------------------------- - -Assumes nginx is installed. - -Then you create a file for the server you want Augur to run under in the -location of your ``sites-enabled`` directory for nginx (In this example, -Augur is running on port 5038: (the long timeouts on the settings page -is for when a user adds a large number of repos or orgs in a single -session to prevent timeouts from nginx) - -:: - - server { - server_name ai.chaoss.io; - - location /api/unstable/ { - proxy_pass http://ai.chaoss.io:5038; - proxy_set_header Host $host; - } - - location / { - proxy_pass http://127.0.0.1:5038; - } - - location /settings { - - proxy_read_timeout 800; - proxy_connect_timeout 800; - proxy_send_timeout 800; - } - - error_log /var/log/nginx/augurview.osshealth.error.log; - access_log /var/log/nginx/augurview.osshealth.access.log; - - } - -Setting up SSL (https) -~~~~~~~~~~~~~~~~~~~~~~ - -Install Certbot: - -:: - - sudo apt update - sudo apt upgrade - sudo apt install certbot - apt-get install python3-certbot-nginx - -Generate a certificate for the specific domain for which you have a file -already in the sites-enabled directory for nginx (located at -``/etc/nginx/sites-enabled`` on Ubuntu): - -:: - - sudo certbot -v --nginx -d ai.chaoss.io - -In the example file above. Your resulting nginx sites-enabled file will -look like this: - -:: - - server { - server_name ai.chaoss.io; - - location /api/unstable/ { - proxy_pass http://ai.chaoss.io:5038; - proxy_set_header Host $host; - } - - location / { - proxy_pass http://127.0.0.1:5038; - } - - location /settings { - - proxy_read_timeout 800; - proxy_connect_timeout 800; - proxy_send_timeout 800; - } - - error_log /var/log/nginx/augurview.osshealth.error.log; - access_log /var/log/nginx/augurview.osshealth.access.log; - - listen 443 ssl; # managed by Certbot - ssl_certificate /etc/letsencrypt/live/ai.chaoss.io/fullchain.pem; # managed by Certbot - ssl_certificate_key /etc/letsencrypt/live/ai.chaoss.io/privkey.pem; # managed by Certbot - include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot - ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot - - } - - server { - if ($host = ai.chaoss.io) { - return 301 https://$host$request_uri; - } # managed by Certbot - - - listen 80; - server_name ai.chaoss.io; - return 404; # managed by Certbot - - - } - -Installing and Configuring Augur! ---------------------------------- - -Create a Python Virtual Environment -``python3 -m venv ~/virtual-env-directory`` - -Activate your Python Virtual Environment -``source ~/virtual-env-directory/bin/activate`` - -From the root of the Augur Directory, type ``make install`` - -You will be prompted to provide your GitHub username and password, your -GitLab username and password, and the postgresql database where you want -to have the Augur Schema built. You will also be prompted to provide a -directory where repositories will be clone into. - -Post Installation of Augur --------------------------- - -Redis Broker Configuration -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -If applications other than Augur are running on the same server, and -using ``redis-server`` it is important to ensure that Augur and these -other applications (or additional instances of Augur) are using distinct -“cache_group”. You can change from the default value of zero by editing -the ``augur_operations.config`` table directly, looking for the “Redis” -section_name, and the “cache_group” setting_name. This SQL is also a -template: - -.. code:: sql - - UPDATE augur_operations.config - SET value = 2 - WHERE - section_name='Redis' - AND - setting_name='cache_group'; +Explanations of Technologies +============================ What does Redis Do? ^^^^^^^^^^^^^^^^^^^ diff --git a/scripts/.DS_Store b/scripts/.DS_Store deleted file mode 100644 index 5feae2ae7e45fde14124639b3f0bf8ffa9a85ddc..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHKOHRW;4E3}XErLaNu*{+oJEYzql$146F91#WiG)-I_PGGJ;0Pctz@95`0k*7p z9y@K*v=R%15V9rDo6Oj;^JX*?6OkL-Cw-zW5p|%9jZHKwgy&gn($JA@p!4sTQch`l z(I3Tm&DsY3A_Kg3ZHlpOH*`k}>z7R=-S$J>BFm#_oaST9bk5(N_nr=44^~+v->{lo z@@^@k`Jfw=AfK47>5Qhbo67m#KH^m77jxP9oaM4)GtXq5O*M~WgaI#CVC5FPoKjlu z>oSug?D%Di#a#LyFU`lH@=|$Emd~?UZ&l6bo%1m<28;n?;HMbi6AaSX613JBFb0f) z83TMi1W?9Mu@Lk}2O3`j09zRLf=P)4iiIF%AWmF?;_9@-aN>^fKO-s%|AEBHj>OQlAHaVw zvvaO(6`M9KT@XU|O1>wt&vj0Yk~k(J)tvSlL^UERqcN5$NGpQJxh+Ltd$?HXIW8!r zgeEkinM}o#Y|W8$L&IM!MthqrFb*j*ZahJD~{d? zUBdCZ?;ZySMLbvQ$^7*3p{4y@Ij5_16W7-nvQGDTxQDMJ=uD9P=OLJ!uj9IUXU_|7 z3+Hw<|Ag=94&GR|#C8r~&E_aq4N4XV!~t<&;Q)UhLNvz6Vs21>I*{}h09Zn|4jl6@ z1=oZYBa68~j3AUL1)5T2w;0Nlqd&B~$YO5Ll#{ZX4`r{c><&fgt7HC9(@8}JC5r>% zfbYPLIj!;f-zwh!`$cjl4u}K)$^li1+i?@OWY^ZEo8z@MMC+h&FfTW#OOWhxtPgl8 c-b2%Yv5*gdk;U8~dJys>ply&O4*aSEpOpHq)&Kwi diff --git a/tests/test_workers/.DS_Store b/tests/test_workers/.DS_Store deleted file mode 100644 index 4d5f243e6b913e4fe02d508cdc8fd3c178087a8d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHKu};H44E2Q$L0u{)#B>7^bN}EdVMpr50&P>Eq9IWWVnT}eCM14bG#%miFk8a$3+WZoR;r3y({h`5Z=chhJdx_R-<{rg8OLW*=298=aZuUzu$i+sHFI z<2+TA0cAiL_$dZZvsvP8Bhi)2Kf76p^PqIDd;~PFunx