Initial commit

.dockerignore

@@ -0,0 +1,2 @@
+/db
+/redis

.github/dependabot.yml

@@ -0,0 +1,12 @@
+version: 2
+updates:
+  - package-ecosystem: "pip"
+    directory: "/app/src"
+    schedule:
+      interval: "daily"
+    open-pull-requests-limit: 0
+  - package-ecosystem: "docker"
+    directory: "/app/envs/prod"
+    schedule:
+      interval: "weekly"
+    open-pull-requests-limit: 0

.github/workflows/cd.yml

@@ -0,0 +1,25 @@
+name: Deploy to AWS on push to certain branches
+
+on:
+  push:
+    branches:
+      - 'deploy-*'
+
+jobs:
+  deploy:
+    env:
+      AWS_ACCESS_KEY_ID:  ${{ secrets.DEPLOYMENT_AWS_ACCESS_KEY_ID }} 
+      AWS_SECRET_ACCESS_KEY:  ${{ secrets.DEPLOYMENT_AWS_SECRET_ACCESS_KEY }} 
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: deploy to aws
+        run: |
+          set -e
+          export ENVIRONMENT=${GITHUB_REF_NAME:7}
+          ./deploy-to-aws.sh $ENVIRONMENT
+          export TAG=deployed-${ENVIRONMENT}-`date -u +"%Y-%m-%dT%H.%M.%S"`
+          git tag $TAG
+          git push origin $TAG

.github/workflows/ci.yml

@@ -0,0 +1,52 @@
+name: Run linter and tests
+
+on:
+  push:
+    branches: [master, main]
+  pull_request:
+    branches: [master, main]
+
+env:
+  PYTHON_DEFAULT_VERSION: "3.11"
+
+jobs:
+  linter:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Set up Python ${{ env.PYTHON_DEFAULT_VERSION }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env.PYTHON_DEFAULT_VERSION }}
+          cache: "pip"
+      - name: Install dependencies
+        run: python -m pip install --upgrade nox 'pdm>=2.12,<3'
+      - name: Run linters
+        run: nox -vs lint
+  test:
+    timeout-minutes: 10
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Set up Python ${{ env.PYTHON_DEFAULT_VERSION }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env.PYTHON_DEFAULT_VERSION }}
+          cache: "pip"
+      - name: Install dependencies
+        run: python -m pip install --upgrade nox 'pdm>=2.12,<3'
+      - name: Prepare environment
+        run: ./setup-dev.sh
+      - name: Run dockerized services
+        run: docker compose up -d --wait
+      - name: Run migrations
+        run: cd app/src && pdm run python manage.py wait_for_database --timeout 120 && pdm run python manage.py migrate
+      - name: Run unit tests
+        run: nox -vs test
+      - name: Stop dockerized services
+        if: success() || failure()
+        run: docker compose down -v

.gitignore

@@ -0,0 +1,19 @@
+*.pyc
+*.sqlite3
+*~
+*.egg-info/
+/docker-compose.yml
+/.idea/
+/redis/
+/db/
+.env
+.venv
+venv
+media/
+.backups/
+.envrc
+.pdm-python
+.terraform.lock.hcl
+.terraform/
+.nox/
+__pycache__

.shellcheckrc

@@ -0,0 +1,6 @@
+# disable common false-positive errors to ease adoption in existing projects
+disable=SC1090-SC1092  # disable errors related to sourcing files
+disable=SC2034  # disable errors related to unused variables
+disable=SC2028  # swapping echo for printf requires more testing to ensure correctness
+# disable errors related to cookiecutter templating:
+disable=SC1054,SC1056,SC1072,SC1073,SC1083,SC1009

README.md

@@ -0,0 +1,190 @@
+# bittensor-panel
+
+Bittensor admin panel
+
+- - -
+
+Skeleton of this project was generated with `cookiecutter-rt-django`, which sometimes gets upgrades that are easy to retrofit into already older projects.
+
+# Base requirements
+
+- docker with [compose plugin](https://docs.docker.com/compose/install/linux/)
+- python 3.11
+- [pdm](https://pdm-project.org)
+- [nox](https://nox.thea.codes)
+
+# Setup development environment
+
+```sh
+$ ./setup-dev.sh
+docker compose up -d
+cd app/src
+pdm run manage.py wait_for_database --timeout 10
+pdm run manage.py migrate
+pdm run manage.py runserver
+```
+
+# Setup production environment (git deployment)
+
+This sets up "deployment by pushing to git storage on remote", so that:
+
+- `git push origin ...` just pushes code to Github / other storage without any consequences;
+- `git push production master` pushes code to a remote server running the app and triggers a git hook to redeploy the application.
+
+```
+Local .git ------------> Origin .git
+                \
+                 ------> Production .git (redeploy on push)
+```
+
+- - -
+
+Use `ssh-keygen` to generate a key pair for the server, then add read-only access to repository in "deployment keys" section (`ssh -A` is easy to use, but not safe).
+
+```sh
+# remote server
+mkdir -p ~/repos
+cd ~/repos
+git init --bare --initial-branch=master bittensor-panel.git
+
+mkdir -p ~/domains/bittensor-panel
+```
+
+```sh
+# locally
+git remote add production root@<server>:~/repos/bittensor-panel.git
+git push production master
+```
+
+```sh
+# remote server
+cd ~/repos/bittensor-panel.git
+
+cat <<'EOT' > hooks/post-receive
+#!/bin/bash
+unset GIT_INDEX_FILE
+export ROOT=/root
+export REPO=bittensor-panel
+while read oldrev newrev ref
+do
+    if [[ $ref =~ .*/master$ ]]; then
+        export GIT_DIR="$ROOT/repos/$REPO.git/"
+        export GIT_WORK_TREE="$ROOT/domains/$REPO/"
+        git checkout -f master
+        cd $GIT_WORK_TREE
+        ./deploy.sh
+    else
+        echo "Doing nothing: only the master branch may be deployed on this server."
+    fi
+done
+EOT
+
+chmod +x hooks/post-receive
+./hooks/post-receive
+cd ~/domains/bittensor-panel
+sudo bin/prepare-os.sh
+./setup-prod.sh
+
+# adjust the `.env` file
+
+mkdir letsencrypt
+./letsencrypt_setup.sh
+./deploy.sh
+```
+
+### Deploy another branch
+
+Only `master` branch is used to redeploy an application.
+If one wants to deploy other branch, force may be used to push desired branch to remote's `master`:
+
+```sh
+git push --force production local-branch-to-deploy:master
+```
+
+## Monitoring execution time of code blocks
+
+Somewhere, probably in `metrics.py`:
+
+```python
+some_calculation_time = prometheus_client.Histogram(
+    'some_calculation_time',
+    'How Long it took to calculate something',
+    namespace='django',
+    unit='seconds',
+    labelnames=['task_type_for_example'],
+    buckets=[0.5, 1, *range(2, 30, 2), *range(30, 75, 5), *range(75, 135, 15)]
+)
+```
+
+Somewhere else:
+
+```python
+with some_calculation_time.labels('blabla').time():
+    do_some_work()
+```
+
+# AWS
+
+Initiate the infrastructure with Terraform:
+TODO
+
+To push a new version of the application to AWS, just push to a branch named `deploy-$(ENVIRONMENT_NAME)`.
+Typical values for `$(ENVIRONMENT_NAME)` are `prod` and `staging`.
+For this to work, GitHub actions needs to be provided with credentials for an account that has the following policies enabled:
+
+- AutoScalingFullAccess
+- AmazonEC2ContainerRegistryFullAccess
+- AmazonS3FullAccess
+
+See `.github/workflows/cd.yml` to find out the secret names.
+
+# Vultr
+
+Initiate the infrastructure with Terraform and cloud-init:
+
+- see Terraform template in `<project>/devops/vultr_tf/core/`
+- see scripts for interacting with Vultr API in `<project>/devops/vultr_scripts/`
+  - note these scripts need `vultr-cli` installed
+
+- for more details see README_vultr.md
+
+# Setting up periodic backups
+
+Add to crontab:
+
+```sh
+# crontab -e
+30 0 * * * cd ~/domains/bittensor-panel && ./bin/backup-db.sh > ~/backup.log 2>&1
+```
+
+Set `BACKUP_LOCAL_ROTATE_KEEP_LAST` to keep only a specific number of most recent backups in local `.backups` directory.
+
+## Configuring offsite targets for backups
+
+Backups are put in `.backups` directory locally, additionally then can be stored offsite in following ways:
+
+**Backblaze**
+
+Set in `.env` file:
+
+- `BACKUP_B2_BUCKET_NAME`
+- `BACKUP_B2_KEY_ID`
+- `BACKUP_B2_KEY_SECRET`
+
+**Email**
+
+Set in `.env` file:
+
+- `EMAIL_HOST`
+- `EMAIL_PORT`
+- `EMAIL_HOST_USER`
+- `EMAIL_HOST_PASSWORD`
+- `EMAIL_TARGET`
+
+# Restoring system from backup after a catastrophical failure
+
+1. Follow the instructions above to set up a new production environment
+2. Restore the database using bin/restore-db.sh
+3. See if everything works
+4. Set up backups on the new machine
+5. Make sure everything is filled up in .env, error reporting integration, email accounts etc