diff --git a/.github/workflows/docker-ci-amd-and-arm.yml b/.github/workflows/docker-ci-amd-and-arm.yml index ddeadd189..2d6f91fd0 100644 --- a/.github/workflows/docker-ci-amd-and-arm.yml +++ b/.github/workflows/docker-ci-amd-and-arm.yml @@ -1,4 +1,4 @@ -name: Docker CI +name: Docker CI Multi-Platform on: workflow_dispatch: @@ -9,44 +9,74 @@ permissions: contents: read jobs: - publish: - name: Build and publish amd64 and arm64 image + settings: runs-on: ubuntu-latest - env: - PUSH_TO_DOCKER: ${{github.ref == 'refs/heads/staging'}} + outputs: + version: ${{steps.env.outputs.version}} steps: - uses: actions/checkout@v3 - - id: version - name: Get version + - name: Setup outputs + id: env run: | - echo "LOGFLARE_VERSION=$(cat VERSION)" >> $GITHUB_ENV - echo "LOGFLARE_VERSION=$(cat VERSION)" >> $GITHUB_OUTPUT - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - name: Login to Docker Hub + echo "version=$(cat ./VERSION)" >> "$GITHUB_OUTPUT" + build: + needs: settings + strategy: + matrix: + include: + - runner: [self-hosted, X64] + arch: amd64 + - runner: arm-runner + arch: arm64 + runs-on: ${{ matrix.runner }} + timeout-minutes: 120 + steps: + # checkout git commit + - uses: actions/checkout@v3 + # Setup docker + # https://github.com/supabase/postgres/blob/develop/.github/workflows/dockerhub-release.yml#LL41C11-L42C44 + - run: docker context create builders + - uses: docker/setup-buildx-action@v2 + with: + endpoint: builders + - name: Login to DockerHub uses: docker/login-action@v2 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - # build for master - - name: Build and push latest, versioned - if: ${{github.ref == 'refs/heads/master'}} + # Build and push + - id: build uses: docker/build-push-action@v3 with: push: true - tags: supabase/logflare:latest, supabase/logflare:${{ env.LOGFLARE_VERSION }} + tags: supabase/logflare:${{ needs.settings.outputs.version }}-${{ matrix.arch }} + platforms: linux/${{ matrix.arch }} + # caching cache-from: type=gha cache-to: type=gha,mode=max - platforms: linux/arm64,linux/amd64 - # build for staging - - name: Build and push staging build - if: ${{github.ref == 'refs/heads/staging'}} - uses: docker/build-push-action@v3 + + merge_publish: + needs: + - settings + - build + runs-on: ubuntu-latest + steps: + - uses: docker/login-action@v2 with: - push: true - tags: supabase/logflare:staging - cache-from: type=gha - cache-to: type=gha,mode=max - platforms: linux/arm64,linux/amd64 + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Setup tags + run: | + echo "tag_amd64=supabase/logflare:${{ needs.settings.outputs.version }}-amd64" >> "$GITHUB_ENV" + echo "tag_arm64=supabase/logflare:${{ needs.settings.outputs.version }}-arm64" >> "$GITHUB_ENV" + echo "tag_ver=supabase/logflare:${{ needs.settings.outputs.version }}" >> "$GITHUB_ENV" + - name: Staging - Merge multi-arch manifests and push + if: github.ref == 'refs/heads/staging' + run: | + docker buildx imagetools create -t supabase/logflare:staging \ + ${{ env.tag_amd64 }} ${{ env.tag_arm64 }} + - name: Master - Merge multi-arch manifests and push + if: github.ref == 'refs/heads/master' + run: | + docker buildx imagetools create -t supabase/logflare:latest -t ${{ env.tag_ver }} \ + ${{ env.tag_amd64 }} ${{ env.tag_arm64 }} diff --git a/.github/workflows/docker-ci.yml b/.github/workflows/docker-ci.yml deleted file mode 100644 index 538e8c291..000000000 --- a/.github/workflows/docker-ci.yml +++ /dev/null @@ -1,91 +0,0 @@ -name: Docker CI - -on: - workflow_dispatch: - push: - branches: [staging, master] - -permissions: - contents: read - -jobs: - publish: - name: Build and publish amd64 image - runs-on: ubuntu-latest - env: - PUSH_TO_DOCKER: ${{github.ref == 'refs/heads/staging'}} - steps: - - uses: actions/checkout@v3 - - id: version - name: Get version - run: | - echo "LOGFLARE_VERSION=$(cat VERSION)" >> $GITHUB_ENV - echo "LOGFLARE_VERSION=$(cat VERSION)" >> $GITHUB_OUTPUT - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - name: Login to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - # build for master - - name: Build and push latest, versioned - if: ${{github.ref == 'refs/heads/master'}} - uses: docker/build-push-action@v3 - with: - push: true - tags: supabase/logflare:latest, supabase/logflare:${{ env.LOGFLARE_VERSION }} - cache-from: type=gha - cache-to: type=gha,mode=max - platforms: linux/amd64 - # build for staging - - name: Build and push staging build - if: ${{github.ref == 'refs/heads/staging'}} - uses: docker/build-push-action@v3 - with: - push: true - tags: supabase/logflare:staging - cache-from: type=gha - cache-to: type=gha,mode=max - platforms: linux/amd64 - trigger_cloudbuild: - name: Trigger Cloud Build in Production - if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/staging' - needs: - - publish - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Set prod envs - if: github.ref == 'refs/heads/master' - run: | - echo "LF_BRANCH=master" >> "$GITHUB_ENV" - echo "LF_CLOUDBUILD_TRIGGER=logflare-master" >> "$GITHUB_ENV" - echo "LF_PROJECT_ID=logflare-232118" >> "$GITHUB_ENV" - echo "LF_GCP_SECRETS=${{ secrets.GCP_PROD_CREDENTIALS }}" >> "$GITHUB_ENV" - - name: Set staging envs - if: github.ref == 'refs/heads/staging' - run: | - echo "LF_BRANCH=staging" >> "$GITHUB_ENV" - echo "LF_CLOUDBUILD_TRIGGER=logflare-app-staging-trigger" >> "$GITHUB_ENV" - echo "LF_PROJECT_ID=logflare-staging" >> "$GITHUB_ENV" - echo "LF_GCP_SECRETS=${{ secrets.GCP_STAGING_CREDENTIALS }}" >> "$GITHUB_ENV" - - uses: actions/setup-python@v4 - with: - python-version: '3.10' - - id: 'auth' - uses: 'google-github-actions/auth@v1' - with: - credentials_json: ${{ env.LF_GCP_SECRETS }} - create_credentials_file: true - export_environment_variables: true - cleanup_credentials: false - - name: 'Set up Cloud SDK' - uses: 'google-github-actions/setup-gcloud@v1' - with: - version: '418.0.0' - project_id: ${{ env.LF_PROJECT_ID}} - - name: 'Trigger Cloud Build' - run: 'gcloud builds triggers run ${{ env.LF_CLOUDBUILD_TRIGGER }} --branch=${{ env.LF_BRANCH }} --format "value(name)"' diff --git a/.github/workflows/docker-test.yml b/.github/workflows/docker-test.yml new file mode 100644 index 000000000..72dad0fbc --- /dev/null +++ b/.github/workflows/docker-test.yml @@ -0,0 +1,24 @@ +name: Docker Build Check + +on: + workflow_dispatch: + pull_request: + branches: [staging] + +permissions: + contents: read + +jobs: + build: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v3 + - uses: docker/setup-buildx-action@v2 + - uses: docker/build-push-action@v3 + with: + push: false + platforms: linux/amd64 + # caching + cache-from: type=gha + cache-to: type=gha,mode=max \ No newline at end of file diff --git a/.github/workflows/elixir-ci.yml b/.github/workflows/elixir-ci.yml index c6e1c3dcb..d719b8406 100644 --- a/.github/workflows/elixir-ci.yml +++ b/.github/workflows/elixir-ci.yml @@ -4,6 +4,9 @@ on: workflow_dispatch: push: branches: [staging, master] + paths: + - '!.github/**' + - '!cloudbuild/**' pull_request: branches: [staging] diff --git a/.github/workflows/trigger-cloudbuild.yml b/.github/workflows/trigger-cloudbuild.yml new file mode 100644 index 000000000..5cd7a535a --- /dev/null +++ b/.github/workflows/trigger-cloudbuild.yml @@ -0,0 +1,58 @@ +name: Trigger Cloudbuild + +on: + workflow_dispatch: + workflow_run: + workflows: [Docker CI Multi-Platform] + types: [completed] + branches: [master, staging] + +permissions: + contents: read + +jobs: + trigger_cloudbuild: + name: Trigger Cloud Build + if: github.event.workflow_run.conclusion == 'success' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + # GCP credentials is a multi-line string, and will result in error when trying to set it to github env. + # directly use it in the auth step instead. + - name: Set prod envs + if: github.ref == 'refs/heads/master' + run: | + echo "LF_BRANCH=master" >> "$GITHUB_ENV" + echo "LF_CLOUDBUILD_TRIGGER=logflare-master" >> "$GITHUB_ENV" + echo "LF_PROJECT_ID=logflare-232118" >> "$GITHUB_ENV" + - name: Set staging envs + if: ${{ github.ref == 'refs/heads/staging' }} + run: | + echo "LF_BRANCH=staging" >> "$GITHUB_ENV" + echo "LF_CLOUDBUILD_TRIGGER=logflare-app-staging-trigger" >> "$GITHUB_ENV" + echo "LF_PROJECT_ID=logflare-staging" >> "$GITHUB_ENV" + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + - if: github.ref == 'refs/heads/master' + uses: 'google-github-actions/auth@v1' + with: + credentials_json: ${{ secrets.GCP_PROD_CREDENTIALS }} + create_credentials_file: true + export_environment_variables: true + cleanup_credentials: false + - if: github.ref == 'refs/heads/staging' + uses: 'google-github-actions/auth@v1' + with: + credentials_json: ${{ secrets.GCP_STAGING_CREDENTIALS }} + create_credentials_file: true + export_environment_variables: true + cleanup_credentials: false + - name: 'Set up Cloud SDK' + uses: 'google-github-actions/setup-gcloud@v1' + with: + version: '418.0.0' + project_id: ${{ env.LF_PROJECT_ID}} + - name: 'Trigger Cloud Build' + run: | + gcloud builds triggers run ${{ env.LF_CLOUDBUILD_TRIGGER }} --branch=${{ env.LF_BRANCH }} --format "value(name)" diff --git a/.gitignore b/.gitignore index aa87e705f..64893de14 100644 --- a/.gitignore +++ b/.gitignore @@ -55,7 +55,15 @@ gcloud_prod.json .google.secret.json .*.env + !.template.env !.docker.env # rustler nif binaries -/priv/native \ No newline at end of file +/priv/native + +# SSL Certificates required for LB to Instance TLS +.*.cacert.* +.*.cert.* +.*.req.* +!.*.cacert.*.enc +!.*.cert.*.enc diff --git a/.prod.cacert.key.enc b/.prod.cacert.key.enc new file mode 100644 index 000000000..ae5143f04 Binary files /dev/null and b/.prod.cacert.key.enc differ diff --git a/.prod.cacert.pem.enc b/.prod.cacert.pem.enc new file mode 100644 index 000000000..f2036b11a Binary files /dev/null and b/.prod.cacert.pem.enc differ diff --git a/.prod.cert.key.enc b/.prod.cert.key.enc new file mode 100644 index 000000000..03f85d948 Binary files /dev/null and b/.prod.cert.key.enc differ diff --git a/.prod.cert.pem.enc b/.prod.cert.pem.enc new file mode 100644 index 000000000..e9cb30a76 Binary files /dev/null and b/.prod.cert.pem.enc differ diff --git a/.prod.env.enc b/.prod.env.enc index b2b5ef1f2..4e9950c27 100644 Binary files a/.prod.env.enc and b/.prod.env.enc differ diff --git a/.staging.cacert.key.enc b/.staging.cacert.key.enc new file mode 100644 index 000000000..bb3fdfab2 Binary files /dev/null and b/.staging.cacert.key.enc differ diff --git a/.staging.cacert.pem.enc b/.staging.cacert.pem.enc new file mode 100644 index 000000000..8083308d0 Binary files /dev/null and b/.staging.cacert.pem.enc differ diff --git a/.staging.cert.key.enc b/.staging.cert.key.enc new file mode 100644 index 000000000..87f77cd96 Binary files /dev/null and b/.staging.cert.key.enc differ diff --git a/.staging.cert.pem.enc b/.staging.cert.pem.enc new file mode 100644 index 000000000..38277c413 Binary files /dev/null and b/.staging.cert.pem.enc differ diff --git a/.staging.env.enc b/.staging.env.enc index c883bdd9b..9087892ef 100644 Binary files a/.staging.env.enc and b/.staging.env.enc differ diff --git a/DEPLOYMENT.md b/DEPLOYMENT.md new file mode 100644 index 000000000..3c1c17994 --- /dev/null +++ b/DEPLOYMENT.md @@ -0,0 +1,31 @@ +## Deployment + +### GRPC Server Setup + +To deploy the GRPC server in a new environment you need the following steps for your ``: + +- **Cloudflare:** Enable GRPC in the `Network` tab +- **Cloudflare:** Generate a CA Certificate to be used on your Origin server: + - SSL/TLS -> Origin Server -> Create Certificate -> Create + - Create .``.cacert.pem with the content from the certificate field + - Create .``.cacert.key with the content from the key field +- **Local:** Generate a self signed certificate for the origin server: + - `openssl req -newkey rsa:2048 -nodes -days 365000 -keyout ..cert.key -out ..req.pem` and set the email to your support email + - `openssl x509 -req -days 12783 -set_serial 1 -in ..req.pem -out ..cert.pem -CA ..cacert.pem -CAkey ..cacert.key` + - Store this files to be pushed into the server +- **Google Cloud:** On your `Instance Template`, allow for HTTPS traffic in the Firewall configuration +- **Google Cloud:** On your `Instance Group`, add a new port onto your `Port Mapping` configuration to be `50051` (do check you change all `Instance Groups`) +- **Google Cloud:** Create your Load Balancer + - Select `HTTP(S) load balancing` + - Select `From Internet to my VMs or serverless services` and `Global HTTP(S) load balancer` + - Frontend Configuration + - Protocol - `HTTPS (includes HTTP/2)` + - IP Address - Create a new IP Address + - Set certificate + - Backend Configuration + - Create `Backend Service` for each `Instance Group` you want to support + - Select the target `Instance Group` and select the GRPC Port set earlier in the popup + - Disable Cloud CDN + - Enable Logging + - Set Health check +- **Cloudflare:** Set a new DNS route with a sub domain pointing to the generated IP of your GRPC LB diff --git a/Dockerfile b/Dockerfile index 3a2d750fa..24d48f3a4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,23 +1,24 @@ FROM elixir:1.14.4-alpine as builder ENV MIX_ENV prod - -RUN apk update && \ - apk add -f curl git build-base nodejs npm rust cargo python3 - -COPY . /logflare - -WORKDIR /logflare # Due to some Rust caveats with SSL on Alpine images, we need to use GIT to fecth cargo registry index ENV CARGO_NET_GIT_FETCH_WITH_CLI=true -RUN mix do local.rebar --force, local.hex --force, deps.get, release - -WORKDIR /logflare/assets -RUN npm install -RUN npm run deploy +# cache intermediate layers for deps compilation +COPY ./VERSION mix.exs mix.lock /logflare/ +COPY assets/package.json assets/package-lock.json /logflare/assets/ WORKDIR /logflare -RUN mix phx.digest +RUN apk update && \ + # all the base dependencies + apk add -f curl git build-base nodejs npm rust cargo python3 && \ + # all the app dependencies + mix do local.rebar --force, local.hex --force, deps.get, deps.compile && \ + npm --prefix assets ci + +COPY . /logflare +RUN mix release && \ + npm run --prefix assets deploy && \ + mix phx.digest # alpine version must match the base erlang image version used # https://github.com/erlef/docker-elixir/blob/master/1.14/alpine/Dockerfile diff --git a/VERSION b/VERSION index e2cac26c1..b966e81a4 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.2.3 \ No newline at end of file +1.2.4 \ No newline at end of file diff --git a/assets/js/app.js b/assets/js/app.js index 311b8fee2..59f19c29a 100644 --- a/assets/js/app.js +++ b/assets/js/app.js @@ -1,50 +1,50 @@ -import "../css/app.scss" -import { Socket } from "phoenix" -import "../css/tailwind.css" -import "bootstrap" -import ClipboardJS from "clipboard" -import * as Dashboard from "./dashboard" -import * as Source from "./source" -import * as Logs from "./logs" -import * as User from "./user" -import BillingHooks from "./billing" -import LiveModalHooks from "./live_modal" -import {LogEventsChart} from "./source_log_chart.jsx" -import Chart from "./admin_dashboard_charts.jsx" -import Loader from "./loader.jsx" -import LiveSocket from "phoenix_live_view" -import LiveReact, {initLiveReact} from "phoenix_live_react" +import "../css/app.scss"; +import { Socket } from "phoenix"; +import "../css/tailwind.css"; +import "bootstrap"; +import ClipboardJS from "clipboard"; +import * as Dashboard from "./dashboard"; +import * as Source from "./source"; +import * as Logs from "./logs"; +import * as User from "./user"; +import BillingHooks from "./billing"; +import LiveModalHooks from "./live_modal"; +import { LogEventsChart } from "./source_log_chart.jsx"; +import Chart from "./admin_dashboard_charts.jsx"; +import Loader from "./loader.jsx"; +import { LiveSocket } from "phoenix_live_view"; +import LiveReact, { initLiveReact } from "phoenix_live_react"; -import sourceLiveViewHooks from "./source_lv_hooks" -import logsLiveViewHooks from "./log_event_live_hooks" +import sourceLiveViewHooks from "./source_lv_hooks"; +import logsLiveViewHooks from "./log_event_live_hooks"; -import moment from "moment" +import moment from "moment"; // set moment globally before daterangepicker -window.moment = moment +window.moment = moment; // import vendor files -import "./vendor/daterangepicker.min.js" -import "./vendor/daterangepicker.css" +import "./vendor/daterangepicker.min.js"; +import "./vendor/daterangepicker.css"; // interfaces -import * as Interfaces from "./interfaces" -import * as Components from "./components" +import * as Interfaces from "./interfaces"; +import * as Components from "./components"; let csrfToken = document .querySelector("meta[name='csrf-token']") - .getAttribute("content") + .getAttribute("content"); -const liveReactHooks = {LiveReact} +const liveReactHooks = { LiveReact }; -window.Components = {LogEventsChart, Loader, AdminChart: Chart} -window.Interfaces = Interfaces +window.Components = { LogEventsChart, Loader, AdminChart: Chart }; +window.Interfaces = Interfaces; // todo: merge with window.Components -window.Comp = Components -window.Dashboard = Dashboard -window.Logs = Logs -window.Source = Source -window.User = User -window.ClipboardJS = ClipboardJS +window.Comp = Components; +window.Dashboard = Dashboard; +window.Logs = Logs; +window.Source = Source; +window.User = User; +window.ClipboardJS = ClipboardJS; const hooks = { ...liveReactHooks, @@ -52,7 +52,7 @@ const hooks = { ...logsLiveViewHooks, ...LiveModalHooks, ...BillingHooks, -} +}; let liveSocket = new LiveSocket("/live", Socket, { hooks, @@ -76,7 +76,7 @@ let liveSocket = new LiveSocket("/live", Socket, { offsetX: e.offsetX, offsetY: e.offsetY, detail: e.detail || 1, - } + }; }, keydown: (e, el) => { return { @@ -91,16 +91,16 @@ let liveSocket = new LiveSocket("/live", Socket, { metaKey: e.metaKey, repeat: e.repeat, shiftKey: e.shiftKey, - } + }; }, }, -}) +}); -liveSocket.connect() +liveSocket.connect(); -window.initLiveReact = initLiveReact -window.liveSocket = liveSocket +window.initLiveReact = initLiveReact; +window.liveSocket = liveSocket; document.addEventListener("DOMContentLoaded", (e) => { - initLiveReact() -}) + initLiveReact(); +}); diff --git a/assets/package-lock.json b/assets/package-lock.json index 839130241..7671fb441 100644 --- a/assets/package-lock.json +++ b/assets/package-lock.json @@ -41,16 +41,19 @@ } }, "../deps/phoenix": { - "version": "0.0.1" + "version": "1.6.16", + "license": "MIT" }, "../deps/phoenix_html": { - "version": "0.0.1" + "version": "3.0.4" }, "../deps/phoenix_live_react": { - "version": "0.0.1" + "version": "0.4.2", + "license": "MIT" }, "../deps/phoenix_live_view": { - "version": "0.0.1" + "version": "0.16.4", + "license": "MIT" }, "node_modules/@babel/code-frame": { "version": "7.18.6", diff --git a/cloudbuild/prod/cloudbuild.yaml b/cloudbuild/prod/cloudbuild.yaml index c572117c1..451982cbc 100644 --- a/cloudbuild/prod/cloudbuild.yaml +++ b/cloudbuild/prod/cloudbuild.yaml @@ -17,6 +17,33 @@ steps: - '--location=us-central1' - '--keyring=logflare-prod-keyring-us-central1' - '--key=logflare-prod-secrets-key' + - name: gcr.io/cloud-builders/gcloud + args: + - kms + - decrypt + - --ciphertext-file=./.prod.cert.pem.enc + - --plaintext-file=./cert.pem + - --location=us-central1 + - --keyring=logflare-prod-keyring-us-central1' + - --key=logflare-prod-secrets-key' + - name: gcr.io/cloud-builders/gcloud + args: + - kms + - decrypt + - --ciphertext-file=./.prod.cert.key.enc + - --plaintext-file=./cert.key + - --location=us-central1 + - --keyring=logflare-prod-keyring-us-central1' + - --key=logflare-prod-secrets-key' + - name: gcr.io/cloud-builders/gcloud + args: + - kms + - decrypt + - --ciphertext-file=./.prod.cacert.pem.enc + - --plaintext-file=./cacert.pem + - --location=us-central1 + - --keyring=logflare-prod-keyring-us-central1' + - --key=logflare-prod-secrets-key' - name: gcr.io/cloud-builders/docker args: - '-c' diff --git a/cloudbuild/secret_setup.Dockerfile b/cloudbuild/secret_setup.Dockerfile index aa26f882d..0ef33ced1 100644 --- a/cloudbuild/secret_setup.Dockerfile +++ b/cloudbuild/secret_setup.Dockerfile @@ -5,6 +5,9 @@ RUN apk add tini COPY .secrets.env /tmp/.secrets.env COPY gcloud.json gcloud.json +COPY cacert.pem cacert.pem +COPY cert.pem cert.pem +COPY cert.key cert.key ENTRYPOINT ["tini", "--"] diff --git a/cloudbuild/staging/cloudbuild.yaml b/cloudbuild/staging/cloudbuild.yaml index 8543bfdbe..e9db3bfec 100644 --- a/cloudbuild/staging/cloudbuild.yaml +++ b/cloudbuild/staging/cloudbuild.yaml @@ -17,11 +17,38 @@ steps: - --location=us-central1 - --keyring=logflare-keyring-us-central1 - --key=logflare-secrets-key + - name: gcr.io/cloud-builders/gcloud + args: + - kms + - decrypt + - --ciphertext-file=./.staging.cert.pem.enc + - --plaintext-file=./cert.pem + - --location=us-central1 + - --keyring=logflare-keyring-us-central1 + - --key=logflare-secrets-key + - name: gcr.io/cloud-builders/gcloud + args: + - kms + - decrypt + - --ciphertext-file=./.staging.cert.key.enc + - --plaintext-file=./cert.key + - --location=us-central1 + - --keyring=logflare-keyring-us-central1 + - --key=logflare-secrets-key + - name: gcr.io/cloud-builders/gcloud + args: + - kms + - decrypt + - --ciphertext-file=./.staging.cacert.pem.enc + - --plaintext-file=./cacert.pem + - --location=us-central1 + - --keyring=logflare-keyring-us-central1 + - --key=logflare-secrets-key - name: "gcr.io/cloud-builders/docker" args: [ "build", - "--build-arg", + "--build-arg", "TAG_VERSION=staging", "-f", "cloudbuild/secret_setup.Dockerfile", diff --git a/config/runtime.exs b/config/runtime.exs index bfd36624a..d082cc660 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -209,3 +209,12 @@ if config_env() != :test do config :goth, json: File.read!("gcloud.json") config :grpc, port: System.get_env("LOGFLARE_GRPC_PORT", "50051") |> String.to_integer() end + +if(File.exists?("cacert.pem") && File.exists?("cert.pem") && File.exists?("cert.key")) do + config :logflare, + ssl: [ + cacertfile: "cacert.pem", + certfile: "cert.pem", + keyfile: "cert.key" + ] +end diff --git a/gcloud_prod.json.enc b/gcloud_prod.json.enc index 1009aa731..1595ba18b 100644 Binary files a/gcloud_prod.json.enc and b/gcloud_prod.json.enc differ diff --git a/gcloud_staging.json.enc b/gcloud_staging.json.enc index 6327a8bd0..e5bf0eb98 100644 Binary files a/gcloud_staging.json.enc and b/gcloud_staging.json.enc differ diff --git a/lib/logflare/application.ex b/lib/logflare/application.ex index 83a585aef..e3f25f6a0 100644 --- a/lib/logflare/application.ex +++ b/lib/logflare/application.ex @@ -79,6 +79,8 @@ defmodule Logflare.Application do topologies = Application.get_env(:libcluster, :topologies, []) grpc_port = Application.get_env(:grpc, :port) + ssl = Application.get_env(:logflare, :ssl) + grpc_creds = if ssl, do: GRPC.Credential.new(ssl: ssl) [ {Task.Supervisor, name: Logflare.TaskSupervisor}, @@ -136,7 +138,7 @@ defmodule Logflare.Application do # If we get a log event and the Source.Supervisor is not up it will 500 LogflareWeb.Endpoint, - {GRPC.Server.Supervisor, {LogflareGrpc.Endpoint, grpc_port}}, + {GRPC.Server.Supervisor, {LogflareGrpc.Endpoint, grpc_port, cred: grpc_creds}}, # Monitor system level metrics Logflare.SystemMetricsSup, diff --git a/lib/logflare/saved_searches/saved_searches.ex b/lib/logflare/saved_searches/saved_searches.ex index 8e1678d2e..108dd8348 100644 --- a/lib/logflare/saved_searches/saved_searches.ex +++ b/lib/logflare/saved_searches/saved_searches.ex @@ -122,7 +122,7 @@ defmodule Logflare.SavedSearches do @doc """ Get a SavedSearch by query string and source """ - @spec get_by_qs_source_id(String.t(), number()) :: SavedSearch.t() + @spec get_by_qs_source_id(String.t(), number()) :: SavedSearch.t() | nil def get_by_qs_source_id(querystring, source_id) do SavedSearch |> where([s], s.querystring == ^querystring) diff --git a/lib/logflare/single_tenant.ex b/lib/logflare/single_tenant.ex index 6e5efffe2..35a7b78a0 100644 --- a/lib/logflare/single_tenant.ex +++ b/lib/logflare/single_tenant.ex @@ -61,9 +61,10 @@ defmodule Logflare.SingleTenant do cache_duration_seconds: 0 }, %{ - name: "charts.usage", + name: "usage.api-counts", query: - Application.app_dir(:logflare, "priv/supabase/endpoints/charts.usage.sql") |> File.read!(), + Application.app_dir(:logflare, "priv/supabase/endpoints/usage.api-counts.sql") + |> File.read!(), sandboxable: true, max_limit: 1000, enable_auth: true, diff --git a/lib/logflare_web/router.ex b/lib/logflare_web/router.ex index 90284daa3..60dc48795 100644 --- a/lib/logflare_web/router.ex +++ b/lib/logflare_web/router.ex @@ -203,7 +203,10 @@ defmodule LogflareWeb.Router do pipe_through([:browser, :require_auth, :set_source, :ensure_source_started]) resources "/", SourceController, except: [:index, :new, :create, :delete] do - live("/rules", Sources.RulesLV, layout: {LogflareWeb.LayoutView, :root}) + live_session(:rules, root_layout: {LogflareWeb.LayoutView, :root}) do + live("/rules", Sources.RulesLV) + end + delete("/saved-searches/:id", SavedSearchesController, :delete) end @@ -287,10 +290,13 @@ defmodule LogflareWeb.Router do scope "/admin", LogflareWeb do pipe_through([:browser, :check_admin]) + live_session(:admin, root_layout: {LayoutView, :root}) do + live("/search", AdminSearchDashboardLive) + end + get("/dashboard", AdminController, :dashboard) get("/sources", AdminController, :sources) get("/accounts", AdminController, :accounts) - live("/search", AdminSearchDashboardLive, layout: {LayoutView, :root}) live("/cluster", Admin.ClusterLive, :index) live("/partner", Admin.PartnerLive, :index) diff --git a/mix.exs b/mix.exs index 5d6449ad4..ecd515f1e 100644 --- a/mix.exs +++ b/mix.exs @@ -61,17 +61,16 @@ defmodule Logflare.Mixfile do defp deps do [ - # Phoenix and LogflareWeb - {:phoenix, "~> 1.5.0", override: true}, + # Phoenix stuff + {:phoenix, "~> 1.6.0"}, + {:phoenix_html, "~> 3.0"}, + {:phoenix_live_view, "~> 0.16.4"}, {:phoenix_pubsub, "~> 2.0.0"}, {:phoenix_ecto, "~> 4.4"}, - {:phoenix_html, "~> 2.10"}, {:phoenix_live_reload, "~> 1.0", only: :dev}, # {:plug, "~> 1.8"}, {:plug_cowboy, "~> 2.6"}, {:plug_crypto, "~> 1.2.2"}, - {:phoenix_live_view, "~> 0.15.3", override: true}, - {:phoenix_live_dashboard, "~> 0.3.0"}, {:cors_plug, "~> 2.0"}, # Oauth @@ -81,7 +80,7 @@ defmodule Logflare.Mixfile do {:oauth2, "~> 2.0.0", override: true}, # Oauth2 provider - {:phoenix_oauth2_provider, "~> 0.5.1"}, + {:phoenix_oauth2_provider, github: "Logflare/phoenix_oauth2_provider"}, {:ex_oauth2_provider, github: "aristamd/ex_oauth2_provider", override: true}, # Ecto and DB @@ -134,7 +133,7 @@ defmodule Logflare.Mixfile do # Pagination {:scrivener_ecto, "~> 2.2"}, {:scrivener_list, "~> 2.0"}, - {:scrivener_html, "~> 1.8"}, + {:scrivener_html, github: "Logflare/scrivener_html"}, # GCP {:google_api_cloud_resource_manager, "~> 0.34.0"}, @@ -148,8 +147,8 @@ defmodule Logflare.Mixfile do # Telemetry & logging {:telemetry, "~> 0.4.0"}, - {:telemetry_poller, "0.5.0"}, - {:telemetry_metrics, "~> 0.6.0", override: true}, + {:telemetry_metrics, "~> 0.6"}, + {:telemetry_poller, "~> 0.5"}, {:logflare_logger_backend, "~> 0.11.1"}, # ETS @@ -249,19 +248,35 @@ defmodule Logflare.Mixfile do "cmd gcloud kms encrypt --ciphertext-file='./.dev.env.enc' --plaintext-file=./.dev.env --location=us-central1 --keyring=logflare-keyring-us-central1 --key=logflare-secrets-key --project=logflare-staging", "decrypt.staging": [ "cmd gcloud kms decrypt --ciphertext-file='./.staging.env.enc' --plaintext-file=./.staging.env --location=us-central1 --keyring=logflare-keyring-us-central1 --key=logflare-secrets-key --project=logflare-staging", - "cmd gcloud kms decrypt --ciphertext-file='./gcloud_staging.json.enc' --plaintext-file=./gcloud_staging.json --location=us-central1 --keyring=logflare-keyring-us-central1 --key=logflare-secrets-key --project=logflare-staging" + "cmd gcloud kms decrypt --ciphertext-file='./gcloud_staging.json.enc' --plaintext-file=./gcloud_staging.json --location=us-central1 --keyring=logflare-keyring-us-central1 --key=logflare-secrets-key --project=logflare-staging", + "cmd gcloud kms decrypt --ciphertext-file='./.staging.cacert.key.enc' --plaintext-file=./.staging.cacert.key --location=us-central1 --keyring=logflare-keyring-us-central1 --key=logflare-secrets-key --project=logflare-staging", + "cmd gcloud kms decrypt --ciphertext-file='./.staging.cacert.pem.enc' --plaintext-file=./.staging.cacert.pem --location=us-central1 --keyring=logflare-keyring-us-central1 --key=logflare-secrets-key --project=logflare-staging", + "cmd gcloud kms decrypt --ciphertext-file='./.staging.cert.key.enc' --plaintext-file=./.staging.cert.key --location=us-central1 --keyring=logflare-keyring-us-central1 --key=logflare-secrets-key --project=logflare-staging", + "cmd gcloud kms decrypt --ciphertext-file='./.staging.cert.pem.enc' --plaintext-file=./.staging.cert.pem --location=us-central1 --keyring=logflare-keyring-us-central1 --key=logflare-secrets-key --project=logflare-staging" ], "encrypt.staging": [ "cmd gcloud kms encrypt --ciphertext-file='./.staging.env.enc' --plaintext-file=./.staging.env --location=us-central1 --keyring=logflare-keyring-us-central1 --key=logflare-secrets-key --project=logflare-staging", - "cmd gcloud kms encrypt --ciphertext-file='./gcloud_staging.json.enc' --plaintext-file=./gcloud_staging.json --location=us-central1 --keyring=logflare-keyring-us-central1 --key=logflare-secrets-key --project=logflare-staging" + "cmd gcloud kms encrypt --ciphertext-file='./gcloud_staging.json.enc' --plaintext-file=./gcloud_staging.json --location=us-central1 --keyring=logflare-keyring-us-central1 --key=logflare-secrets-key --project=logflare-staging", + "cmd gcloud kms encrypt --ciphertext-file='./.staging.cacert.key.enc' --plaintext-file=./.staging.cacert.key --location=us-central1 --keyring=logflare-keyring-us-central1 --key=logflare-secrets-key --project=logflare-staging", + "cmd gcloud kms encrypt --ciphertext-file='./.staging.cacert.pem.enc' --plaintext-file=./.staging.cacert.pem --location=us-central1 --keyring=logflare-keyring-us-central1 --key=logflare-secrets-key --project=logflare-staging", + "cmd gcloud kms encrypt --ciphertext-file='./.staging.cert.key.enc' --plaintext-file=./.staging.cert.key --location=us-central1 --keyring=logflare-keyring-us-central1 --key=logflare-secrets-key --project=logflare-staging", + "cmd gcloud kms encrypt --ciphertext-file='./.staging.cert.pem.enc' --plaintext-file=./.staging.cert.pem --location=us-central1 --keyring=logflare-keyring-us-central1 --key=logflare-secrets-key --project=logflare-staging" ], "decrypt.prod": [ "cmd gcloud kms decrypt --ciphertext-file='./.prod.env.enc' --plaintext-file=./.prod.env --location=us-central1 --keyring=logflare-prod-keyring-us-central1 --key=logflare-prod-secrets-key --project=logflare-232118", - "cmd gcloud kms decrypt --ciphertext-file='./gcloud_prod.json.enc' --plaintext-file=./gcloud_prod.json --location=us-central1 --keyring=logflare-prod-keyring-us-central1 --key=logflare-prod-secrets-key --project=logflare-232118" + "cmd gcloud kms decrypt --ciphertext-file='./gcloud_prod.json.enc' --plaintext-file=./gcloud_prod.json --location=us-central1 --keyring=logflare-prod-keyring-us-central1 --key=logflare-prod-secrets-key --project=logflare-232118", + "cmd gcloud kms decrypt --ciphertext-file='./.prod.cacert.key.enc' --plaintext-file=./.prod.cacert.key --location=us-central1 --keyring=logflare-prod-keyring-us-central1 --key=logflare-prod-secrets-key --project=logflare-232118", + "cmd gcloud kms decrypt --ciphertext-file='./.prod.cacert.pem.enc' --plaintext-file=./.prod.cacert.pem --location=us-central1 --keyring=logflare-prod-keyring-us-central1 --key=logflare-prod-secrets-key --project=logflare-232118", + "cmd gcloud kms decrypt --ciphertext-file='./.prod.cert.key.enc' --plaintext-file=./.prod.cert.key --location=us-central1 --keyring=logflare-prod-keyring-us-central1 --key=logflare-prod-secrets-key --project=logflare-232118", + "cmd gcloud kms decrypt --ciphertext-file='./.prod.cert.pem.enc' --plaintext-file=./.prod.cert.pem --location=us-central1 --keyring=logflare-prod-keyring-us-central1 --key=logflare-prod-secrets-key --project=logflare-232118" ], "encrypt.prod": [ "cmd gcloud kms encrypt --ciphertext-file='./.prod.env.enc' --plaintext-file=./.prod.env --location=us-central1 --keyring=logflare-prod-keyring-us-central1 --key=logflare-prod-secrets-key --project=logflare-232118", - "cmd gcloud kms encrypt --ciphertext-file='./gcloud_prod.json.enc' --plaintext-file=./gcloud_prod.json --location=us-central1 --keyring=logflare-prod-keyring-us-central1 --key=logflare-prod-secrets-key --project=logflare-232118" + "cmd gcloud kms encrypt --ciphertext-file='./gcloud_prod.json.enc' --plaintext-file=./gcloud_prod.json --location=us-central1 --keyring=logflare-prod-keyring-us-central1 --key=logflare-prod-secrets-key --project=logflare-232118", + "cmd gcloud kms encrypt --ciphertext-file='./.prod.cacert.key.enc' --plaintext-file=./.prod.cacert.key --location=us-central1 --keyring=logflare-prod-keyring-us-central1 --key=logflare-prod-secrets-key --project=logflare-232118", + "cmd gcloud kms encrypt --ciphertext-file='./.prod.cacert.pem.enc' --plaintext-file=./.prod.cacert.pem --location=us-central1 --keyring=logflare-prod-keyring-us-central1 --key=logflare-prod-secrets-key --project=logflare-232118", + "cmd gcloud kms encrypt --ciphertext-file='./.prod.cert.key.enc' --plaintext-file=./.prod.cert.key --location=us-central1 --keyring=logflare-prod-keyring-us-central1 --key=logflare-prod-secrets-key --project=logflare-232118", + "cmd gcloud kms encrypt --ciphertext-file='./.prod.cert.pem.enc' --plaintext-file=./.prod.cert.pem --location=us-central1 --keyring=logflare-prod-keyring-us-central1 --key=logflare-prod-secrets-key --project=logflare-232118" ] ] end diff --git a/mix.lock b/mix.lock index e4cb6c278..89afb8386 100644 --- a/mix.lock +++ b/mix.lock @@ -11,12 +11,12 @@ "configcat": {:hex, :configcat, "2.0.1", "cffd7e6ba7a4c41e1e6bbb706379192a1be7cd848bb6b098d4ed054b13c18f9d", [:mix], [{:elixir_uuid, "~> 1.2", [hex: :elixir_uuid, repo: "hexpm", optional: false]}, {:httpoison, "~> 1.7", [hex: :httpoison, repo: "hexpm", optional: false]}, {:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "3e4a239a613d2acbcee7103a6a426c4ae52882ae65bf48cdb5c1247877b65112"}, "contex": {:hex, :contex, "0.3.0", "d390713efee604702600ba801a481bcb8534a9af43e118b29d9d37fe4495fcba", [:mix], [{:nimble_strftime, "~> 0.1.0", [hex: :nimble_strftime, repo: "hexpm", optional: false]}], "hexpm", "3fa7535cc3b265691a4eabc2707fe8622aa60a2565145a14da9aebd613817652"}, "cors_plug": {:hex, :cors_plug, "2.0.3", "316f806d10316e6d10f09473f19052d20ba0a0ce2a1d910ddf57d663dac402ae", [:mix], [{:plug, "~> 1.8", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "ee4ae1418e6ce117fc42c2ba3e6cbdca4e95ecd2fe59a05ec6884ca16d469aea"}, - "cowboy": {:hex, :cowboy, "2.9.0", "865dd8b6607e14cf03282e10e934023a1bd8be6f6bacf921a7e2a96d800cd452", [:make, :rebar3], [{:cowlib, "2.11.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "2c729f934b4e1aa149aff882f57c6372c15399a20d54f65c8d67bef583021bde"}, + "cowboy": {:hex, :cowboy, "2.10.0", "ff9ffeff91dae4ae270dd975642997afe2a1179d94b1887863e43f681a203e26", [:make, :rebar3], [{:cowlib, "2.12.1", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "3afdccb7183cc6f143cb14d3cf51fa00e53db9ec80cdcd525482f5e99bc41d6b"}, "cowboy_telemetry": {:hex, :cowboy_telemetry, "0.3.1", "ebd1a1d7aff97f27c66654e78ece187abdc646992714164380d8a041eda16754", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "3a6efd3366130eab84ca372cbd4a7d3c3a97bdfcfb4911233b035d117063f0af"}, "cowlib": {:hex, :cowlib, "2.12.1", "a9fa9a625f1d2025fe6b462cb865881329b5caff8f1854d1cbc9f9533f00e1e1", [:make, :rebar3], [], "hexpm", "163b73f6367a7341b33c794c4e88e7dbfe6498ac42dcd69ef44c5bc5507c8db0"}, "credo": {:hex, :credo, "1.7.0", "6119bee47272e85995598ee04f2ebbed3e947678dee048d10b5feca139435f75", [:mix], [{:bunt, "~> 0.2.1", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2.8", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "6839fcf63d1f0d1c0f450abc8564a57c43d644077ab96f2934563e68b8a769d7"}, "db_connection": {:hex, :db_connection, "2.5.0", "bb6d4f30d35ded97b29fe80d8bd6f928a1912ca1ff110831edcd238a1973652c", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "c92d5ba26cd69ead1ff7582dbb860adeedfff39774105a4f1c92cbb654b55aa2"}, - "decimal": {:hex, :decimal, "2.0.0", "a78296e617b0f5dd4c6caf57c714431347912ffb1d0842e998e9792b5642d697", [:mix], [], "hexpm", "34666e9c55dea81013e77d9d87370fe6cb6291d1ef32f46a1600230b1d44f577"}, + "decimal": {:hex, :decimal, "2.1.1", "5611dca5d4b2c3dd497dec8f68751f1f1a54755e8ed2a966c2633cf885973ad6", [:mix], [], "hexpm", "53cfe5f497ed0e7771ae1a475575603d77425099ba5faef9394932b35020ffcc"}, "decorator": {:hex, :decorator, "1.4.0", "a57ac32c823ea7e4e67f5af56412d12b33274661bb7640ec7fc882f8d23ac419", [:mix], [], "hexpm", "0a07cedd9083da875c7418dea95b78361197cf2bf3211d743f6f7ce39656597f"}, "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"}, "dialyxir": {:hex, :dialyxir, "1.3.0", "fd1672f0922b7648ff9ce7b1b26fcf0ef56dda964a459892ad15f6b4410b5284", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "00b2a4bcd6aa8db9dcb0b38c1225b7277dca9bc370b6438715667071a304696f"}, @@ -85,15 +85,17 @@ "open_api_spex": {:hex, :open_api_spex, "3.16.2", "1b942ad736c1eb1a3a59bcb1aa626fb658ec5c56219ff94370121c57b5a75001", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:poison, "~> 3.0 or ~> 4.0 or ~> 5.0", [hex: :poison, repo: "hexpm", optional: true]}, {:ymlr, "~> 2.0 or ~> 3.0", [hex: :ymlr, repo: "hexpm", optional: true]}], "hexpm", "59c42ba79e9ae0dafa2668a6f8f73423a04a15353823510692e5c147a825104b"}, "parse_trans": {:hex, :parse_trans, "3.3.1", "16328ab840cc09919bd10dab29e431da3af9e9e7e7e6f0089dd5a2d2820011d8", [:rebar3], [], "hexpm", "07cd9577885f56362d414e8c4c4e6bdf10d43a8767abb92d24cbe8b24c54888b"}, "pgoutput_decoder": {:hex, :pgoutput_decoder, "0.1.0", "d4ffae6e58a563f2e6de8a0495d9f9afbe2f4ac75d6805419cd4a0d05f414c00", [:mix], [], "hexpm", "4dbecbe4eb8de728178fd129ccba810bccafa9a8769c6714c8b7b22963081c27"}, - "phoenix": {:hex, :phoenix, "1.5.14", "2d5db884be496eefa5157505ec0134e66187cb416c072272420c5509d67bf808", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_html, "~> 2.13 or ~> 3.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.0", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:plug, "~> 1.10", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 1.0 or ~> 2.2", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.1.2 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "207f1aa5520320cbb7940d7ff2dde2342162cf513875848f88249ea0ba02fef7"}, + "phoenix": {:hex, :phoenix, "1.6.16", "e5bdd18c7a06da5852a25c7befb72246de4ddc289182285f8685a40b7b5f5451", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.0", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 1.0 or ~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: false]}, {:plug, "~> 1.10", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.2", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "e15989ff34f670a96b95ef6d1d25bad0d9c50df5df40b671d8f4a669e050ac39"}, "phoenix_ecto": {:hex, :phoenix_ecto, "4.4.0", "0672ed4e4808b3fbed494dded89958e22fb882de47a97634c0b13e7b0b5f7720", [:mix], [{:ecto, "~> 3.3", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.14.2 or ~> 3.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "09864e558ed31ee00bd48fcc1d4fc58ae9678c9e81649075431e69dbabb43cc1"}, - "phoenix_html": {:hex, :phoenix_html, "2.14.3", "51f720d0d543e4e157ff06b65de38e13303d5778a7919bcc696599e5934271b8", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "efd697a7fff35a13eeeb6b43db884705cba353a1a41d127d118fda5f90c8e80f"}, + "phoenix_html": {:hex, :phoenix_html, "3.0.4", "232d41884fe6a9c42d09f48397c175cd6f0d443aaa34c7424da47604201df2e1", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "ce17fd3cf815b2ed874114073e743507704b1f5288bb03c304a77458485efc8b"}, "phoenix_live_dashboard": {:hex, :phoenix_live_dashboard, "0.3.6", "6d031e9e5fa8c671e582539e8acd549c4d6e0e90aa704f6644a4a1f5fb334608", [:mix], [{:ecto_psql_extras, "~> 0.4.1 or ~> 0.5", [hex: :ecto_psql_extras, repo: "hexpm", optional: true]}, {:phoenix_html, "~> 2.14.1 or ~> 2.15", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:phoenix_live_view, "~> 0.14.3", [hex: :phoenix_live_view, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.4.0 or ~> 0.5.0 or ~> 0.6.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "55c891eb9cb344d6685c21f452806f54be6a660bbc090c94f65f287e8b4de002"}, "phoenix_live_react": {:hex, :phoenix_live_react, "0.4.2", "8a37f3cccd26c3d992ffd677a7b8ecbd7746980caf560ac5d9c5efb7eb260910", [:mix], [{:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.11 or ~> 3.0", [hex: :phoenix_html, repo: "hexpm", optional: false]}], "hexpm", "eb0fb004afc0904510915b6b3e9458fc3144694dcd5a736d9d2feb00197becdc"}, "phoenix_live_reload": {:hex, :phoenix_live_reload, "1.4.1", "2aff698f5e47369decde4357ba91fc9c37c6487a512b41732818f2204a8ef1d3", [:mix], [{:file_system, "~> 0.2.1 or ~> 0.3", [hex: :file_system, repo: "hexpm", optional: false]}, {:phoenix, "~> 1.4", [hex: :phoenix, repo: "hexpm", optional: false]}], "hexpm", "9bffb834e7ddf08467fe54ae58b5785507aaba6255568ae22b4d46e2bb3615ab"}, - "phoenix_live_view": {:hex, :phoenix_live_view, "0.15.7", "09720b8e5151b3ca8ef739cd7626d4feb987c69ba0b509c9bbdb861d5a365881", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix, "~> 1.5.7", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.14", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.2 or ~> 0.5", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "3a756cf662420272d0f1b3b908cce5222163b5a95aa9bab404f9d29aff53276e"}, - "phoenix_oauth2_provider": {:hex, :phoenix_oauth2_provider, "0.5.1", "6c9bf283ad50ce46714f27b4c7bc588f907b2d97be685638395c6e7af2fdc29c", [:mix], [{:ex_oauth2_provider, "~> 0.5.1", [hex: :ex_oauth2_provider, repo: "hexpm", optional: false]}, {:phoenix, "~> 1.4", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.0", [hex: :phoenix_html, repo: "hexpm", optional: false]}], "hexpm", "745d894c91929157103ae415a0863a39ad32b05fecf4998a7fbec7d720c8bde9"}, + "phoenix_live_view": {:hex, :phoenix_live_view, "0.16.4", "5692edd0bac247a9a816eee7394e32e7a764959c7d0cf9190662fc8b0cd24c97", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix, "~> 1.5.9 or ~> 1.6.0", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 3.0", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.2 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "754ba49aa2e8601afd4f151492c93eb72df69b0b9856bab17711b8397e43bba0"}, + "phoenix_oauth2_provider": {:git, "https://github.com/Logflare/phoenix_oauth2_provider.git", "2b4741acf2dbfb40e615e91a57679448d4d6e200", []}, "phoenix_pubsub": {:hex, :phoenix_pubsub, "2.0.0", "a1ae76717bb168cdeb10ec9d92d1480fec99e3080f011402c0a2d68d47395ffb", [:mix], [], "hexpm", "c52d948c4f261577b9c6fa804be91884b381a7f8f18450c5045975435350f771"}, + "phoenix_template": {:hex, :phoenix_template, "1.0.1", "85f79e3ad1b0180abb43f9725973e3b8c2c3354a87245f91431eec60553ed3ef", [:mix], [{:phoenix_html, "~> 2.14.2 or ~> 3.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}], "hexpm", "157dc078f6226334c91cb32c1865bf3911686f8bcd6bcff86736f6253e6993ee"}, + "phoenix_view": {:hex, :phoenix_view, "2.0.2", "6bd4d2fd595ef80d33b439ede6a19326b78f0f1d8d62b9a318e3d9c1af351098", [:mix], [{:phoenix_html, "~> 2.14.2 or ~> 3.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}], "hexpm", "a929e7230ea5c7ee0e149ffcf44ce7cf7f4b6d2bfe1752dd7c084cdff152d36f"}, "plug": {:hex, :plug, "1.14.2", "cff7d4ec45b4ae176a227acd94a7ab536d9b37b942c8e8fa6dfc0fff98ff4d80", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "842fc50187e13cf4ac3b253d47d9474ed6c296a8732752835ce4a86acdf68d13"}, "plug_cowboy": {:hex, :plug_cowboy, "2.6.1", "9a3bbfceeb65eff5f39dab529e5cd79137ac36e913c02067dba3963a26efe9b2", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "de36e1a21f451a18b790f37765db198075c25875c64834bcc82d90b309eb6613"}, "plug_crypto": {:hex, :plug_crypto, "1.2.5", "918772575e48e81e455818229bf719d4ab4181fcbf7f85b68a35620f78d89ced", [:mix], [], "hexpm", "26549a1d6345e2172eb1c233866756ae44a9609bd33ee6f99147ab3fd87fd842"}, @@ -107,7 +109,7 @@ "rustler": {:hex, :rustler, "0.25.0", "32526b51af7e58a740f61941bf923486ce6415a91c3934cc16c281aa201a2240", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:toml, "~> 0.6", [hex: :toml, repo: "hexpm", optional: false]}], "hexpm", "6b43a11a37fe79c6234d88c4102ab5dfede7a6a764dc5c7b539956cfa02f3cf4"}, "scrivener": {:hex, :scrivener, "2.7.2", "1d913c965ec352650a7f864ad7fd8d80462f76a32f33d57d1e48bc5e9d40aba2", [:mix], [], "hexpm", "7866a0ec4d40274efbee1db8bead13a995ea4926ecd8203345af8f90d2b620d9"}, "scrivener_ecto": {:hex, :scrivener_ecto, "2.7.0", "cf64b8cb8a96cd131cdbcecf64e7fd395e21aaa1cb0236c42a7c2e34b0dca580", [:mix], [{:ecto, "~> 3.3", [hex: :ecto, repo: "hexpm", optional: false]}, {:scrivener, "~> 2.4", [hex: :scrivener, repo: "hexpm", optional: false]}], "hexpm", "e809f171687806b0031129034352f5ae44849720c48dd839200adeaf0ac3e260"}, - "scrivener_html": {:hex, :scrivener_html, "1.8.1", "d2d287cac72bd5405f1d1afe897fffd0f6ff2f2c8874384dda3bea74397e78d6", [:mix], [{:phoenix, ">= 1.0.0 and < 1.5.0", [hex: :phoenix, repo: "hexpm", optional: true]}, {:phoenix_html, "~> 2.2", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:plug, "~> 1.1", [hex: :plug, repo: "hexpm", optional: false]}, {:scrivener, "~> 1.2 or ~> 2.0", [hex: :scrivener, repo: "hexpm", optional: false]}], "hexpm", "04f1bdc463f770a046db0377891f4ee85022259bdb4a9142c64848f8714a3f38"}, + "scrivener_html": {:git, "https://github.com/Logflare/scrivener_html.git", "9224d1f3315e47cb13fa1ed403d3985474a58afa", []}, "scrivener_list": {:hex, :scrivener_list, "2.0.1", "2b3b5c6aaf21d13b76071e755af498b641f37a069e34e68585ba4c624095d719", [:mix], [{:scrivener_ecto, "~> 1.0 or ~> 2.0", [hex: :scrivener_ecto, repo: "hexpm", optional: false]}], "hexpm", "dc82a317268e24b29891b2de659cd82d15654f49ceee86846de2c96b3c4f4e5d"}, "sleeplocks": {:hex, :sleeplocks, "1.1.2", "d45aa1c5513da48c888715e3381211c859af34bee9b8290490e10c90bb6ff0ca", [:rebar3], [], "hexpm", "9fe5d048c5b781d6305c1a3a0f40bb3dfc06f49bf40571f3d2d0c57eaa7f59a5"}, "sobelow": {:hex, :sobelow, "0.12.2", "45f4d500e09f95fdb5a7b94c2838d6b26625828751d9f1127174055a78542cf5", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "2f0b617dce551db651145662b84c8da4f158e7abe049a76daaaae2282df01c5d"}, diff --git a/priv/supabase/endpoints/charts.usage.sql b/priv/supabase/endpoints/charts.usage.sql deleted file mode 100644 index 620b210fd..000000000 --- a/priv/supabase/endpoints/charts.usage.sql +++ /dev/null @@ -1,60 +0,0 @@ -with arr as ( - SELECT `supabase-analytics-eu.logflare.generate_timestamp_trunc_array`(( - CASE - WHEN @interval = 'hourly' THEN 1 - WHEN @interval = 'daily' THEN 7 - WHEN @interval = 'minutely' THEN 1 - ELSE 1 - END - ), ( - CASE - WHEN @interval = 'hourly' THEN "day_hour" - WHEN @interval = 'daily' THEN "day_day" - WHEN @interval = 'minutely' THEN "hour_minute" - ELSE "day_hour" - END - )) as d -), -dates as ( - select - d, - arr.d as arr_d - from arr, unnest(`arr`.d) as d - where d != arr.d[offset(0)] -), -logs as ( -select - ( - CASE - WHEN @interval = 'hourly' THEN timestamp_trunc(f0.timestamp, hour) - WHEN @interval = 'daily' THEN timestamp_trunc(f0.timestamp, day) - WHEN @interval = 'minutely' THEN timestamp_trunc(f0.timestamp, minute) - ELSE timestamp_trunc(f0.timestamp, hour) - END -) as timestamp, - COUNTIF(REGEXP_CONTAINS(f2.path, '/rest')) as total_rest_requests, - COUNTIF(REGEXP_CONTAINS(f2.path, '/storage')) as total_storage_requests, - COUNTIF(REGEXP_CONTAINS(f2.path, '/auth')) as total_auth_requests, - COUNTIF(REGEXP_CONTAINS(f2.path, '/realtime')) as total_realtime_requests, -FROM - `cloudflare.logs.prod` as f0 - LEFT JOIN UNNEST(metadata) AS f1 ON TRUE - LEFT JOIN UNNEST(f1.request) AS f2 ON TRUE -WHERE - project = @project -GROUP BY timestamp -) -SELECT - dates.d as timestamp, - COALESCE(SUM(logs.total_rest_requests), 0) as total_rest_requests, - COALESCE(SUM(logs.total_storage_requests), 0) as total_storage_requests, - COALESCE(SUM(logs.total_auth_requests), 0) as total_auth_requests, - COALESCE(SUM(logs.total_realtime_requests), 0) as total_realtime_requests -FROM - dates - LEFT JOIN logs on dates.d = logs.timestamp - and timestamp >= dates.arr_d[offset(0)] -GROUP BY - timestamp -ORDER BY - timestamp asc; \ No newline at end of file diff --git a/priv/supabase/endpoints/usage.api-counts.sql b/priv/supabase/endpoints/usage.api-counts.sql new file mode 100644 index 000000000..12d7c8803 --- /dev/null +++ b/priv/supabase/endpoints/usage.api-counts.sql @@ -0,0 +1,43 @@ +with +dates as ( + select (case + when @interval = 'hourly' then timestamp_sub(current_timestamp(), interval 1 hour) + when @interval = 'daily' then timestamp_sub(current_timestamp(), interval 7 day) + when @interval = 'minutely' then timestamp_sub(current_timestamp(), interval 60 minute) + end) as start +), +chart_counts as ( +select + (case + when @interval = 'hourly' then timestamp_trunc(f0.timestamp, hour) + when @interval = 'daily' then timestamp_trunc(f0.timestamp, day) + when @interval = 'minutely' then timestamp_trunc(f0.timestamp, minute) + end + ) as timestamp, + COUNTIF(REGEXP_CONTAINS(f2.path, '/rest')) as total_rest_requests, + COUNTIF(REGEXP_CONTAINS(f2.path, '/storage')) as total_storage_requests, + COUNTIF(REGEXP_CONTAINS(f2.path, '/auth')) as total_auth_requests, + COUNTIF(REGEXP_CONTAINS(f2.path, '/realtime')) as total_realtime_requests, +FROM + dates, + `cloudflare.logs.prod` as f0 + LEFT JOIN UNNEST(metadata) AS f1 ON TRUE + LEFT JOIN UNNEST(f1.request) AS f2 ON TRUE +where + REGEXP_CONTAINS(f2.url, @project) AND f0.timestamp >= dates[0] + -- project = @project +GROUP BY + timestamp +) +SELECT + datetime(chart_counts.timestamp, 'UTC') as timestamp, + COALESCE(SUM(chart_counts.total_rest_requests), 0) as total_rest_requests, + COALESCE(SUM(chart_counts.total_storage_requests), 0) as total_storage_requests, + COALESCE(SUM(chart_counts.total_auth_requests), 0) as total_auth_requests, + COALESCE(SUM(chart_counts.total_realtime_requests), 0) as total_realtime_requests, +FROM + chart_counts +GROUP BY + timestamp +ORDER BY + timestamp asc; \ No newline at end of file