test(client): benchmark client compilation (prisma#5301)

Co-authored-by: Joël Galeran <Jolg42@users.noreply.github.com>

.github/workflows/benchmark.yml

@@ -0,0 +1,81 @@
+name: Benchmark
+# Run on push only for master, if not it will trigger push & pull_request on PRs at the same time
+on:
+  push:
+    branches:
+      - master
+    paths-ignore:
+      # Any update here needs to be done for
+      # - `pull_request` see below
+      # - https://github.com/prisma/prisma/blob/master/src/.buildkite/test/buildkite-entry.sh
+      # - https://github.com/prisma/prisma/blob/master/src/.buildkite/publish/buildkite-entry.sh
+      - "*.md"
+      - ".vscode/**"
+      - "docs/**"
+      - "examples/**"
+      - "LICENSE"
+      - "src/.buildkite/**"
+      - "src/.dockerignore"
+      - "src/scripts/ci/publish.ts"
+      - "src/graphs/**"
+  pull_request:
+    paths-ignore:
+      # Any update here needs to be done for
+      # - `push`see before
+      # - https://github.com/prisma/prisma/blob/master/src/.buildkite/test/buildkite-entry.sh
+      # - https://github.com/prisma/prisma/blob/master/src/.buildkite/publish/buildkite-entry.sh
+      - "*.md"
+      - ".vscode/**"
+      - "docs/**"
+      - "examples/**"
+      - "LICENSE"
+      - "src/.buildkite/**"
+      - "src/.dockerignore"
+      - "src/scripts/ci/publish.ts"
+      - "src/graphs/**"
+jobs:
+  benchmark:
+    name: all
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Cache node modules
+        uses: actions/cache@v2
+        env:
+          cache-name: cache-node-modules
+        with:
+          path: ~/.pnpm-store
+          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/pnpm-lock.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-build-${{ env.cache-name }}-
+            ${{ runner.os }}-build-
+            ${{ runner.os }}-
+
+      - uses: actions/setup-node@v2
+        with:
+          node-version: "12"
+
+      - run: bash .github/workflows/setup.sh
+        env:
+          CI: true
+          SKIP_GIT: true
+          GITHUB_CONTEXT: ${{ toJson(github) }}
+      #  This is required as setup.sh can modify pnpm-lock.yml
+      - run: rm -f src/pnpm-lock.yaml
+      - run: rm -f ./pnpm-lock.yaml
+      - name: Run benchmarks
+        run: pnpm run bench
+      - name: Store benchmark result
+        uses: rhysd/github-action-benchmark@v1
+        with:
+          name: Benchmark.js Benchmark
+          tool: "benchmarkjs"
+          output-file-path: output.txt
+          # Use personal access token instead of GITHUB_TOKEN due to https://github.community/t5/GitHub-Actions/Github-action-not-triggering-gh-pages-upon-push/td-p/26869/highlight/false
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          auto-push: true
+          # Show alert with commit comment on detecting possible performance regression
+          alert-threshold: "200%"
+          comment-on-alert: true
+          fail-on-alert: true
+          alert-comment-cc-users: "@williamluke4"

.github/workflows/test.yml

@@ -17,6 +17,7 @@ on:
       - "LICENSE"
       - "src/.buildkite/**"
       - "src/.dockerignore"
+      - "*.bench.ts"
       - "src/scripts/ci/publish.ts"
       - "src/graphs/**"
   pull_request:
@@ -32,6 +33,7 @@ on:
       - "LICENSE"
       - "src/.buildkite/**"
       - "src/.dockerignore"
+      - "*.bench.ts"
       - "src/scripts/ci/publish.ts"
       - "src/graphs/**"
 

package.json

@@ -2,14 +2,17 @@
   "private": true,
   "scripts": {
     "lint": "ts-node scripts/lint.ts",
+    "bench": "ts-node scripts/bench.ts | tee output.txt",
     "precommit-all": "ts-node scripts/lint.ts --staged",
     "eslint": "eslint"
   },
   "devDependencies": {
     "@types/node": "14.14.25",
+    "@types/benchmark": "^2.1.0",
     "@typescript-eslint/eslint-plugin": "4.14.2",
     "@typescript-eslint/parser": "4.14.2",
     "arg": "5.0.0",
+    "benchmark": "^2.1.4",
     "chalk": "4.1.0",
     "eslint": "7.19.0",
     "eslint-config-prettier": "7.2.0",

scripts/bench.ts

@@ -0,0 +1,22 @@
+import execa from "execa";
+import globby from "globby";
+import path from "path";
+
+async function main() {
+  const benchmarks = await globby("./src/packages/**/*.bench.ts", {
+    gitignore: true,
+  });
+  await run(benchmarks);
+}
+async function run(benchmarks: string[]) {
+  for (const location of benchmarks) {
+    await execa.command(`yarn ts-node ${location}`, {
+      cwd: path.join(__dirname, `..`),
+      stdio: "inherit",
+    });
+  }
+}
+main().catch((e) => {
+  console.error(e);
+  process.exit(1);
+});

src/.buildkite/publish/buildkite-entry.sh

@@ -14,7 +14,7 @@ cd ..
 # Any update here needs to be done for 
 # - https://github.com/prisma/prisma/blob/master/.github/workflows/test.yml#L8 GitHub Actions
 # - https://github.com/prisma/prisma/blob/master/src/.buildkite/test/buildkite-entry.sh
-EXCLUDE_LIST="docs,.vscode,examples,src/graphs,README.md,LICENSE,CONTRIBUTING.md,.github"
+EXCLUDE_LIST="*.bench.ts,docs,.vscode,examples,src/graphs,README.md,LICENSE,CONTRIBUTING.md,.github"
 echo $EXCLUDE_LIST
 node last-git-changes/bin.js --exclude="$EXCLUDE_LIST"
 export CHANGED_COUNT=$(node last-git-changes/bin.js --exclude="$EXCLUDE_LIST" | wc -l)

src/.buildkite/test/buildkite-entry.sh

@@ -14,7 +14,7 @@ cd ..
 # Any update here needs to be done for 
 # - https://github.com/prisma/prisma/blob/master/.github/workflows/test.yml#L8 GitHub Actions
 # - https://github.com/prisma/prisma/blob/master/src/.buildkite/publish/buildkite-entry.sh
-EXCLUDE_LIST="docs,.vscode,examples,src/scripts/ci/publish.ts,src/graphs,README.md,LICENSE,CONTRIBUTING.md,.github"
+EXCLUDE_LIST="*.bench.ts,docs,.vscode,examples,src/scripts/ci/publish.ts,src/graphs,README.md,LICENSE,CONTRIBUTING.md,.github"
 echo $EXCLUDE_LIST
 node last-git-changes/bin.js --exclude="$EXCLUDE_LIST"
 export CHANGED_COUNT=$(node last-git-changes/bin.js --exclude="$EXCLUDE_LIST" | wc -l)

src/packages/client/.eslintignore

@@ -12,6 +12,7 @@ sandbox
 /src/__tests__/generation/node_modules
 /src/__tests__/types
 /src/__tests__/runtime-tests
+/src/__tests__/benchmarks
 /src/sandbox
 generated-dmmf.ts
 /src/__tests__/__helpers__/dmmf-types.ts

src/packages/client/jest.config.js

@@ -14,6 +14,8 @@ module.exports = {
     'index.d.ts',
     'index.js',
     'index.test-d.ts',
+    '.bench.ts',
+    '__tests__/benchmarks/',
     '__tests__/types/.*/test.ts',
     'stack.js',
     'runner.js',

src/packages/client/src/__tests__/benchmarks/huge-schema/.gitignore

@@ -0,0 +1 @@
+generated-dmmf.ts

src/packages/client/src/__tests__/benchmarks/huge-schema/builder.ts

@@ -0,0 +1,80 @@
+// This file is used to build large schemas for benchmarking purposes
+
+import fs from 'fs'
+import path from 'path'
+
+function write(location: string, data: string) {
+  if (fs.existsSync(location)) {
+    fs.unlinkSync(location)
+  }
+  fs.writeFileSync(location, data, {})
+}
+
+class Model {
+  name: string
+  body: string
+  constructor({ name, body }: { name: string; body?: string }) {
+    this.name = name
+    this.body = body ?? ``
+  }
+  public build() {
+    return `
+model ${this.name} {
+  ${this.body}
+}
+`
+  }
+}
+
+function main() {
+  let schema = `
+generator client {
+  provider        = "prisma-client-js"
+  previewFeatures = ["groupBy"]
+}
+
+datasource db {
+  provider = "postgresql"
+  url      = env("DATABASE_URL")
+  }
+  
+  `
+  const ts = `
+import { PrismaClient } from  '@prisma/client'
+const client = new PrismaClient();
+
+async function main(){
+  const a = await client.model1.findMany()
+}
+main().catch(err => console.log(err))
+`
+  const modelMap = new Map<string, Model>()
+  for (let i = 1; i < 50; i++) {
+    const modelName = `Model${i}`
+    const model = new Model({
+      name: modelName,
+      body: `   id        Int      @id @default(autoincrement())
+    int             Int
+    optionalInt     Int?
+    float           Float
+    optionalFloat   Float?
+    string          String
+    optionalString  String?
+    json            Json
+    optionalJson    Json?
+    boolean         Boolean
+    optionalBoolean Boolean?`,
+    })
+    modelMap.set(modelName, model)
+  }
+  modelMap.forEach((model) => {
+    schema += model.build()
+  })
+  const schemaPath = path.join(__dirname, 'schema.prisma')
+  const tsPath = path.join(__dirname, 'compile.ts')
+
+  write(schemaPath, schema)
+  write(tsPath, ts)
+}
+
+main()

src/packages/client/src/__tests__/benchmarks/huge-schema/compile.ts

@@ -0,0 +1,7 @@
+import { PrismaClient } from '@prisma/client'
+const client = new PrismaClient()
+
+async function main() {
+  const a = await client.model1.findMany()
+}
+main().catch((err) => console.log(err))

src/packages/client/src/__tests__/benchmarks/huge-schema/huge-schema.bench.ts

@@ -0,0 +1,41 @@
+// @ts-nocheck
+
+import Benchmark from 'benchmark'
+import path from 'path'
+import { compileFile } from '../../../utils/compileFile'
+import { generateTestClient } from '../../../utils/getTestClient'
+
+const suite = new Benchmark.Suite('typescript')
+// @ts-ignore
+suite
+  .add('client generation ~50 Models', {
+    defer: true,
+    fn: function (deferred) {
+      generateTestClient()
+        .then(() => {
+          deferred.resolve()
+        })
+        .catch((err) => {
+          console.error(err)
+          process.exit(1)
+        })
+    },
+  })
+  .add('typescript compilation ~50 Models', {
+    defer: true,
+    fn: function (deferred) {
+      compileFile(path.join(__dirname, './compile.ts'))
+        .then(() => {
+          deferred.resolve()
+        })
+        .catch((err) => {
+          console.error(err)
+          process.exit(1)
+        })
+    },
+  })
+  .on('cycle', (event) => {
+    // Output benchmark result by converting benchmark result to string
+    console.log(String(event.target))
+  })
+  .run()