Merge pull request #14 from Privado-Inc/release/gh_packages

Initial release for gh package build pipeline
Privado-Inc · May 27, 2024 · 95fbdc9 · 95fbdc9
2 parents 0e2e8ef + 3826fe0
commit 95fbdc9
Show file tree

Hide file tree

Showing 26 changed files with 244 additions and 36 deletions.
diff --git a/.github/workflows/gh_release.yml b/.github/workflows/gh_release.yml
@@ -0,0 +1,111 @@
+name: gh packages release
+on:
+  schedule:
+    - cron: "0 7 * * *" # run once daily
+  workflow_dispatch:    # allow to manually trigger this workflow
+jobs:
+  release:
+    if: github.repository_owner == 'Privado-Inc'
+    concurrency: release
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Set up JDK
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: '21'
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: 2.7
+      - name: Install Bundler
+        run: gem install bundler -v 2.4.22
+      - name: Delete `.rustup` directory
+        run: rm -rf /home/runner/.rustup # to save disk space
+        if: runner.os == 'Linux'
+      - name: Delete `.cargo` directory # to save disk space
+        run: rm -rf /home/runner/.cargo
+        if: runner.os == 'Linux'
+      - run: sbt scalafmtCheck test
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - run: ./testDistro.sh
+      - run: |
+          cd joern-cli/target/universal/stage
+          export GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} && ./schema-extender/test.sh
+          cd -
+      - name: sbt createDistribution
+        run: |
+          sbt createDistribution
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: sbt ciReleaseTagNextVersion publish
+        run: |
+          sbt ciReleaseTagNextVersion publish
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - run: sha512sum target/joern-cli.zip > target/joern-cli.zip.sha512
+      - name: Export ENV vars
+        run:
+          echo "LATEST_TAG=$(git describe --tags --abbrev=0)" >> $GITHUB_ENV
+      - run: sbt "querydb/runMain io.joern.dumpq.Main"
+      - name: Create Release
+        id: create_release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          tag_name: ${{ env.LATEST_TAG }}
+          release_name: ${{ env.LATEST_TAG }}
+          draft: false
+          prerelease: false
+      - name: Upload joern-install.sh
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: ./joern-install.sh
+          asset_name: joern-install.sh
+          asset_content_type: text/plain
+      - name: Upload joern-cli.zip
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: target/joern-cli.zip
+          asset_name: joern-cli.zip
+          asset_content_type: application/zip
+      - name: Upload joern-cli.zip.sha512
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: target/joern-cli.zip.sha512
+          asset_name: joern-cli.zip.sha512
+          asset_content_type: text/plain
+      - name: Upload querydb.zip
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: querydb/target/querydb.zip
+          asset_name: querydb.zip
+          asset_content_type: application/zip
+      - name: Upload querydb.json
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: /tmp/querydb.json
+          asset_name: querydb.json
+          asset_content_type: application/json
+
+
diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
@@ -36,8 +36,10 @@ jobs:
             ~/.coursier
           key: ${{ runner.os }}-sbt-${{ hashfiles('**/build.sbt') }}
       - run: sbt scalafmtCheck test
-      - run: ./testDistro.sh
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - run: export GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} && ./testDistro.sh
       - run: |
           cd joern-cli/target/universal/stage
-          ./schema-extender/test.sh
+          export GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} && ./schema-extender/test.sh
           cd -
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
@@ -52,6 +52,8 @@ jobs:
           key: ${{ runner.os }}-sbt-${{ hashfiles('**/build.sbt') }}
       - name: Compile and run tests
         run: sbt clean test
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
   formatting:
     runs-on: ubuntu-latest
     steps:
@@ -71,6 +73,8 @@ jobs:
           key: ${{ runner.os }}-sbt-${{ hashfiles('**/build.sbt') }}
       - name: Check formatting
         run: sbt scalafmtCheck Test/scalafmtCheck
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       - run: echo "Previous step failed because code is not formatted. Run 'sbt scalafmt Test/scalafmt'"
         if: ${{ failure() }}
       - name: Validate CITATION.cff
@@ -93,7 +97,7 @@ jobs:
             ~/.sbt
             ~/.coursier
           key: ${{ runner.os }}-sbt-${{ hashfiles('**/build.sbt') }}
-      - run: ./testDistro.sh
+      - run: export GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} && ./testDistro.sh
       - run: |
           mkdir /tmp/foo
           echo "int foo(int a, int b, int c, int d, int e, int f) {}" > /tmp/foo/foo.c
@@ -103,4 +107,4 @@ jobs:
           ./joern-slice data-flow -o target/slice
       - run: |
           cd joern-cli/target/universal/stage
-          ./schema-extender/test.sh
+          export GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} && ./schema-extender/test.sh
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -5,7 +5,7 @@ on:
   workflow_dispatch:    # allow to manually trigger this workflow
 jobs:
   release:
-    if: github.repository_owner == 'joernio'
+    if: github.repository_owner == 'Privado-Inc'
     concurrency: release
     runs-on: ubuntu-latest
     steps:
@@ -21,6 +21,7 @@ jobs:
       - run: echo $PGP_SECRET | base64 --decode | gpg --batch --import
         env:
           PGP_SECRET: ${{ secrets.PGP_SECRET }}
+      - run: export GPG_TTY=$(tty)
       - name: Set up Ruby
         uses: ruby/setup-ruby@v1
         with:
@@ -33,13 +34,13 @@ jobs:
       - name: Delete `.cargo` directory # to save disk space
         run: rm -rf /home/runner/.cargo
         if: runner.os == 'Linux'
-      - run: sbt scalafmtCheck test
-      - run: ./testDistro.sh
-      - run: |
-          cd joern-cli/target/universal/stage
-          ./schema-extender/test.sh
-          cd -
-      - run: sbt ciReleaseTagNextVersion ciReleaseSonatype createDistribution
+#      - run: sbt scalafmtCheck test
+#      - run: ./testDistro.sh
+#      - run: |
+#          cd joern-cli/target/universal/stage
+#          ./schema-extender/test.sh
+#          cd -
+      - run: sbt printPublishInfo ciReleaseTagNextVersion ciReleaseSonatype createDistribution
         env:
           SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }}
           SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }}

diff --git a/.github/workflows/upgrade-deps.yml b/.github/workflows/upgrade-deps.yml
@@ -37,4 +37,4 @@ jobs:
           ./joern-scan --dump
       - run: |
           cd joern-cli/target/universal/stage
-          ./schema-extender/test.sh
+          export GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} && ./schema-extender/test.sh
diff --git a/build.sbt b/build.sbt
@@ -2,7 +2,7 @@ name                     := "joern"
 ThisBuild / organization := "io.joern"
 ThisBuild / scalaVersion := "3.4.2"
 
-val cpgVersion = "1.6.13"
+val cpgVersion = "0.1.2"
 
 lazy val joerncli          = Projects.joerncli
 lazy val querydb           = Projects.querydb
@@ -84,6 +84,7 @@ createDistribution := {
 }
 
 ThisBuild / resolvers ++= Seq(
+  "Github Package Registry" at "https://maven.pkg.github.com/Privado-Inc/codepropertygraph",
   Resolver.mavenLocal,
   "Sonatype OSS" at "https://oss.sonatype.org/content/repositories/public",
   "Atlassian" at "https://packages.atlassian.com/mvn/maven-atlassian-external",
@@ -113,3 +114,6 @@ lazy val root = project
   .aggregate(aggregatedProjects *)
 
 ThisBuild / Test / packageBin / publishArtifact := true
+
+githubOwner      := "Privado-Inc"
+githubRepository := "joern"
diff --git a/console/build.sbt b/console/build.sbt
@@ -30,3 +30,6 @@ libraryDependencies ++= Seq(
 Test / compile := (Test / compile)
   .dependsOn(Projects.c2cpg / stage, Projects.jssrc2cpg / stage, Projects.swiftsrc2cpg / stage)
   .value
+
+githubOwner      := "Privado-Inc"
+githubRepository := "joern"
diff --git a/dataflowengineoss/build.sbt b/dataflowengineoss/build.sbt
@@ -17,3 +17,6 @@ Antlr4 / antlr4PackageName := Some("io.joern.dataflowengineoss")
 Antlr4 / antlr4Version     := Versions.antlr
 Antlr4 / javaSource        := (Compile / sourceManaged).value
 Compile / doc / sources ~= (_ filter (_ => false))
+
+githubOwner      := "Privado-Inc"
+githubRepository := "joern"
diff --git a/joern-cli/build.sbt b/joern-cli/build.sbt
@@ -132,3 +132,6 @@ generateScaladocs := {
 Universal / packageBin / mappings ++= sbt.Path.directory(new File("joern-cli/src/main/resources/scripts"))
 
 maintainer := "fabs@shiftleft.io"
+
+githubOwner      := "Privado-Inc"
+githubRepository := "joern"
diff --git a/joern-cli/frontends/csharpsrc2cpg/build.sbt b/joern-cli/frontends/csharpsrc2cpg/build.sbt
@@ -20,8 +20,8 @@ lazy val astGenVersion = settingKey[String]("dotnetastgen version")
 astGenVersion := appProperties.value.getString("csharpsrc2cpg.dotnetastgen_version")
 
 libraryDependencies ++= Seq(
-  "io.shiftleft"              %% "codepropertygraph" % Versions.cpg,
-  "org.scalatest"             %% "scalatest"         % Versions.scalatest % Test
+  "io.shiftleft"  %% "codepropertygraph" % Versions.cpg,
+  "org.scalatest" %% "scalatest"         % Versions.scalatest % Test
 )
 
 Compile / doc / scalacOptions ++= Seq("-doc-title", "semanticcpg apidocs", "-doc-version", version.value)
@@ -100,3 +100,6 @@ stage := Def
 
 Universal / packageName       := name.value
 Universal / topLevelDirectory := None
+
+githubOwner      := "Privado-Inc"
+githubRepository := "joern"
diff --git a/joern-cli/frontends/ghidra2cpg/build.sbt b/joern-cli/frontends/ghidra2cpg/build.sbt
@@ -23,3 +23,6 @@ enablePlugins(JavaAppPackaging, LauncherJarPlugin)
 
 fork        := true
 javaOptions := Seq("-Djava.protocol.handler.pkgs=ghidra.framework.protocol")
+
+githubOwner      := "Privado-Inc"
+githubRepository := "joern"
diff --git a/joern-cli/frontends/gosrc2cpg/build.sbt b/joern-cli/frontends/gosrc2cpg/build.sbt
@@ -9,9 +9,9 @@ name := "gosrc2cpg"
 dependsOn(Projects.dataflowengineoss % "compile->compile;test->test", Projects.x2cpg % "compile->compile;test->test")
 
 libraryDependencies ++= Seq(
-  "io.shiftleft"              %% "codepropertygraph" % Versions.cpg,
-  "org.scalatest"             %% "scalatest"         % Versions.scalatest % Test,
-  "com.lihaoyi"               %% "os-lib"            % Versions.osLib
+  "io.shiftleft"  %% "codepropertygraph" % Versions.cpg,
+  "org.scalatest" %% "scalatest"         % Versions.scalatest % Test,
+  "com.lihaoyi"   %% "os-lib"            % Versions.osLib
 )
 
 scalacOptions ++= Seq(
@@ -99,3 +99,6 @@ stage := Def
   .sequential(goAstGenSetAllPlatforms, Universal / stage)
   .andFinally(System.setProperty("ALL_PLATFORMS", "FALSE"))
   .value
+
+githubOwner      := "Privado-Inc"
+githubRepository := "joern"
diff --git a/joern-cli/frontends/javasrc2cpg/build.sbt b/joern-cli/frontends/javasrc2cpg/build.sbt
@@ -50,3 +50,6 @@ packTestCode := {
   }
 }
 packTestCode := packTestCode.triggeredBy(Test / compile).value
+
+githubOwner      := "Privado-Inc"
+githubRepository := "joern"
diff --git a/joern-cli/frontends/jimple2cpg/build.sbt b/joern-cli/frontends/jimple2cpg/build.sbt
@@ -13,3 +13,6 @@ libraryDependencies ++= Seq(
 enablePlugins(JavaAppPackaging, LauncherJarPlugin)
 trapExit    := false
 Test / fork := true
+
+githubOwner      := "Privado-Inc"
+githubRepository := "joern"
diff --git a/joern-cli/frontends/jssrc2cpg/build.sbt b/joern-cli/frontends/jssrc2cpg/build.sbt
@@ -18,8 +18,8 @@ lazy val astGenVersion = settingKey[String]("astgen version")
 astGenVersion := appProperties.value.getString("jssrc2cpg.astgen_version")
 
 libraryDependencies ++= Seq(
-  "io.shiftleft"              %% "codepropertygraph" % Versions.cpg,
-  "org.scalatest"             %% "scalatest"         % Versions.scalatest % Test
+  "io.shiftleft"  %% "codepropertygraph" % Versions.cpg,
+  "org.scalatest" %% "scalatest"         % Versions.scalatest % Test
 )
 
 Compile / doc / scalacOptions ++= Seq("-doc-title", "semanticcpg apidocs", "-doc-version", version.value)
@@ -97,3 +97,6 @@ stage := Def
 
 Universal / packageName       := name.value
 Universal / topLevelDirectory := None
+
+githubOwner      := "Privado-Inc"
+githubRepository := "joern"
diff --git a/joern-cli/frontends/kotlin2cpg/build.sbt b/joern-cli/frontends/kotlin2cpg/build.sbt
@@ -4,8 +4,8 @@ val kotlinVersion = "1.9.23"
 
 dependsOn(
   Projects.dataflowengineoss % "compile->compile;test->test",
-  Projects.x2cpg       % "compile->compile;test->test",
-  Projects.javasrc2cpg % "compile->compile;test->test"
+  Projects.x2cpg             % "compile->compile;test->test",
+  Projects.javasrc2cpg       % "compile->compile;test->test"
 )
 
 libraryDependencies ++= Seq(
@@ -24,3 +24,6 @@ libraryDependencies ++= Seq(
 enablePlugins(JavaAppPackaging, LauncherJarPlugin)
 trapExit    := false
 Test / fork := false
+
+githubOwner      := "Privado-Inc"
+githubRepository := "joern"
diff --git a/joern-cli/frontends/php2cpg/build.sbt b/joern-cli/frontends/php2cpg/build.sbt
@@ -13,11 +13,11 @@ val phpParserDlUrl =
 dependsOn(Projects.dataflowengineoss % "compile->compile;test->test", Projects.x2cpg % "compile->compile;test->test")
 
 libraryDependencies ++= Seq(
-  "com.lihaoyi"   %% "upickle"           % Versions.upickle,
-  "com.lihaoyi"   %% "ujson"             % Versions.upickle,
-  "io.shiftleft"  %% "codepropertygraph" % Versions.cpg,
+  "com.lihaoyi"       %% "upickle"             % Versions.upickle,
+  "com.lihaoyi"       %% "ujson"               % Versions.upickle,
+  "io.shiftleft"      %% "codepropertygraph"   % Versions.cpg,
   "com.github.sh4869" %% "semver-parser-scala" % Versions.semverParser,
-  "org.scalatest" %% "scalatest"         % Versions.scalatest % Test
+  "org.scalatest"     %% "scalatest"           % Versions.scalatest % Test
 )
 
 lazy val phpParseInstallTask = taskKey[Unit]("Install PHP-Parse using PHP Composer")
@@ -37,3 +37,6 @@ Compile / compile := ((Compile / compile) dependsOn phpParseInstallTask).value
 
 enablePlugins(JavaAppPackaging, LauncherJarPlugin)
 Global / onChangedBuildSource := ReloadOnSourceChanges
+
+githubOwner      := "Privado-Inc"
+githubRepository := "joern"
diff --git a/joern-cli/frontends/pysrc2cpg/build.sbt b/joern-cli/frontends/pysrc2cpg/build.sbt
@@ -27,3 +27,6 @@ javaCCTask := {
 }
 
 Compile / sourceGenerators += javaCCTask
+
+githubOwner      := "Privado-Inc"
+githubRepository := "joern"