Merge branch 'master' into kotlin/anonymous-classes

.github/workflows/comparison-results.yml

@@ -0,0 +1,149 @@
+name: Monitoring Stability and Comparing Results
+
+# Triggers when a pull_request is created
+on:
+  pull_request_target:
+    branches:
+      - "**"
+
+jobs:
+  start_workflow:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Send message to slack
+        id: initial-message
+        uses: archive/github-actions-slack@master
+        with:
+          slack-optional-parse: full
+          slack-bot-user-oauth-access-token: ${{ secrets.SLACK_TOKEN }}
+          slack-channel: ${{ secrets.SLACK_CHANNEL_ID }}
+          slack-text: "Comparison workflow started for ${{github.event.pull_request.html_url}}"
+
+      - name: Save output to env
+        id: save-output
+        run: echo "INIT_MSG_TS=${{ fromJson(steps.initial-message.outputs.slack-result).response.message.ts }}" >> $GITHUB_OUTPUT
+    outputs:
+      init_message_ts: ${{steps.save-output.outputs.INIT_MSG_TS}}
+  setup_and_scan:
+    needs: start_workflow
+    strategy:
+      matrix:
+        language: ['java-1', 'java-2' ,'python', 'js', 'ruby-1', 'ruby-2', 'go', 'kotlin']
+    continue-on-error: true
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install JDK-18
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'temurin'
+          java-version: '18'
+
+      - name: Export Java Home Path
+        run: export PATH=$JAVA_HOME/bin:$PATH
+
+      - name: Install sbt
+        run: mkdir -p ~/bin && curl -Ls https://raw.githubusercontent.com/dwijnand/sbt-extras/master/sbt > ~/bin/sbt && chmod 0755 ~/bin/sbt
+      - name: Install Python 3.10
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+
+      - name: Clone standalone-monitoring-stability/main
+        uses: actions/checkout@v3
+        with:
+          repository: Privado-Inc/standalone-monitoring-stability
+          path: ./temp/standalone-monitoring-stability
+          ref: custom-joern-build
+
+      # langauge specific repository file
+      - name: Run the script for ${{github.head_ref}} and ${{github.base_ref}}
+        run: export GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} && cd ./temp/standalone-monitoring-stability && pip install -r requirements.txt && python3 ./run.py -r ./repos/${{matrix.language}}.txt -b main -h main -guf --custom-joern True --custom-joern-base-branch ${{ github.base_ref }} --custom-joern-head-branch ${{ github.head_ref }}
+
+      - name: Run aws-export
+        run: cd ./temp/standalone-monitoring-stability/ && python3 aws-export.py ${{matrix.language}}-${{github.event.number}}
+
+      - name: Move results to a folder
+        run: cd ./temp/standalone-monitoring-stability/ && mkdir results && mv output-${{matrix.language}}-${{github.event.number}}.xlsx ./results/output-${{matrix.language}}-${{github.event.number}}.xlsx && mv ./temp/result-${{matrix.language}}-${{github.event.number}}.zip ./results/result-${{matrix.language}}-${{github.event.number}}.zip && mv slack_summary.txt ./results/slack_summary.txt
+
+      # Zip the results by name
+      - name: Zip the results
+        run: cd /home/runner/work/joern/joern/temp/standalone-monitoring-stability && zip result-${{matrix.language}}-${{github.event.number}}.zip -r ./results
+
+      - name: Set summary variable
+        run: |
+          echo "MESSAGE<<EOF" >> $GITHUB_ENV
+          echo "$(cat /home/runner/work/joern/joern/temp/standalone-monitoring-stability/results/slack_summary.txt)" >> $GITHUB_ENV
+          echo "EOF" >> $GITHUB_ENV
+
+      - name: Post results to slack
+        uses: adrey/slack-file-upload-action@master
+        with:
+          thread_ts: ${{needs.start_workflow.outputs.init_message_ts}}
+          channel: ${{ secrets.SLACK_CHANNEL_ID }} # check
+          path: "/home/runner/work/joern/joern/temp/standalone-monitoring-stability/result-${{matrix.language}}-${{github.event.number}}.zip"
+          initial_comment: "Comparison Results generated on ${{github.event.repository.name}} by PR ${{github.event.number}} from branch ${{github.head_ref}} to ${{github.base_ref}} \nPR link ${{github.event.pull_request.html_url}}\n Language: ${{matrix.language}} \nSummary Report:\n ${{ env.MESSAGE }}"
+          filetype: "zip"
+          token: ${{ secrets.SLACK_TOKEN }}
+
+      - name: Export workflow output
+        run: cd ./temp/standalone-monitoring-stability && python3 ./workflow_check.py /home/runner/work/joern/joern/temp/standalone-monitoring-stability/results/slack_summary.txt
+
+      - name: Set summary variable
+        run: |
+          echo "MESSAGE<<EOF" >> $GITHUB_ENV
+          echo "$(cat ./temp/standalone-monitoring-stability/action_result.txt)" >> $GITHUB_ENV
+          echo "EOF" >> $GITHUB_ENV 
+
+      - name: Print action result
+        run: cat ./temp/standalone-monitoring-stability/action_result.txt
+
+      - name: Upload summary file
+        uses: actions/upload-artifact@master
+        with:
+          name: ${{matrix.language}}
+          path: /home/runner/work/joern/joern/temp/standalone-monitoring-stability/results/slack_summary.txt
+
+      - name: Workflow report analysis
+        if: ${{ env.MESSAGE != 'true' }}
+        run: exit 1
+
+  collate_summary:
+    needs: [start_workflow, setup_and_scan]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Download summary file
+        uses: actions/download-artifact@master
+        with:
+          path: ./language_summary
+
+      - name: Install Python 3.10
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+
+      - name: Clone standalone-monitoring-stability/flow-test
+        uses: actions/checkout@v3
+        with:
+          repository: Privado-Inc/standalone-monitoring-stability
+          path: ./temp/standalone-monitoring-stability
+          ref: main
+
+      - name: Collate summary
+        run: cd ./temp/standalone-monitoring-stability && pip install -r requirements.txt && python3 ./collate_summary.py -s /home/runner/work/joern/joern/language_summary
+
+      - name: Set summary variable
+        run: |
+          echo "MESSAGE<<EOF" >> $GITHUB_ENV
+          echo "$(cat /home/runner/work/joern/joern/temp/standalone-monitoring-stability/global_summary.txt)" >> $GITHUB_ENV
+          echo "EOF" >> $GITHUB_ENV
+
+      - name: Send summary to slack
+        uses: slackapi/slack-github-action@v1.24.0
+        with:
+          update-ts: ${{needs.start_workflow.outputs.init_message_ts}}
+          channel-id: ${{ secrets.SLACK_CHANNEL_ID }}
+          slack-message: "\nComparison Results generated on ${{github.event.repository.name}} by PR ${{github.event.number}} from branch ${{github.head_ref}} to ${{github.base_ref}} \nPR link ${{github.event.pull_request.html_url}}\nLanguage: All \nSummary Report:\n ${{ env.MESSAGE }}"
+        env:
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_TOKEN }}
+

.github/workflows/pr.yml

@@ -107,7 +107,10 @@ jobs:
           ./joern --src /tmp/foo --run scan
           ./joern-scan /tmp/foo
           ./joern-scan --dump
-          ./joern-slice data-flow -o target/slice
+      - run: |
+          mkdir /tmp/slice
+          ./joern-slice data-flow tests/code/javasrc/SliceTest.java -o /tmp/slice/dataflow-slice-javasrc.json
+          ./joern --script "./test-dataflow-slice.sc" --param sliceFile=/tmp/slice/dataflow-slice-javasrc.json | grep -q 'List(boolean b, b, this, s, "MALICIOUS", s, new Foo("MALICIOUS"), s, s, "SAFE", s, b, this, this, b, s, System.out)'
       - run: |
           cd joern-cli/target/universal/stage
           export GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} && ./schema-extender/test.sh

.gitignore

@@ -30,7 +30,6 @@ null
 **/SwiftAstGen-mac
 **/SwiftAstGen-linux
 **/php2cpg/bin
-**/c2cpg/lib
 /foo.c
 /woo.c
 /cpg_*.bin.zip

Dockerfile

@@ -1,11 +1,11 @@
-FROM alpine:3.17.3
+FROM alpine:3.20
 
 # dependencies
 RUN apk update && apk upgrade && apk add --no-cache openjdk17-jdk python3 git curl gnupg bash nss ncurses php
 RUN ln -sf python3 /usr/bin/python
 
 # sbt
-ENV SBT_VERSION 1.8.0
+ENV SBT_VERSION 1.10.0
 ENV SBT_HOME /usr/local/sbt
 ENV PATH ${PATH}:${SBT_HOME}/bin
 RUN curl -sL "https://github.com/sbt/sbt/releases/download/v$SBT_VERSION/sbt-$SBT_VERSION.tgz" | gunzip | tar -x -C /usr/local

README.md

@@ -23,6 +23,7 @@ Specification: https://cpg.joern.io
 
 ## News / Changelog
 
+- Joern v4.0.0 [migrates from overflowdb to flatgraph](changelog/4.0.0-flatgraph.md)
 - Joern v2.0.0 [upgrades from Scala2 to Scala3](changelog/2.0.0-scala3.md)
 - Joern v1.2.0 removes the `overflowdb.traversal.Traversal` class. This change is not completely backwards compatible. See [here](changelog/traversal_removal.md) for a detailed writeup.
 
@@ -125,19 +126,3 @@ sbt stage
 ```
 The last command prints all available queries - add your own in querydb, run the above commands again to see that your query got deployed.
 More details in the [separate querydb readme](querydb/README.md)
-
-## Benchmarks
-
-Various static analysis benchmarks that measure Joern are contained under the `benchmarks`. The benchmarks are
-implemented in ScalaTest and can be run using the `joern-benchmarks` script. The benchmark results can be found on
-the `benchmarks` subproject's `README`. The currently implemented benchmarks along with the language frontends tested
-are:
-
-* [Securibench Micro](http://too4words.github.io/securibench-micro/) [`javasrc2cpg`, `jimple2cpg`]
-* [IFSpec](https://link.springer.com/chapter/10.1007/978-3-030-03638-6_27) ([paper](https://pp.ipd.kit.edu/uploads/publikationen/ifspec18nordsec.pdf)) [`javasrc2cpg`, `jimple2cpg`]
-* [JInfoFlow](https://github.com/plast-lab/JInfoFlow-bench) ([paper](https://yanniss.github.io/ptaint-oopsla17-prelim.pdf)) [`javasrc2cpg`, `jimple2cpg`]
-
-For more instructions on how to run benchmarks individually head over to the `benchmarks` subproject. If you would
-like the benchmark results to be written to a file instead of printed to STDOUT, set the path to the environment
-variable `JOERN_BENCHMARK_RESULT_FILE`.
-