Simplify and document running of scanners (#10)

* Remove `-` from plugin name + work on install.sh

* Fix install script

* Update README.

* Basic output routine for findings

* Test install and scanning

* Fix pr.yml

* Fix pr.yml

* Fix yml?

* Fix pr build?

* Fix?

* test

* test again

* Remove lihayou deps

* Remove debug lines.

.github/workflows/pr.yml

@@ -10,4 +10,9 @@ jobs:
       with:
         java-version: 1.8
     - name: Run tests
-      run: sbt test scalafmtCheck
+      run: |
+        sbt test scalafmtCheck
+        ./install.sh
+        mkdir /tmp/foo
+        echo "int foo(int a, int b, int c, int d, int e, int f) {}" > /tmp/foo/foo.c
+        ./joern --src /tmp/foo --run codequalityscanner

.github/workflows/release.yml

@@ -19,7 +19,7 @@ jobs:
           key: ${{ runner.os }}-sbt-${{ hashfiles('**/build.sbt') }}
       - run: sbt scalafmtCheck test stage
       - run: sbt ciReleaseTagNextVersion createDistribution
-      - run: sha512sum ./query-database.zip > ./query-database.zip.sha512
+      - run: sha512sum ./querydb.zip > ./querydb.zip.sha512
       - name: Export ENV vars
         run:
           echo "LATEST_TAG=$(git describe --tags --abbrev=0)" >> $GITHUB_ENV
@@ -33,21 +33,21 @@ jobs:
           release_name: ${{ env.LATEST_TAG }}
           draft: false
           prerelease: false
-      - name: Upload query-database.zip
+      - name: Upload querydb.zip
         uses: actions/upload-release-asset@v1
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
           upload_url: ${{ steps.create_release.outputs.upload_url }}
-          asset_path: ./query-database.zip
-          asset_name: query-database.zip
+          asset_path: ./querydb.zip
+          asset_name: querydb.zip
           asset_content_type: application/zip
-      - name: Upload query-database.zip.sha512
+      - name: Upload querydb.zip.sha512
         uses: actions/upload-release-asset@v1
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
           upload_url: ${{ steps.create_release.outputs.upload_url }}
-          asset_path: ./query-database.zip.sha512
-          asset_name: query-database.zip.sha512
+          asset_path: ./querydb.zip.sha512
+          asset_name: querydb.zip.sha512
           asset_content_type: text/plain

README.md

@@ -21,6 +21,36 @@ You can fork this project to build your own custom queries and
 scanners or kindly send a PR to to this repo to have them considered
 for inclusion in the default distribution.
 
+## Installing and running
+
+The installation scripts downloads joern and installs it in a sub-directory.
+The query database is installed as an extension.
+
+```
+./install.sh
+```
+
+The query database currently makes available the following scanners:
+
+* codequalityscanner - a code quality scanner for C code
+* cvulnscanner - a vulnerability scanner for C code
+
+You can run scanners as follows:
+
+```
+./joern --src path/to/code --run <scannername> --param k1=v1,...
+```
+
+For example,
+
+```
+mkdir foo
+echo "int foo(int a, int b, int c, int d, int e, int f) {}" > foo/foo.c
+./joern --src foo --run codequalityscanner
+```
+
+runs the code quality scanner and determines that the function `foo` has too many parameters.
+
 ## Database overview
 
 Each scanner is hosted in a sub package of `io.joern.scanners`, that
@@ -135,35 +165,6 @@ sbt scalafmt
 sbt test:scalafmt
 ```
 
-## Installation as a Joern Extension
-
-Make sure Joern is installed, then run
-
-```
-./install.sh
-```
-
-This will install the following scanners:
-
-* cvulnscanner - a vulnerability scanner for C code
-* codequalityscanner - a code quality scanner for C code
-
-## Running scanners
-
-You can run scanners as follows:
-
-```
-joern --src path/to/code --run <scannername> --param k1=v1,...
-```
-
-For example,
-
-```
-joern --src path/to/code --run cvulnscanner
-```
-
-runs the C vulnerability scanner on the code at `path/to/code`.
-
 ## Adding queries to existing scripts
 
 You can add queries to an existing bundles by creating a new query set

build.sbt

@@ -27,14 +27,15 @@ Universal / mappings := (Universal / mappings).value.filterNot {
     path.contains("net.sf.trove4") ||
     path.contains("com.google.guava") ||
     path.contains("org.apache.logging") ||
-    path.contains("com.google.protobuf")
+    path.contains("com.google.protobuf") ||
+    path.contains("com.lihaoyi.u")
 }
 
 lazy val createDistribution = taskKey[Unit]("Create binary distribution of extension")
 createDistribution := {
   (Universal/packageZipTarball).value
   val pkgBin = (Universal/packageBin).value
-  val dstArchive = "./query-database.zip"
+  val dstArchive = "./querydb.zip"
   IO.copy(
     List((pkgBin, file(dstArchive))),
     CopyOptions(overwrite = true, preserveLastModified = true, preserveExecutable = true)

install.sh

@@ -2,32 +2,79 @@
 set -o errexit
 set -o pipefail
 set -o nounset
+set -eu
 
-readonly JOERN_INSTALL=~/bin/joern/joern-cli
-readonly JAR_INSTALL_DIR=${JOERN_INSTALL}/lib/
+readonly JOERN_VERSION="v1.1.63"
+
+if [ "$(uname)" = 'Darwin' ]; then
+  # get script location
+  # https://unix.stackexchange.com/a/96238
+  if [ "${BASH_SOURCE:-x}" != 'x' ]; then
+    this_script=$BASH_SOURCE
+  elif [ "${ZSH_VERSION:-x}" != 'x' ]; then
+    setopt function_argzero
+    this_script=$0
+  elif eval '[[ -n ${.sh.file} ]]' 2>/dev/null; then
+    eval 'this_script=${.sh.file}'
+  else
+    echo 1>&2 "Unsupported shell. Please use bash, ksh93 or zsh."
+    exit 2
+  fi
+  relative_directory=$(dirname "$this_script")
+  SCRIPT_ABS_DIR=$(cd "$relative_directory" && pwd)
+else
+  SCRIPT_ABS_PATH=$(readlink -f "$0")
+  SCRIPT_ABS_DIR=$(dirname "$SCRIPT_ABS_PATH")
+fi
+
+# Check required tools are installed.
+check_installed() {
+  if ! type "$1" > /dev/null; then
+    echo "Please ensure you have $1 installed."
+    exit 1
+  fi
+}
+
+readonly JOERN_INSTALL=$SCRIPT_ABS_DIR/joern-inst/joern-cli
+readonly PLUGIN_DIR=${JOERN_INSTALL}/lib/
 readonly SCHEMA_SRC_DIR=schema/src/main/resources/schema/
 
 echo "Examining Joern installation..."
 
 if [ ! -d "${JOERN_INSTALL}" ]; then
     echo "Cannot find Joern installation at ${JOERN_INSTALL}"
-    echo "Please install Joern first"
-    exit
+    echo "Installing..."
+    check_installed "curl"
+    echo "https://github.com/ShiftLeftSecurity/joern/releases/download/$JOERN_VERSION/joern-cli.zip"
+    curl -L "https://github.com/ShiftLeftSecurity/joern/releases/download/$JOERN_VERSION/joern-cli.zip" -o "$SCRIPT_ABS_DIR/joern-cli.zip"
+    mkdir $SCRIPT_ABS_DIR/"joern-inst"
+    mv "joern-cli.zip" $SCRIPT_ABS_DIR/"joern-inst/"
+    pushd $SCRIPT_ABS_DIR/"joern-inst/"
+      unzip "joern-cli.zip"
+    popd
+    pushd $SCRIPT_ABS_DIR
+    ln -s joern-inst/joern-cli/joern . || true
+    ln -s joern-inst/joern-cli/joern-parse . || true
+    ln -s joern-inst/joern-cli/fuzzyc2cpg.sh . || true
+    popd
 fi
 
-echo "Compiling (sbt stage)..."
-sbt stage
-
-if compgen -G "${JAR_INSTALL_DIR}/io.joern.batteries*.jar" > /dev/null; then
-    echo "Already installed. Uninstalling first."
-    rm ${JAR_INSTALL_DIR}/io.joern.batteries*.jar
-fi
+echo "Compiling (sbt createDistribution)..."
+pushd $SCRIPT_ABS_DIR
+rm lib || true
+sbt createDistribution
+popd
 
-echo "Installing jars into: ${JAR_INSTALL_DIR}"
-cp target/universal/stage/lib/io.joern.batteries*.jar ${JAR_INSTALL_DIR}
+pushd $SCRIPT_ABS_DIR
+  ln -s joern-inst/joern-cli/lib . || true
+  ./joern --remove-plugin querydb
+  ./joern --add-plugin ./querydb.zip
+  rm lib
+popd
 
 echo "Adapting CPG schema"
 cp ${SCHEMA_SRC_DIR}/*.json ${JOERN_INSTALL}/schema-extender/schemas/
 pushd $JOERN_INSTALL
-./schema-extender.sh
+  ./schema-extender.sh
 popd
+

src/main/scala/io/joern/scanners/c/codequality/CodeQualityScanner.scala

@@ -1,5 +1,6 @@
 package io.joern.scanners.c.codequality
 
+import io.joern.scanners.language._
 import io.shiftleft.codepropertygraph.Cpg
 import io.shiftleft.passes.{CpgPass, DiffGraph}
 import io.shiftleft.semanticcpg.layers.{
@@ -32,6 +33,7 @@ class CodeQualityScanner(options: CodeQualityScannerOptions)
   override def create(context: LayerCreatorContext,
                       storeUndoInfo: Boolean): Unit = {
     runPass(new CodeQualityPass(context.cpg), context, storeUndoInfo)
+    outputFindings(context.cpg)
   }
 }
 

src/main/scala/io/joern/scanners/c/codequality/Metrics.scala

@@ -1,6 +1,6 @@
 package io.joern.scanners.c.codequality
 
-import io.joern.scanners.lib.finding
+import io.joern.scanners.language._
 import io.shiftleft.codepropertygraph.Cpg
 import io.shiftleft.codepropertygraph.generated.nodes
 import io.shiftleft.semanticcpg.language._

src/main/scala/io/joern/scanners/c/vulnscan/CVulnScanner.scala

@@ -1,5 +1,6 @@
 package io.joern.scanners.c.vulnscan
 
+import io.joern.scanners.language._
 import io.shiftleft.codepropertygraph.Cpg
 import io.shiftleft.dataflowengineoss.queryengine.EngineContext
 import io.shiftleft.passes.{CpgPass, DiffGraph}
@@ -25,6 +26,7 @@ class CScanner(options: CScannerOptions)(implicit engineContext: EngineContext)
   override def create(context: LayerCreatorContext,
                       storeUndoInfo: Boolean): Unit = {
     runPass(new CScannerPass(context.cpg), context, storeUndoInfo)
+    outputFindings(context.cpg)
   }
 }
 

src/main/scala/io/joern/scanners/c/vulnscan/CopyLoops.scala

@@ -1,6 +1,6 @@
 package io.joern.scanners.c.vulnscan
 
-import io.joern.scanners.lib.finding
+import io.joern.scanners.language._
 import io.shiftleft.codepropertygraph.Cpg
 import io.shiftleft.codepropertygraph.generated.nodes
 import io.shiftleft.semanticcpg.language._

src/main/scala/io/joern/scanners/c/vulnscan/HeapBasedOverflow.scala

@@ -1,6 +1,6 @@
 package io.joern.scanners.c.vulnscan
 
-import io.joern.scanners.lib.finding
+import io.joern.scanners.language._
 import io.shiftleft.codepropertygraph.Cpg
 import io.shiftleft.codepropertygraph.generated.nodes
 import io.shiftleft.semanticcpg.language._

src/main/scala/io/joern/scanners/c/vulnscan/IntegerTruncations.scala

@@ -3,7 +3,7 @@ package io.joern.scanners.c.vulnscan
 import io.shiftleft.codepropertygraph.Cpg
 import io.shiftleft.codepropertygraph.generated.nodes
 import io.shiftleft.semanticcpg.language._
-import io.joern.scanners.lib._
+import io.joern.scanners.language._
 
 object IntegerTruncations {
 

src/main/scala/io/joern/scanners/language/ScannerStarters.scala

@@ -0,0 +1,12 @@
+package io.joern.scanners.language
+
+import io.shiftleft.codepropertygraph.Cpg
+import io.shiftleft.codepropertygraph.generated.{NodeTypes, nodes}
+import overflowdb.traversal._
+
+class ScannerStarters(val cpg: Cpg) extends AnyVal {
+
+  def finding: Traversal[nodes.Finding] =
+    cpg.graph.nodes(NodeTypes.FINDING).cast[nodes.Finding]
+
+}

src/main/scala/io/joern/scanners/language/package.scala

@@ -0,0 +1,69 @@
+package io.joern.scanners
+
+import io.shiftleft.codepropertygraph.Cpg
+import io.shiftleft.codepropertygraph.generated.{NodeTypes, nodes}
+import overflowdb.traversal._
+import io.shiftleft.semanticcpg.language._
+
+package object language {
+
+  implicit def toScannerStarters(cpg: Cpg): ScannerStarters =
+    new ScannerStarters(cpg)
+
+  object FindingKeys {
+    val title = "title"
+    val description = "description"
+    val score = "score"
+  }
+
+  implicit class ScannerFindingStep(val traversal: Traversal[nodes.Finding])
+      extends AnyRef {
+
+    def title: Traversal[String] = traversal.map(_.title)
+
+    def description: Traversal[String] = traversal.map(_.description)
+
+    def score: Traversal[Double] = traversal.map(_.score)
+
+  }
+
+  implicit class ScannerFindingExtension(val node: nodes.Finding)
+      extends AnyRef {
+
+    def title: String = getValue(FindingKeys.title)
+
+    def description: String = getValue(FindingKeys.description)
+
+    def score: Double = getValue(FindingKeys.score).toDouble
+
+    protected def getValue(key: String, default: String = ""): String =
+      node.keyValuePairs.find(_.key == key).map(_.value).getOrElse(default)
+
+  }
+
+  def finding(evidence: nodes.StoredNode,
+              title: String,
+              description: String,
+              score: Double): nodes.NewFinding = {
+    nodes.NewFinding(
+      evidence = List(evidence),
+      keyValuePairs = List(
+        nodes.NewKeyValuePair(FindingKeys.title, title),
+        nodes.NewKeyValuePair(FindingKeys.description, description),
+        nodes.NewKeyValuePair(FindingKeys.score, score.toString)
+      )
+    )
+  }
+
+  def outputFindings(cpg: Cpg): Unit = {
+    cpg.finding.sortBy(_.score.toInt).foreach { finding =>
+      val evidence = finding.evidence.headOption
+        .map { e =>
+          s"${e.location.filename}:${e.location.lineNumber.getOrElse(0)}:${e.location.methodFullName}"
+        }
+        .getOrElse("")
+      println(s"Result: ${finding.score} : ${finding.title}: $evidence")
+    }
+  }
+
+}