Debug codeql scripts

.github/node-dirty.txt

@@ -0,0 +1 @@
+https://github.com/felixge/node-dirty/tree/d7fb4d4ecf0cce144efa21b674965631a7955e61

.github/node-geo-point.txt

@@ -0,0 +1 @@
+https://github.com/rainder/node-geo-point/tree/c839d477ff7a48d1fc6574495cbbc6196161f494

.github/non_trivial_coverage.sh

@@ -61,17 +61,17 @@ report.coverage.total.statements.nonTrivialPct = Math.floor(nonTriviallyCoveredS
 fs.writeFileSync('$output', JSON.stringify(report, null, 2));
 EOF
 
-echo "Running query for computing per-refiner statistics..."
-codeql query run --output $dbdir/RefinerContributions.bqrs -d $dbdir $MY_DIR/../ql/queries/RefinerContributions.ql
-codeql bqrs decode --format json $dbdir/RefinerContributions.bqrs | \
-  jq '[
-    .["#select"].tuples[] |
-    {
-      "key": .[1],
-      "value": {
-        "passingTests": .[2],
-        "coverage": .[3],
-        "nonTrivialCoverage": .[4]
-      }
-    }
-  ] | from_entries' >$1/refiners.json
+# echo "Running query for computing per-refiner statistics..."
+# codeql query run --output $dbdir/RefinerContributions.bqrs -d $dbdir $MY_DIR/../ql/queries/RefinerContributions.ql
+# codeql bqrs decode --format json $dbdir/RefinerContributions.bqrs | \
+#   jq '[
+#     .["#select"].tuples[] |
+#     {
+#       "key": .[1],
+#       "value": {
+#         "passingTests": .[2],
+#         "coverage": .[3],
+#         "nonTrivialCoverage": .[4]
+#       }
+#     }
+#   ] | from_entries' >$1/refiners.json

.github/workflows/run-experiment.yml

@@ -77,20 +77,13 @@ jobs:
         with:
           path: testpilot
 
-      # - name: Check out CodeQL repo
-      #   uses: actions/checkout@v3
-      #   with:
-      #     repository: github/codeql
-      #     ref: codeql-cli/v2.10.0
-      #     path: codeql-repo
-
-      # - name: Install CodeQL 2.10.0
-      #   run: |
-      #     wget -q https://github.com/github/codeql-cli-binaries/releases/download/v2.10.0/codeql-linux64.zip
-      #     unzip codeql-linux64.zip
-      #     echo "$GITHUB_WORKSPACE/codeql" >> $GITHUB_PATH
-      #   env:
-      #     GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Install CodeQL 2.17.6
+        run: |
+          wget -q https://github.com/github/codeql-action/releases/download/codeql-bundle-v2.17.6/codeql-bundle-linux64.tar.gz
+          tar xzf codeql-bundle-linux64.tar.gz
+          echo "$GITHUB_WORKSPACE/codeql" >> $GITHUB_PATH
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Set up Node.js
         uses: actions/setup-node@v3
@@ -101,6 +94,8 @@ jobs:
         run: |
           cd testpilot
           npm run build
+          cd ql
+          codeql pack install
 
       - name: Checkout github package repo
         if: ${{ matrix.package.host == 'github.com' }}
@@ -154,14 +149,14 @@ jobs:
           npm run build || npm run prepack || echo 'Error with npm run build and npm run prepack'
           npm i --no-save mocha
 
-      # - name: Create CodeQL database
-      #   if: ${{ needs.setup.outputs.snippetsFrom == 'code' || needs.setup.outputs.snippetsFrom == 'both' }}
-      #   run: |
-      #     codeql database create --language=javascript "--source-root=$TESTPILOT_PACKAGE_PATH" -- ./db
+      - name: Create CodeQL database
+        if: ${{ needs.setup.outputs.snippetsFrom == 'code' || needs.setup.outputs.snippetsFrom == 'both' }}
+        run: |
+          codeql database create --language=javascript "--source-root=$TESTPILOT_PACKAGE_PATH" -- ./db
 
-      - name: Setup tmate session
-        uses: mxschmitt/action-tmate@v3
-        if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }}    
+      # - name: Setup tmate session
+      #   uses: mxschmitt/action-tmate@v3
+      #   if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }}    
 
       - name: Generate tests
         env:
@@ -192,10 +187,10 @@ jobs:
           node benchmark/editDistance.js --generatedTestsDir  $outputdir --existingTestsDir $TESTPILOT_PACKAGE_PATH --pkgName $TESTPILOT_PACKAGE_NAME
           mv similarityReport.json $outputdir
 
-      # - name: Add non-trivial coverage data
-      #   run: |
-      #     cd testpilot
-      #     ./.github/non_trivial_coverage.sh "results/$TESTPILOT_PACKAGE_NAME"
+      - name: Add non-trivial coverage data
+        run: |
+          cd testpilot
+          ./.github/non_trivial_coverage.sh "results/$TESTPILOT_PACKAGE_NAME"
 
       - name: Zip up results
         run: |
@@ -272,9 +267,13 @@ jobs:
           name: results-all
           path: baseline
 
-      # - name: Setup tmate session
-      #   uses: mxschmitt/action-tmate@v3
-      #   if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }}
+      - name: print toJson(needs.setup.outputs for debugging
+        run: |
+          echo '${{ toJson(needs.setup.outputs) }}'
+
+      - name: Setup tmate session
+        uses: mxschmitt/action-tmate@v3
+        if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }}
 
       - name: Generate report
         run: |
@@ -291,4 +290,4 @@ jobs:
           else
             baseline_artifact=''
           fi
-          node ${GITHUB_WORKSPACE}/benchmark/generate_report.js config.json results/results $baseline_artifact > $GITHUB_STEP_SUMMARY
+          node ${GITHUB_WORKSPACE}/benchmark/generate_report.js ${{ needs.setup.outputs.model }} config.json results/results $baseline_artifact > $GITHUB_STEP_SUMMARY

benchmark/generate_report.ts

@@ -25,21 +25,24 @@ function printCoverageReport(
 ) {
   console.log(`
 # ${title}
-Project | # Snippets Available | # Tests | # Passing Tests |  Statement coverage | # Non-trivial tests | # Non-trivial passing tests | Statement coverage by non-trivial tests
---- |  --: | --: | --: | --: | --: | --: | --:`);
+Project | # Snippets Available | # Tests | # Passing Tests |  Statement coverage | Branch coverage | # Non-trivial tests | # Non-trivial passing tests | Statement coverage by non-trivial tests
+--- |  --: | --: | --: | --: | --: | --: | --: | --:`);
   for (const {
     proj,
     nrUniqueSnippets,
     numTests,
     numPassing,
-    coverage,
+    stmtCoverage,
+    branchCoverage,
     nonTrivialTests,
     nonTrivialPassing,
     nonTrivialCoverage,
   } of Object.values(stats)) {
     console.log(
       `${proj} | ${nrUniqueSnippets} | ${numTests} | ${numPassing} | ${percentage(
-        coverage
+        stmtCoverage
+      )} | ${percentage(
+        branchCoverage
       )} | ${nonTrivialTests} | ${nonTrivialPassing} | ${percentage(
         nonTrivialCoverage
       )}`
@@ -217,33 +220,34 @@ function ppDiff(d: number | string, lowerIsBetter = false) {
   }
 }
 
-if (process.argv.length < 3 || process.argv.length > 5) {
+if (process.argv.length < 4 || process.argv.length > 6) {
   console.error(
-    "Usage: node generate_report.js [<config.json>] <artifact_dir> [<baseline_artifact_dir>]"
+    "Usage: node generate_report.js model [<config.json>] <artifact_dir> [<baseline_artifact_dir>]"
   );
   process.exit(1);
 }
-const hasConfig = fs.lstatSync(process.argv[2]).isFile();
+const model = process.argv[2];
+const hasConfig = fs.lstatSync(process.argv[3]).isFile();
 const config = hasConfig
-  ? JSON.parse(fs.readFileSync(process.argv[2], "utf8"))
+  ? JSON.parse(fs.readFileSync(process.argv[3], "utf8"))
   : {};
-const artifactDir = hasConfig ? process.argv[3] : process.argv[2];
-const baselineArtifactDir = hasConfig ? process.argv[4] : process.argv[3];
+const artifactDir = hasConfig ? process.argv[4] : process.argv[2];
+const baselineArtifactDir = hasConfig ? process.argv[5] : process.argv[4];
 
 console.log(`
 # Parameters
+- model: ${model}
+- temperatures: ${config.temperatures}
 - snippets from: ${config.snippetsFrom}
 - snippet length: ${config.snippetLength}
-- numSnippets: ${config.numSnippets}
-- temperatures: ${config.temperatures}
-- number of completions: ${config.numCompletions}`);
+- numSnippets: ${config.numSnippets}`);
 
 const { coverageStats, failureStats, refinersStats, similarityStats } =
   parseReports(artifactDir);
 
 printCoverageReport("Coverage report", coverageStats);
 printFailureReport("Failure report", failureStats);
-printRefinerReport("Coverage when excluding refiners", refinersStats);
+// printRefinerReport("Coverage when excluding refiners", refinersStats);
 printSimilarityReport(
   "Similarity of generated tests to existing tests",
   similarityStats