Merge branch 'master' of https://github.com/open-goal/jak-project

Author: ImaDoor2785

.github/workflows/inform-pages-repo.yaml

@@ -63,7 +63,7 @@ jobs:
           python ./scripts/docs/append-file-docs.py
 
       - name: Upload Docs Artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: opengoal-docs
           if-no-files-found: error

.github/workflows/linux-build-clang.yaml

@@ -71,7 +71,7 @@ jobs:
           strip ./build/lsp/lsp
 
       - name: Upload artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: ${{ inputs.uploadArtifacts }}
         with:
           name: opengoal-linux-${{ inputs.cachePrefix }}

.github/workflows/macos-build-arm.yaml

@@ -55,7 +55,7 @@ jobs:
         run: ./test.sh
 
       - name: Upload artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: opengoal-macos-${{ inputs.cachePrefix }}
           if-no-files-found: error

.github/workflows/macos-build.yaml

@@ -64,7 +64,7 @@ jobs:
           strip ./build/lsp/lsp
 
       - name: Upload artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: ${{ inputs.uploadArtifacts }}
         with:
           name: opengoal-macos-${{ inputs.cachePrefix }}

.github/workflows/release-pipeline.yaml

@@ -51,7 +51,7 @@ jobs:
       - name: Prepare Artifact Folder
         run: mkdir -p ./ci-artifacts/final
 
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         name: Download all Artifacts
         with:
           path: ./ci-artifacts/

.github/workflows/windows-build-clang.yaml

@@ -71,7 +71,7 @@ jobs:
         run: ./build/bin/goalc-test.exe --gtest_color=yes --gtest_brief=0 --gtest_filter="-*MANUAL_TEST*"
 
       - name: Upload artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: ${{ inputs.uploadArtifacts }}
         with:
           name: opengoal-windows-${{ inputs.cachePrefix }}

.prettierrc

@@ -0,0 +1,11 @@
+{
+  "endOfLine": "auto",
+  "overrides": [
+    {
+      "files": ["**/*.jsonc"],
+      "options": {
+        "parser": "json"
+      }
+    }
+  ]
+}

.prettierrc.json

@@ -68,6 +68,18 @@
         "jak2"
       ]
     },
+    {
+      "type": "default",
+      "project": "CMakeLists.txt",
+      "projectTarget": "offline-test.exe (bin\\offline-test.exe)",
+      "name": "Tests - Offline Tests - Jak 3",
+      "args": [
+        "--iso_data_path",
+        "${workspaceRoot}/iso_data/jak3",
+        "--game",
+        "jak3"
+      ]
+    },
     {
       "type": "default",
       "project": "CMakeLists.txt",
@@ -218,6 +230,19 @@
         "--config-override \"{\\\"decompile_code\\\": false, \\\"levels_extract\\\": true, \\\"allowed_objects\\\": []}\""
       ]
     },
+    {
+      "type": "default",
+      "project": "CMakeLists.txt",
+      "projectTarget": "decompiler.exe (bin\\decompiler.exe)",
+      "name": "Decompiler - Jak 3",
+      "args": [
+        "${workspaceRoot}/decompiler/config/jak3/jak3_config.jsonc",
+        "${workspaceRoot}/iso_data",
+        "${workspaceRoot}/decompiler_out",
+        "--version",
+        "ntsc_v1"
+      ]
+    },
     {
       "type": "default",
       "project": "CMakeLists.txt",

.vs/launch.vs.json

@@ -108,6 +108,16 @@
       "description": "Build with Clang with Debug Symbols",
       "inherits": ["base-windows-debug", "base-clang"]
     },
+    {
+      "name": "Debug-windows-clang-static",
+      "displayName": "Windows Static Debug (clang)",
+      "description": "Build with Clang as Debug but statically linked",
+      "inherits": ["base-windows-debug", "base-clang"],
+      "cacheVariables": {
+        "STATICALLY_LINK": "true",
+        "ZYDIS_BUILD_SHARED_LIB": "OFF"
+      }
+    },
     {
       "name": "RelWithDebInfo-windows-clang",
       "displayName": "Windows RelWithDebInfo (clang)",

CMakePresets.json

@@ -74,24 +74,23 @@ tasks:
         msg: "Couldn't locate compiler executable in '{{.GOALC_BIN_RELEASE_DIR}}/goalc'"
     cmds:
       - "{{.GOALC_BIN_RELEASE_DIR}}/goalc --user-auto --game {{.GAME}}"
-  format:
+  format-json:
     desc: "Format code"
     cmds:
-      - cmd: python ./scripts/cpp/format-includes.py
-      - cmd: python ./third-party/run-clang-format/run-clang-format.py -r common decompiler game goalc test tools lsp -i
       # npm install -g prettier
       - cmd: npx prettier --write ./decompiler/config/jak1/**/*.jsonc
         ignore_error: true
       - cmd: npx prettier --write ./decompiler/config/jak2/**/*.jsonc
         ignore_error: true
-  format-json:
+      - cmd: npx prettier --write ./decompiler/config/jak3/**/*.jsonc
+        ignore_error: true
+  format:
     desc: "Format code"
     cmds:
-      # npm install -g prettier
-      - cmd: npx prettier --write ./decompiler/config/jak1/**/*.jsonc
-        ignore_error: true
-      - cmd: npx prettier --write ./decompiler/config/jak2/**/*.jsonc
-        ignore_error: true
+      - cmd: python ./scripts/cpp/format-includes.py
+      - cmd: python ./third-party/run-clang-format/run-clang-format.py -r common decompiler game goalc test tools lsp -i
+      - task: format-json
+
   # DECOMPILING
   decomp:
     cmds:
@@ -166,9 +165,6 @@ tasks:
         ignore_error: true
       - python ./scripts/update_decomp_reference.py ./failures ./test/decompiler/reference/ --game {{.GAME}}
       - task: offline-test-file
-  type-test-jak1:
-    cmds:
-      - cmd: '{{.GOALCTEST_BIN_RELEASE_DIR}}/goalc-test --gtest_brief=0 --gtest_filter="*Jak1TypeConsistency*" --gtest_break_on_failure'
   type-test:
     cmds:
-      - cmd: '{{.GOALCTEST_BIN_RELEASE_DIR}}/goalc-test --gtest_brief=0 --gtest_filter="*Jak2TypeConsistency*" --gtest_break_on_failure'
+      - cmd: '{{.GOALCTEST_BIN_RELEASE_DIR}}/goalc-test --gtest_brief=0 --gtest_filter="*{{.TYPE_CONSISTENCY_TEST_FILTER}}*" --gtest_break_on_failure'

Taskfile.yml

@@ -67,8 +67,8 @@ void apply_formatting_config(
   //
   // Otherwise, we always default to a hang.
   //
-  // NOTE - any modifications here to child elements could be superseeded later in the recursion
-  // in order to maintain your sanity, only modify things here that _arent_ touched by default
+  // NOTE - any modifications here to child elements could be superseeded later in the recursion!
+  // In order to maintain your sanity, only modify things here that _arent_ touched by default
   // configurations.  These are explicitly prepended with `parent_mutable_`
   if (!predefined_config) {
     if (curr_node.metadata.is_top_level) {
@@ -104,6 +104,8 @@ void apply_formatting_config(
   }
   // If we are hanging, lets determine the indentation width since it is based on the form itself
   if (curr_node.formatting_config.hang_forms) {
+    // TODO - this isn't being calculated for a pre-defined config
+    // TODO - another idea is to do this during consolidation
     curr_node.formatting_config.indentation_width = hang_indentation_width(curr_node);
   }
   // iterate through the refs
@@ -123,6 +125,7 @@ void apply_formatting_config(
   }
 }
 
+// TODO - this doesn't account for paren's width contribution!
 int get_total_form_inlined_width(const FormatterTreeNode& curr_node) {
   if (curr_node.token) {
     return curr_node.token->length();
@@ -191,17 +194,20 @@ bool can_node_be_inlined(const FormatterTreeNode& curr_node, int cursor_pos) {
 }
 
 std::vector<std::string> apply_formatting(const FormatterTreeNode& curr_node,
-                                          std::vector<std::string> output = {},
+                                          std::vector<std::string> /*output*/ = {},
                                           int cursor_pos = 0) {
   using namespace formatter_rules;
   if (!curr_node.token && curr_node.refs.empty()) {
     // special case to handle an empty list
+    if (curr_node.node_prefix) {
+      return {fmt::format("{}()", curr_node.node_prefix.value())};
+    }
     return {"()"};
   }
 
   // If its a token, just print the token and move on
   if (curr_node.token) {
-    return {curr_node.token.value()};
+    return {curr_node.token_str()};
   }
 
   bool inline_form = can_node_be_inlined(curr_node, cursor_pos);
@@ -220,13 +226,19 @@ std::vector<std::string> apply_formatting(const FormatterTreeNode& curr_node,
     // Add new line entry
     if (ref.token) {
       // Cleanup block-comments
-      std::string val = ref.token.value();
+      std::string val = ref.token_str();
       if (ref.metadata.node_type == "block_comment") {
         // TODO - change this sanitization to return a list of lines instead of a single new-lined
         // line
-        val = comments::format_block_comment(ref.token.value());
+        val = comments::format_block_comment(ref.token_str());
       }
       form_lines.push_back(val);
+      if (!curr_node.metadata.is_top_level && i == curr_node.refs.size() - 1 &&
+          (ref.metadata.is_comment)) {
+        // if there's an inline comment at the end of a form, we have to force the paren to the next
+        // line and do a new-line paren this is ugly, but we have no choice!
+        form_lines.push_back("");
+      }
     } else {
       // If it's not a token, we have to recursively build up the form
       // TODO - add the cursor_pos here
@@ -250,6 +262,9 @@ std::vector<std::string> apply_formatting(const FormatterTreeNode& curr_node,
       if ((next_ref.metadata.node_type == "comment" && next_ref.metadata.is_inline) ||
           (curr_node.formatting_config.has_constant_pairs &&
            constant_pairs::is_element_second_in_constant_pair(curr_node, next_ref, i + 1))) {
+        // TODO
+        // has issues with not consolidating first lines, this should probably just be moved to
+        // outside this loop for simplicity, do it later
         if (next_ref.token) {
           form_lines.at(form_lines.size() - 1) += fmt::format(" {}", next_ref.token.value());
           i++;
@@ -260,6 +275,10 @@ std::vector<std::string> apply_formatting(const FormatterTreeNode& curr_node,
           }
           i++;
         }
+        if (!curr_node.metadata.is_top_level && next_ref.metadata.node_type == "comment" &&
+            (i + 1) == (int)curr_node.refs.size()) {
+          form_lines.push_back("");
+        }
       }
     }
     // If we are at the top level, potential separate with a new line
@@ -288,6 +307,9 @@ std::vector<std::string> apply_formatting(const FormatterTreeNode& curr_node,
   // Apply necessary indentation to each line and add parens
   if (!curr_node.metadata.is_top_level) {
     std::string form_surround_start = "(";
+    if (curr_node.node_prefix) {
+      form_surround_start = fmt::format("{}(", curr_node.node_prefix.value());
+    }
     std::string form_surround_end = ")";
     form_lines[0] = fmt::format("{}{}", form_surround_start, form_lines[0]);
     form_lines[form_lines.size() - 1] =

common/formatter/formatter.cpp

@@ -78,16 +78,24 @@ FormatterTree::FormatterTree(const std::string& source, const TSNode& root_node)
   construct_formatter_tree_recursive(source, root_node, root);
 }
 
+const std::unordered_map<std::string, std::vector<std::string>> node_type_ignorable_contents = {
+    {"list_lit", {"(", ")"}},
+    {"quoting_lit", {"'"}},
+    {"unquoting_lit", {","}},
+    {"quasi_quoting_lit", {"`"}}};
+
 // TODO make an imperative version eventually
 void FormatterTree::construct_formatter_tree_recursive(const std::string& source,
                                                        TSNode curr_node,
-                                                       FormatterTreeNode& tree_node) {
+                                                       FormatterTreeNode& tree_node,
+                                                       std::optional<std::string> node_prefix) {
   if (ts_node_child_count(curr_node) == 0) {
     tree_node.refs.push_back(FormatterTreeNode(source, curr_node));
     return;
   }
   const std::string curr_node_type = ts_node_type(curr_node);
   FormatterTreeNode list_node;
+  std::optional<std::string> next_node_prefix;
   if (curr_node_type == "list_lit") {
     list_node = FormatterTreeNode();
   } else if (curr_node_type == "str_lit") {
@@ -97,22 +105,40 @@ void FormatterTree::construct_formatter_tree_recursive(const std::string& source
     tree_node.refs.push_back(FormatterTreeNode(source, curr_node));
     return;
   } else if (curr_node_type == "quoting_lit") {
-    // same story for quoted symbols
-    // TODO - expect to have to add more here
-    tree_node.refs.push_back(FormatterTreeNode(source, curr_node));
-    return;
+    next_node_prefix = "'";
+  } else if (curr_node_type == "unquoting_lit") {
+    next_node_prefix = ",";
+  } else if (curr_node_type == "quasi_quoting_lit") {
+    next_node_prefix = "`";
+  }
+  std::vector<std::string> skippable_nodes = {};
+  if (node_type_ignorable_contents.find(curr_node_type) != node_type_ignorable_contents.end()) {
+    skippable_nodes = node_type_ignorable_contents.at(curr_node_type);
   }
   for (size_t i = 0; i < ts_node_child_count(curr_node); i++) {
     const auto child_node = ts_node_child(curr_node, i);
-    // We skip parens
     const auto contents = get_source_code(source, child_node);
-    if (contents == "(" || contents == ")") {
+    bool skip_node = false;
+    for (const auto& skippable_content : skippable_nodes) {
+      if (skippable_content == contents) {
+        skip_node = true;
+        break;
+      }
+    }
+    if (skip_node) {
       continue;
     }
     if (curr_node_type == "list_lit") {
-      construct_formatter_tree_recursive(source, child_node, list_node);
+      construct_formatter_tree_recursive(source, child_node, list_node, next_node_prefix);
+      if (node_prefix) {
+        list_node.node_prefix = node_prefix;
+      }
     } else {
-      construct_formatter_tree_recursive(source, child_node, tree_node);
+      construct_formatter_tree_recursive(source, child_node, tree_node, next_node_prefix);
+      // TODO - im not sure if this is correct
+      if (node_prefix && !tree_node.refs.empty()) {
+        tree_node.refs.at(tree_node.refs.size() - 1).node_prefix = node_prefix;
+      }
     }
   }
   if (curr_node_type == "list_lit") {

common/formatter/formatter_tree.cpp

@@ -18,8 +18,9 @@
 // we really care about is:
 // - getting all the text tokens for the source code
 // - having them in a proper, nested format
-// The treesitter format is complicated and highly nested, leading to some very hard to understand
-// code. So my solution is a 2-pass format.
+//
+// TLDR - The treesitter format is complicated and highly nested, leading to some very hard to
+// understand code. So my solution is atleast a 2-pass format.
 //
 // Pass 1 - convert the AST into a simplified FormatterTree
 // Pass 2 - use the simplified tree to output the final code
@@ -37,8 +38,9 @@ class FormatterTreeNode {
   std::vector<FormatterTreeNode> refs;
   Metadata metadata;
   // The token is optional because list nodes do not contain a token, they just contain a bunch of
-  // eventually token node refs
+  // eventually-containing token node refs
   std::optional<std::string> token;
+  std::optional<std::string> node_prefix;
 
   formatter_rules::config::FormFormattingConfig formatting_config;
 
@@ -47,6 +49,15 @@ class FormatterTreeNode {
   FormatterTreeNode(const Metadata& _metadata) : metadata(_metadata){};
 
   bool is_list() const { return !token.has_value(); }
+  std::string token_str() const {
+    if (node_prefix && token) {
+      return node_prefix.value() + token.value();
+    }
+    if (token) {
+      return token.value();
+    }
+    return "";
+  }
 };
 
 // A FormatterTree has a very simple and crude tree structure where:
@@ -62,5 +73,6 @@ class FormatterTree {
  private:
   void construct_formatter_tree_recursive(const std::string& source,
                                           TSNode curr_node,
-                                          FormatterTreeNode& tree_node);
+                                          FormatterTreeNode& tree_node,
+                                          std::optional<std::string> node_prefix = {});
 };