From 096659c4c4538c6164cc0663829d91240f23d3d7 Mon Sep 17 00:00:00 2001
From: Navarone Feekery <13634519+navarone-feekery@users.noreply.github.com>
Date: Wed, 29 May 2024 14:49:06 +0200
Subject: [PATCH 1/5] Fix scripts and update README

---
 .buildkite/Dockerfile                         |   3 -
 Jarfile                                       |   2 +-
 README.md                                     |  59 +++-----
 lib/crawler/api/config.rb                     |   2 +-
 lib/crawler/url_validator.rb                  |   4 -
 .../url_request_check_concern.rb              |   2 +-
 script/bundle                                 |   2 +-
 script/development                            |   5 -
 script/environment                            |  16 --
 script/functions.sh                           | 139 ++----------------
 script/setup-rubies                           |  82 -----------
 11 files changed, 38 insertions(+), 278 deletions(-)
 delete mode 100755 script/development
 delete mode 100644 script/setup-rubies

diff --git a/.buildkite/Dockerfile b/.buildkite/Dockerfile
index a0550b2..924a4d9 100644
--- a/.buildkite/Dockerfile
+++ b/.buildkite/Dockerfile
@@ -3,7 +3,4 @@ RUN apt-get update && \
     apt-get install -y --no-install-recommends \
     libicu-dev netbase make
 
-# used for skipping jenv/rbenv setup
-ENV IS_DOCKER=1
-
 ENTRYPOINT [ "/bin/bash" ]
diff --git a/Jarfile b/Jarfile
index 4ac7968..3b5ed04 100644
--- a/Jarfile
+++ b/Jarfile
@@ -2,7 +2,7 @@
 # our java dependencies into vendor/jars (see https://github.com/mkristian/jar-dependencies for details)
 #
 # If you update this file, please run the following command to update the jars cache:
-#    rm -rf Jars.lock vendor/jars && script/development exec script/vendor_jars
+#    make clean install
 #
 # When adding a new dependency, please explain what it is and why we're adding it in a comment.
 #---------------------------------------------------------------------------------------------------
diff --git a/README.md b/README.md
index 3ddd251..8f5f53f 100644
--- a/README.md
+++ b/README.md
@@ -32,42 +32,29 @@ See [Crawling content](#crawling-content) for examples.
 
 #### Running from source
 
-Crawler uses `jenv` and `rbenv` to manage both java and ruby versions when running from source.
-
-1. Install `jenv` and `rbenv`
-    - [Official documentation for installing jenv](https://www.jenv.be/)
-    - [Official documentation for installing rbenv](https://github.com/rbenv/rbenv?tab=readme-ov-file#installation)
-2. Install the required java version (check the file `.java-version`)
-   - Crawler was developed using OpenJDK, so we recommend using an OpenJDK version of Java
-     - [Instructions for installing OpenJDK](https://openjdk.org/install/)
-   - Mac users can also use `brew` to install
-        ```bash
-        # install with brew
-        $ brew install openjdk@21
- 
-        # create symlink
-        $ sudo ln -sfn \
-            /opt/homebrew/opt/openjdk@21/libexec/openjdk.jdk \
-            /Library/Java/JavaVirtualMachines/openjdk-21.jdk
-        ```
-3. Add Java version to `jenv`
-    ```bash
-    # add to jenv and update JAVA_HOME
-    $ jenv add /Library/Java/JavaVirtualMachines/openjdk-21.jdk/Contents/Home
-    $ export JAVA_HOME=$(/usr/libexec/java_home -v21)
-
-    # check java version has been correctly set by `.java-version` file
-    $ java --version
-    ```
-4. Install the required jruby version
-    ```bash
-    # rbenv is easier to use and can install a version based on `.ruby-version` file
-    $ rbenv install
-
-    # check ruby version
-    $ ruby --version
-    ```
-5. Run `make install` to install Crawler dependencies
+Crawler uses both JRuby and Java.
+We recommend using version managers for both.
+When developing Crawler we use `rbenv` and `jenv`.
+There are instructions for setting up these env managers here:
+
+- [Official documentation for installing jenv](https://www.jenv.be/)
+- [Official documentation for installing rbenv](https://github.com/rbenv/rbenv?tab=readme-ov-file#installation)
+
+Go to the root of the Crawler directory and check the expected Java and Ruby versions are being used:
+
+```bash
+# should output the same version as `.ruby-version`
+$ ruby --version
+
+# should output the same version as `.java-version`
+$ java --version
+```
+
+If the versions seem correct, you can install dependencies:
+
+```bash
+$ make install
+```
 
 Crawler should now be functional.
 See [Configuring Crawlers](#configuring-crawlers) to begin crawling web content.
diff --git a/lib/crawler/api/config.rb b/lib/crawler/api/config.rb
index 5611d5f..5bb8d1b 100644
--- a/lib/crawler/api/config.rb
+++ b/lib/crawler/api/config.rb
@@ -344,7 +344,7 @@ def document_mapper
       # Receives a crawler event object and outputs it into relevant systems
       def output_event(event)
         # Log the event
-        event_logger << "#{event.to_json}\n"
+        # event_logger << "#{event.to_json}\n"
 
         # Count stats for the crawl
         stats.update_from_event(event)
diff --git a/lib/crawler/url_validator.rb b/lib/crawler/url_validator.rb
index bb78a8c..4013e42 100644
--- a/lib/crawler/url_validator.rb
+++ b/lib/crawler/url_validator.rb
@@ -67,10 +67,6 @@ class InvalidCrawlConfigError < Error; end
     attr_reader :raw_url, :checks, :results, :url_crawl_result
 
     def initialize(url:, crawl_config:, checks: nil)
-      if configuration && configuration.crawler_domains.empty?
-        raise InvalidCrawlConfigError, 'Please configure at least one domain in the crawl config file.'
-      end
-
       @crawl_config = crawl_config
       # Default to running all checks for the given context
       checks ||= valid_checks
diff --git a/lib/crawler/url_validator/url_request_check_concern.rb b/lib/crawler/url_validator/url_request_check_concern.rb
index 0b9c076..468b26e 100644
--- a/lib/crawler/url_validator/url_request_check_concern.rb
+++ b/lib/crawler/url_validator/url_request_check_concern.rb
@@ -123,7 +123,7 @@ def redirect_validation_result(details) # rubocop:disable Metrics/AbcSize
       end
 
       # If we're running in a domain context, this is an inter-domain redirect that we cannot follow
-      unless configuration
+      unless @crawl_config
         return validation_fail(:url_request, <<~MESSAGE, details)
           The web server at #{url} redirected us to a different domain URL (#{location}).
           If you want to crawl this site, please use #{location.domain_name} as the domain name.
diff --git a/script/bundle b/script/bundle
index 2624b70..4059f12 100755
--- a/script/bundle
+++ b/script/bundle
@@ -16,7 +16,7 @@ function bundle_command() {
     set +x
     echo
     red_echo "ERROR: Bundle command failed!"
-    yellow_echo "Try to run ./script/setup-rubies and then retry this command"
+    yellow_echo "Try to run 'make install' and then retry this command"
     echo
     exit 42
   fi
diff --git a/script/development b/script/development
deleted file mode 100755
index f5f0d02..0000000
--- a/script/development
+++ /dev/null
@@ -1,5 +0,0 @@
-#!/bin/bash
-
-source "$(dirname $0)/environment.sh"
-
-togo "development" "$@"
diff --git a/script/environment b/script/environment
index 59bdaa2..8c759b0 100755
--- a/script/environment
+++ b/script/environment
@@ -5,20 +5,4 @@ source "$(dirname $0)/functions.sh"
 set -e
 
 load_version_constraints
-
-echo "JRuby version required: ${JRUBY_VERSION}"
-echo "Java version required: ${JAVA_VERSION}"
-
-if [ -n "${IS_DOCKER}" ] && [ "${IS_DOCKER}" = "1" ]; then
-  echo "Skipping jenv and rbenv setup because it isn't required for Docker setups."
-else
-  jenv_init
-  ensure_java_installed "$JAVA_VERSION"
-  jenv shell "$JAVA_VERSION"
-
-  rbenv_init
-  ensure_jruby_installed "$JRUBY_VERSION"
-  rbenv shell "$JRUBY_VERSION"
-fi
-
 check_bundle
diff --git a/script/functions.sh b/script/functions.sh
index 93dfee8..6a04edc 100644
--- a/script/functions.sh
+++ b/script/functions.sh
@@ -16,7 +16,6 @@ else
   RESET=$(tput sgr0)
 fi
 
-#---------------------------------------------------------------------------------------------------
 function yellow_echo() {
   echo "${YELLOW}${*}${RESET}"
 }
@@ -41,7 +40,6 @@ function green_echo_date() {
   green_echo "[$(date +"%Y-%m-%d %H:%M:%S")] ${*}"
 }
 
-#---------------------------------------------------------------------------------------------------
 function load_version_constraints() {
   if [[ -z "${PROJECT_ROOT}" ]]; then
     PROJECT_ROOT="$(dirname "${BASH_SOURCE[0]}")/.."
@@ -50,99 +48,21 @@ function load_version_constraints() {
   export BUNDLER_VERSION="$(cat "$PROJECT_ROOT/.bundler-version")"
   export BUNDLER_CONSTRAINT="~> $BUNDLER_VERSION"
 
-  export JRUBY_VERSION="$(cat "$PROJECT_ROOT/.ruby-version")"
+  export RUBY_VERSION="$(cat "$PROJECT_ROOT/.ruby-version")"
   export JAVA_VERSION="$(cat "$PROJECT_ROOT/.java-version")"
-}
-
-#---------------------------------------------------------------------------------------------------
-function rbenv_init() {
-  yellow_echo "Checking if rbenv is installed..."
-  if ! command -v rbenv; then
-    echo "ERROR: rbenv is not installed! Please install it by running 'brew install rbenv' (or use your OS-specific install methods)."
-    exit 2
-  fi
-  green_echo "rbenv: OK"
-  echo
-
-  yellow_echo "Enabling rbenv support..."
-  eval "$(rbenv init -)"
-  green_echo "Done!"
-  echo
-}
-
-#---------------------------------------------------------------------------------------------------
-function jenv_init() {
-  yellow_echo "Checking if jenv is installed..."
-  if ! command -v jenv; then
-    echo "ERROR: jenv is not installed! Please install it by running 'brew install jenv' (or use your OS-specific install methods)."
-    exit 2
-  fi
-  green_echo "jenv: OK"
-  echo
 
-  yellow_echo "Enabling jenv support..."
-  eval "$(jenv init -)"
-  green_echo "Done!"
-  echo
-}
-
-#---------------------------------------------------------------------------------------------------
-function nvm_init() {
-  yellow_echo "Checking if nvm is installed..."
-  if [ ! -s "$NVM_DIR/nvm.sh" ]; then
-    echo "WARNING: nvm is not installed! Skipping..."
-    return 0
-  fi
-  green_echo "nvm: OK"
-  echo
+  RUNNING_RUBY_VERSION=$(ruby --version)
+  RUNNING_JAVA_VERSION=$(java --version)
 
-  yellow_echo "Enabling nvm support..."
-  set +e
-  source "$NVM_DIR/nvm.sh"
-  nvm use
-  rt=$?
-  if [ $rt -eq 3 ]; then
-    try_then_error "Node version from .nvmrc is not installed" "nvm install"
-  elif [ $rt -ne 0 ]; then
-    red_echo "ERROR: something went wrong with 'nvm use'"
-    exit $rt
-  fi
-  set -e
-  green_echo "Done!"
-  echo
+  echo "----"
+  echo "Required Ruby version: ${RUBY_VERSION}"
+  echo "Running Ruby version: ${RUNNING_RUBY_VERSION}"
+  echo "----"
+  echo "Required Java version: ${JAVA_VERSION}"
+  echo "Running Java version: ${RUNNING_JAVA_VERSION}"
+  echo "----"
 }
 
-#---------------------------------------------------------------------------------------------------
-function ensure_java_installed() {
-  JAVA_VERSION="$1"
-  yellow_echo "Checking if JAVA $JAVA_VERSION is installed..."
-  set +e
-  if ! jenv prefix; then
-    red_echo "ERROR: Java version $JAVA_VERSION is not installed! Please install it from homebrew or use your OS-specific install methods."
-    echo
-    yellow_echo "If you are on a mac, you may need to add homebrew-installed java to jenv: "
-    echo
-    echo "  jenv add /Library/Java/JavaVirtualMachines/temurin-$JAVA_VERSION.jdk/Contents/Home && jenv rehash"
-    echo
-    exit 2
-  fi
-  set -e
-  green_echo "Done!"
-  echo
-}
-
-#---------------------------------------------------------------------------------------------------
-function ensure_jruby_installed() {
-  JRUBY_VERSION="$1"
-  yellow_echo "Checking if JRuby $JRUBY_VERSION is installed..."
-  if [ -z "$(rbenv versions --bare | grep "^$JRUBY_VERSION")" ]; then
-    try_then_error "JRuby version $JRUBY_VERSION is not installed" "script/setup-rubies"
-  fi
-  green_echo "Done!"
-  echo
-}
-
-#---------------------------------------------------------------------------------------------------
 function check_bundle() {
   yellow_echo "Checking for missing gems..."
   if ! bundle check > /dev/null; then
@@ -152,32 +72,6 @@ function check_bundle() {
   echo
 }
 
-#---------------------------------------------------------------------------------------------------
-function check_yarn() {
-  yellow_echo "Checking for missing NPM packages..."
-  if ! which yarn; then
-    red_echo "ERROR: yarn is not installed! Please install it by running 'brew install yarn' (or use your OS-specific install methods described here: https://legacy.yarnpkg.com/en/docs/install)."
-    exit 2
-  fi
-  if ! yarn check --integrity > /dev/null; then
-    try_then_error "NPM packages are missing" "yarn install"
-  fi
-  green_echo "Done!"
-  echo
-}
-
-#---------------------------------------------------------------------------------------------------
-function check_git() {
-  yellow_echo "Checking your git tags..."
-  if [ -n "$(git tag | grep -E '^(dal05|v0\.2|show)')" ]; then
-      yellow_echo "Pruning bad tags"
-      git fetch --prune-tags --prune origin
-      yellow_echo "Bad tags pruned"
-  fi
-  green_echo "Done!"
-}
-
-#---------------------------------------------------------------------------------------------------
 function try_then_error() {
   ISSUE="$1"
   COMMAND="$2"
@@ -190,7 +84,6 @@ function try_then_error() {
   fi
 }
 
-#---------------------------------------------------------------------------------------------------
 function __install_macosx_dev_deps() {
   local root_dir
 
@@ -202,7 +95,7 @@ function __install_macosx_dev_deps() {
       exit 1
     fi
 
-    # Doesn't matter where the calling script is called from, this is where we at yo
+    # Doesn't matter where the calling script is called from
     root_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/../"
 
     # Only run homebrew if dependencies have changed since last time
@@ -218,16 +111,6 @@ function __install_macosx_dev_deps() {
       echo "this is fine as the temurin11 recipe needs to install files outside of /usr/local...${RESET}"
       echo
 
-      if [[ "$(uname -s)" == 'Darwin' ]] && [[ "$(arch)" == *'arm64'* ]]
-      then
-        echo "${RED}⚠️  Warning ⚠️"
-        echo "It looks like you're running on an Apple M1 (nice), so you'll need to install Mailhog manually:${RESET}"
-        echo "${BLUE}$ go get github.com/mailhog/MailHog${RESET}"
-	echo
-        echo "${RED}And run it like so:${RESET}"
-        echo "${BLUE}$ ~/go/bin/MailHog${RESET}"
-      fi
-
       sleep 2
 
       ( cd "${root_dir}" && brew bundle --verbose )
diff --git a/script/setup-rubies b/script/setup-rubies
deleted file mode 100644
index ea288d0..0000000
--- a/script/setup-rubies
+++ /dev/null
@@ -1,82 +0,0 @@
-#!/bin/bash
-
-set -e
-
-# Load our common shell functions library
-source "$(dirname $0)/functions.sh"
-
-#---------------------------------------------------------------------------------------------------
-function bundler_installed() {
-  ruby -r bundler -e true 2> /dev/null
-}
-
-function install_bundler() {
-  yellow_echo "Installing bundler version ${BUNDLER_VERSION}"
-  gem install bundler --clear-sources --source=https://rubygems.org -v="${BUNDLER_VERSION}"
-}
-
-function reinstall_bundler() {
-  yellow_echo "Removing all installed bundler versions and installing bundler version ${BUNDLER_VERSION}..."
-  gem uninstall bundler --force --all --ignore-dependencies --executables
-  install_bundler
-}
-
-function check_bundler_version() {
-  ruby -r bundler -e "exit Gem::Requirement.new('${BUNDLER_CONSTRAINT}').satisfied_by?(Gem::Version.new(Bundler::VERSION))"
-}
-
-function ensure_bundler_version() {
-  RUBY_VERSION="$1"
-
-  yellow_echo "Checking bundler version in $RUBY_VERSION..."
-  rbenv shell "$RUBY_VERSION"
-  rbenv rehash
-
-  if ! bundler_installed; then
-    red_echo "No bundler found in $RUBY_VERSION!"
-    install_bundler
-  elif ! check_bundler_version; then
-    red_echo "ERROR: $RUBY_VERSION bundler version does not satisty the constraint: ${BUNDLER_CONSTRAINT}! Installed version: $(bundler --version)"
-    reinstall_bundler
-  fi
-
-  bundler --version
-  green_echo "Done!"
-  echo
-}
-
-#---------------------------------------------------------------------------------------------------
-BUNDLER_VERSION="$(cat .bundler-version)"
-BUNDLER_CONSTRAINT="~> $BUNDLER_VERSION"
-
-JRUBY_VERSION="$(cat .ruby-version)"
-JAVA_VERSION="$(cat .java-version)"
-
-echo "----------------------------------------------------------------------------"
-echo "Jruby version required: ${JRUBY_VERSION}"
-echo "Java version required: ${JAVA_VERSION}"
-echo "Bundler version supported: ${BUNDLER_CONSTRAINT} (recommended: ${BUNDLER_VERSION})"
-echo "----------------------------------------------------------------------------"
-echo
-
-#---------------------------------------------------------------------------------------------------
-# Enable jenv and rbenv support
-rbenv_init
-jenv_init
-
-# Make sure we have the java version we need
-ensure_java_installed "$JAVA_VERSION"
-
-#---------------------------------------------------------------------------------------------------
-yellow_echo "Making sure jruby $JRUBY_VERSION is installed..."
-jenv shell "$JAVA_VERSION"
-rbenv install --skip-existing "$JRUBY_VERSION"
-green_echo "Done!"
-echo
-
-ensure_bundler_version "$JRUBY_VERSION"
-
-#---------------------------------------------------------------------------------------------------
-echo "---------------------------------------------------------------------------------------------"
-green_echo "Your Ruby environment should be ready for working on the Crawler codebase now"
-echo "---------------------------------------------------------------------------------------------"

From 71bfcaaf2da633c5710faa93228e2b19e58e6eca Mon Sep 17 00:00:00 2001
From: Navarone Feekery <13634519+navarone-feekery@users.noreply.github.com>
Date: Wed, 29 May 2024 14:53:13 +0200
Subject: [PATCH 2/5] Remove dev code

---
 lib/crawler/api/config.rb                              | 2 +-
 lib/crawler/url_validator.rb                           | 4 ++++
 lib/crawler/url_validator/url_request_check_concern.rb | 2 +-
 3 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/lib/crawler/api/config.rb b/lib/crawler/api/config.rb
index 5bb8d1b..5611d5f 100644
--- a/lib/crawler/api/config.rb
+++ b/lib/crawler/api/config.rb
@@ -344,7 +344,7 @@ def document_mapper
       # Receives a crawler event object and outputs it into relevant systems
       def output_event(event)
         # Log the event
-        # event_logger << "#{event.to_json}\n"
+        event_logger << "#{event.to_json}\n"
 
         # Count stats for the crawl
         stats.update_from_event(event)
diff --git a/lib/crawler/url_validator.rb b/lib/crawler/url_validator.rb
index 4013e42..bb78a8c 100644
--- a/lib/crawler/url_validator.rb
+++ b/lib/crawler/url_validator.rb
@@ -67,6 +67,10 @@ class InvalidCrawlConfigError < Error; end
     attr_reader :raw_url, :checks, :results, :url_crawl_result
 
     def initialize(url:, crawl_config:, checks: nil)
+      if configuration && configuration.crawler_domains.empty?
+        raise InvalidCrawlConfigError, 'Please configure at least one domain in the crawl config file.'
+      end
+
       @crawl_config = crawl_config
       # Default to running all checks for the given context
       checks ||= valid_checks
diff --git a/lib/crawler/url_validator/url_request_check_concern.rb b/lib/crawler/url_validator/url_request_check_concern.rb
index 468b26e..0b9c076 100644
--- a/lib/crawler/url_validator/url_request_check_concern.rb
+++ b/lib/crawler/url_validator/url_request_check_concern.rb
@@ -123,7 +123,7 @@ def redirect_validation_result(details) # rubocop:disable Metrics/AbcSize
       end
 
       # If we're running in a domain context, this is an inter-domain redirect that we cannot follow
-      unless @crawl_config
+      unless configuration
         return validation_fail(:url_request, <<~MESSAGE, details)
           The web server at #{url} redirected us to a different domain URL (#{location}).
           If you want to crawl this site, please use #{location.domain_name} as the domain name.

From 3773459c4e48c8ee1b45b58797d27c4d5d283df1 Mon Sep 17 00:00:00 2001
From: Navarone Feekery <13634519+navarone-feekery@users.noreply.github.com>
Date: Fri, 31 May 2024 10:50:49 +0200
Subject: [PATCH 3/5] Make env management an option

---
 README.md           |  7 ++++
 script/functions.sh | 78 ++++++++++++++++++++++++++++++++++++++++++---
 2 files changed, 80 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index a672419..ace1b90 100644
--- a/README.md
+++ b/README.md
@@ -54,6 +54,13 @@ If the versions seem correct, you can install dependencies:
 $ make install
 ```
 
+You can also use the env variable `CRAWLER_MANAGE_ENV` to have the install script automatically check whether `rbenv` and `jenv` are installed, and that the correct versions are running on both:
+Doing this requires that you use both `rbenv` and `jenv` in your local setup.
+
+```bash
+$ CRAWLER_MANAGE_ENV=true make install
+```
+
 Crawler should now be functional.
 See [Configuring Crawlers](#configuring-crawlers) to begin crawling web content.
 
diff --git a/script/functions.sh b/script/functions.sh
index 6a04edc..79a51f1 100644
--- a/script/functions.sh
+++ b/script/functions.sh
@@ -51,16 +51,26 @@ function load_version_constraints() {
   export RUBY_VERSION="$(cat "$PROJECT_ROOT/.ruby-version")"
   export JAVA_VERSION="$(cat "$PROJECT_ROOT/.java-version")"
 
+  yellow_echo "Checking version constraints..."
+  green_echo "Required Ruby version: ${RUBY_VERSION}"
+  green_echo "Required Java version: ${JAVA_VERSION}"
+  echo
+
+  if [ "$CRAWLER_MANAGE_ENV" = "true" ]; then
+    rbenv_init
+    ensure_jruby_installed "$RUBY_VERSION"
+    jenv_init
+    ensure_java_installed "$JAVA_VERSION"
+  fi
+
   RUNNING_RUBY_VERSION=$(ruby --version)
   RUNNING_JAVA_VERSION=$(java --version)
 
-  echo "----"
-  echo "Required Ruby version: ${RUBY_VERSION}"
+  yellow_echo "Checking running versions..."
   echo "Running Ruby version: ${RUNNING_RUBY_VERSION}"
-  echo "----"
-  echo "Required Java version: ${JAVA_VERSION}"
   echo "Running Java version: ${RUNNING_JAVA_VERSION}"
-  echo "----"
+  green_echo "Done!"
+  echo
 }
 
 function check_bundle() {
@@ -72,6 +82,64 @@ function check_bundle() {
   echo
 }
 
+function rbenv_init() {
+  yellow_echo "Checking if rbenv is installed..."
+  if ! command -v rbenv; then
+    echo "ERROR: rbenv is not installed! Please install it by running 'brew install rbenv' (or use your OS-specific install methods)."
+    exit 2
+  fi
+  green_echo "rbenv: OK"
+  echo
+
+  yellow_echo "Enabling rbenv support..."
+  eval "$(rbenv init -)"
+  green_echo "Done!"
+  echo
+}
+
+function jenv_init() {
+  yellow_echo "Checking if jenv is installed..."
+  if ! command -v jenv; then
+    echo "ERROR: jenv is not installed! Please install it by running 'brew install jenv' (or use your OS-specific install methods)."
+    exit 2
+  fi
+  green_echo "jenv: OK"
+  echo
+
+  yellow_echo "Enabling jenv support..."
+  eval "$(jenv init -)"
+  green_echo "Done!"
+  echo
+}
+
+function ensure_java_installed() {
+  JAVA_VERSION="$1"
+  yellow_echo "Checking if JAVA $JAVA_VERSION is installed..."
+  set +e
+  if ! jenv prefix; then
+    red_echo "ERROR: Java version $JAVA_VERSION is not installed! Please install it from homebrew or use your OS-specific install methods."
+    echo
+    yellow_echo "If you are on a mac, you may need to add homebrew-installed java to jenv: "
+    echo
+    echo "  jenv add /Library/Java/JavaVirtualMachines/<full-java-version-name>/Contents/Home && jenv rehash"
+    echo
+    exit 2
+  fi
+  set -e
+  green_echo "Done!"
+  echo
+}
+
+function ensure_jruby_installed() {
+  RUBY_VERSION="$1"
+  yellow_echo "Checking if Ruby $RUBY_VERSION is installed..."
+  if [ -z "$(rbenv versions --bare | grep "^$RUBY_VERSION")" ]; then
+    try_then_error "Ruby version $RUBY_VERSION is not installed" "script/setup-rubies"
+  fi
+  green_echo "Done!"
+  echo
+}
+
 function try_then_error() {
   ISSUE="$1"
   COMMAND="$2"

From 0fe04f5a710713bece3d594b33cf11dc1535e885 Mon Sep 17 00:00:00 2001
From: Navarone Feekery <13634519+navarone-feekery@users.noreply.github.com>
Date: Fri, 31 May 2024 12:03:34 +0200
Subject: [PATCH 4/5] Re-implement ruby and bundler installation

---
 script/functions.sh | 94 +++++++++++++++++++++++++++++++++++----------
 1 file changed, 73 insertions(+), 21 deletions(-)

diff --git a/script/functions.sh b/script/functions.sh
index 79a51f1..50aff41 100644
--- a/script/functions.sh
+++ b/script/functions.sh
@@ -20,24 +20,16 @@ function yellow_echo() {
   echo "${YELLOW}${*}${RESET}"
 }
 
-function yellow_echo_date() {
-  yellow_echo "[$(date +"%Y-%m-%d %H:%M:%S")] ${*}"
-}
-
 function red_echo()    {
   echo "${RED}${*}${RESET}"
 }
 
-function red_echo_date() {
-  red_echo "[$(date +"%Y-%m-%d %H:%M:%S")] ${*}"
-}
-
 function green_echo()  {
   echo "${GREEN}${*}${RESET}"
 }
 
-function green_echo_date() {
-  green_echo "[$(date +"%Y-%m-%d %H:%M:%S")] ${*}"
+function blue_echo() {
+  echo "${BLUE}${*}${RESET}"
 }
 
 function load_version_constraints() {
@@ -51,7 +43,7 @@ function load_version_constraints() {
   export RUBY_VERSION="$(cat "$PROJECT_ROOT/.ruby-version")"
   export JAVA_VERSION="$(cat "$PROJECT_ROOT/.java-version")"
 
-  yellow_echo "Checking version constraints..."
+  blue_echo "Checking version constraints..."
   green_echo "Required Ruby version: ${RUBY_VERSION}"
   green_echo "Required Java version: ${JAVA_VERSION}"
   echo
@@ -59,6 +51,7 @@ function load_version_constraints() {
   if [ "$CRAWLER_MANAGE_ENV" = "true" ]; then
     rbenv_init
     ensure_jruby_installed "$RUBY_VERSION"
+    ensure_bundler_version "$RUBY_VERSION"
     jenv_init
     ensure_java_installed "$JAVA_VERSION"
   fi
@@ -66,7 +59,7 @@ function load_version_constraints() {
   RUNNING_RUBY_VERSION=$(ruby --version)
   RUNNING_JAVA_VERSION=$(java --version)
 
-  yellow_echo "Checking running versions..."
+  blue_echo "Checking running versions..."
   echo "Running Ruby version: ${RUNNING_RUBY_VERSION}"
   echo "Running Java version: ${RUNNING_JAVA_VERSION}"
   green_echo "Done!"
@@ -74,7 +67,7 @@ function load_version_constraints() {
 }
 
 function check_bundle() {
-  yellow_echo "Checking for missing gems..."
+  blue_echo "Checking for missing gems..."
   if ! bundle check > /dev/null; then
     try_then_error "Bundle is missing gems" "script/bundle"
   fi
@@ -83,7 +76,7 @@ function check_bundle() {
 }
 
 function rbenv_init() {
-  yellow_echo "Checking if rbenv is installed..."
+  blue_echo "Checking if rbenv is installed..."
   if ! command -v rbenv; then
     echo "ERROR: rbenv is not installed! Please install it by running 'brew install rbenv' (or use your OS-specific install methods)."
     exit 2
@@ -91,14 +84,14 @@ function rbenv_init() {
   green_echo "rbenv: OK"
   echo
 
-  yellow_echo "Enabling rbenv support..."
+  blue_echo "Enabling rbenv support..."
   eval "$(rbenv init -)"
   green_echo "Done!"
   echo
 }
 
 function jenv_init() {
-  yellow_echo "Checking if jenv is installed..."
+  blue_echo "Checking if jenv is installed..."
   if ! command -v jenv; then
     echo "ERROR: jenv is not installed! Please install it by running 'brew install jenv' (or use your OS-specific install methods)."
     exit 2
@@ -106,7 +99,7 @@ function jenv_init() {
   green_echo "jenv: OK"
   echo
 
-  yellow_echo "Enabling jenv support..."
+  blue_echo "Enabling jenv support..."
   eval "$(jenv init -)"
   green_echo "Done!"
   echo
@@ -114,7 +107,7 @@ function jenv_init() {
 
 function ensure_java_installed() {
   JAVA_VERSION="$1"
-  yellow_echo "Checking if JAVA $JAVA_VERSION is installed..."
+  blue_echo "Checking if JAVA $JAVA_VERSION is installed..."
   set +e
   if ! jenv prefix; then
     red_echo "ERROR: Java version $JAVA_VERSION is not installed! Please install it from homebrew or use your OS-specific install methods."
@@ -132,18 +125,77 @@ function ensure_java_installed() {
 
 function ensure_jruby_installed() {
   RUBY_VERSION="$1"
-  yellow_echo "Checking if Ruby $RUBY_VERSION is installed..."
+
+  blue_echo "Checking if RBENV_VERSION has a value..."
+  if [ -z "${RBENV_VERSION+x}" ]; then
+    green_echo "RBENV_VERSION is unset, everything looks good!"
+  else
+    if [ "$RBENV_VERSION" != "$RUBY_VERSION" ]; then
+      red_echo "RBENV_VERSION is set in env to version '$RBENV_VERSION', which is incompatible with '$RUBY_VERSION'"
+      red_echo "Please unset this variable and try again."
+      exit 2
+    else
+      yellow_echo "RBENV_VERSION is set in env to '$RBENV_VERSION', which is the same as the version in '.ruby-version'."
+      yellow_echo "This should not cause issues, however if dependency installation fails, please try unsetting this variable."
+    fi
+  fi
+  unset RBENV_VERSION
+  green_echo "Done!"
+  echo
+
+
+  blue_echo "Checking if Ruby $RUBY_VERSION is installed..."
   if [ -z "$(rbenv versions --bare | grep "^$RUBY_VERSION")" ]; then
-    try_then_error "Ruby version $RUBY_VERSION is not installed" "script/setup-rubies"
+    try_then_error "Ruby version $RUBY_VERSION is not installed" "rbenv install --skip-existing"
   fi
   green_echo "Done!"
   echo
 }
 
+function ensure_bundler_version() {
+  RUBY_VERSION="$1"
+
+  blue_echo "Checking bundler version in $RUBY_VERSION..."
+  rbenv shell "$RUBY_VERSION"
+  rbenv rehash
+
+  if ! bundler_installed; then
+    red_echo "No bundler found in $RUBY_VERSION!"
+    install_bundler
+  elif ! check_bundler_version; then
+    red_echo "ERROR: $RUBY_VERSION bundler version does not satisfy the constraint: ${BUNDLER_CONSTRAINT}! Installed version: $(bundler --version)"
+    reinstall_bundler
+  fi
+
+  bundler --version
+  green_echo "Done!"
+  echo
+}
+
+function bundler_installed() {
+  ruby -r bundler -e true 2> /dev/null
+}
+
+function check_bundler_version() {
+  ruby -r bundler -e "exit Gem::Requirement.new('${BUNDLER_CONSTRAINT}').satisfied_by?(Gem::Version.new(Bundler::VERSION))"
+}
+
+function install_bundler() {
+  blue_echo "Installing bundler version ${BUNDLER_VERSION}"
+  gem install bundler --clear-sources --source=https://rubygems.org -v="${BUNDLER_VERSION}"
+}
+
+function reinstall_bundler() {
+  blue_echo "Removing all installed bundler versions and installing bundler version ${BUNDLER_VERSION}..."
+  gem uninstall bundler --force --all --ignore-dependencies --executables
+  echo
+  install_bundler
+}
+
 function try_then_error() {
   ISSUE="$1"
   COMMAND="$2"
-  yellow_echo "$ISSUE. Running: '$COMMAND'"
+  blue_echo "$ISSUE. Running: '$COMMAND'"
   eval "$2"
   rt=$?
   if [[ $rt -ne 0 ]] ; then

From 5179f0bb3fd71c8fb68c849ec85909863be669a0 Mon Sep 17 00:00:00 2001
From: Navarone Feekery <13634519+navarone-feekery@users.noreply.github.com>
Date: Fri, 31 May 2024 12:06:26 +0200
Subject: [PATCH 5/5] Fix some echo colours

---
 script/bundle | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/script/bundle b/script/bundle
index 4059f12..100267b 100755
--- a/script/bundle
+++ b/script/bundle
@@ -25,10 +25,10 @@ function bundle_command() {
 BUNDLER_VERSION="$(cat .bundler-version)"
 BUNDLER_CONSTRAINT="~> $BUNDLER_VERSION"
 
-yellow_echo "Bundling jruby gems..."
+blue_echo "Bundling jruby gems..."
 bundle_command config cache_all true
 
-yellow_echo "Running the bundle command..."
+blue_echo "Running the bundle command..."
 bundle_command "$@"
 
 green_echo "Done!"