Merge pull request #122 from bluesliverx/main

Add docker image security scanning with trivy

README.rst

@@ -973,6 +973,18 @@ container), you must pass the --publish-ports argument to buildrunner.
 This must never be used on a shared server such as a build server as it could
 cause port mapping conflicts.
 
+Image Security Scans
+====================
+
+Pushed docker images may be automatically scanned for vulnerabilities using the ``security-scan``
+global configuration options or the ``--security-scan-*`` command line options that may override
+the global configuration.  Just set ``security-scan.enabled`` to true to enable automatic scans.
+
+The ``max-score-threshold`` may also be configured to fail the build if the max score of the
+detected vulnerabilities is greater than or equal to the ``max-score-threshold`` value.
+
+Any detected vulnerabilities are added to the ``artifacts.json`` file per Docker image platform,
+along with the detected maximum vulnerability score.
 
 Remote Builds (the 'remote' step attribute)
 ===========================================

buildrunner/__init__.py

@@ -80,7 +80,7 @@ def __init__(
         docker_timeout: int,
         local_images: bool,
         platform: Optional[str],
-        disable_multi_platform: Optional[bool],
+        global_config_overrides: dict,
     ):  # pylint: disable=too-many-statements,too-many-branches,too-many-locals,too-many-arguments
         self.build_dir = build_dir
         self.build_results_dir = build_results_dir
@@ -135,15 +135,10 @@ def __init__(
             log_generated_files=self.log_generated_files,
             build_time=self.build_time,
             tmp_files=self.tmp_files,
+            global_config_overrides=global_config_overrides,
         )
         buildrunner_config = BuildRunnerConfig.get_instance()
 
-        self.disable_multi_platform = (
-            buildrunner_config.global_config.disable_multi_platform
-        )
-        if disable_multi_platform is not None:
-            self.disable_multi_platform = disable_multi_platform
-
         # cleanup local cache
         if self.cleanup_cache:
             self.clean_cache()
@@ -328,18 +323,22 @@ def _exit_message(self, exit_explanation):
         Determine the exit message and output to the log.
         """
         if self.exit_code:
-            exit_message = "\nBuild ERROR."
+            exit_message = "Build ERROR."
+            log_method = self.log.error
         else:
-            exit_message = "\nBuild SUCCESS."
+            exit_message = "Build SUCCESS."
+            log_method = self.log.info
 
         if self.log:
             if exit_explanation:
-                self.log.write("\n" + exit_explanation + "\n")
-            self.log.write(exit_message + "\n")
+                self.log.info("")
+                log_method(exit_explanation)
+            self.log.info("")
+            log_method(exit_message)
         else:
             if exit_explanation:
                 print(f"\n{exit_explanation}")
-            print(exit_message)
+            print(f"\n{exit_message}")
 
     def run(self):  # pylint: disable=too-many-statements,too-many-branches,too-many-locals
         """
@@ -355,7 +354,6 @@ def run(self):  # pylint: disable=too-many-statements,too-many-branches,too-many
                 docker_registry=buildrunner_config.global_config.docker_registry,
                 build_registry=buildrunner_config.global_config.build_registry,
                 temp_dir=buildrunner_config.global_config.temp_dir,
-                disable_multi_platform=self.disable_multi_platform,
                 platform_builders=buildrunner_config.global_config.platform_builders,
                 cache_builders=buildrunner_config.global_config.docker_build_cache.builders,
                 cache_from=buildrunner_config.global_config.docker_build_cache.from_config,
@@ -449,15 +447,12 @@ def run(self):  # pylint: disable=too-many-statements,too-many-branches,too-many
                     self.log.write("\nPush not requested\n")
 
         except BuildRunnerConfigurationError as brce:
-            print("config error")
             exit_explanation = str(brce)
             self.exit_code = os.EX_CONFIG
         except BuildRunnerProcessingError as brpe:
-            print("processing error")
             exit_explanation = str(brpe)
             self.exit_code = 1
         except requests.exceptions.ConnectionError as rce:
-            print("connection error")
             print(str(rce))
             exit_explanation = (
                 "Error communicating with the remote host.\n\tCheck that the "

buildrunner/cli.py

@@ -10,6 +10,9 @@
 import os
 import shutil
 import sys
+from typing import Optional
+
+import yaml
 
 from . import (
     __version__,
@@ -209,6 +212,38 @@ def parse_args(argv):
         help="overrides the 'platforms' configuration and global config; to disable multi-platform builds",
     )
 
+    # Security scan config
+    parser.add_argument(
+        "--security-scan-enabled",
+        default=None,
+        choices=["true", "false"],
+        dest="security_scan_enabled",
+        help="overrides the security-scan.enabled global configuration parameter",
+    )
+    parser.add_argument(
+        "--security-scan-scanner",
+        required=False,
+        choices=["trivy"],
+        dest="security_scan_scanner",
+        help="overrides the security-scan.scanner global configuration parameter",
+    )
+    parser.add_argument(
+        "--security-scan-version",
+        dest="security_scan_version",
+        help="overrides the security-scan.version global configuration parameter",
+    )
+    parser.add_argument(
+        "--security-scan-config-file",
+        dest="security_scan_config_file",
+        help="overrides the security-scan.config global configuration parameter by loading YAML data from the file",
+    )
+    parser.add_argument(
+        "--security-scan-max-score-threshold",
+        type=float,
+        dest="security_scan_max_score_threshold",
+        help="overrides the security-scan.max-score-threshold global configuration parameter",
+    )
+
     args = parser.parse_args(argv[1:])
 
     # Set build results dir if not set
@@ -260,6 +295,70 @@ def _create_results_dir(cleanup_step_artifacts: bool, build_results_dir: str) ->
         sys.exit(os.EX_UNAVAILABLE)
 
 
+def _load_security_scan_config_file(config_file: Optional[str]) -> Optional[dict]:
+    if not config_file:
+        return None
+    if not os.path.exists(config_file):
+        sys.stderr.write(
+            f"ERROR: The specified security scan config file ({config_file}) could not be found"
+        )
+        sys.exit(os.EX_CONFIG)
+    try:
+        with open(config_file, "r", encoding="utf8") as fobj:
+            data = yaml.safe_load(fobj)
+            if not data or not isinstance(data, dict):
+                sys.stderr.write(
+                    f"ERROR: The specified security scan config file ({config_file}) must contain a dictionary"
+                )
+                sys.exit(os.EX_CONFIG)
+            return data
+    except Exception as exc:
+        sys.stderr.write(
+            f"ERROR: The specified security scan config file ({config_file}) could not be loaded: {exc}"
+        )
+        sys.exit(os.EX_CONFIG)
+
+
+def _get_security_scan_options(args: argparse.Namespace) -> dict:
+    security_scan_config = {
+        "enabled": _get_true_value(args.security_scan_enabled),
+        "scanner": args.security_scan_scanner,
+        "version": args.security_scan_version,
+        "config": _load_security_scan_config_file(args.security_scan_config_file),
+        "max-score-threshold": args.security_scan_max_score_threshold,
+    }
+    final_config = {
+        key: value for key, value in security_scan_config.items() if value is not None
+    }
+    if final_config:
+        return {"security-scan": final_config}
+    return {}
+
+
+def _get_global_config_overrides(args: argparse.Namespace) -> dict:
+    """
+    Creates a dictionary of overrides to be deeply merged into the loaded global config file(s) data.
+    Note that these field names must match exact what is stored in the global config file(s) and
+    that any fields listed here will override configured values. In other words, undefined/none values
+    should be filtered from the return data to prevent overriding
+    :param args: the parsed CLI args
+    :return: the overrides (if any specified)
+    """
+    overrides = {}
+    if args.disable_multi_platform is not None:
+        overrides["disable-multi-platform"] = _get_true_value(
+            args.disable_multi_platform
+        )
+    return {
+        **overrides,
+        **_get_security_scan_options(args),
+    }
+
+
+def _get_true_value(value: Optional[str]) -> Optional[bool]:
+    return None if value is None else value == "true"
+
+
 def initialize_br(args: argparse.Namespace) -> BuildRunner:
     _create_results_dir(not args.keep_step_artifacts, args.build_results_dir)
     loggers.initialize_root_logger(
@@ -295,9 +394,7 @@ def initialize_br(args: argparse.Namespace) -> BuildRunner:
         docker_timeout=args.docker_timeout,
         local_images=args.local_images,
         platform=args.platform,
-        disable_multi_platform=None
-        if args.disable_multi_platform is None
-        else args.disable_multi_platform == "true",
+        global_config_overrides=_get_global_config_overrides(args),
     )
 
 

buildrunner/config/__init__.py

@@ -70,6 +70,7 @@ def __init__(
         run_config_file: Optional[str],
         log_generated_files: bool,
         build_time: int,
+        global_config_overrides: dict,
         # May be passed in to add temporary files to this list as they are created
         tmp_files: Optional[List[str]] = None,
         # Used only from CLI commands that do not need a run config
@@ -84,7 +85,9 @@ def __init__(
             self.build_time = epoch_time()
         self.tmp_files = tmp_files
 
-        self.global_config = self._load_global_config(global_config_file)
+        self.global_config = self._load_global_config(
+            global_config_file, global_config_overrides
+        )
         self.env = self._load_env(
             push,
             build_number=build_number,
@@ -96,7 +99,9 @@ def __init__(
             self._load_run_config(run_config_file) if load_run_config else None
         )
 
-    def _load_global_config(self, global_config_file: Optional[str]) -> GlobalConfig:
+    def _load_global_config(
+        self, global_config_file: Optional[str], global_config_overrides: dict
+    ) -> GlobalConfig:
         # load global configuration
         gc_files = DEFAULT_GLOBAL_CONFIG_FILES[:]
         gc_files.append(global_config_file or f"{self.build_dir}/.buildrunner.yaml")
@@ -109,6 +114,7 @@ def _load_global_config(self, global_config_file: Optional[str]) -> GlobalConfig
             **load_global_config_files(
                 build_time=self.build_time,
                 global_config_files=abs_gc_files,
+                global_config_overrides=global_config_overrides,
             )
         )
         if errors:

buildrunner/config/loader.py

@@ -86,7 +86,7 @@ def _add_default_tag_to_tags(config: Union[str, dict], default_tag: str) -> dict
     return config
 
 
-def _set_default_tag(config, default_tag) -> dict:
+def _set_default_tag(config: dict, default_tag: str) -> dict:
     """
     Set default tag if not set for each image
 
@@ -312,10 +312,25 @@ def load_run_file(
     return config_data
 
 
+def _deep_merge_dicts(a_dict: dict, b_dict: dict, path=None) -> dict:
+    if path is None:
+        path = []
+    for key in b_dict:
+        if key in a_dict:
+            if isinstance(a_dict[key], dict) and isinstance(b_dict[key], dict):
+                _deep_merge_dicts(a_dict[key], b_dict[key], path + [str(key)])
+            elif a_dict[key] != b_dict[key]:
+                a_dict[key] = b_dict[key]
+        else:
+            a_dict[key] = b_dict[key]
+    return a_dict
+
+
 def load_global_config_files(
     *,
     build_time: int,
     global_config_files: List[str],
+    global_config_overrides: dict,
 ) -> dict:
     """
     Load global config files templating them with Jinja and parsing the YAML.
@@ -376,9 +391,6 @@ def load_global_config_files(
                         )
                 current_context["local-files"] = scrubbed_local_files
 
-            # Merge local-files from each file
-            if "local-files" in context and "local-files" in current_context:
-                context["local-files"].update(current_context.pop("local-files"))
-            context.update(current_context)
+            _deep_merge_dicts(context, current_context)
 
-    return context
+    return _deep_merge_dicts(context, global_config_overrides)

buildrunner/config/models.py

@@ -12,7 +12,6 @@
 
 from pydantic import BaseModel, Field, field_validator, ValidationError
 
-from buildrunner.docker import multiplatform_image_builder
 from .models_step import Step
 from .validation import (
     get_validation_errors,
@@ -23,6 +22,8 @@
 
 
 DEFAULT_CACHES_ROOT = "~/.buildrunner/caches"
+# Marker for using the local registry instead of an upstream registry
+MP_LOCAL_REGISTRY = "local"
 
 
 class GithubModel(BaseModel, extra="forbid"):
@@ -46,6 +47,20 @@ class DockerBuildCacheConfig(BaseModel, extra="forbid"):
     to_config: Optional[Union[dict, str]] = Field(None, alias="to")
 
 
+class SecurityScanConfig(BaseModel, extra="forbid"):
+    enabled: bool = False
+    scanner: str = "trivy"
+    version: str = "latest"
+    # The local cache directory for the scanner (used if supported by the scanner)
+    cache_dir: Optional[str] = None
+    config: dict = {
+        "timeout": "20m",
+        # Do not error on vulnerabilities by default
+        "exit-code": 0,
+    }
+    max_score_threshold: Optional[float] = Field(None, alias="max-score-threshold")
+
+
 class GlobalConfig(BaseModel, extra="forbid"):
     """Top level global config model"""
 
@@ -76,11 +91,14 @@ class GlobalConfig(BaseModel, extra="forbid"):
         alias="disable-multi-platform", default=None
     )
     build_registry: Optional[str] = Field(
-        alias="build-registry", default=multiplatform_image_builder.LOCAL_REGISTRY
+        alias="build-registry", default=MP_LOCAL_REGISTRY
     )
     platform_builders: Optional[Dict[str, str]] = Field(
         alias="platform-builders", default=None
     )
+    security_scan: SecurityScanConfig = Field(
+        SecurityScanConfig(), alias="security-scan"
+    )
 
     @field_validator("ssh_keys", mode="before")
     @classmethod