Merge pull request #26 from google/gcr

Implement GCR importer

Author: meeehow

README.md

@@ -10,15 +10,16 @@
   - [Requirements](#requirements)
   - [Building HashR binary and running tests](#building-hashr-binary-and-running-tests)
   - [Setting up HashR](#setting-up-hashr)
-    - [OS configuration & required 3rd party tooling](#os-configuration--required-3rd-party-tooling)
+    - [OS configuration \& required 3rd party tooling](#os-configuration--required-3rd-party-tooling)
     - [Setting up storage for processing tasks](#setting-up-storage-for-processing-tasks)
       - [Setting up PostgreSQL storage](#setting-up-postgresql-storage)
       - [Setting up Cloud Spanner](#setting-up-cloud-spanner)
     - [Setting up importers](#setting-up-importers)
       - [TarGz](#targz)
       - [Deb](#deb)
       - [RPM](#rpm)
-      - [GCP](#gcp)
+      - [GCP (Google Cloud Platform)](#gcp-google-cloud-platform)
+      - [GCR (Google Container Registry)](#gcr-google-container-registry)
       - [Windows](#windows)
       - [WSUS](#wsus)
     - [Setting up exporters](#setting-up-exporters)
@@ -219,7 +220,7 @@ This is very similar to the TarGz importer except that it looks for `.rpm` packa
 
 1. `-rpm_repo_path` which should point to the path on the local file system that contains `.rpm` files
 
-#### GCP 
+#### GCP (Google Cloud Platform)
 
 This importer can extract files from GCP disk [images](https://cloud.google.com/compute/docs/images). This is done in few steps: 
 
@@ -230,7 +231,7 @@ This importer can extract files from GCP disk [images](https://cloud.google.com/
 1. Copy raw_disk.tar.gz from GCS to local hashR storage
 1. Extract raw_disk.tar.gz and pass the disk image to Plaso 
 
-List of GCP projects containing public GCP images can be found [here](https://cloud.google.com/compute/docs/images/os-details#general-info). In order to use this importer you need to have a GCP project and follow these steps: 
+List of GCP projects containing public GCP images can be found [here](https://cloud.google.com/compute/docs/images/os-details#general-info). In order to use this importer you need to have a GCP project and follow these steps:
 
 Step 1: Create HashR service account, if this was done while setting up Cloud Spanner please go to step 4.
 
@@ -324,6 +325,37 @@ To use this importer you need to specify the following flag(s):
 1. `-hashrGCPProject` GCP project that will be used to store copy of disk images for processing and also run Cloud Build 
 1. `-hashrGCSBucket` GCS bucket that will be used to store output of Cloud Build (disk images in .tar.gz format)
 
+#### GCR (Google Container Registry) 
+This importer extracts files from container images stored in GCR repositories. In order to set ip up follow these steps: 
+
+Step 1: Create HashR service account, skip to step 4 if this was done while setting up other GCP dependent components.
+
+``` shell
+gcloud iam service-accounts create hashr-sa --description="HashR SA key." --display-name="hashr"
+```
+
+Step 2: Create service account key and store in your home directory. Make sure to  set *<project_name>* to your project name:
+
+``` shell
+gcloud iam service-accounts keys create ~/hashr-sa-private-key.json --iam-account=hashr-sa@<project_name>.iam.gserviceaccount.com
+```
+
+Step 3: Point GOOGLE_APPLICATION_CREDENTIALS env variable to your service account key:
+
+``` shell
+export GOOGLE_APPLICATION_CREDENTIALS=~/hashr-sa-private-key.json
+```
+
+Step 4: Grant hashR service account key required permissions to access given GCR repository. 
+
+``` shell
+gsutil iam ch serviceAccount:hashr-sa@<project_name>.iam.gserviceaccount.com:objectViewer gs://artifacts.<project_name_hosting_gcr_repo>.appspot.com
+```
+
+To use this importer you need to specify the following flag(s): 
+
+1. `-gcr_repos` which should contain comma separated list of GCR repositories from which you want to import the container images.
+   
 #### Windows 
 
 This importer extracts files from official Windows installation media in ISO-13346 format, e.g. the ones you can download from official Microsoft [website](https://www.microsoft.com/en-gb/software-download/windows10ISO). 

go.mod

@@ -5,42 +5,57 @@ go 1.18
 require (
 	cloud.google.com/go/spanner v1.32.0
 	github.com/DATA-DOG/go-sqlmock v1.5.0
-	github.com/Microsoft/go-winio v0.5.3-0.20220712145307-8fca75951feb
+	github.com/Microsoft/go-winio v0.6.0
 	github.com/golang/glog v1.0.0
-	github.com/google/go-cmp v0.5.8
+	github.com/google/go-cmp v0.5.9
+	github.com/google/go-containerregistry v0.12.1
 	github.com/lib/pq v1.10.6
-	google.golang.org/api v0.80.0
-	google.golang.org/genproto v0.0.0-20220519153652-3a47de7e79bd
-	google.golang.org/grpc v1.46.2
-	google.golang.org/protobuf v1.28.0
+	github.com/sassoftware/go-rpmutils v0.2.0
+	golang.org/x/oauth2 v0.1.0
+	google.golang.org/api v0.96.0
+	google.golang.org/genproto v0.0.0-20220915135415-7fd63a7952de
+	google.golang.org/grpc v1.48.0
+	google.golang.org/protobuf v1.28.1
+	pault.ag/go/debian v0.12.0
 )
 
 require (
-	cloud.google.com/go v0.100.2 // indirect
-	cloud.google.com/go/compute v1.6.1 // indirect
+	cloud.google.com/go v0.102.1 // indirect
+	cloud.google.com/go/compute v1.10.0 // indirect
 	github.com/DataDog/zstd v1.4.8 // indirect
 	github.com/census-instrumentation/opencensus-proto v0.3.0 // indirect
 	github.com/cespare/xxhash/v2 v2.1.1 // indirect
 	github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4 // indirect
 	github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1 // indirect
+	github.com/containerd/stargz-snapshotter/estargz v0.12.1 // indirect
+	github.com/docker/cli v20.10.20+incompatible // indirect
+	github.com/docker/distribution v2.8.1+incompatible // indirect
+	github.com/docker/docker v20.10.20+incompatible // indirect
+	github.com/docker/docker-credential-helpers v0.7.0 // indirect
 	github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1 // indirect
 	github.com/envoyproxy/protoc-gen-validate v0.1.0 // indirect
 	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
-	github.com/googleapis/gax-go/v2 v2.3.0 // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.1.0 // indirect
+	github.com/googleapis/gax-go/v2 v2.4.0 // indirect
 	github.com/kjk/lzma v0.0.0-20161016003348-3fd93898850d // indirect
-	github.com/klauspost/compress v1.11.7 // indirect
-	github.com/sassoftware/go-rpmutils v0.2.0 // indirect
+	github.com/klauspost/compress v1.15.11 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.1.0-rc2 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/sirupsen/logrus v1.9.0 // indirect
 	github.com/ulikunitz/xz v0.5.9 // indirect
+	github.com/vbatts/tar-split v0.11.2 // indirect
 	github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
 	go.opencensus.io v0.23.0 // indirect
 	golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad // indirect
-	golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 // indirect
-	golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 // indirect
-	golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6 // indirect
-	golang.org/x/text v0.3.7 // indirect
-	golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f // indirect
+	golang.org/x/net v0.1.0 // indirect
+	golang.org/x/sync v0.1.0 // indirect
+	golang.org/x/sys v0.1.0 // indirect
+	golang.org/x/text v0.4.0 // indirect
+	golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	pault.ag/go/debian v0.12.0 // indirect
 	pault.ag/go/topsort v0.0.0-20160530003732-f98d2ad46e1a // indirect
 )