Merge pull request #5 from google/three-stage-uploads

Switch to three-request uploads.

.travis.yml

@@ -1,9 +1,32 @@
 sudo: required
+
 dist: trusty
+
 language:
   - java
+
 jdk:
   - oraclejdk8  # Building Bazel requires JDK8.
+
+services:
+  - docker
+
+before_install:
+  # Log into DockerHub
+  - docker login -u googlecontainerregistrytesting -p "${GCRT_PASSWORD}"
+  # Log into GCR
+  - docker login -u _json_key -p "${GCR_JSON_KEY}" https://gcr.io
+  # Log into Quay.io
+  - docker login -u googlecontainerregistrytesting -p "${GCRT_PASSWORD}" quay.io
+  # Log into bintray with an API KEY
+  - |
+    docker login -u googlecontainerregistrytesting -p "${BINTRAY_API_KEY}" \
+      googlecontainerregistrytesting-docker-test-repo.bintray.io
+  # Log into Gitlab
+  - |
+    docker login -u googlecontainerregistrytesting -p "${GCRT_PASSWORD}" \
+      registry.gitlab.com
+
 addons:
   apt:
     sources:
@@ -14,4 +37,5 @@ addons:
 
 script:
   - bazel clean && bazel build //...
-  - bazel clean && bazel test --test_output=errors //...
+  # Run outside of the sandbox to get access to Docker credentials registered above.
+  - bazel clean && bazel test --test_output=errors --strategy=TestRunner=standalone //...

BUILD.bazel

@@ -50,3 +50,13 @@ sh_test(
       "testenv.sh",
     ],
 )
+
+sh_test(
+    name = "pusher_test",
+    size = "large",
+    srcs = ["pusher_test.sh"],
+    data = [
+      ":pusher.par",
+      "testenv.sh",
+    ],
+)

client/v2/docker_http_.py

@@ -101,12 +101,8 @@ def response(self):
     return self._resp
 
   @property
-  def http_status_code(self):
-    if self._resp.status:
-      # Check to see if the raw http response was given.
-      return self._resp.status
-    # Return the 'status' contained in an actual dict.
-    return int(self._resp.get('status'))
+  def status(self):
+    return self._resp.status
 
 
 class BadStateException(Exception):

client/v2/docker_image_.py

@@ -153,7 +153,7 @@ def exists(self):
       self.manifest(validate=False)
       return True
     except docker_http.V2DiagnosticException as err:
-      if err.http_status_code == 404:
+      if err.status == httplib.NOT_FOUND:
         return False
       raise
 

client/v2/docker_session_.py

@@ -174,16 +174,20 @@ def _put_blob(
     # or:
     #   POST /v2/<name>/blobs/uploads/        (no body*)
     #   PUT  /v2/<name>/blobs/uploads/<uuid>  (full body)
-    self._put_upload(image, digest)
+    #   Next fastest, but there is a mysterious bad interaction
+    #   with Bintray.  This pattern also hasn't been used in
+    #   clients since 1.8, when they switched to the 3-stage
+    #   method below.
+    # self._put_upload(image, digest)
     # or:
     #   POST   /v2/<name>/blobs/uploads/        (no body*)
     #   PATCH  /v2/<name>/blobs/uploads/<uuid>  (full body)
     #   PUT    /v2/<name>/blobs/uploads/<uuid>  (no body)
-    # self._patch_upload(image, digest)
     #
     # * We attempt to perform a cross-repo mount if any repositories are
     # specified in the "mount" parameter. This does a fast copy from a
     # repository that is known to contain this blob and skips the upload.
+    self._patch_upload(image, digest)
 
   def _remote_tag_digest(self):
     """Check the remote for the given manifest by digest."""

client/v2_2/docker_http_.py

@@ -112,12 +112,8 @@ def response(self):
     return self._resp
 
   @property
-  def http_status_code(self):
-    if self._resp.status:
-      # Check to see if the raw http response was given.
-      return self._resp.status
-    # Return the 'status' contained in an actual dict.
-    return int(self._resp.get('status'))
+  def status(self):
+    return self._resp.status
 
 
 class BadStateException(Exception):

client/v2_2/docker_image_.py

@@ -172,7 +172,7 @@ def exists(self):
       manifest = json.loads(self.manifest(validate=False))
       return manifest['schemaVersion'] == 2
     except docker_http.V2DiagnosticException as err:
-      if err.http_status_code == httplib.NOT_FOUND:
+      if err.status == httplib.NOT_FOUND:
         return False
       raise
 

client/v2_2/docker_session_.py

@@ -187,16 +187,20 @@ def _put_blob(
     # or:
     #   POST /v2/<name>/blobs/uploads/        (no body*)
     #   PUT  /v2/<name>/blobs/uploads/<uuid>  (full body)
-    self._put_upload(image, digest)
+    #   Next fastest, but there is a mysterious bad interaction
+    #   with Bintray.  This pattern also hasn't been used in
+    #   clients since 1.8, when they switched to the 3-stage
+    #   method below.
+    # self._put_upload(image, digest)
     # or:
     #   POST   /v2/<name>/blobs/uploads/        (no body*)
     #   PATCH  /v2/<name>/blobs/uploads/<uuid>  (full body)
     #   PUT    /v2/<name>/blobs/uploads/<uuid>  (no body)
-    # self._patch_upload(image, digest)
     #
     # * We attempt to perform a cross-repo mount if any repositories are
     # specified in the "mount" parameter. This does a fast copy from a
     # repository that is known to contain this blob and skips the upload.
+    self._patch_upload(image, digest)
 
   def _remote_tag_digest(self):
     """Check the remote for the given manifest by digest."""

puller_test.sh

@@ -69,9 +69,8 @@ test_image gcr.io/google-appengine/python:latest
 # Test pulling from DockerHub
 test_image index.docker.io/library/busybox:latest
 
-# TODO(user): Re-enable this once they return content-length on blob HEADs
-# # Test pulling from Quay.io
-# test_image quay.io/coreos/etcd:latest
+# Test pulling from Quay.io
+test_image quay.io/coreos/etcd:latest
 
 # Test pulling from Bintray.io
 test_image jfrog-int-docker-devops-registry.bintray.io/alpine:latest

pusher_test.sh

@@ -0,0 +1,114 @@
+#!/bin/bash -e
+
+# Copyright 2017 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Unit tests for pusher.par
+
+# Trick to chase the symlink before the docker build.
+cp pusher.par pusher2.par
+
+# Generate a fresh random image to avoid completely
+# incremental pushes.
+function generate_image() {
+  local target=$1
+
+  cat > Dockerfile <<EOF
+FROM alpine
+RUN head -c100 /dev/urandom > /tmp/random.txt
+EOF
+  docker build -t random .
+  docker save -o "${target}" random
+  docker rmi -f random
+}
+
+
+# Sanity test that credentials are properly configured.
+function test_sanity() {
+  local image=$1
+  local random_image="direct.$RANDOM.tar"
+  generate_image "${random_image}"
+
+  docker load -i "${random_image}"
+  docker tag random "${image}"
+  docker push "${image}"
+  docker rmi -f random "${image}"
+}
+
+# Test pushing an image by just invoking the pusher
+function test_pusher() {
+  local image=$1
+  local random_image="direct.$RANDOM.tar"
+  generate_image "${random_image}"
+
+  # Test it in our current environment.
+  pusher.par --name="${image}" --tarball="${random_image}"
+}
+
+# Test pushing an image from inside a docker container with a
+# certain base / entrypoint
+function test_base() {
+  local image=$1
+  local entrypoint=$2
+  local base_image=$3
+  local random_image="in-image.$RANDOM.tar"
+  generate_image "${random_image}"
+
+  echo Testing pusher env: ${base_image} entrypoint: ${entrypoint}
+  # TODO(user): use a temp dir
+  cat > Dockerfile <<EOF
+FROM ${base_image}
+ADD pusher2.par /pusher.par
+EOF
+
+  docker build -t pusher_test .
+  docker run -i --rm --entrypoint="${entrypoint}" \
+    -v "${HOME}/.docker/:/root/.docker/" \
+    -v "${random_image}:/${random_image}" pusher_test \
+    /pusher.par --name="${image}" --tarball="/${random_image}"
+  docker rmi pusher_test
+}
+
+function test_image() {
+  local image=$1
+
+  echo "TESTING: ${image}"
+
+  test_sanity "${image}"
+
+  test_pusher "${image}"
+
+  # TODO(user): Test inside of docker images.
+  # test_base "${image}" python2.7 python:2.7
+  # test_base "${image}" python2.7 gcr.io/cloud-builders/bazel
+}
+
+# Test pushing a trivial image.
+# The registered credential only has access to this repository,
+# which is only used for testing.
+test_image gcr.io/containerregistry-releases/pusher-testing:latest
+
+# Test pushing to DockerHub
+test_image index.docker.io/googlecontainerregistrytesting/pusher-testing:latest
+
+# Test pushing to Bintray.io
+test_image googlecontainerregistrytesting-docker-test-repo.bintray.io/testing/pusher-testing:latest
+
+# TODO(user): Enable once Quay supports v2.2
+# # Test pushing to Quay.io
+# test_image quay.io/googlecontainerregistrytesting/pusher-testing:latest
+
+# TODO(user): Enable once SNI issues are resolved.
+# # Test pushing to Gitlab
+# test_image registry.gitlab.com/googlecontainerregistrytesting/pusher-testing/image:latest