Assorted fixes

Add scheme to the domain passed to credential helpers.

Display resolution/result in the "pusher" tool.

Add a .digest() method on v2.x DockerImage (for convenience).

client/docker_creds_.py

@@ -151,7 +151,9 @@ def Get(self):
                          stdout=subprocess.PIPE,
                          stdin=subprocess.PIPE,
                          stderr=subprocess.STDOUT)
-    stdout = p.communicate(input=self._registry)[0]
+    # Some keychains expect a scheme:
+    # https://github.com/bazelbuild/rules_docker/issues/111
+    stdout = p.communicate(input='https://' + self._registry)[0]
 
     output = stdout.decode()
     if output.strip() == _MAGIC_NOT_FOUND_MESSAGE:

client/docker_name_.py

@@ -101,7 +101,10 @@ def __str__(self):
     return self._registry
 
   def __eq__(self, other):
-    return bool(other) and self.registry == other.registry
+    return (bool(other) and
+            # pylint: disable=unidiomatic-typecheck
+            type(self) == type(other) and
+            self.registry == other.registry)
 
   def __ne__(self, other):
     return not self.__eq__(other)
@@ -155,6 +158,8 @@ def __str__(self):
 
   def __eq__(self, other):
     return (bool(other) and
+            # pylint: disable=unidiomatic-typecheck
+            type(self) == type(other) and
             self.registry == other.registry and
             self.repository == other.repository)
 
@@ -213,6 +218,8 @@ def as_repository(self):
 
   def __eq__(self, other):
     return (bool(other) and
+            # pylint: disable=unidiomatic-typecheck
+            type(self) == type(other) and
             self.registry == other.registry and
             self.repository == other.repository and
             self.tag == other.tag)
@@ -258,6 +265,8 @@ def as_repository(self):
 
   def __eq__(self, other):
     return (bool(other) and
+            # pylint: disable=unidiomatic-typecheck
+            type(self) == type(other) and
             self.registry == other.registry and
             self.repository == other.repository and
             self.digest == other.digest)

client/v2/append_.py

@@ -56,18 +56,21 @@ def __init__(
     """
     self._base = base
 
+    unsigned_manifest, unused_signatures = util.DetachSignatures(
+        self._base.manifest())
+    manifest = json.loads(unsigned_manifest)
+    v1_compat = json.loads(manifest['history'][0]['v1Compatibility'])
+
     if tar_gz:
       self._blob = tar_gz
       self._blob_sum = 'sha256:' + hashlib.sha256(self._blob).hexdigest()
+      v1_compat['throwaway'] = False
     else:
       self._blob_sum = _EMPTY_LAYER_TAR_ID
       self._blob = ''
+      v1_compat['throwaway'] = True
 
-    unsigned_manifest, unused_signatures = util.DetachSignatures(
-        self._base.manifest())
-    manifest = json.loads(unsigned_manifest)
     manifest['fsLayers'].insert(0, {'blobSum': self._blob_sum})
-    v1_compat = json.loads(manifest['history'][0]['v1Compatibility'])
     v1_compat['parent'] = v1_compat['id']
     v1_compat['id'] = binascii.hexlify(os.urandom(32))
 

client/v2/docker_http_.py

@@ -291,7 +291,7 @@ def Request(
 
     Raises:
       BadStateException: an unexpected internal state has been encountered.
-      V2DiagnosticException: an error has occured interacting with v2.
+      V2DiagnosticException: an error has occurred interacting with v2.
 
     Returns:
       The response of the HTTP request, and its contents.

client/v2/docker_image_.py

@@ -39,7 +39,7 @@ class DigestMismatchedError(Exception):
 class DockerImage(object):
   """Interface for implementations that interact with Docker images."""
 
-  __metaclass__ = abc.ABCMeta  # For enforcing that methods are overriden.
+  __metaclass__ = abc.ABCMeta  # For enforcing that methods are overridden.
 
   def fs_layers(self):
     """The ordered collection of filesystem layers that comprise this image."""
@@ -50,6 +50,10 @@ def blob_set(self):
     """The unique set of blobs that compose to create the filesystem."""
     return set(self.fs_layers())
 
+  def digest(self):
+    """The digest of the manifest."""
+    return util.Digest(self.manifest())
+
   # pytype: disable=bad-return-type
   @abc.abstractmethod
   def manifest(self):
@@ -92,6 +96,10 @@ def __enter__(self):
   def __exit__(self, unused_type, unused_value, unused_traceback):
     """Close the image."""
 
+  def __str__(self):
+    """A human-readable representation of the image."""
+    return str(type(self))
+
 
 class FromRegistry(DockerImage):
   """This accesses a docker image hosted on a registry (non-local)."""
@@ -236,6 +244,9 @@ def __enter__(self):
   def __exit__(self, unused_type, unused_value, unused_traceback):
     pass
 
+  def __str__(self):
+    return '<docker_image.FromRegistry name: {}>'.format(str(self._name))
+
 
 def _in_whiteout_dir(
     fs,

client/v2/docker_session_.py

@@ -26,7 +26,6 @@
 from containerregistry.client import docker_name
 from containerregistry.client.v2 import docker_http
 from containerregistry.client.v2 import docker_image
-from containerregistry.client.v2 import util
 import httplib2
 
 
@@ -91,13 +90,11 @@ def _blob_exists(self, digest):
 
   def _manifest_exists(self, image):
     """Check the remote for the given manifest by digest."""
-    manifest_digest = util.Digest(image.manifest())
-
     # GET the manifest by digest, and check for 200
     resp, unused_content = self._transport.Request(
         '{base_url}/manifests/{digest}'.format(
             base_url=self._base_url(),
-            digest=manifest_digest),
+            digest=image.digest()),
         method='GET', accepted_codes=[httplib.OK, httplib.NOT_FOUND])
 
     return resp.status == httplib.OK  # pytype: disable=attribute-error
@@ -261,8 +258,7 @@ def upload(self, image):
     # checks (they must exist).
     if self._manifest_exists(image):
       if isinstance(self._name, docker_name.Tag):
-        manifest_digest = util.Digest(image.manifest())
-        if self._remote_tag_digest() == manifest_digest:
+        if self._remote_tag_digest() == image.digest():
           logging.info('Tag points to the right manifest, skipping push.')
           return
         logging.info('Manifest exists, skipping blob uploads and pushing tag.')

client/v2_2/append_.py

@@ -57,13 +57,13 @@ def __init__(
     if tar_gz:
       self._blob = tar_gz
       self._blob_sum = 'sha256:' + hashlib.sha256(self._blob).hexdigest()
-      manifest['layers'].insert(0, {
+      manifest['layers'].append({
           'digest': self._blob_sum,
           'mediaType': docker_http.MANIFEST_SCHEMA2_MIME,
           'size': len(self._blob),
       })
-      config_file['rootfs']['diff_ids'].insert(
-          0, 'sha256:' + hashlib.sha256(
+      config_file['rootfs']['diff_ids'].append(
+          'sha256:' + hashlib.sha256(
               self.uncompressed_blob(self._blob_sum)).hexdigest())
     else:
       self._blob_sum = _EMPTY_LAYER_TAR_ID
@@ -72,10 +72,10 @@ def __init__(
 
     config_file['history'].insert(0, cfg)
 
-    self._config_file = json.dumps(config_file)
+    self._config_file = json.dumps(config_file, sort_keys=True)
     manifest['config']['digest'] = (
         'sha256:' + hashlib.sha256(self._config_file).hexdigest())
-    self._manifest = json.dumps(manifest)
+    self._manifest = json.dumps(manifest, sort_keys=True)
 
   def manifest(self):
     """Override."""

client/v2_2/docker_http_.py

@@ -42,10 +42,20 @@
 MANIFEST_LIST_MIME = 'application/vnd.docker.distribution.manifest.list.v2+json'
 LAYER_MIME = 'application/vnd.docker.image.rootfs.diff.tar.gzip'
 CONFIG_JSON_MIME = 'application/vnd.docker.container.image.v1+json'
+
+OCI_MANIFEST_MIME = 'application/vnd.oci.image.manifest.v1+json'
+OCI_IMAGE_INDEX_MIME = 'application/vnd.oci.image.index.v1+json'
+OCI_LAYER_MIME = 'application/vnd.oci.image.layer.v1.tar+gzip'
+OCI_CONFIG_JSON_MIME = 'application/vnd.oci.image.config.v1+json'
+
 MANIFEST_SCHEMA1_MIMES = [MANIFEST_SCHEMA1_MIME, MANIFEST_SCHEMA1_SIGNED_MIME]
 MANIFEST_SCHEMA2_MIMES = [MANIFEST_SCHEMA2_MIME]
+OCI_MANIFEST_MIMES = [OCI_MANIFEST_MIME]
+
 SUPPORTED_MANIFEST_MIMES = [MANIFEST_SCHEMA1_MIMES, MANIFEST_SCHEMA2_MIME]
 
+MANIFEST_LIST_MIMES = [MANIFEST_LIST_MIME]
+
 
 class Diagnostic(object):
   """Diagnostic encapsulates a Registry v2 diagnostic message.
@@ -307,7 +317,7 @@ def Request(
 
     Raises:
       BadStateException: an unexpected internal state has been encountered.
-      V2DiagnosticException: an error has occured interacting with v2.
+      V2DiagnosticException: an error has occurred interacting with v2.
 
     Returns:
       The response of the HTTP request, and its contents.

client/v2_2/docker_image_.py

@@ -40,7 +40,7 @@ class DigestMismatchedError(Exception):
 class DockerImage(object):
   """Interface for implementations that interact with Docker images."""
 
-  __metaclass__ = abc.ABCMeta  # For enforcing that methods are overriden.
+  __metaclass__ = abc.ABCMeta  # For enforcing that methods are overridden.
 
   def fs_layers(self):
     """The ordered collection of filesystem layers that comprise this image."""
@@ -55,6 +55,16 @@ def blob_set(self):
     """The unique set of blobs that compose to create the filesystem."""
     return set(self.fs_layers() + [self.config_blob()])
 
+  def digest(self):
+    """The digest of the manifest."""
+    return util.Digest(self.manifest())
+
+  def media_type(self):
+    """The media type of the manifest."""
+    manifest = json.loads(self.manifest())
+    # Since 'mediaType' is optional for OCI images, assume OCI if it's missing.
+    return manifest.get('mediaType', docker_http.OCI_MANIFEST_MIME)
+
   # pytype: disable=bad-return-type
   @abc.abstractmethod
   def manifest(self):
@@ -105,6 +115,10 @@ def __enter__(self):
   def __exit__(self, unused_type, unused_value, unused_traceback):
     """Close the image."""
 
+  def __str__(self):
+    """A human-readable representation of the image."""
+    return str(type(self))
+
 
 class FromRegistry(DockerImage):
   """This accesses a docker image hosted on a registry (non-local)."""
@@ -113,10 +127,12 @@ def __init__(
       self,
       name,
       basic_creds,
-      transport):
+      transport,
+      accepted_mimes=docker_http.MANIFEST_SCHEMA2_MIMES):
     self._name = name
     self._creds = basic_creds
     self._original_transport = transport
+    self._accepted_mimes = accepted_mimes
     self._response = {}
 
   def _content(
@@ -170,7 +186,9 @@ def children(self):
   def exists(self):
     try:
       manifest = json.loads(self.manifest(validate=False))
-      return manifest['schemaVersion'] == 2
+      return (manifest['schemaVersion'] == 2 and
+              'layers' in manifest and
+              self.media_type() in self._accepted_mimes)
     except docker_http.V2DiagnosticException as err:
       if err.status == httplib.NOT_FOUND:
         return False
@@ -179,13 +197,12 @@ def exists(self):
   def manifest(self, validate=True):
     """Override."""
     # GET server1/v2/<name>/manifests/<tag_or_digest>
-    accepted_mimes = docker_http.MANIFEST_SCHEMA2_MIMES
 
     if isinstance(self._name, docker_name.Tag):
-      return self._content('manifests/' + self._name.tag, accepted_mimes)
+      return self._content('manifests/' + self._name.tag, self._accepted_mimes)
     else:
       assert isinstance(self._name, docker_name.Digest)
-      c = self._content('manifests/' + self._name.digest, accepted_mimes)
+      c = self._content('manifests/' + self._name.digest, self._accepted_mimes)
       computed = util.Digest(c)
       if validate and computed != self._name.digest:
         raise DigestMismatchedError(
@@ -260,6 +277,9 @@ def __enter__(self):
   def __exit__(self, unused_type, unused_value, unused_traceback):
     pass
 
+  def __str__(self):
+    return '<docker_image.FromRegistry name: {}>'.format(str(self._name))
+
 
 # Gzip injects a timestamp into its output, which makes its output and digest
 # non-deterministic.  To get reproducible pushes, freeze time.
@@ -331,6 +351,7 @@ def _gzipped_content(self, name):
 
   def _populate_manifest_and_blobs(self):
     """Populates self._manifest and self._blob_names."""
+    # TODO(user): Update mimes here for oci_compat.
     manifest = {
         'mediaType': docker_http.MANIFEST_SCHEMA2_MIME,
         'schemaVersion': 2,
@@ -524,6 +545,7 @@ def __enter__(self):
     base_layers = []
     if self._legacy_base:
       base_layers = json.loads(self._legacy_base.manifest())['layers']
+    # TODO(user): Update mimes here for oci_compat.
     self._manifest = json.dumps({
         'schemaVersion': 2,
         'mediaType': docker_http.MANIFEST_SCHEMA2_MIME,

client/v2_2/docker_session_.py

@@ -26,7 +26,6 @@
 from containerregistry.client import docker_name
 from containerregistry.client.v2_2 import docker_http
 from containerregistry.client.v2_2 import docker_image
-from containerregistry.client.v2_2 import util
 import httplib2
 
 
@@ -91,15 +90,13 @@ def _blob_exists(self, digest):
 
   def _manifest_exists(self, image):
     """Check the remote for the given manifest by digest."""
-    manifest_digest = util.Digest(image.manifest())
-
     # GET the manifest by digest, and check for 200
     resp, unused_content = self._transport.Request(
         '{base_url}/manifests/{digest}'.format(
-            base_url=self._base_url(), digest=manifest_digest),
+            base_url=self._base_url(), digest=image.digest()),
         method='GET',
         accepted_codes=[httplib.OK, httplib.NOT_FOUND],
-        accepted_mimes=docker_http.MANIFEST_SCHEMA2_MIMES)
+        accepted_mimes=[image.media_type()])
 
     return resp.status == httplib.OK  # pytype: disable=attribute-error
 
@@ -223,8 +220,9 @@ def _put_manifest(self, image):
         '{base_url}/manifests/{tag_or_digest}'.format(
             base_url=self._base_url(),
             tag_or_digest=_tag_or_digest(self._name)),
-        method='PUT', body=image.manifest(),
-        content_type=docker_http.MANIFEST_SCHEMA2_MIME,
+        method='PUT',
+        body=image.manifest(),
+        content_type=image.media_type(),
         accepted_codes=[httplib.OK, httplib.CREATED, httplib.ACCEPTED])
 
   def _start_upload(
@@ -277,8 +275,7 @@ def upload(self, image):
     # checks (they must exist).
     if self._manifest_exists(image):
       if isinstance(self._name, docker_name.Tag):
-        manifest_digest = util.Digest(image.manifest())
-        if self._remote_tag_digest() == manifest_digest:
+        if self._remote_tag_digest() == image.digest():
           logging.info('Tag points to the right manifest, skipping push.')
           return
         logging.info('Manifest exists, skipping blob uploads and pushing tag.')

tools/fast_pusher_.py

@@ -73,6 +73,10 @@ def Tag(name, files):
 
   formatted_name = name.format(**format_args)
 
+  if files:
+    print('{name} was resolved to {fname}'.format(
+        name=name, fname=formatted_name))
+
   return docker_name.Tag(formatted_name)
 
 
@@ -117,6 +121,8 @@ def main():
     with v2_2_image.FromDisk(config, zip(args.digest or [], args.layer or []),
                              legacy_base=args.tarball) as v2_2_img:
       session.upload(v2_2_img)
+      print('{name} was published with digest: {digest}'.format(
+          name=name, digest=v2_2_img.digest()))
 
 
 if __name__ == '__main__':