Updates to schema v2 (#255)

* Make the v2 schema stricter with additional properties
* Fix the v2 testdata schemas to be compliant with the specification
* Fix the hardcoded feature configs in the unit tests to be compliant with the spec
* Bugfix in converting schemas from v1 to v2
* Update documentation regarding linkage schema v2 
* Add a v2 schema with errors
* Invalid schema exceptions keep more context
* Add cli command to validate a schema and cli schema validation tests
* Avoid a divide by zero when ngrams is empty
* cli can throw nice errors when asked to hash with invalid schema
* Travis: notebook execution runs in the integration test stage
* Expose Schema at top level
* Don't include a default hashing strategy in the schema
* Update tutorial notebooks to use v2 schema (#257)
* Include a notebook sanitizer so travis can test cells match.
* Use the new version of anonlink in api tutorial.
* fine-tuned schemas for jaw dropping performance demonstration
* Schema id now includes version
* Update author attribute
* Note we don't update to latest jsonschema version as Windows exe built with PyInstaller failed.

Closes #254

Author: hardbyte

.travis.yml

@@ -22,7 +22,7 @@ install:
   - travis_retry pip install -e .
 
 script:
-  - if [ "${INCLUDE_NB_TEST}" == "1" ]; then pytest --cov=clkhash --nbval-lax; else pytest --cov=clkhash; fi
+  - pytest --cov=clkhash
   - codecov
 
 
@@ -48,28 +48,33 @@ jobs:
     - python: '3.6'
       env:
         - INCLUDE_CLI=1
-        - INCLUDE_NB_TEST=1
     - python: '2.7'
       env:
         - INCLUDE_CLI=1
-
     # OSX + Python is officially supported by Travis CI as of April 2011
     # https://docs.travis-ci.com/user/reference/osx/
     - os: osx
       osx_image: xcode8.3
       python: "3.6-dev"
 
+    - stage: Integration
+      name: Test Notebooks
+      python: 3.7
+      before_install:
+        - travis_retry pip install -U -r docs/doc-requirements.txt
+      script:
+        - pytest --nbval docs -x --sanitize-with docs/tutorial_sanitize.cfg
+
     - stage: Integration
       python: '3.8-dev'
       env:
         - TEST_ENTITY_SERVICE=https://testing.es.data61.xyz
         - INCLUDE_CLI=1
     - stage: Integration
-      python: '3.6'
+      python: '3.7'
       env:
         - TEST_ENTITY_SERVICE=https://testing.es.data61.xyz
         - INCLUDE_CLI=1
-        - INCLUDE_NB_TEST=1
     - stage: Integration
       python: '2.7'
       env:

CHANGELOG.md

@@ -1,5 +1,6 @@
 ## 0.13.0
 
+- Fix example and test linkage schemas using v2.
 - Fix mismatch between double hash and blake hash key requirement.
 - Update to use newer anonlink-entity-service api.
 - Updates to dependencies.

clkhash/__init__.py

@@ -1,10 +1,11 @@
 import pkg_resources
 
 from . import bloomfilter, field_formats, key_derivation, schema, randomnames, describe
+from .schema import Schema
 
 try:
     __version__ = pkg_resources.get_distribution('clkhash').version
 except pkg_resources.DistributionNotFound:
     __version__ = "development"
 
-__author__ = 'N1 Analytics'
+__author__ = "Data61"

clkhash/bloomfilter.py

@@ -319,9 +319,10 @@ def crypto_bloom_filter(record,      # type: Sequence[Text]
         if fhp:
             ngrams = list(tokenize(field.format_value(entry)))
             hash_function = hashing_function_from_properties(fhp)
-            bloomfilter |= hash_function(ngrams, key,
-                                         fhp.ks(len(ngrams)),
-                                         hash_l, fhp.encoding)
+            if ngrams:
+                bloomfilter |= hash_function(ngrams, key,
+                                             fhp.ks(len(ngrams)),
+                                             hash_l, fhp.encoding)
 
     c1 = bloomfilter.count()
     bloomfilter = fold_xor(bloomfilter, schema.xor_folds)

clkhash/cli.py

@@ -15,6 +15,7 @@
                                  run_get_status, project_create, run_create,
                                  server_get_status, ServiceError,
                                  format_run_status, watch_run_status)
+from clkhash.schema import SchemaError
 
 DEFAULT_SERVICE_URL = 'https://es.data61.xyz'
 
@@ -68,8 +69,11 @@ def hash(pii_csv, keys, schema, clk_json, quiet, no_header, check_header, valida
 
     Use "-" for CLK_JSON to write JSON to stdout.
     """
-
-    schema_object = clkhash.schema.from_json_file(schema_file=schema)
+    try:
+        schema_object = clkhash.schema.from_json_file(schema_file=schema)
+    except SchemaError as e:
+        log(str(e))
+        raise SystemExit(-1)
     header = True
     if not check_header:
         header = 'ignore'
@@ -92,7 +96,7 @@ def hash(pii_csv, keys, schema, clk_json, quiet, no_header, check_header, valida
             log("CLK data written to {}".format(clk_json.name))
 
 
-@cli.command('status', short_help='Get status of entity service')
+@cli.command('status', short_help='get status of entity service')
 @click.option('--server', type=str, default=DEFAULT_SERVICE_URL, help="Server address including protocol")
 @click.option('-o', '--output', type=click.File('w'), default='-')
 @click.option('-v', '--verbose', default=False, is_flag=True, help="Script is more talkative")
@@ -141,7 +145,7 @@ def status(server, output, verbose):
 @click.option('--name', type=str, help="Name to give this project")
 @click.option('--parties', default=2, type=int,
               help="Number of parties in the project")
-@click.option('-o','--output', type=click.File('w'), default='-')
+@click.option('-o', '--output', type=click.File('w'), default='-')
 @click.option('-v', '--verbose', is_flag=True, help="Script is more talkative")
 def create_project(type, schema, server, name, parties, output, verbose):
     """Create a new project on an entity matching server.
@@ -171,7 +175,7 @@ def create_project(type, schema, server, name, parties, output, verbose):
     except ServiceError as e:
         log("Unexpected response - {}".format(e.status_code))
         log(e.text)
-        raise SystemExit
+        raise SystemExit(-1)
     else:
         log("Project created")
 
@@ -318,6 +322,28 @@ def generate_default_schema(output):
     shutil.copyfile(original_path, output)
 
 
+@cli.command('validate-schema', short_help="validate linkage schema")
+@click.argument('schema', type=click.File('r', lazy=True))
+def validate_schema(schema):
+    """Validate a linkage schema
+
+    Given a file containing a linkage schema, verify the schema is valid otherwise
+    print detailed errors.
+    """
+
+    try:
+        clkhash.schema.from_json_file(
+            schema_file=schema,
+            validate=True
+        )
+
+        log("schema is valid", color='green')
+
+    except SchemaError as e:
+        log(str(e))
+        raise SystemExit(-1)
+
+
 if __name__ == "__main__":
     freeze_support()
     cli()

clkhash/data/randomnames-schema.json

@@ -1,4 +1,3 @@
-
 {
   "version": 1,
   "clkConfig": {

clkhash/field_formats.py

@@ -25,11 +25,13 @@ class InvalidEntryError(ValueError):
 
 
 class InvalidSchemaError(ValueError):
-    """ The schema is not valid.
+    """Raised if the schema of a field specification is invalid.
 
-        This exception is raised if, for example, a regular expression
-        included in the schema is not syntactically correct.
+    For example, a regular expression included in the schema is not
+    syntactically correct.
     """
+    json_field_spec = None  # type: Optional[dict]
+    field_spec_index = None # type: Optional[int]
 
 
 class MissingValueSpec(object):
@@ -161,19 +163,17 @@ def fhp_from_json_dict(
     """
     Make a :class:`FieldHashingProperties` object from a dictionary.
 
-        :param dict json_dict:
-            The dictionary must have have an 'ngram' key
-            and one of k or num_bits. It may have
-            'positional' key; if missing a default is used.
-            The encoding is
-            always set to the default value.
-        :return: A :class:`FieldHashingProperties` instance.
+    :param dict json_dict:
+        Conforming to the `hashingConfig` definition
+        in the `v2` linkage schema.
+    :return: A :class:`FieldHashingProperties` instance.
     """
+    hashing_strategy = json_dict['strategy']
     h = json_dict.get('hash', {'type': 'blakeHash'})
-    num_bits = json_dict.get('numBits')
-    k = json_dict.get('k')
-    if not num_bits and not k:
-        num_bits = 200 # default for v2 schema
+
+    num_bits = hashing_strategy.get('numBits')
+    k = hashing_strategy.get('k')
+
     return FieldHashingProperties(
         ngram=json_dict['ngram'],
         positional=json_dict.get(
@@ -263,7 +263,6 @@ def validate(self, str_in):
                 e_new.field_spec = self
                 raise_from(e_new, err)
 
-
     def is_missing_value(self, str_in):
         # type: (Text) -> bool
         """ tests if 'str_in' is the sentinel value for this field
@@ -441,6 +440,7 @@ def from_json_dict(cls,
             except (SyntaxError, re.error) as e:
                 msg = "Invalid regular expression '{}.'".format(pattern)
                 e_new = InvalidSchemaError(msg)
+                e_new.json_field_spec = json_dict
                 raise_from(e_new, e)
             result.regex_based = True
 
@@ -843,9 +843,10 @@ def spec_from_json_dict(
         json_dict  # type: Dict[str, Any]
 ):
     # type: (...) -> FieldSpec
-    """ Turns a dictionary into the appropriate object.
+    """ Turns a dictionary into the appropriate FieldSpec object.
 
         :param dict json_dict: A dictionary with properties.
+        :raises InvalidSchemaError:
         :returns: An initialised instance of the appropriate FieldSpec
             subclass.
     """