Better test coverage and handling for write failures.

geowurster · geowurster · commit ab89bde4f0df · 2015-07-21T21:11:05.000-04:00
diff --git a/README.rst b/README.rst
@@ -26,64 +26,68 @@ Usage
 
       Default settings - buffer geometries in the input CRS:
 
-          $ fio buffer in.geojson out.geojson --dist 10
+          $ fio buffer in.geojson out.geojson --distance 10
 
       Dynamically buffer geometries by a distance stored in the field
-      `magnitude` and write as GeoJSON:
+      'magnitude' and write as GeoJSON:
 
           $ fio buffer \
               in.shp \
               out.geojson \
               --driver GeoJSON \
-              --dist magnitude
+              --distance magnitude
 
       Read geometries from one CRS, buffer in another, and then write to a
       third:
 
           $ fio buffer in.shp out.shp \
-              --dist 10 \
+              --distance 10 \
               --buf-crs EPSG:3857 \
               --dst-crs EPSG:32618
 
       Control cap style, mitre limit, segment resolution, and join style:
 
           $ fio buffer in.geojson out.geojson \
-              --dist 0.1 \
+              --distance 0.1 \
               --res 5 \
               --cap-style flat \
               --join-style mitre \
               --mitre-limit 0.1\
 
     Options:
-      -f, --format, --driver NAME     Output driver name.  [required]
+      --version                       Show the version and exit.
+      -f, --format, --driver NAME     Output driver name.  Derived from the input
+                                      datasource if not given.
       --cap-style [flat|round|square]
                                       Where geometries terminate, use this style.
-                                      (default: round)
+                                      [default: round]
       --join-style [round|mitre|bevel]
                                       Where geometries touch, use this style.
-                                      (default: round)
+                                      [default: round]
       --res INTEGER                   Resolution of the buffer around each vertex
-                                      of the object. (default: 16)
+                                      of the geometry.  [default: 16]
       --mitre-limit FLOAT             When using a mitre join, limit the maximum
                                       length of the join corner according to this
-                                      ratio. (default: 0.5)
-      --dist FLOAT | FIELD            Buffer distance in georeferenced units
-                                      according to --buf-dist.  [required]
+                                      ratio.  [default: 5.0]
+      --distance FLOAT|FIELD          Buffer distance or field containing distance
+                                      values.  Units match --buf-crs.  When
+                                      buffering with a field, feature's with a
+                                      null value are unaltered.  [required]
       --src-crs TEXT                  Specify CRS for input data.  Not needed if
                                       set in input file.
       --buf-crs TEXT                  Perform buffer operations in a different
-                                      CRS. (default: --src-crs)
+                                      CRS. [default: --src-crs]
       --dst-crs TEXT                  Reproject geometries to a different CRS
                                       before writing.  Must be combined with
-                                      --buf-crs. (default: --src-crs)
-      --otype GEOMTYPE                Specify output geometry type. (default:
-                                      MultiPolygon)
+                                      --buf-crs. [default: --src-crs]
+      --geom-type GEOMTYPE            Output layer's geometry type.  [default:
+                                      MultiPolygon]
       --skip-failures                 Skip geometries that fail somewhere in the
                                       processing pipeline.
       --jobs CORES                    Process geometries in parallel across N
-                                      cores.  The goal of this flag is speed so
-                                      feature order and ID's are not preserved.
-                                      (default: 1)
+                                      cores.  Feature ID's and order are not
+                                      preserved if more that 1 cores are used.
+                                      [default: 1]
       --help                          Show this message and exit.
 
 
@@ -114,7 +118,7 @@ Developing
     $ cd fio-buffer
     $ virtualenv venv
     $ source venv/bin/activate
-    $ pip install -e .[test]
+    $ pip install -e .[dev]
     $ py.test tests --cov fio_buffer --cov-report term-missing
 
 
diff --git a/fio_buffer/__init__.py b/fio_buffer/__init__.py
@@ -3,7 +3,7 @@
 """
 
 
-__version__ = '0.1'
+__version__ = '0.1.1'
 __author__ = 'Kevin Wurster'
 __email__ = 'wursterk@gmail.com'
 __source__ = 'https://github.com/geowurster/fio-buffer'
@@ -23,8 +23,9 @@
   this list of conditions and the following disclaimer in the documentation
   and/or other materials provided with the distribution.
 
-* The names of its contributors may not be used to endorse or promote products
-  derived from this software without specific prior written permission.
+* The names of fio-buffer or its contributors may not be used to endorse or
+  promote products derived from this software without specific prior written
+  permission.
 
 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
diff --git a/fio_buffer/core.py b/fio_buffer/core.py
@@ -57,7 +57,7 @@ def _cb_res(ctx, param, value):
 def _cb_dist(ctx, param, value):
 
     """
-    Click callback to ensure `--dist` can be either a float or a field name.
+    Click callback to ensure `--distance` can be either a float or a field name.
     """
 
     try:
@@ -69,7 +69,7 @@ def _cb_dist(ctx, param, value):
 def _processor(args):
 
     """
-    Process a single feature
+    Process a single feature.
 
     Parameters
     ----------
@@ -102,7 +102,13 @@ def _processor(args):
 
     # Support buffering by a field's value
     if not isinstance(buf_args['distance'], (float, int)):
-        buf_args['distance'] = feat['properties'][buf_args['distance']]
+        field_val = feat['properties'][buf_args['distance']]
+
+        # Buffering according to a field but field is None so just return the feature
+        if field_val is None:
+            return feat
+        else:
+            buf_args['distance'] = field_val
 
     try:
         # src_crs -> buf_crs
@@ -123,7 +129,7 @@ def _processor(args):
         return feat
 
     except Exception:
-        logger.exception("Feature with ID %s failed", feat.get('id'))
+        logger.exception("Feature with ID %s failed during buffering", feat.get('id'))
         if not skip_failures:
             raise
 
@@ -133,56 +139,61 @@ def _processor(args):
 @click.argument('infile', required=True)
 @click.argument('outfile', required=True)
 @click.option(
-    '-f', '--format', '--driver', metavar='NAME', required=True,
-    help="Output driver name."
+    '-f', '--format', '--driver', metavar='NAME',
+    help="Output driver name.  Derived from the input datasource if not given."
 )
 @click.option(
-    '--cap-style', type=click.Choice(['flat', 'round', 'square']), default='round',
-    callback=_cb_cap_style, help="Where geometries terminate, use this style. (default: round)"
+    '--cap-style', type=click.Choice(['flat', 'round', 'square']),
+    default='round', show_default=True,
+    callback=_cb_cap_style, help="Where geometries terminate, use this style."
 )
 @click.option(
-    '--join-style', type=click.Choice(['round', 'mitre', 'bevel']), default='round',
-    callback=_cb_join_style, help="Where geometries touch, use this style. (default: round)"
+    '--join-style', type=click.Choice(['round', 'mitre', 'bevel']),
+    default='round', show_default=True,
+    callback=_cb_join_style, help="Where geometries touch, use this style."
 )
 @click.option(
-    '--res', type=click.INT, callback=_cb_res, default=16,
-    help="Resolution of the buffer around each vertex of the object. (default: 16)"
+    '--res', type=click.INT, callback=_cb_res, default=16, show_default=True,
+    help="Resolution of the buffer around each vertex of the geometry."
 )
 @click.option(
-    '--mitre-limit', type=click.FLOAT, default=5.0,
+    '--mitre-limit', type=click.FLOAT, default=5.0, show_default=True,
     help="When using a mitre join, limit the maximum length of the join corner according to "
-         "this ratio. (default: 0.5)"
+         "this ratio."
 )
 @click.option(
-    '--dist', metavar='FLOAT | FIELD', required=True, callback=_cb_dist,
-    help="Buffer distance in georeferenced units according to --buf-dist."
+    '--distance', metavar='FLOAT|FIELD', required=True, callback=_cb_dist,
+    help="Buffer distance or field containing distance values.  Units match --buf-crs.  "
+         "When buffering with a field, feature's with a null value are unaltered."
 )
 @click.option(
     '--src-crs', help="Specify CRS for input data.  Not needed if set in input file."
 )
 @click.option(
-    '--buf-crs', help="Perform buffer operations in a different CRS. (default: --src-crs)"
+    '--buf-crs', help="Perform buffer operations in a different CRS. [default: --src-crs]"
 )
 @click.option(
     '--dst-crs', help="Reproject geometries to a different CRS before writing.  Must be "
-                      "combined with --buf-crs. (default: --src-crs)"
+                      "combined with --buf-crs. [default: --src-crs]"
 )
 @click.option(
-    '--otype', 'output_geom_type', default='MultiPolygon', metavar='GEOMTYPE',
-    help="Specify output geometry type. (default: MultiPolygon)"
+    '--geom-type', 'output_geom_type', default='MultiPolygon',
+    metavar='GEOMTYPE', show_default=True,
+    help="Output layer's geometry type."
 )
 @click.option(
     '--skip-failures', is_flag=True,
     help="Skip geometries that fail somewhere in the processing pipeline."
 )
 @click.option(
-    '--jobs', type=click.IntRange(1, cpu_count()), default=1, metavar="CORES",
-    help="Process geometries in parallel across N cores.  The goal of this flag is speed so "
-         "feature order and ID's are not preserved. (default: 1)"
+    '--jobs', type=click.IntRange(1, cpu_count()), default=1,
+    metavar="CORES", show_default=True,
+    help="Process geometries in parallel across N cores.  Feature ID's and order are not "
+         "preserved if more that 1 cores are used."
 )
 @click.pass_context
 def buffer(ctx, infile, outfile, driver, cap_style, join_style, res, mitre_limit,
-           dist, src_crs, buf_crs, dst_crs, output_geom_type, skip_failures, jobs):
+           distance, src_crs, buf_crs, dst_crs, output_geom_type, skip_failures, jobs):
 
     """
     Geometries can be dilated with a positive distance, eroded with a negative
@@ -195,31 +206,31 @@ def buffer(ctx, infile, outfile, driver, cap_style, join_style, res, mitre_limit
     Default settings - buffer geometries in the input CRS:
 
     \b
-        $ fio buffer in.geojson out.geojson --dist 10
+        $ fio buffer in.geojson out.geojson --distance 10
 
-    Dynamically buffer geometries by a distance stored in the field `magnitude`
+    Dynamically buffer geometries by a distance stored in the field 'magnitude'
     and write as GeoJSON:
 
     \b
         $ fio buffer \\
             in.shp \\
             out.geojson \\
             --driver GeoJSON \\
-            --dist magnitude
+            --distance magnitude
 
     Read geometries from one CRS, buffer in another, and then write to a third:
 
     \b
         $ fio buffer in.shp out.shp \\
-            --dist 10 \\
+            --distance 10 \\
             --buf-crs EPSG:3857 \\
             --dst-crs EPSG:32618
 
     Control cap style, mitre limit, segment resolution, and join style:
 
     \b
         $ fio buffer in.geojson out.geojson \\
-            --dist 0.1 \\
+            --distance 0.1 \\
             --res 5 \\
             --cap-style flat \\
             --join-style mitre \\
@@ -235,15 +246,15 @@ def buffer(ctx, infile, outfile, driver, cap_style, join_style, res, mitre_limit
     if isinstance(getattr(ctx, 'obj'), dict):
         logger.setLevel(ctx.obj.get('verbosity', 1))
 
-    with fio.open(infile, 'r') as src:
+    with fio.open(infile) as src:
 
         logger.debug("Resolving CRS fall backs")
 
         src_crs = src_crs or src.crs
         buf_crs = buf_crs or src_crs
         dst_crs = dst_crs or src_crs
 
-        if src_crs is None:
+        if not src_crs:
             raise click.ClickException(
                 "CRS is not set in input file.  Use --src-crs to specify.")
 
@@ -266,7 +277,7 @@ def buffer(ctx, infile, outfile, driver, cap_style, join_style, res, mitre_limit
 
             # Keyword arguments for `<Geometry>.buffer()`
             buf_args = {
-                'distance': dist,
+                'distance': distance,
                 'resolution': res,
                 'cap_style': cap_style,
                 'join_style': join_style,
@@ -287,6 +298,12 @@ def buffer(ctx, infile, outfile, driver, cap_style, join_style, res, mitre_limit
             logger.debug("Starting processing on %s cores", jobs)
             for o_feat in Pool(jobs).imap_unordered(_processor, task_generator):
                 if o_feat is not None:
-                    dst.write(o_feat)
+                    try:
+                        dst.write(o_feat)
+                    except Exception:
+                        logger.exception(
+                            "Feature with ID %s failed during write", o_feat.get('id'))
+                        if not skip_failures:
+                            raise
 
             logger.debug("Finished processing.")
diff --git a/setup.py b/setup.py
@@ -53,10 +53,14 @@
         buffer=fio_buffer.core:buffer
     """,
     extras_require={
-        'test': ['pytest', 'pytest-cov']
+        'dev': ['pytest', 'pytest-cov']
     },
     include_package_data=True,
-    install_requires=['click>=0.3', 'shapely', 'fiona>=0.6'],
+    install_requires=[
+        'click>=0.3',
+        'shapely',
+        'fiona>=1.6'
+    ],
     keywords='Fiona fio GIS vector buffer plugin',
     license="New BSD",
     long_description=readme,
diff --git a/tests/test_cli.py b/tests/test_cli.py
diff --git a/tests/test_processor.py b/tests/test_processor.py
