Some minor fixes.

setup.cfg

@@ -5,3 +5,9 @@ testpaths = tests
 branch = true
 source =
     src
+
+[pycodestyle]
+max-line-length=120
+
+[flake8]
+max-line-length=120

src/karp5/cli/__init__.py

@@ -1,7 +1,6 @@
 import logging
 import logging.handlers
 import os
-import sys
 
 import click
 
@@ -13,38 +12,20 @@
 
 _logger = logging.getLogger('karp5')
 
-usage = """
-|SCRIPT| --create_metadata
-    Generate 'config/fieldmappings.json' from the 'config/mappings/fieldmappings_*.json' files.
 
-|SCRIPT| --create_mode MODE SUFFIX
-|SCRIPT| --create_empty_index MODE SUFFIX
-|SCRIPT| --import_mode MODE SUFFIX
-|SCRIPT| --publish_mode MODE SUFFIX
-|SCRIPT| --reindex_alias MODE SUFFIX
-|SCRIPT| --getmapping ALIAS OUTFILE
-|SCRIPT| --internalize_lexicon MODE LEXICON1 [LEXICON2 LEXICON3 ... LEXICONM]
-|SCRIPT| --printlatestversion MODE [OUTFILE]
-|SCRIPT| --exportlatestversion MODE [OUTFILE]
-|SCRIPT| --version
-    Prints the version and exits.
-"""
-
-def print_usage(script_name):
-    print(usage.replace('|SCRIPT|', script_name))
-
-
-def setup_cli(config = karp5.Config):
+def setup_cli(config=karp5.Config):
+    print('Setting up logging')
     logger = logging.getLogger('karp5')
     logger.setLevel(config.LOG_LEVEL)
     formatter = logging.Formatter(
-        fmt = config.LOG_FMT,
-        datefmt = config.LOG_DATEFMT
+        fmt=config.LOG_FMT,
+        datefmt=config.LOG_DATEFMT
     )
 
     if config.TESTING or config.DEBUG or config.LOG_TO_STDERR:
         stream_handler = logging.StreamHandler()
         stream_handler.setLevel(config.LOG_LEVEL)
+        stream_handler.setFormatter(formatter)
         logger.addHandler(stream_handler)
     else:
         log_dir = config.LOG_DIR
@@ -58,78 +39,91 @@ def setup_cli(config = karp5.Config):
             backupCount=0
         )
         file_handler.setLevel(config.LOG_LEVEL)
+        file_handler.setFormatter(formatter)
         logger.addHandler(file_handler)
 
         stream_handler = logging.StreamHandler()
         stream_handler.setLevel(logging.WARNING)
+        stream_handler.setFormatter(formatter)
         logger.addHandler(stream_handler)
 
+
 @click.group()
 @click.version_option(
     karp5.get_version(),
     prog_name='karp5'
 )
 def cli():
-    pass
-    # setup_cli()
-
-    # return cli
+    setup_cli()
 
 
-# def register_commands(cli):
 @cli.command(
     'create_metadata',
     short_help='Generate fieldmappings.'
 )
 def create_metadata():
-    """Generate 'config/fieldmappings.json' from the 'config/mappings/fieldmappings_*.json' files."""
+    """
+    Generate 'config/fieldmappings.json'.
+
+    Generate from the 'config/mappings/fieldmappings_*.json' files.
+    NOTE: Not needed since version 5.9.0
+    """
     outpath = 'config/fieldmappings.json'
     metadata.print_all(outpath)
 
+
 @cli.command('create_mode')
 @click.argument('mode')
 @click.argument('suffix')
 def create_mode(mode, suffix):
     upload.create_mode(mode, suffix)
     print('Upload successful')
 
+
 @cli.command('create_empty_index')
 @click.argument('mode')
 @click.argument('suffix')
 def create_empty_index(mode, suffix):
     upload.create_empty_index(mode, suffix)
     print('Index created successfully')
 
+
 @cli.command('import_mode')
 @click.argument('mode')
 @click.argument('suffix')
 def import_mode(mode, suffix):
     upload.create_mode(mode, suffix, with_id=True)
     print('Upload successful')
 
+
 @cli.command('publish_mode')
 @click.argument('mode')
 @click.argument('suffix')
 def publish_mode(mode, suffix):
     """Publish MODE to all modes that contain MODE."""
     upload.publish_mode(mode, suffix)
-    print("Published '{mode}_{suffix}' successfully to '{mode}'".format(mode=mode, suffix=suffix))
+    print("Published '{mode}_{suffix}' successfully to '{mode}'".format(
+        mode=mode,
+        suffix=suffix
+    ))
+
 
 @cli.command('reindex_alias')
 @click.argument('index')
 @click.argument('target_suffix')
 def reindex_alias(index, target_suffix):
-    # target_suffix = argv[3]
     ret = upload.reindex_alias(index, target_suffix)
     if not ret:
         raise click.ClickException('Something went wrong')
 
+
 @cli.command('getmapping')
 @click.argument('alias')
 @click.argument('outfile')
 def getmapping(alias, outfile):
     gm.getmapping(alias, outfile)
 
+
 @cli.command('internalize_lexicon')
 @click.argument('mode', nargs=1)
 @click.argument('to_upload', nargs=-1)
@@ -141,6 +135,7 @@ def internalize_lexicon(mode, to_upload):
     upload.internalize_lexicon(mode, to_upload)
     print('Upload successful')
 
+
 @cli.command('printlatestversion')
 @click.argument('lexicon', metavar='<lexicon>')
 @click.option(
@@ -157,6 +152,7 @@ def printlatestversion(lexicon, output):
     else:
         upload.printlatestversion(lexicon)
 
+
 @cli.command('exportlatestversion')
 @click.argument('lexicon', metavar='<lexicon>')
 @click.option(
@@ -169,10 +165,16 @@ def printlatestversion(lexicon, output):
 def exportlatestversion(lexicon, output):
     if output:
         with open(output, 'w') as fp:
-            upload.printlatestversion(lexicon, debug=False, with_id=True, file=fp)
+            upload.printlatestversion(
+                lexicon,
+                debug=False,
+                with_id=True,
+                file=fp
+            )
     else:
         upload.printlatestversion(lexicon, debug=False, with_id=True)
 
+
 # Commented since dangerous!
 # @cli.command('delete_all')
 # def delete_all():
@@ -190,7 +192,7 @@ def exportlatestversion(lexicon, output):
 #     print('Deletion successful')
 
 
- # Commented since dangerous!
+# Commented since dangerous!
 # @cli.command('delete_mode')
 # @click.argument('mode')
 # def delete_mode(mode):

src/karp5/cli/upload_offline.py

@@ -8,7 +8,7 @@
 
 from elasticsearch import helpers as es_helpers
 from elasticsearch import exceptions as esExceptions
-import elasticsearch_dsl as es_dsl
+# import elasticsearch_dsl as es_dsl
 
 import karp5.dbhandler.dbhandler as db
 from karp5.config import mgr as conf_mgr
@@ -24,8 +24,6 @@
 # ==============================================
 
 
-
-
 def make_indexname(index, suffix):
     return index+'_'+suffix
 
@@ -82,7 +80,7 @@ def upload(informat, name, order, data, elastic, index, typ, sql=False,
         sql_bulk = []
         for res in es_helpers.streaming_bulk(elastic, data, request_timeout=60):
             # res is a tuple, res[0]==True
-            ansname = 'index' # if with_id else 'create' -- changed from ES2
+            ansname = 'index'  # if with_id else 'create' -- changed from ES2
             _id = res[1].get(ansname).get('_id')
             data_doc = data[ok].get('_source')
             if not isinstance(data_doc, dict):
@@ -142,7 +140,7 @@ def get_entries_to_keep_from_sql(lexicons):
                         _logger.debug('|get_entries_to_keep_from_sql| Update entry.')
                         to_keep[_id] = entry
                 else:
-                        to_keep[_id] = entry
+                    to_keep[_id] = entry
             else:
                 _logger.debug('|sql no id| Found entry without id:')
                 _logger.debug('|sql no id| {}}'.format(entry))
@@ -172,7 +170,7 @@ def recover(alias, suffix, lexicon, create_new=True):
     data = bulk.bulkify_sql(to_keep, bulk_info={'index': index, 'type': typ})
     try:
         ok, err = es_helpers.bulk(es, data, request_timeout=30)
-    except:
+    except Exception:
         print(data)
     if err:
         msg = "Error during upload. %s documents successfully uploaded. \
@@ -307,9 +305,6 @@ def create_empty_index(mode, suffix):
     _create_index(mode, make_indexname(mode, suffix))
 
 
-
-
-
 def create_mode(alias, suffix, with_id=False):
     es = conf_mgr.elastic(alias)
     if conf_mgr.modes[alias]['is_index']:
@@ -324,10 +319,12 @@ def create_mode(alias, suffix, with_id=False):
         try:
             lexicons = conf_mgr.get_lexiconlist(index)
             load(lexicons, newname, typ, es, with_id=with_id)
-        except Exception as e:
+        except Exception:
             # delete the index if things did not go well
             ans = es.indices.delete(newname)
-            _logger.error('Any documentes uploaded to ES index %s are removed.' % newname)
+            _logger.error(
+                "Any documentes uploaded to ES index %s are removed. ans = '%s'" % (newname, ans)
+            )
             _logger.error('If data was uploaded to SQL you will have to remove it manually.')
             raise
 
@@ -354,8 +351,10 @@ def add_lexicon(to_add_name, to_add_file, alias, suffix):
     except Exception:
         # delete the index if things did not go well
         ans = es.indices.delete(indexname)
-        #print(ans)
-        _logger.error('Any documentes uploaded to ES index %s are removed.' % indexname)
+        # print(ans)
+        _logger.error(
+            'Any documentes uploaded to ES index %s are removed. ans=%s' % (indexname, ans)
+        )
         _logger.error('If data was uploaded to SQL you will have to remove it manually.')
         raise
 
@@ -373,8 +372,15 @@ def internalize_lexicon(mode, to_add):
         # Go through each lexicon separately
         query = {"query": {"term": {"lexiconName": lex}}}
         # scan and scroll
-        ans = es_helpers.scan(es, query=query, scroll=u'3m', raise_on_error=True,
-                           preserve_order=False, index=mode, request_timeout=30)
+        ans = es_helpers.scan(
+            es,
+            query=query,
+            scroll=u'3m',
+            raise_on_error=True,
+            preserve_order=False,
+            index=mode,
+            request_timeout=30
+        )
         sql_bulk = []
         for hit in ans:  # ans is an iterator of objects from in hits.hits
             _id = hit.get('_id')
@@ -429,7 +435,7 @@ def apply_filter(it, filter_func, field=None):
     filtered = []
     for i in it:
         for x in filter_func(i[field], i['_id']):
-            filtered.append({ field: x})
+            filtered.append({field: x})
     return filtered
 
 
@@ -573,19 +579,20 @@ def make_structure():
     return es.indices.update_aliases('{"actions" : [%s]}'
                                      % ','.join(add_actions), request_timeout=30)
 
+
 def delete_all():
     # delete all indices
     for alias, aliasconf in conf_mgr.modes.items():
         es = conf_mgr.elastic(alias)
         try:
             es.indices.delete('*')
-        except:
+        except Exception:
             print('could not delete es data form mode %s' % alias)
         try:
             # delete all our lexicons in sql
             for name in conf_mgr.get_lexiconlist(alias):
                 db.deletebulk(lexicon=name)
-        except:
+        except Exception:
             print('could not delete sql data form mode %s' % alias)
     print('Successfully deleted all data')
 
@@ -594,14 +601,14 @@ def delete_mode(mode):
     # delete all indices
     es = conf_mgr.elastic(mode)
     try:
-        #print('delete', '%s*' % mode)
+        # print('delete', '%s*' % mode)
         es.indices.delete('%s*' % mode)
-    except:
+    except Exception:
         print('could not delete es data form mode %s' % mode)
     try:
         # delete all our lexicons in sql
         for name in conf_mgr.get_lexiconlist(mode):
             db.deletebulk(lexicon=name)
-    except:
+    except Exception:
         print('could not delete sql data form mode %s' % mode)
     print('Successfully deleted all data')