Merge pull request #24 from johenglisch/error-messages

Error messages

.github/workflows/python-package.yml

@@ -12,12 +12,12 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.7, 3.8, 3.9, "3.10"]
+        python-version: [3.8, 3.9, "3.10", 3.11, 3.12, 3.13]
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies

setup.cfg

@@ -71,7 +71,7 @@ test =
 [flake8]
 ignore = E711,E712,D100,D101,D103,D102,D301,W503
 max-line-length = 100
-exclude = .tox
+exclude = .tox,ENV
 
 [easy_install]
 zip_ok = false
@@ -92,7 +92,7 @@ show_missing = true
 skip_covered = true
 
 [tox:tox]
-envlist = py37, py38, py39, py310, py311
+envlist = py38, py39, py310, py311, py312, py313
 isolated_build = true
 skip_missing_interpreter = true
 

src/pynorare/__main__.py

@@ -11,6 +11,7 @@
 from pyconcepticon import Concepticon
 
 from pynorare import NoRaRe
+from pynorare.util import NoRaReError
 import pynorare.commands
 
 
@@ -58,6 +59,9 @@ def main(args=None, catch_all=False, parsed_args=None):
         except ParserError as e:  # pragma: no cover
             print(e)
             return main([args._command, '-h'])
+        except NoRaReError as e:
+            args.log.error(e)
+            return 1
         except Exception as e:  # pragma: no cover
             if catch_all:  # pragma: no cover
                 print(e)

src/pynorare/api.py

@@ -19,6 +19,7 @@
 import pybtex.database
 
 from pynorare.files import get_mappings, get_excel, download_file
+from pynorare.util import read_wellformed_tsv_or_die
 
 __all__ = ['NoRaRe']
 
@@ -236,7 +237,7 @@ def __init__(self, repos=None, concepticon=None):
                 pass
 
         variables = collections.defaultdict(list)
-        for row in reader(self.repos / 'norare.tsv', delimiter='\t', dicts=True):
+        for row in read_wellformed_tsv_or_die(self.repos / 'norare.tsv'):
             variables[row['DATASET']].append(
                 Variable(**{k.lower(): v for k, v in row.items()}))
 
@@ -249,7 +250,7 @@ def __init__(self, repos=None, concepticon=None):
 
         all_refs = set(self.refs).union(concepticon.bibliography if concepticon else {})
 
-        for row in reader(self.repos / 'datasets.tsv', delimiter='\t', dicts=True):
+        for row in read_wellformed_tsv_or_die(self.repos / 'datasets.tsv'):
             self.datasets[row['ID']] = Dataset(
                 variables=variables[row['ID']],
                 csvwmdpath=datasetsdir / row['ID'] / '{}.tsv-metadata.json'.format(row['ID']),

src/pynorare/cli_util.py

@@ -1,5 +1,7 @@
 import pathlib
 
+from pynorare.util import NoRaReError
+
 
 def add_datasets(parser):
     parser.add_argument(
@@ -11,11 +13,27 @@ def add_datasets(parser):
 
 
 def iter_datasets(args):
-    dsids = [
-        pathlib.Path(dsid).name if pathlib.Path(dsid).exists() else dsid for dsid in args.dataset]
-    for dsid, ds in sorted(args.api.datasets.items(), key=lambda i: i[0]):
-        if args.start_from and dsid < args.start_from:
-            continue  # pragma: no cover
-        if (not dsids) or dsid in dsids:
-            ds.log = args.log
-            yield ds
+    if args.dataset:
+        desired_datasets = [
+            pathlib.Path(dsid).name if pathlib.Path(dsid).exists() else dsid
+            for dsid in args.dataset]
+    else:
+        desired_datasets = args.api.datasets
+    if args.start_from:
+        desired_datasets = [
+            dsid
+            for dsid in desired_datasets
+            if dsid >= args.start_from]
+
+    unknown_datasets = [
+        dsid
+        for dsid in desired_datasets
+        if dsid not in args.api.datasets]
+    if unknown_datasets:
+        raise NoRaReError(
+            'Unknown dataset(s): {}'.format(', '.join(unknown_datasets)))
+
+    for dsid in desired_datasets:
+        ds = args.api.datasets[dsid]
+        ds.log = args.log
+        yield ds

src/pynorare/files.py

@@ -2,11 +2,12 @@
 
 import requests
 from cldfcatalog import Config
-from csvw.dsv import reader
 from pyconcepticon import Concepticon
 import xlrd
 import openpyxl
 
+from pynorare.util import read_wellformed_tsv_or_die
+
 
 def get_mappings(concepticon=None):
     concepticon = concepticon or Concepticon(Config.from_file().get_clone('concepticon'))
@@ -15,7 +16,7 @@ def get_mappings(concepticon=None):
     mappings = {}
     for language, path in paths.items():
         mappings[language] = collections.defaultdict(set)
-        for line in reader(path, delimiter='\t', dicts=True):
+        for line in read_wellformed_tsv_or_die(path):
             gloss = line['GLOSS'].split('///')[1]
             oc = concepticon.conceptsets[line['ID']].ontological_category
             mappings[language][gloss].add((line['ID'], int(line['PRIORITY']), oc))

src/pynorare/util.py

@@ -1,6 +1,47 @@
+import csv
+from itertools import islice, zip_longest
+
 from tqdm import tqdm
 
 
+class NoRaReError(Exception):
+    # Challenge of the day:  Try saying NoRaReError three times in a row
+    # without tripping over your tongue. :P
+    pass
+
+
+def read_wellformed_tsv_or_die(file_name):
+    with open(file_name, encoding='utf-8') as f:
+        rdr = csv.reader(f, delimiter='\t')
+        table = list(rdr)
+
+    if not table:
+        raise NoRaReError(f'{file_name}: file empty')
+    empty_rows = [
+        row_no
+        for row_no, row in enumerate(islice(table, 1, None), 2)
+        if not row or all(not cell for cell in row)]
+    if empty_rows:
+        raise NoRaReError('\n'.join(
+            f'{file_name}: {row_no}: empty row'
+            for row_no in empty_rows))
+
+    header = table[0]
+
+    trailing_cells = [
+        (row_no, len(row))
+        for row_no, row in enumerate(islice(table, 1, None), 2)
+        if any(islice(row, len(header), None))]
+    if trailing_cells:
+        raise NoRaReError('\n'.join(
+            f'{file_name}: {row_no}: trailing cells'
+            for row_no, row_width in trailing_cells))
+
+    return [
+        {h: c for h, c in zip_longest(header, row, fillvalue='') if h}
+        for row in islice(table, 1, None)]
+
+
 def progressbar(iterable=None, **kw):
     kw.setdefault('leave', False)
     kw.setdefault('desc', 'norare')

tests/test_cli.py

@@ -43,5 +43,17 @@ def test_workflow(_main, mocker):
     _main('map', 'ds2')
 
 
+def test_errors(_main, capsys):
+    _main('download', 'non-existing-dataset')
+    _, err = capsys.readouterr()
+    assert 'non-existing-dataset' in err
+    _main('map', 'non-existing-dataset')
+    _, err = capsys.readouterr()
+    assert 'non-existing-dataset' in err
+    _main('validate', 'non-existing-dataset')
+    _, err = capsys.readouterr()
+    assert 'non-existing-dataset' in err
+
+
 def test_check(_main):
     _main('check')