#20 - Resolve flake8 linting and fix readme

Author: patkyn

README.md

@@ -58,19 +58,29 @@ To install published package from testpypi
 pip install -i https://test.pypi.org/simple/ dwcahandler
 ```
  
-### Extensions that are currently supported and have been tested in ALA ingestion:
-Standard Darwin Core Terms and Class  
-Simple Multimedia https://rs.gbif.org/extension/gbif/1.0/multimedia.xml
-Extended Measurement Or Fact http://rs.iobis.org/obis/terms/ExtendedMeasurementOrFact
+### Supported extensions that have been tested in ALA:
+* Standard Darwin Core Terms and Class
+* Simple Multimedia https://rs.gbif.org/extension/gbif/1.0/multimedia.xml
+* Extended Measurement Or Fact http://rs.iobis.org/obis/terms/ExtendedMeasurementOrFact
 
+#### Terms
+* Terms are listed in [terms.csv](src/dwcahandler/dwca/terms/terms.csv)
+```python
+from dwcahandler import DwcaHandler
 
-* List terms that is supported in dwcahandler package in [terms.csv](src/dwcahandler/dwca/terms/terms.csv)
+df_terms, df_class = DwcaHandler.list_terms()
+print(df_terms, df_class)
+```
 
-* Class RowTypes are defined in MetaElementTypes enum class MetaElementTypes.
-  The supported types are defined by the class column in [class-rowtype.csv](src/dwcahandler/dwca/terms/class-rowtype.csv)   
-  For eg: MetaElementTypes.OCCURRENCE
+#### Class
+* Listed in [class-rowtype.csv](src/dwcahandler/dwca/terms/class-rowtype.csv)
+* Used in MetaElementTypes class enum name:
+```python 
+MetaElementTypes.OCCURRENCE
+MetaElementTypes.MULTIMEDIA
+```
 
-To list all the class rowtypes supported
+To list all the class rowtypes
 ```python
 from dwcahandler import DwcaHandler
 
@@ -151,11 +161,3 @@ DwcaHandler.delete_records(dwca_file='/tmp/dwca.zip',
                            output_dwca='/tmp/new-dwca.zip')
 ```
  
-
-```python
-from dwcahandler import DwcaHandler
-
-df_terms, df_class = DwcaHandler.list_terms()
-print(df_terms, df_class)
-```
- 

src/dwcahandler/dwca/core_dwca.py

@@ -351,9 +351,9 @@ def set_keys(self, keys: dict = None):
                     col_term = []
                     for a_key in key_list:
                         if a_key not in dwca_content.df_content.columns.tolist():
-                           col_term.append(Terms.extract_term(a_key))
+                            col_term.append(Terms.extract_term(a_key))
                         else:
-                           col_term.append(a_key)
+                            col_term.append(a_key)
                     dwca_content.keys = col_term
                     set_keys[k] = col_term
 
@@ -881,7 +881,7 @@ def validate_content(self, content_to_validate: dict = None, error_file: str = N
 
                 if not self.check_duplicates(keys_df, content.keys, error_file):
                     log.error("Validation failed for %s %s content for duplicates keys %s",
-                             content.meta_info.core_or_ext_type, content.meta_info.type, content.keys)
+                              content.meta_info.core_or_ext_type, content.meta_info.type, content.keys)
                     validation_content_success = False
 
                 if not self._validate_columns(content):
@@ -891,7 +891,7 @@ def validate_content(self, content_to_validate: dict = None, error_file: str = N
 
                 if validation_content_success:
                     log.info("Validation successful for %s %s content for unique keys %s",
-                         content.meta_info.core_or_ext_type, content.meta_info.type, content.keys)
+                             content.meta_info.core_or_ext_type, content.meta_info.type, content.keys)
                 else:
                     validation_success = False
 

src/dwcahandler/dwca/dwca_meta.py

@@ -19,6 +19,7 @@
 
 MetaElementTypes = Enum("MetaElementTypes", dict(DwcClassRowTypes))
 
+
 def get_meta_class_row_type(row_type_uri: str):
     """
     Find a row type by URI

src/dwcahandler/dwca/terms.py

@@ -1,5 +1,4 @@
 import os
-from pathlib import Path
 from dataclasses import dataclass, field
 import re
 import pandas as pd
@@ -12,10 +11,10 @@
 
 this_dir, this_filename = os.path.split(__file__)
 
-log.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
-                    level=log.DEBUG)
+log.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=log.DEBUG)
 log = log.getLogger("DwcaTerms")
 
+
 def absolute_file_paths(directory):
     """Convert files in a directory into absolute paths and return
     as a generator
@@ -59,10 +58,11 @@ class GbifRegisteredExt(ExtInfo, Enum):
     EXTENDED_MEASUREMENT_OR_FACT = ExtInfo(uri="http://rs.iobis.org/obis/terms/ExtendedMeasurementOrFact",
                                            prefix=NsPrefix.OBIS,
                                            namespace="http://rs.iobis.org/obis/terms/")
-    #AC_MULTIMEDIA = ExtInfo(uri="http://rs.tdwg.org/ac/terms/Multimedia",
+    # AC_MULTIMEDIA = ExtInfo(uri="http://rs.tdwg.org/ac/terms/Multimedia",
     #                        prefix=NsPrefix.AC,
     #                        namespace="http://rs.tdwg.org/ac/terms/")
 
+
 @dataclass
 class Terms:
     """
@@ -119,7 +119,7 @@ def _update_df(self, ns: NsPrefix, updates: pd.DataFrame, df: pd.DataFrame):
         :param updates: dataframe containing the class rows or terms to update
         :param df: dataframe to update
         """
-        def __get_update_info (update_df: pd.DataFrame):
+        def __get_update_info(update_df: pd.DataFrame):
             update_type: str = "term"
             count = len(update_df)
             if 'class' in update_df.columns.tolist():
@@ -137,7 +137,6 @@ def __get_update_info (update_df: pd.DataFrame):
     def get_dwc_source_data() -> pd.DataFrame:
         return pd.read_csv(Terms.DWC_SOURCE_URL, delimiter=",", encoding='utf-8', dtype='str')
 
-    #@staticmethod
     def update_dwc_terms(self):
         """
         Pull the latest terms from gbif dwc csv url and update the darwin core vocab terms in the package
@@ -195,7 +194,6 @@ def get_class_row_types():
         class_list = list(tuple(zip(class_df["class"], class_df["class_uri"])))
         return class_list
 
-    #@staticmethod
     def update_gbif_ext(self):
         """
         Update the class row type and terms specified by GBIF_REGISTERED_EXTENSION and update by prefix
@@ -239,7 +237,7 @@ def _extract_value(text: str):
 
                     df = pd.DataFrame(term_info, columns=["term", "namespace", 'uri'])
                     std_ns = ["http://rs.tdwg.org/dwc/terms/", "http://purl.org/dc/terms/"]
-                    existing_terms = self.terms_df #Terms().terms_df
+                    existing_terms = self.terms_df
                     extra_terms_df = df[(df["namespace"].isin(std_ns)) & (~df["uri"].isin(existing_terms["uri"]))]
                     if len(extra_terms_df) > 0:
                         log.info("Additional standard terms found:\n%s", extra_terms_df)
@@ -266,7 +264,6 @@ def __sort_values(df_to_sort: pd.DataFrame, sorting_column: str) -> pd.DataFrame
             ext_df = df_to_sort[~std_filter_df].copy()
             return pd.concat([std_df, ext_df], ignore_index=True)
 
-
         log.info("Current class and terms")
 
         exclude_update_prefixes = [NsPrefix.DC.value]
@@ -281,8 +278,8 @@ def __sort_values(df_to_sort: pd.DataFrame, sorting_column: str) -> pd.DataFrame
         terms.terms_df = terms.terms_df[terms.terms_df.prefix.isin(exclude_update_prefixes)]
         terms.update_dwc_terms()
         terms.update_gbif_ext()
-        terms.class_df = __sort_values(terms.class_df,  "class")
-        terms.terms_df = __sort_values(terms.terms_df,  "term")
+        terms.class_df = __sort_values(terms.class_df, "class")
+        terms.terms_df = __sort_values(terms.terms_df, "term")
         terms.class_df.to_csv(Terms.CLASS_ROW_TYPE_PATH, index=False)
         terms.terms_df.to_csv(Terms.TERMS_FILE_PATH, index=False)
 

src/dwcahandler/scripts/update_terms.py

@@ -10,6 +10,7 @@
 
 from dwcahandler.dwca.terms import Terms
 
+
 def update_terms():
     """
     Call the update_dwc_terms to get the latest version of tdwg dwc terms

tests/test_create_dwca.py

@@ -155,4 +155,4 @@ def test_create_occurrence_dwca_occurrence_without_ext(self):
 
         assert output_obj
 
-        check_output(output_obj, test_files_folder)
+        check_output(output_obj, test_files_folder)

tests/test_validate_dwca.py

@@ -119,4 +119,3 @@ def test_dwca_with_occ_core_ext_with_duplicates(self, caplog):
         assert "Duplicate ['gbifID'] found. Total rows affected: 3" in caplog.messages
         assert "Duplicate values: ['sample']" in caplog.messages
         assert "Validation failed for extension MetaElementTypes.OCCURRENCE content for duplicates keys ['gbifID']" in caplog.messages
-