fixes warning messages from pandas

Author: patkyn

src/dwcahandler/dwca/core_dwca.py

@@ -235,7 +235,9 @@ def _add_new_columns(self, df_content, delta_df_content):
         delta_df_columns = delta_df_content.columns.to_list()
         new_columns = list(set(delta_df_columns) - set(df_columns))
         if len(new_columns) > 0:
-            df_content[new_columns] = nan
+            # Set to empty string instead of nan to resolve warning message
+            # see https://pandas.pydata.org/pdeps/0006-ban-upcasting.html
+            df_content[new_columns] = ""
 
         return new_columns
 

tests/test_merge_dwca.py

@@ -5,7 +5,7 @@
 from tests import make_dwca
 from dwcahandler import DwcaHandler
 from io import BytesIO
-
+from numpy import nan
 
 class TestMergeDwcaContent:
 
@@ -82,13 +82,13 @@ def test_merge_core_and_ext_records(self):
 
         delta_occ_df = pd.DataFrame(data=[["3", "species3", "-40.0000", "144.0000", "Observation"],
                                           ["4", "species4", "-10.0000", "144.0000", "Observation"],
-                                          ["5", "species5", "-20.0000", "145.0000", ""],
-                                          ["6", "species6", "-30.0000", "146.3048", ""]],
+                                          ["5", "species5", "-20.0000", "145.0000", nan],
+                                          ["6", "species6", "-30.0000", "146.3048", nan]],
                                     columns=["occurrenceID", "scientificName", "decimalLatitude", "decimalLongitude",
                                              "basisOfRecord"])
 
         delta_multimedia_df = pd.DataFrame(data=[["3", "https://new-image3.webp", "image/webp", "StillImage", "RightsHolder3"],
-                                                 ["4", "https://image4.webp", "image/webp", "StillImage", None],
+                                                 ["4", "https://image4.webp", "image/webp", "StillImage", nan],
                                                  ["5", "https://image5.webp", "image/webp", "StillImage", "RightsHolder5"],
                                                  ["6", "https://image6.webp", "image/webp", "StillImage", "RightsHolder6"]],
                                            columns=["occurrenceID", "identifier", "format", "type", "rightsHolder"])
@@ -105,18 +105,19 @@ def test_merge_core_and_ext_records(self):
                                output_dwca_path=output_obj,
                                keys_lookup=keys_lookup)
 
-        expected_occ_df = pd.DataFrame(data=[["1", "species1", "-30.0000", "144.0000", None],
-                                             ["2", "species2", "-28.0000", "115.0000", None],
+        expected_occ_df = pd.DataFrame(data=[["1", "species1", "-30.0000", "144.0000", nan],
+                                             ["2", "species2", "-28.0000", "115.0000", nan],
                                              ["3", "species3", "-40.0000", "144.0000", "Observation"],
                                              ["4", "species4", "-10.0000", "144.0000", "Observation"],
-                                             ["5", "species5", "-20.0000", "145.0000", None],
-                                             ["6", "species6", "-30.0000", "146.3048", None]],
-                                       columns=['occurrenceID', 'scientificName', 'decimalLatitude', 'decimalLongitude', 'basisOfRecord'])
+                                             ["5", "species5", "-20.0000", "145.0000", nan],
+                                             ["6", "species6", "-30.0000", "146.3048", nan]],
+                                       columns=['occurrenceID', 'scientificName', 'decimalLatitude', 'decimalLongitude',
+                                                'basisOfRecord'])
 
-        expected_multimedia_df = pd.DataFrame(data=[["1", "https://image1.jpg", "image/jpeg", "StillImage", None],
-                                                    ["2", "https://image2.jpg", "image/jpeg", "StillImage", None],
+        expected_multimedia_df = pd.DataFrame(data=[["1", "https://image1.jpg", "image/jpeg", "StillImage", nan],
+                                                    ["2", "https://image2.jpg", "image/jpeg", "StillImage", nan],
                                                     ["3", "https://new-image3.webp", "image/webp", "StillImage", "RightsHolder3"],
-                                                    ["4", "https://image4.webp", "image/webp", "StillImage", None],
+                                                    ["4", "https://image4.webp", "image/webp", "StillImage", nan],
                                                     ["5", "https://image5.webp", "image/webp", "StillImage", "RightsHolder5"],
                                                     ["6", "https://image6.webp", "image/webp", "StillImage", "RightsHolder6"]],
                                               columns=["occurrenceID", "identifier", "format", "type", "rightsHolder"])