Merge pull request #322 from iragm/march-2025-fixes

More CSV fixes

Author: iragm

auctions/models.py

@@ -734,6 +734,12 @@ def delete(self, *args, **kwargs):
         self.is_deleted = True
         self.save()
 
+    def save(self, *args, **kwargs):
+        if self.date_start.year < 2000:
+            current_year = timezone.now().year
+            self.date_start = self.date_start.replace(year=current_year)
+        super().save(*args, **kwargs)
+
     @property
     def location_qs(self):
         """All locations associated with this auction"""
@@ -1760,7 +1766,7 @@ def check_number_in_auction(number):
                 # self.print_reminder_email_sent = True
                 # self.second_confirm_email_sent = True
             if self.email and not self.user:
-                self.user = User.objects.filter(active=True, email=self.email).first()
+                self.user = User.objects.filter(is_active=True, email=self.email).first()
         # fill out some fields from user, if set
         # There is a huge security concern here:   <<<< ATTENTION!!!
         # If someone creates an auction and adds every email address that's public

auctions/templates/auctions/bulk_add_users.html

@@ -8,8 +8,8 @@
   <br>
 
   <div class="btn-group">
-  {% if request.user.is_superuser %}<input type="file" name="csv_file" id="csv_file" accept=".csv" hx-encoding="multipart/form-data" hx-post="{% url 'bulk_add_users' auction.slug %}" hx-swap="outerHTML" hx-target="body" style="display: none;">
-  <button class="btn btn-primary btn-sm  ms-2 me-2" onclick="document.getElementById('csv_file').click()"><i class="bi bi-person-fill-up"></i> Import from CSV</button>{% endif %}
+  <input type="file" name="csv_file" id="csv_file" accept=".csv" hx-encoding="multipart/form-data" hx-post="{% url 'bulk_add_users' auction.slug %}" hx-swap="outerHTML" hx-target="body" style="display: none;">
+  <button class="btn btn-primary btn-sm  ms-2 me-2" onclick="document.getElementById('csv_file').click()"><i class="bi bi-person-fill-up"></i> Import from CSV</button>
   <div class="dropdown">
     <button class="btn btn-sm btn-primary dropdown-toggle" type="button" id="dropdownMenuButton" data-bs-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
       <i class="bi bi-person-fill-add"></i> Import from old auctions

auctions/views.py

@@ -2674,14 +2674,13 @@ def extract_info(row, field_name_list, default_response=""):
                     pass
             return default_response
 
-        def columns_exist_in_csv(csv_reader, columns):
+        def columns_exist(field_names, columns):
             """returns True if any value in the list `columns` exists in the file"""
-            first_row = next(csv_reader)
-            result = extract_info(first_row, columns, None)
-            if result is None:
-                return False
-            else:
-                return True
+            case_insensitive_row = {k.lower() for k in field_names}
+            for column in columns:
+                if column in case_insensitive_row:
+                    return True
+            return False
 
         csv_file.seek(0)
         csv_reader = csv.DictReader(TextIOWrapper(csv_file.file))
@@ -2697,73 +2696,81 @@ def columns_exist_in_csv(csv_reader, columns):
         # order matters here - the most important columns should be validated last,
         # so the error refers to the most important missing column
         try:
-            if not columns_exist_in_csv(csv_reader, phone_field_names):
+            if not columns_exist(csv_reader.fieldnames, phone_field_names):
                 error = "Warning: This file does not contain a phone column"
             else:
                 some_columns_exist = True
-            if not columns_exist_in_csv(csv_reader, address_field_names):
+            if not columns_exist(csv_reader.fieldnames, address_field_names):
                 error = "Warning: This file does not contain an address column"
             else:
                 some_columns_exist = True
-            if not columns_exist_in_csv(csv_reader, name_field_names):
+            if not columns_exist(csv_reader.fieldnames, name_field_names):
                 error = "Warning: This file does not contain a name column"
             else:
                 some_columns_exist = True
-            if not columns_exist_in_csv(csv_reader, email_field_names):
+            if not columns_exist(csv_reader.fieldnames, email_field_names):
                 error = "Warning: This file does not contain an email column"
             else:
                 some_columns_exist = True
             if not some_columns_exist:
                 error = (
                     "Unable to read information from this CSV file.  Make sure it contains an email and a name column"
                 )
-        except UnicodeDecodeError as e:
+
+            total_tos = 0
+            total_skipped = 0
+            for row in csv_reader:
+                bidder_number = extract_info(row, bidder_number_fields)
+                email = extract_info(row, email_field_names)
+                name = extract_info(row, name_field_names)
+                phone = extract_info(row, phone_field_names)
+                address = extract_info(row, address_field_names)
+                is_club_member = extract_info(row, is_club_member_fields)
+                if is_club_member.lower() in ["yes", "true", "member", "club member"]:
+                    is_club_member = True
+                else:
+                    is_club_member = False
+                if email or name or phone or address:
+                    if self.tos_is_in_auction(self.auction, name, email):
+                        logger.debug("CSV import skipping %s", name)
+                        total_skipped += 1
+                    else:
+                        logger.debug("CSV import adding %s", name)
+                        if bidder_number:
+                            if AuctionTOS.objects.filter(auction=self.auction, bidder_number=bidder_number).first():
+                                bidder_number = ""
+                        AuctionTOS.objects.create(
+                            auction=self.auction,
+                            pickup_location=self.auction.location_qs.first(),
+                            manually_added=True,
+                            bidder_number=bidder_number,
+                            name=name,
+                            phone_number=phone,
+                            email=email,
+                            address=address,
+                            is_club_member=is_club_member,
+                        )
+                        total_tos += 1
+            if error:
+                messages.error(self.request, error)
+            msg = f"{total_tos} users added"
+            if total_skipped:
+                msg += (
+                    f", {total_skipped} users are already in this auction (matched by email, or name if email not set)"
+                )
+            messages.info(self.request, msg)
+            url = reverse("auction_tos_list", kwargs={"slug": self.auction.slug})
+            response = HttpResponse(status=200)
+            response["HX-Redirect"] = url
+            return response
+        except (UnicodeDecodeError, ValueError) as e:
             messages.error(
                 self.request, f"Unable to read file.  Make sure this is a valid UTF-8 CSV file.  Error was: {e}"
             )
             url = reverse("bulk_add_users", kwargs={"slug": self.auction.slug})
             response = HttpResponse(status=200)
             response["HX-Redirect"] = url
             return response
-        total_tos = 0
-        total_skipped = 0
-        for row in csv_reader:
-            bidder_number = extract_info(row, bidder_number_fields)
-            email = extract_info(row, email_field_names)
-            name = extract_info(row, name_field_names)
-            phone = extract_info(row, phone_field_names)
-            address = extract_info(row, address_field_names)
-            is_club_member = extract_info(row, is_club_member_fields)
-            if is_club_member.lower() in ["yes", "true", "member", "club member"]:
-                is_club_member = True
-            else:
-                is_club_member = False
-            if email or name or phone or address:
-                if self.tos_is_in_auction(self.auction, name, email):
-                    total_skipped += 1
-                else:
-                    AuctionTOS.objects.create(
-                        auction=self.auction,
-                        pickup_location=self.auction.location_qs.first(),
-                        manually_added=True,
-                        bidder_number=bidder_number,
-                        name=name,
-                        phone_number=phone,
-                        email=email,
-                        address=address,
-                        is_club_member=is_club_member,
-                    )
-                    total_tos += 1
-        if error:
-            messages.error(self.request, error)
-        msg = f"{total_tos} users added"
-        if total_skipped:
-            msg += f", {total_skipped} users are already in this auction (matched by email, or name if email not set)"
-        messages.info(self.request, msg)
-        url = reverse("auction_tos_list", kwargs={"slug": self.auction.slug})
-        response = HttpResponse(status=200)
-        response["HX-Redirect"] = url
-        return response
 
         # The code below would populate the formset with the info from the CSV.
         # This process is simply not working, and it fails silently with no log.