Merge branch 'master' into add-cache-tables-script

Author: MonkeyDo

Diff for: frontend/js/src/about/add-data/AddData.tsx

@@ -330,14 +330,6 @@ export default function AddData() {
           </em>
           , a scrobbling application for Android Devices
         </li>
-        <li>
-          <em>
-            <a href="https://github.com/tgwizard/sls">
-              Simple Last.fm Scrobbler
-            </a>
-          </em>
-          , for Android devices
-        </li>
       </ul>
 
       <h4>Scripts</h4>

Diff for: listenbrainz/db/external_service_oauth.py

@@ -1,10 +1,9 @@
-from datetime import datetime
+from datetime import datetime, timezone
 from typing import List, Optional, Union
 
 from sqlalchemy import text
 
 from data.model.external_service import ExternalServiceType
-from listenbrainz import db, utils
 import sqlalchemy
 
 
@@ -32,7 +31,7 @@ def save_token(db_conn, user_id: int, service: ExternalServiceType, access_token
     # to use the new values. any column which does not have a new value to be set should
     # be explicitly set to the default value (which would have been used if the row was
     # inserted instead).
-    token_expires = utils.unix_timestamp_to_datetime(token_expires_ts) if token_expires_ts else None
+    token_expires = datetime.fromtimestamp(token_expires_ts, timezone.utc) if token_expires_ts else None
     result = db_conn.execute(sqlalchemy.text("""
         INSERT INTO external_service_oauth AS eso
         (user_id, external_user_id, service, access_token, refresh_token, token_expires, scopes)
@@ -121,7 +120,7 @@ def update_token(db_conn, user_id: int, service: ExternalServiceType, access_tok
         refresh_token: the new token used to refresh access tokens, if omitted the old token in the database remains unchanged
         expires_at: the unix timestamp at which the access token expires
     """
-    token_expires = utils.unix_timestamp_to_datetime(expires_at)
+    token_expires = datetime.fromtimestamp(expires_at, timezone.utc)
     params = {
         "access_token": access_token,
         "token_expires": token_expires,

Diff for: listenbrainz/db/listens_importer.py

@@ -1,10 +1,9 @@
-import datetime
+from datetime import datetime, timezone
 from typing import Optional, Union
 
 from sqlalchemy import text
 
 from data.model.external_service import ExternalServiceType
-from listenbrainz import utils
 import sqlalchemy
 
 
@@ -51,12 +50,12 @@ def update_latest_listened_at(db_conn, user_id: int, service: ExternalServiceTyp
         """), {
             'user_id': user_id,
             'service': service.value,
-            'timestamp': utils.unix_timestamp_to_datetime(timestamp),
+            'timestamp': datetime.fromtimestamp(timestamp, timezone.utc),
         })
     db_conn.commit()
 
 
-def get_latest_listened_at(db_conn, user_id: int, service: ExternalServiceType) -> Optional[datetime.datetime]:
+def get_latest_listened_at(db_conn, user_id: int, service: ExternalServiceType) -> Optional[datetime]:
     """ Returns the timestamp of the last listen imported for the user with
     specified LB user ID from the given service.
 

Diff for: listenbrainz/db/pinned_recording.py

@@ -1,7 +1,6 @@
 import sqlalchemy
-from datetime import datetime
+from datetime import datetime, timezone
 
-from listenbrainz import db
 from listenbrainz.db.model.pinned_recording import PinnedRecording, WritablePinnedRecording
 from typing import List, Iterable
 
@@ -223,8 +222,8 @@ def get_pins_for_feed(db_conn, user_ids: Iterable[int], min_ts: int, max_ts: int
          LIMIT :count
     """.format(columns=','.join(PINNED_REC_GET_COLUMNS))), {
         "user_ids": tuple(user_ids),
-        "min_ts": datetime.utcfromtimestamp(min_ts),
-        "max_ts": datetime.utcfromtimestamp(max_ts),
+        "min_ts": datetime.fromtimestamp(min_ts, timezone.utc),
+        "max_ts": datetime.fromtimestamp(max_ts, timezone.utc),
         "count": count,
     })
     return [PinnedRecording(**row) for row in result.mappings()]

Diff for: listenbrainz/db/user_relationship.py

@@ -16,7 +16,7 @@
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 
-from datetime import datetime
+from datetime import datetime, timezone
 from typing import List, Iterable
 
 import sqlalchemy
@@ -158,8 +158,8 @@ def get_follow_events(db_conn, user_ids: Iterable[int], min_ts: float, max_ts: f
          LIMIT :count
     """), {
         "user_ids": tuple(user_ids),
-        "min_ts": datetime.utcfromtimestamp(min_ts),
-        "max_ts": datetime.utcfromtimestamp(max_ts),
+        "min_ts": datetime.fromtimestamp(min_ts, timezone.utc),
+        "max_ts": datetime.fromtimestamp(max_ts, timezone.utc),
         "count": count
     })
 

Diff for: listenbrainz/db/user_timeline_event.py

@@ -19,7 +19,7 @@
 import sqlalchemy
 import orjson
 
-from datetime import datetime
+from datetime import datetime, timezone
 
 from sqlalchemy import text
 
@@ -34,9 +34,8 @@
     HiddenUserTimelineEvent,
     PersonalRecordingRecommendationMetadata, WritePersonalRecordingRecommendationMetadata
 )
-from listenbrainz import db
 from listenbrainz.db.exceptions import DatabaseException
-from typing import List, Tuple, Iterable
+from typing import List, Iterable
 
 from listenbrainz.db.model.review import CBReviewTimelineMetadata
 
@@ -196,8 +195,8 @@ def get_user_timeline_events(
          LIMIT :count
     """), {
         "user_ids": tuple(user_ids),
-        "min_ts": datetime.utcfromtimestamp(min_ts),
-        "max_ts": datetime.utcfromtimestamp(max_ts),
+        "min_ts": datetime.fromtimestamp(min_ts, timezone.utc),
+        "max_ts": datetime.fromtimestamp(max_ts, timezone.utc),
         "event_type": event_type.value,
         "count": count,
     })
@@ -255,8 +254,8 @@ def get_personal_recommendation_events_for_feed(db_conn, user_id: int, min_ts: i
          LIMIT :count
     """), {
         "user_id": user_id,
-        "min_ts": datetime.utcfromtimestamp(min_ts),
-        "max_ts": datetime.utcfromtimestamp(max_ts),
+        "min_ts": datetime.fromtimestamp(min_ts, timezone.utc),
+        "max_ts": datetime.fromtimestamp(max_ts, timezone.utc),
         "count": count,
         "event_type": UserTimelineEventType.PERSONAL_RECORDING_RECOMMENDATION.value,
     })
@@ -297,8 +296,8 @@ def get_thanks_events_for_feed(db_conn, user_id: int, min_ts: int, max_ts: int,
          LIMIT :count
     """), {
         "user_id": user_id,
-        "min_ts": datetime.utcfromtimestamp(min_ts),
-        "max_ts": datetime.utcfromtimestamp(max_ts),
+        "min_ts": datetime.fromtimestamp(min_ts, timezone.utc),
+        "max_ts": datetime.fromtimestamp(max_ts, timezone.utc),
         "count": count,
         "event_type": UserTimelineEventType.THANKS.value,
     })

Diff for: listenbrainz/listen.py

@@ -1,7 +1,5 @@
-# coding=utf-8
-import calendar
 from copy import deepcopy
-from datetime import datetime
+from datetime import datetime, timezone
 
 import orjson
 
@@ -76,7 +74,7 @@ def __init__(self, user_id=None, user_name=None, timestamp=None, recording_msid=
         # determine the type of timestamp and do the right thing
         if isinstance(timestamp, int) or isinstance(timestamp, float):
             self.ts_since_epoch = int(timestamp)
-            self.timestamp = datetime.utcfromtimestamp(self.ts_since_epoch)
+            self.timestamp = datetime.fromtimestamp(self.ts_since_epoch, timezone.utc)
         else:
             if timestamp:
                 self.timestamp = timestamp
@@ -105,12 +103,12 @@ def from_json(cls, j):
         """Factory to make Listen() objects from a dict"""
         # Let's go play whack-a-mole with our lovely whicket of timestamp fields. Hopefully one will work!
         try:
-            j['listened_at'] = datetime.utcfromtimestamp(float(j['listened_at']))
+            j['listened_at'] = datetime.fromtimestamp(float(j['listened_at']), timezone.utc)
         except KeyError:
             try:
-                j['listened_at'] = datetime.utcfromtimestamp(float(j['timestamp']))
+                j['listened_at'] = datetime.fromtimestamp(float(j['timestamp']), timezone.utc)
             except KeyError:
-                j['listened_at'] = datetime.utcfromtimestamp(float(j['ts_since_epoch']))
+                j['listened_at'] = datetime.fromtimestamp(float(j['ts_since_epoch']), timezone.utc)
 
         return cls(
             user_id=j.get('user_id'),

Diff for: listenbrainz/listenstore/tests/test_timescale_utils.py

@@ -1,4 +1,4 @@
-from datetime import datetime
+from datetime import datetime, timezone
 
 from sqlalchemy import text
 
@@ -35,8 +35,8 @@ def _get_count_and_timestamp(self, user):
             row = result.fetchone()
             return {
                 "count": row.count,
-                "min_listened_at": row.min_listened_at.replace(tzinfo=None),
-                "max_listened_at": row.max_listened_at.replace(tzinfo=None)
+                "min_listened_at": row.min_listened_at,
+                "max_listened_at": row.max_listened_at
             }
 
     def test_delete_listens_update_metadata(self):
@@ -49,35 +49,51 @@ def test_delete_listens_update_metadata(self):
         update_user_listen_data()
 
         metadata_1 = self._get_count_and_timestamp(user_1)
-        self.assertEqual(metadata_1["min_listened_at"], datetime.utcfromtimestamp(1400000000))
-        self.assertEqual(metadata_1["max_listened_at"], datetime.utcfromtimestamp(1400000200))
+        self.assertEqual(metadata_1["min_listened_at"], datetime.fromtimestamp(1400000000, timezone.utc))
+        self.assertEqual(metadata_1["max_listened_at"], datetime.fromtimestamp(1400000200, timezone.utc))
         self.assertEqual(metadata_1["count"], 5)
 
         metadata_2 = self._get_count_and_timestamp(user_2)
-        self.assertEqual(metadata_2["min_listened_at"], datetime.utcfromtimestamp(1400000000))
-        self.assertEqual(metadata_2["max_listened_at"], datetime.utcfromtimestamp(1400000200))
+        self.assertEqual(metadata_2["min_listened_at"], datetime.fromtimestamp(1400000000, timezone.utc))
+        self.assertEqual(metadata_2["max_listened_at"], datetime.fromtimestamp(1400000200, timezone.utc))
         self.assertEqual(metadata_2["count"], 5)
 
         # to test the case when the update script has not run since delete, so metadata in listen_user_metadata does
         # account for this listen and deleting should not affect it either.
         self._create_test_data(user_1, "timescale_listenstore_test_listens_2.json")
-        self.ls.delete_listen(datetime.utcfromtimestamp(1400000500), user_1["id"], "4269ddbc-9241-46da-935d-4fa9e0f7f371")
+        self.ls.delete_listen(
+            datetime.fromtimestamp(1400000500, timezone.utc),
+            user_1["id"],
+            "4269ddbc-9241-46da-935d-4fa9e0f7f371"
+        )
 
         # test min_listened_at is updated if that listen is deleted for a user
-        self.ls.delete_listen(datetime.utcfromtimestamp(1400000000), user_1["id"], "4269ddbc-9241-46da-935d-4fa9e0f7f371")
+        self.ls.delete_listen(
+            datetime.fromtimestamp(1400000000, timezone.utc),
+            user_1["id"],
+            "4269ddbc-9241-46da-935d-4fa9e0f7f371"
+        )
         # test max_listened_at is updated if that listen is deleted for a user
-        self.ls.delete_listen(datetime.utcfromtimestamp(1400000200), user_1["id"], "db072fa7-0c7f-4f55-b90f-a88da531b219")
+        self.ls.delete_listen(
+            datetime.fromtimestamp(1400000200, timezone.utc),
+            user_1["id"],
+            "db072fa7-0c7f-4f55-b90f-a88da531b219"
+        )
         # test normal listen delete updates correctly
-        self.ls.delete_listen(datetime.utcfromtimestamp(1400000100), user_2["id"], "08ade1eb-800e-4ad8-8184-32941664ac02")
+        self.ls.delete_listen(
+            datetime.fromtimestamp(1400000100, timezone.utc),
+            user_2["id"],
+            "08ade1eb-800e-4ad8-8184-32941664ac02"
+        )
 
         delete_listens()
 
         metadata_1 = self._get_count_and_timestamp(user_1)
-        self.assertEqual(metadata_1["min_listened_at"], datetime.utcfromtimestamp(1400000050))
-        self.assertEqual(metadata_1["max_listened_at"], datetime.utcfromtimestamp(1400000150))
+        self.assertEqual(metadata_1["min_listened_at"], datetime.fromtimestamp(1400000050, timezone.utc))
+        self.assertEqual(metadata_1["max_listened_at"], datetime.fromtimestamp(1400000150, timezone.utc))
         self.assertEqual(metadata_1["count"], 3)
 
         metadata_2 = self._get_count_and_timestamp(user_2)
-        self.assertEqual(metadata_2["min_listened_at"], datetime.utcfromtimestamp(1400000000))
-        self.assertEqual(metadata_2["max_listened_at"], datetime.utcfromtimestamp(1400000200))
+        self.assertEqual(metadata_2["min_listened_at"], datetime.fromtimestamp(1400000000, timezone.utc))
+        self.assertEqual(metadata_2["max_listened_at"], datetime.fromtimestamp(1400000200, timezone.utc))
         self.assertEqual(metadata_2["count"], 4)

Diff for: listenbrainz/testdata/sitewide_top_artists_db.json

@@ -14,5 +14,6 @@
     ],
     "to_ts": 1593043183,
     "stats_range": "all_time",
-    "count": 2
+    "count": 2,
+    "total_artist_count": 123
 }

Diff for: listenbrainz/testdata/sitewide_top_artists_db_data_for_api_test.json

@@ -301,5 +301,6 @@
       "artist_mbid": "12379346-99d9-4941-b4a2-f04144a39863"
     }
   ],
-  "count": 60
-}
+  "count": 60,
+  "total_artist_count": 60
+}

Diff for: listenbrainz/testdata/sitewide_top_artists_db_data_for_api_test_month.json

@@ -301,5 +301,6 @@
       "artist_mbid": "78df245f-0fe0-4a21-b1d4-e79f695354fd"
     }
   ],
-  "count": 60
-}
+  "count": 60,
+  "total_artist_count": 60
+}

Diff for: listenbrainz/testdata/sitewide_top_artists_db_data_for_api_test_too_many.json

@@ -1011,5 +1011,6 @@
       "artist_mbid": "218bbe3d-63cc-46bc-8e43-e56a2f6c9829"
     }
   ],
-  "count": 202
-}
+  "count": 202,
+  "total_artist_count": 202
+}

Diff for: listenbrainz/testdata/sitewide_top_artists_db_data_for_api_test_week.json

@@ -1001,5 +1001,6 @@
       "artist_mbid": "a2c1ed01-991f-4a9d-81e0-9b04c8df1c80"
     }
   ],
-  "count": 200
-}
+  "count": 200,
+  "total_artist_count": 200
+}

Diff for: listenbrainz/testdata/sitewide_top_artists_db_data_for_api_test_year.json

@@ -301,5 +301,6 @@
       "artist_mbid": "a63d3de8-9c39-4320-8199-784df3f4c74c"
     }
   ],
-  "count": 60
-}
+  "count": 60,
+  "total_artist_count": 60
+}

Diff for: listenbrainz/tests/integration/test_stats_api.py

@@ -245,7 +245,9 @@ def assertSitewideStatEqual(self, sent, response, entity, stats_range, count, of
 
         received = orjson.loads(response.data)['payload']
 
-        self.assertEqual(sent['count'], received['count'])
+        singular_entity = entity[:-1] if entity.endswith('s') else entity
+        self.assertEqual(sent[f'total_{singular_entity}_count'], received[f'total_{singular_entity}_count'])
+        self.assertEqual(count, received['count'])
         self.assertEqual(sent['from_ts'], received['from_ts'])
         self.assertEqual(sent['to_ts'], received['to_ts'])
         self.assertEqual(stats_range, received['range'])

Diff for: listenbrainz/tests/unit/test_utils.py

@@ -16,12 +16,9 @@
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 
-import listenbrainz.utils as utils
-import time
 import unittest
 import uuid
 
-from datetime import datetime
 from listenbrainz.webserver import create_app
 from listenbrainz.webserver.views.api_tools import is_valid_uuid
 
@@ -31,12 +28,6 @@ class ListenBrainzUtilsTestCase(unittest.TestCase):
     def setUp(self):
         self.app = create_app(debug=True) # create an app for config value access
 
-    def test_unix_timestamp_to_datetime(self):
-        t = int(time.time())
-        x = utils.unix_timestamp_to_datetime(t)
-        self.assertIsInstance(x, datetime)
-        self.assertEqual(int(x.strftime('%s')), t)
-
     def test_valid_uuid(self):
         self.assertTrue(is_valid_uuid(str(uuid.uuid4())))
         self.assertFalse(is_valid_uuid('hjjkghjk'))

Diff for: listenbrainz/tests/utils.py

@@ -1,9 +1,7 @@
 # coding=utf-8
 
-from datetime import datetime, date
+from datetime import datetime, timezone
 import time
-import sys
-import os
 
 from listenbrainz.listen import Listen
 import uuid
@@ -21,7 +19,7 @@ def generate_data(db_conn, from_date, num_records, user_name):
         item = Listen(
             user_id=user['id'],
             user_name=user_name,
-            timestamp=datetime.utcfromtimestamp(current_date),
+            timestamp=datetime.fromtimestamp(current_date, timezone.utc),
             recording_msid=str(uuid.uuid4()),
             data={
                 'artist_name': 'Test Artist Pls ignore',