From 3504be869d28ab780b431418c5ce01c07d8135cf Mon Sep 17 00:00:00 2001 From: Bryon Roche Date: Thu, 14 Jul 2016 06:29:30 -0700 Subject: [PATCH 01/14] Postgres Support: Core API Implement some core api bits around postgresql support. For this to work, binding formats will have to be changed throughout the rest of the source, hoping this will work for mitigating that. Future commits will do the binding parameter changes, and alter some of the fetches to be consistent with psql's and sqlite's expectations. --- headphones/__init__.py | 296 +++++++++++++++++++++++++++++++++++- headphones/config.py | 2 + headphones/db.py | 189 +++++++++++++++++++++-- headphones/encodingsload.py | 16 ++ requirements-psql.txt | 1 + requirements-pypy.txt | 1 + 6 files changed, 493 insertions(+), 12 deletions(-) create mode 100644 headphones/encodingsload.py create mode 100644 requirements-psql.txt create mode 100644 requirements-pypy.txt diff --git a/headphones/__init__.py b/headphones/__init__.py index 1cb8979b5..97bf72199 100644 --- a/headphones/__init__.py +++ b/headphones/__init__.py @@ -31,6 +31,8 @@ import headphones.config from headphones.softchroot import SoftChroot import headphones.exceptions +import headphones.encodingsload + # (append new extras to the end) POSSIBLE_EXTRAS = [ @@ -125,6 +127,9 @@ def initialize(config_file): if not CONFIG.LOG_DIR: CONFIG.LOG_DIR = os.path.join(DATA_DIR, 'logs') + if CONFIG.USE_POSTGRES: + init_postgres_compat() + if not os.path.exists(CONFIG.LOG_DIR): try: os.makedirs(CONFIG.LOG_DIR) @@ -164,7 +169,10 @@ def initialize(config_file): # Initialize the database logger.info('Checking to see if the database has all tables....') try: - dbcheck() + if CONFIG.USE_POSTGRES: + dbcheck_pgsql() + else: + dbcheck() except Exception as e: logger.error("Can't connect to the database: %s", e) @@ -358,6 +366,277 @@ def sig_handler(signum=None, frame=None): shutdown() +def dbcheck_pgsql(): + # conn = psycopg2.connect(database='headphones', user='headphones', password='headphones', host='127.0.0.1', port='32770') + conn = psycopg2.connect(CONFIG.POSTGRES_DSN) # dbname=headphones user=headphones password=headphones host=127.0.0.1 port=32770 + c = conn.cursor() + c.execute( + 'CREATE TABLE IF NOT EXISTS artists (ArtistID TEXT UNIQUE, ArtistName TEXT, ArtistSortName TEXT, DateAdded TEXT, Status TEXT, IncludeExtras INTEGER, LatestAlbum TEXT, ReleaseDate TEXT, AlbumID TEXT, HaveTracks INTEGER, TotalTracks INTEGER, LastUpdated TEXT, ArtworkURL TEXT, ThumbURL TEXT, Extras TEXT, Type TEXT, MetaCritic TEXT)') + # ReleaseFormat here means CD,Digital,Vinyl, etc. If using the default + # Headphones hybrid release, ReleaseID will equal AlbumID (AlbumID is + # releasegroup id) + c.execute( + 'CREATE TABLE IF NOT EXISTS albums (ArtistID TEXT, ArtistName TEXT, AlbumTitle TEXT, AlbumASIN TEXT, ReleaseDate TEXT, DateAdded TEXT, AlbumID TEXT UNIQUE, Status TEXT, Type TEXT, ArtworkURL TEXT, ThumbURL TEXT, ReleaseID TEXT, ReleaseCountry TEXT, ReleaseFormat TEXT, SearchTerm TEXT, CriticScore TEXT, UserScore TEXT)') + # Format here means mp3, flac, etc. + c.execute( + 'CREATE TABLE IF NOT EXISTS tracks (ArtistID TEXT, ArtistName TEXT, AlbumTitle TEXT, AlbumASIN TEXT, AlbumID TEXT, TrackTitle TEXT, TrackDuration BIGINT, TrackID TEXT, TrackNumber INTEGER, Location TEXT, BitRate INTEGER, CleanName TEXT, Format TEXT, ReleaseID TEXT)') + c.execute( + 'CREATE TABLE IF NOT EXISTS allalbums (ArtistID TEXT, ArtistName TEXT, AlbumTitle TEXT, AlbumASIN TEXT, ReleaseDate TEXT, AlbumID TEXT, Type TEXT, ReleaseID TEXT, ReleaseCountry TEXT, ReleaseFormat TEXT)') + c.execute( + 'CREATE TABLE IF NOT EXISTS alltracks (ArtistID TEXT, ArtistName TEXT, AlbumTitle TEXT, AlbumASIN TEXT, AlbumID TEXT, TrackTitle TEXT, TrackDuration BIGINT, TrackID TEXT, TrackNumber INTEGER, Location TEXT, BitRate INTEGER, CleanName TEXT, Format TEXT, ReleaseID TEXT)') + c.execute( + 'CREATE TABLE IF NOT EXISTS snatched (AlbumID TEXT, Title TEXT, Size INTEGER, URL TEXT, DateAdded TEXT, Status TEXT, FolderName TEXT, Kind TEXT)') + # Matched is a temporary value used to see if there was a match found in + # alltracks + c.execute( + 'CREATE TABLE IF NOT EXISTS have (ArtistName TEXT, AlbumTitle TEXT, TrackNumber TEXT, TrackTitle TEXT, TrackLength TEXT, BitRate TEXT, Genre TEXT, Date TEXT, TrackID TEXT, Location TEXT, CleanName TEXT, Format TEXT, Matched TEXT)') + c.execute( + 'CREATE TABLE IF NOT EXISTS lastfmcloud (ArtistName TEXT, ArtistID TEXT, Count INTEGER)') + c.execute( + 'CREATE TABLE IF NOT EXISTS descriptions (ArtistID TEXT, ReleaseGroupID TEXT, ReleaseID TEXT, Summary TEXT, Content TEXT, LastUpdated TEXT)') + c.execute('CREATE TABLE IF NOT EXISTS blacklist (ArtistID TEXT UNIQUE)') + c.execute('CREATE TABLE IF NOT EXISTS newartists (ArtistName TEXT UNIQUE)') + c.execute( + 'CREATE TABLE IF NOT EXISTS releases (ReleaseID TEXT, ReleaseGroupID TEXT, UNIQUE(ReleaseID, ReleaseGroupID))') + c.execute( + 'CREATE INDEX IF NOT EXISTS tracks_albumid ON tracks(AlbumID ASC)') + c.execute( + 'CREATE INDEX IF NOT EXISTS album_artistid_reldate ON albums(ArtistID ASC, ReleaseDate DESC)') + # Below creates indices to speed up Active Artist updating + # this needs unique because upsert + c.execute( + 'CREATE UNIQUE INDEX IF NOT EXISTS alltracks_relid ON alltracks(ReleaseID ASC, TrackID ASC)') + c.execute( + 'CREATE UNIQUE INDEX IF NOT EXISTS allalbums_relid ON allalbums(ReleaseID ASC)') + c.execute('CREATE UNIQUE INDEX IF NOT EXISTS have_location ON have(Location ASC)') + # Below creates indices to speed up library scanning & matching + c.execute( + 'CREATE INDEX IF NOT EXISTS have_Metadata ON have(ArtistName ASC, AlbumTitle ASC, TrackTitle ASC)') + c.execute( + 'CREATE INDEX IF NOT EXISTS have_CleanName ON have(CleanName ASC)') + c.execute( + 'CREATE INDEX IF NOT EXISTS tracks_Metadata ON tracks(ArtistName ASC, AlbumTitle ASC, TrackTitle ASC)') + c.execute( + 'CREATE INDEX IF NOT EXISTS tracks_CleanName ON tracks(CleanName ASC)') + c.execute( + 'CREATE INDEX IF NOT EXISTS alltracks_Metadata ON alltracks(ArtistName ASC, AlbumTitle ASC, TrackTitle ASC)') + c.execute( + 'CREATE INDEX IF NOT EXISTS alltracks_CleanName ON alltracks(CleanName ASC)') + c.execute( + 'CREATE INDEX IF NOT EXISTS tracks_Location ON tracks(Location ASC)') + c.execute( + 'CREATE INDEX IF NOT EXISTS alltracks_Location ON alltracks(Location ASC)') + c.execute( + 'CREATE INDEX IF NOT EXISTS artist_ciStatus ON artists(lower(Status) ASC)') + c.execute( + 'CREATE INDEX IF NOT EXISTS artist_ciReleaseDate ON artists(lower(ReleaseDate) ASC)') + + # Upsert data integrity tables + c.execute( + 'CREATE UNIQUE INDEX IF NOT EXISTS tracks_uniqid ON tracks(TrackID ASC, AlbumID ASC)') + c.execute( + 'CREATE UNIQUE INDEX IF NOT EXISTS descriptions_uniqid ON descriptions(ArtistID ASC, ReleaseGroupID ASC)') + + try: + c.execute('SELECT IncludeExtras from artists') + except psycopg2.Error: + c.execute( + 'ALTER TABLE artists ADD COLUMN IncludeExtras INTEGER DEFAULT 0') + + try: + c.execute('SELECT LatestAlbum from artists') + except psycopg2.Error: + c.execute('ALTER TABLE artists ADD COLUMN LatestAlbum TEXT') + + try: + c.execute('SELECT ReleaseDate from artists') + except psycopg2.Error: + c.execute('ALTER TABLE artists ADD COLUMN ReleaseDate TEXT') + + try: + c.execute('SELECT AlbumID from artists') + except psycopg2.Error: + c.execute('ALTER TABLE artists ADD COLUMN AlbumID TEXT') + + try: + c.execute('SELECT HaveTracks from artists') + except psycopg2.Error: + c.execute( + 'ALTER TABLE artists ADD COLUMN HaveTracks INTEGER DEFAULT 0') + + try: + c.execute('SELECT TotalTracks from artists') + except psycopg2.Error: + c.execute( + 'ALTER TABLE artists ADD COLUMN TotalTracks INTEGER DEFAULT 0') + + try: + c.execute('SELECT Type from albums') + except psycopg2.Error: + c.execute('ALTER TABLE albums ADD COLUMN Type TEXT DEFAULT "Album"') + + try: + c.execute('SELECT TrackNumber from tracks') + except psycopg2.Error: + c.execute('ALTER TABLE tracks ADD COLUMN TrackNumber INTEGER') + + try: + c.execute('SELECT FolderName from snatched') + except psycopg2.Error: + c.execute('ALTER TABLE snatched ADD COLUMN FolderName TEXT') + + try: + c.execute('SELECT Location from tracks') + except psycopg2.Error: + c.execute('ALTER TABLE tracks ADD COLUMN Location TEXT') + + try: + c.execute('SELECT Location from have') + except psycopg2.Error: + c.execute('ALTER TABLE have ADD COLUMN Location TEXT') + + try: + c.execute('SELECT BitRate from tracks') + except psycopg2.Error: + c.execute('ALTER TABLE tracks ADD COLUMN BitRate INTEGER') + + try: + c.execute('SELECT CleanName from tracks') + except psycopg2.Error: + c.execute('ALTER TABLE tracks ADD COLUMN CleanName TEXT') + + try: + c.execute('SELECT CleanName from have') + except psycopg2.Error: + c.execute('ALTER TABLE have ADD COLUMN CleanName TEXT') + + # Add the Format column + try: + c.execute('SELECT Format from have') + except psycopg2.Error: + c.execute('ALTER TABLE have ADD COLUMN Format TEXT DEFAULT NULL') + + try: + c.execute('SELECT Format from tracks') + except psycopg2.Error: + c.execute('ALTER TABLE tracks ADD COLUMN Format TEXT DEFAULT NULL') + + try: + c.execute('SELECT LastUpdated from artists') + except psycopg2.Error: + c.execute( + 'ALTER TABLE artists ADD COLUMN LastUpdated TEXT DEFAULT NULL') + + try: + c.execute('SELECT ArtworkURL from artists') + except psycopg2.Error: + c.execute( + 'ALTER TABLE artists ADD COLUMN ArtworkURL TEXT DEFAULT NULL') + + try: + c.execute('SELECT ArtworkURL from albums') + except psycopg2.Error: + c.execute('ALTER TABLE albums ADD COLUMN ArtworkURL TEXT DEFAULT NULL') + + try: + c.execute('SELECT ThumbURL from artists') + except psycopg2.Error: + c.execute('ALTER TABLE artists ADD COLUMN ThumbURL TEXT DEFAULT NULL') + + try: + c.execute('SELECT ThumbURL from albums') + except psycopg2.Error: + c.execute('ALTER TABLE albums ADD COLUMN ThumbURL TEXT DEFAULT NULL') + + try: + c.execute('SELECT ArtistID from descriptions') + except psycopg2.Error: + c.execute( + 'ALTER TABLE descriptions ADD COLUMN ArtistID TEXT DEFAULT NULL') + + try: + c.execute('SELECT LastUpdated from descriptions') + except psycopg2.Error: + c.execute( + 'ALTER TABLE descriptions ADD COLUMN LastUpdated TEXT DEFAULT NULL') + + try: + c.execute('SELECT ReleaseID from albums') + except psycopg2.Error: + c.execute('ALTER TABLE albums ADD COLUMN ReleaseID TEXT DEFAULT NULL') + + try: + c.execute('SELECT ReleaseFormat from albums') + except psycopg2.Error: + c.execute( + 'ALTER TABLE albums ADD COLUMN ReleaseFormat TEXT DEFAULT NULL') + + try: + c.execute('SELECT ReleaseCountry from albums') + except psycopg2.Error: + c.execute( + 'ALTER TABLE albums ADD COLUMN ReleaseCountry TEXT DEFAULT NULL') + + try: + c.execute('SELECT ReleaseID from tracks') + except psycopg2.Error: + c.execute('ALTER TABLE tracks ADD COLUMN ReleaseID TEXT DEFAULT NULL') + + try: + c.execute('SELECT Matched from have') + except psycopg2.Error: + c.execute('ALTER TABLE have ADD COLUMN Matched TEXT DEFAULT NULL') + + try: + c.execute('SELECT Extras from artists') + except psycopg2.Error: + c.execute('ALTER TABLE artists ADD COLUMN Extras TEXT DEFAULT NULL') + # Need to update some stuff when people are upgrading and have 'include + # extras' set globally/for an artist + if CONFIG.INCLUDE_EXTRAS: + CONFIG.EXTRAS = "1,2,3,4,5,6,7,8" + logger.info("Copying over current artist IncludeExtras information") + artists = c.execute( + 'SELECT ArtistID, IncludeExtras from artists').fetchall() + for artist in artists: + if artist['IncludeExtras']: + c.execute( + 'UPDATE artists SET Extras=%s WHERE ArtistID=%s', ("1,2,3,4,5,6,7,8", artist['ArtistID'])) + + try: + c.execute('SELECT Kind from snatched') + except psycopg2.Error: + c.execute('ALTER TABLE snatched ADD COLUMN Kind TEXT DEFAULT NULL') + + try: + c.execute('SELECT SearchTerm from albums') + except psycopg2.Error: + c.execute('ALTER TABLE albums ADD COLUMN SearchTerm TEXT DEFAULT NULL') + + try: + c.execute('SELECT CriticScore from albums') + except psycopg2.Error: + c.execute('ALTER TABLE albums ADD COLUMN CriticScore TEXT DEFAULT NULL') + + try: + c.execute('SELECT UserScore from albums') + except psycopg2.Error: + c.execute('ALTER TABLE albums ADD COLUMN UserScore TEXT DEFAULT NULL') + + try: + c.execute('SELECT Type from artists') + except psycopg2.Error: + c.execute('ALTER TABLE artists ADD COLUMN Type TEXT DEFAULT NULL') + + try: + c.execute('SELECT MetaCritic from artists') + except psycopg2.Error: + c.execute('ALTER TABLE artists ADD COLUMN MetaCritic TEXT DEFAULT NULL') + + c.close() + conn.commit() + + def dbcheck(): conn = sqlite3.connect(DB_FILE) c = conn.cursor() @@ -370,11 +649,11 @@ def dbcheck(): 'CREATE TABLE IF NOT EXISTS albums (ArtistID TEXT, ArtistName TEXT, AlbumTitle TEXT, AlbumASIN TEXT, ReleaseDate TEXT, DateAdded TEXT, AlbumID TEXT UNIQUE, Status TEXT, Type TEXT, ArtworkURL TEXT, ThumbURL TEXT, ReleaseID TEXT, ReleaseCountry TEXT, ReleaseFormat TEXT, SearchTerm TEXT, CriticScore TEXT, UserScore TEXT)') # Format here means mp3, flac, etc. c.execute( - 'CREATE TABLE IF NOT EXISTS tracks (ArtistID TEXT, ArtistName TEXT, AlbumTitle TEXT, AlbumASIN TEXT, AlbumID TEXT, TrackTitle TEXT, TrackDuration, TrackID TEXT, TrackNumber INTEGER, Location TEXT, BitRate INTEGER, CleanName TEXT, Format TEXT, ReleaseID TEXT)') + 'CREATE TABLE IF NOT EXISTS tracks (ArtistID TEXT, ArtistName TEXT, AlbumTitle TEXT, AlbumASIN TEXT, AlbumID TEXT, TrackTitle TEXT, TrackDuration TEXT, TrackID TEXT, TrackNumber INTEGER, Location TEXT, BitRate INTEGER, CleanName TEXT, Format TEXT, ReleaseID TEXT)') c.execute( 'CREATE TABLE IF NOT EXISTS allalbums (ArtistID TEXT, ArtistName TEXT, AlbumTitle TEXT, AlbumASIN TEXT, ReleaseDate TEXT, AlbumID TEXT, Type TEXT, ReleaseID TEXT, ReleaseCountry TEXT, ReleaseFormat TEXT)') c.execute( - 'CREATE TABLE IF NOT EXISTS alltracks (ArtistID TEXT, ArtistName TEXT, AlbumTitle TEXT, AlbumASIN TEXT, AlbumID TEXT, TrackTitle TEXT, TrackDuration, TrackID TEXT, TrackNumber INTEGER, Location TEXT, BitRate INTEGER, CleanName TEXT, Format TEXT, ReleaseID TEXT)') + 'CREATE TABLE IF NOT EXISTS alltracks (ArtistID TEXT, ArtistName TEXT, AlbumTitle TEXT, AlbumASIN TEXT, AlbumID TEXT, TrackTitle TEXT, TrackDuration TEXT, TrackID TEXT, TrackNumber INTEGER, Location TEXT, BitRate INTEGER, CleanName TEXT, Format TEXT, ReleaseID TEXT)') c.execute( 'CREATE TABLE IF NOT EXISTS snatched (AlbumID TEXT, Title TEXT, Size INTEGER, URL TEXT, DateAdded TEXT, Status TEXT, FolderName TEXT, Kind TEXT)') # Matched is a temporary value used to see if there was a match found in @@ -647,3 +926,14 @@ def shutdown(restart=False, update=False): subprocess.Popen(popen_list, cwd=os.getcwd()) os._exit(0) + + +def init_postgres_compat(): + if 'PyPy' in sys.subversion: + import psycopg2cffi.compat + psycopg2cffi.compat.register() + + import psycopg2 # pylint: disable=import-error + import psycopg2extensions # pylint: disable=import-error + psycopg2.extensions.register_type(psycopg2.extensions.UNICODE) + psycopg2.extensions.register_type(psycopg2.extensions.UNICODEARRAY) diff --git a/headphones/config.py b/headphones/config.py index 5b8190074..f9583c943 100644 --- a/headphones/config.py +++ b/headphones/config.py @@ -275,6 +275,8 @@ def __repr__(self): 'TWITTER_PREFIX': (str, 'Twitter', 'Headphones'), 'TWITTER_USERNAME': (str, 'Twitter', ''), 'UPDATE_DB_INTERVAL': (int, 'General', 24), + 'USE_POSTGRES': (bool_int, 'Advanced', 0), + 'POSTGRES_DSN': (str, 'Advanced', ''), 'USENET_RETENTION': (int, 'General', '1500'), 'UTORRENT_HOST': (str, 'uTorrent', ''), 'UTORRENT_LABEL': (str, 'uTorrent', ''), diff --git a/headphones/db.py b/headphones/db.py index 8d864abad..12a1afb78 100644 --- a/headphones/db.py +++ b/headphones/db.py @@ -21,8 +21,13 @@ import sqlite3 +import psycopg2 # pylint: disable=import-error +import psycopg2.extras # pylint: disable=import-error + import os import headphones +import threading +from itertools import chain from headphones import logger @@ -38,18 +43,35 @@ def getCacheSize(): return int(headphones.CONFIG.CACHE_SIZEMB) -class DBConnection: +def convert_pgsql_bindparms(query): + # FIXME make a real % parser or something here, hopefully this gets transitioned to an ORM sooner than that + return query.replace('%s', '?') + + +class DBConnection_sqlite: + + dbcache = threading.local() + def __init__(self, filename="headphones.db"): self.filename = filename - self.connection = sqlite3.connect(dbFilename(filename), timeout=20) - # don't wait for the disk to finish writing - self.connection.execute("PRAGMA synchronous = OFF") - # journal disabled since we never do rollbacks - self.connection.execute("PRAGMA journal_mode = %s" % headphones.CONFIG.JOURNAL_MODE) - # 64mb of cache memory,probably need to make it user configurable - self.connection.execute("PRAGMA cache_size=-%s" % (getCacheSize() * 1024)) - self.connection.row_factory = sqlite3.Row + if not hasattr(self.dbcache, 'connection'): + self.dbcache.connection = sqlite3.connect(dbFilename(filename), timeout=20) + # don't wait for the disk to finish writing + self.dbcache.connection.execute("PRAGMA synchronous = OFF") + # journal disabled since we never do rollbacks + self.dbcache.connection.execute("PRAGMA journal_mode = %s" % headphones.CONFIG.JOURNAL_MODE) + # 64mb of cache memory,probably need to make it user configurable + self.dbcache.connection.execute("PRAGMA cache_size=-%s" % (getCacheSize() * 1024)) + + self.dbcache.connection.row_factory = sqlite3.Row + self.connection = self.dbcache.connection + + def commit(self): + self.connection.commit() + + def rollback(self): + self.connection.rollback() def action(self, query, args=None): @@ -63,6 +85,7 @@ def action(self, query, args=None): if args is None: sqlResult = c.execute(query) else: + query = convert_pgsql_bindparms(query) sqlResult = c.execute(query, args) except sqlite3.OperationalError, e: @@ -109,3 +132,151 @@ def genParams(myDict): self.action(insert_query, valueDict.values() + keyDict.values()) except sqlite3.IntegrityError: logger.info('Queries failed: %s and %s', update_query, insert_query) + + +class CIDictCursor(psycopg2.extras.DictCursorBase): + """A cursor that uses a case insensitive fetching dict as the base type for rows. + """ + def __init__(self, *args, **kwargs): + kwargs['row_factory'] = CIDictRow + super(CIDictCursor, self).__init__(*args, **kwargs) + self._prefetch = 0 + + def execute(self, query, vars=None): + self.column_mapping = [] + self._query_executed = 1 + return super(CIDictCursor, self).execute(query, vars) + + def callproc(self, procname, vars=None): + self.column_mapping = [] + self._query_executed = 1 + return super(CIDictCursor, self).callproc(procname, vars) + + def _build_index(self): + if self._query_executed == 1 and self.description: + for i in range(len(self.description)): + self.column_mapping.append(self.description[i][0]) + self._query_executed = 0 + + +class CIDictRow(dict): + """A `!dict` subclass representing a data record.""" + + __slots__ = ('_column_mapping') + + def __init__(self, cursor): + dict.__init__(self) + # Required for named cursors + if cursor.description and not cursor.column_mapping: + cursor._build_index() + + self._column_mapping = cursor.column_mapping + + def __setitem__(self, name, value): + if type(name) == int: + name = self._column_mapping[name] + return dict.__setitem__(self, name, value) + + def __getitem__(self, key): + return dict.get(self, key, dict.get(self, unicode(key).lower())) + + def __getstate__(self): + return (self.copy(), self._column_mapping[:]) + + def __setstate__(self, data): + self.update(data[0]) + self._column_mapping = data[1] + + +class DBConnection_psql: + dbcache = threading.local() + + def __init__(self, filename="headphones.db"): + + self.filename = filename + if not hasattr(self.dbcache, 'connection'): + self.dbcache.connection = psycopg2.connect(database='headphones', user='headphones', password='headphones', host='127.0.0.1', port='32770') + self.connection = self.dbcache.connection + + def commit(self): + self.connection.commit() + + def rollback(self): + self.connection.rollback() + + def action(self, query, args=None): + + if query is None: + return + + sqlResult = None + + try: + with self.connection as c: + cursor = c.cursor(cursor_factory=CIDictCursor) + if args is None: + sqlResult = cursor.execute(query) + else: + sqlResult = cursor.execute(query, args) + + except psycopg2.Warning as e: + logger.warn('Database Error: %s', e) + + except psycopg2.OperationalError as e: + logger.error('Database error: %s', e) + c.rollback() + raise + + except psycopg2.DatabaseError, e: + logger.error('Fatal Error executing %s :: %s', query, e) + c.rollback() + raise + + return cursor + + def select(self, query, args=None): + + sqlResults = self.action(query, args).fetchall() + + if sqlResults is None or sqlResults == [None]: + return [] + + self.connection.commit() + return sqlResults + + def upsert(self, tableName, valueDict, keyDict): + + def genParams(myDict): + return (x + " = %s" for x in myDict.iterkeys()) + + def genUpsertParams(myDict): + return ('.'.join((tableName, x)) + " = %s" for x in myDict.iterkeys()) + + insert_query = ( + "INSERT INTO " + tableName + " (" + ", ".join( + chain(valueDict.iterkeys(), keyDict.iterkeys())) + ")" + + " VALUES (" + ", ".join(["%s"] * (len(valueDict) + len(keyDict))) + + ") ON CONFLICT ( " + ", ".join(keyDict.iterkeys()) + " ) DO UPDATE SET " + + ", ".join(genParams(valueDict)) + " WHERE " + " AND ".join(genUpsertParams(keyDict)) + ) + + vals = chain( + valueDict.itervalues(), + keyDict.itervalues(), + valueDict.itervalues(), + keyDict.itervalues()) + + ret = None + try: + ret = self.action(insert_query, [v for v in vals]) + except psycopg2.IntegrityError: + logger.info('Queries failed: %s and %s', 'was update_query', insert_query) + self.connection.rollback() + return + except psycopg2.ProgrammingError: + logger.exception('Bad query %s', insert_query) + raise + + self.connection.commit() + +DBConnection = DBConnection_psql diff --git a/headphones/encodingsload.py b/headphones/encodingsload.py new file mode 100644 index 000000000..fa119a6aa --- /dev/null +++ b/headphones/encodingsload.py @@ -0,0 +1,16 @@ +"in which we pre-import all the encodings to make sure we don't hit an import lock" + +import encodings.ascii +import encodings.base64_codec +import encodings.charmap +import encodings.idna +import encodings.latin_1 +import encodings.raw_unicode_escape +import encodings.undefined +import encodings.unicode_escape +import encodings.utf_16 +import encodings.utf_32 +import encodings.utf_8 +import encodings.utf_7 + +assert encodings # yay flakes diff --git a/requirements-psql.txt b/requirements-psql.txt new file mode 100644 index 000000000..658130bb2 --- /dev/null +++ b/requirements-psql.txt @@ -0,0 +1 @@ +psycopg2 diff --git a/requirements-pypy.txt b/requirements-pypy.txt new file mode 100644 index 000000000..1eb9917a9 --- /dev/null +++ b/requirements-pypy.txt @@ -0,0 +1 @@ +psycopg2cffi From 56e771976472f7fcf4333e10684cb204d4362985 Mon Sep 17 00:00:00 2001 From: Bryon Roche Date: Thu, 14 Jul 2016 06:33:43 -0700 Subject: [PATCH 02/14] Postgres support part 2: Python usage Convert all the sql uses in the python codebase to use syntax compatible with both cursors, and to do some more explicit transaction management. The transaction management is messy as hell, but at least its a start, and not necessarily as lock/performance intensive as full-on autocommit. --- headphones/albumart.py | 2 +- headphones/albumswitcher.py | 18 +-- headphones/api.py | 33 ++--- headphones/cache.py | 18 +-- headphones/importer.py | 128 +++++++++--------- headphones/lastfm.py | 4 +- headphones/librarysync.py | 73 ++++++----- headphones/mb.py | 34 ++--- headphones/metacritic.py | 2 +- headphones/postprocessor.py | 56 ++++---- headphones/searcher.py | 25 ++-- headphones/torrentfinished.py | 7 +- headphones/updater.py | 4 +- headphones/webserve.py | 238 ++++++++++++++++++++-------------- 14 files changed, 355 insertions(+), 287 deletions(-) diff --git a/headphones/albumart.py b/headphones/albumart.py index 8bbbd425a..444dfe773 100644 --- a/headphones/albumart.py +++ b/headphones/albumart.py @@ -19,7 +19,7 @@ def getAlbumArt(albumid): myDB = db.DBConnection() asin = myDB.action( - 'SELECT AlbumASIN from albums WHERE AlbumID=?', [albumid]).fetchone()[0] + 'SELECT AlbumASIN from albums WHERE AlbumID=%s', [albumid]).fetchone()['AlbumASIN'] if asin: return 'http://ec1.images-amazon.com/images/P/%s.01.LZZZZZZZ.jpg' % asin diff --git a/headphones/albumswitcher.py b/headphones/albumswitcher.py index 74f1077ff..bc8d51ee3 100644 --- a/headphones/albumswitcher.py +++ b/headphones/albumswitcher.py @@ -25,12 +25,12 @@ def switch(AlbumID, ReleaseID): logger.debug('Switching allalbums and alltracks') myDB = db.DBConnection() oldalbumdata = myDB.action( - 'SELECT * from albums WHERE AlbumID=?', [AlbumID]).fetchone() + 'SELECT * from albums WHERE AlbumID=%s', [AlbumID]).fetchone() newalbumdata = myDB.action( - 'SELECT * from allalbums WHERE ReleaseID=?', [ReleaseID]).fetchone() + 'SELECT * from allalbums WHERE ReleaseID=%s', [ReleaseID]).fetchone() newtrackdata = myDB.action( - 'SELECT * from alltracks WHERE ReleaseID=?', [ReleaseID]).fetchall() - myDB.action('DELETE from tracks WHERE AlbumID=?', [AlbumID]) + 'SELECT * from alltracks WHERE ReleaseID=%s', [ReleaseID]).fetchall() + myDB.action('DELETE from tracks WHERE AlbumID=%s', [AlbumID]) controlValueDict = {"AlbumID": AlbumID} @@ -76,19 +76,19 @@ def switch(AlbumID, ReleaseID): # configurable) of the album total_track_count = len(newtrackdata) have_track_count = len(myDB.select( - 'SELECT * from tracks WHERE AlbumID=? AND Location IS NOT NULL', [AlbumID])) + 'SELECT * from tracks WHERE AlbumID=%s AND Location IS NOT NULL', [AlbumID])) if oldalbumdata['Status'] == 'Skipped' and ((have_track_count / float(total_track_count)) >= ( headphones.CONFIG.ALBUM_COMPLETION_PCT / 100.0)): myDB.action( - 'UPDATE albums SET Status=? WHERE AlbumID=?', ['Downloaded', AlbumID]) + 'UPDATE albums SET Status=%s WHERE AlbumID=%s', ['Downloaded', AlbumID]) # Update have track counts on index totaltracks = len(myDB.select( - 'SELECT TrackTitle from tracks WHERE ArtistID=? AND AlbumID IN (SELECT AlbumID FROM albums WHERE Status != "Ignored")', - [newalbumdata['ArtistID']])) + 'SELECT TrackTitle from tracks WHERE ArtistID=%s AND AlbumID IN (SELECT AlbumID FROM albums WHERE Status != %s)', + [newalbumdata['ArtistID'], 'Ignored'])) havetracks = len(myDB.select( - 'SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', + 'SELECT TrackTitle from tracks WHERE ArtistID=%s AND Location IS NOT NULL', [newalbumdata['ArtistID']])) controlValueDict = {"ArtistID": newalbumdata['ArtistID']} diff --git a/headphones/api.py b/headphones/api.py index a8b97c02b..985e518f1 100644 --- a/headphones/api.py +++ b/headphones/api.py @@ -98,10 +98,10 @@ def fetchData(self): else: return self.data - def _dic_from_query(self, query): + def _dic_from_query(self, query, args=None): myDB = db.DBConnection() - rows = myDB.select(query) + rows = myDB.select(query, args) rows_as_dic = [] @@ -114,7 +114,7 @@ def _dic_from_query(self, query): def _getIndex(self, **kwargs): self.data = self._dic_from_query( - 'SELECT * from artists order by ArtistSortName COLLATE NOCASE') + 'SELECT * from artists order by lower(ArtistSortName)') return def _getArtist(self, **kwargs): @@ -126,11 +126,11 @@ def _getArtist(self, **kwargs): self.id = kwargs['id'] artist = self._dic_from_query( - 'SELECT * from artists WHERE ArtistID="' + self.id + '"') + 'SELECT * from artists WHERE ArtistID=%s', [self.id]) albums = self._dic_from_query( - 'SELECT * from albums WHERE ArtistID="' + self.id + '" order by ReleaseDate DESC') + 'SELECT * from albums WHERE ArtistID=%s order by ReleaseDate DESC', [self.id]) description = self._dic_from_query( - 'SELECT * from descriptions WHERE ArtistID="' + self.id + '"') + 'SELECT * from descriptions WHERE ArtistID=%s', [self.id]) self.data = { 'artist': artist, 'albums': albums, 'description': description} @@ -145,11 +145,11 @@ def _getAlbum(self, **kwargs): self.id = kwargs['id'] album = self._dic_from_query( - 'SELECT * from albums WHERE AlbumID="' + self.id + '"') + 'SELECT * from albums WHERE AlbumID=%s', [self.id]) tracks = self._dic_from_query( - 'SELECT * from tracks WHERE AlbumID="' + self.id + '"') + 'SELECT * from tracks WHERE AlbumID=%s', [self.id]) description = self._dic_from_query( - 'SELECT * from descriptions WHERE ReleaseGroupID="' + self.id + '"') + 'SELECT * from descriptions WHERE ReleaseGroupID=%s', [self.id]) self.data = { 'album': album, 'tracks': tracks, 'description': description} @@ -157,7 +157,7 @@ def _getAlbum(self, **kwargs): def _getHistory(self, **kwargs): self.data = self._dic_from_query( - 'SELECT * from snatched WHERE status NOT LIKE "Seed%" order by DateAdded DESC') + 'SELECT * from snatched WHERE status NOT LIKE %s order by DateAdded DESC', ['Seed%']) return def _getUpcoming(self, **kwargs): @@ -167,12 +167,12 @@ def _getUpcoming(self, **kwargs): def _getWanted(self, **kwargs): self.data = self._dic_from_query( - "SELECT * from albums WHERE Status='Wanted'") + "SELECT * from albums WHERE Status=%s", ['Wanted']) return def _getSnatched(self, **kwargs): self.data = self._dic_from_query( - "SELECT * from albums WHERE Status='Snatched'") + "SELECT * from albums WHERE Status=%s", ['Snatched']) return def _getSimilar(self, **kwargs): @@ -232,9 +232,10 @@ def _delArtist(self, **kwargs): self.id = kwargs['id'] myDB = db.DBConnection() - myDB.action('DELETE from artists WHERE ArtistID="' + self.id + '"') - myDB.action('DELETE from albums WHERE ArtistID="' + self.id + '"') - myDB.action('DELETE from tracks WHERE ArtistID="' + self.id + '"') + myDB.action('DELETE from artists WHERE ArtistID=%s', [self.id]) + myDB.action('DELETE from albums WHERE ArtistID=%s', [self.id]) + myDB.action('DELETE from tracks WHERE ArtistID=%s', [self.id]) + myDB.commit() def _pauseArtist(self, **kwargs): if 'id' not in kwargs: @@ -490,5 +491,5 @@ def _download_specific_release(self, **kwargs): if data and bestqual: myDB = db.DBConnection() album = myDB.action( - 'SELECT * from albums WHERE AlbumID=?', [id]).fetchone() + 'SELECT * from albums WHERE AlbumID=%s', [id]).fetchone() searcher.send_to_downloader(data, bestqual, album) diff --git a/headphones/cache.py b/headphones/cache.py index 202b7802d..4c2b95cdb 100644 --- a/headphones/cache.py +++ b/headphones/cache.py @@ -183,13 +183,13 @@ def get_info_from_cache(self, ArtistID=None, AlbumID=None): self.id = ArtistID self.id_type = 'artist' db_info = myDB.action( - 'SELECT Summary, Content, LastUpdated FROM descriptions WHERE ArtistID=?', + 'SELECT Summary, Content, LastUpdated FROM descriptions WHERE ArtistID=%s', [self.id]).fetchone() else: self.id = AlbumID self.id_type = 'album' db_info = myDB.action( - 'SELECT Summary, Content, LastUpdated FROM descriptions WHERE ReleaseGroupID=?', + 'SELECT Summary, Content, LastUpdated FROM descriptions WHERE ReleaseGroupID=%s', [self.id]).fetchone() if not db_info or not db_info['LastUpdated'] or not self._is_current( @@ -315,7 +315,7 @@ def _update_cache(self): else: dbalbum = myDB.action( - 'SELECT ArtistName, AlbumTitle, ReleaseID FROM albums WHERE AlbumID=?', + 'SELECT ArtistName, AlbumTitle, ReleaseID FROM albums WHERE AlbumID=%s', [self.id]).fetchone() if dbalbum['ReleaseID'] != self.id: data = lastfm.request_lastfm("album.getinfo", mbid=dbalbum['ReleaseID'], @@ -355,9 +355,9 @@ def _update_cache(self): # Save the content & summary to the database no matter what if we've # opened up the url if self.id_type == 'artist': - controlValueDict = {"ArtistID": self.id} + controlValueDict = {"ArtistID": self.id, "ReleaseGroupID": None} else: - controlValueDict = {"ReleaseGroupID": self.id} + controlValueDict = {"ReleaseGroupID": self.id, "ArtistID": None} newValueDict = {"Summary": self.info_summary, "Content": self.info_content, @@ -368,17 +368,17 @@ def _update_cache(self): # Save the image URL to the database if image_url: if self.id_type == 'artist': - myDB.action('UPDATE artists SET ArtworkURL=? WHERE ArtistID=?', + myDB.action('UPDATE artists SET ArtworkURL=%s WHERE ArtistID=%s', [image_url, self.id]) else: - myDB.action('UPDATE albums SET ArtworkURL=? WHERE AlbumID=?', [image_url, self.id]) + myDB.action('UPDATE albums SET ArtworkURL=%s WHERE AlbumID=%s', [image_url, self.id]) # Save the thumb URL to the database if thumb_url: if self.id_type == 'artist': - myDB.action('UPDATE artists SET ThumbURL=? WHERE ArtistID=?', [thumb_url, self.id]) + myDB.action('UPDATE artists SET ThumbURL=%s WHERE ArtistID=%s', [thumb_url, self.id]) else: - myDB.action('UPDATE albums SET ThumbURL=? WHERE AlbumID=?', [thumb_url, self.id]) + myDB.action('UPDATE albums SET ThumbURL=%s WHERE AlbumID=%s', [thumb_url, self.id]) # Should we grab the artwork here if we're just grabbing thumbs or # info? Probably not since the files can be quite big diff --git a/headphones/importer.py b/headphones/importer.py index 3b92ca11a..77b2b2c47 100644 --- a/headphones/importer.py +++ b/headphones/importer.py @@ -34,7 +34,7 @@ def is_exists(artistid): myDB = db.DBConnection() # See if the artist is already in the database - artistlist = myDB.select('SELECT ArtistID, ArtistName from artists WHERE ArtistID=?', + artistlist = myDB.select('SELECT ArtistID, ArtistName from artists WHERE ArtistID=%s', [artistid]) if any(artistid in x for x in artistlist): @@ -77,8 +77,10 @@ def artistlist_to_mbids(artistlist, forced=False): myDB = db.DBConnection() if not forced: - bl_artist = myDB.action('SELECT * FROM blacklist WHERE ArtistID=?', + bl_artist = myDB.action('SELECT ArtistID FROM blacklist WHERE ArtistID=%s', [artistid]).fetchone() + if bl_artist: + bl_artist = bl_artist['ArtistID'] if bl_artist or artistid in blacklisted_special_artists: logger.info("Artist ID for '%s' is either blacklisted or Various Artists. To add artist, you must " "do it manually (Artist ID: %s)" % (artist, artistid)) @@ -91,13 +93,14 @@ def artistlist_to_mbids(artistlist, forced=False): # Just update the tracks if it does else: havetracks = len( - myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=?', [artistid])) + len( + myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=%s', [artistid])) + len( myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist])) - myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, artistid]) + myDB.action('UPDATE artists SET HaveTracks=%s WHERE ArtistID=%s', [havetracks, artistid]) # Delete it from the New Artists if the request came from there if forced: - myDB.action('DELETE from newartists WHERE ArtistName=?', [artist]) + myDB.action('DELETE from newartists WHERE ArtistName=%s', [artist]) + myDB.commit() # Update the similar artist tag cloud: logger.info('Updating artist information from Last.fm') @@ -130,14 +133,14 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"): myDB = db.DBConnection() # Delete from blacklist if it's on there - myDB.action('DELETE from blacklist WHERE ArtistID=?', [artistid]) + myDB.action('DELETE from blacklist WHERE ArtistID=%s', [artistid]) # We need the current minimal info in the database instantly # so we don't throw a 500 error when we redirect to the artistPage controlValueDict = {"ArtistID": artistid} # Don't replace a known artist name with an "Artist ID" placeholder - dbartist = myDB.action('SELECT * FROM artists WHERE ArtistID=?', [artistid]).fetchone() + dbartist = myDB.action('SELECT * FROM artists WHERE ArtistID=%s', [artistid]).fetchone() # Only modify the Include Extras stuff if it's a new artist. We need it early so we know what to fetch if not dbartist: @@ -161,10 +164,10 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"): if artist and artist.get('artist_name') in blacklisted_special_artist_names: logger.warn('Cannot import blocked special purpose artist: %s' % artist.get('artist_name')) - myDB.action('DELETE from artists WHERE ArtistID=?', [artistid]) + myDB.action('DELETE from artists WHERE ArtistID=%s', [artistid]) # in case it's already in the db - myDB.action('DELETE from albums WHERE ArtistID=?', [artistid]) - myDB.action('DELETE from tracks WHERE ArtistID=?', [artistid]) + myDB.action('DELETE from albums WHERE ArtistID=%s', [artistid]) + myDB.action('DELETE from tracks WHERE ArtistID=%s', [artistid]) return if not artist: @@ -194,7 +197,7 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"): # See if we need to grab extras. Artist specific extras take precedence # over global option. Global options are set when adding a new artist try: - db_artist = myDB.action('SELECT IncludeExtras, Extras from artists WHERE ArtistID=?', + db_artist = myDB.action('SELECT IncludeExtras, Extras from artists WHERE ArtistID=%s', [artistid]).fetchone() includeExtras = db_artist['IncludeExtras'] except IndexError: @@ -211,19 +214,19 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"): group_list.append(groups['id']) if not extrasonly: remove_missing_groups_from_albums = myDB.select( - "SELECT AlbumID FROM albums WHERE ArtistID=?", [artistid]) + "SELECT AlbumID FROM albums WHERE ArtistID=%s", [artistid]) else: remove_missing_groups_from_albums = myDB.select( - 'SELECT AlbumID FROM albums WHERE ArtistID=? AND Status="Skipped" AND Type!="Album"', - [artistid]) + 'SELECT AlbumID FROM albums WHERE ArtistID=%s AND Status=%s AND Type!=%s', + [artistid, 'Skipped', 'Album']) for items in remove_missing_groups_from_albums: if items['AlbumID'] not in group_list: # Remove all from albums/tracks that aren't in release groups - myDB.action("DELETE FROM albums WHERE AlbumID=?", [items['AlbumID']]) - myDB.action("DELETE FROM allalbums WHERE AlbumID=?", [items['AlbumID']]) - myDB.action("DELETE FROM tracks WHERE AlbumID=?", [items['AlbumID']]) - myDB.action("DELETE FROM alltracks WHERE AlbumID=?", [items['AlbumID']]) - myDB.action('DELETE from releases WHERE ReleaseGroupID=?', [items['AlbumID']]) + myDB.action("DELETE FROM albums WHERE AlbumID=%s", [items['AlbumID']]) + myDB.action("DELETE FROM allalbums WHERE AlbumID=%s", [items['AlbumID']]) + myDB.action("DELETE FROM tracks WHERE AlbumID=%s", [items['AlbumID']]) + myDB.action("DELETE FROM alltracks WHERE AlbumID=%s", [items['AlbumID']]) + myDB.action('DELETE from releases WHERE ReleaseGroupID=%s', [items['AlbumID']]) logger.info("[%s] Removing all references to release group %s to reflect MusicBrainz refresh" % ( artist['artist_name'], items['AlbumID'])) if not extrasonly: @@ -245,7 +248,7 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"): # Make a user configurable variable to skip update of albums with release dates older than this date (in days) pause_delta = headphones.CONFIG.MB_IGNORE_AGE - rg_exists = myDB.action("SELECT * from albums WHERE AlbumID=?", [rg['id']]).fetchone() + rg_exists = myDB.action("SELECT * from albums WHERE AlbumID=%s", [rg['id']]).fetchone() if not forcefull: new_release_group = False @@ -296,16 +299,16 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"): if new_releases != 0: # Dump existing hybrid release since we're repackaging/replacing it - myDB.action("DELETE from albums WHERE ReleaseID=?", [rg['id']]) - myDB.action("DELETE from allalbums WHERE ReleaseID=?", [rg['id']]) - myDB.action("DELETE from tracks WHERE ReleaseID=?", [rg['id']]) - myDB.action("DELETE from alltracks WHERE ReleaseID=?", [rg['id']]) - myDB.action('DELETE from releases WHERE ReleaseGroupID=?', [rg['id']]) + myDB.action("DELETE from albums WHERE ReleaseID=%s", [rg['id']]) + myDB.action("DELETE from allalbums WHERE ReleaseID=%s", [rg['id']]) + myDB.action("DELETE from tracks WHERE ReleaseID=%s", [rg['id']]) + myDB.action("DELETE from alltracks WHERE ReleaseID=%s", [rg['id']]) + myDB.action('DELETE from releases WHERE ReleaseGroupID=%s', [rg['id']]) # This will be used later to build a hybrid release fullreleaselist = [] # Search for releases within a release group - find_hybrid_releases = myDB.action("SELECT * from allalbums WHERE AlbumID=?", + find_hybrid_releases = myDB.action("SELECT * from allalbums WHERE AlbumID=%s", [rg['id']]) # Build the dictionary for the fullreleaselist @@ -323,7 +326,7 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"): "ReleaseCountry": items['ReleaseCountry'], "ReleaseFormat": items['ReleaseFormat'] } - find_hybrid_tracks = myDB.action("SELECT * from alltracks WHERE ReleaseID=?", + find_hybrid_tracks = myDB.action("SELECT * from alltracks WHERE ReleaseID=%s", [hybrid_release_id]) totalTracks = 1 hybrid_track_array = [] @@ -386,21 +389,21 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"): "CleanName": cleanname } - match = myDB.action('SELECT Location, BitRate, Format from have WHERE CleanName=?', + match = myDB.action('SELECT Location, BitRate, Format from have WHERE CleanName=%s', [cleanname]).fetchone() if not match: match = myDB.action( - 'SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', + 'SELECT Location, BitRate, Format from have WHERE ArtistName LIKE %s AND AlbumTitle LIKE %s AND TrackTitle LIKE %s', [artist['artist_name'], rg['title'], track['title']]).fetchone() # if not match: - # match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone() + # match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=%s', [track['id']]).fetchone() if match: newValueDict['Location'] = match['Location'] newValueDict['BitRate'] = match['BitRate'] newValueDict['Format'] = match['Format'] - # myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']]) - myDB.action('UPDATE have SET Matched=? WHERE Location=?', + # myDB.action('UPDATE have SET Matched="True" WHERE Location=%s', [match['Location']]) + myDB.action('UPDATE have SET Matched=%s WHERE Location=%s', (rg['id'], match['Location'])) myDB.upsert("alltracks", newValueDict, controlValueDict) @@ -418,7 +421,7 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"): if not releaseid: releaseid = rg['id'] - album = myDB.action('SELECT * from allalbums WHERE ReleaseID=?', [releaseid]).fetchone() + album = myDB.action('SELECT * from allalbums WHERE ReleaseID=%s', [releaseid]).fetchone() controlValueDict = {"AlbumID": rg['id']} @@ -457,7 +460,7 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"): myDB.upsert("albums", newValueDict, controlValueDict) - tracks = myDB.action('SELECT * from alltracks WHERE ReleaseID=?', + tracks = myDB.action('SELECT * from alltracks WHERE ReleaseID=%s', [releaseid]).fetchall() # This is used to see how many tracks you have from an album - to @@ -492,7 +495,7 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"): # Mark albums as downloaded if they have at least 80% (by default, configurable) of the album have_track_count = len( - myDB.select('SELECT * from tracks WHERE AlbumID=? AND Location IS NOT NULL', + myDB.select('SELECT * from tracks WHERE AlbumID=%s AND Location IS NOT NULL', [rg['id']])) marked_as_downloaded = False @@ -500,13 +503,13 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"): if rg_exists['Status'] == 'Skipped' and ( (have_track_count / float(total_track_count)) >= ( headphones.CONFIG.ALBUM_COMPLETION_PCT / 100.0)): - myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?', + myDB.action('UPDATE albums SET Status=%s WHERE AlbumID=%s', ['Downloaded', rg['id']]) marked_as_downloaded = True else: if (have_track_count / float(total_track_count)) >= ( headphones.CONFIG.ALBUM_COMPLETION_PCT / 100.0): - myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?', + myDB.action('UPDATE albums SET Status=%s WHERE AlbumID=%s', ['Downloaded', rg['id']]) marked_as_downloaded = True @@ -538,7 +541,7 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"): "[%s] Finished updating artist: %s but with errors, so not marking it as updated in the database" % ( artist['artist_name'], artist['artist_name'])) else: - myDB.action('DELETE FROM newartists WHERE ArtistName = ?', [artist['artist_name']]) + myDB.action('DELETE FROM newartists WHERE ArtistName = %s', [artist['artist_name']]) logger.info(u"Updating complete for: %s" % artist['artist_name']) # Start searching for newly added albums @@ -549,6 +552,8 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"): for album_search in album_searches: searcher.searchforalbum(albumid=album_search) + myDB.commit() + def finalize_update(artistid, artistname, errors=False): # Moving this little bit to it's own function so we can update have tracks & latest album when deleting extras @@ -556,17 +561,17 @@ def finalize_update(artistid, artistname, errors=False): myDB = db.DBConnection() latestalbum = myDB.action( - 'SELECT AlbumTitle, ReleaseDate, AlbumID from albums WHERE ArtistID=? order by ReleaseDate DESC', + 'SELECT AlbumTitle, ReleaseDate, AlbumID from albums WHERE ArtistID=%s order by ReleaseDate DESC', [artistid]).fetchone() totaltracks = len(myDB.select( - 'SELECT TrackTitle from tracks WHERE ArtistID=? AND AlbumID IN (SELECT AlbumID FROM albums WHERE Status != "Ignored")', - [artistid])) - # havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [artistid])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist['artist_name']])) + 'SELECT TrackTitle from tracks WHERE ArtistID=%s AND AlbumID IN (SELECT AlbumID FROM albums WHERE Status != %s)', + [artistid, 'Ignored'])) + # havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=%s AND Location IS NOT NULL', [artistid])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist['artist_name']])) havetracks = len( - myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', + myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=%s AND Location IS NOT NULL', [artistid])) + len( - myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND Matched = "Failed"', - [artistname])) + myDB.select('SELECT TrackTitle from have WHERE ArtistName like %s AND Matched = %s', + [artistname, 'Failed'])) controlValueDict = {"ArtistID": artistid} @@ -594,7 +599,7 @@ def addReleaseById(rid, rgid=None): # Create minimum info upfront if added from searchresults status = '' if rgid: - dbalbum = myDB.select("SELECT * from albums WHERE AlbumID=?", [rgid]) + dbalbum = myDB.select("SELECT * from albums WHERE AlbumID=%s", [rgid]) if not dbalbum: status = 'Loading' controlValueDict = {"AlbumID": rgid} @@ -608,7 +613,7 @@ def addReleaseById(rid, rgid=None): artistid = None release_dict = None results = myDB.select( - "SELECT albums.ArtistID, releases.ReleaseGroupID from releases, albums WHERE releases.ReleaseID=? and releases.ReleaseGroupID=albums.AlbumID LIMIT 1", + "SELECT albums.ArtistID, releases.ReleaseGroupID from releases, albums WHERE releases.ReleaseID=%s and releases.ReleaseGroupID=albums.AlbumID LIMIT 1", [rid]) for result in results: rgid = result['ReleaseGroupID'] @@ -624,22 +629,22 @@ def addReleaseById(rid, rgid=None): except Exception as e: logger.info('Unable to get release information for Release %s: %s', rid, e) if status == 'Loading': - myDB.action("DELETE FROM albums WHERE AlbumID=?", [rgid]) + myDB.action("DELETE FROM albums WHERE AlbumID=%s", [rgid]) return if not release_dict: logger.info('Unable to get release information for Release %s: no dict', rid) if status == 'Loading': - myDB.action("DELETE FROM albums WHERE AlbumID=?", [rgid]) + myDB.action("DELETE FROM albums WHERE AlbumID=%s", [rgid]) return rgid = release_dict['rgid'] artistid = release_dict['artist_id'] # we don't want to make more calls to MB here unless we have to, could be happening quite a lot - rg_exists = myDB.select("SELECT * from albums WHERE AlbumID=?", [rgid]) + rg_exists = myDB.select("SELECT * from albums WHERE AlbumID=%s", [rgid]) # make sure the artist exists since I don't know what happens later if it doesn't - artist_exists = myDB.select("SELECT * from artists WHERE ArtistID=?", [artistid]) + artist_exists = myDB.select("SELECT * from artists WHERE ArtistID=%s", [artistid]) if not artist_exists and release_dict: if release_dict['artist_name'].startswith('The '): @@ -676,7 +681,7 @@ def addReleaseById(rid, rgid=None): logger.error( "Artist does not exist in the database and did not get a valid response from MB. Skipping release.") if status == 'Loading': - myDB.action("DELETE FROM albums WHERE AlbumID=?", [rgid]) + myDB.action("DELETE FROM albums WHERE AlbumID=%s", [rgid]) return if not rg_exists and release_dict or status == 'Loading' and release_dict: # it should never be the case that we have an rg and not the artist @@ -702,7 +707,7 @@ def addReleaseById(rid, rgid=None): myDB.upsert("albums", newValueDict, controlValueDict) # keep a local cache of these so that external programs that are adding releasesByID don't hammer MB - myDB.action('INSERT INTO releases VALUES( ?, ?)', [rid, release_dict['rgid']]) + myDB.action('INSERT INTO releases VALUES( %s, %s)', [rid, release_dict['rgid']]) for track in release_dict['tracks']: cleanname = helpers.clean_name( @@ -721,27 +726,27 @@ def addReleaseById(rid, rgid=None): } match = myDB.action( - 'SELECT Location, BitRate, Format, Matched from have WHERE CleanName=?', + 'SELECT Location, BitRate, Format, Matched from have WHERE CleanName=%s', [cleanname]).fetchone() if not match: match = myDB.action( - 'SELECT Location, BitRate, Format, Matched from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', + 'SELECT Location, BitRate, Format, Matched from have WHERE ArtistName LIKE %s AND AlbumTitle LIKE %s AND TrackTitle LIKE %s', [release_dict['artist_name'], release_dict['rg_title'], track['title']]).fetchone() # if not match: - # match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone() + # match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=%s', [track['id']]).fetchone() if match: newValueDict['Location'] = match['Location'] newValueDict['BitRate'] = match['BitRate'] newValueDict['Format'] = match['Format'] - # myDB.action('DELETE from have WHERE Location=?', [match['Location']]) + # myDB.action('DELETE from have WHERE Location=%s', [match['Location']]) # If the album has been scanned before adding the release it will be unmatched, update to matched if match['Matched'] == 'Failed': - myDB.action('UPDATE have SET Matched=? WHERE Location=?', + myDB.action('UPDATE have SET Matched=%s WHERE Location=%s', (release_dict['rgid'], match['Location'])) myDB.upsert("tracks", newValueDict, controlValueDict) @@ -760,11 +765,14 @@ def addReleaseById(rid, rgid=None): import searcher searcher.searchforalbum(rgid, False) + myDB.commit() + elif not rg_exists and not release_dict: logger.error( "ReleaseGroup does not exist in the database and did not get a valid response from MB. Skipping release.") if status == 'Loading': - myDB.action("DELETE FROM albums WHERE AlbumID=?", [rgid]) + myDB.action("DELETE FROM albums WHERE AlbumID=%s", [rgid]) + myDB.commit() return else: logger.info('Release ' + str(rid) + " already exists in the database!") @@ -781,7 +789,7 @@ def updateFormat(): except Exception as e: logger.info("Exception from MediaFile for: " + track['Location'] + " : " + str(e)) continue - controlValueDict = {"TrackID": track['TrackID']} + controlValueDict = {"TrackID": track['TrackID'], "AlbumID": track['AlbumID']} newValueDict = {"Format": f.format} myDB.upsert("tracks", newValueDict, controlValueDict) logger.info('Finished finding media format for %s files' % len(tracks)) diff --git a/headphones/lastfm.py b/headphones/lastfm.py index 89ab74483..a478271f6 100644 --- a/headphones/lastfm.py +++ b/headphones/lastfm.py @@ -101,7 +101,9 @@ def getSimilar(): artist_name, artist_mbid = item[0] count = item[1] - myDB.action("INSERT INTO lastfmcloud VALUES( ?, ?, ?)", [artist_name, artist_mbid, count]) + myDB.action("INSERT INTO lastfmcloud VALUES( %s, %s, %s)", [artist_name, artist_mbid, count]) + + myDB.commit() logger.debug("Inserted %d artists into Last.FM tag cloud", len(top_list)) diff --git a/headphones/librarysync.py b/headphones/librarysync.py index ba246ff0e..10098fb92 100644 --- a/headphones/librarysync.py +++ b/headphones/librarysync.py @@ -57,9 +57,9 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, for track in tracks: encoded_track_string = track['Location'].encode(headphones.SYS_ENCODING, 'replace') if not os.path.isfile(encoded_track_string): - myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE Location=?', + myDB.action('UPDATE tracks SET Location=%s, BitRate=%s, Format=%s WHERE Location=%s', [None, None, None, track['Location']]) - myDB.action('UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE Location=?', + myDB.action('UPDATE alltracks SET Location=%s, BitRate=%s, Format=%s WHERE Location=%s', [None, None, None, track['Location']]) del_have_tracks = myDB.select('SELECT Location, Matched, ArtistName from have') @@ -70,7 +70,7 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, if track['ArtistName']: # Make sure deleted files get accounted for when updating artist track counts new_artists.append(track['ArtistName']) - myDB.action('DELETE FROM have WHERE Location=?', [track['Location']]) + myDB.action('DELETE FROM have WHERE Location=%s', [track['Location']]) logger.info( 'File %s removed from Headphones, as it is no longer on disk' % encoded_track_string.decode( headphones.SYS_ENCODING, 'replace')) @@ -159,7 +159,7 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, } # song_list.append(song_dict) - check_exist_song = myDB.action("SELECT * FROM have WHERE Location=?", + check_exist_song = myDB.action("SELECT * FROM have WHERE Location=%s", [unicode_song_path]).fetchone() # Only attempt to match songs that are new, haven't yet been matched, or metadata has changed. if not check_exist_song: @@ -183,10 +183,10 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, newValueDict['Matched'] = None myDB.upsert("have", newValueDict, controlValueDict) myDB.action( - 'UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE Location=?', + 'UPDATE tracks SET Location=%s, BitRate=%s, Format=%s WHERE Location=%s', [None, None, None, unicode_song_path]) myDB.action( - 'UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE Location=?', + 'UPDATE alltracks SET Location=%s, BitRate=%s, Format=%s WHERE Location=%s', [None, None, None, unicode_song_path]) new_song_count += 1 else: @@ -198,11 +198,12 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, # Now we start track matching logger.info("%s new/modified songs found and added to the database" % new_song_count) - song_list = myDB.action("SELECT * FROM have WHERE Matched IS NULL AND LOCATION LIKE ?", + + song_list = myDB.action("SELECT * FROM have WHERE Matched IS NULL AND LOCATION LIKE %s", [dir.decode(headphones.SYS_ENCODING, 'replace') + "%"]) total_number_of_songs = \ - myDB.action("SELECT COUNT(*) FROM have WHERE Matched IS NULL AND LOCATION LIKE ?", - [dir.decode(headphones.SYS_ENCODING, 'replace') + "%"]).fetchone()[0] + myDB.action("SELECT COUNT(*) as c FROM have WHERE Matched IS NULL AND LOCATION LIKE %s", + [dir.decode(headphones.SYS_ENCODING, 'replace') + "%"]).fetchone()['c'] logger.info("Found " + str(total_number_of_songs) + " new/modified tracks in: '" + dir.decode( headphones.SYS_ENCODING, 'replace') + "'. Matching tracks to the appropriate releases....") @@ -240,13 +241,12 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, if song['ArtistName'] and song['AlbumTitle'] and song['TrackTitle']: track = myDB.action( - 'SELECT ArtistName, AlbumTitle, TrackTitle, AlbumID from tracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', + 'SELECT ArtistName, AlbumTitle, TrackTitle, AlbumID, TrackID from tracks WHERE ArtistName LIKE %s AND AlbumTitle LIKE %s AND TrackTitle LIKE %s', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone() have_updated = False if track: - controlValueDict = {'ArtistName': track['ArtistName'], - 'AlbumTitle': track['AlbumTitle'], - 'TrackTitle': track['TrackTitle']} + controlValueDict = {'TrackID': track['TrackID'], + 'AlbumID': track['AlbumID']} newValueDict = {'Location': song['Location'], 'BitRate': song['BitRate'], 'Format': song['Format']} @@ -257,10 +257,11 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, myDB.upsert("have", newValueDict2, controlValueDict2) have_updated = True else: - track = myDB.action('SELECT CleanName, AlbumID from tracks WHERE CleanName LIKE ?', + track = myDB.action('SELECT CleanName, AlbumID, TrackID from tracks WHERE CleanName LIKE %s', [song['CleanName']]).fetchone() if track: - controlValueDict = {'CleanName': track['CleanName']} + controlValueDict = {'TrackID': track['TrackID'], + 'AlbumID': track['AlbumID']} newValueDict = {'Location': song['Location'], 'BitRate': song['BitRate'], 'Format': song['Format']} @@ -277,12 +278,11 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, have_updated = True alltrack = myDB.action( - 'SELECT ArtistName, AlbumTitle, TrackTitle, AlbumID from alltracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', + 'SELECT ArtistName, AlbumTitle, TrackTitle, AlbumID, ReleaseID, TrackID from alltracks WHERE ArtistName LIKE %s AND AlbumTitle LIKE %s AND TrackTitle LIKE %s', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone() if alltrack: - controlValueDict = {'ArtistName': alltrack['ArtistName'], - 'AlbumTitle': alltrack['AlbumTitle'], - 'TrackTitle': alltrack['TrackTitle']} + controlValueDict = {'ReleaseID': alltrack['ReleaseID'], + 'TrackID': alltrack['TrackID']} newValueDict = {'Location': song['Location'], 'BitRate': song['BitRate'], 'Format': song['Format']} @@ -293,10 +293,11 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, myDB.upsert("have", newValueDict2, controlValueDict2) else: alltrack = myDB.action( - 'SELECT CleanName, AlbumID from alltracks WHERE CleanName LIKE ?', + 'SELECT CleanName, AlbumID, ReleaseID, TrackID from alltracks WHERE CleanName LIKE %s', [song['CleanName']]).fetchone() if alltrack: - controlValueDict = {'CleanName': alltrack['CleanName']} + controlValueDict = {'ReleaseID': alltrack['ReleaseID'], + 'TrackID': alltrack['TrackID']} newValueDict = {'Location': song['Location'], 'BitRate': song['BitRate'], 'Format': song['Format']} @@ -317,7 +318,7 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, newValueDict2 = {'Matched': "Failed"} myDB.upsert("have", newValueDict2, controlValueDict2) - # myDB.action('INSERT INTO have (ArtistName, AlbumTitle, TrackNumber, TrackTitle, TrackLength, BitRate, Genre, Date, TrackID, Location, CleanName, Format) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [song['ArtistName'], song['AlbumTitle'], song['TrackNumber'], song['TrackTitle'], song['TrackLength'], song['BitRate'], song['Genre'], song['Date'], song['TrackID'], song['Location'], CleanName, song['Format']]) + # myDB.action('INSERT INTO have (ArtistName, AlbumTitle, TrackNumber, TrackTitle, TrackLength, BitRate, Genre, Date, TrackID, Location, CleanName, Format) VALUES( %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)', [song['ArtistName'], song['AlbumTitle'], song['TrackNumber'], song['TrackTitle'], song['TrackLength'], song['BitRate'], song['Genre'], song['Date'], song['TrackID'], song['Location'], CleanName, song['Format']]) logger.info('Completed matching tracks from directory: %s' % dir.decode(headphones.SYS_ENCODING, 'replace')) @@ -333,14 +334,14 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, artist_list = [ x for x in unique_artists if helpers.clean_name(x).lower() not in [ - helpers.clean_name(y[0]).lower() + helpers.clean_name(y['ArtistName']).lower() for y in current_artists ] ] artists_checked = [ x for x in unique_artists if helpers.clean_name(x).lower() in [ - helpers.clean_name(y[0]).lower() + helpers.clean_name(y['ArtistName']).lower() for y in current_artists ] ] @@ -351,14 +352,14 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, # We update the track count upon an album switch to compliment this havetracks = ( len(myDB.select( - 'SELECT TrackTitle from tracks WHERE ArtistName like ? AND Location IS NOT NULL', + 'SELECT TrackTitle from tracks WHERE ArtistName like %s AND Location IS NOT NULL', [artist])) + len(myDB.select( - 'SELECT TrackTitle from have WHERE ArtistName like ? AND Matched = "Failed"', - [artist])) + 'SELECT TrackTitle from have WHERE ArtistName like %s AND Matched = %s', + ['Failed', artist])) ) # Note: some people complain about having "artist have tracks" > # of tracks total in artist official releases # (can fix by getting rid of second len statement) - myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistName=?', [havetracks, artist]) + myDB.action('UPDATE artists SET HaveTracks=%s WHERE ArtistName=%s', [havetracks, artist]) logger.info('Found %i new artists' % len(artist_list)) @@ -370,7 +371,7 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, logger.info('To add these artists, go to Manage->Manage New Artists') # myDB.action('DELETE from newartists') for artist in artist_list: - myDB.action('INSERT OR IGNORE INTO newartists VALUES (?)', [artist]) + myDB.action('INSERT INTO newartists VALUES (%s) ON CONFLICT DO NOTHING', [artist]) if headphones.CONFIG.DETECT_BITRATE and bitrates: headphones.CONFIG.PREFERRED_BITRATE = sum(bitrates) / len(bitrates) / 1000 @@ -380,11 +381,11 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, logger.info('Updating artist track counts') havetracks = len( - myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', + myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=%s AND Location IS NOT NULL', [ArtistID])) + len(myDB.select( - 'SELECT TrackTitle from have WHERE ArtistName like ? AND Matched = "Failed"', - [ArtistName])) - myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, ArtistID]) + 'SELECT TrackTitle from have WHERE ArtistName like %s AND Matched = %s', + ['Failed', ArtistName])) + myDB.action('UPDATE artists SET HaveTracks=%s WHERE ArtistID=%s', [havetracks, ArtistID]) if not append: update_album_status() @@ -393,6 +394,7 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, lastfm.getSimilar() logger.info('Library scan complete') + myDB.commit() # ADDED THIS SECTION TO MARK ALBUMS AS DOWNLOADED IF ARTISTS ARE ADDED EN MASSE BEFORE LIBRARY IS SCANNED @@ -403,11 +405,11 @@ def update_album_status(AlbumID=None): logger.info('Counting matched tracks to mark albums as skipped/downloaded') if AlbumID: album_status_updater = myDB.action( - 'SELECT AlbumID, AlbumTitle, Status from albums WHERE AlbumID=?', [AlbumID]) + 'SELECT AlbumID, AlbumTitle, Status from albums WHERE AlbumID=%s', [AlbumID]) else: album_status_updater = myDB.action('SELECT AlbumID, AlbumTitle, Status from albums') for album in album_status_updater: - track_counter = myDB.action('SELECT Location from tracks where AlbumID=?', + track_counter = myDB.action('SELECT Location from tracks where AlbumID=%s', [album['AlbumID']]) total_tracks = 0 have_tracks = 0 @@ -439,4 +441,5 @@ def update_album_status(AlbumID=None): myDB.upsert("albums", {'Status': new_album_status}, {'AlbumID': album['AlbumID']}) if new_album_status != album['Status']: logger.info('Album %s changed to %s' % (album['AlbumTitle'], new_album_status)) + myDB.commit() logger.info('Album status update complete') diff --git a/headphones/mb.py b/headphones/mb.py index 397e50087..1a50bec13 100644 --- a/headphones/mb.py +++ b/headphones/mb.py @@ -315,7 +315,7 @@ def getArtist(artistid, extrasonly=False): myDB = db.DBConnection() try: - db_artist = myDB.action('SELECT IncludeExtras, Extras from artists WHERE ArtistID=?', + db_artist = myDB.action('SELECT IncludeExtras, Extras from artists WHERE ArtistID=%s', [artistid]).fetchone() includeExtras = db_artist['IncludeExtras'] except IndexError: @@ -371,6 +371,7 @@ def getArtist(artistid, extrasonly=False): 'type': unicode(rg_type) }) artist_dict['releasegroups'] = releasegroups + myDB.commit() return artist_dict @@ -542,16 +543,16 @@ def get_new_releases(rgid, includeExtras=False, forcefull=False): for release_mark in results: release_list.append(unicode(release_mark['id'])) release_title = release_mark['title'] - remove_missing_releases = myDB.action("SELECT ReleaseID FROM allalbums WHERE AlbumID=?", + remove_missing_releases = myDB.action("SELECT ReleaseID FROM allalbums WHERE AlbumID=%s", [rgid]) - if remove_missing_releases: + if remove_missing_releases.rowcount: for items in remove_missing_releases: if items['ReleaseID'] not in release_list and items['ReleaseID'] != rgid: # Remove all from albums/tracks that aren't in release - myDB.action("DELETE FROM albums WHERE ReleaseID=?", [items['ReleaseID']]) - myDB.action("DELETE FROM tracks WHERE ReleaseID=?", [items['ReleaseID']]) - myDB.action("DELETE FROM allalbums WHERE ReleaseID=?", [items['ReleaseID']]) - myDB.action("DELETE FROM alltracks WHERE ReleaseID=?", [items['ReleaseID']]) + myDB.action("DELETE FROM albums WHERE ReleaseID=%s", [items['ReleaseID']]) + myDB.action("DELETE FROM tracks WHERE ReleaseID=%s", [items['ReleaseID']]) + myDB.action("DELETE FROM allalbums WHERE ReleaseID=%s", [items['ReleaseID']]) + myDB.action("DELETE FROM alltracks WHERE ReleaseID=%s", [items['ReleaseID']]) logger.info( "Removing all references to release %s to reflect MusicBrainz" % items[ 'ReleaseID']) @@ -566,12 +567,12 @@ def get_new_releases(rgid, includeExtras=False, forcefull=False): release = {} rel_id_check = releasedata['id'] - album_checker = myDB.action('SELECT * from allalbums WHERE ReleaseID=?', + album_checker = myDB.action('SELECT * from allalbums WHERE ReleaseID=%s', [rel_id_check]).fetchone() if not album_checker or forcefull: # DELETE all references to this release since we're updating it anyway. - myDB.action('DELETE from allalbums WHERE ReleaseID=?', [rel_id_check]) - myDB.action('DELETE from alltracks WHERE ReleaseID=?', [rel_id_check]) + myDB.action('DELETE from allalbums WHERE ReleaseID=%s', [rel_id_check]) + myDB.action('DELETE from alltracks WHERE ReleaseID=%s', [rel_id_check]) release['AlbumTitle'] = unicode(releasedata['title']) release['AlbumID'] = unicode(rgid) release['AlbumASIN'] = unicode(releasedata['asin']) if 'asin' in releasedata else None @@ -654,21 +655,21 @@ def get_new_releases(rgid, includeExtras=False, forcefull=False): "CleanName": cleanname } - match = myDB.action('SELECT Location, BitRate, Format from have WHERE CleanName=?', + match = myDB.action('SELECT Location, BitRate, Format from have WHERE CleanName=%s', [cleanname]).fetchone() if not match: match = myDB.action( - 'SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', + 'SELECT Location, BitRate, Format from have WHERE ArtistName LIKE %s AND AlbumTitle LIKE %s AND TrackTitle LIKE %s', [release['ArtistName'], release['AlbumTitle'], track['title']]).fetchone() # if not match: - # match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone() + # match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=%s', [track['id']]).fetchone() if match: newValueDict['Location'] = match['Location'] newValueDict['BitRate'] = match['BitRate'] newValueDict['Format'] = match['Format'] - # myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']]) - myDB.action('UPDATE have SET Matched=? WHERE Location=?', + # myDB.action('UPDATE have SET Matched="True" WHERE Location=%s', [match['Location']]) + myDB.action('UPDATE have SET Matched=%s WHERE Location=%s', (release['AlbumID'], match['Location'])) myDB.upsert("alltracks", newValueDict, controlValueDict) @@ -686,6 +687,7 @@ def get_new_releases(rgid, includeExtras=False, forcefull=False): else: num_new_releases = num_new_releases + myDB.commit() return num_new_releases @@ -716,7 +718,7 @@ def findArtistbyAlbum(name): myDB = db.DBConnection() artist = myDB.action( - 'SELECT AlbumTitle from have WHERE ArtistName=? AND AlbumTitle IS NOT NULL ORDER BY RANDOM()', + 'SELECT AlbumTitle from have WHERE ArtistName=%s AND AlbumTitle IS NOT NULL ORDER BY RANDOM()', [name]).fetchone() if not artist: diff --git a/headphones/metacritic.py b/headphones/metacritic.py index d482786fe..79e26d68a 100644 --- a/headphones/metacritic.py +++ b/headphones/metacritic.py @@ -47,7 +47,7 @@ def update(artistid, artist_name, release_groups): logger.info("Unable to get metacritic scores for: %s" % artist_name) myDB = db.DBConnection() - artist = myDB.action('SELECT * FROM artists WHERE ArtistID=?', [artistid]).fetchone() + artist = myDB.action('SELECT * FROM artists WHERE ArtistID=%s', [artistid]).fetchone() score_list = [] diff --git a/headphones/postprocessor.py b/headphones/postprocessor.py index ae38bdfb8..dcf3f9b7c 100755 --- a/headphones/postprocessor.py +++ b/headphones/postprocessor.py @@ -40,7 +40,7 @@ def checkFolder(): with postprocessor_lock: myDB = db.DBConnection() - snatched = myDB.select('SELECT * from snatched WHERE Status="Snatched"') + snatched = myDB.select('SELECT * from snatched WHERE Status=%s', ['Snatched']) for album in snatched: if album['FolderName']: @@ -68,8 +68,8 @@ def checkFolder(): def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=False): myDB = db.DBConnection() - release = myDB.action('SELECT * from albums WHERE AlbumID=?', [albumid]).fetchone() - tracks = myDB.select('SELECT * from tracks WHERE AlbumID=?', [albumid]) + release = myDB.action('SELECT * from albums WHERE AlbumID=%s', [albumid]).fetchone() + tracks = myDB.select('SELECT * from tracks WHERE AlbumID=%s', [albumid]) if not release or not tracks: release_list = None @@ -105,7 +105,7 @@ def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=Fal # this check is skipped, since it is assumed the user wants this. if headphones.CONFIG.FREEZE_DB and not forced: artist = myDB.select( - "SELECT ArtistName, ArtistID FROM artists WHERE ArtistId=? OR ArtistName=?", + "SELECT ArtistName, ArtistID FROM artists WHERE ArtistId=%s OR ArtistName=%s", [release_dict['artist_id'], release_dict['artist_name']]) if not artist: @@ -115,8 +115,8 @@ def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=Fal release_dict['artist_id'], albumid) myDB.action( - 'UPDATE snatched SET status = "Frozen" WHERE status NOT LIKE "Seed%" and AlbumID=?', - [albumid]) + 'UPDATE snatched SET status = "Frozen" WHERE status NOT LIKE %s and AlbumID=%s', + ['Seed%', albumid]) frozen = re.search(r' \(Frozen\)(?:\[\d+\])?', albumpath) if not frozen: if headphones.CONFIG.RENAME_FROZEN: @@ -124,6 +124,7 @@ def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=Fal else: logger.warn(u"Won't rename %s to mark as 'Frozen', because it is disabled.", albumpath.decode(headphones.SYS_ENCODING, 'replace')) + myDB.commit() return logger.info(u"Now adding/updating artist: " + release_dict['artist_name']) @@ -165,7 +166,7 @@ def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=Fal myDB.upsert("albums", newValueDict, controlValueDict) # Delete existing tracks associated with this AlbumID since we're going to replace them and don't want any extras - myDB.action('DELETE from tracks WHERE AlbumID=?', [albumid]) + myDB.action('DELETE from tracks WHERE AlbumID=%s', [albumid]) for track in release_dict['tracks']: controlValueDict = {"TrackID": track['id'], "AlbumID": albumid} @@ -188,8 +189,8 @@ def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=Fal logger.info(u"Addition complete for: " + release_dict['title'] + " - " + release_dict[ 'artist_name']) - release = myDB.action('SELECT * from albums WHERE AlbumID=?', [albumid]).fetchone() - tracks = myDB.select('SELECT * from tracks WHERE AlbumID=?', [albumid]) + release = myDB.action('SELECT * from albums WHERE AlbumID=%s', [albumid]).fetchone() + tracks = myDB.select('SELECT * from tracks WHERE AlbumID=%s', [albumid]) downloaded_track_list = [] downloaded_cuecount = 0 @@ -301,9 +302,10 @@ def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=Fal logger.warn(u'Could not identify album: %s. It may not be the intended album.', albumpath.decode(headphones.SYS_ENCODING, 'replace')) myDB.action( - 'UPDATE snatched SET status = "Unprocessed" WHERE status NOT LIKE "Seed%" and AlbumID=?', - [albumid]) + 'UPDATE snatched SET status = %s WHERE status NOT LIKE %s and AlbumID=%s', + ['Unprocessed', 'Seed%', albumid]) processed = re.search(r' \(Unprocessed\)(?:\[\d+\])?', albumpath) + myDB.commit() if not processed: if headphones.CONFIG.RENAME_UNPROCESSED: renameUnprocessedFolder(albumpath, tag="Unprocessed") @@ -440,16 +442,16 @@ def doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list, updateFilePermissions(albumpaths) myDB = db.DBConnection() - myDB.action('UPDATE albums SET status = "Downloaded" WHERE AlbumID=?', [albumid]) + myDB.action('UPDATE albums SET status = %s WHERE AlbumID=%s', ['Downloaded', albumid]) myDB.action( - 'UPDATE snatched SET status = "Processed" WHERE Status NOT LIKE "Seed%" and AlbumID=?', - [albumid]) + 'UPDATE snatched SET status = %s WHERE Status NOT LIKE %s and AlbumID=%s', + ['Processed', 'Seed%', albumid]) # Check if torrent has finished seeding if headphones.CONFIG.TORRENT_DOWNLOADER == 1 or headphones.CONFIG.TORRENT_DOWNLOADER == 2: seed_snatched = myDB.action( - 'SELECT * from snatched WHERE Status="Seed_Snatched" and AlbumID=?', - [albumid]).fetchone() + 'SELECT * from snatched WHERE Status=%s and AlbumID=%s', + ['Seed_Snatched', albumid]).fetchone() if seed_snatched: hash = seed_snatched['FolderName'] torrent_removed = False @@ -464,12 +466,12 @@ def doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list, # Torrent removed, delete the snatched record, else update Status for scheduled job to check if torrent_removed: - myDB.action('DELETE from snatched WHERE status = "Seed_Snatched" and AlbumID=?', - [albumid]) + myDB.action('DELETE from snatched WHERE status = %s and AlbumID=%s', + ['Seed_Snatched', albumid]) else: myDB.action( - 'UPDATE snatched SET status = "Seed_Processed" WHERE status = "Seed_Snatched" and AlbumID=?', - [albumid]) + 'UPDATE snatched SET status = %s WHERE status = %s and AlbumID=%s', + ['Seed_Processed', 'Seed_Snatched', albumid]) # Update the have tracks for all created dirs: for albumpath in albumpaths: @@ -583,6 +585,8 @@ def doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list, if new_folder: shutil.rmtree(new_folder) + myDB.commit() + def embedAlbumArt(artwork, downloaded_track_list): logger.info('Embedding album art') @@ -1171,7 +1175,7 @@ def forcePostProcess(dir=None, expand_subfolders=True, album_dir=None, keep_orig # spaces/underscores came from sab replacing values logger.debug('Attempting to find album in the snatched table') snatched = myDB.action( - 'SELECT AlbumID, Title, Kind, Status from snatched WHERE FolderName LIKE ?', + 'SELECT AlbumID, Title, Kind, Status from snatched WHERE FolderName LIKE %s', [folder_basename]).fetchone() if snatched: @@ -1201,7 +1205,7 @@ def forcePostProcess(dir=None, expand_subfolders=True, album_dir=None, keep_orig if rgid: rgid = possible_rgid release = myDB.action( - 'SELECT ArtistName, AlbumTitle, AlbumID from albums WHERE AlbumID=?', + 'SELECT ArtistName, AlbumTitle, AlbumID from albums WHERE AlbumID=%s', [rgid]).fetchone() if release: logger.info( @@ -1226,7 +1230,7 @@ def forcePostProcess(dir=None, expand_subfolders=True, album_dir=None, keep_orig if name and album: release = myDB.action( - 'SELECT AlbumID, ArtistName, AlbumTitle from albums WHERE ArtistName LIKE ? and AlbumTitle LIKE ?', + 'SELECT AlbumID, ArtistName, AlbumTitle from albums WHERE ArtistName LIKE %s and AlbumTitle LIKE %s', [name, album]).fetchone() if release: logger.info( @@ -1266,7 +1270,7 @@ def forcePostProcess(dir=None, expand_subfolders=True, album_dir=None, keep_orig if name and album: release = myDB.action( - 'SELECT AlbumID, ArtistName, AlbumTitle from albums WHERE ArtistName LIKE ? and AlbumTitle LIKE ?', + 'SELECT AlbumID, ArtistName, AlbumTitle from albums WHERE ArtistName LIKE %s and AlbumTitle LIKE %s', [name, album]).fetchone() if release: logger.info( @@ -1295,7 +1299,7 @@ def forcePostProcess(dir=None, expand_subfolders=True, album_dir=None, keep_orig if '-' not in folder_basename: release = myDB.action( - 'SELECT AlbumID, ArtistName, AlbumTitle from albums WHERE AlbumTitle LIKE ?', + 'SELECT AlbumID, ArtistName, AlbumTitle from albums WHERE AlbumTitle LIKE %s', [folder_basename]).fetchone() if release: logger.info( @@ -1323,3 +1327,5 @@ def forcePostProcess(dir=None, expand_subfolders=True, album_dir=None, keep_orig "albums from another source, they must be in an 'Artist - Album " "[Year]' format, or end with the musicbrainz release group id.", folder_basename) + + myDB.commit() diff --git a/headphones/searcher.py b/headphones/searcher.py index c8f16184f..25061e06b 100644 --- a/headphones/searcher.py +++ b/headphones/searcher.py @@ -190,7 +190,8 @@ def searchforalbum(albumid=None, new=False, losslessOnly=False, if not albumid: results = myDB.select( - 'SELECT * from albums WHERE Status="Wanted" OR Status="Wanted Lossless"') + 'SELECT * from albums WHERE Status=%s OR Status=%s', + ['Wanted','Wanted Lossless']) for album in results: @@ -221,13 +222,13 @@ def searchforalbum(albumid=None, new=False, losslessOnly=False, do_sorted_search(album, new, losslessOnly) elif albumid and choose_specific_download: - album = myDB.action('SELECT * from albums WHERE AlbumID=?', [albumid]).fetchone() + album = myDB.action('SELECT * from albums WHERE AlbumID=%s', [albumid]).fetchone() logger.info('Searching for "%s - %s"' % (album['ArtistName'], album['AlbumTitle'])) results = do_sorted_search(album, new, losslessOnly, choose_specific_download=True) return results else: - album = myDB.action('SELECT * from albums WHERE AlbumID=?', [albumid]).fetchone() + album = myDB.action('SELECT * from albums WHERE AlbumID=%s', [albumid]).fetchone() logger.info('Searching for "%s - %s" since it was marked as wanted' % ( album['ArtistName'], album['AlbumTitle'])) do_sorted_search(album, new, losslessOnly) @@ -257,8 +258,8 @@ def do_sorted_search(album, new, losslessOnly, choose_specific_download=False): results = [] myDB = db.DBConnection() - albumlength = myDB.select('SELECT sum(TrackDuration) from tracks WHERE AlbumID=?', - [album['AlbumID']])[0][0] + albumlength = myDB.select('SELECT sum(TrackDuration) as tsum from tracks WHERE AlbumID=%s', + [album['AlbumID']])[0]['tsum'] if headphones.CONFIG.PREFER_TORRENTS == 0 and not choose_specific_download: @@ -372,7 +373,7 @@ def more_filtering(results, album, albumlength, new): continue if new: - alreadydownloaded = myDB.select('SELECT * from snatched WHERE URL=?', [result[2]]) + alreadydownloaded = myDB.select('SELECT * from snatched WHERE URL=%s', [result[2]]) if len(alreadydownloaded): logger.info( @@ -976,16 +977,16 @@ def send_to_downloader(data, bestqual, album): utorrent.setSeedRatio(torrentid, seed_ratio) myDB = db.DBConnection() - myDB.action('UPDATE albums SET status = "Snatched" WHERE AlbumID=?', [album['AlbumID']]) - myDB.action('INSERT INTO snatched VALUES( ?, ?, ?, ?, DATETIME("NOW", "localtime"), ?, ?, ?)', - [album['AlbumID'], bestqual[0], bestqual[1], bestqual[2], "Snatched", folder_name, + myDB.action('UPDATE albums SET status = %s WHERE AlbumID=%s', [album['Snatched' ,'AlbumID']]) + myDB.action('INSERT INTO snatched VALUES( %s, %s, %s, %s, now(), %s, %s, %s)', + [album['AlbumID'], bestqual[0], bestqual[1], bestqual[2], 'Snatched', folder_name, kind]) # Store the torrent id so we can check later if it's finished seeding and can be removed if seed_ratio is not None and seed_ratio != 0 and torrentid: myDB.action( - 'INSERT INTO snatched VALUES( ?, ?, ?, ?, DATETIME("NOW", "localtime"), ?, ?, ?)', - [album['AlbumID'], bestqual[0], bestqual[1], bestqual[2], "Seed_Snatched", torrentid, + 'INSERT INTO snatched VALUES( %s, %s, %s, %s, now(), %s, %s, %s)', + [album['AlbumID'], bestqual[0], bestqual[1], bestqual[2], 'Seed_Snatched', torrentid, kind]) # notify @@ -1051,6 +1052,8 @@ def send_to_downloader(data, bestqual, album): message = 'Snatched from ' + provider + '. ' + name email.notify("Snatched: " + title, message) + myDB.commit() + def verifyresult(title, artistterm, term, lossless): title = re.sub('[\.\-\/\_]', ' ', title) diff --git a/headphones/torrentfinished.py b/headphones/torrentfinished.py index bc622291c..dbda04f5f 100644 --- a/headphones/torrentfinished.py +++ b/headphones/torrentfinished.py @@ -30,7 +30,7 @@ def checkTorrentFinished(): with postprocessor_lock: myDB = db.DBConnection() - results = myDB.select('SELECT * from snatched WHERE Status="Seed_Processed"') + results = myDB.select('SELECT * from snatched WHERE Status=%s', ['Seed_Processed']) for album in results: hash = album['FolderName'] @@ -42,7 +42,8 @@ def checkTorrentFinished(): torrent_removed = utorrent.removeTorrent(hash, True) if torrent_removed: - myDB.action('DELETE from snatched WHERE status = "Seed_Processed" and AlbumID=?', - [albumid]) + myDB.action('DELETE from snatched WHERE status = %s and AlbumID=%s', + ['Seed_Processed', albumid]) + myDB.commit() logger.info("Checking finished torrents completed") diff --git a/headphones/updater.py b/headphones/updater.py index 0dc49e526..869b4a19b 100644 --- a/headphones/updater.py +++ b/headphones/updater.py @@ -20,11 +20,11 @@ def dbUpdate(forcefull=False): myDB = db.DBConnection() active_artists = myDB.select( - 'SELECT ArtistID, ArtistName from artists WHERE Status="Active" or Status="Loading" order by LastUpdated ASC') + 'SELECT ArtistID, ArtistName from artists WHERE Status=%s or Status=%s order by LastUpdated ASC', ['Active', 'Loading']) logger.info('Starting update for %i active artists', len(active_artists)) for artist in active_artists: - artistid = artist[0] + artistid = artist['ArtistID'] importer.addArtisttoDB(artistid=artistid, extrasonly=False, forcefull=forcefull) logger.info('Active artist update complete') diff --git a/headphones/webserve.py b/headphones/webserve.py index 249bf9f95..56c8e218e 100644 --- a/headphones/webserve.py +++ b/headphones/webserve.py @@ -49,7 +49,7 @@ def serve_template(templatename, **kwargs): interface_dir = os.path.join(str(headphones.PROG_DIR), 'data/interfaces/') template_dir = os.path.join(str(interface_dir), headphones.CONFIG.INTERFACE) - _hplookup = TemplateLookup(directories=[template_dir]) + _hplookup = TemplateLookup(directories=[template_dir], input_encoding='utf-8', output_encoding='utf-8') try: template = _hplookup.get_template(templatename) @@ -66,7 +66,8 @@ def index(self): @cherrypy.expose def home(self): myDB = db.DBConnection() - artists = myDB.select('SELECT * from artists order by ArtistSortName COLLATE NOCASE') + artists = myDB.select('SELECT * from artists order by lower(ArtistSortName)') + myDB.commit() return serve_template(templatename="index.html", title="Home", artists=artists) @cherrypy.expose @@ -77,7 +78,7 @@ def threads(self): @cherrypy.expose def artistPage(self, ArtistID): myDB = db.DBConnection() - artist = myDB.action('SELECT * FROM artists WHERE ArtistID=?', [ArtistID]).fetchone() + artist = myDB.action('SELECT * FROM artists WHERE ArtistID=%s', [ArtistID]).fetchone() # Don't redirect to the artist page until it has the bare minimum info inserted # Redirect to the home page if we still can't get it after 5 seconds @@ -85,13 +86,13 @@ def artistPage(self, ArtistID): while not artist and retry < 5: time.sleep(1) - artist = myDB.action('SELECT * FROM artists WHERE ArtistID=?', [ArtistID]).fetchone() + artist = myDB.action('SELECT * FROM artists WHERE ArtistID=%s', [ArtistID]).fetchone() retry += 1 if not artist: raise cherrypy.HTTPRedirect("home") - albums = myDB.select('SELECT * from albums WHERE ArtistID=? order by ReleaseDate DESC', + albums = myDB.select("SELECT * from albums WHERE ArtistID=%s order by to_date(ReleaseDate,'YYYY-MM-DD') DESC", [ArtistID]) # Serve the extras up as a dict to make things easier for new templates (append new extras to the end) @@ -111,19 +112,20 @@ def artistPage(self, ArtistID): extras_dict[extra] = "" i += 1 + myDB.commit() return serve_template(templatename="artist.html", title=artist['ArtistName'], artist=artist, albums=albums, extras=extras_dict) @cherrypy.expose def albumPage(self, AlbumID): myDB = db.DBConnection() - album = myDB.action('SELECT * from albums WHERE AlbumID=?', [AlbumID]).fetchone() + album = myDB.action('SELECT * from albums WHERE AlbumID=%s', [AlbumID]).fetchone() retry = 0 while retry < 5: if not album: time.sleep(1) - album = myDB.action('SELECT * from albums WHERE AlbumID=?', [AlbumID]).fetchone() + album = myDB.action('SELECT * from albums WHERE AlbumID=%s', [AlbumID]).fetchone() retry += 1 else: break @@ -132,8 +134,8 @@ def albumPage(self, AlbumID): raise cherrypy.HTTPRedirect("home") tracks = myDB.select( - 'SELECT * from tracks WHERE AlbumID=? ORDER BY CAST(TrackNumber AS INTEGER)', [AlbumID]) - description = myDB.action('SELECT * from descriptions WHERE ReleaseGroupID=?', + 'SELECT * from tracks WHERE AlbumID=%s ORDER BY CAST(TrackNumber AS INTEGER)', [AlbumID]) + description = myDB.action('SELECT * from descriptions WHERE ReleaseGroupID=%s', [AlbumID]).fetchone() if not album['ArtistName']: @@ -144,6 +146,7 @@ def albumPage(self, AlbumID): title = title + "" else: title = title + album['AlbumTitle'] + myDB.commit() return serve_template(templatename="album.html", title=title, album=album, tracks=tracks, description=description) @@ -210,18 +213,18 @@ def removeExtras(self, ArtistID, ArtistName): newValueDict = {'IncludeExtras': 0} myDB.upsert("artists", newValueDict, controlValueDict) extraalbums = myDB.select( - 'SELECT AlbumID from albums WHERE ArtistID=? AND Status="Skipped" AND Type!="Album"', - [ArtistID]) + 'SELECT AlbumID from albums WHERE ArtistID=%s AND Status=%s AND Type!=%s', + ['Skipped', 'Album', ArtistID]) for album in extraalbums: - myDB.action('DELETE from tracks WHERE ArtistID=? AND AlbumID=?', + myDB.action('DELETE from tracks WHERE ArtistID=%s AND AlbumID=%s', [ArtistID, album['AlbumID']]) - myDB.action('DELETE from albums WHERE ArtistID=? AND AlbumID=?', + myDB.action('DELETE from albums WHERE ArtistID=%s AND AlbumID=%s', [ArtistID, album['AlbumID']]) - myDB.action('DELETE from allalbums WHERE ArtistID=? AND AlbumID=?', + myDB.action('DELETE from allalbums WHERE ArtistID=%s AND AlbumID=%s', [ArtistID, album['AlbumID']]) - myDB.action('DELETE from alltracks WHERE ArtistID=? AND AlbumID=?', + myDB.action('DELETE from alltracks WHERE ArtistID=%s AND AlbumID=%s', [ArtistID, album['AlbumID']]) - myDB.action('DELETE from releases WHERE ReleaseGroupID=?', [album['AlbumID']]) + myDB.action('DELETE from releases WHERE ReleaseGroupID=%s', [album['AlbumID']]) from headphones import cache c = cache.Cache() c.remove_from_cache(AlbumID=album['AlbumID']) @@ -248,34 +251,35 @@ def resumeArtist(self, ArtistID): def removeArtist(self, ArtistID): myDB = db.DBConnection() - namecheck = myDB.select('SELECT ArtistName from artists where ArtistID=?', [ArtistID]) + namecheck = myDB.select('SELECT ArtistName from artists where ArtistID=%s', [ArtistID]) for name in namecheck: artistname = name['ArtistName'] logger.info(u"Deleting all traces of artist: " + artistname) - myDB.action('DELETE from artists WHERE ArtistID=?', [ArtistID]) + myDB.action('DELETE from artists WHERE ArtistID=%s', [ArtistID]) from headphones import cache c = cache.Cache() rgids = myDB.select( - 'SELECT AlbumID FROM albums WHERE ArtistID=? UNION SELECT AlbumID FROM allalbums WHERE ArtistID=?', + 'SELECT AlbumID FROM albums WHERE ArtistID=%s UNION SELECT AlbumID FROM allalbums WHERE ArtistID=%s', [ArtistID, ArtistID]) for rgid in rgids: albumid = rgid['AlbumID'] - myDB.action('DELETE from releases WHERE ReleaseGroupID=?', [albumid]) - myDB.action('DELETE from have WHERE Matched=?', [albumid]) + myDB.action('DELETE from releases WHERE ReleaseGroupID=%s', [albumid]) + myDB.action('DELETE from have WHERE Matched=%s', [albumid]) c.remove_from_cache(AlbumID=albumid) - myDB.action('DELETE from descriptions WHERE ReleaseGroupID=?', [albumid]) + myDB.action('DELETE from descriptions WHERE ReleaseGroupID=%s', [albumid]) - myDB.action('DELETE from albums WHERE ArtistID=?', [ArtistID]) - myDB.action('DELETE from tracks WHERE ArtistID=?', [ArtistID]) + myDB.action('DELETE from albums WHERE ArtistID=%s', [ArtistID]) + myDB.action('DELETE from tracks WHERE ArtistID=%s', [ArtistID]) - myDB.action('DELETE from allalbums WHERE ArtistID=?', [ArtistID]) - myDB.action('DELETE from alltracks WHERE ArtistID=?', [ArtistID]) - myDB.action('DELETE from have WHERE ArtistName=?', [artistname]) + myDB.action('DELETE from allalbums WHERE ArtistID=%s', [ArtistID]) + myDB.action('DELETE from alltracks WHERE ArtistID=%s', [ArtistID]) + myDB.action('DELETE from have WHERE ArtistName=%s', [artistname]) c.remove_from_cache(ArtistID=ArtistID) - myDB.action('DELETE from descriptions WHERE ArtistID=?', [ArtistID]) - myDB.action('INSERT OR REPLACE into blacklist VALUES (?)', [ArtistID]) + myDB.action('DELETE from descriptions WHERE ArtistID=%s', [ArtistID]) + myDB.action('INSERT into blacklist VALUES (%s) ON CONFLICT DO NOTHING', [ArtistID]) + myDB.commit() @cherrypy.expose def deleteArtist(self, ArtistID): @@ -286,7 +290,7 @@ def deleteArtist(self, ArtistID): def scanArtist(self, ArtistID): myDB = db.DBConnection() - artist_name = myDB.select('SELECT DISTINCT ArtistName FROM artists WHERE ArtistID=?', [ArtistID])[0][0] + artist_name = myDB.select('SELECT DISTINCT ArtistName FROM artists WHERE ArtistID=%s', [ArtistID])[0]['ArtistName'] logger.info(u"Scanning artist: %s", artist_name) @@ -352,6 +356,7 @@ def scanArtist(self, ArtistID): threading.Thread(target=librarysync.libraryScan, kwargs={"dir": artistfolder, "artistScan": True, "ArtistID": ArtistID, "ArtistName": artist_name}).start() + myDB.commit() raise cherrypy.HTTPRedirect("artistPage?ArtistID=%s" % ArtistID) @cherrypy.expose @@ -391,10 +396,11 @@ def markAlbums(self, ArtistID=None, action=None, **args): if ArtistID: ArtistIDT = ArtistID else: - ArtistIDT = myDB.action('SELECT ArtistID FROM albums WHERE AlbumID=?', [mbid]).fetchone()[0] + ArtistIDT = myDB.action('SELECT ArtistID FROM albums WHERE AlbumID=%s', [mbid]).fetchone()['ArtistID'] myDB.action( - 'UPDATE artists SET TotalTracks=(SELECT COUNT(*) FROM tracks WHERE ArtistID = ? AND AlbumTitle IN (SELECT AlbumTitle FROM albums WHERE Status != "Ignored")) WHERE ArtistID = ?', - [ArtistIDT, ArtistIDT]) + 'UPDATE artists SET TotalTracks=(SELECT COUNT(*) FROM tracks WHERE ArtistID = %s AND AlbumTitle IN (SELECT AlbumTitle FROM albums WHERE Status != %s)) WHERE ArtistID = %s', + [ArtistIDT, 'Ignored', ArtistIDT]) + myDB.commit() if ArtistID: raise cherrypy.HTTPRedirect("artistPage?ArtistID=%s" % ArtistID) else: @@ -407,11 +413,12 @@ def addArtists(self, action=None, **args): if action == "ignore": myDB = db.DBConnection() for artist in args: - myDB.action('DELETE FROM newartists WHERE ArtistName=?', - [artist.decode(headphones.SYS_ENCODING, 'replace')]) - myDB.action('UPDATE have SET Matched="Ignored" WHERE ArtistName=?', + myDB.action('DELETE FROM newartists WHERE ArtistName=%s', [artist.decode(headphones.SYS_ENCODING, 'replace')]) + myDB.action('UPDATE have SET Matched=%s WHERE ArtistName=%s', + ['Ignored', artist.decode(headphones.SYS_ENCODING, 'replace')]) logger.info("Artist %s removed from new artist list and set to ignored" % artist) + myDB.commit() raise cherrypy.HTTPRedirect("home") @cherrypy.expose @@ -465,7 +472,8 @@ def download_specific_release(self, AlbumID, title, size, url, provider, kind, * if data and bestqual: myDB = db.DBConnection() - album = myDB.action('SELECT * from albums WHERE AlbumID=?', [AlbumID]).fetchone() + album = myDB.action('SELECT * from albums WHERE AlbumID=%s', [AlbumID]).fetchone() + myDB.commit() searcher.send_to_downloader(data, bestqual, album) return json.dumps({'result': 'success'}) else: @@ -485,25 +493,27 @@ def deleteAlbum(self, AlbumID, ArtistID=None): logger.info(u"Deleting all traces of album: " + AlbumID) myDB = db.DBConnection() - myDB.action('DELETE from have WHERE Matched=?', [AlbumID]) - album = myDB.action('SELECT ArtistID, ArtistName, AlbumTitle from albums where AlbumID=?', + myDB.action('DELETE from have WHERE Matched=%s', [AlbumID]) + album = myDB.action('SELECT ArtistID, ArtistName, AlbumTitle from albums where AlbumID=%s', [AlbumID]).fetchone() if album: ArtistID = album['ArtistID'] - myDB.action('DELETE from have WHERE ArtistName=? AND AlbumTitle=?', + myDB.action('DELETE from have WHERE ArtistName=%s AND AlbumTitle=%s', [album['ArtistName'], album['AlbumTitle']]) - myDB.action('DELETE from albums WHERE AlbumID=?', [AlbumID]) - myDB.action('DELETE from tracks WHERE AlbumID=?', [AlbumID]) - myDB.action('DELETE from allalbums WHERE AlbumID=?', [AlbumID]) - myDB.action('DELETE from alltracks WHERE AlbumID=?', [AlbumID]) - myDB.action('DELETE from releases WHERE ReleaseGroupID=?', [AlbumID]) - myDB.action('DELETE from descriptions WHERE ReleaseGroupID=?', [AlbumID]) + myDB.action('DELETE from albums WHERE AlbumID=%s', [AlbumID]) + myDB.action('DELETE from tracks WHERE AlbumID=%s', [AlbumID]) + myDB.action('DELETE from allalbums WHERE AlbumID=%s', [AlbumID]) + myDB.action('DELETE from alltracks WHERE AlbumID=%s', [AlbumID]) + myDB.action('DELETE from releases WHERE ReleaseGroupID=%s', [AlbumID]) + myDB.action('DELETE from descriptions WHERE ReleaseGroupID=%s', [AlbumID]) from headphones import cache c = cache.Cache() c.remove_from_cache(AlbumID=AlbumID) + myDB.commit() + if ArtistID: raise cherrypy.HTTPRedirect("artistPage?ArtistID=%s" % ArtistID) else: @@ -532,8 +542,9 @@ def editSearchTerm(self, AlbumID, SearchTerm): def upcoming(self): myDB = db.DBConnection() upcoming = myDB.select( - "SELECT * from albums WHERE ReleaseDate > date('now') order by ReleaseDate ASC") - wanted = myDB.select("SELECT * from albums WHERE Status='Wanted'") + "SELECT * from albums WHERE to_date(ReleaseDate,'YYYY-MM-DD') > date('now') order by ReleaseDate ASC") + wanted = myDB.select("SELECT * from albums WHERE Status=%s", ['Wanted']) + myDB.commit() return serve_template(templatename="upcoming.html", title="Upcoming", upcoming=upcoming, wanted=wanted) @@ -541,12 +552,14 @@ def upcoming(self): def manage(self): myDB = db.DBConnection() emptyArtists = myDB.select("SELECT * FROM artists WHERE LatestAlbum IS NULL") + myDB.commit() return serve_template(templatename="manage.html", title="Manage", emptyArtists=emptyArtists) @cherrypy.expose def manageArtists(self): myDB = db.DBConnection() - artists = myDB.select('SELECT * from artists order by ArtistSortName COLLATE NOCASE') + artists = myDB.select('SELECT * from artists order by lower(ArtistSortName)') + myDB.commit() return serve_template(templatename="manageartists.html", title="Manage Artists", artists=artists) @@ -554,11 +567,12 @@ def manageArtists(self): def manageAlbums(self, Status=None): myDB = db.DBConnection() if Status == "Upcoming": - albums = myDB.select("SELECT * from albums WHERE ReleaseDate > date('now')") + albums = myDB.select("SELECT * from albums WHERE to_date(ReleaseDate,'YYYY-MM-DD') > date('now')") elif Status: - albums = myDB.select('SELECT * from albums WHERE Status=?', [Status]) + albums = myDB.select('SELECT * from albums WHERE Status=%s', [Status]) else: albums = myDB.select('SELECT * from albums') + myDB.commit() return serve_template(templatename="managealbums.html", title="Manage Albums", albums=albums) @@ -566,6 +580,7 @@ def manageAlbums(self, Status=None): def manageNew(self): myDB = db.DBConnection() newartists = myDB.select('SELECT * from newartists') + myDB.commit() return serve_template(templatename="managenew.html", title="Manage New Artists", newartists=newartists) @@ -575,7 +590,7 @@ def manageUnmatched(self): have_album_dictionary = [] headphones_album_dictionary = [] have_albums = myDB.select( - 'SELECT ArtistName, AlbumTitle, TrackTitle, CleanName from have WHERE Matched = "Failed" GROUP BY AlbumTitle ORDER BY ArtistName') + 'SELECT ArtistName, AlbumTitle, TrackTitle, CleanName from have WHERE Matched = %s GROUP BY AlbumTitle, TrackTitle, CleanName, ArtistName ORDER BY ArtistName', ('Failed',)) for albums in have_albums: # Have to skip over manually matched tracks if albums['ArtistName'] and albums['AlbumTitle'] and albums['TrackTitle']: @@ -604,6 +619,7 @@ def manageUnmatched(self): clean_name(d['ArtistName']).lower(), clean_name(d['AlbumTitle']).lower()) not in check] + myDB.commit() return serve_template(templatename="manageunmatched.html", title="Manage Unmatched Items", unmatchedalbums=unmatchedalbums) @@ -615,22 +631,22 @@ def markUnmatched(self, action=None, existing_artist=None, existing_album=None, if action == "ignoreArtist": artist = existing_artist myDB.action( - 'UPDATE have SET Matched="Ignored" WHERE ArtistName=? AND Matched = "Failed"', - [artist]) + 'UPDATE have SET Matched=%s WHERE ArtistName=%s AND Matched = %s', + ['Ignored' ,artist, 'Failed']) elif action == "ignoreAlbum": artist = existing_artist album = existing_album myDB.action( - 'UPDATE have SET Matched="Ignored" WHERE ArtistName=? AND AlbumTitle=? AND Matched = "Failed"', - (artist, album)) + 'UPDATE have SET Matched=%s WHERE ArtistName=%s AND AlbumTitle=%s AND Matched = %s', + ('Ignored', artist, album, 'Failed')) elif action == "matchArtist": existing_artist_clean = helpers.clean_name(existing_artist).lower() new_artist_clean = helpers.clean_name(new_artist).lower() if new_artist_clean != existing_artist_clean: have_tracks = myDB.action( - 'SELECT Matched, CleanName, Location, BitRate, Format FROM have WHERE ArtistName=?', + 'SELECT Matched, CleanName, Location, BitRate, Format FROM have WHERE ArtistName=%s', [existing_artist]) update_count = 0 for entry in have_tracks: @@ -639,25 +655,28 @@ def markUnmatched(self, action=None, existing_artist=None, existing_album=None, new_clean_filename = old_clean_filename.replace(existing_artist_clean, new_artist_clean, 1) myDB.action( - 'UPDATE have SET CleanName=? WHERE ArtistName=? AND CleanName=?', + 'UPDATE have SET CleanName=%s WHERE ArtistName=%s AND CleanName=%s', [new_clean_filename, existing_artist, old_clean_filename]) - controlValueDict = {"CleanName": new_clean_filename} newValueDict = {"Location": entry['Location'], "BitRate": entry['BitRate'], "Format": entry['Format'] } # Attempt to match tracks with new CleanName match_alltracks = myDB.action( - 'SELECT CleanName from alltracks WHERE CleanName=?', + 'SELECT CleanName, ReleaseID, TrackID from alltracks WHERE CleanName=%s', [new_clean_filename]).fetchone() if match_alltracks: + controlValueDict = {"TrackID": match_alltracks['TrackID'], + "ReleaseID": match_alltracks['ReleaseID']} myDB.upsert("alltracks", newValueDict, controlValueDict) match_tracks = myDB.action( - 'SELECT CleanName, AlbumID from tracks WHERE CleanName=?', + 'SELECT CleanName, AlbumID from tracks WHERE CleanName=%s', [new_clean_filename]).fetchone() if match_tracks: + controlValueDict = {"TrackID": match_tracks['TrackID'], + "AlbumID": match_tracks['AlbumID']} myDB.upsert("tracks", newValueDict, controlValueDict) - myDB.action('UPDATE have SET Matched="Manual" WHERE CleanName=?', + myDB.action('UPDATE have SET Matched="Manual" WHERE CleanName=%s', [new_clean_filename]) update_count += 1 # This was throwing errors and I don't know why, but it seems to be working fine. @@ -679,7 +698,7 @@ def markUnmatched(self, action=None, existing_artist=None, existing_album=None, new_clean_string = new_artist_clean + " " + new_album_clean if existing_clean_string != new_clean_string: have_tracks = myDB.action( - 'SELECT Matched, CleanName, Location, BitRate, Format FROM have WHERE ArtistName=? AND AlbumTitle=?', + 'SELECT Matched, CleanName, Location, BitRate, Format FROM have WHERE ArtistName=%s AND AlbumTitle=%s', (existing_artist, existing_album)) update_count = 0 for entry in have_tracks: @@ -688,26 +707,29 @@ def markUnmatched(self, action=None, existing_artist=None, existing_album=None, new_clean_filename = old_clean_filename.replace(existing_clean_string, new_clean_string, 1) myDB.action( - 'UPDATE have SET CleanName=? WHERE ArtistName=? AND AlbumTitle=? AND CleanName=?', + 'UPDATE have SET CleanName=%s WHERE ArtistName=%s AND AlbumTitle=%s AND CleanName=%s', [new_clean_filename, existing_artist, existing_album, old_clean_filename]) - controlValueDict = {"CleanName": new_clean_filename} newValueDict = {"Location": entry['Location'], "BitRate": entry['BitRate'], "Format": entry['Format'] } # Attempt to match tracks with new CleanName match_alltracks = myDB.action( - 'SELECT CleanName from alltracks WHERE CleanName=?', + 'SELECT CleanName, TrackID, ReleaseID from alltracks WHERE CleanName=%s', [new_clean_filename]).fetchone() if match_alltracks: + controlValueDict = {"TrackID": match_alltracks["TrackID"], + "ReleaseID": match_alltracks["ReleaseID"],} myDB.upsert("alltracks", newValueDict, controlValueDict) match_tracks = myDB.action( - 'SELECT CleanName, AlbumID from tracks WHERE CleanName=?', + 'SELECT CleanName, TrackID, AlbumID from tracks WHERE CleanName=%s', [new_clean_filename]).fetchone() if match_tracks: + controlValueDict = {"TrackID": match_tracks["TrackID"], + "AlbumID": match_tracks["AlbumID"],} myDB.upsert("tracks", newValueDict, controlValueDict) - myDB.action('UPDATE have SET Matched="Manual" WHERE CleanName=?', + myDB.action('UPDATE have SET Matched="Manual" WHERE CleanName=%s', [new_clean_filename]) album_id = match_tracks['AlbumID'] update_count += 1 @@ -722,6 +744,7 @@ def markUnmatched(self, action=None, existing_artist=None, existing_album=None, logger.info( "Artist %s / Album %s already named appropriately; nothing to modify" % ( existing_artist, existing_album)) + myDB.commit() @cherrypy.expose def manageManual(self): @@ -745,6 +768,7 @@ def manageManual(self): manual_albums.append(manual_dict) manual_albums_sorted = sorted(manual_albums, key=itemgetter('ArtistName', 'AlbumTitle')) + myDB.commit() return serve_template(templatename="managemanual.html", title="Manage Manual Items", manualalbums=manual_albums_sorted) @@ -753,22 +777,22 @@ def markManual(self, action=None, existing_artist=None, existing_album=None): myDB = db.DBConnection() if action == "unignoreArtist": artist = existing_artist - myDB.action('UPDATE have SET Matched="Failed" WHERE ArtistName=? AND Matched="Ignored"', - [artist]) + myDB.action('UPDATE have SET Matched=%s WHERE ArtistName=%s AND Matched=%s', + ['Failed', artist, 'Ignored']) logger.info("Artist: %s successfully restored to unmatched list" % artist) elif action == "unignoreAlbum": artist = existing_artist album = existing_album myDB.action( - 'UPDATE have SET Matched="Failed" WHERE ArtistName=? AND AlbumTitle=? AND Matched="Ignored"', - (artist, album)) + 'UPDATE have SET Matched=%s WHERE ArtistName=%s AND AlbumTitle=%s AND Matched=%s', + ('Failed', artist, album, 'Ignored')) logger.info("Album: %s successfully restored to unmatched list" % album) elif action == "unmatchArtist": artist = existing_artist update_clean = myDB.select( - 'SELECT ArtistName, AlbumTitle, TrackTitle, CleanName, Matched from have WHERE ArtistName=?', + 'SELECT ArtistName, AlbumTitle, TrackTitle, CleanName, Matched from have WHERE ArtistName=%s', [artist]) update_count = 0 for tracks in update_clean: @@ -779,13 +803,13 @@ def markManual(self, action=None, existing_artist=None, existing_album=None): track_title = tracks['TrackTitle'] if tracks['CleanName'] != original_clean: myDB.action( - 'UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE CleanName=?', + 'UPDATE tracks SET Location=%s, BitRate=%s, Format=%s WHERE CleanName=%s', [None, None, None, tracks['CleanName']]) myDB.action( - 'UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE CleanName=?', + 'UPDATE alltracks SET Location=%s, BitRate=%s, Format=%s WHERE CleanName=%s', [None, None, None, tracks['CleanName']]) myDB.action( - 'UPDATE have SET CleanName=?, Matched="Failed" WHERE ArtistName=? AND AlbumTitle=? AND TrackTitle=?', + 'UPDATE have SET CleanName=%s, Matched="Failed" WHERE ArtistName=%s AND AlbumTitle=%s AND TrackTitle=%s', (original_clean, artist, album, track_title)) update_count += 1 if update_count > 0: @@ -796,7 +820,7 @@ def markManual(self, action=None, existing_artist=None, existing_album=None): artist = existing_artist album = existing_album update_clean = myDB.select( - 'SELECT ArtistName, AlbumTitle, TrackTitle, CleanName, Matched from have WHERE ArtistName=? AND AlbumTitle=?', + 'SELECT ArtistName, AlbumTitle, TrackTitle, CleanName, Matched from have WHERE ArtistName=%s AND AlbumTitle=%s', (artist, album)) update_count = 0 for tracks in update_clean: @@ -805,23 +829,24 @@ def markManual(self, action=None, existing_artist=None, existing_album=None): 'TrackTitle']).lower() track_title = tracks['TrackTitle'] if tracks['CleanName'] != original_clean: - album_id_check = myDB.action('SELECT AlbumID from tracks WHERE CleanName=?', + album_id_check = myDB.action('SELECT AlbumID from tracks WHERE CleanName=%s', [tracks['CleanName']]).fetchone() if album_id_check: - album_id = album_id_check[0] + album_id = album_id_check['AlbumID'] myDB.action( - 'UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE CleanName=?', + 'UPDATE tracks SET Location=%s, BitRate=%s, Format=%s WHERE CleanName=%s', [None, None, None, tracks['CleanName']]) myDB.action( - 'UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE CleanName=?', + 'UPDATE alltracks SET Location=%s, BitRate=%s, Format=%s WHERE CleanName=%s', [None, None, None, tracks['CleanName']]) myDB.action( - 'UPDATE have SET CleanName=?, Matched="Failed" WHERE ArtistName=? AND AlbumTitle=? AND TrackTitle=?', - (original_clean, artist, album, track_title)) + 'UPDATE have SET CleanName=%s, Matched=%s WHERE ArtistName=%s AND AlbumTitle=%s AND TrackTitle=%s', + (original_clean, 'Failed', artist, album, track_title)) update_count += 1 if update_count > 0: librarysync.update_album_status(album_id) logger.info("Album: %s successfully restored to unmatched list" % album) + myDB.commit() @cherrypy.expose def markArtists(self, action=None, **args): @@ -843,6 +868,7 @@ def markArtists(self, action=None, **args): if len(artistsToAdd) > 0: logger.debug("Refreshing artists: %s" % artistsToAdd) threading.Thread(target=importer.addArtistIDListToDB, args=[artistsToAdd]).start() + myDB.commit() raise cherrypy.HTTPRedirect("home") @cherrypy.expose @@ -918,7 +944,8 @@ def checkGithub(self): def history(self): myDB = db.DBConnection() history = myDB.select( - '''SELECT AlbumID, Title, Size, URL, DateAdded, Status, Kind, ifnull(FolderName, '?') FolderName FROM snatched WHERE Status NOT LIKE "Seed%" ORDER BY DateAdded DESC''') + '''SELECT AlbumID, Title, Size, URL, DateAdded, Status, Kind, quote_nullable(FolderName) FolderName FROM snatched WHERE Status NOT LIKE %s ORDER BY DateAdded DESC''', ['Seed%']) + myDB.commit() return serve_template(templatename="history.html", title="History", history=history) @cherrypy.expose @@ -980,22 +1007,29 @@ def getArtists_json(self, iDisplayStart=0, iDisplayLength=100, sSearch="", iSort sortcolumn = 'ArtistSortName' sortbyhavepercent = False + releasedate_type = "to_date(ReleaseDate,'YYYY-MM-DD')" if iSortCol_0 == '2': sortcolumn = 'Status' elif iSortCol_0 == '3': - sortcolumn = 'ReleaseDate' + sortcolumn = releasedate_type elif iSortCol_0 == '4': sortbyhavepercent = True + if sortcolumn == releasedate_type: + searchstr = "%s" + else: + searchstr = "lower(%s)" + if sSearch == "": - query = 'SELECT * from artists order by %s COLLATE NOCASE %s' % (sortcolumn, sSortDir_0) + query = ('SELECT * from artists order by ' + searchstr + ' %s') % (sortcolumn, sSortDir_0) filtered = myDB.select(query) totalcount = len(filtered) else: - query = 'SELECT * from artists WHERE ArtistSortName LIKE "%' + sSearch + '%" OR LatestAlbum LIKE "%' + sSearch + '%"' + 'ORDER BY %s COLLATE NOCASE %s' % ( + sTerm = '%' + sSearch + '%' + query = ('SELECT * from artists WHERE ArtistSortName LIKE %%s OR LatestAlbum LIKE %%s ORDER BY ' + searchstr + ' %s') % ( sortcolumn, sSortDir_0) - filtered = myDB.select(query) - totalcount = myDB.select('SELECT COUNT(*) from artists')[0][0] + filtered = myDB.select(query, [sTerm, sTerm]) + totalcount = myDB.select('SELECT COUNT(*) as c from artists')[0]['c'] if sortbyhavepercent: filtered.sort(key=lambda x: ( @@ -1004,7 +1038,7 @@ def getArtists_json(self, iDisplayStart=0, iDisplayLength=100, sSearch="", iSort # can't figure out how to change the datatables default sorting order when its using an ajax datasource so ill # just reverse it here and the first click on the "Latest Album" header will sort by descending release date - if sortcolumn == 'ReleaseDate': + if sortcolumn == releasedate_type: filtered.reverse() artists = filtered[iDisplayStart:(iDisplayStart + iDisplayLength)] @@ -1037,6 +1071,7 @@ def getArtists_json(self, iDisplayStart=0, iDisplayLength=100, sSearch="", iSort rows.append(row) + myDB.commit() dict = {'iTotalDisplayRecords': len(filtered), 'iTotalRecords': totalcount, 'aaData': rows, @@ -1050,10 +1085,11 @@ def getAlbumsByArtist_json(self, artist=None): myDB = db.DBConnection() album_json = {} counter = 0 - album_list = myDB.select("SELECT AlbumTitle from albums WHERE ArtistName=?", [artist]) + album_list = myDB.select("SELECT AlbumTitle from albums WHERE ArtistName=%s", [artist]) for album in album_list: album_json[counter] = album['AlbumTitle'] counter += 1 + myDB.commit() json_albums = json.dumps(album_json) cherrypy.response.headers['Content-type'] = 'application/json' @@ -1062,7 +1098,8 @@ def getAlbumsByArtist_json(self, artist=None): @cherrypy.expose def getArtistjson(self, ArtistID, **kwargs): myDB = db.DBConnection() - artist = myDB.action('SELECT * FROM artists WHERE ArtistID=?', [ArtistID]).fetchone() + artist = myDB.action('SELECT * FROM artists WHERE ArtistID=%s', [ArtistID]).fetchone() + myDB.commit() artist_json = json.dumps({ 'ArtistName': artist['ArtistName'], 'Status': artist['Status'] @@ -1072,7 +1109,8 @@ def getArtistjson(self, ArtistID, **kwargs): @cherrypy.expose def getAlbumjson(self, AlbumID, **kwargs): myDB = db.DBConnection() - album = myDB.action('SELECT * from albums WHERE AlbumID=?', [AlbumID]).fetchone() + album = myDB.action('SELECT * from albums WHERE AlbumID=%s', [AlbumID]).fetchone() + myDB.commit() album_json = json.dumps({ 'AlbumTitle': album['AlbumTitle'], 'ArtistName': album['ArtistName'], @@ -1086,15 +1124,16 @@ def clearhistory(self, type=None, date_added=None, title=None): if type: if type == 'all': logger.info(u"Clearing all history") - myDB.action('DELETE from snatched WHERE Status NOT LIKE "Seed%"') + myDB.action('DELETE from snatched WHERE Status NOT LIKE %s', ['Seed%']) else: logger.info(u"Clearing history where status is %s" % type) - myDB.action('DELETE from snatched WHERE Status=?', [type]) + myDB.action('DELETE from snatched WHERE Status=%s', [type]) else: logger.info(u"Deleting '%s' from history" % title) myDB.action( - 'DELETE from snatched WHERE Status NOT LIKE "Seed%" AND Title=? AND DateAdded=?', - [title, date_added]) + 'DELETE from snatched WHERE Status NOT LIKE %s AND Title=%s AND DateAdded=%s', + ['Seed%', title, date_added]) + myDB.commit() raise cherrypy.HTTPRedirect("history") @cherrypy.expose @@ -1116,12 +1155,14 @@ def forceScan(self, keepmatched=None): myDB.action('UPDATE artists SET HaveTracks=NULL') logger.info('Reset track counts for all artists') myDB.action( - 'UPDATE albums SET Status="Skipped" WHERE Status="Skipped" OR Status="Downloaded"') + 'UPDATE albums SET Status=%s WHERE Status=%s OR Status=%s', + ['Skipped', 'Skipped', 'Downloaded']) logger.info('Marking all unwanted albums as Skipped') try: threading.Thread(target=librarysync.libraryScan).start() except Exception as e: logger.error('Unable to complete the scan: %s' % e) + myDB.commit() raise cherrypy.HTTPRedirect("home") @cherrypy.expose @@ -1583,6 +1624,7 @@ def update(self): def extras(self): myDB = db.DBConnection() cloudlist = myDB.select('SELECT * from lastfmcloud') + myDB.commit() return serve_template(templatename="extras.html", title="Extras", cloudlist=cloudlist) @cherrypy.expose From 8832f93bd896ab15bfd453c3924784902362498f Mon Sep 17 00:00:00 2001 From: Bryon Roche Date: Thu, 14 Jul 2016 06:35:04 -0700 Subject: [PATCH 03/14] Postgres support part 3: template usage Also fix pythony sqlite-specific bits here. --- data/interfaces/default/album.html | 12 ++++++------ data/interfaces/default/artist.html | 10 +++++----- data/interfaces/default/managealbums.html | 10 +++++----- data/interfaces/default/manageunmatched.html | 2 +- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/data/interfaces/default/album.html b/data/interfaces/default/album.html index 132237431..1a8af92f4 100644 --- a/data/interfaces/default/album.html +++ b/data/interfaces/default/album.html @@ -26,7 +26,7 @@