fix: add DB migration & endpoint to download missing TMDB People (#1203)

Added a new database migration and endpoint to download missing TMDB people in case the database gets into a bad state for whatever reason (though most likely because it failed to completely download one or more TMDB shows/movies).

Fixes #1202

Author: fearnlj01

Shoko.Server/API/v3/Controllers/ActionController.cs

@@ -192,6 +192,16 @@ public ActionResult UpdateAllTmdbShows()
         return Ok();
     }
 
+    /// <summary>
+    /// Download any missing TMDB People.
+    /// </summary>
+    [HttpGet("DownloadMissingTmdbPeople")]
+    public ActionResult DownloadMissingTmdbPeople()
+    {
+        Task.Factory.StartNew(() => _tmdbService.RepairMissingPeople());
+        return Ok();
+    }
+
     /// <summary>
     /// Purge all unused TMDB Shows that are not linked to any AniDB anime.
     /// </summary>

Shoko.Server/Databases/DatabaseFixes.cs

@@ -837,4 +837,10 @@ public static void ClearQuartzQueue()
         var queueHandler = Utils.ServiceContainer.GetRequiredService<QueueHandler>();
         queueHandler.Clear().ConfigureAwait(false).GetAwaiter().GetResult();
     }
+
+    public static void RepairMissingTMDBPersons()
+    {
+        var service = Utils.ServiceContainer.GetRequiredService<TmdbMetadataService>();
+        service.RepairMissingPeople().ConfigureAwait(false).GetAwaiter().GetResult();
+    }
 }

Shoko.Server/Databases/MySQL.cs

@@ -27,7 +27,7 @@ namespace Shoko.Server.Databases;
 public class MySQL : BaseDatabase<MySqlConnection>
 {
     public override string Name { get; } = "MySQL";
-    public override int RequiredVersion { get; } = 139;
+    public override int RequiredVersion { get; } = 140;
 
     private List<DatabaseCommand> createVersionTable = new()
     {
@@ -849,6 +849,7 @@ public class MySQL : BaseDatabase<MySqlConnection>
         new(139, 10, "ALTER TABLE Trakt_Show ADD COLUMN TmdbShowID INT NULL;"),
         new(139, 11, DatabaseFixes.CleanupAfterRemovingTvDB),
         new(139, 12, DatabaseFixes.ClearQuartzQueue),
+        new(140, 1, DatabaseFixes.RepairMissingTMDBPersons)
     };
 
     private DatabaseCommand linuxTableVersionsFix = new("RENAME TABLE versions TO Versions;");

Shoko.Server/Databases/SQLServer.cs

@@ -28,7 +28,7 @@ namespace Shoko.Server.Databases;
 public class SQLServer : BaseDatabase<SqlConnection>
 {
     public override string Name { get; } = "SQLServer";
-    public override int RequiredVersion { get; } = 131;
+    public override int RequiredVersion { get; } = 132;
 
     public override void BackupDatabase(string fullfilename)
     {
@@ -779,6 +779,7 @@ public override bool HasVersionsTable()
         new DatabaseCommand(131, 10, "ALTER TABLE Trakt_Show ADD TmdbShowID INT NULL;"),
         new DatabaseCommand(131, 11, DatabaseFixes.CleanupAfterRemovingTvDB),
         new DatabaseCommand(131, 12, DatabaseFixes.ClearQuartzQueue),
+        new DatabaseCommand(132, 1, DatabaseFixes.RepairMissingTMDBPersons)
     };
 
     private static void AlterImdbMovieIDType()

Shoko.Server/Databases/SQLite.cs

@@ -28,7 +28,7 @@ public class SQLite : BaseDatabase<SqliteConnection>
 {
     public override string Name => "SQLite";
 
-    public override int RequiredVersion => 123;
+    public override int RequiredVersion => 124;
 
     public override void BackupDatabase(string fullfilename)
     {
@@ -774,6 +774,7 @@ public override void CreateDatabase()
         new(123, 10, "ALTER TABLE Trakt_Show ADD COLUMN TmdbShowID INTEGER NULL;"),
         new(123, 11, DatabaseFixes.CleanupAfterRemovingTvDB),
         new(123, 12, DatabaseFixes.ClearQuartzQueue),
+        new(124, 1, DatabaseFixes.RepairMissingTMDBPersons)
     };
 
     private static Tuple<bool, string> MigrateRenamers(object connection)

Shoko.Server/Providers/TMDB/TmdbMetadataService.cs

@@ -2031,6 +2031,39 @@ private void PurgeCompany(int companyId, bool removeImageFiles = true)
 
     #region People
 
+    public async Task RepairMissingPeople()
+    {
+        var missingIds = new HashSet<int>();
+        var updateCount = 0;
+        var skippedCount = 0;
+        
+        var peopleIds = _tmdbPeople.GetAll().Select(person => person.TmdbPersonID).ToHashSet();
+
+        foreach (var person in _tmdbEpisodeCast.GetAll())
+            if (!peopleIds.Contains(person.TmdbPersonID)) missingIds.Add(person.TmdbPersonID);
+        foreach (var person in _tmdbEpisodeCrew.GetAll())
+            if (!peopleIds.Contains(person.TmdbPersonID)) missingIds.Add(person.TmdbPersonID);
+        
+        foreach (var person in _tmdbMovieCast.GetAll())
+            if (!peopleIds.Contains(person.TmdbPersonID)) missingIds.Add(person.TmdbPersonID);
+        foreach (var person in _tmdbMovieCrew.GetAll())
+            if (!peopleIds.Contains(person.TmdbPersonID)) missingIds.Add(person.TmdbPersonID);
+
+        _logger.LogDebug("Found {@Count} unique missing TMDB People for Episode & Movie staff", missingIds.Count);
+        
+        foreach (var personId in missingIds)
+        {
+            var (_, updated) = await UpdatePerson(personId, forceRefresh: true);
+            if (updated) 
+                updateCount++;
+            else
+                skippedCount++;
+        }
+        
+        _logger.LogInformation("Updated missing TMDB People: Found/Updated/Skipped {@Found}/{@Updated}/{@Skipped}",
+            missingIds.Count, updateCount, skippedCount);
+    }
+    
     public async Task<(bool added, bool updated)> UpdatePerson(int personId, bool forceRefresh = false, bool downloadImages = false)
     {
         using (await GetLockForEntity(ForeignEntityType.Person, personId, "metadata & images", "Update").ConfigureAwait(false))
@@ -2048,6 +2081,12 @@ private void PurgeCompany(int companyId, bool removeImageFiles = true)
                 methods |= PersonMethods.Images;
             var newlyAdded = tmdbPerson.TMDB_PersonID is 0;
             var person = await UseClient(c => c.GetPersonAsync(personId, methods), $"Get person {personId}");
+            if (person is null)
+            {
+                _logger.LogDebug("Unable to update staff; Scheduling refresh of related links. (Person={PersonId})", personId);
+                await ScheduleUpdateMoviesAndShowsByPerson(personId);
+                return (false, false);
+            }
             var updated = tmdbPerson.Populate(person);
             if (updated)
             {
@@ -2062,6 +2101,28 @@ private void PurgeCompany(int companyId, bool removeImageFiles = true)
         }
     }
 
+    private async Task ScheduleUpdateMoviesAndShowsByPerson(int tmdbPersonId)
+    {
+        var showIds = new HashSet<int>();
+        var movieIds = new HashSet<int>();
+
+        foreach (var staff in _tmdbEpisodeCast.GetByTmdbPersonID(tmdbPersonId))
+            showIds.Add(staff.TmdbShowID);
+        foreach (var staff in _tmdbEpisodeCrew.GetByTmdbPersonID(tmdbPersonId))
+            showIds.Add(staff.TmdbShowID);
+
+        foreach (var staff in _tmdbMovieCast.GetByTmdbPersonID(tmdbPersonId))
+            movieIds.Add(staff.TmdbMovieID);
+        foreach (var staff in _tmdbMovieCrew.GetByTmdbPersonID(tmdbPersonId))
+            movieIds.Add(staff.TmdbMovieID);
+
+        foreach (var showId in showIds)
+            await ScheduleUpdateOfShow(showId, downloadCrewAndCast: true, forceRefresh: true);
+
+        foreach (var movieId in movieIds)
+            await ScheduleUpdateOfMovie(movieId, downloadCrewAndCast: true, forceRefresh: true);
+    }
+
     private async Task DownloadPersonImages(int personId, ProfileImages images, bool forceDownload = false)
     {
         var settings = _settingsProvider.GetSettings();