Skip to content

Commit

Permalink
Youtube playlist parsing for better audio'd entries (#1861)
Browse files Browse the repository at this point in the history 
* Youtube playlist parsing for better audio'd entries

* Move better audio replacement into sproc

* lint

* Only apply better audio to main/audio types
  • Loading branch information
@Brainicism
Brainicism authored Feb 4, 2024
1 parent 21fa18f commit 99b39b8
Show file tree
Hide file tree
Showing 11 changed files with 60 additions and 57 deletions.
20 changes: 20 additions & 0 deletions sql/procedures/create_kmq_data_tables_procedure.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,23 @@ CREATE PROCEDURE CreateKmqDataTables(
IN maxRank INT
)
BEGIN
/* replace songs with better audio counterpart */
ALTER TABLE kpop_videos.app_kpop ADD COLUMN IF NOT EXISTS original_vlink VARCHAR(255);
DROP TEMPORARY TABLE IF EXISTS temp_tbl;
CREATE TEMPORARY TABLE temp_tbl
SELECT a.id as original_id, a.original_name as original_name, a.vlink as original_link, b.vlink as better_audio_link
FROM kpop_videos.app_kpop as a
LEFT JOIN kpop_videos.app_kpop as b ON a.id_better_audio = b.id
WHERE b.vlink is not null
AND a.vtype IN ('main', 'audio');

DELETE kpop_videos.app_kpop FROM kpop_videos.app_kpop
JOIN temp_tbl tt on kpop_videos.app_kpop.vlink = tt.better_audio_link
WHERE kpop_videos.app_kpop.vlink = tt.better_audio_link;

UPDATE kpop_videos.app_kpop JOIN temp_tbl tt on kpop_videos.app_kpop.id = tt.original_id
SET kpop_videos.app_kpop.vlink = tt.better_audio_link, kpop_videos.app_kpop.original_vlink = tt.original_link;

/* update available_songs table */
DROP TABLE IF EXISTS available_songs_temp;
CREATE TABLE available_songs_temp (
Expand All @@ -14,6 +31,7 @@ BEGIN
clean_song_name_ko VARCHAR(255) NOT NULL,
song_aliases VARCHAR(255) NOT NULL,
link VARCHAR(255) NOT NULL,
original_link VARCHAR(255),
artist_name_en VARCHAR(255) NOT NULL,
original_artist_name_en VARCHAR(255) NOT NULL,
artist_name_ko VARCHAR(255),
Expand Down Expand Up @@ -43,6 +61,7 @@ BEGIN
TRIM(SUBSTRING_INDEX(kpop_videos.app_kpop.kname, '(', 1)) AS clean_song_name_ko,
kpop_videos.app_kpop.alias AS song_aliases,
vlink AS link,
kpop_videos.app_kpop.original_vlink AS original_link,
TRIM(kpop_videos.app_kpop_group.name) AS artist_name_en,
TRIM(kpop_videos.app_kpop_group.original_name) AS original_artist_name_en,
TRIM(kpop_videos.app_kpop_group.kname) AS artist_name_ko,
Expand Down Expand Up @@ -81,6 +100,7 @@ BEGIN
TRIM(SUBSTRING_INDEX(kpop_videos.app_kpop.kname, '(', 1)) AS clean_song_name_ko,
kpop_videos.app_kpop.alias AS song_aliases,
vlink AS link,
null,
TRIM(kpop_videos.app_kpop_group.name) AS artist_name_en,
TRIM(kpop_videos.app_kpop_group.original_name) AS original_artist_name_en,
TRIM(kpop_videos.app_kpop_group.kname) AS artist_name_ko,
Expand Down
2 changes: 2 additions & 0 deletions src/helpers/management_utils.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -251,13 +251,15 @@ export async function reloadAliases(): Promise<void> {
for (const mapping of songAliasMapping) {
songAliases[mapping["link"]] = mapping["song_aliases"]
.split(";")
.map((x) => x.trim())
.filter((x: string) => x);
}

const artistAliases: { [artistName: string]: string[] } = {};
for (const mapping of artistAliasMapping) {
const aliases: Array<string> = mapping["artist_aliases"]
.split(";")
.map((x) => x.trim())
.filter((x: string) => x);

const previousNameEn = mapping["previous_name_en"];
Expand Down
23 changes: 18 additions & 5 deletions src/helpers/playlist_manager.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -363,18 +363,31 @@ export default class PlaylistManager {
matchedSongs = await dbContext.kmq
.selectFrom("available_songs")
.select(SongSelector.QueriedSongFields)
.where(
"link",
"in",
youtubePlaylistVideoIDs.map((x) => x.videoId),
.where((eb) =>
eb.or([
eb(
"link",
"in",
youtubePlaylistVideoIDs.map((x) => x.videoId),
),
eb(
"original_link",
"in",
youtubePlaylistVideoIDs.map((x) => x.videoId),
),
]),
)
.execute();

unmatchedSongs = youtubePlaylistVideoIDs
.filter(
(x) =>
!matchedSongs
.map((y) => y.youtubeLink)
.flatMap((y) =>
y.originalLink
? [y.originalLink, y.youtubeLink]
: [y.youtubeLink],
)
.includes(x.videoId),
)
.map((x) => `${x.title} (${x.videoId})`);
Expand Down
1 change: 1 addition & 0 deletions src/interfaces/queried_song.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ export default interface QueriedSong {
artistName: string;
hangulArtistName: string | null;
youtubeLink: string;
originalLink: string | null;
publishDate: Date;
members: AvailableGenders;
artistID: number;
Expand Down
40 changes: 0 additions & 40 deletions src/seed/seed_db.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,6 @@ import fs from "fs";
import path from "path";
import util from "util";
import type { DatabaseContext } from "../database_context";
import type { KpopVideosDB } from "../typings/kpop_videos_db";
import type { Kysely } from "kysely";

const exec = util.promisify(cp.exec);

Expand Down Expand Up @@ -87,39 +85,6 @@ export async function tableExists(
);
}

async function replaceBetterAudioSongs(
db: Kysely<KpopVideosDB>,
): Promise<void> {
const songsWithBetterAudioCounterpart = await db
.selectFrom("app_kpop as a")
.leftJoin("app_kpop as b", "a.id_better_audio", "b.id")
.select([
"a.id as original_id",
"a.original_name as original_name",
"b.vlink as better_audio_link",
])
.where("b.vlink", "is not", null)
.$narrowType<{ better_audio_link: string }>()
.execute();

for (const betterAudioSong of songsWithBetterAudioCounterpart) {
// remove 'better' audio entry to not consume b-side limit
await db
.deleteFrom("app_kpop")
.where("vlink", "=", betterAudioSong.better_audio_link)
.execute();

// replace main video with better audio entry
await db
.updateTable("app_kpop")
.where("id", "=", betterAudioSong.original_id)
.set({
vlink: betterAudioSong.better_audio_link,
})
.execute();
}
}

async function listTables(
db: DatabaseContext,
databaseName: string,
Expand Down Expand Up @@ -346,8 +311,6 @@ async function validateSqlDump(
`mysql -u ${process.env.DB_USER} -p${process.env.DB_PASS} -h ${process.env.DB_HOST} --port ${process.env.DB_PORT} kpop_videos_validation < ${mvSeedFilePath}`,
);

await replaceBetterAudioSongs(db.kpopVideosValidation);

logger.info("Validating MV song count");
const mvSongCount = (await db.kpopVideosValidation
.selectFrom("app_kpop")
Expand Down Expand Up @@ -530,9 +493,6 @@ async function seedDb(db: DatabaseContext, bootstrap: boolean): Promise<void> {
await sql`DROP TABLE IF EXISTS kpop_videos.old;`.execute(db.agnostic);
await sql`DROP DATABASE IF EXISTS kpop_videos_tmp;`.execute(db.agnostic);

logger.info("Substituting songs with better audio");
await replaceBetterAudioSongs(db.kpopVideos);

// override queries
logger.info("Performing data overrides");
const overrideQueries = await getOverrideQueries(db);
Expand Down
1 change: 1 addition & 0 deletions src/structures/song_selector.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ export default class SongSelector {
"available_songs.artist_name_en as artistName",
"available_songs.artist_name_ko as hangulArtistName",
"available_songs.link as youtubeLink",
"available_songs.original_link as originalLink",
"available_songs.publishedon as publishDate",
"available_songs.members",
"available_songs.id_artist as artistID",
Expand Down
1 change: 1 addition & 0 deletions src/test/ci/exp.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -445,6 +445,7 @@ describe("exp command", () => {
artistName: "x",
hangulArtistName: "x",
youtubeLink: "x",
originalLink: null,
publishDate: new Date(),
members: "female",
artistID: 1,
Expand Down
10 changes: 10 additions & 0 deletions src/test/ci/game_round.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ describe("game round", () => {
artistName: "Jisoo",
hangulArtistName: "지수",
youtubeLink: "abcde",
originalLink: null,
publishDate: new Date(),
members: "female",
artistID: 1,
Expand Down Expand Up @@ -60,6 +61,7 @@ describe("game round", () => {
artistName: "IU + Blackpink",
hangulArtistName: "아이유+블랙핑크",
youtubeLink: "abcde",
originalLink: null,
publishDate: new Date(),
members: "female",
artistID: 2,
Expand Down Expand Up @@ -93,6 +95,7 @@ describe("game round", () => {
artistName: "Yena (Choi Yena)",
hangulArtistName: "최예나 (나)",
youtubeLink: "abcde",
originalLink: null,
publishDate: new Date(),
members: "female",
artistID: 3,
Expand Down Expand Up @@ -126,6 +129,7 @@ describe("game round", () => {
artistName: " Blackpink + IU ",
hangulArtistName: " 블랙핑크+아이유 ",
youtubeLink: "abcde",
originalLink: null,
publishDate: new Date(),
members: "female",
artistID: 3,
Expand Down Expand Up @@ -171,6 +175,7 @@ describe("game round", () => {
artistName: "A really epic person",
hangulArtistName: "정말 서사시인",
youtubeLink: "abcde",
originalLink: null,
publishDate: new Date(),
members: "male",
artistID: 4,
Expand Down Expand Up @@ -211,6 +216,7 @@ describe("game round", () => {
artistName: "Person2",
hangulArtistName: "2인칭",
youtubeLink: "abcde",
originalLink: null,
publishDate: new Date(),
members: "female",
artistID: 4,
Expand Down Expand Up @@ -306,6 +312,7 @@ describe("game round", () => {
artistName: "5",
hangulArtistName: "6",
youtubeLink: "7",
originalLink: null,
publishDate: new Date(2015, 0),
members: "coed",
artistID: 4,
Expand Down Expand Up @@ -369,6 +376,7 @@ describe("game round", () => {
artistName: "artist",
hangulArtistName: "예술가",
youtubeLink: "a1b2c3",
originalLink: null,
publishDate: new Date(2015, 0),
members: "male",
artistID: 4,
Expand Down Expand Up @@ -661,6 +669,7 @@ describe("game round", () => {
artistName: "artist",
hangulArtistName: "예술가",
youtubeLink: "a1b2c3",
originalLink: null,
publishDate: new Date(2015),
members: "female",
artistID: 4,
Expand Down Expand Up @@ -715,6 +724,7 @@ describe("game round", () => {
artistName: "artist",
hangulArtistName: "예술가",
youtubeLink: "a1b2c3",
originalLink: null,
publishDate: new Date(2015, 0),
members: "male",
artistID: 4,
Expand Down
17 changes: 5 additions & 12 deletions src/test/ci/management_utils.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,19 +11,12 @@ describe("management utils", () => {

describe("song aliases", () => {
it("should add the song aliases", () => {
const videoID = "0rtV5esQT6I";
const expectedAliases = [
"Like OOH AHH",
"Like OOH-AHH",
"OOH AHH하게",
"우아하게",
];
const videoID = "lQaclKRINdA";
const expectedAliases = ["It Is War", "It's War"];

assert.ok(
setIntersection(
State.aliases.song[videoID],
expectedAliases,
).size === expectedAliases.length,
assert.deepStrictEqual(
State.aliases.song[videoID],
expectedAliases,
);
});
});
Expand Down
1 change: 1 addition & 0 deletions src/typings/kmq_db.d.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ export interface AvailableSongs {
link: string;
members: "coed" | "female" | "male";
original_artist_name_en: string;
original_link: Generated<string | null>;
previous_name_en: Generated<string | null>;
previous_name_ko: Generated<string | null>;
publishedon: Date;
Expand Down
1 change: 1 addition & 0 deletions src/typings/kpop_videos_db.d.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ export interface AppKpop {
likes: Generated<number>;
name: Generated<string>;
original_name: Generated<string>;
original_vlink: Generated<string | null>;
promotedcharted: number;
promotedviews_yc: Generated<number>;
promotedweeks: number;
Expand Down

0 comments on commit 99b39b8

Please sign in to comment.