feat: improve converting subtitles to txt, remove duplicate lines

Author: christian-fei

lib/subtitles-summary.js

@@ -29,13 +29,16 @@ export async function summarizeVideo(id, repo, callback = () => {}) {
   })
 }
 export function cleanTranscript (transcript) {
-  return transcript
-  .split('\n')
-  .map(line => {
-    if (/^\d+/.test(line)) return
-    line = line.replace(/\d{2}:\d{2}:\d{2},\d{3} --> \d{2}:\d{2}:\d{2},\d{3}/g, '')
-    return line.replace(/<[^>]+>/g, '').trim()
-  }).filter(Boolean).join('\n').trim()
+  const lines = []
+  for (let line of transcript.split('\n')) {
+    line = line.trim()
+    if (/^\d+/.test(line)) continue
+    if (line.match(/^\d+:\d{2}:\d{2},\d{3} --> \d{2}:\d{2}:\d{2},\d{3}/)) continue
+    if (lines.length === 0 || !lines[lines.length - 1].startsWith(line)) {
+      lines.push(line)
+    }
+  }
+  return lines.filter(Boolean).join('\n').trim()
 }
 
 

test/subtitles.summary.test.js

@@ -5,21 +5,38 @@ import { cleanTranscript } from '../lib/subtitles-summary.js'
 test('cleans transcript', () => {
   const transcript = `
 1
-00:00:01,240 --> 00:00:06,879
-<font color="white" size=".72c">when we are about to do something</font>
+00:00:01,240 --> 00:00:04,630
+
+Open ai writes to the
 
 2
-00:00:04,160 --> 00:00:10,519
-<font color="white" size=".72c">new</font>
+00:00:04,630 --> 00:00:04,640
+Open ai writes to the
+ 
 
 3
-00:00:06,879 --> 00:00:14,080
-<font color="white" size=".72c">Eh after finishing</font>
+00:00:04,640 --> 00:00:09,110
+Open ai writes to the
+US administration their position Well it is
+
+4
+00:00:09,110 --> 00:00:09,120
+US administration their position Well it is
+ 
+
+5
+00:00:09,120 --> 00:00:12,509
+US administration their position Well it is
+necessary that it be free from copyright from
+
+6
+00:00:12,509 --> 00:00:12,519
+necessary that it be free from copyright from
   `.trim()
   const expectedCleanTranscript = `
-when we are about to do something
-new
-Eh after finishing
+Open ai writes to the
+US administration their position Well it is
+necessary that it be free from copyright from
   `.trim()
 
   assert.strictEqual(cleanTranscript(transcript), expectedCleanTranscript)