Add CLOUDFLARE_AI_GATEWAY_URL environment variable and update generateAudio function in openai.service.ts

Author: nicobytes

Diff for: apps/api/src/bindings.ts

@@ -2,6 +2,7 @@ export type Bindings = {
   DB: D1Database;
   CLOUDFLARE_ACCOUNT_ID: string;
   CLOUDFLARE_API_TOKEN: string;
+  CLOUDFLARE_AI_GATEWAY_URL: string;
   OPENAI_API_KEY: string;
   AI: any;
   BUCKET: R2Bucket;

Diff for: apps/api/src/index.ts

@@ -9,8 +9,9 @@ import createSimulation from '@src/routes/createSimulation';
 import createQuestion from '@src/routes/createQuestion';
 import createFeedback from '@src/routes/createFeedback';
 import createTranscript from '@src/routes/createTranscript';
+import { App } from "@src/types";
 
-const app = new OpenAPIHono();
+const app = new OpenAPIHono<App>();
 app.use("*", cors());
 app.use('*', prettyJSON());
 app.notFound((c) => c.json({ message: 'Not Found', ok: false }, 404));
@@ -22,6 +23,51 @@ app.route('/api/v1/questions', createQuestion);
 app.route('/api/v1/feedback', createFeedback);
 app.route('/api/v1/transcript', createTranscript);
 
+/*
+import { Ai } from '@cloudflare/ai';
+
+app.post('/api/v1/demo1', async (c) => {
+  const ai = new Ai(c.env.AI);
+  const arrayBuffer = await c.req.arrayBuffer();
+  const audio = [...new Uint8Array(arrayBuffer)];
+  const { text } = await ai.run("@cf/openai/whisper", {audio });
+  return c.json({text});
+});
+
+app.post('/api/v1/demo2', async (c) => {
+  const ai = new Ai(c.env.AI);
+  const data = await c.req.formData();
+  const file = data.get('file') as unknown as File;
+  const arrayBuffer = await file.arrayBuffer();
+  const audio = [...new Uint8Array(arrayBuffer)];
+  const { text } = await ai.run("@cf/openai/whisper", { audio });
+  return c.json({text});
+});
+
+app.post('/api/v1/demo3', async (c) => {
+  const data = await c.req.formData();
+  const file = data.get('file') as unknown as File;
+  const text = await generateTranscription(file, c.env.OPENAI_API_KEY);
+  return c.json({text});
+});
+
+app.post('/api/v1/demo4', async (c) => {
+  const data = await c.req.formData();
+  const file = data.get('file') as unknown as File;
+  const arrayBuffer = await file.arrayBuffer();
+  const response = await fetch('https://gateway.ai.cloudflare.com/v1/b2bb1719bede14df8732870a3974b263/gateway/workers-ai/@cf/openai/whisper', {
+    method: 'POST',
+    headers: {
+      'Authorization': `Bearer ${c.env.CLOUDFLARE_API_TOKEN}`,
+      'Content-Type': 'application/octet-stream',
+    },
+    body: arrayBuffer,
+  });
+  const result = await response.json();
+  return c.json({result});
+});
+*/
+
 app.get("/", swaggerUI({ url: "/docs" }));
 app.doc("/docs", {
   info: {

Diff for: apps/api/src/routes/createQuestion.ts

@@ -45,7 +45,7 @@ app.openapi(route, async (c) => {
     cloudflareApiToken: c.env.CLOUDFLARE_API_TOKEN,
   }, c.env.DB);
 
-  const filename = await generateAudio(response, c.env.OPENAI_API_KEY, c.env.BUCKET);
+  const filename = await generateAudio(response, c.env.OPENAI_API_KEY, c.env.CLOUDFLARE_AI_GATEWAY_URL, c.env.BUCKET);
 
   return c.json({
     id: `${Date.now()}`,

Diff for: apps/api/src/routes/createTranscript.ts

@@ -1,6 +1,6 @@
 import { OpenAPIHono, createRoute } from "@hono/zod-openapi";
 import { MessageSchema } from '@src/dtos/message.dto';
-import { generateTranscription } from '@src/services/whisper.service';
+import { generateTranscription } from '@src/services/openai.service';
 import { App } from "@src/types";
 
 const app = new OpenAPIHono<App>();
@@ -23,9 +23,9 @@ const route = createRoute({
 });
 
 app.openapi(route, async (c) => {
-  const body = await c.req.parseBody();
-  const file = body.file as File;
-  const answer = await generateTranscription(file, c.env.AI);
+  const data = await c.req.formData();
+  const file = data.get('file') as unknown as File;
+  const answer = await generateTranscription(file, c.env.OPENAI_API_KEY, c.env.CLOUDFLARE_AI_GATEWAY_URL);
 
   return c.json({
     id: `${Date.now()}`,

Diff for: apps/api/src/services/openai.service.ts

@@ -1,9 +1,7 @@
 import OpenAI from "openai";
 
-export const generateTranscription = async (file: File, key: string) => {
-  const openai = new OpenAI({
-    apiKey: key,
-  });
+export const generateTranscription = async (file: File, apiKey: string, baseURL: string) => {
+  const openai = new OpenAI({ apiKey, baseURL: `${baseURL}/openai`});
   console.log('name', file.name);
   console.log('type', file.type);
   const transcription = await openai.audio.transcriptions.create({
@@ -14,10 +12,8 @@ export const generateTranscription = async (file: File, key: string) => {
   return transcription.text;
 }
 
-export const generateAudio = async (input: string, key: string, bucket: R2Bucket) => {
-  const openai = new OpenAI({
-    apiKey: key,
-  });
+export const generateAudio = async (input: string, apiKey: string, baseURL: string, bucket: R2Bucket) => {
+  const openai = new OpenAI({ apiKey, baseURL: `${baseURL}/openai`});
   const file = await openai.audio.speech.create({
     model: "tts-1",
     voice: "alloy",

Diff for: apps/webapp/src/app/app.routes.ts

@@ -13,6 +13,10 @@ export const routes: Routes = [
     path: 'simulator/:id',
     loadComponent: () => import('./pages/simulator/simulator.component')
   },
+  {
+    path: 'demo',
+    loadComponent: () => import('./pages/demo/demo.component')
+  },
   {
     path: '**',
     redirectTo: ''

Diff for: apps/webapp/src/app/components/modal-recording/modal-recording.component.ts

@@ -31,7 +31,7 @@ export class ModalRecordingComponent implements AfterViewInit {
   @ViewChild('videoRta', { static: false }) videoRta!: ElementRef<HTMLVideoElement>;
   private mediaRecorder!: MediaRecorder;
   status = signal<'init' | 'recording' | 'success' | 'processing' | 'streaming'>('init');
-  file = signal<File | null>(null);
+  file = signal<Blob | null>(null);
   intervalId = signal<number | null>(null);
   duration = signal<number>(0);
   formatDuration = computed(() => {
@@ -77,11 +77,9 @@ export class ModalRecordingComponent implements AfterViewInit {
   previewVideo() {
     const chunks = this.recordedChunks();
     const blob = new Blob(chunks, { type: 'video/webm' });
-    const filename = `${Date.now()}.webm`;
-    const file = new File([blob], filename, { type: blob.type, lastModified: Date.now() });
-    const url = URL.createObjectURL(file);
+    const url = URL.createObjectURL(blob);
     this.status.set('success');
-    this.file.set(file);
+    this.file.set(blob);
     this.videoRta.nativeElement.src = url;
     this.cdRef.detectChanges();
   }

Diff for: apps/webapp/src/app/pages/demo/demo.component.html

@@ -0,0 +1,16 @@
+<div class="p-10">
+  <p>
+    <button class="btn btn-large btn-primary" type="button" (click)="startRecording()">start</button>
+  </p>
+  <p>
+    <button class="btn btn-large"  type="button" (click)="stopRecording()">stop</button>
+  </p>
+  @if(url()) {
+    <p>
+      <video controls>
+        <source [src]="url()" type="video/webm">
+      </video>
+    </p>
+  }
+
+</div>

Diff for: apps/webapp/src/app/pages/demo/demo.component.ts

@@ -0,0 +1,75 @@
+import { Component, inject, signal } from '@angular/core';
+import { HttpClient } from '@angular/common/http';
+import { environment } from '@env/environment';
+
+@Component({
+  selector: 'app-demo',
+  standalone: true,
+  imports: [],
+  templateUrl: './demo.component.html',
+  styles: ``
+})
+export default class DemoComponent {
+  recordedChunks = signal<Blob[]>([]);
+  url = signal<string | null>(null);
+  private mediaRecorder!: MediaRecorder;
+
+  private http = inject(HttpClient);
+
+  async startRecording() {
+    const stream = await navigator.mediaDevices.getUserMedia({
+      video: true,
+      audio: true,
+    });
+    this.mediaRecorder = new MediaRecorder(stream, { mimeType: 'video/webm;codecs=vp9,opus' });
+    this.mediaRecorder.ondataavailable = (event) => {
+      if (event.data.size > 0) {
+        this.recordedChunks.update((chunks) => [...chunks, event.data]);
+      }
+    };
+    this.mediaRecorder.start();
+  }
+
+  stopRecording() {
+    this.mediaRecorder.stop();
+    setTimeout(() => {
+      this.processRecording();
+    }, 300);
+  }
+
+  processRecording() {
+    const chunks = this.recordedChunks();
+    console.log(chunks);
+    const blob = new Blob(chunks, { type: 'video/webm' });
+    const url = URL.createObjectURL(blob);
+    this.url.set(url);
+    this.requestOpenAI(blob).subscribe((rta) => {
+      console.log(rta);
+      this.recordedChunks.set([]);
+      this.url.set(null);
+    });
+  }
+
+  requestBlob(file: Blob) {
+    return this.http.post(`${environment.apiUrl}/demo1`, file);
+  }
+
+  requestFormData(file: Blob) {
+    const formData = new FormData();
+    formData.append('file', file, 'file.webm');
+    return this.http.post(`${environment.apiUrl}/demo2`, formData);
+  }
+
+  requestOpenAI(file: Blob) {
+    const formData = new FormData();
+    formData.append('file', file, 'file.webm');
+    return this.http.post(`${environment.apiUrl}/demo3`, formData);
+  }
+
+  requestGateway(file: Blob) {
+    const formData = new FormData();
+    formData.append('file', file, 'file.webm');
+    return this.http.post(`${environment.apiUrl}/demo4`, formData);
+  }
+
+}

Diff for: apps/webapp/src/app/pages/simulator/simulator.component.ts

@@ -63,7 +63,7 @@ export default class SimulatorComponent implements OnInit {
     });
   }
 
-  createTranscript(file: File, question: string) {
+  createTranscript(file: Blob, question: string) {
     this.mode.set('loading');
     this.apiService.createTranscript(file).subscribe({
       next: (newMessage) => {

Diff for: apps/webapp/src/app/services/api.service.ts

@@ -30,9 +30,10 @@ export class ApiService {
     });
   }
 
-  createTranscript(file: File) {
+  createTranscript(file: Blob) {
     const formData = new FormData();
-    formData.append('file', file);
+    const filename = `file-${Date.now()}.webm`;
+    formData.append('file', file, filename);
     return this.http.post<Message>(`${environment.apiUrl}/transcript`, formData)
     .pipe(
       map((response) => ({