Update configurations and dependencies for webapp and API components

Author: nicobytes

Diff for: apps/api/package.json

@@ -8,12 +8,14 @@
     "migrations:apply:prod": "wrangler d1 migrations apply interviews_db --remote"
   },
   "dependencies": {
+    "@cloudflare/ai": "1.1.0",
     "@hono/swagger-ui": "^0.2.1",
     "@hono/zod-openapi": "^0.11.0",
     "@langchain/cloudflare": "^0.0.4",
     "drizzle-orm": "^0.30.8",
     "hono": "4.2.4",
-    "langchain": "^0.1.33"
+    "langchain": "^0.1.33",
+    "openai": "^4.33.1"
   },
   "devDependencies": {
     "@cloudflare/workers-types": "4.20240405.0",

Diff for: apps/api/src/bindings.ts

@@ -2,5 +2,8 @@ export type Bindings = {
   DB: D1Database;
   CLOUDFLARE_ACCOUNT_ID: string;
   CLOUDFLARE_API_TOKEN: string;
+  OPENAI_API_KEY: string;
   AI: any;
+  BUCKET: R2Bucket;
+  R2_URL: string;
 }

Diff for: apps/api/src/dtos/feedback.dto.ts

@@ -1,15 +1,19 @@
 import { z } from '@hono/zod-openapi';
 
-export const FeedbackSchema = z
-  .object({
-    id: z.string().openapi({
-      example: '1',
+export const GenerateFeedbackSchema = z.object({
+  answer: z
+    .string()
+    .min(3)
+    .openapi({
+      example: 'Message',
     }),
-    from: z.string().openapi({
-      example: 'user',
+  question: z
+    .string()
+    .min(3)
+    .openapi({
+      example: 'Message',
     }),
-    text: z.string().openapi({
-      example: 'message',
+  simulationId: z.string().openapi({
+      example: '1',
     }),
-  })
-  .openapi('Feedback');
+});

Diff for: apps/api/src/dtos/message.dto.ts

@@ -26,6 +26,7 @@ export const MessageSchema = z
     type: z.string().openapi({
       example: 'feedback',
     }),
+    file: z.any().optional()
   })
   .openapi('Message');
 

Diff for: apps/api/src/index.ts

@@ -8,6 +8,7 @@ import { dbMiddleware } from '@src/middlewares/db.middleware';
 import createSimulation from '@src/routes/createSimulation';
 import createQuestion from '@src/routes/createQuestion';
 import createFeedback from '@src/routes/createFeedback';
+import createTranscript from '@src/routes/createTranscript';
 
 const app = new OpenAPIHono();
 app.use("*", cors());
@@ -19,6 +20,7 @@ app.use('/api/v1/*', dbMiddleware);
 app.route('/api/v1/simulations', createSimulation);
 app.route('/api/v1/questions', createQuestion);
 app.route('/api/v1/feedback', createFeedback);
+app.route('/api/v1/transcript', createTranscript);
 
 app.get("/", swaggerUI({ url: "/docs" }));
 app.doc("/docs", {

Diff for: apps/api/src/routes/createFeedback.ts

@@ -1,39 +1,45 @@
 import { OpenAPIHono, createRoute } from "@hono/zod-openapi";
-import { MessageListSchema } from '@src/dtos/message.dto';
-import { generateTranscription } from '@src/services/whisper.service';
+import { MessageSchema } from '@src/dtos/message.dto';
+import { GenerateFeedbackSchema } from '@src/dtos/feedback.dto';
 import { generateFeedback } from '@src/services/llm.service';
 import { App } from "@src/types";
 import { HTTPException } from "hono/http-exception";
 import { getSimulationById } from '@src/services/simulation.service';
+import { generateAudio } from '@src/services/openai.service';
 
 const app = new OpenAPIHono<App>();
 
 const route = createRoute({
   tags: ['simulation'],
   method: 'post',
   path: '/',
+  request: {
+    body: {
+      content: {
+        'application/json': {
+          schema: GenerateFeedbackSchema,
+        },
+      },
+    },
+  },
   responses: {
     200: {
       content: {
         'application/json': {
-          schema: MessageListSchema,
+          schema: MessageSchema,
         },
       },
-      description: 'Retrieve new simulation',
+      description: 'Retrieve new feedback',
     },
   },
   public: true,
 });
 
 app.openapi(route, async (c) => {
   const db = c.get('db');
-  const body = await c.req.parseBody();
-  const file = body.file as File;
-  const question = body.question as string;
-  const simulationId = body.simulationId as string;
+  const { question, answer, simulationId } = c.req.valid('json');
   const entity = await getSimulationById(db, +simulationId);
 
-  const answer = await generateTranscription(file, c.env.AI);
   const response = await generateFeedback({
     role: entity.role,
     question,
@@ -47,23 +53,12 @@ app.openapi(route, async (c) => {
     throw new HTTPException(500, { message: `Complex messsage` })
   }
 
-  const messages = [
-    {
-      id: `${Date.now()}`,
-      from: 'user',
-      text: answer,
-      type: 'response',
-    },
-    {
-      id: `${Date.now() + 1}`,
-      from: 'bot',
-      text: content,
-      type: 'feedback',
-    },
-
-  ];
-
-  return c.json(messages);
+  return c.json({
+    id: `${Date.now() + 1}`,
+    from: 'bot',
+    text: content,
+    type: 'feedback'
+  },);
 });
 
 export default app;

Diff for: apps/api/src/routes/createQuestion.ts

@@ -2,6 +2,7 @@ import { OpenAPIHono, createRoute } from "@hono/zod-openapi";
 import { MessageSchema, NewMessageSchema } from '@src/dtos/message.dto';
 import { getSimulationById } from '@src/services/simulation.service';
 import { generateQuestion } from '@src/services/llm.service';
+import { generateAudio } from '@src/services/openai.service';
 import { App } from "@src/types";
 
 const app = new OpenAPIHono<App>();
@@ -44,11 +45,14 @@ app.openapi(route, async (c) => {
     cloudflareApiToken: c.env.CLOUDFLARE_API_TOKEN,
   }, c.env.DB);
 
+  const filename = await generateAudio(response, c.env.OPENAI_API_KEY, c.env.BUCKET);
+
   return c.json({
     id: `${Date.now()}`,
     from: 'bot',
     text: response,
     type: 'response',
+    audio: `${c.env.R2_URL}/${filename}`,
   });
 });
 

Diff for: apps/api/src/routes/createTranscript.ts

@@ -0,0 +1,38 @@
+import { OpenAPIHono, createRoute } from "@hono/zod-openapi";
+import { MessageSchema } from '@src/dtos/message.dto';
+import { generateTranscription } from '@src/services/whisper.service';
+import { App } from "@src/types";
+
+const app = new OpenAPIHono<App>();
+
+const route = createRoute({
+  tags: ['simulation'],
+  method: 'post',
+  path: '/',
+  responses: {
+    200: {
+      content: {
+        'application/json': {
+          schema: MessageSchema,
+        },
+      },
+      description: 'Retrieve new simulation',
+    },
+  },
+  public: true,
+});
+
+app.openapi(route, async (c) => {
+  const body = await c.req.parseBody();
+  const file = body.file as File;
+  const answer = await generateTranscription(file, c.env.AI);
+
+  return c.json({
+    id: `${Date.now()}`,
+    from: 'user',
+    text: answer,
+    type: 'transcription'
+  },);
+});
+
+export default app;

Diff for: apps/api/src/services/openai.service.ts

@@ -0,0 +1,30 @@
+import OpenAI from "openai";
+
+export const generateTranscription = async (file: File, key: string) => {
+  const openai = new OpenAI({
+    apiKey: key,
+  });
+  console.log('name', file.name);
+  console.log('type', file.type);
+  const transcription = await openai.audio.transcriptions.create({
+    file: file,
+    model: "whisper-1",
+  });
+
+  return transcription.text;
+}
+
+export const generateAudio = async (input: string, key: string, bucket: R2Bucket) => {
+  const openai = new OpenAI({
+    apiKey: key,
+  });
+  const file = await openai.audio.speech.create({
+    model: "tts-1",
+    voice: "alloy",
+    input,
+  });
+  const blob = await file.blob();
+  const fileName = `audios/${Date.now()}.mp3`;
+  await bucket.put(fileName, blob);
+  return fileName;
+}

Diff for: apps/api/src/services/whisper.service.ts

@@ -2,7 +2,8 @@ import { Ai } from '@cloudflare/ai';
 import { HTTPException } from 'hono/http-exception';
 
 export const generateTranscription = async (file: File, binding: any) => {
-
+  console.log('name', file.name);
+  console.log('type', file.type);
   const blob = await file.arrayBuffer();
   const input = {
     audio: [...new Uint8Array(blob)],
@@ -18,3 +19,4 @@ export const generateTranscription = async (file: File, binding: any) => {
 
   return response.text;
 }
+

Diff for: apps/api/wrangler.toml

@@ -9,3 +9,8 @@ database_id = "ca1cba03-9281-4bab-ac79-0ae16b39b88a"
 
 [ai]
 binding = "AI"
+
+[[r2_buckets]]
+binding = "BUCKET"
+bucket_name = "files"
+preview_bucket_name = "preview-files"

Diff for: apps/webapp/angular.json

@@ -51,7 +51,8 @@
               "src/assets"
             ],
             "styles": [
-              "src/styles.scss"
+              "src/styles.scss",
+              "node_modules/ngx-toastr/toastr.css"
             ],
             "scripts": []
           },