Merge pull request #582 from aws-samples/development

Release 0.20.2
aws-samples · Nov 28, 2023 · daa39b9 · daa39b9
2 parents 4f0e883 + 041d4e9
commit daa39b9
Show file tree

Hide file tree

Showing 10 changed files with 204 additions and 108 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## [0.20.1] - 2023-11-28
+- Adjust handling of Elicit Intent response to account for no interpretations from Lex. Precreate mp3 audio files needed for voice response as default un-authenticated role can't use Polly to create these responses dynamically.
+- Dependency upgrades to fix critical vulnerabilities.
+
 ## [0.20.1] - 2023-10-24
 - Removed breaking change of adding CSP configurations into Cloudfront. CSP will remain in place on index.html file but Cloudfront CSPs will need to be manually configured. As a result removed MarkdownSupportDomains parameter.
 - Minor bug fixes.

diff --git a/Makefile b/Makefile
@@ -152,6 +152,8 @@ sync-website: create-iframe-snippet
 		--metadata-directive REPLACE --cache-control max-age=0 \
 		--include 'lex-web-ui-loader-config.json' \
 		--include 'initial_speech*.*' \
+		--include 'all_done*.*' \
+		--include 'there_was_an_error*.*' \
 		"$(CONFIG_DIR)" s3://$(WEBAPP_BUCKET)
 	@echo "[INFO] all done deploying"
 .PHONY: sync-website
diff --git a/build/update-lex-web-ui-config.js b/build/update-lex-web-ui-config.js
@@ -157,8 +157,70 @@ const lexV2BotLocaleVoices = {
   'HELP_INTENT',
   'MIN_BUTTON_TOOLTIP_CONTENT',
 ].forEach(function (envVar) {
-  console.log('[INFO] Env var - %s: [%s]', envVar, process.env[envVar]);
+  console.info('[INFO] Env var - %s: [%s]', envVar, process.env[envVar]);
 });
+
+/**
+ * Create an Mp3 file in the specified output folder for the given text, languageCode, and voiceId
+ * using AWS Polly.
+ * @param text
+ * @param languageCode
+ * @param voiceId
+ * @param output
+ */
+function createMp3(text, languageCode, voiceId, output) {
+  let lcDefinition = (languageCode.length > 0) ? `--language-code ${languageCode}` : '';
+  const cmd = `aws polly synthesize-speech --text "${text}" ${lcDefinition} --voice-id "${voiceId}"  --output-format mp3 --text-type text "${output}"`
+  console.info(`createMp3 cmd is \n${cmd}`);
+  exec(cmd, (error, stdout, stderr) => {
+    if (error) {
+      console.error(`createMp3 error: ${error.message}`);
+    }
+    if (stderr) {
+      console.error(`createMp3 stderr: ${stderr}`);
+    }
+    console.info(`createMp3 stdout: ${stdout}`);
+  });
+}
+
+/**
+ * Translate the specified text to the specified localeId and create an Mp3 in
+ * the specified output folder.
+ * @param localeId
+ * @param text
+ * @param output
+ */
+function translateAndCreateMp3(localeId, text, output) {
+  console.info(`translate '${text}' to ${localeId.trim()} with output of ${output}`);
+  lid = localeId.trim()
+  if (lid === 'en_US') {
+    return;
+  }
+  let targetPollyVoiceConfig = lexV2BotLocaleVoices[lid]
+  let enUSPollyVoiceConfig = lexV2BotLocaleVoices["en_US"];
+  console.info(`targetPollyVoiceConfig ${JSON.stringify(targetPollyVoiceConfig,null,4)}`);
+  if (targetPollyVoiceConfig) {
+    // translate the english text defined in CF template to the target language.
+    const targetTranslateLang = lid.split("_")[0];
+    const translateCmd = `aws translate translate-text --text "${text}" --source-language-code auto --target-language-code ${targetTranslateLang} --output json --query 'TranslatedText'`
+    console.info(`translate cmd is \n${translateCmd}`);
+    exec(translateCmd, (error, stdout, stderr) => {
+      if (error) {
+        console.error(`translate error: ${error.message}`);
+      }
+      if (stderr) {
+        console.error(`translate stderr: ${stderr}`);
+      }
+      console.info(`translate stdout: ${stdout.trim()}`);
+      // if a language code for the target locale exists, specify this for the polly command
+      createMp3(stdout.trim().replace(/['"]+/g, ''), targetPollyVoiceConfig.languageCode, targetPollyVoiceConfig.voiceId, output)
+    });
+  } else { // the specified locale can't be translated as it is not in the map. Generate an english version for this locale.
+    console.info(`Could not find specified locale "${lid}"`)
+    createMp3(text.trim().replace(/['"]+/g, ''), enUSPollyVoiceConfig.languageCode, enUSPollyVoiceConfig.voiceId, output)
+  }
+}
+
 Object.keys(config)
 .map(function (confKey) { return config[confKey]; })
 .forEach(function (item) {
@@ -167,67 +229,47 @@ Object.keys(config)
       console.error('[ERROR] could not write file: ', err);
       process.exit(1);
     }
-    console.log('[INFO] Updated file: ', item.file);
-    console.log('[INFO] Config contents: ', JSON.stringify(item.conf));
+
+    // This following code pre-creates mp3 files needed for voice interaction. These files need to be pre-created
+    // and made available to the lex-web-ui for voice mode as the unauthenticated IAM role built for lex-web-ui no
+    // longer has access to Polly dynamically. The build IAM role does have access to Polly and Translate. The files
+    // are made available in the lex-web-ui web app bucket alongside of other UI assets. The files created are used
+    // for initial voice, the "All done" verbal response, and the "There was an error" verbal response.
+
+    console.info('[INFO] Updated file: ', item.file);
+    console.info('[INFO] Config contents: ', JSON.stringify(item.conf));
     revisedConfig = item.conf;
+    let enUSPollyVoiceConfig = lexV2BotLocaleVoices["en_US"];
+    const {exec} = require("child_process");
+    const path = require('path');
+    const configDir = path.parse(item.file).dir;
+    console.info('[INFO] Config dir is: ', configDir);
+
+    // if an initial speach is set in the configuration, generate mp3 files for english and other configured locales
     if (revisedConfig.lex && revisedConfig.lex.initialSpeechInstruction && revisedConfig.lex.initialSpeechInstruction.length > 0) {
-      const {exec} = require("child_process");
-      const path = require('path');
-      const configDir = path.parse(item.file).dir;
-      console.log('[INFO] Config dir is: ', configDir);
       // always generate an en_US mp3 if initial speech is defined
-      const cmd = `aws polly synthesize-speech --text "${revisedConfig.lex.initialSpeechInstruction.replace(/['"]+/g, '')}" --language-code "en-US" --voice-id "${revisedConfig.polly.voiceId}"  --output-format mp3 --text-type text "${configDir}/initial_speech_en_US.mp3"`
-      exec(cmd, (error, stdout, stderr) => {
-        if (error) {
-          console.log(`error: ${error.message}`);
-        }
-        if (stderr) {
-          console.log(`stderr: ${stderr}`);
-        }
-        console.log(`stdout: ${stdout}`);
-      });
+      createMp3(revisedConfig.lex.initialSpeechInstruction.replace(/['"]+/g, ''), "en-US", enUSPollyVoiceConfig.voiceId,`${configDir}/initial_speech_en_US.mp3`);
+
       // Iterate through the map of the configured v2BotLocaleIds and generate mp3 files with initial speech.
       // This is only supported for LexV2 bots.
-      revisedConfig.lex.v2BotLocaleId.split(",").map((origLocaleId) => {
-        let targetPollyVoiceConfig = lexV2BotLocaleVoices[origLocaleId];
-        if (targetPollyVoiceConfig && origLocaleId !== 'en_US') {
-          // translate the english text defined in CF template to the target language.
-          const targetTranslateLang = origLocaleId.split("_")[0];
-          const translateCmd = `aws translate translate-text --text "${revisedConfig.lex.initialSpeechInstruction}" --source-language-code auto --target-language-code ${targetTranslateLang} --output json --query 'TranslatedText'`
-          exec(translateCmd, (error, stdout, stderr) => {
-            if (error) {
-              console.log(`error: ${error.message}`);
-            }
-            if (stderr) {
-              console.log(`stderr: ${stderr}`);
-            }
-            console.log(`stdout: ${stdout.trim()}`);
-            // if a language code for the target locale exists, specify this for the polly command
-            let lcDefinition = (targetPollyVoiceConfig.languageCode.length > 0) ? `--language-code ${targetPollyVoiceConfig.languageCode}` : '';
-            const pollyCmd = `aws polly synthesize-speech --text ${stdout.trim()} ${lcDefinition} --voice-id "${targetPollyVoiceConfig.voiceId}"  --engine "${targetPollyVoiceConfig.engine}" --output-format mp3 --text-type text "${configDir}/initial_speech_${origLocaleId}.mp3"`
-            exec(pollyCmd, (error, stdout, stderr) => {
-              if (error) {
-                console.log(`error: ${error.message}`);
-              }
-              if (stderr) {
-                console.log(`stderr: ${stderr}`);
-              }
-              console.log(`stdout: ${stdout}`);
-            });
-          });
-        } else { // the specified local can't be translated as it is not in the map. Generate an english version for this locale.
-          const defaultPollyCmd = `aws polly synthesize-speech --text "${revisedConfig.lex.initialSpeechInstruction.replace(/['"]+/g, '')}" --language-code "en-US" --voice-id "${revisedConfig.polly.voiceId}"  --output-format mp3 --text-type text "${configDir}/initial_speech_${origLocaleId}.mp3"`
-          exec(defaultPollyCmd, (error, stdout, stderr) => {
-            if (error) {
-              console.log(`error: ${error.message}`);
-            }
-            if (stderr) {
-              console.log(`stderr: ${stderr}`);
-            }
-            console.log(`stdout: ${stdout}`);
-          });
+      revisedConfig.lex.v2BotLocaleId.split(",").map((localeId) => {
+        lid = localeId.trim();
+        if (lid != "en_US") {
+          translateAndCreateMp3(lid, revisedConfig.lex.initialSpeechInstruction.replace(/['"]+/g, ''), `${configDir}/initial_speech_${lid}.mp3`)
         }
       });
     }
+
+    // create mp3 audio files for other prompts used by lex-web-ui in english and other locales
+    if (revisedConfig && revisedConfig.lex) {
+      // Create special case MP3s that lexwebui might utilize
+      createMp3('All done', "en-US", enUSPollyVoiceConfig.voiceId, `${configDir}/all_done_en_US.mp3`);
+      createMp3('There was an error', "en-US", enUSPollyVoiceConfig.voiceId, `${configDir}/there_was_an_error_en_US.mp3`);
+      revisedConfig.lex.v2BotLocaleId.split(",").map((localeId) => {
+        let lid = localeId.trim();
+        translateAndCreateMp3(localeId, 'All done', `${configDir}/all_done_${lid}.mp3`)
+        translateAndCreateMp3(localeId, 'There was an error', `${configDir}/there_was_an_error_${lid}.mp3`)
+      });
+    }
   });
 });
diff --git a/lex-web-ui/package-lock.json b/lex-web-ui/package-lock.json
diff --git a/lex-web-ui/package.json b/lex-web-ui/package.json
@@ -1,6 +1,6 @@
 {
   "name": "lex-web-ui",
-  "version": "0.20.1",
+  "version": "0.20.2",
   "description": "Amazon Lex Web Interface",
   "author": "AWS",
   "license": "Amazon Software License",

diff --git a/lex-web-ui/src/lib/lex/client.js b/lex-web-ui/src/lib/lex/client.js
@@ -263,11 +263,16 @@ export default class {
             res.slotToElicit = oState.dialogAction.slotToElicit;
           }
           else {  // Fallback for some responses that do not have an intent (ElicitIntent, etc)
-            res.intentName = oState.interpretations[0].intent.name;
-            res.slots = oState.interpretations[0].intent.slots;
+            if ("interpretations" in oState) {
+              res.intentName = oState.interpretations[0].intent.name;
+              res.slots = oState.interpretations[0].intent.slots;
+            } else {
+              res.intentName = '';
+              res.slots = '';
+            }
             res.dialogState = '';
             res.slotToElicit = '';
-          }          
+          }
           res.inputTranscript = res.inputTranscript
             && b64CompressedToString(res.inputTranscript);
           res.interpretations = res.interpretations

diff --git a/lex-web-ui/src/store/actions.js b/lex-web-ui/src/store/actions.js
@@ -38,6 +38,9 @@ let lexClient;
 let audio;
 let recorder;
 let liveChatSession;
+let pollyInitialSpeechBlob = {};
+let pollyAllDoneBlob = {};
+let pollyThereWasAnErrorBlob = {};
 
 export default {
   /***********************************************************************
@@ -428,11 +431,44 @@ export default {
       .then(audioUrl => context.dispatch('playAudio', audioUrl));
   },
   pollySynthesizeInitialSpeech(context) {
-    const localeId = localStorage.getItem('selectedLocale') ? localStorage.getItem('selectedLocale') : context.state.config.lex.v2BotLocaleId.split(',')[0];
-    return fetch(`./initial_speech_${localeId}.mp3`)
-      .then(data => data.blob())
-      .then(blob => context.dispatch('getAudioUrl', blob))
-      .then(audioUrl => context.dispatch('playAudio', audioUrl));
+    const localeId = localStorage.getItem('selectedLocale') ? localStorage.getItem('selectedLocale') : context.state.config.lex.v2BotLocaleId.split(',')[0].trim();
+    if (localeId in pollyInitialSpeechBlob) {
+      return Promise.resolve(pollyInitialSpeechBlob[localeId]);
+    } else {
+      return fetch(`./initial_speech_${localeId}.mp3`)
+        .then(data => data.blob())
+        .then((blob) => {
+          pollyInitialSpeechBlob[localeId] = blob;
+          return context.dispatch('getAudioUrl', blob)
+        })
+        .then(audioUrl => context.dispatch('playAudio', audioUrl));
+    }
+  },
+  pollySynthesizeAllDone: function (context) {
+    const localeId = localStorage.getItem('selectedLocale') ? localStorage.getItem('selectedLocale') : context.state.config.lex.v2BotLocaleId.split(',')[0].trim();
+    if (localeId in pollyAllDoneBlob) {
+      return Promise.resolve(pollyAllDoneBlob[localeId]);
+    } else {
+      return fetch(`./all_done_${localeId}.mp3`)
+        .then(data => data.blob())
+        .then(blob => {
+          pollyAllDoneBlob[localeId] = blob;
+          return Promise.resolve(blob)
+        })
+    }
+  },
+  pollySynthesizeThereWasAnError(context) {
+    const localeId = localStorage.getItem('selectedLocale') ? localStorage.getItem('selectedLocale') : context.state.config.lex.v2BotLocaleId.split(',')[0].trim();
+    if (localeId in pollyThereWasAnErrorBlob) {
+      return Promise.resolve(pollyThereWasAnErrorBlob[localeId]);
+    } else {
+      return fetch(`./there_was_an_error_${localeId}.mp3`)
+        .then(data => data.blob())
+        .then(blob => {
+          pollyThereWasAnErrorBlob[localeId] = blob;
+          return Promise.resolve(blob)
+        })
+    }
   },
   interruptSpeechConversation(context) {
     if (!context.state.recState.isConversationGoing &&
@@ -714,13 +750,14 @@ export default {
     return Promise.resolve()
       .then(() => {
         if (!audioStream || !audioStream.length) {
-          const text = (dialogState === 'ReadyForFulfillment') ?
-            'All done' :
-            'There was an error';
-          return context.dispatch('pollyGetBlob', text);
+          if (dialogState === 'ReadyForFulfillment') {
+            return context.dispatch('pollySynthesizeAllDone');
+          } else {
+            return context.dispatch('pollySynthesizeThereWasAnError');
+          }
+        } else {
+          return Promise.resolve(new Blob([audioStream], {type: contentType}));
         }
-
-        return Promise.resolve(new Blob([audioStream], { type: contentType }));
       });
   },
   updateLexState(context, lexState) {

diff --git a/lex-web-ui/src/store/mutations.js b/lex-web-ui/src/store/mutations.js
@@ -527,6 +527,6 @@ export default {
     state.lex.isPostTextRetry = bool;
   },
   updateLocaleIds(state, data) {
-    state.config.lex.v2BotLocaleId = data;
+    state.config.lex.v2BotLocaleId = data.trim().replace(/ /g, '');
   },
 };