From da906074d5569546a6e0a93289340851e56d3f3b Mon Sep 17 00:00:00 2001 From: damencho Date: Tue, 10 Sep 2024 09:07:29 -0500 Subject: [PATCH 1/3] feat(transcription): Adds additional languages via metadata and handle visitors. --- .../java/org/jitsi/jigasi/JvbConference.java | 13 +++++- .../jigasi/TranscriptionGatewaySession.java | 43 ++++++++++++++++++- .../jigasi/transcription/Transcriber.java | 8 ++++ .../transcription/TranslationManager.java | 8 +++- 4 files changed, 69 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/jitsi/jigasi/JvbConference.java b/src/main/java/org/jitsi/jigasi/JvbConference.java index 977884514..1f5963c67 100644 --- a/src/main/java/org/jitsi/jigasi/JvbConference.java +++ b/src/main/java/org/jitsi/jigasi/JvbConference.java @@ -2212,10 +2212,21 @@ private void processRoomMetadataJson(String json) if (data.get("type").equals("room_metadata")) { + TranscriptionGatewaySession transcriptionSession = (TranscriptionGatewaySession)this.gatewaySession; + JSONObject metadataObj = (JSONObject)data.getOrDefault("metadata", new JSONObject()); JSONObject recordingObj = (JSONObject)metadataObj.getOrDefault("recording", new JSONObject()); - ((TranscriptionGatewaySession)this.gatewaySession).setBackendTranscribingEnabled( + transcriptionSession.setBackendTranscribingEnabled( (boolean)recordingObj.getOrDefault("isTranscribingEnabled", false)); + + JSONObject visitorsObj = (JSONObject)metadataObj.getOrDefault("visitors", new JSONObject()); + String languages = (String)visitorsObj.get("transcribingLanguages"); + if (StringUtils.isNotEmpty(languages)) + { + transcriptionSession.updateTranslateLanguages(languages.split(",")); + } + transcriptionSession.setVisitorsCountRequestingTranscription( + Integer.parseInt((String)visitorsObj.get("transcribingCount"))); } } } diff --git a/src/main/java/org/jitsi/jigasi/TranscriptionGatewaySession.java b/src/main/java/org/jitsi/jigasi/TranscriptionGatewaySession.java index 06ee29097..015d02dec 100644 --- a/src/main/java/org/jitsi/jigasi/TranscriptionGatewaySession.java +++ b/src/main/java/org/jitsi/jigasi/TranscriptionGatewaySession.java @@ -126,6 +126,17 @@ public class TranscriptionGatewaySession */ private int numberOfScheduledParticipantsLeaving = 0; + /** + * The currently used additional languages for translation. + * Used for visitors to announce the languages used by visitors. + */ + private List additionalLanguages = new ArrayList<>(); + + /** + * The number of visitors that are requesting transcriptions. + */ + private int numberOfVisitorsRequestingTranscription = 0; + /** * Create a TranscriptionGatewaySession which can handle the transcription * of a JVB conference @@ -359,7 +370,9 @@ void notifyChatRoomMemberUpdated(ChatRoomMember chatMember, Presence presence) private boolean isTranscriptionRequested() { - return transcriber.isAnyParticipantRequestingTranscription() || isBackendTranscribingEnabled; + return transcriber.isAnyParticipantRequestingTranscription() + || isBackendTranscribingEnabled + || this.numberOfVisitorsRequestingTranscription > 0; } private void maybeStopTranscription() @@ -753,4 +766,32 @@ public void setBackendTranscribingEnabled(boolean backendTranscribingEnabled) this.maybeStopTranscription(); } + + /** + * @param count The count of visitors that are requesting transcriptions. + */ + public void setVisitorsCountRequestingTranscription(int count) + { + this.numberOfVisitorsRequestingTranscription = count; + + this.maybeStopTranscription(); + } + + /** + * To update all participant languages with the additional that are provided. + * @param additionalLanguages languages to add to those from the participants. + */ + void updateTranslateLanguages(String[] additionalLanguages) + { + List oldLangs = this.additionalLanguages; + this.additionalLanguages = Arrays.asList(additionalLanguages); + + // remove unused streams + oldLangs.stream().filter(s -> !this.additionalLanguages.contains(s)) + .forEach(lang -> this.transcriber.getTranslationManager().removeLanguage(lang)); + + // add new languages + this.additionalLanguages.stream().filter(s -> !oldLangs.contains(s)) + .forEach(lang -> this.transcriber.getTranslationManager().addLanguage(lang)); + } } diff --git a/src/main/java/org/jitsi/jigasi/transcription/Transcriber.java b/src/main/java/org/jitsi/jigasi/transcription/Transcriber.java index 27bc578fd..6588c3733 100644 --- a/src/main/java/org/jitsi/jigasi/transcription/Transcriber.java +++ b/src/main/java/org/jitsi/jigasi/transcription/Transcriber.java @@ -834,6 +834,14 @@ public TranscriptionService getTranscriptionService() return transcriptionService; } + /** + * @return the {@link TranslationManager}. + */ + public TranslationManager getTranslationManager() + { + return this.translationManager; + } + /** * Notifies all of the listeners of this {@link Transcriber} of a new * {@link TranscriptionResult} which was received. diff --git a/src/main/java/org/jitsi/jigasi/transcription/TranslationManager.java b/src/main/java/org/jitsi/jigasi/transcription/TranslationManager.java index cc8a8f4b8..31ad65974 100644 --- a/src/main/java/org/jitsi/jigasi/transcription/TranslationManager.java +++ b/src/main/java/org/jitsi/jigasi/transcription/TranslationManager.java @@ -140,9 +140,15 @@ private List getTranslations( { for (String targetLanguage : translationLanguages) { + String sourceLang = result.getParticipant().getSourceLanguage(); + if (sourceLang != null && sourceLang.equals(targetLanguage)) + { + continue; + } + String translatedText = translationService.translate( alternatives.iterator().next().getTranscription(), - result.getParticipant().getSourceLanguage(), + sourceLang, targetLanguage); translatedResults.add(new TranslationResult( From f9755efd88f7e47b75eb0861facef4c603579b36 Mon Sep 17 00:00:00 2001 From: damencho Date: Tue, 10 Sep 2024 13:04:09 -0500 Subject: [PATCH 2/3] fix: NoSuchElementException observed sometimes on leave. 2024-09-10 16:59:58.093 SEVERE: [58] JvbConference.lambda$static$0#206: Error processing xmpp queue item java.util.NoSuchElementException at java.base/java.util.ArrayList$Itr.next(ArrayList.java:970) at java.base/java.util.Collections$UnmodifiableCollection$1.next(Collections.java:1054) at org.jitsi.jigasi.JvbConference.memberPresenceChangedInternal(JvbConference.java:1361) --- src/main/java/org/jitsi/jigasi/JvbConference.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/jitsi/jigasi/JvbConference.java b/src/main/java/org/jitsi/jigasi/JvbConference.java index 1f5963c67..6a8231497 100644 --- a/src/main/java/org/jitsi/jigasi/JvbConference.java +++ b/src/main/java/org/jitsi/jigasi/JvbConference.java @@ -1358,7 +1358,8 @@ else if (ChatRoomMemberPresenceChangeEvent.MEMBER_UPDATED + " " + member.getContactAddress()); CallPeer peer; - if (jvbCall != null && (peer = jvbCall.getCallPeers().next()) instanceof MediaAwareCallPeer) + if (jvbCall != null && jvbCall.getCallPeerCount() > 0 + && (peer = jvbCall.getCallPeers().next()) instanceof MediaAwareCallPeer) { MediaAwareCallPeer peerMedia = (MediaAwareCallPeer) peer; peerMedia.getConferenceMembers().forEach(confMember -> From 2b04f56d822c54f14a42137601d1bd53e274ae09 Mon Sep 17 00:00:00 2001 From: damencho Date: Tue, 10 Sep 2024 13:04:50 -0500 Subject: [PATCH 3/3] fix: Problem joining when jicofo is not in brewery. In this case the following error is observed: org.jivesoftware.smackx.xdata.provider.DataFormProvider.parseField: Unknown form field child element {jabber:x:data}desc ignored 2024-09-10 17:14:49.523 WARNING: [1089] org.jivesoftware.smack.parsing.ExceptionThrowingCallbackWithHint.handleUnparsableStanza: Parsing exception "javax.xml.stream.XMLStreamException: ParseError at [row,col]:[1,2982] Message: found: CHARACTERS, expected START_ELEMENT or END_ELEMENT" encountered. This exception will be re-thrown, leading to a disconnect. You can change this behavior by setting a different ParsingExceptionCallback using setParsingExceptionCallback(). More information an be found in AbstractXMPPConnection's javadoc. 2024-09-10 17:14:49.524 WARNING: [1096] org.jivesoftware.smack.AbstractXMPPConnection.callConnectionClosedOnErrorListener: Connection XMPPTCPConnection[jigasi@auth.mydomain.com/jitsi-3f1ep5l] (10) closed with error java.io.IOException: org.jivesoftware.smack.xml.XmlPullParserException: javax.xml.stream.XMLStreamException: ParseError at [row,col]:[1,2982] Message: found: CHARACTERS, expected START_ELEMENT or END_ELEMENT --- src/main/java/org/jitsi/jigasi/Main.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/org/jitsi/jigasi/Main.java b/src/main/java/org/jitsi/jigasi/Main.java index ae7f31d80..1c172665f 100644 --- a/src/main/java/org/jitsi/jigasi/Main.java +++ b/src/main/java/org/jitsi/jigasi/Main.java @@ -152,7 +152,6 @@ public class Main "org.jivesoftware.smackx.si", "org.jivesoftware.smackx.vcardtemp", "org.jivesoftware.smackx.xhtmlim", - "org.jivesoftware.smackx.xdata", "org.jivesoftware.smackx.eme", "org.jivesoftware.smackx.iqprivate", "org.jivesoftware.smackx.bookmarks",