feat(ios/android/sdk): delegate transcription chunk received event (#14516)

* feat(subtitles): created separate helpers for delegating transcription chunks events 
on web and mobile

android/sdk/src/main/java/org/jitsi/meet/sdk/BroadcastEvent.java

@@ -89,7 +89,8 @@ public class BroadcastEvent {
         CHAT_MESSAGE_RECEIVED("org.jitsi.meet.CHAT_MESSAGE_RECEIVED"),
         CHAT_TOGGLED("org.jitsi.meet.CHAT_TOGGLED"),
         VIDEO_MUTED_CHANGED("org.jitsi.meet.VIDEO_MUTED_CHANGED"),
-        READY_TO_CLOSE("org.jitsi.meet.READY_TO_CLOSE");
+        READY_TO_CLOSE("org.jitsi.meet.READY_TO_CLOSE"),
+        TRANSCRIPTION_CHUNK_RECEIVED("org.jitsi.meet.TRANSCRIPTION_CHUNK_RECEIVED");
 
         private static final String CONFERENCE_BLURRED_NAME = "CONFERENCE_BLURRED";
         private static final String CONFERENCE_FOCUSED_NAME = "CONFERENCE_FOCUSED";
@@ -106,6 +107,7 @@ public class BroadcastEvent {
         private static final String CHAT_TOGGLED_NAME = "CHAT_TOGGLED";
         private static final String VIDEO_MUTED_CHANGED_NAME = "VIDEO_MUTED_CHANGED";
         private static final String READY_TO_CLOSE_NAME = "READY_TO_CLOSE";
+        private static final String TRANSCRIPTION_CHUNK_RECEIVED_NAME = "TRANSCRIPTION_CHUNK_RECEIVED";
 
         private final String action;
 
@@ -158,6 +160,8 @@ public class BroadcastEvent {
                     return VIDEO_MUTED_CHANGED;
                 case READY_TO_CLOSE_NAME:
                     return READY_TO_CLOSE;
+                case TRANSCRIPTION_CHUNK_RECEIVED_NAME:
+                    return TRANSCRIPTION_CHUNK_RECEIVED;
             }
 
             return null;

android/sdk/src/main/java/org/jitsi/meet/sdk/JitsiMeetActivity.java

@@ -255,6 +255,10 @@ public class JitsiMeetActivity extends AppCompatActivity
         finish();
     }
 
+//    protected void onTranscriptionChunkReceived(HashMap<String, Object> extraData) {
+//        JitsiMeetLogger.i("Transcription chunk received: " + extraData);
+//    }
+
     // Activity lifecycle methods
     //
 
@@ -338,6 +342,9 @@ public class JitsiMeetActivity extends AppCompatActivity
                 case READY_TO_CLOSE:
                     onReadyToClose();
                     break;
+//                case TRANSCRIPTION_CHUNK_RECEIVED:
+//                    onTranscriptionChunkReceived(event.getData());
+//                    break;
             }
         }
     }

ios/app/src/ViewController.m

@@ -98,6 +98,10 @@
     [self _onJitsiMeetViewDelegateEvent:@"READY_TO_CLOSE" withData:data];
 }
 
+// - (void)transcriptionChunkReceived:(NSDictionary *)data {
+//     [self _onJitsiMeetViewDelegateEvent:@"TRANSCRIPTION_CHUNK_RECEIVED" withData:data];
+// }
+
 - (void)participantJoined:(NSDictionary *)data {
   NSLog(@"%@%@", @"Participant joined: ", data[@"participantId"]);
 }
@@ -130,6 +134,7 @@
   NSLog(@"%@%@", @"Video muted changed: ", data[@"muted"]);
 }
 
+
 #pragma mark - Helpers
 
 - (void)terminate {

ios/sdk/src/JitsiMeetView.h

@@ -26,7 +26,7 @@
 @property (nonatomic, nullable, weak) id<JitsiMeetViewDelegate> delegate;
 
 /**
- * Joins the conference specified by the given options. The gievn options will
+ * Joins the conference specified by the given options. The given options will
  * be merged with the defaultConferenceOptions (if set) in JitsiMeet. If there
  * is an already active conference it will be automatically left prior to
  * joining the new one.

ios/sdk/src/JitsiMeetViewDelegate.h

@@ -116,4 +116,11 @@
  */
 - (void)readyToClose:(NSDictionary *)data;
 
+/**
+ * Called when the transcription chunk was received.
+ *
+ * The `data` dictionary contains a `messageID`, `language`, `participant` key.
+ */
+- (void)transcriptionChunkReceived:(NSDictionary *)data;
+
 @end

react/features/mobile/external-api/middleware.ts

@@ -86,7 +86,7 @@ const CONFERENCE_TERMINATED = 'CONFERENCE_TERMINATED';
 const ENDPOINT_TEXT_MESSAGE_RECEIVED = 'ENDPOINT_TEXT_MESSAGE_RECEIVED';
 
 /**
- * Event which will be emitted on the native side to indicate a participant togggles
+ * Event which will be emitted on the native side to indicate a participant toggles
  * the screen share.
  */
 const SCREEN_SHARE_TOGGLED = 'SCREEN_SHARE_TOGGLED';

react/features/subtitles/components/AbstractClosedCaptionButton.tsx

@@ -4,7 +4,7 @@ import { IReduxState } from '../../app/types';
 import { MEET_FEATURES } from '../../base/jwt/constants';
 import AbstractButton, { IProps as AbstractButtonProps } from '../../base/toolbox/components/AbstractButton';
 import { maybeShowPremiumFeatureDialog } from '../../jaas/actions';
-import { canStartSubtitles } from '../functions';
+import { canStartSubtitles } from '../functions.any';
 
 export interface IAbstractProps extends AbstractButtonProps {
 

react/features/subtitles/functions.native.ts

@@ -0,0 +1,30 @@
+/* eslint-disable max-params, max-len */
+
+import { sendEvent } from '../mobile/external-api/functions';
+
+
+/**
+ * Event which will be emitted on the native side to indicate that the transcription chunk was received.
+ */
+const TRANSCRIPTION_CHUNK_RECEIVED = 'TRANSCRIPTION_CHUNK_RECEIVED';
+
+/**
+ * Logs when about the received transcription chunk.
+ *
+ * @param {string} transcriptMessageID - Transcription message id.
+ * @param {string} language - The language of the transcribed message.
+ * @param {Object} participant - The participant who send the message.
+ * @param {any} text - The message text.
+ * @param {any} _store - The store.
+ * @returns {Event}
+ */
+export const notifyTranscriptionChunkReceived = (transcriptMessageID: string, language: string, participant: Object, text: any, _store?: any) =>
+    sendEvent(
+        _store,
+        TRANSCRIPTION_CHUNK_RECEIVED,
+        {
+            messageID: transcriptMessageID,
+            language,
+            participant,
+            text
+        });

react/features/subtitles/functions.web.ts

@@ -0,0 +1,19 @@
+/* eslint-disable max-params, max-len */
+
+/**
+ * Logs when about the received transcription chunk.
+ *
+ * @param {string} transcriptMessageID - Transcription message id.
+ * @param {string} language - The language of the transcribed message.
+ * @param {Object} participant - The participant who send the message.
+ * @param {any} text - The message text.
+ * @param {any} _store - The store.
+ * @returns {Event}
+ */
+export const notifyTranscriptionChunkReceived = (transcriptMessageID: string, language: string, participant: Object, text: any, _store?: any) =>
+    APP.API.notifyTranscriptionChunkReceived({
+        messageID: transcriptMessageID,
+        language,
+        participant,
+        ...text
+    });

react/features/subtitles/middleware.ts

@@ -12,6 +12,8 @@ import {
     removeTranscriptMessage,
     updateTranscriptMessage
 } from './actions.any';
+import { notifyTranscriptionChunkReceived } from './functions';
+
 
 /**
  * The type of json-message which indicates that json carries a
@@ -43,7 +45,7 @@ const P_NAME_TRANSLATION_LANGUAGE = 'translation_language';
 const REMOVE_AFTER_MS = 3000;
 
 /**
- * Stability factor for a trancription. We'll treat a transcript as stable
+ * Stability factor for a transcription. We'll treat a transcript as stable
  * beyond this value.
  */
 const STABLE_TRANSCRIPTION_FACTOR = 0.85;
@@ -89,13 +91,14 @@ MiddlewareRegistry.register(store => next => action => {
  * @private
  * @returns {Object} The value returned by {@code next(action)}.
  */
-function _endpointMessageReceived({ dispatch, getState }: IStore, next: Function, action: AnyAction) {
+function _endpointMessageReceived(store: IStore, next: Function, action: AnyAction) {
     const { data: json } = action;
 
     if (![ JSON_TYPE_TRANSCRIPTION_RESULT, JSON_TYPE_TRANSLATION_RESULT ].includes(json?.type)) {
         return next(action);
     }
 
+    const { dispatch, getState } = store;
     const state = getState();
     const language
         = state['features/base/conference'].conference
@@ -129,7 +132,7 @@ function _endpointMessageReceived({ dispatch, getState }: IStore, next: Function
         const { text } = json.transcript[0];
 
         // First, notify the external API.
-        if (typeof APP !== 'undefined' && !(json.is_interim && skipInterimTranscriptions)) {
+        if (!(json.is_interim && skipInterimTranscriptions)) {
             const txt: any = {};
 
             if (!json.is_interim) {
@@ -140,26 +143,31 @@ function _endpointMessageReceived({ dispatch, getState }: IStore, next: Function
                 txt.unstable = text;
             }
 
-            APP.API.notifyTranscriptionChunkReceived({
-                messageID: transcriptMessageID,
-                language: json.language,
+            notifyTranscriptionChunkReceived(
+                transcriptMessageID,
+                json.language,
                 participant,
-                ...txt
-            });
-
-            // Dump transcript in a <transcript> element for debugging purposes.
-            if (!json.is_interim && dumpTranscript) {
-                try {
-                    let elem = document.body.getElementsByTagName('transcript')[0];
-
-                    if (!elem) {
-                        elem = document.createElement('transcript');
-                        document.body.appendChild(elem);
+                txt,
+                store
+            );
+
+            if (navigator.product !== 'ReactNative') {
+
+                // Dump transcript in a <transcript> element for debugging purposes.
+                if (!json.is_interim && dumpTranscript) {
+                    try {
+                        let elem = document.body.getElementsByTagName('transcript')[0];
+
+                        // eslint-disable-next-line max-depth
+                        if (!elem) {
+                            elem = document.createElement('transcript');
+                            document.body.appendChild(elem);
+                        }
+
+                        elem.append(`${new Date(json.timestamp).toISOString()} ${participant.name}: ${text}`);
+                    } catch (_) {
+                        // Ignored.
                     }
-
-                    elem.append(`${new Date(json.timestamp).toISOString()} ${participant.name}: ${text}`);
-                } catch (_) {
-                    // Ignored.
                 }
             }
         }

react/features/subtitles/reducer.ts

@@ -2,7 +2,9 @@ import ReducerRegistry from '../base/redux/ReducerRegistry';
 
 import {
     REMOVE_TRANSCRIPT_MESSAGE,
-    SET_REQUESTING_SUBTITLES, TOGGLE_REQUESTING_SUBTITLES, UPDATE_TRANSCRIPT_MESSAGE
+    SET_REQUESTING_SUBTITLES,
+    TOGGLE_REQUESTING_SUBTITLES,
+    UPDATE_TRANSCRIPT_MESSAGE
 } from './actionTypes';
 
 /**