Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix error when spamming browser speech recognition button #881

Merged
merged 1 commit into from
Mar 7, 2018
Merged

Fix error when spamming browser speech recognition button #881

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion src/CognitiveServices/SpeechRecognition.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,7 @@ export class SpeechRecognizer implements Speech.ISpeechRecognizer {
});
}

this.actualRecognizer.Recognize(eventhandler, speechContext);
return this.actualRecognizer.Recognize(eventhandler, speechContext);
}

public speechIsAvailable(){
Expand All @@ -143,7 +143,10 @@ export class SpeechRecognizer implements Speech.ISpeechRecognizer {
if (this.actualRecognizer != null) {
this.actualRecognizer.AudioSource.TurnOff();
}

this.isStreamingToService = false;

return Promise.resolve();
}

private log(message: string) {
Expand Down
24 changes: 12 additions & 12 deletions src/Shell.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,12 @@ import { classList } from './Chat';
import { Dispatch, connect } from 'react-redux';
import { Strings } from './Strings';
import { Speech } from './SpeechModule'
import { ChatActions, sendMessage, sendFiles } from './Store';
import { ChatActions, ListeningState, sendMessage, sendFiles } from './Store';

interface Props {
inputText: string,
strings: Strings,
listening: boolean,
listeningState: ListeningState,

onChangeText: (inputText: string) => void

Expand Down Expand Up @@ -66,15 +66,15 @@ class ShellContainer extends React.Component<Props> implements ShellFunctions {
}

private onTextInputFocus(){
if (this.props.listening) {
if (this.props.listeningState === ListeningState.STARTED) {
this.props.stopListening();
}
}

private onClickMic() {
if (this.props.listening) {
if (this.props.listeningState === ListeningState.STARTED) {
this.props.stopListening();
} else {
} else if (this.props.listeningState === ListeningState.STOPPED) {
this.props.startListening();
}
}
Expand All @@ -93,7 +93,7 @@ class ShellContainer extends React.Component<Props> implements ShellFunctions {
this.props.inputText.length > 0 && 'has-text'
);

const showMicButton = this.props.listening || (Speech.SpeechRecognizer.speechIsAvailable() && !this.props.inputText.length);
const showMicButton = this.props.listeningState !== ListeningState.STOPPED || (Speech.SpeechRecognizer.speechIsAvailable() && !this.props.inputText.length);

const sendButtonClassName = classList(
'wc-send',
Expand All @@ -103,11 +103,11 @@ class ShellContainer extends React.Component<Props> implements ShellFunctions {
const micButtonClassName = classList(
'wc-mic',
!showMicButton && 'hidden',
this.props.listening && 'active',
!this.props.listening && 'inactive'
this.props.listeningState === ListeningState.STARTED && 'active',
this.props.listeningState !== ListeningState.STARTED && 'inactive'
);

const placeholder = this.props.listening ? this.props.strings.listeningIndicator : this.props.strings.consolePlaceholder;
const placeholder = this.props.listeningState === ListeningState.STARTED ? this.props.strings.listeningIndicator : this.props.strings.consolePlaceholder;

return (
<div className={ className }>
Expand Down Expand Up @@ -184,11 +184,11 @@ export const Shell = connect(
// only used to create helper functions below
locale: state.format.locale,
user: state.connection.user,
listening : state.shell.listening
listeningState: state.shell.listeningState
}), {
// passed down to ShellContainer
onChangeText: (input: string) => ({ type: 'Update_Input', input, source: "text" } as ChatActions),
stopListening: () => ({ type: 'Listening_Stop' }),
stopListening: () => ({ type: 'Listening_Stopping' }),
startListening: () => ({ type: 'Listening_Starting' }),
// only used to create helper functions below
sendMessage,
Expand All @@ -197,7 +197,7 @@ export const Shell = connect(
// from stateProps
inputText: stateProps.inputText,
strings: stateProps.strings,
listening : stateProps.listening,
listeningState: stateProps.listeningState,
// from dispatchProps
onChangeText: dispatchProps.onChangeText,
// helper functions
Expand Down
35 changes: 22 additions & 13 deletions src/SpeechModule.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@ export module Speech {
onRecognitionFailed: Action;

warmup(): void;
startRecognizing(): void;
stopRecognizing(): void;
startRecognizing(): Promise<void>;
stopRecognizing(): Promise<void>;
speechIsAvailable() : boolean;
}

Expand Down Expand Up @@ -53,19 +53,22 @@ export module Speech {
SpeechRecognizer.instance.onFinalResult = onFinalResult;
SpeechRecognizer.instance.onAudioStreamingToService = onAudioStreamStarted;
SpeechRecognizer.instance.onRecognitionFailed = onRecognitionFailed;
SpeechRecognizer.instance.startRecognizing();

return SpeechRecognizer.instance.startRecognizing();
}

public static stopRecognizing() {
if (!SpeechRecognizer.speechIsAvailable())
if (!SpeechRecognizer.speechIsAvailable()) {
return;
}

SpeechRecognizer.instance.stopRecognizing();
return SpeechRecognizer.instance.stopRecognizing();
}

public static warmup() {
if (!SpeechRecognizer.speechIsAvailable())
if (!SpeechRecognizer.speechIsAvailable()) {
return;
}

SpeechRecognizer.instance.warmup();
}
Expand Down Expand Up @@ -118,7 +121,7 @@ export module Speech {
console.error("This browser does not support speech recognition");
return;
}

this.recognizer = new (<any>window).webkitSpeechRecognition();
this.recognizer.lang = 'en-US';
this.recognizer.interimResults = true;
Expand Down Expand Up @@ -163,11 +166,17 @@ export module Speech {
}

public startRecognizing() {
this.recognizer.start();
return new Promise<void>(resolve => {
this.recognizer.onstart = () => resolve();
this.recognizer.start();
});
}

public stopRecognizing() {
this.recognizer.stop();
return new Promise<void>(resolve => {
this.recognizer.onend = () => resolve();
this.recognizer.stop();
});
}
}

Expand Down Expand Up @@ -292,10 +301,10 @@ export module Speech {
}
}

// process SSML markup into an array of either
// process SSML markup into an array of either
// * utterenance
// * number which is delay in msg
// * url which is an audio file
// * url which is an audio file
private processNodes(nodes: NodeList, output: any[]): void {
for (let i = 0; i < nodes.length; i++) {
const node = nodes[i];
Expand Down Expand Up @@ -327,7 +336,7 @@ export module Speech {
break;
case 'say-as':
case 'prosody': // ToDo: handle via msg.rate
case 'emphasis': // ToDo: can probably emulate via prosody + pitch
case 'emphasis': // ToDo: can probably emulate via prosody + pitch
case 'w':
case 'phoneme': //
case 'voice':
Expand Down Expand Up @@ -375,4 +384,4 @@ export module Speech {
get speakChunks(): any[] { return this._speakChunks; }
get lang(): string { return this._lang; }
}
}
}
68 changes: 46 additions & 22 deletions src/Store.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,13 @@ import * as konsole from './Konsole';

import { Reducer } from 'redux';

export enum ListeningState {
STOPPED,
STARTING,
STARTED,
STOPPING
}

export const sendMessage = (text: string, from: User, locale: string) => ({
type: 'Send_Message',
activity: {
Expand Down Expand Up @@ -46,7 +53,7 @@ const attachmentsFromFiles = (files: FileList) => {
export interface ShellState {
sendTyping: boolean
input: string
listening: boolean
listeningState: ListeningState
lastInputViaSpeech : boolean
}

Expand All @@ -58,6 +65,8 @@ export type ShellAction = {
type: 'Listening_Starting'
} | {
type: 'Listening_Start'
} | {
type: 'Listening_Stopping'
} | {
type: 'Listening_Stop'
} | {
Expand All @@ -81,51 +90,62 @@ export const shell: Reducer<ShellState> = (
state: ShellState = {
input: '',
sendTyping: false,
listening : false,
listeningState: ListeningState.STOPPED,
lastInputViaSpeech : false
},
action: ShellAction
) => {
switch (action.type) {
case 'Update_Input':
return {
... state,
...state,
input: action.input,
lastInputViaSpeech : action.source == "speech"
};

case 'Listening_Start':
return {
... state,
listening: true
...state,
listeningState: ListeningState.STARTED
};

case 'Listening_Stop':
return {
... state,
listening: false
...state,
listeningState: ListeningState.STOPPED
};

case 'Listening_Starting':
return {
...state,
listeningState: ListeningState.STARTING
};

case 'Listening_Stopping':
return {
...state,
listeningState: ListeningState.STOPPING
};

case 'Send_Message':
return {
... state,
...state,
input: ''
};

case 'Set_Send_Typing':
return {
... state,
...state,
sendTyping: action.sendTyping
};

case 'Card_Action_Clicked':
case 'Card_Action_Clicked':
return {
... state,
lastInputViaSpeech : false
...state,
lastInputViaSpeech : false
};

default:
case 'Listening_Starting':
return state;
}
}
Expand Down Expand Up @@ -570,7 +590,7 @@ const speakSSMLEpic: Epic<ChatActions, ChatState> = (action$, store) =>
return call$.map(onSpeakingFinished)
.catch(error => Observable.of(nullAction));
})
.merge(action$.ofType('Speak_SSML').map(_ => ({ type: 'Listening_Stop' } as ShellAction)));
.merge(action$.ofType('Speak_SSML').map(_ => ({ type: 'Listening_Stopping' } as ShellAction)));

const speakOnMessageReceivedEpic: Epic<ChatActions, ChatState> = (action$, store) =>
action$.ofType('Receive_Message')
Expand All @@ -588,13 +608,17 @@ const stopSpeakingEpic: Epic<ChatActions, ChatState> = (action$) =>
.do(Speech.SpeechSynthesizer.stopSpeaking)
.map(_ => nullAction)

const stopListeningEpic: Epic<ChatActions, ChatState> = (action$) =>
const stopListeningEpic: Epic<ChatActions, ChatState> = (action$, store) =>
action$.ofType(
'Listening_Stop',
'Listening_Stopping',
'Card_Action_Clicked'
)
.do(Speech.SpeechRecognizer.stopRecognizing)
.map(_ => nullAction)
.do(() => {
Speech.SpeechRecognizer.stopRecognizing().then(() => {
store.dispatch({ type: 'Listening_Stop' });
});
})
.map(_ => nullAction);

const startListeningEpic: Epic<ChatActions, ChatState> = (action$, store) =>
action$.ofType('Listening_Starting')
Expand All @@ -604,21 +628,21 @@ const startListeningEpic: Epic<ChatActions, ChatState> = (action$, store) =>
var onFinalResult = (srText : string) => {
srText = srText.replace(/^[.\s]+|[.\s]+$/g, "");
onIntermediateResult(srText);
store.dispatch({ type: 'Listening_Stop' });
store.dispatch({ type: 'Listening_Stopping' });
store.dispatch(sendMessage(srText, store.getState().connection.user, locale));
};
var onAudioStreamStart = () => { store.dispatch({ type: 'Listening_Start' }) };
var onRecognitionFailed = () => { store.dispatch({ type: 'Listening_Stop' })};
var onRecognitionFailed = () => { store.dispatch({ type: 'Listening_Stopping' })};
Speech.SpeechRecognizer.startRecognizing(locale, onIntermediateResult, onFinalResult, onAudioStreamStart, onRecognitionFailed);
})
.map(_ => nullAction)

const listeningSilenceTimeoutEpic: Epic<ChatActions, ChatState> = (action$, store) =>
{
const cancelMessages$ = action$.ofType('Update_Input', 'Listening_Stop');
const cancelMessages$ = action$.ofType('Update_Input', 'Listening_Stopping');
return action$.ofType('Listening_Start')
.mergeMap((action) =>
Observable.of(({ type: 'Listening_Stop' }) as ShellAction)
Observable.of(({ type: 'Listening_Stopping' }) as ShellAction)
.delay(5000)
.takeUntil(cancelMessages$));
};
Expand Down