Add support background filter for ios 16+ (#2484)

* Add background filter support to iOS

* Support background filter for iOS 16+

CHANGELOG.md

@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 
 - Add the audio output gain and frequency to the meeting readiness checker's configuration. The readiness checker uses this value to set the "Play Tone" gain and frequency.
+- Add support for background filter starting from iOS 16 for major browsers Safari, Chrome, and Firefox (except on iPad).
 
 ### Removed
 

demos/browser/app/meetingV2/meetingV2.ts

@@ -408,7 +408,7 @@ export class DemoMeetingApp
     }
 
     (document.getElementById('sdk-version') as HTMLSpanElement).innerText =
-        'amazon-chime-sdk-js@' + Versioning.sdkVersion;
+      'amazon-chime-sdk-js@' + Versioning.sdkVersion;
     this.initEventListeners();
     this.initParameters();
     this.setMediaRegion();

docs/modules/backgroundfilter_video_processor.html

@@ -89,8 +89,45 @@ <h2>Can I use a background filter in my application?</h2>
 					<a href="#browser-compatibility" id="browser-compatibility" style="color: inherit; text-decoration: none;">
 						<h3>Browser compatibility</h3>
 					</a>
-					<p>A background filter in the Amazon Chime SDK for JavaScript works in Firefox, Chrome, and Chromium-based browsers (including Electron) on desktop, Android, and Safari 14.1 and later on macOS.</p>
-					<p>There is a known issue with <code>VideoFrameProcessor</code> in Safari 15: see <a href="https://github.com/aws/amazon-chime-sdk-js/issues/1059">github issue 1059</a></p>
+					<p>The APIs for video processing in Amazon Chime SDK for JavaScript work in Firefox, Chrome, Chromium-based browsers
+					(including Electron) on desktop, Android and iOS operating systems. A full compatibility table is below.</p>
+					<table>
+						<thead>
+							<tr>
+								<th>Browser</th>
+								<th>Minimum supported version</th>
+							</tr>
+						</thead>
+						<tbody><tr>
+								<td>Firefox</td>
+								<td>76</td>
+							</tr>
+							<tr>
+								<td>Chromium-based browsers and environments, including Edge and Electron</td>
+								<td>78</td>
+							</tr>
+							<tr>
+								<td>Android Chrome</td>
+								<td>78</td>
+							</tr>
+							<tr>
+								<td>Safari on MacOS</td>
+								<td>13.0</td>
+							</tr>
+							<tr>
+								<td>iOS Safari</td>
+								<td>16</td>
+							</tr>
+							<tr>
+								<td>iOS Chrome</td>
+								<td>16</td>
+							</tr>
+							<tr>
+								<td>iOS Firefox (Except on iPad)</td>
+								<td>16</td>
+							</tr>
+					</tbody></table>
+					<p>Note that there is a known issue with <code>VideoFrameProcessor</code> in Safari 15: see <a href="https://github.com/aws/amazon-chime-sdk-js/issues/1059">github issue 1059</a>. This has been fixed with Safari 16.</p>
 					<p>A background filter can be CPU-intensive and the web runtime affects performance. As such, not all mobile devices or lower-spec laptop or desktop computers will be sufficiently powerful, or will be able to use a background filter while also supporting multiple video streams and rich application functionality.</p>
 					<p>See the sections “<a href="#checking-for-support-before-offering-a-background-filter">Checking for support before offering a background filter</a>” for more details about checking for support.</p>
 					<a href="#simd-support" id="simd-support" style="color: inherit; text-decoration: none;">

docs/modules/videoprocessor.html

@@ -87,7 +87,8 @@ <h2>Introduction</h2>
 					<a href="#browser-compatibility" id="browser-compatibility" style="color: inherit; text-decoration: none;">
 						<h3>Browser compatibility</h3>
 					</a>
-					<p>The APIs for video processing in Amazon Chime SDK for JavaScript work in Firefox, Chrome, Chromium-based browsers (including Electron) on desktop, and Android operating systems. A full compatibility table is below. Currently, the APIs for video processing do not support Safari/Chrome/Firefox on iOS devices due to <a href="https://bugs.webkit.org/show_bug.cgi?id=181663">Webkit Bug 181663</a>.</p>
+					<p>The APIs for video processing in Amazon Chime SDK for JavaScript work in Firefox, Chrome, Chromium-based browsers
+					(including Electron) on desktop, Android and iOS operating systems. A full compatibility table is below.</p>
 					<table>
 						<thead>
 							<tr>
@@ -113,17 +114,18 @@ <h3>Browser compatibility</h3>
 							</tr>
 							<tr>
 								<td>iOS Safari</td>
-								<td>Not supported</td>
+								<td>16</td>
 							</tr>
 							<tr>
 								<td>iOS Chrome</td>
-								<td>Not supported</td>
+								<td>16</td>
 							</tr>
 							<tr>
-								<td>iOS Firefox</td>
-								<td>Not supported</td>
+								<td>iOS Firefox (Except on iPad)</td>
+								<td>16</td>
 							</tr>
 					</tbody></table>
+					<p>Note that there is a known issue with <code>VideoFrameProcessor</code> in Safari 15: see <a href="https://github.com/aws/amazon-chime-sdk-js/issues/1059">github issue 1059</a>. This has been fixed with Safari 16.</p>
 					<a href="#video-processing-apis" id="video-processing-apis" style="color: inherit; text-decoration: none;">
 						<h2>Video Processing APIs</h2>
 					</a>

guides/10_Video_Processor.md

@@ -16,17 +16,20 @@ A typical workflow would be:
 
 ### Browser compatibility
 
-The APIs for video processing in Amazon Chime SDK for JavaScript work in Firefox, Chrome, Chromium-based browsers (including Electron) on desktop, and Android operating systems. A full compatibility table is below. Currently, the APIs for video processing do not support Safari/Chrome/Firefox on iOS devices due to [Webkit Bug 181663](https://bugs.webkit.org/show_bug.cgi?id=181663).
+The APIs for video processing in Amazon Chime SDK for JavaScript work in Firefox, Chrome, Chromium-based browsers 
+(including Electron) on desktop, Android and iOS operating systems. A full compatibility table is below.
 
 |Browser                                                                |Minimum supported version  
 |---                                                                    |---
 |Firefox                                                                |76
 |Chromium-based browsers and environments, including Edge and Electron  |78
 |Android Chrome                                                         |78
 |Safari on MacOS                                                        |13.0
-|iOS Safari                                                             |Not supported
-|iOS Chrome                                                             |Not supported
-|iOS Firefox                                                            |Not supported
+|iOS Safari                                                             |16
+|iOS Chrome                                                             |16
+|iOS Firefox (Except on iPad)                                           |16
+
+Note that there is a known issue with `VideoFrameProcessor` in Safari 15: see [github issue 1059](https://github.com/aws/amazon-chime-sdk-js/issues/1059). This has been fixed with Safari 16.
 
 ## Video Processing APIs
 

guides/15_Background_Filter_Video_Processor.md

@@ -17,9 +17,20 @@ Background replacement is available as part of the Amazon Chime SDK for JavaScri
 
 ### Browser compatibility
 
-A background filter in the Amazon Chime SDK for JavaScript works in Firefox, Chrome, and Chromium-based browsers (including Electron) on desktop, Android, and Safari 14.1 and later on macOS. 
-
-There is a known issue with `VideoFrameProcessor` in Safari 15: see [github issue 1059](https://github.com/aws/amazon-chime-sdk-js/issues/1059)
+The APIs for video processing in Amazon Chime SDK for JavaScript work in Firefox, Chrome, Chromium-based browsers
+(including Electron) on desktop, Android and iOS operating systems. A full compatibility table is below.
+
+|Browser                                                                |Minimum supported version
+|---                                                                    |---
+|Firefox                                                                |76
+|Chromium-based browsers and environments, including Edge and Electron  |78
+|Android Chrome                                                         |78
+|Safari on MacOS                                                        |13.0
+|iOS Safari                                                             |16
+|iOS Chrome                                                             |16
+|iOS Firefox (Except on iPad)                                           |16
+
+Note that there is a known issue with `VideoFrameProcessor` in Safari 15: see [github issue 1059](https://github.com/aws/amazon-chime-sdk-js/issues/1059). This has been fixed with Safari 16.
 
 A background filter can be CPU-intensive and the web runtime affects performance. As such, not all mobile devices or lower-spec laptop or desktop computers will be sufficiently powerful, or will be able to use a background filter while also supporting multiple video streams and rich application functionality.
 

src/browserbehavior/DefaultBrowserBehavior.ts

@@ -2,12 +2,15 @@
 // SPDX-License-Identifier: Apache-2.0
 
 import { detect } from 'detect-browser';
+import { UAParser } from 'ua-parser-js';
 
 import BrowserBehavior from './BrowserBehavior';
 import ExtendedBrowserBehavior from './ExtendedBrowserBehavior';
 
 export default class DefaultBrowserBehavior implements BrowserBehavior, ExtendedBrowserBehavior {
   private readonly browser = detect();
+  private readonly uaParserResult =
+    navigator && navigator.userAgent ? new UAParser(navigator.userAgent).getResult() : null;
 
   private browserSupport: { [id: string]: number } = {
     chrome: 78,
@@ -48,6 +51,7 @@ export default class DefaultBrowserBehavior implements BrowserBehavior, Extended
   ];
 
   private webkitBrowsers: string[] = ['crios', 'fxios', 'safari', 'ios', 'ios-webview'];
+  private static MIN_IOS_SUPPORT_CANVAS_STREAM_PLAYBACK = 16;
 
   version(): string {
     return this.browser.version;
@@ -57,6 +61,10 @@ export default class DefaultBrowserBehavior implements BrowserBehavior, Extended
     return parseInt(this.version().split('.')[0]);
   }
 
+  osMajorVersion(): number {
+    return parseInt(this.uaParserResult.os.version.split('.')[0]);
+  }
+
   name(): string {
     return this.browser.name;
   }
@@ -88,7 +96,10 @@ export default class DefaultBrowserBehavior implements BrowserBehavior, Extended
   }
 
   supportsCanvasCapturedStreamPlayback(): boolean {
-    return !this.isIOSSafari() && !this.isIOSChrome() && !this.isIOSFirefox();
+    return (
+      (!this.isIOSSafari() && !this.isIOSChrome() && !this.isIOSFirefox()) ||
+      this.osMajorVersion() >= DefaultBrowserBehavior.MIN_IOS_SUPPORT_CANVAS_STREAM_PLAYBACK
+    );
   }
 
   supportsBackgroundFilter(): boolean {

test/backgroundprocessor/BackgroundBlurProcessor.test.ts

@@ -690,6 +690,9 @@ describe('BackgroundBlurProcessor', () => {
         const supportedUserAgents = [
           'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3865.75 Safari/537.36',
           'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.2 Safari/605.1.15',
+          'Mozilla/5.0 (iPhone; CPU iPhone OS 16_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.0 Mobile/15E148 Safari/604.1',
+          'Mozilla/5.0 (iPhone; CPU iPhone OS 16_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/107.0.5304.66 Mobile/15E148 Safari/604.1',
+          'Mozilla/5.0 (iPhone; CPU iPhone OS 16_0_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) FxiOS/106.0 Mobile/15E148 Safari/605.1.15',
         ];
         for (const supportedUserAgent of supportedUserAgents) {
           setUserAgent(supportedUserAgent);

test/browserbehavior/DefaultBrowserBehavior.test.ts

@@ -78,6 +78,17 @@ describe('DefaultBrowserBehavior', () => {
     mockBuilder.cleanup();
   });
 
+  describe('construction', () => {
+    it('can be constructed without user agent', () => {
+      const userAgent = navigator.userAgent;
+      // @ts-ignore
+      delete navigator.userAgent;
+      new DefaultBrowserBehavior();
+      // @ts-ignore
+      navigator.userAgent = userAgent;
+    });
+  });
+
   describe('platforms', () => {
     it('can detect Firefox', () => {
       setUserAgent(FIREFOX_MAC_USER_AGENT);

test/task/CleanRestartedSessionTask.test.ts

@@ -6,6 +6,7 @@ import * as chai from 'chai';
 import { NScaleVideoUplinkBandwidthPolicy } from '../../src';
 import AudioVideoControllerState from '../../src/audiovideocontroller/AudioVideoControllerState';
 import NoOpAudioVideoController from '../../src/audiovideocontroller/NoOpAudioVideoController';
+import BrowserBehavior from '../../src/browserbehavior/BrowserBehavior';
 import DefaultBrowserBehavior from '../../src/browserbehavior/DefaultBrowserBehavior';
 import ConnectionHealthData from '../../src/connectionhealthpolicy/ConnectionHealthData';
 import SignalingAndMetricsConnectionMonitor from '../../src/connectionmonitor/SignalingAndMetricsConnectionMonitor';
@@ -37,11 +38,12 @@ describe('CleanRestartedSessionTask', () => {
   let domMockBuilder: DOMMockBuilder;
   let domMockBehavior: DOMMockBehavior;
   let task: Task;
-  const browserBehavior = new DefaultBrowserBehavior();
+  let browserBehavior: BrowserBehavior;
 
   beforeEach(() => {
     domMockBehavior = new DOMMockBehavior();
     domMockBuilder = new DOMMockBuilder(domMockBehavior);
+    browserBehavior = new DefaultBrowserBehavior();
     context = new AudioVideoControllerState();
     context.audioVideoController = new NoOpAudioVideoController();
     context.logger = context.audioVideoController.logger;

test/task/CleanStoppedSessionTask.test.ts

@@ -8,6 +8,7 @@ import AudioVideoController from '../../src/audiovideocontroller/AudioVideoContr
 import AudioVideoControllerState from '../../src/audiovideocontroller/AudioVideoControllerState';
 import NoOpAudioVideoController from '../../src/audiovideocontroller/NoOpAudioVideoController';
 import FullJitterBackoff from '../../src/backoff/FullJitterBackoff';
+import BrowserBehavior from '../../src/browserbehavior/BrowserBehavior';
 import DefaultBrowserBehavior from '../../src/browserbehavior/DefaultBrowserBehavior';
 import ConnectionMonitor from '../../src/connectionmonitor/ConnectionMonitor';
 import Logger from '../../src/logger/Logger';
@@ -40,10 +41,10 @@ describe('CleanStoppedSessionTask', () => {
   const RECONNECT_SHORT_BACKOFF_MS = 1 * 1000;
   const RECONNECT_LONG_BACKOFF_MS = 5 * 1000;
   const behavior = new DOMMockBehavior();
-  const browserBehavior = new DefaultBrowserBehavior();
 
   let context: AudioVideoControllerState;
   let domMockBuilder: DOMMockBuilder | null = null;
+  let browserBehavior: BrowserBehavior;
   let task: Task;
   let webSocketAdapter: DefaultWebSocketAdapter;
   let signalingClient: SignalingClient;
@@ -66,6 +67,7 @@ describe('CleanStoppedSessionTask', () => {
 
   beforeEach(async () => {
     domMockBuilder = new DOMMockBuilder(behavior);
+    browserBehavior = new DefaultBrowserBehavior();
     context = new AudioVideoControllerState();
     context.audioVideoController = new NoOpAudioVideoController();
     context.logger = context.audioVideoController.logger;

test/task/CreatePeerConnectionTask.test.ts

@@ -43,7 +43,7 @@ describe('CreatePeerConnectionTask', () => {
   let context: AudioVideoControllerState;
   let domMockBuilder: DOMMockBuilder | null = null;
   let task: Task;
-  const browser: BrowserBehavior = new DefaultBrowserBehavior();
+  let browser: BrowserBehavior;
 
   function makeICEEvent(candidateStr: string | null): RTCPeerConnectionIceEvent {
     let iceCandidate: RTCIceCandidate = null;
@@ -61,6 +61,7 @@ describe('CreatePeerConnectionTask', () => {
   beforeEach(() => {
     domMockBehavior = new DOMMockBehavior();
     domMockBuilder = new DOMMockBuilder(domMockBehavior);
+    browser = new DefaultBrowserBehavior();
 
     context = new AudioVideoControllerState();
     context.audioVideoController = new NoOpAudioVideoController();

test/task/CreateSDPTask.test.ts

@@ -22,12 +22,13 @@ describe('CreateSDPTask', () => {
   let context: AudioVideoControllerState;
   let domMockBuilder: DOMMockBuilder;
   let domMockBehavior: DOMMockBehavior;
-  const browser: BrowserBehavior = new DefaultBrowserBehavior();
+  let browser: BrowserBehavior;
   let task: Task;
 
   beforeEach(() => {
     domMockBehavior = new DOMMockBehavior();
     domMockBuilder = new DOMMockBuilder(domMockBehavior);
+    browser = new DefaultBrowserBehavior();
     context = new AudioVideoControllerState();
     context.audioVideoController = new NoOpAudioVideoController();
     context.transceiverController = new DefaultTransceiverController(context.logger, browser);

test/task/LeaveAndReceiveLeaveAckTask.test.ts

@@ -33,7 +33,7 @@ describe('LeaveAndReceiveLeaveAckTask', () => {
   let signalingClient: SignalingClient;
   let leaveAckBuffer: Uint8Array;
   let request: SignalingClientConnectionRequest;
-  const browser: BrowserBehavior = new DefaultBrowserBehavior();
+  let browser: BrowserBehavior;
 
   function makeLeaveAckFrame(): Uint8Array {
     const frame = SdkLeaveAckFrame.create();
@@ -50,6 +50,7 @@ describe('LeaveAndReceiveLeaveAckTask', () => {
 
   beforeEach(() => {
     domMockBuilder = new DOMMockBuilder(behavior);
+    browser = new DefaultBrowserBehavior();
     context = new AudioVideoControllerState();
     context.audioVideoController = new NoOpAudioVideoController();
     context.transceiverController = new DefaultTransceiverController(context.logger, browser);

test/task/PromoteToPrimaryMeetingTask.test.ts

@@ -39,7 +39,7 @@ describe('PromoteToPrimaryMeetingTask', () => {
   let signalingClient: SignalingClient;
   let primaryMeetingJoinAck: Uint8Array;
   let request: SignalingClientConnectionRequest;
-  const browser: BrowserBehavior = new DefaultBrowserBehavior();
+  let browser: BrowserBehavior;
 
   function makePrimaryMeetingJoinAckFrame(errorStatus: number = undefined): Uint8Array {
     const frame = SdkPrimaryMeetingJoinAckFrame.create();
@@ -61,6 +61,7 @@ describe('PromoteToPrimaryMeetingTask', () => {
 
   beforeEach(() => {
     domMockBuilder = new DOMMockBuilder(behavior);
+    browser = new DefaultBrowserBehavior();
     context = new AudioVideoControllerState();
     context.audioVideoController = new NoOpAudioVideoController();
     context.transceiverController = new DefaultTransceiverController(context.logger, browser);