W3C WebRTC
WG Meeting
July 22, 2021
10:00 AM - Noon Pacific Time
1
Chairs: Bernard Aboba
Harald Alvestrand
Jan-Ivar Bruaroey
W3C WG IPR Policy
2
Welcome!
3
About this Virtual Meeting
4
Understanding Document Status
5
W3C Code of Conduct
6
Virtual Interim Meeting Tips
This session is being recorded
7
Issues for Discussion Today
Time control:
8
#29 MediaStreamTrack Transfer: Summary
9
#29 MediaStreamTrack Transfer: editorial finalization
10
Discussion (10:15 to 10:25)
11
Mediacapture-transform Alternative Proposal
(10:25 - 10:45)
12
#23 Towards more JS in MediaStreamTrack
13
#23 Why only focusing on video?
14
#23 Why not using WhatWG streams?
15
#23 Goal 1: Reading a video MediaStreamTrack (1/3)
16
async function transform(frame) {� ...�}�// start processing�track.processVideoFrame = transform(frame)�// finish processing�track.processVideoFrame = null;�
partial interface MediaStreamTrack {� [Exposed=Worker] attribute VideoFrameCallback processVideoFrame;�};�callback VideoFrameCallback = Promise<undefined>(VideoFrame);�
#23 Goal 1: Reading a video MediaStreamTrack (2/3)
17
async function transform(f) {� // Use ‘f’ as long as you are in� ...� // ‘f’ will be closed when exiting�}��track.processVideoFrame = (f) => {� // transform returns a promise� return transform(f);�}�
#23 Goal 1: Reading a video MediaStreamTrack (3/3)
18
async function transform(frame) {� webCodecEncoder.encode(frame);�}��track.processVideoFrame = transform;
async function transform(frame) {� webCodecEncoder.encode(frame);�}��const processor = new MediaStreamTrackProcess(track);�const reader = processor.readable.getReader();�let chunk = await reader.read();�while (!chunk.done) {� await transform(chunk.data);� // Probably need try/catch to ensure frame gets always closed� chunk.data.close();� chunk = await reader.read();�};
#23 Goal 2: MediaStreamTrack JS Source - level 1
�
19
const stream = await getUserMedia({video: true});�const originalTrack = stream.getVideoTracks()[0];��// originalTrack --> transform --> transformedTrack�const source = {� start: (controller) => {� originalTrack.processVideoFrame = async (frame) => {� // Tranform frame from originalTrack� const newFrame = await transform(frame);� // Enqueue newFrame in transformedTrack pipeline� controller.enqueue(newFrame);� };� // Stop track when originalTrack is stopped� originalTrack.onended = () => controller.stop();� },� // When track is stopped, stop frame transforming� stop: () => originalTrack.processVideoFrame = null�};�const transformedTrack = MediaStreamTrack.createVideoTrack(source);��// Use transformedTrack instead of originalTrack�pc.addTrack(transformedTrack, stream);
partial interface MediaStreamTrack {� [Exposed=Worker] static MediaStreamTrack� createVideoTrack(VideoSource source);�};��dictionary VideoSource {� // Called synchronously by createVideoTrack� VideoSourceStartCallback start;� // Called when track.stop is called� VideoSourceStopCallback stop;�};��[Exposed=Worker]�interface VideoSourceController {� undefined enqueue(VideoFrame frame);� undefined stop();�};��callback VideoSourceStartCallback =� any(VideoSourceController);�callback VideoSourceStopCallback = any();�
#23 MediaStreamTrack & WhatWG streams
20
// MediaStreamTrackGenerator equivalent��function createMyMediaStreamTrackGenerator() {� const controllers = { };� // Create a MediaStreamTrack object� const generatedTrack = MediaStreamTrack.createVideoTrack({� start: (c) => controllers.track = c,� stop: () => controllers.stream.error()� });� // Create a WritableStream� generatedTrack.writable = new WritableStream({� start : (c) => controllers.stream = c,� write : (f) => controllers.track.enqueue(f),� close : () => controllers.track.stop(),� abort : () => controllers.track.stop()� });� return generatedTrack;�}
// MediaStreamTrackProcessor equivalent��class MyMediaStreamTrackProcessor {� constructor(track) {� this.readable = new ReadableStream({� start : (c) => {� track.onended = () => c.close();� const p = new Promise(r => this.firstFrame = r);� track.processVideoFrame = (frame) => {� c.enqueue(frame.clone());� this.firstFrame();� return new Promise(r => this.nextFrame = r);� };� return p;� },� pull : (c) => this.nextFrame(),� close : () => track.processVideoFrame = null,� });� }�}
#23 Goal 2: MediaStreamTrack JS Source - level 1
�
21
MediaStreamTrack.createVideoTrack(VideoTrackSource source)
vs.
MediaStreamTrack.createVideoTrack(WritableStream stream)
#23 Goal 2: MediaStreamTrack JS Source - level 2
22
const transformedTrack = MediaStreamTrack.createVideoTrack({� start: (c) => {� originalTrack.processVideoFrame = transform;� // Handle camera track muted state.� originalTrack.onmuted = () => c.muted = true;� originalTrack.onunmuted = () => c.muted = false;� }�});�...�transformedTrack.onmuted = () => { ... };�pc.addTrack(transformedTrack, stream);
partial interface VideoSourceController {� attribute boolean muted;�};�
#23 Goal 2: MediaStreamTrack JS Source - level 3
23
#23 MediaStreamTrack JS Source - the whole thing!
24
// video-call-with-a-dragon-head.html�async function getCameraTrackWithDragonHead()�{� const stream = await� navigator.mediaDevices.getUserMedia({video:true});� const source = stream.getVideoTracks()[0];� const worker = new Worker("dragon-head-worker.js");� let resolve;� const promise = new Promise(r => resolve = r);� worker.onmessage = event => resolve(event.data.track);� worker.postMessage({track:source}, [source]);� return promise;�}��const track = await getCameraTrackWithDragonHead();��// Use track as if coming from getUserMedia, including:�// - Using clone, applyConstraints et al�// - Receiving muted/unmuted events from source�// - Set enabled=true/false, propagating to source�...�track.onmuted = () => { ... };
// dragon-head-worker.js�onmessage = (event) => {� const track = event.data.track;� const transformed = MediaStreamTrack.createVideoTrack({� start: (c) => {� this.controller = c;� track.onended = () => c.stop();� // transform then enqueue each camera frame� track.processVideoFrame =� async (f) => c.enqueue(await transformHead(f));� // propagate camera muted state to track� track.onmuted = () => c.muted = true;� track.onunmuted = () => c.muted = false;� // propagate camera settings to track� c.capabilities = track.capabilities();� c.constraints = track.constraints();� c.settings = track.settings();� },� applyConstraints: async (constraints) => {� // propagate applyConstraints to camera� await track.applyConstraints(constraints);� this.controller.constraints = track.constraints();� this.controller.settings = track.settings();� },� // propagate enabled to camera� enabledChanged: (value) => track.enabled = value,� stop: () => track.processVideoFrame = null� });� self.postMessage({track:transformed}, [transformed]);�});
#23 Beyond Goal 2
25
#23 Tentative conclusion
26
Discussion (10:45 - 11:00)
27
Screen Capture (Jan-Ivar)
28
Screen-capture
#1 use case:
“Screen sharing
Using WebRTC” =
VC Presentations
2021 =
web presentations
#1 use case is�unsafe!
We must fix this!
From presentation I gave @ W3C 2021 AC Meeting
#182 Safer presentations even in getDisplayMedia
Today’s choices, “Entire Screen”, “Window”, “Tab” are all unsafe!*�*Over-sharing (desktop, accidental back button or tab flip), even active malicious attacks on web’s same-origin policy
New (safe) choice: “Web Page”
#182 Safer presentations even in getDisplayMedia
What does the spec need to facilitate this new choice in User Agents?�
#158 crop MediaStream from the share-this-tab API
Proposal from Youenn:
navigator.mediaDevices.getTabViewportMedia() // tab viewport
document.getViewportMedia() // captures document viewport
iframe.getViewportMedia() // captures iframe viewport
All 3 call the same underlying algorithm taking a viewport as internal parameter,�which is responsible of permission policy, prompting, creation of track.
With this approach, there is no default option to select in the spec and no surprise from web developers on what they will get.
Easy feature detection of gradual support. Follow-up Q: Permissions policy?�Has site-isolation & capture opt-in been resolved?
Discussion (11:15-11:30)
34
Wrap-up and Next Steps (11:30-12:00)
35
For extra credit
36
Name the bird!
Thank you
Special thanks to:
WG Participants, Editors & Chairs
The mammal
37