DEV Community 👩‍💻👨‍💻

Masui Masanori
Masui Masanori

Posted on

[Pion/WebRTC] Enabling and disabling the video track

Intro

When participating in an online meeting such as Microsoft Teams or Zoom, I can choose to share or not share video.
This time, I will try implementing it on my application.

Examples

Modifying a MediaStream

To change media streams during a session, it must be negotiated by sending offers and answers as at initiating connection.
Since it is the answer-side(client-side) that wants to change the media streams, I added a function to send messages to the offer-side(server-side) to request updating the session.

sseHub.go

...
func handleReceivedMessage(h *SSEHub, message ClientMessage) {
    switch message.Event {
    case TextEvent:
...
    case CandidateEvent:
...
    case AnswerEvent:
...
    case UpdateEvent:
        // when the offer-side is received this type messages,
        // it will start updating the peer connections.
        signalPeerConnections(h)
    }
}
func signalPeerConnections(h *SSEHub) {
    defer func() {
        dispatchKeyFrame(h)
    }()
    for syncAttempt := 0; ; syncAttempt++ {
        if syncAttempt == 25 {
            // Release the lock and attempt a sync in 3 seconds. We might be blocking a RemoveTrack or AddTrack
            go func() {
                time.Sleep(time.Second * 3)
                signalPeerConnections(h)
            }()
            return
        }
        if !attemptSync(h) {
            break
        }
    }
}
...
Enter fullscreen mode Exit fullscreen mode

Enabling / disabling video tracks

If the client-side application does not share any video during the session, the "getUserMedia" constraint can disable the use of video.

webrtc.controller.ts

...
navigator.mediaDevices.getUserMedia({ video: false, audio: true })
    .then(stream => {
        this.webcamStream = stream;
    });
...
Enter fullscreen mode Exit fullscreen mode

Adding a video track the first time

If I want to set the video enabled, I can execute "getUserMedia" again and add a video track into the MediaStream.

webrtc.controller.ts

...
private addVideoTrack(peerConnection: RTCPeerConnection) {
    navigator.mediaDevices.getUserMedia({ video: true })
        .then(stream => {
            const newVideoTracks = stream.getVideoTracks();
            if (this.webcamStream == null ||
                newVideoTracks.length <= 0) {
                return;
            }
            this.localVideo.srcObject = stream;
            this.localVideo.play();
            for (const v of newVideoTracks) {
                this.webcamStream.addTrack(v);
                peerConnection.addTrack(v, this.webcamStream);
            }
            if (this.connectionUpdatedEvent != null) {
                this.connectionUpdatedEvent();
            }
        });
}
...
Enter fullscreen mode Exit fullscreen mode

Removing the video and re-adding the video

I can stop sharing the video by "removeTrack".
However, if the local MediaStream video track is stopped, it will not be shared as a remote video track when it is re-added, so the local MediaStream is only paused.

webrtc.controller.ts

...
    public switchLocalVideoUsage(used: boolean): void {
        if (this.peerConnection == null ||
            this.webcamStream == null) {
            return;
        }
        const tracks = this.webcamStream.getVideoTracks();
        if (used) {
            if (tracks.length > 0 &&
                tracks[0] != null) {
                this.replaceVideoTrack(this.peerConnection, tracks[0]);
            } else {
                this.addVideoTrack(this.peerConnection);
            }
        } else {
            this.removeVideoTrack(this.peerConnection);
        }
    }
...
    /** for re-adding the video */
    private replaceVideoTrack(peerConnection: RTCPeerConnection, track: MediaStreamTrack) {
        this.localVideo.play();
        for (const s of peerConnection.getSenders()) {
            if (s.track == null || s.track.kind === "video") {
                s.replaceTrack(track);
            }
        }
        for (const t of peerConnection.getTransceivers()) {
            if (t.sender.track?.kind == null ||
                t.sender.track.kind === "video") {
                t.direction = "sendrecv";
            }
        }
        if (this.connectionUpdatedEvent != null) {
            this.connectionUpdatedEvent();
        }
    }
    private removeVideoTrack(peerConnection: RTCPeerConnection) {
        const senders = peerConnection.getSenders();
        if (senders.length > 0) {
            this.localVideo.pause();
            for (const s of senders) {
                if (s.track?.kind === "video") {
                    peerConnection.removeTrack(s);
                }
            }
            if (this.connectionUpdatedEvent != null) {
                this.connectionUpdatedEvent();
            }
        }
    }
...
Enter fullscreen mode Exit fullscreen mode

Note that even after re-adding, the Transceiver direction will not be changed from "recvonly" and will be treated as "inactive" in the Answer's SDP, so it must be changed individually.

Adding / removing MediaStreams into the DOM

Previously, only MediaStreamTracks whose "kind" was "video" in all received were added as DOM elements.

This time, if there are no video tracks, audio tracks must be added as the audio elements.

Also, if the track is removed, the corresponding element must be deleted, but if only the video track is removed, the audio track must be taken from the "srcObject" of the video element and re-added as an audio element.

webrtc.controller.ts

import * as urlParam from "./urlParamGetter";
type RemoteTrack = {
    id: string,
    kind: "video"|"audio",
    element: HTMLElement,
};
export class MainView {
...
    public addRemoteTrack(stream: MediaStream, kind: "video"|"audio", id?: string): void {
        if(this.tracks.some(t => t.id === stream.id)) {
            if(kind === "audio") {
                return;
            }
            this.removeRemoteTrack(stream.id, "audio");
        }
        const remoteTrack = document.createElement(kind);
        remoteTrack.srcObject = stream;
        remoteTrack.autoplay = true;
        remoteTrack.controls = false;
        this.remoteTrackArea.appendChild(remoteTrack);
        this.tracks.push({
            id: (id == null)? stream.id: id,
            kind,
            element: remoteTrack,
        });        
    }
    public removeRemoteTrack(id: string, kind: "video"|"audio"): void {
        const targets = this.tracks.filter(t => t.id === id);
        if(targets.length <= 0) {
            return;
        }
        if(kind === "video") {
            // the audio tracks must be re-added as audio elements.
            const audioTrack = this.getAudioTrack(targets[0]?.element);
            if(audioTrack != null) {
                this.addRemoteTrack(new MediaStream([audioTrack]), "audio", id);
            }
        }
        for(const t of targets) {
            this.remoteTrackArea.removeChild(t.element);
        }
        const newTracks = new Array<RemoteTrack>();
        for(const t of this.tracks.filter(t => t.id !== id || (t.id === id && t.kind !== kind))) {
            newTracks.push(t);
        }
        this.tracks = newTracks;
    }
    /** get audio track from "srcObject" of HTMLVideoElements */
    private getAudioTrack(target: HTMLElement|null|undefined): MediaStreamTrack|null {
        if(target == null ||
            !(target instanceof HTMLVideoElement)){
            return null;
        }
        if(target.srcObject == null ||
            !("getAudioTracks" in target.srcObject) ||
            (typeof target.srcObject.getAudioTracks !== "function")) {
            return null;
        }
        const tracks = target.srcObject.getAudioTracks();
        if(tracks.length <= 0 ||
            tracks[0] == null) {
            return null;
        }
        return tracks[0];
    }
}
Enter fullscreen mode Exit fullscreen mode

Resources

Top comments (0)

🌚 Browsing with dark mode makes you a better developer by a factor of exactly 40.

It's a scientific fact.