Adds simulcast support in meet.

This commit is contained in:
George Politis 2014-08-21 11:58:33 +02:00
parent 233f18cb78
commit ffaa9a62b8
11 changed files with 1117 additions and 59 deletions

96
app.js
View File

@ -70,18 +70,18 @@ function init() {
}
obtainAudioAndVideoPermissions(function (stream) {
var audioStream = new webkitMediaStream(stream);
var videoStream = new webkitMediaStream(stream);
var videoTracks = stream.getVideoTracks();
var audioStream = new webkitMediaStream();
var videoStream = new webkitMediaStream();
var audioTracks = stream.getAudioTracks();
for (var i = 0; i < videoTracks.length; i++) {
audioStream.removeTrack(videoTracks[i]);
var videoTracks = stream.getVideoTracks();
for (var i = 0; i < audioTracks.length; i++) {
audioStream.addTrack(audioTracks[i]);
}
VideoLayout.changeLocalAudio(audioStream);
startLocalRtpStatsCollector(audioStream);
for (i = 0; i < audioTracks.length; i++) {
videoStream.removeTrack(audioTracks[i]);
for (i = 0; i < videoTracks.length; i++) {
videoStream.addTrack(videoTracks[i]);
}
VideoLayout.changeLocalVideo(videoStream, true);
maybeDoJoin();
@ -237,7 +237,9 @@ function waitForRemoteVideo(selector, ssrc, stream) {
if (stream.id === 'mixedmslabel') return;
if (selector[0].currentTime > 0) {
RTC.attachMediaStream(selector, stream); // FIXME: why do i have to do this for FF?
var simulcast = new Simulcast();
var videoStream = simulcast.getReceivingVideoStream(stream);
RTC.attachMediaStream(selector, videoStream); // FIXME: why do i have to do this for FF?
// FIXME: add a class that will associate peer Jid, video.src, it's ssrc and video type
// in order to get rid of too many maps
@ -256,18 +258,40 @@ function waitForRemoteVideo(selector, ssrc, stream) {
}
$(document).bind('remotestreamadded.jingle', function (event, data, sid) {
waitForPresence(data, sid);
});
function waitForPresence(data, sid) {
var sess = connection.jingle.sessions[sid];
var thessrc;
// look up an associated JID for a stream id
if (data.stream.id.indexOf('mixedmslabel') === -1) {
// look only at a=ssrc: and _not_ at a=ssrc-group: lines
var ssrclines
= SDPUtil.find_lines(sess.peerconnection.remoteDescription.sdp, 'a=ssrc');
= SDPUtil.find_lines(sess.peerconnection.remoteDescription.sdp, 'a=ssrc:');
ssrclines = ssrclines.filter(function (line) {
return line.indexOf('mslabel:' + data.stream.label) !== -1;
});
if (ssrclines.length) {
thessrc = ssrclines[0].substring(7).split(' ')[0];
// We signal our streams (through Jingle to the focus) before we set
// our presence (through which peers associate remote streams to
// jids). So, it might arrive that a remote stream is added but
// ssrc2jid is not yet updated and thus data.peerjid cannot be
// successfully set. Here we wait for up to a second for the
// presence to arrive.
if (!ssrc2jid[thessrc]) {
setTimeout(function(d, s) {
return function() {
waitForPresence(d, s);
}
}(data, sid), 250);
return;
}
// ok to overwrite the one from focus? might save work in colibri.js
console.log('associated jid', ssrc2jid[thessrc], data.peerjid);
if (ssrc2jid[thessrc]) {
@ -276,6 +300,9 @@ $(document).bind('remotestreamadded.jingle', function (event, data, sid) {
}
}
// NOTE(gp) now that we have simulcast, a media stream can have more than 1
// ssrc. We should probably take that into account in our MediaStream
// wrapper.
mediaStreams.push(new MediaStream(data, sid, thessrc));
var container;
@ -322,7 +349,7 @@ $(document).bind('remotestreamadded.jingle', function (event, data, sid) {
sendKeyframe(sess.peerconnection);
}, 3000);
}
});
}
/**
* Returns the JID of the user to whom given <tt>videoSrc</tt> belongs.
@ -532,40 +559,35 @@ $(document).bind('callterminated.jingle', function (event, sid, jid, reason) {
$(document).bind('setLocalDescription.jingle', function (event, sid) {
// put our ssrcs into presence so other clients can identify our stream
var sess = connection.jingle.sessions[sid];
var newssrcs = {};
var directions = {};
var localSDP = new SDP(sess.peerconnection.localDescription.sdp);
localSDP.media.forEach(function (media) {
var type = SDPUtil.parse_mid(SDPUtil.find_line(media, 'a=mid:'));
var newssrcs = [];
var simulcast = new Simulcast();
var media = simulcast.parseMedia(sess.peerconnection.localDescription);
media.forEach(function (media) {
if (SDPUtil.find_line(media, 'a=ssrc:')) {
// assumes a single local ssrc
var ssrc = SDPUtil.find_line(media, 'a=ssrc:').substring(7).split(' ')[0];
newssrcs[type] = ssrc;
directions[type] = (
SDPUtil.find_line(media, 'a=sendrecv') ||
SDPUtil.find_line(media, 'a=recvonly') ||
SDPUtil.find_line(media, 'a=sendonly') ||
SDPUtil.find_line(media, 'a=inactive') ||
'a=sendrecv').substr(2);
}
// TODO(gp) maybe exclude FID streams?
Object.keys(media.sources).forEach(function(ssrc) {
newssrcs.push({
'ssrc': ssrc,
'type': media.type,
'direction': media.direction
});
});
});
console.log('new ssrcs', newssrcs);
// Have to clear presence map to get rid of removed streams
connection.emuc.clearPresenceMedia();
var i = 0;
Object.keys(newssrcs).forEach(function (mtype) {
i++;
var type = mtype;
// Change video type to screen
if (mtype === 'video' && isUsingScreenStream) {
type = 'screen';
if (newssrcs.length > 0) {
for (var i = 1; i <= newssrcs.length; i ++) {
// Change video type to screen
if (newssrcs[i-1].type === 'video' && isUsingScreenStream) {
newssrcs[i-1].type = 'screen';
}
connection.emuc.addMediaToPresence(i,
newssrcs[i-1].type, newssrcs[i-1].ssrc, newssrcs[i-1].direction);
}
connection.emuc.addMediaToPresence(i, type, newssrcs[mtype], directions[mtype]);
});
if (i > 0) {
connection.emuc.sendPresence();
}
});

View File

@ -22,5 +22,7 @@ var config = {
useBundle: true,
enableRecording: false,
enableWelcomePage: false,
enableSimulcast: false,
useNativeSimulcast: false,
isBrand: false
};

View File

@ -84,6 +84,11 @@ function onDataChannel(event)
'lastnchanged',
[lastNEndpoints, endpointsEnteringLastN, stream]);
}
else if ("SimulcastLayersChangedEvent" === colibriClass)
{
var endpointSimulcastLayers = obj.endpointSimulcastLayers;
$(document).trigger('simulcastlayerschanged', [endpointSimulcastLayers]);
}
else
{
console.debug("Data channel JSON-formatted message: ", obj);

View File

@ -10,6 +10,7 @@
<meta itemprop="description" content="Join a WebRTC video conference powered by the Jitsi Videobridge"/>
<meta itemprop="image" content="/images/jitsilogo.png"/>
<script src="libs/jquery-2.1.1.min.js"></script>
<script src="simulcast.js?v=1"></script><!-- simulcast handling -->
<script src="libs/strophe/strophe.jingle.adapter.js?v=1"></script><!-- strophe.jingle bundles -->
<script src="libs/strophe/strophe.jingle.bundle.js?v=8"></script>
<script src="libs/strophe/strophe.jingle.js?v=1"></script>

View File

@ -42,6 +42,7 @@ function ColibriFocus(connection, bridgejid) {
this.bridgejid = bridgejid;
this.peers = [];
this.remoteStreams = [];
this.confid = null;
/**
@ -142,6 +143,7 @@ ColibriFocus.prototype.makeConference = function (peers) {
event.peerjid = jid;
}
});
self.remoteStreams.push(event.stream);
$(document).trigger('remotestreamadded.jingle', [event, self.sid]);
};
this.peerconnection.onicecandidate = function (event) {
@ -525,9 +527,11 @@ ColibriFocus.prototype.createdConference = function (result) {
}
}
bridgeSDP.raw = bridgeSDP.session + bridgeSDP.media.join('');
var bridgeDesc = new RTCSessionDescription({type: 'offer', sdp: bridgeSDP.raw});
var simulcast = new Simulcast();
var bridgeDesc = simulcast.transformBridgeDescription(bridgeDesc);
this.peerconnection.setRemoteDescription(
new RTCSessionDescription({type: 'offer', sdp: bridgeSDP.raw}),
this.peerconnection.setRemoteDescription(bridgeDesc,
function () {
console.log('setRemoteDescription success');
self.peerconnection.createAnswer(
@ -553,6 +557,24 @@ ColibriFocus.prototype.createdConference = function (result) {
endpoint: self.myMucResource
});
// signal (through COLIBRI) to the bridge
// the SSRC groups of the participant
// that plays the role of the focus
var ssrc_group_lines = SDPUtil.find_lines(media, 'a=ssrc-group:');
var idx = 0;
ssrc_group_lines.forEach(function(line) {
idx = line.indexOf(' ');
var semantics = line.substr(0, idx).substr(13);
var ssrcs = line.substr(14 + semantics.length).split(' ');
if (ssrcs.length != 0) {
elem.c('ssrc-group', { semantics: semantics, xmlns: 'urn:xmpp:jingle:apps:rtp:ssma:0' });
ssrcs.forEach(function(ssrc) {
elem.c('source', { ssrc: ssrc })
.up();
});
elem.up();
}
});
// FIXME: should reuse code from .toJingle
for (var j = 0; j < mline.fmt.length; j++)
{
@ -646,6 +668,7 @@ ColibriFocus.prototype.initiate = function (peer, isInitiator) {
sdp.removeMediaLines(i, 'a=rtcp-mux');
}
sdp.removeMediaLines(i, 'a=ssrc:');
sdp.removeMediaLines(i, 'a=ssrc-group:');
sdp.removeMediaLines(i, 'a=crypto:');
sdp.removeMediaLines(i, 'a=candidate:');
sdp.removeMediaLines(i, 'a=ice-options:google-ice');
@ -655,14 +678,20 @@ ColibriFocus.prototype.initiate = function (peer, isInitiator) {
sdp.removeMediaLines(i, 'a=setup:');
if (1) { //i > 0) { // not for audio FIXME: does not work as intended
// re-add all remote a=ssrcs
// re-add all remote a=ssrcs _and_ a=ssrc-group
for (var jid in this.remotessrc) {
if (jid == peer || !this.remotessrc[jid][i])
continue;
sdp.media[i] += this.remotessrc[jid][i];
}
// and local a=ssrc lines
sdp.media[i] += SDPUtil.find_lines(localSDP.media[i], 'a=ssrc').join('\r\n') + '\r\n';
// add local a=ssrc-group: lines
lines = SDPUtil.find_lines(localSDP.media[i], 'a=ssrc-group:');
if (lines.length != 0)
sdp.media[i] += lines.join('\r\n') + '\r\n';
// and local a=ssrc: lines
sdp.media[i] += SDPUtil.find_lines(localSDP.media[i], 'a=ssrc:').join('\r\n') + '\r\n';
}
}
sdp.raw = sdp.session + sdp.media.join('');
@ -864,6 +893,24 @@ ColibriFocus.prototype.updateChannel = function (remoteSDP, participant) {
expire: self.channelExpire
});
// signal (throught COLIBRI) to the bridge the SSRC groups of this
// participant
var ssrc_group_lines = SDPUtil.find_lines(remoteSDP.media[channel], 'a=ssrc-group:');
var idx = 0;
ssrc_group_lines.forEach(function(line) {
idx = line.indexOf(' ');
var semantics = line.substr(0, idx).substr(13);
var ssrcs = line.substr(14 + semantics.length).split(' ');
if (ssrcs.length != 0) {
change.c('ssrc-group', { semantics: semantics, xmlns: 'urn:xmpp:jingle:apps:rtp:ssma:0' });
ssrcs.forEach(function(ssrc) {
change.c('source', { ssrc: ssrc })
.up();
});
change.up();
}
});
var rtpmap = SDPUtil.find_lines(remoteSDP.media[channel], 'a=rtpmap:');
rtpmap.forEach(function (val) {
// TODO: too much copy-paste
@ -1028,20 +1075,33 @@ ColibriFocus.prototype.setRemoteDescription = function (session, elem, desctype)
if (!remoteSDP.media[channel])
continue;
var lines = SDPUtil.find_lines(remoteSDP.media[channel], 'a=ssrc-group:');
if (lines.length != 0)
// prepend ssrc-groups
this.remotessrc[session.peerjid][channel] = lines.join('\r\n') + '\r\n';
if (SDPUtil.find_lines(remoteSDP.media[channel], 'a=ssrc:').length)
{
this.remotessrc[session.peerjid][channel] =
if (!this.remotessrc[session.peerjid][channel])
this.remotessrc[session.peerjid][channel] = '';
this.remotessrc[session.peerjid][channel] +=
SDPUtil.find_lines(remoteSDP.media[channel], 'a=ssrc:')
.join('\r\n') + '\r\n';
}
}
// ACT 4: add new a=ssrc lines to local remotedescription
// ACT 4: add new a=ssrc and s=ssrc-group lines to local remotedescription
for (channel = 0; channel < this.channels[participant].length; channel++) {
//if (channel == 0) continue; FIXME: does not work as intended
if (!remoteSDP.media[channel])
continue;
var lines = SDPUtil.find_lines(remoteSDP.media[channel], 'a=ssrc-group:');
if (lines.length != 0)
this.peerconnection.enqueueAddSsrc(
channel, SDPUtil.find_lines(remoteSDP.media[channel], 'a=ssrc-group:').join('\r\n') + '\r\n');
if (SDPUtil.find_lines(remoteSDP.media[channel], 'a=ssrc:').length) {
this.peerconnection.enqueueAddSsrc(
channel,
@ -1311,6 +1371,9 @@ ColibriFocus.prototype.sendTerminate = function (session, reason, text) {
ColibriFocus.prototype.setRTCPTerminationStrategy = function (strategyFQN) {
var self = this;
// TODO(gp) maybe move the RTCP termination strategy element under the
// content or channel element.
var strategyIQ = $iq({to: this.bridgejid, type: 'set'});
strategyIQ.c('conference', {
xmlns: 'http://jitsi.org/protocol/colibri',
@ -1378,3 +1441,50 @@ ColibriFocus.prototype.setChannelLastN = function (channelLastN) {
}
};
/**
* Sets the default value of the channel simulcast layer attribute in this
* conference and updates/patches the existing channels.
*/
ColibriFocus.prototype.setReceiveSimulcastLayer = function (receiveSimulcastLayer) {
if (('number' === typeof(receiveSimulcastLayer))
&& (this.receiveSimulcastLayer !== receiveSimulcastLayer))
{
// TODO(gp) be able to set the receiving simulcast layer on a per
// sender basis.
this.receiveSimulcastLayer = receiveSimulcastLayer;
// Update/patch the existing channels.
var patch = $iq({ to: this.bridgejid, type: 'set' });
patch.c(
'conference',
{ xmlns: 'http://jitsi.org/protocol/colibri', id: this.confid });
patch.c('content', { name: 'video' });
patch.c(
'channel',
{
id: $(this.mychannel[1 /* video */]).attr('id'),
'receive-simulcast-layer': this.receiveSimulcastLayer
});
patch.up(); // end of channel
for (var p = 0; p < this.channels.length; p++)
{
patch.c(
'channel',
{
id: $(this.channels[p][1 /* video */]).attr('id'),
'receive-simulcast-layer': this.receiveSimulcastLayer
});
patch.up(); // end of channel
}
this.connection.sendIQ(
patch,
function (res) {
console.info('Set channel simulcast receive layer succeeded:', res);
},
function (err) {
console.error('Set channel simulcast receive layer failed:', err);
});
}
};

View File

@ -128,7 +128,11 @@ dumpSDP = function(description) {
if (TraceablePeerConnection.prototype.__defineGetter__ !== undefined) {
TraceablePeerConnection.prototype.__defineGetter__('signalingState', function() { return this.peerconnection.signalingState; });
TraceablePeerConnection.prototype.__defineGetter__('iceConnectionState', function() { return this.peerconnection.iceConnectionState; });
TraceablePeerConnection.prototype.__defineGetter__('localDescription', function() { return this.peerconnection.localDescription; });
TraceablePeerConnection.prototype.__defineGetter__('localDescription', function() {
var simulcast = new Simulcast();
var publicLocalDescription = simulcast.makeLocalDescriptionPublic(this.peerconnection.localDescription);
return publicLocalDescription;
});
TraceablePeerConnection.prototype.__defineGetter__('remoteDescription', function() { return this.peerconnection.remoteDescription; });
}
@ -149,6 +153,8 @@ TraceablePeerConnection.prototype.createDataChannel = function (label, opts) {
TraceablePeerConnection.prototype.setLocalDescription = function (description, successCallback, failureCallback) {
var self = this;
var simulcast = new Simulcast();
description = simulcast.transformLocalDescription(description);
this.trace('setLocalDescription', dumpSDP(description));
this.peerconnection.setLocalDescription(description,
function () {
@ -169,6 +175,8 @@ TraceablePeerConnection.prototype.setLocalDescription = function (description, s
TraceablePeerConnection.prototype.setRemoteDescription = function (description, successCallback, failureCallback) {
var self = this;
var simulcast = new Simulcast();
description = simulcast.transformRemoteDescription(description);
this.trace('setRemoteDescription', dumpSDP(description));
this.peerconnection.setRemoteDescription(description,
function () {
@ -208,6 +216,16 @@ TraceablePeerConnection.prototype.addSource = function (elem) {
$(elem).each(function (idx, content) {
var name = $(content).attr('name');
var lines = '';
tmp = $(content).find('ssrc-group[xmlns="urn:xmpp:jingle:apps:rtp:ssma:0"]').each(function() {
var semantics = this.getAttribute('semantics');
var ssrcs = $(this).find('>source').map(function () {
return this.getAttribute('ssrc');
}).get();
if (ssrcs.length != 0) {
lines += 'a=ssrc-group:' + semantics + ' ' + ssrcs.join(' ') + '\r\n';
}
});
tmp = $(content).find('source[xmlns="urn:xmpp:jingle:apps:rtp:ssma:0"]'); // can handle both >source and >description>source
tmp.each(function () {
var ssrc = $(this).attr('ssrc');
@ -254,6 +272,16 @@ TraceablePeerConnection.prototype.removeSource = function (elem) {
$(elem).each(function (idx, content) {
var name = $(content).attr('name');
var lines = '';
tmp = $(content).find('ssrc-group[xmlns="urn:xmpp:jingle:apps:rtp:ssma:0"]').each(function() {
var semantics = this.getAttribute('semantics');
var ssrcs = $(this).find('>source').map(function () {
return this.getAttribute('ssrc');
}).get();
if (ssrcs.length != 0) {
lines += 'a=ssrc-group:' + semantics + ' ' + ssrcs.join(' ') + '\r\n';
}
});
tmp = $(content).find('source[xmlns="urn:xmpp:jingle:apps:rtp:ssma:0"]'); // can handle both >source and >description>source
tmp.each(function () {
var ssrc = $(this).attr('ssrc');
@ -413,6 +441,8 @@ TraceablePeerConnection.prototype.createAnswer = function (successCallback, fail
this.trace('createAnswer', JSON.stringify(constraints, null, ' '));
this.peerconnection.createAnswer(
function (answer) {
var simulcast = new Simulcast();
answer = simulcast.transformAnswer(answer);
self.trace('createAnswerOnSuccess', dumpSDP(answer));
successCallback(answer);
},
@ -628,18 +658,43 @@ function getUserMediaWithConstraints(um, success_callback, failure_callback, res
constraints.video.mandatory.minFrameRate = fps;
}
var isFF = navigator.userAgent.toLowerCase().indexOf('firefox') > -1;
try {
RTC.getUserMedia(constraints,
function (stream) {
console.log('onUserMediaSuccess');
success_callback(stream);
},
function (error) {
console.warn('Failed to get access to local media. Error ', error);
if(failure_callback) {
failure_callback(error);
}
});
if (config.enableSimulcast
&& constraints.video
&& constraints.video.chromeMediaSource !== 'screen'
&& constraints.video.chromeMediaSource !== 'desktop'
&& !isAndroid
// We currently do not support FF, as it doesn't have multistream support.
&& !isFF) {
var simulcast = new Simulcast();
simulcast.getUserMedia(constraints, function (stream) {
console.log('onUserMediaSuccess');
success_callback(stream);
},
function (error) {
console.warn('Failed to get access to local media. Error ', error);
if (failure_callback) {
failure_callback(error);
}
});
} else {
RTC.getUserMedia(constraints,
function (stream) {
console.log('onUserMediaSuccess');
success_callback(stream);
},
function (error) {
console.warn('Failed to get access to local media. Error ', error);
if (failure_callback) {
failure_callback(error);
}
});
}
} catch (e) {
console.error('GUM failed: ', e);
if(failure_callback) {

View File

@ -31,6 +31,15 @@ SDP.prototype.getMediaSsrcMap = function() {
}
channel.ssrcs[linessrc].lines.push(line);
});
tmp = SDPUtil.find_lines(self.media[channelNum], 'a=ssrc-group:');
tmp.forEach(function(line){
var semantics = line.substr(0, idx).substr(13);
var ssrcs = line.substr(14 + semantics.length).split(' ');
if (ssrcs.length != 0) {
var ssrcGroup = new ChannelSsrcGroup(semantics, ssrcs);
channel.ssrcGroups.push(ssrcGroup);
}
});
}
return media_ssrcs;
}
@ -56,6 +65,32 @@ SDP.prototype.containsSSRC = function(ssrc) {
* @param otherSdp the other SDP to check ssrc with.
*/
SDP.prototype.getNewMedia = function(otherSdp) {
// this could be useful in Array.prototype.
function arrayEquals(array) {
// if the other array is a falsy value, return
if (!array)
return false;
// compare lengths - can save a lot of time
if (this.length != array.length)
return false;
for (var i = 0, l=this.length; i < l; i++) {
// Check if we have nested arrays
if (this[i] instanceof Array && array[i] instanceof Array) {
// recurse into the nested arrays
if (!this[i].equals(array[i]))
return false;
}
else if (this[i] != array[i]) {
// Warning - two different object instances will never be equal: {x:20} != {x:20}
return false;
}
}
return true;
};
var myMedia = this.getMediaSsrcMap();
var othersMedia = otherSdp.getMediaSsrcMap();
var newMedia = {};
@ -77,6 +112,32 @@ SDP.prototype.getNewMedia = function(otherSdp) {
newMedia[channelNum].ssrcs[ssrc] = othersChannel.ssrcs[ssrc];
}
})
// Look for new ssrc groups across the channels
othersChannel.ssrcGroups.forEach(function(otherSsrcGroup){
// try to match the other ssrc-group with an ssrc-group of ours
var matched = false;
for (var i = 0; i < myChannel.ssrcGroups.length; i++) {
var mySsrcGroup = myChannel.ssrcGroups[i];
if (otherSsrcGroup.semantics == mySsrcGroup
&& arrayEquals.apply(otherSsrcGroup.ssrcs, [mySsrcGroup.ssrcs])) {
matched = true;
break;
}
}
if (!matched) {
// Allocate channel if we've found an ssrc-group that doesn't
// exist in our channel
if(!newMedia[channelNum]){
newMedia[channelNum] = new MediaChannel(othersChannel.chNumber, othersChannel.mediaType);
}
newMedia[channelNum].ssrcGroups.push(otherSsrcGroup);
}
});
});
return newMedia;
}
@ -241,6 +302,22 @@ SDP.prototype.toJingle = function (elem, thecreator) {
tmp.xmlns = 'http://estos.de/ns/ssrc';
tmp.ssrc = ssrc;
elem.c('ssrc', tmp).up(); // ssrc is part of description
// XEP-0339 handle ssrc-group attributes
var ssrc_group_lines = SDPUtil.find_lines(this.media[i], 'a=ssrc-group:');
ssrc_group_lines.forEach(function(line) {
idx = line.indexOf(' ');
var semantics = line.substr(0, idx).substr(13);
var ssrcs = line.substr(14 + semantics.length).split(' ');
if (ssrcs.length != 0) {
elem.c('ssrc-group', { semantics: semantics, xmlns: 'urn:xmpp:jingle:apps:rtp:ssma:0' });
ssrcs.forEach(function(ssrc) {
elem.c('source', { ssrc: ssrc })
.up();
});
elem.up();
}
});
}
if (SDPUtil.find_line(this.media[i], 'a=rtcp-mux')) {
@ -578,6 +655,18 @@ SDP.prototype.jingle2media = function (content) {
media += SDPUtil.candidateFromJingle(this);
});
// XEP-0339 handle ssrc-group attributes
tmp = content.find('description>ssrc-group[xmlns="urn:xmpp:jingle:apps:rtp:ssma:0"]').each(function() {
var semantics = this.getAttribute('semantics');
var ssrcs = $(this).find('>source').map(function() {
return this.getAttribute('ssrc');
}).get();
if (ssrcs.length != 0) {
media += 'a=ssrc-group:' + semantics + ' ' + ssrcs.join(' ') + '\r\n';
}
});
tmp = content.find('description>source[xmlns="urn:xmpp:jingle:apps:rtp:ssma:0"]');
tmp.each(function () {
var ssrc = this.getAttribute('ssrc');

View File

@ -15,6 +15,17 @@ function ChannelSsrc(ssrc, type) {
this.lines = [];
}
/**
* Class holds a=ssrc-group: lines
* @param semantics
* @param ssrcs
* @constructor
*/
function ChannelSsrcGroup(semantics, ssrcs, line) {
this.semantics = semantics;
this.ssrcs = ssrcs;
}
/**
* Helper class represents media channel. Is a container for ChannelSsrc, holds channel idx and media type.
* @param channelNumber channel idx in SDP media array.
@ -36,6 +47,12 @@ function MediaChannel(channelNumber, mediaType) {
* The maps of ssrc numbers to ChannelSsrc objects.
*/
this.ssrcs = {};
/**
* The array of ChannelSsrcGroup objects.
* @type {Array}
*/
this.ssrcGroups = [];
}
SDPUtil = {

View File

@ -119,6 +119,8 @@ JingleSession.prototype.accept = function () {
// FIXME: change any inactive to sendrecv or whatever they were originally
pranswer.sdp = pranswer.sdp.replace('a=inactive', 'a=sendrecv');
}
var simulcast = new Simulcast();
pranswer = simulcast.makeLocalDescriptionPublic(pranswer);
var prsdp = new SDP(pranswer.sdp);
var accept = $iq({to: this.peerjid,
type: 'set'})
@ -565,7 +567,10 @@ JingleSession.prototype.createdAnswer = function (sdp, provisional) {
initiator: this.initiator,
responder: this.responder,
sid: this.sid });
this.localSDP.toJingle(accept, this.initiator == this.me ? 'initiator' : 'responder');
var simulcast = new Simulcast();
var publicLocalDesc = simulcast.makeLocalDescriptionPublic(sdp);
var publicLocalSDP = new SDP(publicLocalDesc.sdp);
publicLocalSDP.toJingle(accept, this.initiator == this.me ? 'initiator' : 'responder');
this.connection.sendIQ(accept,
function () {
var ack = {};

669
simulcast.js Normal file
View File

@ -0,0 +1,669 @@
/*jslint plusplus: true */
/*jslint nomen: true*/
/**
* Created by gp on 11/08/14.
*/
function Simulcast() {
"use strict";
// TODO(gp) split the Simulcast class in two classes : NativeSimulcast and ClassicSimulcast.
this.debugLvl = 1;
}
(function () {
"use strict";
// global state for all transformers.
var localExplosionMap = {}, localVideoSourceCache, emptyCompoundIndex,
remoteMaps = {
msid2Quality: {},
ssrc2Msid: {},
receivingVideoStreams: {}
}, localMaps = {
msids: [],
msid2ssrc: {}
};
Simulcast.prototype._generateGuid = (function () {
function s4() {
return Math.floor((1 + Math.random()) * 0x10000)
.toString(16)
.substring(1);
}
return function () {
return s4() + s4() + '-' + s4() + '-' + s4() + '-' +
s4() + '-' + s4() + s4() + s4();
};
}());
Simulcast.prototype._cacheVideoSources = function (lines) {
localVideoSourceCache = this._getVideoSources(lines);
};
Simulcast.prototype._restoreVideoSources = function (lines) {
this._replaceVideoSources(lines, localVideoSourceCache);
};
Simulcast.prototype._replaceVideoSources = function (lines, videoSources) {
var i, inVideo = false, index = -1, howMany = 0;
if (this.debugLvl) {
console.info('Replacing video sources...');
}
for (i = 0; i < lines.length; i++) {
if (inVideo && lines[i].substring(0, 'm='.length) === 'm=') {
// Out of video.
break;
}
if (!inVideo && lines[i].substring(0, 'm=video '.length) === 'm=video ') {
// In video.
inVideo = true;
}
if (inVideo && (lines[i].substring(0, 'a=ssrc:'.length) === 'a=ssrc:'
|| lines[i].substring(0, 'a=ssrc-group:'.length) === 'a=ssrc-group:')) {
if (index === -1) {
index = i;
}
howMany++;
}
}
// efficiency baby ;)
lines.splice.apply(lines,
[index, howMany].concat(videoSources));
};
Simulcast.prototype._getVideoSources = function (lines) {
var i, inVideo = false, sb = [];
if (this.debugLvl) {
console.info('Getting video sources...');
}
for (i = 0; i < lines.length; i++) {
if (inVideo && lines[i].substring(0, 'm='.length) === 'm=') {
// Out of video.
break;
}
if (!inVideo && lines[i].substring(0, 'm=video '.length) === 'm=video ') {
// In video.
inVideo = true;
}
if (inVideo && lines[i].substring(0, 'a=ssrc:'.length) === 'a=ssrc:') {
// In SSRC.
sb.push(lines[i]);
}
if (inVideo && lines[i].substring(0, 'a=ssrc-group:'.length) === 'a=ssrc-group:') {
sb.push(lines[i]);
}
}
return sb;
};
Simulcast.prototype._parseMedia = function (lines, mediatypes) {
var i, res = [], type, cur_media, idx, ssrcs, cur_ssrc, ssrc,
ssrc_attribute, group, semantics, skip;
if (this.debugLvl) {
console.info('Parsing media sources...');
}
for (i = 0; i < lines.length; i++) {
if (lines[i].substring(0, 'm='.length) === 'm=') {
type = lines[i]
.substr('m='.length, lines[i].indexOf(' ') - 'm='.length);
skip = mediatypes !== undefined && mediatypes.indexOf(type) === -1;
if (!skip) {
cur_media = {
'type': type,
'sources': {},
'groups': []
};
res.push(cur_media);
}
} else if (!skip && lines[i].substring(0, 'a=ssrc:'.length) === 'a=ssrc:') {
idx = lines[i].indexOf(' ');
ssrc = lines[i].substring('a=ssrc:'.length, idx);
if (cur_media.sources[ssrc] === undefined) {
cur_ssrc = {'ssrc': ssrc};
cur_media.sources[ssrc] = cur_ssrc;
}
ssrc_attribute = lines[i].substr(idx + 1).split(':', 2)[0];
cur_ssrc[ssrc_attribute] = lines[i].substr(idx + 1).split(':', 2)[1];
if (cur_media.base === undefined) {
cur_media.base = cur_ssrc;
}
} else if (!skip && lines[i].substring(0, 'a=ssrc-group:'.length) === 'a=ssrc-group:') {
idx = lines[i].indexOf(' ');
semantics = lines[i].substr(0, idx).substr('a=ssrc-group:'.length);
ssrcs = lines[i].substr(idx).trim().split(' ');
group = {
'semantics': semantics,
'ssrcs': ssrcs
};
cur_media.groups.push(group);
} else if (!skip && (lines[i].substring(0, 'a=sendrecv'.length) === 'a=sendrecv' ||
lines[i].substring(0, 'a=recvonly'.length) === 'a=recvonly' ||
lines[i].substring(0, 'a=sendonly'.length) === 'a=sendonly' ||
lines[i].substring(0, 'a=inactive'.length) === 'a=inactive')) {
cur_media.direction = lines[i].substring('a='.length, 8);
}
}
return res;
};
// Returns a random integer between min (included) and max (excluded)
// Using Math.round() will give you a non-uniform distribution!
Simulcast.prototype._generateRandomSSRC = function () {
var min = 0, max = 0xffffffff;
return Math.floor(Math.random() * (max - min)) + min;
};
function CompoundIndex(obj) {
if (obj !== undefined) {
this.row = obj.row;
this.column = obj.column;
}
}
emptyCompoundIndex = new CompoundIndex();
Simulcast.prototype._indexOfArray = function (needle, haystack, start) {
var length = haystack.length, idx, i;
if (!start) {
start = 0;
}
for (i = start; i < length; i++) {
idx = haystack[i].indexOf(needle);
if (idx !== -1) {
return new CompoundIndex({row: i, column: idx});
}
}
return emptyCompoundIndex;
};
Simulcast.prototype._removeSimulcastGroup = function (lines) {
var i;
for (i = lines.length - 1; i >= 0; i--) {
if (lines[i].indexOf('a=ssrc-group:SIM') !== -1) {
lines.splice(i, 1);
}
}
};
Simulcast.prototype._explodeLocalSimulcastSources = function (lines) {
var sb, msid, sid, tid, videoSources, self;
if (this.debugLvl) {
console.info('Exploding local video sources...');
}
videoSources = this._parseMedia(lines, ['video'])[0];
self = this;
if (videoSources.groups && videoSources.groups.length !== 0) {
videoSources.groups.forEach(function (group) {
if (group.semantics === 'SIM') {
group.ssrcs.forEach(function (ssrc) {
// Get the msid for this ssrc..
if (localExplosionMap[ssrc]) {
// .. either from the explosion map..
msid = localExplosionMap[ssrc];
} else {
// .. or generate a new one (msid).
sid = videoSources.sources[ssrc].msid
.substring(0, videoSources.sources[ssrc].msid.indexOf(' '));
tid = self._generateGuid();
msid = [sid, tid].join(' ');
localExplosionMap[ssrc] = msid;
}
// Assign it to the source object.
videoSources.sources[ssrc].msid = msid;
// TODO(gp) Change the msid of associated sources.
});
}
});
}
sb = this._compileVideoSources(videoSources);
this._replaceVideoSources(lines, sb);
};
Simulcast.prototype._groupLocalVideoSources = function (lines) {
var sb, videoSources, ssrcs = [], ssrc;
if (this.debugLvl) {
console.info('Grouping local video sources...');
}
videoSources = this._parseMedia(lines, ['video'])[0];
for (ssrc in videoSources.sources) {
// jitsi-meet destroys/creates streams at various places causing
// the original local stream ids to change. The only thing that
// remains unchanged is the trackid.
localMaps.msid2ssrc[videoSources.sources[ssrc].msid.split(' ')[1]] = ssrc;
}
// TODO(gp) add only "free" sources.
localMaps.msids.forEach(function (msid) {
ssrcs.push(localMaps.msid2ssrc[msid]);
});
if (!videoSources.groups) {
videoSources.groups = [];
}
videoSources.groups.push({
'semantics': 'SIM',
'ssrcs': ssrcs
});
sb = this._compileVideoSources(videoSources);
this._replaceVideoSources(lines, sb);
};
Simulcast.prototype._appendSimulcastGroup = function (lines) {
var videoSources, ssrcGroup, simSSRC, numOfSubs = 3, i, sb, msid;
if (this.debugLvl) {
console.info('Appending simulcast group...');
}
// Get the primary SSRC information.
videoSources = this._parseMedia(lines, ['video'])[0];
// Start building the SIM SSRC group.
ssrcGroup = ['a=ssrc-group:SIM'];
// The video source buffer.
sb = [];
// Create the simulcast sub-streams.
for (i = 0; i < numOfSubs; i++) {
// TODO(gp) prevent SSRC collision.
simSSRC = this._generateRandomSSRC();
ssrcGroup.push(simSSRC);
sb.splice.apply(sb, [sb.length, 0].concat(
[["a=ssrc:", simSSRC, " cname:", videoSources.base.cname].join(''),
["a=ssrc:", simSSRC, " msid:", videoSources.base.msid].join('')]
));
if (this.debugLvl) {
console.info(['Generated substream ', i, ' with SSRC ', simSSRC, '.'].join(''));
}
}
// Add the group sim layers.
sb.splice(0, 0, ssrcGroup.join(' '))
this._replaceVideoSources(lines, sb);
};
// Does the actual patching.
Simulcast.prototype._ensureSimulcastGroup = function (lines) {
if (this.debugLvl) {
console.info('Ensuring simulcast group...');
}
if (this._indexOfArray('a=ssrc-group:SIM', lines) === emptyCompoundIndex) {
this._appendSimulcastGroup(lines);
this._cacheVideoSources(lines);
} else {
// verify that the ssrcs participating in the SIM group are present
// in the SDP (needed for presence).
this._restoreVideoSources(lines);
}
};
Simulcast.prototype._ensureGoogConference = function (lines) {
var sb;
if (this.debugLvl) {
console.info('Ensuring x-google-conference flag...')
}
if (this._indexOfArray('a=x-google-flag:conference', lines) === emptyCompoundIndex) {
// Add the google conference flag
sb = this._getVideoSources(lines);
sb = ['a=x-google-flag:conference'].concat(sb);
this._replaceVideoSources(lines, sb);
}
};
Simulcast.prototype._compileVideoSources = function (videoSources) {
var sb = [], ssrc, addedSSRCs = [];
if (this.debugLvl) {
console.info('Compiling video sources...');
}
// Add the groups
if (videoSources.groups && videoSources.groups.length !== 0) {
videoSources.groups.forEach(function (group) {
if (group.ssrcs && group.ssrcs.length !== 0) {
sb.push([['a=ssrc-group:', group.semantics].join(''), group.ssrcs.join(' ')].join(' '));
// if (group.semantics !== 'SIM') {
group.ssrcs.forEach(function (ssrc) {
addedSSRCs.push(ssrc);
sb.splice.apply(sb, [sb.length, 0].concat([
["a=ssrc:", ssrc, " cname:", videoSources.sources[ssrc].cname].join(''),
["a=ssrc:", ssrc, " msid:", videoSources.sources[ssrc].msid].join('')]));
});
//}
}
});
}
// Then add any free sources.
if (videoSources.sources) {
for (ssrc in videoSources.sources) {
if (addedSSRCs.indexOf(ssrc) === -1) {
sb.splice.apply(sb, [sb.length, 0].concat([
["a=ssrc:", ssrc, " cname:", videoSources.sources[ssrc].cname].join(''),
["a=ssrc:", ssrc, " msid:", videoSources.sources[ssrc].msid].join('')]));
}
}
}
return sb;
};
Simulcast.prototype.transformAnswer = function (desc) {
if (config.enableSimulcast && config.useNativeSimulcast) {
var sb = desc.sdp.split('\r\n');
// Even if we have enabled native simulcasting previously
// (with a call to SLD with an appropriate SDP, for example),
// createAnswer seems to consistently generate incomplete SDP
// with missing SSRCS.
//
// So, subsequent calls to SLD will have missing SSRCS and presence
// won't have the complete list of SRCs.
this._ensureSimulcastGroup(sb);
desc = new RTCSessionDescription({
type: desc.type,
sdp: sb.join('\r\n')
});
if (this.debugLvl && this.debugLvl > 1) {
console.info('Transformed answer');
console.info(desc.sdp);
}
}
return desc;
};
Simulcast.prototype.makeLocalDescriptionPublic = function (desc) {
var sb;
if (!desc || desc == null)
return desc;
if (config.enableSimulcast) {
if (config.useNativeSimulcast) {
sb = desc.sdp.split('\r\n');
this._explodeLocalSimulcastSources(sb);
desc = new RTCSessionDescription({
type: desc.type,
sdp: sb.join('\r\n')
});
if (this.debugLvl && this.debugLvl > 1) {
console.info('Exploded local video sources');
console.info(desc.sdp);
}
} else {
sb = desc.sdp.split('\r\n');
this._groupLocalVideoSources(sb);
desc = new RTCSessionDescription({
type: desc.type,
sdp: sb.join('\r\n')
});
if (this.debugLvl && this.debugLvl > 1) {
console.info('Grouped local video sources');
console.info(desc.sdp);
}
}
}
return desc;
};
Simulcast.prototype._ensureOrder = function (lines) {
var videoSources, sb;
videoSources = this._parseMedia(lines, ['video'])[0];
sb = this._compileVideoSources(videoSources);
this._replaceVideoSources(lines, sb);
};
Simulcast.prototype.transformBridgeDescription = function (desc) {
if (config.enableSimulcast && config.useNativeSimulcast) {
var sb = desc.sdp.split('\r\n');
this._ensureGoogConference(sb);
desc = new RTCSessionDescription({
type: desc.type,
sdp: sb.join('\r\n')
});
if (this.debugLvl && this.debugLvl > 1) {
console.info('Transformed bridge description');
console.info(desc.sdp);
}
}
return desc;
};
Simulcast.prototype._updateRemoteMaps = function (lines) {
var remoteVideoSources = this._parseMedia(lines, ['video'])[0], videoSource, quality;
if (remoteVideoSources.groups && remoteVideoSources.groups.length !== 0) {
remoteVideoSources.groups.forEach(function (group) {
if (group.semantics === 'SIM' && group.ssrcs && group.ssrcs.length !== 0) {
quality = 0;
group.ssrcs.forEach(function (ssrc) {
videoSource = remoteVideoSources.sources[ssrc];
remoteMaps.msid2Quality[videoSource.msid] = quality++;
remoteMaps.ssrc2Msid[videoSource.ssrc] = videoSource.msid;
});
}
});
}
};
Simulcast.prototype.transformLocalDescription = function (desc) {
if (config.enableSimulcast && !config.useNativeSimulcast) {
var sb = desc.sdp.split('\r\n');
this._removeSimulcastGroup(sb);
desc = new RTCSessionDescription({
type: desc.type,
sdp: sb.join('\r\n')
});
if (this.debugLvl && this.debugLvl > 1) {
console.info('Transformed local description');
console.info(desc.sdp);
}
}
return desc;
};
Simulcast.prototype.transformRemoteDescription = function (desc) {
if (config.enableSimulcast) {
var sb = desc.sdp.split('\r\n');
this._updateRemoteMaps(sb);
this._removeSimulcastGroup(sb); // NOTE(gp) this needs to be called after updateRemoteMaps!
this._ensureGoogConference(sb);
desc = new RTCSessionDescription({
type: desc.type,
sdp: sb.join('\r\n')
});
if (this.debugLvl && this.debugLvl > 1) {
console.info('Transformed remote description');
console.info(desc.sdp);
}
}
return desc;
};
Simulcast.prototype.setReceivingVideoStream = function (ssrc) {
var receivingTrack = remoteMaps.ssrc2Msid[ssrc],
msidParts = receivingTrack.split(' ');
remoteMaps.receivingVideoStreams[msidParts[0]] = msidParts[1];
};
Simulcast.prototype.getReceivingVideoStream = function (stream) {
var tracks, track, i, electedTrack, msid, quality = 1, receivingTrackId;
if (config.enableSimulcast) {
if (remoteMaps.receivingVideoStreams[stream.id])
{
receivingTrackId = remoteMaps.receivingVideoStreams[stream.id];
tracks = stream.getVideoTracks();
for (i = 0; i < tracks.length; i++) {
if (receivingTrackId === tracks[i].id) {
electedTrack = tracks[i];
break;
}
}
}
if (!electedTrack) {
tracks = stream.getVideoTracks();
for (i = 0; i < tracks.length; i++) {
track = tracks[i];
msid = [stream.id, track.id].join(' ');
if (remoteMaps.msid2Quality[msid] === quality) {
electedTrack = track;
break;
}
}
}
}
return (electedTrack)
? new webkitMediaStream([electedTrack])
: stream;
};
Simulcast.prototype.getUserMedia = function (constraints, success, err) {
// TODO(gp) what if we request a resolution not supported by the hardware?
// TODO(gp) make the lq stream configurable; although this wouldn't work with native simulcast
var lqConstraints = {
audio: false,
video: {
mandatory: {
maxWidth: 320,
maxHeight: 180
}
}
};
if (config.enableSimulcast && !config.useNativeSimulcast) {
// NOTE(gp) if we request the lq stream first webkitGetUserMedia fails randomly. Tested with Chrome 37.
navigator.webkitGetUserMedia(constraints, function (hqStream) {
// reset local maps.
localMaps.msids = [];
localMaps.msid2ssrc = {};
// add hq trackid to local map
localMaps.msids.push(hqStream.getVideoTracks()[0].id);
navigator.webkitGetUserMedia(lqConstraints, function (lqStream) {
// add lq trackid to local map
localMaps.msids.push(lqStream.getVideoTracks()[0].id);
hqStream.addTrack(lqStream.getVideoTracks()[0]);
success(hqStream);
}, err);
}, err);
} else {
// There's nothing special to do for native simulcast, so just do a normal GUM.
navigator.webkitGetUserMedia(constraints, function (hqStream) {
// reset local maps.
localMaps.msids = [];
localMaps.msid2ssrc = {};
// add hq stream to local map
localMaps.msids.push(hqStream.getVideoTracks()[0].id);
success(hqStream);
}, err);
}
};
Simulcast.prototype.getRemoteVideoStreamIdBySSRC = function (primarySSRC) {
return remoteMaps.ssrc2Msid[primarySSRC];
};
Simulcast.prototype.parseMedia = function (desc, mediatypes) {
var lines = desc.sdp.split('\r\n');
return this._parseMedia(lines, mediatypes);
};
}());

View File

@ -381,7 +381,9 @@ var VideoLayout = (function (my) {
// If the container is currently visible we attach the stream.
if (!isVideo
|| (container.offsetParent !== null && isVideo)) {
RTC.attachMediaStream(sel, stream);
var simulcast = new Simulcast();
var videoStream = simulcast.getReceivingVideoStream(stream);
RTC.attachMediaStream(sel, videoStream);
if (isVideo)
waitForRemoteVideo(sel, thessrc, stream);
@ -1248,7 +1250,9 @@ var VideoLayout = (function (my) {
&& mediaStream.type === mediaStream.VIDEO_TYPE) {
var sel = $('#participant_' + resourceJid + '>video');
RTC.attachMediaStream(sel, mediaStream.stream);
var simulcast = new Simulcast();
var videoStream = simulcast.getReceivingVideoStream(mediaStream.stream);
RTC.attachMediaStream(sel, videoStream);
waitForRemoteVideo(
sel,
mediaStream.ssrc,
@ -1288,5 +1292,84 @@ var VideoLayout = (function (my) {
}
});
/**
* On simulcast layers changed event.
*/
$(document).bind('simulcastlayerschanged', function (event, endpointSimulcastLayers) {
var simulcast = new Simulcast();
endpointSimulcastLayers.forEach(function (esl) {
var primarySSRC = esl.simulcastLayer.primarySSRC;
simulcast.setReceivingVideoStream(primarySSRC);
var msid = simulcast.getRemoteVideoStreamIdBySSRC(primarySSRC);
// Get session and stream from msid.
var session, electedStream;
var i, j, k;
if (connection.jingle) {
var keys = Object.keys(connection.jingle.sessions);
for (i = 0; i < keys.length; i++) {
var sid = keys[i];
if (electedStream) {
// stream found, stop.
break;
}
session = connection.jingle.sessions[sid];
if (session.remoteStreams) {
for (j = 0; j < session.remoteStreams.length; j++) {
var remoteStream = session.remoteStreams[j];
if (electedStream) {
// stream found, stop.
break;
}
var tracks = remoteStream.getVideoTracks();
if (tracks) {
for (k = 0; k < tracks.length; k++) {
var track = tracks[k];
if (msid === [remoteStream.id, track.id].join(' ')) {
electedStream = new webkitMediaStream([track]);
// stream found, stop.
break;
}
}
}
}
}
}
}
if (session && electedStream) {
console.info('Switching simulcast substream.');
console.info([esl, primarySSRC, msid, session, electedStream]);
var msidParts = msid.split(' ');
var selRemoteVideo = $(['#', 'remoteVideo_', session.sid, '_', msidParts[0]].join(''));
var updateLargeVideo = (ssrc2jid[videoSrcToSsrc[selRemoteVideo.attr('src')]]
== ssrc2jid[videoSrcToSsrc[$('#largeVideo').attr('src')]]);
var updateFocusedVideoSrc = (selRemoteVideo.attr('src') == focusedVideoSrc);
var electedStreamUrl = webkitURL.createObjectURL(electedStream);
selRemoteVideo.attr('src', electedStreamUrl);
videoSrcToSsrc[selRemoteVideo.attr('src')] = primarySSRC;
if (updateLargeVideo) {
VideoLayout.updateLargeVideo(electedStreamUrl);
}
if (updateFocusedVideoSrc) {
focusedVideoSrc = electedStreamUrl;
}
} else {
console.error('Could not find a stream or a session.');
}
});
});
return my;
}(VideoLayout || {}));