haskal
/
jiti-meet
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Adds simulcast support in meet.

This commit is contained in:
George Politis 2014-08-21 11:58:33 +02:00
parent 233f18cb78
commit ffaa9a62b8
11 changed files with 1117 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

94
app.js
View File

@ -70,18 +70,18 @@ function init() {
}
obtainAudioAndVideoPermissions(function (stream) {
var audioStream = new webkitMediaStream(stream);
var videoStream = new webkitMediaStream(stream);
var videoTracks = stream.getVideoTracks();
var audioStream = new webkitMediaStream();
var videoStream = new webkitMediaStream();
var audioTracks = stream.getAudioTracks();
for (var i = 0; i < videoTracks.length; i++) {
audioStream.removeTrack(videoTracks[i]);
var videoTracks = stream.getVideoTracks();
for (var i = 0; i < audioTracks.length; i++) {
audioStream.addTrack(audioTracks[i]);
}
VideoLayout.changeLocalAudio(audioStream);
startLocalRtpStatsCollector(audioStream);
for (i = 0; i < audioTracks.length; i++) {
videoStream.removeTrack(audioTracks[i]);
for (i = 0; i < videoTracks.length; i++) {
videoStream.addTrack(videoTracks[i]);
}
VideoLayout.changeLocalVideo(videoStream, true);
maybeDoJoin();
@ -237,7 +237,9 @@ function waitForRemoteVideo(selector, ssrc, stream) {
if (stream.id === 'mixedmslabel') return;
if (selector[0].currentTime > 0) {
RTC.attachMediaStream(selector, stream); // FIXME: why do i have to do this for FF?
var simulcast = new Simulcast();
var videoStream = simulcast.getReceivingVideoStream(stream);
RTC.attachMediaStream(selector, videoStream); // FIXME: why do i have to do this for FF?
// FIXME: add a class that will associate peer Jid, video.src, it's ssrc and video type
// in order to get rid of too many maps
@ -256,18 +258,40 @@ function waitForRemoteVideo(selector, ssrc, stream) {
}
$(document).bind('remotestreamadded.jingle', function (event, data, sid) {
waitForPresence(data, sid);
});
function waitForPresence(data, sid) {
var sess = connection.jingle.sessions[sid];
var thessrc;
// look up an associated JID for a stream id
if (data.stream.id.indexOf('mixedmslabel') === -1) {
// look only at a=ssrc: and _not_ at a=ssrc-group: lines
var ssrclines
= SDPUtil.find_lines(sess.peerconnection.remoteDescription.sdp, 'a=ssrc');
= SDPUtil.find_lines(sess.peerconnection.remoteDescription.sdp, 'a=ssrc:');
ssrclines = ssrclines.filter(function (line) {
return line.indexOf('mslabel:' + data.stream.label) !== -1;
});
if (ssrclines.length) {
thessrc = ssrclines[0].substring(7).split(' ')[0];
// We signal our streams (through Jingle to the focus) before we set
// our presence (through which peers associate remote streams to
// jids). So, it might arrive that a remote stream is added but
// ssrc2jid is not yet updated and thus data.peerjid cannot be
// successfully set. Here we wait for up to a second for the
// presence to arrive.
if (!ssrc2jid[thessrc]) {
setTimeout(function(d, s) {
return function() {
waitForPresence(d, s);
}
}(data, sid), 250);
return;
}
// ok to overwrite the one from focus? might save work in colibri.js
console.log('associated jid', ssrc2jid[thessrc], data.peerjid);
if (ssrc2jid[thessrc]) {
@ -276,6 +300,9 @@ $(document).bind('remotestreamadded.jingle', function (event, data, sid) {
}
}
// NOTE(gp) now that we have simulcast, a media stream can have more than 1
// ssrc. We should probably take that into account in our MediaStream
// wrapper.
mediaStreams.push(new MediaStream(data, sid, thessrc));
var container;
@ -322,7 +349,7 @@ $(document).bind('remotestreamadded.jingle', function (event, data, sid) {
sendKeyframe(sess.peerconnection);
}, 3000);
}
});
}
/**
* Returns the JID of the user to whom given <tt>videoSrc</tt> belongs.
@ -532,40 +559,35 @@ $(document).bind('callterminated.jingle', function (event, sid, jid, reason) {
$(document).bind('setLocalDescription.jingle', function (event, sid) {
// put our ssrcs into presence so other clients can identify our stream
var sess = connection.jingle.sessions[sid];
var newssrcs = {};
var directions = {};
var localSDP = new SDP(sess.peerconnection.localDescription.sdp);
localSDP.media.forEach(function (media) {
var type = SDPUtil.parse_mid(SDPUtil.find_line(media, 'a=mid:'));
var newssrcs = [];
var simulcast = new Simulcast();
var media = simulcast.parseMedia(sess.peerconnection.localDescription);
media.forEach(function (media) {
if (SDPUtil.find_line(media, 'a=ssrc:')) {
// assumes a single local ssrc
var ssrc = SDPUtil.find_line(media, 'a=ssrc:').substring(7).split(' ')[0];
newssrcs[type] = ssrc;
directions[type] = (
SDPUtil.find_line(media, 'a=sendrecv') ||
SDPUtil.find_line(media, 'a=recvonly') ||
SDPUtil.find_line(media, 'a=sendonly') ||
SDPUtil.find_line(media, 'a=inactive') ||
'a=sendrecv').substr(2);
}
// TODO(gp) maybe exclude FID streams?
Object.keys(media.sources).forEach(function(ssrc) {
newssrcs.push({
'ssrc': ssrc,
'type': media.type,
'direction': media.direction
});
});
});
console.log('new ssrcs', newssrcs);
// Have to clear presence map to get rid of removed streams
connection.emuc.clearPresenceMedia();
var i = 0;
Object.keys(newssrcs).forEach(function (mtype) {
i++;
var type = mtype;
if (newssrcs.length > 0) {
for (var i = 1; i <= newssrcs.length; i ++) {
// Change video type to screen
if (mtype === 'video' && isUsingScreenStream) {
type = 'screen';
if (newssrcs[i-1].type === 'video' && isUsingScreenStream) {
newssrcs[i-1].type = 'screen';
}
connection.emuc.addMediaToPresence(i, type, newssrcs[mtype], directions[mtype]);
});
if (i > 0) {
connection.emuc.addMediaToPresence(i,
newssrcs[i-1].type, newssrcs[i-1].ssrc, newssrcs[i-1].direction);
}
connection.emuc.sendPresence();
}
});

2
config.js
View File

@ -22,5 +22,7 @@ var config = {
useBundle: true,
enableRecording: false,
enableWelcomePage: false,
enableSimulcast: false,
useNativeSimulcast: false,
isBrand: false
};

5
data_channels.js
View File

@ -84,6 +84,11 @@ function onDataChannel(event)
'lastnchanged',
[lastNEndpoints, endpointsEnteringLastN, stream]);
}
else if ("SimulcastLayersChangedEvent" === colibriClass)
{
var endpointSimulcastLayers = obj.endpointSimulcastLayers;
$(document).trigger('simulcastlayerschanged', [endpointSimulcastLayers]);
}
else
{
console.debug("Data channel JSON-formatted message: ", obj);

1
index.html
View File

@ -10,6 +10,7 @@
<meta itemprop="description" content="Join a WebRTC video conference powered by the Jitsi Videobridge"/>
<meta itemprop="image" content="/images/jitsilogo.png"/>
<script src="libs/jquery-2.1.1.min.js"></script>
<script src="simulcast.js?v=1"></script><!-- simulcast handling -->
<script src="libs/strophe/strophe.jingle.adapter.js?v=1"></script><!-- strophe.jingle bundles -->
<script src="libs/strophe/strophe.jingle.bundle.js?v=8"></script>
<script src="libs/strophe/strophe.jingle.js?v=1"></script>

124
libs/colibri/colibri.focus.js
View File

@ -42,6 +42,7 @@ function ColibriFocus(connection, bridgejid) {
this.bridgejid = bridgejid;
this.peers = [];
this.remoteStreams = [];
this.confid = null;
/**
@ -142,6 +143,7 @@ ColibriFocus.prototype.makeConference = function (peers) {
event.peerjid = jid;
}
});
self.remoteStreams.push(event.stream);
$(document).trigger('remotestreamadded.jingle', [event, self.sid]);
};
this.peerconnection.onicecandidate = function (event) {
@ -525,9 +527,11 @@ ColibriFocus.prototype.createdConference = function (result) {
}
}
bridgeSDP.raw = bridgeSDP.session + bridgeSDP.media.join('');
var bridgeDesc = new RTCSessionDescription({type: 'offer', sdp: bridgeSDP.raw});
var simulcast = new Simulcast();
var bridgeDesc = simulcast.transformBridgeDescription(bridgeDesc);
this.peerconnection.setRemoteDescription(
new RTCSessionDescription({type: 'offer', sdp: bridgeSDP.raw}),
this.peerconnection.setRemoteDescription(bridgeDesc,
function () {
console.log('setRemoteDescription success');
self.peerconnection.createAnswer(
@ -553,6 +557,24 @@ ColibriFocus.prototype.createdConference = function (result) {
endpoint: self.myMucResource
});
// signal (through COLIBRI) to the bridge
// the SSRC groups of the participant
// that plays the role of the focus
var ssrc_group_lines = SDPUtil.find_lines(media, 'a=ssrc-group:');
var idx = 0;
ssrc_group_lines.forEach(function(line) {
idx = line.indexOf(' ');
var semantics = line.substr(0, idx).substr(13);
var ssrcs = line.substr(14 + semantics.length).split(' ');
if (ssrcs.length != 0) {
elem.c('ssrc-group', { semantics: semantics, xmlns: 'urn:xmpp:jingle:apps:rtp:ssma:0' });
ssrcs.forEach(function(ssrc) {
elem.c('source', { ssrc: ssrc })
.up();
});
elem.up();
}
});
// FIXME: should reuse code from .toJingle
for (var j = 0; j < mline.fmt.length; j++)
{
@ -646,6 +668,7 @@ ColibriFocus.prototype.initiate = function (peer, isInitiator) {
sdp.removeMediaLines(i, 'a=rtcp-mux');
}
sdp.removeMediaLines(i, 'a=ssrc:');
sdp.removeMediaLines(i, 'a=ssrc-group:');
sdp.removeMediaLines(i, 'a=crypto:');
sdp.removeMediaLines(i, 'a=candidate:');
sdp.removeMediaLines(i, 'a=ice-options:google-ice');
@ -655,14 +678,20 @@ ColibriFocus.prototype.initiate = function (peer, isInitiator) {
sdp.removeMediaLines(i, 'a=setup:');
if (1) { //i > 0) { // not for audio FIXME: does not work as intended
// re-add all remote a=ssrcs
// re-add all remote a=ssrcs _and_ a=ssrc-group
for (var jid in this.remotessrc) {
if (jid == peer || !this.remotessrc[jid][i])
continue;
sdp.media[i] += this.remotessrc[jid][i];
}
// and local a=ssrc lines
sdp.media[i] += SDPUtil.find_lines(localSDP.media[i], 'a=ssrc').join('\r\n') + '\r\n';
// add local a=ssrc-group: lines
lines = SDPUtil.find_lines(localSDP.media[i], 'a=ssrc-group:');
if (lines.length != 0)
sdp.media[i] += lines.join('\r\n') + '\r\n';
// and local a=ssrc: lines
sdp.media[i] += SDPUtil.find_lines(localSDP.media[i], 'a=ssrc:').join('\r\n') + '\r\n';
}
}
sdp.raw = sdp.session + sdp.media.join('');
@ -864,6 +893,24 @@ ColibriFocus.prototype.updateChannel = function (remoteSDP, participant) {
expire: self.channelExpire
});
// signal (throught COLIBRI) to the bridge the SSRC groups of this
// participant
var ssrc_group_lines = SDPUtil.find_lines(remoteSDP.media[channel], 'a=ssrc-group:');
var idx = 0;
ssrc_group_lines.forEach(function(line) {
idx = line.indexOf(' ');
var semantics = line.substr(0, idx).substr(13);
var ssrcs = line.substr(14 + semantics.length).split(' ');
if (ssrcs.length != 0) {
change.c('ssrc-group', { semantics: semantics, xmlns: 'urn:xmpp:jingle:apps:rtp:ssma:0' });
ssrcs.forEach(function(ssrc) {
change.c('source', { ssrc: ssrc })
.up();
});
change.up();
}
});
var rtpmap = SDPUtil.find_lines(remoteSDP.media[channel], 'a=rtpmap:');
rtpmap.forEach(function (val) {
// TODO: too much copy-paste
@ -1028,20 +1075,33 @@ ColibriFocus.prototype.setRemoteDescription = function (session, elem, desctype)
if (!remoteSDP.media[channel])
continue;
var lines = SDPUtil.find_lines(remoteSDP.media[channel], 'a=ssrc-group:');
if (lines.length != 0)
// prepend ssrc-groups
this.remotessrc[session.peerjid][channel] = lines.join('\r\n') + '\r\n';
if (SDPUtil.find_lines(remoteSDP.media[channel], 'a=ssrc:').length)
{
this.remotessrc[session.peerjid][channel] =
if (!this.remotessrc[session.peerjid][channel])
this.remotessrc[session.peerjid][channel] = '';
this.remotessrc[session.peerjid][channel] +=
SDPUtil.find_lines(remoteSDP.media[channel], 'a=ssrc:')
.join('\r\n') + '\r\n';
}
}
// ACT 4: add new a=ssrc lines to local remotedescription
// ACT 4: add new a=ssrc and s=ssrc-group lines to local remotedescription
for (channel = 0; channel < this.channels[participant].length; channel++) {
//if (channel == 0) continue; FIXME: does not work as intended
if (!remoteSDP.media[channel])
continue;
var lines = SDPUtil.find_lines(remoteSDP.media[channel], 'a=ssrc-group:');
if (lines.length != 0)
this.peerconnection.enqueueAddSsrc(
channel, SDPUtil.find_lines(remoteSDP.media[channel], 'a=ssrc-group:').join('\r\n') + '\r\n');
if (SDPUtil.find_lines(remoteSDP.media[channel], 'a=ssrc:').length) {
this.peerconnection.enqueueAddSsrc(
channel,
@ -1311,6 +1371,9 @@ ColibriFocus.prototype.sendTerminate = function (session, reason, text) {
ColibriFocus.prototype.setRTCPTerminationStrategy = function (strategyFQN) {
var self = this;
// TODO(gp) maybe move the RTCP termination strategy element under the
// content or channel element.
var strategyIQ = $iq({to: this.bridgejid, type: 'set'});
strategyIQ.c('conference', {
xmlns: 'http://jitsi.org/protocol/colibri',
@ -1378,3 +1441,50 @@ ColibriFocus.prototype.setChannelLastN = function (channelLastN) {
}
};
/**
* Sets the default value of the channel simulcast layer attribute in this
* conference and updates/patches the existing channels.
*/
ColibriFocus.prototype.setReceiveSimulcastLayer = function (receiveSimulcastLayer) {
if (('number' === typeof(receiveSimulcastLayer))
&& (this.receiveSimulcastLayer !== receiveSimulcastLayer))
{
// TODO(gp) be able to set the receiving simulcast layer on a per
// sender basis.
this.receiveSimulcastLayer = receiveSimulcastLayer;
// Update/patch the existing channels.
var patch = $iq({ to: this.bridgejid, type: 'set' });
patch.c(
'conference',
{ xmlns: 'http://jitsi.org/protocol/colibri', id: this.confid });
patch.c('content', { name: 'video' });
patch.c(
'channel',
{
id: $(this.mychannel[1 /* video */]).attr('id'),
'receive-simulcast-layer': this.receiveSimulcastLayer
});
patch.up(); // end of channel
for (var p = 0; p < this.channels.length; p++)
{
patch.c(
'channel',
{
id: $(this.channels[p][1 /* video */]).attr('id'),
'receive-simulcast-layer': this.receiveSimulcastLayer
});
patch.up(); // end of channel
}
this.connection.sendIQ(
patch,
function (res) {
console.info('Set channel simulcast receive layer succeeded:', res);
},
function (err) {
console.error('Set channel simulcast receive layer failed:', err);
});
}
};

57
libs/strophe/strophe.jingle.adapter.js
View File

@ -128,7 +128,11 @@ dumpSDP = function(description) {
if (TraceablePeerConnection.prototype.__defineGetter__ !== undefined) {
TraceablePeerConnection.prototype.__defineGetter__('signalingState', function() { return this.peerconnection.signalingState; });
TraceablePeerConnection.prototype.__defineGetter__('iceConnectionState', function() { return this.peerconnection.iceConnectionState; });
TraceablePeerConnection.prototype.__defineGetter__('localDescription', function() { return this.peerconnection.localDescription; });
TraceablePeerConnection.prototype.__defineGetter__('localDescription', function() {
var simulcast = new Simulcast();
var publicLocalDescription = simulcast.makeLocalDescriptionPublic(this.peerconnection.localDescription);
return publicLocalDescription;
});
TraceablePeerConnection.prototype.__defineGetter__('remoteDescription', function() { return this.peerconnection.remoteDescription; });
}
@ -149,6 +153,8 @@ TraceablePeerConnection.prototype.createDataChannel = function (label, opts) {
TraceablePeerConnection.prototype.setLocalDescription = function (description, successCallback, failureCallback) {
var self = this;
var simulcast = new Simulcast();
description = simulcast.transformLocalDescription(description);
this.trace('setLocalDescription', dumpSDP(description));
this.peerconnection.setLocalDescription(description,
function () {
@ -169,6 +175,8 @@ TraceablePeerConnection.prototype.setLocalDescription = function (description, s
TraceablePeerConnection.prototype.setRemoteDescription = function (description, successCallback, failureCallback) {
var self = this;
var simulcast = new Simulcast();
description = simulcast.transformRemoteDescription(description);
this.trace('setRemoteDescription', dumpSDP(description));
this.peerconnection.setRemoteDescription(description,
function () {
@ -208,6 +216,16 @@ TraceablePeerConnection.prototype.addSource = function (elem) {
$(elem).each(function (idx, content) {
var name = $(content).attr('name');
var lines = '';
tmp = $(content).find('ssrc-group[xmlns="urn:xmpp:jingle:apps:rtp:ssma:0"]').each(function() {
var semantics = this.getAttribute('semantics');
var ssrcs = $(this).find('>source').map(function () {
return this.getAttribute('ssrc');
}).get();
if (ssrcs.length != 0) {
lines += 'a=ssrc-group:' + semantics + ' ' + ssrcs.join(' ') + '\r\n';
}
});
tmp = $(content).find('source[xmlns="urn:xmpp:jingle:apps:rtp:ssma:0"]'); // can handle both >source and >description>source
tmp.each(function () {
var ssrc = $(this).attr('ssrc');
@ -254,6 +272,16 @@ TraceablePeerConnection.prototype.removeSource = function (elem) {
$(elem).each(function (idx, content) {
var name = $(content).attr('name');
var lines = '';
tmp = $(content).find('ssrc-group[xmlns="urn:xmpp:jingle:apps:rtp:ssma:0"]').each(function() {
var semantics = this.getAttribute('semantics');
var ssrcs = $(this).find('>source').map(function () {
return this.getAttribute('ssrc');
}).get();
if (ssrcs.length != 0) {
lines += 'a=ssrc-group:' + semantics + ' ' + ssrcs.join(' ') + '\r\n';
}
});
tmp = $(content).find('source[xmlns="urn:xmpp:jingle:apps:rtp:ssma:0"]'); // can handle both >source and >description>source
tmp.each(function () {
var ssrc = $(this).attr('ssrc');
@ -413,6 +441,8 @@ TraceablePeerConnection.prototype.createAnswer = function (successCallback, fail
this.trace('createAnswer', JSON.stringify(constraints, null, ' '));
this.peerconnection.createAnswer(
function (answer) {
var simulcast = new Simulcast();
answer = simulcast.transformAnswer(answer);
self.trace('createAnswerOnSuccess', dumpSDP(answer));
successCallback(answer);
},
@ -628,7 +658,30 @@ function getUserMediaWithConstraints(um, success_callback, failure_callback, res
constraints.video.mandatory.minFrameRate = fps;
}
var isFF = navigator.userAgent.toLowerCase().indexOf('firefox') > -1;
try {
if (config.enableSimulcast
&& constraints.video
&& constraints.video.chromeMediaSource !== 'screen'
&& constraints.video.chromeMediaSource !== 'desktop'
&& !isAndroid
// We currently do not support FF, as it doesn't have multistream support.
&& !isFF) {
var simulcast = new Simulcast();
simulcast.getUserMedia(constraints, function (stream) {
console.log('onUserMediaSuccess');
success_callback(stream);
},
function (error) {
console.warn('Failed to get access to local media. Error ', error);
if (failure_callback) {
failure_callback(error);
}
});
} else {
RTC.getUserMedia(constraints,
function (stream) {
console.log('onUserMediaSuccess');
@ -640,6 +693,8 @@ function getUserMediaWithConstraints(um, success_callback, failure_callback, res
failure_callback(error);
}
});
}
} catch (e) {
console.error('GUM failed: ', e);
if(failure_callback) {

89
libs/strophe/strophe.jingle.sdp.js
View File

@ -31,6 +31,15 @@ SDP.prototype.getMediaSsrcMap = function() {
}
channel.ssrcs[linessrc].lines.push(line);
});
tmp = SDPUtil.find_lines(self.media[channelNum], 'a=ssrc-group:');
tmp.forEach(function(line){
var semantics = line.substr(0, idx).substr(13);
var ssrcs = line.substr(14 + semantics.length).split(' ');
if (ssrcs.length != 0) {
var ssrcGroup = new ChannelSsrcGroup(semantics, ssrcs);
channel.ssrcGroups.push(ssrcGroup);
}
});
}
return media_ssrcs;
}
@ -56,6 +65,32 @@ SDP.prototype.containsSSRC = function(ssrc) {
* @param otherSdp the other SDP to check ssrc with.
*/
SDP.prototype.getNewMedia = function(otherSdp) {
// this could be useful in Array.prototype.
function arrayEquals(array) {
// if the other array is a falsy value, return
if (!array)
return false;
// compare lengths - can save a lot of time
if (this.length != array.length)
return false;
for (var i = 0, l=this.length; i < l; i++) {
// Check if we have nested arrays
if (this[i] instanceof Array && array[i] instanceof Array) {
// recurse into the nested arrays
if (!this[i].equals(array[i]))
return false;
}
else if (this[i] != array[i]) {
// Warning - two different object instances will never be equal: {x:20} != {x:20}
return false;
}
}
return true;
};
var myMedia = this.getMediaSsrcMap();
var othersMedia = otherSdp.getMediaSsrcMap();
var newMedia = {};
@ -77,6 +112,32 @@ SDP.prototype.getNewMedia = function(otherSdp) {
newMedia[channelNum].ssrcs[ssrc] = othersChannel.ssrcs[ssrc];
}
})
// Look for new ssrc groups across the channels
othersChannel.ssrcGroups.forEach(function(otherSsrcGroup){
// try to match the other ssrc-group with an ssrc-group of ours
var matched = false;
for (var i = 0; i < myChannel.ssrcGroups.length; i++) {
var mySsrcGroup = myChannel.ssrcGroups[i];
if (otherSsrcGroup.semantics == mySsrcGroup
&& arrayEquals.apply(otherSsrcGroup.ssrcs, [mySsrcGroup.ssrcs])) {
matched = true;
break;
}
}
if (!matched) {
// Allocate channel if we've found an ssrc-group that doesn't
// exist in our channel
if(!newMedia[channelNum]){
newMedia[channelNum] = new MediaChannel(othersChannel.chNumber, othersChannel.mediaType);
}
newMedia[channelNum].ssrcGroups.push(otherSsrcGroup);
}
});
});
return newMedia;
}
@ -241,6 +302,22 @@ SDP.prototype.toJingle = function (elem, thecreator) {
tmp.xmlns = 'http://estos.de/ns/ssrc';
tmp.ssrc = ssrc;
elem.c('ssrc', tmp).up(); // ssrc is part of description
// XEP-0339 handle ssrc-group attributes
var ssrc_group_lines = SDPUtil.find_lines(this.media[i], 'a=ssrc-group:');
ssrc_group_lines.forEach(function(line) {
idx = line.indexOf(' ');
var semantics = line.substr(0, idx).substr(13);
var ssrcs = line.substr(14 + semantics.length).split(' ');
if (ssrcs.length != 0) {
elem.c('ssrc-group', { semantics: semantics, xmlns: 'urn:xmpp:jingle:apps:rtp:ssma:0' });
ssrcs.forEach(function(ssrc) {
elem.c('source', { ssrc: ssrc })
.up();
});
elem.up();
}
});
}
if (SDPUtil.find_line(this.media[i], 'a=rtcp-mux')) {
@ -578,6 +655,18 @@ SDP.prototype.jingle2media = function (content) {
media += SDPUtil.candidateFromJingle(this);
});
// XEP-0339 handle ssrc-group attributes
tmp = content.find('description>ssrc-group[xmlns="urn:xmpp:jingle:apps:rtp:ssma:0"]').each(function() {
var semantics = this.getAttribute('semantics');
var ssrcs = $(this).find('>source').map(function() {
return this.getAttribute('ssrc');
}).get();
if (ssrcs.length != 0) {
media += 'a=ssrc-group:' + semantics + ' ' + ssrcs.join(' ') + '\r\n';
}
});
tmp = content.find('description>source[xmlns="urn:xmpp:jingle:apps:rtp:ssma:0"]');
tmp.each(function () {
var ssrc = this.getAttribute('ssrc');

17
libs/strophe/strophe.jingle.sdp.util.js
View File

@ -15,6 +15,17 @@ function ChannelSsrc(ssrc, type) {
this.lines = [];
}
/**
* Class holds a=ssrc-group: lines
* @param semantics
* @param ssrcs
* @constructor
*/
function ChannelSsrcGroup(semantics, ssrcs, line) {
this.semantics = semantics;
this.ssrcs = ssrcs;
}
/**
* Helper class represents media channel. Is a container for ChannelSsrc, holds channel idx and media type.
* @param channelNumber channel idx in SDP media array.
@ -36,6 +47,12 @@ function MediaChannel(channelNumber, mediaType) {
* The maps of ssrc numbers to ChannelSsrc objects.
*/
this.ssrcs = {};
/**
* The array of ChannelSsrcGroup objects.
* @type {Array}
*/
this.ssrcGroups = [];
}
SDPUtil = {

7
libs/strophe/strophe.jingle.session.js
View File

@ -119,6 +119,8 @@ JingleSession.prototype.accept = function () {
// FIXME: change any inactive to sendrecv or whatever they were originally
pranswer.sdp = pranswer.sdp.replace('a=inactive', 'a=sendrecv');
}
var simulcast = new Simulcast();
pranswer = simulcast.makeLocalDescriptionPublic(pranswer);
var prsdp = new SDP(pranswer.sdp);
var accept = $iq({to: this.peerjid,
type: 'set'})
@ -565,7 +567,10 @@ JingleSession.prototype.createdAnswer = function (sdp, provisional) {
initiator: this.initiator,
responder: this.responder,
sid: this.sid });
this.localSDP.toJingle(accept, this.initiator == this.me ? 'initiator' : 'responder');
var simulcast = new Simulcast();
var publicLocalDesc = simulcast.makeLocalDescriptionPublic(sdp);
var publicLocalSDP = new SDP(publicLocalDesc.sdp);
publicLocalSDP.toJingle(accept, this.initiator == this.me ? 'initiator' : 'responder');
this.connection.sendIQ(accept,
function () {
var ack = {};

669
simulcast.js Normal file
View File

@ -0,0 +1,669 @@
/*jslint plusplus: true */
/*jslint nomen: true*/
/**
* Created by gp on 11/08/14.
*/
function Simulcast() {
"use strict";
// TODO(gp) split the Simulcast class in two classes : NativeSimulcast and ClassicSimulcast.
this.debugLvl = 1;
}
(function () {
"use strict";
// global state for all transformers.
var localExplosionMap = {}, localVideoSourceCache, emptyCompoundIndex,
remoteMaps = {
msid2Quality: {},
ssrc2Msid: {},
receivingVideoStreams: {}
}, localMaps = {
msids: [],
msid2ssrc: {}
};
Simulcast.prototype._generateGuid = (function () {
function s4() {
return Math.floor((1 + Math.random()) * 0x10000)
.toString(16)
.substring(1);
}
return function () {
return s4() + s4() + '-' + s4() + '-' + s4() + '-' +
s4() + '-' + s4() + s4() + s4();
};
}());
Simulcast.prototype._cacheVideoSources = function (lines) {
localVideoSourceCache = this._getVideoSources(lines);
};
Simulcast.prototype._restoreVideoSources = function (lines) {
this._replaceVideoSources(lines, localVideoSourceCache);
};
Simulcast.prototype._replaceVideoSources = function (lines, videoSources) {
var i, inVideo = false, index = -1, howMany = 0;
if (this.debugLvl) {
console.info('Replacing video sources...');
}
for (i = 0; i < lines.length; i++) {
if (inVideo && lines[i].substring(0, 'm='.length) === 'm=') {
// Out of video.
break;
}
if (!inVideo && lines[i].substring(0, 'm=video '.length) === 'm=video ') {
// In video.
inVideo = true;
}
if (inVideo && (lines[i].substring(0, 'a=ssrc:'.length) === 'a=ssrc:'
|| lines[i].substring(0, 'a=ssrc-group:'.length) === 'a=ssrc-group:')) {
if (index === -1) {
index = i;
}
howMany++;
}
}
// efficiency baby ;)
lines.splice.apply(lines,
[index, howMany].concat(videoSources));
};
Simulcast.prototype._getVideoSources = function (lines) {
var i, inVideo = false, sb = [];
if (this.debugLvl) {
console.info('Getting video sources...');
}
for (i = 0; i < lines.length; i++) {
if (inVideo && lines[i].substring(0, 'm='.length) === 'm=') {
// Out of video.
break;
}
if (!inVideo && lines[i].substring(0, 'm=video '.length) === 'm=video ') {
// In video.
inVideo = true;
}
if (inVideo && lines[i].substring(0, 'a=ssrc:'.length) === 'a=ssrc:') {
// In SSRC.
sb.push(lines[i]);
}
if (inVideo && lines[i].substring(0, 'a=ssrc-group:'.length) === 'a=ssrc-group:') {
sb.push(lines[i]);
}
}
return sb;
};
Simulcast.prototype._parseMedia = function (lines, mediatypes) {
var i, res = [], type, cur_media, idx, ssrcs, cur_ssrc, ssrc,
ssrc_attribute, group, semantics, skip;
if (this.debugLvl) {
console.info('Parsing media sources...');
}
for (i = 0; i < lines.length; i++) {
if (lines[i].substring(0, 'm='.length) === 'm=') {
type = lines[i]
.substr('m='.length, lines[i].indexOf(' ') - 'm='.length);
skip = mediatypes !== undefined && mediatypes.indexOf(type) === -1;
if (!skip) {
cur_media = {
'type': type,
'sources': {},
'groups': []
};
res.push(cur_media);
}
} else if (!skip && lines[i].substring(0, 'a=ssrc:'.length) === 'a=ssrc:') {
idx = lines[i].indexOf(' ');
ssrc = lines[i].substring('a=ssrc:'.length, idx);
if (cur_media.sources[ssrc] === undefined) {
cur_ssrc = {'ssrc': ssrc};
cur_media.sources[ssrc] = cur_ssrc;
}
ssrc_attribute = lines[i].substr(idx + 1).split(':', 2)[0];
cur_ssrc[ssrc_attribute] = lines[i].substr(idx + 1).split(':', 2)[1];
if (cur_media.base === undefined) {
cur_media.base = cur_ssrc;
}
} else if (!skip && lines[i].substring(0, 'a=ssrc-group:'.length) === 'a=ssrc-group:') {
idx = lines[i].indexOf(' ');
semantics = lines[i].substr(0, idx).substr('a=ssrc-group:'.length);
ssrcs = lines[i].substr(idx).trim().split(' ');
group = {
'semantics': semantics,
'ssrcs': ssrcs
};
cur_media.groups.push(group);
} else if (!skip && (lines[i].substring(0, 'a=sendrecv'.length) === 'a=sendrecv' ||
lines[i].substring(0, 'a=recvonly'.length) === 'a=recvonly' ||
lines[i].substring(0, 'a=sendonly'.length) === 'a=sendonly' ||
lines[i].substring(0, 'a=inactive'.length) === 'a=inactive')) {
cur_media.direction = lines[i].substring('a='.length, 8);
}
}
return res;
};
// Returns a random integer between min (included) and max (excluded)
// Using Math.round() will give you a non-uniform distribution!
Simulcast.prototype._generateRandomSSRC = function () {
var min = 0, max = 0xffffffff;
return Math.floor(Math.random() * (max - min)) + min;
};
function CompoundIndex(obj) {
if (obj !== undefined) {
this.row = obj.row;
this.column = obj.column;
}
}
emptyCompoundIndex = new CompoundIndex();
Simulcast.prototype._indexOfArray = function (needle, haystack, start) {
var length = haystack.length, idx, i;
if (!start) {
start = 0;
}
for (i = start; i < length; i++) {
idx = haystack[i].indexOf(needle);
if (idx !== -1) {
return new CompoundIndex({row: i, column: idx});
}
}
return emptyCompoundIndex;
};
Simulcast.prototype._removeSimulcastGroup = function (lines) {
var i;
for (i = lines.length - 1; i >= 0; i--) {
if (lines[i].indexOf('a=ssrc-group:SIM') !== -1) {
lines.splice(i, 1);
}
}
};
Simulcast.prototype._explodeLocalSimulcastSources = function (lines) {
var sb, msid, sid, tid, videoSources, self;
if (this.debugLvl) {
console.info('Exploding local video sources...');
}
videoSources = this._parseMedia(lines, ['video'])[0];
self = this;
if (videoSources.groups && videoSources.groups.length !== 0) {
videoSources.groups.forEach(function (group) {
if (group.semantics === 'SIM') {
group.ssrcs.forEach(function (ssrc) {
// Get the msid for this ssrc..
if (localExplosionMap[ssrc]) {
// .. either from the explosion map..
msid = localExplosionMap[ssrc];
} else {
// .. or generate a new one (msid).
sid = videoSources.sources[ssrc].msid
.substring(0, videoSources.sources[ssrc].msid.indexOf(' '));
tid = self._generateGuid();
msid = [sid, tid].join(' ');
localExplosionMap[ssrc] = msid;
}
// Assign it to the source object.
videoSources.sources[ssrc].msid = msid;
// TODO(gp) Change the msid of associated sources.
});
}
});
}
sb = this._compileVideoSources(videoSources);
this._replaceVideoSources(lines, sb);
};
Simulcast.prototype._groupLocalVideoSources = function (lines) {
var sb, videoSources, ssrcs = [], ssrc;
if (this.debugLvl) {
console.info('Grouping local video sources...');
}
videoSources = this._parseMedia(lines, ['video'])[0];
for (ssrc in videoSources.sources) {
// jitsi-meet destroys/creates streams at various places causing
// the original local stream ids to change. The only thing that
// remains unchanged is the trackid.
localMaps.msid2ssrc[videoSources.sources[ssrc].msid.split(' ')[1]] = ssrc;
}
// TODO(gp) add only "free" sources.
localMaps.msids.forEach(function (msid) {
ssrcs.push(localMaps.msid2ssrc[msid]);
});
if (!videoSources.groups) {
videoSources.groups = [];
}
videoSources.groups.push({
'semantics': 'SIM',
'ssrcs': ssrcs
});
sb = this._compileVideoSources(videoSources);
this._replaceVideoSources(lines, sb);
};
Simulcast.prototype._appendSimulcastGroup = function (lines) {
var videoSources, ssrcGroup, simSSRC, numOfSubs = 3, i, sb, msid;
if (this.debugLvl) {
console.info('Appending simulcast group...');
}
// Get the primary SSRC information.
videoSources = this._parseMedia(lines, ['video'])[0];
// Start building the SIM SSRC group.
ssrcGroup = ['a=ssrc-group:SIM'];
// The video source buffer.
sb = [];
// Create the simulcast sub-streams.
for (i = 0; i < numOfSubs; i++) {
// TODO(gp) prevent SSRC collision.
simSSRC = this._generateRandomSSRC();
ssrcGroup.push(simSSRC);
sb.splice.apply(sb, [sb.length, 0].concat(
[["a=ssrc:", simSSRC, " cname:", videoSources.base.cname].join(''),
["a=ssrc:", simSSRC, " msid:", videoSources.base.msid].join('')]
));
if (this.debugLvl) {
console.info(['Generated substream ', i, ' with SSRC ', simSSRC, '.'].join(''));
}
}
// Add the group sim layers.
sb.splice(0, 0, ssrcGroup.join(' '))
this._replaceVideoSources(lines, sb);
};
// Does the actual patching.
Simulcast.prototype._ensureSimulcastGroup = function (lines) {
if (this.debugLvl) {
console.info('Ensuring simulcast group...');
}
if (this._indexOfArray('a=ssrc-group:SIM', lines) === emptyCompoundIndex) {
this._appendSimulcastGroup(lines);
this._cacheVideoSources(lines);
} else {
// verify that the ssrcs participating in the SIM group are present
// in the SDP (needed for presence).
this._restoreVideoSources(lines);
}
};
Simulcast.prototype._ensureGoogConference = function (lines) {
var sb;
if (this.debugLvl) {
console.info('Ensuring x-google-conference flag...')
}
if (this._indexOfArray('a=x-google-flag:conference', lines) === emptyCompoundIndex) {
// Add the google conference flag
sb = this._getVideoSources(lines);
sb = ['a=x-google-flag:conference'].concat(sb);
this._replaceVideoSources(lines, sb);
}
};
Simulcast.prototype._compileVideoSources = function (videoSources) {
var sb = [], ssrc, addedSSRCs = [];
if (this.debugLvl) {
console.info('Compiling video sources...');
}
// Add the groups
if (videoSources.groups && videoSources.groups.length !== 0) {
videoSources.groups.forEach(function (group) {
if (group.ssrcs && group.ssrcs.length !== 0) {
sb.push([['a=ssrc-group:', group.semantics].join(''), group.ssrcs.join(' ')].join(' '));
// if (group.semantics !== 'SIM') {
group.ssrcs.forEach(function (ssrc) {
addedSSRCs.push(ssrc);
sb.splice.apply(sb, [sb.length, 0].concat([
["a=ssrc:", ssrc, " cname:", videoSources.sources[ssrc].cname].join(''),
["a=ssrc:", ssrc, " msid:", videoSources.sources[ssrc].msid].join('')]));
});
//}
}
});
}
// Then add any free sources.
if (videoSources.sources) {
for (ssrc in videoSources.sources) {
if (addedSSRCs.indexOf(ssrc) === -1) {
sb.splice.apply(sb, [sb.length, 0].concat([
["a=ssrc:", ssrc, " cname:", videoSources.sources[ssrc].cname].join(''),
["a=ssrc:", ssrc, " msid:", videoSources.sources[ssrc].msid].join('')]));
}
}
}
return sb;
};
Simulcast.prototype.transformAnswer = function (desc) {
if (config.enableSimulcast && config.useNativeSimulcast) {
var sb = desc.sdp.split('\r\n');
// Even if we have enabled native simulcasting previously
// (with a call to SLD with an appropriate SDP, for example),
// createAnswer seems to consistently generate incomplete SDP
// with missing SSRCS.
//
// So, subsequent calls to SLD will have missing SSRCS and presence
// won't have the complete list of SRCs.
this._ensureSimulcastGroup(sb);
desc = new RTCSessionDescription({
type: desc.type,
sdp: sb.join('\r\n')
});
if (this.debugLvl && this.debugLvl > 1) {
console.info('Transformed answer');
console.info(desc.sdp);
}
}
return desc;
};
Simulcast.prototype.makeLocalDescriptionPublic = function (desc) {
var sb;
if (!desc || desc == null)
return desc;
if (config.enableSimulcast) {
if (config.useNativeSimulcast) {
sb = desc.sdp.split('\r\n');
this._explodeLocalSimulcastSources(sb);
desc = new RTCSessionDescription({
type: desc.type,
sdp: sb.join('\r\n')
});
if (this.debugLvl && this.debugLvl > 1) {
console.info('Exploded local video sources');
console.info(desc.sdp);
}
} else {
sb = desc.sdp.split('\r\n');
this._groupLocalVideoSources(sb);
desc = new RTCSessionDescription({
type: desc.type,
sdp: sb.join('\r\n')
});
if (this.debugLvl && this.debugLvl > 1) {
console.info('Grouped local video sources');
console.info(desc.sdp);
}
}
}
return desc;
};
Simulcast.prototype._ensureOrder = function (lines) {
var videoSources, sb;
videoSources = this._parseMedia(lines, ['video'])[0];
sb = this._compileVideoSources(videoSources);
this._replaceVideoSources(lines, sb);
};
Simulcast.prototype.transformBridgeDescription = function (desc) {
if (config.enableSimulcast && config.useNativeSimulcast) {
var sb = desc.sdp.split('\r\n');
this._ensureGoogConference(sb);
desc = new RTCSessionDescription({
type: desc.type,
sdp: sb.join('\r\n')
});
if (this.debugLvl && this.debugLvl > 1) {
console.info('Transformed bridge description');
console.info(desc.sdp);
}
}
return desc;
};
Simulcast.prototype._updateRemoteMaps = function (lines) {
var remoteVideoSources = this._parseMedia(lines, ['video'])[0], videoSource, quality;
if (remoteVideoSources.groups && remoteVideoSources.groups.length !== 0) {
remoteVideoSources.groups.forEach(function (group) {
if (group.semantics === 'SIM' && group.ssrcs && group.ssrcs.length !== 0) {
quality = 0;
group.ssrcs.forEach(function (ssrc) {
videoSource = remoteVideoSources.sources[ssrc];
remoteMaps.msid2Quality[videoSource.msid] = quality++;
remoteMaps.ssrc2Msid[videoSource.ssrc] = videoSource.msid;
});
}
});
}
};
Simulcast.prototype.transformLocalDescription = function (desc) {
if (config.enableSimulcast && !config.useNativeSimulcast) {
var sb = desc.sdp.split('\r\n');
this._removeSimulcastGroup(sb);
desc = new RTCSessionDescription({
type: desc.type,
sdp: sb.join('\r\n')
});
if (this.debugLvl && this.debugLvl > 1) {
console.info('Transformed local description');
console.info(desc.sdp);
}
}
return desc;
};
Simulcast.prototype.transformRemoteDescription = function (desc) {
if (config.enableSimulcast) {
var sb = desc.sdp.split('\r\n');
this._updateRemoteMaps(sb);
this._removeSimulcastGroup(sb); // NOTE(gp) this needs to be called after updateRemoteMaps!
this._ensureGoogConference(sb);
desc = new RTCSessionDescription({
type: desc.type,
sdp: sb.join('\r\n')
});
if (this.debugLvl && this.debugLvl > 1) {
console.info('Transformed remote description');
console.info(desc.sdp);
}
}
return desc;
};
Simulcast.prototype.setReceivingVideoStream = function (ssrc) {
var receivingTrack = remoteMaps.ssrc2Msid[ssrc],
msidParts = receivingTrack.split(' ');
remoteMaps.receivingVideoStreams[msidParts[0]] = msidParts[1];
};
Simulcast.prototype.getReceivingVideoStream = function (stream) {
var tracks, track, i, electedTrack, msid, quality = 1, receivingTrackId;
if (config.enableSimulcast) {
if (remoteMaps.receivingVideoStreams[stream.id])
{
receivingTrackId = remoteMaps.receivingVideoStreams[stream.id];
tracks = stream.getVideoTracks();
for (i = 0; i < tracks.length; i++) {
if (receivingTrackId === tracks[i].id) {
electedTrack = tracks[i];
break;
}
}
}
if (!electedTrack) {
tracks = stream.getVideoTracks();
for (i = 0; i < tracks.length; i++) {
track = tracks[i];
msid = [stream.id, track.id].join(' ');
if (remoteMaps.msid2Quality[msid] === quality) {
electedTrack = track;
break;
}
}
}
}
return (electedTrack)
? new webkitMediaStream([electedTrack])
: stream;
};
Simulcast.prototype.getUserMedia = function (constraints, success, err) {
// TODO(gp) what if we request a resolution not supported by the hardware?
// TODO(gp) make the lq stream configurable; although this wouldn't work with native simulcast
var lqConstraints = {
audio: false,
video: {
mandatory: {
maxWidth: 320,
maxHeight: 180
}
}
};
if (config.enableSimulcast && !config.useNativeSimulcast) {
// NOTE(gp) if we request the lq stream first webkitGetUserMedia fails randomly. Tested with Chrome 37.
navigator.webkitGetUserMedia(constraints, function (hqStream) {
// reset local maps.
localMaps.msids = [];
localMaps.msid2ssrc = {};
// add hq trackid to local map
localMaps.msids.push(hqStream.getVideoTracks()[0].id);
navigator.webkitGetUserMedia(lqConstraints, function (lqStream) {
// add lq trackid to local map
localMaps.msids.push(lqStream.getVideoTracks()[0].id);
hqStream.addTrack(lqStream.getVideoTracks()[0]);
success(hqStream);
}, err);
}, err);
} else {
// There's nothing special to do for native simulcast, so just do a normal GUM.
navigator.webkitGetUserMedia(constraints, function (hqStream) {
// reset local maps.
localMaps.msids = [];
localMaps.msid2ssrc = {};
// add hq stream to local map
localMaps.msids.push(hqStream.getVideoTracks()[0].id);
success(hqStream);
}, err);
}
};
Simulcast.prototype.getRemoteVideoStreamIdBySSRC = function (primarySSRC) {
return remoteMaps.ssrc2Msid[primarySSRC];
};
Simulcast.prototype.parseMedia = function (desc, mediatypes) {
var lines = desc.sdp.split('\r\n');
return this._parseMedia(lines, mediatypes);
};
}());

87
videolayout.js
View File

@ -381,7 +381,9 @@ var VideoLayout = (function (my) {
// If the container is currently visible we attach the stream.
if (!isVideo
|| (container.offsetParent !== null && isVideo)) {
RTC.attachMediaStream(sel, stream);
var simulcast = new Simulcast();
var videoStream = simulcast.getReceivingVideoStream(stream);
RTC.attachMediaStream(sel, videoStream);
if (isVideo)
waitForRemoteVideo(sel, thessrc, stream);
@ -1248,7 +1250,9 @@ var VideoLayout = (function (my) {
&& mediaStream.type === mediaStream.VIDEO_TYPE) {
var sel = $('#participant_' + resourceJid + '>video');
RTC.attachMediaStream(sel, mediaStream.stream);
var simulcast = new Simulcast();
var videoStream = simulcast.getReceivingVideoStream(mediaStream.stream);
RTC.attachMediaStream(sel, videoStream);
waitForRemoteVideo(
sel,
mediaStream.ssrc,
@ -1288,5 +1292,84 @@ var VideoLayout = (function (my) {
}
});
/**
* On simulcast layers changed event.
*/
$(document).bind('simulcastlayerschanged', function (event, endpointSimulcastLayers) {
var simulcast = new Simulcast();
endpointSimulcastLayers.forEach(function (esl) {
var primarySSRC = esl.simulcastLayer.primarySSRC;
simulcast.setReceivingVideoStream(primarySSRC);
var msid = simulcast.getRemoteVideoStreamIdBySSRC(primarySSRC);
// Get session and stream from msid.
var session, electedStream;
var i, j, k;
if (connection.jingle) {
var keys = Object.keys(connection.jingle.sessions);
for (i = 0; i < keys.length; i++) {
var sid = keys[i];
if (electedStream) {
// stream found, stop.
break;
}
session = connection.jingle.sessions[sid];
if (session.remoteStreams) {
for (j = 0; j < session.remoteStreams.length; j++) {
var remoteStream = session.remoteStreams[j];
if (electedStream) {
// stream found, stop.
break;
}
var tracks = remoteStream.getVideoTracks();
if (tracks) {
for (k = 0; k < tracks.length; k++) {
var track = tracks[k];
if (msid === [remoteStream.id, track.id].join(' ')) {
electedStream = new webkitMediaStream([track]);
// stream found, stop.
break;
}
}
}
}
}
}
}
if (session && electedStream) {
console.info('Switching simulcast substream.');
console.info([esl, primarySSRC, msid, session, electedStream]);
var msidParts = msid.split(' ');
var selRemoteVideo = $(['#', 'remoteVideo_', session.sid, '_', msidParts[0]].join(''));
var updateLargeVideo = (ssrc2jid[videoSrcToSsrc[selRemoteVideo.attr('src')]]
== ssrc2jid[videoSrcToSsrc[$('#largeVideo').attr('src')]]);
var updateFocusedVideoSrc = (selRemoteVideo.attr('src') == focusedVideoSrc);
var electedStreamUrl = webkitURL.createObjectURL(electedStream);
selRemoteVideo.attr('src', electedStreamUrl);
videoSrcToSsrc[selRemoteVideo.attr('src')] = primarySSRC;
if (updateLargeVideo) {
VideoLayout.updateLargeVideo(electedStreamUrl);
}
if (updateFocusedVideoSrc) {
focusedVideoSrc = electedStreamUrl;
}
} else {
console.error('Could not find a stream or a session.');
}
});
});
return my;
}(VideoLayout || {}));