haskal
/
gst-meet
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

make jingle more generic over codecs 

This adds a very simple Codec struct, which describes each codec
supported by a RTP description.

It will make it much easier to add newer codecs in the future, like AV1.

This commit is best viewed with git log -p -w to ignore indent changes.
This commit is contained in:
Emmanuel Gil Peyrot 2021-10-23 19:31:04 +02:00 committed by Jasper
parent 5db78b11bf
commit 7fa31bd796
1 changed files with 292 additions and 318 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

428
lib-gst-meet/src/jingle.rs
View File

@ -41,18 +41,82 @@ const RTP_HDREXT_TRANSPORT_CC: &str =
const DEFAULT_STUN_PORT: u16 = 3478;
const DEFAULT_TURNS_PORT: u16 = 5349;
#[derive(Clone, PartialEq)]
enum CodecName {
Opus,
H264,
Vp8,
Vp9,
}
#[derive(Clone)]
struct Codec {
name: CodecName,
pt: u8,
rtx_pt: Option<u8>,
rtcp_fbs: Vec<RtcpFb>,
}
impl Codec {
fn is(&self, pt: u8) -> bool {
self.pt == pt
}
fn is_rtx(&self, rtx_pt: u8) -> bool {
if let Some(pt) = self.rtx_pt {
pt == rtx_pt
} else {
false
}
}
fn is_audio(&self) -> bool {
self.name == CodecName::Opus
}
fn is_video(&self) -> bool {
self.name != CodecName::Opus
}
fn is_codec(&self, name: &str) -> bool {
match name {
"h264" => self.name == CodecName::H264,
"vp8" => self.name == CodecName::Vp8,
"vp9" => self.name == CodecName::Vp9,
_ => false,
}
}
fn encoding_name(&self) -> &'static str {
match self.name {
CodecName::Opus => "opus",
CodecName::H264 => "H264",
CodecName::Vp8 => "VP8",
CodecName::Vp9 => "VP9",
}
}
fn make_depay_name(&self) -> &'static str {
match self.name {
CodecName::Opus => "rtpopusdepay",
CodecName::H264 => "rtph264depay",
CodecName::Vp8 => "rtpvp8depay",
CodecName::Vp9 => "rtpvp9depay",
}
}
fn make_pay_name(&self) -> &'static str {
match self.name {
CodecName::Opus => "rtpopuspay",
CodecName::H264 => "rtph264pay",
CodecName::Vp8 => "rtpvp8pay",
CodecName::Vp9 => "rtpvp9pay",
}
}
}
struct ParsedRtpDescription {
opus_payload_type: Option<u8>,
opus_rtcp_fbs: Option<Vec<RtcpFb>>,
h264_payload_type: Option<u8>,
h264_rtx_payload_type: Option<u8>,
h264_rtcp_fbs: Option<Vec<RtcpFb>>,
vp8_payload_type: Option<u8>,
vp8_rtx_payload_type: Option<u8>,
vp8_rtcp_fbs: Option<Vec<RtcpFb>>,
vp9_payload_type: Option<u8>,
vp9_rtx_payload_type: Option<u8>,
vp9_rtcp_fbs: Option<Vec<RtcpFb>>,
codecs: Vec<Codec>,
audio_hdrext_ssrc_audio_level: Option<u8>,
audio_hdrext_transport_cc: Option<u8>,
video_hdrext_abs_send_time: Option<u8>,
@ -111,17 +175,10 @@ impl JingleSession {
}
fn parse_rtp_description(description: &RtpDescription, remote_ssrc_map: &mut HashMap<u32, Source>) -> Result<Option<ParsedRtpDescription>> {
let mut opus_payload_type = None;
let mut opus_rtcp_fbs = None;
let mut h264_payload_type = None;
let mut h264_rtx_payload_type = None;
let mut h264_rtcp_fbs = None;
let mut vp8_payload_type = None;
let mut vp8_rtx_payload_type = None;
let mut vp8_rtcp_fbs = None;
let mut vp9_payload_type = None;
let mut vp9_rtx_payload_type = None;
let mut vp9_rtcp_fbs = None;
let mut opus = None;
let mut h264 = None;
let mut vp8 = None;
let mut vp9 = None;
let mut audio_hdrext_ssrc_audio_level = None;
let mut audio_hdrext_transport_cc = None;
let mut video_hdrext_abs_send_time = None;
@ -133,8 +190,12 @@ impl JingleSession {
if let Some(name) = &pt.name {
match name.as_str() {
"opus" => {
opus_payload_type = Some(pt.id);
opus_rtcp_fbs = Some(pt.rtcp_fbs.clone());
opus = Some(Codec {
name: CodecName::Opus,
pt: pt.id,
rtx_pt: None,
rtcp_fbs: pt.rtcp_fbs.clone(),
});
}
_ => (),
}
@ -157,16 +218,28 @@ impl JingleSession {
if let Some(name) = &pt.name {
match name.as_str() {
"H264" => {
h264_payload_type = Some(pt.id);
h264_rtcp_fbs = Some(pt.rtcp_fbs.clone());
h264 = Some(Codec {
name: CodecName::H264,
pt: pt.id,
rtx_pt: None,
rtcp_fbs: pt.rtcp_fbs.clone(),
});
}
"VP8" => {
vp8_payload_type = Some(pt.id);
vp8_rtcp_fbs = Some(pt.rtcp_fbs.clone());
vp8 = Some(Codec {
name: CodecName::Vp8,
pt: pt.id,
rtx_pt: None,
rtcp_fbs: pt.rtcp_fbs.clone(),
});
}
"VP9" => {
vp9_payload_type = Some(pt.id);
vp9_rtcp_fbs = Some(pt.rtcp_fbs.clone());
vp9 = Some(Codec {
name: CodecName::Vp9,
pt: pt.id,
rtx_pt: None,
rtcp_fbs: pt.rtcp_fbs.clone(),
});
}
_ => (),
}
@ -177,18 +250,21 @@ impl JingleSession {
if name == "rtx" {
for param in pt.parameters.iter() {
if param.name == "apt" {
let pt: u8 = param.value.parse()?;
if Some(pt) == h264_payload_type {
h264_rtx_payload_type = Some(pt);
let apt_pt: u8 = param.value.parse()?;
if let Some(h264) = &mut h264 {
if apt_pt == h264.pt {
h264.rtx_pt = Some(pt.id);
}
else if Some(pt) == h264_payload_type {
h264_rtx_payload_type = Some(pt);
}
else if Some(pt) == h264_payload_type {
h264_rtx_payload_type = Some(pt);
if let Some(vp8) = &mut vp8 {
if apt_pt == vp8.pt {
vp8.rtx_pt = Some(pt.id);
}
}
if let Some(vp9) = &mut vp9 {
if apt_pt == vp9.pt {
vp9.rtx_pt = Some(pt.id);
}
else {
bail!("unknown rtx apt: {}", pt);
}
}
}
@ -211,6 +287,8 @@ impl JingleSession {
return Ok(None);
}
let codecs = [opus, h264, vp8, vp9].iter().flatten().cloned().collect();
for ssrc in &description.ssrcs {
let owner = ssrc
.info
@ -246,17 +324,7 @@ impl JingleSession {
);
}
Ok(Some(ParsedRtpDescription {
opus_payload_type,
opus_rtcp_fbs,
h264_payload_type,
h264_rtx_payload_type,
h264_rtcp_fbs,
vp8_payload_type,
vp8_rtx_payload_type,
vp8_rtcp_fbs,
vp9_payload_type,
vp9_rtx_payload_type,
vp9_rtcp_fbs,
codecs,
audio_hdrext_ssrc_audio_level,
audio_hdrext_transport_cc,
video_hdrext_abs_send_time,
@ -396,17 +464,7 @@ impl JingleSession {
debug!("Received Jingle session-initiate from {}", initiator);
let mut ice_transport = None;
let mut opus_payload_type = None;
let mut opus_rtcp_fbs = None;
let mut h264_payload_type = None;
let mut h264_rtx_payload_type = None;
let mut h264_rtcp_fbs = None;
let mut vp8_payload_type = None;
let mut vp8_rtx_payload_type = None;
let mut vp8_rtcp_fbs = None;
let mut vp9_payload_type = None;
let mut vp9_rtx_payload_type = None;
let mut vp9_rtcp_fbs = None;
let mut codecs = vec![];
let mut audio_hdrext_ssrc_audio_level = None;
let mut audio_hdrext_transport_cc = None;
let mut video_hdrext_abs_send_time = None;
@ -417,17 +475,7 @@ impl JingleSession {
for content in &jingle.contents {
if let Some(Description::Rtp(description)) = &content.description {
if let Some(description) = JingleSession::parse_rtp_description(description, &mut remote_ssrc_map)? {
opus_payload_type = opus_payload_type.or(description.opus_payload_type);
opus_rtcp_fbs = opus_rtcp_fbs.or(description.opus_rtcp_fbs);
h264_payload_type = h264_payload_type.or(description.h264_payload_type);
h264_rtx_payload_type = h264_rtx_payload_type.or(description.h264_rtx_payload_type);
h264_rtcp_fbs = h264_rtcp_fbs.or(description.h264_rtcp_fbs);
vp8_payload_type = vp8_payload_type.or(description.vp8_payload_type);
vp8_rtx_payload_type = vp8_rtx_payload_type.or(description.vp8_rtx_payload_type);
vp8_rtcp_fbs = vp8_rtcp_fbs.or(description.vp8_rtcp_fbs);
vp9_payload_type = vp9_payload_type.or(description.vp9_payload_type);
vp9_rtx_payload_type = vp9_rtx_payload_type.or(description.vp9_rtx_payload_type);
vp9_rtcp_fbs = vp9_rtcp_fbs.or(description.vp9_rtcp_fbs);
codecs.extend(description.codecs);
audio_hdrext_ssrc_audio_level = audio_hdrext_ssrc_audio_level.or(description.audio_hdrext_ssrc_audio_level);
audio_hdrext_transport_cc = audio_hdrext_transport_cc.or(description.audio_hdrext_transport_cc);
video_hdrext_abs_send_time = video_hdrext_abs_send_time.or(description.video_hdrext_abs_send_time);
@ -522,12 +570,16 @@ impl JingleSession {
)?;
pipeline.add(&dtlssrtpdec)?;
{
let codecs = codecs.clone();
rtpbin.connect("request-pt-map", false, move |values| {
let f = || {
debug!("rtpbin request-pt-map {:?}", values);
let pt = values[2].get::<u32>()? as u8;
let mut caps = gstreamer::Caps::builder("application/x-rtp");
if Some(pt) == opus_payload_type {
for codec in codecs.iter() {
if codec.is(pt) {
if codec.is_audio() {
caps = caps
.field("media", "audio")
.field("encoding-name", "OPUS")
@ -538,67 +590,34 @@ impl JingleSession {
if let Some(hdrext) = audio_hdrext_transport_cc {
caps = caps.field(&format!("extmap-{}", hdrext), &RTP_HDREXT_TRANSPORT_CC);
}
Ok::<_, anyhow::Error>(Some(caps.build()))
}
else if Some(pt) == h264_payload_type
|| Some(pt) == vp8_payload_type
|| Some(pt) == vp9_payload_type
{
else {
// A video codec, as the only audio codec we support is Opus.
caps = caps
.field("media", "video")
.field("clock-rate", 90000)
.field(
"encoding-name",
if Some(pt) == h264_payload_type {
"H264"
}
else if Some(pt) == vp8_payload_type {
"VP8"
}
else if Some(pt) == vp9_payload_type {
"VP9"
}
else {
unreachable!()
},
);
.field("encoding-name", codec.encoding_name());
// if let Some(hdrext) = video_hdrext_abs_send_time {
// caps = caps.field(&format!("extmap-{}", hdrext), &RTP_HDREXT_ABS_SEND_TIME);
// }
if let Some(hdrext) = video_hdrext_transport_cc {
caps = caps.field(&format!("extmap-{}", hdrext), &RTP_HDREXT_TRANSPORT_CC);
}
Ok(Some(caps.build()))
}
else if Some(pt) == vp8_rtx_payload_type
|| Some(pt) == vp9_rtx_payload_type
|| Some(pt) == h264_rtx_payload_type
{
return Ok::<_, anyhow::Error>(Some(caps.build()));
}
else if codec.is_rtx(pt) {
caps = caps
.field("media", "video")
.field("clock-rate", 90000)
.field("encoding-name", "RTX")
.field(
"apt",
if Some(pt) == vp8_rtx_payload_type {
vp8_payload_type.context("missing VP8 payload type")?
.field("apt", codec.pt);
return Ok(Some(caps.build()));
}
else if Some(pt) == vp9_rtx_payload_type {
vp9_payload_type.context("missing VP9 payload type")?
}
else if Some(pt) == h264_rtx_payload_type {
h264_payload_type.context("missing H264 payload type")?
}
else {
unreachable!()
},
);
Ok(Some(caps.build()))
}
else {
warn!("unknown payload type: {}", pt);
Ok(None)
}
};
match f() {
Ok(Some(caps)) => {
@ -612,6 +631,7 @@ impl JingleSession {
},
}
})?;
}
let handle = Handle::current();
let jingle_session = conference.jingle_session.clone();
@ -653,20 +673,26 @@ impl JingleSession {
None
})?;
let pts: Vec<(String, u32)> = codecs.iter()
.filter(|codec| codec.is_video())
.flat_map(|codec| {
if let Some(rtx_pt) = codec.rtx_pt {
Some((codec.pt.to_string(), rtx_pt as u32))
} else {
None
}
})
.collect();
{
let pts = pts.clone();
rtpbin.connect("request-aux-sender", false, move |values| {
let f = move || {
let f = || {
let session: u32 = values[1].get()?;
debug!("creating RTX sender for session {}", session);
let mut pt_map = gstreamer::Structure::builder("application/x-rtp-pt-map");
let mut ssrc_map = gstreamer::Structure::builder("application/x-rtp-ssrc-map");
if let (Some(pt), Some(rtx_pt)) = (vp8_payload_type, vp8_rtx_payload_type) {
pt_map = pt_map.field(&pt.to_string(), &(rtx_pt as u32));
}
if let (Some(pt), Some(rtx_pt)) = (vp9_payload_type, vp9_rtx_payload_type) {
pt_map = pt_map.field(&pt.to_string(), &(rtx_pt as u32));
}
if let (Some(pt), Some(rtx_pt)) = (h264_payload_type, h264_rtx_payload_type) {
pt_map = pt_map.field(&pt.to_string(), &(rtx_pt as u32));
for (pt, rtx_pt) in pts.iter() {
pt_map = pt_map.field(pt, rtx_pt);
}
ssrc_map = ssrc_map.field(&video_ssrc.to_string(), &(video_rtx_ssrc as u32));
let bin = gstreamer::Bin::new(None);
@ -696,20 +722,15 @@ impl JingleSession {
},
}
})?;
}
rtpbin.connect("request-aux-receiver", false, move |values| {
let f = move || {
let f = || {
let session: u32 = values[1].get()?;
debug!("creating RTX receiver for session {}", session);
let mut pt_map = gstreamer::Structure::builder("application/x-rtp-pt-map");
if let (Some(pt), Some(rtx_pt)) = (vp8_payload_type, vp8_rtx_payload_type) {
pt_map = pt_map.field(&pt.to_string(), &(rtx_pt as u32));
}
if let (Some(pt), Some(rtx_pt)) = (vp9_payload_type, vp9_rtx_payload_type) {
pt_map = pt_map.field(&pt.to_string(), &(rtx_pt as u32));
}
if let (Some(pt), Some(rtx_pt)) = (h264_payload_type, h264_rtx_payload_type) {
pt_map = pt_map.field(&pt.to_string(), &(rtx_pt as u32));
for (pt, rtx_pt) in pts.iter() {
pt_map = pt_map.field(pt, rtx_pt);
}
let bin = gstreamer::Bin::new(None);
let rtx_receiver = gstreamer::ElementFactory::make("rtprtxreceive", None)?;
@ -743,6 +764,7 @@ impl JingleSession {
let conference = conference.clone();
let pipeline = pipeline.clone();
let rtpbin_ = rtpbin.clone();
let codecs = codecs.clone();
rtpbin.connect("pad-added", false, move |values| {
let rtpbin = &rtpbin_;
let f = || {
@ -772,26 +794,22 @@ impl JingleSession {
let source_element = match source.media_type {
MediaType::Audio => {
if Some(pt) == opus_payload_type {
gstreamer::ElementFactory::make("rtpopusdepay", None)?
let codec = codecs.iter()
.filter(|codec| codec.is_audio())
.find(|codec| codec.is(pt));
if let Some(codec) = codec {
gstreamer::ElementFactory::make(codec.make_depay_name(), None)?
}
else {
bail!("received audio with unsupported PT {}", pt);
}
},
MediaType::Video => {
if Some(pt) == h264_payload_type {
let element = gstreamer::ElementFactory::make("rtph264depay", None)?;
element.set_property("request-keyframe", true)?;
element
}
else if Some(pt) == vp8_payload_type {
let element = gstreamer::ElementFactory::make("rtpvp8depay", None)?;
element.set_property("request-keyframe", true)?;
element
}
else if Some(pt) == vp9_payload_type {
let element = gstreamer::ElementFactory::make("rtpvp9depay", None)?;
let codec = codecs.iter()
.filter(|codec| codec.is_video())
.find(|codec| codec.is(pt));
if let Some(codec) = codec {
let element = gstreamer::ElementFactory::make(codec.make_depay_name(), None)?;
element.set_property("request-keyframe", true)?;
element
}
@ -897,11 +915,14 @@ impl JingleSession {
})?;
}
let audio_sink_element = gstreamer::ElementFactory::make("rtpopuspay", None)?;
audio_sink_element.set_property(
"pt",
opus_payload_type.context("no opus payload type in jingle session-initiate")? as u32,
)?;
let opus = codecs.iter().find(|codec| codec.name == CodecName::Opus);
let audio_sink_element = if let Some(opus) = opus {
let audio_sink_element = gstreamer::ElementFactory::make(opus.make_pay_name(), None)?;
audio_sink_element.set_property("pt", opus.pt as u32)?;
audio_sink_element
} else {
bail!("no opus payload type in jingle session-initiate");
};
audio_sink_element.set_property("min-ptime", 10i64 * 1000 * 1000)?;
audio_sink_element.set_property("ssrc", audio_ssrc)?;
if audio_sink_element.has_property("auto-header-extension", None) {
@ -943,35 +964,21 @@ impl JingleSession {
}
pipeline.add(&audio_sink_element)?;
let video_sink_element = match conference.config.video_codec.as_str() {
"h264" => {
let element = gstreamer::ElementFactory::make("rtph264pay", None)?;
element.set_property(
"pt",
h264_payload_type.context("no h264 payload type in jingle session-initiate")? as u32,
)?;
let codec_name = conference.config.video_codec.as_str();
let codec = codecs.iter().find(|codec| codec.is_codec(codec_name));
let video_sink_element = if let Some(codec) = codec {
let element = gstreamer::ElementFactory::make(codec.make_pay_name(), None)?;
element.set_property("pt", codec.pt as u32)?;
if codec.name == CodecName::H264 {
element.set_property_from_str("aggregate-mode", "zero-latency");
element
},
"vp8" => {
let element = gstreamer::ElementFactory::make("rtpvp8pay", None)?;
element.set_property(
"pt",
vp8_payload_type.context("no vp8 payload type in jingle session-initiate")? as u32,
)?;
}
else {
element.set_property_from_str("picture-id-mode", "15-bit");
}
element
},
"vp9" => {
let element = gstreamer::ElementFactory::make("rtpvp9pay", None)?;
element.set_property(
"pt",
vp9_payload_type.context("no vp9 payload type in jingle session-initiate")? as u32,
)?;
element.set_property_from_str("picture-id-mode", "15-bit");
element
},
other => bail!("unsupported video codec: {}", other),
}
else {
bail!("unsupported video codec: {}", codec_name);
};
video_sink_element.set_property("ssrc", video_ssrc)?;
if video_sink_element.has_property("auto-header-extension", None) {
@ -1114,73 +1121,40 @@ impl JingleSession {
let mut description = RtpDescription::new(initiate_content.name.0.clone());
description.payload_types = if initiate_content.name.0 == "audio" {
let codec = codecs.iter().find(|codec| codec.name == CodecName::Opus);
if let Some(codec) = codec {
let mut pt = PayloadType::new(
opus_payload_type.context("no opus payload type in jingle session-initiate")?,
codec.pt,
"opus".to_owned(),
48000,
2,
);
pt.rtcp_fbs = opus_rtcp_fbs.clone().unwrap_or_default();
pt.rtcp_fbs = codec.rtcp_fbs.clone();
vec![pt]
}
else {
bail!("no opus payload type in jingle session-initiate");
}
}
else {
let mut pts = vec![];
match conference.config.video_codec.as_str() {
"h264" => {
if let Some(h264_pt) = h264_payload_type {
let mut pt = PayloadType::new(h264_pt, "H264".to_owned(), 90000, 1);
pt.rtcp_fbs = h264_rtcp_fbs.clone().unwrap_or_default();
let codec_name = conference.config.video_codec.as_str();
let codec = codecs.iter().find(|codec| codec.is_codec(codec_name));
if let Some(codec) = codec {
let mut pt = PayloadType::new(codec.pt, codec.encoding_name().to_owned(), 90000, 1);
pt.rtcp_fbs = codec.rtcp_fbs.clone();
pts.push(pt);
if let Some(rtx_pt) = h264_rtx_payload_type {
if let Some(rtx_pt) = codec.rtx_pt {
let mut rtx_pt = PayloadType::new(rtx_pt, "rtx".to_owned(), 90000, 1);
rtx_pt.parameters = vec![jingle_rtp::Parameter {
name: "apt".to_owned(),
value: h264_pt.to_string(),
value: codec.pt.to_string(),
}];
pts.push(rtx_pt);
}
}
else {
bail!("no h264 payload type in jingle session-initiate");
}
},
"vp8" => {
if let Some(vp8_pt) = vp8_payload_type {
let mut pt = PayloadType::new(vp8_pt, "VP8".to_owned(), 90000, 1);
pt.rtcp_fbs = vp8_rtcp_fbs.clone().unwrap_or_default();
pts.push(pt);
if let Some(rtx_pt) = vp8_rtx_payload_type {
let mut rtx_pt = PayloadType::new(rtx_pt, "rtx".to_owned(), 90000, 1);
rtx_pt.parameters = vec![jingle_rtp::Parameter {
name: "apt".to_owned(),
value: vp8_pt.to_string(),
}];
pts.push(rtx_pt);
}
}
else {
bail!("no vp8 payload type in jingle session-initiate");
}
},
"vp9" => {
if let Some(vp9_pt) = vp9_payload_type {
let mut pt = PayloadType::new(vp9_pt, "VP9".to_owned(), 90000, 1);
pt.rtcp_fbs = vp9_rtcp_fbs.clone().unwrap_or_default();
pts.push(pt);
if let Some(rtx_pt) = vp9_rtx_payload_type {
let mut rtx_pt = PayloadType::new(rtx_pt, "rtx".to_owned(), 90000, 1);
rtx_pt.parameters = vec![jingle_rtp::Parameter {
name: "apt".to_owned(),
value: vp9_pt.to_string(),
}];
pts.push(rtx_pt);
}
}
else {
bail!("no vp9 payload type in jingle session-initiate");
}
},
other => bail!("unsupported video codec: {}", other),
bail!("unsupported video codec: {}", codec_name);
}
pts
};