be-gay-do-crimes
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Releases Wiki Activity

Fix dash parser and more refactor 

- Add new itags
This commit is contained in:
Mauricio Colli 2017-07-11 00:08:03 -03:00
parent bda65e83d6
commit b1989c0a83
41 changed files with 802 additions and 751 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Extractor.java
View File

@ -14,7 +14,7 @@ public abstract class Extractor implements Serializable {
this.urlIdHandler = urlIdHandler; this.urlIdHandler = urlIdHandler;
this.serviceId = serviceId; this.serviceId = serviceId;
this.url = url; this.url = url;
this.previewInfoCollector = new StreamInfoItemCollector(urlIdHandler, serviceId); this.previewInfoCollector = new StreamInfoItemCollector(serviceId);
} }
public String getUrl() { public String getUrl() {

6
Info.java
View File

@ -11,9 +11,9 @@ public abstract class Info implements Serializable {
* Id of this Info object <br> * Id of this Info object <br>
* e.g. Youtube: https://www.youtube.com/watch?v=RER5qCTzZ7 > RER5qCTzZ7 * e.g. Youtube: https://www.youtube.com/watch?v=RER5qCTzZ7 > RER5qCTzZ7
*/ */
public String id = ""; public String id;
public String url = ""; public String url;
public String name = ""; public String name;
public List<Throwable> errors = new Vector<>(); public List<Throwable> errors = new Vector<>();
} }

19
InfoItem.java
View File

@ -1,7 +1,5 @@
package org.schabi.newpipe.extractor; package org.schabi.newpipe.extractor;
import java.io.Serializable;
/* /*
* Created by the-scrabi on 11.02.17. * Created by the-scrabi on 11.02.17.
* *
@ -22,14 +20,21 @@ import java.io.Serializable;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>. * along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/ */
public interface InfoItem extends Serializable { import java.io.Serializable;
enum InfoType {
public abstract class InfoItem implements Serializable {
public enum InfoType {
STREAM, STREAM,
PLAYLIST, PLAYLIST,
CHANNEL CHANNEL
} }
InfoType infoType(); public InfoItem(InfoType infoType) {
String getTitle(); this.info_type = infoType;
String getLink(); }
public final InfoType info_type;
public int service_id = -1;
public String url;
public String name;
} }

2
InfoItemCollector.java
View File

@ -25,7 +25,7 @@ import java.util.Vector;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>. * along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/ */
public class InfoItemCollector { public abstract class InfoItemCollector {
private List<InfoItem> itemList = new Vector<>(); private List<InfoItem> itemList = new Vector<>();
private List<Throwable> errors = new Vector<>(); private List<Throwable> errors = new Vector<>();
private int serviceId = -1; private int serviceId = -1;

13
MediaFormat.java
View File

@ -91,4 +91,17 @@ public enum MediaFormat {
} }
return ""; return "";
} }
/**
* Return the MediaFormat with the supplied mime type
*
* @return MediaFormat associated with this mime type,
* or null if none match it.
*/
public static MediaFormat getFromMimeType(String mimeType) {
for (MediaFormat vf : MediaFormat.values()) {
if (vf.mimeType.equals(mimeType)) return vf;
}
return null;
}
} }

1
channel/ChannelExtractor.java
View File

@ -1,6 +1,5 @@
package org.schabi.newpipe.extractor.channel; package org.schabi.newpipe.extractor.channel;
import org.schabi.newpipe.extractor.Extractor;
import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;

21
channel/ChannelInfo.java
View File

@ -29,21 +29,16 @@ import java.util.List;
public class ChannelInfo extends Info { public class ChannelInfo extends Info {
public static ChannelInfo getInfo(ChannelExtractor extractor) public static ChannelInfo getInfo(ChannelExtractor extractor) throws ParsingException {
throws ParsingException {
ChannelInfo info = new ChannelInfo(); ChannelInfo info = new ChannelInfo();
// important data // important data
info.service_id = extractor.getServiceId(); info.service_id = extractor.getServiceId();
info.url = extractor.getUrl(); info.url = extractor.getUrl();
info.id = extractor.getChannelId();
info.name = extractor.getChannelName(); info.name = extractor.getChannelName();
info.hasMoreStreams = extractor.hasMoreStreams(); info.has_more_streams = extractor.hasMoreStreams();
try {
info.id = extractor.getChannelId();
} catch (Exception e) {
info.errors.add(e);
}
try { try {
info.avatar_url = extractor.getAvatarUrl(); info.avatar_url = extractor.getAvatarUrl();
} catch (Exception e) { } catch (Exception e) {
@ -75,10 +70,10 @@ public class ChannelInfo extends Info {
return info; return info;
} }
public String avatar_url = ""; public String avatar_url;
public String banner_url = ""; public String banner_url;
public String feed_url = ""; public String feed_url;
public List<InfoItem> related_streams = null; public List<InfoItem> related_streams;
public long subscriber_count = -1; public long subscriber_count = -1;
public boolean hasMoreStreams = false; public boolean has_more_streams = false;
} }

25
channel/ChannelInfoItem.java
View File

@ -22,25 +22,14 @@ import org.schabi.newpipe.extractor.InfoItem;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>. * along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/ */
public class ChannelInfoItem implements InfoItem { public class ChannelInfoItem extends InfoItem {
public int serviceId = -1; public String thumbnail_url;
public String channelName = ""; public String description;
public String thumbnailUrl = ""; public long subscriber_count = -1;
public String webPageUrl = ""; public long view_count = -1;
public String description = "";
public long subscriberCount = -1;
public long viewCount = -1;
public InfoType infoType() { public ChannelInfoItem() {
return InfoType.CHANNEL; super(InfoType.CHANNEL);
}
public String getTitle() {
return channelName;
}
public String getLink() {
return webPageUrl;
} }
} }

12
channel/ChannelInfoItemCollector.java
View File

@ -31,24 +31,24 @@ public class ChannelInfoItemCollector extends InfoItemCollector {
public ChannelInfoItem extract(ChannelInfoItemExtractor extractor) throws ParsingException { public ChannelInfoItem extract(ChannelInfoItemExtractor extractor) throws ParsingException {
ChannelInfoItem resultItem = new ChannelInfoItem(); ChannelInfoItem resultItem = new ChannelInfoItem();
// important information // important information
resultItem.channelName = extractor.getChannelName(); resultItem.name = extractor.getChannelName();
resultItem.serviceId = getServiceId(); resultItem.service_id = getServiceId();
resultItem.webPageUrl = extractor.getWebPageUrl(); resultItem.url = extractor.getWebPageUrl();
// optional information // optional information
try { try {
resultItem.subscriberCount = extractor.getSubscriberCount(); resultItem.subscriber_count = extractor.getSubscriberCount();
} catch (Exception e) { } catch (Exception e) {
addError(e); addError(e);
} }
try { try {
resultItem.viewCount = extractor.getViewCount(); resultItem.view_count = extractor.getViewCount();
} catch (Exception e) { } catch (Exception e) {
addError(e); addError(e);
} }
try { try {
resultItem.thumbnailUrl = extractor.getThumbnailUrl(); resultItem.thumbnail_url = extractor.getThumbnailUrl();
} catch (Exception e) { } catch (Exception e) {
addError(e); addError(e);
} }

11
exceptions/ContentNotAvailableException.java Normal file
View File

@ -0,0 +1,11 @@
package org.schabi.newpipe.extractor.exceptions;
public class ContentNotAvailableException extends ParsingException {
public ContentNotAvailableException(String message) {
super(message);
}
public ContentNotAvailableException(String message, Throwable cause) {
super(message, cause);
}
}

1
playlist/PlaylistExtractor.java
View File

@ -1,6 +1,5 @@
package org.schabi.newpipe.extractor.playlist; package org.schabi.newpipe.extractor.playlist;
import org.schabi.newpipe.extractor.Extractor;
import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;

22
playlist/PlaylistInfo.java
View File

@ -14,14 +14,10 @@ public class PlaylistInfo extends Info {
info.service_id = extractor.getServiceId(); info.service_id = extractor.getServiceId();
info.url = extractor.getUrl(); info.url = extractor.getUrl();
info.id = extractor.getPlaylistId();
info.name = extractor.getPlaylistName(); info.name = extractor.getPlaylistName();
info.hasMoreStreams = extractor.hasMoreStreams(); info.has_more_streams = extractor.hasMoreStreams();
try {
info.id = extractor.getPlaylistId();
} catch (Exception e) {
info.errors.add(e);
}
try { try {
info.streams_count = extractor.getStreamsCount(); info.streams_count = extractor.getStreamsCount();
} catch (Exception e) { } catch (Exception e) {
@ -63,12 +59,12 @@ public class PlaylistInfo extends Info {
return info; return info;
} }
public String avatar_url = ""; public String avatar_url;
public String banner_url = ""; public String banner_url;
public String uploader_url = ""; public String uploader_url;
public String uploader_name = ""; public String uploader_name;
public String uploader_avatar_url = ""; public String uploader_avatar_url;
public long streams_count = 0; public long streams_count = 0;
public List<InfoItem> related_streams = null; public List<InfoItem> related_streams;
public boolean hasMoreStreams = false; public boolean has_more_streams;
} }

23
playlist/PlaylistInfoItem.java
View File

@ -2,22 +2,15 @@ package org.schabi.newpipe.extractor.playlist;
import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.InfoItem;
public class PlaylistInfoItem implements InfoItem { public class PlaylistInfoItem extends InfoItem {
public int serviceId = -1; public String thumbnail_url;
public String name = ""; /**
public String thumbnailUrl = ""; * How many streams this playlist have
public String webPageUrl = ""; */
public long streams_count = 0;
public InfoType infoType() { public PlaylistInfoItem() {
return InfoType.PLAYLIST; super(InfoType.PLAYLIST);
}
public String getTitle() {
return name;
}
public String getLink() {
return webPageUrl;
} }
} }

13
playlist/PlaylistInfoItemCollector.java
View File

@ -1,6 +1,7 @@
package org.schabi.newpipe.extractor.playlist; package org.schabi.newpipe.extractor.playlist;
import org.schabi.newpipe.extractor.InfoItemCollector; import org.schabi.newpipe.extractor.InfoItemCollector;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
public class PlaylistInfoItemCollector extends InfoItemCollector { public class PlaylistInfoItemCollector extends InfoItemCollector {
@ -12,10 +13,16 @@ public class PlaylistInfoItemCollector extends InfoItemCollector {
final PlaylistInfoItem resultItem = new PlaylistInfoItem(); final PlaylistInfoItem resultItem = new PlaylistInfoItem();
resultItem.name = extractor.getPlaylistName(); resultItem.name = extractor.getPlaylistName();
resultItem.serviceId = getServiceId(); resultItem.service_id = getServiceId();
resultItem.webPageUrl = extractor.getWebPageUrl(); resultItem.url = extractor.getWebPageUrl();
try { try {
resultItem.thumbnailUrl = extractor.getThumbnailUrl(); resultItem.thumbnail_url = extractor.getThumbnailUrl();
} catch (Exception e) {
addError(e);
}
try {
resultItem.streams_count = extractor.getStreamsCount();
} catch (Exception e) { } catch (Exception e) {
addError(e); addError(e);
} }

1
playlist/PlaylistInfoItemExtractor.java
View File

@ -6,4 +6,5 @@ public interface PlaylistInfoItemExtractor {
String getThumbnailUrl() throws ParsingException; String getThumbnailUrl() throws ParsingException;
String getPlaylistName() throws ParsingException; String getPlaylistName() throws ParsingException;
String getWebPageUrl() throws ParsingException; String getWebPageUrl() throws ParsingException;
long getStreamsCount() throws ParsingException;
} }

9
search/InfoItemSearchCollector.java
View File

@ -1,7 +1,6 @@
package org.schabi.newpipe.extractor.search; package org.schabi.newpipe.extractor.search;
import org.schabi.newpipe.extractor.InfoItemCollector; import org.schabi.newpipe.extractor.InfoItemCollector;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemCollector; import org.schabi.newpipe.extractor.channel.ChannelInfoItemCollector;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
@ -30,15 +29,15 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
*/ */
public class InfoItemSearchCollector extends InfoItemCollector { public class InfoItemSearchCollector extends InfoItemCollector {
private String suggestion = ""; private String suggestion;
private StreamInfoItemCollector streamCollector; private StreamInfoItemCollector streamCollector;
private ChannelInfoItemCollector channelCollector; private ChannelInfoItemCollector channelCollector;
SearchResult result = new SearchResult(); private SearchResult result = new SearchResult();
InfoItemSearchCollector(UrlIdHandler handler, int serviceId) { InfoItemSearchCollector(int serviceId) {
super(serviceId); super(serviceId);
streamCollector = new StreamInfoItemCollector(handler, serviceId); streamCollector = new StreamInfoItemCollector(serviceId);
channelCollector = new ChannelInfoItemCollector(serviceId); channelCollector = new ChannelInfoItemCollector(serviceId);
} }

7
search/SearchEngine.java
View File

@ -1,6 +1,5 @@
package org.schabi.newpipe.extractor.search; package org.schabi.newpipe.extractor.search;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import java.io.IOException; import java.io.IOException;
@ -28,7 +27,7 @@ import java.util.EnumSet;
public abstract class SearchEngine { public abstract class SearchEngine {
public enum Filter { public enum Filter {
STREAM, CHANNEL, PLAY_LIST STREAM, CHANNEL, PLAYLIST
} }
public static class NothingFoundException extends ExtractionException { public static class NothingFoundException extends ExtractionException {
@ -39,8 +38,8 @@ public abstract class SearchEngine {
private InfoItemSearchCollector collector; private InfoItemSearchCollector collector;
public SearchEngine(UrlIdHandler urlIdHandler, int serviceId) { public SearchEngine(int serviceId) {
collector = new InfoItemSearchCollector(urlIdHandler, serviceId); collector = new InfoItemSearchCollector(serviceId);
} }
protected InfoItemSearchCollector getInfoItemSearchCollector() { protected InfoItemSearchCollector getInfoItemSearchCollector() {

2
search/SearchResult.java
View File

@ -49,7 +49,7 @@ public class SearchResult {
return result; return result;
} }
public String suggestion = ""; public String suggestion;
public List<InfoItem> resultList = new Vector<>(); public List<InfoItem> resultList = new Vector<>();
public List<Throwable> errors = new Vector<>(); public List<Throwable> errors = new Vector<>();
} }

160
services/youtube/ItagItem.java Normal file
View File

@ -0,0 +1,160 @@
package org.schabi.newpipe.extractor.services.youtube;
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import static org.schabi.newpipe.extractor.MediaFormat.M4A;
import static org.schabi.newpipe.extractor.MediaFormat.MPEG_4;
import static org.schabi.newpipe.extractor.MediaFormat.WEBM;
import static org.schabi.newpipe.extractor.MediaFormat.WEBMA;
import static org.schabi.newpipe.extractor.MediaFormat.v3GPP;
import static org.schabi.newpipe.extractor.services.youtube.ItagItem.ItagType.AUDIO;
import static org.schabi.newpipe.extractor.services.youtube.ItagItem.ItagType.VIDEO;
import static org.schabi.newpipe.extractor.services.youtube.ItagItem.ItagType.VIDEO_ONLY;
public class ItagItem {
/**
* List can be found here https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/youtube.py#L360
*/
private static final ItagItem[] ITAG_LIST = {
/////////////////////////////////////////////////////
// VIDEO ID Type Format Resolution FPS ///
///////////////////////////////////////////////////
new ItagItem(17, VIDEO, v3GPP, "144p"),
new ItagItem(36, VIDEO, v3GPP, "240p"),
new ItagItem(18, VIDEO, MPEG_4, "360p"),
new ItagItem(34, VIDEO, MPEG_4, "360p"),
new ItagItem(35, VIDEO, MPEG_4, "480p"),
new ItagItem(59, VIDEO, MPEG_4, "480p"),
new ItagItem(78, VIDEO, MPEG_4, "480p"),
new ItagItem(22, VIDEO, MPEG_4, "720p"),
new ItagItem(37, VIDEO, MPEG_4, "1080p"),
new ItagItem(38, VIDEO, MPEG_4, "1080p"),
new ItagItem(43, VIDEO, WEBM, "360p"),
new ItagItem(44, VIDEO, WEBM, "480p"),
new ItagItem(45, VIDEO, WEBM, "720p"),
new ItagItem(46, VIDEO, WEBM, "1080p"),
////////////////////////////////////////////////////////////////////
// AUDIO ID ItagType Format Bitrate ///
//////////////////////////////////////////////////////////////////
// Disable Opus codec as it's not well supported in older devices
// new ItagItem(249, AUDIO, WEBMA, 50),
// new ItagItem(250, AUDIO, WEBMA, 70),
// new ItagItem(251, AUDIO, WEBMA, 16),
new ItagItem(171, AUDIO, WEBMA, 128),
new ItagItem(172, AUDIO, WEBMA, 256),
new ItagItem(139, AUDIO, M4A, 48),
new ItagItem(140, AUDIO, M4A, 128),
new ItagItem(141, AUDIO, M4A, 256),
/// VIDEO ONLY ////////////////////////////////////////////
// ID Type Format Resolution FPS ///
/////////////////////////////////////////////////////////
// Don't add VideoOnly streams that have normal variants
new ItagItem(160, VIDEO_ONLY, MPEG_4, "144p"),
new ItagItem(133, VIDEO_ONLY, MPEG_4, "240p"),
// new ItagItem(134, VIDEO_ONLY, MPEG_4, "360p"),
new ItagItem(135, VIDEO_ONLY, MPEG_4, "480p"),
new ItagItem(212, VIDEO_ONLY, MPEG_4, "480p"),
// new ItagItem(136, VIDEO_ONLY, MPEG_4, "720p"),
new ItagItem(298, VIDEO_ONLY, MPEG_4, "720p60", 60),
new ItagItem(137, VIDEO_ONLY, MPEG_4, "1080p"),
new ItagItem(299, VIDEO_ONLY, MPEG_4, "1080p60", 60),
new ItagItem(266, VIDEO_ONLY, MPEG_4, "2160p"),
new ItagItem(278, VIDEO_ONLY, WEBM, "144p"),
new ItagItem(242, VIDEO_ONLY, WEBM, "240p"),
// new ItagItem(243, VIDEO_ONLY, WEBM, "360p"),
new ItagItem(244, VIDEO_ONLY, WEBM, "480p"),
new ItagItem(245, VIDEO_ONLY, WEBM, "480p"),
new ItagItem(246, VIDEO_ONLY, WEBM, "480p"),
new ItagItem(247, VIDEO_ONLY, WEBM, "720p"),
new ItagItem(248, VIDEO_ONLY, WEBM, "1080p"),
new ItagItem(271, VIDEO_ONLY, WEBM, "1440p"),
// #272 is either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
new ItagItem(272, VIDEO_ONLY, WEBM, "2160p"),
new ItagItem(302, VIDEO_ONLY, WEBM, "720p60", 60),
new ItagItem(303, VIDEO_ONLY, WEBM, "1080p60", 60),
new ItagItem(308, VIDEO_ONLY, WEBM, "1440p60", 60),
new ItagItem(313, VIDEO_ONLY, WEBM, "2160p"),
new ItagItem(315, VIDEO_ONLY, WEBM, "2160p60", 60)
};
/*//////////////////////////////////////////////////////////////////////////
// Utils
//////////////////////////////////////////////////////////////////////////*/
public static boolean isSupported(int itag) {
for (ItagItem item : ITAG_LIST) {
if (itag == item.id) {
return true;
}
}
return false;
}
public static ItagItem getItag(int itagId) throws ParsingException {
for (ItagItem item : ITAG_LIST) {
if (itagId == item.id) {
return item;
}
}
throw new ParsingException("itag=" + Integer.toString(itagId) + " not supported");
}
/*//////////////////////////////////////////////////////////////////////////
// Contructors and misc
//////////////////////////////////////////////////////////////////////////*/
public enum ItagType {
AUDIO,
VIDEO,
VIDEO_ONLY
}
/**
* Call {@link #ItagItem(int, ItagType, MediaFormat, String, int)} with the fps set to 30.
*/
public ItagItem(int id, ItagType type, MediaFormat format, String resolution) {
this.id = id;
this.itagType = type;
this.mediaFormatId = format.id;
this.resolutionString = resolution;
this.fps = 30;
}
/**
* Constructor for videos.
*
* @param resolution string that will be used in the frontend
*/
public ItagItem(int id, ItagType type, MediaFormat format, String resolution, int fps) {
this.id = id;
this.itagType = type;
this.mediaFormatId = format.id;
this.resolutionString = resolution;
this.fps = fps;
}
public ItagItem(int id, ItagType type, MediaFormat format, int avgBitrate) {
this.id = id;
this.itagType = type;
this.mediaFormatId = format.id;
this.avgBitrate = avgBitrate;
}
public int id;
public ItagType itagType;
public int mediaFormatId;
// Audio fields
public int avgBitrate = -1;
// Video fields
public String resolutionString;
public int fps = -1;
}

13
services/youtube/YoutubeChannelExtractor.java
View File

@ -13,10 +13,11 @@ import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.stream.AbstractStreamInfo;
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector; import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException; import java.io.IOException;
@ -135,7 +136,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
if (subscriberCount == -1) { if (subscriberCount == -1) {
Element el = doc.select("span[class*=\"yt-subscription-button-subscriber-count\"]").first(); Element el = doc.select("span[class*=\"yt-subscription-button-subscriber-count\"]").first();
if (el != null) { if (el != null) {
subscriberCount = Long.parseLong(el.text().replaceAll("\\D+", "")); subscriberCount = Long.parseLong(Utils.removeNonDigitCharacters(el.text()));
} else { } else {
throw new ParsingException("Could not get subscriber count"); throw new ParsingException("Could not get subscriber count");
} }
@ -164,7 +165,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
throw new ExtractionException("Channel doesn't have more streams"); throw new ExtractionException("Channel doesn't have more streams");
} }
StreamInfoItemCollector collector = new StreamInfoItemCollector(getUrlIdHandler(), getServiceId()); StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
setupNextStreamsAjax(NewPipe.getDownloader()); setupNextStreamsAjax(NewPipe.getDownloader());
collectStreamsFrom(collector, nextStreamsAjax.select("body").first()); collectStreamsFrom(collector, nextStreamsAjax.select("body").first());
@ -223,8 +224,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
if (li.select("div[class=\"feed-item-dismissable\"]").first() != null) { if (li.select("div[class=\"feed-item-dismissable\"]").first() != null) {
collector.commit(new StreamInfoItemExtractor() { collector.commit(new StreamInfoItemExtractor() {
@Override @Override
public AbstractStreamInfo.StreamType getStreamType() throws ParsingException { public StreamType getStreamType() throws ParsingException {
return AbstractStreamInfo.StreamType.VIDEO_STREAM; return StreamType.VIDEO_STREAM;
} }
@Override @Override
@ -302,7 +303,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
return -1; return -1;
} }
output = input.replaceAll("\\D+", ""); output = Utils.removeNonDigitCharacters(input);
try { try {
return Long.parseLong(output); return Long.parseLong(output);

5
services/youtube/YoutubeChannelInfoItemExtractor.java
View File

@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor.services.youtube;
import org.jsoup.nodes.Element; import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.utils.Utils;
/* /*
* Created by Christian Schabesberger on 12.02.17. * Created by Christian Schabesberger on 12.02.17.
@ -62,7 +63,7 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor
if (subsEl == null) { if (subsEl == null) {
return 0; return 0;
} else { } else {
return Long.parseLong(subsEl.text().replaceAll("\\D+", "")); return Long.parseLong(Utils.removeNonDigitCharacters(subsEl.text()));
} }
} }
@ -72,7 +73,7 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor
if (metaEl == null) { if (metaEl == null) {
return 0; return 0;
} else { } else {
return Long.parseLong(metaEl.text().replaceAll("\\D+", "")); return Long.parseLong(Utils.removeNonDigitCharacters(metaEl.text()));
} }
} }

13
services/youtube/YoutubeChannelUrlIdHandler.java
View File

@ -26,18 +26,29 @@ import org.schabi.newpipe.extractor.utils.Parser;
public class YoutubeChannelUrlIdHandler implements UrlIdHandler { public class YoutubeChannelUrlIdHandler implements UrlIdHandler {
private static final YoutubeChannelUrlIdHandler instance = new YoutubeChannelUrlIdHandler();
private static final String ID_PATTERN = "/(user/[A-Za-z0-9_-]*|channel/[A-Za-z0-9_-]*)";
public static YoutubeChannelUrlIdHandler getInstance() {
return instance;
}
@Override
public String getUrl(String channelId) { public String getUrl(String channelId) {
return "https://www.youtube.com/" + channelId; return "https://www.youtube.com/" + channelId;
} }
@Override
public String getId(String siteUrl) throws ParsingException { public String getId(String siteUrl) throws ParsingException {
return Parser.matchGroup1("/(user/[A-Za-z0-9_-]*|channel/[A-Za-z0-9_-]*)", siteUrl); return Parser.matchGroup1(ID_PATTERN, siteUrl);
} }
@Override
public String cleanUrl(String siteUrl) throws ParsingException { public String cleanUrl(String siteUrl) throws ParsingException {
return getUrl(getId(siteUrl)); return getUrl(getId(siteUrl));
} }
@Override
public boolean acceptUrl(String videoUrl) { public boolean acceptUrl(String videoUrl) {
return (videoUrl.contains("youtube") || return (videoUrl.contains("youtube") ||
videoUrl.contains("youtu.be")) && videoUrl.contains("youtu.be")) &&

11
services/youtube/YoutubePlaylistExtractor.java
View File

@ -12,10 +12,11 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.stream.AbstractStreamInfo;
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector; import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException; import java.io.IOException;
@ -157,7 +158,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
} }
try { try {
streamsCount = Long.parseLong(input.replaceAll("\\D+", "")); streamsCount = Long.parseLong(Utils.removeNonDigitCharacters(input));
} catch (NumberFormatException e) { } catch (NumberFormatException e) {
// When there's no videos in a playlist, there's no number in the "innerHtml", // When there's no videos in a playlist, there's no number in the "innerHtml",
// all characters that is not a number is removed, so we try to parse a empty string // all characters that is not a number is removed, so we try to parse a empty string
@ -186,7 +187,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
throw new ExtractionException("Playlist doesn't have more streams"); throw new ExtractionException("Playlist doesn't have more streams");
} }
StreamInfoItemCollector collector = new StreamInfoItemCollector(getUrlIdHandler(), getServiceId()); StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
setupNextStreamsAjax(NewPipe.getDownloader()); setupNextStreamsAjax(NewPipe.getDownloader());
collectStreamsFrom(collector, nextStreamsAjax.select("tbody[id=\"pl-load-more-destination\"]").first()); collectStreamsFrom(collector, nextStreamsAjax.select("tbody[id=\"pl-load-more-destination\"]").first());
@ -244,8 +245,8 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
for (final Element li : element.children()) { for (final Element li : element.children()) {
collector.commit(new StreamInfoItemExtractor() { collector.commit(new StreamInfoItemExtractor() {
@Override @Override
public AbstractStreamInfo.StreamType getStreamType() throws ParsingException { public StreamType getStreamType() throws ParsingException {
return AbstractStreamInfo.StreamType.VIDEO_STREAM; return StreamType.VIDEO_STREAM;
} }
@Override @Override

5
services/youtube/YoutubePlaylistUrlIdHandler.java
View File

@ -7,8 +7,13 @@ import org.schabi.newpipe.extractor.utils.Parser;
public class YoutubePlaylistUrlIdHandler implements UrlIdHandler { public class YoutubePlaylistUrlIdHandler implements UrlIdHandler {
private static final YoutubePlaylistUrlIdHandler instance = new YoutubePlaylistUrlIdHandler();
private static final String ID_PATTERN = "([\\-a-zA-Z0-9_]{34})"; private static final String ID_PATTERN = "([\\-a-zA-Z0-9_]{34})";
public static YoutubePlaylistUrlIdHandler getInstance() {
return instance;
}
@Override @Override
public String getUrl(String listId) { public String getUrl(String listId) {
return "https://www.youtube.com/playlist?list=" + listId; return "https://www.youtube.com/playlist?list=" + listId;

5
services/youtube/YoutubeSearchEngine.java
View File

@ -5,7 +5,6 @@ import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element; import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.Downloader; import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.search.InfoItemSearchCollector; import org.schabi.newpipe.extractor.search.InfoItemSearchCollector;
import org.schabi.newpipe.extractor.search.SearchEngine; import org.schabi.newpipe.extractor.search.SearchEngine;
@ -40,8 +39,8 @@ public class YoutubeSearchEngine extends SearchEngine {
private static final String TAG = YoutubeSearchEngine.class.toString(); private static final String TAG = YoutubeSearchEngine.class.toString();
public static final String CHARSET_UTF_8 = "UTF-8"; public static final String CHARSET_UTF_8 = "UTF-8";
public YoutubeSearchEngine(UrlIdHandler urlIdHandler, int serviceId) { public YoutubeSearchEngine(int serviceId) {
super(urlIdHandler, serviceId); super(serviceId);
} }
@Override @Override

6
services/youtube/YoutubeService.java
View File

@ -58,7 +58,7 @@ public class YoutubeService extends StreamingService {
@Override @Override
public SearchEngine getSearchEngineInstance() { public SearchEngine getSearchEngineInstance() {
return new YoutubeSearchEngine(getStreamUrlIdHandlerInstance(), getServiceId()); return new YoutubeSearchEngine(getServiceId());
} }
@Override @Override
@ -68,13 +68,13 @@ public class YoutubeService extends StreamingService {
@Override @Override
public UrlIdHandler getChannelUrlIdHandlerInstance() { public UrlIdHandler getChannelUrlIdHandlerInstance() {
return new YoutubeChannelUrlIdHandler(); return YoutubeChannelUrlIdHandler.getInstance();
} }
@Override @Override
public UrlIdHandler getPlaylistUrlIdHandlerInstance() { public UrlIdHandler getPlaylistUrlIdHandlerInstance() {
return new YoutubePlaylistUrlIdHandler(); return YoutubePlaylistUrlIdHandler.getInstance();
} }
@Override @Override

692
services/youtube/YoutubeStreamExtractor.java
View File

@ -9,20 +9,21 @@ import org.mozilla.javascript.Context;
import org.mozilla.javascript.Function; import org.mozilla.javascript.Function;
import org.mozilla.javascript.ScriptableObject; import org.mozilla.javascript.ScriptableObject;
import org.schabi.newpipe.extractor.Downloader; import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.stream.AbstractStreamInfo;
import org.schabi.newpipe.extractor.stream.AudioStream; import org.schabi.newpipe.extractor.stream.AudioStream;
import org.schabi.newpipe.extractor.stream.Stream;
import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfo;
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector; import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.stream.VideoStream; import org.schabi.newpipe.extractor.stream.VideoStream;
import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException; import java.io.IOException;
import java.util.List; import java.util.List;
@ -52,12 +53,11 @@ import java.util.regex.Pattern;
*/ */
public class YoutubeStreamExtractor extends StreamExtractor { public class YoutubeStreamExtractor extends StreamExtractor {
public static final String URL_ENCODED_FMT_STREAM_MAP = "url_encoded_fmt_stream_map"; private static final String TAG = YoutubeStreamExtractor.class.getSimpleName();
public static final String HTTPS = "https:";
public static final String CONTENT = "content";
public static final String REGEX_INT = "[^\\d]";
// exceptions /*//////////////////////////////////////////////////////////////////////////
// Exceptions
//////////////////////////////////////////////////////////////////////////*/
public class DecryptException extends ParsingException { public class DecryptException extends ParsingException {
DecryptException(String message, Throwable cause) { DecryptException(String message, Throwable cause) {
@ -65,8 +65,6 @@ public class YoutubeStreamExtractor extends StreamExtractor {
} }
} }
// special content not available exceptions
public class GemaException extends ContentNotAvailableException { public class GemaException extends ContentNotAvailableException {
GemaException(String message) { GemaException(String message) {
super(message); super(message);
@ -79,267 +77,27 @@ public class YoutubeStreamExtractor extends StreamExtractor {
} }
} }
// ---------------- /*//////////////////////////////////////////////////////////////////////////*/
// Sometimes if the html page of youtube is already downloaded, youtube web page will internally private Document doc;
// download the /get_video_info page. Since a certain date dashmpd url is only available over private final String dirtyUrl;
// this /get_video_info page, so we always need to download this one to.
// %%video_id%% will be replaced by the actual video id
// $$el_type$$ will be replaced by the actual el_type (se the declarations below)
private static final String GET_VIDEO_INFO_URL =
"https://www.youtube.com/get_video_info?video_id=%%video_id%%$$el_type$$&ps=default&eurl=&gl=US&hl=en";
// eltype is necessary for the url above
private static final String EL_INFO = "el=info";
public enum ItagType { public YoutubeStreamExtractor(UrlIdHandler urlIdHandler, String pageUrl, int serviceId) throws ExtractionException, IOException {
AUDIO, super(urlIdHandler, urlIdHandler.cleanUrl(pageUrl), serviceId);
VIDEO, dirtyUrl = pageUrl;
VIDEO_ONLY fetchDocument();
} }
private static class ItagItem { /*//////////////////////////////////////////////////////////////////////////
public ItagItem(int id, ItagType type, MediaFormat format, String res, int fps) { // Impl
this.id = id; //////////////////////////////////////////////////////////////////////////*/
this.itagType = type;
this.mediaFormatId = format.id;
this.resolutionString = res;
this.fps = fps;
}
public ItagItem(int id, ItagType type, MediaFormat format, int samplingRate, int bandWidth) { @Override
this(id, type, format, 0, samplingRate, bandWidth); public String getId() throws ParsingException {
}
public ItagItem(int id, ItagType type, MediaFormat format, int avgBitrate, int samplingRate, int bandWidth) {
this.id = id;
this.itagType = type;
this.mediaFormatId = format.id;
this.avgBitrate = avgBitrate;
this.samplingRate = samplingRate;
this.bandWidth = bandWidth;
}
public int id;
public ItagType itagType;
public int mediaFormatId;
public String resolutionString;
public int fps = -1;
public int avgBitrate = -1;
public int samplingRate = -1;
public int bandWidth = -1;
}
/**
* List can be found here https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/youtube.py#L360
*/
private static final ItagItem[] itagList = {
//////////////////////////////////////////////////////////////////////////
// VIDEO ID ItagType Format Resolution FPS ///
////////////////////////////////////////////////////////////////////////
new ItagItem(17, ItagType.VIDEO, MediaFormat.v3GPP, "144p" , 12),
new ItagItem(18, ItagType.VIDEO, MediaFormat.MPEG_4, "360p" , 24),
new ItagItem(22, ItagType.VIDEO, MediaFormat.MPEG_4, "720p" , 24),
new ItagItem(36, ItagType.VIDEO, MediaFormat.v3GPP, "240p" , 24),
new ItagItem(37, ItagType.VIDEO, MediaFormat.MPEG_4, "1080p" , 24),
new ItagItem(38, ItagType.VIDEO, MediaFormat.MPEG_4, "1080p" , 24),
new ItagItem(43, ItagType.VIDEO, MediaFormat.WEBM, "360p" , 24),
new ItagItem(44, ItagType.VIDEO, MediaFormat.WEBM, "480p" , 24),
new ItagItem(45, ItagType.VIDEO, MediaFormat.WEBM, "720p" , 24),
new ItagItem(46, ItagType.VIDEO, MediaFormat.WEBM, "1080p" , 24),
//////////////////////////////////////////////////////////////////////////////////////////
// AUDIO ID ItagType Format Bitrate SamplingR Bandwidth ///
////////////////////////////////////////////////////////////////////////////////////////
// Disable Opus codec as it's not well supported in older devices
// new ItagItem(249, ItagType.AUDIO, MediaFormat.WEBMA, 50, 0, 0),
// new ItagItem(250, ItagType.AUDIO, MediaFormat.WEBMA, 70, 0, 0),
// new ItagItem(251, ItagType.AUDIO, MediaFormat.WEBMA, 160, 0, 0),
new ItagItem(171, ItagType.AUDIO, MediaFormat.WEBMA, 128, 0, 0),
new ItagItem(172, ItagType.AUDIO, MediaFormat.WEBMA, 256, 0, 0),
new ItagItem(140, ItagType.AUDIO, MediaFormat.M4A, 128, 0, 0),
new ItagItem(141, ItagType.AUDIO, MediaFormat.M4A, 256, 0, 0),
/// VIDEO ONLY ///////////////////////////////////////////////////////////////////
// ID ItagType Format Resolution FPS ///
////////////////////////////////////////////////////////////////////////////////
// Don't add VideoOnly streams that have normal variants
// new ItagItem(160, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "144p" , 24),
// new ItagItem(133, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "240p" , 24),
// new ItagItem(134, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "360p" , 24),
new ItagItem(135, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "480p" , 30),
// new ItagItem(136, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "720p" , 30),
new ItagItem(298, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "720p60" , 60),
new ItagItem(137, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "1080p" , 30),
new ItagItem(299, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "1080p60" , 60),
new ItagItem(266, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "2160p" , 30),
// new ItagItem(243, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "360p" , 30),
new ItagItem(244, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "480p" , 30),
new ItagItem(245, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "480p" , 30),
new ItagItem(246, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "480p" , 30),
new ItagItem(247, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "720p" , 30),
new ItagItem(248, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "1080p" , 30),
new ItagItem(271, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "1440p" , 30),
// #272 is either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
new ItagItem(272, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "2160p" , 30),
new ItagItem(302, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "720p60" , 60),
new ItagItem(303, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "1080p60" , 60),
new ItagItem(308, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "1440p60" , 60),
new ItagItem(313, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "2160p" , 30),
new ItagItem(315, ItagType.VIDEO_ONLY, MediaFormat.WEBM, "2160p60" , 60)
};
public static boolean itagIsSupported(int itag) {
for (ItagItem item : itagList) {
if (itag == item.id) {
return true;
}
}
return false;
}
public static ItagItem getItagItem(int itag) throws ParsingException {
for (ItagItem item : itagList) {
if (itag == item.id) {
return item;
}
}
throw new ParsingException("itag=" + Integer.toString(itag) + " not supported");
}
private static final String TAG = YoutubeStreamExtractor.class.toString();
private final Document doc;
private JSONObject playerArgs;
private boolean isAgeRestricted;
private Map<String, String> videoInfoPage;
// static values
private static final String DECRYPTION_FUNC_NAME = "decrypt";
// cached values
private static volatile String decryptionCode = "";
UrlIdHandler urlidhandler = YoutubeStreamUrlIdHandler.getInstance();
String pageUrl = "";
public YoutubeStreamExtractor(UrlIdHandler urlIdHandler, String pageUrl, int serviceId)
throws ExtractionException, IOException {
super(urlIdHandler, pageUrl, serviceId);
//most common videoInfo fields are now set in our superclass, for all services
this.pageUrl = pageUrl;
Downloader downloader = NewPipe.getDownloader();
String pageContent = downloader.download(urlidhandler.cleanUrl(pageUrl));
doc = Jsoup.parse(pageContent, pageUrl);
JSONObject ytPlayerConfig;
String playerUrl;
// Check if the video is age restricted
if (pageContent.contains("<meta property=\"og:restrictions:age")) {
String videoInfoUrl = GET_VIDEO_INFO_URL.replace("%%video_id%%",
urlidhandler.getId(pageUrl)).replace("$$el_type$$", "&" + EL_INFO);
String videoInfoPageString = downloader.download(videoInfoUrl);
videoInfoPage = Parser.compatParseMap(videoInfoPageString);
playerUrl = getPlayerUrlFromRestrictedVideo(pageUrl);
isAgeRestricted = true;
} else {
ytPlayerConfig = getPlayerConfig(pageContent);
playerArgs = getPlayerArgs(ytPlayerConfig);
playerUrl = getPlayerUrl(ytPlayerConfig);
isAgeRestricted = false;
}
if (decryptionCode.isEmpty()) {
decryptionCode = loadDecryptionCode(playerUrl);
}
}
private JSONObject getPlayerConfig(String pageContent) throws ParsingException {
try { try {
String ytPlayerConfigRaw = return getUrlIdHandler().getId(getUrl());
Parser.matchGroup1("ytplayer.config\\s*=\\s*(\\{.*?\\});", pageContent); } catch (Exception e) {
return new JSONObject(ytPlayerConfigRaw); throw new ParsingException("Could not get stream id");
} catch (Parser.RegexException e) {
String errorReason = getErrorMessage();
switch(errorReason) {
case "GEMA":
throw new GemaException(errorReason);
case "":
throw new ContentNotAvailableException("Content not available: player config empty", e);
default:
throw new ContentNotAvailableException("Content not available", e);
}
} catch (JSONException e) {
throw new ParsingException("Could not parse yt player config", e);
}
}
private JSONObject getPlayerArgs(JSONObject playerConfig) throws ParsingException {
JSONObject playerArgs;
//attempt to load the youtube js player JSON arguments
boolean isLiveStream = false; //used to determine if this is a livestream or not
try {
playerArgs = playerConfig.getJSONObject("args");
// check if we have a live stream. We need to filter it, since its not yet supported.
if ((playerArgs.has("ps") && playerArgs.get("ps").toString().equals("live"))
|| (playerArgs.get(URL_ENCODED_FMT_STREAM_MAP).toString().isEmpty())) {
isLiveStream = true;
}
} catch (JSONException e) {
throw new ParsingException("Could not parse yt player config", e);
}
if (isLiveStream) {
throw new LiveStreamException("This is a Life stream. Can't use those right now.");
}
return playerArgs;
}
private String getPlayerUrl(JSONObject playerConfig) throws ParsingException {
try {
// The Youtube service needs to be initialized by downloading the
// js-Youtube-player. This is done in order to get the algorithm
// for decrypting cryptic signatures inside certain stream urls.
String playerUrl = "";
JSONObject ytAssets = playerConfig.getJSONObject("assets");
playerUrl = ytAssets.getString("js");
if (playerUrl.startsWith("//")) {
playerUrl = HTTPS + playerUrl;
}
return playerUrl;
} catch (JSONException e) {
throw new ParsingException(
"Could not load decryption code for the Youtube service.", e);
}
}
private String getPlayerUrlFromRestrictedVideo(String pageUrl) throws ParsingException, ReCaptchaException {
try {
Downloader downloader = NewPipe.getDownloader();
String playerUrl = "";
String videoId = urlidhandler.getId(pageUrl);
String embedUrl = "https://www.youtube.com/embed/" + videoId;
String embedPageContent = downloader.download(embedUrl);
//todo: find out if this can be reapaced by Parser.matchGroup1()
Pattern assetsPattern = Pattern.compile("\"assets\":.+?\"js\":\\s*(\"[^\"]+\")");
Matcher patternMatcher = assetsPattern.matcher(embedPageContent);
while (patternMatcher.find()) {
playerUrl = patternMatcher.group(1);
}
playerUrl = playerUrl.replace("\\", "").replace("\"", "");
if (playerUrl.startsWith("//")) {
playerUrl = HTTPS + playerUrl;
}
return playerUrl;
} catch (IOException e) {
throw new ParsingException(
"Could load decryption code form restricted video for the Youtube service.", e);
} catch (ReCaptchaException e) {
throw new ReCaptchaException("reCaptcha Challenge requested");
} }
} }
@ -457,7 +215,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Override @Override
public String getDashMpdUrl() throws ParsingException { public String getDashMpdUrl() throws ParsingException {
try { try {
String dashManifestUrl = ""; String dashManifestUrl;
if (videoInfoPage != null && videoInfoPage.containsKey("dashmpd")) { if (videoInfoPage != null && videoInfoPage.containsKey("dashmpd")) {
dashManifestUrl = videoInfoPage.get("dashmpd"); dashManifestUrl = videoInfoPage.get("dashmpd");
} else if (playerArgs.has("dashmpd")) { } else if (playerArgs.has("dashmpd")) {
@ -479,7 +237,6 @@ public class YoutubeStreamExtractor extends StreamExtractor {
} }
} }
@Override @Override
public List<AudioStream> getAudioStreams() throws ParsingException { public List<AudioStream> getAudioStreams() throws ParsingException {
Vector<AudioStream> audioStreams = new Vector<>(); Vector<AudioStream> audioStreams = new Vector<>();
@ -507,9 +264,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
int itag = Integer.parseInt(tags.get("itag")); int itag = Integer.parseInt(tags.get("itag"));
if (itagIsSupported(itag)) { if (ItagItem.isSupported(itag)) {
ItagItem itagItem = getItagItem(itag); ItagItem itagItem = ItagItem.getItag(itag);
if (itagItem.itagType == ItagType.AUDIO) { if (itagItem.itagType == ItagItem.ItagType.AUDIO) {
String streamUrl = tags.get("url"); String streamUrl = tags.get("url");
// if video has a signature: decrypt it and add it to the url // if video has a signature: decrypt it and add it to the url
if (tags.get("s") != null) { if (tags.get("s") != null) {
@ -517,11 +274,10 @@ public class YoutubeStreamExtractor extends StreamExtractor {
+ decryptSignature(tags.get("s"), decryptionCode); + decryptSignature(tags.get("s"), decryptionCode);
} }
audioStreams.add(new AudioStream(streamUrl, AudioStream audioStream = new AudioStream(streamUrl, itagItem.mediaFormatId, itagItem.avgBitrate);
itagItem.mediaFormatId, if (!Stream.containSimilarStream(audioStream, audioStreams)) {
itagItem.avgBitrate, audioStreams.add(audioStream);
itagItem.bandWidth, }
itagItem.samplingRate));
} }
} }
} }
@ -552,19 +308,20 @@ public class YoutubeStreamExtractor extends StreamExtractor {
int itag = Integer.parseInt(tags.get("itag")); int itag = Integer.parseInt(tags.get("itag"));
if (itagIsSupported(itag)) { if (ItagItem.isSupported(itag)) {
ItagItem itagItem = getItagItem(itag); ItagItem itagItem = ItagItem.getItag(itag);
if (itagItem.itagType == ItagType.VIDEO) { if (itagItem.itagType == ItagItem.ItagType.VIDEO) {
String streamUrl = tags.get("url"); String streamUrl = tags.get("url");
// if video has a signature: decrypt it and add it to the url // if video has a signature: decrypt it and add it to the url
if (tags.get("s") != null) { if (tags.get("s") != null) {
streamUrl = streamUrl + "&signature=" streamUrl = streamUrl + "&signature="
+ decryptSignature(tags.get("s"), decryptionCode); + decryptSignature(tags.get("s"), decryptionCode);
} }
videoStreams.add(new VideoStream(
streamUrl, VideoStream videoStream = new VideoStream(streamUrl, itagItem.mediaFormatId, itagItem.resolutionString);
itagItem.mediaFormatId, if (!Stream.containSimilarStream(videoStream, videoStreams)) {
itagItem.resolutionString)); videoStreams.add(videoStream);
}
} }
} }
} catch (Exception e) { } catch (Exception e) {
@ -612,9 +369,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
int itag = Integer.parseInt(tags.get("itag")); int itag = Integer.parseInt(tags.get("itag"));
if (itagIsSupported(itag)) { if (ItagItem.isSupported(itag)) {
ItagItem itagItem = getItagItem(itag); ItagItem itagItem = ItagItem.getItag(itag);
if (itagItem.itagType == ItagType.VIDEO_ONLY) { if (itagItem.itagType == ItagItem.ItagType.VIDEO_ONLY) {
String streamUrl = tags.get("url"); String streamUrl = tags.get("url");
// if video has a signature: decrypt it and add it to the url // if video has a signature: decrypt it and add it to the url
if (tags.get("s") != null) { if (tags.get("s") != null) {
@ -622,11 +379,10 @@ public class YoutubeStreamExtractor extends StreamExtractor {
+ decryptSignature(tags.get("s"), decryptionCode); + decryptSignature(tags.get("s"), decryptionCode);
} }
videoOnlyStreams.add(new VideoStream( VideoStream videoStream = new VideoStream(streamUrl, itagItem.mediaFormatId, itagItem.resolutionString, true);
true, //isVideoOnly if (!Stream.containSimilarStream(videoStream, videoOnlyStreams)) {
streamUrl, videoOnlyStreams.add(videoStream);
itagItem.mediaFormatId, }
itagItem.resolutionString));
} }
} }
} }
@ -649,7 +405,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
public int getTimeStamp() throws ParsingException { public int getTimeStamp() throws ParsingException {
String timeStamp; String timeStamp;
try { try {
timeStamp = Parser.matchGroup1("((#|&|\\?)t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)", pageUrl); timeStamp = Parser.matchGroup1("((#|&|\\?)t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)", dirtyUrl);
} catch (Parser.RegexException e) { } catch (Parser.RegexException e) {
// catch this instantly since an url does not necessarily have to have a time stamp // catch this instantly since an url does not necessarily have to have a time stamp
@ -730,7 +486,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
//if this ckicks in our button has no content and thefore likes/dislikes are disabled //if this ckicks in our button has no content and thefore likes/dislikes are disabled
return -1; return -1;
} }
return Integer.parseInt(likesString.replaceAll(REGEX_INT, "")); return Integer.parseInt(Utils.removeNonDigitCharacters(likesString));
} catch (NumberFormatException nfe) { } catch (NumberFormatException nfe) {
throw new ParsingException( throw new ParsingException(
"failed to parse likesString \"" + likesString + "\" as integers", nfe); "failed to parse likesString \"" + likesString + "\" as integers", nfe);
@ -750,7 +506,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
//if this kicks in our button has no content and therefore likes/dislikes are disabled //if this kicks in our button has no content and therefore likes/dislikes are disabled
return -1; return -1;
} }
return Integer.parseInt(dislikesString.replaceAll(REGEX_INT, "")); return Integer.parseInt(Utils.removeNonDigitCharacters(dislikesString));
} catch (NumberFormatException nfe) { } catch (NumberFormatException nfe) {
throw new ParsingException( throw new ParsingException(
"failed to parse dislikesString \"" + dislikesString + "\" as integers", nfe); "failed to parse dislikesString \"" + dislikesString + "\" as integers", nfe);
@ -788,11 +544,6 @@ public class YoutubeStreamExtractor extends StreamExtractor {
} }
} }
@Override
public String getPageUrl() {
return pageUrl;
}
@Override @Override
public String getChannelUrl() throws ParsingException { public String getChannelUrl() throws ParsingException {
try { try {
@ -804,92 +555,186 @@ public class YoutubeStreamExtractor extends StreamExtractor {
} }
@Override @Override
public StreamInfo.StreamType getStreamType() throws ParsingException { public StreamType getStreamType() throws ParsingException {
//todo: if implementing livestream support this value should be generated dynamically //todo: if implementing livestream support this value should be generated dynamically
return StreamInfo.StreamType.VIDEO_STREAM; return StreamType.VIDEO_STREAM;
} }
/** /**
* Provides information about links to other videos on the video page, such as related videos. * {@inheritDoc}
* This is encapsulated in a StreamInfoItem object,
* which is a subset of the fields in a full StreamInfo.
*/ */
private StreamInfoItemExtractor extractVideoPreviewInfo(final Element li) { @Override
return new StreamInfoItemExtractor() { public String getErrorMessage() {
@Override String errorMessage = doc.select("h1[id=\"unavailable-message\"]").first().text();
public AbstractStreamInfo.StreamType getStreamType() throws ParsingException { StringBuilder errorReason;
return AbstractStreamInfo.StreamType.VIDEO_STREAM;
}
@Override if (errorMessage == null || errorMessage.isEmpty()) {
public boolean isAd() throws ParsingException { errorReason = null;
return !li.select("span[class*=\"icon-not-available\"]").isEmpty(); } else if (errorMessage.contains("GEMA")) {
} // Gema sometimes blocks youtube music content in germany:
// https://www.gema.de/en/
// Detailed description:
// https://en.wikipedia.org/wiki/GEMA_%28German_organization%29
errorReason = new StringBuilder("GEMA");
} else {
errorReason = new StringBuilder(errorMessage);
errorReason.append(" ");
errorReason.append(doc.select("[id=\"unavailable-submessage\"]").first().text());
}
@Override return errorReason != null ? errorReason.toString() : null;
public String getWebPageUrl() throws ParsingException {
return li.select("a.content-link").first().attr("abs:href");
}
@Override
public String getTitle() throws ParsingException {
//todo: check NullPointerException causing
return li.select("span.title").first().text();
//this page causes the NullPointerException, after finding it by searching for "tjvg":
//https://www.youtube.com/watch?v=Uqg0aEhLFAg
}
@Override
public int getDuration() throws ParsingException {
return YoutubeParsingHelper.parseDurationString(
li.select("span.video-time").first().text());
}
@Override
public String getUploader() throws ParsingException {
return li.select("span.g-hovercard").first().text();
}
@Override
public String getUploadDate() throws ParsingException {
return null;
}
@Override
public long getViewCount() throws ParsingException {
//this line is unused
//String views = li.select("span.view-count").first().text();
//Log.i(TAG, "title:"+info.title);
//Log.i(TAG, "view count:"+views);
try {
return Long.parseLong(li.select("span.view-count")
.first().text().replaceAll(REGEX_INT, ""));
} catch (Exception e) {
//related videos sometimes have no view count
return 0;
}
}
@Override
public String getThumbnailUrl() throws ParsingException {
Element img = li.select("img").first();
String thumbnailUrl = img.attr("abs:src");
// Sometimes youtube sends links to gif files which somehow seem to not exist
// anymore. Items with such gif also offer a secondary image source. So we are going
// to use that if we caught such an item.
if (thumbnailUrl.contains(".gif")) {
thumbnailUrl = img.attr("data-thumb");
}
if (thumbnailUrl.startsWith("//")) {
thumbnailUrl = HTTPS + thumbnailUrl;
}
return thumbnailUrl;
}
};
} }
/*//////////////////////////////////////////////////////////////////////////
// Utils
//////////////////////////////////////////////////////////////////////////*/
private JSONObject playerArgs;
private boolean isAgeRestricted;
private Map<String, String> videoInfoPage;
private static final String URL_ENCODED_FMT_STREAM_MAP = "url_encoded_fmt_stream_map";
private static final String HTTPS = "https:";
private static final String CONTENT = "content";
/**
* Sometimes if the html page of youtube is already downloaded, youtube web page will internally
* download the /get_video_info page. Since a certain date dashmpd url is only available over
* this /get_video_info page, so we always need to download this one to.
* <p>
* %%video_id%% will be replaced by the actual video id
* $$el_type$$ will be replaced by the actual el_type (se the declarations below)
*/
private static final String GET_VIDEO_INFO_URL =
"https://www.youtube.com/get_video_info?video_id=%%video_id%%$$el_type$$&ps=default&eurl=&gl=US&hl=en";
// eltype is necessary for the url above
private static final String EL_INFO = "el=info";
// static values
private static final String DECRYPTION_FUNC_NAME = "decrypt";
// cached values
private static volatile String decryptionCode = "";
private void fetchDocument() throws IOException, ReCaptchaException, ParsingException {
Downloader downloader = NewPipe.getDownloader();
String pageContent = downloader.download(getUrl());
doc = Jsoup.parse(pageContent, getUrl());
JSONObject ytPlayerConfig;
String playerUrl;
String videoInfoUrl = GET_VIDEO_INFO_URL.replace("%%video_id%%", getId()).replace("$$el_type$$", "&" + EL_INFO);
String videoInfoPageString = downloader.download(videoInfoUrl);
videoInfoPage = Parser.compatParseMap(videoInfoPageString);
// Check if the video is age restricted
if (pageContent.contains("<meta property=\"og:restrictions:age")) {
playerUrl = getPlayerUrlFromRestrictedVideo(getUrl());
isAgeRestricted = true;
} else {
ytPlayerConfig = getPlayerConfig(pageContent);
playerArgs = getPlayerArgs(ytPlayerConfig);
playerUrl = getPlayerUrl(ytPlayerConfig);
isAgeRestricted = false;
}
if (decryptionCode.isEmpty()) {
decryptionCode = loadDecryptionCode(playerUrl);
}
}
private JSONObject getPlayerConfig(String pageContent) throws ParsingException {
try {
String ytPlayerConfigRaw =
Parser.matchGroup1("ytplayer.config\\s*=\\s*(\\{.*?\\});", pageContent);
return new JSONObject(ytPlayerConfigRaw);
} catch (Parser.RegexException e) {
String errorReason = getErrorMessage();
switch (errorReason) {
case "GEMA":
throw new GemaException(errorReason);
case "":
throw new ContentNotAvailableException("Content not available: player config empty", e);
default:
throw new ContentNotAvailableException("Content not available", e);
}
} catch (JSONException e) {
throw new ParsingException("Could not parse yt player config", e);
}
}
private JSONObject getPlayerArgs(JSONObject playerConfig) throws ParsingException {
JSONObject playerArgs;
//attempt to load the youtube js player JSON arguments
boolean isLiveStream = false; //used to determine if this is a livestream or not
try {
playerArgs = playerConfig.getJSONObject("args");
// check if we have a live stream. We need to filter it, since its not yet supported.
if ((playerArgs.has("ps") && playerArgs.get("ps").toString().equals("live"))
|| (playerArgs.get(URL_ENCODED_FMT_STREAM_MAP).toString().isEmpty())) {
isLiveStream = true;
}
} catch (JSONException e) {
throw new ParsingException("Could not parse yt player config", e);
}
if (isLiveStream) {
throw new LiveStreamException("This is a Life stream. Can't use those right now.");
}
return playerArgs;
}
private String getPlayerUrl(JSONObject playerConfig) throws ParsingException {
try {
// The Youtube service needs to be initialized by downloading the
// js-Youtube-player. This is done in order to get the algorithm
// for decrypting cryptic signatures inside certain stream urls.
String playerUrl;
JSONObject ytAssets = playerConfig.getJSONObject("assets");
playerUrl = ytAssets.getString("js");
if (playerUrl.startsWith("//")) {
playerUrl = HTTPS + playerUrl;
}
return playerUrl;
} catch (JSONException e) {
throw new ParsingException(
"Could not load decryption code for the Youtube service.", e);
}
}
private String getPlayerUrlFromRestrictedVideo(String pageUrl) throws ParsingException, ReCaptchaException {
try {
Downloader downloader = NewPipe.getDownloader();
String playerUrl = "";
String videoId = getUrlIdHandler().getId(pageUrl);
String embedUrl = "https://www.youtube.com/embed/" + videoId;
String embedPageContent = downloader.download(embedUrl);
//todo: find out if this can be reapaced by Parser.matchGroup1()
Pattern assetsPattern = Pattern.compile("\"assets\":.+?\"js\":\\s*(\"[^\"]+\")");
Matcher patternMatcher = assetsPattern.matcher(embedPageContent);
while (patternMatcher.find()) {
playerUrl = patternMatcher.group(1);
}
playerUrl = playerUrl.replace("\\", "").replace("\"", "");
if (playerUrl.startsWith("//")) {
playerUrl = HTTPS + playerUrl;
}
return playerUrl;
} catch (IOException e) {
throw new ParsingException(
"Could load decryption code form restricted video for the Youtube service.", e);
} catch (ReCaptchaException e) {
throw new ReCaptchaException("reCaptcha Challenge requested");
}
}
private String loadDecryptionCode(String playerUrl) throws DecryptException { private String loadDecryptionCode(String playerUrl) throws DecryptException {
String decryptionFuncName; String decryptionFuncName;
@ -935,8 +780,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
return decryptionCode; return decryptionCode;
} }
private String decryptSignature(String encryptedSig, String decryptionCode) private String decryptSignature(String encryptedSig, String decryptionCode) throws DecryptException {
throws DecryptException {
Context context = Context.enter(); Context context = Context.enter();
context.setOptimizationLevel(-1); context.setOptimizationLevel(-1);
Object result = null; Object result = null;
@ -953,28 +797,84 @@ public class YoutubeStreamExtractor extends StreamExtractor {
return result == null ? "" : result.toString(); return result == null ? "" : result.toString();
} }
/** /**
* {@inheritDoc} * Provides information about links to other videos on the video page, such as related videos.
* This is encapsulated in a StreamInfoItem object,
* which is a subset of the fields in a full StreamInfo.
*/ */
public String getErrorMessage() { private StreamInfoItemExtractor extractVideoPreviewInfo(final Element li) {
String errorMessage = doc.select("h1[id=\"unavailable-message\"]").first().text(); return new StreamInfoItemExtractor() {
StringBuilder errorReason; @Override
public StreamType getStreamType() throws ParsingException {
return StreamType.VIDEO_STREAM;
}
if (errorMessage == null || errorMessage.isEmpty()) { @Override
errorReason = null; public boolean isAd() throws ParsingException {
} else if(errorMessage.contains("GEMA")) { return !li.select("span[class*=\"icon-not-available\"]").isEmpty();
// Gema sometimes blocks youtube music content in germany: }
// https://www.gema.de/en/
// Detailed description:
// https://en.wikipedia.org/wiki/GEMA_%28German_organization%29
errorReason = new StringBuilder("GEMA");
} else {
errorReason = new StringBuilder(errorMessage);
errorReason.append(" ");
errorReason.append(doc.select("[id=\"unavailable-submessage\"]").first().text());
}
return errorReason != null ? errorReason.toString() : null; @Override
public String getWebPageUrl() throws ParsingException {
return li.select("a.content-link").first().attr("abs:href");
}
@Override
public String getTitle() throws ParsingException {
//todo: check NullPointerException causing
return li.select("span.title").first().text();
//this page causes the NullPointerException, after finding it by searching for "tjvg":
//https://www.youtube.com/watch?v=Uqg0aEhLFAg
}
@Override
public int getDuration() throws ParsingException {
return YoutubeParsingHelper.parseDurationString(
li.select("span.video-time").first().text());
}
@Override
public String getUploader() throws ParsingException {
return li.select("span.g-hovercard").first().text();
}
@Override
public String getUploadDate() throws ParsingException {
return null;
}
@Override
public long getViewCount() throws ParsingException {
//this line is unused
//String views = li.select("span.view-count").first().text();
//Log.i(TAG, "title:"+info.title);
//Log.i(TAG, "view count:"+views);
try {
return Long.parseLong(Utils.removeNonDigitCharacters(
li.select("span.view-count").first().text()));
} catch (Exception e) {
//related videos sometimes have no view count
return 0;
}
}
@Override
public String getThumbnailUrl() throws ParsingException {
Element img = li.select("img").first();
String thumbnailUrl = img.attr("abs:src");
// Sometimes youtube sends links to gif files which somehow seem to not exist
// anymore. Items with such gif also offer a secondary image source. So we are going
// to use that if we caught such an item.
if (thumbnailUrl.contains(".gif")) {
thumbnailUrl = img.attr("data-thumb");
}
if (thumbnailUrl.startsWith("//")) {
thumbnailUrl = HTTPS + thumbnailUrl;
}
return thumbnailUrl;
}
};
} }
} }

11
services/youtube/YoutubeStreamInfoItemExtractor.java
View File

@ -3,8 +3,9 @@ package org.schabi.newpipe.extractor.services.youtube;
import org.jsoup.nodes.Element; import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.exceptions.FoundAdException; import org.schabi.newpipe.extractor.exceptions.FoundAdException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.stream.AbstractStreamInfo;
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.utils.Utils;
/* /*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org> * Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
@ -116,7 +117,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
} }
} }
output = input.replaceAll("[^0-9]+", ""); output = Utils.removeNonDigitCharacters(input);
try { try {
return Long.parseLong(output); return Long.parseLong(output);
@ -150,11 +151,11 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
} }
@Override @Override
public AbstractStreamInfo.StreamType getStreamType() { public StreamType getStreamType() {
if (isLiveStream(item)) { if (isLiveStream(item)) {
return AbstractStreamInfo.StreamType.LIVE_STREAM; return StreamType.LIVE_STREAM;
} else { } else {
return AbstractStreamInfo.StreamType.VIDEO_STREAM; return StreamType.VIDEO_STREAM;
} }
} }

1
services/youtube/YoutubeStreamUrlIdHandler.java
View File

@ -140,6 +140,7 @@ public class YoutubeStreamUrlIdHandler implements UrlIdHandler {
return Parser.matchGroup1("ci=" + ID_PATTERN, uri.getQuery()); return Parser.matchGroup1("ci=" + ID_PATTERN, uri.getQuery());
} }
@Override
public String cleanUrl(String complexUrl) throws ParsingException { public String cleanUrl(String complexUrl) throws ParsingException {
return getUrl(getId(complexUrl)); return getUrl(getId(complexUrl));
} }

41
stream/AbstractStreamInfo.java
View File

@ -1,41 +0,0 @@
package org.schabi.newpipe.extractor.stream;
/*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* AbstractStreamInfo.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
import org.schabi.newpipe.extractor.Info;
/**
* Common properties between StreamInfo and StreamInfoItem.
*/
public abstract class AbstractStreamInfo extends Info {
public enum StreamType {
NONE, // placeholder to check if stream type was checked or not
VIDEO_STREAM,
AUDIO_STREAM,
LIVE_STREAM,
AUDIO_LIVE_STREAM,
FILE
}
public StreamType stream_type;
public String uploader = "";
public String thumbnail_url = "";
public String upload_date = "";
public long view_count = -1;
}

34
stream/AudioStream.java
View File

@ -1,7 +1,5 @@
package org.schabi.newpipe.extractor.stream; package org.schabi.newpipe.extractor.stream;
import java.io.Serializable;
/* /*
* Created by Christian Schabesberger on 04.03.16. * Created by Christian Schabesberger on 04.03.16.
* *
@ -22,31 +20,17 @@ import java.io.Serializable;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>. * along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/ */
public class AudioStream implements Serializable { public class AudioStream extends Stream {
public String url = ""; public int average_bitrate = -1;
public int format = -1;
public int bandwidth = -1;
public int sampling_rate = -1;
public int avgBitrate = -1;
public AudioStream(String url, int format, int avgBitrate, int bandwidth, int samplingRate) { public AudioStream(String url, int format, int averageBitrate) {
this.url = url; super(url, format);
this.format = format; this.average_bitrate = averageBitrate;
this.avgBitrate = avgBitrate;
this.bandwidth = bandwidth;
this.sampling_rate = samplingRate;
} }
// reveals whether two streams are the same, but have different urls @Override
public boolean equalStats(AudioStream cmp) { public boolean equalStats(Stream cmp) {
return format == cmp.format return super.equalStats(cmp) && cmp instanceof AudioStream &&
&& bandwidth == cmp.bandwidth average_bitrate == ((AudioStream) cmp).average_bitrate;
&& sampling_rate == cmp.sampling_rate
&& avgBitrate == cmp.avgBitrate;
}
// reveals whether two streams are equal
public boolean equals(AudioStream cmp) {
return cmp != null && equalStats(cmp) && url.equals(cmp.url);
} }
} }

39
stream/Stream.java Normal file
View File

@ -0,0 +1,39 @@
package org.schabi.newpipe.extractor.stream;
import java.io.Serializable;
import java.util.List;
public abstract class Stream implements Serializable {
public String url;
public int format = -1;
public Stream(String url, int format) {
this.url = url;
this.format = format;
}
/**
* Reveals whether two streams are the same, but have different urls
*/
public boolean equalStats(Stream cmp) {
return cmp != null && format == cmp.format;
}
/**
* Reveals whether two Streams are equal
*/
public boolean equals(Stream cmp) {
return equalStats(cmp) && url.equals(cmp.url);
}
/**
* Check if the list already contains one stream with equals stats
*/
public static boolean containSimilarStream(Stream stream, List<? extends Stream> streamList) {
if (stream == null || streamList == null) return false;
for (Stream cmpStream : streamList) {
if (stream.equalStats(cmpStream)) return true;
}
return false;
}
}

14
stream/StreamExtractor.java
View File

@ -31,20 +31,11 @@ import java.util.List;
*/ */
public abstract class StreamExtractor extends Extractor { public abstract class StreamExtractor extends Extractor {
public static class ContentNotAvailableException extends ParsingException {
public ContentNotAvailableException(String message) {
super(message);
}
public ContentNotAvailableException(String message, Throwable cause) {
super(message, cause);
}
}
public StreamExtractor(UrlIdHandler urlIdHandler, String url, int serviceId) { public StreamExtractor(UrlIdHandler urlIdHandler, String url, int serviceId) {
super(urlIdHandler, serviceId, url); super(urlIdHandler, serviceId, url);
} }
public abstract String getId() throws ParsingException;
public abstract int getTimeStamp() throws ParsingException; public abstract int getTimeStamp() throws ParsingException;
public abstract String getTitle() throws ParsingException; public abstract String getTitle() throws ParsingException;
public abstract String getDescription() throws ParsingException; public abstract String getDescription() throws ParsingException;
@ -65,8 +56,7 @@ public abstract class StreamExtractor extends Extractor {
public abstract int getDislikeCount() throws ParsingException; public abstract int getDislikeCount() throws ParsingException;
public abstract StreamInfoItemExtractor getNextVideo() throws ParsingException; public abstract StreamInfoItemExtractor getNextVideo() throws ParsingException;
public abstract StreamInfoItemCollector getRelatedVideos() throws ParsingException; public abstract StreamInfoItemCollector getRelatedVideos() throws ParsingException;
public abstract String getPageUrl(); public abstract StreamType getStreamType() throws ParsingException;
public abstract StreamInfo.StreamType getStreamType() throws ParsingException;
/** /**
* Analyses the webpage's document and extracts any error message there might be. * Analyses the webpage's document and extracts any error message there might be.

146
stream/StreamInfo.java
View File

@ -1,7 +1,8 @@
package org.schabi.newpipe.extractor.stream; package org.schabi.newpipe.extractor.stream;
import org.schabi.newpipe.extractor.Info;
import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.utils.DashMpdParser; import org.schabi.newpipe.extractor.utils.DashMpdParser;
@ -31,11 +32,11 @@ import java.util.Vector;
/** /**
* Info object for opened videos, ie the video ready to play. * Info object for opened videos, ie the video ready to play.
*/ */
@SuppressWarnings("ALL") @SuppressWarnings("WeakerAccess")
public class StreamInfo extends AbstractStreamInfo { public class StreamInfo extends Info {
public static class StreamExctractException extends ExtractionException { public static class StreamExtractException extends ExtractionException {
StreamExctractException(String message) { StreamExtractException(String message) {
super(message); super(message);
} }
} }
@ -43,43 +44,11 @@ public class StreamInfo extends AbstractStreamInfo {
public StreamInfo() { public StreamInfo() {
} }
/**
* Creates a new StreamInfo object from an existing AbstractVideoInfo.
* All the shared properties are copied to the new StreamInfo.
*/
@SuppressWarnings("WeakerAccess")
public StreamInfo(AbstractStreamInfo avi) {
this.id = avi.id;
this.url = avi.url;
this.name = avi.name;
this.uploader = avi.uploader;
this.thumbnail_url = avi.thumbnail_url;
this.upload_date = avi.upload_date;
this.upload_date = avi.upload_date;
this.view_count = avi.view_count;
//todo: better than this
if (avi instanceof StreamInfoItem) {
//shitty String to convert code
/*
String dur = ((StreamInfoItem)avi).duration;
int minutes = Integer.parseInt(dur.substring(0, dur.indexOf(":")));
int seconds = Integer.parseInt(dur.substring(dur.indexOf(":")+1, dur.length()));
*/
this.duration = ((StreamInfoItem) avi).duration;
}
}
public void addException(Exception e) {
errors.add(e);
}
/** /**
* Fills out the video info fields which are common to all services. * Fills out the video info fields which are common to all services.
* Probably needs to be overridden by subclasses * Probably needs to be overridden by subclasses
*/ */
public static StreamInfo getVideoInfo(StreamExtractor extractor) public static StreamInfo getVideoInfo(StreamExtractor extractor) throws ExtractionException {
throws ExtractionException, StreamExtractor.ContentNotAvailableException {
StreamInfo streamInfo = new StreamInfo(); StreamInfo streamInfo = new StreamInfo();
try { try {
@ -87,15 +56,15 @@ public class StreamInfo extends AbstractStreamInfo {
streamInfo = extractStreams(streamInfo, extractor); streamInfo = extractStreams(streamInfo, extractor);
streamInfo = extractOptionalData(streamInfo, extractor); streamInfo = extractOptionalData(streamInfo, extractor);
} catch (ExtractionException e) { } catch (ExtractionException e) {
// Currently YouTube does not distinguish between age restricted videos and videos blocked // Currently YouTube does not distinguish between age restricted videos and videos blocked
// by country. This means that during the initialisation of the extractor, the extractor // by country. This means that during the initialisation of the extractor, the extractor
// will assume that a video is age restricted while in reality it it blocked by country. // will assume that a video is age restricted while in reality it it blocked by country.
// //
// We will now detect whether the video is blocked by country or not. // We will now detect whether the video is blocked by country or not.
String errorMsg = extractor.getErrorMessage(); String errorMsg = extractor.getErrorMessage();
if (errorMsg != null) { if (errorMsg != null) {
throw new StreamExtractor.ContentNotAvailableException(errorMsg); throw new ContentNotAvailableException(errorMsg);
} else { } else {
throw e; throw e;
} }
@ -104,18 +73,14 @@ public class StreamInfo extends AbstractStreamInfo {
return streamInfo; return streamInfo;
} }
private static StreamInfo extractImportantData( private static StreamInfo extractImportantData(StreamInfo streamInfo, StreamExtractor extractor) throws ExtractionException {
StreamInfo streamInfo, StreamExtractor extractor)
throws ExtractionException {
/* ---- important data, withoug the video can't be displayed goes here: ---- */ /* ---- important data, withoug the video can't be displayed goes here: ---- */
// if one of these is not available an exception is meant to be thrown directly into the frontend. // if one of these is not available an exception is meant to be thrown directly into the frontend.
UrlIdHandler uiconv = extractor.getUrlIdHandler();
streamInfo.service_id = extractor.getServiceId(); streamInfo.service_id = extractor.getServiceId();
streamInfo.url = extractor.getPageUrl(); streamInfo.url = extractor.getUrl();
streamInfo.stream_type = extractor.getStreamType(); streamInfo.stream_type = extractor.getStreamType();
streamInfo.id = uiconv.getId(extractor.getPageUrl()); streamInfo.id = extractor.getId();
streamInfo.name = extractor.getTitle(); streamInfo.name = extractor.getTitle();
streamInfo.age_limit = extractor.getAgeLimit(); streamInfo.age_limit = extractor.getAgeLimit();
@ -130,9 +95,7 @@ public class StreamInfo extends AbstractStreamInfo {
return streamInfo; return streamInfo;
} }
private static StreamInfo extractStreams( private static StreamInfo extractStreams(StreamInfo streamInfo, StreamExtractor extractor) throws ExtractionException {
StreamInfo streamInfo, StreamExtractor extractor)
throws ExtractionException {
/* ---- stream extraction goes here ---- */ /* ---- stream extraction goes here ---- */
// At least one type of stream has to be available, // At least one type of stream has to be available,
// otherwise an exception will be thrown directly into the frontend. // otherwise an exception will be thrown directly into the frontend.
@ -149,34 +112,33 @@ public class StreamInfo extends AbstractStreamInfo {
} catch (Exception e) { } catch (Exception e) {
streamInfo.addException(new ExtractionException("Couldn't get audio streams", e)); streamInfo.addException(new ExtractionException("Couldn't get audio streams", e));
} }
// also try to get streams from the dashMpd
if (streamInfo.dashMpdUrl != null && !streamInfo.dashMpdUrl.isEmpty()) {
if (streamInfo.audio_streams == null) {
streamInfo.audio_streams = new Vector<>();
}
//todo: make this quick and dirty solution a real fallback
// same as the quick and dirty above
try {
streamInfo.audio_streams.addAll(
DashMpdParser.getAudioStreams(streamInfo.dashMpdUrl));
} catch (Exception e) {
streamInfo.addException(
new ExtractionException("Couldn't get audio streams from dash mpd", e));
}
}
/* Extract video stream url*/ /* Extract video stream url*/
try { try {
streamInfo.video_streams = extractor.getVideoStreams(); streamInfo.video_streams = extractor.getVideoStreams();
} catch (Exception e) { } catch (Exception e) {
streamInfo.addException( streamInfo.addException(new ExtractionException("Couldn't get video streams", e));
new ExtractionException("Couldn't get video streams", e));
} }
/* Extract video only stream url*/ /* Extract video only stream url*/
try { try {
streamInfo.video_only_streams = extractor.getVideoOnlyStreams(); streamInfo.video_only_streams = extractor.getVideoOnlyStreams();
} catch (Exception e) { } catch (Exception e) {
streamInfo.addException( streamInfo.addException(new ExtractionException("Couldn't get video only streams", e));
new ExtractionException("Couldn't get video only streams", e)); }
// Lists can be null if a exception was thrown during extraction
if (streamInfo.video_streams == null) streamInfo.video_streams = new Vector<>();
if (streamInfo.video_only_streams == null) streamInfo.video_only_streams = new Vector<>();
if (streamInfo.audio_streams == null) streamInfo.audio_streams = new Vector<>();
if (streamInfo.dashMpdUrl != null && !streamInfo.dashMpdUrl.isEmpty()) {
try {
// Will try to find in the dash manifest for any stream that the ItagItem has (by the id),
// it has video, video only and audio streams and will only add to the list if it don't
// find a similar stream in the respective lists (calling Stream#equalStats).
DashMpdParser.getStreams(streamInfo);
} catch (Exception e) {
streamInfo.addException(new ExtractionException("Couldn't get streams from dash mpd", e));
}
} }
// either dash_mpd audio_only or video has to be available, otherwise we didn't get a stream, // either dash_mpd audio_only or video has to be available, otherwise we didn't get a stream,
@ -184,15 +146,14 @@ public class StreamInfo extends AbstractStreamInfo {
if ((streamInfo.video_streams == null || streamInfo.video_streams.isEmpty()) if ((streamInfo.video_streams == null || streamInfo.video_streams.isEmpty())
&& (streamInfo.audio_streams == null || streamInfo.audio_streams.isEmpty()) && (streamInfo.audio_streams == null || streamInfo.audio_streams.isEmpty())
&& (streamInfo.dashMpdUrl == null || streamInfo.dashMpdUrl.isEmpty())) { && (streamInfo.dashMpdUrl == null || streamInfo.dashMpdUrl.isEmpty())) {
throw new StreamExctractException( throw new StreamExtractException(
"Could not get any stream. See error variable to get further details."); "Could not get any stream. See error variable to get further details.");
} }
return streamInfo; return streamInfo;
} }
private static StreamInfo extractOptionalData( private static StreamInfo extractOptionalData(StreamInfo streamInfo, StreamExtractor extractor) {
StreamInfo streamInfo, StreamExtractor extractor) {
/* ---- optional data goes here: ---- */ /* ---- optional data goes here: ---- */
// If one of these fails, the frontend needs to handle that they are not available. // If one of these fails, the frontend needs to handle that they are not available.
// Exceptions are therefore not thrown into the frontend, but stored into the error List, // Exceptions are therefore not thrown into the frontend, but stored into the error List,
@ -259,8 +220,7 @@ public class StreamInfo extends AbstractStreamInfo {
streamInfo.addException(e); streamInfo.addException(e);
} }
try { try {
StreamInfoItemCollector c = new StreamInfoItemCollector( StreamInfoItemCollector c = new StreamInfoItemCollector(extractor.getServiceId());
extractor.getUrlIdHandler(), extractor.getServiceId());
StreamInfoItemExtractor nextVideo = extractor.getNextVideo(); StreamInfoItemExtractor nextVideo = extractor.getNextVideo();
c.commit(nextVideo); c.commit(nextVideo);
if (c.getItemList().size() != 0) { if (c.getItemList().size() != 0) {
@ -282,26 +242,36 @@ public class StreamInfo extends AbstractStreamInfo {
return streamInfo; return streamInfo;
} }
public String uploader_thumbnail_url = ""; public void addException(Exception e) {
public String channel_url = ""; errors.add(e);
public String description = ""; }
public List<VideoStream> video_streams = null; public StreamType stream_type;
public List<AudioStream> audio_streams = null; public String uploader;
public List<VideoStream> video_only_streams = null; public String thumbnail_url;
public String upload_date;
public long view_count = -1;
public String uploader_thumbnail_url;
public String channel_url;
public String description;
public List<VideoStream> video_streams;
public List<AudioStream> audio_streams;
public List<VideoStream> video_only_streams;
// video streams provided by the dash mpd do not need to be provided as VideoStream. // video streams provided by the dash mpd do not need to be provided as VideoStream.
// Later on this will also aplly to audio streams. Since dash mpd is standarized, // Later on this will also aplly to audio streams. Since dash mpd is standarized,
// crawling such a file is not service dependent. Therefore getting audio only streams by yust // crawling such a file is not service dependent. Therefore getting audio only streams by yust
// providing the dash mpd fille will be possible in the future. // providing the dash mpd fille will be possible in the future.
public String dashMpdUrl = ""; public String dashMpdUrl;
public int duration = -1; public int duration = -1;
public int age_limit = -1; public int age_limit = -1;
public int like_count = -1; public int like_count = -1;
public int dislike_count = -1; public int dislike_count = -1;
public String average_rating = ""; public String average_rating;
public StreamInfoItem next_video = null; public StreamInfoItem next_video;
public List<InfoItem> related_streams = null; public List<InfoItem> related_streams = new Vector<>();
//in seconds. some metadata is not passed using a StreamInfo object! //in seconds. some metadata is not passed using a StreamInfo object!
public int start_position = 0; public int start_position = 0;
} }

20
stream/StreamInfoItem.java
View File

@ -25,18 +25,16 @@ import org.schabi.newpipe.extractor.InfoItem;
/** /**
* Info object for previews of unopened videos, eg search results, related videos * Info object for previews of unopened videos, eg search results, related videos
*/ */
public class StreamInfoItem extends AbstractStreamInfo implements InfoItem { public class StreamInfoItem extends InfoItem {
public int duration; public StreamType stream_type;
public InfoType infoType() { public String uploader;
return InfoType.STREAM; public String thumbnail_url;
} public String upload_date;
public long view_count = -1;
public int duration = -1;
public String getTitle() { public StreamInfoItem() {
return name; super(InfoType.STREAM);
}
public String getLink() {
return url;
} }
} }

19
stream/StreamInfoItemCollector.java
View File

@ -1,8 +1,6 @@
package org.schabi.newpipe.extractor.stream; package org.schabi.newpipe.extractor.stream;
import org.schabi.newpipe.extractor.InfoItemCollector; import org.schabi.newpipe.extractor.InfoItemCollector;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.UrlIdHandler;
import org.schabi.newpipe.extractor.exceptions.FoundAdException; import org.schabi.newpipe.extractor.exceptions.FoundAdException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
@ -28,15 +26,8 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
public class StreamInfoItemCollector extends InfoItemCollector { public class StreamInfoItemCollector extends InfoItemCollector {
private UrlIdHandler urlIdHandler; public StreamInfoItemCollector(int serviceId) {
public StreamInfoItemCollector(UrlIdHandler handler, int serviceId) {
super(serviceId); super(serviceId);
urlIdHandler = handler;
}
private UrlIdHandler getUrlIdHandler() {
return urlIdHandler;
} }
public StreamInfoItem extract(StreamInfoItemExtractor extractor) throws Exception { public StreamInfoItem extract(StreamInfoItemExtractor extractor) throws Exception {
@ -48,13 +39,7 @@ public class StreamInfoItemCollector extends InfoItemCollector {
// important information // important information
resultItem.service_id = getServiceId(); resultItem.service_id = getServiceId();
resultItem.url = extractor.getWebPageUrl(); resultItem.url = extractor.getWebPageUrl();
if (getUrlIdHandler() == null) {
throw new ParsingException("Error: UrlIdHandler not set");
} else if (!resultItem.url.isEmpty()) {
resultItem.id = NewPipe.getService(getServiceId())
.getStreamUrlIdHandlerInstance()
.getId(resultItem.url);
}
resultItem.name = extractor.getTitle(); resultItem.name = extractor.getTitle();
resultItem.stream_type = extractor.getStreamType(); resultItem.stream_type = extractor.getStreamType();

2
stream/StreamInfoItemExtractor.java
View File

@ -23,7 +23,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
*/ */
public interface StreamInfoItemExtractor { public interface StreamInfoItemExtractor {
AbstractStreamInfo.StreamType getStreamType() throws ParsingException; StreamType getStreamType() throws ParsingException;
String getWebPageUrl() throws ParsingException; String getWebPageUrl() throws ParsingException;
String getTitle() throws ParsingException; String getTitle() throws ParsingException;
int getDuration() throws ParsingException; int getDuration() throws ParsingException;

10
stream/StreamType.java Normal file
View File

@ -0,0 +1,10 @@
package org.schabi.newpipe.extractor.stream;
public enum StreamType {
NONE, // placeholder to check if stream type was checked or not
VIDEO_STREAM,
AUDIO_STREAM,
LIVE_STREAM,
AUDIO_LIVE_STREAM,
FILE
}

31
stream/VideoStream.java
View File

@ -1,7 +1,5 @@
package org.schabi.newpipe.extractor.stream; package org.schabi.newpipe.extractor.stream;
import java.io.Serializable;
/* /*
* Created by Christian Schabesberger on 04.03.16. * Created by Christian Schabesberger on 04.03.16.
* *
@ -22,31 +20,24 @@ import java.io.Serializable;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>. * along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/ */
public class VideoStream implements Serializable { public class VideoStream extends Stream {
//url of the stream public String resolution;
public String url = ""; public boolean isVideoOnly;
public int format = -1;
public String resolution = "";
public boolean isVideoOnly = false;
public VideoStream(String url, int format, String res) { public VideoStream(String url, int format, String res) {
this(false, url, format, res); this(url, format, res, false);
} }
public VideoStream(boolean isVideoOnly, String url, int format, String res) { public VideoStream(String url, int format, String res, boolean isVideoOnly) {
this.url = url; super(url, format);
this.format = format;
this.resolution = res; this.resolution = res;
this.isVideoOnly = isVideoOnly; this.isVideoOnly = isVideoOnly;
} }
// reveals whether two streams are the same, but have different urls @Override
public boolean equalStats(VideoStream cmp) { public boolean equalStats(Stream cmp) {
return format == cmp.format && resolution.equals(cmp.resolution); return super.equalStats(cmp) && cmp instanceof VideoStream &&
} resolution.equals(((VideoStream) cmp).resolution) &&
isVideoOnly == ((VideoStream) cmp).isVideoOnly;
// reveals whether two streams are equal
public boolean equals(VideoStream cmp) {
return cmp != null && equalStats(cmp) && url.equals(cmp.url);
} }
} }

65
utils/DashMpdParser.java
View File

@ -5,7 +5,11 @@ import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.services.youtube.ItagItem;
import org.schabi.newpipe.extractor.stream.AudioStream; import org.schabi.newpipe.extractor.stream.AudioStream;
import org.schabi.newpipe.extractor.stream.Stream;
import org.schabi.newpipe.extractor.stream.StreamInfo;
import org.schabi.newpipe.extractor.stream.VideoStream;
import org.w3c.dom.Document; import org.w3c.dom.Document;
import org.w3c.dom.Element; import org.w3c.dom.Element;
import org.w3c.dom.NodeList; import org.w3c.dom.NodeList;
@ -13,8 +17,6 @@ import org.w3c.dom.NodeList;
import java.io.ByteArrayInputStream; import java.io.ByteArrayInputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.util.List;
import java.util.Vector;
import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.DocumentBuilderFactory;
@ -44,24 +46,25 @@ public class DashMpdParser {
private DashMpdParser() { private DashMpdParser() {
} }
static class DashMpdParsingException extends ParsingException { public static class DashMpdParsingException extends ParsingException {
DashMpdParsingException(String message, Exception e) { DashMpdParsingException(String message, Exception e) {
super(message, e); super(message, e);
} }
} }
public static List<AudioStream> getAudioStreams(String dashManifestUrl) /**
throws DashMpdParsingException, ReCaptchaException { * Download manifest and return nodelist with elements of tag "AdaptationSet"
*/
public static void getStreams(StreamInfo streamInfo) throws DashMpdParsingException, ReCaptchaException {
String dashDoc; String dashDoc;
Downloader downloader = NewPipe.getDownloader(); Downloader downloader = NewPipe.getDownloader();
try { try {
dashDoc = downloader.download(dashManifestUrl); dashDoc = downloader.download(streamInfo.dashMpdUrl);
} catch (IOException ioe) { } catch (IOException ioe) {
throw new DashMpdParsingException("Could not get dash mpd: " + dashManifestUrl, ioe); throw new DashMpdParsingException("Could not get dash mpd: " + streamInfo.dashMpdUrl, ioe);
} catch (ReCaptchaException e) { } catch (ReCaptchaException e) {
throw new ReCaptchaException("reCaptcha Challenge needed"); throw new ReCaptchaException("reCaptcha Challenge needed");
} }
Vector<AudioStream> audioStreams = new Vector<>();
try { try {
DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
@ -69,27 +72,43 @@ public class DashMpdParser {
InputStream stream = new ByteArrayInputStream(dashDoc.getBytes()); InputStream stream = new ByteArrayInputStream(dashDoc.getBytes());
Document doc = builder.parse(stream); Document doc = builder.parse(stream);
NodeList adaptationSetList = doc.getElementsByTagName("AdaptationSet"); NodeList representationList = doc.getElementsByTagName("Representation");
for (int i = 0; i < adaptationSetList.getLength(); i++) {
Element adaptationSet = (Element) adaptationSetList.item(i); for (int i = 0; i < representationList.getLength(); i++) {
String memeType = adaptationSet.getAttribute("mimeType"); Element representation = ((Element) representationList.item(i));
if (memeType.contains("audio")) { try {
Element representation = (Element) adaptationSet.getElementsByTagName("Representation").item(0); String mimeType = ((Element) representation.getParentNode()).getAttribute("mimeType");
String id = representation.getAttribute("id");
String url = representation.getElementsByTagName("BaseURL").item(0).getTextContent(); String url = representation.getElementsByTagName("BaseURL").item(0).getTextContent();
int bandwidth = Integer.parseInt(representation.getAttribute("bandwidth")); ItagItem itag = ItagItem.getItag(Integer.parseInt(id));
int samplingRate = Integer.parseInt(representation.getAttribute("audioSamplingRate")); if (itag != null) {
int format = -1; MediaFormat mediaFormat = MediaFormat.getFromMimeType(mimeType);
if (memeType.equals(MediaFormat.WEBMA.mimeType)) { int format = mediaFormat != null ? mediaFormat.id : -1;
format = MediaFormat.WEBMA.id;
} else if (memeType.equals(MediaFormat.M4A.mimeType)) { if (itag.itagType.equals(ItagItem.ItagType.AUDIO)) {
format = MediaFormat.M4A.id; AudioStream audioStream = new AudioStream(url, format, itag.avgBitrate);
if (!Stream.containSimilarStream(audioStream, streamInfo.audio_streams)) {
streamInfo.audio_streams.add(audioStream);
}
} else {
boolean isVideoOnly = itag.itagType.equals(ItagItem.ItagType.VIDEO_ONLY);
VideoStream videoStream = new VideoStream(url, format, itag.resolutionString, isVideoOnly);
if (isVideoOnly) {
if (!Stream.containSimilarStream(videoStream, streamInfo.video_only_streams)) {
streamInfo.video_only_streams.add(videoStream);
}
} else if (!Stream.containSimilarStream(videoStream, streamInfo.video_streams)) {
streamInfo.video_streams.add(videoStream);
}
}
} }
audioStreams.add(new AudioStream(url, format, 0, bandwidth, samplingRate)); } catch (Exception ignored) {
} }
} }
} catch (Exception e) { } catch (Exception e) {
throw new DashMpdParsingException("Could not parse Dash mpd", e); throw new DashMpdParsingException("Could not parse Dash mpd", e);
} }
return audioStreams;
} }
} }

20
utils/Utils.java Normal file
View File

@ -0,0 +1,20 @@
package org.schabi.newpipe.extractor.utils;
public class Utils {
private Utils() {
//no instance
}
/**
* Remove all non-digit characters from a string.<p>
* Examples:<br/>
* <ul><li>1 234 567 views -> 1234567</li>
* <li>$ 31,133.124 -> 31133124</li></ul>
*
* @param toRemove string to remove non-digit chars
* @return a string that contains only digits
*/
public static String removeNonDigitCharacters(String toRemove) {
return toRemove.replaceAll("\\D+", "");
}
}