be-gay-do-crimes
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Releases Wiki Activity

merged master

This commit is contained in:
Ritvik Saraf 2018-12-25 19:12:22 +05:30
parent 4883b6fb48 c77050dc70
commit df0db8468d
40 changed files with 763 additions and 321 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -1,6 +1,6 @@
# NewPipe Extractor # NewPipe Extractor
[![Build Status](https://travis-ci.org/TeamNewPipe/NewPipeExtractor.svg?branch=master)](https://travis-ci.org/TeamNewPipe/NewPipeExtractor) [![JIT Pack Badge](https://jitpack.io/v/TeamNewPipe/NewPipeExtractor.svg)](https://jitpack.io/#TeamNewPipe/NewPipeExtractor) [Documentation](https://teamnewpipe.github.io/NewPipeExtractor/javadoc/) [![Build Status](https://travis-ci.org/TeamNewPipe/NewPipeExtractor.svg?branch=master)](https://travis-ci.org/TeamNewPipe/NewPipeExtractor) [![JIT Pack Badge](https://jitpack.io/v/TeamNewPipe/NewPipeExtractor.svg)](https://jitpack.io/#TeamNewPipe/NewPipeExtractor) [Documentation](https://teamnewpipe.github.io/documentation/)
NewPipe Extractor is a library for extracting things from streaming sites. It is a core component of [NewPipe](https://github.com/TeamNewPipe/NewPipe), but could be used independently. NewPipe Extractor is a library for extracting things from streaming sites. It is a core component of [NewPipe](https://github.com/TeamNewPipe/NewPipe), but could be used independently.

9
extractor/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java
View File

@ -36,7 +36,14 @@ public enum MediaFormat {
M4A (0x3, "m4a", "m4a", "audio/mp4"), M4A (0x3, "m4a", "m4a", "audio/mp4"),
WEBMA (0x4, "WebM", "webm", "audio/webm"), WEBMA (0x4, "WebM", "webm", "audio/webm"),
MP3 (0x5, "MP3", "mp3", "audio/mpeg"), MP3 (0x5, "MP3", "mp3", "audio/mpeg"),
OPUS (0x6, "opus", "opus", "audio/opus"); OPUS (0x6, "opus", "opus", "audio/opus"),
// subtitles formats
VTT (0x7, "WebVTT", "vtt", "text/vtt"),
TTML (0x8, "Timed Text Markup Language", "ttml", "application/ttml+xml"),
TRANSCRIPT1 (0x9, "TranScript v1", "srv1", "text/xml"),
TRANSCRIPT2 (0xA, "TranScript v2", "srv2", "text/xml"),
TRANSCRIPT3 (0xB, "TranScript v3", "srv3", "text/xml"),
SRT (0xC, "SubRip file format", "srt", "text/srt");
public final int id; public final int id;
public final String name; public final String name;

2
extractor/src/main/java/org/schabi/newpipe/extractor/NewPipe.java
View File

@ -103,7 +103,7 @@ public class NewPipe {
NewPipe.localization = localization; NewPipe.localization = localization;
} }
public static Localization getLocalization() { public static Localization getPreferredLocalization() {
return localization; return localization;
} }
} }

22
extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java
View File

@ -9,6 +9,24 @@ import java.util.List;
import static java.util.Arrays.asList; import static java.util.Arrays.asList;
import static java.util.Collections.unmodifiableList; import static java.util.Collections.unmodifiableList;
/*
* Copyright (C) Christian Schabesberger 2018 <chris.schabesberger@mailbox.org>
* ServiceList.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
/** /**
* A list of supported services. * A list of supported services.
*/ */
@ -21,6 +39,10 @@ public final class ServiceList {
public static final SoundcloudService SoundCloud; public static final SoundcloudService SoundCloud;
public static final PeertubeService PeerTube; public static final PeertubeService PeerTube;
/**
* When creating a new service, put this service in the end of this list,
* and give it the next free id.
*/
private static final List<StreamingService> SERVICES = unmodifiableList( private static final List<StreamingService> SERVICES = unmodifiableList(
asList( asList(
YouTube = new YoutubeService(0), YouTube = new YoutubeService(0),

149
extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java
View File

@ -20,12 +20,39 @@ import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
import org.schabi.newpipe.extractor.utils.Localization; import org.schabi.newpipe.extractor.utils.Localization;
/*
* Copyright (C) Christian Schabesberger 2018 <chris.schabesberger@mailbox.org>
* StreamingService.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public abstract class StreamingService { public abstract class StreamingService {
/**
* This class holds meta information about the service implementation.
*/
public static class ServiceInfo { public static class ServiceInfo {
private String name; private String name;
private final List<MediaCapability> mediaCapabilities; private final List<MediaCapability> mediaCapabilities;
/**
* Creates a new instance of a ServiceInfo
* @param name the name of the service
* @param mediaCapabilities the type of media this service can handle
*/
public ServiceInfo(String name, List<MediaCapability> mediaCapabilities) { public ServiceInfo(String name, List<MediaCapability> mediaCapabilities) {
this.name = name; this.name = name;
this.mediaCapabilities = Collections.unmodifiableList(mediaCapabilities); this.mediaCapabilities = Collections.unmodifiableList(mediaCapabilities);
@ -48,6 +75,10 @@ public abstract class StreamingService {
} }
} }
/**
* LinkType will be used to determine which type of URL you are handling, and therefore which part
* of NewPipe should handle a certain URL.
*/
public enum LinkType { public enum LinkType {
NONE, NONE,
STREAM, STREAM,
@ -58,6 +89,16 @@ public abstract class StreamingService {
private final int serviceId; private final int serviceId;
private final ServiceInfo serviceInfo; private final ServiceInfo serviceInfo;
/**
* Creates a new Streaming service.
* If you Implement one do not set id within your implementation of this extractor, instead
* set the id when you put the extractor into
* <a href="https://teamnewpipe.github.io/NewPipeExtractor/javadoc/org/schabi/newpipe/extractor/ServiceList.html">ServiceList</a>.
* All other parameters can be set directly from the overriding constructor.
* @param id the number of the service to identify him within the NewPipe frontend
* @param name the name of the service
* @param capabilities the type of media this service can handle
*/
public StreamingService(int id, String name, List<ServiceInfo.MediaCapability> capabilities) { public StreamingService(int id, String name, List<ServiceInfo.MediaCapability> capabilities) {
this.serviceId = id; this.serviceId = id;
this.serviceInfo = new ServiceInfo(name, capabilities); this.serviceInfo = new ServiceInfo(name, capabilities);
@ -79,9 +120,31 @@ public abstract class StreamingService {
//////////////////////////////////////////// ////////////////////////////////////////////
// Url Id handler // Url Id handler
//////////////////////////////////////////// ////////////////////////////////////////////
/**
* Must return a new instance of an implementation of LinkHandlerFactory for streams.
* @return an instance of a LinkHandlerFactory for streams
*/
public abstract LinkHandlerFactory getStreamLHFactory(); public abstract LinkHandlerFactory getStreamLHFactory();
/**
* Must return a new instance of an implementation of ListLinkHandlerFactory for channels.
* If support for channels is not given null must be returned.
* @return an instance of a ListLinkHandlerFactory for channels or null
*/
public abstract ListLinkHandlerFactory getChannelLHFactory(); public abstract ListLinkHandlerFactory getChannelLHFactory();
/**
* Must return a new instance of an implementation of ListLinkHandlerFactory for playlists.
* If support for playlists is not given null must be returned.
* @return an instance of a ListLinkHandlerFactory for playlists or null
*/
public abstract ListLinkHandlerFactory getPlaylistLHFactory(); public abstract ListLinkHandlerFactory getPlaylistLHFactory();
/**
* Must return an instance of an implementation of SearchQueryHandlerFactory.
* @return an instance of a SearchQueryHandlerFactory
*/
public abstract SearchQueryHandlerFactory getSearchQHFactory(); public abstract SearchQueryHandlerFactory getSearchQHFactory();
public abstract ListLinkHandlerFactory getCommentsLHFactory(); public abstract ListLinkHandlerFactory getCommentsLHFactory();
@ -89,15 +152,62 @@ public abstract class StreamingService {
//////////////////////////////////////////// ////////////////////////////////////////////
// Extractor // Extractor
//////////////////////////////////////////// ////////////////////////////////////////////
public abstract SearchExtractor getSearchExtractor(SearchQueryHandler queryHandler, Localization localization);
public abstract SuggestionExtractor getSuggestionExtractor(Localization localization);
public abstract SubscriptionExtractor getSubscriptionExtractor();
public abstract KioskList getKioskList(Localization localization) throws ExtractionException;
/**
* Must create a new instance of a SearchExtractor implementation.
* @param queryHandler specifies the keyword lock for, and the filters which should be applied.
* @param localization specifies the language/country for the extractor.
* @return a new SearchExtractor instance
*/
public abstract SearchExtractor getSearchExtractor(SearchQueryHandler queryHandler, Localization localization);
/**
* Must create a new instance of a SuggestionExtractor implementation.
* @param localization specifies the language/country for the extractor.
* @return a new SuggestionExtractor instance
*/
public abstract SuggestionExtractor getSuggestionExtractor(Localization localization);
/**
* Outdated or obsolete. null can be returned.
* @return just null
*/
public abstract SubscriptionExtractor getSubscriptionExtractor();
/**
* Must create a new instance of a KioskList implementation.
* @return a new KioskList instance
* @throws ExtractionException
*/
public abstract KioskList getKioskList() throws ExtractionException;
/**
* Must create a new instance of a ChannelExtractor implementation.
* @param linkHandler is pointing to the channel which should be handled by this new instance.
* @param localization specifies the language used for the request.
* @return a new ChannelExtractor
* @throws ExtractionException
*/
public abstract ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler, public abstract ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler,
Localization localization) throws ExtractionException; Localization localization) throws ExtractionException;
/**
* Must crete a new instance of a PlaylistExtractor implementation.
* @param linkHandler is pointing to the playlist which should be handled by this new instance.
* @param localization specifies the language used for the request.
* @return a new PlaylistExtractor
* @throws ExtractionException
*/
public abstract PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler, public abstract PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler,
Localization localization) throws ExtractionException; Localization localization) throws ExtractionException;
/**
* Must create a new instance of a StreamExtractor implementation.
* @param linkHandler is pointing to the stream which should be handled by this new instance.
* @param localization specifies the language used for the request.
* @return a new StreamExtractor
* @throws ExtractionException
*/
public abstract StreamExtractor getStreamExtractor(LinkHandler linkHandler, public abstract StreamExtractor getStreamExtractor(LinkHandler linkHandler,
Localization localization) throws ExtractionException; Localization localization) throws ExtractionException;
public abstract CommentsExtractor getCommentsExtractor(ListLinkHandler linkHandler, public abstract CommentsExtractor getCommentsExtractor(ListLinkHandler linkHandler,
@ -107,31 +217,27 @@ public abstract class StreamingService {
//////////////////////////////////////////// ////////////////////////////////////////////
public SearchExtractor getSearchExtractor(SearchQueryHandler queryHandler) { public SearchExtractor getSearchExtractor(SearchQueryHandler queryHandler) {
return getSearchExtractor(queryHandler, NewPipe.getLocalization()); return getSearchExtractor(queryHandler, NewPipe.getPreferredLocalization());
} }
public SuggestionExtractor getSuggestionExtractor() { public SuggestionExtractor getSuggestionExtractor() {
return getSuggestionExtractor(NewPipe.getLocalization()); return getSuggestionExtractor(NewPipe.getPreferredLocalization());
}
public KioskList getKioskList() throws ExtractionException {
return getKioskList(NewPipe.getLocalization());
} }
public ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler) throws ExtractionException { public ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler) throws ExtractionException {
return getChannelExtractor(linkHandler, NewPipe.getLocalization()); return getChannelExtractor(linkHandler, NewPipe.getPreferredLocalization());
} }
public PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler) throws ExtractionException { public PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler) throws ExtractionException {
return getPlaylistExtractor(linkHandler, NewPipe.getLocalization()); return getPlaylistExtractor(linkHandler, NewPipe.getPreferredLocalization());
} }
public StreamExtractor getStreamExtractor(LinkHandler linkHandler) throws ExtractionException { public StreamExtractor getStreamExtractor(LinkHandler linkHandler) throws ExtractionException {
return getStreamExtractor(linkHandler, NewPipe.getLocalization()); return getStreamExtractor(linkHandler, NewPipe.getPreferredLocalization());
} }
public CommentsExtractor getCommentsExtractor(ListLinkHandler urlIdHandler) throws ExtractionException { public CommentsExtractor getCommentsExtractor(ListLinkHandler urlIdHandler) throws ExtractionException {
return getCommentsExtractor(urlIdHandler, NewPipe.getLocalization()); return getCommentsExtractor(urlIdHandler, NewPipe.getPreferredLocalization());
} }
//////////////////////////////////////////// ////////////////////////////////////////////
@ -172,19 +278,19 @@ public abstract class StreamingService {
//////////////////////////////////////////// ////////////////////////////////////////////
public SearchExtractor getSearchExtractor(String query) throws ExtractionException { public SearchExtractor getSearchExtractor(String query) throws ExtractionException {
return getSearchExtractor(getSearchQHFactory().fromQuery(query), NewPipe.getLocalization()); return getSearchExtractor(getSearchQHFactory().fromQuery(query), NewPipe.getPreferredLocalization());
} }
public ChannelExtractor getChannelExtractor(String url) throws ExtractionException { public ChannelExtractor getChannelExtractor(String url) throws ExtractionException {
return getChannelExtractor(getChannelLHFactory().fromUrl(url), NewPipe.getLocalization()); return getChannelExtractor(getChannelLHFactory().fromUrl(url), NewPipe.getPreferredLocalization());
} }
public PlaylistExtractor getPlaylistExtractor(String url) throws ExtractionException { public PlaylistExtractor getPlaylistExtractor(String url) throws ExtractionException {
return getPlaylistExtractor(getPlaylistLHFactory().fromUrl(url), NewPipe.getLocalization()); return getPlaylistExtractor(getPlaylistLHFactory().fromUrl(url), NewPipe.getPreferredLocalization());
} }
public StreamExtractor getStreamExtractor(String url) throws ExtractionException { public StreamExtractor getStreamExtractor(String url) throws ExtractionException {
return getStreamExtractor(getStreamLHFactory().fromUrl(url), NewPipe.getLocalization()); return getStreamExtractor(getStreamLHFactory().fromUrl(url), NewPipe.getPreferredLocalization());
} }
public CommentsExtractor getCommentsExtractor(String url) throws ExtractionException { public CommentsExtractor getCommentsExtractor(String url) throws ExtractionException {
@ -192,7 +298,7 @@ public abstract class StreamingService {
if(null == llhf) { if(null == llhf) {
return null; return null;
} }
return getCommentsExtractor(llhf.fromUrl(url), NewPipe.getLocalization()); return getCommentsExtractor(llhf.fromUrl(url), NewPipe.getPreferredLocalization());
} }
public abstract boolean isCommentsSupported(); public abstract boolean isCommentsSupported();
@ -203,6 +309,11 @@ public abstract class StreamingService {
/** /**
* figure out where the link is pointing to (a channel, video, playlist, etc.) * figure out where the link is pointing to (a channel, video, playlist, etc.)
/**
* Figures out where the link is pointing to (a channel, a video, a playlist, etc.)
* @param url the url on which it should be decided of which link type it is
* @return the link type of url
* @throws ParsingException
*/ */
public final LinkType getLinkTypeByUrl(String url) throws ParsingException { public final LinkType getLinkTypeByUrl(String url) throws ParsingException {
LinkHandlerFactory sH = getStreamLHFactory(); LinkHandlerFactory sH = getStreamLHFactory();

34
extractor/src/main/java/org/schabi/newpipe/extractor/Subtitles.java
View File

@ -1,34 +0,0 @@
package org.schabi.newpipe.extractor;
import org.schabi.newpipe.extractor.stream.SubtitlesFormat;
import java.io.Serializable;
import java.util.Locale;
public class Subtitles implements Serializable {
private final SubtitlesFormat format;
private final Locale locale;
private final String URL;
private final boolean autoGenerated;
public Subtitles(SubtitlesFormat format, Locale locale, String URL, boolean autoGenerated) {
this.format = format;
this.locale = locale;
this.URL = URL;
this.autoGenerated = autoGenerated;
}
public SubtitlesFormat getFileType() { return format; }
public Locale getLocale() {
return locale;
}
public String getURL() {
return URL;
}
public boolean isAutoGenerated() {
return autoGenerated;
}
}

3
extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java
View File

@ -51,8 +51,7 @@ public class ChannelInfo extends ListInfo<StreamInfoItem> {
public static InfoItemsPage<StreamInfoItem> getMoreItems(StreamingService service, public static InfoItemsPage<StreamInfoItem> getMoreItems(StreamingService service,
String url, String url,
String pageUrl, String pageUrl) throws IOException, ExtractionException {
Localization localization) throws IOException, ExtractionException {
return service.getChannelExtractor(url).getPage(pageUrl); return service.getChannelExtractor(url).getPage(pageUrl);
} }

17
extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java
View File

@ -30,7 +30,6 @@ import org.schabi.newpipe.extractor.utils.Localization;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
public abstract class KioskExtractor extends ListExtractor<StreamInfoItem> { public abstract class KioskExtractor extends ListExtractor<StreamInfoItem> {
private String contentCountry = null;
private final String id; private final String id;
public KioskExtractor(StreamingService streamingService, public KioskExtractor(StreamingService streamingService,
@ -41,17 +40,6 @@ public abstract class KioskExtractor extends ListExtractor<StreamInfoItem> {
this.id = kioskId; this.id = kioskId;
} }
/**
* For certain Websites the content of a kiosk will be different depending
* on the country you want to poen the website in. Therefore you should
* set the contentCountry.
* @param contentCountry Set the country decoded as Country Code: http://www.1728.org/countries.htm
*/
public void setContentCountry(String contentCountry) {
this.contentCountry = contentCountry;
}
@Nonnull @Nonnull
@Override @Override
public String getId() { public String getId() {
@ -69,9 +57,4 @@ public abstract class KioskExtractor extends ListExtractor<StreamInfoItem> {
@Nonnull @Nonnull
@Override @Override
public abstract String getName() throws ParsingException; public abstract String getName() throws ParsingException;
public String getContentCountry() {
return contentCountry;
}
} }

13
extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskInfo.java
View File

@ -37,25 +37,20 @@ public class KioskInfo extends ListInfo<StreamInfoItem> {
public static ListExtractor.InfoItemsPage<StreamInfoItem> getMoreItems(StreamingService service, public static ListExtractor.InfoItemsPage<StreamInfoItem> getMoreItems(StreamingService service,
String url, String url,
String pageUrl, String pageUrl) throws IOException, ExtractionException {
String contentCountry) throws IOException, ExtractionException {
KioskList kl = service.getKioskList(); KioskList kl = service.getKioskList();
KioskExtractor extractor = kl.getExtractorByUrl(url, pageUrl); KioskExtractor extractor = kl.getExtractorByUrl(url, pageUrl);
extractor.setContentCountry(contentCountry);
return extractor.getPage(pageUrl); return extractor.getPage(pageUrl);
} }
public static KioskInfo getInfo(String url, public static KioskInfo getInfo(String url) throws IOException, ExtractionException {
String contentCountry) throws IOException, ExtractionException { return getInfo(NewPipe.getServiceByUrl(url), url);
return getInfo(NewPipe.getServiceByUrl(url), url, contentCountry);
} }
public static KioskInfo getInfo(StreamingService service, public static KioskInfo getInfo(StreamingService service,
String url, String url) throws IOException, ExtractionException {
String contentCountry) throws IOException, ExtractionException {
KioskList kl = service.getKioskList(); KioskList kl = service.getKioskList();
KioskExtractor extractor = kl.getExtractorByUrl(url, null); KioskExtractor extractor = kl.getExtractorByUrl(url, null);
extractor.setContentCountry(contentCountry);
extractor.fetchPage(); extractor.fetchPage();
return getInfo(extractor); return getInfo(extractor);
} }

25
extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java
View File

@ -23,7 +23,6 @@ public class KioskList {
private final int service_id; private final int service_id;
private final HashMap<String, KioskEntry> kioskList = new HashMap<>(); private final HashMap<String, KioskEntry> kioskList = new HashMap<>();
private String defaultKiosk = null; private String defaultKiosk = null;
private final Localization localization;
private class KioskEntry { private class KioskEntry {
public KioskEntry(KioskExtractorFactory ef, ListLinkHandlerFactory h) { public KioskEntry(KioskExtractorFactory ef, ListLinkHandlerFactory h) {
@ -34,9 +33,8 @@ public class KioskList {
final ListLinkHandlerFactory handlerFactory; final ListLinkHandlerFactory handlerFactory;
} }
public KioskList(int service_id, Localization localization) { public KioskList(int service_id) {
this.service_id = service_id; this.service_id = service_id;
this.localization = localization;
} }
public void addKioskEntry(KioskExtractorFactory extractorFactory, ListLinkHandlerFactory handlerFactory, String id) public void addKioskEntry(KioskExtractorFactory extractorFactory, ListLinkHandlerFactory handlerFactory, String id)
@ -53,13 +51,18 @@ public class KioskList {
public KioskExtractor getDefaultKioskExtractor(String nextPageUrl) public KioskExtractor getDefaultKioskExtractor(String nextPageUrl)
throws ExtractionException, IOException { throws ExtractionException, IOException {
return getDefaultKioskExtractor(nextPageUrl, NewPipe.getPreferredLocalization());
}
public KioskExtractor getDefaultKioskExtractor(String nextPageUrl, Localization localization)
throws ExtractionException, IOException {
if(defaultKiosk != null && !defaultKiosk.equals("")) { if(defaultKiosk != null && !defaultKiosk.equals("")) {
return getExtractorById(defaultKiosk, nextPageUrl); return getExtractorById(defaultKiosk, nextPageUrl, localization);
} else { } else {
if(!kioskList.isEmpty()) { if(!kioskList.isEmpty()) {
// if not set get any entry // if not set get any entry
Object[] keySet = kioskList.keySet().toArray(); Object[] keySet = kioskList.keySet().toArray();
return getExtractorById(keySet[0].toString(), nextPageUrl); return getExtractorById(keySet[0].toString(), nextPageUrl, localization);
} else { } else {
return null; return null;
} }
@ -72,6 +75,11 @@ public class KioskList {
public KioskExtractor getExtractorById(String kioskId, String nextPageUrl) public KioskExtractor getExtractorById(String kioskId, String nextPageUrl)
throws ExtractionException, IOException { throws ExtractionException, IOException {
return getExtractorById(kioskId, nextPageUrl, NewPipe.getPreferredLocalization());
}
public KioskExtractor getExtractorById(String kioskId, String nextPageUrl, Localization localization)
throws ExtractionException, IOException {
KioskEntry ke = kioskList.get(kioskId); KioskEntry ke = kioskList.get(kioskId);
if(ke == null) { if(ke == null) {
throw new ExtractionException("No kiosk found with the type: " + kioskId); throw new ExtractionException("No kiosk found with the type: " + kioskId);
@ -86,11 +94,16 @@ public class KioskList {
} }
public KioskExtractor getExtractorByUrl(String url, String nextPageUrl) public KioskExtractor getExtractorByUrl(String url, String nextPageUrl)
throws ExtractionException, IOException{
return getExtractorByUrl(url, nextPageUrl, NewPipe.getPreferredLocalization());
}
public KioskExtractor getExtractorByUrl(String url, String nextPageUrl, Localization localization)
throws ExtractionException, IOException { throws ExtractionException, IOException {
for(Map.Entry<String, KioskEntry> e : kioskList.entrySet()) { for(Map.Entry<String, KioskEntry> e : kioskList.entrySet()) {
KioskEntry ke = e.getValue(); KioskEntry ke = e.getValue();
if(ke.handlerFactory.acceptUrl(url)) { if(ke.handlerFactory.acceptUrl(url)) {
return getExtractorById(e.getKey(), nextPageUrl); return getExtractorById(e.getKey(), nextPageUrl, localization);
} }
} }
throw new ExtractionException("Could not find a kiosk that fits to the url: " + url); throw new ExtractionException("Could not find a kiosk that fits to the url: " + url);

60
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java
View File

@ -97,36 +97,6 @@ public class PeertubeService extends StreamingService {
return null; return null;
} }
@Override
public KioskList getKioskList(Localization localization) throws ExtractionException {
KioskList.KioskExtractorFactory kioskFactory = new KioskList.KioskExtractorFactory() {
@Override
public KioskExtractor createNewKiosk(StreamingService streamingService,
String url,
String id,
Localization local)
throws ExtractionException {
return new PeertubeTrendingExtractor(PeertubeService.this,
new PeertubeTrendingLinkHandlerFactory().fromId(id), id, local);
}
};
KioskList list = new KioskList(getServiceId(), localization);
// add kiosks here e.g.:
final PeertubeTrendingLinkHandlerFactory h = new PeertubeTrendingLinkHandlerFactory();
try {
list.addKioskEntry(kioskFactory, h, PeertubeTrendingLinkHandlerFactory.KIOSK_TRENDING);
list.addKioskEntry(kioskFactory, h, PeertubeTrendingLinkHandlerFactory.KIOSK_RECENT);
list.addKioskEntry(kioskFactory, h, PeertubeTrendingLinkHandlerFactory.KIOSK_LOCAL);
list.setDefaultKiosk(PeertubeTrendingLinkHandlerFactory.KIOSK_TRENDING);
} catch (Exception e) {
throw new ExtractionException(e);
}
return list;
}
@Override @Override
public ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler, Localization localization) public ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler, Localization localization)
throws ExtractionException { throws ExtractionException {
@ -171,5 +141,35 @@ public class PeertubeService extends StreamingService {
} }
} }
@Override
public KioskList getKioskList() throws ExtractionException {
KioskList.KioskExtractorFactory kioskFactory = new KioskList.KioskExtractorFactory() {
@Override
public KioskExtractor createNewKiosk(StreamingService streamingService,
String url,
String id,
Localization local)
throws ExtractionException {
return new PeertubeTrendingExtractor(PeertubeService.this,
new PeertubeTrendingLinkHandlerFactory().fromId(id), id, local);
}
};
KioskList list = new KioskList(getServiceId());
// add kiosks here e.g.:
final PeertubeTrendingLinkHandlerFactory h = new PeertubeTrendingLinkHandlerFactory();
try {
list.addKioskEntry(kioskFactory, h, PeertubeTrendingLinkHandlerFactory.KIOSK_TRENDING);
list.addKioskEntry(kioskFactory, h, PeertubeTrendingLinkHandlerFactory.KIOSK_RECENT);
list.addKioskEntry(kioskFactory, h, PeertubeTrendingLinkHandlerFactory.KIOSK_LOCAL);
list.setDefaultKiosk(PeertubeTrendingLinkHandlerFactory.KIOSK_TRENDING);
} catch (Exception e) {
throw new ExtractionException(e);
}
return list;
}
} }

11
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java
View File

@ -11,7 +11,6 @@ import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.ServiceList; import org.schabi.newpipe.extractor.ServiceList;
import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.Subtitles;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
@ -25,7 +24,7 @@ import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamType; import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.stream.SubtitlesFormat; import org.schabi.newpipe.extractor.stream.SubtitlesStream;
import org.schabi.newpipe.extractor.stream.VideoStream; import org.schabi.newpipe.extractor.stream.VideoStream;
import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Localization; import org.schabi.newpipe.extractor.utils.Localization;
@ -170,12 +169,12 @@ public class PeertubeStreamExtractor extends StreamExtractor {
} }
@Override @Override
public List<Subtitles> getSubtitlesDefault() throws IOException, ExtractionException { public List<SubtitlesStream> getSubtitlesDefault() throws IOException, ExtractionException {
return Collections.emptyList(); return Collections.emptyList();
} }
@Override @Override
public List<Subtitles> getSubtitles(SubtitlesFormat format) throws IOException, ExtractionException { public List<SubtitlesStream> getSubtitles(final MediaFormat format) throws IOException, ExtractionException {
return Collections.emptyList(); return Collections.emptyList();
} }
@ -185,12 +184,12 @@ public class PeertubeStreamExtractor extends StreamExtractor {
} }
@Override @Override
public StreamInfoItem getNextVideo() throws IOException, ExtractionException { public StreamInfoItem getNextStream() throws IOException, ExtractionException {
return null; return null;
} }
@Override @Override
public StreamInfoItemsCollector getRelatedVideos() throws IOException, ExtractionException { public StreamInfoItemsCollector getRelatedStreams() throws IOException, ExtractionException {
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
//TODO fetch related videos not trending //TODO fetch related videos not trending

4
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java
View File

@ -69,7 +69,7 @@ public class SoundcloudService extends StreamingService {
} }
@Override @Override
public KioskList getKioskList(Localization localization) throws ExtractionException { public KioskList getKioskList() throws ExtractionException {
KioskList.KioskExtractorFactory chartsFactory = new KioskList.KioskExtractorFactory() { KioskList.KioskExtractorFactory chartsFactory = new KioskList.KioskExtractorFactory() {
@Override @Override
public KioskExtractor createNewKiosk(StreamingService streamingService, public KioskExtractor createNewKiosk(StreamingService streamingService,
@ -82,7 +82,7 @@ public class SoundcloudService extends StreamingService {
} }
}; };
KioskList list = new KioskList(getServiceId(), localization); KioskList list = new KioskList(getServiceId());
// add kiosks here e.g.: // add kiosks here e.g.:
final SoundcloudChartsLinkHandlerFactory h = new SoundcloudChartsLinkHandlerFactory(); final SoundcloudChartsLinkHandlerFactory h = new SoundcloudChartsLinkHandlerFactory();

8
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java
View File

@ -172,13 +172,13 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
@Override @Override
@Nonnull @Nonnull
public List<Subtitles> getSubtitlesDefault() throws IOException, ExtractionException { public List<SubtitlesStream> getSubtitlesDefault() throws IOException, ExtractionException {
return Collections.emptyList(); return Collections.emptyList();
} }
@Override @Override
@Nonnull @Nonnull
public List<Subtitles> getSubtitles(SubtitlesFormat format) throws IOException, ExtractionException { public List<SubtitlesStream> getSubtitles(MediaFormat format) throws IOException, ExtractionException {
return Collections.emptyList(); return Collections.emptyList();
} }
@ -188,12 +188,12 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
} }
@Override @Override
public StreamInfoItem getNextVideo() throws IOException, ExtractionException { public StreamInfoItem getNextStream() throws IOException, ExtractionException {
return null; return null;
} }
@Override @Override
public StreamInfoItemsCollector getRelatedVideos() throws IOException, ExtractionException { public StreamInfoItemsCollector getRelatedStreams() throws IOException, ExtractionException {
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
String apiUrl = "https://api-v2.soundcloud.com/tracks/" + urlEncode(getId()) + "/related" String apiUrl = "https://api-v2.soundcloud.com/tracks/" + urlEncode(getId()) + "/related"

4
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java
View File

@ -111,8 +111,8 @@ public class YoutubeService extends StreamingService {
} }
@Override @Override
public KioskList getKioskList(final Localization localization) throws ExtractionException { public KioskList getKioskList() throws ExtractionException {
KioskList list = new KioskList(getServiceId(), localization); KioskList list = new KioskList(getServiceId());
// add kiosks here e.g.: // add kiosks here e.g.:
try { try {

21
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java
View File

@ -7,6 +7,7 @@ import java.util.Arrays;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.regex.Pattern;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
@ -25,6 +26,7 @@ import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Localization; import org.schabi.newpipe.extractor.utils.Localization;
import org.schabi.newpipe.extractor.utils.Parser;
import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonObject;
@ -34,6 +36,7 @@ import com.grack.nanojson.JsonParser;
public class YoutubeCommentsExtractor extends CommentsExtractor { public class YoutubeCommentsExtractor extends CommentsExtractor {
private static final String USER_AGENT = "Mozilla/5.0 (Android 8.1.0; Mobile; rv:62.0) Gecko/62.0 Firefox/62.0"; private static final String USER_AGENT = "Mozilla/5.0 (Android 8.1.0; Mobile; rv:62.0) Gecko/62.0 Firefox/62.0";
private static final Pattern YT_CLIENT_NAME_PATTERN = Pattern.compile("INNERTUBE_CONTEXT_CLIENT_NAME\\\":(.*?)[,}]");
private String ytClientVersion; private String ytClientVersion;
private String ytClientName; private String ytClientName;
@ -62,7 +65,7 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
JsonArray arr; JsonArray arr;
try { try {
arr = (JsonArray) JsonUtils.getValue(ajaxJson, "response.continuationContents.commentSectionContinuation.continuations"); arr = JsonUtils.getArray(ajaxJson, "response.continuationContents.commentSectionContinuation.continuations");
} catch (Exception e) { } catch (Exception e) {
return ""; return "";
} }
@ -71,7 +74,7 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
} }
String continuation; String continuation;
try { try {
continuation = (String) JsonUtils.getValue(arr.getObject(0), "nextContinuationData.continuation"); continuation = JsonUtils.getString(arr.getObject(0), "nextContinuationData.continuation");
} catch (Exception e) { } catch (Exception e) {
return ""; return "";
} }
@ -111,7 +114,7 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
JsonArray contents; JsonArray contents;
try { try {
contents = (JsonArray) JsonUtils.getValue(ajaxJson, "response.continuationContents.commentSectionContinuation.items"); contents = JsonUtils.getArray(ajaxJson, "response.continuationContents.commentSectionContinuation.items");
}catch(Exception e) { }catch(Exception e) {
//no comments //no comments
return; return;
@ -135,7 +138,7 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
private void fetchTitle(JsonArray contents) { private void fetchTitle(JsonArray contents) {
if(null == title) { if(null == title) {
try { try {
title = getYoutubeText((JsonObject) JsonUtils.getValue(contents.getObject(0), "commentThreadRenderer.commentTargetTitle")); title = getYoutubeText(JsonUtils.getObject(contents.getObject(0), "commentThreadRenderer.commentTargetTitle"));
} catch (Exception e) { } catch (Exception e) {
title = "Youtube Comments"; title = "Youtube Comments";
} }
@ -150,7 +153,7 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
DownloadResponse response = downloader.get(getUrl(), request); DownloadResponse response = downloader.get(getUrl(), request);
String responseBody = response.getResponseBody(); String responseBody = response.getResponseBody();
ytClientVersion = findValue(responseBody, "INNERTUBE_CONTEXT_CLIENT_VERSION\":\"", "\""); ytClientVersion = findValue(responseBody, "INNERTUBE_CONTEXT_CLIENT_VERSION\":\"", "\"");
ytClientName = findValue(responseBody, "INNERTUBE_CONTEXT_CLIENT_NAME\":", ","); ytClientName = Parser.matchGroup1(YT_CLIENT_NAME_PATTERN, responseBody);
String commentsTokenInside = findValue(responseBody, "commentSectionRenderer", "}"); String commentsTokenInside = findValue(responseBody, "commentSectionRenderer", "}");
String commentsToken = findValue(commentsTokenInside, "continuation\":\"", "\""); String commentsToken = findValue(commentsTokenInside, "continuation\":\"", "\"");
initPage = getPage(getNextPageUrl(commentsToken)); initPage = getPage(getNextPageUrl(commentsToken));
@ -196,17 +199,17 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
public static String getYoutubeText(@Nonnull JsonObject object) throws ParsingException { public static String getYoutubeText(@Nonnull JsonObject object) throws ParsingException {
try { try {
return (String) JsonUtils.getValue(object, "simpleText"); return JsonUtils.getString(object, "simpleText");
} catch (Exception e1) { } catch (Exception e1) {
try { try {
JsonArray arr = (JsonArray) JsonUtils.getValue(object, "runs"); JsonArray arr = JsonUtils.getArray(object, "runs");
String result = ""; String result = "";
for(int i=0; i<arr.size();i++) { for(int i=0; i<arr.size();i++) {
result = result + (String) JsonUtils.getValue(arr.getObject(i), "text"); result = result + JsonUtils.getString(arr.getObject(i), "text");
} }
return result; return result;
} catch (Exception e2) { } catch (Exception e2) {
throw new ParsingException("Could not get text", e2); return "";
} }
} }
} }

24
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java
View File

@ -7,7 +7,6 @@ import org.schabi.newpipe.extractor.utils.JsonUtils;
import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonObject;
public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtractor { public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtractor {
private final JsonObject json; private final JsonObject json;
@ -26,8 +25,8 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
@Override @Override
public String getThumbnailUrl() throws ParsingException { public String getThumbnailUrl() throws ParsingException {
try { try {
JsonArray arr = (JsonArray) JsonUtils.getValue(json, "authorThumbnail.thumbnails"); JsonArray arr = JsonUtils.getArray(json, "authorThumbnail.thumbnails");
return (String) JsonUtils.getValue(arr.getObject(2), "url"); return JsonUtils.getString(arr.getObject(2), "url");
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get thumbnail url", e); throw new ParsingException("Could not get thumbnail url", e);
} }
@ -36,7 +35,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
@Override @Override
public String getName() throws ParsingException { public String getName() throws ParsingException {
try { try {
return YoutubeCommentsExtractor.getYoutubeText((JsonObject) JsonUtils.getValue(json, "authorText")); return YoutubeCommentsExtractor.getYoutubeText(JsonUtils.getObject(json, "authorText"));
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get author name", e); throw new ParsingException("Could not get author name", e);
} }
@ -45,7 +44,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
@Override @Override
public String getPublishedTime() throws ParsingException { public String getPublishedTime() throws ParsingException {
try { try {
return YoutubeCommentsExtractor.getYoutubeText((JsonObject) JsonUtils.getValue(json, "publishedTimeText")); return YoutubeCommentsExtractor.getYoutubeText(JsonUtils.getObject(json, "publishedTimeText"));
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get publishedTimeText", e); throw new ParsingException("Could not get publishedTimeText", e);
} }
@ -54,7 +53,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
@Override @Override
public Integer getLikeCount() throws ParsingException { public Integer getLikeCount() throws ParsingException {
try { try {
return (Integer) JsonUtils.getValue(json, "likeCount"); return JsonUtils.getNumber(json, "likeCount").intValue();
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get like count", e); throw new ParsingException("Could not get like count", e);
} }
@ -63,7 +62,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
@Override @Override
public String getCommentText() throws ParsingException { public String getCommentText() throws ParsingException {
try { try {
return YoutubeCommentsExtractor.getYoutubeText((JsonObject) JsonUtils.getValue(json, "contentText")); return YoutubeCommentsExtractor.getYoutubeText(JsonUtils.getObject(json, "contentText"));
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get comment text", e); throw new ParsingException("Could not get comment text", e);
} }
@ -72,7 +71,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
@Override @Override
public String getCommentId() throws ParsingException { public String getCommentId() throws ParsingException {
try { try {
return (String) JsonUtils.getValue(json, "commentId"); return JsonUtils.getString(json, "commentId");
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get comment id", e); throw new ParsingException("Could not get comment id", e);
} }
@ -81,8 +80,8 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
@Override @Override
public String getAuthorThumbnail() throws ParsingException { public String getAuthorThumbnail() throws ParsingException {
try { try {
JsonArray arr = (JsonArray) JsonUtils.getValue(json, "authorThumbnail.thumbnails"); JsonArray arr = JsonUtils.getArray(json, "authorThumbnail.thumbnails");
return (String) JsonUtils.getValue(arr.getObject(2), "url"); return JsonUtils.getString(arr.getObject(2), "url");
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get author thumbnail", e); throw new ParsingException("Could not get author thumbnail", e);
} }
@ -91,7 +90,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
@Override @Override
public String getAuthorName() throws ParsingException { public String getAuthorName() throws ParsingException {
try { try {
return YoutubeCommentsExtractor.getYoutubeText((JsonObject) JsonUtils.getValue(json, "authorText")); return YoutubeCommentsExtractor.getYoutubeText(JsonUtils.getObject(json, "authorText"));
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get author name", e); throw new ParsingException("Could not get author name", e);
} }
@ -100,8 +99,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
@Override @Override
public String getAuthorEndpoint() throws ParsingException { public String getAuthorEndpoint() throws ParsingException {
try { try {
return "https://youtube.com" return "https://youtube.com" + JsonUtils.getString(json, "authorEndpoint.browseEndpoint.canonicalBaseUrl");
+ (String) JsonUtils.getValue(json, "authorEndpoint.browseEndpoint.canonicalBaseUrl");
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get author endpoint", e); throw new ParsingException("Could not get author endpoint", e);
} }

21
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java
View File

@ -54,21 +54,20 @@ public class YoutubeSearchExtractor extends SearchExtractor {
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
final String site; final String site;
final String url = getUrl(); final String url = getUrl();
final String contentCountry = getLocalization().getCountry();
//String url = builder.build().toString(); //String url = builder.build().toString();
//if we've been passed a valid language code, append it to the URL //if we've been passed a valid language code, append it to the URL
if (!contentCountry.isEmpty()) { site = downloader.download(url, getLocalization());
//assert Pattern.matches("[a-z]{2}(-([A-Z]{2}|[0-9]{1,3}))?", languageCode);
site = downloader.download(url, getLocalization());
} else {
site = downloader.download(url);
}
doc = Jsoup.parse(site, url); doc = Jsoup.parse(site, url);
} }
@Override @Override
public String getSearchSuggestion() throws ParsingException { public String getUrl() throws ParsingException {
return super.getUrl() + "&gl="+ getLocalization().getCountry();
}
@Override
public String getSearchSuggestion() {
final Element el = doc.select("div[class*=\"spell-correction\"]").first(); final Element el = doc.select("div[class*=\"spell-correction\"]").first();
if (el != null) { if (el != null) {
return el.select("a").first().text(); return el.select("a").first().text();
@ -79,13 +78,13 @@ public class YoutubeSearchExtractor extends SearchExtractor {
@Nonnull @Nonnull
@Override @Override
public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionException { public InfoItemsPage<InfoItem> getInitialPage() throws ExtractionException {
return new InfoItemsPage<>(collectItems(doc), getNextPageUrl()); return new InfoItemsPage<>(collectItems(doc), getNextPageUrl());
} }
@Override @Override
public String getNextPageUrl() throws ExtractionException { public String getNextPageUrl() throws ExtractionException {
return getUrl() + "&page=" + Integer.toString( 2); return getUrl() + "&page=" + 2;
} }
@Override @Override
@ -104,7 +103,7 @@ public class YoutubeSearchExtractor extends SearchExtractor {
.getQuery()) .getQuery())
.get("page")); .get("page"));
return currentUrl.replace("&page=" + Integer.toString( pageNr), return currentUrl.replace("&page=" + pageNr,
"&page=" + Integer.toString(pageNr + 1)); "&page=" + Integer.toString(pageNr + 1));
} }

30
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java
View File

@ -7,6 +7,7 @@ import com.grack.nanojson.JsonParserException;
import org.jsoup.Jsoup; import org.jsoup.Jsoup;
import org.jsoup.nodes.Document; import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element; import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.mozilla.javascript.Context; import org.mozilla.javascript.Context;
import org.mozilla.javascript.Function; import org.mozilla.javascript.Function;
import org.mozilla.javascript.ScriptableObject; import org.mozilla.javascript.ScriptableObject;
@ -460,15 +461,15 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Override @Override
@Nonnull @Nonnull
public List<Subtitles> getSubtitlesDefault() throws IOException, ExtractionException { public List<SubtitlesStream> getSubtitlesDefault() throws IOException, ExtractionException {
return getSubtitles(SubtitlesFormat.TTML); return getSubtitles(MediaFormat.TTML);
} }
@Override @Override
@Nonnull @Nonnull
public List<Subtitles> getSubtitles(final SubtitlesFormat format) throws IOException, ExtractionException { public List<SubtitlesStream> getSubtitles(final MediaFormat format) throws IOException, ExtractionException {
assertPageFetched(); assertPageFetched();
List<Subtitles> subtitles = new ArrayList<>(); List<SubtitlesStream> subtitles = new ArrayList<>();
for (final SubtitlesInfo subtitlesInfo : subtitlesInfos) { for (final SubtitlesInfo subtitlesInfo : subtitlesInfos) {
subtitles.add(subtitlesInfo.getSubtitle(format)); subtitles.add(subtitlesInfo.getSubtitle(format));
} }
@ -490,13 +491,17 @@ public class YoutubeStreamExtractor extends StreamExtractor {
} }
@Override @Override
public StreamInfoItem getNextVideo() throws IOException, ExtractionException { public StreamInfoItem getNextStream() throws IOException, ExtractionException {
assertPageFetched(); assertPageFetched();
try { try {
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
collector.commit(extractVideoPreviewInfo(doc.select("div[class=\"watch-sidebar-section\"]")
.first().select("li").first()));
Elements watch = doc.select("div[class=\"watch-sidebar-section\"]");
if (watch.size() < 1) {
return null;// prevent the snackbar notification "report error" on age-restricted videos
}
collector.commit(extractVideoPreviewInfo(watch.first().select("li").first()));
return collector.getItems().get(0); return collector.getItems().get(0);
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get next video", e); throw new ParsingException("Could not get next video", e);
@ -504,7 +509,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
} }
@Override @Override
public StreamInfoItemsCollector getRelatedVideos() throws IOException, ExtractionException { public StreamInfoItemsCollector getRelatedStreams() throws IOException, ExtractionException {
assertPageFetched(); assertPageFetched();
try { try {
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
@ -815,21 +820,16 @@ public class YoutubeStreamExtractor extends StreamExtractor {
final String languageCode; final String languageCode;
final boolean isGenerated; final boolean isGenerated;
final Locale locale;
public SubtitlesInfo(final String baseUrl, final String languageCode, final boolean isGenerated) { public SubtitlesInfo(final String baseUrl, final String languageCode, final boolean isGenerated) {
this.cleanUrl = baseUrl this.cleanUrl = baseUrl
.replaceAll("&fmt=[^&]*", "") // Remove preexisting format if exists .replaceAll("&fmt=[^&]*", "") // Remove preexisting format if exists
.replaceAll("&tlang=[^&]*", ""); // Remove translation language .replaceAll("&tlang=[^&]*", ""); // Remove translation language
this.languageCode = languageCode; this.languageCode = languageCode;
this.isGenerated = isGenerated; this.isGenerated = isGenerated;
final String[] splits = languageCode.split("-");
this.locale = splits.length == 2 ? new Locale(splits[0], splits[1]) : new Locale(languageCode);
} }
public Subtitles getSubtitle(final SubtitlesFormat format) { public SubtitlesStream getSubtitle(final MediaFormat format) {
return new Subtitles(format, locale, cleanUrl + "&fmt=" + format.getExtension(), isGenerated); return new SubtitlesStream(format, languageCode, cleanUrl + "&fmt=" + format.getSuffix(), isGenerated);
} }
} }

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java
View File

@ -50,7 +50,7 @@ public class YoutubeTrendingExtractor extends KioskExtractor {
@Override @Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
final String contentCountry = getContentCountry(); final String contentCountry = getLocalization().getCountry();
String url = getUrl(); String url = getUrl();
if(contentCountry != null && !contentCountry.isEmpty()) { if(contentCountry != null && !contentCountry.isEmpty()) {
url += "?gl=" + contentCountry; url += "?gl=" + contentCountry;

262
extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java
View File

@ -1,7 +1,7 @@
package org.schabi.newpipe.extractor.stream; package org.schabi.newpipe.extractor.stream;
/* /*
* Created by Christian Schabesberger on 10.08.15. * Created by Christian Schabesberger on 10.08.18.
* *
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org> * Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* StreamExtractor.java is part of NewPipe. * StreamExtractor.java is part of NewPipe.
@ -21,8 +21,8 @@ package org.schabi.newpipe.extractor.stream;
*/ */
import org.schabi.newpipe.extractor.Extractor; import org.schabi.newpipe.extractor.Extractor;
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.Subtitles;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
@ -34,7 +34,7 @@ import java.io.IOException;
import java.util.List; import java.util.List;
/** /**
* Scrapes information from a video streaming service (eg, YouTube). * Scrapes information from a video/audio streaming service (eg, YouTube).
*/ */
public abstract class StreamExtractor extends Extractor { public abstract class StreamExtractor extends Extractor {
@ -44,22 +44,235 @@ public abstract class StreamExtractor extends Extractor {
super(service, linkHandler, localization); super(service, linkHandler, localization);
} }
/**
* The day on which the stream got uploaded/created. The return information should be in the format
* dd.mm.yyyy, however it NewPipe will not crash if its not.
* @return The day on which the stream got uploaded.
* @throws ParsingException
*/
@Nonnull @Nonnull
public abstract String getUploadDate() throws ParsingException; public abstract String getUploadDate() throws ParsingException;
/**
* This will return the url to the thumbnail of the stream. Try to return the medium resolution here.
* @return The url of the thumbnail.
* @throws ParsingException
*/
@Nonnull @Nonnull
public abstract String getThumbnailUrl() throws ParsingException; public abstract String getThumbnailUrl() throws ParsingException;
/**
* This is the stream description. On YouTube this is the video description. You can return simple HTML here.
* @return The description of the stream/video.
* @throws ParsingException
*/
@Nonnull @Nonnull
public abstract String getDescription() throws ParsingException; public abstract String getDescription() throws ParsingException;
/** /**
* Get the age limit * Get the age limit.
* @return The age which limits the content or {@value NO_AGE_LIMIT} if there is no limit * @return The age which limits the content or {@value NO_AGE_LIMIT} if there is no limit
* @throws ParsingException if an error occurs while parsing * @throws ParsingException if an error occurs while parsing
*/ */
public abstract int getAgeLimit() throws ParsingException; public abstract int getAgeLimit() throws ParsingException;
/**
* This should return the length of a video in seconds.
* @return The length of the stream in seconds.
* @throws ParsingException
*/
public abstract long getLength() throws ParsingException; public abstract long getLength() throws ParsingException;
/**
* If the url you are currently handling contains a time stamp/seek, you can return the
* position it represents here.
* If the url has no time stamp simply return zero.
* @return the timestamp in seconds
* @throws ParsingException
*/
public abstract long getTimeStamp() throws ParsingException; public abstract long getTimeStamp() throws ParsingException;
/**
* The count of how many people have watched the video/listened to the audio stream.
* If the current stream has no view count or its not available simply return -1
* @return amount of views.
* @throws ParsingException
*/
public abstract long getViewCount() throws ParsingException;
/**
* The Amount of likes a video/audio stream got.
* If the current stream has no likes or its not available simply return -1
* @return the amount of likes the stream got
* @throws ParsingException
*/
public abstract long getLikeCount() throws ParsingException;
/**
* The Amount of dislikes a video/audio stream got.
* If the current stream has no dislikes or its not available simply return -1
* @return the amount of likes the stream got
* @throws ParsingException
*/
public abstract long getDislikeCount() throws ParsingException;
/**
* The Url to the page of the creator/uploader of the stream. This must not be a homepage,
* but the page offered by the service the extractor handles. This url will be handled by the
* <a href="https://teamnewpipe.github.io/documentation/03_Implement_a_service/#channel">ChannelExtractor</a>,
* so be sure to implement that one before you return a value here, otherwise NewPipe will crash if one selects
* this url.
* @return the url to the page of the creator/uploader of the stream or an empty String
* @throws ParsingException
*/
@Nonnull
public abstract String getUploaderUrl() throws ParsingException;
/**
* The name of the creator/uploader of the stream.
* If the name is not available you can simply return an empty string.
* @return the name of the creator/uploader of the stream or an empty String
* @throws ParsingException
*/
@Nonnull
public abstract String getUploaderName() throws ParsingException;
/**
* The url to the image file/profile picture/avatar of the creator/uploader of the stream.
* If the url is not available you can return an empty String.
* @return The url of the image file of the uploader or an empty String
* @throws ParsingException
*/
@Nonnull
public abstract String getUploaderAvatarUrl() throws ParsingException;
/**
* Get the dash mpd url. If you don't know what a dash MPD is you can read about it
* <a href="https://www.brendanlong.com/the-structure-of-an-mpeg-dash-mpd.html">here</a>.
* @return the url as a string or an empty string
* @throws ParsingException if an error occurs while reading
*/
@Nonnull public abstract String getDashMpdUrl() throws ParsingException;
/**
* I am not sure if this is in use, and how this is used. However the frontend is missing support
* for HLS streams. Prove me if I am wrong. Please open an
* <a href="https://github.com/teamnewpipe/newpipe/issues">issue</a>,
* or fix this description if you know whats up with this.
* @return The Url to the hls stream.
* @throws ParsingException
*/
@Nonnull public abstract String getHlsUrl() throws ParsingException;
/**
* This should return a list of available
* <a href="https://teamnewpipe.github.io/NewPipeExtractor/javadoc/org/schabi/newpipe/extractor/stream/AudioStream.html">AudioStream</a>s
* You can also return null or an empty list, however be aware that if you don't return anything
* in getVideoStreams(), getVideoOnlyStreams() and getDashMpdUrl() either the Collector will handle this as
* a failed extraction procedure.
* @return a list of audio only streams in the format of AudioStream
* @throws IOException
* @throws ExtractionException
*/
public abstract List<AudioStream> getAudioStreams() throws IOException, ExtractionException;
/**
* This should return a list of available
* <a href="https://teamnewpipe.github.io/NewPipeExtractor/javadoc/org/schabi/newpipe/extractor/stream/VideoStream.html">VideoStream</a>s
* Be aware this is the list of video streams which do contain an audio stream.
* You can also return null or an empty list, however be aware that if you don't return anything
* in getAudioStreams(), getVideoOnlyStreams() and getDashMpdUrl() either the Collector will handle this as
* a failed extraction procedure.
* @return a list of combined video and streams in the format of AudioStream
* @throws IOException
* @throws ExtractionException
*/
public abstract List<VideoStream> getVideoStreams() throws IOException, ExtractionException;
/**
* This should return a list of available
* <a href="https://teamnewpipe.github.io/NewPipeExtractor/javadoc/org/schabi/newpipe/extractor/stream/VideoStream.html">VideoStream</a>s.
* Be aware this is the list of video streams which do NOT contain an audio stream.
* You can also return null or an empty list, however be aware that if you don't return anything
* in getAudioStreams(), getVideoStreams() and getDashMpdUrl() either the Collector will handle this as
* a failed extraction procedure.
* @return a list of video and streams in the format of AudioStream
* @throws IOException
* @throws ExtractionException
*/
public abstract List<VideoStream> getVideoOnlyStreams() throws IOException, ExtractionException;
/**
* This will return a list of available
* <a href="https://teamnewpipe.github.io/NewPipeExtractor/javadoc/org/schabi/newpipe/extractor/stream/Subtitles.html">Subtitles</a>s.
* If no subtitles are available an empty list can returned.
* @return a list of available subtitles or an empty list
* @throws IOException
* @throws ExtractionException
*/
@Nonnull
public abstract List<SubtitlesStream> getSubtitlesDefault() throws IOException, ExtractionException;
/**
* This will return a list of available
* <a href="https://teamnewpipe.github.io/NewPipeExtractor/javadoc/org/schabi/newpipe/extractor/stream/Subtitles.html">Subtitles</a>s.
* given by a specific type.
* If no subtitles in that specific format are available an empty list can returned.
* @param format the media format by which the subtitles should be filtered
* @return a list of available subtitles or an empty list
* @throws IOException
* @throws ExtractionException
*/
@Nonnull
public abstract List<SubtitlesStream> getSubtitles(MediaFormat format) throws IOException, ExtractionException;
/**
* Get the <a href="https://teamnewpipe.github.io/NewPipeExtractor/javadoc/">StreamType</a>.
* @return the type of the stream
* @throws ParsingException
*/
public abstract StreamType getStreamType() throws ParsingException;
/**
* should return the url of the next stream. NewPipe will automatically play
* the next stream if the user wants that.
* If the next stream is is not available simply return null
* @return the InfoItem of the next stream
* @throws IOException
* @throws ExtractionException
*/
public abstract StreamInfoItem getNextStream() throws IOException, ExtractionException;
/**
* Should return a list of streams related to the current handled. Many services show suggested
* streams. If you don't like suggested streams you should implement them anyway since they can
* be disabled by the user later in the frontend.
* This list MUST NOT contain the next available video as this should be return through getNextStream()
* If is is not available simply return null
* @return a list of InfoItems showing the related videos/streams
* @throws IOException
* @throws ExtractionException
*/
public abstract StreamInfoItemsCollector getRelatedStreams() throws IOException, ExtractionException;
/**
* Should analyse the webpage's document and extracts any error message there might be. (e.g. GEMA block)
*
* @return Error message; null if there is no error message.
*/
public abstract String getErrorMessage();
//////////////////////////////////////////////////////////////////
/// Helper
//////////////////////////////////////////////////////////////////
/**
* Override this function if the format of time stamp in the url is not the same format as that form youtube.
* Honestly I don't even know the time stamp fromat of youtube.
* @param regexPattern
* @return the sime stamp/seek for the video in seconds
* @throws ParsingException
*/
protected long getTimestampSeconds(String regexPattern) throws ParsingException { protected long getTimestampSeconds(String regexPattern) throws ParsingException {
String timeStamp; String timeStamp;
try { try {
@ -103,43 +316,6 @@ public abstract class StreamExtractor extends Extractor {
} }
} else { } else {
return 0; return 0;
}}; }
}
public abstract long getViewCount() throws ParsingException;
public abstract long getLikeCount() throws ParsingException;
public abstract long getDislikeCount() throws ParsingException;
@Nonnull
public abstract String getUploaderUrl() throws ParsingException;
@Nonnull
public abstract String getUploaderName() throws ParsingException;
@Nonnull
public abstract String getUploaderAvatarUrl() throws ParsingException;
/**
* Get the dash mpd url
* @return the url as a string or an empty string
* @throws ParsingException if an error occurs while reading
*/
@Nonnull public abstract String getDashMpdUrl() throws ParsingException;
@Nonnull public abstract String getHlsUrl() throws ParsingException;
public abstract List<AudioStream> getAudioStreams() throws IOException, ExtractionException;
public abstract List<VideoStream> getVideoStreams() throws IOException, ExtractionException;
public abstract List<VideoStream> getVideoOnlyStreams() throws IOException, ExtractionException;
@Nonnull
public abstract List<Subtitles> getSubtitlesDefault() throws IOException, ExtractionException;
@Nonnull
public abstract List<Subtitles> getSubtitles(SubtitlesFormat format) throws IOException, ExtractionException;
public abstract StreamType getStreamType() throws ParsingException;
public abstract StreamInfoItem getNextVideo() throws IOException, ExtractionException;
public abstract StreamInfoItemsCollector getRelatedVideos() throws IOException, ExtractionException;
/**
* Analyses the webpage's document and extracts any error message there might be.
*
* @return Error message; null if there is no error message.
*/
public abstract String getErrorMessage();
} }

43
extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java
View File

@ -8,13 +8,10 @@ import org.schabi.newpipe.extractor.Info;
import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.Subtitles;
import org.schabi.newpipe.extractor.comments.CommentsInfo;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.utils.DashMpdParser; import org.schabi.newpipe.extractor.utils.DashMpdParser;
import org.schabi.newpipe.extractor.utils.ExtractorHelper; import org.schabi.newpipe.extractor.utils.ExtractorHelper;
import org.schabi.newpipe.extractor.utils.Localization;
/* /*
* Created by Christian Schabesberger on 26.08.15. * Created by Christian Schabesberger on 26.08.15.
@ -163,6 +160,9 @@ public class StreamInfo extends Info {
streamInfo.getVideoOnlyStreams().addAll(result.getVideoOnlyStreams()); streamInfo.getVideoOnlyStreams().addAll(result.getVideoOnlyStreams());
streamInfo.getAudioStreams().addAll(result.getAudioStreams()); streamInfo.getAudioStreams().addAll(result.getAudioStreams());
streamInfo.getVideoStreams().addAll(result.getVideoStreams()); streamInfo.getVideoStreams().addAll(result.getVideoStreams());
streamInfo.segmentedVideoOnlyStreams = result.getSegmentedVideoOnlyStreams();
streamInfo.segmentedAudioStreams = result.getSegmentedAudioStreams();
streamInfo.segmentedVideoStreams = result.getSegmentedVideoStreams();
} catch (Exception e) { } catch (Exception e) {
// Sometimes we receive 403 (forbidden) error when trying to download the // Sometimes we receive 403 (forbidden) error when trying to download the
// manifest (similar to what happens with youtube-dl), // manifest (similar to what happens with youtube-dl),
@ -254,7 +254,7 @@ public class StreamInfo extends Info {
streamInfo.addError(e); streamInfo.addError(e);
} }
try { try {
streamInfo.setNextVideo(extractor.getNextVideo()); streamInfo.setNextVideo(extractor.getNextStream());
} catch (Exception e) { } catch (Exception e) {
streamInfo.addError(e); streamInfo.addError(e);
} }
@ -289,12 +289,17 @@ public class StreamInfo extends Info {
private List<VideoStream> videoOnlyStreams; private List<VideoStream> videoOnlyStreams;
private String dashMpdUrl; private String dashMpdUrl;
private List<VideoStream> segmentedVideoStreams;
private List<AudioStream> segmentedAudioStreams;
private List<VideoStream> segmentedVideoOnlyStreams;
private String hlsUrl; private String hlsUrl;
private StreamInfoItem nextVideo; private StreamInfoItem nextVideo;
private List<InfoItem> relatedStreams; private List<InfoItem> relatedStreams;
private long startPosition = 0; private long startPosition = 0;
private List<Subtitles> subtitles; private List<SubtitlesStream> subtitles;
/** /**
* Get the stream type * Get the stream type
@ -449,6 +454,30 @@ public class StreamInfo extends Info {
this.dashMpdUrl = dashMpdUrl; this.dashMpdUrl = dashMpdUrl;
} }
public List<VideoStream> getSegmentedVideoStreams() {
return segmentedVideoStreams;
}
public void setSegmentedVideoStreams(List<VideoStream> segmentedVideoStreams) {
this.segmentedVideoStreams = segmentedVideoStreams;
}
public List<AudioStream> getSegmentedAudioStreams() {
return segmentedAudioStreams;
}
public void setSegmentedAudioStreams(List<AudioStream> segmentedAudioStreams) {
this.segmentedAudioStreams = segmentedAudioStreams;
}
public List<VideoStream> getSegmentedVideoOnlyStreams() {
return segmentedVideoOnlyStreams;
}
public void setSegmentedVideoOnlyStreams(List<VideoStream> segmentedVideoOnlyStreams) {
this.segmentedVideoOnlyStreams = segmentedVideoOnlyStreams;
}
public String getHlsUrl() { public String getHlsUrl() {
return hlsUrl; return hlsUrl;
} }
@ -481,11 +510,11 @@ public class StreamInfo extends Info {
this.startPosition = startPosition; this.startPosition = startPosition;
} }
public List<Subtitles> getSubtitles() { public List<SubtitlesStream> getSubtitles() {
return subtitles; return subtitles;
} }
public void setSubtitles(List<Subtitles> subtitles) { public void setSubtitles(List<SubtitlesStream> subtitles) {
this.subtitles = subtitles; this.subtitles = subtitles;
} }

25
extractor/src/main/java/org/schabi/newpipe/extractor/stream/SubtitlesFormat.java
View File

@ -1,25 +0,0 @@
package org.schabi.newpipe.extractor.stream;
public enum SubtitlesFormat {
// YouTube subtitles formats
// TRANSCRIPT(3) is default YT format based on TTML,
// but unlike VTT or TTML, it is NOT W3 standard
// TRANSCRIPT subtitles are NOT supported by ExoPlayer, only VTT and TTML
VTT (0x0, "vtt"),
TTML (0x1, "ttml"),
TRANSCRIPT1 (0x2, "srv1"),
TRANSCRIPT2 (0x3, "srv2"),
TRANSCRIPT3 (0x4, "srv3");
private final int id;
private final String extension;
SubtitlesFormat(int id, String extension) {
this.id = id;
this.extension = extension;
}
public String getExtension() {
return extension;
}
}

73
extractor/src/main/java/org/schabi/newpipe/extractor/stream/SubtitlesStream.java Normal file
View File

@ -0,0 +1,73 @@
package org.schabi.newpipe.extractor.stream;
import org.schabi.newpipe.extractor.MediaFormat;
import java.io.Serializable;
import java.util.Locale;
public class SubtitlesStream extends Stream implements Serializable {
private final MediaFormat format;
private final Locale locale;
private final String url;
private final boolean autoGenerated;
private final String code;
public SubtitlesStream(MediaFormat format, String languageCode, String url, boolean autoGenerated) {
super(url, format);
/*
* Locale.forLanguageTag only for API >= 21
* Locale.Builder only for API >= 21
* Country codes doesn't work well without
*/
final String[] splits = languageCode.split("-");
switch (splits.length) {
default:
this.locale = new Locale(splits[0]);
break;
case 3:
this.locale = new Locale(splits[0], splits[1], splits[2]);// complex variants doesn't work!
break;
case 2:
this.locale = new Locale(splits[0], splits[1]);
break;
}
this.code = languageCode;
this.format = format;
this.url = url;
this.autoGenerated = autoGenerated;
}
public String getExtension() {
return format.suffix;
}
public String getURL() {
return url;
}
public boolean isAutoGenerated() {
return autoGenerated;
}
@Override
public boolean equalStats(Stream cmp) {
return super.equalStats(cmp)&&
cmp instanceof SubtitlesStream &&
code.equals(((SubtitlesStream) cmp).code) &&
autoGenerated == ((SubtitlesStream) cmp).autoGenerated;
}
public String getDisplayLanguageName() {
return locale.getDisplayName(locale);
}
public String getLanguageTag() {
return code;
}
public Locale getLocale() {
return locale;
}
}

91
extractor/src/main/java/org/schabi/newpipe/extractor/utils/DashMpdParser.java
View File

@ -59,10 +59,23 @@ public class DashMpdParser {
private final List<AudioStream> audioStreams; private final List<AudioStream> audioStreams;
private final List<VideoStream> videoOnlyStreams; private final List<VideoStream> videoOnlyStreams;
public ParserResult(List<VideoStream> videoStreams, List<AudioStream> audioStreams, List<VideoStream> videoOnlyStreams) { private final List<VideoStream> segmentedVideoStreams;
private final List<AudioStream> segmentedAudioStreams;
private final List<VideoStream> segmentedVideoOnlyStreams;
public ParserResult(List<VideoStream> videoStreams,
List<AudioStream> audioStreams,
List<VideoStream> videoOnlyStreams,
List<VideoStream> segmentedVideoStreams,
List<AudioStream> segmentedAudioStreams,
List<VideoStream> segmentedVideoOnlyStreams) {
this.videoStreams = videoStreams; this.videoStreams = videoStreams;
this.audioStreams = audioStreams; this.audioStreams = audioStreams;
this.videoOnlyStreams = videoOnlyStreams; this.videoOnlyStreams = videoOnlyStreams;
this.segmentedVideoStreams = segmentedVideoStreams;
this.segmentedAudioStreams = segmentedAudioStreams;
this.segmentedVideoOnlyStreams = segmentedVideoOnlyStreams;
} }
public List<VideoStream> getVideoStreams() { public List<VideoStream> getVideoStreams() {
@ -76,10 +89,22 @@ public class DashMpdParser {
public List<VideoStream> getVideoOnlyStreams() { public List<VideoStream> getVideoOnlyStreams() {
return videoOnlyStreams; return videoOnlyStreams;
} }
public List<VideoStream> getSegmentedVideoStreams() {
return segmentedVideoStreams;
}
public List<AudioStream> getSegmentedAudioStreams() {
return segmentedAudioStreams;
}
public List<VideoStream> getSegmentedVideoOnlyStreams() {
return segmentedVideoOnlyStreams;
}
} }
/** /**
* Will try to download (using {@link StreamInfo#dashMpdUrl}) and parse the dash manifest, * Will try to download (using {@link StreamInfo#getDashMpdUrl()}) and parse the dash manifest,
* then it will search for any stream that the ItagItem has (by the id). * then it will search for any stream that the ItagItem has (by the id).
* <p> * <p>
* It has video, video only and audio streams and will only add to the list if it don't * It has video, video only and audio streams and will only add to the list if it don't
@ -90,7 +115,8 @@ public class DashMpdParser {
* *
* @param streamInfo where the parsed streams will be added * @param streamInfo where the parsed streams will be added
*/ */
public static ParserResult getStreams(final StreamInfo streamInfo) throws DashMpdParsingException, ReCaptchaException { public static ParserResult getStreams(final StreamInfo streamInfo)
throws DashMpdParsingException, ReCaptchaException {
String dashDoc; String dashDoc;
Downloader downloader = NewPipe.getDownloader(); Downloader downloader = NewPipe.getDownloader();
try { try {
@ -113,6 +139,10 @@ public class DashMpdParser {
final List<AudioStream> audioStreams = new ArrayList<>(); final List<AudioStream> audioStreams = new ArrayList<>();
final List<VideoStream> videoOnlyStreams = new ArrayList<>(); final List<VideoStream> videoOnlyStreams = new ArrayList<>();
final List<VideoStream> segmentedVideoStreams = new ArrayList<>();
final List<AudioStream> segmentedAudioStreams = new ArrayList<>();
final List<VideoStream> segmentedVideoOnlyStreams = new ArrayList<>();
for (int i = 0; i < representationList.getLength(); i++) { for (int i = 0; i < representationList.getLength(); i++) {
final Element representation = (Element) representationList.item(i); final Element representation = (Element) representationList.item(i);
try { try {
@ -126,34 +156,61 @@ public class DashMpdParser {
// instead we need to add the "media=" value from the <SegementURL/> tags inside the <SegmentList/> // instead we need to add the "media=" value from the <SegementURL/> tags inside the <SegmentList/>
// tag in order to get a full working url. However each of these is just pointing to a part of the // tag in order to get a full working url. However each of these is just pointing to a part of the
// video, so we can not return a URL with a working stream here. // video, so we can not return a URL with a working stream here.
// We decided not to ignore such streams for the moment. // Instead of putting those streams into the list of regular stream urls wie put them in a
if (itag != null && segmentationList == null) { // for example "segmentedVideoStreams" list.
if (itag != null) {
final MediaFormat mediaFormat = MediaFormat.getFromMimeType(mimeType); final MediaFormat mediaFormat = MediaFormat.getFromMimeType(mimeType);
if (itag.itagType.equals(ItagItem.ItagType.AUDIO)) { if (itag.itagType.equals(ItagItem.ItagType.AUDIO)) {
final AudioStream audioStream = new AudioStream(url, mediaFormat, itag.avgBitrate); if(segmentationList == null) {
final AudioStream audioStream = new AudioStream(url, mediaFormat, itag.avgBitrate);
if (!Stream.containSimilarStream(audioStream, streamInfo.getAudioStreams())) { if (!Stream.containSimilarStream(audioStream, streamInfo.getAudioStreams())) {
audioStreams.add(audioStream); audioStreams.add(audioStream);
}
} else {
segmentedAudioStreams.add(
new AudioStream(id, mediaFormat, itag.avgBitrate));
} }
} else { } else {
boolean isVideoOnly = itag.itagType.equals(ItagItem.ItagType.VIDEO_ONLY); boolean isVideoOnly = itag.itagType.equals(ItagItem.ItagType.VIDEO_ONLY);
final VideoStream videoStream = new VideoStream(url, mediaFormat, itag.resolutionString, isVideoOnly);
if (isVideoOnly) { if(segmentationList == null) {
if (!Stream.containSimilarStream(videoStream, streamInfo.getVideoOnlyStreams())) { final VideoStream videoStream = new VideoStream(url,
streamInfo.getVideoOnlyStreams().add(videoStream); mediaFormat,
videoOnlyStreams.add(videoStream); itag.resolutionString,
isVideoOnly);
if (isVideoOnly) {
if (!Stream.containSimilarStream(videoStream, streamInfo.getVideoOnlyStreams())) {
videoOnlyStreams.add(videoStream);
}
} else if (!Stream.containSimilarStream(videoStream, streamInfo.getVideoStreams())) {
videoStreams.add(videoStream);
}
} else {
final VideoStream videoStream = new VideoStream(id,
mediaFormat,
itag.resolutionString,
isVideoOnly);
if(isVideoOnly) {
segmentedVideoOnlyStreams.add(videoStream);
} else {
segmentedVideoStreams.add(videoStream);
} }
} else if (!Stream.containSimilarStream(videoStream, streamInfo.getVideoStreams())) {
videoStreams.add(videoStream);
} }
} }
} }
} catch (Exception ignored) { } catch (Exception ignored) {
} }
} }
return new ParserResult(videoStreams, audioStreams, videoOnlyStreams); return new ParserResult(
videoStreams,
audioStreams,
videoOnlyStreams,
segmentedVideoStreams,
segmentedAudioStreams,
segmentedVideoOnlyStreams);
} catch (Exception e) { } catch (Exception e) {
throw new DashMpdParsingException("Could not parse Dash mpd", e); throw new DashMpdParsingException("Could not parse Dash mpd", e);
} }

2
extractor/src/main/java/org/schabi/newpipe/extractor/utils/ExtractorHelper.java
View File

@ -30,7 +30,7 @@ public class ExtractorHelper {
public static List<InfoItem> getRelatedVideosOrLogError(StreamInfo info, StreamExtractor extractor) { public static List<InfoItem> getRelatedVideosOrLogError(StreamInfo info, StreamExtractor extractor) {
try { try {
InfoItemsCollector<? extends InfoItem, ?> collector = extractor.getRelatedVideos(); InfoItemsCollector<? extends InfoItem, ?> collector = extractor.getRelatedStreams();
info.addAllErrors(collector.getErrors()); info.addAllErrors(collector.getErrors());
//noinspection unchecked //noinspection unchecked

23
extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java
View File

@ -32,7 +32,7 @@ public class JsonUtils {
public static String getString(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{ public static String getString(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{
Object value = getValue(object, path); Object value = getValue(object, path);
if(value instanceof String) { if(value instanceof String) {
return (String) getValue(object, path); return (String) value;
}else { }else {
throw new ParsingException("Unable to get " + path); throw new ParsingException("Unable to get " + path);
} }
@ -42,12 +42,31 @@ public class JsonUtils {
public static Number getNumber(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{ public static Number getNumber(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{
Object value = getValue(object, path); Object value = getValue(object, path);
if(value instanceof Number) { if(value instanceof Number) {
return (Number) getValue(object, path); return (Number) value;
}else { }else {
throw new ParsingException("Unable to get " + path); throw new ParsingException("Unable to get " + path);
} }
} }
@Nonnull
public static JsonObject getObject(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{
Object value = getValue(object, path);
if(value instanceof JsonObject) {
return (JsonObject) value;
}else {
throw new ParsingException("Unable to get " + path);
}
}
@Nonnull
public static JsonArray getArray(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{
Object value = getValue(object, path);
if(value instanceof JsonArray) {
return (JsonArray) value;
}else {
throw new ParsingException("Unable to get " + path);
}
}
@Nonnull @Nonnull
public static List<Object> getValues(@Nonnull JsonArray array, @Nonnull String path) throws ParsingException { public static List<Object> getValues(@Nonnull JsonArray array, @Nonnull String path) throws ParsingException {

22
extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java
View File

@ -1,10 +1,5 @@
package org.schabi.newpipe.extractor.utils; package org.schabi.newpipe.extractor.utils;
import org.nibor.autolink.LinkExtractor;
import org.nibor.autolink.LinkSpan;
import org.nibor.autolink.LinkType;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import java.io.UnsupportedEncodingException; import java.io.UnsupportedEncodingException;
import java.net.URLDecoder; import java.net.URLDecoder;
import java.util.ArrayList; import java.util.ArrayList;
@ -14,6 +9,11 @@ import java.util.Map;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import org.nibor.autolink.LinkExtractor;
import org.nibor.autolink.LinkSpan;
import org.nibor.autolink.LinkType;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
/* /*
* Created by Christian Schabesberger on 02.02.16. * Created by Christian Schabesberger on 02.02.16.
* *
@ -52,17 +52,25 @@ public class Parser {
return matchGroup(pattern, input, 1); return matchGroup(pattern, input, 1);
} }
public static String matchGroup1(Pattern pattern, String input) throws RegexException {
return matchGroup(pattern, input, 1);
}
public static String matchGroup(String pattern, String input, int group) throws RegexException { public static String matchGroup(String pattern, String input, int group) throws RegexException {
Pattern pat = Pattern.compile(pattern); Pattern pat = Pattern.compile(pattern);
return matchGroup(pat, input, group);
}
public static String matchGroup(Pattern pat, String input, int group) throws RegexException {
Matcher mat = pat.matcher(input); Matcher mat = pat.matcher(input);
boolean foundMatch = mat.find(); boolean foundMatch = mat.find();
if (foundMatch) { if (foundMatch) {
return mat.group(group); return mat.group(group);
} else { } else {
if (input.length() > 1024) { if (input.length() > 1024) {
throw new RegexException("failed to find pattern \"" + pattern); throw new RegexException("failed to find pattern \"" + pat.pattern());
} else { } else {
throw new RegexException("failed to find pattern \"" + pattern + " inside of " + input + "\""); throw new RegexException("failed to find pattern \"" + pat.pattern() + " inside of " + input + "\"");
} }
} }
} }

30
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java
View File

@ -1,5 +1,13 @@
package org.schabi.newpipe.extractor.services.peertube; package org.schabi.newpipe.extractor.services.peertube;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
import java.io.IOException;
import org.junit.BeforeClass; import org.junit.BeforeClass;
import org.junit.Test; import org.junit.Test;
import org.schabi.newpipe.Downloader; import org.schabi.newpipe.Downloader;
@ -12,12 +20,6 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamType; import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.utils.Localization; import org.schabi.newpipe.extractor.utils.Localization;
import java.io.IOException;
import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
/** /**
* Test for {@link StreamExtractor} * Test for {@link StreamExtractor}
*/ */
@ -27,7 +29,7 @@ public class PeertubeStreamExtractorDefaultTest {
@BeforeClass @BeforeClass
public static void setUp() throws Exception { public static void setUp() throws Exception {
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en")); NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
extractor = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://peertube.mastodon.host/videos/watch/86fe4f24-64c3-4ab4-9e7e-66177219ed21"); extractor = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://peertube.mastodon.host/videos/watch/04af977f-4201-4697-be67-a8d8cae6fa7a");
extractor.fetchPage(); extractor.fetchPage();
} }
@ -39,22 +41,22 @@ public class PeertubeStreamExtractorDefaultTest {
@Test @Test
public void testGetTitle() throws ParsingException { public void testGetTitle() throws ParsingException {
assertEquals(extractor.getName(), "Pažadėtoji 1 Sezonas 1050 Serija - Ziuri.me"); assertEquals(extractor.getName(), "The Internet's Own Boy");
} }
@Test @Test
public void testGetDescription() throws ParsingException { public void testGetDescription() throws ParsingException {
assertEquals(extractor.getDescription(), "Serialo veiksmas vyksta Radžastane. kur vis dar gyvos liaudies tradicijos. o jo centre - Anandi gimusi varguolių šeimoje. Mergaitė privalės ištekėti už turtingo paveldėtojo vos tik sulaukusi aštuonerių metų ir priprasti prie naujojo nuotakos vaidm..."); assertEquals(extractor.getDescription(), "The story of programming prodigy and information activist Aaron Swartz, who took his own life at the age of 26.");
} }
@Test @Test
public void testGetUploaderName() throws ParsingException { public void testGetUploaderName() throws ParsingException {
assertEquals(extractor.getUploaderName(), "djsets"); assertEquals(extractor.getUploaderName(), "root");
} }
@Test @Test
public void testGetLength() throws ParsingException { public void testGetLength() throws ParsingException {
assertEquals(extractor.getLength(), 1238); assertEquals(extractor.getLength(), 6299);
} }
@Test @Test
@ -65,13 +67,13 @@ public class PeertubeStreamExtractorDefaultTest {
@Test @Test
public void testGetUploadDate() throws ParsingException { public void testGetUploadDate() throws ParsingException {
assertEquals("2018-10-06", extractor.getUploadDate()); assertEquals("2017-10-17", extractor.getUploadDate());
} }
@Test @Test
public void testGetUploaderUrl() throws ParsingException { public void testGetUploaderUrl() throws ParsingException {
assertIsSecureUrl(extractor.getUploaderUrl()); assertIsSecureUrl(extractor.getUploaderUrl());
assertEquals("https://peertube.mastodon.host/api/v1/accounts/djsets@hostyour.tv", extractor.getUploaderUrl()); assertEquals("https://peertube.mastodon.host/api/v1/accounts/root@peertube2.cpy.re", extractor.getUploaderUrl());
} }
@Test @Test
@ -96,7 +98,7 @@ public class PeertubeStreamExtractorDefaultTest {
@Test @Test
public void testGetRelatedVideos() throws ExtractionException, IOException { public void testGetRelatedVideos() throws ExtractionException, IOException {
StreamInfoItemsCollector relatedVideos = extractor.getRelatedVideos(); StreamInfoItemsCollector relatedVideos = extractor.getRelatedStreams();
assertFalse(relatedVideos.getItems().isEmpty()); assertFalse(relatedVideos.getItems().isEmpty());
assertTrue(relatedVideos.getErrors().isEmpty()); assertTrue(relatedVideos.getErrors().isEmpty());
} }

2
extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java
View File

@ -101,7 +101,7 @@ public class SoundcloudStreamExtractorDefaultTest {
@Test @Test
public void testGetRelatedVideos() throws ExtractionException, IOException { public void testGetRelatedVideos() throws ExtractionException, IOException {
StreamInfoItemsCollector relatedVideos = extractor.getRelatedVideos(); StreamInfoItemsCollector relatedVideos = extractor.getRelatedStreams();
assertFalse(relatedVideos.getItems().isEmpty()); assertFalse(relatedVideos.getItems().isEmpty());
assertTrue(relatedVideos.getErrors().isEmpty()); assertTrue(relatedVideos.getErrors().isEmpty());
} }

2
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java
View File

@ -219,7 +219,7 @@ public class YoutubePlaylistExtractorTest {
@Test @Test
public void testUploaderName() throws Exception { public void testUploaderName() throws Exception {
assertEquals("Tomas Nilsson", extractor.getUploaderName()); assertEquals("Tomas Nilsson TOMPA571", extractor.getUploaderName());
} }
@Test @Test

4
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorAgeRestrictedTest.java
View File

@ -4,13 +4,13 @@ import org.junit.BeforeClass;
import org.junit.Ignore; import org.junit.Ignore;
import org.junit.Test; import org.junit.Test;
import org.schabi.newpipe.Downloader; import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.SubtitlesFormat;
import org.schabi.newpipe.extractor.stream.VideoStream; import org.schabi.newpipe.extractor.stream.VideoStream;
import org.schabi.newpipe.extractor.utils.Localization; import org.schabi.newpipe.extractor.utils.Localization;
@ -131,6 +131,6 @@ public class YoutubeStreamExtractorAgeRestrictedTest {
@Test @Test
public void testGetSubtitlesList() throws IOException, ExtractionException { public void testGetSubtitlesList() throws IOException, ExtractionException {
// Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null
assertTrue(extractor.getSubtitles(SubtitlesFormat.TTML).isEmpty()); assertTrue(extractor.getSubtitles(MediaFormat.TTML).isEmpty());
} }
} }

4
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorControversialTest.java
View File

@ -4,13 +4,13 @@ import org.junit.BeforeClass;
import org.junit.Ignore; import org.junit.Ignore;
import org.junit.Test; import org.junit.Test;
import org.schabi.newpipe.Downloader; import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.SubtitlesFormat;
import org.schabi.newpipe.extractor.stream.VideoStream; import org.schabi.newpipe.extractor.stream.VideoStream;
import org.schabi.newpipe.extractor.utils.Localization; import org.schabi.newpipe.extractor.utils.Localization;
@ -124,6 +124,6 @@ public class YoutubeStreamExtractorControversialTest {
@Test @Test
public void testGetSubtitlesList() throws IOException, ExtractionException { public void testGetSubtitlesList() throws IOException, ExtractionException {
// Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null
assertTrue(!extractor.getSubtitles(SubtitlesFormat.TTML).isEmpty()); assertTrue(!extractor.getSubtitles(MediaFormat.TTML).isEmpty());
} }
} }

10
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDASHTest.java
View File

@ -47,15 +47,21 @@ public class YoutubeStreamExtractorDASHTest {
@Test @Test
public void testGetDashMpd() { public void testGetDashMpd() {
System.out.println(info.getDashMpdUrl());
assertTrue(info.getDashMpdUrl(), assertTrue(info.getDashMpdUrl(),
info.getDashMpdUrl() != null && !info.getDashMpdUrl().isEmpty()); info.getDashMpdUrl() != null && !info.getDashMpdUrl().isEmpty());
} }
@Test @Test
public void testDashMpdParser() { public void testRegularStreams() {
assertEquals(0, info.getAudioStreams().size()); assertEquals(0, info.getAudioStreams().size());
assertEquals(0, info.getVideoOnlyStreams().size()); assertEquals(0, info.getVideoOnlyStreams().size());
assertEquals(4, info.getVideoStreams().size()); assertEquals(4, info.getVideoStreams().size());
} }
@Test
public void testSegmentedStreams() {
assertEquals(2, info.getSegmentedAudioStreams().size());
assertEquals(3, info.getSegmentedVideoOnlyStreams().size());
assertEquals(0, info.getSegmentedVideoStreams().size());
}
} }

9
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java
View File

@ -3,12 +3,12 @@ package org.schabi.newpipe.extractor.services.youtube;
import org.junit.BeforeClass; import org.junit.BeforeClass;
import org.junit.Test; import org.junit.Test;
import org.schabi.newpipe.Downloader; import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
import org.schabi.newpipe.extractor.stream.*; import org.schabi.newpipe.extractor.stream.*;
import org.schabi.newpipe.extractor.utils.DashMpdParser;
import org.schabi.newpipe.extractor.utils.Localization; import org.schabi.newpipe.extractor.utils.Localization;
import org.schabi.newpipe.extractor.utils.Utils; import org.schabi.newpipe.extractor.utils.Utils;
@ -17,7 +17,6 @@ import java.io.IOException;
import static org.junit.Assert.*; import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
import static org.schabi.newpipe.extractor.ServiceList.YouTube; import static org.schabi.newpipe.extractor.ServiceList.YouTube;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeTrendingExtractorTest.extractor;
/* /*
* Created by Christian Schabesberger on 30.12.15. * Created by Christian Schabesberger on 30.12.15.
@ -151,7 +150,7 @@ public class YoutubeStreamExtractorDefaultTest {
@Test @Test
public void testGetRelatedVideos() throws ExtractionException, IOException { public void testGetRelatedVideos() throws ExtractionException, IOException {
StreamInfoItemsCollector relatedVideos = extractor.getRelatedVideos(); StreamInfoItemsCollector relatedVideos = extractor.getRelatedStreams();
Utils.printErrors(relatedVideos.getErrors()); Utils.printErrors(relatedVideos.getErrors());
assertFalse(relatedVideos.getItems().isEmpty()); assertFalse(relatedVideos.getItems().isEmpty());
assertTrue(relatedVideos.getErrors().isEmpty()); assertTrue(relatedVideos.getErrors().isEmpty());
@ -166,7 +165,7 @@ public class YoutubeStreamExtractorDefaultTest {
@Test @Test
public void testGetSubtitlesList() throws IOException, ExtractionException { public void testGetSubtitlesList() throws IOException, ExtractionException {
// Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null
assertTrue(extractor.getSubtitles(SubtitlesFormat.TTML).isEmpty()); assertTrue(extractor.getSubtitles(MediaFormat.TTML).isEmpty());
} }
} }
@ -177,7 +176,7 @@ public class YoutubeStreamExtractorDefaultTest {
public static void setUp() throws Exception { public static void setUp() throws Exception {
NewPipe.init(Downloader.getInstance(), new Localization("GB", "en")); NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
extractor = (YoutubeStreamExtractor) YouTube extractor = (YoutubeStreamExtractor) YouTube
.getStreamExtractor("https://www.youtube.com/watch?v=LzR8Sf5PK2Q"); .getStreamExtractor("https://www.youtube.com/watch?v=fBc4Q_htqPg");
extractor.fetchPage(); extractor.fetchPage();
} }

1
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractorTest.java
View File

@ -51,7 +51,6 @@ public class YoutubeTrendingExtractorTest {
.getKioskList() .getKioskList()
.getExtractorById("Trending", null); .getExtractorById("Trending", null);
extractor.fetchPage(); extractor.fetchPage();
extractor.setContentCountry("de");
} }
@Test @Test

2
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingKioskInfoTest.java
View File

@ -46,7 +46,7 @@ public class YoutubeTrendingKioskInfoTest {
StreamingService service = YouTube; StreamingService service = YouTube;
LinkHandlerFactory LinkHandlerFactory = service.getKioskList().getListLinkHandlerFactoryByType("Trending"); LinkHandlerFactory LinkHandlerFactory = service.getKioskList().getListLinkHandlerFactoryByType("Trending");
kioskInfo = KioskInfo.getInfo(YouTube, LinkHandlerFactory.fromId("Trending").getUrl(), null); kioskInfo = KioskInfo.getInfo(YouTube, LinkHandlerFactory.fromId("Trending").getUrl());
} }
@Test @Test

2
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchCountTest.java
View File

@ -28,7 +28,7 @@ public class YoutubeSearchCountTest {
public void testViewCount() { public void testViewCount() {
ChannelInfoItem ci = (ChannelInfoItem) itemsPage.getItems().get(0); ChannelInfoItem ci = (ChannelInfoItem) itemsPage.getItems().get(0);
assertTrue("Count does not fit: " + Long.toString(ci.getSubscriberCount()), assertTrue("Count does not fit: " + Long.toString(ci.getSubscriberCount()),
65043316 < ci.getSubscriberCount() && ci.getSubscriberCount() < 68043316); 69043316 < ci.getSubscriberCount() && ci.getSubscriberCount() < 73043316);
} }
} }
} }

4
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java
View File

@ -45,12 +45,12 @@ public class YoutubeSearchExtractorChannelOnlyTest extends YoutubeSearchExtracto
} }
assertFalse("First and second page are equal", equals); assertFalse("First and second page are equal", equals);
assertEquals("https://www.youtube.com/results?q=pewdiepie&sp=EgIQAlAU&page=3", secondPage.getNextPageUrl()); assertEquals("https://www.youtube.com/results?q=pewdiepie&sp=EgIQAlAU&gl=GB&page=3", secondPage.getNextPageUrl());
} }
@Test @Test
public void testGetSecondPageUrl() throws Exception { public void testGetSecondPageUrl() throws Exception {
assertEquals("https://www.youtube.com/results?q=pewdiepie&sp=EgIQAlAU&page=2", extractor.getNextPageUrl()); assertEquals("https://www.youtube.com/results?q=pewdiepie&sp=EgIQAlAU&gl=GB&page=2", extractor.getNextPageUrl());
} }
@Test @Test

12
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java
View File

@ -49,18 +49,22 @@ public class YoutubeSearchExtractorDefaultTest extends YoutubeSearchExtractorBas
itemsPage = extractor.getInitialPage(); itemsPage = extractor.getInitialPage();
} }
@Test
public void testGetUrl() throws Exception {
assertEquals("https://www.youtube.com/results?q=pewdiepie&gl=GB", extractor.getUrl());
}
@Test @Test
public void testGetSecondPageUrl() throws Exception { public void testGetSecondPageUrl() throws Exception {
assertEquals("https://www.youtube.com/results?q=pewdiepie&page=2", extractor.getNextPageUrl()); assertEquals("https://www.youtube.com/results?q=pewdiepie&gl=GB&page=2", extractor.getNextPageUrl());
} }
@Test @Test
public void testResultList_FirstElement() { public void testResultList_FirstElement() {
InfoItem firstInfoItem = itemsPage.getItems().get(0); InfoItem firstInfoItem = itemsPage.getItems().get(1);
// THe channel should be the first item // The channel should be the first item
assertTrue(firstInfoItem instanceof ChannelInfoItem); assertTrue(firstInfoItem instanceof ChannelInfoItem);
assertEquals("name", "PewDiePie", firstInfoItem.getName()); assertEquals("name", "PewDiePie", firstInfoItem.getName());
assertEquals("url","https://www.youtube.com/user/PewDiePie", firstInfoItem.getUrl()); assertEquals("url","https://www.youtube.com/user/PewDiePie", firstInfoItem.getUrl());
@ -96,7 +100,7 @@ public class YoutubeSearchExtractorDefaultTest extends YoutubeSearchExtractorBas
} }
assertFalse("First and second page are equal", equals); assertFalse("First and second page are equal", equals);
assertEquals("https://www.youtube.com/results?q=pewdiepie&page=3", secondPage.getNextPageUrl()); assertEquals("https://www.youtube.com/results?q=pewdiepie&gl=GB&page=3", secondPage.getNextPageUrl());
} }
@Test @Test