be-gay-do-crimes
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #314 from wb9688/remove-getnextpageurl 

Next page stuff
This commit is contained in:
Tobias Groza 2020-07-07 20:49:38 +02:00 committed by GitHub
parent e5d23a89de 0a5a905bc7
commit a70cb0283f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
53 changed files with 801 additions and 955 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
extractor/src/main/java/org/schabi/newpipe/extractor/ListExtractor.java
View File

@ -2,7 +2,6 @@ package org.schabi.newpipe.extractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
import java.util.Collections;
@ -10,13 +9,11 @@ import java.util.List;
import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
/**
* Base class to extractors that have a list (e.g. playlists, users).
*/
public abstract class ListExtractor<R extends InfoItem> extends Extractor {
/**
* Constant that should be returned whenever
* a list has an unknown number of items.
@ -38,36 +35,22 @@ public abstract class ListExtractor<R extends InfoItem> extends Extractor {
}
/**
* A {@link InfoItemsPage InfoItemsPage} corresponding to the initial page where the items are from the initial request and
* the nextPageUrl relative to it.
* A {@link InfoItemsPage InfoItemsPage} corresponding to the initial page
* where the items are from the initial request and the nextPage relative to it.
*
* @return a {@link InfoItemsPage} corresponding to the initial page
*/
@Nonnull
public abstract InfoItemsPage<R> getInitialPage() throws IOException, ExtractionException;
/**
* Returns an url that can be used to get the next page relative to the initial one.
* <p>Usually, these links will only work in the implementation itself.</p>
*
* @return an url pointing to the next page relative to the initial page
* @see #getPage(String)
*/
public abstract String getNextPageUrl() throws IOException, ExtractionException;
/**
* Get a list of items corresponding to the specific requested page.
*
* @param pageUrl any page url got from the exclusive implementation of the list extractor
* @param page any page got from the exclusive implementation of the list extractor
* @return a {@link InfoItemsPage} corresponding to the requested page
* @see #getNextPageUrl()
* @see InfoItemsPage#getNextPageUrl()
* @see InfoItemsPage#getNextPage()
*/
public abstract InfoItemsPage<R> getPage(final String pageUrl) throws IOException, ExtractionException;
public boolean hasNextPage() throws IOException, ExtractionException {
return !isNullOrEmpty(getNextPageUrl());
}
public abstract InfoItemsPage<R> getPage(final Page page) throws IOException, ExtractionException;
@Override
public ListLinkHandler getLinkHandler() {
@ -80,23 +63,22 @@ public abstract class ListExtractor<R extends InfoItem> extends Extractor {
/**
* A class that is used to wrap a list of gathered items and eventual errors, it
* also contains a field that points to the next available page ({@link #nextPageUrl}).
* also contains a field that points to the next available page ({@link #nextPage}).
*/
public static class InfoItemsPage<T extends InfoItem> {
private static final InfoItemsPage<InfoItem> EMPTY =
new InfoItemsPage<>(Collections.<InfoItem>emptyList(), "", Collections.<Throwable>emptyList());
new InfoItemsPage<>(Collections.<InfoItem>emptyList(), null, Collections.<Throwable>emptyList());
/**
* A convenient method that returns a representation of an empty page.
*
* @return a type-safe page with the list of items and errors empty and the nextPageUrl set to an empty string.
* @return a type-safe page with the list of items and errors empty and the nextPage set to {@code null}.
*/
public static <T extends InfoItem> InfoItemsPage<T> emptyPage() {
//noinspection unchecked
return (InfoItemsPage<T>) EMPTY;
}
/**
* The current list of items of this page
*/
@ -105,40 +87,40 @@ public abstract class ListExtractor<R extends InfoItem> extends Extractor {
/**
* Url pointing to the next page relative to this one
*
* @see ListExtractor#getPage(String)
* @see ListExtractor#getPage(Page)
* @see Page
*/
private final String nextPageUrl;
private final Page nextPage;
/**
* Errors that happened during the extraction
*/
private final List<Throwable> errors;
public InfoItemsPage(InfoItemsCollector<T, ?> collector, String nextPageUrl) {
this(collector.getItems(), nextPageUrl, collector.getErrors());
public InfoItemsPage(InfoItemsCollector<T, ?> collector, Page nextPage) {
this(collector.getItems(), nextPage, collector.getErrors());
}
public InfoItemsPage(List<T> itemsList, String nextPageUrl, List<Throwable> errors) {
public InfoItemsPage(List<T> itemsList, Page nextPage, List<Throwable> errors) {
this.itemsList = itemsList;
this.nextPageUrl = nextPageUrl;
this.nextPage = nextPage;
this.errors = errors;
}
public boolean hasNextPage() {
return !isNullOrEmpty(nextPageUrl);
return Page.isValid(nextPage);
}
public List<T> getItems() {
return itemsList;
}
public String getNextPageUrl() {
return nextPageUrl;
public Page getNextPage() {
return nextPage;
}
public List<Throwable> getErrors() {
return errors;
}
}
}

14
extractor/src/main/java/org/schabi/newpipe/extractor/ListInfo.java
View File

@ -4,11 +4,9 @@ import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import java.util.List;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public abstract class ListInfo<T extends InfoItem> extends Info {
private List<T> relatedItems;
private String nextPageUrl = null;
private Page nextPage = null;
private final List<String> contentFilters;
private final String sortFilter;
@ -39,15 +37,15 @@ public abstract class ListInfo<T extends InfoItem> extends Info {
}
public boolean hasNextPage() {
return !isNullOrEmpty(nextPageUrl);
return Page.isValid(nextPage);
}
public String getNextPageUrl() {
return nextPageUrl;
public Page getNextPage() {
return nextPage;
}
public void setNextPageUrl(String pageUrl) {
this.nextPageUrl = pageUrl;
public void setNextPage(Page page) {
this.nextPage = page;
}
public List<String> getContentFilters() {

52
extractor/src/main/java/org/schabi/newpipe/extractor/Page.java Normal file
View File

@ -0,0 +1,52 @@
package org.schabi.newpipe.extractor;
import java.io.Serializable;
import java.util.List;
import java.util.Map;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class Page implements Serializable {
private final String url;
private final List<String> ids;
private final Map<String, String> cookies;
public Page(final String url, final List<String> ids, final Map<String, String> cookies) {
this.url = url;
this.ids = ids;
this.cookies = cookies;
}
public Page(final String url) {
this(url, null, null);
}
public Page(final String url, final Map<String, String> cookies) {
this(url, null, cookies);
}
public Page(final List<String> ids) {
this(null, ids, null);
}
public Page(final List<String> ids, final Map<String, String> cookies) {
this(null, ids, cookies);
}
public String getUrl() {
return url;
}
public List<String> getIds() {
return ids;
}
public Map<String, String> getCookies() {
return cookies;
}
public static boolean isValid(final Page page) {
return page != null && (!isNullOrEmpty(page.getUrl())
|| !isNullOrEmpty(page.getIds()));
}
}

7
extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java
View File

@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor.channel;
import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage;
import org.schabi.newpipe.extractor.ListInfo;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
@ -49,8 +50,8 @@ public class ChannelInfo extends ListInfo<StreamInfoItem> {
public static InfoItemsPage<StreamInfoItem> getMoreItems(StreamingService service,
String url,
String pageUrl) throws IOException, ExtractionException {
return service.getChannelExtractor(url).getPage(pageUrl);
Page page) throws IOException, ExtractionException {
return service.getChannelExtractor(url).getPage(page);
}
public static ChannelInfo getInfo(ChannelExtractor extractor) throws IOException, ExtractionException {
@ -81,7 +82,7 @@ public class ChannelInfo extends ListInfo<StreamInfoItem> {
final InfoItemsPage<StreamInfoItem> itemsPage = ExtractorHelper.getItemsPageOrLogError(info, extractor);
info.setRelatedItems(itemsPage.getItems());
info.setNextPageUrl(itemsPage.getNextPageUrl());
info.setNextPage(itemsPage.getNextPage());
try {
info.setSubscriberCount(extractor.getSubscriberCount());

11
extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfo.java
View File

@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor.comments;
import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage;
import org.schabi.newpipe.extractor.ListInfo;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
@ -39,23 +40,23 @@ public class CommentsInfo extends ListInfo<CommentsInfoItem> {
InfoItemsPage<CommentsInfoItem> initialCommentsPage = ExtractorHelper.getItemsPageOrLogError(commentsInfo,
commentsExtractor);
commentsInfo.setRelatedItems(initialCommentsPage.getItems());
commentsInfo.setNextPageUrl(initialCommentsPage.getNextPageUrl());
commentsInfo.setNextPage(initialCommentsPage.getNextPage());
return commentsInfo;
}
public static InfoItemsPage<CommentsInfoItem> getMoreItems(CommentsInfo commentsInfo, String pageUrl)
public static InfoItemsPage<CommentsInfoItem> getMoreItems(CommentsInfo commentsInfo, Page page)
throws ExtractionException, IOException {
return getMoreItems(NewPipe.getService(commentsInfo.getServiceId()), commentsInfo, pageUrl);
return getMoreItems(NewPipe.getService(commentsInfo.getServiceId()), commentsInfo, page);
}
public static InfoItemsPage<CommentsInfoItem> getMoreItems(StreamingService service, CommentsInfo commentsInfo,
String pageUrl) throws IOException, ExtractionException {
Page page) throws IOException, ExtractionException {
if (null == commentsInfo.getCommentsExtractor()) {
commentsInfo.setCommentsExtractor(service.getCommentsExtractor(commentsInfo.getUrl()));
commentsInfo.getCommentsExtractor().fetchPage();
}
return commentsInfo.getCommentsExtractor().getPage(pageUrl);
return commentsInfo.getCommentsExtractor().getPage(page);
}
private transient CommentsExtractor commentsExtractor;

2
extractor/src/main/java/org/schabi/newpipe/extractor/feed/FeedInfo.java
View File

@ -45,7 +45,7 @@ public class FeedInfo extends ListInfo<StreamInfoItem> {
final InfoItemsPage<StreamInfoItem> itemsPage = ExtractorHelper.getItemsPageOrLogError(info, extractor);
info.setRelatedItems(itemsPage.getItems());
info.setNextPageUrl(itemsPage.getNextPageUrl());
info.setNextPage(itemsPage.getNextPage());
return info;
}

10
extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskInfo.java
View File

@ -23,6 +23,7 @@ package org.schabi.newpipe.extractor.kiosk;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.ListInfo;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
@ -33,18 +34,17 @@ import org.schabi.newpipe.extractor.utils.ExtractorHelper;
import java.io.IOException;
public class KioskInfo extends ListInfo<StreamInfoItem> {
private KioskInfo(int serviceId, ListLinkHandler linkHandler, String name) throws ParsingException {
super(serviceId, linkHandler, name);
}
public static ListExtractor.InfoItemsPage<StreamInfoItem> getMoreItems(StreamingService service,
String url,
String pageUrl)
Page page)
throws IOException, ExtractionException {
KioskList kl = service.getKioskList();
KioskExtractor extractor = kl.getExtractorByUrl(url, pageUrl);
return extractor.getPage(pageUrl);
KioskExtractor extractor = kl.getExtractorByUrl(url, page);
return extractor.getPage(page);
}
public static KioskInfo getInfo(String url) throws IOException, ExtractionException {
@ -71,7 +71,7 @@ public class KioskInfo extends ListInfo<StreamInfoItem> {
final ListExtractor.InfoItemsPage<StreamInfoItem> itemsPage = ExtractorHelper.getItemsPageOrLogError(info, extractor);
info.setRelatedItems(itemsPage.getItems());
info.setNextPageUrl(itemsPage.getNextPageUrl());
info.setNextPage(itemsPage.getNextPage());
return info;
}

27
extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java
View File

@ -1,6 +1,7 @@
package org.schabi.newpipe.extractor.kiosk;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
@ -59,23 +60,23 @@ public class KioskList {
public KioskExtractor getDefaultKioskExtractor()
throws ExtractionException, IOException {
return getDefaultKioskExtractor("");
return getDefaultKioskExtractor(null);
}
public KioskExtractor getDefaultKioskExtractor(String nextPageUrl)
public KioskExtractor getDefaultKioskExtractor(Page nextPage)
throws ExtractionException, IOException {
return getDefaultKioskExtractor(nextPageUrl, NewPipe.getPreferredLocalization());
return getDefaultKioskExtractor(nextPage, NewPipe.getPreferredLocalization());
}
public KioskExtractor getDefaultKioskExtractor(String nextPageUrl, Localization localization)
public KioskExtractor getDefaultKioskExtractor(Page nextPage, Localization localization)
throws ExtractionException, IOException {
if (defaultKiosk != null && !defaultKiosk.equals("")) {
return getExtractorById(defaultKiosk, nextPageUrl, localization);
return getExtractorById(defaultKiosk, nextPage, localization);
} else {
if (!kioskList.isEmpty()) {
// if not set get any entry
Object[] keySet = kioskList.keySet().toArray();
return getExtractorById(keySet[0].toString(), nextPageUrl, localization);
return getExtractorById(keySet[0].toString(), nextPage, localization);
} else {
return null;
}
@ -86,12 +87,12 @@ public class KioskList {
return defaultKiosk;
}
public KioskExtractor getExtractorById(String kioskId, String nextPageUrl)
public KioskExtractor getExtractorById(String kioskId, Page nextPage)
throws ExtractionException, IOException {
return getExtractorById(kioskId, nextPageUrl, NewPipe.getPreferredLocalization());
return getExtractorById(kioskId, nextPage, NewPipe.getPreferredLocalization());
}
public KioskExtractor getExtractorById(String kioskId, String nextPageUrl, Localization localization)
public KioskExtractor getExtractorById(String kioskId, Page nextPage, Localization localization)
throws ExtractionException, IOException {
KioskEntry ke = kioskList.get(kioskId);
if (ke == null) {
@ -111,17 +112,17 @@ public class KioskList {
return kioskList.keySet();
}
public KioskExtractor getExtractorByUrl(String url, String nextPageUrl)
public KioskExtractor getExtractorByUrl(String url, Page nextPage)
throws ExtractionException, IOException {
return getExtractorByUrl(url, nextPageUrl, NewPipe.getPreferredLocalization());
return getExtractorByUrl(url, nextPage, NewPipe.getPreferredLocalization());
}
public KioskExtractor getExtractorByUrl(String url, String nextPageUrl, Localization localization)
public KioskExtractor getExtractorByUrl(String url, Page nextPage, Localization localization)
throws ExtractionException, IOException {
for (Map.Entry<String, KioskEntry> e : kioskList.entrySet()) {
KioskEntry ke = e.getValue();
if (ke.handlerFactory.acceptUrl(url)) {
return getExtractorById(ke.handlerFactory.getId(url), nextPageUrl, localization);
return getExtractorById(ke.handlerFactory.getId(url), nextPage, localization);
}
}
throw new ExtractionException("Could not find a kiosk that fits to the url: " + url);

7
extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java
View File

@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor.playlist;
import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage;
import org.schabi.newpipe.extractor.ListInfo;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
@ -32,8 +33,8 @@ public class PlaylistInfo extends ListInfo<StreamInfoItem> {
public static InfoItemsPage<StreamInfoItem> getMoreItems(StreamingService service,
String url,
String pageUrl) throws IOException, ExtractionException {
return service.getPlaylistExtractor(url).getPage(pageUrl);
Page page) throws IOException, ExtractionException {
return service.getPlaylistExtractor(url).getPage(page);
}
/**
@ -112,7 +113,7 @@ public class PlaylistInfo extends ListInfo<StreamInfoItem> {
final InfoItemsPage<StreamInfoItem> itemsPage = ExtractorHelper.getItemsPageOrLogError(info, extractor);
info.setRelatedItems(itemsPage.getItems());
info.setNextPageUrl(itemsPage.getNextPageUrl());
info.setNextPage(itemsPage.getNextPage());
return info;
}

9
extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchInfo.java
View File

@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor.search;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.ListInfo;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
@ -10,9 +11,7 @@ import org.schabi.newpipe.extractor.utils.ExtractorHelper;
import java.io.IOException;
public class SearchInfo extends ListInfo<InfoItem> {
private String searchString;
private String searchSuggestion;
private boolean isCorrectedSearch;
@ -55,7 +54,7 @@ public class SearchInfo extends ListInfo<InfoItem> {
ListExtractor.InfoItemsPage<InfoItem> page = ExtractorHelper.getItemsPageOrLogError(info, extractor);
info.setRelatedItems(page.getItems());
info.setNextPageUrl(page.getNextPageUrl());
info.setNextPage(page.getNextPage());
return info;
}
@ -63,9 +62,9 @@ public class SearchInfo extends ListInfo<InfoItem> {
public static ListExtractor.InfoItemsPage<InfoItem> getMoreItems(StreamingService service,
SearchQueryHandler query,
String pageUrl)
Page page)
throws IOException, ExtractionException {
return service.getSearchExtractor(query).getPage(pageUrl);
return service.getSearchExtractor(query).getPage(page);
}
// Getter

10
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceExtractor.java
View File

@ -5,6 +5,7 @@ import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.downloader.Downloader;
@ -79,13 +80,8 @@ public class MediaCCCConferenceExtractor extends ChannelExtractor {
}
@Override
public String getNextPageUrl() {
return null;
}
@Override
public InfoItemsPage<StreamInfoItem> getPage(final String pageUrl) {
return null;
public InfoItemsPage<StreamInfoItem> getPage(final Page page) {
return InfoItemsPage.emptyPage();
}
@Override

9
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceKiosk.java
View File

@ -5,6 +5,7 @@ import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelInfoItem;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemsCollector;
@ -37,16 +38,12 @@ public class MediaCCCConferenceKiosk extends KioskExtractor<ChannelInfoItem> {
collector.commit(new MediaCCCConferenceInfoItemExtractor(conferences.getObject(i)));
}
return new InfoItemsPage<>(collector, "");
return new InfoItemsPage<>(collector, null);
}
@Override
public String getNextPageUrl() {
return "";
}
@Override
public InfoItemsPage<ChannelInfoItem> getPage(final String pageUrl) {
public InfoItemsPage<ChannelInfoItem> getPage(final Page page) {
return InfoItemsPage.emptyPage();
}

8
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java
View File

@ -6,6 +6,7 @@ import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelInfoItem;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
@ -80,12 +81,7 @@ public class MediaCCCSearchExtractor extends SearchExtractor {
}
@Override
public String getNextPageUrl() {
return "";
}
@Override
public InfoItemsPage<InfoItem> getPage(final String pageUrl) {
public InfoItemsPage<InfoItem> getPage(final Page page) {
return InfoItemsPage.emptyPage();
}

48
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java
View File

@ -1,9 +1,14 @@
package org.schabi.newpipe.extractor.services.peertube;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.InfoItemsCollector;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeStreamInfoItemExtractor;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils;
@ -14,7 +19,6 @@ import java.util.Date;
import java.util.TimeZone;
public class PeertubeParsingHelper {
public static final String START_KEY = "start";
public static final String COUNT_KEY = "count";
public static final int ITEMS_PER_PAGE = 12;
@ -23,17 +27,17 @@ public class PeertubeParsingHelper {
private PeertubeParsingHelper() {
}
public static void validate(JsonObject json) throws ContentNotAvailableException {
String error = json.getString("error");
public static void validate(final JsonObject json) throws ContentNotAvailableException {
final String error = json.getString("error");
if (!Utils.isBlank(error)) {
throw new ContentNotAvailableException(error);
}
}
public static Calendar parseDateFrom(String textualUploadDate) throws ParsingException {
Date date;
public static Calendar parseDateFrom(final String textualUploadDate) throws ParsingException {
final Date date;
try {
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'");
final SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'");
sdf.setTimeZone(TimeZone.getTimeZone("GMT"));
date = sdf.parse(textualUploadDate);
} catch (ParseException e) {
@ -45,26 +49,42 @@ public class PeertubeParsingHelper {
return uploadDate;
}
public static String getNextPageUrl(String prevPageUrl, long total) {
String prevStart;
public static Page getNextPage(final String prevPageUrl, final long total) {
final String prevStart;
try {
prevStart = Parser.matchGroup1(START_PATTERN, prevPageUrl);
} catch (Parser.RegexException e) {
return "";
return null;
}
if (Utils.isBlank(prevStart)) return "";
long nextStart = 0;
if (Utils.isBlank(prevStart)) return null;
final long nextStart;
try {
nextStart = Long.parseLong(prevStart) + ITEMS_PER_PAGE;
} catch (NumberFormatException e) {
return "";
return null;
}
if (nextStart >= total) {
return "";
return null;
} else {
return prevPageUrl.replace(START_KEY + "=" + prevStart, START_KEY + "=" + nextStart);
return new Page(prevPageUrl.replace(START_KEY + "=" + prevStart, START_KEY + "=" + nextStart));
}
}
public static void collectStreamsFrom(final InfoItemsCollector collector, final JsonObject json, final String baseUrl) throws ParsingException {
final JsonArray contents;
try {
contents = (JsonArray) JsonUtils.getValue(json, "data");
} catch (Exception e) {
throw new ParsingException("Unable to extract list info", e);
}
for (final Object c : contents) {
if (c instanceof JsonObject) {
final JsonObject item = (JsonObject) c;
final PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item, baseUrl);
collector.commit(extractor);
}
}
}
}

98
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java
View File

@ -1,10 +1,10 @@
package org.schabi.newpipe.extractor.services.peertube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.downloader.Downloader;
@ -20,23 +20,23 @@ import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.*;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.collectStreamsFrom;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class PeertubeAccountExtractor extends ChannelExtractor {
private InfoItemsPage<StreamInfoItem> initPage;
private long total;
private JsonObject json;
private final String baseUrl;
public PeertubeAccountExtractor(StreamingService service, ListLinkHandler linkHandler) throws ParsingException {
public PeertubeAccountExtractor(final StreamingService service, final ListLinkHandler linkHandler) throws ParsingException {
super(service, linkHandler);
this.baseUrl = getBaseUrl();
}
@Override
public String getAvatarUrl() throws ParsingException {
public String getAvatarUrl() {
String value;
try {
value = JsonUtils.getString(json, "avatar.path");
@ -47,7 +47,7 @@ public class PeertubeAccountExtractor extends ChannelExtractor {
}
@Override
public String getBannerUrl() throws ParsingException {
public String getBannerUrl() {
return null;
}
@ -57,13 +57,12 @@ public class PeertubeAccountExtractor extends ChannelExtractor {
}
@Override
public long getSubscriberCount() throws ParsingException {
Number number = JsonUtils.getNumber(json, "followersCount");
return number.longValue();
public long getSubscriberCount() {
return json.getLong("followersCount");
}
@Override
public String getDescription() throws ParsingException {
public String getDescription() {
try {
return JsonUtils.getString(json, "description");
} catch (ParsingException e) {
@ -72,93 +71,73 @@ public class PeertubeAccountExtractor extends ChannelExtractor {
}
@Override
public String getParentChannelName() throws ParsingException {
public String getParentChannelName() {
return "";
}
@Override
public String getParentChannelUrl() throws ParsingException {
public String getParentChannelUrl() {
return "";
}
@Override
public String getParentChannelAvatarUrl() throws ParsingException {
public String getParentChannelAvatarUrl() {
return "";
}
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
super.fetchPage();
return initPage;
}
private void collectStreamsFrom(StreamInfoItemsCollector collector, JsonObject json, String pageUrl) throws ParsingException {
JsonArray contents;
try {
contents = (JsonArray) JsonUtils.getValue(json, "data");
} catch (Exception e) {
throw new ParsingException("unable to extract channel streams", e);
}
for (Object c : contents) {
if (c instanceof JsonObject) {
final JsonObject item = (JsonObject) c;
PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item, baseUrl);
collector.commit(extractor);
}
}
final String pageUrl = getUrl() + "/videos?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE;
return getPage(new Page(pageUrl));
}
@Override
public String getNextPageUrl() throws IOException, ExtractionException {
super.fetchPage();
return initPage.getNextPageUrl();
}
public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException, ExtractionException {
if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL");
}
final Response response = getDownloader().get(page.getUrl());
@Override
public InfoItemsPage<StreamInfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
Response response = getDownloader().get(pageUrl);
JsonObject json = null;
if (response != null && !Utils.isBlank(response.responseBody())) {
try {
json = JsonParser.object().from(response.responseBody());
} catch (Exception e) {
throw new ParsingException("Could not parse json data for kiosk info", e);
throw new ParsingException("Could not parse json data for account info", e);
}
}
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
if (json != null) {
PeertubeParsingHelper.validate(json);
total = JsonUtils.getNumber(json, "total").longValue();
collectStreamsFrom(collector, json, pageUrl);
final long total = json.getLong("total");
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
collectStreamsFrom(collector, json, getBaseUrl());
return new InfoItemsPage<>(collector, PeertubeParsingHelper.getNextPage(page.getUrl(), total));
} else {
throw new ExtractionException("Unable to get PeerTube kiosk info");
throw new ExtractionException("Unable to get PeerTube account info");
}
return new InfoItemsPage<>(collector, PeertubeParsingHelper.getNextPageUrl(pageUrl, total));
}
@Override
public void onFetchPage(Downloader downloader) throws IOException, ExtractionException {
Response response = downloader.get(getUrl());
if (null != response && null != response.responseBody()) {
public void onFetchPage(final Downloader downloader) throws IOException, ExtractionException {
final Response response = downloader.get(getUrl());
if (response != null && response.responseBody() != null) {
setInitialData(response.responseBody());
} else {
throw new ExtractionException("Unable to extract PeerTube channel data");
throw new ExtractionException("Unable to extract PeerTube account data");
}
String pageUrl = getUrl() + "/videos?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE;
this.initPage = getPage(pageUrl);
}
private void setInitialData(String responseBody) throws ExtractionException {
private void setInitialData(final String responseBody) throws ExtractionException {
try {
json = JsonParser.object().from(responseBody);
} catch (JsonParserException e) {
throw new ExtractionException("Unable to extract PeerTube channel data", e);
throw new ExtractionException("Unable to extract PeerTube account data", e);
}
if (json == null) throw new ExtractionException("Unable to extract PeerTube channel data");
if (json == null) throw new ExtractionException("Unable to extract PeerTube account data");
}
@Override
@ -170,5 +149,4 @@ public class PeertubeAccountExtractor extends ChannelExtractor {
public String getOriginalUrl() throws ParsingException {
return baseUrl + "/" + getId();
}
}

92
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java
View File

@ -1,10 +1,10 @@
package org.schabi.newpipe.extractor.services.peertube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.downloader.Downloader;
@ -16,29 +16,28 @@ import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Parser.RegexException;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.*;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.collectStreamsFrom;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class PeertubeChannelExtractor extends ChannelExtractor {
private InfoItemsPage<StreamInfoItem> initPage;
private long total;
private JsonObject json;
private final String baseUrl;
public PeertubeChannelExtractor(StreamingService service, ListLinkHandler linkHandler) throws ParsingException {
public PeertubeChannelExtractor(final StreamingService service, final ListLinkHandler linkHandler) throws ParsingException {
super(service, linkHandler);
this.baseUrl = getBaseUrl();
}
@Override
public String getAvatarUrl() throws ParsingException {
public String getAvatarUrl() {
String value;
try {
value = JsonUtils.getString(json, "avatar.path");
@ -49,7 +48,7 @@ public class PeertubeChannelExtractor extends ChannelExtractor {
}
@Override
public String getBannerUrl() throws ParsingException {
public String getBannerUrl() {
return null;
}
@ -59,13 +58,12 @@ public class PeertubeChannelExtractor extends ChannelExtractor {
}
@Override
public long getSubscriberCount() throws ParsingException {
Number number = JsonUtils.getNumber(json, "followersCount");
return number.longValue();
public long getSubscriberCount() {
return json.getLong("followersCount");
}
@Override
public String getDescription() throws ParsingException {
public String getDescription() {
try {
return JsonUtils.getString(json, "description");
} catch (ParsingException e) {
@ -84,7 +82,7 @@ public class PeertubeChannelExtractor extends ChannelExtractor {
}
@Override
public String getParentChannelAvatarUrl() throws ParsingException {
public String getParentChannelAvatarUrl() {
String value;
try {
value = JsonUtils.getString(json, "ownerAccount.avatar.path");
@ -96,74 +94,55 @@ public class PeertubeChannelExtractor extends ChannelExtractor {
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
super.fetchPage();
return initPage;
}
private void collectStreamsFrom(StreamInfoItemsCollector collector, JsonObject json, String pageUrl) throws ParsingException {
JsonArray contents;
try {
contents = (JsonArray) JsonUtils.getValue(json, "data");
} catch (Exception e) {
throw new ParsingException("unable to extract channel streams", e);
}
for (Object c : contents) {
if (c instanceof JsonObject) {
final JsonObject item = (JsonObject) c;
PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item, baseUrl);
collector.commit(extractor);
}
}
final String pageUrl = getUrl() + "/videos?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE;
return getPage(new Page(pageUrl));
}
@Override
public String getNextPageUrl() throws IOException, ExtractionException {
super.fetchPage();
return initPage.getNextPageUrl();
}
public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException, ExtractionException {
if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL");
}
final Response response = getDownloader().get(page.getUrl());
@Override
public InfoItemsPage<StreamInfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
Response response = getDownloader().get(pageUrl);
JsonObject json = null;
if (response != null && !Utils.isBlank(response.responseBody())) {
try {
json = JsonParser.object().from(response.responseBody());
} catch (Exception e) {
throw new ParsingException("Could not parse json data for kiosk info", e);
throw new ParsingException("Could not parse json data for channel info", e);
}
}
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
if (json != null) {
PeertubeParsingHelper.validate(json);
this.total = JsonUtils.getNumber(json, "total").longValue();
collectStreamsFrom(collector, json, pageUrl);
final long total = json.getLong("total");
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
collectStreamsFrom(collector, json, getBaseUrl());
return new InfoItemsPage<>(collector, PeertubeParsingHelper.getNextPage(page.getUrl(), total));
} else {
throw new ExtractionException("Unable to get PeerTube kiosk info");
throw new ExtractionException("Unable to get PeerTube channel info");
}
return new InfoItemsPage<>(collector, PeertubeParsingHelper.getNextPageUrl(pageUrl, total));
}
@Override
public void onFetchPage(Downloader downloader) throws IOException, ExtractionException {
Response response = downloader.get(getUrl());
if (null != response && null != response.responseBody()) {
public void onFetchPage(final Downloader downloader) throws IOException, ExtractionException {
final Response response = downloader.get(getUrl());
if (response != null && response.responseBody() != null) {
setInitialData(response.responseBody());
} else {
throw new ExtractionException("Unable to extract PeerTube channel data");
}
this.initPage = getPage(getUrl() + "/videos?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE);
}
private void setInitialData(String responseBody) throws ExtractionException {
private void setInitialData(final String responseBody) throws ExtractionException {
try {
json = JsonParser.object().from(responseBody);
} catch (JsonParserException e) {
throw new ExtractionException("Unable to extract peertube channel data", e);
throw new ExtractionException("Unable to extract PeerTube channel data", e);
}
if (json == null) throw new ExtractionException("Unable to extract PeerTube channel data");
}
@ -177,5 +156,4 @@ public class PeertubeChannelExtractor extends ChannelExtractor {
public String getOriginalUrl() throws ParsingException {
return baseUrl + "/" + getId();
}
}

38
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsExtractor.java
View File

@ -4,6 +4,7 @@ import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.comments.CommentsExtractor;
import org.schabi.newpipe.extractor.comments.CommentsInfoItem;
@ -22,19 +23,17 @@ import java.io.IOException;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class PeertubeCommentsExtractor extends CommentsExtractor {
private InfoItemsPage<CommentsInfoItem> initPage;
private long total;
public PeertubeCommentsExtractor(final StreamingService service, final ListLinkHandler uiHandler) {
super(service, uiHandler);
}
@Override
public InfoItemsPage<CommentsInfoItem> getInitialPage() throws IOException, ExtractionException {
super.fetchPage();
return initPage;
final String pageUrl = getUrl() + "?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE;
return getPage(new Page(pageUrl));
}
private void collectCommentsFrom(final CommentsInfoItemsCollector collector, final JsonObject json) throws ParsingException {
@ -52,14 +51,13 @@ public class PeertubeCommentsExtractor extends CommentsExtractor {
}
@Override
public String getNextPageUrl() throws IOException, ExtractionException {
super.fetchPage();
return initPage.getNextPageUrl();
}
public InfoItemsPage<CommentsInfoItem> getPage(final Page page) throws IOException, ExtractionException {
if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL");
}
final Response response = getDownloader().get(page.getUrl());
@Override
public InfoItemsPage<CommentsInfoItem> getPage(final String pageUrl) throws IOException, ExtractionException {
final Response response = getDownloader().get(pageUrl);
JsonObject json = null;
if (response != null && !Utils.isBlank(response.responseBody())) {
try {
@ -69,19 +67,19 @@ public class PeertubeCommentsExtractor extends CommentsExtractor {
}
}
final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId());
if (json != null) {
final Number number = JsonUtils.getNumber(json, "total");
if (number != null) this.total = number.longValue();
PeertubeParsingHelper.validate(json);
final long total = json.getLong("total");
final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId());
collectCommentsFrom(collector, json);
return new InfoItemsPage<>(collector, PeertubeParsingHelper.getNextPage(page.getUrl(), total));
} else {
throw new ExtractionException("Unable to get peertube comments info");
throw new ExtractionException("Unable to get PeerTube kiosk info");
}
return new InfoItemsPage<>(collector, PeertubeParsingHelper.getNextPageUrl(pageUrl, total));
}
@Override
public void onFetchPage(Downloader downloader) throws IOException, ExtractionException {
this.initPage = getPage(getUrl() + "?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE);
}
public void onFetchPage(Downloader downloader) { }
}

27
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java
View File

@ -1,6 +1,7 @@
package org.schabi.newpipe.extractor.services.peertube.extractors;
import com.grack.nanojson.JsonObject;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.schabi.newpipe.extractor.ServiceList;
@ -10,14 +11,14 @@ import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import java.util.Objects;
public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtractor {
private final JsonObject item;
private final String url;
private final String baseUrl;
public PeertubeCommentsInfoItemExtractor(JsonObject item, PeertubeCommentsExtractor extractor) throws ParsingException {
public PeertubeCommentsInfoItemExtractor(final JsonObject item, final PeertubeCommentsExtractor extractor) throws ParsingException {
this.item = item;
this.url = extractor.getUrl();
this.baseUrl = extractor.getBaseUrl();
@ -29,7 +30,7 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac
}
@Override
public String getThumbnailUrl() throws ParsingException {
public String getThumbnailUrl() {
String value;
try {
value = JsonUtils.getString(item, "account.avatar.path");
@ -51,20 +52,20 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac
@Override
public DateWrapper getUploadDate() throws ParsingException {
String textualUploadDate = getTextualUploadDate();
final String textualUploadDate = getTextualUploadDate();
return new DateWrapper(PeertubeParsingHelper.parseDateFrom(textualUploadDate));
}
@Override
public int getLikeCount() throws ParsingException {
public int getLikeCount() {
return -1;
}
@Override
public String getCommentText() throws ParsingException {
String htmlText = JsonUtils.getString(item, "text");
final String htmlText = JsonUtils.getString(item, "text");
try {
Document doc = Jsoup.parse(htmlText);
final Document doc = Jsoup.parse(htmlText);
return doc.body().text();
} catch (Exception e) {
return htmlText.replaceAll("(?s)<[^>]*>(\\s*<[^>]*>)*", "");
@ -72,13 +73,12 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac
}
@Override
public String getCommentId() throws ParsingException {
Number value = JsonUtils.getNumber(item, "id");
return value.toString();
public String getCommentId() {
return Objects.toString(item.getLong("id"), null);
}
@Override
public String getUploaderAvatarUrl() throws ParsingException {
public String getUploaderAvatarUrl() {
String value;
try {
value = JsonUtils.getString(item, "account.avatar.path");
@ -95,9 +95,8 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac
@Override
public String getUploaderUrl() throws ParsingException {
String name = JsonUtils.getString(item, "account.name");
String host = JsonUtils.getString(item, "account.host");
final String name = JsonUtils.getString(item, "account.name");
final String host = JsonUtils.getString(item, "account.host");
return ServiceList.PeerTube.getChannelLHFactory().fromId("accounts/" + name + "@" + host, baseUrl).getUrl();
}
}

90
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubePlaylistExtractor.java
View File

@ -1,9 +1,10 @@
package org.schabi.newpipe.extractor.services.peertube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
@ -14,22 +15,22 @@ import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull;
import java.io.IOException;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.*;
import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.collectStreamsFrom;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class PeertubePlaylistExtractor extends PlaylistExtractor {
private JsonObject playlistInfo;
private JsonObject playlistVideos;
private String initialPageUrl;
private long total;
public PeertubePlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) {
public PeertubePlaylistExtractor(final StreamingService service, final ListLinkHandler linkHandler) {
super(service, linkHandler);
}
@ -39,17 +40,17 @@ public class PeertubePlaylistExtractor extends PlaylistExtractor {
}
@Override
public String getBannerUrl() throws ParsingException {
public String getBannerUrl() {
return null;
}
@Override
public String getUploaderUrl() throws ParsingException {
public String getUploaderUrl() {
return playlistInfo.getObject("ownerAccount").getString("url");
}
@Override
public String getUploaderName() throws ParsingException {
public String getUploaderName() {
return playlistInfo.getObject("ownerAccount").getString("displayName");
}
@ -59,19 +60,19 @@ public class PeertubePlaylistExtractor extends PlaylistExtractor {
}
@Override
public long getStreamCount() throws ParsingException {
return playlistInfo.getNumber("videosLength").longValue();
public long getStreamCount() {
return playlistInfo.getLong("videosLength");
}
@Nonnull
@Override
public String getSubChannelName() throws ParsingException {
public String getSubChannelName() {
return playlistInfo.getObject("videoChannel").getString("displayName");
}
@Nonnull
@Override
public String getSubChannelUrl() throws ParsingException {
public String getSubChannelUrl() {
return playlistInfo.getObject("videoChannel").getString("url");
}
@ -84,47 +85,48 @@ public class PeertubePlaylistExtractor extends PlaylistExtractor {
@Nonnull
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
return getPage(initialPageUrl);
return getPage(new Page(getUrl() + "/videos?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE));
}
@Override
public String getNextPageUrl() throws IOException, ExtractionException {
return PeertubeParsingHelper.getNextPageUrl(initialPageUrl, total);
}
@Override
public InfoItemsPage<StreamInfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
Response response = getDownloader().get(pageUrl);
try {
playlistVideos = JsonParser.object().from(response.responseBody());
} catch (JsonParserException jpe) {
throw new ExtractionException("Could not parse json", jpe);
}
PeertubeParsingHelper.validate(playlistVideos);
this.total = JsonUtils.getNumber(playlistVideos, "total").longValue();
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
JsonArray videos = playlistVideos.getArray("data");
for (Object o : videos) {
JsonObject video = ((JsonObject) o).getObject("video");
collector.commit(new PeertubeStreamInfoItemExtractor(video, getBaseUrl()));
public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException, ExtractionException {
if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL");
}
return new InfoItemsPage<>(collector, PeertubeParsingHelper.getNextPageUrl(pageUrl, total));
final Response response = getDownloader().get(page.getUrl());
JsonObject json = null;
if (response != null && !Utils.isBlank(response.responseBody())) {
try {
json = JsonParser.object().from(response.responseBody());
} catch (Exception e) {
throw new ParsingException("Could not parse json data for playlist info", e);
}
}
if (json != null) {
PeertubeParsingHelper.validate(json);
final long total = json.getLong("total");
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
collectStreamsFrom(collector, json, getBaseUrl());
return new InfoItemsPage<>(collector, PeertubeParsingHelper.getNextPage(page.getUrl(), total));
} else {
throw new ExtractionException("Unable to get PeerTube playlist info");
}
}
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
Response response = downloader.get(getUrl());
public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException {
final Response response = downloader.get(getUrl());
try {
playlistInfo = JsonParser.object().from(response.responseBody());
} catch (JsonParserException jpe) {
throw new ExtractionException("Could not parse json", jpe);
}
PeertubeParsingHelper.validate(playlistInfo);
initialPageUrl = getUrl() + "/videos?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE;
}
@Nonnull

79
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java
View File

@ -1,12 +1,10 @@
package org.schabi.newpipe.extractor.services.peertube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.InfoItemExtractor;
import org.schabi.newpipe.extractor.InfoItemsCollector;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
@ -16,28 +14,26 @@ import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Parser.RegexException;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull;
import java.io.IOException;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.*;
import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.collectStreamsFrom;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class PeertubeSearchExtractor extends SearchExtractor {
private InfoItemsPage<InfoItem> initPage;
private long total;
public PeertubeSearchExtractor(StreamingService service, SearchQueryHandler linkHandler) {
super(service, linkHandler);
}
@Nonnull
@Override
public String getSearchSuggestion() throws ParsingException {
public String getSearchSuggestion() {
return "";
}
@ -48,44 +44,20 @@ public class PeertubeSearchExtractor extends SearchExtractor {
@Override
public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionException {
super.fetchPage();
return initPage;
}
private InfoItemsCollector<InfoItem, InfoItemExtractor> collectStreamsFrom(JsonObject json) throws ParsingException {
final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId());
JsonArray contents;
try {
contents = (JsonArray) JsonUtils.getValue(json, "data");
} catch (Exception e) {
throw new ParsingException("unable to extract search info", e);
}
String baseUrl = getBaseUrl();
for (Object c : contents) {
if (c instanceof JsonObject) {
final JsonObject item = (JsonObject) c;
PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item, baseUrl);
collector.commit(extractor);
}
}
return collector;
final String pageUrl = getUrl() + "&" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE;
return getPage(new Page(pageUrl));
}
@Override
public String getNextPageUrl() throws IOException, ExtractionException {
super.fetchPage();
return initPage.getNextPageUrl();
}
public InfoItemsPage<InfoItem> getPage(final Page page) throws IOException, ExtractionException {
if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL");
}
final Response response = getDownloader().get(page.getUrl());
@Override
public InfoItemsPage<InfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
Response response = getDownloader().get(pageUrl);
JsonObject json = null;
if (null != response && !Utils.isBlank(response.responseBody())) {
if (response != null && !Utils.isBlank(response.responseBody())) {
try {
json = JsonParser.object().from(response.responseBody());
} catch (Exception e) {
@ -94,15 +66,18 @@ public class PeertubeSearchExtractor extends SearchExtractor {
}
if (json != null) {
total = JsonUtils.getNumber(json, "total").longValue();
return new InfoItemsPage<>(collectStreamsFrom(json), PeertubeParsingHelper.getNextPageUrl(pageUrl, total));
PeertubeParsingHelper.validate(json);
final long total = json.getLong("total");
final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId());
collectStreamsFrom(collector, json, getBaseUrl());
return new InfoItemsPage<>(collector, PeertubeParsingHelper.getNextPage(page.getUrl(), total));
} else {
throw new ExtractionException("Unable to get peertube search info");
throw new ExtractionException("Unable to get PeerTube search info");
}
}
@Override
public void onFetchPage(Downloader downloader) throws IOException, ExtractionException {
initPage = getPage(getUrl() + "&" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE);
}
public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException { }
}

153
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java
View File

@ -40,13 +40,11 @@ import java.util.Locale;
import javax.annotation.Nonnull;
public class PeertubeStreamExtractor extends StreamExtractor {
private final String baseUrl;
private JsonObject json;
private List<SubtitlesStream> subtitles = new ArrayList<>();
public PeertubeStreamExtractor(StreamingService service, LinkHandler linkHandler) throws ParsingException {
public PeertubeStreamExtractor(final StreamingService service, final LinkHandler linkHandler) throws ParsingException {
super(service, linkHandler);
this.baseUrl = getBaseUrl();
}
@ -82,10 +80,10 @@ public class PeertubeStreamExtractor extends StreamExtractor {
}
if (text.length() == 250 && text.substring(247).equals("...")) {
//if description is shortened, get full description
Downloader dl = NewPipe.getDownloader();
final Downloader dl = NewPipe.getDownloader();
try {
Response response = dl.get(getUrl() + "/description");
JsonObject jsonObject = JsonParser.object().from(response.responseBody());
final Response response = dl.get(getUrl() + "/description");
final JsonObject jsonObject = JsonParser.object().from(response.responseBody());
text = JsonUtils.getString(jsonObject, "description");
} catch (ReCaptchaException | IOException | JsonParserException e) {
e.printStackTrace();
@ -96,7 +94,7 @@ public class PeertubeStreamExtractor extends StreamExtractor {
@Override
public int getAgeLimit() throws ParsingException {
boolean isNSFW = JsonUtils.getBoolean(json, "nsfw");
final boolean isNSFW = JsonUtils.getBoolean(json, "nsfw");
if (isNSFW) {
return 18;
} else {
@ -105,39 +103,35 @@ public class PeertubeStreamExtractor extends StreamExtractor {
}
@Override
public long getLength() throws ParsingException {
Number value = JsonUtils.getNumber(json, "duration");
return value.longValue();
public long getLength() {
return json.getLong("duration");
}
@Override
public long getTimeStamp() throws ParsingException {
public long getTimeStamp() {
//TODO fetch timestamp from url if present;
return 0;
}
@Override
public long getViewCount() throws ParsingException {
Number value = JsonUtils.getNumber(json, "views");
return value.longValue();
public long getViewCount() {
return json.getLong("views");
}
@Override
public long getLikeCount() throws ParsingException {
Number value = JsonUtils.getNumber(json, "likes");
return value.longValue();
public long getLikeCount() {
return json.getLong("likes");
}
@Override
public long getDislikeCount() throws ParsingException {
Number value = JsonUtils.getNumber(json, "dislikes");
return value.longValue();
public long getDislikeCount() {
return json.getLong("dislikes");
}
@Override
public String getUploaderUrl() throws ParsingException {
String name = JsonUtils.getString(json, "account.name");
String host = JsonUtils.getString(json, "account.host");
final String name = JsonUtils.getString(json, "account.name");
final String host = JsonUtils.getString(json, "account.host");
return getService().getChannelLHFactory().fromId("accounts/" + name + "@" + host, baseUrl).getUrl();
}
@ -147,7 +141,7 @@ public class PeertubeStreamExtractor extends StreamExtractor {
}
@Override
public String getUploaderAvatarUrl() throws ParsingException {
public String getUploaderAvatarUrl() {
String value;
try {
value = JsonUtils.getString(json, "account.avatar.path");
@ -170,7 +164,7 @@ public class PeertubeStreamExtractor extends StreamExtractor {
@Nonnull
@Override
public String getSubChannelAvatarUrl() throws ParsingException {
public String getSubChannelAvatarUrl() {
String value;
try {
value = JsonUtils.getString(json, "channel.avatar.path");
@ -181,35 +175,35 @@ public class PeertubeStreamExtractor extends StreamExtractor {
}
@Override
public String getDashMpdUrl() throws ParsingException {
public String getDashMpdUrl() {
return "";
}
@Override
public String getHlsUrl() throws ParsingException {
public String getHlsUrl() {
return "";
}
@Override
public List<AudioStream> getAudioStreams() throws IOException, ExtractionException {
public List<AudioStream> getAudioStreams() {
return null;
}
@Override
public List<VideoStream> getVideoStreams() throws IOException, ExtractionException {
public List<VideoStream> getVideoStreams() throws ExtractionException {
assertPageFetched();
List<VideoStream> videoStreams = new ArrayList<>();
final List<VideoStream> videoStreams = new ArrayList<>();
try {
JsonArray streams = json.getArray("files");
for (Object s : streams) {
final JsonArray streams = json.getArray("files");
for (final Object s : streams) {
if (!(s instanceof JsonObject)) continue;
JsonObject stream = (JsonObject) s;
String url = JsonUtils.getString(stream, "fileUrl");
String torrentUrl = JsonUtils.getString(stream, "torrentUrl");
String resolution = JsonUtils.getString(stream, "resolution.label");
String extension = url.substring(url.lastIndexOf(".") + 1);
MediaFormat format = MediaFormat.getFromSuffix(extension);
VideoStream videoStream = new VideoStream(url, torrentUrl, format, resolution);
final JsonObject stream = (JsonObject) s;
final String url = JsonUtils.getString(stream, "fileUrl");
final String torrentUrl = JsonUtils.getString(stream, "torrentUrl");
final String resolution = JsonUtils.getString(stream, "resolution.label");
final String extension = url.substring(url.lastIndexOf(".") + 1);
final MediaFormat format = MediaFormat.getFromSuffix(extension);
final VideoStream videoStream = new VideoStream(url, torrentUrl, format, resolution);
if (!Stream.containSimilarStream(videoStream, videoStreams)) {
videoStreams.add(videoStream);
}
@ -223,20 +217,19 @@ public class PeertubeStreamExtractor extends StreamExtractor {
@Override
public List<VideoStream> getVideoOnlyStreams() throws IOException, ExtractionException {
// TODO Auto-generated method stub
return null;
public List<VideoStream> getVideoOnlyStreams() {
return Collections.emptyList();
}
@Override
public List<SubtitlesStream> getSubtitlesDefault() throws IOException, ExtractionException {
public List<SubtitlesStream> getSubtitlesDefault() {
return subtitles;
}
@Override
public List<SubtitlesStream> getSubtitles(final MediaFormat format) throws IOException, ExtractionException {
List<SubtitlesStream> filteredSubs = new ArrayList<>();
for (SubtitlesStream sub : subtitles) {
public List<SubtitlesStream> getSubtitles(final MediaFormat format) {
final List<SubtitlesStream> filteredSubs = new ArrayList<>();
for (final SubtitlesStream sub : subtitles) {
if (sub.getFormat() == format) {
filteredSubs.add(sub);
}
@ -245,20 +238,20 @@ public class PeertubeStreamExtractor extends StreamExtractor {
}
@Override
public StreamType getStreamType() throws ParsingException {
public StreamType getStreamType() {
return StreamType.VIDEO_STREAM;
}
@Override
public StreamInfoItem getNextStream() throws IOException, ExtractionException {
public StreamInfoItem getNextStream() {
return null;
}
@Override
public StreamInfoItemsCollector getRelatedStreams() throws IOException, ExtractionException {
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
List<String> tags = getTags();
String apiUrl = null;
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final List<String> tags = getTags();
final String apiUrl;
if (!tags.isEmpty()) {
apiUrl = getRelatedStreamsUrl(tags);
@ -280,7 +273,7 @@ public class PeertubeStreamExtractor extends StreamExtractor {
@Nonnull
@Override
public String getSupportInfo() throws ParsingException {
public String getSupportInfo() {
try {
return JsonUtils.getString(json, "support");
} catch (ParsingException e) {
@ -288,21 +281,21 @@ public class PeertubeStreamExtractor extends StreamExtractor {
}
}
private String getRelatedStreamsUrl(List<String> tags) throws UnsupportedEncodingException {
String url = baseUrl + PeertubeSearchQueryHandlerFactory.SEARCH_ENDPOINT;
StringBuilder params = new StringBuilder();
private String getRelatedStreamsUrl(final List<String> tags) throws UnsupportedEncodingException {
final String url = baseUrl + PeertubeSearchQueryHandlerFactory.SEARCH_ENDPOINT;
final StringBuilder params = new StringBuilder();
params.append("start=0&count=8&sort=-createdAt");
for (String tag : tags) {
for (final String tag : tags) {
params.append("&tagsOneOf=");
params.append(URLEncoder.encode(tag, "UTF-8"));
}
return url + "?" + params.toString();
}
private void getStreamsFromApi(StreamInfoItemsCollector collector, String apiUrl) throws ReCaptchaException, IOException, ParsingException {
Response response = getDownloader().get(apiUrl);
private void getStreamsFromApi(final StreamInfoItemsCollector collector, final String apiUrl) throws ReCaptchaException, IOException, ParsingException {
final Response response = getDownloader().get(apiUrl);
JsonObject relatedVideosJson = null;
if (null != response && !Utils.isBlank(response.responseBody())) {
if (response != null && !Utils.isBlank(response.responseBody())) {
try {
relatedVideosJson = JsonParser.object().from(response.responseBody());
} catch (JsonParserException e) {
@ -315,66 +308,64 @@ public class PeertubeStreamExtractor extends StreamExtractor {
}
}
private void collectStreamsFrom(StreamInfoItemsCollector collector, JsonObject json) throws ParsingException {
JsonArray contents;
private void collectStreamsFrom(final StreamInfoItemsCollector collector, final JsonObject json) throws ParsingException {
final JsonArray contents;
try {
contents = (JsonArray) JsonUtils.getValue(json, "data");
} catch (Exception e) {
throw new ParsingException("unable to extract related videos", e);
}
for (Object c : contents) {
for (final Object c : contents) {
if (c instanceof JsonObject) {
final JsonObject item = (JsonObject) c;
PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item, baseUrl);
final PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item, baseUrl);
//do not add the same stream in related streams
if (!extractor.getUrl().equals(getUrl())) collector.commit(extractor);
}
}
}
@Override
public String getErrorMessage() {
return null;
}
@Override
public void onFetchPage(Downloader downloader) throws IOException, ExtractionException {
Response response = downloader.get(getUrl());
if (null != response && null != response.responseBody()) {
public void onFetchPage(final Downloader downloader) throws IOException, ExtractionException {
final Response response = downloader.get(getUrl());
if (response != null && response.responseBody() != null) {
setInitialData(response.responseBody());
} else {
throw new ExtractionException("Unable to extract peertube channel data");
throw new ExtractionException("Unable to extract PeerTube channel data");
}
loadSubtitles();
}
private void setInitialData(String responseBody) throws ExtractionException {
private void setInitialData(final String responseBody) throws ExtractionException {
try {
json = JsonParser.object().from(responseBody);
} catch (JsonParserException e) {
throw new ExtractionException("Unable to extract peertube stream data", e);
throw new ExtractionException("Unable to extract PeerTube stream data", e);
}
if (null == json) throw new ExtractionException("Unable to extract peertube stream data");
if (json == null) throw new ExtractionException("Unable to extract PeerTube stream data");
PeertubeParsingHelper.validate(json);
}
private void loadSubtitles() {
if (subtitles.isEmpty()) {
try {
Response response = getDownloader().get(getUrl() + "/captions");
JsonObject captionsJson = JsonParser.object().from(response.responseBody());
JsonArray captions = JsonUtils.getArray(captionsJson, "data");
for (Object c : captions) {
final Response response = getDownloader().get(getUrl() + "/captions");
final JsonObject captionsJson = JsonParser.object().from(response.responseBody());
final JsonArray captions = JsonUtils.getArray(captionsJson, "data");
for (final Object c : captions) {
if (c instanceof JsonObject) {
JsonObject caption = (JsonObject) c;
String url = baseUrl + JsonUtils.getString(caption, "captionPath");
String languageCode = JsonUtils.getString(caption, "language.id");
String ext = url.substring(url.lastIndexOf(".") + 1);
MediaFormat fmt = MediaFormat.getFromSuffix(ext);
final JsonObject caption = (JsonObject) c;
final String url = baseUrl + JsonUtils.getString(caption, "captionPath");
final String languageCode = JsonUtils.getString(caption, "language.id");
final String ext = url.substring(url.lastIndexOf(".") + 1);
final MediaFormat fmt = MediaFormat.getFromSuffix(ext);
if (fmt != null && languageCode != null)
subtitles.add(new SubtitlesStream(fmt, languageCode, url, false));
}
@ -416,7 +407,7 @@ public class PeertubeStreamExtractor extends StreamExtractor {
}
@Override
public Locale getLanguageInfo() throws ParsingException {
public Locale getLanguageInfo() {
try {
return new Locale(JsonUtils.getString(json, "language.id"));
} catch (ParsingException e) {

27
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamInfoItemExtractor.java
View File

@ -1,6 +1,7 @@
package org.schabi.newpipe.extractor.services.peertube.extractors;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.ServiceList;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.localization.DateWrapper;
@ -10,24 +11,23 @@ import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.utils.JsonUtils;
public class PeertubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
protected final JsonObject item;
private final String baseUrl;
public PeertubeStreamInfoItemExtractor(JsonObject item, String baseUrl) {
public PeertubeStreamInfoItemExtractor(final JsonObject item, final String baseUrl) {
this.item = item;
this.baseUrl = baseUrl;
}
@Override
public String getUrl() throws ParsingException {
String uuid = JsonUtils.getString(item, "uuid");
final String uuid = JsonUtils.getString(item, "uuid");
return ServiceList.PeerTube.getStreamLHFactory().fromId(uuid, baseUrl).getUrl();
}
@Override
public String getThumbnailUrl() throws ParsingException {
String value = JsonUtils.getString(item, "thumbnailPath");
final String value = JsonUtils.getString(item, "thumbnailPath");
return baseUrl + value;
}
@ -37,20 +37,19 @@ public class PeertubeStreamInfoItemExtractor implements StreamInfoItemExtractor
}
@Override
public boolean isAd() throws ParsingException {
public boolean isAd() {
return false;
}
@Override
public long getViewCount() throws ParsingException {
Number value = JsonUtils.getNumber(item, "views");
return value.longValue();
public long getViewCount() {
return item.getLong("views");
}
@Override
public String getUploaderUrl() throws ParsingException {
String name = JsonUtils.getString(item, "account.name");
String host = JsonUtils.getString(item, "account.host");
final String name = JsonUtils.getString(item, "account.name");
final String host = JsonUtils.getString(item, "account.host");
return ServiceList.PeerTube.getChannelLHFactory().fromId("accounts/" + name + "@" + host, baseUrl).getUrl();
}
@ -77,14 +76,12 @@ public class PeertubeStreamInfoItemExtractor implements StreamInfoItemExtractor
}
@Override
public StreamType getStreamType() throws ParsingException {
public StreamType getStreamType() {
return StreamType.VIDEO_STREAM;
}
@Override
public long getDuration() throws ParsingException {
Number value = JsonUtils.getNumber(item, "duration");
return value.longValue();
public long getDuration() {
return item.getLong("duration");
}
}

21
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSubscriptionExtractor.java
View File

@ -1,21 +0,0 @@
package org.schabi.newpipe.extractor.services.peertube.extractors;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
import java.util.List;
public class PeertubeSubscriptionExtractor extends SubscriptionExtractor {
public PeertubeSubscriptionExtractor(StreamingService service, List<ContentSource> supportedSources) {
super(service, supportedSources);
// TODO Auto-generated constructor stub
}
@Override
public String getRelatedUrl() {
// TODO Auto-generated method stub
return null;
}
}

8
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSuggestionExtractor.java
View File

@ -1,22 +1,18 @@
package org.schabi.newpipe.extractor.services.peertube.extractors;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
public class PeertubeSuggestionExtractor extends SuggestionExtractor {
public PeertubeSuggestionExtractor(StreamingService service) {
public PeertubeSuggestionExtractor(final StreamingService service) {
super(service);
}
@Override
public List<String> suggestionList(String query) throws IOException, ExtractionException {
public List<String> suggestionList(final String query) {
return Collections.emptyList();
}
}

72
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java
View File

@ -1,9 +1,9 @@
package org.schabi.newpipe.extractor.services.peertube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
@ -14,19 +14,20 @@ import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.*;
import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.collectStreamsFrom;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class PeertubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
private InfoItemsPage<StreamInfoItem> initPage;
private long total;
public PeertubeTrendingExtractor(StreamingService streamingService, ListLinkHandler linkHandler, String kioskId) {
public PeertubeTrendingExtractor(final StreamingService streamingService, final ListLinkHandler linkHandler, final String kioskId) {
super(streamingService, linkHandler, kioskId);
}
@ -37,38 +38,18 @@ public class PeertubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
super.fetchPage();
return initPage;
}
private void collectStreamsFrom(StreamInfoItemsCollector collector, JsonObject json, String pageUrl) throws ParsingException {
JsonArray contents;
try {
contents = (JsonArray) JsonUtils.getValue(json, "data");
} catch (Exception e) {
throw new ParsingException("Unable to extract kiosk info", e);
}
String baseUrl = getBaseUrl();
for (Object c : contents) {
if (c instanceof JsonObject) {
final JsonObject item = (JsonObject) c;
PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item, baseUrl);
collector.commit(extractor);
}
}
final String pageUrl = getUrl() + "&" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE;
return getPage(new Page(pageUrl));
}
@Override
public String getNextPageUrl() throws IOException, ExtractionException {
super.fetchPage();
return initPage.getNextPageUrl();
}
public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException, ExtractionException {
if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL");
}
final Response response = getDownloader().get(page.getUrl());
@Override
public InfoItemsPage<StreamInfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
Response response = getDownloader().get(pageUrl);
JsonObject json = null;
if (response != null && !Utils.isBlank(response.responseBody())) {
try {
@ -78,20 +59,19 @@ public class PeertubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
}
}
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
if (json != null) {
Number number = JsonUtils.getNumber(json, "total");
if (number != null) this.total = number.longValue();
collectStreamsFrom(collector, json, pageUrl);
PeertubeParsingHelper.validate(json);
final long total = json.getLong("total");
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
collectStreamsFrom(collector, json, getBaseUrl());
return new InfoItemsPage<>(collector, PeertubeParsingHelper.getNextPage(page.getUrl(), total));
} else {
throw new ExtractionException("Unable to get peertube kiosk info");
throw new ExtractionException("Unable to get PeerTube kiosk info");
}
return new InfoItemsPage<>(collector, PeertubeParsingHelper.getNextPageUrl(pageUrl, total));
}
@Override
public void onFetchPage(Downloader downloader) throws IOException, ExtractionException {
this.initPage = getPage(getUrl() + "&" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE);
}
public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException { }
}

60
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelExtractor.java
View File

@ -1,9 +1,10 @@
package org.schabi.newpipe.extractor.services.soundcloud.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.downloader.Downloader;
@ -14,9 +15,10 @@ import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import javax.annotation.Nonnull;
import java.io.IOException;
import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
@ -25,21 +27,18 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
private String userId;
private JsonObject user;
private StreamInfoItemsCollector streamInfoItemsCollector = null;
private String nextPageUrl = null;
public SoundcloudChannelExtractor(StreamingService service, ListLinkHandler linkHandler) {
public SoundcloudChannelExtractor(final StreamingService service, final ListLinkHandler linkHandler) {
super(service, linkHandler);
}
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException {
userId = getLinkHandler().getId();
String apiUrl = "https://api-v2.soundcloud.com/users/" + userId +
final String apiUrl = "https://api-v2.soundcloud.com/users/" + userId +
"?client_id=" + SoundcloudParsingHelper.clientId();
String response = downloader.get(apiUrl, getExtractorLocalization()).responseBody();
final String response = downloader.get(apiUrl, getExtractorLocalization()).responseBody();
try {
user = JsonParser.object().from(response);
} catch (JsonParserException e) {
@ -76,7 +75,7 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
@Override
public long getSubscriberCount() {
return user.getNumber("followers_count", 0).longValue();
return user.getLong("followers_count", 0);
}
@Override
@ -85,61 +84,48 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
}
@Override
public String getParentChannelName() throws ParsingException {
public String getParentChannelName() {
return "";
}
@Override
public String getParentChannelUrl() throws ParsingException {
public String getParentChannelUrl() {
return "";
}
@Override
public String getParentChannelAvatarUrl() throws ParsingException {
public String getParentChannelAvatarUrl() {
return "";
}
@Nonnull
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws ExtractionException {
if (streamInfoItemsCollector == null) {
computeNextPageAndGetStreams();
}
return new InfoItemsPage<>(streamInfoItemsCollector, getNextPageUrl());
}
@Override
public String getNextPageUrl() throws ExtractionException {
if (nextPageUrl == null) {
computeNextPageAndGetStreams();
}
return nextPageUrl;
}
private void computeNextPageAndGetStreams() throws ExtractionException {
try {
streamInfoItemsCollector = new StreamInfoItemsCollector(getServiceId());
final StreamInfoItemsCollector streamInfoItemsCollector = new StreamInfoItemsCollector(getServiceId());
String apiUrl = "https://api-v2.soundcloud.com/users/" + getId() + "/tracks"
final String apiUrl = "https://api-v2.soundcloud.com/users/" + getId() + "/tracks"
+ "?client_id=" + SoundcloudParsingHelper.clientId()
+ "&limit=20"
+ "&linked_partitioning=1";
nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, streamInfoItemsCollector, apiUrl);
final String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, streamInfoItemsCollector, apiUrl);
return new InfoItemsPage<>(streamInfoItemsCollector, new Page(nextPageUrl));
} catch (Exception e) {
throw new ExtractionException("Could not get next page", e);
}
}
@Override
public InfoItemsPage<StreamInfoItem> getPage(final String pageUrl) throws IOException, ExtractionException {
if (isNullOrEmpty(pageUrl)) {
throw new ExtractionException(new IllegalArgumentException("Page url is empty or null"));
public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException, ExtractionException {
if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL");
}
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, pageUrl);
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, page.getUrl());
return new InfoItemsPage<>(collector, nextPageUrl);
return new InfoItemsPage<>(collector, new Page(nextPageUrl));
}
}

4
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelInfoItemExtractor.java
View File

@ -32,12 +32,12 @@ public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtrac
@Override
public long getSubscriberCount() {
return itemObject.getNumber("followers_count", 0).longValue();
return itemObject.getLong("followers_count");
}
@Override
public long getStreamCount() {
return itemObject.getNumber("track_count", 0).longValue();
return itemObject.getLong("track_count");
}
@Override

48
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChartsExtractor.java
View File

@ -1,5 +1,6 @@
package org.schabi.newpipe.extractor.services.soundcloud.extractors;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
@ -9,16 +10,14 @@ import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import javax.annotation.Nonnull;
import java.io.IOException;
import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class SoundcloudChartsExtractor extends KioskExtractor<StreamInfoItem> {
private StreamInfoItemsCollector collector = null;
private String nextPageUrl = null;
public SoundcloudChartsExtractor(StreamingService service,
ListLinkHandler linkHandler,
String kioskId) {
@ -36,20 +35,21 @@ public class SoundcloudChartsExtractor extends KioskExtractor<StreamInfoItem> {
}
@Override
public InfoItemsPage<StreamInfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
if (isNullOrEmpty(pageUrl)) {
throw new ExtractionException(new IllegalArgumentException("Page url is empty or null"));
public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException, ExtractionException {
if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL");
}
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, pageUrl, true);
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, page.getUrl(), true);
return new InfoItemsPage<>(collector, nextPageUrl);
return new InfoItemsPage<>(collector, new Page(nextPageUrl));
}
private void computeNextPageAndStreams() throws IOException, ExtractionException {
collector = new StreamInfoItemsCollector(getServiceId());
@Nonnull
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
String apiUrl = "https://api-v2.soundcloud.com/charts" +
"?genre=soundcloud:genres:all-music" +
@ -61,27 +61,11 @@ public class SoundcloudChartsExtractor extends KioskExtractor<StreamInfoItem> {
apiUrl += "&kind=trending";
}
String contentCountry = SoundCloud.getContentCountry().getCountryCode();
final String contentCountry = SoundCloud.getContentCountry().getCountryCode();
apiUrl += "&region=soundcloud:regions:" + contentCountry;
nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl, true);
}
final String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl, true);
@Override
public String getNextPageUrl() throws IOException, ExtractionException {
if (nextPageUrl == null) {
computeNextPageAndStreams();
}
return nextPageUrl;
}
@Nonnull
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
if (collector == null) {
computeNextPageAndStreams();
}
return new InfoItemsPage<>(collector, getNextPageUrl());
return new InfoItemsPage<>(collector, new Page(nextPageUrl));
}
}

79
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsExtractor.java
View File

@ -4,7 +4,9 @@ import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.comments.CommentsExtractor;
import org.schabi.newpipe.extractor.comments.CommentsInfoItem;
@ -15,58 +17,63 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import javax.annotation.Nonnull;
import java.io.IOException;
import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class SoundcloudCommentsExtractor extends CommentsExtractor {
private JsonObject json;
public SoundcloudCommentsExtractor(StreamingService service, ListLinkHandler uiHandler) {
public SoundcloudCommentsExtractor(final StreamingService service, final ListLinkHandler uiHandler) {
super(service, uiHandler);
}
@Nonnull
@Override
public InfoItemsPage<CommentsInfoItem> getInitialPage() throws IOException, ExtractionException {
final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId());
public InfoItemsPage<CommentsInfoItem> getInitialPage() throws ExtractionException, IOException {
final Downloader downloader = NewPipe.getDownloader();
final Response response = downloader.get(getUrl());
collectStreamsFrom(collector, json.getArray("collection"));
return new InfoItemsPage<>(collector, getNextPageUrl());
}
@Override
public String getNextPageUrl() throws IOException, ExtractionException {
return json.getString("next_href");
}
@Override
public InfoItemsPage<CommentsInfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
Downloader dl = NewPipe.getDownloader();
Response rp = dl.get(pageUrl);
try {
json = JsonParser.object().from(rp.responseBody());
} catch (JsonParserException e) {
throw new ParsingException("Could not parse json", e);
}
final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId());
collectStreamsFrom(collector, json.getArray("collection"));
return new InfoItemsPage<>(collector, getNextPageUrl());
}
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
Response response = downloader.get(getUrl());
final JsonObject json;
try {
json = JsonParser.object().from(response.responseBody());
} catch (JsonParserException e) {
throw new ParsingException("Could not parse json", e);
}
final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId());
collectStreamsFrom(collector, json.getArray("collection"));
return new InfoItemsPage<>(collector, new Page(json.getString("next_href")));
}
@Override
public InfoItemsPage<CommentsInfoItem> getPage(final Page page) throws ExtractionException, IOException {
if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL");
}
final Downloader downloader = NewPipe.getDownloader();
final Response response = downloader.get(page.getUrl());
final JsonObject json;
try {
json = JsonParser.object().from(response.responseBody());
} catch (JsonParserException e) {
throw new ParsingException("Could not parse json", e);
}
final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId());
collectStreamsFrom(collector, json.getArray("collection"));
return new InfoItemsPage<>(collector, new Page(json.getString("next_href")));
}
@Override
public void onFetchPage(@Nonnull final Downloader downloader) { }
private void collectStreamsFrom(final CommentsInfoItemsCollector collector, final JsonArray entries) throws ParsingException {
final String url = getUrl();
for (Object comment : entries) {

21
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java
View File

@ -6,10 +6,11 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper;
import java.util.Objects;
import javax.annotation.Nullable;
public class SoundcloudCommentsInfoItemExtractor implements CommentsInfoItemExtractor {
private JsonObject json;
private String url;
@ -19,32 +20,32 @@ public class SoundcloudCommentsInfoItemExtractor implements CommentsInfoItemExtr
}
@Override
public String getCommentId() throws ParsingException {
return json.getNumber("id").toString();
public String getCommentId() {
return Objects.toString(json.getLong("id"), null);
}
@Override
public String getCommentText() throws ParsingException {
public String getCommentText() {
return json.getString("body");
}
@Override
public String getUploaderName() throws ParsingException {
public String getUploaderName() {
return json.getObject("user").getString("username");
}
@Override
public String getUploaderAvatarUrl() throws ParsingException {
public String getUploaderAvatarUrl() {
return json.getObject("user").getString("avatar_url");
}
@Override
public String getUploaderUrl() throws ParsingException {
public String getUploaderUrl() {
return json.getObject("user").getString("permalink_url");
}
@Override
public String getTextualUploadDate() throws ParsingException {
public String getTextualUploadDate() {
return json.getString("created_at");
}
@ -55,7 +56,7 @@ public class SoundcloudCommentsInfoItemExtractor implements CommentsInfoItemExtr
}
@Override
public int getLikeCount() throws ParsingException {
public int getLikeCount() {
return -1;
}
@ -70,7 +71,7 @@ public class SoundcloudCommentsInfoItemExtractor implements CommentsInfoItemExtr
}
@Override
public String getThumbnailUrl() throws ParsingException {
public String getThumbnailUrl() {
return json.getObject("user").getString("avatar_url");
}
}

95
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistExtractor.java
View File

@ -6,6 +6,7 @@ import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
@ -15,24 +16,23 @@ import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
@SuppressWarnings("WeakerAccess")
public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
private static final int streamsPerRequestedPage = 15;
private static final int STREAMS_PER_REQUESTED_PAGE = 15;
private String playlistId;
private JsonObject playlist;
private StreamInfoItemsCollector streamInfoItemsCollector;
private String nextPageUrl;
public SoundcloudPlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) {
super(service, linkHandler);
}
@ -113,98 +113,73 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
@Override
public long getStreamCount() {
return playlist.getNumber("track_count", 0).longValue();
return playlist.getLong("track_count");
}
@Nonnull
@Override
public String getSubChannelName() throws ParsingException {
public String getSubChannelName() {
return "";
}
@Nonnull
@Override
public String getSubChannelUrl() throws ParsingException {
public String getSubChannelUrl() {
return "";
}
@Nonnull
@Override
public String getSubChannelAvatarUrl() throws ParsingException {
public String getSubChannelAvatarUrl() {
return "";
}
@Nonnull
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
if (streamInfoItemsCollector == null) {
computeInitialTracksAndNextPageUrl();
}
return new InfoItemsPage<>(streamInfoItemsCollector, nextPageUrl);
}
public InfoItemsPage<StreamInfoItem> getInitialPage() {
final StreamInfoItemsCollector streamInfoItemsCollector = new StreamInfoItemsCollector(getServiceId());
final List<String> ids = new ArrayList<>();
private void computeInitialTracksAndNextPageUrl() throws IOException, ExtractionException {
streamInfoItemsCollector = new StreamInfoItemsCollector(getServiceId());
StringBuilder nextPageUrlBuilder = new StringBuilder("https://api-v2.soundcloud.com/tracks?client_id=");
nextPageUrlBuilder.append(SoundcloudParsingHelper.clientId());
nextPageUrlBuilder.append("&ids=");
JsonArray tracks = playlist.getArray("tracks");
final JsonArray tracks = playlist.getArray("tracks");
for (Object o : tracks) {
if (o instanceof JsonObject) {
JsonObject track = (JsonObject) o;
final JsonObject track = (JsonObject) o;
if (track.has("title")) { // i.e. if full info is available
streamInfoItemsCollector.commit(new SoundcloudStreamInfoItemExtractor(track));
} else {
// %09d would be enough, but a 0 before the number does not create problems, so let's be sure
nextPageUrlBuilder.append(String.format("%010d,", track.getInt("id")));
ids.add(String.format("%010d", track.getInt("id")));
}
}
}
nextPageUrlBuilder.setLength(nextPageUrlBuilder.length() - 1); // remove trailing ,
nextPageUrl = nextPageUrlBuilder.toString();
if (nextPageUrl.endsWith("&ids")) {
// there are no other videos
nextPageUrl = "";
}
return new InfoItemsPage<>(streamInfoItemsCollector, new Page(ids));
}
@Override
public String getNextPageUrl() throws IOException, ExtractionException {
if (nextPageUrl == null) {
computeInitialTracksAndNextPageUrl();
}
return nextPageUrl;
}
@Override
public InfoItemsPage<StreamInfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
if (isNullOrEmpty(pageUrl)) {
throw new ExtractionException(new IllegalArgumentException("Page url is empty or null"));
public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException, ExtractionException {
if (page == null || isNullOrEmpty(page.getIds())) {
throw new IllegalArgumentException("Page doesn't contain IDs");
}
// see computeInitialTracksAndNextPageUrl
final int lengthFirstPartOfUrl = ("https://api-v2.soundcloud.com/tracks?client_id="
+ SoundcloudParsingHelper.clientId()
+ "&ids=").length();
final int lengthOfEveryStream = 11;
String currentPageUrl, nextUrl;
int lengthMaxStreams = lengthFirstPartOfUrl + lengthOfEveryStream * streamsPerRequestedPage;
if (pageUrl.length() <= lengthMaxStreams) {
currentPageUrl = pageUrl; // fetch every remaining video, there are less than the max
nextUrl = ""; // afterwards the list is complete
final List<String> currentIds;
final List<String> nextIds;
if (page.getIds().size() <= STREAMS_PER_REQUESTED_PAGE) {
// Fetch every remaining stream, there are less than the max
currentIds = page.getIds();
nextIds = null;
} else {
currentPageUrl = pageUrl.substring(0, lengthMaxStreams);
nextUrl = pageUrl.substring(0, lengthFirstPartOfUrl) + pageUrl.substring(lengthMaxStreams);
currentIds = page.getIds().subList(0, STREAMS_PER_REQUESTED_PAGE);
nextIds = page.getIds().subList(STREAMS_PER_REQUESTED_PAGE, page.getIds().size());
}
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
String response = NewPipe.getDownloader().get(currentPageUrl, getExtractorLocalization()).responseBody();
final String currentPageUrl = "https://api-v2.soundcloud.com/tracks?client_id="
+ SoundcloudParsingHelper.clientId()
+ "&ids=" + Utils.join(",", currentIds);
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final String response = NewPipe.getDownloader().get(currentPageUrl, getExtractorLocalization()).responseBody();
try {
JsonArray tracks = JsonParser.array().from(response);
final JsonArray tracks = JsonParser.array().from(response);
for (Object track : tracks) {
if (track instanceof JsonObject) {
collector.commit(new SoundcloudStreamInfoItemExtractor((JsonObject) track));
@ -214,6 +189,6 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
throw new ParsingException("Could not parse json response", e);
}
return new InfoItemsPage<>(collector, nextUrl);
return new InfoItemsPage<>(collector, new Page(nextIds));
}
}

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistInfoItemExtractor.java
View File

@ -81,6 +81,6 @@ public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtr
@Override
public long getStreamCount() {
return itemObject.getNumber("track_count", 0).longValue();
return itemObject.getLong("track_count");
}
}

29
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java
View File

@ -4,9 +4,11 @@ import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.InfoItemExtractor;
import org.schabi.newpipe.extractor.InfoItemsCollector;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
@ -16,17 +18,18 @@ import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.utils.Parser;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.net.URL;
import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.services.soundcloud.linkHandler.SoundcloudSearchQueryHandlerFactory.ITEMS_PER_PAGE;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class SoundcloudSearchExtractor extends SearchExtractor {
private JsonArray searchCollection;
public SoundcloudSearchExtractor(StreamingService service, SearchQueryHandler linkHandler) {
@ -47,25 +50,24 @@ public class SoundcloudSearchExtractor extends SearchExtractor {
@Nonnull
@Override
public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionException {
return new InfoItemsPage<>(collectItems(searchCollection), getNextPageUrl());
return new InfoItemsPage<>(collectItems(searchCollection), getNextPageFromCurrentUrl(getUrl()));
}
@Override
public String getNextPageUrl() throws IOException, ExtractionException {
return getNextPageUrlFromCurrentUrl(getUrl());
}
public InfoItemsPage<InfoItem> getPage(final Page page) throws IOException, ExtractionException {
if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL");
}
@Override
public InfoItemsPage<InfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
final Downloader dl = getDownloader();
try {
final String response = dl.get(pageUrl, getExtractorLocalization()).responseBody();
final String response = dl.get(page.getUrl(), getExtractorLocalization()).responseBody();
searchCollection = JsonParser.object().from(response).getArray("collection");
} catch (JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}
return new InfoItemsPage<>(collectItems(searchCollection), getNextPageUrlFromCurrentUrl(pageUrl));
return new InfoItemsPage<>(collectItems(searchCollection), getNextPageFromCurrentUrl(page.getUrl()));
}
@Override
@ -108,7 +110,7 @@ public class SoundcloudSearchExtractor extends SearchExtractor {
return collector;
}
private String getNextPageUrlFromCurrentUrl(String currentUrl)
private Page getNextPageFromCurrentUrl(String currentUrl)
throws MalformedURLException, UnsupportedEncodingException {
final int pageOffset = Integer.parseInt(
Parser.compatParseMap(
@ -116,8 +118,7 @@ public class SoundcloudSearchExtractor extends SearchExtractor {
.getQuery())
.get("offset"));
return currentUrl.replace("&offset=" +
Integer.toString(pageOffset),
"&offset=" + Integer.toString(pageOffset + ITEMS_PER_PAGE));
return new Page(currentUrl.replace("&offset=" + pageOffset,
"&offset=" + (pageOffset + ITEMS_PER_PAGE)));
}
}

6
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java
View File

@ -102,7 +102,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
@Override
public long getLength() {
return track.getNumber("duration", 0).longValue() / 1000L;
return track.getLong("duration") / 1000L;
}
@Override
@ -112,12 +112,12 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
@Override
public long getViewCount() {
return track.getNumber("playback_count", 0).longValue();
return track.getLong("playback_count");
}
@Override
public long getLikeCount() {
return track.getNumber("favoritings_count", -1).longValue();
return track.getLong("favoritings_count", -1);
}
@Override

8
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamInfoItemExtractor.java
View File

@ -30,7 +30,7 @@ public class SoundcloudStreamInfoItemExtractor implements StreamInfoItemExtracto
@Override
public long getDuration() {
return itemObject.getNumber("duration", 0).longValue() / 1000L;
return itemObject.getLong("duration") / 1000L;
}
@Override
@ -53,13 +53,9 @@ public class SoundcloudStreamInfoItemExtractor implements StreamInfoItemExtracto
return new DateWrapper(SoundcloudParsingHelper.parseDateFrom(getTextualUploadDate()));
}
private String getCreatedAt() {
return itemObject.getString("created_at");
}
@Override
public long getViewCount() {
return itemObject.getNumber("playback_count", 0).longValue();
return itemObject.getLong("playback_count");
}
@Override

60
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java
View File

@ -2,6 +2,8 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.downloader.Downloader;
@ -16,10 +18,13 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull;
import java.io.IOException;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
@ -104,15 +109,6 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
YoutubeParsingHelper.defaultAlertsCheck(initialData);
}
@Override
public String getNextPageUrl() throws ExtractionException {
if (getVideoTab() == null) return "";
return getNextPageUrlFrom(getVideoTab().getObject("content").getObject("sectionListRenderer")
.getArray("contents").getObject(0).getObject("itemSectionRenderer")
.getArray("contents").getObject(0).getObject("gridRenderer").getArray("continuations"));
}
@Nonnull
@Override
public String getUrl() throws ParsingException {
@ -231,22 +227,27 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Nonnull
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws ExtractionException {
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
Page nextPage = null;
if (getVideoTab() != null) {
JsonArray videos = getVideoTab().getObject("content").getObject("sectionListRenderer").getArray("contents")
.getObject(0).getObject("itemSectionRenderer").getArray("contents").getObject(0)
.getObject("gridRenderer").getArray("items");
collectStreamsFrom(collector, videos);
final JsonObject gridRenderer = getVideoTab().getObject("content").getObject("sectionListRenderer")
.getArray("contents").getObject(0).getObject("itemSectionRenderer")
.getArray("contents").getObject(0).getObject("gridRenderer");
collectStreamsFrom(collector, gridRenderer.getArray("items"));
nextPage = getNextPageFrom(gridRenderer.getArray("continuations"));
}
return new InfoItemsPage<>(collector, getNextPageUrl());
return new InfoItemsPage<>(collector, nextPage);
}
@Override
public InfoItemsPage<StreamInfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
if (isNullOrEmpty(pageUrl)) {
throw new ExtractionException(new IllegalArgumentException("Page url is empty or null"));
public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException, ExtractionException {
if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL");
}
// Unfortunately, we have to fetch the page even if we are only getting next streams,
@ -254,27 +255,26 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
fetchPage();
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization());
final JsonArray ajaxJson = getJsonResponse(page.getUrl(), getExtractorLocalization());
JsonObject sectionListContinuation = ajaxJson.getObject(1).getObject("response")
.getObject("continuationContents").getObject("gridContinuation");
collectStreamsFrom(collector, sectionListContinuation.getArray("items"));
return new InfoItemsPage<>(collector, getNextPageUrlFrom(sectionListContinuation.getArray("continuations")));
return new InfoItemsPage<>(collector, getNextPageFrom(sectionListContinuation.getArray("continuations")));
}
private String getNextPageUrlFrom(JsonArray continuations) {
private Page getNextPageFrom(final JsonArray continuations) {
if (isNullOrEmpty(continuations)) {
return "";
return null;
}
JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData");
String continuation = nextContinuationData.getString("continuation");
String clickTrackingParams = nextContinuationData.getString("clickTrackingParams");
return "https://www.youtube.com/browse_ajax?ctoken=" + continuation + "&continuation=" + continuation
+ "&itct=" + clickTrackingParams;
final JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData");
final String continuation = nextContinuationData.getString("continuation");
final String clickTrackingParams = nextContinuationData.getString("clickTrackingParams");
return new Page("https://www.youtube.com/browse_ajax?ctoken=" + continuation
+ "&continuation=" + continuation + "&itct=" + clickTrackingParams);
}
private void collectStreamsFrom(StreamInfoItemsCollector collector, JsonArray videos) throws ParsingException {

74
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java
View File

@ -3,6 +3,8 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.comments.CommentsExtractor;
import org.schabi.newpipe.extractor.comments.CommentsInfoItem;
@ -17,7 +19,6 @@ import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Parser;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
@ -26,19 +27,19 @@ import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;
import javax.annotation.Nonnull;
import static java.util.Collections.singletonList;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class YoutubeCommentsExtractor extends CommentsExtractor {
// using the mobile site for comments because it loads faster and uses get requests instead of post
private static final String USER_AGENT = "Mozilla/5.0 (Android 8.1.0; Mobile; rv:62.0) Gecko/62.0 Firefox/62.0";
private static final Pattern YT_CLIENT_NAME_PATTERN = Pattern.compile("INNERTUBE_CONTEXT_CLIENT_NAME\\\":(.*?)[,}]");
private String ytClientVersion;
private String ytClientName;
private InfoItemsPage<CommentsInfoItem> initPage;
private String responseBody;
public YoutubeCommentsExtractor(StreamingService service, ListLinkHandler uiHandler) {
super(service, uiHandler);
@ -46,56 +47,49 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
@Override
public InfoItemsPage<CommentsInfoItem> getInitialPage() throws IOException, ExtractionException {
// initial page does not load any comments but is required to get comments token
super.fetchPage();
return initPage;
String commentsTokenInside = findValue(responseBody, "commentSectionRenderer", "}");
String commentsToken = findValue(commentsTokenInside, "continuation\":\"", "\"");
return getPage(getNextPage(commentsToken));
}
@Override
public String getNextPageUrl() throws IOException, ExtractionException {
// initial page does not load any comments but is required to get comments token
super.fetchPage();
return initPage.getNextPageUrl();
}
private String getNextPageUrl(JsonObject ajaxJson) throws IOException, ParsingException {
private Page getNextPage(JsonObject ajaxJson) throws ParsingException {
JsonArray arr;
try {
arr = JsonUtils.getArray(ajaxJson, "response.continuationContents.commentSectionContinuation.continuations");
} catch (Exception e) {
return "";
return null;
}
if (arr.isEmpty()) {
return "";
return null;
}
String continuation;
try {
continuation = JsonUtils.getString(arr.getObject(0), "nextContinuationData.continuation");
} catch (Exception e) {
return "";
return null;
}
return getNextPageUrl(continuation);
return getNextPage(continuation);
}
private String getNextPageUrl(String continuation) throws ParsingException {
private Page getNextPage(String continuation) throws ParsingException {
Map<String, String> params = new HashMap<>();
params.put("action_get_comments", "1");
params.put("pbj", "1");
params.put("ctoken", continuation);
try {
return "https://m.youtube.com/watch_comment?" + getDataString(params);
return new Page("https://m.youtube.com/watch_comment?" + getDataString(params));
} catch (UnsupportedEncodingException e) {
throw new ParsingException("Could not get next page url", e);
}
}
@Override
public InfoItemsPage<CommentsInfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
if (isNullOrEmpty(pageUrl)) {
throw new ExtractionException(new IllegalArgumentException("Page url is empty or null"));
public InfoItemsPage<CommentsInfoItem> getPage(final Page page) throws IOException, ExtractionException {
if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL");
}
String ajaxResponse = makeAjaxRequest(pageUrl);
String ajaxResponse = makeAjaxRequest(page.getUrl());
JsonObject ajaxJson;
try {
ajaxJson = JsonParser.array().from(ajaxResponse).getObject(1);
@ -104,11 +98,10 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
}
CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId());
collectCommentsFrom(collector, ajaxJson);
return new InfoItemsPage<>(collector, getNextPageUrl(ajaxJson));
return new InfoItemsPage<>(collector, getNextPage(ajaxJson));
}
private void collectCommentsFrom(CommentsInfoItemsCollector collector, JsonObject ajaxJson) throws ParsingException {
JsonArray contents;
try {
contents = JsonUtils.getArray(ajaxJson, "response.continuationContents.commentSectionContinuation.items");
@ -136,16 +129,13 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
final Map<String, List<String>> requestHeaders = new HashMap<>();
requestHeaders.put("User-Agent", singletonList(USER_AGENT));
final Response response = downloader.get(getUrl(), requestHeaders, getExtractorLocalization());
String responseBody = response.responseBody();
responseBody = response.responseBody();
ytClientVersion = findValue(responseBody, "INNERTUBE_CONTEXT_CLIENT_VERSION\":\"", "\"");
ytClientName = Parser.matchGroup1(YT_CLIENT_NAME_PATTERN, responseBody);
String commentsTokenInside = findValue(responseBody, "commentSectionRenderer", "}");
String commentsToken = findValue(commentsTokenInside, "continuation\":\"", "\"");
initPage = getPage(getNextPageUrl(commentsToken));
}
private String makeAjaxRequest(String siteUrl) throws IOException, ReCaptchaException {
private String makeAjaxRequest(String siteUrl) throws IOException, ReCaptchaException {
Map<String, List<String>> requestHeaders = new HashMap<>();
requestHeaders.put("Accept", singletonList("*/*"));
requestHeaders.put("User-Agent", singletonList(USER_AGENT));
@ -174,22 +164,4 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
int endIndex = doc.indexOf(end, beginIndex);
return doc.substring(beginIndex, endIndex);
}
public static String getYoutubeText(@Nonnull JsonObject object) throws ParsingException {
try {
return JsonUtils.getString(object, "simpleText");
} catch (Exception e1) {
try {
JsonArray arr = JsonUtils.getArray(object, "runs");
String result = "";
for (int i = 0; i < arr.size(); i++) {
result = result + JsonUtils.getString(arr.getObject(i), "text");
}
return result;
} catch (Exception e2) {
return "";
}
}
}
}

13
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java
View File

@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.localization.DateWrapper;
@ -11,7 +12,7 @@ import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtractor {
@ -43,7 +44,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
@Override
public String getName() throws ParsingException {
try {
return YoutubeCommentsExtractor.getYoutubeText(JsonUtils.getObject(json, "authorText"));
return getTextFromObject(JsonUtils.getObject(json, "authorText"));
} catch (Exception e) {
return "";
}
@ -52,7 +53,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
@Override
public String getTextualUploadDate() throws ParsingException {
try {
return YoutubeCommentsExtractor.getYoutubeText(JsonUtils.getObject(json, "publishedTimeText"));
return getTextFromObject(JsonUtils.getObject(json, "publishedTimeText"));
} catch (Exception e) {
throw new ParsingException("Could not get publishedTimeText", e);
}
@ -72,7 +73,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
@Override
public int getLikeCount() throws ParsingException {
try {
return JsonUtils.getNumber(json, "likeCount").intValue();
return json.getInt("likeCount");
} catch (Exception e) {
throw new ParsingException("Could not get like count", e);
}
@ -81,7 +82,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
@Override
public String getCommentText() throws ParsingException {
try {
String commentText = YoutubeCommentsExtractor.getYoutubeText(JsonUtils.getObject(json, "contentText"));
String commentText = getTextFromObject(JsonUtils.getObject(json, "contentText"));
// youtube adds U+FEFF in some comments. eg. https://www.youtube.com/watch?v=Nj4F63E59io<feff>
return Utils.removeUTF8BOM(commentText);
} catch (Exception e) {
@ -111,7 +112,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
@Override
public String getUploaderName() throws ParsingException {
try {
return YoutubeCommentsExtractor.getYoutubeText(JsonUtils.getObject(json, "authorText"));
return getTextFromObject(JsonUtils.getObject(json, "authorText"));
} catch (Exception e) {
return "";
}

18
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedExtractor.java
View File

@ -5,6 +5,7 @@ import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
@ -15,9 +16,10 @@ import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import javax.annotation.Nonnull;
import java.io.IOException;
import javax.annotation.Nonnull;
public class YoutubeFeedExtractor extends FeedExtractor {
public YoutubeFeedExtractor(StreamingService service, ListLinkHandler linkHandler) {
super(service, linkHandler);
@ -66,17 +68,7 @@ public class YoutubeFeedExtractor extends FeedExtractor {
}
@Override
public String getNextPageUrl() {
return null;
}
@Override
public InfoItemsPage<StreamInfoItem> getPage(String pageUrl) {
return null;
}
@Override
public boolean hasNextPage() {
return false;
public InfoItemsPage<StreamInfoItem> getPage(final Page page) {
return InfoItemsPage.emptyPage();
}
}

41
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java
View File

@ -7,6 +7,7 @@ import com.grack.nanojson.JsonParserException;
import com.grack.nanojson.JsonWriter;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
@ -169,32 +170,25 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
final JsonArray contents = initialData.getObject("contents").getObject("sectionListRenderer").getArray("contents");
for (Object content : contents) {
if (((JsonObject) content).has("musicShelfRenderer")) {
collectMusicStreamsFrom(collector, ((JsonObject) content).getObject("musicShelfRenderer").getArray("contents"));
}
}
return new InfoItemsPage<>(collector, getNextPageUrl());
}
@Override
public String getNextPageUrl() throws ExtractionException, IOException {
final JsonArray contents = initialData.getObject("contents").getObject("sectionListRenderer").getArray("contents");
Page nextPage = null;
for (Object content : contents) {
if (((JsonObject) content).has("musicShelfRenderer")) {
return getNextPageUrlFrom(((JsonObject) content).getObject("musicShelfRenderer").getArray("continuations"));
final JsonObject musicShelfRenderer = ((JsonObject) content).getObject("musicShelfRenderer");
collectMusicStreamsFrom(collector, musicShelfRenderer.getArray("contents"));
nextPage = getNextPageFrom(musicShelfRenderer.getArray("continuations"));
}
}
return "";
return new InfoItemsPage<>(collector, nextPage);
}
@Override
public InfoItemsPage<InfoItem> getPage(final String pageUrl) throws IOException, ExtractionException {
if (isNullOrEmpty(pageUrl)) {
throw new ExtractionException(new IllegalArgumentException("Page url is empty or null"));
public InfoItemsPage<InfoItem> getPage(final Page page) throws IOException, ExtractionException {
if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL");
}
final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId());
@ -236,7 +230,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
headers.put("Referer", Collections.singletonList("music.youtube.com"));
headers.put("Content-Type", Collections.singletonList("application/json"));
final String responseBody = getValidJsonResponseBody(getDownloader().post(pageUrl, headers, json));
final String responseBody = getValidJsonResponseBody(getDownloader().post(page.getUrl(), headers, json));
final JsonObject ajaxJson;
try {
@ -250,7 +244,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
collectMusicStreamsFrom(collector, musicShelfContinuation.getArray("contents"));
final JsonArray continuations = musicShelfContinuation.getArray("continuations");
return new InfoItemsPage<>(collector, getNextPageUrlFrom(continuations));
return new InfoItemsPage<>(collector, getNextPageFrom(continuations));
}
private void collectMusicStreamsFrom(final InfoItemsSearchCollector collector, final JsonArray videos) {
@ -495,16 +489,17 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
}
}
private String getNextPageUrlFrom(final JsonArray continuations) throws ParsingException, IOException, ReCaptchaException {
private Page getNextPageFrom(final JsonArray continuations) throws ParsingException, IOException, ReCaptchaException {
if (isNullOrEmpty(continuations)) {
return "";
return null;
}
final JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData");
final String continuation = nextContinuationData.getString("continuation");
final String clickTrackingParams = nextContinuationData.getString("clickTrackingParams");
return "https://music.youtube.com/youtubei/v1/search?ctoken=" + continuation + "&continuation=" + continuation
+ "&itct=" + clickTrackingParams + "&alt=json&key=" + YoutubeParsingHelper.getYoutubeMusicKeys()[0];
return new Page("https://music.youtube.com/youtubei/v1/search?ctoken=" + continuation
+ "&continuation=" + continuation + "&itct=" + clickTrackingParams + "&alt=json"
+ "&key=" + YoutubeParsingHelper.getYoutubeMusicKeys()[0]);
}
}

56
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java
View File

@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
@ -53,7 +54,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
}
private JsonObject getUploaderInfo() throws ParsingException {
JsonArray items = initialData.getObject("sidebar").getObject("playlistSidebarRenderer").getArray("items");
final JsonArray items = initialData.getObject("sidebar").getObject("playlistSidebarRenderer").getArray("items");
JsonObject videoOwner = items.getObject(1).getObject("playlistSidebarSecondaryInfoRenderer").getObject("videoOwner");
if (videoOwner.has("videoOwnerRenderer")) {
@ -77,19 +78,10 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
}
}
@Override
public String getNextPageUrl() {
return getNextPageUrlFrom(initialData.getObject("contents").getObject("twoColumnBrowseResultsRenderer")
.getArray("tabs").getObject(0).getObject("tabRenderer").getObject("content")
.getObject("sectionListRenderer").getArray("contents").getObject(0)
.getObject("itemSectionRenderer").getArray("contents").getObject(0)
.getObject("playlistVideoListRenderer").getArray("continuations"));
}
@Nonnull
@Override
public String getName() throws ParsingException {
String name = getTextFromObject(playlistInfo.getObject("title"));
final String name = getTextFromObject(playlistInfo.getObject("title"));
if (name != null && !name.isEmpty()) return name;
return initialData.getObject("microformat").getObject("microformatDataRenderer").getString("title");
@ -138,7 +130,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Override
public String getUploaderAvatarUrl() throws ParsingException {
try {
String url = getUploaderInfo().getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url");
final String url = getUploaderInfo().getObject("thumbnail").getArray("thumbnails").getObject(0).getString("url");
return fixThumbnailUrl(url);
} catch (Exception e) {
@ -149,7 +141,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Override
public long getStreamCount() throws ParsingException {
try {
String viewsText = getTextFromObject(getPlaylistInfo().getArray("stats").getObject(0));
final String viewsText = getTextFromObject(getPlaylistInfo().getArray("stats").getObject(0));
return Long.parseLong(Utils.removeNonDigitCharacters(viewsText));
} catch (Exception e) {
throw new ParsingException("Could not get video count from playlist", e);
@ -178,6 +170,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() {
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
Page nextPage = null;
final JsonArray contents = initialData.getObject("contents").getObject("twoColumnBrowseResultsRenderer")
.getArray("tabs").getObject(0).getObject("tabRenderer").getObject("content")
@ -193,48 +186,51 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
.getObject("videoList").getObject("playlistVideoListRenderer").getArray("contents"));
}
}
return new InfoItemsPage<>(collector, null);
} else if (contents.getObject(0).has("playlistVideoListRenderer")) {
final JsonArray videos = contents.getObject(0)
.getObject("playlistVideoListRenderer").getArray("contents");
collectStreamsFrom(collector, videos);
final JsonObject videos = contents.getObject(0).getObject("playlistVideoListRenderer");
collectStreamsFrom(collector, videos.getArray("contents"));
nextPage = getNextPageFrom(videos.getArray("continuations"));
}
return new InfoItemsPage<>(collector, getNextPageUrl());
return new InfoItemsPage<>(collector, nextPage);
}
@Override
public InfoItemsPage<StreamInfoItem> getPage(final String pageUrl) throws IOException, ExtractionException {
if (isNullOrEmpty(pageUrl)) {
throw new ExtractionException(new IllegalArgumentException("Page url is empty or null"));
public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException, ExtractionException {
if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL");
}
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization());
final JsonArray ajaxJson = getJsonResponse(page.getUrl(), getExtractorLocalization());
final JsonObject sectionListContinuation = ajaxJson.getObject(1).getObject("response")
.getObject("continuationContents").getObject("playlistVideoListContinuation");
collectStreamsFrom(collector, sectionListContinuation.getArray("contents"));
return new InfoItemsPage<>(collector, getNextPageUrlFrom(sectionListContinuation.getArray("continuations")));
return new InfoItemsPage<>(collector, getNextPageFrom(sectionListContinuation.getArray("continuations")));
}
private String getNextPageUrlFrom(final JsonArray continuations) {
private Page getNextPageFrom(final JsonArray continuations) {
if (isNullOrEmpty(continuations)) {
return "";
return null;
}
JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData");
String continuation = nextContinuationData.getString("continuation");
String clickTrackingParams = nextContinuationData.getString("clickTrackingParams");
return "https://www.youtube.com/browse_ajax?ctoken=" + continuation + "&continuation=" + continuation
+ "&itct=" + clickTrackingParams;
final JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData");
final String continuation = nextContinuationData.getString("continuation");
final String clickTrackingParams = nextContinuationData.getString("clickTrackingParams");
return new Page("https://www.youtube.com/browse_ajax?ctoken=" + continuation + "&continuation=" + continuation
+ "&itct=" + clickTrackingParams);
}
private void collectStreamsFrom(final StreamInfoItemsCollector collector, final JsonArray videos) {
final TimeAgoParser timeAgoParser = getTimeAgoParser();
for (Object video : videos) {
for (final Object video : videos) {
if (((JsonObject) video).has("playlistVideoRenderer")) {
collector.commit(new YoutubeStreamInfoItemExtractor(((JsonObject) video).getObject("playlistVideoRenderer"), timeAgoParser) {
@Override

36
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java
View File

@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
@ -101,28 +102,27 @@ public class YoutubeSearchExtractor extends SearchExtractor {
final JsonArray sections = initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer")
.getObject("primaryContents").getObject("sectionListRenderer").getArray("contents");
Page nextPage = null;
for (Object section : sections) {
collectStreamsFrom(collector, ((JsonObject) section).getObject("itemSectionRenderer").getArray("contents"));
final JsonObject itemSectionRenderer = ((JsonObject) section).getObject("itemSectionRenderer");
collectStreamsFrom(collector, itemSectionRenderer.getArray("contents"));
nextPage = getNextPageFrom(itemSectionRenderer.getArray("continuations"));
}
return new InfoItemsPage<>(collector, getNextPageUrl());
return new InfoItemsPage<>(collector, nextPage);
}
@Override
public String getNextPageUrl() throws ExtractionException {
return getNextPageUrlFrom(initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer")
.getObject("primaryContents").getObject("sectionListRenderer").getArray("contents")
.getObject(0).getObject("itemSectionRenderer").getArray("continuations"));
}
@Override
public InfoItemsPage<InfoItem> getPage(final String pageUrl) throws IOException, ExtractionException {
if (isNullOrEmpty(pageUrl)) {
throw new ExtractionException(new IllegalArgumentException("Page url is empty or null"));
public InfoItemsPage<InfoItem> getPage(final Page page) throws IOException, ExtractionException {
if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL");
}
final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId());
final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization());
final JsonArray ajaxJson = getJsonResponse(page.getUrl(), getExtractorLocalization());
final JsonObject itemSectionRenderer = ajaxJson.getObject(1).getObject("response")
.getObject("continuationContents").getObject("itemSectionContinuation");
@ -130,7 +130,7 @@ public class YoutubeSearchExtractor extends SearchExtractor {
collectStreamsFrom(collector, itemSectionRenderer.getArray("contents"));
final JsonArray continuations = itemSectionRenderer.getArray("continuations");
return new InfoItemsPage<>(collector, getNextPageUrlFrom(continuations));
return new InfoItemsPage<>(collector, getNextPageFrom(continuations));
}
private void collectStreamsFrom(final InfoItemsSearchCollector collector, final JsonArray videos) throws NothingFoundException, ParsingException {
@ -150,16 +150,16 @@ public class YoutubeSearchExtractor extends SearchExtractor {
}
}
private String getNextPageUrlFrom(final JsonArray continuations) throws ParsingException {
private Page getNextPageFrom(final JsonArray continuations) throws ParsingException {
if (isNullOrEmpty(continuations)) {
return "";
return null;
}
final JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData");
final String continuation = nextContinuationData.getString("continuation");
final String clickTrackingParams = nextContinuationData.getString("clickTrackingParams");
return getUrl() + "&pbj=1&ctoken=" + continuation + "&continuation=" + continuation
+ "&itct=" + clickTrackingParams;
return new Page(getUrl() + "&pbj=1&ctoken=" + continuation + "&continuation=" + continuation
+ "&itct=" + clickTrackingParams);
}
}

13
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java
View File

@ -23,6 +23,7 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
@ -61,13 +62,8 @@ public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
}
@Override
public String getNextPageUrl() {
return "";
}
@Override
public InfoItemsPage<StreamInfoItem> getPage(String pageUrl) {
return null;
public InfoItemsPage<StreamInfoItem> getPage(final Page page) {
return InfoItemsPage.emptyPage();
}
@Nonnull
@ -98,6 +94,7 @@ public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
collector.commit(new YoutubeStreamInfoItemExtractor(videoInfo, timeAgoParser));
}
}
return new InfoItemsPage<>(collector, getNextPageUrl());
return new InfoItemsPage<>(collector, null);
}
}

13
extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java
View File

@ -7,6 +7,7 @@ import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLDecoder;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@ -222,4 +223,16 @@ public class Utils {
return true;
}
public static String join(final CharSequence delimiter, final Iterable<? extends CharSequence> elements) {
final StringBuilder stringBuilder = new StringBuilder();
final Iterator<? extends CharSequence> iterator = elements.iterator();
while (iterator.hasNext()) {
stringBuilder.append(iterator.next());
if (iterator.hasNext()) {
stringBuilder.append(delimiter);
}
}
return stringBuilder.toString();
}
}

15
extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultTests.java
View File

@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor.services;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelInfoItem;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
@ -84,9 +85,8 @@ public final class DefaultTests {
}
public static <T extends InfoItem> void assertNoMoreItems(ListExtractor<T> extractor) throws Exception {
assertFalse("More items available when it shouldn't", extractor.hasNextPage());
final String nextPageUrl = extractor.getNextPageUrl();
assertTrue("Next page is not empty or null", isNullOrEmpty(nextPageUrl));
final ListExtractor.InfoItemsPage<T> initialPage = extractor.getInitialPage();
assertFalse("More items available when it shouldn't", initialPage.hasNextPage());
}
public static void assertNoDuplicatedItems(StreamingService expectedService,
@ -118,8 +118,9 @@ public final class DefaultTests {
}
public static <T extends InfoItem> ListExtractor.InfoItemsPage<T> defaultTestMoreItems(ListExtractor<T> extractor) throws Exception {
assertTrue("Doesn't have more items", extractor.hasNextPage());
ListExtractor.InfoItemsPage<T> nextPage = extractor.getPage(extractor.getNextPageUrl());
final ListExtractor.InfoItemsPage<T> initialPage = extractor.getInitialPage();
assertTrue("Doesn't have more items", initialPage.hasNextPage());
ListExtractor.InfoItemsPage<T> nextPage = extractor.getPage(initialPage.getNextPage());
final List<T> items = nextPage.getItems();
assertFalse("Next page is empty", items.isEmpty());
assertEmptyErrors("Next page have errors", nextPage.getErrors());
@ -129,9 +130,9 @@ public final class DefaultTests {
}
public static void defaultTestGetPageInNewExtractor(ListExtractor<? extends InfoItem> extractor, ListExtractor<? extends InfoItem> newExtractor) throws Exception {
final String nextPageUrl = extractor.getNextPageUrl();
final Page nextPage = extractor.getInitialPage().getNextPage();
final ListExtractor.InfoItemsPage<? extends InfoItem> page = newExtractor.getPage(nextPageUrl);
final ListExtractor.InfoItemsPage<? extends InfoItem> page = newExtractor.getPage(nextPage);
defaultTestListOfItems(extractor.getService(), page.getItems(), page.getErrors());
}
}

36
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java
View File

@ -5,6 +5,7 @@ import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.comments.CommentsInfo;
import org.schabi.newpipe.extractor.comments.CommentsInfoItem;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
@ -14,7 +15,9 @@ import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
import java.util.List;
import static org.junit.Assert.*;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
public class PeertubeCommentsExtractorTest {
@ -31,11 +34,10 @@ public class PeertubeCommentsExtractorTest {
@Test
public void testGetComments() throws IOException, ExtractionException {
InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
assertTrue(comments.getErrors().isEmpty());
boolean result = findInComments(comments, "@root A great documentary on a great guy.");
while (comments.hasNextPage() && !result) {
comments = extractor.getPage(comments.getNextPageUrl());
comments = extractor.getPage(comments.getNextPage());
result = findInComments(comments, "@root A great documentary on a great guy.");
}
@ -44,24 +46,26 @@ public class PeertubeCommentsExtractorTest {
@Test
public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException {
final CommentsInfo commentsInfo = CommentsInfo.getInfo("https://framatube.org/videos/watch/a8ea95b8-0396-49a6-8f30-e25e25fb2828");
assertTrue(commentsInfo.getErrors().isEmpty());
CommentsInfo commentsInfo = CommentsInfo.getInfo("https://framatube.org/videos/watch/a8ea95b8-0396-49a6-8f30-e25e25fb2828");
assertEquals("Comments", commentsInfo.getName());
boolean result = findInComments(commentsInfo.getRelatedItems(), "Loved it!!!");
String nextPage = commentsInfo.getNextPageUrl();
while (!Utils.isBlank(nextPage) && !result) {
final InfoItemsPage<CommentsInfoItem> moreItems = CommentsInfo.getMoreItems(PeerTube, commentsInfo, nextPage);
Page nextPage = commentsInfo.getNextPage();
InfoItemsPage<CommentsInfoItem> moreItems = new InfoItemsPage<>(null, nextPage, null);
while (moreItems.hasNextPage() && !result) {
moreItems = CommentsInfo.getMoreItems(PeerTube, commentsInfo, nextPage);
result = findInComments(moreItems.getItems(), "Loved it!!!");
nextPage = moreItems.getNextPageUrl();
nextPage = moreItems.getNextPage();
}
assertTrue(result);
}
@Test
public void testGetCommentsAllData() throws IOException, ExtractionException {
final InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
for (final CommentsInfoItem c : comments.getItems()) {
InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
for (CommentsInfoItem c : comments.getItems()) {
assertFalse(Utils.isBlank(c.getUploaderUrl()));
assertFalse(Utils.isBlank(c.getUploaderName()));
assertFalse(Utils.isBlank(c.getUploaderAvatarUrl()));
@ -71,16 +75,16 @@ public class PeertubeCommentsExtractorTest {
assertFalse(Utils.isBlank(c.getTextualUploadDate()));
assertFalse(Utils.isBlank(c.getThumbnailUrl()));
assertFalse(Utils.isBlank(c.getUrl()));
assertEquals(-1, c.getLikeCount());
assertFalse(c.getLikeCount() != -1);
}
}
private boolean findInComments(final InfoItemsPage<CommentsInfoItem> comments, final String comment) {
private boolean findInComments(InfoItemsPage<CommentsInfoItem> comments, String comment) {
return findInComments(comments.getItems(), comment);
}
private boolean findInComments(final List<CommentsInfoItem> comments, final String comment) {
for (final CommentsInfoItem c : comments) {
private boolean findInComments(List<CommentsInfoItem> comments, String comment) {
for (CommentsInfoItem c : comments) {
if (c.getCommentText().contains(comment)) {
return true;
}

2
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/search/PeertubeSearchExtractorTest.java
View File

@ -51,7 +51,7 @@ public class PeertubeSearchExtractorTest {
extractor.fetchPage();
final InfoItemsPage<InfoItem> page1 = extractor.getInitialPage();
final InfoItemsPage<InfoItem> page2 = extractor.getPage(page1.getNextPageUrl());
final InfoItemsPage<InfoItem> page2 = extractor.getPage(page1.getNextPage());
assertNoDuplicatedItems(PeerTube, page1, page2);
}

2
extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java
View File

@ -268,7 +268,7 @@ public class SoundcloudPlaylistExtractorTest {
ListExtractor.InfoItemsPage<StreamInfoItem> currentPage = defaultTestMoreItems(extractor);
// Test for 2 more levels
for (int i = 0; i < 2; i++) {
currentPage = extractor.getPage(currentPage.getNextPageUrl());
currentPage = extractor.getPage(currentPage.getNextPage());
defaultTestListOfItems(SoundCloud, currentPage.getItems(), currentPage.getErrors());
}
}

2
extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorTest.java
View File

@ -119,7 +119,7 @@ public class SoundcloudSearchExtractorTest {
extractor.fetchPage();
final InfoItemsPage<InfoItem> page1 = extractor.getInitialPage();
final InfoItemsPage<InfoItem> page2 = extractor.getPage(page1.getNextPageUrl());
final InfoItemsPage<InfoItem> page2 = extractor.getPage(page1.getNextPage());
assertNoDuplicatedItems(SoundCloud, page1, page2);
}

24
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java
View File

@ -5,6 +5,7 @@ import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.comments.CommentsInfo;
import org.schabi.newpipe.extractor.comments.CommentsInfoItem;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
@ -22,7 +23,6 @@ import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
public class YoutubeCommentsExtractorTest {
private static final String urlYT = "https://www.youtube.com/watch?v=D00Au7k3i6o";
private static final String urlInvidious = "https://invidio.us/watch?v=D00Au7k3i6o";
private static YoutubeCommentsExtractor extractorYT;
@ -33,6 +33,7 @@ public class YoutubeCommentsExtractorTest {
NewPipe.init(DownloaderTestImpl.getInstance());
extractorYT = (YoutubeCommentsExtractor) YouTube
.getCommentsExtractor(urlYT);
extractorYT.fetchPage();
extractorInvidious = (YoutubeCommentsExtractor) YouTube
.getCommentsExtractor(urlInvidious);
}
@ -44,12 +45,11 @@ public class YoutubeCommentsExtractorTest {
}
private boolean getCommentsHelper(YoutubeCommentsExtractor extractor) throws IOException, ExtractionException {
boolean result;
InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
result = findInComments(comments, "s1ck m3m3");
boolean result = findInComments(comments, "s1ck m3m3");
while (comments.hasNextPage() && !result) {
comments = extractor.getPage(comments.getNextPageUrl());
comments = extractor.getPage(comments.getNextPage());
result = findInComments(comments, "s1ck m3m3");
}
@ -63,16 +63,18 @@ public class YoutubeCommentsExtractorTest {
}
private boolean getCommentsFromCommentsInfoHelper(String url) throws IOException, ExtractionException {
boolean result = false;
CommentsInfo commentsInfo = CommentsInfo.getInfo(url);
result = findInComments(commentsInfo.getRelatedItems(), "s1ck m3m3");
/* String nextPage = commentsInfo.getNextPageUrl();
while (!Utils.isBlank(nextPage) && !result) {
InfoItemsPage<CommentsInfoItem> moreItems = CommentsInfo.getMoreItems(YouTube, commentsInfo, nextPage);
assertEquals("Comments", commentsInfo.getName());
boolean result = findInComments(commentsInfo.getRelatedItems(), "s1ck m3m3");
Page nextPage = commentsInfo.getNextPage();
InfoItemsPage<CommentsInfoItem> moreItems = new InfoItemsPage<>(null, nextPage, null);
while (moreItems.hasNextPage() && !result) {
moreItems = CommentsInfo.getMoreItems(YouTube, commentsInfo, nextPage);
result = findInComments(moreItems.getItems(), "s1ck m3m3");
nextPage = moreItems.getNextPageUrl();
}*/
nextPage = moreItems.getNextPage();
}
return result;
}

2
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java
View File

@ -209,7 +209,7 @@ public class YoutubePlaylistExtractorTest {
// test for 2 more levels
for (int i = 0; i < 2; i++) {
currentPage = extractor.getPage(currentPage.getNextPageUrl());
currentPage = extractor.getPage(currentPage.getNextPage());
defaultTestListOfItems(YouTube, currentPage.getItems(), currentPage.getErrors());
}
}

14
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java
View File

@ -19,8 +19,9 @@ import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmptyErrors;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoDuplicatedItems;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.*;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.CHANNELS;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.PLAYLISTS;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.VIDEOS;
public class YoutubeSearchExtractorTest {
public static class All extends DefaultSearchExtractorTest {
@ -186,15 +187,14 @@ public class YoutubeSearchExtractorTest {
@Test
public void testMoreRelatedItems() throws Exception {
final ListExtractor.InfoItemsPage<InfoItem> initialPage = extractor().getInitialPage();
// YouTube actually gives us an empty next page, but after that, no more pages.
assertTrue(extractor.hasNextPage());
final ListExtractor.InfoItemsPage<InfoItem> nextEmptyPage = extractor.getPage(extractor.getNextPageUrl());
assertTrue(initialPage.hasNextPage());
final ListExtractor.InfoItemsPage<InfoItem> nextEmptyPage = extractor.getPage(initialPage.getNextPage());
assertEquals(0, nextEmptyPage.getItems().size());
assertEmptyErrors("Empty page has errors", nextEmptyPage.getErrors());
assertFalse("More items available when it shouldn't", nextEmptyPage.hasNextPage());
final String nextPageUrl = nextEmptyPage.getNextPageUrl();
assertTrue("Next page is not empty or null", isNullOrEmpty(nextPageUrl));
}
}
@ -206,7 +206,7 @@ public class YoutubeSearchExtractorTest {
extractor.fetchPage();
final ListExtractor.InfoItemsPage<InfoItem> page1 = extractor.getInitialPage();
final ListExtractor.InfoItemsPage<InfoItem> page2 = extractor.getPage(page1.getNextPageUrl());
final ListExtractor.InfoItemsPage<InfoItem> page2 = extractor.getPage(page1.getNextPage());
assertNoDuplicatedItems(YouTube, page1, page2);
}

10
extractor/src/test/java/org/schabi/newpipe/extractor/utils/UtilsTest.java
View File

@ -1,18 +1,24 @@
package org.schabi.newpipe.extractor.utils;
import com.grack.nanojson.JsonParserException;
import org.junit.Test;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import java.util.Arrays;
import static org.junit.Assert.assertEquals;
public class UtilsTest {
@Test
public void testMixedNumberWordToLong() throws JsonParserException, ParsingException {
public void testMixedNumberWordToLong() throws ParsingException {
assertEquals(10, Utils.mixedNumberWordToLong("10"));
assertEquals(10.5e3, Utils.mixedNumberWordToLong("10.5K"), 0.0);
assertEquals(10.5e6, Utils.mixedNumberWordToLong("10.5M"), 0.0);
assertEquals(10.5e6, Utils.mixedNumberWordToLong("10,5M"), 0.0);
assertEquals(1.5e9, Utils.mixedNumberWordToLong("1,5B"), 0.0);
}
@Test
public void testJoin() {
assertEquals("some,random,stuff", Utils.join(",", Arrays.asList("some", "random", "stuff")));
}
}