be-gay-do-crimes
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge branch 'dev' into bandcamp

This commit is contained in:
TobiGr 2021-02-15 22:09:41 +01:00
parent 07570554b6 416cf17b7a
commit e062c8cb0d
245 changed files with 19792 additions and 2318 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
.github/workflows/ci.yml vendored
View File

@ -1,6 +1,9 @@
name: CI
on:
schedule:
# once per day
- cron: 0 0 * * *
push:
branches:
- dev
@ -25,5 +28,11 @@ jobs:
key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }}
restore-keys: ${{ runner.os }}-gradle
# See gradle file for difference between downloaders
- name: Build and run Tests
run: ./gradlew check --stacktrace -Ddownloader=MOCK
run: |
if [[ $GITHUB_EVENT_NAME == 'schedule' ]]; then
./gradlew check --stacktrace -Ddownloader=REAL
else
./gradlew check --stacktrace -Ddownloader=MOCK
fi

2
README.md
View File

@ -11,7 +11,7 @@ NewPipe Extractor is available at JitPack's Maven repo.
If you're using Gradle, you could add NewPipe Extractor as a dependency with the following steps:
1. Add `maven { url 'https://jitpack.io' }` to the `repositories` in your `build.gradle`.
2. Add `implementation 'com.github.TeamNewPipe:NewPipeExtractor:v0.20.8'`the `dependencies` in your `build.gradle`. Replace `v0.20.8` with the latest release.
2. Add `implementation 'com.github.TeamNewPipe:NewPipeExtractor:v0.20.10'`the `dependencies` in your `build.gradle`. Replace `v0.20.10` with the latest release.
**Note:** To use NewPipe Extractor in projects with a `minSdkVersion` below 26, [API desugaring](https://developer.android.com/studio/write/java8-support#library-desugaring) is required.

2
build.gradle
View File

@ -5,7 +5,7 @@ allprojects {
sourceCompatibility = 1.8
targetCompatibility = 1.8
version 'v0.20.8'
version 'v0.20.10'
group 'com.github.TeamNewPipe'
repositories {

2
extractor/build.gradle
View File

@ -15,6 +15,6 @@ dependencies {
implementation 'org.nibor.autolink:autolink:0.10.0'
testImplementation 'junit:junit:4.13.1'
testImplementation "com.squareup.okhttp3:okhttp:3.12.11"
testImplementation "com.squareup.okhttp3:okhttp:3.12.13"
testImplementation 'com.google.code.gson:gson:2.8.6'
}

2
extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfo.java
View File

@ -25,7 +25,7 @@ public class CommentsInfo extends ListInfo<CommentsInfoItem> {
return getInfo(serviceByUrl.getCommentsExtractor(url));
}
private static CommentsInfo getInfo(CommentsExtractor commentsExtractor) throws IOException, ExtractionException {
public static CommentsInfo getInfo(CommentsExtractor commentsExtractor) throws IOException, ExtractionException {
// for services which do not have a comments extractor
if (null == commentsExtractor) {
return null;

11
extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItem.java
View File

@ -17,6 +17,7 @@ public class CommentsInfoItem extends InfoItem {
private DateWrapper uploadDate;
private int likeCount;
private boolean heartedByUploader;
private boolean pinned;
public CommentsInfoItem(int serviceId, String url, String name) {
super(InfoType.COMMENT, serviceId, url, name);
@ -91,7 +92,15 @@ public class CommentsInfoItem extends InfoItem {
this.heartedByUploader = isHeartedByUploader;
}
public boolean getHeartedByUploader() {
public boolean isHeartedByUploader() {
return this.heartedByUploader;
}
public boolean isPinned() {
return pinned;
}
public void setPinned(boolean pinned) {
this.pinned = pinned;
}
}

10
extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItemExtractor.java
View File

@ -11,6 +11,7 @@ public interface CommentsInfoItemExtractor extends InfoItemExtractor {
/**
* Return the like count of the comment, or -1 if it's unavailable
*
* @see StreamExtractor#getLikeCount()
*/
int getLikeCount() throws ParsingException;
@ -22,12 +23,14 @@ public interface CommentsInfoItemExtractor extends InfoItemExtractor {
/**
* The upload date given by the service, unmodified
*
* @see StreamExtractor#getTextualUploadDate()
*/
String getTextualUploadDate() throws ParsingException;
/**
* The upload date wrapped with DateWrapper class
*
* @see StreamExtractor#getUploadDate()
*/
@Nullable
@ -44,5 +47,10 @@ public interface CommentsInfoItemExtractor extends InfoItemExtractor {
/**
* Whether the comment has been hearted by the uploader
*/
boolean getHeartedByUploader() throws ParsingException;
boolean isHeartedByUploader() throws ParsingException;
/**
* Whether the comment is pinned
*/
boolean isPinned() throws ParsingException;
}

8
extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfoItemsCollector.java
View File

@ -71,7 +71,13 @@ public class CommentsInfoItemsCollector extends InfoItemsCollector<CommentsInfoI
}
try {
resultItem.setHeartedByUploader(extractor.getHeartedByUploader());
resultItem.setHeartedByUploader(extractor.isHeartedByUploader());
} catch (Exception e) {
addError(e);
}
try {
resultItem.setPinned(extractor.isPinned());
} catch (Exception e) {
addError(e);
}

4
extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java
View File

@ -14,6 +14,8 @@ import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class KioskList {
public interface KioskExtractorFactory {
@ -70,7 +72,7 @@ public class KioskList {
public KioskExtractor getDefaultKioskExtractor(Page nextPage, Localization localization)
throws ExtractionException, IOException {
if (defaultKiosk != null && !defaultKiosk.equals("")) {
if (!isNullOrEmpty(defaultKiosk)) {
return getExtractorById(defaultKiosk, nextPage, localization);
} else {
if (!kioskList.isEmpty()) {

3
extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandlerFactory.java
View File

@ -50,6 +50,9 @@ public abstract class LinkHandlerFactory {
* @return a {@link LinkHandler} complete with information
*/
public LinkHandler fromUrl(final String url) throws ParsingException {
if (Utils.isNullOrEmpty(url)) {
throw new IllegalArgumentException("The url is null or empty");
}
final String polishedUrl = Utils.followGoogleRedirectIfNeeded(url);
final String baseUrl = Utils.getBaseUrl(polishedUrl);
return fromUrl(polishedUrl, baseUrl);

12
extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/SearchQueryHandlerFactory.java
View File

@ -5,6 +5,8 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
import java.util.ArrayList;
import java.util.List;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
public abstract class SearchQueryHandlerFactory extends ListLinkHandlerFactory {
///////////////////////////////////
@ -12,7 +14,7 @@ public abstract class SearchQueryHandlerFactory extends ListLinkHandlerFactory {
///////////////////////////////////
@Override
public abstract String getUrl(String querry, List<String> contentFilter, String sortFilter) throws ParsingException;
public abstract String getUrl(String query, List<String> contentFilter, String sortFilter) throws ParsingException;
public String getSearchString(String url) {
return "";
@ -28,14 +30,14 @@ public abstract class SearchQueryHandlerFactory extends ListLinkHandlerFactory {
}
@Override
public SearchQueryHandler fromQuery(String querry,
public SearchQueryHandler fromQuery(String query,
List<String> contentFilter,
String sortFilter) throws ParsingException {
return new SearchQueryHandler(super.fromQuery(querry, contentFilter, sortFilter));
return new SearchQueryHandler(super.fromQuery(query, contentFilter, sortFilter));
}
public SearchQueryHandler fromQuery(String querry) throws ParsingException {
return fromQuery(querry, new ArrayList<String>(0), "");
public SearchQueryHandler fromQuery(String query) throws ParsingException {
return fromQuery(query, new ArrayList<String>(0), EMPTY_STRING);
}
/**

4
extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCSearchQueryHandlerFactory.java
View File

@ -7,6 +7,8 @@ import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.List;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
public class MediaCCCSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
public static final String ALL = "all";
public static final String CONFERENCES = "conferences";
@ -31,7 +33,7 @@ public class MediaCCCSearchQueryHandlerFactory extends SearchQueryHandlerFactory
final String sortFilter) throws ParsingException {
try {
return "https://media.ccc.de/public/events/search?q="
+ URLEncoder.encode(query, "UTF-8");
+ URLEncoder.encode(query, UTF_8);
} catch (UnsupportedEncodingException e) {
throw new ParsingException("Could not create search string with query: " + query, e);
}

12
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java
View File

@ -1,7 +1,6 @@
package org.schabi.newpipe.extractor.services.peertube.extractors;
import com.grack.nanojson.JsonObject;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.schabi.newpipe.extractor.ServiceList;
@ -13,6 +12,8 @@ import org.schabi.newpipe.extractor.utils.JsonUtils;
import java.util.Objects;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtractor {
private final JsonObject item;
private final String url;
@ -68,7 +69,7 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac
final Document doc = Jsoup.parse(htmlText);
return doc.body().text();
} catch (Exception e) {
return htmlText.replaceAll("(?s)<[^>]*>(\\s*<[^>]*>)*", "");
return htmlText.replaceAll("(?s)<[^>]*>(\\s*<[^>]*>)*", EMPTY_STRING);
}
}
@ -89,7 +90,12 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac
}
@Override
public boolean getHeartedByUploader() throws ParsingException {
public boolean isHeartedByUploader() throws ParsingException {
return false;
}
@Override
public boolean isPinned() throws ParsingException {
return false;
}

14
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java
View File

@ -18,15 +18,7 @@ import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeSearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeStreamLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.AudioStream;
import org.schabi.newpipe.extractor.stream.Description;
import org.schabi.newpipe.extractor.stream.Stream;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamSegment;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.stream.SubtitlesStream;
import org.schabi.newpipe.extractor.stream.VideoStream;
import org.schabi.newpipe.extractor.stream.*;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Utils;
@ -40,6 +32,8 @@ import java.util.Collections;
import java.util.List;
import java.util.Locale;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
public class PeertubeStreamExtractor extends StreamExtractor {
private final String baseUrl;
private JsonObject json;
@ -322,7 +316,7 @@ public class PeertubeStreamExtractor extends StreamExtractor {
params.append("start=0&count=8&sort=-createdAt");
for (final String tag : tags) {
params.append("&tagsOneOf=");
params.append(URLEncoder.encode(tag, "UTF-8"));
params.append(URLEncoder.encode(tag, UTF_8));
}
return url + "?" + params.toString();
}

5
extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeSearchQueryHandlerFactory.java
View File

@ -8,9 +8,10 @@ import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.List;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
public class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
public static final String CHARSET_UTF_8 = "UTF-8";
public static final String VIDEOS = "videos";
public static final String SEPIA_VIDEOS = "sepia_videos"; // sepia is the global index
public static final String SEPIA_BASE_URL = "https://sepiasearch.org";
@ -35,7 +36,7 @@ public class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory
public String getUrl(String searchString, List<String> contentFilters, String sortFilter, String baseUrl) throws ParsingException {
try {
final String url = baseUrl + SEARCH_ENDPOINT
+ "?search=" + URLEncoder.encode(searchString, CHARSET_UTF_8);
+ "?search=" + URLEncoder.encode(searchString, UTF_8);
return url;
} catch (UnsupportedEncodingException e) {

22
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java
View File

@ -37,9 +37,7 @@ import java.util.List;
import static java.util.Collections.singletonList;
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
import static org.schabi.newpipe.extractor.utils.Utils.*;
public class SoundcloudParsingHelper {
private static final String HARDCODED_CLIENT_ID = "H2c34Q0E7hftqnuDHGsk88DbNqhYpgMm"; // Updated on 24/06/20
@ -117,7 +115,7 @@ public class SoundcloudParsingHelper {
*/
public static JsonObject resolveFor(Downloader downloader, String url) throws IOException, ExtractionException {
String apiUrl = "https://api-v2.soundcloud.com/resolve"
+ "?url=" + URLEncoder.encode(url, "UTF-8")
+ "?url=" + URLEncoder.encode(url, UTF_8)
+ "&client_id=" + clientId();
try {
@ -136,7 +134,7 @@ public class SoundcloudParsingHelper {
public static String resolveUrlWithEmbedPlayer(String apiUrl) throws IOException, ReCaptchaException, ParsingException {
String response = NewPipe.getDownloader().get("https://w.soundcloud.com/player/?url="
+ URLEncoder.encode(apiUrl, "UTF-8"), SoundCloud.getLocalization()).responseBody();
+ URLEncoder.encode(apiUrl, UTF_8), SoundCloud.getLocalization()).responseBody();
return Jsoup.parse(response).select("link[rel=\"canonical\"]").first().attr("abs:href");
}
@ -148,17 +146,17 @@ public class SoundcloudParsingHelper {
*/
public static String resolveIdWithEmbedPlayer(String urlString) throws IOException, ReCaptchaException, ParsingException {
// Remove the tailing slash from URLs due to issues with the SoundCloud API
if (urlString.charAt(urlString.length() -1) == '/') urlString = urlString.substring(0, urlString.length()-1);
if (urlString.charAt(urlString.length() - 1) == '/') urlString = urlString.substring(0, urlString.length() - 1);
URL url;
try {
url = Utils.stringToURL(urlString);
} catch (MalformedURLException e){
} catch (MalformedURLException e) {
throw new IllegalArgumentException("The given URL is not valid");
}
String response = NewPipe.getDownloader().get("https://w.soundcloud.com/player/?url="
+ URLEncoder.encode(url.toString(), "UTF-8"), SoundCloud.getLocalization()).responseBody();
+ URLEncoder.encode(url.toString(), UTF_8), SoundCloud.getLocalization()).responseBody();
// handle playlists / sets different and get playlist id via uir field in JSON
if (url.getPath().contains("/sets/") && !url.getPath().endsWith("/sets"))
return Parser.matchGroup1("\"uri\":\\s*\"https:\\/\\/api\\.soundcloud\\.com\\/playlists\\/((\\d)*?)\"", response);
@ -240,10 +238,14 @@ public class SoundcloudParsingHelper {
* @return the next streams url, empty if don't have
*/
public static String getStreamsFromApi(StreamInfoItemsCollector collector, String apiUrl, boolean charts) throws IOException, ReCaptchaException, ParsingException {
String response = NewPipe.getDownloader().get(apiUrl, SoundCloud.getLocalization()).responseBody();
final Response response = NewPipe.getDownloader().get(apiUrl, SoundCloud.getLocalization());
if (response.responseCode() >= 400) {
throw new IOException("Could not get streams from API, HTTP " + response.responseCode());
}
JsonObject responseObject;
try {
responseObject = JsonParser.object().from(response);
responseObject = JsonParser.object().from(response.responseBody());
} catch (JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}

6
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelExtractor.java
View File

@ -3,7 +3,6 @@ package org.schabi.newpipe.extractor.services.soundcloud.extractors;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
@ -15,11 +14,10 @@ import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import javax.annotation.Nonnull;
import java.io.IOException;
import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
@SuppressWarnings("WeakerAccess")

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelInfoItemExtractor.java
View File

@ -3,7 +3,7 @@ package org.schabi.newpipe.extractor.services.soundcloud.extractors;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtractor {

20
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChartsExtractor.java
View File

@ -6,13 +6,13 @@ import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.localization.ContentCountry;
import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import java.io.IOException;
import javax.annotation.Nonnull;
import java.io.IOException;
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
@ -61,10 +61,20 @@ public class SoundcloudChartsExtractor extends KioskExtractor<StreamInfoItem> {
apiUrl += "&kind=trending";
}
final String contentCountry = SoundCloud.getContentCountry().getCountryCode();
apiUrl += "&region=soundcloud:regions:" + contentCountry;
final ContentCountry contentCountry = SoundCloud.getContentCountry();
String apiUrlWithRegion = null;
if (getService().getSupportedCountries().contains(contentCountry)) {
apiUrlWithRegion = apiUrl + "&region=soundcloud:regions:" + contentCountry.getCountryCode();
}
final String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl, true);
String nextPageUrl;
try {
nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrlWithRegion == null ? apiUrl : apiUrlWithRegion, true);
} catch (IOException e) {
// Request to other region may be geo-restricted. See https://github.com/TeamNewPipe/NewPipeExtractor/issues/537
// we retry without the specified region.
nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl, true);
}
return new InfoItemsPage<>(collector, new Page(nextPageUrl));
}

7
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java
View File

@ -40,7 +40,12 @@ public class SoundcloudCommentsInfoItemExtractor implements CommentsInfoItemExtr
}
@Override
public boolean getHeartedByUploader() throws ParsingException {
public boolean isHeartedByUploader() throws ParsingException {
return false;
}
@Override
public boolean isPinned() throws ParsingException {
return false;
}

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistInfoItemExtractor.java
View File

@ -4,7 +4,7 @@ import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtractor {

15
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java
View File

@ -4,13 +4,7 @@ import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.InfoItemExtractor;
import org.schabi.newpipe.extractor.InfoItemsCollector;
import org.schabi.newpipe.extractor.MetaInfo;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.*;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
@ -19,6 +13,7 @@ import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.utils.Parser;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
@ -26,10 +21,8 @@ import java.net.URL;
import java.util.Collections;
import java.util.List;
import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.services.soundcloud.linkHandler.SoundcloudSearchQueryHandlerFactory.ITEMS_PER_PAGE;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class SoundcloudSearchExtractor extends SearchExtractor {
@ -50,7 +43,7 @@ public class SoundcloudSearchExtractor extends SearchExtractor {
return false;
}
@Nonnull
@Nonnull
@Override
public List<MetaInfo> getMetaInfo() {
return Collections.emptyList();

26
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java
View File

@ -4,7 +4,6 @@ import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.MetaInfo;
import org.schabi.newpipe.extractor.NewPipe;
@ -17,15 +16,10 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper;
import org.schabi.newpipe.extractor.stream.AudioStream;
import org.schabi.newpipe.extractor.stream.Description;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamSegment;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.stream.SubtitlesStream;
import org.schabi.newpipe.extractor.stream.VideoStream;
import org.schabi.newpipe.extractor.stream.*;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
@ -34,11 +28,7 @@ import java.util.Collections;
import java.util.List;
import java.util.Locale;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import static org.schabi.newpipe.extractor.utils.Utils.*;
public class SoundcloudStreamExtractor extends StreamExtractor {
private JsonObject track;
@ -60,7 +50,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
@Nonnull
@Override
public String getId() {
return track.getInt("id") + "";
return track.getInt("id") + EMPTY_STRING;
}
@Nonnull
@ -73,8 +63,8 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
@Override
public String getTextualUploadDate() {
return track.getString("created_at")
.replace("T"," ")
.replace("Z", "");
.replace("T", " ")
.replace("Z", EMPTY_STRING);
}
@Nonnull
@ -232,7 +222,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
private static String urlEncode(String value) {
try {
return URLEncoder.encode(value, "UTF-8");
return URLEncoder.encode(value, UTF_8);
} catch (UnsupportedEncodingException e) {
throw new IllegalStateException(e);
}

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamInfoItemExtractor.java
View File

@ -7,7 +7,7 @@ import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper;
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamType;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
public class SoundcloudStreamInfoItemExtractor implements StreamInfoItemExtractor {

6
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSuggestionExtractor.java
View File

@ -17,9 +17,9 @@ import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.List;
public class SoundcloudSuggestionExtractor extends SuggestionExtractor {
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
public static final String CHARSET_UTF_8 = "UTF-8";
public class SoundcloudSuggestionExtractor extends SuggestionExtractor {
public SoundcloudSuggestionExtractor(StreamingService service) {
super(service);
@ -32,7 +32,7 @@ public class SoundcloudSuggestionExtractor extends SuggestionExtractor {
Downloader dl = NewPipe.getDownloader();
String url = "https://api-v2.soundcloud.com/search/queries"
+ "?q=" + URLEncoder.encode(query, CHARSET_UTF_8)
+ "?q=" + URLEncoder.encode(query, UTF_8)
+ "&client_id=" + SoundcloudParsingHelper.clientId()
+ "&limit=10";

5
extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudSearchQueryHandlerFactory.java
View File

@ -11,8 +11,9 @@ import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.List;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
public class SoundcloudSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
public static final String CHARSET_UTF_8 = "UTF-8";
public static final String TRACKS = "tracks";
public static final String USERS = "users";
@ -43,7 +44,7 @@ public class SoundcloudSearchQueryHandlerFactory extends SearchQueryHandlerFacto
}
}
return url + "?q=" + URLEncoder.encode(id, CHARSET_UTF_8)
return url + "?q=" + URLEncoder.encode(id, UTF_8)
+ "&client_id=" + SoundcloudParsingHelper.clientId()
+ "&limit=" + ITEMS_PER_PAGE
+ "&offset=0";

65
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
View File

@ -1,14 +1,6 @@
package org.schabi.newpipe.extractor.services.youtube;
import com.grack.nanojson.*;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import com.grack.nanojson.JsonWriter;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.schabi.newpipe.extractor.MetaInfo;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.downloader.Response;
@ -21,27 +13,27 @@ import org.schabi.newpipe.extractor.stream.Description;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLDecoder;
import java.nio.charset.StandardCharsets;
import java.time.LocalDate;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.time.format.DateTimeParseException;
import java.util.*;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.NewPipe.getDownloader;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.*;
import static org.schabi.newpipe.extractor.utils.Utils.HTTP;
import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import static org.schabi.newpipe.extractor.utils.Utils.join;
/*
* Created by Christian Schabesberger on 02.03.16.
@ -111,7 +103,7 @@ public class YoutubeParsingHelper {
return host.equalsIgnoreCase("invidio.us")
|| host.equalsIgnoreCase("dev.invidio.us")
|| host.equalsIgnoreCase("www.invidio.us")
|| host.equalsIgnoreCase("vid.encryptionin.space")
|| host.equalsIgnoreCase("redirect.invidious.io")
|| host.equalsIgnoreCase("invidious.snopyta.org")
|| host.equalsIgnoreCase("yewtu.be")
|| host.equalsIgnoreCase("tube.connect.cafe")
@ -122,11 +114,16 @@ public class YoutubeParsingHelper {
|| host.equalsIgnoreCase("invidious.xyz")
|| host.equalsIgnoreCase("vid.mint.lgbt")
|| host.equalsIgnoreCase("invidiou.site")
|| host.equalsIgnoreCase("invidious.fdn.fr");
|| host.equalsIgnoreCase("invidious.fdn.fr")
|| host.equalsIgnoreCase("invidious.048596.xyz")
|| host.equalsIgnoreCase("invidious.zee.li")
|| host.equalsIgnoreCase("vid.puffyan.us")
|| host.equalsIgnoreCase("ytprivate.com");
}
/**
* Parses the duration string of the video expecting ":" or "." as separators
*
* @return the duration in seconds
* @throws ParsingException when more than 3 separators are found
*/
@ -196,6 +193,7 @@ public class YoutubeParsingHelper {
/**
* Checks if the given playlist id is a YouTube Mix (auto-generated playlist)
* Ids from a YouTube Mix start with "RD"
*
* @param playlistId
* @return Whether given id belongs to a YouTube Mix
*/
@ -206,15 +204,18 @@ public class YoutubeParsingHelper {
/**
* Checks if the given playlist id is a YouTube Music Mix (auto-generated playlist)
* Ids from a YouTube Music Mix start with "RDAMVM" or "RDCLAK"
*
* @param playlistId
* @return Whether given id belongs to a YouTube Music Mix
*/
public static boolean isYoutubeMusicMixId(final String playlistId) {
return playlistId.startsWith("RDAMVM") || playlistId.startsWith("RDCLAK");
}
/**
* Checks if the given playlist id is a YouTube Channel Mix (auto-generated playlist)
* Ids from a YouTube channel Mix start with "RDCM"
*
* @return Whether given id belongs to a YouTube Channel Mix
*/
public static boolean isYoutubeChannelMixId(final String playlistId) {
@ -223,6 +224,7 @@ public class YoutubeParsingHelper {
/**
* Extracts the video id from the playlist id for Mixes.
*
* @throws ParsingException If the playlistId is a Channel Mix or not a mix.
*/
public static String extractVideoIdFromMixId(final String playlistId) throws ParsingException {
@ -314,7 +316,8 @@ public class YoutubeParsingHelper {
clientVersion = contextClientVersion;
break;
}
} catch (Parser.RegexException ignored) { }
} catch (Parser.RegexException ignored) {
}
}
if (!isNullOrEmpty(clientVersion) && !isNullOrEmpty(shortClientVersion)) {
@ -326,7 +329,8 @@ public class YoutubeParsingHelper {
} catch (Parser.RegexException e) {
try {
key = Parser.matchGroup1("innertubeApiKey\":\"([0-9a-zA-Z_-]+?)\"", html);
} catch (Parser.RegexException ignored) { }
} catch (Parser.RegexException ignored) {
}
}
}
@ -358,7 +362,7 @@ public class YoutubeParsingHelper {
*
* Quick-and-dirty solution to reset global state in between test classes.
*/
static void resetClientVersionAndKey() {
public static void resetClientVersionAndKey() {
clientVersion = null;
key = null;
}
@ -393,7 +397,7 @@ public class YoutubeParsingHelper {
.end()
.value("query", "test")
.value("params", "Eg-KAQwIARAAGAAgACgAMABqChAEEAUQAxAKEAk%3D")
.end().done().getBytes("UTF-8");
.end().done().getBytes(UTF_8);
// @formatter:on
final Map<String, List<String>> headers = new HashMap<>();
@ -438,10 +442,17 @@ public class YoutubeParsingHelper {
return youtubeMusicKeys = new String[]{key, clientName, clientVersion};
}
@Nullable
public static String getUrlFromNavigationEndpoint(JsonObject navigationEndpoint) throws ParsingException {
if (navigationEndpoint.has("urlEndpoint")) {
String internUrl = navigationEndpoint.getObject("urlEndpoint").getString("url");
if (internUrl.startsWith("https://www.youtube.com/redirect?")) {
// remove https://www.youtube.com part to fall in the next if block
internUrl = internUrl.substring(23);
}
if (internUrl.startsWith("/redirect?")) {
// q parameter can be the first parameter
internUrl = internUrl.substring(10);
@ -450,7 +461,7 @@ public class YoutubeParsingHelper {
if (param.split("=")[0].equals("q")) {
String url;
try {
url = URLDecoder.decode(param.split("=")[1], "UTF-8");
url = URLDecoder.decode(param.split("=")[1], UTF_8);
} catch (UnsupportedEncodingException e) {
return null;
}
@ -459,6 +470,8 @@ public class YoutubeParsingHelper {
}
} else if (internUrl.startsWith("http")) {
return internUrl;
} else if (internUrl.startsWith("/channel") || internUrl.startsWith("/user") || internUrl.startsWith("/watch")) {
return "https://www.youtube.com" + internUrl;
}
} else if (navigationEndpoint.has("browseEndpoint")) {
final JsonObject browseEndpoint = navigationEndpoint.getObject("browseEndpoint");
@ -496,6 +509,7 @@ public class YoutubeParsingHelper {
/**
* Get the text from a JSON object that has either a simpleText or a runs array.
*
* @param textObject JSON object to get the text from
* @param html whether to return HTML, by parsing the navigationEndpoint
* @return text in the JSON object or {@code null}
@ -723,7 +737,7 @@ public class YoutubeParsingHelper {
final String title = YoutubeParsingHelper.getTextFromObject(clarificationRenderer.getObject("contentTitle"));
final String text = YoutubeParsingHelper.getTextFromObject(clarificationRenderer.getObject("text"));
if (title == null || text == null) {
if (title == null || text == null) {
throw new ParsingException("Could not extract clarification renderer content");
}
metaInfo.setTitle(title);
@ -767,6 +781,7 @@ public class YoutubeParsingHelper {
/**
* Sometimes, YouTube provides URLs which use Google's cache. They look like
* {@code https://webcache.googleusercontent.com/search?q=cache:CACHED_URL}
*
* @param url the URL which might refer to the Google's webcache
* @return the URL which is referring to the original site
*/

47
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java
View File

@ -20,10 +20,8 @@ import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull;
import java.io.IOException;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
/*
@ -230,9 +228,9 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
.getArray("contents").getObject(0).getObject("itemSectionRenderer")
.getArray("contents").getObject(0).getObject("gridRenderer");
collectStreamsFrom(collector, gridRenderer.getArray("items"));
final JsonObject continuation = collectStreamsFrom(collector, gridRenderer.getArray("items"));
nextPage = getNextPageFrom(gridRenderer.getArray("continuations"));
nextPage = getNextPageFrom(continuation);
}
return new InfoItemsPage<>(collector, nextPage);
@ -252,36 +250,47 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
final JsonArray ajaxJson = getJsonResponse(page.getUrl(), getExtractorLocalization());
JsonObject sectionListContinuation = ajaxJson.getObject(1).getObject("response")
.getObject("continuationContents").getObject("gridContinuation");
.getArray("onResponseReceivedActions").getObject(0).getObject("appendContinuationItemsAction");
collectStreamsFrom(collector, sectionListContinuation.getArray("items"));
final JsonObject continuation = collectStreamsFrom(collector, sectionListContinuation.getArray("continuationItems"));
return new InfoItemsPage<>(collector, getNextPageFrom(sectionListContinuation.getArray("continuations")));
return new InfoItemsPage<>(collector, getNextPageFrom(continuation));
}
private Page getNextPageFrom(final JsonArray continuations) {
private Page getNextPageFrom(final JsonObject continuations) {
if (isNullOrEmpty(continuations)) {
return null;
}
final JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData");
final String continuation = nextContinuationData.getString("continuation");
final String clickTrackingParams = nextContinuationData.getString("clickTrackingParams");
final JsonObject continuationEndpoint = continuations.getObject("continuationEndpoint");
final String continuation = continuationEndpoint.getObject("continuationCommand").getString("token");
final String clickTrackingParams = continuationEndpoint.getString("clickTrackingParams");
return new Page("https://www.youtube.com/browse_ajax?ctoken=" + continuation
+ "&continuation=" + continuation + "&itct=" + clickTrackingParams);
}
private void collectStreamsFrom(StreamInfoItemsCollector collector, JsonArray videos) throws ParsingException {
/**
* Collect streams from an array of items
*
* @param collector the collector where videos will be commited
* @param videos the array to get videos from
* @return the continuation object
* @throws ParsingException if an error happened while extracting
*/
private JsonObject collectStreamsFrom(StreamInfoItemsCollector collector, JsonArray videos) throws ParsingException {
collector.reset();
final String uploaderName = getName();
final String uploaderUrl = getUrl();
final TimeAgoParser timeAgoParser = getTimeAgoParser();
for (Object video : videos) {
if (((JsonObject) video).has("gridVideoRenderer")) {
JsonObject continuation = null;
for (Object object : videos) {
final JsonObject video = (JsonObject) object;
if (video.has("gridVideoRenderer")) {
collector.commit(new YoutubeStreamInfoItemExtractor(
((JsonObject) video).getObject("gridVideoRenderer"), timeAgoParser) {
video.getObject("gridVideoRenderer"), timeAgoParser) {
@Override
public String getUploaderName() {
return uploaderName;
@ -292,8 +301,12 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
return uploaderUrl;
}
});
} else if (video.has("continuationItemRenderer")) {
continuation = video.getObject("continuationItemRenderer");
}
}
return continuation;
}
private JsonObject getVideoTab() throws ParsingException {

9
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java
View File

@ -3,7 +3,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.comments.CommentsExtractor;
@ -19,6 +18,7 @@ import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Parser;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
@ -27,9 +27,8 @@ import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;
import javax.annotation.Nonnull;
import static java.util.Collections.singletonList;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class YoutubeCommentsExtractor extends CommentsExtractor {
@ -152,9 +151,9 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
first = false;
else
result.append("&");
result.append(URLEncoder.encode(entry.getKey(), "UTF-8"));
result.append(URLEncoder.encode(entry.getKey(), UTF_8));
result.append("=");
result.append(URLEncoder.encode(entry.getValue(), "UTF-8"));
result.append(URLEncoder.encode(entry.getValue(), UTF_8));
}
return result.toString();
}

17
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java
View File

@ -2,7 +2,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.localization.DateWrapper;
@ -13,6 +12,7 @@ import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtractor {
@ -46,7 +46,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
try {
return getTextFromObject(JsonUtils.getObject(json, "authorText"));
} catch (Exception e) {
return "";
return EMPTY_STRING;
}
}
@ -86,7 +86,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
if (contentText.isEmpty()) {
// completely empty comments as described in
// https://github.com/TeamNewPipe/NewPipeExtractor/issues/380#issuecomment-668808584
return "";
return EMPTY_STRING;
}
final String commentText = getTextFromObject(contentText);
// youtube adds U+FEFF in some comments. eg. https://www.youtube.com/watch?v=Nj4F63E59io<feff>
@ -116,16 +116,21 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
}
@Override
public boolean getHeartedByUploader() throws ParsingException {
public boolean isHeartedByUploader() throws ParsingException {
return json.has("creatorHeart");
}
@Override
public boolean isPinned() {
return json.has("pinnedCommentBadge");
}
@Override
public String getUploaderName() throws ParsingException {
try {
return getTextFromObject(JsonUtils.getObject(json, "authorText"));
} catch (Exception e) {
return "";
return EMPTY_STRING;
}
}
@ -134,7 +139,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
try {
return "https://youtube.com/channel/" + JsonUtils.getString(json, "authorEndpoint.browseEndpoint.browseId");
} catch (Exception e) {
return "";
return EMPTY_STRING;
}
}

44
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java
View File

@ -1,11 +1,6 @@
package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import com.grack.nanojson.JsonWriter;
import com.grack.nanojson.*;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.MetaInfo;
import org.schabi.newpipe.extractor.Page;
@ -20,27 +15,20 @@ import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ALBUMS;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ARTISTS;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_PLAYLISTS;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_SONGS;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_VIDEOS;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.*;
import static org.schabi.newpipe.extractor.utils.Utils.*;
public class YoutubeMusicSearchExtractor extends SearchExtractor {
private JsonObject initialData;
@ -105,7 +93,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
.end()
.value("query", getSearchString())
.value("params", params)
.end().done().getBytes("UTF-8");
.end().done().getBytes(UTF_8);
// @formatter:on
final Map<String, List<String>> headers = new HashMap<>();
@ -229,7 +217,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
.value("enableSafetyMode", false)
.end()
.end()
.end().done().getBytes("UTF-8");
.end().done().getBytes(UTF_8);
// @formatter:on
final Map<String, List<String>> headers = new HashMap<>();
@ -365,7 +353,12 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
.getObject(descriptionElements.size() - 3)
.getString("text");
if (!isNullOrEmpty(viewCount)) {
return Utils.mixedNumberWordToLong(viewCount);
try {
return Utils.mixedNumberWordToLong(viewCount);
} catch (final Parser.RegexException e) {
// probably viewCount == "No views" or similar
return 0;
}
}
throw new ParsingException("Could not get view count");
}
@ -421,10 +414,15 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
@Override
public long getSubscriberCount() throws ParsingException {
final String viewCount = getTextFromObject(info.getArray("flexColumns").getObject(2)
final String subscriberCount = getTextFromObject(info.getArray("flexColumns").getObject(2)
.getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text"));
if (!isNullOrEmpty(viewCount)) {
return Utils.mixedNumberWordToLong(viewCount);
if (!isNullOrEmpty(subscriberCount)) {
try {
return Utils.mixedNumberWordToLong(subscriberCount);
} catch (final Parser.RegexException ignored) {
// probably subscriberCount == "No subscribers" or similar
return 0;
}
}
throw new ParsingException("Could not get subscriber count");
}

2
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java
View File

@ -82,7 +82,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Override
public String getName() throws ParsingException {
final String name = getTextFromObject(playlistInfo.getObject("title"));
if (name != null && !name.isEmpty()) return name;
if (!isNullOrEmpty(name)) return name;
return initialData.getObject("microformat").getObject("microformatDataRenderer").getString("title");
}

24
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java
View File

@ -1,11 +1,6 @@
package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import com.grack.nanojson.JsonWriter;
import com.grack.nanojson.*;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.MetaInfo;
import org.schabi.newpipe.extractor.Page;
@ -20,16 +15,15 @@ import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import java.io.IOException;
import java.util.*;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getClientVersion;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
/*
@ -171,7 +165,7 @@ public class YoutubeSearchExtractor extends SearchExtractor {
.object("user").end()
.end()
.value("continuation", page.getId())
.end().done().getBytes("UTF-8");
.end().done().getBytes(UTF_8);
// @formatter:on
final Map<String, List<String>> headers = new HashMap<>();

65
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java
View File

@ -4,7 +4,6 @@ import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
@ -29,17 +28,7 @@ import org.schabi.newpipe.extractor.localization.TimeAgoPatternsManager;
import org.schabi.newpipe.extractor.services.youtube.ItagItem;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.AudioStream;
import org.schabi.newpipe.extractor.stream.Description;
import org.schabi.newpipe.extractor.stream.Frameset;
import org.schabi.newpipe.extractor.stream.Stream;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamSegment;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.stream.SubtitlesStream;
import org.schabi.newpipe.extractor.stream.VideoStream;
import org.schabi.newpipe.extractor.stream.*;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils;
@ -47,26 +36,13 @@ import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.net.URL;
import java.time.LocalDate;
import java.time.OffsetDateTime;
import java.time.format.DateTimeFormatter;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.*;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
/*
@ -102,17 +78,21 @@ public class YoutubeStreamExtractor extends StreamExtractor {
/*//////////////////////////////////////////////////////////////////////////*/
@Nullable private static String cachedDeobfuscationCode = null;
@Nullable private String playerJsUrl = null;
@Nullable
private static String cachedDeobfuscationCode = null;
@Nullable
private String playerJsUrl = null;
private JsonArray initialAjaxJson;
private JsonObject initialData;
@Nonnull private final Map<String, String> videoInfoPage = new HashMap<>();
@Nonnull
private final Map<String, String> videoInfoPage = new HashMap<>();
private JsonObject playerResponse;
private JsonObject videoPrimaryInfoRenderer;
private JsonObject videoSecondaryInfoRenderer;
private int ageLimit = -1;
@Nullable private List<SubtitlesStream> subtitles = null;
@Nullable
private List<SubtitlesStream> subtitles = null;
public YoutubeStreamExtractor(StreamingService service, LinkHandler linkHandler) {
super(service, linkHandler);
@ -224,7 +204,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Nonnull
@Override
public Description getDescription() {
public Description getDescription() throws ParsingException {
assertPageFetched();
// description with more info on links
try {
@ -234,8 +214,15 @@ public class YoutubeStreamExtractor extends StreamExtractor {
// age-restricted videos cause a ParsingException here
}
String description = playerResponse.getObject("videoDetails").getString("shortDescription");
if (description == null) {
final JsonObject descriptionObject = playerResponse.getObject("microformat")
.getObject("playerMicroformatRenderer").getObject("description");
description = getTextFromObject(descriptionObject);
}
// raw non-html description
return new Description(playerResponse.getObject("videoDetails").getString("shortDescription"), Description.PLAIN_TEXT);
return new Description(description, Description.PLAIN_TEXT);
}
@Override
@ -780,8 +767,6 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
private String getDeobfuscationFuncName(final String playerCode) throws DeobfuscateException {
Parser.RegexException exception = null;
for (final String regex : REGEXES) {
@ -1079,8 +1064,10 @@ public class YoutubeStreamExtractor extends StreamExtractor {
// Search for correct panel containing the data
for (int i = 0; i < panels.size(); i++) {
if (panels.getObject(i).getObject("engagementPanelSectionListRenderer")
.getString("panelIdentifier").equals("engagement-panel-macro-markers")) {
final String panelIdentifier = panels.getObject(i).getObject("engagementPanelSectionListRenderer")
.getString("panelIdentifier");
if (panelIdentifier.equals("engagement-panel-macro-markers-description-chapters")
|| panelIdentifier.equals("engagement-panel-macro-markers")) {
segmentsArray = panels.getObject(i).getObject("engagementPanelSectionListRenderer")
.getObject("content").getObject("macroMarkersListRenderer").getArray("contents");
break;
@ -1129,6 +1116,6 @@ public class YoutubeStreamExtractor extends StreamExtractor {
public List<MetaInfo> getMetaInfo() throws ParsingException {
return YoutubeParsingHelper.getMetaInfo(
initialData.getObject("contents").getObject("twoColumnWatchNextResults")
.getObject("results").getObject("results").getArray("contents"));
.getObject("results").getObject("results").getArray("contents"));
}
}

6
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java
View File

@ -17,10 +17,8 @@ import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
/*

8
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java
View File

@ -15,6 +15,8 @@ import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.List;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
/*
* Created by Christian Schabesberger on 28.09.16.
*
@ -37,8 +39,6 @@ import java.util.List;
public class YoutubeSuggestionExtractor extends SuggestionExtractor {
public static final String CHARSET_UTF_8 = "UTF-8";
public YoutubeSuggestionExtractor(StreamingService service) {
super(service);
}
@ -52,8 +52,8 @@ public class YoutubeSuggestionExtractor extends SuggestionExtractor {
+ "?client=" + "youtube" //"firefox" for JSON, 'toolbar' for xml
+ "&jsonp=" + "JP"
+ "&ds=" + "yt"
+ "&gl=" + URLEncoder.encode(getExtractorContentCountry().getCountryCode(), CHARSET_UTF_8)
+ "&q=" + URLEncoder.encode(query, CHARSET_UTF_8);
+ "&gl=" + URLEncoder.encode(getExtractorContentCountry().getCountryCode(), UTF_8)
+ "&q=" + URLEncoder.encode(query, UTF_8);
String response = dl.get(url, getExtractorLocalization()).responseBody();
// trim JSONP part "JP(...)"

10
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java
View File

@ -69,7 +69,15 @@ public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
@Nonnull
@Override
public String getName() throws ParsingException {
String name = getTextFromObject(initialData.getObject("header").getObject("feedTabbedHeaderRenderer").getObject("title"));
final JsonObject header = initialData.getObject("header");
JsonObject title = null;
if (header.has("feedTabbedHeaderRenderer")) {
title = header.getObject("feedTabbedHeaderRenderer").getObject("title");
} else if (header.has("c4TabbedHeaderRenderer")) {
title = header.getObject("c4TabbedHeaderRenderer").getObject("title");
}
String name = getTextFromObject(title);
if (!isNullOrEmpty(name)) {
return name;
}

13
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java
View File

@ -7,8 +7,9 @@ import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.List;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
public class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
public static final String CHARSET_UTF_8 = "UTF-8";
public static final String ALL = "all";
public static final String VIDEOS = "videos";
@ -37,21 +38,21 @@ public class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory
default:
break;
case VIDEOS:
return SEARCH_URL + URLEncoder.encode(searchString, CHARSET_UTF_8) + "&sp=EgIQAQ%253D%253D";
return SEARCH_URL + URLEncoder.encode(searchString, UTF_8) + "&sp=EgIQAQ%253D%253D";
case CHANNELS:
return SEARCH_URL + URLEncoder.encode(searchString, CHARSET_UTF_8) + "&sp=EgIQAg%253D%253D";
return SEARCH_URL + URLEncoder.encode(searchString, UTF_8) + "&sp=EgIQAg%253D%253D";
case PLAYLISTS:
return SEARCH_URL + URLEncoder.encode(searchString, CHARSET_UTF_8) + "&sp=EgIQAw%253D%253D";
return SEARCH_URL + URLEncoder.encode(searchString, UTF_8) + "&sp=EgIQAw%253D%253D";
case MUSIC_SONGS:
case MUSIC_VIDEOS:
case MUSIC_ALBUMS:
case MUSIC_PLAYLISTS:
case MUSIC_ARTISTS:
return MUSIC_SEARCH_URL + URLEncoder.encode(searchString, CHARSET_UTF_8);
return MUSIC_SEARCH_URL + URLEncoder.encode(searchString, UTF_8);
}
}
return SEARCH_URL + URLEncoder.encode(searchString, CHARSET_UTF_8);
return SEARCH_URL + URLEncoder.encode(searchString, UTF_8);
} catch (UnsupportedEncodingException e) {
throw new ParsingException("Could not encode query", e);
}

14
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java
View File

@ -36,7 +36,7 @@ import java.util.regex.Pattern;
public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
private static final Pattern YOUTUBE_VIDEO_ID_REGEX_PATTERN = Pattern.compile("([a-zA-Z0-9_-]{11})");
private static final Pattern YOUTUBE_VIDEO_ID_REGEX_PATTERN = Pattern.compile("^([a-zA-Z0-9_-]{11})");
private static final YoutubeStreamLinkHandlerFactory instance = new YoutubeStreamLinkHandlerFactory();
private YoutubeStreamLinkHandlerFactory() {
@ -183,10 +183,10 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
// there is no break-statement here on purpose so the next code-block gets also run for hooktube
}
case "WWW.INVIDIO.US":
case "DEV.INVIDIO.US":
case "INVIDIO.US":
case "VID.ENCRYPTIONIN.SPACE":
case "DEV.INVIDIO.US":
case "WWW.INVIDIO.US":
case "REDIRECT.INVIDIOUS.IO":
case "INVIDIOUS.SNOPYTA.ORG":
case "YEWTU.BE":
case "TUBE.CONNECT.CAFE":
@ -197,7 +197,11 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
case "INVIDIOUS.XYZ":
case "VID.MINT.LGBT":
case "INVIDIOU.SITE":
case "INVIDIOUS.FDN.FR": { // code-block for hooktube.com and Invidious instances
case "INVIDIOUS.FDN.FR":
case "INVIDIOUS.048596.XYZ":
case "INVIDIOUS.ZEE.LI":
case "VID.PUFFYAN.US":
case "YTPRIVATE.COM": { // code-block for hooktube.com and Invidious instances
if (path.equals("watch")) {
String viewQueryValue = Utils.getQueryValue(url, "v");
if (viewQueryValue != null) {

18
extractor/src/main/java/org/schabi/newpipe/extractor/stream/Description.java
View File

@ -1,13 +1,16 @@
package org.schabi.newpipe.extractor.stream;
import java.io.Serializable;
import java.util.Objects;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
public class Description implements Serializable {
public static final int HTML = 1;
public static final int MARKDOWN = 2;
public static final int PLAIN_TEXT = 3;
public static final Description emptyDescription = new Description("", PLAIN_TEXT);
public static final Description emptyDescription = new Description(EMPTY_STRING, PLAIN_TEXT);
private String content;
private int type;
@ -28,4 +31,17 @@ public class Description implements Serializable {
public int getType() {
return type;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
Description that = (Description) o;
return type == that.type && Objects.equals(content, that.content);
}
@Override
public int hashCode() {
return Objects.hash(content, type);
}
}

2
extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java
View File

@ -64,7 +64,7 @@ public class StreamInfo extends Info {
return getInfo(service.getStreamExtractor(url));
}
private static StreamInfo getInfo(StreamExtractor extractor) throws ExtractionException, IOException {
public static StreamInfo getInfo(StreamExtractor extractor) throws ExtractionException, IOException {
extractor.fetchPage();
StreamInfo streamInfo;
try {

1
extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java
View File

@ -13,7 +13,6 @@ import java.util.List;
public class JsonUtils {
public static final JsonObject EMPTY_OBJECT = new JsonObject();
public static final JsonArray EMPTY_ARRAY = new JsonArray();
public static final String EMPTY_STRING = "";
private JsonUtils() {
}

8
extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java
View File

@ -14,6 +14,8 @@ import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
/*
* Created by Christian Schabesberger on 02.02.16.
*
@ -69,9 +71,9 @@ public class Parser {
} else {
// only pass input to exception message when it is not too long
if (input.length() > 1024) {
throw new RegexException("failed to find pattern \"" + pat.pattern());
throw new RegexException("failed to find pattern \"" + pat.pattern() + "\"");
} else {
throw new RegexException("failed to find pattern \"" + pat.pattern() + " inside of " + input + "\"");
throw new RegexException("failed to find pattern \"" + pat.pattern() + "\" inside of \"" + input + "\"");
}
}
}
@ -87,7 +89,7 @@ public class Parser {
for (String arg : input.split("&")) {
String[] splitArg = arg.split("=");
if (splitArg.length > 1) {
map.put(splitArg[0], URLDecoder.decode(splitArg[1], "UTF-8"));
map.put(splitArg[0], URLDecoder.decode(splitArg[1], UTF_8));
} else {
map.put(splitArg[0], "");
}

23
extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java
View File

@ -6,16 +6,14 @@ import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLDecoder;
import java.util.Collection;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.*;
public class Utils {
public static final String HTTP = "http://";
public static final String HTTPS = "https://";
public static final String UTF_8 = "UTF-8";
public static final String EMPTY_STRING = "";
private Utils() {
//no instance
@ -117,7 +115,7 @@ public class Utils {
String query;
try {
query = URLDecoder.decode(params[0], "UTF-8");
query = URLDecoder.decode(params[0], UTF_8);
} catch (UnsupportedEncodingException e) {
System.err.println("Cannot decode string with UTF-8. using the string without decoding");
e.printStackTrace();
@ -126,7 +124,7 @@ public class Utils {
if (query.equals(parameterName)) {
try {
return URLDecoder.decode(params[1], "UTF-8");
return URLDecoder.decode(params[1], UTF_8);
} catch (UnsupportedEncodingException e) {
System.err.println("Cannot decode string with UTF-8. using the string without decoding");
e.printStackTrace();
@ -200,6 +198,7 @@ public class Utils {
/**
* If the provided url is a Google search redirect, then the actual url is extracted from the
* {@code url=} query value and returned, otherwise the original url is returned.
*
* @param url the url which can possibly be a Google search redirect
* @return an url with no Google search redirects
*/
@ -208,7 +207,7 @@ public class Utils {
try {
final URL decoded = Utils.stringToURL(url);
if (decoded.getHost().contains("google") && decoded.getPath().equals("/url")) {
return URLDecoder.decode(Parser.matchGroup1("&url=([^&]+)(?:&|$)", url), "UTF-8");
return URLDecoder.decode(Parser.matchGroup1("&url=([^&]+)(?:&|$)", url), UTF_8);
}
} catch (final Exception ignored) {
}
@ -231,12 +230,12 @@ public class Utils {
return map == null || map.isEmpty();
}
public static boolean isWhitespace(final int c){
public static boolean isWhitespace(final int c) {
return c == ' ' || c == '\t' || c == '\n' || c == '\f' || c == '\r';
}
public static boolean isBlank(final String string) {
if (string == null || string.isEmpty()) {
if (isNullOrEmpty(string)) {
return true;
}
@ -263,10 +262,10 @@ public class Utils {
}
public static String join(final String delimiter, final String mapJoin,
final Map<? extends CharSequence, ? extends CharSequence> elements) {
final Map<? extends CharSequence, ? extends CharSequence> elements) {
final List<String> list = new LinkedList<>();
for (final Map.Entry<? extends CharSequence, ? extends CharSequence> entry : elements
.entrySet()) {
.entrySet()) {
list.add(entry.getKey() + mapJoin + entry.getValue());
}
return join(delimiter, list);

3
extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCRecentListExtractorTest.java
View File

@ -26,16 +26,13 @@ public class MediaCCCRecentListExtractorTest {
}
@Test
@Ignore("TODO fix")
public void testStreamList() throws Exception {
final List<StreamInfoItem> items = extractor.getInitialPage().getItems();
assertEquals(100, items.size());
for (final StreamInfoItem item: items) {
assertFalse(isNullOrEmpty(item.getName()));
assertTrue(item.getDuration() > 0);
assertTrue(isNullOrEmpty(item.getUploaderName())); // we do not get the uploader name
assertTrue(item.getUploadDate().offsetDateTime().isBefore(OffsetDateTime.now()));
assertTrue(item.getUploadDate().offsetDateTime().isAfter(OffsetDateTime.now().minusYears(1)));
}
}

1
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeAccountExtractorTest.java
View File

@ -101,7 +101,6 @@ public class PeertubeAccountExtractorTest {
}
@Test
@Ignore("TODO fix")
public void testSubscriberCount() throws ParsingException {
assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 500);
}

5
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistExtractorTest.java
View File

@ -4,6 +4,7 @@ import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.ExtractorAsserts;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubePlaylistExtractor;
@ -51,9 +52,8 @@ public class PeertubePlaylistExtractorTest {
}
@Test
@Ignore("TODO fix")
public void testGetStreamCount() throws ParsingException {
assertEquals(35, extractor.getStreamCount());
ExtractorAsserts.assertAtLeast(39, extractor.getStreamCount());
}
@Test
@ -62,7 +62,6 @@ public class PeertubePlaylistExtractorTest {
}
@Test
@Ignore("TODO fix")
public void testGetSubChannelName() throws ParsingException {
assertEquals("SHOCKING !", extractor.getSubChannelName());
}

7
extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorTest.java
View File

@ -94,13 +94,6 @@ public class PeertubeStreamExtractorTest {
@Override public Locale expectedLanguageInfo() { return Locale.forLanguageTag("en"); }
@Override public List<String> expectedTags() { return Arrays.asList("framasoft", "peertube"); }
@Override public int expectedStreamSegmentsCount() { return 0; }
@Override
@Test
@Ignore("TODO fix")
public void testSubChannelName() throws Exception {
super.testSubChannelName();
}
}
@Ignore("TODO fix")

1
extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java
View File

@ -42,7 +42,6 @@ public class SoundcloudPlaylistExtractorTest {
}
@Test
@Ignore("TODO fix")
public void testName() {
assertEquals("THE PERFECT LUV TAPE®️", extractor.getName());
}

1
extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamLinkHandlerFactoryTest.java
View File

@ -26,7 +26,6 @@ public class SoundcloudStreamLinkHandlerFactoryTest {
}
@Test(expected = IllegalArgumentException.class)
@Ignore("TODO fix")
public void getIdWithNullAsUrl() throws ParsingException {
linkHandler.fromUrl(null).getId();
}

15
extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorTest.java
View File

@ -19,6 +19,7 @@ import static java.util.Collections.singletonList;
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoDuplicatedItems;
import static org.schabi.newpipe.extractor.services.soundcloud.linkHandler.SoundcloudSearchQueryHandlerFactory.*;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
public class SoundcloudSearchExtractorTest {
@ -33,6 +34,7 @@ public class SoundcloudSearchExtractorTest {
extractor.fetchPage();
}
// @formatter:off
@Override public SearchExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return SoundCloud; }
@Override public String expectedName() { return QUERY; }
@ -41,6 +43,7 @@ public class SoundcloudSearchExtractorTest {
@Override public String expectedOriginalUrlContains() { return "soundcloud.com/search?q=" + urlEncode(QUERY); }
@Override public String expectedSearchString() { return QUERY; }
@Nullable @Override public String expectedSearchSuggestion() { return null; }
// @formatter:on
}
public static class Tracks extends DefaultSearchExtractorTest {
@ -54,6 +57,7 @@ public class SoundcloudSearchExtractorTest {
extractor.fetchPage();
}
// @formatter:off
@Override public SearchExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return SoundCloud; }
@Override public String expectedName() { return QUERY; }
@ -62,8 +66,8 @@ public class SoundcloudSearchExtractorTest {
@Override public String expectedOriginalUrlContains() { return "soundcloud.com/search/tracks?q=" + urlEncode(QUERY); }
@Override public String expectedSearchString() { return QUERY; }
@Nullable @Override public String expectedSearchSuggestion() { return null; }
@Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; }
// @formatter:on
}
public static class Users extends DefaultSearchExtractorTest {
@ -77,6 +81,7 @@ public class SoundcloudSearchExtractorTest {
extractor.fetchPage();
}
// @formatter:off
@Override public SearchExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return SoundCloud; }
@Override public String expectedName() { return QUERY; }
@ -85,8 +90,8 @@ public class SoundcloudSearchExtractorTest {
@Override public String expectedOriginalUrlContains() { return "soundcloud.com/search/users?q=" + urlEncode(QUERY); }
@Override public String expectedSearchString() { return QUERY; }
@Nullable @Override public String expectedSearchSuggestion() { return null; }
@Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.CHANNEL; }
// @formatter:on
}
public static class Playlists extends DefaultSearchExtractorTest {
@ -100,6 +105,7 @@ public class SoundcloudSearchExtractorTest {
extractor.fetchPage();
}
// @formatter:off
@Override public SearchExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return SoundCloud; }
@Override public String expectedName() { return QUERY; }
@ -108,13 +114,12 @@ public class SoundcloudSearchExtractorTest {
@Override public String expectedOriginalUrlContains() { return "soundcloud.com/search/playlists?q=" + urlEncode(QUERY); }
@Override public String expectedSearchString() { return QUERY; }
@Nullable @Override public String expectedSearchSuggestion() { return null; }
@Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.PLAYLIST; }
// @formatter:on
}
public static class PagingTest {
@Test
@Ignore("TODO fix")
public void duplicatedItemsCheck() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
final SearchExtractor extractor = SoundCloud.getSearchExtractor("cirque du soleil", singletonList(TRACKS), "");
@ -129,7 +134,7 @@ public class SoundcloudSearchExtractorTest {
private static String urlEncode(String value) {
try {
return URLEncoder.encode(value, CHARSET_UTF_8);
return URLEncoder.encode(value, UTF_8);
} catch (UnsupportedEncodingException e) {
throw new RuntimeException(e);
}

1
extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchQHTest.java
View File

@ -24,7 +24,6 @@ public class SoundcloudSearchQHTest {
}
@Test
@Ignore("TODO fix")
public void testRegularValues() throws Exception {
assertEquals("https://api-v2.soundcloud.com/search?q=asdf&limit=10&offset=0",
removeClientId(SoundCloud.getSearchQHFactory().fromQuery("asdf").getUrl()));

71
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java
View File

@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor.services.youtube;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.schabi.newpipe.downloader.DownloaderFactory;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
@ -12,21 +13,29 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor;
import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmpty;
import java.io.IOException;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
import static org.schabi.newpipe.extractor.services.DefaultTests.*;
import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestGetPageInNewExtractor;
import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestMoreItems;
import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRelatedItems;
/**
* Test for {@link ChannelExtractor}
*/
public class YoutubeChannelExtractorTest {
private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/channel/";
public static class NotAvailable {
@BeforeClass
public static void setUp() {
NewPipe.init(DownloaderTestImpl.getInstance());
public static void setUp() throws IOException {
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "notAvailable"));
}
@Test(expected = ContentNotAvailableException.class)
@ -46,8 +55,9 @@ public class YoutubeChannelExtractorTest {
public static class NotSupported {
@BeforeClass
public static void setUp() {
NewPipe.init(DownloaderTestImpl.getInstance());
public static void setUp() throws IOException {
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "notSupported"));
}
@Test(expected = ContentNotSupportedException.class)
@ -63,7 +73,8 @@ public class YoutubeChannelExtractorTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "gronkh"));
extractor = (YoutubeChannelExtractor) YouTube
.getChannelExtractor("http://www.youtube.com/user/Gronkh");
extractor.fetchPage();
@ -116,8 +127,8 @@ public class YoutubeChannelExtractorTest {
// ChannelExtractor
//////////////////////////////////////////////////////////////////////////*/
@Ignore("TODO fix, character ü makes problems")
@Test
@Ignore("TODO fix")
public void testDescription() throws Exception {
assertTrue(extractor.getDescription().contains("Zart im Schmelz und süffig im Abgang. Ungebremster Spieltrieb"));
}
@ -154,7 +165,8 @@ public class YoutubeChannelExtractorTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "VSauce"));
extractor = (YoutubeChannelExtractor) YouTube
.getChannelExtractor("https://www.youtube.com/user/Vsauce");
extractor.fetchPage();
@ -245,22 +257,13 @@ public class YoutubeChannelExtractorTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "kurzgesagt"));
extractor = (YoutubeChannelExtractor) YouTube
.getChannelExtractor("https://www.youtube.com/channel/UCsXVk37bltHxD1rDPwtNM8Q");
extractor.fetchPage();
}
/*//////////////////////////////////////////////////////////////////////////
// Additional Testing
//////////////////////////////////////////////////////////////////////////*/
@Test
public void testGetPageInNewExtractor() throws Exception {
final ChannelExtractor newExtractor = YouTube.getChannelExtractor(extractor.getUrl());
defaultTestGetPageInNewExtractor(extractor, newExtractor);
}
/*//////////////////////////////////////////////////////////////////////////
// Extractor
//////////////////////////////////////////////////////////////////////////*/
@ -342,12 +345,33 @@ public class YoutubeChannelExtractorTest {
}
}
public static class KurzgesagtAdditional {
private static YoutubeChannelExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
// Test is not deterministic, mocks can't be used
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (YoutubeChannelExtractor) YouTube
.getChannelExtractor("https://www.youtube.com/channel/UCsXVk37bltHxD1rDPwtNM8Q");
extractor.fetchPage();
}
@Test
public void testGetPageInNewExtractor() throws Exception {
final ChannelExtractor newExtractor = YouTube.getChannelExtractor(extractor.getUrl());
defaultTestGetPageInNewExtractor(extractor, newExtractor);
}
}
public static class CaptainDisillusion implements BaseChannelExtractorTest {
private static YoutubeChannelExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "captainDisillusion"));
extractor = (YoutubeChannelExtractor) YouTube
.getChannelExtractor("https://www.youtube.com/user/CaptainDisillusion/videos");
extractor.fetchPage();
@ -436,7 +460,8 @@ public class YoutubeChannelExtractorTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "random"));
extractor = (YoutubeChannelExtractor) YouTube
.getChannelExtractor("https://www.youtube.com/channel/UCUaQMQS9lY5lit3vurpXQ6w");
extractor.fetchPage();

10
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLocalizationTest.java
View File

@ -1,8 +1,7 @@
package org.schabi.newpipe.extractor.services.youtube;
import org.junit.Ignore;
import org.junit.Test;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.downloader.DownloaderFactory;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
@ -23,14 +22,15 @@ import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRela
/**
* A class that tests multiple channels and ranges of "time ago".
*/
@Ignore("Should be ran manually from time to time, as it's too time consuming.")
public class YoutubeChannelLocalizationTest {
private static final boolean DEBUG = true;
private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/channel/";
private static final boolean DEBUG = false;
private final DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm");
@Test
public void testAllSupportedLocalizations() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "localization"));
testLocalizationsFor("https://www.youtube.com/user/NBCNews");
testLocalizationsFor("https://www.youtube.com/channel/UCcmpeVbSSQlZRvHfdC-CRwg/videos");

59
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java
View File

@ -1,8 +1,9 @@
package org.schabi.newpipe.extractor.services.youtube;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.downloader.DownloaderFactory;
import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.Page;
@ -16,10 +17,15 @@ import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
import java.util.List;
import static org.junit.Assert.*;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
public class YoutubeCommentsExtractorTest {
private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/comments/";
/**
* Test a "normal" YouTube
*/
@ -30,7 +36,8 @@ public class YoutubeCommentsExtractorTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "thomas"));
extractor = (YoutubeCommentsExtractor) YouTube
.getCommentsExtractor(url);
extractor.fetchPage();
@ -117,13 +124,15 @@ public class YoutubeCommentsExtractorTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "empty"));
extractor = (YoutubeCommentsExtractor) YouTube
.getCommentsExtractor(url);
extractor.fetchPage();
}
@Test
@Ignore("TODO fix")
public void testGetCommentsAllData() throws IOException, ExtractionException {
final InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
@ -155,7 +164,8 @@ public class YoutubeCommentsExtractorTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "hearted"));
extractor = (YoutubeCommentsExtractor) YouTube
.getCommentsExtractor(url);
extractor.fetchPage();
@ -181,7 +191,7 @@ public class YoutubeCommentsExtractorTest {
assertFalse(Utils.isBlank(c.getUrl()));
assertFalse(c.getLikeCount() < 0);
assertFalse(Utils.isBlank(c.getCommentText()));
if (c.getHeartedByUploader()) {
if (c.isHeartedByUploader()) {
heartedByUploader = true;
}
}
@ -189,4 +199,41 @@ public class YoutubeCommentsExtractorTest {
}
}
public static class Pinned {
private final static String url = "https://www.youtube.com/watch?v=bjFtFMilb34";
private static YoutubeCommentsExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "pinned"));
extractor = (YoutubeCommentsExtractor) YouTube
.getCommentsExtractor(url);
extractor.fetchPage();
}
@Test
public void testGetCommentsAllData() throws IOException, ExtractionException {
final InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
DefaultTests.defaultTestListOfItems(YouTube, comments.getItems(), comments.getErrors());
for (CommentsInfoItem c : comments.getItems()) {
assertFalse(Utils.isBlank(c.getUploaderUrl()));
assertFalse(Utils.isBlank(c.getUploaderName()));
assertFalse(Utils.isBlank(c.getUploaderAvatarUrl()));
assertFalse(Utils.isBlank(c.getCommentId()));
assertFalse(Utils.isBlank(c.getName()));
assertFalse(Utils.isBlank(c.getTextualUploadDate()));
assertNotNull(c.getUploadDate());
assertFalse(Utils.isBlank(c.getThumbnailUrl()));
assertFalse(Utils.isBlank(c.getUrl()));
assertFalse(c.getLikeCount() < 0);
assertFalse(Utils.isBlank(c.getCommentText()));
}
assertTrue("First comment isn't pinned", comments.getItems().get(0).isPinned());
}
}
}

11
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeFeedExtractorTest.java
View File

@ -2,24 +2,29 @@ package org.schabi.newpipe.extractor.services.youtube;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.downloader.DownloaderFactory;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.BaseListExtractorTest;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeFeedExtractor;
import static org.junit.Assert.*;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoMoreItems;
import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRelatedItems;
public class YoutubeFeedExtractorTest {
private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/feed/";
public static class Kurzgesagt implements BaseListExtractorTest {
private static YoutubeFeedExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH));
extractor = (YoutubeFeedExtractor) YouTube
.getFeedExtractor("https://www.youtube.com/user/Kurzgesagt");
extractor.fetchPage();

8
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeKioskExtractorTest.java
View File

@ -2,7 +2,7 @@ package org.schabi.newpipe.extractor.services.youtube;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.downloader.DownloaderFactory;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.BaseListExtractorTest;
@ -14,12 +14,16 @@ import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoMoreIte
import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRelatedItems;
public class YoutubeKioskExtractorTest {
private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/kiosk/";
public static class Trending implements BaseListExtractorTest {
private static YoutubeTrendingExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "trending"));
extractor = (YoutubeTrendingExtractor) YouTube.getKioskList().getDefaultKioskExtractor();
extractor.fetchPage();
}

49
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java
View File

@ -1,11 +1,5 @@
package org.schabi.newpipe.extractor.services.youtube;
import java.io.File;
import java.io.IOException;
import java.util.Collections;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import org.hamcrest.MatcherAssert;
import org.junit.BeforeClass;
import org.junit.Test;
@ -26,6 +20,12 @@ import org.schabi.newpipe.extractor.services.youtube.YoutubeMixPlaylistExtractor
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMixPlaylistExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import java.io.IOException;
import java.util.Collections;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.CoreMatchers.startsWith;
import static org.hamcrest.MatcherAssert.assertThat;
@ -43,7 +43,7 @@ public class YoutubeMixPlaylistExtractorTest {
private static final String VIDEO_ID = "_AzeUSL9lZc";
private static final String VIDEO_TITLE =
"Most Beautiful And Emotional Piano: Anime Music Shigatsu wa Kimi no Uso OST IMO";
private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/mix/";
private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/mix/";
private static final Map<String, String> dummyCookie
= Collections.singletonMap(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
@ -134,9 +134,9 @@ public class YoutubeMixPlaylistExtractorTest {
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "mixWithIndex"));
extractor = (YoutubeMixPlaylistExtractor) YouTube
.getPlaylistExtractor(
"https://www.youtube.com/watch?v=" + VIDEO_ID_NUMBER_13 + "&list=RD"
+ VIDEO_ID + INDEX);
.getPlaylistExtractor(
"https://www.youtube.com/watch?v=" + VIDEO_ID_NUMBER_13 + "&list=RD"
+ VIDEO_ID + INDEX);
extractor.fetchPage();
}
@ -165,8 +165,8 @@ public class YoutubeMixPlaylistExtractorTest {
@Test
public void getPage() throws Exception {
final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(
new Page("https://www.youtube.com/watch?v=" + VIDEO_ID_NUMBER_13 + "&list=RD"
+ VIDEO_ID + INDEX + PBJ, dummyCookie));
new Page("https://www.youtube.com/watch?v=" + VIDEO_ID_NUMBER_13 + "&list=RD"
+ VIDEO_ID + INDEX + PBJ, dummyCookie));
assertFalse(streams.getItems().isEmpty());
assertTrue(streams.hasNextPage());
}
@ -204,9 +204,9 @@ public class YoutubeMixPlaylistExtractorTest {
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "myMix"));
extractor = (YoutubeMixPlaylistExtractor) YouTube
.getPlaylistExtractor(
"https://www.youtube.com/watch?v=" + VIDEO_ID + "&list=RDMM"
+ VIDEO_ID);
.getPlaylistExtractor(
"https://www.youtube.com/watch?v=" + VIDEO_ID + "&list=RDMM"
+ VIDEO_ID);
extractor.fetchPage();
}
@ -238,11 +238,12 @@ public class YoutubeMixPlaylistExtractorTest {
@Test
public void getPage() throws Exception {
final InfoItemsPage<StreamInfoItem> streams =
extractor.getPage(new Page("https://www.youtube.com/watch?v=" + VIDEO_ID
+ "&list=RDMM" + VIDEO_ID + PBJ, dummyCookie));
extractor.getPage(new Page("https://www.youtube.com/watch?v=" + VIDEO_ID
+ "&list=RDMM" + VIDEO_ID + PBJ, dummyCookie));
assertFalse(streams.getItems().isEmpty());
assertTrue(streams.hasNextPage());
}
@Test
public void getContinuations() throws Exception {
InfoItemsPage<StreamInfoItem> streams = extractor.getInitialPage();
@ -290,8 +291,8 @@ public class YoutubeMixPlaylistExtractorTest {
@Test(expected = ExtractionException.class)
public void invalidVideoId() throws Exception {
extractor = (YoutubeMixPlaylistExtractor) YouTube
.getPlaylistExtractor(
"https://www.youtube.com/watch?v=" + "abcde" + "&list=RD" + "abcde");
.getPlaylistExtractor(
"https://www.youtube.com/watch?v=" + "abcde" + "&list=RD" + "abcde");
extractor.fetchPage();
extractor.getName();
}
@ -309,9 +310,9 @@ public class YoutubeMixPlaylistExtractorTest {
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "channelMix"));
extractor = (YoutubeMixPlaylistExtractor) YouTube
.getPlaylistExtractor(
"https://www.youtube.com/watch?v=" + VIDEO_ID_OF_CHANNEL
+ "&list=RDCM" + CHANNEL_ID);
.getPlaylistExtractor(
"https://www.youtube.com/watch?v=" + VIDEO_ID_OF_CHANNEL
+ "&list=RDCM" + CHANNEL_ID);
extractor.fetchPage();
}
@ -339,8 +340,8 @@ public class YoutubeMixPlaylistExtractorTest {
@Test
public void getPage() throws Exception {
final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(
new Page("https://www.youtube.com/watch?v=" + VIDEO_ID_OF_CHANNEL
+ "&list=RDCM" + CHANNEL_ID + PBJ, dummyCookie));
new Page("https://www.youtube.com/watch?v=" + VIDEO_ID_OF_CHANNEL
+ "&list=RDCM" + CHANNEL_ID + PBJ, dummyCookie));
assertFalse(streams.getItems().isEmpty());
assertTrue(streams.hasNextPage());
}

10
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java
View File

@ -2,7 +2,7 @@ package org.schabi.newpipe.extractor.services.youtube;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.downloader.DownloaderFactory;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
@ -13,9 +13,13 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
public class YoutubeParsingHelperTest {
private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/";
@BeforeClass
public static void setUp() {
NewPipe.init(DownloaderTestImpl.getInstance());
public static void setUp() throws IOException {
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "youtubeParsingHelper"));
}
@Test

33
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java
View File

@ -6,10 +6,9 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
import org.junit.runners.Suite.SuiteClasses;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.downloader.DownloaderFactory;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
@ -22,13 +21,18 @@ import org.schabi.newpipe.extractor.services.youtube.YoutubePlaylistExtractorTes
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubePlaylistExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import java.io.IOException;
import static junit.framework.TestCase.assertFalse;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
import static org.schabi.newpipe.extractor.services.DefaultTests.*;
import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoMoreItems;
import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestGetPageInNewExtractor;
import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestListOfItems;
import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestMoreItems;
import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRelatedItems;
/**
* Test for {@link YoutubePlaylistExtractor}
@ -38,10 +42,13 @@ import static org.schabi.newpipe.extractor.services.DefaultTests.*;
LearningPlaylist.class, ContinuationsTests.class})
public class YoutubePlaylistExtractorTest {
private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/playlist/";
public static class NotAvailable {
@BeforeClass
public static void setUp() {
NewPipe.init(DownloaderTestImpl.getInstance());
public static void setUp() throws IOException {
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "notAvailable"));
}
@Test(expected = ContentNotAvailableException.class)
@ -65,7 +72,8 @@ public class YoutubePlaylistExtractorTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "TimelessPopHits"));
extractor = (YoutubePlaylistExtractor) YouTube
.getPlaylistExtractor("http://www.youtube.com/watch?v=lp-EO5I60KA&list=PLMC9KNkIncKtPzgY-5rmhvj7fax8fdxoj");
extractor.fetchPage();
@ -162,7 +170,8 @@ public class YoutubePlaylistExtractorTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "huge"));
extractor = (YoutubePlaylistExtractor) YouTube
.getPlaylistExtractor("https://www.youtube.com/watch?v=8SbUC-UaAxE&list=PLWwAypAcFRgKAIIFqBr9oy-ZYZnixa_Fj");
extractor.fetchPage();
@ -274,7 +283,8 @@ public class YoutubePlaylistExtractorTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "learning"));
extractor = (YoutubePlaylistExtractor) YouTube
.getPlaylistExtractor("https://www.youtube.com/playlist?list=PL8dPuuaLjXtOAKed_MxxWBNaPno5h3Zs8");
extractor.fetchPage();
@ -370,8 +380,9 @@ public class YoutubePlaylistExtractorTest {
public static class ContinuationsTests {
@BeforeClass
public static void setUp() {
NewPipe.init(DownloaderTestImpl.getInstance());
public static void setUp() throws IOException {
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "continuations"));
}
@Test

2
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java
View File

@ -42,6 +42,8 @@ public class YoutubeStreamLinkHandlerFactoryTest {
invalidUrls.add("https://www.youtube.com/watch?v=jZViOEv90d");
invalidUrls.add("https://www.youtube.com/watchjZViOEv90d");
invalidUrls.add("https://www.youtube.com/");
invalidUrls.add("https://www.youtube.com/channel/UCBR8-60-B28hp2BmDPdntcQ");
invalidUrls.add("https://invidio.us/channel/UCBR8-60-B28hp2BmDPdntcQ");
for (String invalidUrl : invalidUrls) {
Throwable exception = null;
try {

18
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractorTest.java
View File

@ -12,25 +12,25 @@ import org.schabi.newpipe.extractor.subscription.SubscriptionItem;
import java.io.ByteArrayInputStream;
import java.io.FileInputStream;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.List;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.junit.Assert.*;
import static org.schabi.newpipe.FileUtils.resolveTestResource;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
/**
* Test for {@link YoutubeSubscriptionExtractor}
*/
public class YoutubeSubscriptionExtractorTest {
private static YoutubeSubscriptionExtractor subscriptionExtractor;
private static LinkHandlerFactory urlHandler;
@BeforeClass
public static void setupClass() {
//Doesn't make network requests
NewPipe.init(DownloaderTestImpl.getInstance());
subscriptionExtractor = new YoutubeSubscriptionExtractor(ServiceList.YouTube);
urlHandler = ServiceList.YouTube.getChannelLHFactory();
@ -53,7 +53,7 @@ public class YoutubeSubscriptionExtractorTest {
@Test
public void testEmptySourceException() throws Exception {
final List<SubscriptionItem> items = subscriptionExtractor.fromInputStream(
new ByteArrayInputStream("[]".getBytes(StandardCharsets.UTF_8)));
new ByteArrayInputStream("[]".getBytes(UTF_8)));
assertTrue(items.isEmpty());
}
@ -61,7 +61,7 @@ public class YoutubeSubscriptionExtractorTest {
public void testSubscriptionWithEmptyTitleInSource() throws Exception {
final String source = "[{\"snippet\":{\"resourceId\":{\"channelId\":\"UCEOXxzW2vU0P-0THehuIIeg\"}}}]";
final List<SubscriptionItem> items = subscriptionExtractor.fromInputStream(
new ByteArrayInputStream(source.getBytes(StandardCharsets.UTF_8)));
new ByteArrayInputStream(source.getBytes(UTF_8)));
assertEquals(1, items.size());
assertEquals(ServiceList.YouTube.getServiceId(), items.get(0).getServiceId());
@ -74,7 +74,7 @@ public class YoutubeSubscriptionExtractorTest {
final String source = "[{\"snippet\":{\"resourceId\":{\"channelId\":\"gibberish\"},\"title\":\"name1\"}}," +
"{\"snippet\":{\"resourceId\":{\"channelId\":\"UCEOXxzW2vU0P-0THehuIIeg\"},\"title\":\"name2\"}}]";
final List<SubscriptionItem> items = subscriptionExtractor.fromInputStream(
new ByteArrayInputStream(source.getBytes(StandardCharsets.UTF_8)));
new ByteArrayInputStream(source.getBytes(UTF_8)));
assertEquals(1, items.size());
assertEquals(ServiceList.YouTube.getServiceId(), items.get(0).getServiceId());
@ -98,7 +98,7 @@ public class YoutubeSubscriptionExtractorTest {
for (String invalidContent : invalidList) {
try {
byte[] bytes = invalidContent.getBytes(StandardCharsets.UTF_8);
byte[] bytes = invalidContent.getBytes(UTF_8);
subscriptionExtractor.fromInputStream(new ByteArrayInputStream(bytes));
fail("Extracting from \"" + invalidContent + "\" didn't throw an exception");
} catch (final Exception e) {

8
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSuggestionExtractorTest.java
View File

@ -22,7 +22,7 @@ package org.schabi.newpipe.extractor.services.youtube;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.downloader.DownloaderFactory;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.localization.Localization;
@ -37,11 +37,15 @@ import static org.schabi.newpipe.extractor.ServiceList.YouTube;
* Test for {@link SuggestionExtractor}
*/
public class YoutubeSuggestionExtractorTest {
private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/suggestions/";
private static SuggestionExtractor suggestionExtractor;
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance(), new Localization("de", "DE"));
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + ""), new Localization("de", "DE"));
suggestionExtractor = YouTube.getSuggestionExtractor();
}

11
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingKioskInfoTest.java
View File

@ -22,7 +22,7 @@ package org.schabi.newpipe.extractor.services.youtube;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.downloader.DownloaderFactory;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.kiosk.KioskInfo;
@ -36,14 +36,17 @@ import static org.schabi.newpipe.extractor.ServiceList.YouTube;
* Test for {@link KioskInfo}
*/
public class YoutubeTrendingKioskInfoTest {
private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "kiosk";
static KioskInfo kioskInfo;
@BeforeClass
public static void setUp()
throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
StreamingService service = YouTube;
LinkHandlerFactory LinkHandlerFactory = service.getKioskList().getListLinkHandlerFactoryByType("Trending");
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH));
LinkHandlerFactory LinkHandlerFactory = ((StreamingService) YouTube).getKioskList().getListLinkHandlerFactoryByType("Trending");
kioskInfo = KioskInfo.getInfo(YouTube, LinkHandlerFactory.fromId("Trending").getUrl());
}

4
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeMusicSearchExtractorTest.java
View File

@ -10,12 +10,14 @@ import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.DefaultSearchExtractorTest;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory;
import javax.annotation.Nullable;
import java.net.URLEncoder;
import javax.annotation.Nullable;
import static java.util.Collections.singletonList;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
// Doesn't work with mocks. Makes request with different `dataToSend` i think
public class YoutubeMusicSearchExtractorTest {
public static class MusicSongs extends DefaultSearchExtractorTest {
private static SearchExtractor extractor;

48
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java
View File

@ -1,24 +1,26 @@
package org.schabi.newpipe.extractor.services.youtube.search;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.*;
import org.schabi.newpipe.downloader.DownloaderFactory;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.MetaInfo;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.DefaultSearchExtractorTest;
import org.schabi.newpipe.extractor.services.youtube.YoutubeService;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.stream.Description;
import javax.annotation.Nullable;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import javax.annotation.Nullable;
import static java.util.Collections.singletonList;
import static junit.framework.TestCase.assertFalse;
import static org.junit.Assert.assertEquals;
@ -29,15 +31,20 @@ import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoDuplica
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.CHANNELS;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.PLAYLISTS;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.VIDEOS;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
public class YoutubeSearchExtractorTest {
private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/search/";
public static class All extends DefaultSearchExtractorTest {
private static SearchExtractor extractor;
private static final String QUERY = "test";
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "all"));
extractor = YouTube.getSearchExtractor(QUERY);
extractor.fetchPage();
}
@ -58,7 +65,8 @@ public class YoutubeSearchExtractorTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "channel"));
extractor = YouTube.getSearchExtractor(QUERY, singletonList(CHANNELS), "");
extractor.fetchPage();
}
@ -81,7 +89,8 @@ public class YoutubeSearchExtractorTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "playlist"));
extractor = YouTube.getSearchExtractor(QUERY, singletonList(PLAYLISTS), "");
extractor.fetchPage();
}
@ -104,7 +113,8 @@ public class YoutubeSearchExtractorTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "videos"));
extractor = YouTube.getSearchExtractor(QUERY, singletonList(VIDEOS), "");
extractor.fetchPage();
}
@ -128,7 +138,8 @@ public class YoutubeSearchExtractorTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "suggestions"));
extractor = YouTube.getSearchExtractor(QUERY, singletonList(VIDEOS), "");
extractor.fetchPage();
}
@ -151,7 +162,8 @@ public class YoutubeSearchExtractorTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "corrected"));
extractor = YouTube.getSearchExtractor(QUERY, singletonList(VIDEOS), "");
extractor.fetchPage();
}
@ -174,7 +186,8 @@ public class YoutubeSearchExtractorTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "random"));
extractor = YouTube.getSearchExtractor(QUERY);
extractor.fetchPage();
}
@ -208,7 +221,8 @@ public class YoutubeSearchExtractorTest {
public static class PagingTest {
@Test
public void duplicatedItemsCheck() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "paging"));
final SearchExtractor extractor = YouTube.getSearchExtractor("cirque du soleil", singletonList(VIDEOS), "");
extractor.fetchPage();
@ -219,14 +233,14 @@ public class YoutubeSearchExtractorTest {
}
}
@Ignore("TODO fix")
public static class MetaInfoTest extends DefaultSearchExtractorTest {
private static SearchExtractor extractor;
private static final String QUERY = "Covid";
@Test
public void clarificationTest() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "metaInfo"));
extractor = YouTube.getSearchExtractor(QUERY, singletonList(VIDEOS), "");
extractor.fetchPage();
}

27
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java
View File

@ -7,6 +7,7 @@ import static java.util.Arrays.asList;
import static org.junit.Assert.assertEquals;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.*;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
public class YoutubeSearchQHTest {
@ -19,37 +20,37 @@ public class YoutubeSearchQHTest {
assertEquals("https://www.youtube.com/results?search_query=G%C3%BCl%C3%BCm", YouTube.getSearchQHFactory().fromQuery("Gülüm").getUrl());
assertEquals("https://www.youtube.com/results?search_query=%3Fj%24%29H%C2%A7B", YouTube.getSearchQHFactory().fromQuery("?j$)H§B").getUrl());
assertEquals("https://music.youtube.com/search?q=asdf", YouTube.getSearchQHFactory().fromQuery("asdf", asList(new String[]{MUSIC_SONGS}), "").getUrl());
assertEquals("https://music.youtube.com/search?q=hans", YouTube.getSearchQHFactory().fromQuery("hans", asList(new String[]{MUSIC_SONGS}), "").getUrl());
assertEquals("https://music.youtube.com/search?q=Poifj%26jaijf", YouTube.getSearchQHFactory().fromQuery("Poifj&jaijf", asList(new String[]{MUSIC_SONGS}), "").getUrl());
assertEquals("https://music.youtube.com/search?q=G%C3%BCl%C3%BCm", YouTube.getSearchQHFactory().fromQuery("Gülüm", asList(new String[]{MUSIC_SONGS}), "").getUrl());
assertEquals("https://music.youtube.com/search?q=%3Fj%24%29H%C2%A7B", YouTube.getSearchQHFactory().fromQuery("?j$)H§B", asList(new String[]{MUSIC_SONGS}), "").getUrl());
assertEquals("https://music.youtube.com/search?q=asdf", YouTube.getSearchQHFactory().fromQuery("asdf", asList(new String[]{MUSIC_SONGS}), EMPTY_STRING).getUrl());
assertEquals("https://music.youtube.com/search?q=hans", YouTube.getSearchQHFactory().fromQuery("hans", asList(new String[]{MUSIC_SONGS}), EMPTY_STRING).getUrl());
assertEquals("https://music.youtube.com/search?q=Poifj%26jaijf", YouTube.getSearchQHFactory().fromQuery("Poifj&jaijf", asList(new String[]{MUSIC_SONGS}), EMPTY_STRING).getUrl());
assertEquals("https://music.youtube.com/search?q=G%C3%BCl%C3%BCm", YouTube.getSearchQHFactory().fromQuery("Gülüm", asList(new String[]{MUSIC_SONGS}), EMPTY_STRING).getUrl());
assertEquals("https://music.youtube.com/search?q=%3Fj%24%29H%C2%A7B", YouTube.getSearchQHFactory().fromQuery("?j$)H§B", asList(new String[]{MUSIC_SONGS}), EMPTY_STRING).getUrl());
}
@Test
public void testGetContentFilter() throws Exception {
assertEquals(VIDEOS, YouTube.getSearchQHFactory()
.fromQuery("", asList(new String[]{VIDEOS}), "").getContentFilters().get(0));
.fromQuery(EMPTY_STRING, asList(new String[]{VIDEOS}), EMPTY_STRING).getContentFilters().get(0));
assertEquals(CHANNELS, YouTube.getSearchQHFactory()
.fromQuery("asdf", asList(new String[]{CHANNELS}), "").getContentFilters().get(0));
.fromQuery("asdf", asList(new String[]{CHANNELS}), EMPTY_STRING).getContentFilters().get(0));
assertEquals(MUSIC_SONGS, YouTube.getSearchQHFactory()
.fromQuery("asdf", asList(new String[]{MUSIC_SONGS}), "").getContentFilters().get(0));
.fromQuery("asdf", asList(new String[]{MUSIC_SONGS}), EMPTY_STRING).getContentFilters().get(0));
}
@Test
public void testWithContentfilter() throws Exception {
assertEquals("https://www.youtube.com/results?search_query=asdf&sp=EgIQAQ%253D%253D", YouTube.getSearchQHFactory()
.fromQuery("asdf", asList(new String[]{VIDEOS}), "").getUrl());
.fromQuery("asdf", asList(new String[]{VIDEOS}), EMPTY_STRING).getUrl());
assertEquals("https://www.youtube.com/results?search_query=asdf&sp=EgIQAg%253D%253D", YouTube.getSearchQHFactory()
.fromQuery("asdf", asList(new String[]{CHANNELS}), "").getUrl());
.fromQuery("asdf", asList(new String[]{CHANNELS}), EMPTY_STRING).getUrl());
assertEquals("https://www.youtube.com/results?search_query=asdf&sp=EgIQAw%253D%253D", YouTube.getSearchQHFactory()
.fromQuery("asdf", asList(new String[]{PLAYLISTS}), "").getUrl());
.fromQuery("asdf", asList(new String[]{PLAYLISTS}), EMPTY_STRING).getUrl());
assertEquals("https://www.youtube.com/results?search_query=asdf", YouTube.getSearchQHFactory()
.fromQuery("asdf", asList(new String[]{"fjiijie"}), "").getUrl());
.fromQuery("asdf", asList(new String[]{"fjiijie"}), EMPTY_STRING).getUrl());
assertEquals("https://music.youtube.com/search?q=asdf", YouTube.getSearchQHFactory()
.fromQuery("asdf", asList(new String[]{MUSIC_SONGS}), "").getUrl());
.fromQuery("asdf", asList(new String[]{MUSIC_SONGS}), EMPTY_STRING).getUrl());
}
@Test

7
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java
View File

@ -1,10 +1,11 @@
package org.schabi.newpipe.extractor.services.youtube.stream;
import org.junit.BeforeClass;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.downloader.DownloaderFactory;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamType;
@ -16,6 +17,7 @@ import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
public class YoutubeStreamExtractorAgeRestrictedTest extends DefaultStreamExtractorTest {
private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/stream/";
private static final String ID = "MmBeUZqv1QA";
private static final int TIMESTAMP = 196;
private static final String URL = YoutubeStreamExtractorDefaultTest.BASE_URL + ID + "&t=" + TIMESTAMP;
@ -23,7 +25,8 @@ public class YoutubeStreamExtractorAgeRestrictedTest extends DefaultStreamExtrac
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "ageRestricted"));
extractor = YouTube.getStreamExtractor(URL);
extractor.fetchPage();
}

16
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java
View File

@ -2,11 +2,11 @@ package org.schabi.newpipe.extractor.services.youtube.stream;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.downloader.DownloaderFactory;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamType;
@ -21,14 +21,17 @@ import static org.schabi.newpipe.extractor.ServiceList.YouTube;
/**
* Test for {@link YoutubeStreamLinkHandlerFactory}
*/
@Ignore("Video is not available anymore")
public class YoutubeStreamExtractorControversialTest extends DefaultStreamExtractorTest {
private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/stream/";
private static final String ID = "T4XJQO3qol8";
private static final String URL = YoutubeStreamExtractorDefaultTest.BASE_URL + ID;
private static StreamExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "controversial"));
extractor = YouTube.getStreamExtractor(URL);
extractor.fetchPage();
}
@ -53,11 +56,4 @@ public class YoutubeStreamExtractorControversialTest extends DefaultStreamExtrac
@Nullable @Override public String expectedTextualUploadDate() { return "2010-09-09"; }
@Override public long expectedLikeCountAtLeast() { return 13300; }
@Override public long expectedDislikeCountAtLeast() { return 2600; }
@Override
@Test
@Ignore("TODO fix")
public void testErrorMessage() throws Exception {
super.testErrorMessage();
}
}

85
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java
View File

@ -3,29 +3,31 @@ package org.schabi.newpipe.extractor.services.youtube.stream;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.downloader.DownloaderFactory;
import org.schabi.newpipe.extractor.MetaInfo;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.stream.Description;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamSegment;
import org.schabi.newpipe.extractor.stream.StreamType;
import javax.annotation.Nullable;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import javax.annotation.Nullable;
import static org.junit.Assert.*;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
/*
* Created by Christian Schabesberger on 30.12.15.
@ -47,12 +49,14 @@ import static org.schabi.newpipe.extractor.ServiceList.YouTube;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class YoutubeStreamExtractorDefaultTest {
private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/stream/";
static final String BASE_URL = "https://www.youtube.com/watch?v=";
public static class NotAvailable {
@BeforeClass
public static void setUp() {
NewPipe.init(DownloaderTestImpl.getInstance());
public static void setUp() throws IOException {
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "notAvailable"));
}
@Test(expected = ContentNotAvailableException.class)
@ -78,11 +82,13 @@ public class YoutubeStreamExtractorDefaultTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "pewdiwpie"));
extractor = YouTube.getStreamExtractor(URL);
extractor.fetchPage();
}
// @formatter:off
@Override public StreamExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return YouTube; }
@Override public String expectedName() { return "Marzia & Felix - Wedding 19.08.2019"; }
@ -105,9 +111,9 @@ public class YoutubeStreamExtractorDefaultTest {
@Override public long expectedLikeCountAtLeast() { return 5212900; }
@Override public long expectedDislikeCountAtLeast() { return 30600; }
@Override public int expectedStreamSegmentsCount() { return 0; }
// @formatter:on
}
@Ignore("TODO fix")
public static class DescriptionTestUnboxing extends DefaultStreamExtractorTest {
private static final String ID = "cV5TjZCJkuA";
private static final String URL = BASE_URL + ID;
@ -115,11 +121,13 @@ public class YoutubeStreamExtractorDefaultTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "unboxing"));
extractor = YouTube.getStreamExtractor(URL);
extractor.fetchPage();
}
// @formatter:off
@Override public StreamExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return YouTube; }
@Override public String expectedName() { return "This Smartphone Changes Everything..."; }
@ -142,6 +150,14 @@ public class YoutubeStreamExtractorDefaultTest {
@Nullable @Override public String expectedTextualUploadDate() { return "2018-06-19"; }
@Override public long expectedLikeCountAtLeast() { return 340100; }
@Override public long expectedDislikeCountAtLeast() { return 18700; }
// @formatter:on
@Override
@Test
@Ignore("TODO fix")
public void testDescription() throws Exception {
super.testDescription();
}
}
@Ignore("TODO fix")
@ -153,11 +169,13 @@ public class YoutubeStreamExtractorDefaultTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "ratingsDisabled"));
extractor = YouTube.getStreamExtractor(URL);
extractor.fetchPage();
}
// @formatter:off
@Override public StreamExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return YouTube; }
@Override public String expectedName() { return "AlphaOmegaSin Fanboy Logic: Likes/Dislikes Disabled = Point Invalid Lol wtf?"; }
@ -176,6 +194,7 @@ public class YoutubeStreamExtractorDefaultTest {
@Nullable @Override public String expectedTextualUploadDate() { return "2019-01-02"; }
@Override public long expectedLikeCountAtLeast() { return -1; }
@Override public long expectedDislikeCountAtLeast() { return -1; }
// @formatter:on
}
public static class StreamSegmentsTestOstCollection extends DefaultStreamExtractorTest {
@ -186,11 +205,13 @@ public class YoutubeStreamExtractorDefaultTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "streamSegmentsOstCollection"));
extractor = YouTube.getStreamExtractor(URL);
extractor.fetchPage();
}
// @formatter:off
@Override public StreamExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return YouTube; }
@Override public String expectedName() { return "1 Hour - Most Epic Anime Mix - Battle Anime OST"; }
@ -221,6 +242,7 @@ public class YoutubeStreamExtractorDefaultTest {
assertEquals(BASE_URL + ID + "?t=589", segment.getUrl());
assertNotNull(segment.getPreviewUrl());
}
// @formatter:on
}
public static class StreamSegmentsTestMaiLab extends DefaultStreamExtractorTest {
@ -231,11 +253,13 @@ public class YoutubeStreamExtractorDefaultTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "streamSegmentsMaiLab"));
extractor = YouTube.getStreamExtractor(URL);
extractor.fetchPage();
}
// @formatter:off
@Override public StreamExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return YouTube; }
@Override public String expectedName() { return "Vitamin D wissenschaftlich gepr\u00fcft"; }
@ -246,9 +270,7 @@ public class YoutubeStreamExtractorDefaultTest {
@Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; }
@Override public String expectedUploaderName() { return "maiLab"; }
@Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCyHDQ5C6z1NDmJ4g6SerW8g"; }
@Override public List<String> expectedDescriptionContains() {
return Arrays.asList("Vitamin", "2:44", "Was ist Vitamin D?");
}
@Override public List<String> expectedDescriptionContains() {return Arrays.asList("Vitamin", "2:44", "Was ist Vitamin D?");}
@Override public long expectedLength() { return 1010; }
@Override public long expectedViewCountAtLeast() { return 815500; }
@Nullable @Override public String expectedUploadDate() { return "2020-11-18 00:00:00.000"; }
@ -256,10 +278,10 @@ public class YoutubeStreamExtractorDefaultTest {
@Override public long expectedLikeCountAtLeast() { return 48500; }
@Override public long expectedDislikeCountAtLeast() { return 20000; }
@Override public boolean expectedHasSubtitles() { return true; }
@Override public int expectedStreamSegmentsCount() { return 7; }
// @formatter:on
@Test
@Ignore("TODO fix")
public void testStreamSegment() throws Exception {
final StreamSegment segment = extractor.getStreamSegments().get(1);
assertEquals(164, segment.getStartTimeSeconds());
@ -267,9 +289,15 @@ public class YoutubeStreamExtractorDefaultTest {
assertEquals(BASE_URL + ID + "?t=164", segment.getUrl());
assertNotNull(segment.getPreviewUrl());
}
@Override
@Test
@Ignore("encoding problem")
public void testName() throws Exception {
super.testName();
}
}
@Ignore("TODO fix")
public static class PublicBroadcasterTest extends DefaultStreamExtractorTest {
private static final String ID = "q6fgbYWsMgw";
private static final int TIMESTAMP = 0;
@ -278,11 +306,13 @@ public class YoutubeStreamExtractorDefaultTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "publicBroadcast"));
extractor = YouTube.getStreamExtractor(URL);
extractor.fetchPage();
}
// @formatter:off
@Override public StreamExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return YouTube; }
@Override public String expectedName() { return "Was verbirgt sich am tiefsten Punkt des Ozeans?"; }
@ -303,12 +333,27 @@ public class YoutubeStreamExtractorDefaultTest {
@Override public long expectedDislikeCountAtLeast() { return 500; }
@Override public List<MetaInfo> expectedMetaInfo() throws MalformedURLException {
return Collections.singletonList(new MetaInfo(
"",
EMPTY_STRING,
new Description("Funk is a German public broadcast service.", Description.PLAIN_TEXT),
Collections.singletonList(new URL("https://de.wikipedia.org/wiki/Funk_(Medienangebot)?wprov=yicw1")),
Collections.singletonList("Wikipedia (German)")
));
}
// @formatter:on
@Override
@Ignore("TODO fix")
@Test
public void testUploaderName() throws Exception {
super.testUploaderName();
}
@Override
@Ignore("TODO fix")
@Test
public void testMetaInfo() throws Exception {
super.testMetaInfo();
}
}
}

7
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java
View File

@ -1,10 +1,11 @@
package org.schabi.newpipe.extractor.services.youtube.stream;
import org.junit.BeforeClass;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.downloader.DownloaderFactory;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamType;
@ -16,6 +17,7 @@ import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
public class YoutubeStreamExtractorLivestreamTest extends DefaultStreamExtractorTest {
private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/stream/";
private static final String ID = "5qap5aO4i9A";
private static final int TIMESTAMP = 1737;
private static final String URL = YoutubeStreamExtractorDefaultTest.BASE_URL + ID + "&t=" + TIMESTAMP;
@ -23,7 +25,8 @@ public class YoutubeStreamExtractorLivestreamTest extends DefaultStreamExtractor
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "live"));
extractor = YouTube.getStreamExtractor(URL);
extractor.fetchPage();
}

7
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorUnlistedTest.java
View File

@ -1,10 +1,11 @@
package org.schabi.newpipe.extractor.services.youtube.stream;
import org.junit.BeforeClass;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.downloader.DownloaderFactory;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamType;
@ -16,13 +17,15 @@ import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
public class YoutubeStreamExtractorUnlistedTest extends DefaultStreamExtractorTest {
private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/stream/";
static final String ID = "udsB8KnIJTg";
static final String URL = YoutubeStreamExtractorDefaultTest.BASE_URL + ID;
private static StreamExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "unlisted"));
extractor = YouTube.getStreamExtractor(URL);
extractor.fetchPage();
}

77
extractor/src/test/resources/org/schabi/newpipe/extractor/kiosk/generated_mock_0.json Normal file
View File

File diff suppressed because one or more lines are too long

77
extractor/src/test/resources/org/schabi/newpipe/extractor/kiosk/generated_mock_1.json Normal file
View File

File diff suppressed because one or more lines are too long

77
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/VSauce/generated_mock_0.json Normal file
View File

File diff suppressed because one or more lines are too long

77
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/VSauce/generated_mock_1.json Normal file
View File

File diff suppressed because one or more lines are too long

73
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/VSauce/generated_mock_2.json Normal file
View File

File diff suppressed because one or more lines are too long

77
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/captainDisillusion/generated_mock_0.json Normal file
View File

File diff suppressed because one or more lines are too long

77
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/captainDisillusion/generated_mock_1.json Normal file
View File

File diff suppressed because one or more lines are too long

73
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/captainDisillusion/generated_mock_2.json Normal file
View File

File diff suppressed because one or more lines are too long

77
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/gronkh/generated_mock_0.json Normal file
View File

File diff suppressed because one or more lines are too long

77
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/gronkh/generated_mock_1.json Normal file
View File

File diff suppressed because one or more lines are too long

73
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/gronkh/generated_mock_2.json Normal file
View File

File diff suppressed because one or more lines are too long

77
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/kurzgesagt/generated_mock_0.json Normal file
View File

File diff suppressed because one or more lines are too long

77
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/kurzgesagt/generated_mock_1.json Normal file
View File

File diff suppressed because one or more lines are too long

73
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/kurzgesagt/generated_mock_2.json Normal file
View File

File diff suppressed because one or more lines are too long

77
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/localization/generated_mock_0.json Normal file
View File

File diff suppressed because one or more lines are too long

77
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/localization/generated_mock_1.json Normal file
View File

File diff suppressed because one or more lines are too long

77
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/localization/generated_mock_2.json Normal file
View File

File diff suppressed because one or more lines are too long

77
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/localization/generated_mock_3.json Normal file
View File

File diff suppressed because one or more lines are too long

77
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/localization/generated_mock_4.json Normal file
View File

File diff suppressed because one or more lines are too long

77
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/notAvailable/generated_mock_0.json Normal file
View File

File diff suppressed because one or more lines are too long

77
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/notAvailable/generated_mock_1.json Normal file
View File

File diff suppressed because one or more lines are too long

69
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/notAvailable/generated_mock_2.json Normal file
View File

@ -0,0 +1,69 @@
{
"request": {
"httpMethod": "GET",
"url": "https://www.youtube.com/channel/DOESNT-EXIST/videos?pbj\u003d1\u0026view\u003d0\u0026flow\u003dgrid",
"headers": {
"Accept-Language": [
"en-GB, en;q\u003d0.9"
],
"X-YouTube-Client-Name": [
"1"
],
"X-YouTube-Client-Version": [
"2.20200214.04.00"
]
},
"localization": {
"languageCode": "en",
"countryCode": "GB"
}
},
"response": {
"responseCode": 404,
"responseMessage": "",
"responseHeaders": {
"alt-svc": [
"h3-29\u003d\":443\"; ma\u003d2592000,h3-T051\u003d\":443\"; ma\u003d2592000,h3-Q050\u003d\":443\"; ma\u003d2592000,h3-Q046\u003d\":443\"; ma\u003d2592000,h3-Q043\u003d\":443\"; ma\u003d2592000,quic\u003d\":443\"; ma\u003d2592000; v\u003d\"46,43\""
],
"cache-control": [
"no-cache, no-store, max-age\u003d0, must-revalidate"
],
"content-type": [
"text/html; charset\u003dutf-8"
],
"date": [
"Sat, 13 Feb 2021 19:13:39 GMT"
],
"expires": [
"Mon, 01 Jan 1990 00:00:00 GMT"
],
"p3p": [
"CP\u003d\"This is not a P3P policy! See http://support.google.com/accounts/answer/151657?hl\u003den-GB for more info.\""
],
"pragma": [
"no-cache"
],
"server": [
"ESF"
],
"set-cookie": [
"YSC\u003dbFLZyqvSsDQ; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
"VISITOR_INFO1_LIVE\u003dYgNWpn3F6Ww; Domain\u003d.youtube.com; Expires\u003dThu, 12-Aug-2021 19:13:39 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone"
],
"strict-transport-security": [
"max-age\u003d31536000"
],
"x-content-type-options": [
"nosniff"
],
"x-frame-options": [
"SAMEORIGIN"
],
"x-xss-protection": [
"0"
]
},
"responseBody": "\u003chtml lang\u003d\"en-GB\" dir\u003d\"ltr\"\u003e\u003chead\u003e\u003ctitle\u003e404 Not Found\u003c/title\u003e\u003cstyle nonce\u003d\"AkiRKVBSwx7zy61zR/BBpg\"\u003e*{margin:0;padding:0;border:0}html,body{height:100%;}\u003c/style\u003e\u003clink rel\u003d\"shortcut icon\" href\u003d\"https://www.youtube.com/img/favicon.ico\" type\u003d\"image/x-icon\"\u003e\u003clink rel\u003d\"icon\" href\u003d\"https://www.youtube.com/img/favicon_32.png\" sizes\u003d\"32x32\"\u003e\u003clink rel\u003d\"icon\" href\u003d\"https://www.youtube.com/img/favicon_48.png\" sizes\u003d\"48x48\"\u003e\u003clink rel\u003d\"icon\" href\u003d\"https://www.youtube.com/img/favicon_96.png\" sizes\u003d\"96x96\"\u003e\u003clink rel\u003d\"icon\" href\u003d\"https://www.youtube.com/img/favicon_144.png\" sizes\u003d\"144x144\"\u003e\u003c/head\u003e\u003cbody\u003e\u003ciframe style\u003d\"display:block;border:0;\" src\u003d\"/error?src\u003d404\u0026amp;ifr\u003d1\u0026amp;error\u003d\" width\u003d\"100%\" height\u003d\"100%\" frameborder\u003d\"\\\" scrolling\u003d\"no\"\u003e\u003c/iframe\u003e\u003c/body\u003e\u003c/html\u003e",
"latestUrl": "https://www.youtube.com/channel/DOESNT-EXIST/videos?pbj\u003d1\u0026view\u003d0\u0026flow\u003dgrid"
}
}

77
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/notSupported/generated_mock_0.json Normal file
View File

File diff suppressed because one or more lines are too long

77
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/notSupported/generated_mock_1.json Normal file
View File

File diff suppressed because one or more lines are too long

77
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/random/generated_mock_0.json Normal file
View File

File diff suppressed because one or more lines are too long

77
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/random/generated_mock_1.json Normal file
View File

File diff suppressed because one or more lines are too long

68
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/comments/hearted/generated_mock_0.json Normal file
View File

File diff suppressed because one or more lines are too long

80
extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/comments/hearted/generated_mock_1.json Normal file
View File

File diff suppressed because one or more lines are too long

Some files were not shown because too many files have changed in this diff Show More