be-gay-do-crimes
/
NewPipeExtractor
mirror of https://github.com/TeamNewPipe/NewPipeExtractor
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #291 from wb9688/yt-music-search 

Add support for YouTube Music search
This commit is contained in:
Tobias Groza 2020-04-03 17:55:34 +02:00 committed by GitHub
parent cef7c96d7d bce27a0e22
commit b81e22ddc9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 881 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
extractor/src/main/java/org/schabi/newpipe/extractor/ListExtractor.java
View File

@ -3,16 +3,33 @@ package org.schabi.newpipe.extractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import javax.annotation.Nonnull;
/**
* Base class to extractors that have a list (e.g. playlists, users).
*/
public abstract class ListExtractor<R extends InfoItem> extends Extractor {
/**
* Constant that should be returned whenever
* a list has an unknown number of items.
*/
public static final long ITEM_COUNT_UNKNOWN = -1;
/**
* Constant that should be returned whenever a list has an
* infinite number of items. For example a YouTube mix.
*/
public static final long ITEM_COUNT_INFINITE = -2;
/**
* Constant that should be returned whenever a list
* has an unknown number of items bigger than 100.
*/
public static final long ITEM_COUNT_MORE_THAN_100 = -3;
public ListExtractor(StreamingService service, ListLinkHandler linkHandler) {
super(service, linkHandler);
}

41
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java
View File

@ -7,22 +7,45 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.feed.FeedExtractor;
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
import org.schabi.newpipe.extractor.kiosk.KioskList;
import org.schabi.newpipe.extractor.linkhandler.*;
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.localization.ContentCountry;
import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.*;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.*;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeCommentsExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeFeedExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMusicSearchExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubePlaylistExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSubscriptionExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSuggestionExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeTrendingExtractor;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeCommentsLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeTrendingLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
import javax.annotation.Nonnull;
import java.util.List;
import javax.annotation.Nonnull;
import static java.util.Arrays.asList;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.*;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.COMMENTS;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.LIVE;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.VIDEO;
/*
* Created by Christian Schabesberger on 23.08.15.
@ -92,7 +115,13 @@ public class YoutubeService extends StreamingService {
@Override
public SearchExtractor getSearchExtractor(SearchQueryHandler query) {
return new YoutubeSearchExtractor(this, query);
final List<String> contentFilters = query.getContentFilters();
if (contentFilters.size() > 0 && contentFilters.get(0).startsWith("music_")) {
return new YoutubeMusicSearchExtractor(this, query);
} else {
return new YoutubeSearchExtractor(this, query);
}
}
@Override

494
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java Normal file
View File

@ -0,0 +1,494 @@
package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import com.grack.nanojson.JsonWriter;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getValidJsonResponseBody;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ALBUMS;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ARTISTS;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_PLAYLISTS;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_SONGS;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_VIDEOS;
public class YoutubeMusicSearchExtractor extends SearchExtractor {
private JsonObject initialData;
public YoutubeMusicSearchExtractor(final StreamingService service, final SearchQueryHandler linkHandler) {
super(service, linkHandler);
}
@Override
public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException {
final String[] youtubeMusicKeys = YoutubeParsingHelper.getYoutubeMusicKeys();
final String url = "https://music.youtube.com/youtubei/v1/search?alt=json&key=" + youtubeMusicKeys[0];
final String params;
switch (getLinkHandler().getContentFilters().get(0)) {
case MUSIC_SONGS:
params = "Eg-KAQwIARAAGAAgACgAMABqChAEEAUQAxAKEAk%3D";
break;
case MUSIC_VIDEOS:
params = "Eg-KAQwIABABGAAgACgAMABqChAEEAUQAxAKEAk%3D";
break;
case MUSIC_ALBUMS:
params = "Eg-KAQwIABAAGAEgACgAMABqChAEEAUQAxAKEAk%3D";
break;
case MUSIC_PLAYLISTS:
params = "Eg-KAQwIABAAGAAgACgBMABqChAEEAUQAxAKEAk%3D";
break;
case MUSIC_ARTISTS:
params = "Eg-KAQwIABAAGAAgASgAMABqChAEEAUQAxAKEAk%3D";
break;
default:
params = null;
break;
}
// @formatter:off
byte[] json = JsonWriter.string()
.object()
.object("context")
.object("client")
.value("clientName", "WEB_REMIX")
.value("clientVersion", youtubeMusicKeys[2])
.value("hl", "en")
.value("gl", getExtractorContentCountry().getCountryCode())
.array("experimentIds").end()
.value("experimentsToken", "")
.value("utcOffsetMinutes", 0)
.object("locationInfo").end()
.object("musicAppInfo").end()
.end()
.object("capabilities").end()
.object("request")
.array("internalExperimentFlags").end()
.object("sessionIndex").end()
.end()
.object("activePlayers").end()
.object("user")
.value("enableSafetyMode", false)
.end()
.end()
.value("query", getSearchString())
.value("params", params)
.end().done().getBytes("UTF-8");
// @formatter:on
final Map<String, List<String>> headers = new HashMap<>();
headers.put("X-YouTube-Client-Name", Collections.singletonList(youtubeMusicKeys[1]));
headers.put("X-YouTube-Client-Version", Collections.singletonList(youtubeMusicKeys[2]));
headers.put("Origin", Collections.singletonList("https://music.youtube.com"));
headers.put("Referer", Collections.singletonList("music.youtube.com"));
headers.put("Content-Type", Collections.singletonList("application/json"));
final String responseBody = getValidJsonResponseBody(getDownloader().post(url, headers, json));
try {
initialData = JsonParser.object().from(responseBody);
} catch (JsonParserException e) {
throw new ParsingException("Could not parse JSON", e);
}
}
@Nonnull
@Override
public String getUrl() throws ParsingException {
return super.getUrl();
}
@Override
public String getSearchSuggestion() throws ParsingException {
final JsonObject itemSectionRenderer = initialData.getObject("contents").getObject("sectionListRenderer")
.getArray("contents").getObject(0).getObject("itemSectionRenderer");
if (itemSectionRenderer == null) {
return "";
}
final JsonObject didYouMeanRenderer = itemSectionRenderer.getArray("contents")
.getObject(0).getObject("didYouMeanRenderer");
if (didYouMeanRenderer == null) {
return "";
}
return getTextFromObject(didYouMeanRenderer.getObject("correctedQuery"));
}
@Nonnull
@Override
public InfoItemsPage<InfoItem> getInitialPage() throws ExtractionException, IOException {
final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId());
final JsonArray contents = initialData.getObject("contents").getObject("sectionListRenderer").getArray("contents");
for (Object content : contents) {
if (((JsonObject) content).getObject("musicShelfRenderer") != null) {
collectMusicStreamsFrom(collector, ((JsonObject) content).getObject("musicShelfRenderer").getArray("contents"));
}
}
return new InfoItemsPage<>(collector, getNextPageUrl());
}
@Override
public String getNextPageUrl() throws ExtractionException, IOException {
final JsonArray contents = initialData.getObject("contents").getObject("sectionListRenderer").getArray("contents");
for (Object content : contents) {
if (((JsonObject) content).getObject("musicShelfRenderer") != null) {
return getNextPageUrlFrom(((JsonObject) content).getObject("musicShelfRenderer").getArray("continuations"));
}
}
return "";
}
@Override
public InfoItemsPage<InfoItem> getPage(final String pageUrl) throws IOException, ExtractionException {
if (pageUrl == null || pageUrl.isEmpty()) {
throw new ExtractionException(new IllegalArgumentException("Page url is empty or null"));
}
final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId());
final String[] youtubeMusicKeys = YoutubeParsingHelper.getYoutubeMusicKeys();
// @formatter:off
byte[] json = JsonWriter.string()
.object()
.object("context")
.object("client")
.value("clientName", "WEB_REMIX")
.value("clientVersion", youtubeMusicKeys[2])
.value("hl", "en")
.value("gl", getExtractorContentCountry().getCountryCode())
.array("experimentIds").end()
.value("experimentsToken", "")
.value("utcOffsetMinutes", 0)
.object("locationInfo").end()
.object("musicAppInfo").end()
.end()
.object("capabilities").end()
.object("request")
.array("internalExperimentFlags").end()
.object("sessionIndex").end()
.end()
.object("activePlayers").end()
.object("user")
.value("enableSafetyMode", false)
.end()
.end()
.end().done().getBytes("UTF-8");
// @formatter:on
final Map<String, List<String>> headers = new HashMap<>();
headers.put("X-YouTube-Client-Name", Collections.singletonList(youtubeMusicKeys[1]));
headers.put("X-YouTube-Client-Version", Collections.singletonList(youtubeMusicKeys[2]));
headers.put("Origin", Collections.singletonList("https://music.youtube.com"));
headers.put("Referer", Collections.singletonList("music.youtube.com"));
headers.put("Content-Type", Collections.singletonList("application/json"));
final String responseBody = getValidJsonResponseBody(getDownloader().post(pageUrl, headers, json));
final JsonObject ajaxJson;
try {
ajaxJson = JsonParser.object().from(responseBody);
} catch (JsonParserException e) {
throw new ParsingException("Could not parse JSON", e);
}
if (ajaxJson.getObject("continuationContents") == null) {
return InfoItemsPage.emptyPage();
}
final JsonObject musicShelfContinuation = ajaxJson.getObject("continuationContents").getObject("musicShelfContinuation");
collectMusicStreamsFrom(collector, musicShelfContinuation.getArray("contents"));
final JsonArray continuations = musicShelfContinuation.getArray("continuations");
return new InfoItemsPage<>(collector, getNextPageUrlFrom(continuations));
}
private void collectMusicStreamsFrom(final InfoItemsSearchCollector collector, final JsonArray videos) {
final TimeAgoParser timeAgoParser = getTimeAgoParser();
for (Object item : videos) {
final JsonObject info = ((JsonObject) item).getObject("musicResponsiveListItemRenderer");
if (info != null) {
final String searchType = getLinkHandler().getContentFilters().get(0);
if (searchType.equals(MUSIC_SONGS) || searchType.equals(MUSIC_VIDEOS)) {
collector.commit(new YoutubeStreamInfoItemExtractor(info, timeAgoParser) {
@Override
public String getUrl() throws ParsingException {
final String url = getUrlFromNavigationEndpoint(info.getObject("doubleTapCommand"));
if (url != null && !url.isEmpty()) {
return url;
}
throw new ParsingException("Could not get url");
}
@Override
public String getName() throws ParsingException {
final String name = getTextFromObject(info.getArray("flexColumns").getObject(0)
.getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text"));
if (name != null && !name.isEmpty()) {
return name;
}
throw new ParsingException("Could not get name");
}
@Override
public long getDuration() throws ParsingException {
final String duration = getTextFromObject(info.getArray("flexColumns").getObject(3)
.getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text"));
if (duration != null && !duration.isEmpty()) {
return YoutubeParsingHelper.parseDurationString(duration);
}
throw new ParsingException("Could not get duration");
}
@Override
public String getUploaderName() throws ParsingException {
final String name = getTextFromObject(info.getArray("flexColumns").getObject(1)
.getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text"));
if (name != null && !name.isEmpty()) {
return name;
}
throw new ParsingException("Could not get uploader name");
}
@Override
public String getUploaderUrl() throws ParsingException {
if (searchType.equals(MUSIC_VIDEOS)) {
JsonArray items = info.getObject("menu").getObject("menuRenderer").getArray("items");
for (Object item : items) {
final JsonObject menuNavigationItemRenderer = ((JsonObject) item).getObject("menuNavigationItemRenderer");
if (menuNavigationItemRenderer != null && menuNavigationItemRenderer.getObject("icon").getString("iconType").equals("ARTIST")) {
return getUrlFromNavigationEndpoint(menuNavigationItemRenderer.getObject("navigationEndpoint"));
}
}
return null;
} else {
final JsonObject navigationEndpoint = info.getArray("flexColumns")
.getObject(1).getObject("musicResponsiveListItemFlexColumnRenderer")
.getObject("text").getArray("runs").getObject(0).getObject("navigationEndpoint");
if (navigationEndpoint == null) {
return null;
}
final String url = getUrlFromNavigationEndpoint(navigationEndpoint);
if (url != null && !url.isEmpty()) {
return url;
}
throw new ParsingException("Could not get uploader url");
}
}
@Override
public String getTextualUploadDate() {
return null;
}
@Override
public DateWrapper getUploadDate() {
return null;
}
@Override
public long getViewCount() throws ParsingException {
if (searchType.equals(MUSIC_SONGS)) {
return -1;
}
final String viewCount = getTextFromObject(info.getArray("flexColumns").getObject(2)
.getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text"));
if (viewCount != null && !viewCount.isEmpty()) {
return Utils.mixedNumberWordToLong(viewCount);
}
throw new ParsingException("Could not get view count");
}
@Override
public String getThumbnailUrl() throws ParsingException {
try {
final JsonArray thumbnails = info.getObject("thumbnail").getObject("musicThumbnailRenderer")
.getObject("thumbnail").getArray("thumbnails");
// the last thumbnail is the one with the highest resolution
final String url = thumbnails.getObject(thumbnails.size() - 1).getString("url");
return fixThumbnailUrl(url);
} catch (Exception e) {
throw new ParsingException("Could not get thumbnail url", e);
}
}
});
} else if (searchType.equals(MUSIC_ARTISTS)) {
collector.commit(new YoutubeChannelInfoItemExtractor(info) {
@Override
public String getThumbnailUrl() throws ParsingException {
try {
final JsonArray thumbnails = info.getObject("thumbnail").getObject("musicThumbnailRenderer")
.getObject("thumbnail").getArray("thumbnails");
// the last thumbnail is the one with the highest resolution
final String url = thumbnails.getObject(thumbnails.size() - 1).getString("url");
return fixThumbnailUrl(url);
} catch (Exception e) {
throw new ParsingException("Could not get thumbnail url", e);
}
}
@Override
public String getName() throws ParsingException {
final String name = getTextFromObject(info.getArray("flexColumns").getObject(0)
.getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text"));
if (name != null && !name.isEmpty()) {
return name;
}
throw new ParsingException("Could not get name");
}
@Override
public String getUrl() throws ParsingException {
final String url = getUrlFromNavigationEndpoint(info.getObject("navigationEndpoint"));
if (url != null && !url.isEmpty()) {
return url;
}
throw new ParsingException("Could not get url");
}
@Override
public long getSubscriberCount() throws ParsingException {
final String viewCount = getTextFromObject(info.getArray("flexColumns").getObject(2)
.getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text"));
if (viewCount != null && !viewCount.isEmpty()) {
return Utils.mixedNumberWordToLong(viewCount);
}
throw new ParsingException("Could not get subscriber count");
}
@Override
public long getStreamCount() {
return -1;
}
@Override
public String getDescription() {
return null;
}
});
} else if (searchType.equals(MUSIC_ALBUMS) || searchType.equals(MUSIC_PLAYLISTS)) {
collector.commit(new YoutubePlaylistInfoItemExtractor(info) {
@Override
public String getThumbnailUrl() throws ParsingException {
try {
final JsonArray thumbnails = info.getObject("thumbnail").getObject("musicThumbnailRenderer")
.getObject("thumbnail").getArray("thumbnails");
// the last thumbnail is the one with the highest resolution
final String url = thumbnails.getObject(thumbnails.size() - 1).getString("url");
return fixThumbnailUrl(url);
} catch (Exception e) {
throw new ParsingException("Could not get thumbnail url", e);
}
}
@Override
public String getName() throws ParsingException {
final String name = getTextFromObject(info.getArray("flexColumns").getObject(0)
.getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text"));
if (name != null && !name.isEmpty()) {
return name;
}
throw new ParsingException("Could not get name");
}
@Override
public String getUrl() throws ParsingException {
final String url = getUrlFromNavigationEndpoint(info.getObject("doubleTapCommand"));
if (url != null && !url.isEmpty()) {
return url;
}
throw new ParsingException("Could not get url");
}
@Override
public String getUploaderName() throws ParsingException {
final String name;
if (searchType.equals(MUSIC_ALBUMS)) {
name = getTextFromObject(info.getArray("flexColumns").getObject(2)
.getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text"));
} else {
name = getTextFromObject(info.getArray("flexColumns").getObject(1)
.getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text"));
}
if (name != null && !name.isEmpty()) {
return name;
}
throw new ParsingException("Could not get uploader name");
}
@Override
public long getStreamCount() throws ParsingException {
if (searchType.equals(MUSIC_ALBUMS)) {
return ITEM_COUNT_UNKNOWN;
}
final String count = getTextFromObject(info.getArray("flexColumns").getObject(2)
.getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text"));
if (count != null && !count.isEmpty()) {
if (count.contains("100+")) {
return ITEM_COUNT_MORE_THAN_100;
} else {
return Long.parseLong(Utils.removeNonDigitCharacters(count));
}
}
throw new ParsingException("Could not get count");
}
});
}
}
}
}
private String getNextPageUrlFrom(final JsonArray continuations) throws ParsingException, IOException, ReCaptchaException {
if (continuations == null) {
return "";
}
final JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData");
final String continuation = nextContinuationData.getString("continuation");
final String clickTrackingParams = nextContinuationData.getString("clickTrackingParams");
return "https://music.youtube.com/youtubei/v1/search?ctoken=" + continuation + "&continuation=" + continuation
+ "&itct=" + clickTrackingParams + "&alt=json&key=" + YoutubeParsingHelper.getYoutubeMusicKeys()[0];
}
}

32
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java
View File

@ -43,12 +43,12 @@ import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeP
public class YoutubeSearchExtractor extends SearchExtractor {
private JsonObject initialData;
public YoutubeSearchExtractor(StreamingService service, SearchQueryHandler linkHandler) {
public YoutubeSearchExtractor(final StreamingService service, final SearchQueryHandler linkHandler) {
super(service, linkHandler);
}
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException {
final String url = getUrl() + "&pbj=1";
final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization());
@ -64,23 +64,23 @@ public class YoutubeSearchExtractor extends SearchExtractor {
@Override
public String getSearchSuggestion() throws ParsingException {
JsonObject showingResultsForRenderer = initialData.getObject("contents")
final JsonObject showingResultsForRenderer = initialData.getObject("contents")
.getObject("twoColumnSearchResultsRenderer").getObject("primaryContents")
.getObject("sectionListRenderer").getArray("contents").getObject(0)
.getObject("itemSectionRenderer").getArray("contents").getObject(0)
.getObject("showingResultsForRenderer");
if (showingResultsForRenderer == null) {
return "";
} else {
return getTextFromObject(showingResultsForRenderer.getObject("correctedQuery"));
}
return getTextFromObject(showingResultsForRenderer.getObject("correctedQuery"));
}
@Nonnull
@Override
public InfoItemsPage<InfoItem> getInitialPage() throws ExtractionException {
final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId());
JsonArray sections = initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer")
final JsonArray sections = initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer")
.getObject("primaryContents").getObject("sectionListRenderer").getArray("contents");
for (Object section : sections) {
@ -98,7 +98,7 @@ public class YoutubeSearchExtractor extends SearchExtractor {
}
@Override
public InfoItemsPage<InfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
public InfoItemsPage<InfoItem> getPage(final String pageUrl) throws IOException, ExtractionException {
if (pageUrl == null || pageUrl.isEmpty()) {
throw new ExtractionException(new IllegalArgumentException("Page url is empty or null"));
}
@ -106,17 +106,16 @@ public class YoutubeSearchExtractor extends SearchExtractor {
final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId());
final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization());
JsonObject itemSectionRenderer = ajaxJson.getObject(1).getObject("response")
final JsonObject itemSectionRenderer = ajaxJson.getObject(1).getObject("response")
.getObject("continuationContents").getObject("itemSectionContinuation");
collectStreamsFrom(collector, itemSectionRenderer.getArray("contents"));
final JsonArray continuations = itemSectionRenderer.getArray("continuations");
return new InfoItemsPage<>(collector, getNextPageUrlFrom(itemSectionRenderer.getArray("continuations")));
return new InfoItemsPage<>(collector, getNextPageUrlFrom(continuations));
}
private void collectStreamsFrom(InfoItemsSearchCollector collector, JsonArray videos) throws NothingFoundException, ParsingException {
collector.reset();
private void collectStreamsFrom(final InfoItemsSearchCollector collector, final JsonArray videos) throws NothingFoundException, ParsingException {
final TimeAgoParser timeAgoParser = getTimeAgoParser();
for (Object item : videos) {
@ -133,14 +132,15 @@ public class YoutubeSearchExtractor extends SearchExtractor {
}
}
private String getNextPageUrlFrom(JsonArray continuations) throws ParsingException {
private String getNextPageUrlFrom(final JsonArray continuations) throws ParsingException {
if (continuations == null) {
return "";
}
JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData");
String continuation = nextContinuationData.getString("continuation");
String clickTrackingParams = nextContinuationData.getString("clickTrackingParams");
final JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData");
final String continuation = nextContinuationData.getString("continuation");
final String clickTrackingParams = nextContinuationData.getString("clickTrackingParams");
return getUrl() + "&pbj=1&ctoken=" + continuation + "&continuation=" + continuation
+ "&itct=" + clickTrackingParams;
}

123
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java
View File

@ -5,6 +5,8 @@ import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import com.grack.nanojson.JsonWriter;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.schabi.newpipe.extractor.downloader.Response;
@ -18,6 +20,7 @@ import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLDecoder;
import java.text.ParseException;
@ -62,6 +65,9 @@ public class YoutubeParsingHelper {
private static final String HARDCODED_CLIENT_VERSION = "2.20200214.04.00";
private static String clientVersion;
private static final String[] HARDCODED_YOUTUBE_MUSIC_KEYS = {"AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30", "67", "0.1"};
private static String[] youtubeMusicKeys;
private static final String FEED_BASE_CHANNEL_ID = "https://www.youtube.com/feeds/videos.xml?channel_id=";
private static final String FEED_BASE_USER = "https://www.youtube.com/feeds/videos.xml?user=";
@ -196,11 +202,7 @@ public class YoutubeParsingHelper {
*/
public static String getClientVersion() throws IOException, ExtractionException {
if (clientVersion != null && !clientVersion.isEmpty()) return clientVersion;
if (isHardcodedClientVersionValid()) {
clientVersion = HARDCODED_CLIENT_VERSION;
return clientVersion;
}
if (isHardcodedClientVersionValid()) return clientVersion = HARDCODED_CLIENT_VERSION;
final String url = "https://www.youtube.com/results?search_query=test";
final String html = getDownloader().get(url).responseBody();
@ -217,8 +219,7 @@ public class YoutubeParsingHelper {
JsonObject p = (JsonObject) param;
String key = p.getString("key");
if (key != null && key.equals("cver")) {
clientVersion = p.getString("value");
return clientVersion;
return clientVersion = p.getString("value");
}
}
} else if (s.getString("service").equals("ECATCHER")) {
@ -244,21 +245,94 @@ public class YoutubeParsingHelper {
try {
contextClientVersion = Parser.matchGroup1(pattern, html);
if (contextClientVersion != null && !contextClientVersion.isEmpty()) {
clientVersion = contextClientVersion;
return clientVersion;
return clientVersion = contextClientVersion;
}
} catch (Exception ignored) {
}
}
if (shortClientVersion != null) {
clientVersion = shortClientVersion;
return clientVersion;
return clientVersion = shortClientVersion;
}
throw new ParsingException("Could not get client version");
}
public static boolean areHardcodedYoutubeMusicKeysValid() throws IOException, ReCaptchaException {
final String url = "https://music.youtube.com/youtubei/v1/search?alt=json&key=" + HARDCODED_YOUTUBE_MUSIC_KEYS[0];
// @formatter:off
byte[] json = JsonWriter.string()
.object()
.object("context")
.object("client")
.value("clientName", "WEB_REMIX")
.value("clientVersion", HARDCODED_YOUTUBE_MUSIC_KEYS[2])
.value("hl", "en")
.value("gl", "GB")
.array("experimentIds").end()
.value("experimentsToken", "")
.value("utcOffsetMinutes", 0)
.object("locationInfo").end()
.object("musicAppInfo").end()
.end()
.object("capabilities").end()
.object("request")
.array("internalExperimentFlags").end()
.object("sessionIndex").end()
.end()
.object("activePlayers").end()
.object("user")
.value("enableSafetyMode", false)
.end()
.end()
.value("query", "test")
.value("params", "Eg-KAQwIARAAGAAgACgAMABqChAEEAUQAxAKEAk%3D")
.end().done().getBytes("UTF-8");
// @formatter:on
Map<String, List<String>> headers = new HashMap<>();
headers.put("X-YouTube-Client-Name", Collections.singletonList(HARDCODED_YOUTUBE_MUSIC_KEYS[1]));
headers.put("X-YouTube-Client-Version", Collections.singletonList(HARDCODED_YOUTUBE_MUSIC_KEYS[2]));
headers.put("Origin", Collections.singletonList("https://music.youtube.com"));
headers.put("Referer", Collections.singletonList("music.youtube.com"));
headers.put("Content-Type", Collections.singletonList("application/json"));
String response = getDownloader().post(url, headers, json).responseBody();
return response.length() > 50; // ensure to have a valid response
}
public static String[] getYoutubeMusicKeys() throws IOException, ReCaptchaException, Parser.RegexException {
if (youtubeMusicKeys != null && youtubeMusicKeys.length == 3) return youtubeMusicKeys;
if (areHardcodedYoutubeMusicKeysValid()) return youtubeMusicKeys = HARDCODED_YOUTUBE_MUSIC_KEYS;
final String url = "https://music.youtube.com/";
final String html = getDownloader().get(url).responseBody();
String key;
try {
key = Parser.matchGroup1("INNERTUBE_API_KEY\":\"([0-9a-zA-Z_-]+?)\"", html);
} catch (Parser.RegexException e) {
key = Parser.matchGroup1("innertube_api_key\":\"([0-9a-zA-Z_-]+?)\"", html);
}
final String clientName = Parser.matchGroup1("INNERTUBE_CONTEXT_CLIENT_NAME\":([0-9]+?),", html);
String clientVersion;
try {
clientVersion = Parser.matchGroup1("INNERTUBE_CONTEXT_CLIENT_VERSION\":\"([0-9\\.]+?)\"", html);
} catch (Parser.RegexException e) {
try {
clientVersion = Parser.matchGroup1("INNERTUBE_CLIENT_VERSION\":\"([0-9\\.]+?)\"", html);
} catch (Parser.RegexException ee) {
clientVersion = Parser.matchGroup1("innertube_context_client_version\":\"([0-9\\.]+?)\"", html);
}
}
return youtubeMusicKeys = new String[]{key, clientName, clientVersion};
}
public static String getUrlFromNavigationEndpoint(JsonObject navigationEndpoint) throws ParsingException {
if (navigationEndpoint.getObject("urlEndpoint") != null) {
String internUrl = navigationEndpoint.getObject("urlEndpoint").getString("url");
@ -303,6 +377,9 @@ public class YoutubeParsingHelper {
if (navigationEndpoint.getObject("watchEndpoint").has("startTimeSeconds"))
url.append("&amp;t=").append(navigationEndpoint.getObject("watchEndpoint").getInt("startTimeSeconds"));
return url.toString();
} else if (navigationEndpoint.getObject("watchPlaylistEndpoint") != null) {
return "https://www.youtube.com/playlist?list=" +
navigationEndpoint.getObject("watchPlaylistEndpoint").getString("playlistId");
}
return null;
}
@ -351,12 +428,8 @@ public class YoutubeParsingHelper {
return thumbnailUrl;
}
public static JsonArray getJsonResponse(String url, Localization localization) throws IOException, ExtractionException {
Map<String, List<String>> headers = new HashMap<>();
headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion()));
final Response response = getDownloader().get(url, headers, localization);
public static String getValidJsonResponseBody(final Response response)
throws ParsingException, MalformedURLException {
if (response.responseCode() == 404) {
throw new ContentNotAvailableException("Not found" +
" (\"" + response.responseCode() + " " + response.responseMessage() + "\")");
@ -377,11 +450,24 @@ public class YoutubeParsingHelper {
}
final String responseContentType = response.getHeader("Content-Type");
if (responseContentType != null && responseContentType.toLowerCase().contains("text/html")) {
if (responseContentType != null
&& responseContentType.toLowerCase().contains("text/html")) {
throw new ParsingException("Got HTML document, expected JSON response" +
" (latest url was: \"" + response.latestUrl() + "\")");
}
return responseBody;
}
public static JsonArray getJsonResponse(final String url, final Localization localization)
throws IOException, ExtractionException {
Map<String, List<String>> headers = new HashMap<>();
headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion()));
final Response response = getDownloader().get(url, headers, localization);
final String responseBody = getValidJsonResponseBody(response);
try {
return JsonParser.array().from(responseBody);
} catch (JsonParserException e) {
@ -393,6 +479,7 @@ public class YoutubeParsingHelper {
* Shared alert detection function, multiple endpoints return the error similarly structured.
* <p>
* Will check if the object has an alert of the type "ERROR".
* </p>
*
* @param initialData the object which will be checked if an alert is present
* @throws ContentNotAvailableException if an alert is detected

41
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java
View File

@ -8,13 +8,21 @@ import java.net.URLEncoder;
import java.util.List;
public class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
public static final String CHARSET_UTF_8 = "UTF-8";
public static final String ALL = "all";
public static final String VIDEOS = "videos";
public static final String CHANNELS = "channels";
public static final String PLAYLISTS = "playlists";
public static final String ALL = "all";
public static final String MUSIC_SONGS = "music_songs";
public static final String MUSIC_VIDEOS = "music_videos";
public static final String MUSIC_ALBUMS = "music_albums";
public static final String MUSIC_PLAYLISTS = "music_playlists";
public static final String MUSIC_ARTISTS = "music_artists";
private static final String SEARCH_URL = "https://www.youtube.com/results?search_query=";
private static final String MUSIC_SEARCH_URL = "https://music.youtube.com/search?q=";
public static YoutubeSearchQueryHandlerFactory getInstance() {
return new YoutubeSearchQueryHandlerFactory();
@ -23,20 +31,27 @@ public class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory
@Override
public String getUrl(String searchString, List<String> contentFilters, String sortFilter) throws ParsingException {
try {
final String url = "https://www.youtube.com/results"
+ "?search_query=" + URLEncoder.encode(searchString, CHARSET_UTF_8);
if (contentFilters.size() > 0) {
switch (contentFilters.get(0)) {
case VIDEOS: return url + "&sp=EgIQAQ%253D%253D";
case CHANNELS: return url + "&sp=EgIQAg%253D%253D";
case PLAYLISTS: return url + "&sp=EgIQAw%253D%253D";
case ALL:
default:
break;
case VIDEOS:
return SEARCH_URL + URLEncoder.encode(searchString, CHARSET_UTF_8) + "&sp=EgIQAQ%253D%253D";
case CHANNELS:
return SEARCH_URL + URLEncoder.encode(searchString, CHARSET_UTF_8) + "&sp=EgIQAg%253D%253D";
case PLAYLISTS:
return SEARCH_URL + URLEncoder.encode(searchString, CHARSET_UTF_8) + "&sp=EgIQAw%253D%253D";
case MUSIC_SONGS:
case MUSIC_VIDEOS:
case MUSIC_ALBUMS:
case MUSIC_PLAYLISTS:
case MUSIC_ARTISTS:
return MUSIC_SEARCH_URL + URLEncoder.encode(searchString, CHARSET_UTF_8);
}
}
return url;
return SEARCH_URL + URLEncoder.encode(searchString, CHARSET_UTF_8);
} catch (UnsupportedEncodingException e) {
throw new ParsingException("Could not encode query", e);
}
@ -48,6 +63,12 @@ public class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory
ALL,
VIDEOS,
CHANNELS,
PLAYLISTS};
PLAYLISTS,
MUSIC_SONGS,
MUSIC_VIDEOS,
MUSIC_ALBUMS,
MUSIC_PLAYLISTS
// MUSIC_ARTISTS
};
}
}

9
extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultTests.java
View File

@ -37,11 +37,14 @@ public final class DefaultTests {
if (item instanceof StreamInfoItem) {
StreamInfoItem streamInfoItem = (StreamInfoItem) item;
assertNotEmpty("Uploader name not set: " + item, streamInfoItem.getUploaderName());
assertNotEmpty("Uploader url not set: " + item, streamInfoItem.getUploaderUrl());
assertIsSecureUrl(streamInfoItem.getUploaderUrl());
// assertNotEmpty("Uploader url not set: " + item, streamInfoItem.getUploaderUrl());
if (streamInfoItem.getUploaderUrl() != null && !streamInfoItem.getUploaderUrl().isEmpty()) {
assertIsSecureUrl(streamInfoItem.getUploaderUrl());
assertExpectedLinkType(expectedService, streamInfoItem.getUploaderUrl(), LinkType.CHANNEL);
}
assertExpectedLinkType(expectedService, streamInfoItem.getUrl(), LinkType.STREAM);
assertExpectedLinkType(expectedService, streamInfoItem.getUploaderUrl(), LinkType.CHANNEL);
final String textualUploadDate = streamInfoItem.getTextualUploadDate();
if (textualUploadDate != null && !textualUploadDate.isEmpty()) {

6
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java
View File

@ -22,4 +22,10 @@ public class YoutubeParsingHelperTest {
assertTrue("Hardcoded client version is not valid anymore",
YoutubeParsingHelper.isHardcodedClientVersionValid());
}
@Test
public void testAreHardcodedYoutubeMusicKeysValid() throws IOException, ExtractionException {
assertTrue("Hardcoded YouTube Music keys are not valid anymore",
YoutubeParsingHelper.areHardcodedYoutubeMusicKeysValid());
}
}

153
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeMusicSearchExtractorTest.java Normal file
View File

@ -0,0 +1,153 @@
package org.schabi.newpipe.extractor.services.youtube.search;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.DefaultSearchExtractorTest;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory;
import java.net.URLEncoder;
import javax.annotation.Nullable;
import static java.util.Collections.singletonList;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
public class YoutubeMusicSearchExtractorTest {
public static class MusicSongs extends DefaultSearchExtractorTest {
private static SearchExtractor extractor;
private static final String QUERY = "mocromaniac";
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = YouTube.getSearchExtractor(QUERY, singletonList(YoutubeSearchQueryHandlerFactory.MUSIC_SONGS), "");
extractor.fetchPage();
}
@Override public SearchExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return YouTube; }
@Override public String expectedName() { return QUERY; }
@Override public String expectedId() { return QUERY; }
@Override public String expectedUrlContains() { return "music.youtube.com/search?q=" + QUERY; }
@Override public String expectedOriginalUrlContains() { return "music.youtube.com/search?q=" + QUERY; }
@Override public String expectedSearchString() { return QUERY; }
@Nullable @Override public String expectedSearchSuggestion() { return null; }
@Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; }
}
public static class MusicVideos extends DefaultSearchExtractorTest {
private static SearchExtractor extractor;
private static final String QUERY = "fresku";
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = YouTube.getSearchExtractor(QUERY, singletonList(YoutubeSearchQueryHandlerFactory.MUSIC_VIDEOS), "");
extractor.fetchPage();
}
@Override public SearchExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return YouTube; }
@Override public String expectedName() { return QUERY; }
@Override public String expectedId() { return QUERY; }
@Override public String expectedUrlContains() { return "music.youtube.com/search?q=" + QUERY; }
@Override public String expectedOriginalUrlContains() { return "music.youtube.com/search?q=" + QUERY; }
@Override public String expectedSearchString() { return QUERY; }
@Nullable @Override public String expectedSearchSuggestion() { return null; }
@Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; }
}
public static class MusicAlbums extends DefaultSearchExtractorTest {
private static SearchExtractor extractor;
private static final String QUERY = "johnny sellah";
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = YouTube.getSearchExtractor(QUERY, singletonList(YoutubeSearchQueryHandlerFactory.MUSIC_ALBUMS), "");
extractor.fetchPage();
}
@Override public SearchExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return YouTube; }
@Override public String expectedName() { return QUERY; }
@Override public String expectedId() { return QUERY; }
@Override public String expectedUrlContains() { return "music.youtube.com/search?q=" + URLEncoder.encode(QUERY); }
@Override public String expectedOriginalUrlContains() { return "music.youtube.com/search?q=" + URLEncoder.encode(QUERY); }
@Override public String expectedSearchString() { return QUERY; }
@Nullable @Override public String expectedSearchSuggestion() { return null; }
@Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.PLAYLIST; }
}
public static class MusicPlaylists extends DefaultSearchExtractorTest {
private static SearchExtractor extractor;
private static final String QUERY = "louivos";
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = YouTube.getSearchExtractor(QUERY, singletonList(YoutubeSearchQueryHandlerFactory.MUSIC_PLAYLISTS), "");
extractor.fetchPage();
}
@Override public SearchExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return YouTube; }
@Override public String expectedName() { return QUERY; }
@Override public String expectedId() { return QUERY; }
@Override public String expectedUrlContains() { return "music.youtube.com/search?q=" + QUERY; }
@Override public String expectedOriginalUrlContains() { return "music.youtube.com/search?q=" + QUERY; }
@Override public String expectedSearchString() { return QUERY; }
@Nullable @Override public String expectedSearchSuggestion() { return null; }
@Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.PLAYLIST; }
}
@Ignore
public static class MusicArtists extends DefaultSearchExtractorTest {
private static SearchExtractor extractor;
private static final String QUERY = "kevin";
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = YouTube.getSearchExtractor(QUERY, singletonList(YoutubeSearchQueryHandlerFactory.MUSIC_ARTISTS), "");
extractor.fetchPage();
}
@Override public SearchExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return YouTube; }
@Override public String expectedName() { return QUERY; }
@Override public String expectedId() { return QUERY; }
@Override public String expectedUrlContains() { return "music.youtube.com/search?q=" + QUERY; }
@Override public String expectedOriginalUrlContains() { return "music.youtube.com/search?q=" + QUERY; }
@Override public String expectedSearchString() { return QUERY; }
@Nullable @Override public String expectedSearchSuggestion() { return null; }
@Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.CHANNEL; }
}
public static class Suggestion extends DefaultSearchExtractorTest {
private static SearchExtractor extractor;
private static final String QUERY = "megaman x3";
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = YouTube.getSearchExtractor(QUERY, singletonList(YoutubeSearchQueryHandlerFactory.MUSIC_SONGS), "");
extractor.fetchPage();
}
@Override public SearchExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return YouTube; }
@Override public String expectedName() { return QUERY; }
@Override public String expectedId() { return QUERY; }
@Override public String expectedUrlContains() { return "music.youtube.com/search?q=" + URLEncoder.encode(QUERY); }
@Override public String expectedOriginalUrlContains() { return "music.youtube.com/search?q=" + URLEncoder.encode(QUERY); }
@Override public String expectedSearchString() { return QUERY; }
@Nullable @Override public String expectedSearchSuggestion() { return "mega man x3"; }
@Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; }
}
}

18
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java
View File

@ -16,6 +16,12 @@ public class YoutubeSearchQHTest {
assertEquals("https://www.youtube.com/results?search_query=Poifj%26jaijf", YouTube.getSearchQHFactory().fromQuery("Poifj&jaijf").getUrl());
assertEquals("https://www.youtube.com/results?search_query=G%C3%BCl%C3%BCm", YouTube.getSearchQHFactory().fromQuery("Gülüm").getUrl());
assertEquals("https://www.youtube.com/results?search_query=%3Fj%24%29H%C2%A7B", YouTube.getSearchQHFactory().fromQuery("?j$)H§B").getUrl());
assertEquals("https://music.youtube.com/search?q=asdf", YouTube.getSearchQHFactory().fromQuery("asdf", asList(new String[]{MUSIC_SONGS}), "").getUrl());
assertEquals("https://music.youtube.com/search?q=hans", YouTube.getSearchQHFactory().fromQuery("hans", asList(new String[]{MUSIC_SONGS}), "").getUrl());
assertEquals("https://music.youtube.com/search?q=Poifj%26jaijf", YouTube.getSearchQHFactory().fromQuery("Poifj&jaijf", asList(new String[]{MUSIC_SONGS}), "").getUrl());
assertEquals("https://music.youtube.com/search?q=G%C3%BCl%C3%BCm", YouTube.getSearchQHFactory().fromQuery("Gülüm", asList(new String[]{MUSIC_SONGS}), "").getUrl());
assertEquals("https://music.youtube.com/search?q=%3Fj%24%29H%C2%A7B", YouTube.getSearchQHFactory().fromQuery("?j$)H§B", asList(new String[]{MUSIC_SONGS}), "").getUrl());
}
@Test
@ -24,6 +30,9 @@ public class YoutubeSearchQHTest {
.fromQuery("", asList(new String[]{VIDEOS}), "").getContentFilters().get(0));
assertEquals(CHANNELS, YouTube.getSearchQHFactory()
.fromQuery("asdf", asList(new String[]{CHANNELS}), "").getContentFilters().get(0));
assertEquals(MUSIC_SONGS, YouTube.getSearchQHFactory()
.fromQuery("asdf", asList(new String[]{MUSIC_SONGS}), "").getContentFilters().get(0));
}
@Test
@ -36,16 +45,23 @@ public class YoutubeSearchQHTest {
.fromQuery("asdf", asList(new String[]{PLAYLISTS}), "").getUrl());
assertEquals("https://www.youtube.com/results?search_query=asdf", YouTube.getSearchQHFactory()
.fromQuery("asdf", asList(new String[]{"fjiijie"}), "").getUrl());
assertEquals("https://music.youtube.com/search?q=asdf", YouTube.getSearchQHFactory()
.fromQuery("asdf", asList(new String[]{MUSIC_SONGS}), "").getUrl());
}
@Test
public void testGetAvailableContentFilter() {
final String[] contentFilter = YouTube.getSearchQHFactory().getAvailableContentFilter();
assertEquals(4, contentFilter.length);
assertEquals(8, contentFilter.length);
assertEquals("all", contentFilter[0]);
assertEquals("videos", contentFilter[1]);
assertEquals("channels", contentFilter[2]);
assertEquals("playlists", contentFilter[3]);
assertEquals("music_songs", contentFilter[4]);
assertEquals("music_videos", contentFilter[5]);
assertEquals("music_albums", contentFilter[6]);
assertEquals("music_playlists", contentFilter[7]);
}
@Test