From e65333c3cea86b6fa8197044b8969b21df47291e Mon Sep 17 00:00:00 2001 From: Mauricio Colli Date: Sat, 29 Feb 2020 21:52:25 -0300 Subject: [PATCH] [YouTube] Detect deleted/nonexistent/invalid channels and playlists - Added tests for these cases. --- .../extractors/YoutubeChannelExtractor.java | 2 +- .../extractors/YoutubePlaylistExtractor.java | 3 ++ .../linkHandler/YoutubeParsingHelper.java | 35 +++++++++++++++++++ .../youtube/YoutubeChannelExtractorTest.java | 23 ++++++++++++ .../youtube/YoutubePlaylistExtractorTest.java | 23 ++++++++++++ 5 files changed, 85 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index 91450d474..11dd8985f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -58,8 +58,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor { final String url = super.getUrl() + "/videos?pbj=1&view=0&flow=grid"; final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization()); - initialData = ajaxJson.getObject(1).getObject("response"); + YoutubeParsingHelper.defaultAlertsCheck(initialData); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index 0f57a21a7..eef85aa9d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -10,6 +10,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.utils.Utils; @@ -39,6 +40,8 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization()); initialData = ajaxJson.getObject(1).getObject("response"); + YoutubeParsingHelper.defaultAlertsCheck(initialData); + playlistInfo = getPlaylistInfo(); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java index 87d37f8e5..d2b0a7da6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java @@ -362,10 +362,45 @@ public class YoutubeParsingHelper { throw new ParsingException("JSON response is too short"); } + // Check if the request was redirected to the error page. + final URL latestUrl = new URL(response.latestUrl()); + if (latestUrl.getHost().equalsIgnoreCase("www.youtube.com")) { + final String path = latestUrl.getPath(); + if (path.equalsIgnoreCase("/oops") || path.equalsIgnoreCase("/error")) { + throw new ContentNotAvailableException("Content unavailable"); + } + } + + final String responseContentType = response.getHeader("Content-Type"); + if (responseContentType != null && responseContentType.toLowerCase().contains("text/html")) { + throw new ParsingException("Got HTML document, expected JSON response" + + " (latest url was: \"" + response.latestUrl() + "\")"); + } + try { return JsonParser.array().from(responseBody); } catch (JsonParserException e) { throw new ParsingException("Could not parse JSON", e); } } + + /** + * Shared alert detection function, multiple endpoints return the error similarly structured. + *

+ * Will check if the object has an alert of the type "ERROR". + * + * @param initialData the object which will be checked if an alert is present + * @throws ContentNotAvailableException if an alert is detected + */ + public static void defaultAlertsCheck(JsonObject initialData) throws ContentNotAvailableException { + final JsonArray alerts = initialData.getArray("alerts"); + if (alerts != null && !alerts.isEmpty()) { + final JsonObject alertRenderer = alerts.getObject(0).getObject("alertRenderer"); + final String alertText = alertRenderer.getObject("text").getString("simpleText"); + final String alertType = alertRenderer.getString("type"); + if (alertType.equalsIgnoreCase("ERROR")) { + throw new ContentNotAvailableException("Got error: \"" + alertText + "\""); + } + } + } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java index fc4ffff31..4d69a6c89 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java @@ -6,6 +6,7 @@ import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.ServiceList; import org.schabi.newpipe.extractor.channel.ChannelExtractor; +import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor; @@ -19,6 +20,28 @@ import static org.schabi.newpipe.extractor.services.DefaultTests.*; * Test for {@link ChannelExtractor} */ public class YoutubeChannelExtractorTest { + + public static class NotAvailable { + @BeforeClass + public static void setUp() { + NewPipe.init(DownloaderTestImpl.getInstance()); + } + + @Test(expected = ContentNotAvailableException.class) + public void deletedFetch() throws Exception { + final ChannelExtractor extractor = + YouTube.getChannelExtractor("https://www.youtube.com/channel/UCAUc4iz6edWerIjlnL8OSSw"); + extractor.fetchPage(); + } + + @Test(expected = ContentNotAvailableException.class) + public void nonExistentFetch() throws Exception { + final ChannelExtractor extractor = + YouTube.getChannelExtractor("https://www.youtube.com/channel/DOESNT-EXIST"); + extractor.fetchPage(); + } + } + public static class Gronkh implements BaseChannelExtractorTest { private static YoutubeChannelExtractor extractor; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java index d4de9175e..67584195c 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java @@ -7,6 +7,7 @@ import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.ServiceList; +import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.services.BasePlaylistExtractorTest; @@ -23,6 +24,28 @@ import static org.schabi.newpipe.extractor.services.DefaultTests.*; * Test for {@link YoutubePlaylistExtractor} */ public class YoutubePlaylistExtractorTest { + + public static class NotAvailable { + @BeforeClass + public static void setUp() { + NewPipe.init(DownloaderTestImpl.getInstance()); + } + + @Test(expected = ContentNotAvailableException.class) + public void nonExistentFetch() throws Exception { + final PlaylistExtractor extractor = + YouTube.getPlaylistExtractor("https://www.youtube.com/playlist?list=PL11111111111111111111111111111111"); + extractor.fetchPage(); + } + + @Test(expected = ContentNotAvailableException.class) + public void invalidId() throws Exception { + final PlaylistExtractor extractor = + YouTube.getPlaylistExtractor("https://www.youtube.com/playlist?list=INVALID_ID"); + extractor.fetchPage(); + } + } + public static class TimelessPopHits implements BasePlaylistExtractorTest { private static YoutubePlaylistExtractor extractor;