Merge pull request #580 from TeamNewPipe/accountTerminated

Add AccountTerminatedException for terminated channels

extractor/src/main/java/org/schabi/newpipe/extractor/exceptions/AccountTerminatedException.java

@@ -0,0 +1,31 @@
+package org.schabi.newpipe.extractor.exceptions;
+
+public class AccountTerminatedException extends ContentNotAvailableException {
+
+    private Reason reason = Reason.UNKNOWN;
+
+    public AccountTerminatedException(final String message) {
+        super(message);
+    }
+
+    public AccountTerminatedException(final String message, final Reason reason) {
+        super(message);
+        this.reason = reason;
+    }
+
+    public AccountTerminatedException(final String message, final Throwable cause) {
+        super(message, cause);
+    }
+
+    /**
+     * The reason for the violation. There should also be more info in the exception's message.
+     */
+    public Reason getReason() {
+        return reason;
+    }
+
+    public enum Reason {
+        UNKNOWN,
+        VIOLATION
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java

@@ -10,10 +10,7 @@ import com.grack.nanojson.JsonWriter;
 import org.schabi.newpipe.extractor.MetaInfo;
 import org.schabi.newpipe.extractor.Page;
 import org.schabi.newpipe.extractor.downloader.Response;
-import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
+import org.schabi.newpipe.extractor.exceptions.*;
-import org.schabi.newpipe.extractor.exceptions.ExtractionException;
-import org.schabi.newpipe.extractor.exceptions.ParsingException;
-import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
 import org.schabi.newpipe.extractor.localization.Localization;
 import org.schabi.newpipe.extractor.stream.Description;
 import org.schabi.newpipe.extractor.utils.JsonUtils;
@@ -762,6 +759,23 @@ public class YoutubeParsingHelper {
             final String alertText = getTextFromObject(alertRenderer.getObject("text"));
             final String alertType = alertRenderer.getString("type", EMPTY_STRING);
             if (alertType.equalsIgnoreCase("ERROR")) {
+                if (alertText != null && alertText.contains("This account has been terminated")) {
+                    if (alertText.contains("violation") || alertText.contains("violating")
+                            || alertText.contains("infringement")) {
+                        // possible error messages:
+                        // "This account has been terminated for a violation of YouTube's Terms of Service."
+                        // "This account has been terminated due to multiple or severe violations of YouTube's policy prohibiting hate speech."
+                        // "This account has been terminated due to multiple or severe violations of YouTube's policy prohibiting content designed to harass, bully or threaten."
+                        // "This account has been terminated due to multiple or severe violations of YouTube's policy against spam, deceptive practices and misleading content or other Terms of Service violations."
+                        // "This account has been terminated due to multiple or severe violations of YouTube's policy on nudity or sexual content."
+                        // "This account has been terminated for violating YouTube's Community Guidelines."
+                        // "This account has been terminated because we received multiple third-party claims of copyright infringement regarding material that the user posted."
+                        // "This account has been terminated because it is linked to an account that received multiple third-party claims of copyright infringement."
+                        throw new AccountTerminatedException(alertText, AccountTerminatedException.Reason.VIOLATION);
+                    } else {
+                        throw new AccountTerminatedException(alertText);
+                    }
+                }
                 throw new ContentNotAvailableException("Got error: \"" + alertText + "\"");
             }
         }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedExtractor.java

@@ -9,6 +9,7 @@ import org.schabi.newpipe.extractor.Page;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.downloader.Downloader;
 import org.schabi.newpipe.extractor.downloader.Response;
+import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.feed.FeedExtractor;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
@@ -33,6 +34,9 @@ public class YoutubeFeedExtractor extends FeedExtractor {
         final String feedUrl = YoutubeParsingHelper.getFeedUrlFrom(channelIdOrUser);
 
         final Response response = downloader.get(feedUrl);
+        if (response.responseCode() == 404) {
+            throw new ContentNotAvailableException("Could not get feed: 404 - not found");
+        }
         document = Jsoup.parse(response.responseBody());
     }
 

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java

@@ -6,6 +6,7 @@ import org.schabi.newpipe.downloader.DownloaderFactory;
 import org.schabi.newpipe.downloader.DownloaderTestImpl;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.channel.ChannelExtractor;
+import org.schabi.newpipe.extractor.exceptions.AccountTerminatedException;
 import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
 import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
@@ -50,6 +51,90 @@ public class YoutubeChannelExtractorTest {
                     YouTube.getChannelExtractor("https://www.youtube.com/channel/DOESNT-EXIST");
             extractor.fetchPage();
         }
+
+        @Test(expected = AccountTerminatedException.class)
+        public void accountTerminatedTOSFetch() throws Exception {
+            // "This account has been terminated for a violation of YouTube's Terms of Service."
+            final ChannelExtractor extractor =
+                    YouTube.getChannelExtractor("https://www.youtube.com/channel/UCTGjY2I-ZUGnwVoWAGRd7XQ");
+            try {
+                extractor.fetchPage();
+            } catch (AccountTerminatedException e) {
+                assertEquals(e.getReason(), AccountTerminatedException.Reason.VIOLATION);
+                throw e;
+            }
+        }
+
+        @Test(expected = AccountTerminatedException.class)
+        public void accountTerminatedCommunityFetch() throws Exception {
+            // "This account has been terminated for violating YouTube's Community Guidelines."
+            final ChannelExtractor extractor =
+                    YouTube.getChannelExtractor("https://www.youtube.com/channel/UC0AuOxCr9TZ0TtEgL1zpIgA");
+            try {
+                extractor.fetchPage();
+            } catch (AccountTerminatedException e) {
+                assertEquals(e.getReason(), AccountTerminatedException.Reason.VIOLATION);
+                throw e;
+            }
+        }
+
+        @Test(expected = AccountTerminatedException.class)
+        public void accountTerminatedHateFetch() throws Exception {
+            // "This account has been terminated due to multiple or severe violations
+            // of YouTube's policy prohibiting hate speech."
+            final ChannelExtractor extractor =
+                    YouTube.getChannelExtractor("https://www.youtube.com/channel/UCPWXIOPK-9myzek6jHR5yrg");
+            try {
+                extractor.fetchPage();
+            } catch (AccountTerminatedException e) {
+                assertEquals(e.getReason(), AccountTerminatedException.Reason.VIOLATION);
+                throw e;
+            }
+        }
+
+        @Test(expected = AccountTerminatedException.class)
+        public void accountTerminatedBullyFetch() throws Exception {
+            // "This account has been terminated due to multiple or severe violations
+            // of YouTube's policy prohibiting content designed to harass, bully or threaten."
+            final ChannelExtractor extractor =
+                    YouTube.getChannelExtractor("https://youtube.com/channel/UCB1o7_gbFp2PLsamWxFenBg");
+            try {
+                extractor.fetchPage();
+            } catch (AccountTerminatedException e) {
+                assertEquals(e.getReason(), AccountTerminatedException.Reason.VIOLATION);
+                throw e;
+            }
+        }
+
+        @Test(expected = AccountTerminatedException.class)
+        public void accountTerminatedSpamFetch() throws Exception {
+            // "This account has been terminated due to multiple or severe violations
+            // of YouTube's policy against spam, deceptive practices and misleading content
+            // or other Terms of Service violations."
+            final ChannelExtractor extractor =
+                    YouTube.getChannelExtractor("https://www.youtube.com/channel/UCoaO4U_p7G7AwalqSbGCZOA");
+            try {
+                extractor.fetchPage();
+            } catch (AccountTerminatedException e) {
+                assertEquals(e.getReason(), AccountTerminatedException.Reason.VIOLATION);
+                throw e;
+            }
+        }
+
+        @Test(expected = AccountTerminatedException.class)
+        public void accountTerminatedCopyrightFetch() throws Exception {
+            // "This account has been terminated because we received multiple third-party claims
+            // of copyright infringement regarding material that the user posted."
+            final ChannelExtractor extractor =
+                    YouTube.getChannelExtractor("https://www.youtube.com/channel/UCpExuV8qJMfCaSQNL1YG6bQ");
+            try {
+                extractor.fetchPage();
+            } catch (AccountTerminatedException e) {
+                assertEquals(e.getReason(), AccountTerminatedException.Reason.VIOLATION);
+                throw e;
+            }
+        }
+
     }
 
     public static class NotSupported {

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeFeedExtractorTest.java

@@ -4,10 +4,12 @@ import org.junit.BeforeClass;
 import org.junit.Test;
 import org.schabi.newpipe.downloader.DownloaderFactory;
 import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.services.BaseListExtractorTest;
 import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeFeedExtractor;
 
+import java.io.IOException;
 import java.util.Random;
 
 import static org.junit.Assert.assertEquals;
@@ -77,4 +79,20 @@ public class YoutubeFeedExtractorTest {
             assertNoMoreItems(extractor);
         }
     }
+
+    public static class NotAvailable {
+
+        @BeforeClass
+        public static void setUp() throws IOException {
+            NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "notAvailable/"));
+        }
+
+        @Test(expected = ContentNotAvailableException.class)
+        public void AccountTerminatedFetch() throws Exception {
+            YoutubeFeedExtractor extractor = (YoutubeFeedExtractor) YouTube
+                    .getFeedExtractor("https://www.youtube.com/channel/UCTGjY2I-ZUGnwVoWAGRd7XQ");
+            extractor.fetchPage();
+        }
+
+    }
 }

extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/notAvailable/generated_mock_8.json

@@ -0,0 +1,74 @@
+{
+  "request": {
+    "httpMethod": "GET",
+    "url": "https://www.youtube.com/channel/DOESNT-EXIST/videos?pbj\u003d1\u0026view\u003d0\u0026flow\u003dgrid",
+    "headers": {
+      "Accept-Language": [
+        "en-GB, en;q\u003d0.9"
+      ],
+      "Cookie": [
+        "CONSENT\u003dPENDING+100406"
+      ],
+      "X-YouTube-Client-Name": [
+        "1"
+      ],
+      "X-YouTube-Client-Version": [
+        "2.20200214.04.00"
+      ]
+    },
+    "localization": {
+      "languageCode": "en",
+      "countryCode": "GB"
+    }
+  },
+  "response": {
+    "responseCode": 404,
+    "responseMessage": "",
+    "responseHeaders": {
+      "alt-svc": [
+        "h3-29\u003d\":443\"; ma\u003d2592000,h3-T051\u003d\":443\"; ma\u003d2592000,h3-Q050\u003d\":443\"; ma\u003d2592000,h3-Q046\u003d\":443\"; ma\u003d2592000,h3-Q043\u003d\":443\"; ma\u003d2592000,quic\u003d\":443\"; ma\u003d2592000; v\u003d\"46,43\""
+      ],
+      "cache-control": [
+        "no-cache, no-store, max-age\u003d0, must-revalidate"
+      ],
+      "content-type": [
+        "text/html; charset\u003dutf-8"
+      ],
+      "date": [
+        "Sat, 01 May 2021 15:47:25 GMT"
+      ],
+      "expires": [
+        "Mon, 01 Jan 1990 00:00:00 GMT"
+      ],
+      "p3p": [
+        "CP\u003d\"This is not a P3P policy! See http://support.google.com/accounts/answer/151657?hl\u003den-GB for more info.\""
+      ],
+      "permissions-policy": [
+        "ch-ua-full-version\u003d*, ch-ua-platform\u003d*, ch-ua-platform-version\u003d*, ch-ua-arch\u003d*, ch-ua-model\u003d*"
+      ],
+      "pragma": [
+        "no-cache"
+      ],
+      "server": [
+        "ESF"
+      ],
+      "set-cookie": [
+        "YSC\u003dgPx3EMoLzV4; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone"
+      ],
+      "strict-transport-security": [
+        "max-age\u003d31536000"
+      ],
+      "x-content-type-options": [
+        "nosniff"
+      ],
+      "x-frame-options": [
+        "SAMEORIGIN"
+      ],
+      "x-xss-protection": [
+        "0"
+      ]
+    },
+    "responseBody": "\u003chtml lang\u003d\"en-GB\" dir\u003d\"ltr\"\u003e\u003chead\u003e\u003ctitle\u003e404 Not Found\u003c/title\u003e\u003cstyle nonce\u003d\"J6HBrtOLDsO+k15HZ0o11w\"\u003e*{margin:0;padding:0;border:0}html,body{height:100%;}\u003c/style\u003e\u003clink rel\u003d\"shortcut icon\" href\u003d\"https://www.youtube.com/img/favicon.ico\" type\u003d\"image/x-icon\"\u003e\u003clink rel\u003d\"icon\" href\u003d\"https://www.youtube.com/img/favicon_32.png\" sizes\u003d\"32x32\"\u003e\u003clink rel\u003d\"icon\" href\u003d\"https://www.youtube.com/img/favicon_48.png\" sizes\u003d\"48x48\"\u003e\u003clink rel\u003d\"icon\" href\u003d\"https://www.youtube.com/img/favicon_96.png\" sizes\u003d\"96x96\"\u003e\u003clink rel\u003d\"icon\" href\u003d\"https://www.youtube.com/img/favicon_144.png\" sizes\u003d\"144x144\"\u003e\u003c/head\u003e\u003cbody\u003e\u003ciframe style\u003d\"display:block;border:0;\" src\u003d\"/error?src\u003d404\u0026amp;ifr\u003d1\u0026amp;error\u003d\" width\u003d\"100%\" height\u003d\"100%\" frameborder\u003d\"\\\" scrolling\u003d\"no\"\u003e\u003c/iframe\u003e\u003c/body\u003e\u003c/html\u003e",
+    "latestUrl": "https://www.youtube.com/channel/DOESNT-EXIST/videos?pbj\u003d1\u0026view\u003d0\u0026flow\u003dgrid"
+  }
+}

extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/feed/notAvailable/generated_mock_0.json

@@ -0,0 +1,44 @@
+{
+  "request": {
+    "httpMethod": "GET",
+    "url": "https://www.youtube.com/feeds/videos.xml?channel_id\u003dUCTGjY2I-ZUGnwVoWAGRd7XQ",
+    "headers": {
+      "Accept-Language": [
+        "en-GB, en;q\u003d0.9"
+      ]
+    },
+    "localization": {
+      "languageCode": "en",
+      "countryCode": "GB"
+    }
+  },
+  "response": {
+    "responseCode": 404,
+    "responseMessage": "",
+    "responseHeaders": {
+      "alt-svc": [
+        "h3-29\u003d\":443\"; ma\u003d2592000,h3-T051\u003d\":443\"; ma\u003d2592000,h3-Q050\u003d\":443\"; ma\u003d2592000,h3-Q046\u003d\":443\"; ma\u003d2592000,h3-Q043\u003d\":443\"; ma\u003d2592000,quic\u003d\":443\"; ma\u003d2592000; v\u003d\"46,43\""
+      ],
+      "content-length": [
+        "1613"
+      ],
+      "content-type": [
+        "text/html; charset\u003dUTF-8"
+      ],
+      "date": [
+        "Tue, 30 Mar 2021 08:33:07 GMT"
+      ],
+      "server": [
+        "YouTube RSS Feeds server"
+      ],
+      "x-frame-options": [
+        "SAMEORIGIN"
+      ],
+      "x-xss-protection": [
+        "0"
+      ]
+    },
+    "responseBody": "\u003c!DOCTYPE html\u003e\n\u003chtml lang\u003den\u003e\n  \u003cmeta charset\u003dutf-8\u003e\n  \u003cmeta name\u003dviewport content\u003d\"initial-scale\u003d1, minimum-scale\u003d1, width\u003ddevice-width\"\u003e\n  \u003ctitle\u003eError 404 (Not Found)!!1\u003c/title\u003e\n  \u003cstyle\u003e\n    *{margin:0;padding:0}html,code{font:15px/22px arial,sans-serif}html{background:#fff;color:#222;padding:15px}body{margin:7% auto 0;max-width:390px;min-height:180px;padding:30px 0 15px}* \u003e body{background:url(//www.google.com/images/errors/robot.png) 100% 5px no-repeat;padding-right:205px}p{margin:11px 0 22px;overflow:hidden}ins{color:#777;text-decoration:none}a img{border:0}@media screen and (max-width:772px){body{background:none;margin-top:0;max-width:none;padding-right:0}}#logo{background:url(//www.google.com/images/branding/googlelogo/1x/googlelogo_color_150x54dp.png) no-repeat;margin-left:-5px}@media only screen and (min-resolution:192dpi){#logo{background:url(//www.google.com/images/branding/googlelogo/2x/googlelogo_color_150x54dp.png) no-repeat 0% 0%/100% 100%;-moz-border-image:url(//www.google.com/images/branding/googlelogo/2x/googlelogo_color_150x54dp.png) 0}}@media only screen and (-webkit-min-device-pixel-ratio:2){#logo{background:url(//www.google.com/images/branding/googlelogo/2x/googlelogo_color_150x54dp.png) no-repeat;-webkit-background-size:100% 100%}}#logo{display:inline-block;height:54px;width:150px}\n  \u003c/style\u003e\n  \u003ca href\u003d//www.google.com/\u003e\u003cspan id\u003dlogo aria-label\u003dGoogle\u003e\u003c/span\u003e\u003c/a\u003e\n  \u003cp\u003e\u003cb\u003e404.\u003c/b\u003e \u003cins\u003eThat’s an error.\u003c/ins\u003e\n  \u003cp\u003eThe requested URL \u003ccode\u003e/feeds/videos.xml?channel_id\u003dUCTGjY2I-ZUGnwVoWAGRd7XQ\u003c/code\u003e was not found on this server.  \u003cins\u003eThat’s all we know.\u003c/ins\u003e\n",
+    "latestUrl": "https://www.youtube.com/feeds/videos.xml?channel_id\u003dUCTGjY2I-ZUGnwVoWAGRd7XQ"
+  }
+}