From 94e7f0d3ab34376d05bd25f522f85228fb128b28 Mon Sep 17 00:00:00 2001 From: Stypox Date: Tue, 27 Aug 2019 12:01:00 +0200 Subject: [PATCH 1/7] Fix fallback method is not tried on exception in YoutubeChannelInfoItem.getUrl() --- .../YoutubeChannelInfoItemExtractor.java | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java index d5247cad8..f6eab98f3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java @@ -56,19 +56,25 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor @Override public String getUrl() throws ParsingException { - String buttonTrackingUrl = el.select("button[class*=\"yt-uix-button\"]").first() - .attr("abs:data-href"); + try { + String buttonTrackingUrl = el.select("button[class*=\"yt-uix-button\"]").first() + .attr("abs:data-href"); - Pattern channelIdPattern = Pattern.compile("(?:.*?)\\%252Fchannel\\%252F([A-Za-z0-9\\-\\_]+)(?:.*)"); - Matcher match = channelIdPattern.matcher(buttonTrackingUrl); + Pattern channelIdPattern = Pattern.compile("(?:.*?)\\%252Fchannel\\%252F([A-Za-z0-9\\-\\_]+)(?:.*)"); + Matcher match = channelIdPattern.matcher(buttonTrackingUrl); - if (match.matches()) { - return YoutubeChannelExtractor.CHANNEL_URL_BASE + match.group(1); - } else { + if (match.matches()) { + return YoutubeChannelExtractor.CHANNEL_URL_BASE + match.group(1); + } + } catch(Throwable ignored) {} + + try { // fallback method just in case youtube changes things; it should never run and tests will fail // provides an url with "/user/NAME", that is inconsistent with stream and channel extractor return el.select("a[class*=\"yt-uix-tile-link\"]").first() .attr("abs:href"); + } catch (Throwable e) { + throw new ParsingException("Could not get channel url", e); } } From db3596c8189f9463086bd8178233e360af7bfbc1 Mon Sep 17 00:00:00 2001 From: Stypox Date: Tue, 27 Aug 2019 13:15:06 +0200 Subject: [PATCH 2/7] Fix "Could not get id" for channels w/o "Subscribe" button --- .../services/youtube/extractors/YoutubeChannelExtractor.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index 9ac93deec..41f354fd9 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -82,6 +82,11 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Nonnull @Override public String getId() throws ParsingException { + try { + return doc.select("meta[itemprop=\"channelId\"]").first().attr("content"); + } catch (Exception ignored) {} + + // fallback method; does not work with channels that have no "Subscribe" button (e.g. EminemVEVO) try { Element element = doc.getElementsByClass("yt-uix-subscription-button").first(); if (element == null) element = doc.getElementsByClass("yt-uix-subscription-preferences-button").first(); From 35921345d9cf227b84858d186203034e83b7c74f Mon Sep 17 00:00:00 2001 From: Stypox Date: Tue, 27 Aug 2019 13:16:08 +0200 Subject: [PATCH 3/7] Use Exception instead of Throwable (more consistent) --- .../youtube/extractors/YoutubeChannelInfoItemExtractor.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java index f6eab98f3..03a95abba 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java @@ -66,14 +66,14 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor if (match.matches()) { return YoutubeChannelExtractor.CHANNEL_URL_BASE + match.group(1); } - } catch(Throwable ignored) {} + } catch(Exception ignored) {} try { // fallback method just in case youtube changes things; it should never run and tests will fail // provides an url with "/user/NAME", that is inconsistent with stream and channel extractor return el.select("a[class*=\"yt-uix-tile-link\"]").first() .attr("abs:href"); - } catch (Throwable e) { + } catch (Exception e) { throw new ParsingException("Could not get channel url", e); } } From f6088c4fc1605315fc2c24b2eca96361c667a6a9 Mon Sep 17 00:00:00 2001 From: Stypox Date: Tue, 27 Aug 2019 13:27:11 +0200 Subject: [PATCH 4/7] Add test for Eminem channel (it has no "Subscribe" button) --- .../youtube/YoutubeChannelExtractorTest.java | 99 ++++++++++++++++++- 1 file changed, 97 insertions(+), 2 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java index 9e8737722..c25fdd9cc 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java @@ -392,6 +392,100 @@ public class YoutubeChannelExtractorTest { } } + // this channel has no "Subscribe" button + public static class EminemVEVO implements BaseChannelExtractorTest { + private static YoutubeChannelExtractor extractor; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(Downloader.getInstance(), new Localization("GB", "en")); + extractor = (YoutubeChannelExtractor) YouTube + .getChannelExtractor("https://www.youtube.com/user/EminemVEVO/"); + extractor.fetchPage(); + } + + /*////////////////////////////////////////////////////////////////////////// + // Extractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testServiceId() { + assertEquals(YouTube.getServiceId(), extractor.getServiceId()); + } + + @Test + public void testName() throws Exception { + assertEquals("EminemVEVO", extractor.getName()); + } + + @Test + public void testId() throws Exception { + assertEquals("UC20vb-R_px4CguHzzBPhoyQ", extractor.getId()); + } + + @Test + public void testUrl() throws ParsingException { + assertEquals("https://www.youtube.com/channel/UC20vb-R_px4CguHzzBPhoyQ", extractor.getUrl()); + } + + @Test + public void testOriginalUrl() throws ParsingException { + assertEquals("https://www.youtube.com/user/EminemVEVO/", extractor.getOriginalUrl()); + } + + /*////////////////////////////////////////////////////////////////////////// + // ListExtractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testRelatedItems() throws Exception { + defaultTestRelatedItems(extractor, YouTube.getServiceId()); + } + + @Test + public void testMoreRelatedItems() throws Exception { + defaultTestMoreItems(extractor, YouTube.getServiceId()); + } + + /*////////////////////////////////////////////////////////////////////////// + // ChannelExtractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testDescription() throws Exception { + final String description = extractor.getDescription(); + assertTrue(description, description.contains("Eminem on Vevo")); + } + + @Test + public void testAvatarUrl() throws Exception { + String avatarUrl = extractor.getAvatarUrl(); + assertIsSecureUrl(avatarUrl); + assertTrue(avatarUrl, avatarUrl.contains("yt3")); + } + + @Test + public void testBannerUrl() throws Exception { + String bannerUrl = extractor.getBannerUrl(); + assertIsSecureUrl(bannerUrl); + assertTrue(bannerUrl, bannerUrl.contains("yt3")); + } + + @Test + public void testFeedUrl() throws Exception { + assertEquals("https://www.youtube.com/feeds/videos.xml?channel_id=UC20vb-R_px4CguHzzBPhoyQ", extractor.getFeedUrl()); + } + + @Test + public void testSubscriberCount() throws Exception { + // there is no "Subscribe" button + long subscribers = extractor.getSubscriberCount(); + assertEquals("Wrong subscriber count", -1, subscribers); + } + } + + + public static class RandomChannel implements BaseChannelExtractorTest { private static YoutubeChannelExtractor extractor; @@ -483,8 +577,9 @@ public class YoutubeChannelExtractorTest { @Test public void testSubscriberCount() throws Exception { - assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 50); + long subscribers = extractor.getSubscriberCount(); + assertTrue("Wrong subscriber count: " + subscribers, subscribers >= 50); } } -}; +} From d1cd341592718dc6735adedac34f0637a4f80610 Mon Sep 17 00:00:00 2001 From: Stypox Date: Tue, 27 Aug 2019 13:56:19 +0200 Subject: [PATCH 5/7] Change comment --- .../youtube/extractors/YoutubeChannelInfoItemExtractor.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java index 03a95abba..a687c0504 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java @@ -68,9 +68,9 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor } } catch(Exception ignored) {} + // fallback method for channels without "Subscribe" button (or just in case yt changes things) + // provides an url with "/user/NAME", inconsistent with stream and channel extractor: tests will fail try { - // fallback method just in case youtube changes things; it should never run and tests will fail - // provides an url with "/user/NAME", that is inconsistent with stream and channel extractor return el.select("a[class*=\"yt-uix-tile-link\"]").first() .attr("abs:href"); } catch (Exception e) { From 0710f31a3914e3ac60ff0511e06363b636d9c532 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Mon, 23 Sep 2019 10:44:17 +0200 Subject: [PATCH 6/7] Fix TeamNewPipe/NewPipeExtractor#197 --- .../extractors/YoutubeStreamExtractor.java | 27 +++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index bec12af88..5351e6112 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -575,21 +575,26 @@ public class YoutubeStreamExtractor extends StreamExtractor { */ @Override public String getErrorMessage() { - String errorMessage = doc.select("h1[id=\"unavailable-message\"]").first().text(); StringBuilder errorReason; + Element errorElement = doc.select("h1[id=\"unavailable-message\"]").first(); - if (errorMessage == null || errorMessage.isEmpty()) { + if (errorElement == null) { errorReason = null; - } else if (errorMessage.contains("GEMA")) { - // Gema sometimes blocks youtube music content in germany: - // https://www.gema.de/en/ - // Detailed description: - // https://en.wikipedia.org/wiki/GEMA_%28German_organization%29 - errorReason = new StringBuilder("GEMA"); } else { - errorReason = new StringBuilder(errorMessage); - errorReason.append(" "); - errorReason.append(doc.select("[id=\"unavailable-submessage\"]").first().text()); + String errorMessage = errorElement.text(); + if (errorMessage == null || errorMessage.isEmpty()) { + errorReason = null; + } else if (errorMessage.contains("GEMA")) { + // Gema sometimes blocks youtube music content in germany: + // https://www.gema.de/en/ + // Detailed description: + // https://en.wikipedia.org/wiki/GEMA_%28German_organization%29 + errorReason = new StringBuilder("GEMA"); + } else { + errorReason = new StringBuilder(errorMessage); + errorReason.append(" "); + errorReason.append(doc.select("[id=\"unavailable-submessage\"]").first().text()); + } } return errorReason != null ? errorReason.toString() : null; From 8ab48c62b95fa3d879533c113f9f265c965d784f Mon Sep 17 00:00:00 2001 From: TobiGr Date: Wed, 25 Sep 2019 08:56:39 +0200 Subject: [PATCH 7/7] [YouTube] Fix NPE in ChennelExtractor.getSubsciberCount() --- .../youtube/extractors/YoutubeChannelExtractor.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index 41f354fd9..38722fa52 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -140,11 +140,12 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Override public long getSubscriberCount() throws ParsingException { - final String el = doc.select("span[class*=\"yt-subscription-button-subscriber-count\"]") - .first().attr("title"); + + final Element el = doc.select("span[class*=\"yt-subscription-button-subscriber-count\"]").first(); if (el != null) { + String elTitle = el.attr("title"); try { - return Utils.mixedNumberWordToLong(el); + return Utils.mixedNumberWordToLong(elTitle); } catch (NumberFormatException e) { throw new ParsingException("Could not get subscriber count", e); }