From 9af07a25620edf38211a7caf71b9288bb0ea6370 Mon Sep 17 00:00:00 2001 From: Bleuzen Date: Tue, 24 Jul 2018 17:25:42 +0200 Subject: [PATCH 01/20] re-enable opus --- .../schabi/newpipe/extractor/services/youtube/ItagItem.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ItagItem.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ItagItem.java index 3be29c981..ba40c2dd5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ItagItem.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ItagItem.java @@ -34,10 +34,6 @@ public class ItagItem { //////////////////////////////////////////////////////////////////// // AUDIO ID ItagType Format Bitrate /// ////////////////////////////////////////////////////////////////// - // Disable Opus codec as it's not well supported in older devices -// new ItagItem(249, AUDIO, WEBMA, 50), -// new ItagItem(250, AUDIO, WEBMA, 70), -// new ItagItem(251, AUDIO, WEBMA, 160), new ItagItem(171, AUDIO, WEBMA, 128), new ItagItem(172, AUDIO, WEBMA, 256), new ItagItem(139, AUDIO, M4A, 48), @@ -45,7 +41,7 @@ public class ItagItem { new ItagItem(141, AUDIO, M4A, 256), new ItagItem(249, AUDIO, OPUS, 50), new ItagItem(250, AUDIO, OPUS, 70), - new ItagItem(160, AUDIO, OPUS, 160), + new ItagItem(251, AUDIO, OPUS, 160), /// VIDEO ONLY //////////////////////////////////////////// // ID Type Format Resolution FPS /// From 701666f4982514e06a6afa0ed7eaf6b07251159b Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Sat, 4 Aug 2018 14:20:27 +0200 Subject: [PATCH 02/20] block yt premium videos fix logig error for block yt premium videos --- .../YoutubeStreamInfoItemExtractor.java | 15 ++- .../youtube/YoutubeChannelExtractorTest.java | 97 +++++++++++++++++++ 2 files changed, 111 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java index a948bfcff..1aca59399 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java @@ -45,7 +45,17 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { @Override public boolean isAd() throws ParsingException { return !item.select("span[class*=\"icon-not-available\"]").isEmpty() - || !item.select("span[class*=\"yt-badge-ad\"]").isEmpty(); + || !item.select("span[class*=\"yt-badge-ad\"]").isEmpty() + || isPremiumVideo(); + } + + private boolean isPremiumVideo() { + Element premiumSpan = item.select("span[class=\"standalone-collection-badge-renderer-red-text\"]").first(); + if(premiumSpan == null) return false; + + // if this span has text it most likely says ("Free Video") so we can play this + if(premiumSpan.hasText()) return false; + return true; } @Override @@ -119,6 +129,9 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { Element meta = item.select("div[class=\"yt-lockup-meta\"]").first(); if (meta == null) return ""; + Element li = meta.select("li").first(); + if(li == null) return ""; + return meta.select("li").first().text(); } catch (Exception e) { throw new ParsingException("Could not get upload date", e); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java index c9ab333ff..22ec1dd04 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java @@ -114,6 +114,103 @@ public class YoutubeChannelExtractorTest { } } + // Youtube RED/Premium ad blocking test + public static class VSauce implements BaseChannelExtractorTest { + private static YoutubeChannelExtractor extractor; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(Downloader.getInstance()); + extractor = (YoutubeChannelExtractor) YouTube + .getChannelExtractor("https://www.youtube.com/user/Vsauce"); + extractor.fetchPage(); + } + + /*////////////////////////////////////////////////////////////////////////// + // Extractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testServiceId() { + assertEquals(YouTube.getServiceId(), extractor.getServiceId()); + } + + @Test + public void testName() throws Exception { + assertEquals("Vsauce", extractor.getName()); + } + + @Test + public void testId() throws Exception { + assertEquals("UC6nSFpj9HTCZ5t-N3Rm3-HA", extractor.getId()); + } + + @Test + public void testUrl() throws ParsingException { + assertEquals("https://www.youtube.com/channel/UC6nSFpj9HTCZ5t-N3Rm3-HA", extractor.getUrl()); + } + + @Test + public void testOriginalUrl() throws ParsingException { + assertEquals("https://www.youtube.com/user/Vsauce", extractor.getOriginalUrl()); + } + + /*////////////////////////////////////////////////////////////////////////// + // ListExtractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testRelatedItems() throws Exception { + defaultTestRelatedItems(extractor, YouTube.getServiceId()); + } + + @Test + public void testMoreRelatedItems() throws Exception { + defaultTestMoreItems(extractor, ServiceList.YouTube.getServiceId()); + } + + /*////////////////////////////////////////////////////////////////////////// + // ChannelExtractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testDescription() throws Exception { + assertTrue("What it actually was: " + extractor.getDescription(), + extractor.getDescription().contains("Our World is Amazing. Questions? Ideas? Tweet me:")); + } + + @Test + public void testAvatarUrl() throws Exception { + String avatarUrl = extractor.getAvatarUrl(); + assertIsSecureUrl(avatarUrl); + assertTrue(avatarUrl, avatarUrl.contains("yt3")); + } + + @Test + public void testBannerUrl() throws Exception { + String bannerUrl = extractor.getBannerUrl(); + assertIsSecureUrl(bannerUrl); + assertTrue(bannerUrl, bannerUrl.contains("yt3")); + } + + @Test + public void testFeedUrl() throws Exception { + assertEquals("https://www.youtube.com/feeds/videos.xml?channel_id=UC6nSFpj9HTCZ5t-N3Rm3-HA", extractor.getFeedUrl()); + } + + @Test + public void testSubscriberCount() throws Exception { + assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 0); + } + + @Test + public void testChannelDonation() throws Exception { + // this needs to be ignored since wed have to upgrade channel extractor to the new yt interface + // in order to make this work + assertTrue(extractor.getDonationLinks().length == 0); + } + } + public static class Kurzgesagt implements BaseChannelExtractorTest { private static YoutubeChannelExtractor extractor; From aeb813840d6732528ecf43fa454a57a4dd413e21 Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Sun, 5 Aug 2018 14:14:36 +0200 Subject: [PATCH 03/20] fix search querry tests --- .../newpipe/extractor/StreamingService.java | 28 ++++----- .../soundcloud/SoundcloudService.java | 8 +-- .../SoundcloudSubscriptionExtractor.java | 2 +- .../services/youtube/YoutubeService.java | 8 +-- .../extractors/YoutubePlaylistExtractor.java | 2 +- .../YoutubePlaylistInfoItemExtractor.java | 18 +++--- .../SoundcloudSubscriptionExtractorTest.java | 2 +- ...HTest.java => SoundcloudSearchQHTest.java} | 45 +++++++------- .../YoutubeSubscriptionExtractorTest.java | 2 +- .../youtube/search/YoutubeSearchQHTest.java | 59 +++++++++++++++++++ .../youtube/search/YoutubeSearchQUHTest.java | 55 ----------------- 11 files changed, 120 insertions(+), 109 deletions(-) rename extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/{SoundcloudSearchQUHTest.java => SoundcloudSearchQHTest.java} (57%) create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java delete mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQUHTest.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java index e3dd32243..03e465b6c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java @@ -67,10 +67,10 @@ public abstract class StreamingService { //////////////////////////////////////////// // Url Id handler //////////////////////////////////////////// - public abstract LinkHandlerFactory getStreamUIHFactory(); - public abstract ListLinkHandlerFactory getChannelUIHFactory(); - public abstract ListLinkHandlerFactory getPlaylistUIHFactory(); - public abstract SearchQueryHandlerFactory getSearchQIHFactory(); + public abstract LinkHandlerFactory getStreamLHFactory(); + public abstract ListLinkHandlerFactory getChannelLHFactory(); + public abstract ListLinkHandlerFactory getPlaylistLHFactory(); + public abstract SearchQueryHandlerFactory getSearchQHFactory(); //////////////////////////////////////////// @@ -86,31 +86,31 @@ public abstract class StreamingService { public abstract StreamExtractor getStreamExtractor(LinkHandler UIHFactory) throws ExtractionException; public SearchExtractor getSearchExtractor(String query, List contentFilter, String sortFilter, String contentCountry) throws ExtractionException { - return getSearchExtractor(getSearchQIHFactory().fromQuery(query, contentFilter, sortFilter), contentCountry); + return getSearchExtractor(getSearchQHFactory().fromQuery(query, contentFilter, sortFilter), contentCountry); } public ChannelExtractor getChannelExtractor(String id, List contentFilter, String sortFilter) throws ExtractionException { - return getChannelExtractor(getChannelUIHFactory().fromQuery(id, contentFilter, sortFilter)); + return getChannelExtractor(getChannelLHFactory().fromQuery(id, contentFilter, sortFilter)); } public PlaylistExtractor getPlaylistExtractor(String id, List contentFilter, String sortFilter) throws ExtractionException { - return getPlaylistExtractor(getPlaylistUIHFactory().fromQuery(id, contentFilter, sortFilter)); + return getPlaylistExtractor(getPlaylistLHFactory().fromQuery(id, contentFilter, sortFilter)); } public SearchExtractor getSearchExtractor(String query, String contentCountry) throws ExtractionException { - return getSearchExtractor(getSearchQIHFactory().fromQuery(query), contentCountry); + return getSearchExtractor(getSearchQHFactory().fromQuery(query), contentCountry); } public ChannelExtractor getChannelExtractor(String url) throws ExtractionException { - return getChannelExtractor(getChannelUIHFactory().fromUrl(url)); + return getChannelExtractor(getChannelLHFactory().fromUrl(url)); } public PlaylistExtractor getPlaylistExtractor(String url) throws ExtractionException { - return getPlaylistExtractor(getPlaylistUIHFactory().fromUrl(url)); + return getPlaylistExtractor(getPlaylistLHFactory().fromUrl(url)); } public StreamExtractor getStreamExtractor(String url) throws ExtractionException { - return getStreamExtractor(getStreamUIHFactory().fromUrl(url)); + return getStreamExtractor(getStreamLHFactory().fromUrl(url)); } @@ -119,9 +119,9 @@ public abstract class StreamingService { * figure out where the link is pointing to (a channel, video, playlist, etc.) */ public final LinkType getLinkTypeByUrl(String url) throws ParsingException { - LinkHandlerFactory sH = getStreamUIHFactory(); - LinkHandlerFactory cH = getChannelUIHFactory(); - LinkHandlerFactory pH = getPlaylistUIHFactory(); + LinkHandlerFactory sH = getStreamLHFactory(); + LinkHandlerFactory cH = getChannelLHFactory(); + LinkHandlerFactory pH = getPlaylistLHFactory(); if (sH.acceptUrl(url)) { return LinkType.STREAM; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java index 17aa0284b..c533b951e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java @@ -26,22 +26,22 @@ public class SoundcloudService extends StreamingService { } @Override - public SearchQueryHandlerFactory getSearchQIHFactory() { + public SearchQueryHandlerFactory getSearchQHFactory() { return new SoundcloudSearchQueryHandlerFactory(); } @Override - public LinkHandlerFactory getStreamUIHFactory() { + public LinkHandlerFactory getStreamLHFactory() { return SoundcloudStreamLinkHandlerFactory.getInstance(); } @Override - public ListLinkHandlerFactory getChannelUIHFactory() { + public ListLinkHandlerFactory getChannelLHFactory() { return SoundcloudChannelLinkHandlerFactory.getInstance(); } @Override - public ListLinkHandlerFactory getPlaylistUIHFactory() { + public ListLinkHandlerFactory getPlaylistLHFactory() { return SoundcloudPlaylistLinkHandlerFactory.getInstance(); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSubscriptionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSubscriptionExtractor.java index b5bdcb535..f65bf98b7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSubscriptionExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSubscriptionExtractor.java @@ -31,7 +31,7 @@ public class SoundcloudSubscriptionExtractor extends SubscriptionExtractor { String id; try { - id = service.getChannelUIHFactory().fromUrl(getUrlFrom(channelUrl)).getId(); + id = service.getChannelLHFactory().fromUrl(getUrlFrom(channelUrl)).getId(); } catch (ExtractionException e) { throw new InvalidSourceException(e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java index 228827bf0..51a68f0b1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java @@ -49,22 +49,22 @@ public class YoutubeService extends StreamingService { } @Override - public LinkHandlerFactory getStreamUIHFactory() { + public LinkHandlerFactory getStreamLHFactory() { return YoutubeStreamLinkHandlerFactory.getInstance(); } @Override - public ListLinkHandlerFactory getChannelUIHFactory() { + public ListLinkHandlerFactory getChannelLHFactory() { return YoutubeChannelLinkHandlerFactory.getInstance(); } @Override - public ListLinkHandlerFactory getPlaylistUIHFactory() { + public ListLinkHandlerFactory getPlaylistLHFactory() { return YoutubePlaylistLinkHandlerFactory.getInstance(); } @Override - public SearchQueryHandlerFactory getSearchQIHFactory() { + public SearchQueryHandlerFactory getSearchQHFactory() { return YoutubeSearchQueryHandlerFactory.getInstance(); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index 4852ceee5..2ef559db0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -175,7 +175,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { private void collectStreamsFrom(StreamInfoItemsCollector collector, Element element) { collector.reset(); - final LinkHandlerFactory streamLinkHandlerFactory = getService().getStreamUIHFactory(); + final LinkHandlerFactory streamLinkHandlerFactory = getService().getStreamLHFactory(); for (final Element li : element.children()) { if(isDeletedItem(li)) { continue; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java index 1b5fa4492..2d084909b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistInfoItemExtractor.java @@ -48,18 +48,22 @@ public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtract @Override public String getUrl() throws ParsingException { - String url; - try { - final Element href = el.select("div[class=\"yt-lockup-meta\"]").first() - .select("a").first(); + final Element div = el.select("div[class=\"yt-lockup-meta\"]").first(); + + if(div != null) { + final Element a = div.select("a").first(); + return a.attr("abs:href"); + } + + // this is for yt premium playlists + return el.select("h3[class=\"yt-lockup-title\"").first() + .select("a").first() + .attr("abs:href"); - url = href.attr("abs:href"); } catch (Exception e) { throw new ParsingException("Failed to extract playlist url", e); } - - return url; } @Override diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSubscriptionExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSubscriptionExtractorTest.java index 6af125346..d0000a559 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSubscriptionExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSubscriptionExtractorTest.java @@ -27,7 +27,7 @@ public class SoundcloudSubscriptionExtractorTest { public static void setupClass() { NewPipe.init(Downloader.getInstance()); subscriptionExtractor = new SoundcloudSubscriptionExtractor(ServiceList.SoundCloud); - urlHandler = ServiceList.SoundCloud.getChannelUIHFactory(); + urlHandler = ServiceList.SoundCloud.getChannelLHFactory(); } @Test diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchQUHTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchQHTest.java similarity index 57% rename from extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchQUHTest.java rename to extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchQHTest.java index a21db1138..5282ccad6 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchQUHTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchQHTest.java @@ -8,8 +8,11 @@ import org.schabi.newpipe.extractor.NewPipe; import static java.util.Arrays.asList; import static org.junit.Assert.assertEquals; import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; +import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudSearchQueryHandlerFactory.PLAYLISTS; +import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudSearchQueryHandlerFactory.TRACKS; +import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudSearchQueryHandlerFactory.USERS; -public class SoundcloudSearchQUHTest { +public class SoundcloudSearchQHTest { @BeforeClass public static void setUpClass() throws Exception { @@ -24,50 +27,50 @@ public class SoundcloudSearchQUHTest { @Test public void testRegularValues() throws Exception { assertEquals("https://api-v2.soundcloud.com/search?q=asdf&limit=10&offset=0", - removeClientId(SoundCloud.getSearchQIHFactory().fromQuery("asdf").getUrl())); + removeClientId(SoundCloud.getSearchQHFactory().fromQuery("asdf").getUrl())); assertEquals("https://api-v2.soundcloud.com/search?q=hans&limit=10&offset=0", - removeClientId(SoundCloud.getSearchQIHFactory().fromQuery("hans").getUrl())); + removeClientId(SoundCloud.getSearchQHFactory().fromQuery("hans").getUrl())); assertEquals("https://api-v2.soundcloud.com/search?q=Poifj%26jaijf&limit=10&offset=0", - removeClientId(SoundCloud.getSearchQIHFactory().fromQuery("Poifj&jaijf").getUrl())); + removeClientId(SoundCloud.getSearchQHFactory().fromQuery("Poifj&jaijf").getUrl())); assertEquals("https://api-v2.soundcloud.com/search?q=G%C3%BCl%C3%BCm&limit=10&offset=0", - removeClientId(SoundCloud.getSearchQIHFactory().fromQuery("Gülüm").getUrl())); + removeClientId(SoundCloud.getSearchQHFactory().fromQuery("Gülüm").getUrl())); assertEquals("https://api-v2.soundcloud.com/search?q=%3Fj%24%29H%C2%A7B&limit=10&offset=0", - removeClientId(SoundCloud.getSearchQIHFactory().fromQuery("?j$)H§B").getUrl())); + removeClientId(SoundCloud.getSearchQHFactory().fromQuery("?j$)H§B").getUrl())); } @Test public void testGetContentFilter() throws Exception { - assertEquals("tracks", SoundCloud.getSearchQIHFactory() + assertEquals("tracks", SoundCloud.getSearchQHFactory() .fromQuery("", asList(new String[]{"tracks"}), "").getContentFilters().get(0)); - assertEquals("users", SoundCloud.getSearchQIHFactory() + assertEquals("users", SoundCloud.getSearchQHFactory() .fromQuery("asdf", asList(new String[]{"users"}), "").getContentFilters().get(0)); } @Test public void testWithContentfilter() throws Exception { - assertEquals("https://api-v2.soundcloud.com/search/tracks?q=asdf&limit=10&offset=0", removeClientId(SoundCloud.getSearchQIHFactory() - .fromQuery("asdf", asList(new String[]{"tracks"}), "").getUrl())); - assertEquals("https://api-v2.soundcloud.com/search/users?q=asdf&limit=10&offset=0", removeClientId(SoundCloud.getSearchQIHFactory() - .fromQuery("asdf", asList(new String[]{"users"}), "").getUrl())); - assertEquals("https://api-v2.soundcloud.com/search/playlists?q=asdf&limit=10&offset=0", removeClientId(SoundCloud.getSearchQIHFactory() - .fromQuery("asdf", asList(new String[]{"playlist"}), "").getUrl())); - assertEquals("https://api-v2.soundcloud.com/search?q=asdf&limit=10&offset=0", removeClientId(SoundCloud.getSearchQIHFactory() + assertEquals("https://api-v2.soundcloud.com/search/tracks?q=asdf&limit=10&offset=0", removeClientId(SoundCloud.getSearchQHFactory() + .fromQuery("asdf", asList(new String[]{TRACKS}), "").getUrl())); + assertEquals("https://api-v2.soundcloud.com/search/users?q=asdf&limit=10&offset=0", removeClientId(SoundCloud.getSearchQHFactory() + .fromQuery("asdf", asList(new String[]{USERS}), "").getUrl())); + assertEquals("https://api-v2.soundcloud.com/search/playlists?q=asdf&limit=10&offset=0", removeClientId(SoundCloud.getSearchQHFactory() + .fromQuery("asdf", asList(new String[]{PLAYLISTS}), "").getUrl())); + assertEquals("https://api-v2.soundcloud.com/search?q=asdf&limit=10&offset=0", removeClientId(SoundCloud.getSearchQHFactory() .fromQuery("asdf", asList(new String[]{"fjiijie"}), "").getUrl())); } @Test public void testGetAvailableContentFilter() { - final String[] contentFilter = SoundCloud.getSearchQIHFactory().getAvailableContentFilter(); + final String[] contentFilter = SoundCloud.getSearchQHFactory().getAvailableContentFilter(); assertEquals(4, contentFilter.length); - assertEquals("tracks", contentFilter[0]); - assertEquals("users", contentFilter[1]); - assertEquals("playlist", contentFilter[2]); - assertEquals("any", contentFilter[3]); + assertEquals("all", contentFilter[0]); + assertEquals("tracks", contentFilter[1]); + assertEquals("users", contentFilter[2]); + assertEquals("playlists", contentFilter[3]); } @Test public void testGetAvailableSortFilter() { - final String[] contentFilter = SoundCloud.getSearchQIHFactory().getAvailableSortFilter(); + final String[] contentFilter = SoundCloud.getSearchQHFactory().getAvailableSortFilter(); assertEquals(0, contentFilter.length); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractorTest.java index a4ab4eeb9..57b493b04 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSubscriptionExtractorTest.java @@ -29,7 +29,7 @@ public class YoutubeSubscriptionExtractorTest { public static void setupClass() { NewPipe.init(Downloader.getInstance()); subscriptionExtractor = new YoutubeSubscriptionExtractor(ServiceList.YouTube); - urlHandler = ServiceList.YouTube.getChannelUIHFactory(); + urlHandler = ServiceList.YouTube.getChannelLHFactory(); } @Test diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java new file mode 100644 index 000000000..c3d3f93c4 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQHTest.java @@ -0,0 +1,59 @@ +package org.schabi.newpipe.extractor.services.youtube.search; + +import org.junit.Test; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory; + +import static java.util.Arrays.asList; +import static org.junit.Assert.assertEquals; +import static org.schabi.newpipe.extractor.ServiceList.YouTube; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.CHANNELS; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.PLAYLISTS; +import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.VIDEOS; + +public class YoutubeSearchQHTest { + + @Test + public void testRegularValues() throws Exception { + assertEquals("https://www.youtube.com/results?q=asdf", YouTube.getSearchQHFactory().fromQuery("asdf").getUrl()); + assertEquals("https://www.youtube.com/results?q=hans",YouTube.getSearchQHFactory().fromQuery("hans").getUrl()); + assertEquals("https://www.youtube.com/results?q=Poifj%26jaijf", YouTube.getSearchQHFactory().fromQuery("Poifj&jaijf").getUrl()); + assertEquals("https://www.youtube.com/results?q=G%C3%BCl%C3%BCm", YouTube.getSearchQHFactory().fromQuery("Gülüm").getUrl()); + assertEquals("https://www.youtube.com/results?q=%3Fj%24%29H%C2%A7B", YouTube.getSearchQHFactory().fromQuery("?j$)H§B").getUrl()); + } + + @Test + public void testGetContentFilter() throws Exception { + assertEquals(VIDEOS, YouTube.getSearchQHFactory() + .fromQuery("", asList(new String[]{VIDEOS}), "").getContentFilters().get(0)); + assertEquals(CHANNELS, YouTube.getSearchQHFactory() + .fromQuery("asdf", asList(new String[]{CHANNELS}), "").getContentFilters().get(0)); + } + + @Test + public void testWithContentfilter() throws Exception { + assertEquals("https://www.youtube.com/results?q=asdf&sp=EgIQAVAU", YouTube.getSearchQHFactory() + .fromQuery("asdf", asList(new String[]{VIDEOS}), "").getUrl()); + assertEquals("https://www.youtube.com/results?q=asdf&sp=EgIQAlAU", YouTube.getSearchQHFactory() + .fromQuery("asdf", asList(new String[]{CHANNELS}), "").getUrl()); + assertEquals("https://www.youtube.com/results?q=asdf&sp=EgIQA1AU", YouTube.getSearchQHFactory() + .fromQuery("asdf", asList(new String[]{PLAYLISTS}), "").getUrl()); + assertEquals("https://www.youtube.com/results?q=asdf", YouTube.getSearchQHFactory() + .fromQuery("asdf", asList(new String[]{"fjiijie"}), "").getUrl()); + } + + @Test + public void testGetAvailableContentFilter() { + final String[] contentFilter = YouTube.getSearchQHFactory().getAvailableContentFilter(); + assertEquals(4, contentFilter.length); + assertEquals("all", contentFilter[0]); + assertEquals("videos", contentFilter[1]); + assertEquals("channels", contentFilter[2]); + assertEquals("playlists", contentFilter[3]); + } + + @Test + public void testGetAvailableSortFilter() { + final String[] contentFilter = YouTube.getSearchQHFactory().getAvailableSortFilter(); + assertEquals(0, contentFilter.length); + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQUHTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQUHTest.java deleted file mode 100644 index eddf4310b..000000000 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchQUHTest.java +++ /dev/null @@ -1,55 +0,0 @@ -package org.schabi.newpipe.extractor.services.youtube.search; - -import org.junit.Test; - -import static java.util.Arrays.asList; -import static org.junit.Assert.assertEquals; -import static org.schabi.newpipe.extractor.ServiceList.YouTube; - -public class YoutubeSearchQUHTest { - - @Test - public void testRegularValues() throws Exception { - assertEquals("https://www.youtube.com/results?q=asdf", YouTube.getSearchQIHFactory().fromQuery("asdf").getUrl()); - assertEquals("https://www.youtube.com/results?q=hans",YouTube.getSearchQIHFactory().fromQuery("hans").getUrl()); - assertEquals("https://www.youtube.com/results?q=Poifj%26jaijf", YouTube.getSearchQIHFactory().fromQuery("Poifj&jaijf").getUrl()); - assertEquals("https://www.youtube.com/results?q=G%C3%BCl%C3%BCm", YouTube.getSearchQIHFactory().fromQuery("Gülüm").getUrl()); - assertEquals("https://www.youtube.com/results?q=%3Fj%24%29H%C2%A7B", YouTube.getSearchQIHFactory().fromQuery("?j$)H§B").getUrl()); - } - - @Test - public void testGetContentFilter() throws Exception { - assertEquals("stream", YouTube.getSearchQIHFactory() - .fromQuery("", asList(new String[]{"stream"}), "").getContentFilters().get(0)); - assertEquals("channel", YouTube.getSearchQIHFactory() - .fromQuery("asdf", asList(new String[]{"channel"}), "").getContentFilters().get(0)); - } - - @Test - public void testWithContentfilter() throws Exception { - assertEquals("https://www.youtube.com/results?q=asdf&sp=EgIQAVAU", YouTube.getSearchQIHFactory() - .fromQuery("asdf", asList(new String[]{"stream"}), "").getUrl()); - assertEquals("https://www.youtube.com/results?q=asdf&sp=EgIQAlAU", YouTube.getSearchQIHFactory() - .fromQuery("asdf", asList(new String[]{"channel"}), "").getUrl()); - assertEquals("https://www.youtube.com/results?q=asdf&sp=EgIQA1AU", YouTube.getSearchQIHFactory() - .fromQuery("asdf", asList(new String[]{"playlist"}), "").getUrl()); - assertEquals("https://www.youtube.com/results?q=asdf", YouTube.getSearchQIHFactory() - .fromQuery("asdf", asList(new String[]{"fjiijie"}), "").getUrl()); - } - - @Test - public void testGetAvailableContentFilter() { - final String[] contentFilter = YouTube.getSearchQIHFactory().getAvailableContentFilter(); - assertEquals(4, contentFilter.length); - assertEquals("stream", contentFilter[0]); - assertEquals("channel", contentFilter[1]); - assertEquals("playlist", contentFilter[2]); - assertEquals("any", contentFilter[3]); - } - - @Test - public void testGetAvailableSortFilter() { - final String[] contentFilter = YouTube.getSearchQIHFactory().getAvailableSortFilter(); - assertEquals(0, contentFilter.length); - } -} From 389a87fc897aa44ac6347de41deb02b0fe9a5e1d Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Thu, 16 Aug 2018 17:11:18 +0200 Subject: [PATCH 04/20] add link parsing for youtube description --- .../extractors/YoutubeStreamExtractor.java | 18 ++++++++++++++++-- .../YoutubeStreamExtractorDefaultTest.java | 7 +++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index bd44ecf3d..4709f2577 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -18,13 +18,15 @@ import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.services.youtube.ItagItem; import org.schabi.newpipe.extractor.stream.*; -import org.schabi.newpipe.extractor.utils.DonationLinkHelper; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Utils; import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.MalformedURLException; +import java.net.URL; import java.util.*; /* @@ -152,12 +154,24 @@ public class YoutubeStreamExtractor extends StreamExtractor { public String getDescription() throws ParsingException { assertPageFetched(); try { - return doc.select("p[id=\"eow-description\"]").first().html(); + return parseHtmlAndGetFullLinks(doc.select("p[id=\"eow-description\"]").first().html()); } catch (Exception e) {//todo: add fallback method <-- there is no ... as long as i know throw new ParsingException("Could not get the description", e); } } + private String parseHtmlAndGetFullLinks(String descriptionHtml) + throws MalformedURLException, UnsupportedEncodingException, ParsingException { + final Document description = Jsoup.parse(descriptionHtml, getUrl()); + for(Element a : description.select("a")) { + final URL redirectLink = new URL( + a.attr("abs:href")); + final String link = Parser.compatParseMap(redirectLink.getQuery()).get("q"); + a.text(link); + } + return description.select("body").first().html(); + } + @Override public int getAgeLimit() throws ParsingException { assertPageFetched(); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java index 982ceb986..669032ec0 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java @@ -73,6 +73,13 @@ public class YoutubeStreamExtractorDefaultTest { assertFalse(extractor.getDescription().isEmpty()); } + @Test + public void testGetFullLinksInDescriptlion() throws ParsingException { + assertTrue(extractor.getDescription().contains("http://smarturl.it/SubscribeAdele?IQid=yt")); + assertFalse(extractor.getDescription().contains("http://smarturl.it/SubscribeAdele?IQi...")); + System.out.println(extractor.getDescription()); + } + @Test public void testGetUploaderName() throws ParsingException { assertNotNull(extractor.getUploaderName()); From 981cb333b50bf16e73131eff0652123bf2e2f5e8 Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Thu, 16 Aug 2018 17:37:19 +0200 Subject: [PATCH 05/20] fix link parsing for yt internal links blub --- .../youtube/extractors/YoutubeStreamExtractor.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 4709f2577..bf39dafcc 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -155,7 +155,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { assertPageFetched(); try { return parseHtmlAndGetFullLinks(doc.select("p[id=\"eow-description\"]").first().html()); - } catch (Exception e) {//todo: add fallback method <-- there is no ... as long as i know + } catch (Exception e) { throw new ParsingException("Could not get the description", e); } } @@ -166,8 +166,14 @@ public class YoutubeStreamExtractor extends StreamExtractor { for(Element a : description.select("a")) { final URL redirectLink = new URL( a.attr("abs:href")); - final String link = Parser.compatParseMap(redirectLink.getQuery()).get("q"); - a.text(link); + final String queryString = redirectLink.getQuery(); + if(queryString != null) { + // if the query string is null we are not dealing with a redirect link, + // so we don't need to override it. + final String link = + Parser.compatParseMap(queryString).get("q"); + a.text(link); + } } return description.select("body").first().html(); } From c4e16c733769099144bd25110a651e380d71aeb8 Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Fri, 17 Aug 2018 16:22:50 +0200 Subject: [PATCH 06/20] fix hashtag parsing --- .../services/youtube/extractors/YoutubeStreamExtractor.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index bf39dafcc..eae8cc5c0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -172,7 +172,11 @@ public class YoutubeStreamExtractor extends StreamExtractor { // so we don't need to override it. final String link = Parser.compatParseMap(queryString).get("q"); - a.text(link); + + // if link is null the a tag is a hashtag. They refer to the youtube search. We do not handle them. + if(link != null) { + a.text(link); + } } } return description.select("body").first().html(); From e662c97433c9507e2f1d01bc00555df9040ae022 Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Tue, 21 Aug 2018 17:23:56 +0200 Subject: [PATCH 07/20] make dash parser ignore segmented streams --- .../extractors/YoutubeStreamExtractor.java | 2 +- .../newpipe/extractor/stream/StreamInfo.java | 5 +- .../extractor/utils/DashMpdParser.java | 77 +++++++++++++++---- .../YoutubeStreamExtractorDASHText.java | 60 +++++++++++++++ .../YoutubeStreamExtractorDefaultTest.java | 4 +- 5 files changed, 128 insertions(+), 20 deletions(-) create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDASHText.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index eae8cc5c0..0551e0f92 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -433,7 +433,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { } @Override - public List getVideoOnlyStreams() throws IOException, ExtractionException { + public List getVideoOnlyStreams() throws ExtractionException { assertPageFetched(); List videoOnlyStreams = new ArrayList<>(); try { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java index f2633533f..b6da4076d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java @@ -147,7 +147,10 @@ public class StreamInfo extends Info { Exception dashMpdError = null; if (streamInfo.getDashMpdUrl() != null && !streamInfo.getDashMpdUrl().isEmpty()) { try { - DashMpdParser.getStreams(streamInfo); + DashMpdParser.ParserResult result = DashMpdParser.getStreams(streamInfo); + streamInfo.getVideoOnlyStreams().addAll(result.getVideoOnlyStreams()); + streamInfo.getAudioStreams().addAll(result.getAudioStreams()); + streamInfo.getVideoStreams().addAll(result.getVideoStreams()); } catch (Exception e) { // Sometimes we receive 403 (forbidden) error when trying to download the manifest (similar to what happens with youtube-dl), // just skip the exception (but store it somewhere), as we later check if we have streams anyway. diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/DashMpdParser.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/DashMpdParser.java index 1fb32c13c..6c656d772 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/DashMpdParser.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/DashMpdParser.java @@ -12,6 +12,7 @@ import org.schabi.newpipe.extractor.stream.StreamInfo; import org.schabi.newpipe.extractor.stream.VideoStream; import org.w3c.dom.Document; import org.w3c.dom.Element; +import org.w3c.dom.Node; import org.w3c.dom.NodeList; import javax.xml.parsers.DocumentBuilder; @@ -19,6 +20,8 @@ import javax.xml.parsers.DocumentBuilderFactory; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; /* * Created by Christian Schabesberger on 02.02.16. @@ -51,6 +54,30 @@ public class DashMpdParser { } } + public static class ParserResult { + private final List videoStreams; + private final List audioStreams; + private final List videoOnlyStreams; + + public ParserResult(List videoStreams, List audioStreams, List videoOnlyStreams) { + this.videoStreams = videoStreams; + this.audioStreams = audioStreams; + this.videoOnlyStreams = videoOnlyStreams; + } + + public List getVideoStreams() { + return videoStreams; + } + + public List getAudioStreams() { + return audioStreams; + } + + public List getVideoOnlyStreams() { + return videoOnlyStreams; + } + } + /** * Will try to download (using {@link StreamInfo#dashMpdUrl}) and parse the dash manifest, * then it will search for any stream that the ItagItem has (by the id). @@ -58,9 +85,12 @@ public class DashMpdParser { * It has video, video only and audio streams and will only add to the list if it don't * find a similar stream in the respective lists (calling {@link Stream#equalStats}). * + * Info about dash MPD can be found here + * @see www.brendanlog.com + * * @param streamInfo where the parsed streams will be added */ - public static void getStreams(StreamInfo streamInfo) throws DashMpdParsingException, ReCaptchaException { + public static ParserResult getStreams(final StreamInfo streamInfo) throws DashMpdParsingException, ReCaptchaException { String dashDoc; Downloader downloader = NewPipe.getDownloader(); try { @@ -72,45 +102,58 @@ public class DashMpdParser { } try { - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - DocumentBuilder builder = factory.newDocumentBuilder(); - InputStream stream = new ByteArrayInputStream(dashDoc.getBytes()); + final DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + final DocumentBuilder builder = factory.newDocumentBuilder(); + final InputStream stream = new ByteArrayInputStream(dashDoc.getBytes()); - Document doc = builder.parse(stream); - NodeList representationList = doc.getElementsByTagName("Representation"); + final Document doc = builder.parse(stream); + final NodeList representationList = doc.getElementsByTagName("Representation"); + + final List videoStreams = new ArrayList<>(); + final List audioStreams = new ArrayList<>(); + final List videoOnlyStreams = new ArrayList<>(); for (int i = 0; i < representationList.getLength(); i++) { - Element representation = ((Element) representationList.item(i)); + final Element representation = (Element) representationList.item(i); try { - String mimeType = ((Element) representation.getParentNode()).getAttribute("mimeType"); - String id = representation.getAttribute("id"); - String url = representation.getElementsByTagName("BaseURL").item(0).getTextContent(); - ItagItem itag = ItagItem.getItag(Integer.parseInt(id)); - if (itag != null) { - MediaFormat mediaFormat = MediaFormat.getFromMimeType(mimeType); + final String mimeType = ((Element) representation.getParentNode()).getAttribute("mimeType"); + final String id = representation.getAttribute("id"); + final String url = representation.getElementsByTagName("BaseURL").item(0).getTextContent(); + final ItagItem itag = ItagItem.getItag(Integer.parseInt(id)); + final Node segmentationList = representation.getElementsByTagName("SegmentList").item(0); + + // if SegmentList is not null this means that BaseUrl is not representing the url to the stream. + // instead we need to add the "media=" value from the tags inside the + // tag in order to get a full working url. However each of these is just pointing to a part of the + // video, so we can not return a URL with a working stream here. + // We decided not to ignore such streams for the moment. + if (itag != null && segmentationList == null) { + final MediaFormat mediaFormat = MediaFormat.getFromMimeType(mimeType); if (itag.itagType.equals(ItagItem.ItagType.AUDIO)) { - AudioStream audioStream = new AudioStream(url, mediaFormat, itag.avgBitrate); + final AudioStream audioStream = new AudioStream(url, mediaFormat, itag.avgBitrate); if (!Stream.containSimilarStream(audioStream, streamInfo.getAudioStreams())) { - streamInfo.getAudioStreams().add(audioStream); + audioStreams.add(audioStream); } } else { boolean isVideoOnly = itag.itagType.equals(ItagItem.ItagType.VIDEO_ONLY); - VideoStream videoStream = new VideoStream(url, mediaFormat, itag.resolutionString, isVideoOnly); + final VideoStream videoStream = new VideoStream(url, mediaFormat, itag.resolutionString, isVideoOnly); if (isVideoOnly) { if (!Stream.containSimilarStream(videoStream, streamInfo.getVideoOnlyStreams())) { streamInfo.getVideoOnlyStreams().add(videoStream); + videoOnlyStreams.add(videoStream); } } else if (!Stream.containSimilarStream(videoStream, streamInfo.getVideoStreams())) { - streamInfo.getVideoStreams().add(videoStream); + videoStreams.add(videoStream); } } } } catch (Exception ignored) { } } + return new ParserResult(videoStreams, audioStreams, videoOnlyStreams); } catch (Exception e) { throw new DashMpdParsingException("Could not parse Dash mpd", e); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDASHText.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDASHText.java new file mode 100644 index 000000000..8bb6453ee --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDASHText.java @@ -0,0 +1,60 @@ +package org.schabi.newpipe.extractor.services.youtube; + + +/* + * Created by Christian Schabesberger on 30.12.15. + * + * Copyright (C) Christian Schabesberger 2015 + * YoutubeVideoExtractorDefault.java is part of NewPipe. + * + * NewPipe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe. If not, see . + */ + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.stream.StreamExtractor; +import org.schabi.newpipe.extractor.stream.StreamInfo; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.schabi.newpipe.extractor.ServiceList.YouTube; + +/** + * Test for {@link StreamExtractor} + */ +public class YoutubeStreamExtractorDASHText { + private static StreamInfo info; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(Downloader.getInstance()); + info = StreamInfo.getInfo(YouTube, "https://www.youtube.com/watch?v=00Q4SUnVQK4"); + } + + @Test + public void testGetDashMpd() { + System.out.println(info.getDashMpdUrl()); + assertTrue(info.getDashMpdUrl(), + info.getDashMpdUrl() != null && !info.getDashMpdUrl().isEmpty()); + } + + @Test + public void testDashMpdParser() { + assertEquals(0, info.getAudioStreams().size()); + assertEquals(0, info.getVideoOnlyStreams().size()); + assertEquals(4, info.getVideoStreams().size()); + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java index 669032ec0..484c0e67c 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java @@ -8,6 +8,7 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor; import org.schabi.newpipe.extractor.stream.*; +import org.schabi.newpipe.extractor.utils.DashMpdParser; import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; @@ -140,8 +141,9 @@ public class YoutubeStreamExtractorDefaultTest { @Test public void testGetDashMpd() throws ParsingException { + // we dont expect this particular video to have a DASH file. For this purpouse we use a different test class. assertTrue(extractor.getDashMpdUrl(), - extractor.getDashMpdUrl() != null || !extractor.getDashMpdUrl().isEmpty()); + extractor.getDashMpdUrl() != null && extractor.getDashMpdUrl().isEmpty()); } @Test From 3aa7ff6d0e5d7a5d849f4e00bd747154dd7931dc Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Mon, 27 Aug 2018 20:45:43 +0200 Subject: [PATCH 08/20] make ytdescription alter href instead of the text of links --- .../youtube/extractors/YoutubeStreamExtractor.java | 10 ++++++++-- .../youtube/YoutubeStreamExtractorDefaultTest.java | 5 ++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 0551e0f92..65c4bf18f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -173,9 +173,15 @@ public class YoutubeStreamExtractor extends StreamExtractor { final String link = Parser.compatParseMap(queryString).get("q"); - // if link is null the a tag is a hashtag. They refer to the youtube search. We do not handle them. if(link != null) { - a.text(link); + // if link is null the a tag is a hashtag. + // They refer to the youtube search. We do not handle them. + a.attr("abs:href", link); + + } else if(redirectLink.toString().contains("watch?v=")) { + // Another posibility is that this link is pointing to another video + // we need to put the redirectLink in here explicitly in order to add the domain part to the link. + a.attr("abs:href", redirectLink.toString()); } } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java index 484c0e67c..da589aa0a 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java @@ -58,7 +58,7 @@ public class YoutubeStreamExtractorDefaultTest { } @Test - public void testGetValidTimeStamp() throws IOException, ExtractionException { + public void testGetValidTimeStamp() throws ExtractionException { StreamExtractor extractor = YouTube.getStreamExtractor("https://youtu.be/FmG385_uUys?t=174"); assertEquals(extractor.getTimeStamp() + "", "174"); } @@ -77,8 +77,7 @@ public class YoutubeStreamExtractorDefaultTest { @Test public void testGetFullLinksInDescriptlion() throws ParsingException { assertTrue(extractor.getDescription().contains("http://smarturl.it/SubscribeAdele?IQid=yt")); - assertFalse(extractor.getDescription().contains("http://smarturl.it/SubscribeAdele?IQi...")); - System.out.println(extractor.getDescription()); + assertTrue(extractor.getDescription().contains("http://smarturl.it/SubscribeAdele?IQi...")); } @Test From fef71aeccc3765c284977f3a51612a27fc28f519 Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Tue, 28 Aug 2018 11:55:12 +0200 Subject: [PATCH 09/20] roleback using href for description links --- .../services/youtube/extractors/YoutubeStreamExtractor.java | 4 ++-- .../services/youtube/YoutubeStreamExtractorDefaultTest.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 65c4bf18f..a8c7c097d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -176,12 +176,12 @@ public class YoutubeStreamExtractor extends StreamExtractor { if(link != null) { // if link is null the a tag is a hashtag. // They refer to the youtube search. We do not handle them. - a.attr("abs:href", link); + a.text(link); } else if(redirectLink.toString().contains("watch?v=")) { // Another posibility is that this link is pointing to another video // we need to put the redirectLink in here explicitly in order to add the domain part to the link. - a.attr("abs:href", redirectLink.toString()); + a.text(redirectLink.toString()); } } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java index da589aa0a..0c4bbf713 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java @@ -77,7 +77,7 @@ public class YoutubeStreamExtractorDefaultTest { @Test public void testGetFullLinksInDescriptlion() throws ParsingException { assertTrue(extractor.getDescription().contains("http://smarturl.it/SubscribeAdele?IQid=yt")); - assertTrue(extractor.getDescription().contains("http://smarturl.it/SubscribeAdele?IQi...")); + assertFalse(extractor.getDescription().contains("http://smarturl.it/SubscribeAdele?IQi...")); } @Test From f498dd78757334ba8550a6176ea5d85cee93d992 Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Tue, 28 Aug 2018 14:33:05 +0200 Subject: [PATCH 10/20] fix failing unit tests fix yt share url error --- .../extractors/YoutubePlaylistExtractor.java | 2 +- .../YoutubeStreamLinkHandlerFactory.java | 52 ------------------- .../youtube/YoutubePlaylistExtractorTest.java | 18 ++++--- ...tubeStreamExtractorAgeRestrictedTest.java} | 4 +- ...utubeStreamExtractorControversialTest.java | 5 +- .../YoutubeStreamLinkHandlerFactoryTest.java | 18 ------- 6 files changed, 16 insertions(+), 83 deletions(-) rename extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/{YoutubeStreamExtractorRestrictedTest.java => YoutubeStreamExtractorAgeRestrictedTest.java} (98%) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index 2ef559db0..671767290 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -118,7 +118,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { @Nonnull @Override - public InfoItemsPage getInitialPage() throws IOException, ExtractionException { + public InfoItemsPage getInitialPage() throws ExtractionException { StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); Element tbody = doc.select("tbody[id=\"pl-load-more-destination\"]").first(); collectStreamsFrom(collector, tbody); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java index b159e3ad8..fca495854 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java @@ -71,8 +71,6 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory { } catch (UnsupportedEncodingException uee) { throw new ParsingException("Could not parse attribution_link", uee); } - } else if (lowercaseUrl.contains("youtube.com/shared?ci=")) { - return getRealIdFromSharedLink(url); } else if (url.contains("vnd.youtube")) { id = Parser.matchGroup1(ID_PATTERN, url); } else if (url.contains("embed")) { @@ -113,56 +111,6 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory { } } - /** - * Get the real url from a shared uri. - *

- * Shared URI's look like this: - * 

-     *     * https://www.youtube.com/shared?ci=PJICrTByb3E
-     *     * vnd.youtube://www.youtube.com/shared?ci=PJICrTByb3E&feature=twitter-deep-link
-     *
- * - * @param url The shared url - * @return the id of the stream - * @throws ParsingException - */ - private String getRealIdFromSharedLink(String url) throws ParsingException { - URI uri; - try { - uri = new URI(url); - } catch (URISyntaxException e) { - throw new ParsingException("Invalid shared link", e); - } - String sharedId = getSharedId(uri); - Downloader downloader = NewPipe.getDownloader(); - String content; - try { - content = downloader.download("https://www.youtube.com/shared?ci=" + sharedId); - } catch (IOException | ReCaptchaException e) { - throw new ParsingException("Unable to resolve shared link", e); - } - final Document document = Jsoup.parse(content); - - final Element element = document.select("link[rel=\"canonical\"]").first(); - final String urlWithRealId = (element != null) - ? element.attr("abs:href") - : document.select("meta[property=\"og:url\"]").first() - .attr("abs:content"); - - String realId = Parser.matchGroup1(ID_PATTERN, urlWithRealId); - if (sharedId.equals(realId)) { - throw new ParsingException("Got same id for as shared info_id: " + sharedId); - } - return realId; - } - - private String getSharedId(URI uri) throws ParsingException { - if (!"/shared".equals(uri.getPath())) { - throw new ParsingException("Not a shared link: " + uri.toString() + " (path != " + uri.getPath() + ")"); - } - return Parser.matchGroup1("ci=" + ID_PATTERN, uri.getQuery()); - } - @Override public boolean onAcceptUrl(final String url) throws FoundAdException { final String lowercaseUrl = url.toLowerCase(); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java index ff57344cc..7a7f5ace9 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java @@ -127,7 +127,7 @@ public class YoutubePlaylistExtractorTest { public static void setUp() throws Exception { NewPipe.init(Downloader.getInstance()); extractor = (YoutubePlaylistExtractor) YouTube - .getPlaylistExtractor("https://www.youtube.com/watch?v=lH1caqoAGe0&list=PL45xb3ujEhqUexNt53jb9WT2mS-uUaUrn"); + .getPlaylistExtractor("https://www.youtube.com/watch?v=8SbUC-UaAxE&list=PLWwAypAcFRgKAIIFqBr9oy-ZYZnixa_Fj"); extractor.fetchPage(); } @@ -152,23 +152,23 @@ public class YoutubePlaylistExtractorTest { @Test public void testName() throws Exception { - String name = extractor.getName(); - assertTrue(name, name.contains("Korrekte Aussprache - Lektion 1")); + final String name = extractor.getName(); + assertEquals("I Wanna Rock Super Gigantic Playlist 1: Hardrock, AOR, Metal and more !!! 5000 music videos !!!", name); } @Test public void testId() throws Exception { - assertEquals("PL45xb3ujEhqUexNt53jb9WT2mS-uUaUrn", extractor.getId()); + assertEquals("PLWwAypAcFRgKAIIFqBr9oy-ZYZnixa_Fj", extractor.getId()); } @Test public void testUrl() throws ParsingException { - assertEquals("https://www.youtube.com/playlist?list=PL45xb3ujEhqUexNt53jb9WT2mS-uUaUrn", extractor.getUrl()); + assertEquals("https://www.youtube.com/playlist?list=PLWwAypAcFRgKAIIFqBr9oy-ZYZnixa_Fj", extractor.getUrl()); } @Test public void testOriginalUrl() throws ParsingException { - assertEquals("https://www.youtube.com/watch?v=lH1caqoAGe0&list=PL45xb3ujEhqUexNt53jb9WT2mS-uUaUrn", extractor.getOriginalUrl()); + assertEquals("https://www.youtube.com/watch?v=8SbUC-UaAxE&list=PLWwAypAcFRgKAIIFqBr9oy-ZYZnixa_Fj", extractor.getOriginalUrl()); } /*////////////////////////////////////////////////////////////////////////// @@ -182,8 +182,10 @@ public class YoutubePlaylistExtractorTest { @Test public void testMoreRelatedItems() throws Exception { - ListExtractor.InfoItemsPage currentPage = defaultTestMoreItems(extractor, ServiceList.YouTube.getServiceId()); + ListExtractor.InfoItemsPage currentPage + = defaultTestMoreItems(extractor, ServiceList.YouTube.getServiceId()); // Test for 2 more levels + for (int i = 0; i < 2; i++) { currentPage = extractor.getPage(currentPage.getNextPageUrl()); defaultTestListOfItems(YouTube.getServiceId(), currentPage.getItems(), currentPage.getErrors()); @@ -216,7 +218,7 @@ public class YoutubePlaylistExtractorTest { @Test public void testUploaderName() throws Exception { - assertEquals("Luksan Wunder", extractor.getUploaderName()); + assertEquals("Tomas Nilsson", extractor.getUploaderName()); } @Test diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorRestrictedTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorAgeRestrictedTest.java similarity index 98% rename from extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorRestrictedTest.java rename to extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorAgeRestrictedTest.java index b8bee1a32..50e1fba56 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorRestrictedTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorAgeRestrictedTest.java @@ -24,7 +24,7 @@ import static org.schabi.newpipe.extractor.ServiceList.YouTube; /** * Test for {@link YoutubeStreamLinkHandlerFactory} */ -public class YoutubeStreamExtractorRestrictedTest { +public class YoutubeStreamExtractorAgeRestrictedTest { public static final String HTTPS = "https://"; private static YoutubeStreamExtractor extractor; @@ -32,7 +32,7 @@ public class YoutubeStreamExtractorRestrictedTest { public static void setUp() throws Exception { NewPipe.init(Downloader.getInstance()); extractor = (YoutubeStreamExtractor) YouTube - .getStreamExtractor("https://www.youtube.com/watch?v=i6JTvzrpBy0"); + .getStreamExtractor("https://www.youtube.com/watch?v=MmBeUZqv1QA"); extractor.fetchPage(); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorControversialTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorControversialTest.java index 1f5948064..42ca746fc 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorControversialTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorControversialTest.java @@ -61,7 +61,7 @@ public class YoutubeStreamExtractorControversialTest { @Test public void testGetDescription() throws ParsingException { assertNotNull(extractor.getDescription()); - assertFalse(extractor.getDescription().isEmpty()); +// assertFalse(extractor.getDescription().isEmpty()); } @Test @@ -112,13 +112,14 @@ public class YoutubeStreamExtractorControversialTest { assertTrue(streams.size() > 0); } - + @Ignore @Test public void testGetSubtitlesListDefault() throws IOException, ExtractionException { // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null assertTrue(!extractor.getSubtitlesDefault().isEmpty()); } + @Ignore @Test public void testGetSubtitlesList() throws IOException, ExtractionException { // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java index c71d19390..fdcbeca0e 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java @@ -81,16 +81,6 @@ public class YoutubeStreamLinkHandlerFactoryTest { assertEquals("jZViOEv90dI", urlIdHandler.fromUrl("vnd.youtube:jZViOEv90dI").getId()); } - @Test - public void getIdfromSharedLinksYt() throws Exception { - String sharedId = "7JIArTByb3E"; - String realId = "Q7JsK50NGaA"; - assertEquals(realId, urlIdHandler.fromUrl("vnd.youtube://www.YouTube.com/shared?ci=" + sharedId + "&feature=twitter-deep-link").getId()); - assertEquals(realId, urlIdHandler.fromUrl("vnd.youtube://www.youtube.com/shared?ci=" + sharedId).getId()); - assertEquals(realId, urlIdHandler.fromUrl("https://www.youtube.com/shared?ci=" + sharedId).getId()); - } - - @Test public void testAcceptYtUrl() throws ParsingException { assertTrue(urlIdHandler.acceptUrl("https://www.youtube.com/watch?v=jZViOEv90dI")); @@ -111,14 +101,6 @@ public class YoutubeStreamLinkHandlerFactoryTest { assertTrue(urlIdHandler.acceptUrl("vnd.youtube:jZViOEv90dI")); } - @Test - public void testAcceptSharedYtUrl() throws ParsingException { - String sharedId = "8A940MXKFmQ"; - assertTrue(urlIdHandler.acceptUrl("vnd.youtube://www.youtube.com/shared?ci=" + sharedId + "&feature=twitter-deep-link")); - assertTrue(urlIdHandler.acceptUrl("vnd.youtube://www.youtube.com/shared?ci=" + sharedId)); - assertTrue(urlIdHandler.acceptUrl("https://www.youtube.com/shared?ci=" + sharedId)); - } - @Test public void testAcceptHookUrl() throws ParsingException { assertTrue(urlIdHandler.acceptUrl("https://hooktube.com/watch?v=TglNG-yjabU")); From 0400ae026f3653e0bdd0dbe8f6b33cc71edd1ce5 Mon Sep 17 00:00:00 2001 From: Mauricio Colli Date: Mon, 3 Sep 2018 22:37:31 -0300 Subject: [PATCH 11/20] Fix channels with subscription count disabled Related: TeamNewPipe/NewPipe#1649 --- .../extractors/YoutubeChannelExtractor.java | 17 ++++++++++++----- .../YoutubeChannelInfoItemExtractor.java | 13 +++++++++---- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index e8680d11f..9f9fc549b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -7,13 +7,15 @@ import com.grack.nanojson.JsonParserException; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; -import org.schabi.newpipe.extractor.*; +import org.schabi.newpipe.extractor.Downloader; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; -import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.utils.DonationLinkHelper; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Utils; @@ -131,11 +133,16 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Override public long getSubscriberCount() throws ParsingException { - Element el = doc.select("span[class*=\"yt-subscription-button-subscriber-count\"]").first(); + final Element el = doc.select("span[class*=\"yt-subscription-button-subscriber-count\"]").first(); if (el != null) { - return Long.parseLong(Utils.removeNonDigitCharacters(el.text())); + try { + return Long.parseLong(Utils.removeNonDigitCharacters(el.text())); + } catch (NumberFormatException e) { + throw new ParsingException("Could not get subscriber count", e); + } } else { - throw new ParsingException("Could not get subscriber count"); + // If the element is null, the channel have the subscriber count disabled + return -1; } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java index 0c1e5254d..d62ce82d2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java @@ -59,11 +59,16 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor @Override public long getSubscriberCount() throws ParsingException { - Element subsEl = el.select("span[class*=\"yt-subscriber-count\"]").first(); - if (subsEl == null) { - return 0; + final Element subsEl = el.select("span[class*=\"yt-subscriber-count\"]").first(); + if (subsEl != null) { + try { + return Long.parseLong(Utils.removeNonDigitCharacters(el.text())); + } catch (NumberFormatException e) { + throw new ParsingException("Could not get subscriber count", e); + } } else { - return Long.parseLong(Utils.removeNonDigitCharacters(subsEl.text())); + // If the element is null, the channel have the subscriber count disabled + return -1; } } From 834382111b98e6292ac961eee7c1f2b126762792 Mon Sep 17 00:00:00 2001 From: Mauricio Colli Date: Mon, 3 Sep 2018 22:37:31 -0300 Subject: [PATCH 12/20] Fix NPE when playlist is empty --- .../youtube/extractors/YoutubePlaylistExtractor.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index 671767290..d20c39652 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -6,7 +6,8 @@ import com.grack.nanojson.JsonParserException; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; -import org.schabi.newpipe.extractor.*; +import org.schabi.newpipe.extractor.Downloader; +import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; @@ -19,6 +20,7 @@ import org.schabi.newpipe.extractor.stream.StreamType; import org.schabi.newpipe.extractor.utils.Utils; import javax.annotation.Nonnull; +import javax.annotation.Nullable; import java.io.IOException; @SuppressWarnings("WeakerAccess") @@ -172,9 +174,13 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { } } - private void collectStreamsFrom(StreamInfoItemsCollector collector, Element element) { + private void collectStreamsFrom(@Nonnull StreamInfoItemsCollector collector, @Nullable Element element) { collector.reset(); + if (element == null) { + return; + } + final LinkHandlerFactory streamLinkHandlerFactory = getService().getStreamLHFactory(); for (final Element li : element.children()) { if(isDeletedItem(li)) { From 850670917fce3e3fb532ecefe5b30fd062d1c1e3 Mon Sep 17 00:00:00 2001 From: Mauricio Colli Date: Wed, 5 Sep 2018 07:25:24 -0300 Subject: [PATCH 13/20] Quick fix for the kiosks in SoundCloud --- .../services/soundcloud/SoundcloudChartsExtractor.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java index 12223b8a5..93f043a62 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java @@ -57,11 +57,11 @@ public class SoundcloudChartsExtractor extends KioskExtractor { apiUrl += "&kind=trending"; } - List supportedCountries = Arrays.asList("AU", "CA", "FR", "DE", "IE", "NL", "NZ", "GB", "US"); + /*List supportedCountries = Arrays.asList("AU", "CA", "FR", "DE", "IE", "NL", "NZ", "GB", "US"); String contentCountry = getContentCountry(); if (supportedCountries.contains(contentCountry)) { apiUrl += "®ion=soundcloud:regions:" + contentCountry; - } + }*/ nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl, true); } From 119843bfaca2036192e8e0a30dab6e63185c705b Mon Sep 17 00:00:00 2001 From: John Zhen Mo Date: Fri, 7 Sep 2018 11:34:14 -0700 Subject: [PATCH 14/20] -Fix signature extraction parse exception. --- .../services/youtube/extractors/YoutubeStreamExtractor.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index a8c7c097d..f8d36c497 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -698,8 +698,10 @@ public class YoutubeStreamExtractor extends StreamExtractor { } String playerCode = downloader.download(playerUrl); - decryptionFuncName = - Parser.matchGroup("([\"\\'])signature\\1\\s*,\\s*([a-zA-Z0-9$]+)\\(", playerCode, 2); + decryptionFuncName = Parser.matchGroup( + // Look for a function with the first line containing pattern of: [var]=[var].split("") + "(\\w+)\\s*=\\s*function\\((\\w+)\\)\\{\\s*\\2=\\s*\\2\\.split\\(\"\"\\)\\s*;", + playerCode, 1); String functionPattern = "(" + decryptionFuncName.replace("$", "\\$") From 4469d1130799220c2d070d4896e70ea4ff1c4507 Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Fri, 7 Sep 2018 21:40:36 +0200 Subject: [PATCH 15/20] fix channel links in description --- .../services/youtube/extractors/YoutubeStreamExtractor.java | 3 ++- .../services/youtube/YoutubePlaylistExtractorTest.java | 2 +- ...ractorDASHText.java => YoutubeStreamExtractorDASHTest.java} | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) rename extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/{YoutubeStreamExtractorDASHText.java => YoutubeStreamExtractorDASHTest.java} (97%) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index f8d36c497..64e053a76 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -178,7 +178,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { // They refer to the youtube search. We do not handle them. a.text(link); - } else if(redirectLink.toString().contains("watch?v=")) { + } else if(redirectLink.toString().contains("watch?v=") + || redirectLink.toString().contains("https://www.youtube.com/")) { // Another posibility is that this link is pointing to another video // we need to put the redirectLink in here explicitly in order to add the domain part to the link. a.text(redirectLink.toString()); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java index 7a7f5ace9..e89db1ffc 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java @@ -120,7 +120,7 @@ public class YoutubePlaylistExtractorTest { } } - public static class ImportantVideos implements BasePlaylistExtractorTest { + public static class HugePlaylist implements BasePlaylistExtractorTest { private static YoutubePlaylistExtractor extractor; @BeforeClass diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDASHText.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDASHTest.java similarity index 97% rename from extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDASHText.java rename to extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDASHTest.java index 8bb6453ee..073df2a3d 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDASHText.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDASHTest.java @@ -35,7 +35,7 @@ import static org.schabi.newpipe.extractor.ServiceList.YouTube; /** * Test for {@link StreamExtractor} */ -public class YoutubeStreamExtractorDASHText { +public class YoutubeStreamExtractorDASHTest { private static StreamInfo info; @BeforeClass From 66c3c3f45241d4b0c909ff527a5570c2213d9c52 Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Fri, 7 Sep 2018 22:18:22 +0200 Subject: [PATCH 16/20] fix channel links in description part 2 --- .../extractors/YoutubeStreamExtractor.java | 10 +- .../YoutubeStreamExtractorDefaultTest.java | 233 ++++++++++-------- 2 files changed, 139 insertions(+), 104 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 64e053a76..d63eee1d6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -178,12 +178,12 @@ public class YoutubeStreamExtractor extends StreamExtractor { // They refer to the youtube search. We do not handle them. a.text(link); - } else if(redirectLink.toString().contains("watch?v=") - || redirectLink.toString().contains("https://www.youtube.com/")) { - // Another posibility is that this link is pointing to another video - // we need to put the redirectLink in here explicitly in order to add the domain part to the link. - a.text(redirectLink.toString()); } + } else if(redirectLink.toString().contains("watch?v=") + || redirectLink.toString().contains("https://www.youtube.com/")) { + // Another posibility is that this link is pointing to another video + // we need to put the redirectLink in here explicitly in order to add the domain part to the link. + a.text(redirectLink.toString()); } } return description.select("body").first().html(); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java index 0c4bbf713..2b5f6058c 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java @@ -16,6 +16,7 @@ import java.io.IOException; import static org.junit.Assert.*; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ServiceList.YouTube; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeTrendingExtractorTest.extractor; /* * Created by Christian Schabesberger on 30.12.15. @@ -41,127 +42,161 @@ import static org.schabi.newpipe.extractor.ServiceList.YouTube; * Test for {@link StreamExtractor} */ public class YoutubeStreamExtractorDefaultTest { - private static YoutubeStreamExtractor extractor; - @BeforeClass - public static void setUp() throws Exception { - NewPipe.init(Downloader.getInstance()); - extractor = (YoutubeStreamExtractor) YouTube - .getStreamExtractor("https://www.youtube.com/watch?v=rYEDA3JcQqw"); - extractor.fetchPage(); - } + public static class AdeleHello { + private static YoutubeStreamExtractor extractor; - @Test - public void testGetInvalidTimeStamp() throws ParsingException { - assertTrue(extractor.getTimeStamp() + "", - extractor.getTimeStamp() <= 0); - } + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(Downloader.getInstance()); + extractor = (YoutubeStreamExtractor) YouTube + .getStreamExtractor("https://www.youtube.com/watch?v=rYEDA3JcQqw"); + extractor.fetchPage(); + } - @Test - public void testGetValidTimeStamp() throws ExtractionException { - StreamExtractor extractor = YouTube.getStreamExtractor("https://youtu.be/FmG385_uUys?t=174"); - assertEquals(extractor.getTimeStamp() + "", "174"); - } + @Test + public void testGetInvalidTimeStamp() throws ParsingException { + assertTrue(extractor.getTimeStamp() + "", + extractor.getTimeStamp() <= 0); + } - @Test - public void testGetTitle() throws ParsingException { - assertFalse(extractor.getName().isEmpty()); - } + @Test + public void testGetValidTimeStamp() throws ExtractionException { + StreamExtractor extractor = YouTube.getStreamExtractor("https://youtu.be/FmG385_uUys?t=174"); + assertEquals(extractor.getTimeStamp() + "", "174"); + } - @Test - public void testGetDescription() throws ParsingException { - assertNotNull(extractor.getDescription()); - assertFalse(extractor.getDescription().isEmpty()); - } + @Test + public void testGetTitle() throws ParsingException { + assertFalse(extractor.getName().isEmpty()); + } - @Test - public void testGetFullLinksInDescriptlion() throws ParsingException { - assertTrue(extractor.getDescription().contains("http://smarturl.it/SubscribeAdele?IQid=yt")); - assertFalse(extractor.getDescription().contains("http://smarturl.it/SubscribeAdele?IQi...")); - } + @Test + public void testGetDescription() throws ParsingException { + assertNotNull(extractor.getDescription()); + assertFalse(extractor.getDescription().isEmpty()); + } - @Test - public void testGetUploaderName() throws ParsingException { - assertNotNull(extractor.getUploaderName()); - assertFalse(extractor.getUploaderName().isEmpty()); - } + @Test + public void testGetFullLinksInDescriptlion() throws ParsingException { + assertTrue(extractor.getDescription().contains("http://smarturl.it/SubscribeAdele?IQid=yt")); + assertFalse(extractor.getDescription().contains("http://smarturl.it/SubscribeAdele?IQi...")); + } + + @Test + public void testGetUploaderName() throws ParsingException { + assertNotNull(extractor.getUploaderName()); + assertFalse(extractor.getUploaderName().isEmpty()); + } - @Test - public void testGetLength() throws ParsingException { - assertTrue(extractor.getLength() > 0); - } + @Test + public void testGetLength() throws ParsingException { + assertTrue(extractor.getLength() > 0); + } - @Test - public void testGetViewCount() throws ParsingException { - Long count = extractor.getViewCount(); - assertTrue(Long.toString(count), count >= /* specific to that video */ 1220025784); - } + @Test + public void testGetViewCount() throws ParsingException { + Long count = extractor.getViewCount(); + assertTrue(Long.toString(count), count >= /* specific to that video */ 1220025784); + } - @Test - public void testGetUploadDate() throws ParsingException { - assertTrue(extractor.getUploadDate().length() > 0); - } + @Test + public void testGetUploadDate() throws ParsingException { + assertTrue(extractor.getUploadDate().length() > 0); + } - @Test - public void testGetUploaderUrl() throws ParsingException { - assertTrue(extractor.getUploaderUrl().length() > 0); - } + @Test + public void testGetUploaderUrl() throws ParsingException { + assertTrue(extractor.getUploaderUrl().length() > 0); + } - @Test - public void testGetThumbnailUrl() throws ParsingException { - assertIsSecureUrl(extractor.getThumbnailUrl()); - } + @Test + public void testGetThumbnailUrl() throws ParsingException { + assertIsSecureUrl(extractor.getThumbnailUrl()); + } - @Test - public void testGetUploaderAvatarUrl() throws ParsingException { - assertIsSecureUrl(extractor.getUploaderAvatarUrl()); - } + @Test + public void testGetUploaderAvatarUrl() throws ParsingException { + assertIsSecureUrl(extractor.getUploaderAvatarUrl()); + } - @Test - public void testGetAudioStreams() throws IOException, ExtractionException { - assertFalse(extractor.getAudioStreams().isEmpty()); - } + @Test + public void testGetAudioStreams() throws IOException, ExtractionException { + assertFalse(extractor.getAudioStreams().isEmpty()); + } - @Test - public void testGetVideoStreams() throws IOException, ExtractionException { - for (VideoStream s : extractor.getVideoStreams()) { - assertIsSecureUrl(s.url); - assertTrue(s.resolution.length() > 0); - assertTrue(Integer.toString(s.getFormatId()), - 0 <= s.getFormatId() && s.getFormatId() <= 4); + @Test + public void testGetVideoStreams() throws IOException, ExtractionException { + for (VideoStream s : extractor.getVideoStreams()) { + assertIsSecureUrl(s.url); + assertTrue(s.resolution.length() > 0); + assertTrue(Integer.toString(s.getFormatId()), + 0 <= s.getFormatId() && s.getFormatId() <= 4); + } + } + + @Test + public void testStreamType() throws ParsingException { + assertTrue(extractor.getStreamType() == StreamType.VIDEO_STREAM); + } + + @Test + public void testGetDashMpd() throws ParsingException { + // we dont expect this particular video to have a DASH file. For this purpouse we use a different test class. + assertTrue(extractor.getDashMpdUrl(), + extractor.getDashMpdUrl() != null && extractor.getDashMpdUrl().isEmpty()); + } + + @Test + public void testGetRelatedVideos() throws ExtractionException, IOException { + StreamInfoItemsCollector relatedVideos = extractor.getRelatedVideos(); + Utils.printErrors(relatedVideos.getErrors()); + assertFalse(relatedVideos.getItems().isEmpty()); + assertTrue(relatedVideos.getErrors().isEmpty()); + } + + @Test + public void testGetSubtitlesListDefault() throws IOException, ExtractionException { + // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null + assertTrue(extractor.getSubtitlesDefault().isEmpty()); + } + + @Test + public void testGetSubtitlesList() throws IOException, ExtractionException { + // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null + assertTrue(extractor.getSubtitles(SubtitlesFormat.TTML).isEmpty()); } } - @Test - public void testStreamType() throws ParsingException { - assertTrue(extractor.getStreamType() == StreamType.VIDEO_STREAM); - } + public static class DescriptionTestPewdiepie { + private static YoutubeStreamExtractor extractor; - @Test - public void testGetDashMpd() throws ParsingException { - // we dont expect this particular video to have a DASH file. For this purpouse we use a different test class. - assertTrue(extractor.getDashMpdUrl(), - extractor.getDashMpdUrl() != null && extractor.getDashMpdUrl().isEmpty()); - } + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(Downloader.getInstance()); + extractor = (YoutubeStreamExtractor) YouTube + .getStreamExtractor("https://www.youtube.com/watch?v=dJY8iT341F4"); + extractor.fetchPage(); + } - @Test - public void testGetRelatedVideos() throws ExtractionException, IOException { - StreamInfoItemsCollector relatedVideos = extractor.getRelatedVideos(); - Utils.printErrors(relatedVideos.getErrors()); - assertFalse(relatedVideos.getItems().isEmpty()); - assertTrue(relatedVideos.getErrors().isEmpty()); - } + @Test + public void testGetDescription() throws ParsingException { + assertNotNull(extractor.getDescription()); + assertFalse(extractor.getDescription().isEmpty()); + } - @Test - public void testGetSubtitlesListDefault() throws IOException, ExtractionException { - // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null - assertTrue(extractor.getSubtitlesDefault().isEmpty()); - } + @Test + public void testGetFullLinksInDescriptlion() throws ParsingException { + assertTrue(extractor.getDescription().contains("https://www.reddit.com/r/PewdiepieSubmissions/")); + assertTrue(extractor.getDescription().contains("https://www.youtube.com/channel/UC3e8EMTOn4g6ZSKggHTnNng")); - @Test - public void testGetSubtitlesList() throws IOException, ExtractionException { - // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null - assertTrue(extractor.getSubtitles(SubtitlesFormat.TTML).isEmpty()); + assertFalse(extractor.getDescription().contains("https://www.reddit.com/r/PewdiepieSub...")); + assertFalse(extractor.getDescription().contains("https://usa.clutchchairz.com/product/...")); + assertFalse(extractor.getDescription().contains("https://europe.clutchchairz.com/en/pr...")); + assertFalse(extractor.getDescription().contains("https://canada.clutchchairz.com/produ...")); + assertFalse(extractor.getDescription().contains("http://store.steampowered.com/app/703...")); + assertFalse(extractor.getDescription().contains("https://www.youtube.com/channel/UC3e8...")); + } } } From ed73ae55f1f88981961d982dafde9e5917156376 Mon Sep 17 00:00:00 2001 From: John Zhen Mo Date: Fri, 7 Sep 2018 22:25:07 -0700 Subject: [PATCH 17/20] -Added more decrypt function name matching regex. -Cleaned up decryption code generation method. --- .../extractors/YoutubeStreamExtractor.java | 53 ++++++++++--------- 1 file changed, 28 insertions(+), 25 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index f8d36c497..6e43868e8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -560,6 +560,13 @@ public class YoutubeStreamExtractor extends StreamExtractor { private static final String VERIFIED_URL_PARAMS = "&has_verified=1&bpctr=9999999999"; + private final static String DECYRYPTION_SIGNATURE_FUNCTION_REGEX = + "(\\w+)\\s*=\\s*function\\((\\w+)\\)\\{\\s*\\2=\\s*\\2\\.split\\(\"\"\\)\\s*;"; + private final static String DECRYPTION_AKAMAIZED_STRING_REGEX = + "yt\\.akamaized\\.net/\\)\\s*\\|\\|\\s*.*?\\s*c\\s*&&\\s*d\\.set\\([^,]+\\s*,\\s*([a-zA-Z0-9$]+)\\("; + private final static String DECRYPTION_AKAMAIZED_SHORT_STRING_REGEX = + "\\bc\\s*&&\\s*d\\.set\\([^,]+\\s*,\\s*([a-zA-Z0-9$]+)\\("; + private volatile String decryptionCode = ""; private String pageHtml = null; @@ -682,13 +689,6 @@ public class YoutubeStreamExtractor extends StreamExtractor { } private String loadDecryptionCode(String playerUrl) throws DecryptException { - String decryptionFuncName; - String decryptionFunc; - String helperObjectName; - String helperObject; - String callerFunc = "function " + DECRYPTION_FUNC_NAME + "(a){return %%(a);}"; - String decryptionCode; - try { Downloader downloader = NewPipe.getDownloader(); if (!playerUrl.contains("https://youtube.com")) { @@ -696,35 +696,38 @@ public class YoutubeStreamExtractor extends StreamExtractor { //than we have to add it by hand playerUrl = "https://youtube.com" + playerUrl; } - String playerCode = downloader.download(playerUrl); - decryptionFuncName = Parser.matchGroup( - // Look for a function with the first line containing pattern of: [var]=[var].split("") - "(\\w+)\\s*=\\s*function\\((\\w+)\\)\\{\\s*\\2=\\s*\\2\\.split\\(\"\"\\)\\s*;", - playerCode, 1); + final String playerCode = downloader.download(playerUrl); - String functionPattern = "(" - + decryptionFuncName.replace("$", "\\$") + final String decryptionFunctionName; + if (Parser.isMatch(DECRYPTION_AKAMAIZED_SHORT_STRING_REGEX, playerCode)) { + decryptionFunctionName = Parser.matchGroup1(DECRYPTION_AKAMAIZED_SHORT_STRING_REGEX, playerCode); + } else if (Parser.isMatch(DECRYPTION_AKAMAIZED_STRING_REGEX, playerCode)) { + decryptionFunctionName = Parser.matchGroup1(DECRYPTION_AKAMAIZED_STRING_REGEX, playerCode); + } else { + decryptionFunctionName = Parser.matchGroup1(DECYRYPTION_SIGNATURE_FUNCTION_REGEX, playerCode); + } + final String functionPattern = "(" + + decryptionFunctionName.replace("$", "\\$") + "=function\\([a-zA-Z0-9_]+\\)\\{.+?\\})"; - decryptionFunc = "var " + Parser.matchGroup1(functionPattern, playerCode) + ";"; + final String decryptionFunction = "var " + Parser.matchGroup1(functionPattern, playerCode) + ";"; - helperObjectName = Parser - .matchGroup1(";([A-Za-z0-9_\\$]{2})\\...\\(", decryptionFunc); + final String helperObjectName = + Parser.matchGroup1(";([A-Za-z0-9_\\$]{2})\\...\\(", decryptionFunction); + final String helperPattern = + "(var " + helperObjectName.replace("$", "\\$") + "=\\{.+?\\}\\};)"; + final String helperObject = + Parser.matchGroup1(helperPattern, playerCode.replace("\n", "")); - String helperPattern = "(var " - + helperObjectName.replace("$", "\\$") + "=\\{.+?\\}\\};)"; - helperObject = Parser.matchGroup1(helperPattern, playerCode.replace("\n", "")); + final String callerFunction = + "function " + DECRYPTION_FUNC_NAME + "(a){return " + decryptionFunctionName + "(a);}"; - - callerFunc = callerFunc.replace("%%", decryptionFuncName); - decryptionCode = helperObject + decryptionFunc + callerFunc; + return helperObject + decryptionFunction + callerFunction; } catch (IOException ioe) { throw new DecryptException("Could not load decrypt function", ioe); } catch (Exception e) { throw new DecryptException("Could not parse decrypt function ", e); } - - return decryptionCode; } private String decryptSignature(String encryptedSig, String decryptionCode) throws DecryptException { From a83d0209449949735954b6eecc50027f64faefd4 Mon Sep 17 00:00:00 2001 From: skil3z <43092256+skil3z@users.noreply.github.com> Date: Sun, 9 Sep 2018 12:53:10 +0300 Subject: [PATCH 18/20] Accommodate time formatting for different countries If there's a . in the time format, this detects it and uses is instead of : This removes errors and lag related to "Could not get duration" while using NewPipe in countries with official time formatting with . (dot) instead of : (punctuation colon) Tested to compile and work on real device --- .../youtube/linkHandler/YoutubeParsingHelper.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java index fca0584ca..fc16333f2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java @@ -30,7 +30,17 @@ public class YoutubeParsingHelper { public static long parseDurationString(String input) throws ParsingException, NumberFormatException { - String[] splitInput = input.split(":"); + + String[] splitInput; + + // If time separator : is not detected, try . instead + + if (input.contains(":")) { + splitInput = input.split(":"); + } else { + splitInput = input.split("\\."); + } + String days = "0"; String hours = "0"; String minutes = "0"; From 8a4afe2548fb466b6d2b72a634fc512f0b7988f9 Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Sun, 9 Sep 2018 14:01:39 +0200 Subject: [PATCH 19/20] refactor split time parsing --- .../linkHandler/YoutubeParsingHelper.java | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java index fc16333f2..84f1f1351 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java @@ -31,20 +31,16 @@ public class YoutubeParsingHelper { public static long parseDurationString(String input) throws ParsingException, NumberFormatException { - String[] splitInput; - // If time separator : is not detected, try . instead - - if (input.contains(":")) { - splitInput = input.split(":"); - } else { - splitInput = input.split("\\."); - } - + + final String[] splitInput = input.contains(":") + ? input.split(":") + : input.split("\\."); + String days = "0"; String hours = "0"; String minutes = "0"; - String seconds; + final String seconds; switch (splitInput.length) { case 4: From 217d13b10280ab7685bea984b76618d826ad1560 Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Tue, 11 Sep 2018 15:14:22 +0200 Subject: [PATCH 20/20] fix wrong subscription count --- .../YoutubeChannelInfoItemExtractor.java | 2 +- .../search/YoutubeSearchCountTest.java | 35 +++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchCountTest.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java index d62ce82d2..9e0e975f7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java @@ -62,7 +62,7 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor final Element subsEl = el.select("span[class*=\"yt-subscriber-count\"]").first(); if (subsEl != null) { try { - return Long.parseLong(Utils.removeNonDigitCharacters(el.text())); + return Long.parseLong(Utils.removeNonDigitCharacters(subsEl.text())); } catch (NumberFormatException e) { throw new ParsingException("Could not get subscriber count", e); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchCountTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchCountTest.java new file mode 100644 index 000000000..745b6a0a1 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchCountTest.java @@ -0,0 +1,35 @@ +package org.schabi.newpipe.extractor.services.youtube.search; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.channel.ChannelInfoItem; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory; + +import static java.util.Collections.singletonList; +import static junit.framework.TestCase.assertTrue; +import static org.schabi.newpipe.extractor.ServiceList.YouTube; + +public class YoutubeSearchCountTest { + public static class YoutubeChannelViewCountTest extends YoutubeSearchExtractorBaseTest { + @BeforeClass + public static void setUpClass() throws Exception { + NewPipe.init(Downloader.getInstance()); + extractor = (YoutubeSearchExtractor) YouTube.getSearchExtractor("pewdiepie", + singletonList(YoutubeSearchQueryHandlerFactory.CHANNELS), null,"de"); + extractor.fetchPage(); + itemsPage = extractor.getInitialPage(); + } + + @Test + public void testViewCount() throws Exception { + boolean foundKnownChannel = false; + ChannelInfoItem ci = (ChannelInfoItem) itemsPage.getItems().get(0); + assertTrue("Count does not fit: " + Long.toString(ci.getSubscriberCount()), + 65043316 < ci.getSubscriberCount() && ci.getSubscriberCount() < 68043316); + } + } +}