From 98f49852d79712b4c839c92e7f75c8094315cb6c Mon Sep 17 00:00:00 2001 From: Connectety-W Date: Sun, 13 Jan 2019 12:52:07 +0100 Subject: [PATCH 01/24] refactored YouTube-linkHandler to use less regex and more URL-methods --- .../YoutubeChannelLinkHandlerFactory.java | 48 ++++- .../linkHandler/YoutubeParsingHelper.java | 38 ++++ .../YoutubePlaylistLinkHandlerFactory.java | 35 +++- .../YoutubeStreamLinkHandlerFactory.java | 180 +++++++++++------- .../YoutubeTrendingLinkHandlerFactory.java | 13 +- .../schabi/newpipe/extractor/utils/Utils.java | 43 ++++- .../YoutubeStreamLinkHandlerFactoryTest.java | 12 +- 7 files changed, 277 insertions(+), 92 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java index 950bab2b9..da207b278 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java @@ -1,9 +1,9 @@ package org.schabi.newpipe.extractor.services.youtube.linkHandler; -import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.utils.Parser; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; +import java.net.URL; import java.util.List; /* @@ -29,25 +29,53 @@ import java.util.List; public class YoutubeChannelLinkHandlerFactory extends ListLinkHandlerFactory { private static final YoutubeChannelLinkHandlerFactory instance = new YoutubeChannelLinkHandlerFactory(); - private static final String ID_PATTERN = "/(user/[A-Za-z0-9_-]*|channel/[A-Za-z0-9_-]*)"; public static YoutubeChannelLinkHandlerFactory getInstance() { return instance; } - @Override - public String getId(String url) throws ParsingException { - return Parser.matchGroup1(ID_PATTERN, url); - } - @Override public String getUrl(String id, List contentFilters, String searchFilter) { return "https://www.youtube.com/" + id; } + @Override + public String getId(String url) throws ParsingException { + try { + URL urlObj = new URL(url); + String path = urlObj.getPath(); + + if (!(YoutubeParsingHelper.isYoutubeURL(urlObj) || urlObj.getHost().equalsIgnoreCase("hooktube.com"))) { + throw new ParsingException("the URL given is not a Youtube-URL"); + } + + if (!path.startsWith("/user/") && !path.startsWith("/channel/")) { + throw new ParsingException("the URL given is neither a channel nor an user"); + } + + // remove leading "/" + path = path.substring(1); + + String[] splitPath = path.split("/"); + String id = splitPath[1]; + + if (id == null || !id.matches("[A-Za-z0-9_-]+")) { + throw new ParsingException("The given id is not a Youtube-Video-ID"); + } + + return splitPath[0] + "/" + id; + } catch (final Exception exception) { + throw new ParsingException("Error could not parse url :" + exception.getMessage(), exception); + } + } + @Override public boolean onAcceptUrl(String url) { - return (url.contains("youtube") || url.contains("youtu.be") || url.contains("hooktube.com")) - && (url.contains("/user/") || url.contains("/channel/")); + try { + getId(url); + } catch (ParsingException e) { + return false; + } + return true; } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java index 84f1f1351..335bc5bf6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java @@ -3,6 +3,8 @@ package org.schabi.newpipe.extractor.services.youtube.linkHandler; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import java.net.URL; + /* * Created by Christian Schabesberger on 02.03.16. * @@ -28,6 +30,42 @@ public class YoutubeParsingHelper { private YoutubeParsingHelper() { } + private static boolean isHTTP(URL url) { + // make sure its http or https + String protocol = url.getProtocol(); + if (!protocol.equals("http") && !protocol.equals("https")) { + return false; + } + + boolean usesDefaultPort = url.getPort() == url.getDefaultPort(); + boolean setsNoPort = url.getPort() == -1; + + return setsNoPort || usesDefaultPort; + } + + public static boolean isYoutubeURL(URL url) { + // make sure its http or https + if (!isHTTP(url)) + return false; + + // make sure its a known youtube url + String host = url.getHost(); + return host.equalsIgnoreCase("youtube.com") || host.equalsIgnoreCase("www.youtube.com") + || host.equalsIgnoreCase("m.youtube.com"); + } + + public static boolean isYoutubeALikeURL(URL url) { + // make sure its http or https + if (!isHTTP(url)) + return false; + + // make sure its a known youtube url + String host = url.getHost(); + return host.equalsIgnoreCase("youtube.com") || host.equalsIgnoreCase("www.youtube.com") + || host.equalsIgnoreCase("m.youtube.com") || host.equalsIgnoreCase("www.youtube-nocookie.com") + || host.equalsIgnoreCase("youtu.be") || host.equalsIgnoreCase("hooktube.com"); + } + public static long parseDurationString(String input) throws ParsingException, NumberFormatException { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java index 9954634fc..91944122d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java @@ -1,16 +1,15 @@ package org.schabi.newpipe.extractor.services.youtube.linkHandler; - -import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.utils.Parser; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; +import org.schabi.newpipe.extractor.utils.Utils; +import java.net.URL; import java.util.List; public class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory { private static final YoutubePlaylistLinkHandlerFactory instance = new YoutubePlaylistLinkHandlerFactory(); - private static final String ID_PATTERN = "([\\-a-zA-Z0-9_]{10,})"; public static YoutubePlaylistLinkHandlerFactory getInstance() { return instance; @@ -24,17 +23,35 @@ public class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory { @Override public String getId(String url) throws ParsingException { try { - return Parser.matchGroup1("list=" + ID_PATTERN, url); + URL urlObj = new URL(url); + + if (!YoutubeParsingHelper.isYoutubeURL(urlObj)) { + throw new ParsingException("the url given is not a Youtube-URL"); + } + + String listID = Utils.getQueryValue(urlObj, "list"); + + if (listID == null) { + throw new ParsingException("the url given does not include a playlist"); + } + + if (!listID.matches("[a-zA-Z0-9_-]{10,}")) { + throw new ParsingException("the list-ID given in the URL does not match the list pattern"); + } + + return listID; } catch (final Exception exception) { throw new ParsingException("Error could not parse url :" + exception.getMessage(), exception); } } - @Override public boolean onAcceptUrl(final String url) { - final boolean hasNotEmptyUrl = url != null && !url.isEmpty(); - final boolean isYoutubeDomain = hasNotEmptyUrl && (url.contains("youtube") || url.contains("youtu.be")); - return isYoutubeDomain && url.contains("list="); + try { + getId(url); + } catch (ParsingException e) { + return false; + } + return true; } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java index 5d07779cf..e479e74c9 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java @@ -1,21 +1,14 @@ package org.schabi.newpipe.extractor.services.youtube.linkHandler; -import org.jsoup.Jsoup; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.schabi.newpipe.extractor.Downloader; -import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; import org.schabi.newpipe.extractor.exceptions.FoundAdException; import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; -import org.schabi.newpipe.extractor.utils.Parser; +import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; +import org.schabi.newpipe.extractor.utils.Utils; -import java.io.IOException; -import java.io.UnsupportedEncodingException; +import java.net.MalformedURLException; import java.net.URI; import java.net.URISyntaxException; -import java.net.URLDecoder; +import java.net.URL; /* * Created by Christian Schabesberger on 02.02.16. @@ -40,7 +33,6 @@ import java.net.URLDecoder; public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory { private static final YoutubeStreamLinkHandlerFactory instance = new YoutubeStreamLinkHandlerFactory(); - private static final String ID_PATTERN = "([\\-a-zA-Z0-9_]{11})"; private YoutubeStreamLinkHandlerFactory() { } @@ -49,78 +41,138 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory { return instance; } + private static String assertIsID(String id) throws ParsingException { + if (id == null || !id.matches("[a-zA-Z0-9_-]{11}")) { + throw new ParsingException("The given string is not a Youtube-Video-ID"); + } + + return id; + } + @Override public String getUrl(String id) { return "https://www.youtube.com/watch?v=" + id; } @Override - public String getId(String url) throws ParsingException, IllegalArgumentException { - if (url.isEmpty()) { - throw new IllegalArgumentException("The url parameter should not be empty"); - } + public String getId(String urlString) throws ParsingException, IllegalArgumentException { + try { + URI uri = new URI(urlString); - String lowercaseUrl = url.toLowerCase(); - if (lowercaseUrl.contains("youtube")) { - if (lowercaseUrl.contains("list=")) { - throw new ParsingException("Error no suitable url: " + url); - } - if (url.contains("attribution_link")) { - try { - String escapedQuery = Parser.matchGroup1("u=(.[^&|$]*)", url); - String query = URLDecoder.decode(escapedQuery, "UTF-8"); - return Parser.matchGroup1("v=" + ID_PATTERN, query); - } catch (UnsupportedEncodingException uee) { - throw new ParsingException("Could not parse attribution_link", uee); + if (uri.getScheme().equals("vnd.youtube")) { + String scheme = uri.getSchemeSpecificPart(); + if (scheme.startsWith("//")) { + urlString = "https:" + scheme; + } else { + return assertIsID(scheme); } } - if (url.contains("vnd.youtube")) { - return Parser.matchGroup1(ID_PATTERN, url); - } - if (url.contains("embed")) { - return Parser.matchGroup1("embed/" + ID_PATTERN, url); - } - if (url.contains("googleads")) { - throw new FoundAdException("Error found add: " + url); - } - return Parser.matchGroup1("[?&]v=" + ID_PATTERN, url); + } catch (URISyntaxException ignored) { } - if (lowercaseUrl.contains("youtu.be")) { - if (lowercaseUrl.contains("list=")) { - throw new ParsingException("Error no suitable url: " + url); - } - if (url.contains("v=")) { - return Parser.matchGroup1("v=" + ID_PATTERN, url); - } - return Parser.matchGroup1("[Yy][Oo][Uu][Tt][Uu]\\.[Bb][Ee]/" + ID_PATTERN, url); + + URL url; + try { + url = new URL(urlString); + } catch (MalformedURLException e) { + throw new IllegalArgumentException("The given URL is not valid"); } - if (lowercaseUrl.contains("hooktube")) { - if (lowercaseUrl.contains("&v=") - || lowercaseUrl.contains("?v=")) { - return Parser.matchGroup1("[?&]v=" + ID_PATTERN, url); + + String host = url.getHost(); + String path = url.getPath(); + // remove leading "/" of URL-path if URL-path is given + if (!path.isEmpty()) { + path = path.substring(1); + } + + if (!YoutubeParsingHelper.isYoutubeALikeURL(url)) { + if (host.equalsIgnoreCase("googleads.g.doubleclick.net")) { + throw new FoundAdException("Error found ad: " + urlString); } - if (url.contains("/embed/")) { - return Parser.matchGroup1("embed/" + ID_PATTERN, url); + + throw new ParsingException("The url is not a Youtube-URL"); + } + + if (YoutubePlaylistLinkHandlerFactory.getInstance().acceptUrl(urlString)) { + throw new ParsingException("Error no suitable url: " + urlString); + } + + // using uppercase instead of lowercase, because toLowercase replaces some unicode characters + // with their lowercase ASCII equivalent. Using toLowercase could result in faultily matching unicode urls. + switch (host.toUpperCase()) { + case "WWW.YOUTUBE-NOCOOKIE.COM": { + if (path.startsWith("embed/")) { + String id = path.split("/")[1]; + + return assertIsID(id); + } } - if (url.contains("/v/")) { - return Parser.matchGroup1("v/" + ID_PATTERN, url); + + case "YOUTUBE.COM": + case "WWW.YOUTUBE.COM": + case "M.YOUTUBE.COM": { + if (path.equals("attribution_link")) { + String uQueryValue = Utils.getQueryValue(url, "u"); + + URL decodedURL; + try { + decodedURL = new URL("http://www.youtube.com" + uQueryValue); + } catch (MalformedURLException e) { + throw new ParsingException("Error no suitable url: " + urlString); + } + + String viewQueryValue = Utils.getQueryValue(decodedURL, "v"); + return assertIsID(viewQueryValue); + } + + if (path.startsWith("embed/")) { + String id = path.split("/")[1]; + + return assertIsID(id); + } + + String viewQueryValue = Utils.getQueryValue(url, "v"); + return assertIsID(viewQueryValue); } - if (url.contains("/watch/")) { - return Parser.matchGroup1("watch/" + ID_PATTERN, url); + + case "YOUTU.BE": { + String viewQueryValue = Utils.getQueryValue(url, "v"); + if (viewQueryValue != null) { + return assertIsID(viewQueryValue); + } + + return assertIsID(path); + } + + case "HOOKTUBE.COM": { + if (path.equals("watch")) { + String viewQueryValue = Utils.getQueryValue(url, "v"); + if (viewQueryValue != null) { + return assertIsID(viewQueryValue); + } + } + if (path.startsWith("embed/")) { + String id = path.substring("embed/".length()); + + return assertIsID(id); + } + if (path.startsWith("v/")) { + String id = path.substring("v/".length()); + + return assertIsID(id); + } + if (path.startsWith("watch/")) { + String id = path.substring("watch/".length()); + + return assertIsID(id); + } } } - throw new ParsingException("Error no suitable url: " + url); + + throw new ParsingException("Error no suitable url: " + urlString); } @Override public boolean onAcceptUrl(final String url) throws FoundAdException { - final String lowercaseUrl = url.toLowerCase(); - if (!lowercaseUrl.contains("youtube") && - !lowercaseUrl.contains("youtu.be") && - !lowercaseUrl.contains("hooktube")) { - return false; - // bad programming I know <-- nice meme - } try { getId(url); return true; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeTrendingLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeTrendingLinkHandlerFactory.java index e61693b08..123da9d1a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeTrendingLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeTrendingLinkHandlerFactory.java @@ -21,8 +21,9 @@ package org.schabi.newpipe.extractor.services.youtube.linkHandler; */ import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; -import org.schabi.newpipe.extractor.utils.Parser; +import java.net.MalformedURLException; +import java.net.URL; import java.util.List; public class YoutubeTrendingLinkHandlerFactory extends ListLinkHandlerFactory { @@ -38,6 +39,14 @@ public class YoutubeTrendingLinkHandlerFactory extends ListLinkHandlerFactory { @Override public boolean onAcceptUrl(final String url) { - return Parser.isMatch("^(https://|http://|)(www.|m.|)youtube.com/feed/trending(|\\?.*)$", url); + URL urlObj; + try { + urlObj = new URL(url); + } catch (MalformedURLException e) { + return false; + } + + String urlPath = urlObj.getPath(); + return YoutubeParsingHelper.isYoutubeURL(urlObj) && urlPath.equals("/feed/trending"); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java index 663fd093b..65f3ce24c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java @@ -2,6 +2,9 @@ package org.schabi.newpipe.extractor.utils; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import java.io.UnsupportedEncodingException; +import java.net.URL; +import java.net.URLDecoder; import java.util.List; public class Utils { @@ -57,5 +60,43 @@ public class Utils { } return url; } -} + /** + * get the value of a URL-query by name. + * if a url-query is give multiple times, only the value of the first query is returned + * + * @param url the url to be used + * @param parameterName the pattern that will be used to check the url + * @return a string that contains the value of the query parameter or null if nothing was found + */ + public static String getQueryValue(URL url, String parameterName) { + String urlQuery = url.getQuery(); + + if (urlQuery != null) { + for (String param : urlQuery.split("&")) { + String[] params = param.split("=", 2); + + String query; + try { + query = URLDecoder.decode(params[0], "UTF-8"); + } catch (UnsupportedEncodingException e) { + System.err.println("Cannot decode string with UTF-8. using the string without decoding"); + e.printStackTrace(); + query = params[0]; + } + + if (query.equals(parameterName)) { + try { + return URLDecoder.decode(params[1], "UTF-8"); + } catch (UnsupportedEncodingException e) { + System.err.println("Cannot decode string with UTF-8. using the string without decoding"); + e.printStackTrace(); + return params[1]; + } + } + } + } + + return null; + } +} \ No newline at end of file diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java index 519eb0efb..f06ad319d 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java @@ -60,9 +60,9 @@ public class YoutubeStreamLinkHandlerFactoryTest { public void getIdfromYt() throws Exception { assertEquals("jZViOEv90dI", linkHandler.fromUrl("https://www.youtube.com/watch?v=jZViOEv90dI").getId()); assertEquals("W-fFHeTX70Q", linkHandler.fromUrl("https://www.youtube.com/watch?v=W-fFHeTX70Q").getId()); - assertEquals("jZViOEv90dI", linkHandler.fromUrl("https://www.youtube.com/watch?v=jZViOEv90dI?t=100").getId()); - assertEquals("jZViOEv90dI", linkHandler.fromUrl("https://WWW.YouTube.com/watch?v=jZViOEv90dI?t=100").getId()); - assertEquals("jZViOEv90dI", linkHandler.fromUrl("HTTPS://www.youtube.com/watch?v=jZViOEv90dI?t=100").getId()); + assertEquals("jZViOEv90dI", linkHandler.fromUrl("https://www.youtube.com/watch?v=jZViOEv90dI&t=100").getId()); + assertEquals("jZViOEv90dI", linkHandler.fromUrl("https://WWW.YouTube.com/watch?v=jZViOEv90dI&t=100").getId()); + assertEquals("jZViOEv90dI", linkHandler.fromUrl("HTTPS://www.youtube.com/watch?v=jZViOEv90dI&t=100").getId()); assertEquals("jZViOEv90dI", linkHandler.fromUrl("https://youtu.be/jZViOEv90dI?t=9s").getId()); assertEquals("jZViOEv90dI", linkHandler.fromUrl("HTTPS://Youtu.be/jZViOEv90dI?t=9s").getId()); assertEquals("uEJuoEs1UxY", linkHandler.fromUrl("http://www.youtube.com/watch_popup?v=uEJuoEs1UxY").getId()); @@ -85,9 +85,9 @@ public class YoutubeStreamLinkHandlerFactoryTest { @Test public void testAcceptYtUrl() throws ParsingException { assertTrue(linkHandler.acceptUrl("https://www.youtube.com/watch?v=jZViOEv90dI")); - assertTrue(linkHandler.acceptUrl("https://www.youtube.com/watch?v=jZViOEv90dI?t=100")); - assertTrue(linkHandler.acceptUrl("https://WWW.YouTube.com/watch?v=jZViOEv90dI?t=100")); - assertTrue(linkHandler.acceptUrl("HTTPS://www.youtube.com/watch?v=jZViOEv90dI?t=100")); + assertTrue(linkHandler.acceptUrl("https://www.youtube.com/watch?v=jZViOEv90dI&t=100")); + assertTrue(linkHandler.acceptUrl("https://WWW.YouTube.com/watch?v=jZViOEv90dI&t=100")); + assertTrue(linkHandler.acceptUrl("HTTPS://www.youtube.com/watch?v=jZViOEv90dI&t=100")); assertTrue(linkHandler.acceptUrl("https://youtu.be/jZViOEv90dI?t=9s")); assertTrue(linkHandler.acceptUrl("https://www.youtube.com/embed/jZViOEv90dI")); assertTrue(linkHandler.acceptUrl("https://www.youtube-nocookie.com/embed/jZViOEv90dI")); From 50c4783f71ecf4d8841289d0ef96f2f3e5eca0f7 Mon Sep 17 00:00:00 2001 From: Ritvik Saraf <13ritvik@gmail.com> Date: Sat, 19 Jan 2019 17:31:21 +0530 Subject: [PATCH 02/24] fixed extracting hls url --- .../extractors/YoutubeStreamExtractor.java | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 4801fa461..3b614fc36 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -5,6 +5,7 @@ import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; import org.jsoup.Jsoup; +import org.jsoup.helper.StringUtil; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; @@ -386,13 +387,18 @@ public class YoutubeStreamExtractor extends StreamExtractor { public String getHlsUrl() throws ParsingException { assertPageFetched(); try { - String hlsvp; - if (playerArgs != null && playerArgs.isString("hlsvp")) { - hlsvp = playerArgs.getString("hlsvp", ""); - } else { - return ""; + String hlsvp = ""; + if (playerArgs != null) { + if( playerArgs.isString("hlsvp") ) { + hlsvp = playerArgs.getString("hlsvp", ""); + }else { + hlsvp = JsonParser.object() + .from(playerArgs.getString("player_response", "{}")) + .getObject("streamingData", new JsonObject()) + .getString("hlsManifestUrl", ""); + } } - + return hlsvp; } catch (Exception e) { throw new ParsingException("Could not get hls manifest url", e); From c60d6ec0abb0e8fee9ec1f93398d3a69bf643a09 Mon Sep 17 00:00:00 2001 From: Ritvik Saraf <13ritvik@gmail.com> Date: Sat, 19 Jan 2019 18:20:02 +0530 Subject: [PATCH 03/24] removed unused import --- .../services/youtube/extractors/YoutubeStreamExtractor.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 3b614fc36..4b21a06ae 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -5,7 +5,6 @@ import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; import org.jsoup.Jsoup; -import org.jsoup.helper.StringUtil; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; @@ -398,7 +397,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { .getString("hlsManifestUrl", ""); } } - + return hlsvp; } catch (Exception e) { throw new ParsingException("Could not get hls manifest url", e); From a6c972eff8ae69d85c1dd74cb43fdcf59dd142c0 Mon Sep 17 00:00:00 2001 From: Connectety-W Date: Sun, 20 Jan 2019 01:31:30 +0100 Subject: [PATCH 04/24] fixed tests by prepending HTTP to URLs without protocol and adding a check for null. --- .../YoutubeChannelLinkHandlerFactory.java | 3 ++- .../YoutubePlaylistLinkHandlerFactory.java | 2 +- .../YoutubeStreamLinkHandlerFactory.java | 15 +++++------ .../YoutubeTrendingLinkHandlerFactory.java | 3 ++- .../schabi/newpipe/extractor/utils/Utils.java | 25 +++++++++++++++++-- 5 files changed, 36 insertions(+), 12 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java index da207b278..e3522b313 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java @@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor.services.youtube.linkHandler; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; +import org.schabi.newpipe.extractor.utils.Utils; import java.net.URL; import java.util.List; @@ -42,7 +43,7 @@ public class YoutubeChannelLinkHandlerFactory extends ListLinkHandlerFactory { @Override public String getId(String url) throws ParsingException { try { - URL urlObj = new URL(url); + URL urlObj = Utils.stringToURL(url); String path = urlObj.getPath(); if (!(YoutubeParsingHelper.isYoutubeURL(urlObj) || urlObj.getHost().equalsIgnoreCase("hooktube.com"))) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java index 91944122d..008aeb933 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java @@ -23,7 +23,7 @@ public class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory { @Override public String getId(String url) throws ParsingException { try { - URL urlObj = new URL(url); + URL urlObj = Utils.stringToURL(url); if (!YoutubeParsingHelper.isYoutubeURL(urlObj)) { throw new ParsingException("the url given is not a Youtube-URL"); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java index e479e74c9..3e793960f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java @@ -58,13 +58,14 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory { public String getId(String urlString) throws ParsingException, IllegalArgumentException { try { URI uri = new URI(urlString); + String scheme = uri.getScheme(); - if (uri.getScheme().equals("vnd.youtube")) { - String scheme = uri.getSchemeSpecificPart(); - if (scheme.startsWith("//")) { - urlString = "https:" + scheme; + if (scheme != null && scheme.equals("vnd.youtube")) { + String schemeSpecificPart = uri.getSchemeSpecificPart(); + if (schemeSpecificPart.startsWith("//")) { + urlString = "https:" + schemeSpecificPart; } else { - return assertIsID(scheme); + return assertIsID(schemeSpecificPart); } } } catch (URISyntaxException ignored) { @@ -72,7 +73,7 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory { URL url; try { - url = new URL(urlString); + url = Utils.stringToURL(urlString); } catch (MalformedURLException e) { throw new IllegalArgumentException("The given URL is not valid"); } @@ -115,7 +116,7 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory { URL decodedURL; try { - decodedURL = new URL("http://www.youtube.com" + uQueryValue); + decodedURL = Utils.stringToURL("http://www.youtube.com" + uQueryValue); } catch (MalformedURLException e) { throw new ParsingException("Error no suitable url: " + urlString); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeTrendingLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeTrendingLinkHandlerFactory.java index 123da9d1a..253e9cd8a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeTrendingLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeTrendingLinkHandlerFactory.java @@ -21,6 +21,7 @@ package org.schabi.newpipe.extractor.services.youtube.linkHandler; */ import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; +import org.schabi.newpipe.extractor.utils.Utils; import java.net.MalformedURLException; import java.net.URL; @@ -41,7 +42,7 @@ public class YoutubeTrendingLinkHandlerFactory extends ListLinkHandlerFactory { public boolean onAcceptUrl(final String url) { URL urlObj; try { - urlObj = new URL(url); + urlObj = Utils.stringToURL(url); } catch (MalformedURLException e) { return false; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java index 65f3ce24c..d4b8db432 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java @@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor.utils; import org.schabi.newpipe.extractor.exceptions.ParsingException; import java.io.UnsupportedEncodingException; +import java.net.MalformedURLException; import java.net.URL; import java.net.URLDecoder; import java.util.List; @@ -43,7 +44,7 @@ public class Utils { } public static void printErrors(List errors) { - for(Throwable e : errors) { + for (Throwable e : errors) { e.printStackTrace(); System.err.println("----------------"); } @@ -55,7 +56,7 @@ public class Utils { public static String replaceHttpWithHttps(final String url) { if (url == null) return null; - if(!url.isEmpty() && url.startsWith(HTTP)) { + if (!url.isEmpty() && url.startsWith(HTTP)) { return HTTPS + url.substring(HTTP.length()); } return url; @@ -99,4 +100,24 @@ public class Utils { return null; } + + /** + * converts a string to a URL-Object. + * defaults to HTTP if no protocol is given + * + * @param url the string to be converted to a URL-Object + * @return a URL-Object containing the url + */ + public static URL stringToURL(String url) throws MalformedURLException { + try { + return new URL(url); + } catch (MalformedURLException e) { + // if no protocol is given try prepending "http://" + if (e.getMessage().equals("no protocol: " + url)) { + return new URL(HTTP + url); + } + + throw e; + } + } } \ No newline at end of file From 19288c1456d667644e3d0ec12bf15d3d21b88447 Mon Sep 17 00:00:00 2001 From: Connectety-W Date: Sun, 20 Jan 2019 14:39:06 +0100 Subject: [PATCH 05/24] added forgotten break statements --- .../youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java index 3e793960f..d9d9e93a0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java @@ -106,6 +106,8 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory { return assertIsID(id); } + + break; } case "YOUTUBE.COM": @@ -167,6 +169,8 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory { return assertIsID(id); } } + + break; } throw new ParsingException("Error no suitable url: " + urlString); From ef576341c8ac9f3c7186565e8182bb9df4afd328 Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Sun, 23 Dec 2018 14:59:26 +0100 Subject: [PATCH 06/24] implement searchextractor for mediaccc --- extractor/.attach_pid31246 | 0 .../schabi/newpipe/extractor/ServiceList.java | 5 +- .../services/media_ccc/MediaCCCService.java | 88 +++++++++++++++++++ .../extractors/MediaCCCSearchExtractor.java | 67 ++++++++++++++ .../MediaCCCStreamInfoItemExtractor.java | 67 ++++++++++++++ .../MediaCCCSearchQueryHandlerFactory.java | 30 +++++++ .../MediaCCCSearchExtractorTest.java | 36 ++++++++ 7 files changed, 292 insertions(+), 1 deletion(-) create mode 100644 extractor/.attach_pid31246 create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCStreamInfoItemExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCSearchQueryHandlerFactory.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorTest.java diff --git a/extractor/.attach_pid31246 b/extractor/.attach_pid31246 new file mode 100644 index 000000000..e69de29bb diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java b/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java index 36690438f..a171db71d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java @@ -1,5 +1,6 @@ package org.schabi.newpipe.extractor; +import org.schabi.newpipe.extractor.services.media_ccc.MediaCCCService; import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudService; import org.schabi.newpipe.extractor.services.youtube.YoutubeService; @@ -36,6 +37,7 @@ public final class ServiceList { public static final YoutubeService YouTube; public static final SoundcloudService SoundCloud; + public static final MediaCCCService MediaCCC; /** * When creating a new service, put this service in the end of this list, @@ -44,7 +46,8 @@ public final class ServiceList { private static final List SERVICES = unmodifiableList( asList( YouTube = new YoutubeService(0), - SoundCloud = new SoundcloudService(1) + SoundCloud = new SoundcloudService(1), + MediaCCC = new MediaCCCService(2) )); /** diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java new file mode 100644 index 000000000..1a78110b5 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java @@ -0,0 +1,88 @@ +package org.schabi.newpipe.extractor.services.media_ccc; + +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.SuggestionExtractor; +import org.schabi.newpipe.extractor.channel.ChannelExtractor; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.kiosk.KioskList; +import org.schabi.newpipe.extractor.linkhandler.*; +import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; +import org.schabi.newpipe.extractor.search.SearchExtractor; +import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCSearchExtractor; +import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCSearchQueryHandlerFactory; +import org.schabi.newpipe.extractor.stream.StreamExtractor; +import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; +import org.schabi.newpipe.extractor.utils.Localization; + +import static java.util.Arrays.asList; +import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.*; + +public class MediaCCCService extends StreamingService { + public MediaCCCService(int id) { + super(id, "Media.CCC", asList(AUDIO, VIDEO)); + } + + @Override + public SearchExtractor getSearchExtractor(SearchQueryHandler query, Localization localization) { + return new MediaCCCSearchExtractor(this, query, localization); + } + + @Override + public LinkHandlerFactory getStreamLHFactory() { + return null; + } + + @Override + public ListLinkHandlerFactory getChannelLHFactory() { + return null; + } + + @Override + public ListLinkHandlerFactory getPlaylistLHFactory() { + return null; + } + + @Override + public SearchQueryHandlerFactory getSearchQHFactory() { + return new MediaCCCSearchQueryHandlerFactory(); + } + + @Override + public StreamExtractor getStreamExtractor(LinkHandler linkHandler, Localization localization) { + return null; + } + + @Override + public ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler, Localization localization) { + return null; + } + + @Override + public PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler, Localization localization) { + return null; + } + + @Override + public SuggestionExtractor getSuggestionExtractor(Localization localization) { + return null; + } + + @Override + public KioskList getKioskList() throws ExtractionException { + KioskList list = new KioskList(getServiceId()); + + // add kiosks here e.g.: + try { + // Add kiosk here + } catch (Exception e) { + throw new ExtractionException(e); + } + + return list; + } + + @Override + public SubscriptionExtractor getSubscriptionExtractor() { + return null; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java new file mode 100644 index 000000000..2eb0dee7f --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java @@ -0,0 +1,67 @@ +package org.schabi.newpipe.extractor.services.media_ccc.extractors; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; +import org.schabi.newpipe.extractor.Downloader; +import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.InfoItemsCollector; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; +import org.schabi.newpipe.extractor.search.SearchExtractor; +import org.schabi.newpipe.extractor.services.media_ccc.extractors.infoItems.MediaCCCStreamInfoItemExtractor; +import org.schabi.newpipe.extractor.utils.Localization; + +import javax.annotation.Nonnull; +import java.io.IOException; + +public class MediaCCCSearchExtractor extends SearchExtractor { + + JsonObject doc; + + public MediaCCCSearchExtractor(StreamingService service, SearchQueryHandler linkHandler, Localization localization) { + super(service, linkHandler, localization); + } + + @Override + public String getSearchSuggestion() throws ParsingException { + return null; + } + + @Nonnull + @Override + public InfoItemsPage getInitialPage() throws IOException, ExtractionException { + InfoItemsCollector searchItems = getInfoItemSearchCollector(); + JsonArray events = doc.getArray("events"); + for(int i = 0; i < events.size(); i++) { + searchItems.commit(new MediaCCCStreamInfoItemExtractor( + events.getObject(i))); + } + return new InfoItemsPage<>(searchItems, null); + } + + @Override + public String getNextPageUrl() throws IOException, ExtractionException { + return null; + } + + @Override + public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { + return null; + } + + @Override + public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { + final String site; + final String url = getUrl(); + site = downloader.download(url, getLocalization()); + try { + doc = JsonParser.object().from(site); + } catch (JsonParserException jpe) { + throw new ExtractionException("Could not parse json.", jpe); + } + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCStreamInfoItemExtractor.java new file mode 100644 index 000000000..adb57e886 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCStreamInfoItemExtractor.java @@ -0,0 +1,67 @@ +package org.schabi.newpipe.extractor.services.media_ccc.extractors.infoItems; + +import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; +import org.schabi.newpipe.extractor.stream.StreamType; + +public class MediaCCCStreamInfoItemExtractor implements StreamInfoItemExtractor { + + JsonObject event; + + public MediaCCCStreamInfoItemExtractor(JsonObject event) { + this.event = event; + } + + @Override + public StreamType getStreamType() throws ParsingException { + return StreamType.VIDEO_STREAM; + } + + @Override + public boolean isAd() throws ParsingException { + return false; + } + + @Override + public long getDuration() throws ParsingException { + return event.getInt("length"); + } + + @Override + public long getViewCount() throws ParsingException { + return event.getInt("view_count"); + } + + @Override + public String getUploaderName() throws ParsingException { + return event.getString("conference_url") + .replace("https://api.media.ccc.de/public/conferences/", ""); + } + + @Override + public String getUploaderUrl() throws ParsingException { + return event.getString("conference_url"); + } + + @Override + public String getUploadDate() throws ParsingException { + return event.getString("release_date"); + } + + @Override + public String getName() throws ParsingException { + return event.getString("title"); + } + + @Override + public String getUrl() throws ParsingException { + return "https://api.media.ccc.de/public/events/" + + event.getString("guid"); + } + + @Override + public String getThumbnailUrl() throws ParsingException { + return event.getString("thumbnails_url"); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCSearchQueryHandlerFactory.java new file mode 100644 index 000000000..f94c253e3 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCSearchQueryHandlerFactory.java @@ -0,0 +1,30 @@ +package org.schabi.newpipe.extractor.services.media_ccc.linkHandler; + +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory; + +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.util.List; + +public class MediaCCCSearchQueryHandlerFactory extends SearchQueryHandlerFactory { + + @Override + public String[] getAvailableContentFilter() { + return null; + } + + @Override + public String[] getAvailableSortFilter() { + return null; + } + + @Override + public String getUrl(String querry, List contentFilter, String sortFilter) throws ParsingException { + try { + return "https://api.media.ccc.de/public/events/search?q=" + URLEncoder.encode(querry, "UTF-8"); + } catch (UnsupportedEncodingException e) { + throw new ParsingException("Could not create search string with querry: " + querry, e); + } + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorTest.java new file mode 100644 index 000000000..c88c0bca5 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorTest.java @@ -0,0 +1,36 @@ +package org.schabi.newpipe.extractor.services.media_ccc; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.ListExtractor; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.search.SearchExtractor; +import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCSearchExtractor; +import org.schabi.newpipe.extractor.utils.Localization; + +import static junit.framework.TestCase.assertTrue; +import static org.schabi.newpipe.extractor.ServiceList.MediaCCC; + +/** + * Test for {@link MediaCCCSearchExtractor} + */ +public class MediaCCCSearchExtractorTest { + private static SearchExtractor extractor; + private static ListExtractor.InfoItemsPage itemsPage; + + @BeforeClass + public static void setUpClass() throws Exception { + NewPipe.init(Downloader.getInstance(), new Localization("GB", "en")); + extractor = MediaCCC.getSearchExtractor("source"); + extractor.fetchPage(); + itemsPage = extractor.getInitialPage(); + } + + @Test + public void testCount() throws Exception { + assertTrue(Integer.toString(itemsPage.getItems().size()), + itemsPage.getItems().size() >= 25); + } +} From 2148edc7d7b199d4910ba83c429b9c0882bd7739 Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Sun, 23 Dec 2018 19:41:18 +0100 Subject: [PATCH 07/24] add channel and strem extractor for mediaccc --- .../extractor/channel/ChannelExtractor.java | 1 - .../extractor/channel/ChannelInfo.java | 5 - .../services/media_ccc/MediaCCCService.java | 12 +- .../MediaCCCConferenceExtractor.java | 94 ++++++++ .../extractors/MediaCCCStreamExtractor.java | 222 ++++++++++++++++++ .../MediaCCCSuggestionExtractor.java | 21 ++ .../MediaCCCStreamInfoItemExtractor.java | 2 +- .../MediaCCCConferenceLinkHandlerFactory.java | 32 +++ .../MediaCCCSearchQueryHandlerFactory.java | 4 +- .../MediaCCCStreamLinkHandlerFactory.java | 27 +++ .../SoundcloudChannelExtractor.java | 5 - .../extractors/YoutubeChannelExtractor.java | 23 -- .../MediaCCCConferenceExtractorTest.java | 51 ++++ .../MediaCCCSearchExtractorTest.java | 25 ++ .../MediaCCCStreamExtractorTest.java | 83 +++++++ .../youtube/YoutubeChannelExtractorTest.java | 23 -- .../YoutubeSearchExtractorDefaultTest.java | 14 +- 17 files changed, 575 insertions(+), 69 deletions(-) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSuggestionExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferenceLinkHandlerFactory.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCStreamLinkHandlerFactory.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceExtractorTest.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java index 5b74826e6..5172e0977 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java @@ -38,5 +38,4 @@ public abstract class ChannelExtractor extends ListExtractor { public abstract String getFeedUrl() throws ParsingException; public abstract long getSubscriberCount() throws ParsingException; public abstract String getDescription() throws ParsingException; - public abstract String[] getDonationLinks() throws ParsingException; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java index 20abdfe42..a848725f6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java @@ -92,11 +92,6 @@ public class ChannelInfo extends ListInfo { } catch (Exception e) { info.addError(e); } - try { - info.setDonationLinks(extractor.getDonationLinks()); - } catch (Exception e) { - info.addError(e); - } return info; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java index 1a78110b5..af8b21a26 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java @@ -8,8 +8,12 @@ import org.schabi.newpipe.extractor.kiosk.KioskList; import org.schabi.newpipe.extractor.linkhandler.*; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.search.SearchExtractor; +import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCConferenceExtractor; import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCSearchExtractor; +import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCStreamExtractor; +import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCConferenceLinkHandlerFactory; import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCSearchQueryHandlerFactory; +import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCStreamLinkHandlerFactory; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; import org.schabi.newpipe.extractor.utils.Localization; @@ -29,12 +33,12 @@ public class MediaCCCService extends StreamingService { @Override public LinkHandlerFactory getStreamLHFactory() { - return null; + return new MediaCCCStreamLinkHandlerFactory(); } @Override public ListLinkHandlerFactory getChannelLHFactory() { - return null; + return new MediaCCCConferenceLinkHandlerFactory(); } @Override @@ -49,12 +53,12 @@ public class MediaCCCService extends StreamingService { @Override public StreamExtractor getStreamExtractor(LinkHandler linkHandler, Localization localization) { - return null; + return new MediaCCCStreamExtractor(this, linkHandler, localization); } @Override public ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler, Localization localization) { - return null; + return new MediaCCCConferenceExtractor(this, linkHandler, localization); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceExtractor.java new file mode 100644 index 000000000..faa4b444c --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceExtractor.java @@ -0,0 +1,94 @@ +package org.schabi.newpipe.extractor.services.media_ccc.extractors; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; +import org.schabi.newpipe.extractor.Downloader; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.channel.ChannelExtractor; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.services.media_ccc.extractors.infoItems.MediaCCCStreamInfoItemExtractor; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; +import org.schabi.newpipe.extractor.utils.Localization; + +import javax.annotation.Nonnull; +import java.io.IOException; + +public class MediaCCCConferenceExtractor extends ChannelExtractor { + + private JsonObject conferenceData; + + public MediaCCCConferenceExtractor(StreamingService service, ListLinkHandler linkHandler, Localization localization) { + super(service, linkHandler, localization); + } + + @Override + public String getAvatarUrl() throws ParsingException { + return conferenceData.getString("logo_url"); + } + + @Override + public String getBannerUrl() throws ParsingException { + return conferenceData.getString("logo_url"); + } + + @Override + public String getFeedUrl() throws ParsingException { + return null; + } + + @Override + public long getSubscriberCount() throws ParsingException { + return -1; + } + + @Override + public String getDescription() throws ParsingException { + return null; + } + + @Nonnull + @Override + public InfoItemsPage getInitialPage() throws IOException, ExtractionException { + StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + JsonArray events = conferenceData.getArray("events"); + for(int i = 0; i < events.size(); i++) { + collector.commit(new MediaCCCStreamInfoItemExtractor(events.getObject(i))); + } + return new InfoItemsPage<>(collector, null); + } + + @Override + public String getNextPageUrl() throws IOException, ExtractionException { + return null; + } + + @Override + public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { + return null; + } + + @Override + public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { + try { + conferenceData = JsonParser.object().from(downloader.download(getUrl())); + } catch (JsonParserException jpe) { + throw new ExtractionException("Could not parse json returnd by url: " + getUrl()); + } + } + + @Nonnull + @Override + public String getName() throws ParsingException { + return conferenceData.getString("title"); + } + + @Override + public String getOriginalUrl() throws ParsingException { + return "https://media.ccc.de/c/" + conferenceData.getString("acronym"); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java new file mode 100644 index 000000000..6c0e0fad8 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java @@ -0,0 +1,222 @@ +package org.schabi.newpipe.extractor.services.media_ccc.extractors; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; +import org.schabi.newpipe.extractor.Downloader; +import org.schabi.newpipe.extractor.MediaFormat; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.LinkHandler; +import org.schabi.newpipe.extractor.stream.*; +import org.schabi.newpipe.extractor.utils.Localization; +import org.schabi.newpipe.extractor.utils.Parser; + +import javax.annotation.Nonnull; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class MediaCCCStreamExtractor extends StreamExtractor { + + private JsonObject data; + private JsonObject conferenceData; + + public MediaCCCStreamExtractor(StreamingService service, LinkHandler linkHandler, Localization localization) { + super(service, linkHandler, localization); + } + + @Nonnull + @Override + public String getUploadDate() throws ParsingException { + return data.getString("release_date"); + } + + @Nonnull + @Override + public String getThumbnailUrl() throws ParsingException { + return data.getString("thumb_url"); + } + + @Nonnull + @Override + public String getDescription() throws ParsingException { + return data.getString("description"); + } + + @Override + public int getAgeLimit() throws ParsingException { + return 0; + } + + @Override + public long getLength() throws ParsingException { + return data.getInt("length"); + } + + @Override + public long getTimeStamp() throws ParsingException { + return 0; + } + + @Override + public long getViewCount() throws ParsingException { + return data.getInt("view_count"); + } + + @Override + public long getLikeCount() throws ParsingException { + return -1; + } + + @Override + public long getDislikeCount() throws ParsingException { + return -1; + } + + @Nonnull + @Override + public String getUploaderUrl() throws ParsingException { + return data.getString("conference_url"); + } + + @Nonnull + @Override + public String getUploaderName() throws ParsingException { + return data.getString("conference_url") + .replace("https://api.media.ccc.de/public/conferences/", ""); + } + + @Nonnull + @Override + public String getUploaderAvatarUrl() throws ParsingException { + return conferenceData.getString("logo_url"); + } + + @Nonnull + @Override + public String getDashMpdUrl() throws ParsingException { + return null; + } + + @Nonnull + @Override + public String getHlsUrl() throws ParsingException { + return null; + } + + @Override + public List getAudioStreams() throws IOException, ExtractionException { + final JsonArray recordings = data.getArray("recordings"); + final List audioStreams = new ArrayList<>(); + for(int i = 0; i < recordings.size(); i++) { + final JsonObject recording = recordings.getObject(i); + final String mimeType = recording.getString("mime_type"); + if(mimeType.startsWith("audio")) { + //first we need to resolve the actual video data from CDN + final MediaFormat mediaFormat; + if(mimeType.endsWith("opus")) { + mediaFormat = MediaFormat.OPUS; + } else if(mimeType.endsWith("mpeg")) { + mediaFormat = MediaFormat.MP3; + } else { + throw new ExtractionException("Unknown media format: " + mimeType); + } + + audioStreams.add(new AudioStream(recording.getString("recording_url"), mediaFormat, -1)); + } + } + return audioStreams; + } + + @Override + public List getVideoStreams() throws IOException, ExtractionException { + final JsonArray recordings = data.getArray("recordings"); + final List videoStreams = new ArrayList<>(); + for(int i = 0; i < recordings.size(); i++) { + final JsonObject recording = recordings.getObject(i); + final String mimeType = recording.getString("mime_type"); + if(mimeType.startsWith("video")) { + //first we need to resolve the actual video data from CDN + + final MediaFormat mediaFormat; + if(mimeType.endsWith("webm")) { + mediaFormat = MediaFormat.WEBM; + } else if(mimeType.endsWith("mp4")) { + mediaFormat = MediaFormat.MPEG_4; + } else { + throw new ExtractionException("Unknown media format: " + mimeType); + } + + + videoStreams.add(new VideoStream(recording.getString("recording_url"), + mediaFormat, + Integer.toString(recording.getInt("height")))); + } + } + return videoStreams; + } + + @Override + public List getVideoOnlyStreams() throws IOException, ExtractionException { + return null; + } + + @Nonnull + @Override + public List getSubtitlesDefault() throws IOException, ExtractionException { + return null; + } + + @Nonnull + @Override + public List getSubtitles(MediaFormat format) throws IOException, ExtractionException { + return null; + } + + @Override + public StreamType getStreamType() throws ParsingException { + return StreamType.VIDEO_STREAM; + } + + @Override + public StreamInfoItem getNextStream() throws IOException, ExtractionException { + return null; + } + + @Override + public StreamInfoItemsCollector getRelatedStreams() throws IOException, ExtractionException { + return null; + } + + @Override + public String getErrorMessage() { + return null; + } + + @Override + public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { + try { + data = JsonParser.object().from( + downloader.download(getLinkHandler().getUrl())); + conferenceData = JsonParser.object() + .from(downloader.download(getUploaderUrl())); + } catch (JsonParserException jpe) { + throw new ExtractionException("Could not parse json returned by url: " + getLinkHandler().getUrl(), jpe); + } + + } + + @Nonnull + @Override + public String getName() throws ParsingException { + return data.getString("title"); + } + + @Override + public String getOriginalUrl() throws ParsingException { + return data.getString("frontend_link"); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSuggestionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSuggestionExtractor.java new file mode 100644 index 000000000..9c6b4742d --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSuggestionExtractor.java @@ -0,0 +1,21 @@ +package org.schabi.newpipe.extractor.services.media_ccc.extractors; + +import org.schabi.newpipe.extractor.SuggestionExtractor; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.utils.Localization; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class MediaCCCSuggestionExtractor extends SuggestionExtractor { + + public MediaCCCSuggestionExtractor(int serviceId, Localization localization) { + super(serviceId, localization); + } + + @Override + public List suggestionList(String query) throws IOException, ExtractionException { + return new ArrayList<>(0); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCStreamInfoItemExtractor.java index adb57e886..99bd45adb 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCStreamInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCStreamInfoItemExtractor.java @@ -62,6 +62,6 @@ public class MediaCCCStreamInfoItemExtractor implements StreamInfoItemExtractor @Override public String getThumbnailUrl() throws ParsingException { - return event.getString("thumbnails_url"); + return event.getString("thumb_url"); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferenceLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferenceLinkHandlerFactory.java new file mode 100644 index 000000000..b1207a9a2 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferenceLinkHandlerFactory.java @@ -0,0 +1,32 @@ +package org.schabi.newpipe.extractor.services.media_ccc.linkHandler; + +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; +import org.schabi.newpipe.extractor.utils.Parser; + +import java.util.List; + +public class MediaCCCConferenceLinkHandlerFactory extends ListLinkHandlerFactory { + + @Override + public String getUrl(String id, List contentFilter, String sortFilter) throws ParsingException { + return "https://api.media.ccc.de/public/conferences/" + id; + } + + @Override + public String getId(String url) throws ParsingException { + if(url.startsWith("https://api.media.ccc.de/public/conferences/")) { + return url.replace("https://api.media.ccc.de/public/conferences/", ""); + } else if(url.startsWith("https://media.ccc.de/c/")) { + return Parser.matchGroup1("https://media.ccc.de/c/([^?#]*)", url); + } else { + throw new ParsingException("Could not get id from url: " + url); + } + } + + @Override + public boolean onAcceptUrl(String url) throws ParsingException { + return url.startsWith("https://api.media.ccc.de/public/conferences/") + || url.startsWith("https://media.ccc.de/c/"); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCSearchQueryHandlerFactory.java index f94c253e3..b9747dc8f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCSearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCSearchQueryHandlerFactory.java @@ -11,12 +11,12 @@ public class MediaCCCSearchQueryHandlerFactory extends SearchQueryHandlerFactory @Override public String[] getAvailableContentFilter() { - return null; + return new String[0]; } @Override public String[] getAvailableSortFilter() { - return null; + return new String[0]; } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCStreamLinkHandlerFactory.java new file mode 100644 index 000000000..fc6bf39e4 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCStreamLinkHandlerFactory.java @@ -0,0 +1,27 @@ +package org.schabi.newpipe.extractor.services.media_ccc.linkHandler; + +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; + +public class MediaCCCStreamLinkHandlerFactory extends LinkHandlerFactory { + + @Override + public String getId(String url) throws ParsingException { + if(url.startsWith("https://api.media.ccc.de/public/events/") && + !url.contains("?q=")) { + return url.replace("https://api.media.ccc.de/public/events/", ""); + } + throw new ParsingException("Could not get id from url: " + url); + } + + @Override + public String getUrl(String id) throws ParsingException { + return "https://api.media.ccc.de/public/events/" + id; + } + + @Override + public boolean onAcceptUrl(String url) throws ParsingException { + return url.startsWith("https://api.media.ccc.de/public/events/") && + !url.contains("?q="); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java index 888432ef5..1942a04d1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java @@ -127,9 +127,4 @@ public class SoundcloudChannelExtractor extends ChannelExtractor { return new InfoItemsPage<>(collector, nextPageUrl); } - - @Override - public String[] getDonationLinks() { - return new String[0]; - } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index b8a8c7bda..d48b57a6a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -189,29 +189,6 @@ public class YoutubeChannelExtractor extends ChannelExtractor { return new InfoItemsPage<>(collector, getNextPageUrlFromAjaxPage(ajaxJson, pageUrl)); } - @Override - public String[] getDonationLinks() throws ParsingException { - try { - ArrayList links = new ArrayList<>(); - Element linkHolder = doc.select("div[id=\"header-links\"]").first(); - if(linkHolder == null) { - // this occures if no links are embeded into the channel - return new String[0]; - } - for(Element a : linkHolder.select("a")) { - String link = a.attr("abs:href"); - if(DonationLinkHelper.getDonatoinServiceByLink(link) != DonationLinkHelper.DonationService.NO_DONATION) { - links.add(link); - } - } - String[] retLinks = new String[links.size()]; - retLinks = links.toArray(retLinks); - return retLinks; - } catch (Exception e) { - throw new ParsingException("Could not get donation links", e); - } - } - private String getNextPageUrlFromAjaxPage(final JsonObject ajaxJson, final String pageUrl) throws ParsingException { String loadMoreHtmlDataRaw = ajaxJson.getString("load_more_widget_html"); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceExtractorTest.java new file mode 100644 index 000000000..3ea6aaaae --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceExtractorTest.java @@ -0,0 +1,51 @@ +package org.schabi.newpipe.extractor.services.media_ccc; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.channel.ChannelExtractor; +import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCConferenceExtractor; +import org.schabi.newpipe.extractor.utils.Localization; + +import static junit.framework.TestCase.assertEquals; +import static org.schabi.newpipe.extractor.ServiceList.MediaCCC; + +/** + * Test {@link MediaCCCConferenceExtractor} + */ +public class MediaCCCConferenceExtractorTest { + private static ChannelExtractor extractor; + + @BeforeClass + public static void setUpClass() throws Exception { + NewPipe.init(Downloader.getInstance(), new Localization("en", "en_GB")); + extractor = MediaCCC.getChannelExtractor("https://api.media.ccc.de/public/conferences/froscon2017"); + extractor.fetchPage(); + } + + @Test + public void testName() throws Exception { + assertEquals("FrOSCon 2017", extractor.getName()); + } + + @Test + public void testGetUrl() throws Exception { + assertEquals("https://api.media.ccc.de/public/conferences/froscon2017", extractor.getUrl()); + } + + @Test + public void testGetOriginalUrl() throws Exception { + assertEquals("https://media.ccc.de/c/froscon2017", extractor.getOriginalUrl()); + } + + @Test + public void testGetThumbnailUrl() throws Exception { + assertEquals("https://static.media.ccc.de/media/events/froscon/2017/logo.png", extractor.getAvatarUrl()); + } + + @Test + public void testGetInitalPage() throws Exception { + assertEquals(97,extractor.getInitialPage().getItems().size()); + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorTest.java index c88c0bca5..c08777a43 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorTest.java @@ -10,7 +10,9 @@ import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCSearchExtractor; import org.schabi.newpipe.extractor.utils.Localization; +import static junit.framework.TestCase.assertEquals; import static junit.framework.TestCase.assertTrue; +import static org.junit.Assert.assertFalse; import static org.schabi.newpipe.extractor.ServiceList.MediaCCC; /** @@ -33,4 +35,27 @@ public class MediaCCCSearchExtractorTest { assertTrue(Integer.toString(itemsPage.getItems().size()), itemsPage.getItems().size() >= 25); } + + @Test + public void testServiceId() throws Exception { + assertEquals(2, extractor.getServiceId()); + } + + @Test + public void testName() throws Exception { + assertFalse(itemsPage.getItems().get(0).getName(), itemsPage.getItems().get(0).getName().isEmpty()); + } + + @Test + public void testUrl() throws Exception { + assertTrue("Url should start with: https://api.media.ccc.de/public/events/", + itemsPage.getItems().get(0).getUrl().startsWith("https://api.media.ccc.de/public/events/")); + } + + @Test + public void testThumbnailUrl() throws Exception { + assertTrue(itemsPage.getItems().get(0).getThumbnailUrl(), + itemsPage.getItems().get(0).getThumbnailUrl().startsWith("https://static.media.ccc.de/media/") + && itemsPage.getItems().get(0).getThumbnailUrl().endsWith(".jpg")); + } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java new file mode 100644 index 000000000..8aba91535 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java @@ -0,0 +1,83 @@ +package org.schabi.newpipe.extractor.services.media_ccc; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.services.BaseExtractorTest; +import org.schabi.newpipe.extractor.stream.StreamExtractor; +import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCStreamExtractor; +import org.schabi.newpipe.extractor.utils.Localization; + +import static junit.framework.TestCase.assertEquals; +import static org.schabi.newpipe.extractor.ServiceList.MediaCCC; + +/** + * Test {@link MediaCCCStreamExtractor} + */ +public class MediaCCCStreamExtractorTest implements BaseExtractorTest { + private static StreamExtractor extractor; + + @BeforeClass + public static void setUpClass() throws Exception { + NewPipe.init(Downloader.getInstance(), new Localization("GB", "en")); + + extractor = MediaCCC.getStreamExtractor("https://api.media.ccc.de/public/events/8afc16c2-d76a-53f6-85e4-90494665835d"); + extractor.fetchPage(); + } + + @Override + public void testServiceId() throws Exception { + assertEquals(2, extractor.getServiceId()); + } + + @Override + public void testName() throws Exception { + assertEquals("tmux - Warum ein schwarzes Fenster am Bildschirm reicht", extractor.getName()); + } + + @Override + public void testId() throws Exception { + assertEquals("", extractor.getId()); + } + + @Override + public void testUrl() throws Exception { + assertEquals("", extractor.getUrl()); + } + + @Override + public void testOriginalUrl() throws Exception { + assertEquals("", extractor.getOriginalUrl()); + } + + @Test + public void testThumbnail() throws Exception { + assertEquals("https://static.media.ccc.de/media/events/gpn/gpn18/105-hd.jpg", extractor.getThumbnailUrl()); + } + + @Test + public void testUploaderName() throws Exception { + assertEquals("gpn18", extractor.getUploaderName()); + } + + @Test + public void testUploaderUrl() throws Exception { + assertEquals("https://api.media.ccc.de/public/conferences/gpn18", extractor.getUploaderUrl()); + } + + @Test + public void testUploaderAvatarUrl() throws Exception { + assertEquals("https://static.media.ccc.de/media/events/gpn/gpn18/logo.png", extractor.getUploaderAvatarUrl()); + } + + @Test + public void testVideoStreams() throws Exception { + assertEquals(4, extractor.getVideoStreams().size()); + } + + @Test + public void testAudioStreams() throws Exception { + assertEquals(2, extractor.getAudioStreams().size()); + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java index 622514416..efa8cbbea 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java @@ -106,13 +106,6 @@ public class YoutubeChannelExtractorTest { public void testSubscriberCount() throws Exception { assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 0); } - - @Test - public void testChannelDonation() throws Exception { - // this needs to be ignored since wed have to upgrade channel extractor to the new yt interface - // in order to make this work - assertTrue(extractor.getDonationLinks().length == 0); - } } // Youtube RED/Premium ad blocking test @@ -204,12 +197,6 @@ public class YoutubeChannelExtractorTest { assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 0); } - @Test - public void testChannelDonation() throws Exception { - // this needs to be ignored since wed have to upgrade channel extractor to the new yt interface - // in order to make this work - assertTrue(extractor.getDonationLinks().length == 0); - } } public static class Kurzgesagt implements BaseChannelExtractorTest { @@ -312,11 +299,6 @@ public class YoutubeChannelExtractorTest { public void testSubscriberCount() throws Exception { assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 5e6); } - - @Test - public void testChannelDonation() throws Exception { - assertTrue(extractor.getDonationLinks().length == 1); - } } public static class CaptainDisillusion implements BaseChannelExtractorTest { @@ -501,11 +483,6 @@ public class YoutubeChannelExtractorTest { public void testSubscriberCount() throws Exception { assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 50); } - - @Test - public void testChannelDonation() throws Exception { - assertTrue(extractor.getDonationLinks().length == 0); - } } }; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java index 63cd349a1..f25b00197 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorDefaultTest.java @@ -6,6 +6,7 @@ import org.schabi.newpipe.Downloader; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.channel.ChannelInfoItem; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItem; @@ -63,13 +64,16 @@ public class YoutubeSearchExtractorDefaultTest extends YoutubeSearchExtractorBas @Test public void testResultList_FirstElement() { InfoItem firstInfoItem = itemsPage.getItems().get(0); - if(! (firstInfoItem instanceof ChannelInfoItem)) - firstInfoItem = itemsPage.getItems().get(1); + InfoItem secondInfoItem = itemsPage.getItems().get(1); + + InfoItem channelItem = firstInfoItem instanceof ChannelInfoItem ? firstInfoItem + : secondInfoItem; // The channel should be the first item - assertTrue(firstInfoItem instanceof ChannelInfoItem); - assertEquals("name", "PewDiePie", firstInfoItem.getName()); - assertEquals("url","https://www.youtube.com/user/PewDiePie", firstInfoItem.getUrl()); + assertTrue((firstInfoItem instanceof ChannelInfoItem) + || (secondInfoItem instanceof ChannelInfoItem)); + assertEquals("name", "PewDiePie", channelItem.getName()); + assertEquals("url","https://www.youtube.com/user/PewDiePie", channelItem.getUrl()); } @Test From 7fcb3f579008565a80fc9542f13f96fee29e5d4f Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Sun, 23 Dec 2018 20:25:19 +0100 Subject: [PATCH 08/24] fix crash when return null as a related streams collector --- .../services/media_ccc/extractors/MediaCCCStreamExtractor.java | 2 +- .../org/schabi/newpipe/extractor/utils/ExtractorHelper.java | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java index 6c0e0fad8..e95435851 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java @@ -188,7 +188,7 @@ public class MediaCCCStreamExtractor extends StreamExtractor { @Override public StreamInfoItemsCollector getRelatedStreams() throws IOException, ExtractionException { - return null; + return new StreamInfoItemsCollector(getServiceId()); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ExtractorHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ExtractorHelper.java index 445c52c77..0fc6a3120 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ExtractorHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ExtractorHelper.java @@ -30,6 +30,7 @@ public class ExtractorHelper { public static List getRelatedVideosOrLogError(StreamInfo info, StreamExtractor extractor) { try { InfoItemsCollector collector = extractor.getRelatedStreams(); + if(collector == null) return Collections.emptyList(); info.addAllErrors(collector.getErrors()); //noinspection unchecked From cf3ea227531fc13c1fd2c69fe8bf6d65202bc953 Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Sun, 23 Dec 2018 21:16:05 +0100 Subject: [PATCH 09/24] make stream size have a p in the end --- .../media_ccc/extractors/MediaCCCStreamExtractor.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java index e95435851..f26747d6a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java @@ -150,10 +150,8 @@ public class MediaCCCStreamExtractor extends StreamExtractor { throw new ExtractionException("Unknown media format: " + mimeType); } - videoStreams.add(new VideoStream(recording.getString("recording_url"), - mediaFormat, - Integer.toString(recording.getInt("height")))); + mediaFormat, recording.getInt("height") + "p")); } } return videoStreams; From 50a17afc056cfc9ccb14a72f3ff5a944d6f9418d Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Sun, 23 Dec 2018 21:52:40 +0100 Subject: [PATCH 10/24] fix issue with originalUrl --- .../src/main/java/org/schabi/newpipe/extractor/Info.java | 8 +++++++- .../org/schabi/newpipe/extractor/channel/ChannelInfo.java | 6 +++++- .../schabi/newpipe/extractor/playlist/PlaylistInfo.java | 5 +++++ .../org/schabi/newpipe/extractor/search/SearchInfo.java | 5 +++++ 4 files changed, 22 insertions(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/Info.java b/extractor/src/main/java/org/schabi/newpipe/extractor/Info.java index 853fb1189..7bdb23aba 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/Info.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/Info.java @@ -27,7 +27,7 @@ public abstract class Info implements Serializable { * * @see Extractor#getOriginalUrl() */ - private final String originalUrl; + private String originalUrl; private final String name; private final List errors = new ArrayList<>(); @@ -62,6 +62,12 @@ public abstract class Info implements Serializable { return getClass().getSimpleName() + "[url=\"" + url + "\"" + ifDifferentString + ", name=\"" + name + "\"]"; } + // if you use an api and want to handle the website url + // overriding original url is essential + public void setOriginalUrl(String url) { + originalUrl = url; + } + public int getServiceId() { return serviceId; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java index a848725f6..0832b601b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java @@ -61,7 +61,11 @@ public class ChannelInfo extends ListInfo { extractor.getLinkHandler(), extractor.getName()); - + try { + info.setOriginalUrl(extractor.getOriginalUrl()); + } catch (Exception e) { + info.addError(e); + } try { info.setAvatarUrl(extractor.getAvatarUrl()); } catch (Exception e) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java index e613c888b..978fdcdb4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java @@ -47,6 +47,11 @@ public class PlaylistInfo extends ListInfo { extractor.getLinkHandler(), extractor.getName()); + try { + info.setOriginalUrl(extractor.getOriginalUrl()); + } catch (Exception e) { + info.addError(e); + } try { info.setStreamCount(extractor.getStreamCount()); } catch (Exception e) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchInfo.java index 474c61090..6362b6c65 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchInfo.java @@ -37,6 +37,11 @@ public class SearchInfo extends ListInfo { extractor.getLinkHandler(), extractor.getSearchString()); + try { + info.setOriginalUrl(extractor.getOriginalUrl()); + } catch (Exception e) { + info.addError(e); + } try { info.searchSuggestion = extractor.getSearchSuggestion(); } catch (Exception e) { From 73232a7bad9ea122be2a50eb81d09db5b550db64 Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Sun, 23 Dec 2018 22:31:38 +0100 Subject: [PATCH 11/24] stream names may not contain dots --- .../newpipe/extractor/services/media_ccc/MediaCCCService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java index af8b21a26..cab5936d7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java @@ -23,7 +23,7 @@ import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCap public class MediaCCCService extends StreamingService { public MediaCCCService(int id) { - super(id, "Media.CCC", asList(AUDIO, VIDEO)); + super(id, "MediaCCC", asList(AUDIO, VIDEO)); } @Override From cb51254aa36240b606814b40e2c72f0c065a4b7f Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Sat, 26 Jan 2019 22:55:34 +0100 Subject: [PATCH 12/24] return one null less --- .../services/media_ccc/extractors/MediaCCCSearchExtractor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java index 2eb0dee7f..6d9b2cbff 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java @@ -45,7 +45,7 @@ public class MediaCCCSearchExtractor extends SearchExtractor { @Override public String getNextPageUrl() throws IOException, ExtractionException { - return null; + return ""; } @Override From 150345929202a0aae40633bf64d0d25a9c2cfae4 Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Mon, 28 Jan 2019 12:18:16 +0100 Subject: [PATCH 13/24] add kiosk for conferences --- .../extractor/kiosk/KioskExtractor.java | 3 +- .../newpipe/extractor/kiosk/KioskList.java | 5 ++ .../services/media_ccc/MediaCCCService.java | 17 ++++- .../extractors/MediaCCCConferenceKiosk.java | 69 +++++++++++++++++++ .../extractors/MediaCCCSearchExtractor.java | 4 +- .../MediaCCCConferenceInfoItemExtractor.java | 44 ++++++++++++ ...aCCCConferencesListLinkHandlerFactory.java | 25 +++++++ .../soundcloud/SoundcloudChartsExtractor.java | 2 +- .../extractors/YoutubeTrendingExtractor.java | 2 +- .../MediaCCCConferenceListExtractorTest.java | 59 ++++++++++++++++ 10 files changed, 224 insertions(+), 6 deletions(-) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceKiosk.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCConferenceInfoItemExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferencesListLinkHandlerFactory.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceListExtractorTest.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java index 481bb63ca..e2786bbac 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskExtractor.java @@ -20,6 +20,7 @@ package org.schabi.newpipe.extractor.kiosk; * along with NewPipe. If not, see . */ +import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -29,7 +30,7 @@ import org.schabi.newpipe.extractor.utils.Localization; import javax.annotation.Nonnull; -public abstract class KioskExtractor extends ListExtractor { +public abstract class KioskExtractor extends ListExtractor { private final String id; public KioskExtractor(StreamingService streamingService, diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java b/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java index 22a9f2ca8..3636110d7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/kiosk/KioskList.java @@ -49,6 +49,11 @@ public class KioskList { defaultKiosk = kioskType; } + public KioskExtractor getDefaultKioskExtractor() + throws ExtractionException, IOException { + return getDefaultKioskExtractor(""); + } + public KioskExtractor getDefaultKioskExtractor(String nextPageUrl) throws ExtractionException, IOException { return getDefaultKioskExtractor(nextPageUrl, NewPipe.getPreferredLocalization()); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java index cab5936d7..4dbf8e7f6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java @@ -4,20 +4,25 @@ import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.SuggestionExtractor; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.kiosk.KioskExtractor; import org.schabi.newpipe.extractor.kiosk.KioskList; import org.schabi.newpipe.extractor.linkhandler.*; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCConferenceExtractor; +import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCConferenceKiosk; import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCSearchExtractor; import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCStreamExtractor; import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCConferenceLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCConferencesListLinkHandlerFactory; import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCSearchQueryHandlerFactory; import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCStreamLinkHandlerFactory; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; import org.schabi.newpipe.extractor.utils.Localization; +import java.io.IOException; + import static java.util.Arrays.asList; import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.*; @@ -77,7 +82,17 @@ public class MediaCCCService extends StreamingService { // add kiosks here e.g.: try { - // Add kiosk here + list.addKioskEntry(new KioskList.KioskExtractorFactory() { + @Override + public KioskExtractor createNewKiosk(StreamingService streamingService, + String url, + String kioskId, + Localization localization) throws ExtractionException, IOException { + return new MediaCCCConferenceKiosk(MediaCCCService.this, + new MediaCCCConferencesListLinkHandlerFactory().fromUrl(url), kioskId, localization); + } + }, new MediaCCCConferencesListLinkHandlerFactory(), "conferences"); + list.setDefaultKiosk("conferences"); } catch (Exception e) { throw new ExtractionException(e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceKiosk.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceKiosk.java new file mode 100644 index 000000000..7eac10c11 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceKiosk.java @@ -0,0 +1,69 @@ +package org.schabi.newpipe.extractor.services.media_ccc.extractors; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; +import org.schabi.newpipe.extractor.Downloader; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.channel.ChannelInfoItem; +import org.schabi.newpipe.extractor.channel.ChannelInfoItemsCollector; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.kiosk.KioskExtractor; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.services.media_ccc.extractors.infoItems.MediaCCCConferenceInfoItemExtractor; +import org.schabi.newpipe.extractor.utils.Localization; + +import javax.annotation.Nonnull; +import java.io.IOException; + +public class MediaCCCConferenceKiosk extends KioskExtractor { + + private JsonObject doc; + + public MediaCCCConferenceKiosk(StreamingService streamingService, + ListLinkHandler linkHandler, + String kioskId, + Localization localization) { + super(streamingService, linkHandler, kioskId, localization); + } + + @Nonnull + @Override + public InfoItemsPage getInitialPage() throws IOException, ExtractionException { + JsonArray conferences = doc.getArray("conferences"); + ChannelInfoItemsCollector collector = new ChannelInfoItemsCollector(getServiceId()); + for(int i = 0; i < conferences.size(); i++) { + collector.commit(new MediaCCCConferenceInfoItemExtractor(conferences.getObject(i))); + } + + return new InfoItemsPage<>(collector, ""); + } + + @Override + public String getNextPageUrl() throws IOException, ExtractionException { + return ""; + } + + @Override + public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { + return InfoItemsPage.emptyPage(); + } + + @Override + public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { + String site = downloader.download(getLinkHandler().getUrl()); + try { + doc = JsonParser.object().from(site); + } catch (JsonParserException jpe) { + throw new ExtractionException("Could not parse json.", jpe); + } + } + + @Nonnull + @Override + public String getName() throws ParsingException { + return doc.getString("Conferences"); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java index 6d9b2cbff..4e1a4f1c0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java @@ -20,7 +20,7 @@ import java.io.IOException; public class MediaCCCSearchExtractor extends SearchExtractor { - JsonObject doc; + private JsonObject doc; public MediaCCCSearchExtractor(StreamingService service, SearchQueryHandler linkHandler, Localization localization) { super(service, linkHandler, localization); @@ -50,7 +50,7 @@ public class MediaCCCSearchExtractor extends SearchExtractor { @Override public InfoItemsPage getPage(String pageUrl) throws IOException, ExtractionException { - return null; + return InfoItemsPage.emptyPage(); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCConferenceInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCConferenceInfoItemExtractor.java new file mode 100644 index 000000000..05914dd75 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCConferenceInfoItemExtractor.java @@ -0,0 +1,44 @@ +package org.schabi.newpipe.extractor.services.media_ccc.extractors.infoItems; + +import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; +import org.schabi.newpipe.extractor.exceptions.ParsingException; + +public class MediaCCCConferenceInfoItemExtractor implements ChannelInfoItemExtractor { + + JsonObject conference; + + public MediaCCCConferenceInfoItemExtractor(JsonObject conference) { + this.conference = conference; + } + + @Override + public String getDescription() throws ParsingException { + return ""; + } + + @Override + public long getSubscriberCount() throws ParsingException { + return -1; + } + + @Override + public long getStreamCount() throws ParsingException { + return -1; + } + + @Override + public String getName() throws ParsingException { + return conference.getString("title"); + } + + @Override + public String getUrl() throws ParsingException { + return conference.getString("url"); + } + + @Override + public String getThumbnailUrl() throws ParsingException { + return conference.getString("logo_url"); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferencesListLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferencesListLinkHandlerFactory.java new file mode 100644 index 000000000..603f1ea2d --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferencesListLinkHandlerFactory.java @@ -0,0 +1,25 @@ +package org.schabi.newpipe.extractor.services.media_ccc.linkHandler; + +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; + +import java.util.List; + +public class MediaCCCConferencesListLinkHandlerFactory extends ListLinkHandlerFactory { + @Override + public String getId(String url) throws ParsingException { + return "conferences"; + } + + @Override + public String getUrl(String id, List contentFilter, String sortFilter) throws ParsingException { + return "https://api.media.ccc.de/public/conferences"; + } + + @Override + public boolean onAcceptUrl(String url) throws ParsingException { + return url.equals("https://media.ccc.de/b/conferences") + || url.equals("https://api.media.ccc.de/public/conferences"); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java index 8f02b1c68..1a02166e4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java @@ -14,7 +14,7 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; -public class SoundcloudChartsExtractor extends KioskExtractor { +public class SoundcloudChartsExtractor extends KioskExtractor { private StreamInfoItemsCollector collector = null; private String nextPageUrl = null; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java index f33f7b8a5..b065aa630 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java @@ -37,7 +37,7 @@ import org.schabi.newpipe.extractor.utils.Localization; import javax.annotation.Nonnull; import java.io.IOException; -public class YoutubeTrendingExtractor extends KioskExtractor { +public class YoutubeTrendingExtractor extends KioskExtractor { private Document doc; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceListExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceListExtractorTest.java new file mode 100644 index 000000000..2116916f2 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCConferenceListExtractorTest.java @@ -0,0 +1,59 @@ +package org.schabi.newpipe.extractor.services.media_ccc; +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.channel.ChannelInfoItem; +import org.schabi.newpipe.extractor.kiosk.KioskExtractor; +import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCConferenceKiosk; +import org.schabi.newpipe.extractor.utils.Localization; + +import java.util.List; + +import static org.schabi.newpipe.extractor.ServiceList.MediaCCC; +import static org.junit.Assert.assertTrue; + + +/** + * Test {@link MediaCCCConferenceKiosk} + */ +public class MediaCCCConferenceListExtractorTest { + + private static KioskExtractor extractor; + + @BeforeClass + public static void setUpClass() throws Exception { + NewPipe.init(Downloader.getInstance(), new Localization("en", "en_GB")); + extractor = MediaCCC.getKioskList().getDefaultKioskExtractor(); + extractor.fetchPage(); + } + + @Test + public void getConferencesListTest() throws Exception { + assertTrue("returned list was to small", + extractor.getInitialPage().getItems().size() >= 174); + } + + @Test + public void conferenceTypeTest() throws Exception { + assertTrue(contains(extractor.getInitialPage().getItems(), "FrOSCon 2016")); + assertTrue(contains(extractor.getInitialPage().getItems(), "ChaosWest @ 35c3")); + assertTrue(contains(extractor.getInitialPage().getItems(), "CTreffOS chaOStalks")); + assertTrue(contains(extractor.getInitialPage().getItems(), "Datenspuren 2015")); + assertTrue(contains(extractor.getInitialPage().getItems(), "Chaos Singularity 2017")); + assertTrue(contains(extractor.getInitialPage().getItems(), "SIGINT10")); + assertTrue(contains(extractor.getInitialPage().getItems(), "Vintage Computing Festival Berlin 2015")); + assertTrue(contains(extractor.getInitialPage().getItems(), "FIfFKon 2015")); + assertTrue(contains(extractor.getInitialPage().getItems(), "33C3: trailers")); + assertTrue(contains(extractor.getInitialPage().getItems(), "Blinkenlights")); + } + + private boolean contains(List itemList, String name) { + for(InfoItem item : itemList) { + if(item.getName().equals(name)) + return true; + } + return false; + } +} From 14e1ccdb86f1fa47c8c123dacd9785e0aea56c6b Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Mon, 28 Jan 2019 15:30:17 +0100 Subject: [PATCH 14/24] add conferences search --- .../extractors/MediaCCCSearchExtractor.java | 101 +++++++++++++++--- .../MediaCCCSearchQueryHandlerFactory.java | 10 +- .../MediaCCCSearchExtractorAllTest.java | 60 +++++++++++ ...ediaCCCSearchExtractorConferencesTest.java | 50 +++++++++ ...=> MediaCCCSearchExtractorEventsTest.java} | 17 ++- .../SoundcloudSearchExtractorDefaultTest.java | 21 ++-- 6 files changed, 231 insertions(+), 28 deletions(-) create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorAllTest.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorConferencesTest.java rename extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/{MediaCCCSearchExtractorTest.java => MediaCCCSearchExtractorEventsTest.java} (76%) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java index 4e1a4f1c0..ef008afd6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java @@ -8,22 +8,39 @@ import org.schabi.newpipe.extractor.Downloader; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.InfoItemsCollector; import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.channel.ChannelInfoItem; +import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; +import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector; import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.services.media_ccc.extractors.infoItems.MediaCCCStreamInfoItemExtractor; +import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCConferenceLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCConferencesListLinkHandlerFactory; import org.schabi.newpipe.extractor.utils.Localization; - +import static org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCSearchQueryHandlerFactory.CONFERENCES; +import static org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCSearchQueryHandlerFactory.EVENTS; +import static org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCSearchQueryHandlerFactory.ALL; import javax.annotation.Nonnull; import java.io.IOException; +import java.util.List; public class MediaCCCSearchExtractor extends SearchExtractor { private JsonObject doc; + private MediaCCCConferenceKiosk conferenceKiosk; public MediaCCCSearchExtractor(StreamingService service, SearchQueryHandler linkHandler, Localization localization) { super(service, linkHandler, localization); + try { + conferenceKiosk = new MediaCCCConferenceKiosk(service, + new MediaCCCConferencesListLinkHandlerFactory().fromId("conferences"), + "conferences", + localization); + } catch (Exception e) { + e.printStackTrace(); + } } @Override @@ -34,11 +51,22 @@ public class MediaCCCSearchExtractor extends SearchExtractor { @Nonnull @Override public InfoItemsPage getInitialPage() throws IOException, ExtractionException { - InfoItemsCollector searchItems = getInfoItemSearchCollector(); - JsonArray events = doc.getArray("events"); - for(int i = 0; i < events.size(); i++) { - searchItems.commit(new MediaCCCStreamInfoItemExtractor( - events.getObject(i))); + InfoItemsSearchCollector searchItems = getInfoItemSearchCollector(); + + if(getLinkHandler().getContentFilters().contains(CONFERENCES) + || getLinkHandler().getContentFilters().contains(ALL)) { + searchConferences(getSearchString(), + conferenceKiosk.getInitialPage().getItems(), + searchItems); + } + + if(getLinkHandler().getContentFilters().contains(EVENTS) + || getLinkHandler().getContentFilters().contains(ALL)) { + JsonArray events = doc.getArray("events"); + for (int i = 0; i < events.size(); i++) { + searchItems.commit(new MediaCCCStreamInfoItemExtractor( + events.getObject(i))); + } } return new InfoItemsPage<>(searchItems, null); } @@ -55,13 +83,60 @@ public class MediaCCCSearchExtractor extends SearchExtractor { @Override public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { - final String site; - final String url = getUrl(); - site = downloader.download(url, getLocalization()); - try { - doc = JsonParser.object().from(site); - } catch (JsonParserException jpe) { - throw new ExtractionException("Could not parse json.", jpe); + if(getLinkHandler().getContentFilters().contains(EVENTS) + || getLinkHandler().getContentFilters().contains(ALL)) { + final String site; + final String url = getUrl(); + site = downloader.download(url, getLocalization()); + try { + doc = JsonParser.object().from(site); + } catch (JsonParserException jpe) { + throw new ExtractionException("Could not parse json.", jpe); + } + } + if(getLinkHandler().getContentFilters().contains(CONFERENCES) + || getLinkHandler().getContentFilters().contains(ALL)) + conferenceKiosk.fetchPage(); + } + + private void searchConferences(String searchString, + List channelItems, + InfoItemsSearchCollector collector) { + for(final ChannelInfoItem item : channelItems) { + if(item.getName().toUpperCase().contains( + searchString.toUpperCase())) { + collector.commit(new ChannelInfoItemExtractor() { + @Override + public String getDescription() throws ParsingException { + return item.getDescription(); + } + + @Override + public long getSubscriberCount() throws ParsingException { + return item.getSubscriberCount(); + } + + @Override + public long getStreamCount() throws ParsingException { + return item.getStreamCount(); + } + + @Override + public String getName() throws ParsingException { + return item.getName(); + } + + @Override + public String getUrl() throws ParsingException { + return item.getUrl(); + } + + @Override + public String getThumbnailUrl() throws ParsingException { + return item.getThumbnailUrl(); + } + }); + } } } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCSearchQueryHandlerFactory.java index b9747dc8f..fb6c705a5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCSearchQueryHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCSearchQueryHandlerFactory.java @@ -9,9 +9,17 @@ import java.util.List; public class MediaCCCSearchQueryHandlerFactory extends SearchQueryHandlerFactory { + public static final String ALL = "all"; + public static final String CONFERENCES = "conferences"; + public static final String EVENTS = "events"; + @Override public String[] getAvailableContentFilter() { - return new String[0]; + return new String[] { + ALL, + CONFERENCES, + EVENTS + }; } @Override diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorAllTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorAllTest.java new file mode 100644 index 000000000..dc0c64035 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorAllTest.java @@ -0,0 +1,60 @@ +package org.schabi.newpipe.extractor.services.media_ccc; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.ListExtractor; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.channel.ChannelInfoItem; +import org.schabi.newpipe.extractor.search.SearchExtractor; +import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCSearchExtractor; +import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCSearchQueryHandlerFactory; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import org.schabi.newpipe.extractor.utils.Localization; + +import java.util.Arrays; + +import static junit.framework.TestCase.assertTrue; +import static org.schabi.newpipe.extractor.ServiceList.MediaCCC; + +/** + * Test for {@link MediaCCCSearchExtractor} + */ +public class MediaCCCSearchExtractorAllTest { + + private static SearchExtractor extractor; + private static ListExtractor.InfoItemsPage itemsPage; + + @BeforeClass + public static void setUpClass() throws Exception { + NewPipe.init(Downloader.getInstance(), new Localization("GB", "en")); + extractor = MediaCCC.getSearchExtractor( new MediaCCCSearchQueryHandlerFactory() + .fromQuery("c", Arrays.asList(new String[] {"all"}), "") + ,new Localization("GB", "en")); + extractor.fetchPage(); + itemsPage = extractor.getInitialPage(); + } + + @Test + public void testIfChannelInfoItemsAvailable() { + boolean isAvialable = false; + for(InfoItem item : itemsPage.getItems()) { + if(item instanceof ChannelInfoItem) { + isAvialable = true; + } + } + assertTrue("ChannelInfoItem not in all list", isAvialable); + } + + @Test + public void testIfStreamInfoitemsAvailable() { + boolean isAvialable = false; + for(InfoItem item : itemsPage.getItems()) { + if(item instanceof StreamInfoItem) { + isAvialable = true; + } + } + assertTrue("ChannelInfoItem not in all list", isAvialable); + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorConferencesTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorConferencesTest.java new file mode 100644 index 000000000..4b7a8b266 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorConferencesTest.java @@ -0,0 +1,50 @@ +package org.schabi.newpipe.extractor.services.media_ccc; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.ListExtractor; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.channel.ChannelInfoItem; +import org.schabi.newpipe.extractor.search.SearchExtractor; +import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCSearchExtractor; +import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCSearchQueryHandlerFactory; +import org.schabi.newpipe.extractor.utils.Localization; + +import java.util.Arrays; + +import static junit.framework.TestCase.assertTrue; +import static org.schabi.newpipe.extractor.ServiceList.MediaCCC; + +/** + * Test for {@link MediaCCCSearchExtractor} + */ +public class MediaCCCSearchExtractorConferencesTest { + + private static SearchExtractor extractor; + private static ListExtractor.InfoItemsPage itemsPage; + + @BeforeClass + public static void setUpClass() throws Exception { + NewPipe.init(Downloader.getInstance(), new Localization("GB", "en")); + extractor = MediaCCC.getSearchExtractor( new MediaCCCSearchQueryHandlerFactory() + .fromQuery("c3", Arrays.asList(new String[] {"conferences"}), "") + ,new Localization("GB", "en")); + extractor.fetchPage(); + itemsPage = extractor.getInitialPage(); + } + + @Test + public void testReturnTypeChannel() { + for(InfoItem item : itemsPage.getItems()) { + assertTrue("Item is not of type channel", item instanceof ChannelInfoItem); + } + } + + @Test + public void testItemCount() { + assertTrue("Count is to hight: " + itemsPage.getItems().size(), itemsPage.getItems().size() < 127); + assertTrue("Countis to low: " + itemsPage.getItems().size(), itemsPage.getItems().size() >= 29); + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorEventsTest.java similarity index 76% rename from extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorTest.java rename to extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorEventsTest.java index c08777a43..11b1fbdfb 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorEventsTest.java @@ -8,8 +8,12 @@ import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCSearchExtractor; +import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCSearchQueryHandlerFactory; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.utils.Localization; +import java.util.Arrays; + import static junit.framework.TestCase.assertEquals; import static junit.framework.TestCase.assertTrue; import static org.junit.Assert.assertFalse; @@ -18,14 +22,16 @@ import static org.schabi.newpipe.extractor.ServiceList.MediaCCC; /** * Test for {@link MediaCCCSearchExtractor} */ -public class MediaCCCSearchExtractorTest { +public class MediaCCCSearchExtractorEventsTest { private static SearchExtractor extractor; private static ListExtractor.InfoItemsPage itemsPage; @BeforeClass public static void setUpClass() throws Exception { NewPipe.init(Downloader.getInstance(), new Localization("GB", "en")); - extractor = MediaCCC.getSearchExtractor("source"); + extractor = MediaCCC.getSearchExtractor( new MediaCCCSearchQueryHandlerFactory() + .fromQuery("linux", Arrays.asList(new String[] {"events"}), "") + ,new Localization("GB", "en")); extractor.fetchPage(); itemsPage = extractor.getInitialPage(); } @@ -58,4 +64,11 @@ public class MediaCCCSearchExtractorTest { itemsPage.getItems().get(0).getThumbnailUrl().startsWith("https://static.media.ccc.de/media/") && itemsPage.getItems().get(0).getThumbnailUrl().endsWith(".jpg")); } + + @Test + public void testReturnTypeStream() throws Exception { + for(InfoItem item : itemsPage.getItems()) { + assertTrue("Item is not of type StreamInfoItem", item instanceof StreamInfoItem); + } + } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorDefaultTest.java index 683a15c55..682641de8 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorDefaultTest.java @@ -8,10 +8,13 @@ import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.channel.ChannelInfoItem; import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudSearchExtractor; +import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudSearchQueryHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.utils.Localization; +import java.util.Arrays; + import static org.junit.Assert.*; import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; import static org.schabi.newpipe.extractor.ServiceList.YouTube; @@ -44,26 +47,20 @@ public class SoundcloudSearchExtractorDefaultTest extends SoundcloudSearchExtrac @BeforeClass public static void setUpClass() throws Exception { NewPipe.init(Downloader.getInstance(), new Localization("GB", "en")); - extractor = (SoundcloudSearchExtractor) SoundCloud.getSearchExtractor("lill uzi vert"); + extractor = (SoundcloudSearchExtractor) SoundCloud.getSearchExtractor( + new SoundcloudSearchQueryHandlerFactory().fromQuery("lill uzi vert", + Arrays.asList(new String[]{"tracks"}), ""), + new Localization("GB", "en")); extractor.fetchPage(); itemsPage = extractor.getInitialPage(); } @Test public void testGetSecondPageUrl() throws Exception { - assertEquals("https://api-v2.soundcloud.com/search?q=lill+uzi+vert&limit=10&offset=10", + assertEquals("https://api-v2.soundcloud.com/search/tracks?q=lill+uzi+vert&limit=10&offset=10", removeClientId(extractor.getNextPageUrl())); } - @Test - public void testResultList_FirstElement() { - InfoItem firstInfoItem = itemsPage.getItems().get(0); - - // THe channel should be the first item - assertEquals("name", "Bad and Boujee (Feat. Lil Uzi Vert) [Prod. By Metro Boomin]", firstInfoItem.getName()); - assertEquals("url","https://soundcloud.com/migosatl/bad-and-boujee-feat-lil-uzi-vert-prod-by-metro-boomin", firstInfoItem.getUrl()); - } - @Test public void testResultListCheckIfContainsStreamItems() { boolean hasStreams = false; @@ -94,7 +91,7 @@ public class SoundcloudSearchExtractorDefaultTest extends SoundcloudSearchExtrac } assertFalse("First and second page are equal", equals); - assertEquals("https://api-v2.soundcloud.com/search?q=lill+uzi+vert&limit=10&offset=20", + assertEquals("https://api-v2.soundcloud.com/search/tracks?q=lill+uzi+vert&limit=10&offset=20", removeClientId(secondPage.getNextPageUrl())); } From 644da4fc3cb3e9b63cbb27fe50ec56e0d5f9e1b7 Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Mon, 28 Jan 2019 15:43:21 +0100 Subject: [PATCH 15/24] add ogg as audioformat --- .../schabi/newpipe/extractor/MediaFormat.java | 25 ++++++++++--------- .../extractors/MediaCCCStreamExtractor.java | 2 ++ ...utubeStreamExtractorAgeRestrictedTest.java | 2 +- .../YoutubeStreamExtractorDefaultTest.java | 2 +- 4 files changed, 17 insertions(+), 14 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java b/extractor/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java index 21ead42c4..bb8366510 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/MediaFormat.java @@ -30,20 +30,21 @@ public enum MediaFormat { //video and audio combined formats // id name suffix mime type MPEG_4 (0x0, "MPEG-4", "mp4", "video/mp4"), - v3GPP (0x1, "3GPP", "3gp", "video/3gpp"), - WEBM (0x2, "WebM", "webm", "video/webm"), + v3GPP (0x10, "3GPP", "3gp", "video/3gpp"), + WEBM (0x20, "WebM", "webm", "video/webm"), // audio formats - M4A (0x3, "m4a", "m4a", "audio/mp4"), - WEBMA (0x4, "WebM", "webm", "audio/webm"), - MP3 (0x5, "MP3", "mp3", "audio/mpeg"), - OPUS (0x6, "opus", "opus", "audio/opus"), + M4A (0x100, "m4a", "m4a", "audio/mp4"), + WEBMA (0x200, "WebM", "webm", "audio/webm"), + MP3 (0x300, "MP3", "mp3", "audio/mpeg"), + OPUS (0x400, "opus", "opus", "audio/opus"), + OGG (0x500, "ogg", "ogg", "audio/ogg"), // subtitles formats - VTT (0x7, "WebVTT", "vtt", "text/vtt"), - TTML (0x8, "Timed Text Markup Language", "ttml", "application/ttml+xml"), - TRANSCRIPT1 (0x9, "TranScript v1", "srv1", "text/xml"), - TRANSCRIPT2 (0xA, "TranScript v2", "srv2", "text/xml"), - TRANSCRIPT3 (0xB, "TranScript v3", "srv3", "text/xml"), - SRT (0xC, "SubRip file format", "srt", "text/srt"); + VTT (0x1000, "WebVTT", "vtt", "text/vtt"), + TTML (0x2000, "Timed Text Markup Language", "ttml", "application/ttml+xml"), + TRANSCRIPT1 (0x3000, "TranScript v1", "srv1", "text/xml"), + TRANSCRIPT2 (0x4000, "TranScript v2", "srv2", "text/xml"), + TRANSCRIPT3 (0x5000, "TranScript v3", "srv3", "text/xml"), + SRT (0x6000, "SubRip file format", "srt", "text/srt"); public final int id; public final String name; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java index f26747d6a..5c9900236 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java @@ -146,6 +146,8 @@ public class MediaCCCStreamExtractor extends StreamExtractor { mediaFormat = MediaFormat.WEBM; } else if(mimeType.endsWith("mp4")) { mediaFormat = MediaFormat.MPEG_4; + } else if(mimeType.endsWith("ogg")){ + mediaFormat = MediaFormat.OGG; } else { throw new ExtractionException("Unknown media format: " + mimeType); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorAgeRestrictedTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorAgeRestrictedTest.java index dbf8d6d61..8a91887af 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorAgeRestrictedTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorAgeRestrictedTest.java @@ -117,7 +117,7 @@ public class YoutubeStreamExtractorAgeRestrictedTest { s.getUrl().contains(HTTPS)); assertTrue(s.resolution.length() > 0); assertTrue(Integer.toString(s.getFormatId()), - 0 <= s.getFormatId() && s.getFormatId() <= 4); + 0 <= s.getFormatId() && s.getFormatId() <= 0x100); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java index 24e8b7fae..f4d8a3540 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java @@ -135,7 +135,7 @@ public class YoutubeStreamExtractorDefaultTest { assertIsSecureUrl(s.url); assertTrue(s.resolution.length() > 0); assertTrue(Integer.toString(s.getFormatId()), - 0 <= s.getFormatId() && s.getFormatId() <= 4); + 0 <= s.getFormatId() && s.getFormatId() <= 0x100); } } From d0efe8bd47cfbee8fcc6768f53911facb624ff23 Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Tue, 29 Jan 2019 16:04:19 +0100 Subject: [PATCH 16/24] fix ogg --- .../extractors/MediaCCCStreamExtractor.java | 4 +- .../services/media_ccc/MediaCCCOggTest.java | 42 +++++++++++++++++++ .../MediaCCCSearchExtractorAllTest.java | 4 +- 3 files changed, 47 insertions(+), 3 deletions(-) create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCOggTest.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java index 5c9900236..616a3b865 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java @@ -121,6 +121,8 @@ public class MediaCCCStreamExtractor extends StreamExtractor { mediaFormat = MediaFormat.OPUS; } else if(mimeType.endsWith("mpeg")) { mediaFormat = MediaFormat.MP3; + } else if(mimeType.endsWith("ogg")){ + mediaFormat = MediaFormat.OGG; } else { throw new ExtractionException("Unknown media format: " + mimeType); } @@ -146,8 +148,6 @@ public class MediaCCCStreamExtractor extends StreamExtractor { mediaFormat = MediaFormat.WEBM; } else if(mimeType.endsWith("mp4")) { mediaFormat = MediaFormat.MPEG_4; - } else if(mimeType.endsWith("ogg")){ - mediaFormat = MediaFormat.OGG; } else { throw new ExtractionException("Unknown media format: " + mimeType); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCOggTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCOggTest.java new file mode 100644 index 000000000..8f075ed90 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCOggTest.java @@ -0,0 +1,42 @@ +package org.schabi.newpipe.extractor.services.media_ccc; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCStreamExtractor; +import org.schabi.newpipe.extractor.stream.AudioStream; +import org.schabi.newpipe.extractor.stream.StreamExtractor; +import org.schabi.newpipe.extractor.utils.Localization; + + +import static junit.framework.TestCase.assertEquals; +import static org.schabi.newpipe.extractor.ServiceList.MediaCCC; + +/** + * Test {@link MediaCCCStreamExtractor} + */ +public class MediaCCCOggTest { + // test against https://api.media.ccc.de/public/events/1317 + private static StreamExtractor extractor; + + @BeforeClass + public static void setUpClass() throws Exception { + NewPipe.init(Downloader.getInstance(), new Localization("GB", "en")); + + extractor = MediaCCC.getStreamExtractor("https://api.media.ccc.de/public/events/1317"); + extractor.fetchPage(); + } + + @Test + public void getAudioStreamsCount() throws Exception { + assertEquals(1, extractor.getAudioStreams().size()); + } + + @Test + public void getAudioStreamsContainOgg() throws Exception { + for(AudioStream stream : extractor.getAudioStreams()) { + System.out.println(stream.getFormat()); + } + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorAllTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorAllTest.java index dc0c64035..9ce475b4a 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorAllTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorAllTest.java @@ -30,7 +30,7 @@ public class MediaCCCSearchExtractorAllTest { public static void setUpClass() throws Exception { NewPipe.init(Downloader.getInstance(), new Localization("GB", "en")); extractor = MediaCCC.getSearchExtractor( new MediaCCCSearchQueryHandlerFactory() - .fromQuery("c", Arrays.asList(new String[] {"all"}), "") + .fromQuery("c3", Arrays.asList(new String[] {"all"}), "") ,new Localization("GB", "en")); extractor.fetchPage(); itemsPage = extractor.getInitialPage(); @@ -40,6 +40,7 @@ public class MediaCCCSearchExtractorAllTest { public void testIfChannelInfoItemsAvailable() { boolean isAvialable = false; for(InfoItem item : itemsPage.getItems()) { + System.out.println(item); if(item instanceof ChannelInfoItem) { isAvialable = true; } @@ -51,6 +52,7 @@ public class MediaCCCSearchExtractorAllTest { public void testIfStreamInfoitemsAvailable() { boolean isAvialable = false; for(InfoItem item : itemsPage.getItems()) { + System.out.println(item); if(item instanceof StreamInfoItem) { isAvialable = true; } From 79b0a19d1af158a07abfd067e155d75f413fb834 Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Tue, 29 Jan 2019 16:49:37 +0100 Subject: [PATCH 17/24] make search filter empty be equal to all --- .../extractors/MediaCCCSearchExtractor.java | 14 ++++++++------ .../media_ccc/MediaCCCSearchExtractorAllTest.java | 4 +--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java index ef008afd6..af6724b62 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java @@ -6,7 +6,6 @@ import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; import org.schabi.newpipe.extractor.Downloader; import org.schabi.newpipe.extractor.InfoItem; -import org.schabi.newpipe.extractor.InfoItemsCollector; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelInfoItem; import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; @@ -16,7 +15,6 @@ import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector; import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.services.media_ccc.extractors.infoItems.MediaCCCStreamInfoItemExtractor; -import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCConferenceLinkHandlerFactory; import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCConferencesListLinkHandlerFactory; import org.schabi.newpipe.extractor.utils.Localization; import static org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCSearchQueryHandlerFactory.CONFERENCES; @@ -54,14 +52,16 @@ public class MediaCCCSearchExtractor extends SearchExtractor { InfoItemsSearchCollector searchItems = getInfoItemSearchCollector(); if(getLinkHandler().getContentFilters().contains(CONFERENCES) - || getLinkHandler().getContentFilters().contains(ALL)) { + || getLinkHandler().getContentFilters().contains(ALL) + || getLinkHandler().getContentFilters().isEmpty()) { searchConferences(getSearchString(), conferenceKiosk.getInitialPage().getItems(), searchItems); } if(getLinkHandler().getContentFilters().contains(EVENTS) - || getLinkHandler().getContentFilters().contains(ALL)) { + || getLinkHandler().getContentFilters().contains(ALL) + || getLinkHandler().getContentFilters().isEmpty()) { JsonArray events = doc.getArray("events"); for (int i = 0; i < events.size(); i++) { searchItems.commit(new MediaCCCStreamInfoItemExtractor( @@ -84,7 +84,8 @@ public class MediaCCCSearchExtractor extends SearchExtractor { @Override public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { if(getLinkHandler().getContentFilters().contains(EVENTS) - || getLinkHandler().getContentFilters().contains(ALL)) { + || getLinkHandler().getContentFilters().contains(ALL) + || getLinkHandler().getContentFilters().isEmpty()) { final String site; final String url = getUrl(); site = downloader.download(url, getLocalization()); @@ -95,7 +96,8 @@ public class MediaCCCSearchExtractor extends SearchExtractor { } } if(getLinkHandler().getContentFilters().contains(CONFERENCES) - || getLinkHandler().getContentFilters().contains(ALL)) + || getLinkHandler().getContentFilters().contains(ALL) + || getLinkHandler().getContentFilters().isEmpty()) conferenceKiosk.fetchPage(); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorAllTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorAllTest.java index 9ce475b4a..8dfdb0a92 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorAllTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCSearchExtractorAllTest.java @@ -30,7 +30,7 @@ public class MediaCCCSearchExtractorAllTest { public static void setUpClass() throws Exception { NewPipe.init(Downloader.getInstance(), new Localization("GB", "en")); extractor = MediaCCC.getSearchExtractor( new MediaCCCSearchQueryHandlerFactory() - .fromQuery("c3", Arrays.asList(new String[] {"all"}), "") + .fromQuery("c3", Arrays.asList(new String[0]), "") ,new Localization("GB", "en")); extractor.fetchPage(); itemsPage = extractor.getInitialPage(); @@ -40,7 +40,6 @@ public class MediaCCCSearchExtractorAllTest { public void testIfChannelInfoItemsAvailable() { boolean isAvialable = false; for(InfoItem item : itemsPage.getItems()) { - System.out.println(item); if(item instanceof ChannelInfoItem) { isAvialable = true; } @@ -52,7 +51,6 @@ public class MediaCCCSearchExtractorAllTest { public void testIfStreamInfoitemsAvailable() { boolean isAvialable = false; for(InfoItem item : itemsPage.getItems()) { - System.out.println(item); if(item instanceof StreamInfoItem) { isAvialable = true; } From a97b6dfa7a72f978452a23b217b6619fb82edfea Mon Sep 17 00:00:00 2001 From: Ritvik Saraf <13ritvik@gmail.com> Date: Sat, 16 Feb 2019 00:41:23 +0530 Subject: [PATCH 18/24] mediaccc merge --- .../services/media_ccc/MediaCCCService.java | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java index 4dbf8e7f6..4096569c2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java @@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor.services.media_ccc; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.SuggestionExtractor; import org.schabi.newpipe.extractor.channel.ChannelExtractor; +import org.schabi.newpipe.extractor.comments.CommentsExtractor; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.kiosk.KioskExtractor; import org.schabi.newpipe.extractor.kiosk.KioskList; @@ -104,4 +105,20 @@ public class MediaCCCService extends StreamingService { public SubscriptionExtractor getSubscriptionExtractor() { return null; } + + @Override + public ListLinkHandlerFactory getCommentsLHFactory() { + return null; + } + + @Override + public CommentsExtractor getCommentsExtractor(ListLinkHandler linkHandler, Localization localization) + throws ExtractionException { + return null; + } + + @Override + public boolean isCommentsSupported() { + return false; + } } From 3dadf63028e3ebbcf46ad555c494826c39db1882 Mon Sep 17 00:00:00 2001 From: Ritvik Saraf <13ritvik@gmail.com> Date: Sat, 16 Feb 2019 00:57:00 +0530 Subject: [PATCH 19/24] refactored comments capability --- .../newpipe/extractor/StreamingService.java | 5 +---- .../services/media_ccc/MediaCCCService.java | 22 ++++++++++--------- .../soundcloud/SoundcloudService.java | 21 +++++++++--------- .../services/youtube/YoutubeService.java | 18 ++++++--------- 4 files changed, 31 insertions(+), 35 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java index 8818ea362..86bd9d1f8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java @@ -66,7 +66,7 @@ public abstract class StreamingService { } public enum MediaCapability { - AUDIO, VIDEO, LIVE + AUDIO, VIDEO, LIVE, COMMENTS } } @@ -296,9 +296,6 @@ public abstract class StreamingService { return getCommentsExtractor(llhf.fromUrl(url), NewPipe.getPreferredLocalization()); } - public abstract boolean isCommentsSupported(); - - /** * Figures out where the link is pointing to (a channel, a video, a playlist, etc.) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java index 4096569c2..a23295a97 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java @@ -1,5 +1,11 @@ package org.schabi.newpipe.extractor.services.media_ccc; +import static java.util.Arrays.asList; +import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO; +import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.VIDEO; + +import java.io.IOException; + import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.SuggestionExtractor; import org.schabi.newpipe.extractor.channel.ChannelExtractor; @@ -7,7 +13,12 @@ import org.schabi.newpipe.extractor.comments.CommentsExtractor; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.kiosk.KioskExtractor; import org.schabi.newpipe.extractor.kiosk.KioskList; -import org.schabi.newpipe.extractor.linkhandler.*; +import org.schabi.newpipe.extractor.linkhandler.LinkHandler; +import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; +import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; +import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCConferenceExtractor; @@ -22,11 +33,6 @@ import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; import org.schabi.newpipe.extractor.utils.Localization; -import java.io.IOException; - -import static java.util.Arrays.asList; -import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.*; - public class MediaCCCService extends StreamingService { public MediaCCCService(int id) { super(id, "MediaCCC", asList(AUDIO, VIDEO)); @@ -117,8 +123,4 @@ public class MediaCCCService extends StreamingService { return null; } - @Override - public boolean isCommentsSupported() { - return false; - } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java index e62fadda1..74d38be57 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java @@ -1,21 +1,27 @@ package org.schabi.newpipe.extractor.services.soundcloud; -import org.schabi.newpipe.extractor.*; -import org.schabi.newpipe.extractor.linkhandler.*; +import static java.util.Collections.singletonList; +import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO; + +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.SuggestionExtractor; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.comments.CommentsExtractor; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.kiosk.KioskExtractor; import org.schabi.newpipe.extractor.kiosk.KioskList; +import org.schabi.newpipe.extractor.linkhandler.LinkHandler; +import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; +import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; +import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; import org.schabi.newpipe.extractor.utils.Localization; -import static java.util.Collections.singletonList; -import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO; - public class SoundcloudService extends StreamingService { public SoundcloudService(int id) { @@ -113,9 +119,4 @@ public class SoundcloudService extends StreamingService { return null; } - @Override - public boolean isCommentsSupported() { - return false; - } - } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java index e5569775d..3b621b8f0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java @@ -1,5 +1,11 @@ package org.schabi.newpipe.extractor.services.youtube; +import static java.util.Arrays.asList; +import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO; +import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.COMMENTS; +import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.LIVE; +import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.VIDEO; + import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.SuggestionExtractor; import org.schabi.newpipe.extractor.channel.ChannelExtractor; @@ -33,11 +39,6 @@ import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; import org.schabi.newpipe.extractor.utils.Localization; -import static java.util.Arrays.asList; -import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO; -import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.LIVE; -import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.VIDEO; - /* * Created by Christian Schabesberger on 23.08.15. * @@ -61,7 +62,7 @@ import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCap public class YoutubeService extends StreamingService { public YoutubeService(int id) { - super(id, "YouTube", asList(AUDIO, VIDEO, LIVE)); + super(id, "YouTube", asList(AUDIO, VIDEO, LIVE, COMMENTS)); } @Override @@ -150,9 +151,4 @@ public class YoutubeService extends StreamingService { return new YoutubeCommentsExtractor(this, urlIdHandler, localization); } - @Override - public boolean isCommentsSupported() { - return true; - } - } From 04460bba2c48d504ba163c037ed72669c53c7bc5 Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Tue, 19 Feb 2019 16:10:46 +0100 Subject: [PATCH 20/24] add mediaCCC to supported sites in description. --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index b13f04ff0..715ec2fb8 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ The following sites are currently supported: - YouTube - SoundCloud +- MediaCCC ## License From 948a7826ab2fa27425593ed07bb3a1617185493d Mon Sep 17 00:00:00 2001 From: Ritvik Saraf <13ritvik@gmail.com> Date: Sat, 23 Feb 2019 18:20:12 +0530 Subject: [PATCH 21/24] fetch channel id for channel url --- .../youtube/extractors/YoutubeCommentsInfoItemExtractor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java index 93c4395b7..471cafd9c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java @@ -99,7 +99,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract @Override public String getAuthorEndpoint() throws ParsingException { try { - return "https://youtube.com" + JsonUtils.getString(json, "authorEndpoint.browseEndpoint.canonicalBaseUrl"); + return "https://youtube.com/channel/" + JsonUtils.getString(json, "authorEndpoint.browseEndpoint.browseId"); } catch (Exception e) { throw new ParsingException("Could not get author endpoint", e); } From 9eff18252b4e0912fb0be7b616588b806ec31805 Mon Sep 17 00:00:00 2001 From: Ritvik Saraf <13ritvik@gmail.com> Date: Thu, 28 Feb 2019 22:55:18 +0530 Subject: [PATCH 22/24] fix npe in fetching service by url --- .../java/org/schabi/newpipe/extractor/StreamingService.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java index 86bd9d1f8..26e84da97 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java @@ -308,11 +308,11 @@ public abstract class StreamingService { LinkHandlerFactory cH = getChannelLHFactory(); LinkHandlerFactory pH = getPlaylistLHFactory(); - if (sH.acceptUrl(url)) { + if (sH != null && sH.acceptUrl(url)) { return LinkType.STREAM; - } else if (cH.acceptUrl(url)) { + } else if (cH != null && cH.acceptUrl(url)) { return LinkType.CHANNEL; - } else if (pH.acceptUrl(url)) { + } else if (pH != null && pH.acceptUrl(url)) { return LinkType.PLAYLIST; } else { return LinkType.NONE; From 72262707bf250620f7d4221cc4e342afd4934235 Mon Sep 17 00:00:00 2001 From: Ritvik Saraf <13ritvik@gmail.com> Date: Fri, 1 Mar 2019 00:10:29 +0530 Subject: [PATCH 23/24] fixed youtube comment utf8 bom --- .../YoutubeCommentsInfoItemExtractor.java | 5 ++++- .../org/schabi/newpipe/extractor/utils/Utils.java | 14 ++++++++++++-- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java index 471cafd9c..1d447bd55 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java @@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.utils.JsonUtils; +import org.schabi.newpipe.extractor.utils.Utils; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; @@ -62,7 +63,9 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract @Override public String getCommentText() throws ParsingException { try { - return YoutubeCommentsExtractor.getYoutubeText(JsonUtils.getObject(json, "contentText")); + String commentText = YoutubeCommentsExtractor.getYoutubeText(JsonUtils.getObject(json, "contentText")); + // youtube adds U+FEFF in some comments. eg. https://www.youtube.com/watch?v=Nj4F63E59io + return Utils.removeUTF8BOM(commentText); } catch (Exception e) { throw new ParsingException("Could not get comment text", e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java index d4b8db432..fde334fa7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java @@ -1,13 +1,13 @@ package org.schabi.newpipe.extractor.utils; -import org.schabi.newpipe.extractor.exceptions.ParsingException; - import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; import java.net.URL; import java.net.URLDecoder; import java.util.List; +import org.schabi.newpipe.extractor.exceptions.ParsingException; + public class Utils { private Utils() { @@ -120,4 +120,14 @@ public class Utils { throw e; } } + + public static String removeUTF8BOM(String s) { + if (s.startsWith("\uFEFF")) { + s = s.substring(1); + } + if (s.endsWith("\uFEFF")) { + s = s.substring(0, s.length()-1); + } + return s; + } } \ No newline at end of file From a2735c4f9765ff28bd967483dda2154ae3a2e8e6 Mon Sep 17 00:00:00 2001 From: Ritvik Saraf <13ritvik@gmail.com> Date: Sat, 2 Mar 2019 02:48:05 +0530 Subject: [PATCH 24/24] fix comment url --- .../youtube/extractors/YoutubeCommentsExtractor.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java index 85d150014..aba43b2e8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java @@ -107,11 +107,11 @@ public class YoutubeCommentsExtractor extends CommentsExtractor { throw new ParsingException("Could not parse json data for comments", e); } CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId()); - collectCommentsFrom(collector, ajaxJson, pageUrl); + collectCommentsFrom(collector, ajaxJson); return new InfoItemsPage<>(collector, getNextPageUrl(ajaxJson)); } - private void collectCommentsFrom(CommentsInfoItemsCollector collector, JsonObject ajaxJson, String pageUrl) throws ParsingException { + private void collectCommentsFrom(CommentsInfoItemsCollector collector, JsonObject ajaxJson) throws ParsingException { JsonArray contents; try { @@ -130,7 +130,7 @@ public class YoutubeCommentsExtractor extends CommentsExtractor { for(Object c: comments) { if(c instanceof JsonObject) { - CommentsInfoItemExtractor extractor = new YoutubeCommentsInfoItemExtractor((JsonObject) c, pageUrl); + CommentsInfoItemExtractor extractor = new YoutubeCommentsInfoItemExtractor((JsonObject) c, getUrl()); collector.commit(extractor); } }