From 5a2dcdce166e1acb8dc3de2b4b9ad95674a9c779 Mon Sep 17 00:00:00 2001 From: Andrew <30773181+2secslater@users.noreply.github.com> Date: Thu, 8 Aug 2019 00:19:02 +0000 Subject: [PATCH 01/25] Add Invidious instances Added all publicly listed Invidious instances from omarroth/invidious wiki page to the link handler factory for YouTube. --- .../YoutubeStreamLinkHandlerFactory.java | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java index 521fa47c3..478ebe2d1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java @@ -163,7 +163,20 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory { } case "WWW.INVIDIO.US": - case "INVIDIO.US": { // code-block for hooktube.com and invidio.us + case "DEV.INVIDIO.US" + case "INVIDIO.US": + case "INVIDIOUS.SNOPYTA.ORG": + case "DE.INVIDIOUS.SNOPYTA.ORG": + case "FI.INVIDIOUS.SNOPYTA.ORG": + case "VID.WXZM.SX": + case "INVIDIOUS.KABI.TK": + case "INVIDIOU.SH": + case "WWW.INVIDIOU.SH": + case "NO.INVIDIOU.SH": + case "INVIDIOUS.ENKIRTON.NET": + case "TUBE.POAL.CO": + case "INVIDIOUS.13AD.DE": + case "YT.ELUKERIO.ORG": { // code-block for hooktube.com and Invidious instances if (path.equals("watch")) { String viewQueryValue = Utils.getQueryValue(url, "v"); if (viewQueryValue != null) { From 7fb17684f5d979cef0aeb7cece303c1ab7ad1f09 Mon Sep 17 00:00:00 2001 From: Andrew <30773181+2secslater@users.noreply.github.com> Date: Thu, 8 Aug 2019 00:25:42 +0000 Subject: [PATCH 02/25] Fixed missing colon causing builds to fail --- .../youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java index 478ebe2d1..853f77fe5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java @@ -163,7 +163,7 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory { } case "WWW.INVIDIO.US": - case "DEV.INVIDIO.US" + case "DEV.INVIDIO.US": case "INVIDIO.US": case "INVIDIOUS.SNOPYTA.ORG": case "DE.INVIDIOUS.SNOPYTA.ORG": From d0f1c31b34f8e7a9f1367e2c8a11ad50638586f8 Mon Sep 17 00:00:00 2001 From: Andrew <30773181+2secslater@users.noreply.github.com> Date: Tue, 10 Sep 2019 17:54:32 +0100 Subject: [PATCH 03/25] Add Invidious instances to parsing helper for YouTube --- .../services/youtube/linkHandler/YoutubeParsingHelper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java index 7464b7b44..8174dc493 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java @@ -48,7 +48,7 @@ public class YoutubeParsingHelper { public static boolean isInvidioURL(URL url) { String host = url.getHost(); - return host.equalsIgnoreCase("invidio.us") || host.equalsIgnoreCase("www.invidio.us"); + return host.equalsIgnoreCase("invidio.us") || host.equalsIgnoreCase("dev.invidio.us") || host.equalsIgnoreCase("www.invidio.us") || host.equalsIgnoreCase("invidious.snopyta.org") || host.equalsIgnoreCase("de.invidious.snopyta.org") || host.equalsIgnoreCase("fi.invidious.snopyta.org") || host.equalsIgnoreCase("vid.wxzm.sx") || host.equalsIgnoreCase("invidious.kabi.tk") || host.equalsIgnoreCase("invidiou.sh") || host.equalsIgnoreCase("www.invidiou.sh") || host.equalsIgnoreCase("no.invidiou.sh") || host.equalsIgnoreCase("invidious.enkirton.net") || host.equalsIgnoreCase("tube.poal.co") || host.equalsIgnoreCase("invidious.13ad.de") || host.equalsIgnoreCase("yt.elukerio.org"); } public static long parseDurationString(String input) From bf017bf5b9d75c40ef5a1b1f9cdba3bb1f7f5b81 Mon Sep 17 00:00:00 2001 From: Stypox Date: Wed, 11 Sep 2019 19:05:41 +0200 Subject: [PATCH 04/25] Fix TeamNewPipe/NewPipe#2615 --- .../extractors/YoutubeStreamExtractor.java | 66 ++++++++----------- 1 file changed, 29 insertions(+), 37 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 44e77b01e..08bf50582 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -85,6 +85,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { private JsonObject playerArgs; @Nonnull private final Map videoInfoPage = new HashMap<>(); + private JsonObject playerResponse; @Nonnull private List subtitlesInfos = new ArrayList<>(); @@ -486,7 +487,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { assertPageFetched(); List videoStreams = new ArrayList<>(); try { - for (Map.Entry entry : getItags(URL_ENCODED_FMT_STREAM_MAP, ItagItem.ItagType.VIDEO).entrySet()) { + for (Map.Entry entry : getItags(ADAPTIVE_FMTS, ItagItem.ItagType.VIDEO).entrySet()) { ItagItem itag = entry.getValue(); VideoStream videoStream = new VideoStream(entry.getKey(), itag.getMediaFormat(), itag.resolutionString); @@ -620,7 +621,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { //////////////////////////////////////////////////////////////////////////*/ private static final String URL_ENCODED_FMT_STREAM_MAP = "url_encoded_fmt_stream_map"; - private static final String ADAPTIVE_FMTS = "adaptive_fmts"; + private static final String ADAPTIVE_FMTS = "adaptiveFormats"; private static final String HTTPS = "https:"; private static final String CONTENT = "content"; private static final String DECRYPTION_FUNC_NAME = "decrypt"; @@ -667,6 +668,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { playerUrl = getPlayerUrl(ytPlayerConfig); isAgeRestricted = false; } + playerResponse = getPlayerResponse(); if (decryptionCode.isEmpty()) { decryptionCode = loadDecryptionCode(playerUrl); @@ -728,6 +730,20 @@ public class YoutubeStreamExtractor extends StreamExtractor { } } + private JsonObject getPlayerResponse() throws ParsingException { + try { + String playerResponseStr; + if(playerArgs != null) { + playerResponseStr = playerArgs.getString("player_response"); + } else { + playerResponseStr = videoInfoPage.get("player_response"); + } + return JsonParser.object().from(playerResponseStr); + } catch (Exception e) { + throw new ParsingException("Could not parse yt player response", e); + } + } + @Nonnull private EmbeddedInfo getEmbeddedInfo() throws ParsingException, ReCaptchaException { try { @@ -924,45 +940,21 @@ public class YoutubeStreamExtractor extends StreamExtractor { "&sts=" + sts + "&ps=default&gl=US&hl=en"; } - private Map getItags(String encodedUrlMapKey, ItagItem.ItagType itagTypeWanted) throws ParsingException { + private Map getItags(String streamingDataKey, ItagItem.ItagType itagTypeWanted) throws ParsingException { Map urlAndItags = new LinkedHashMap<>(); - String encodedUrlMap = ""; - if (playerArgs != null && playerArgs.isString(encodedUrlMapKey)) { - encodedUrlMap = playerArgs.getString(encodedUrlMapKey, ""); - } else if (videoInfoPage.containsKey(encodedUrlMapKey)) { - encodedUrlMap = videoInfoPage.get(encodedUrlMapKey); - } + JsonArray formats = playerResponse.getObject("streamingData").getArray(streamingDataKey); + for (int i = 0; i != formats.size(); ++i) { + JsonObject formatData = formats.getObject(i); + int itag = formatData.getInt("itag"); - for (String url_data_str : encodedUrlMap.split(",")) { - try { - // This loop iterates through multiple streams, therefore tags - // is related to one and the same stream at a time. - Map tags = Parser.compatParseMap( - org.jsoup.parser.Parser.unescapeEntities(url_data_str, true)); - - int itag = Integer.parseInt(tags.get("itag")); - - if (ItagItem.isSupported(itag)) { - ItagItem itagItem = ItagItem.getItag(itag); - if (itagItem.itagType == itagTypeWanted) { - String streamUrl = tags.get("url"); - // if video has a signature: decrypt it and add it to the url - if (tags.get("s") != null) { - if (tags.get("sp") == null) { - // fallback for urls not conaining the "sp" tag - streamUrl = streamUrl + "&signature=" + decryptSignature(tags.get("s"), decryptionCode); - } - else { - streamUrl = streamUrl + "&" + tags.get("sp") + "=" + decryptSignature(tags.get("s"), decryptionCode); - } - } - urlAndItags.put(streamUrl, itagItem); - } + if (ItagItem.isSupported(itag)) { + ItagItem itagItem = ItagItem.getItag(itag); + if (itagItem.itagType == itagTypeWanted) { + String streamUrl = formatData.getString("url"); + System.out.println(streamUrl); + urlAndItags.put(streamUrl, itagItem); } - } catch (DecryptException e) { - throw e; - } catch (Exception ignored) { } } From 63a37c48e3bfce4f2a0bb2472fda4afd75f8903a Mon Sep 17 00:00:00 2001 From: Stypox Date: Wed, 11 Sep 2019 19:31:39 +0200 Subject: [PATCH 05/25] Remove println left behind --- .../services/youtube/extractors/YoutubeStreamExtractor.java | 1 - 1 file changed, 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 08bf50582..32fe8d8bb 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -952,7 +952,6 @@ public class YoutubeStreamExtractor extends StreamExtractor { ItagItem itagItem = ItagItem.getItag(itag); if (itagItem.itagType == itagTypeWanted) { String streamUrl = formatData.getString("url"); - System.out.println(streamUrl); urlAndItags.put(streamUrl, itagItem); } } From d9570d8634754ca03755268bac6f25c54ac37010 Mon Sep 17 00:00:00 2001 From: Stypox Date: Wed, 11 Sep 2019 19:35:08 +0200 Subject: [PATCH 06/25] Use pre-generated playerResponse field everywhere in YtStreamExtractor --- .../extractors/YoutubeStreamExtractor.java | 28 +++---------------- 1 file changed, 4 insertions(+), 24 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 32fe8d8bb..b4c8058e8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -254,20 +254,6 @@ public class YoutubeStreamExtractor extends StreamExtractor { public long getLength() throws ParsingException { assertPageFetched(); - final JsonObject playerResponse; - try { - final String pr; - if(playerArgs != null) { - pr = playerArgs.getString("player_response"); - } else { - pr = videoInfoPage.get("player_response"); - } - playerResponse = JsonParser.object() - .from(pr); - } catch (Exception e) { - throw new ParsingException("Could not get playerResponse", e); - } - // try getting duration from playerargs try { String durationMs = playerResponse @@ -859,19 +845,13 @@ public class YoutubeStreamExtractor extends StreamExtractor { } catch (IOException | ExtractionException e) { throw new SubtitlesException("Unable to download player configs", e); } - final String playerResponse = playerConfig.getObject("args", new JsonObject()) - .getString("player_response"); final JsonObject captions; - try { - if (playerResponse == null || !JsonParser.object().from(playerResponse).has("captions")) { - // Captions does not exist - return Collections.emptyList(); - } - captions = JsonParser.object().from(playerResponse).getObject("captions"); - } catch (JsonParserException e) { - throw new SubtitlesException("Unable to parse subtitles listing", e); + if (!playerResponse.has("captions")) { + // Captions does not exist + return Collections.emptyList(); } + captions = playerResponse.getObject("captions"); final JsonObject renderer = captions.getObject("playerCaptionsTracklistRenderer", new JsonObject()); final JsonArray captionsArray = renderer.getArray("captionTracks", new JsonArray()); From e5e8c66686589de0c452fa75c68143152a3e1c4f Mon Sep 17 00:00:00 2001 From: Stypox Date: Wed, 11 Sep 2019 19:56:16 +0200 Subject: [PATCH 07/25] Readd signature decryption in YtStreamExtractor --- .../extractors/YoutubeStreamExtractor.java | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index b4c8058e8..82836ce8d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -929,10 +929,22 @@ public class YoutubeStreamExtractor extends StreamExtractor { int itag = formatData.getInt("itag"); if (ItagItem.isSupported(itag)) { - ItagItem itagItem = ItagItem.getItag(itag); - if (itagItem.itagType == itagTypeWanted) { - String streamUrl = formatData.getString("url"); - urlAndItags.put(streamUrl, itagItem); + try { + ItagItem itagItem = ItagItem.getItag(itag); + if (itagItem.itagType == itagTypeWanted) { + String streamUrl; + if (formatData.has("url")) { + streamUrl = formatData.getString("url"); + } else { + // this url has an encrypted signature + Map cipher = Parser.compatParseMap(formatData.getString("cipher")); + streamUrl = cipher.get("url") + "&" + cipher.get("sp") + "=" + decryptSignature(cipher.get("s"), decryptionCode); + } + + urlAndItags.put(streamUrl, itagItem); + } + } catch (UnsupportedEncodingException ignored) { + } } } From 9c423a0a4084d5497409798221a0871b648fd36f Mon Sep 17 00:00:00 2001 From: Stypox Date: Wed, 11 Sep 2019 20:04:28 +0200 Subject: [PATCH 08/25] Use FORMATS to get video+audio streams on yt Not ADAPTIVE_FORMATS --- .../extractors/YoutubeStreamExtractor.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 82836ce8d..8f5c0c8cd 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -449,11 +449,11 @@ public class YoutubeStreamExtractor extends StreamExtractor { } @Override - public List getAudioStreams() throws IOException, ExtractionException { + public List getAudioStreams() throws ExtractionException { assertPageFetched(); List audioStreams = new ArrayList<>(); try { - for (Map.Entry entry : getItags(ADAPTIVE_FMTS, ItagItem.ItagType.AUDIO).entrySet()) { + for (Map.Entry entry : getItags(ADAPTIVE_FORMATS, ItagItem.ItagType.AUDIO).entrySet()) { ItagItem itag = entry.getValue(); AudioStream audioStream = new AudioStream(entry.getKey(), itag.getMediaFormat(), itag.avgBitrate); @@ -469,11 +469,11 @@ public class YoutubeStreamExtractor extends StreamExtractor { } @Override - public List getVideoStreams() throws IOException, ExtractionException { + public List getVideoStreams() throws ExtractionException { assertPageFetched(); List videoStreams = new ArrayList<>(); try { - for (Map.Entry entry : getItags(ADAPTIVE_FMTS, ItagItem.ItagType.VIDEO).entrySet()) { + for (Map.Entry entry : getItags(FORMATS, ItagItem.ItagType.VIDEO).entrySet()) { ItagItem itag = entry.getValue(); VideoStream videoStream = new VideoStream(entry.getKey(), itag.getMediaFormat(), itag.resolutionString); @@ -493,7 +493,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { assertPageFetched(); List videoOnlyStreams = new ArrayList<>(); try { - for (Map.Entry entry : getItags(ADAPTIVE_FMTS, ItagItem.ItagType.VIDEO_ONLY).entrySet()) { + for (Map.Entry entry : getItags(ADAPTIVE_FORMATS, ItagItem.ItagType.VIDEO_ONLY).entrySet()) { ItagItem itag = entry.getValue(); VideoStream videoStream = new VideoStream(entry.getKey(), itag.getMediaFormat(), itag.resolutionString, true); @@ -530,7 +530,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { assertPageFetched(); try { if (playerArgs != null && (playerArgs.has("ps") && playerArgs.get("ps").toString().equals("live") || - playerArgs.get(URL_ENCODED_FMT_STREAM_MAP).toString().isEmpty())) { + playerResponse.getObject("streamingData").getArray(FORMATS).isEmpty())) { return StreamType.LIVE_STREAM; } } catch (Exception e) { @@ -606,8 +606,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { // Fetch page //////////////////////////////////////////////////////////////////////////*/ - private static final String URL_ENCODED_FMT_STREAM_MAP = "url_encoded_fmt_stream_map"; - private static final String ADAPTIVE_FMTS = "adaptiveFormats"; + private static final String FORMATS = "formats"; + private static final String ADAPTIVE_FORMATS = "adaptiveFormats"; private static final String HTTPS = "https:"; private static final String CONTENT = "content"; private static final String DECRYPTION_FUNC_NAME = "decrypt"; From 24a37b88a9b59519c606f15e11f5e2c2164c2fdc Mon Sep 17 00:00:00 2001 From: Stypox Date: Wed, 11 Sep 2019 20:12:30 +0200 Subject: [PATCH 09/25] Use pre-generated playerResponse field in yt's getHlsUrl() Also refactored code to always throw exception when the url can't be found --- .../extractors/YoutubeStreamExtractor.java | 21 +++++++------------ 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 8f5c0c8cd..8983c1a2f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -429,22 +429,15 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public String getHlsUrl() throws ParsingException { assertPageFetched(); - try { - String hlsvp = ""; - if (playerArgs != null) { - if( playerArgs.isString("hlsvp") ) { - hlsvp = playerArgs.getString("hlsvp", ""); - }else { - hlsvp = JsonParser.object() - .from(playerArgs.getString("player_response", "{}")) - .getObject("streamingData", new JsonObject()) - .getString("hlsManifestUrl", ""); - } - } - return hlsvp; + try { + return playerResponse.getObject("streamingData").getString("hlsManifestUrl"); } catch (Exception e) { - throw new ParsingException("Could not get hls manifest url", e); + if (playerArgs != null && playerArgs.isString("hlsvp")) { + return playerArgs.getString("hlsvp"); + } else { + throw new ParsingException("Could not get hls manifest url", e); + } } } From 5f8e76eb873e35d0b0a6dbd3c95b8a276ff6c12a Mon Sep 17 00:00:00 2001 From: Stypox Date: Thu, 12 Sep 2019 14:36:42 +0200 Subject: [PATCH 10/25] Move stream-related youtube tests to subfolder --- .../{ => stream}/YoutubeStreamExtractorAgeRestrictedTest.java | 2 +- .../{ => stream}/YoutubeStreamExtractorControversialTest.java | 2 +- .../youtube/{ => stream}/YoutubeStreamExtractorDefaultTest.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) rename extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/{ => stream}/YoutubeStreamExtractorAgeRestrictedTest.java (98%) rename extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/{ => stream}/YoutubeStreamExtractorControversialTest.java (98%) rename extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/{ => stream}/YoutubeStreamExtractorDefaultTest.java (99%) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorAgeRestrictedTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java similarity index 98% rename from extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorAgeRestrictedTest.java rename to extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java index f41750d7e..6a24ec5f2 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorAgeRestrictedTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java @@ -1,4 +1,4 @@ -package org.schabi.newpipe.extractor.services.youtube; +package org.schabi.newpipe.extractor.services.youtube.stream; import org.junit.BeforeClass; import org.junit.Ignore; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorControversialTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java similarity index 98% rename from extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorControversialTest.java rename to extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java index 8fd991154..261d521c1 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorControversialTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java @@ -1,4 +1,4 @@ -package org.schabi.newpipe.extractor.services.youtube; +package org.schabi.newpipe.extractor.services.youtube.stream; import org.junit.BeforeClass; import org.junit.Ignore; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java similarity index 99% rename from extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java rename to extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java index 9bf0344a1..2ef10cf6d 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java @@ -1,4 +1,4 @@ -package org.schabi.newpipe.extractor.services.youtube; +package org.schabi.newpipe.extractor.services.youtube.stream; import org.junit.BeforeClass; import org.junit.Test; From 4453a6344763155e85bdac2d465a9b7774f86562 Mon Sep 17 00:00:00 2001 From: Stypox Date: Thu, 12 Sep 2019 15:05:22 +0200 Subject: [PATCH 11/25] Add test for YouTube livestreams The current livestream is https://www.youtube.com/watch?v=EcEMX-63PKY --- .../YoutubeStreamExtractorLivestreamTest.java | 141 ++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java new file mode 100644 index 000000000..106ec928b --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java @@ -0,0 +1,141 @@ +package org.schabi.newpipe.extractor.services.youtube.stream; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.MediaFormat; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor; +import org.schabi.newpipe.extractor.stream.StreamExtractor; +import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; +import org.schabi.newpipe.extractor.stream.StreamType; +import org.schabi.newpipe.extractor.stream.VideoStream; +import org.schabi.newpipe.extractor.utils.Localization; +import org.schabi.newpipe.extractor.utils.Utils; + +import java.io.IOException; + +import static org.junit.Assert.*; +import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; +import static org.schabi.newpipe.extractor.ServiceList.YouTube; + +public class YoutubeStreamExtractorLivestreamTest { + private static YoutubeStreamExtractor extractor; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(Downloader.getInstance(), new Localization("GB", "en")); + extractor = (YoutubeStreamExtractor) YouTube + .getStreamExtractor("https://www.youtube.com/watch?v=EcEMX-63PKY"); + extractor.fetchPage(); + } + + @Test + public void testGetInvalidTimeStamp() throws ParsingException { + assertTrue(extractor.getTimeStamp() + "", + extractor.getTimeStamp() <= 0); + } + + @Test + public void testGetTitle() throws ParsingException { + assertFalse(extractor.getName().isEmpty()); + } + + @Test + public void testGetDescription() throws ParsingException { + assertNotNull(extractor.getDescription()); + assertFalse(extractor.getDescription().isEmpty()); + } + + @Test + public void testGetFullLinksInDescription() throws ParsingException { + assertTrue(extractor.getDescription().contains("https://www.instagram.com/nathalie.baraton/")); + assertFalse(extractor.getDescription().contains("https://www.instagram.com/nathalie.ba...")); + } + + @Test + public void testGetUploaderName() throws ParsingException { + assertNotNull(extractor.getUploaderName()); + assertFalse(extractor.getUploaderName().isEmpty()); + } + + + @Test + public void testGetLength() throws ParsingException { + assertEquals(0, extractor.getLength()); + } + + @Test + public void testGetViewCount() throws ParsingException { + long count = extractor.getViewCount(); + assertTrue(Long.toString(count), count >= 7148995); + } + + @Test + public void testGetUploadDate() throws ParsingException { + assertTrue(extractor.getUploadDate().length() > 0); + } + + @Test + public void testGetUploaderUrl() throws ParsingException { + assertEquals("https://www.youtube.com/channel/UCSJ4gkVC6NrvII8umztf0Ow", extractor.getUploaderUrl()); + } + + @Test + public void testGetThumbnailUrl() throws ParsingException { + assertIsSecureUrl(extractor.getThumbnailUrl()); + } + + @Test + public void testGetUploaderAvatarUrl() throws ParsingException { + assertIsSecureUrl(extractor.getUploaderAvatarUrl()); + } + + @Test + public void testGetAudioStreams() throws ExtractionException { + assertFalse(extractor.getAudioStreams().isEmpty()); + } + + @Test + public void testGetVideoStreams() throws ExtractionException { + for (VideoStream s : extractor.getVideoStreams()) { + assertIsSecureUrl(s.url); + assertTrue(s.resolution.length() > 0); + assertTrue(Integer.toString(s.getFormatId()), + 0 <= s.getFormatId() && s.getFormatId() <= 0x100); + } + } + + @Test + public void testStreamType() throws ParsingException { + assertTrue(extractor.getStreamType() == StreamType.LIVE_STREAM); + } + + @Test + public void testGetDashMpd() throws ParsingException { + // we dont expect this particular video to have a DASH file. For this purpouse we use a different test class. + assertTrue(extractor.getDashMpdUrl(), extractor.getDashMpdUrl().isEmpty()); + } + + @Test + public void testGetRelatedVideos() throws ExtractionException, IOException { + StreamInfoItemsCollector relatedVideos = extractor.getRelatedStreams(); + Utils.printErrors(relatedVideos.getErrors()); + assertFalse(relatedVideos.getItems().isEmpty()); + assertTrue(relatedVideos.getErrors().isEmpty()); + } + + @Test + public void testGetSubtitlesListDefault() throws IOException, ExtractionException { + // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null + assertTrue(extractor.getSubtitlesDefault().isEmpty()); + } + + @Test + public void testGetSubtitlesList() throws IOException, ExtractionException { + // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null + assertTrue(extractor.getSubtitles(MediaFormat.TTML).isEmpty()); + } +} From 0d8fb65003d404be95c9f57020d4d52268b6cf9c Mon Sep 17 00:00:00 2001 From: Stypox Date: Thu, 12 Sep 2019 15:07:07 +0200 Subject: [PATCH 12/25] Fix NPE on determining whether stream is live on Youtube --- .../services/youtube/extractors/YoutubeStreamExtractor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 8983c1a2f..ada39450c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -523,7 +523,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { assertPageFetched(); try { if (playerArgs != null && (playerArgs.has("ps") && playerArgs.get("ps").toString().equals("live") || - playerResponse.getObject("streamingData").getArray(FORMATS).isEmpty())) { + (!playerResponse.getObject("streamingData").has(FORMATS)))) { return StreamType.LIVE_STREAM; } } catch (Exception e) { From 3f1ba93be527969d2aabf9e46193bfafbd3e2448 Mon Sep 17 00:00:00 2001 From: Stypox Date: Thu, 12 Sep 2019 15:08:17 +0200 Subject: [PATCH 13/25] Fix NPE when extracting itags with non-existing streamingData key --- .../services/youtube/extractors/YoutubeStreamExtractor.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index ada39450c..bec12af88 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -915,8 +915,12 @@ public class YoutubeStreamExtractor extends StreamExtractor { private Map getItags(String streamingDataKey, ItagItem.ItagType itagTypeWanted) throws ParsingException { Map urlAndItags = new LinkedHashMap<>(); + JsonObject streamingData = playerResponse.getObject("streamingData"); + if (!streamingData.has(streamingDataKey)) { + return urlAndItags; + } - JsonArray formats = playerResponse.getObject("streamingData").getArray(streamingDataKey); + JsonArray formats = streamingData.getArray(streamingDataKey); for (int i = 0; i != formats.size(); ++i) { JsonObject formatData = formats.getObject(i); int itag = formatData.getInt("itag"); From 1a1672248a7f6917d28637e47fbd7a67f3acec70 Mon Sep 17 00:00:00 2001 From: Stypox Date: Thu, 12 Sep 2019 15:10:23 +0200 Subject: [PATCH 14/25] Eliminate Android Studio warnings in livestream test --- .../youtube/stream/YoutubeStreamExtractorLivestreamTest.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java index 106ec928b..3e24b7e36 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java @@ -8,7 +8,6 @@ import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor; -import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.stream.StreamType; import org.schabi.newpipe.extractor.stream.VideoStream; @@ -110,7 +109,7 @@ public class YoutubeStreamExtractorLivestreamTest { @Test public void testStreamType() throws ParsingException { - assertTrue(extractor.getStreamType() == StreamType.LIVE_STREAM); + assertSame(extractor.getStreamType(), StreamType.LIVE_STREAM); } @Test @@ -129,13 +128,11 @@ public class YoutubeStreamExtractorLivestreamTest { @Test public void testGetSubtitlesListDefault() throws IOException, ExtractionException { - // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null assertTrue(extractor.getSubtitlesDefault().isEmpty()); } @Test public void testGetSubtitlesList() throws IOException, ExtractionException { - // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null assertTrue(extractor.getSubtitles(MediaFormat.TTML).isEmpty()); } } From 075e6d51d6cda7232133184986dca3b80ec436dc Mon Sep 17 00:00:00 2001 From: toehead2001 Date: Wed, 11 Sep 2019 20:43:49 -0600 Subject: [PATCH 15/25] Add music.youtube.com to link handler --- .../services/youtube/linkHandler/YoutubeParsingHelper.java | 2 +- .../youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java index 8174dc493..4c3655340 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java @@ -33,7 +33,7 @@ public class YoutubeParsingHelper { public static boolean isYoutubeURL(URL url) { String host = url.getHost(); return host.equalsIgnoreCase("youtube.com") || host.equalsIgnoreCase("www.youtube.com") - || host.equalsIgnoreCase("m.youtube.com"); + || host.equalsIgnoreCase("m.youtube.com") || host.equalsIgnoreCase("music.youtube.com"); } public static boolean isYoutubeServiceURL(URL url) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java index 853f77fe5..e3ac4d520 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java @@ -114,7 +114,8 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory { case "YOUTUBE.COM": case "WWW.YOUTUBE.COM": - case "M.YOUTUBE.COM": { + case "M.YOUTUBE.COM": + case "MUSIC.YOUTUBE.COM": { if (path.equals("attribution_link")) { String uQueryValue = Utils.getQueryValue(url, "u"); From b709529cb64199ac7bc967503fb3428157f8e567 Mon Sep 17 00:00:00 2001 From: toehead2001 Date: Thu, 12 Sep 2019 12:14:16 -0600 Subject: [PATCH 16/25] Add link handler tests for music.youtube.com --- .../youtube/YoutubePlaylistLinkHandlerFactoryTest.java | 2 ++ .../services/youtube/YoutubeStreamLinkHandlerFactoryTest.java | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistLinkHandlerFactoryTest.java index f04974a7c..4e2d148cd 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistLinkHandlerFactoryTest.java @@ -40,6 +40,7 @@ public class YoutubePlaylistLinkHandlerFactoryTest { assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("https://youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId()); assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("www.youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId()); assertEquals("PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV", linkHandler.fromUrl("www.youtube.com/playlist?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV").getId()); + assertEquals("OLAK5uy_lEBUW9iTwqf0IlYPxZ8LrzpgqjAHZgZpM", linkHandler.fromUrl("https://music.youtube.com/playlist?list=OLAK5uy_lEBUW9iTwqf0IlYPxZ8LrzpgqjAHZgZpM").getId()); } @Test @@ -54,6 +55,7 @@ public class YoutubePlaylistLinkHandlerFactoryTest { assertTrue(linkHandler.acceptUrl("https://youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC")); assertTrue(linkHandler.acceptUrl("www.youtube.com/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC")); assertTrue(linkHandler.acceptUrl("www.youtube.com/playlist?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV")); + assertTrue(linkHandler.acceptUrl("https://music.youtube.com/playlist?list=OLAK5uy_lEBUW9iTwqf0IlYPxZ8LrzpgqjAHZgZpM")); } @Test diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java index a7bc43b3f..154dcdd26 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java @@ -80,6 +80,7 @@ public class YoutubeStreamLinkHandlerFactoryTest { assertEquals("EhxJLojIE_o", linkHandler.fromUrl("http://www.youtube.com/attribution_link?a=JdfC0C9V6ZI&u=%2Fwatch%3Fv%3DEhxJLojIE_o%26feature%3Dshare").getId()); assertEquals("jZViOEv90dI", linkHandler.fromUrl("vnd.youtube://www.youtube.com/watch?v=jZViOEv90dI").getId()); assertEquals("jZViOEv90dI", linkHandler.fromUrl("vnd.youtube:jZViOEv90dI").getId()); + assertEquals("O0EDx9WAelc", linkHandler.fromUrl("https://music.youtube.com/watch?v=O0EDx9WAelc").getId()); } @Test @@ -98,8 +99,8 @@ public class YoutubeStreamLinkHandlerFactoryTest { assertTrue(linkHandler.acceptUrl("http://www.youtube.com/attribution_link?a=JdfC0C9V6ZI&u=%2Fwatch%3Fv%3DEhxJLojIE_o%26feature%3Dshare")); assertTrue(linkHandler.acceptUrl("vnd.youtube://www.youtube.com/watch?v=jZViOEv90dI")); assertTrue(linkHandler.acceptUrl("vnd.youtube:jZViOEv90dI")); - assertTrue(linkHandler.acceptUrl("vnd.youtube.launch:jZViOEv90dI")); + assertTrue(linkHandler.acceptUrl("https://music.youtube.com/watch?v=O0EDx9WAelc")); } @Test From 06016d1ae3dc4cf50ab5943c6593dd7cb05ad585 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Mon, 16 Sep 2019 23:15:54 +0200 Subject: [PATCH 17/25] Fix YouTube subscriber count Modify test to fail on too small subscriber count --- .../extractors/YoutubeChannelExtractor.java | 7 +++-- .../schabi/newpipe/extractor/utils/Utils.java | 29 +++++++++++++++++++ .../youtube/YoutubeChannelExtractorTest.java | 2 ++ 3 files changed, 35 insertions(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index 624cba670..9ac93deec 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -49,7 +49,7 @@ import java.util.ArrayList; public class YoutubeChannelExtractor extends ChannelExtractor { /*package-private*/ static final String CHANNEL_URL_BASE = "https://www.youtube.com/channel/"; private static final String CHANNEL_FEED_BASE = "https://www.youtube.com/feeds/videos.xml?channel_id="; - private static final String CHANNEL_URL_PARAMETERS = "/videos?view=0&flow=list&sort=dd&live_view=10000"; + private static final String CHANNEL_URL_PARAMETERS = "/videos?view=0&flow=list&sort=dd&live_view=10000&gl=US&hl=en"; private Document doc; @@ -135,10 +135,11 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Override public long getSubscriberCount() throws ParsingException { - final Element el = doc.select("span[class*=\"yt-subscription-button-subscriber-count\"]").first(); + final String el = doc.select("span[class*=\"yt-subscription-button-subscriber-count\"]") + .first().attr("title"); if (el != null) { try { - return Long.parseLong(Utils.removeNonDigitCharacters(el.text())); + return Utils.mixedNumberWordToLong(el); } catch (NumberFormatException e) { throw new ParsingException("Could not get subscriber count", e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java index 3dd01c49c..ecf4017d1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java @@ -27,6 +27,35 @@ public class Utils { return toRemove.replaceAll("\\D+", ""); } + /** + *

Convert a mixed number word to a long.

+ *

Examples:

+ *
    + *
  • 123 -> 123
  • + *
  • 1.23K -> 1230
  • + *
  • 1.23M -> 1230000
  • + *
+ * @param numberWord string to be converted to a long + * @return a long + * @throws NumberFormatException + * @throws ParsingException + */ + public static long mixedNumberWordToLong(String numberWord) throws NumberFormatException, ParsingException { + String multiplier = ""; + try { + multiplier = Parser.matchGroup("[\\d]+([\\.,][\\d]+)?([KMkm])+", numberWord, 2); + } catch(ParsingException ignored) {} + double count = Double.parseDouble(Parser.matchGroup1("([\\d]+([\\.,][\\d]+)?)", numberWord)); + switch (multiplier.toUpperCase()) { + case "K": + return (long) (count * 1e3); + case "M": + return (long) (count * 1e6); + default: + return (long) (count); + } + } + /** * Check if the url matches the pattern. * diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java index f124bed7c..9e8737722 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java @@ -105,6 +105,7 @@ public class YoutubeChannelExtractorTest { @Test public void testSubscriberCount() throws Exception { assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 0); + assertTrue("Subscriber count too small", extractor.getSubscriberCount() >= 4e6); } } @@ -195,6 +196,7 @@ public class YoutubeChannelExtractorTest { @Test public void testSubscriberCount() throws Exception { assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 0); + assertTrue("Subscriber count too small", extractor.getSubscriberCount() >= 10e6); } } From 6d504e08836b0b65fb737f92d28bfc413cfb34c0 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Tue, 17 Sep 2019 09:14:59 +0200 Subject: [PATCH 18/25] Add test for mixedNumberWordToLong method Add Billion to mixedNumberWordToLong --- .../schabi/newpipe/extractor/utils/Utils.java | 7 +++++-- .../newpipe/extractor/utils/UtilsTest.java | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/utils/UtilsTest.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java index ecf4017d1..1f8da13b2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java @@ -43,14 +43,17 @@ public class Utils { public static long mixedNumberWordToLong(String numberWord) throws NumberFormatException, ParsingException { String multiplier = ""; try { - multiplier = Parser.matchGroup("[\\d]+([\\.,][\\d]+)?([KMkm])+", numberWord, 2); + multiplier = Parser.matchGroup("[\\d]+([\\.,][\\d]+)?([KMBkmb])+", numberWord, 2); } catch(ParsingException ignored) {} - double count = Double.parseDouble(Parser.matchGroup1("([\\d]+([\\.,][\\d]+)?)", numberWord)); + double count = Double.parseDouble(Parser.matchGroup1("([\\d]+([\\.,][\\d]+)?)", numberWord) + .replace(",", ".")); switch (multiplier.toUpperCase()) { case "K": return (long) (count * 1e3); case "M": return (long) (count * 1e6); + case "B": + return (long) (count * 1e9); default: return (long) (count); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/utils/UtilsTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/utils/UtilsTest.java new file mode 100644 index 000000000..578867445 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/utils/UtilsTest.java @@ -0,0 +1,18 @@ +package org.schabi.newpipe.extractor.utils; + +import com.grack.nanojson.JsonParserException; +import org.junit.Test; +import org.schabi.newpipe.extractor.exceptions.ParsingException; + +import static org.junit.Assert.assertEquals; + +public class UtilsTest { + @Test + public void testMixedNumberWordToLong() throws JsonParserException, ParsingException { + assertEquals(10, Utils.mixedNumberWordToLong("10")); + assertEquals(10.5e3, Utils.mixedNumberWordToLong("10.5K"), 0.0); + assertEquals(10.5e6, Utils.mixedNumberWordToLong("10.5M"), 0.0); + assertEquals(10.5e6, Utils.mixedNumberWordToLong("10,5M"), 0.0); + assertEquals(1.5e9, Utils.mixedNumberWordToLong("1,5B"), 0.0); + } +} From 94e7f0d3ab34376d05bd25f522f85228fb128b28 Mon Sep 17 00:00:00 2001 From: Stypox Date: Tue, 27 Aug 2019 12:01:00 +0200 Subject: [PATCH 19/25] Fix fallback method is not tried on exception in YoutubeChannelInfoItem.getUrl() --- .../YoutubeChannelInfoItemExtractor.java | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java index d5247cad8..f6eab98f3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java @@ -56,19 +56,25 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor @Override public String getUrl() throws ParsingException { - String buttonTrackingUrl = el.select("button[class*=\"yt-uix-button\"]").first() - .attr("abs:data-href"); + try { + String buttonTrackingUrl = el.select("button[class*=\"yt-uix-button\"]").first() + .attr("abs:data-href"); - Pattern channelIdPattern = Pattern.compile("(?:.*?)\\%252Fchannel\\%252F([A-Za-z0-9\\-\\_]+)(?:.*)"); - Matcher match = channelIdPattern.matcher(buttonTrackingUrl); + Pattern channelIdPattern = Pattern.compile("(?:.*?)\\%252Fchannel\\%252F([A-Za-z0-9\\-\\_]+)(?:.*)"); + Matcher match = channelIdPattern.matcher(buttonTrackingUrl); - if (match.matches()) { - return YoutubeChannelExtractor.CHANNEL_URL_BASE + match.group(1); - } else { + if (match.matches()) { + return YoutubeChannelExtractor.CHANNEL_URL_BASE + match.group(1); + } + } catch(Throwable ignored) {} + + try { // fallback method just in case youtube changes things; it should never run and tests will fail // provides an url with "/user/NAME", that is inconsistent with stream and channel extractor return el.select("a[class*=\"yt-uix-tile-link\"]").first() .attr("abs:href"); + } catch (Throwable e) { + throw new ParsingException("Could not get channel url", e); } } From db3596c8189f9463086bd8178233e360af7bfbc1 Mon Sep 17 00:00:00 2001 From: Stypox Date: Tue, 27 Aug 2019 13:15:06 +0200 Subject: [PATCH 20/25] Fix "Could not get id" for channels w/o "Subscribe" button --- .../services/youtube/extractors/YoutubeChannelExtractor.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index 9ac93deec..41f354fd9 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -82,6 +82,11 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Nonnull @Override public String getId() throws ParsingException { + try { + return doc.select("meta[itemprop=\"channelId\"]").first().attr("content"); + } catch (Exception ignored) {} + + // fallback method; does not work with channels that have no "Subscribe" button (e.g. EminemVEVO) try { Element element = doc.getElementsByClass("yt-uix-subscription-button").first(); if (element == null) element = doc.getElementsByClass("yt-uix-subscription-preferences-button").first(); From 35921345d9cf227b84858d186203034e83b7c74f Mon Sep 17 00:00:00 2001 From: Stypox Date: Tue, 27 Aug 2019 13:16:08 +0200 Subject: [PATCH 21/25] Use Exception instead of Throwable (more consistent) --- .../youtube/extractors/YoutubeChannelInfoItemExtractor.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java index f6eab98f3..03a95abba 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java @@ -66,14 +66,14 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor if (match.matches()) { return YoutubeChannelExtractor.CHANNEL_URL_BASE + match.group(1); } - } catch(Throwable ignored) {} + } catch(Exception ignored) {} try { // fallback method just in case youtube changes things; it should never run and tests will fail // provides an url with "/user/NAME", that is inconsistent with stream and channel extractor return el.select("a[class*=\"yt-uix-tile-link\"]").first() .attr("abs:href"); - } catch (Throwable e) { + } catch (Exception e) { throw new ParsingException("Could not get channel url", e); } } From f6088c4fc1605315fc2c24b2eca96361c667a6a9 Mon Sep 17 00:00:00 2001 From: Stypox Date: Tue, 27 Aug 2019 13:27:11 +0200 Subject: [PATCH 22/25] Add test for Eminem channel (it has no "Subscribe" button) --- .../youtube/YoutubeChannelExtractorTest.java | 99 ++++++++++++++++++- 1 file changed, 97 insertions(+), 2 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java index 9e8737722..c25fdd9cc 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java @@ -392,6 +392,100 @@ public class YoutubeChannelExtractorTest { } } + // this channel has no "Subscribe" button + public static class EminemVEVO implements BaseChannelExtractorTest { + private static YoutubeChannelExtractor extractor; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(Downloader.getInstance(), new Localization("GB", "en")); + extractor = (YoutubeChannelExtractor) YouTube + .getChannelExtractor("https://www.youtube.com/user/EminemVEVO/"); + extractor.fetchPage(); + } + + /*////////////////////////////////////////////////////////////////////////// + // Extractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testServiceId() { + assertEquals(YouTube.getServiceId(), extractor.getServiceId()); + } + + @Test + public void testName() throws Exception { + assertEquals("EminemVEVO", extractor.getName()); + } + + @Test + public void testId() throws Exception { + assertEquals("UC20vb-R_px4CguHzzBPhoyQ", extractor.getId()); + } + + @Test + public void testUrl() throws ParsingException { + assertEquals("https://www.youtube.com/channel/UC20vb-R_px4CguHzzBPhoyQ", extractor.getUrl()); + } + + @Test + public void testOriginalUrl() throws ParsingException { + assertEquals("https://www.youtube.com/user/EminemVEVO/", extractor.getOriginalUrl()); + } + + /*////////////////////////////////////////////////////////////////////////// + // ListExtractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testRelatedItems() throws Exception { + defaultTestRelatedItems(extractor, YouTube.getServiceId()); + } + + @Test + public void testMoreRelatedItems() throws Exception { + defaultTestMoreItems(extractor, YouTube.getServiceId()); + } + + /*////////////////////////////////////////////////////////////////////////// + // ChannelExtractor + //////////////////////////////////////////////////////////////////////////*/ + + @Test + public void testDescription() throws Exception { + final String description = extractor.getDescription(); + assertTrue(description, description.contains("Eminem on Vevo")); + } + + @Test + public void testAvatarUrl() throws Exception { + String avatarUrl = extractor.getAvatarUrl(); + assertIsSecureUrl(avatarUrl); + assertTrue(avatarUrl, avatarUrl.contains("yt3")); + } + + @Test + public void testBannerUrl() throws Exception { + String bannerUrl = extractor.getBannerUrl(); + assertIsSecureUrl(bannerUrl); + assertTrue(bannerUrl, bannerUrl.contains("yt3")); + } + + @Test + public void testFeedUrl() throws Exception { + assertEquals("https://www.youtube.com/feeds/videos.xml?channel_id=UC20vb-R_px4CguHzzBPhoyQ", extractor.getFeedUrl()); + } + + @Test + public void testSubscriberCount() throws Exception { + // there is no "Subscribe" button + long subscribers = extractor.getSubscriberCount(); + assertEquals("Wrong subscriber count", -1, subscribers); + } + } + + + public static class RandomChannel implements BaseChannelExtractorTest { private static YoutubeChannelExtractor extractor; @@ -483,8 +577,9 @@ public class YoutubeChannelExtractorTest { @Test public void testSubscriberCount() throws Exception { - assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 50); + long subscribers = extractor.getSubscriberCount(); + assertTrue("Wrong subscriber count: " + subscribers, subscribers >= 50); } } -}; +} From d1cd341592718dc6735adedac34f0637a4f80610 Mon Sep 17 00:00:00 2001 From: Stypox Date: Tue, 27 Aug 2019 13:56:19 +0200 Subject: [PATCH 23/25] Change comment --- .../youtube/extractors/YoutubeChannelInfoItemExtractor.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java index 03a95abba..a687c0504 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java @@ -68,9 +68,9 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor } } catch(Exception ignored) {} + // fallback method for channels without "Subscribe" button (or just in case yt changes things) + // provides an url with "/user/NAME", inconsistent with stream and channel extractor: tests will fail try { - // fallback method just in case youtube changes things; it should never run and tests will fail - // provides an url with "/user/NAME", that is inconsistent with stream and channel extractor return el.select("a[class*=\"yt-uix-tile-link\"]").first() .attr("abs:href"); } catch (Exception e) { From 0710f31a3914e3ac60ff0511e06363b636d9c532 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Mon, 23 Sep 2019 10:44:17 +0200 Subject: [PATCH 24/25] Fix TeamNewPipe/NewPipeExtractor#197 --- .../extractors/YoutubeStreamExtractor.java | 27 +++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index bec12af88..5351e6112 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -575,21 +575,26 @@ public class YoutubeStreamExtractor extends StreamExtractor { */ @Override public String getErrorMessage() { - String errorMessage = doc.select("h1[id=\"unavailable-message\"]").first().text(); StringBuilder errorReason; + Element errorElement = doc.select("h1[id=\"unavailable-message\"]").first(); - if (errorMessage == null || errorMessage.isEmpty()) { + if (errorElement == null) { errorReason = null; - } else if (errorMessage.contains("GEMA")) { - // Gema sometimes blocks youtube music content in germany: - // https://www.gema.de/en/ - // Detailed description: - // https://en.wikipedia.org/wiki/GEMA_%28German_organization%29 - errorReason = new StringBuilder("GEMA"); } else { - errorReason = new StringBuilder(errorMessage); - errorReason.append(" "); - errorReason.append(doc.select("[id=\"unavailable-submessage\"]").first().text()); + String errorMessage = errorElement.text(); + if (errorMessage == null || errorMessage.isEmpty()) { + errorReason = null; + } else if (errorMessage.contains("GEMA")) { + // Gema sometimes blocks youtube music content in germany: + // https://www.gema.de/en/ + // Detailed description: + // https://en.wikipedia.org/wiki/GEMA_%28German_organization%29 + errorReason = new StringBuilder("GEMA"); + } else { + errorReason = new StringBuilder(errorMessage); + errorReason.append(" "); + errorReason.append(doc.select("[id=\"unavailable-submessage\"]").first().text()); + } } return errorReason != null ? errorReason.toString() : null; From 8ab48c62b95fa3d879533c113f9f265c965d784f Mon Sep 17 00:00:00 2001 From: TobiGr Date: Wed, 25 Sep 2019 08:56:39 +0200 Subject: [PATCH 25/25] [YouTube] Fix NPE in ChennelExtractor.getSubsciberCount() --- .../youtube/extractors/YoutubeChannelExtractor.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index 41f354fd9..38722fa52 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -140,11 +140,12 @@ public class YoutubeChannelExtractor extends ChannelExtractor { @Override public long getSubscriberCount() throws ParsingException { - final String el = doc.select("span[class*=\"yt-subscription-button-subscriber-count\"]") - .first().attr("title"); + + final Element el = doc.select("span[class*=\"yt-subscription-button-subscriber-count\"]").first(); if (el != null) { + String elTitle = el.attr("title"); try { - return Utils.mixedNumberWordToLong(el); + return Utils.mixedNumberWordToLong(elTitle); } catch (NumberFormatException e) { throw new ParsingException("Could not get subscriber count", e); }