diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 2ad2f9049..a49943915 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -166,8 +166,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { throws MalformedURLException, UnsupportedEncodingException, ParsingException { final Document description = Jsoup.parse(descriptionHtml, getUrl()); for(Element a : description.select("a")) { - final URL redirectLink = new URL( - a.attr("abs:href")); + final String rawUrl = a.attr("abs:href"); + final URL redirectLink = new URL(rawUrl); final String queryString = redirectLink.getQuery(); if(queryString != null) { // if the query string is null we are not dealing with a redirect link, @@ -179,11 +179,15 @@ public class YoutubeStreamExtractor extends StreamExtractor { // if link is null the a tag is a hashtag. // They refer to the youtube search. We do not handle them. a.text(link); + a.attr("href", link); } else if(redirectLink.toString().contains("https://www.youtube.com/")) { a.text(redirectLink.toString()); + a.attr("href", redirectLink.toString()); } } else if(redirectLink.toString().contains("https://www.youtube.com/")) { + descriptionHtml = descriptionHtml.replace(rawUrl, redirectLink.toString()); a.text(redirectLink.toString()); + a.attr("href", redirectLink.toString()); } } return description.select("body").first().html(); @@ -206,29 +210,40 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public long getLength() throws ParsingException { assertPageFetched(); - if(playerArgs != null) { - try { - long returnValue = Long.parseLong(playerArgs.get("length_seconds") + ""); - if (returnValue >= 0) return returnValue; - } catch (Exception ignored) { - // Try other method... + + final JsonObject playerResponse; + try { + final String pr; + if(playerArgs != null) { + pr = playerArgs.getString("player_response"); + } else { + pr = videoInfoPage.get("player_response"); } - } - - String lengthString = videoInfoPage.get("length_seconds"); - try { - return Long.parseLong(lengthString); - } catch (Exception ignored) { - // Try other method... - } - - // TODO: 25.11.17 Implement a way to get the length for age restricted videos #44 - try { - // Fallback to HTML method - return Long.parseLong(doc.select("div[class~=\"ytp-progress-bar\"][role=\"slider\"]").first() - .attr("aria-valuemax")); + playerResponse = JsonParser.object() + .from(pr); } catch (Exception e) { - throw new ParsingException("Could not get video length", e); + throw new ParsingException("Could not get playerResponse", e); + } + + // try getting duration from playerargs + try { + String durationMs = playerResponse + .getObject("streamingData") + .getArray("formats") + .getObject(0) + .getString("approxDurationMs"); + return Long.parseLong(durationMs)/1000; + } catch (Exception e) { + } + + //try getting value from age gated video + try { + String duration = playerResponse + .getObject("videoDetails") + .getString("lengthSeconds"); + return Long.parseLong(duration); + } catch (Exception e) { + throw new ParsingException("Every methode to get the duration has failed: ", e); } } @@ -597,6 +612,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { final String playerUrl; // Check if the video is age restricted if (pageContent.contains(" currentPage = defaultTestMoreItems(extractor, ServiceList.SoundCloud.getServiceId()); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java index 3970efde8..c8a2a841c 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java @@ -39,27 +39,27 @@ public class SoundcloudStreamExtractorDefaultTest { @Test public void testGetValidTimeStamp() throws IOException, ExtractionException { StreamExtractor extractor = SoundCloud.getStreamExtractor("https://soundcloud.com/liluzivert/do-what-i-want-produced-by-maaly-raw-don-cannon#t=69"); - assertEquals(extractor.getTimeStamp() + "", "69"); + assertEquals("69", extractor.getTimeStamp() + ""); } @Test public void testGetTitle() throws ParsingException { - assertEquals(extractor.getName(), "Do What I Want [Produced By Maaly Raw + Don Cannon]"); + assertEquals("Do What I Want [Produced By Maaly Raw + Don Cannon]", extractor.getName()); } @Test public void testGetDescription() throws ParsingException { - assertEquals(extractor.getDescription(), "The Perfect LUV Tape®️"); + assertEquals("The Perfect LUV Tape®️", extractor.getDescription()); } @Test public void testGetUploaderName() throws ParsingException { - assertEquals(extractor.getUploaderName(), "LIL UZI VERT"); + assertEquals("LIL UZI VERT", extractor.getUploaderName()); } @Test public void testGetLength() throws ParsingException { - assertEquals(extractor.getLength(), 175); + assertEquals(175, extractor.getLength()); } @Test diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorAgeRestrictedTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorAgeRestrictedTest.java index 8a91887af..f41750d7e 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorAgeRestrictedTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorAgeRestrictedTest.java @@ -71,10 +71,9 @@ public class YoutubeStreamExtractorAgeRestrictedTest { assertFalse(extractor.getUploaderName().isEmpty()); } - @Ignore // Currently there is no way get the length from restricted videos @Test public void testGetLength() throws ParsingException { - assertTrue(extractor.getLength() > 0); + assertEquals(1789, extractor.getLength()); } @Test @@ -97,8 +96,6 @@ public class YoutubeStreamExtractorAgeRestrictedTest { assertIsSecureUrl(extractor.getUploaderAvatarUrl()); } - // FIXME: 25.11.17 Are there no streams or are they not listed? - @Ignore @Test public void testGetAudioStreams() throws IOException, ExtractionException { // audio streams are not always necessary diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorControversialTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorControversialTest.java index a300d6228..8fd991154 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorControversialTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorControversialTest.java @@ -71,10 +71,9 @@ public class YoutubeStreamExtractorControversialTest { assertFalse(extractor.getUploaderName().isEmpty()); } - @Ignore // Currently there is no way get the length from restricted videos @Test public void testGetLength() throws ParsingException { - assertTrue(extractor.getLength() > 0); + assertEquals(219, extractor.getLength()); } @Test @@ -97,8 +96,6 @@ public class YoutubeStreamExtractorControversialTest { assertIsSecureUrl(extractor.getUploaderAvatarUrl()); } - // FIXME: 25.11.17 Are there no streams or are they not listed? - @Ignore @Test public void testGetAudioStreams() throws IOException, ExtractionException { // audio streams are not always necessary @@ -113,17 +110,15 @@ public class YoutubeStreamExtractorControversialTest { assertTrue(streams.size() > 0); } - @Ignore @Test public void testGetSubtitlesListDefault() throws IOException, ExtractionException { // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null - assertTrue(!extractor.getSubtitlesDefault().isEmpty()); + assertFalse(extractor.getSubtitlesDefault().isEmpty()); } - @Ignore @Test public void testGetSubtitlesList() throws IOException, ExtractionException { // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null - assertTrue(!extractor.getSubtitles(MediaFormat.TTML).isEmpty()); + assertFalse(extractor.getSubtitles(MediaFormat.TTML).isEmpty()); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java index f4d8a3540..bb160afcd 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java @@ -53,7 +53,7 @@ public class YoutubeStreamExtractorDefaultTest { public static void setUp() throws Exception { NewPipe.init(Downloader.getInstance(), new Localization("GB", "en")); extractor = (YoutubeStreamExtractor) YouTube - .getStreamExtractor("https://www.youtube.com/watch?v=rYEDA3JcQqw"); + .getStreamExtractor("https://www.youtube.com/watch?v=YQHsXMglC9A"); extractor.fetchPage(); } @@ -82,7 +82,7 @@ public class YoutubeStreamExtractorDefaultTest { @Test public void testGetFullLinksInDescriptlion() throws ParsingException { - assertTrue(extractor.getDescription().contains("http://smarturl.it/SubscribeAdele?IQid=yt")); + assertTrue(extractor.getDescription().contains("http://adele.com")); assertFalse(extractor.getDescription().contains("http://smarturl.it/SubscribeAdele?IQi...")); } @@ -95,7 +95,7 @@ public class YoutubeStreamExtractorDefaultTest { @Test public void testGetLength() throws ParsingException { - assertTrue(extractor.getLength() > 0); + assertEquals(366, extractor.getLength()); } @Test diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java index 14b94b510..9439312b2 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorChannelOnlyTest.java @@ -1,6 +1,7 @@ package org.schabi.newpipe.extractor.services.youtube.search; import org.junit.BeforeClass; +import org.junit.Ignore; import org.junit.Test; import org.schabi.newpipe.Downloader; import org.schabi.newpipe.extractor.InfoItem; @@ -53,6 +54,7 @@ public class YoutubeSearchExtractorChannelOnlyTest extends YoutubeSearchExtracto assertEquals("https://www.youtube.com/results?q=pewdiepie&sp=EgIQAlAU&gl=GB&page=2", extractor.getNextPageUrl()); } + @Ignore @Test public void testOnlyContainChannels() { for(InfoItem item : itemsPage.getItems()) { diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index ed88a042a..5c2d1cf01 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index ade732376..4b7e1f3d3 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,5 @@ -#Fri Jan 18 11:51:40 CET 2019 distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-5.5.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-4.2.1-all.zip diff --git a/gradlew b/gradlew index cccdd3d51..8e25e6c19 100755 --- a/gradlew +++ b/gradlew @@ -1,5 +1,21 @@ #!/usr/bin/env sh +# +# Copyright 2015 the original author or authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + ############################################################################## ## ## Gradle start up script for UN*X @@ -28,7 +44,7 @@ APP_NAME="Gradle" APP_BASE_NAME=`basename "$0"` # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS="" +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD="maximum" diff --git a/gradlew.bat b/gradlew.bat index e95643d6a..24467a141 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -1,3 +1,19 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + @if "%DEBUG%" == "" @echo off @rem ########################################################################## @rem @@ -14,7 +30,7 @@ set APP_BASE_NAME=%~n0 set APP_HOME=%DIRNAME% @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -set DEFAULT_JVM_OPTS= +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" @rem Find java.exe if defined JAVA_HOME goto findJavaFromJavaHome