From 645ee5a5ff773ff9fd845fd17e01489cc0b879bc Mon Sep 17 00:00:00 2001 From: John Zhen M Date: Tue, 15 Aug 2017 17:11:38 -0700 Subject: [PATCH] -Added playlist info item extraction for Youtube and Soundcloud. -Added playlist collection into search engine. -Fixed stream info duration exception when parsing 0 second video. --- .../search/InfoItemSearchCollector.java | 15 +++ .../SoundcloudPlaylistInfoItemExtractor.java | 89 ++++++++++++++++++ .../soundcloud/SoundcloudSearchEngine.java | 8 +- .../YoutubePlaylistInfoItemExtractor.java | 92 +++++++++++++++++++ .../services/youtube/YoutubeSearchEngine.java | 13 ++- .../YoutubeStreamInfoItemExtractor.java | 5 +- .../SoundcloudSearchEnginePlaylistTest.java | 80 ++++++++++++++++ .../youtube/YoutubeSearchEngineAllTest.java | 3 +- .../YoutubeSearchEnginePlaylistTest.java | 81 ++++++++++++++++ 9 files changed, 378 insertions(+), 8 deletions(-) create mode 100644 src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistInfoItemExtractor.java create mode 100644 src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistInfoItemExtractor.java create mode 100644 src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEnginePlaylistTest.java create mode 100644 src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEnginePlaylistTest.java diff --git a/src/main/java/org/schabi/newpipe/extractor/search/InfoItemSearchCollector.java b/src/main/java/org/schabi/newpipe/extractor/search/InfoItemSearchCollector.java index 5c5eff884..8507f0ac8 100644 --- a/src/main/java/org/schabi/newpipe/extractor/search/InfoItemSearchCollector.java +++ b/src/main/java/org/schabi/newpipe/extractor/search/InfoItemSearchCollector.java @@ -5,6 +5,8 @@ import org.schabi.newpipe.extractor.channel.ChannelInfoItemCollector; import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.FoundAdException; +import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemCollector; +import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector; import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; @@ -32,6 +34,7 @@ public class InfoItemSearchCollector extends InfoItemCollector { private String suggestion = ""; private StreamInfoItemCollector streamCollector; private ChannelInfoItemCollector userCollector; + private PlaylistInfoItemCollector playlistCollector; private SearchResult result = new SearchResult(); @@ -39,6 +42,7 @@ public class InfoItemSearchCollector extends InfoItemCollector { super(serviceId); streamCollector = new StreamInfoItemCollector(serviceId); userCollector = new ChannelInfoItemCollector(serviceId); + playlistCollector = new PlaylistInfoItemCollector(serviceId); } public void setSuggestion(String suggestion) { @@ -49,6 +53,7 @@ public class InfoItemSearchCollector extends InfoItemCollector { addFromCollector(userCollector); addFromCollector(streamCollector); + addFromCollector(playlistCollector); result.suggestion = suggestion; result.errors = getErrors(); @@ -74,4 +79,14 @@ public class InfoItemSearchCollector extends InfoItemCollector { addError(e); } } + + public void commit(PlaylistInfoItemExtractor extractor) { + try { + result.resultList.add(playlistCollector.extract(extractor)); + } catch (FoundAdException ae) { + System.err.println("Found ad"); + } catch (Exception e) { + addError(e); + } + } } diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistInfoItemExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistInfoItemExtractor.java new file mode 100644 index 000000000..fcfd8851e --- /dev/null +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistInfoItemExtractor.java @@ -0,0 +1,89 @@ +package org.schabi.newpipe.extractor.services.soundcloud; + +import com.github.openjson.JSONArray; +import com.github.openjson.JSONObject; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor; + +public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtractor { + private static final String USER_KEY = "user"; + private static final String AVATAR_URL_KEY = "avatar_url"; + private static final String ARTWORK_URL_KEY = "artwork_url"; + private static final String NULL_VALUE = "null"; + + private JSONObject searchResult; + + public SoundcloudPlaylistInfoItemExtractor(JSONObject searchResult) { + this.searchResult = searchResult; + } + + @Override + public String getName() throws ParsingException { + try { + return searchResult.getString("title"); + } catch (Exception e) { + throw new ParsingException("Failed to extract playlist name", e); + } + } + + @Override + public String getUrl() throws ParsingException { + try { + return searchResult.getString("permalink_url"); + } catch (Exception e) { + throw new ParsingException("Failed to extract playlist name", e); + } + } + + @Override + public String getThumbnailUrl() throws ParsingException { + // Over-engineering at its finest + try { + final String artworkUrl = searchResult.optString(ARTWORK_URL_KEY); + if (!artworkUrl.isEmpty() && !artworkUrl.equals(NULL_VALUE)) return artworkUrl; + + // Look for artwork url inside the track list + final JSONArray tracks = searchResult.optJSONArray("tracks"); + if (tracks == null) return null; + for (int i = 0; i < tracks.length(); i++) { + if (tracks.isNull(i)) continue; + final JSONObject track = tracks.optJSONObject(i); + if (track == null) continue; + + // First look for track artwork url + final String url = track.optString(ARTWORK_URL_KEY); + if (!url.isEmpty() && !url.equals(NULL_VALUE)) return url; + + // Then look for track creator avatar url + final JSONObject creator = track.getJSONObject(USER_KEY); + final String creatorAvatar = creator.optString(AVATAR_URL_KEY); + if (!creatorAvatar.isEmpty() && !creatorAvatar.equals(NULL_VALUE)) return creatorAvatar; + } + + // Last resort, use user avatar url. If still not found, then throw exception. + final JSONObject user = searchResult.getJSONObject(USER_KEY); + return user.getString(AVATAR_URL_KEY); + } catch (Exception e) { + throw new ParsingException("Failed to extract playlist thumbnail url", e); + } + } + + @Override + public String getUploaderName() throws ParsingException { + try { + final JSONObject user = searchResult.getJSONObject(USER_KEY); + return user.optString("username"); + } catch (Exception e) { + throw new ParsingException("Failed to extract playlist uploader", e); + } + } + + @Override + public long getStreamCount() throws ParsingException { + try { + return Long.parseLong(searchResult.optString("track_count")); + } catch (Exception e) { + throw new ParsingException("Failed to extract playlist stream count", e); + } + } +} diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEngine.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEngine.java index 2197b2551..9a9169a45 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEngine.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEngine.java @@ -27,10 +27,12 @@ public class SoundcloudSearchEngine extends SearchEngine { String url = "https://api-v2.soundcloud.com/search"; - if (filter.contains(Filter.STREAM) && !filter.contains(Filter.CHANNEL)) { + if (filter.contains(Filter.STREAM) && filter.size() == 1) { url += "/tracks"; - } else if (!filter.contains(Filter.STREAM) && filter.contains(Filter.CHANNEL)) { + } else if (filter.contains(Filter.CHANNEL) && filter.size() == 1) { url += "/users"; + } else if (filter.contains(Filter.PLAYLIST) && filter.size() == 1) { + url += "/playlists"; } url += "?q=" + URLEncoder.encode(query, CHARSET_UTF_8) @@ -53,6 +55,8 @@ public class SoundcloudSearchEngine extends SearchEngine { collector.commit(new SoundcloudChannelInfoItemExtractor(searchResult)); } else if (kind.equals("track")) { collector.commit(new SoundcloudStreamInfoItemExtractor(searchResult)); + } else if (kind.equals("playlist")) { + collector.commit(new SoundcloudPlaylistInfoItemExtractor(searchResult)); } } diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistInfoItemExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistInfoItemExtractor.java new file mode 100644 index 000000000..552f58f41 --- /dev/null +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistInfoItemExtractor.java @@ -0,0 +1,92 @@ +package org.schabi.newpipe.extractor.services.youtube; + +import org.jsoup.nodes.Element; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor; +import org.schabi.newpipe.extractor.utils.Utils; + +public class YoutubePlaylistInfoItemExtractor implements PlaylistInfoItemExtractor { + private Element el; + + public YoutubePlaylistInfoItemExtractor(Element el) { + this.el = el; + } + + @Override + public String getThumbnailUrl() throws ParsingException { + String url; + + try { + Element te = el.select("div[class=\"yt-thumb video-thumb\"]").first() + .select("img").first(); + url = te.attr("abs:src"); + + if (url.contains(".gif")) { + url = te.attr("abs:data-thumb"); + } + } catch (Exception e) { + throw new ParsingException("Failed to extract playlist thumbnail url", e); + } + + return url; + } + + @Override + public String getName() throws ParsingException { + String name; + try { + final Element title = el.select("[class=\"yt-lockup-title\"]").first() + .select("a").first(); + + name = title == null ? "" : title.text(); + } catch (Exception e) { + throw new ParsingException("Failed to extract playlist name", e); + } + + return name; + } + + @Override + public String getUrl() throws ParsingException { + String url; + + try { + final Element href = el.select("div[class=\"yt-lockup-meta\"]").first() + .select("a").first(); + + url = href.attr("abs:href"); + } catch (Exception e) { + throw new ParsingException("Failed to extract playlist url", e); + } + + return url; + } + + @Override + public String getUploaderName() throws ParsingException { + String name; + + try { + final Element div = el.select("div[class=\"yt-lockup-byline\"]").first() + .select("a").first(); + + name = div.text(); + } catch (Exception e) { + throw new ParsingException("Failed to extract playlist uploader", e); + } + + return name; + } + + @Override + public long getStreamCount() throws ParsingException { + try { + final Element count = el.select("span[class=\"formatted-video-count-label\"]").first() + .select("b").first(); + + return count == null ? 0 : Long.parseLong(Utils.removeNonDigitCharacters(count.text())); + } catch (Exception e) { + throw new ParsingException("Failed to extract playlist stream count", e); + } + } +} diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngine.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngine.java index 62beae1dd..bc1c8684f 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngine.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngine.java @@ -57,10 +57,13 @@ public class YoutubeSearchEngine extends SearchEngine { String url = "https://www.youtube.com/results" + "?q=" + URLEncoder.encode(query, CHARSET_UTF_8) + "&page=" + Integer.toString(page + 1); - if (filter.contains(Filter.STREAM) && !filter.contains(Filter.CHANNEL)) { - url += "&sp=EgIQAQ%253D%253D"; - } else if (!filter.contains(Filter.STREAM) && filter.contains(Filter.CHANNEL)) { - url += "&sp=EgIQAg%253D%253D"; + + if (filter.contains(Filter.STREAM) && filter.size() == 1) { + url += "&sp=EgIQAVAU"; + } else if (filter.contains(Filter.CHANNEL) && filter.size() == 1) { + url += "&sp=EgIQAlAU"; //EgIQA( lowercase L )AU + } else if (filter.contains(Filter.PLAYLIST) && filter.size() == 1) { + url += "&sp=EgIQA1AU"; //EgIQA( one )AU } String site; @@ -105,6 +108,8 @@ public class YoutubeSearchEngine extends SearchEngine { collector.commit(new YoutubeStreamInfoItemExtractor(el)); } else if ((el = item.select("div[class*=\"yt-lockup-channel\"]").first()) != null) { collector.commit(new YoutubeChannelInfoItemExtractor(el)); + } else if ((el = item.select("div[class*=\"yt-lockup-playlist\"]").first()) != null) { + collector.commit(new YoutubePlaylistInfoItemExtractor(el)); } else { // noinspection ConstantConditions // simply ignore not known items diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamInfoItemExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamInfoItemExtractor.java index 2e4e7cff1..8b48c1f28 100644 --- a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamInfoItemExtractor.java +++ b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamInfoItemExtractor.java @@ -74,7 +74,10 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor { try { if (getStreamType() == StreamType.LIVE_STREAM) return -1; - return YoutubeParsingHelper.parseDurationString(item.select("span[class*=\"video-time\"]").first().text()); + final Element duration = item.select("span[class*=\"video-time\"]").first(); + // apparently on youtube, video-time element will not show up if the video has a duration of 00:00 + // see: https://www.youtube.com/results?sp=EgIQAVAU&q=asdfgf + return duration == null ? 0 : YoutubeParsingHelper.parseDurationString(duration.text()); } catch (Exception e) { throw new ParsingException("Could not get Duration: " + getUrl(), e); } diff --git a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEnginePlaylistTest.java b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEnginePlaylistTest.java new file mode 100644 index 000000000..df83ac455 --- /dev/null +++ b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEnginePlaylistTest.java @@ -0,0 +1,80 @@ +package org.schabi.newpipe.extractor.services.soundcloud; + +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.search.SearchEngine; +import org.schabi.newpipe.extractor.search.SearchResult; + +import java.util.EnumSet; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; + + +/* + * Created by Christian Schabesberger on 29.12.15. + * + * Copyright (C) Christian Schabesberger 2015 + * YoutubeSearchEngineStreamTest.java is part of NewPipe. + * + * NewPipe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe. If not, see . + */ + +/** + * Test for {@link SearchEngine} + */ +public class SoundcloudSearchEnginePlaylistTest { + private SearchResult result; + + @Before + public void setUp() throws Exception { + NewPipe.init(Downloader.getInstance()); + SearchEngine engine = SoundCloud.getService().getSearchEngine(); + + // Search by country not yet implemented + result = engine.search("parkmemme", 0, "", EnumSet.of(SearchEngine.Filter.PLAYLIST)) + .getSearchResult(); + } + + @Test + public void testResultList() { + assertFalse(result.resultList.isEmpty()); + } + + @Test + public void testUserItemType() { + for (InfoItem infoItem : result.resultList) { + assertEquals(InfoItem.InfoType.PLAYLIST, infoItem.info_type); + } + } + + @Test + public void testResultErrors() { + if (!result.errors.isEmpty()) for (Throwable error : result.errors) error.printStackTrace(); + assertTrue(result.errors == null || result.errors.isEmpty()); + } + + @Ignore + @Test + public void testSuggestion() { + //todo write a real test + assertTrue(result.suggestion != null); + } +} diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngineAllTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngineAllTest.java index 0e2c61ec8..daaa6b698 100644 --- a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngineAllTest.java +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngineAllTest.java @@ -50,7 +50,8 @@ public class YoutubeSearchEngineAllTest { // keep in mind that the suggestions can change by country (the parameter "de") result = engine.search("asdgff", 0, "de", EnumSet.of(SearchEngine.Filter.CHANNEL, - SearchEngine.Filter.STREAM)).getSearchResult(); + SearchEngine.Filter.STREAM, + SearchEngine.Filter.PLAYLIST)).getSearchResult(); } @Test diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEnginePlaylistTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEnginePlaylistTest.java new file mode 100644 index 000000000..5e6e2a5b4 --- /dev/null +++ b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEnginePlaylistTest.java @@ -0,0 +1,81 @@ +package org.schabi.newpipe.extractor.services.youtube; + +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.search.SearchEngine; +import org.schabi.newpipe.extractor.search.SearchResult; + +import java.util.EnumSet; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.schabi.newpipe.extractor.ServiceList.YouTube; + + +/* + * Created by Christian Schabesberger on 29.12.15. + * + * Copyright (C) Christian Schabesberger 2015 + * YoutubeSearchEngineStreamTest.java is part of NewPipe. + * + * NewPipe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe. If not, see . + */ + +/** + * Test for {@link SearchEngine} + */ +public class YoutubeSearchEnginePlaylistTest { + private SearchResult result; + + @Before + public void setUp() throws Exception { + NewPipe.init(Downloader.getInstance()); + SearchEngine engine = YouTube.getService().getSearchEngine(); + + // Youtube will suggest "gronkh" instead of "grrunkh" + // keep in mind that the suggestions can change by country (the parameter "de") + result = engine.search("grrunkh", 0, "de", + EnumSet.of(SearchEngine.Filter.PLAYLIST)).getSearchResult(); + } + + @Test + public void testResultList() { + assertFalse(result.resultList.isEmpty()); + } + + @Test + public void testUserItemType() { + for (InfoItem infoItem : result.resultList) { + assertEquals(InfoItem.InfoType.PLAYLIST, infoItem.info_type); + } + } + + @Test + public void testResultErrors() { + if (!result.errors.isEmpty()) for (Throwable error : result.errors) error.printStackTrace(); + assertTrue(result.errors == null || result.errors.isEmpty()); + } + + @Ignore + @Test + public void testSuggestion() { + //todo write a real test + assertTrue(result.suggestion != null); + } +}