From 812a78581142b23de930630566bff8aeb47f47de Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Mon, 2 Jul 2018 13:47:39 +0200 Subject: [PATCH] remove searchengine --- .../schabi/newpipe/extractor/ListInfo.java | 8 +- .../extractor/playlist/PlaylistInfo.java | 2 +- .../extractor/search/SearchEngine.java | 50 ------- .../newpipe/extractor/search/SearchInfo.java | 12 +- .../extractor/search/SearchResult.java | 71 ---------- .../soundcloud/SoundcloudSearchEngine.java | 84 ------------ .../soundcloud/SoundcloudSearchExtractor.java | 3 +- .../extractors/YoutubeSearchEngine.java | 124 ------------------ 8 files changed, 10 insertions(+), 344 deletions(-) delete mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchEngine.java delete mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchResult.java delete mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEngine.java delete mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchEngine.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/ListInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/ListInfo.java index 7a2d6e872..5f9a59bd5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/ListInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/ListInfo.java @@ -1,16 +1,14 @@ package org.schabi.newpipe.extractor; -import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.uih.ListUIHandler; -import java.util.ArrayList; import java.util.List; public abstract class ListInfo extends Info { private List relatedItems; private String nextPageUrl = null; - private List contentFilters = new ArrayList<>(); - private String sortFilter = ""; + private final List contentFilters; + private final String sortFilter; public ListInfo(int serviceId, String id, @@ -24,7 +22,7 @@ public abstract class ListInfo extends Info { this.sortFilter = sortFilter; } - public ListInfo(int serviceId, ListUIHandler listUrlIdHandler, String name) throws ParsingException { + public ListInfo(int serviceId, ListUIHandler listUrlIdHandler, String name) { super(serviceId, listUrlIdHandler, name); this.contentFilters = listUrlIdHandler.getContentFilters(); this.sortFilter = listUrlIdHandler.getSortFilter(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java index 5e3f94ddf..b4f3ebc1f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java @@ -15,7 +15,7 @@ import java.io.IOException; public class PlaylistInfo extends ListInfo { - public PlaylistInfo(int serviceId, ListUIHandler urlIdHandler, String name) throws ParsingException { + private PlaylistInfo(int serviceId, ListUIHandler urlIdHandler, String name) throws ParsingException { super(serviceId, urlIdHandler, name); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchEngine.java b/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchEngine.java deleted file mode 100644 index ce32c5f6f..000000000 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchEngine.java +++ /dev/null @@ -1,50 +0,0 @@ -package org.schabi.newpipe.extractor.search; - -import org.schabi.newpipe.extractor.exceptions.ExtractionException; - -import java.io.IOException; - -/* - * Created by Christian Schabesberger on 10.08.15. - * - * Copyright (C) Christian Schabesberger 2015 - * SearchEngine.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - -public abstract class SearchEngine { - public enum Filter { - ANY, STREAM, CHANNEL, PLAYLIST - } - - public static class NothingFoundException extends ExtractionException { - public NothingFoundException(String message) { - super(message); - } - } - - private final InfoItemsSearchCollector collector; - - public SearchEngine(int serviceId) { - collector = new InfoItemsSearchCollector(serviceId); - } - - protected InfoItemsSearchCollector getInfoItemSearchCollector() { - return collector; - } - - public abstract InfoItemsSearchCollector search(String query, int page, String contentCountry, Filter filter) - throws IOException, ExtractionException; -} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchInfo.java index 5c1583c7a..2d36bdf7a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchInfo.java @@ -2,21 +2,19 @@ package org.schabi.newpipe.extractor.search; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.ListInfo; -import org.schabi.newpipe.extractor.uih.ListUIHFactory; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.uih.ListUIHandler; +import org.schabi.newpipe.extractor.uih.SearchQIHandler; public class SearchInfo extends ListInfo { - private String searchString = ""; - private String searchSuggestion = ""; - + private String searchString; + private String searchSuggestion; public SearchInfo(int serviceId, - ListUIHandler urlIdHandler, + SearchQIHandler qIHandler, String searchString) throws ParsingException { - super(serviceId, urlIdHandler, "Search"); + super(serviceId, qIHandler, "Search"); this.searchString = searchString; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchResult.java b/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchResult.java deleted file mode 100644 index 0d4f1ac1a..000000000 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchResult.java +++ /dev/null @@ -1,71 +0,0 @@ -package org.schabi.newpipe.extractor.search; - -import org.schabi.newpipe.extractor.InfoItem; -import org.schabi.newpipe.extractor.exceptions.ExtractionException; - -import javax.annotation.Nonnull; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -/* - * Created by Christian Schabesberger on 29.02.16. - * - * Copyright (C) Christian Schabesberger 2016 - * SearchResult.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - -public class SearchResult { - private final int serviceId; - public final String suggestion; - @Nonnull - public final List resultList; - @Nonnull - public final List errors; - - public SearchResult(int serviceId, String suggestion, List results, List errors) { - this.serviceId = serviceId; - this.suggestion = suggestion; - this.resultList = Collections.unmodifiableList(new ArrayList<>(results)); - this.errors = Collections.unmodifiableList(new ArrayList<>(errors)); - } - - public static SearchResult getSearchResult(@Nonnull final SearchEngine engine, final String query, final int page, - final String languageCode, final SearchEngine.Filter filter) - throws IOException, ExtractionException { - return null; - } - - public String getSuggestion() { - return suggestion; - } - - - @Nonnull - public List getResults() { - return Collections.unmodifiableList(resultList); - } - - @Nonnull - public List getErrors() { - return errors; - } - - public int getServiceId() { - return serviceId; - } -} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEngine.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEngine.java deleted file mode 100644 index 3cd27031a..000000000 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEngine.java +++ /dev/null @@ -1,84 +0,0 @@ -package org.schabi.newpipe.extractor.services.soundcloud; - -import com.grack.nanojson.JsonArray; -import com.grack.nanojson.JsonObject; -import com.grack.nanojson.JsonParser; -import com.grack.nanojson.JsonParserException; -import org.schabi.newpipe.extractor.Downloader; -import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.exceptions.ExtractionException; -import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector; -import org.schabi.newpipe.extractor.search.SearchEngine; - -import java.io.IOException; -import java.net.URLEncoder; - -public class SoundcloudSearchEngine extends SearchEngine { - public static final String CHARSET_UTF_8 = "UTF-8"; - - public SoundcloudSearchEngine(int serviceId) { - super(serviceId); - } - - @Override - public InfoItemsSearchCollector search(String query, int page, String languageCode, Filter filter) throws IOException, ExtractionException { - InfoItemsSearchCollector collector = getInfoItemSearchCollector(); - - Downloader dl = NewPipe.getDownloader(); - - String url = "https://api-v2.soundcloud.com/search"; - - switch (filter) { - case STREAM: - url += "/tracks"; - break; - case CHANNEL: - url += "/users"; - break; - case PLAYLIST: - url += "/playlists"; - break; - case ANY: - // Don't append any parameter to search for everything - default: - break; - } - - url += "?q=" + URLEncoder.encode(query, CHARSET_UTF_8) - + "&client_id=" + SoundcloudParsingHelper.clientId() - + "&limit=10" - + "&offset=" + Integer.toString(page * 10); - - JsonArray searchCollection; - try { - searchCollection = JsonParser.object().from(dl.download(url)).getArray("collection"); - } catch (JsonParserException e) { - throw new ParsingException("Could not parse json response", e); - } - - if (searchCollection.size() == 0) { - throw new NothingFoundException("Nothing found"); - } - - for (Object result : searchCollection) { - if (!(result instanceof JsonObject)) continue; - //noinspection ConstantConditions - JsonObject searchResult = (JsonObject) result; - String kind = searchResult.getString("kind", ""); - switch (kind) { - case "user": - collector.commit(new SoundcloudChannelInfoItemExtractor(searchResult)); - break; - case "track": - collector.commit(new SoundcloudStreamInfoItemExtractor(searchResult)); - break; - case "playlist": - collector.commit(new SoundcloudPlaylistInfoItemExtractor(searchResult)); - break; - } - } - - return collector; - } -} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchExtractor.java index 8c5476398..c976ce2d8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchExtractor.java @@ -8,7 +8,6 @@ import org.schabi.newpipe.extractor.*; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector; -import org.schabi.newpipe.extractor.search.SearchEngine; import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.uih.SearchQIHandler; import org.schabi.newpipe.extractor.utils.Parser; @@ -70,7 +69,7 @@ public class SoundcloudSearchExtractor extends SearchExtractor { } if (searchCollection.size() == 0) { - throw new SearchEngine.NothingFoundException("Nothing found"); + throw new SearchExtractor.NothingFoundException("Nothing found"); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchEngine.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchEngine.java deleted file mode 100644 index 7fda65dde..000000000 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchEngine.java +++ /dev/null @@ -1,124 +0,0 @@ -package org.schabi.newpipe.extractor.services.youtube.extractors; - -import org.jsoup.Jsoup; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.schabi.newpipe.extractor.Downloader; -import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.exceptions.ExtractionException; -import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector; -import org.schabi.newpipe.extractor.search.SearchEngine; - -import java.io.IOException; -import java.net.URLEncoder; - - -/* - * Created by Christian Schabesberger on 09.08.15. - * - * Copyright (C) Christian Schabesberger 2015 - * YoutubeSearchEngine.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - -public class YoutubeSearchEngine extends SearchEngine { - - private static final String TAG = YoutubeSearchEngine.class.toString(); - public static final String CHARSET_UTF_8 = "UTF-8"; - - public YoutubeSearchEngine(int serviceId) { - super(serviceId); - } - - @Override - public InfoItemsSearchCollector search(String query, int page, String languageCode, Filter filter) - throws IOException, ExtractionException { - InfoItemsSearchCollector collector = getInfoItemSearchCollector(); - Downloader downloader = NewPipe.getDownloader(); - - String url = "https://www.youtube.com/results" - + "?q=" + URLEncoder.encode(query, CHARSET_UTF_8) - + "&page=" + Integer.toString(page + 1); - - switch (filter) { - case STREAM: - url += "&sp=EgIQAVAU"; - break; - case CHANNEL: - url += "&sp=EgIQAlAU"; //EgIQA( lowercase L )AU - break; - case PLAYLIST: - url += "&sp=EgIQA1AU"; //EgIQA( one )AU - break; - case ANY: - // Don't append any parameter to search for everything - default: - break; - } - - String site; - //String url = builder.build().toString(); - //if we've been passed a valid language code, append it to the URL - if (!languageCode.isEmpty()) { - //assert Pattern.matches("[a-z]{2}(-([A-Z]{2}|[0-9]{1,3}))?", languageCode); - site = downloader.download(url, languageCode); - } else { - site = downloader.download(url); - } - - Document doc = Jsoup.parse(site, url); - Element list = doc.select("ol[class=\"item-section\"]").first(); - - for (Element item : list.children()) { - /* First we need to determine which kind of item we are working with. - Youtube depicts five different kinds of items on its search result page. These are - regular videos, playlists, channels, two types of video suggestions, and a "no video - found" item. Since we only want videos, we need to filter out all the others. - An example for this can be seen here: - https://www.youtube.com/results?search_query=asdf&page=1 - - We already applied a filter to the url, so we don't need to care about channels and - playlists now. - */ - - Element el; - - // both types of spell correction item - if ((el = item.select("div[class*=\"spell-correction\"]").first()) != null) { - if (list.children().size() == 1) { - throw new NothingFoundException("Did you mean: " + el.select("a").first().text()); - } - // search message item - } else if ((el = item.select("div[class*=\"search-message\"]").first()) != null) { - throw new NothingFoundException(el.text()); - - // video item type - } else if ((el = item.select("div[class*=\"yt-lockup-video\"]").first()) != null) { - collector.commit(new YoutubeStreamInfoItemExtractor(el)); - } else if ((el = item.select("div[class*=\"yt-lockup-channel\"]").first()) != null) { - collector.commit(new YoutubeChannelInfoItemExtractor(el)); - } else if ((el = item.select("div[class*=\"yt-lockup-playlist\"]").first()) != null && - item.select(".yt-pl-icon-mix").isEmpty()) { - collector.commit(new YoutubePlaylistInfoItemExtractor(el)); - } else { - // noinspection ConstantConditions - // simply ignore not known items - // throw new ExtractionException("unexpected element found: \"" + item + "\""); - } - } - - return collector; - } -}