Merge pull request #19 from mauriciocolli/refactor

Refactor and improvements

build.gradle

@@ -10,5 +10,8 @@ dependencies {
     implementation 'org.mozilla:rhino:1.7.7.1'
 
     testImplementation 'junit:junit:4.12'
+
+    sourceCompatibility = 1.7
+    targetCompatibility = 1.7
 }
 

src/main/java/org/schabi/newpipe/extractor/Extractor.java

@@ -1,35 +1,73 @@
 package org.schabi.newpipe.extractor;
 
-import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
 
-import java.io.Serializable;
+import java.io.IOException;
 
-public abstract class Extractor implements Serializable {
-    private final int serviceId;
-    private final String url;
-    private final UrlIdHandler urlIdHandler;
-    private final StreamInfoItemCollector previewInfoCollector;
+public abstract class Extractor {
+    /**
+     * {@link StreamingService} currently related to this extractor.<br/>
+     * Useful for getting other things from a service (like the url handlers for cleaning/accepting/get id from urls).
+     */
+    private final StreamingService service;
 
-    public Extractor(UrlIdHandler urlIdHandler, int serviceId, String url) {
-        this.urlIdHandler = urlIdHandler;
-        this.serviceId = serviceId;
-        this.url = url;
-        this.previewInfoCollector = new StreamInfoItemCollector(serviceId);
+    /**
+     * Dirty/original url that was passed in the constructor.
+     * <p>
+     * What makes a url "dirty" or not is, for example, the additional parameters
+     * (not important as—in this case—the id):
+     * <pre>
+     *     https://www.youtube.com/watch?v=a9Zf_258aTI<i>&t=4s</i>  →  <i><b>&t=4s</b></i>
+     * </pre>
+     * But as you can imagine, the time parameter is very important when calling, for example, {@link org.schabi.newpipe.extractor.stream.StreamExtractor#getTimeStamp()}.
+     */
+    private final String originalUrl;
+
+    /**
+     * The cleaned url, result of passing the {@link #originalUrl} to the associated urlIdHandler ({@link #getUrlIdHandler()}).
+     * <p>
+     * Is lazily-cleaned by calling {@link #getCleanUrl()}
+     */
+    private String cleanUrl;
+
+    public Extractor(StreamingService service, String url) throws ExtractionException {
+        this.service = service;
+        this.originalUrl = url;
     }
 
-    public String getUrl() {
-        return url;
+    /**
+     * @return a {@link UrlIdHandler} of the current extractor type (e.g. a ChannelExtractor should return a channel url handler).
+     */
+    protected abstract UrlIdHandler getUrlIdHandler() throws ParsingException;
+
+    /**
+     * Fetch the current page.
+     */
+    public abstract void fetchPage() throws IOException, ExtractionException;
+
+    public String getOriginalUrl() {
+        return originalUrl;
     }
 
-    public UrlIdHandler getUrlIdHandler() {
-        return urlIdHandler;
+    public String getCleanUrl() {
+        if (cleanUrl != null && !cleanUrl.isEmpty()) return cleanUrl;
+
+        try {
+            cleanUrl = getUrlIdHandler().cleanUrl(originalUrl);
+        } catch (Exception e) {
+            cleanUrl = null;
+            // Fallback to the original url
+            return originalUrl;
+        }
+        return cleanUrl;
+    }
+
+    public StreamingService getService() {
+        return service;
     }
 
     public int getServiceId() {
-        return serviceId;
-    }
-
-    protected StreamInfoItemCollector getStreamPreviewInfoCollector() {
-        return previewInfoCollector;
+        return service.getServiceId();
     }
 }

src/main/java/org/schabi/newpipe/extractor/Info.java

@@ -1,8 +1,8 @@
 package org.schabi.newpipe.extractor;
 
 import java.io.Serializable;
+import java.util.ArrayList;
 import java.util.List;
-import java.util.Vector;
 
 public abstract class Info implements Serializable {
 
@@ -15,5 +15,5 @@ public abstract class Info implements Serializable {
     public String url;
     public String name;
 
-    public List<Throwable> errors = new Vector<>();
+    public List<Throwable> errors = new ArrayList<>();
 }

src/main/java/org/schabi/newpipe/extractor/InfoItemCollector.java

@@ -2,8 +2,8 @@ package org.schabi.newpipe.extractor;
 
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 
+import java.util.ArrayList;
 import java.util.List;
-import java.util.Vector;
 
 /*
  * Created by Christian Schabesberger on 12.02.17.
@@ -26,8 +26,8 @@ import java.util.Vector;
  */
 
 public abstract class InfoItemCollector {
-    private List<InfoItem> itemList = new Vector<>();
-    private List<Throwable> errors = new Vector<>();
+    private List<InfoItem> itemList = new ArrayList<>();
+    private List<Throwable> errors = new ArrayList<>();
     private int serviceId = -1;
 
     public InfoItemCollector(int serviceId) {
@@ -46,7 +46,7 @@ public abstract class InfoItemCollector {
         if (serviceId != otherC.serviceId) {
             throw new ExtractionException("Service Id does not equal: "
                     + NewPipe.getNameOfService(serviceId)
-                    + " and " + NewPipe.getNameOfService(otherC.serviceId));
+                    + " and " + NewPipe.getNameOfService((otherC.serviceId)));
         }
         errors.addAll(otherC.errors);
         itemList.addAll(otherC.itemList);

src/main/java/org/schabi/newpipe/extractor/ListExtractor.java

@@ -4,6 +4,7 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
 
 import java.io.IOException;
+import java.util.List;
 
 /**
  * Base class to extractors that have a list (e.g. playlists, channels).
@@ -11,16 +12,40 @@ import java.io.IOException;
 public abstract class ListExtractor extends Extractor {
     protected String nextStreamsUrl;
 
-    public ListExtractor(UrlIdHandler urlIdHandler, int serviceId, String url) {
-        super(urlIdHandler, serviceId, url);
+    /**
+     * Get a new ListExtractor with the given nextStreamsUrl set.
+     * <p>
+     * The extractor <b>WILL</b> fetch the page if {@link #fetchPageUponCreation()} return true, otherwise, it will <b>NOT</b>.
+     * <p>
+     * You can call {@link #fetchPage()} later, but this is mainly used just to get more items, so we don't waste bandwidth
+     * downloading the whole page, but if the service that is being implemented need it, just do its own logic in {@link #fetchPageUponCreation()}.
+     */
+    public ListExtractor(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException {
+        super(service, url);
+        setNextStreamsUrl(nextStreamsUrl);
+
+        if (fetchPageUponCreation()) {
+            fetchPage();
+        }
     }
 
-    public boolean hasMoreStreams(){
+    /**
+     * Decide if the page will be fetched upon creation.
+     * <p>
+     * The default implementation checks if the nextStreamsUrl is null or empty (indication that the caller
+     * don't need or know what is the next page, thus, fetch the page).
+     */
+    protected boolean fetchPageUponCreation() {
+        return nextStreamsUrl == null || nextStreamsUrl.isEmpty();
+    }
+
+    public abstract StreamInfoItemCollector getStreams() throws IOException, ExtractionException;
+    public abstract NextItemsResult getNextStreams() throws IOException, ExtractionException;
+
+    public boolean hasMoreStreams() {
         return nextStreamsUrl != null && !nextStreamsUrl.isEmpty();
     }
 
-    public abstract StreamInfoItemCollector getNextStreams() throws ExtractionException, IOException;
-
     public String getNextStreamsUrl() {
         return nextStreamsUrl;
     }
@@ -29,4 +54,29 @@ public abstract class ListExtractor extends Extractor {
         this.nextStreamsUrl = nextStreamsUrl;
     }
 
+    /*//////////////////////////////////////////////////////////////////////////
+    // Inner
+    //////////////////////////////////////////////////////////////////////////*/
+
+    public static class NextItemsResult {
+        /**
+         * The current list of items to this result
+         */
+        public final List<InfoItem> nextItemsList;
+
+        /**
+         * Next url to fetch more items
+         */
+        public final String nextItemsUrl;
+
+        public NextItemsResult(List<InfoItem> nextItemsList, String nextItemsUrl) {
+            this.nextItemsList = nextItemsList;
+            this.nextItemsUrl = nextItemsUrl;
+        }
+
+        public boolean hasMoreStreams() {
+            return nextItemsUrl != null && !nextItemsUrl.isEmpty();
+        }
+    }
+
 }

src/main/java/org/schabi/newpipe/extractor/ListInfo.java

@@ -0,0 +1,9 @@
+package org.schabi.newpipe.extractor;
+
+import java.util.List;
+
+public abstract class ListInfo extends Info {
+    public List<InfoItem> related_streams;
+    public boolean has_more_streams;
+    public String next_streams_url;
+}

src/main/java/org/schabi/newpipe/extractor/NewPipe.java

@@ -1,7 +1,5 @@
 package org.schabi.newpipe.extractor;
 
-import org.schabi.newpipe.extractor.exceptions.ExtractionException;
-
 /*
  * Created by Christian Schabesberger on 23.08.15.
  *
@@ -22,54 +20,16 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
  * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-/**
- * Provides access to the video streaming services supported by NewPipe.
- * Currently only Youtube until the API becomes more stable.
- */
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 
-@SuppressWarnings("ALL")
+/**
+ * Provides access to streaming services supported by NewPipe.
+ */
 public class NewPipe {
     private static final String TAG = NewPipe.class.toString();
-
-    private NewPipe() {
-    }
-
     private static Downloader downloader = null;
 
-    public static StreamingService[] getServices() {
-        return ServiceList.serviceList;
-    }
-
-    public static StreamingService getService(int serviceId) throws ExtractionException {
-        for (StreamingService s : ServiceList.serviceList) {
-            if (s.getServiceId() == serviceId) {
-                return s;
-            }
-        }
-        return null;
-    }
-
-    public static StreamingService getService(String serviceName) throws ExtractionException {
-        return ServiceList.serviceList[getIdOfService(serviceName)];
-    }
-
-    public static String getNameOfService(int id) {
-        try {
-            return getService(id).getServiceInfo().name;
-        } catch (Exception e) {
-            System.err.println("Service id not known");
-            e.printStackTrace();
-            return "";
-        }
-    }
-
-    public static int getIdOfService(String serviceName) {
-        for (int i = 0; i < ServiceList.serviceList.length; i++) {
-            if (ServiceList.serviceList[i].getServiceInfo().name.equals(serviceName)) {
-                return i;
-            }
-        }
-        return -1;
+    private NewPipe() {
     }
 
     public static void init(Downloader d) {
@@ -80,12 +40,63 @@ public class NewPipe {
         return downloader;
     }
 
-    public static StreamingService getServiceByUrl(String url) {
-        for (StreamingService s : ServiceList.serviceList) {
-            if (s.getLinkTypeByUrl(url) != StreamingService.LinkType.NONE) {
-                return s;
+    /*//////////////////////////////////////////////////////////////////////////
+    // Utils
+    //////////////////////////////////////////////////////////////////////////*/
+
+    public static StreamingService[] getServices() {
+        final ServiceList[] values = ServiceList.values();
+        final StreamingService[] streamingServices = new StreamingService[values.length];
+
+        for (int i = 0; i < values.length; i++) streamingServices[i] = values[i].getService();
+
+        return streamingServices;
+    }
+
+    public static StreamingService getService(int serviceId) throws ExtractionException {
+        for (ServiceList item : ServiceList.values()) {
+            if (item.getService().getServiceId() == serviceId) {
+                return item.getService();
             }
         }
-        return null;
+        throw new ExtractionException("There's no service with the id = \"" + serviceId + "\"");
+    }
+
+    public static StreamingService getService(String serviceName) throws ExtractionException {
+        for (ServiceList item : ServiceList.values()) {
+            if (item.getService().getServiceInfo().name.equals(serviceName)) {
+                return item.getService();
+            }
+        }
+        throw new ExtractionException("There's no service with the name = \"" + serviceName + "\"");
+    }
+
+    public static StreamingService getServiceByUrl(String url) throws ExtractionException {
+        for (ServiceList item : ServiceList.values()) {
+            if (item.getService().getLinkTypeByUrl(url) != StreamingService.LinkType.NONE) {
+                return item.getService();
+            }
+        }
+        throw new ExtractionException("No service can handle the url = \"" + url + "\"");
+    }
+
+    public static int getIdOfService(String serviceName) {
+        try {
+            //noinspection ConstantConditions
+            return getService(serviceName).getServiceId();
+        } catch (ExtractionException ignored) {
+            return -1;
+        }
+    }
+
+    public static String getNameOfService(int id) {
+        try {
+            //noinspection ConstantConditions
+            return getService(id).getServiceInfo().name;
+        } catch (Exception e) {
+            System.err.println("Service id not known");
+            e.printStackTrace();
+            return "<unknown>";
+        }
     }
 }

src/main/java/org/schabi/newpipe/extractor/ServiceList.java

@@ -1,15 +1,36 @@
 package org.schabi.newpipe.extractor;
 
-import org.schabi.newpipe.extractor.services.youtube.YoutubeService;
 import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudService;
+import org.schabi.newpipe.extractor.services.youtube.YoutubeService;
 
-/*
- * Created by the-scrabi on 18.02.17.
+/**
+ * A list of supported services.
  */
+public enum ServiceList {
+    Youtube(new YoutubeService(0, "Youtube")),
+    SoundCloud(new SoundcloudService(1, "SoundCloud"));
+//  DailyMotion(new DailyMotionService(2, "DailyMotion"));
 
-class ServiceList {
-    public static final StreamingService[] serviceList = {
-            new YoutubeService(0),
-            new SoundcloudService(1)
-    };
+    private final StreamingService service;
+
+    ServiceList(StreamingService service) {
+        this.service = service;
+    }
+
+    public StreamingService getService() {
+        return service;
+    }
+
+    public StreamingService.ServiceInfo getServiceInfo() {
+        return service.getServiceInfo();
+    }
+
+    public int getId() {
+        return service.getServiceId();
+    }
+
+    @Override
+    public String toString() {
+        return service.getServiceInfo().name;
+    }
 }

src/main/java/org/schabi/newpipe/extractor/StreamingService.java

@@ -10,7 +10,11 @@ import java.io.IOException;
 
 public abstract class StreamingService {
     public class ServiceInfo {
-        public String name = "";
+        public final String name;
+
+        public ServiceInfo(String name) {
+            this.name = name;
+        }
     }
 
     public enum LinkType {
@@ -20,35 +24,46 @@ public abstract class StreamingService {
         PLAYLIST
     }
 
-    private int serviceId;
+    private final int serviceId;
+    private final ServiceInfo serviceInfo;
 
-    public StreamingService(int id) {
-        serviceId = id;
+    public StreamingService(int id, String name) {
+        this.serviceId = id;
+        this.serviceInfo = new ServiceInfo(name);
     }
 
-    public abstract ServiceInfo getServiceInfo();
-
-    public abstract UrlIdHandler getStreamUrlIdHandlerInstance();
-    public abstract UrlIdHandler getChannelUrlIdHandlerInstance();
-    public abstract UrlIdHandler getPlaylistUrlIdHandlerInstance();
-    public abstract SearchEngine getSearchEngineInstance();
-    public abstract SuggestionExtractor getSuggestionExtractorInstance();
-    public abstract StreamExtractor getStreamExtractorInstance(String url) throws IOException, ExtractionException;
-    public abstract ChannelExtractor getChannelExtractorInstance(String url) throws ExtractionException, IOException;
-    public abstract PlaylistExtractor getPlaylistExtractorInstance(String url) throws ExtractionException, IOException;
-
-
     public final int getServiceId() {
         return serviceId;
     }
 
+    public ServiceInfo getServiceInfo() {
+        return serviceInfo;
+    }
+
+    public abstract UrlIdHandler getStreamUrlIdHandler();
+    public abstract UrlIdHandler getChannelUrlIdHandler();
+    public abstract UrlIdHandler getPlaylistUrlIdHandler();
+    public abstract SearchEngine getSearchEngine();
+    public abstract SuggestionExtractor getSuggestionExtractor();
+    public abstract StreamExtractor getStreamExtractor(String url) throws IOException, ExtractionException;
+    public abstract ChannelExtractor getChannelExtractor(String url, String nextStreamsUrl) throws IOException, ExtractionException;
+    public abstract PlaylistExtractor getPlaylistExtractor(String url, String nextStreamsUrl) throws IOException, ExtractionException;
+
+    public ChannelExtractor getChannelExtractor(String url) throws IOException, ExtractionException {
+        return getChannelExtractor(url, null);
+    }
+
+    public PlaylistExtractor getPlaylistExtractor(String url) throws IOException, ExtractionException {
+        return getPlaylistExtractor(url, null);
+    }
+
     /**
      * figure out where the link is pointing to (a channel, video, playlist, etc.)
      */
     public final LinkType getLinkTypeByUrl(String url) {
-        UrlIdHandler sH = getStreamUrlIdHandlerInstance();
-        UrlIdHandler cH = getChannelUrlIdHandlerInstance();
-        UrlIdHandler pH = getPlaylistUrlIdHandlerInstance();
+        UrlIdHandler sH = getStreamUrlIdHandler();
+        UrlIdHandler cH = getChannelUrlIdHandler();
+        UrlIdHandler pH = getPlaylistUrlIdHandler();
 
         if (sH.acceptUrl(url)) {
             return LinkType.STREAM;

src/main/java/org/schabi/newpipe/extractor/SuggestionExtractor.java

@@ -33,8 +33,7 @@ public abstract class SuggestionExtractor {
         this.serviceId = serviceId;
     }
 
-    public abstract List<String> suggestionList(String query, String contentCountry)
-            throws ExtractionException, IOException;
+    public abstract List<String> suggestionList(String query, String contentCountry) throws IOException, ExtractionException;
 
     public int getServiceId() {
         return serviceId;

src/main/java/org/schabi/newpipe/extractor/UrlIdHandler.java

@@ -1,9 +1,6 @@
 package org.schabi.newpipe.extractor;
 
-import java.io.IOException;
-
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
-import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
 
 /*
  * Created by Christian Schabesberger on 26.07.16.

src/main/java/org/schabi/newpipe/extractor/channel/ChannelExtractor.java

@@ -1,11 +1,10 @@
 package org.schabi.newpipe.extractor.channel;
 
 import org.schabi.newpipe.extractor.ListExtractor;
+import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.UrlIdHandler;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
-import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
-import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
 
 import java.io.IOException;
 
@@ -31,8 +30,13 @@ import java.io.IOException;
 
 public abstract class ChannelExtractor extends ListExtractor {
 
-    public ChannelExtractor(UrlIdHandler urlIdHandler, String url, int serviceId) throws ExtractionException, IOException {
-        super(urlIdHandler, serviceId, url);
+    public ChannelExtractor(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException {
+        super(service, url, nextStreamsUrl);
+    }
+
+    @Override
+    protected UrlIdHandler getUrlIdHandler() throws ParsingException {
+        return getService().getChannelUrlIdHandler();
     }
 
     public abstract String getChannelId() throws ParsingException;
@@ -40,7 +44,6 @@ public abstract class ChannelExtractor extends ListExtractor {
     public abstract String getAvatarUrl() throws ParsingException;
     public abstract String getBannerUrl() throws ParsingException;
     public abstract String getFeedUrl() throws ParsingException;
-    public abstract StreamInfoItemCollector getStreams() throws ParsingException, ReCaptchaException, IOException;
     public abstract long getSubscriberCount() throws ParsingException;
 
 }

src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfo.java

@@ -1,11 +1,16 @@
 package org.schabi.newpipe.extractor.channel;
 
-import org.schabi.newpipe.extractor.Info;
-import org.schabi.newpipe.extractor.InfoItem;
+import org.schabi.newpipe.extractor.ListExtractor.NextItemsResult;
+import org.schabi.newpipe.extractor.ListInfo;
+import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.ServiceList;
+import org.schabi.newpipe.extractor.StreamingService;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
 
-import java.util.List;
+import java.io.IOException;
+import java.util.ArrayList;
 
 /*
  * Created by Christian Schabesberger on 31.07.16.
@@ -27,17 +32,36 @@ import java.util.List;
  * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-public class ChannelInfo extends Info {
+public class ChannelInfo extends ListInfo {
+
+    public static NextItemsResult getMoreItems(ServiceList serviceItem, String nextStreamsUrl) throws IOException, ExtractionException {
+        return getMoreItems(serviceItem.getService(), nextStreamsUrl);
+    }
+
+    public static NextItemsResult getMoreItems(StreamingService service, String nextStreamsUrl) throws IOException, ExtractionException {
+        return service.getChannelExtractor(null, nextStreamsUrl).getNextStreams();
+    }
+
+    public static ChannelInfo getInfo(String url) throws IOException, ExtractionException {
+        return getInfo(NewPipe.getServiceByUrl(url), url);
+    }
+
+    public static ChannelInfo getInfo(ServiceList serviceItem, String url) throws IOException, ExtractionException {
+        return getInfo(serviceItem.getService(), url);
+    }
+
+    public static ChannelInfo getInfo(StreamingService service, String url) throws IOException, ExtractionException {
+        return getInfo(service.getChannelExtractor(url));
+    }
 
     public static ChannelInfo getInfo(ChannelExtractor extractor) throws ParsingException {
         ChannelInfo info = new ChannelInfo();
 
         // important data
         info.service_id = extractor.getServiceId();
-        info.url = extractor.getUrl();
+        info.url = extractor.getCleanUrl();
         info.id = extractor.getChannelId();
         info.name = extractor.getChannelName();
-        info.has_more_streams = extractor.hasMoreStreams();
 
         try {
             info.avatar_url = extractor.getAvatarUrl();
@@ -67,13 +91,16 @@ public class ChannelInfo extends Info {
             info.errors.add(e);
         }
 
+        // Lists can be null if a exception was thrown during extraction
+        if (info.related_streams == null) info.related_streams = new ArrayList<>();
+
+        info.has_more_streams = extractor.hasMoreStreams();
+        info.next_streams_url = extractor.getNextStreamsUrl();
         return info;
     }
 
     public String avatar_url;
     public String banner_url;
     public String feed_url;
-    public List<InfoItem> related_streams;
     public long subscriber_count = -1;
-    public boolean has_more_streams = false;
 }

src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItem.java

@@ -27,7 +27,7 @@ public class ChannelInfoItem extends InfoItem {
     public String thumbnail_url;
     public String description;
     public long subscriber_count = -1;
-    public long view_count = -1;
+    public long stream_count = -1;
 
     public ChannelInfoItem() {
         super(InfoType.CHANNEL);

src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItemCollector.java

@@ -43,7 +43,7 @@ public class ChannelInfoItemCollector extends InfoItemCollector {
             addError(e);
         }
         try {
-            resultItem.view_count = extractor.getViewCount();
+            resultItem.stream_count = extractor.getStreamCount();
         } catch (Exception e) {
             addError(e);
         }

src/main/java/org/schabi/newpipe/extractor/channel/ChannelInfoItemExtractor.java

@@ -28,5 +28,5 @@ public interface ChannelInfoItemExtractor {
     String getWebPageUrl() throws ParsingException;
     String getDescription() throws ParsingException;
     long getSubscriberCount() throws ParsingException;
-    long getViewCount() throws ParsingException;
+    long getStreamCount() throws ParsingException;
 }

src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistExtractor.java

@@ -1,18 +1,22 @@
 package org.schabi.newpipe.extractor.playlist;
 
 import org.schabi.newpipe.extractor.ListExtractor;
+import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.UrlIdHandler;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
-import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
-import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
 
 import java.io.IOException;
 
 public abstract class PlaylistExtractor extends ListExtractor {
 
-    public PlaylistExtractor(UrlIdHandler urlIdHandler, String url, int serviceId) throws ExtractionException, IOException {
-        super(urlIdHandler, serviceId, url);
+    public PlaylistExtractor(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException {
+        super(service, url, nextStreamsUrl);
+    }
+
+    @Override
+    protected UrlIdHandler getUrlIdHandler() throws ParsingException {
+        return getService().getPlaylistUrlIdHandler();
     }
 
     public abstract String getPlaylistId() throws ParsingException;
@@ -22,6 +26,5 @@ public abstract class PlaylistExtractor extends ListExtractor {
     public abstract String getUploaderUrl() throws ParsingException;
     public abstract String getUploaderName() throws ParsingException;
     public abstract String getUploaderAvatarUrl() throws ParsingException;
-    public abstract StreamInfoItemCollector getStreams() throws ParsingException, ReCaptchaException, IOException;
-    public abstract long getStreamsCount() throws ParsingException;
+    public abstract long getStreamCount() throws ParsingException;
 }

src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfo.java

@@ -1,25 +1,49 @@
 package org.schabi.newpipe.extractor.playlist;
 
-import org.schabi.newpipe.extractor.Info;
-import org.schabi.newpipe.extractor.InfoItem;
+import org.schabi.newpipe.extractor.ListExtractor.NextItemsResult;
+import org.schabi.newpipe.extractor.ListInfo;
+import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.ServiceList;
+import org.schabi.newpipe.extractor.StreamingService;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
 
-import java.util.List;
+import java.io.IOException;
+import java.util.ArrayList;
 
-public class PlaylistInfo extends Info {
+public class PlaylistInfo extends ListInfo {
+
+    public static NextItemsResult getMoreItems(ServiceList serviceItem, String nextStreamsUrl) throws IOException, ExtractionException {
+        return getMoreItems(serviceItem.getService(), nextStreamsUrl);
+    }
+
+    public static NextItemsResult getMoreItems(StreamingService service, String nextStreamsUrl) throws IOException, ExtractionException {
+        return service.getPlaylistExtractor(null, nextStreamsUrl).getNextStreams();
+    }
+
+    public static PlaylistInfo getInfo(String url) throws IOException, ExtractionException {
+        return getInfo(NewPipe.getServiceByUrl(url), url);
+    }
+
+    public static PlaylistInfo getInfo(ServiceList serviceItem, String url) throws IOException, ExtractionException {
+        return getInfo(serviceItem.getService(), url);
+    }
+
+    public static PlaylistInfo getInfo(StreamingService service, String url) throws IOException, ExtractionException {
+        return getInfo(service.getPlaylistExtractor(url));
+    }
 
     public static PlaylistInfo getInfo(PlaylistExtractor extractor) throws ParsingException {
         PlaylistInfo info = new PlaylistInfo();
 
         info.service_id = extractor.getServiceId();
-        info.url = extractor.getUrl();
+        info.url = extractor.getCleanUrl();
         info.id = extractor.getPlaylistId();
         info.name = extractor.getPlaylistName();
-        info.has_more_streams = extractor.hasMoreStreams();
 
         try {
-            info.streams_count = extractor.getStreamsCount();
+            info.stream_count = extractor.getStreamCount();
         } catch (Exception e) {
             info.errors.add(e);
         }
@@ -56,6 +80,11 @@ public class PlaylistInfo extends Info {
             info.errors.add(e);
         }
 
+        // Lists can be null if a exception was thrown during extraction
+        if (info.related_streams == null) info.related_streams = new ArrayList<>();
+
+        info.has_more_streams = extractor.hasMoreStreams();
+        info.next_streams_url = extractor.getNextStreamsUrl();
         return info;
     }
 
@@ -64,7 +93,5 @@ public class PlaylistInfo extends Info {
     public String uploader_url;
     public String uploader_name;
     public String uploader_avatar_url;
-    public long streams_count = 0;
-    public List<InfoItem> related_streams;
-    public boolean has_more_streams;
+    public long stream_count = 0;
 }

src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfoItem.java

@@ -8,7 +8,7 @@ public class PlaylistInfoItem extends InfoItem {
     /**
      * How many streams this playlist have
      */
-    public long streams_count = 0;
+    public long stream_count = 0;
 
     public PlaylistInfoItem() {
         super(InfoType.PLAYLIST);

src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfoItemCollector.java

@@ -1,7 +1,6 @@
 package org.schabi.newpipe.extractor.playlist;
 
 import org.schabi.newpipe.extractor.InfoItemCollector;
-import org.schabi.newpipe.extractor.UrlIdHandler;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 
 public class PlaylistInfoItemCollector extends InfoItemCollector {
@@ -22,7 +21,7 @@ public class PlaylistInfoItemCollector extends InfoItemCollector {
             addError(e);
         }
         try {
-            resultItem.streams_count = extractor.getStreamsCount();
+            resultItem.stream_count = extractor.getStreamCount();
         } catch (Exception e) {
             addError(e);
         }

src/main/java/org/schabi/newpipe/extractor/playlist/PlaylistInfoItemExtractor.java

@@ -6,5 +6,5 @@ public interface PlaylistInfoItemExtractor {
     String getThumbnailUrl() throws ParsingException;
     String getPlaylistName() throws ParsingException;
     String getWebPageUrl() throws ParsingException;
-    long getStreamsCount() throws ParsingException;
+    long getStreamCount() throws ParsingException;
 }

src/main/java/org/schabi/newpipe/extractor/search/InfoItemSearchCollector.java

@@ -59,7 +59,7 @@ public class InfoItemSearchCollector extends InfoItemCollector {
         try {
             result.resultList.add(streamCollector.extract(extractor));
         } catch (FoundAdException ae) {
-            System.err.println("Found add");
+            System.err.println("Found ad");
         } catch (Exception e) {
             addError(e);
         }
@@ -69,7 +69,7 @@ public class InfoItemSearchCollector extends InfoItemCollector {
         try {
             result.resultList.add(channelCollector.extract(extractor));
         } catch (FoundAdException ae) {
-            System.err.println("Found add");
+            System.err.println("Found ad");
         } catch (Exception e) {
             addError(e);
         }

src/main/java/org/schabi/newpipe/extractor/search/SearchEngine.java

@@ -49,5 +49,5 @@ public abstract class SearchEngine {
     //Result search(String query, int page);
     public abstract InfoItemSearchCollector search(
             String query, int page, String contentCountry, EnumSet<Filter> filter)
-            throws ExtractionException, IOException;
+            throws IOException, ExtractionException;
 }

src/main/java/org/schabi/newpipe/extractor/search/SearchResult.java

@@ -4,9 +4,9 @@ import org.schabi.newpipe.extractor.InfoItem;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.EnumSet;
 import java.util.List;
-import java.util.Vector;
 
 /*
  * Created by Christian Schabesberger on 29.02.16.
@@ -31,7 +31,7 @@ import java.util.Vector;
 public class SearchResult {
     public static SearchResult getSearchResult(SearchEngine engine, String query,
                                                int page, String languageCode, EnumSet<SearchEngine.Filter> filter)
-            throws ExtractionException, IOException {
+            throws IOException, ExtractionException {
 
         SearchResult result = engine
                 .search(query, page, languageCode, filter)
@@ -50,6 +50,6 @@ public class SearchResult {
     }
 
     public String suggestion;
-    public List<InfoItem> resultList = new Vector<>();
-    public List<Throwable> errors = new Vector<>();
+    public List<InfoItem> resultList = new ArrayList<>();
+    public List<Throwable> errors = new ArrayList<>();
 }

src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java

@@ -1,37 +1,46 @@
 package org.schabi.newpipe.extractor.services.soundcloud;
 
-import java.io.IOException;
-
-import org.json.JSONArray;
 import org.json.JSONObject;
 import org.schabi.newpipe.extractor.Downloader;
 import org.schabi.newpipe.extractor.NewPipe;
-import org.schabi.newpipe.extractor.UrlIdHandler;
+import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.channel.ChannelExtractor;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
-import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
 
+import java.io.IOException;
+
 @SuppressWarnings("WeakerAccess")
 public class SoundcloudChannelExtractor extends ChannelExtractor {
     private String channelId;
     private JSONObject channel;
-    private String nextUrl;
 
-    public SoundcloudChannelExtractor(UrlIdHandler urlIdHandler, String url, int serviceId) throws ExtractionException, IOException {
-        super(urlIdHandler, url, serviceId);
+    public SoundcloudChannelExtractor(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException {
+        super(service, url, nextStreamsUrl);
+    }
 
+    @Override
+    public void fetchPage() throws IOException, ExtractionException {
         Downloader dl = NewPipe.getDownloader();
 
-        channelId = urlIdHandler.getId(url);
-        String apiUrl = "https://api-v2.soundcloud.com/users/" + channelId
-                + "?client_id=" + SoundcloudParsingHelper.clientId();
+        channelId = getUrlIdHandler().getId(getOriginalUrl());
+        String apiUrl = "https://api.soundcloud.com/users/" + channelId +
+                "?client_id=" + SoundcloudParsingHelper.clientId();
 
         String response = dl.download(apiUrl);
         channel = new JSONObject(response);
     }
 
+    @Override
+    public String getCleanUrl() {
+        try {
+            return channel.getString("permalink_url");
+        } catch (Exception e) {
+            return getOriginalUrl();
+        }
+    }
+
     @Override
     public String getChannelId() {
         return channelId;
@@ -56,32 +65,6 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
         }
     }
 
-    @Override
-    public StreamInfoItemCollector getStreams() throws ReCaptchaException, IOException, ParsingException {
-        StreamInfoItemCollector collector = getStreamPreviewInfoCollector();
-        Downloader dl = NewPipe.getDownloader();
-
-        String apiUrl = "https://api-v2.soundcloud.com/users/" + channelId + "/tracks"
-                + "?client_id=" + SoundcloudParsingHelper.clientId()
-                + "&limit=10"
-                + "&offset=0"
-                + "&linked_partitioning=1";
-
-        String response = dl.download(apiUrl);
-        JSONObject responseObject = new JSONObject(response);
-
-        nextUrl = responseObject.getString("next_href")
-                + "&client_id=" + SoundcloudParsingHelper.clientId()
-                + "&linked_partitioning=1";
-
-        JSONArray responseCollection = responseObject.getJSONArray("collection");
-        for (int i = 0; i < responseCollection.length(); i++) {
-            JSONObject track = responseCollection.getJSONObject(i);
-            collector.commit(new SoundcloudStreamInfoItemExtractor(track));
-        }
-        return collector;
-    }
-
     @Override
     public long getSubscriberCount() {
         return channel.getLong("followers_count");
@@ -93,26 +76,27 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
     }
 
     @Override
-    public StreamInfoItemCollector getNextStreams() throws ExtractionException, IOException {
-        if (nextUrl.equals("")) {
+    public StreamInfoItemCollector getStreams() throws IOException, ExtractionException {
+        StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
+
+        String apiUrl = "https://api-v2.soundcloud.com/users/" + getChannelId() + "/tracks"
+                + "?client_id=" + SoundcloudParsingHelper.clientId()
+                + "&limit=20"
+                + "&linked_partitioning=1";
+
+        nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, apiUrl);
+        return collector;
+    }
+
+    @Override
+    public NextItemsResult getNextStreams() throws IOException, ExtractionException {
+        if (!hasMoreStreams()) {
             throw new ExtractionException("Channel doesn't have more streams");
         }
 
-        StreamInfoItemCollector collector = getStreamPreviewInfoCollector();
-        Downloader dl = NewPipe.getDownloader();
+        StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
+        nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, nextStreamsUrl);
 
-        String response = dl.download(nextUrl);
-        JSONObject responseObject = new JSONObject(response);
-
-        nextUrl = responseObject.getString("next_href")
-                + "&client_id=" + SoundcloudParsingHelper.clientId()
-                + "&linked_partitioning=1";
-
-        JSONArray responseCollection = responseObject.getJSONArray("collection");
-        for (int i = 0; i < responseCollection.length(); i++) {
-            JSONObject track = responseCollection.getJSONObject(i);
-            collector.commit(new SoundcloudStreamInfoItemExtractor(track));
-        }
-        return collector;
+        return new NextItemsResult(collector.getItemList(), nextStreamsUrl);
     }
 }

src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelInfoItemExtractor.java

@@ -31,7 +31,7 @@ public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtrac
     }
 
     @Override
-    public long getViewCount() {
+    public long getStreamCount() {
         return searchResult.getLong("track_count");
     }
 

src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelUrlIdHandler.java

@@ -1,10 +1,7 @@
 package org.schabi.newpipe.extractor.services.soundcloud;
 
-import org.json.JSONObject;
 import org.jsoup.Jsoup;
-import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
-import org.schabi.newpipe.extractor.Downloader;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.UrlIdHandler;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
@@ -21,47 +18,28 @@ public class SoundcloudChannelUrlIdHandler implements UrlIdHandler {
     @Override
     public String getUrl(String channelId) throws ParsingException {
         try {
-            Downloader dl = NewPipe.getDownloader();
-
-            String response = dl.download("https://api-v2.soundcloud.com/user/" + channelId
-                    + "?client_id=" + SoundcloudParsingHelper.clientId());
-            JSONObject responseObject = new JSONObject(response);
-
-            return responseObject.getString("permalink_url");
+            return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/users/" + channelId);
         } catch (Exception e) {
             throw new ParsingException(e.getMessage(), e);
         }
     }
 
     @Override
-    public String getId(String siteUrl) throws ParsingException {
+    public String getId(String url) throws ParsingException {
         try {
-            Downloader dl = NewPipe.getDownloader();
-
-            String response = dl.download(siteUrl);
-            Document doc = Jsoup.parse(response);
-
-            Element androidElement = doc.select("meta[property=al:android:url]").first();
-            String id = androidElement.attr("content").substring(19);
-
-            return id;
+            return SoundcloudParsingHelper.resolveIdWithEmbedPlayer(url);
         } catch (Exception e) {
             throw new ParsingException(e.getMessage(), e);
         }
     }
 
     @Override
-    public String cleanUrl(String siteUrl) throws ParsingException {
+    public String cleanUrl(String complexUrl) throws ParsingException {
         try {
-            Downloader dl = NewPipe.getDownloader();
+            Element ogElement = Jsoup.parse(NewPipe.getDownloader().download(complexUrl))
+                    .select("meta[property=og:url]").first();
 
-            String response = dl.download(siteUrl);
-            Document doc = Jsoup.parse(response);
-
-            Element ogElement = doc.select("meta[property=og:url]").first();
-            String url = ogElement.attr("content");
-
-            return url;
+            return ogElement.attr("content");
         } catch (Exception e) {
             throw new ParsingException(e.getMessage(), e);
         }
@@ -71,6 +49,5 @@ public class SoundcloudChannelUrlIdHandler implements UrlIdHandler {
     public boolean acceptUrl(String channelUrl) {
         String regex = "^https?://(www\\.)?soundcloud.com/[0-9a-z_-]+(/((tracks|albums|sets|reposts|followers|following)/?)?)?([#?].*)?$";
         return Parser.isMatch(regex, channelUrl.toLowerCase());
-
     }
 }

src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java

@@ -1,13 +1,7 @@
 package org.schabi.newpipe.extractor.services.soundcloud;
 
-import java.io.IOException;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.Arrays;
-import java.util.Date;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-
+import org.json.JSONArray;
+import org.json.JSONObject;
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
@@ -15,61 +9,51 @@ import org.schabi.newpipe.extractor.Downloader;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
+import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
 import org.schabi.newpipe.extractor.utils.Parser;
 import org.schabi.newpipe.extractor.utils.Parser.RegexException;
 
+import java.io.IOException;
+import java.net.URLEncoder;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+
 public class SoundcloudParsingHelper {
+    private static String clientId;
+
     private SoundcloudParsingHelper() {
     }
 
-    public static final String clientId() throws ReCaptchaException, IOException, RegexException {
+    public static String clientId() throws ReCaptchaException, IOException, RegexException {
+        if (clientId != null && !clientId.isEmpty()) return clientId;
+
         Downloader dl = NewPipe.getDownloader();
 
         String response = dl.download("https://soundcloud.com");
         Document doc = Jsoup.parse(response);
 
+        // TODO: Find a less heavy way to get the client_id
+        // Currently we are downloading a 1MB file (!) just to get the client_id,
+        // youtube-dl don't have a way too, they are just hardcoding and updating it when it becomes invalid.
+        // The embed mode has a way to get it, but we still have to download a heavy file (~800KB).
         Element jsElement = doc.select("script[src^=https://a-v2.sndcdn.com/assets/app]").first();
         String js = dl.download(jsElement.attr("src"));
 
-        String clientId = Parser.matchGroup1(",client_id:\"(.*?)\"", js);
+        clientId = Parser.matchGroup1(",client_id:\"(.*?)\"", js);
         return clientId;
     }
 
-    public static String toTimeAgoString(String time) throws ParsingException {
-        try {
-            List<Long> times = Arrays.asList(TimeUnit.DAYS.toMillis(365), TimeUnit.DAYS.toMillis(30),
-                    TimeUnit.DAYS.toMillis(7), TimeUnit.HOURS.toMillis(1), TimeUnit.MINUTES.toMillis(1),
-                    TimeUnit.SECONDS.toMillis(1));
-            List<String> timesString = Arrays.asList("year", "month", "week", "day", "hour", "minute", "second");
-
-            SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
-
-            long timeAgo = System.currentTimeMillis() - dateFormat.parse(time).getTime();
-
-            StringBuilder timeAgoString = new StringBuilder();
-
-            for (int i = 0; i < times.size(); i++) {
-                Long current = times.get(i);
-                long currentAmount = timeAgo / current;
-                if (currentAmount > 0) {
-                    timeAgoString.append(currentAmount).append(" ").append(timesString.get(i))
-                            .append(currentAmount != 1 ? "s ago" : " ago");
-                    break;
-                }
-            }
-            if (timeAgoString.toString().equals("")) {
-                timeAgoString.append("Just now");
-            }
-            return timeAgoString.toString();
-        } catch (ParseException e) {
-            throw new ParsingException(e.getMessage(), e);
-        }
-    }
-
     public static String toDateString(String time) throws ParsingException {
         try {
-            SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
-            Date date = dateFormat.parse(time);
+            Date date;
+            // Have two date formats, one for the 'api.soundc...' and the other 'api-v2.soundc...'.
+            try {
+                date = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'").parse(time);
+            } catch (Exception e) {
+                date = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss +0000").parse(time);
+            }
+
             SimpleDateFormat newDateFormat = new SimpleDateFormat("yyyy-MM-dd");
             return newDateFormat.format(date);
         } catch (ParseException e) {
@@ -77,4 +61,83 @@ public class SoundcloudParsingHelper {
         }
     }
 
+    /**
+     * Call the endpoint "/resolve" of the api.<br/>
+     * See https://developers.soundcloud.com/docs/api/reference#resolve
+     */
+    public static JSONObject resolveFor(String url) throws IOException, ReCaptchaException, ParsingException {
+        String apiUrl = "https://api.soundcloud.com/resolve"
+                + "?url=" + URLEncoder.encode(url, "UTF-8")
+                + "&client_id=" + clientId();
+
+        return new JSONObject(NewPipe.getDownloader().download(apiUrl));
+    }
+
+    /**
+     * Fetch the embed player with the apiUrl and return the canonical url (like the permalink_url from the json api).<br/>
+     *
+     * @return the url resolved
+     */
+    public static String resolveUrlWithEmbedPlayer(String apiUrl) throws IOException, ReCaptchaException, ParsingException {
+
+        String response = NewPipe.getDownloader().download("https://w.soundcloud.com/player/?url="
+                + URLEncoder.encode(apiUrl, "UTF-8"));
+
+        return Jsoup.parse(response).select("link[rel=\"canonical\"]").first().attr("abs:href");
+    }
+
+    /**
+     * Fetch the embed player with the url and return the id (like the id from the json api).<br/>
+     *
+     * @return the id resolved
+     */
+    public static String resolveIdWithEmbedPlayer(String url) throws IOException, ReCaptchaException, ParsingException {
+
+        String response = NewPipe.getDownloader().download("https://w.soundcloud.com/player/?url="
+                + URLEncoder.encode(url, "UTF-8"));
+        return Parser.matchGroup1(",\"id\":(.*?),", response);
+    }
+
+    /**
+     * Fetch the streams from the given api and commit each of them to the collector.
+     * <p>
+     * This differ from {@link #getStreamsFromApi(StreamInfoItemCollector, String)} in the sense that they will always
+     * get MIN_ITEMS or more items.
+     *
+     * @param minItems the method will return only when it have extracted that many items (equal or more)
+     */
+    public static String getStreamsFromApiMinItems(int minItems, StreamInfoItemCollector collector, String apiUrl) throws IOException, ReCaptchaException, ParsingException {
+        String nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl);
+
+        while (!nextStreamsUrl.isEmpty() && collector.getItemList().size() < minItems) {
+            nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, nextStreamsUrl);
+        }
+
+        return nextStreamsUrl;
+    }
+
+    /**
+     * Fetch the streams from the given api and commit each of them to the collector.
+     *
+     * @return the next streams url, empty if don't have
+     */
+    public static String getStreamsFromApi(StreamInfoItemCollector collector, String apiUrl) throws IOException, ReCaptchaException, ParsingException {
+        String response = NewPipe.getDownloader().download(apiUrl);
+        JSONObject responseObject = new JSONObject(response);
+
+        JSONArray responseCollection = responseObject.getJSONArray("collection");
+        for (int i = 0; i < responseCollection.length(); i++) {
+            collector.commit(new SoundcloudStreamInfoItemExtractor(responseCollection.getJSONObject(i)));
+        }
+
+        String nextStreamsUrl;
+        try {
+            nextStreamsUrl = responseObject.getString("next_href");
+            if (!nextStreamsUrl.contains("client_id=")) nextStreamsUrl += "&client_id=" + SoundcloudParsingHelper.clientId();
+        } catch (Exception ignored) {
+            nextStreamsUrl = "";
+        }
+
+        return nextStreamsUrl;
+    }
 }

src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java

@@ -1,38 +1,46 @@
 package org.schabi.newpipe.extractor.services.soundcloud;
 
-import java.io.IOException;
-import java.util.List;
-
-import org.json.JSONArray;
 import org.json.JSONObject;
 import org.schabi.newpipe.extractor.Downloader;
 import org.schabi.newpipe.extractor.NewPipe;
-import org.schabi.newpipe.extractor.UrlIdHandler;
+import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
-import org.schabi.newpipe.extractor.exceptions.ParsingException;
-import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
 import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
 
+import java.io.IOException;
+
 @SuppressWarnings("WeakerAccess")
 public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
     private String playlistId;
     private JSONObject playlist;
-    private List<String> nextTracks;
 
-    public SoundcloudPlaylistExtractor(UrlIdHandler urlIdHandler, String url, int serviceId) throws IOException, ExtractionException {
-        super(urlIdHandler, url, serviceId);
+    public SoundcloudPlaylistExtractor(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException {
+        super(service, url, nextStreamsUrl);
+    }
 
+    @Override
+    public void fetchPage() throws IOException, ExtractionException {
         Downloader dl = NewPipe.getDownloader();
-        playlistId = urlIdHandler.getId(url);
 
-        String apiUrl = "https://api-v2.soundcloud.com/users/" + playlistId
-                + "?client_id=" + SoundcloudParsingHelper.clientId();
+        playlistId = getUrlIdHandler().getId(getOriginalUrl());
+        String apiUrl = "https://api.soundcloud.com/playlists/" + playlistId +
+                "?client_id=" + SoundcloudParsingHelper.clientId() +
+                "&representation=compact";
 
         String response = dl.download(apiUrl);
         playlist = new JSONObject(response);
     }
 
+    @Override
+    public String getCleanUrl() {
+        try {
+            return playlist.getString("permalink_url");
+        } catch (Exception e) {
+            return getOriginalUrl();
+        }
+    }
+
     @Override
     public String getPlaylistId() {
         return playlistId;
@@ -69,61 +77,33 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
     }
 
     @Override
-    public long getStreamsCount() {
+    public long getStreamCount() {
         return playlist.getLong("track_count");
     }
 
     @Override
-    public StreamInfoItemCollector getStreams() throws ParsingException, ReCaptchaException, IOException {
-        StreamInfoItemCollector collector = getStreamPreviewInfoCollector();
-        Downloader dl = NewPipe.getDownloader();
+    public StreamInfoItemCollector getStreams() throws IOException, ExtractionException {
+        StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
 
-        String apiUrl = "https://api-v2.soundcloud.com/playlists/" + playlistId
-                + "?client_id=" + SoundcloudParsingHelper.clientId();
+        // Note the "api", NOT "api-v2"
+        String apiUrl = "https://api.soundcloud.com/playlists/" + getPlaylistId() + "/tracks"
+                + "?client_id=" + SoundcloudParsingHelper.clientId()
+                + "&limit=20"
+                + "&linked_partitioning=1";
 
-        String response = dl.download(apiUrl);
-        JSONObject responseObject = new JSONObject(response);
-        JSONArray responseCollection = responseObject.getJSONArray("collection");
-
-        for (int i = 0; i < responseCollection.length(); i++) {
-            JSONObject track = responseCollection.getJSONObject(i);
-            try {
-                collector.commit(new SoundcloudStreamInfoItemExtractor(track));
-            } catch (Exception e) {
-                nextTracks.add(track.getString("id"));
-            }
-        }
+        nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, apiUrl);
         return collector;
     }
 
     @Override
-    public StreamInfoItemCollector getNextStreams() throws ReCaptchaException, IOException, ParsingException {
-        if (nextTracks.equals(null)) {
-            return null;
+    public NextItemsResult getNextStreams() throws IOException, ExtractionException {
+        if (!hasMoreStreams()) {
+            throw new ExtractionException("Playlist doesn't have more streams");
         }
 
-        StreamInfoItemCollector collector = getStreamPreviewInfoCollector();
-        Downloader dl = NewPipe.getDownloader();
+        StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
+        nextStreamsUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, nextStreamsUrl);
 
-        // TODO: Do this per 10 tracks, instead of all tracks at once
-        String apiUrl = "https://api-v2.soundcloud.com/tracks?ids=";
-        for (String id : nextTracks) {
-            apiUrl += id;
-            if (!id.equals(nextTracks.get(nextTracks.size() - 1))) {
-                apiUrl += ",";
-            }
-        }
-        apiUrl += "&client_id=" + SoundcloudParsingHelper.clientId();
-
-        String response = dl.download(apiUrl);
-        JSONObject responseObject = new JSONObject(response);
-        JSONArray responseCollection = responseObject.getJSONArray("collection");
-
-        for (int i = 0; i < responseCollection.length(); i++) {
-            JSONObject track = responseCollection.getJSONObject(i);
-            collector.commit(new SoundcloudStreamInfoItemExtractor(track));
-        }
-        nextTracks = null;
-        return collector;
+        return new NextItemsResult(collector.getItemList(), nextStreamsUrl);
     }
 }

src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistUrlIdHandler.java

@@ -1,10 +1,7 @@
 package org.schabi.newpipe.extractor.services.soundcloud;
 
-import org.json.JSONObject;
 import org.jsoup.Jsoup;
-import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
-import org.schabi.newpipe.extractor.Downloader;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.UrlIdHandler;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
@@ -21,13 +18,7 @@ public class SoundcloudPlaylistUrlIdHandler implements UrlIdHandler {
     @Override
     public String getUrl(String listId) throws ParsingException {
         try {
-            Downloader dl = NewPipe.getDownloader();
-
-            String response = dl.download("https://api-v2.soundcloud.com/playlists/" + listId
-                    + "?client_id=" + SoundcloudParsingHelper.clientId());
-            JSONObject responseObject = new JSONObject(response);
-
-            return responseObject.getString("permalink_url");
+            return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/playlists/" + listId);
         } catch (Exception e) {
             throw new ParsingException(e.getMessage(), e);
         }
@@ -36,15 +27,7 @@ public class SoundcloudPlaylistUrlIdHandler implements UrlIdHandler {
     @Override
     public String getId(String url) throws ParsingException {
         try {
-            Downloader dl = NewPipe.getDownloader();
-
-            String response = dl.download(url);
-            Document doc = Jsoup.parse(response);
-
-            Element androidElement = doc.select("meta[property=al:android:url]").first();
-            String id = androidElement.attr("content").substring(23);
-
-            return id;
+            return SoundcloudParsingHelper.resolveIdWithEmbedPlayer(url);
         } catch (Exception e) {
             throw new ParsingException(e.getMessage(), e);
         }
@@ -53,15 +36,10 @@ public class SoundcloudPlaylistUrlIdHandler implements UrlIdHandler {
     @Override
     public String cleanUrl(String complexUrl) throws ParsingException {
         try {
-            Downloader dl = NewPipe.getDownloader();
+            Element ogElement = Jsoup.parse(NewPipe.getDownloader().download(complexUrl))
+                    .select("meta[property=og:url]").first();
 
-            String response = dl.download(complexUrl);
-            Document doc = Jsoup.parse(response);
-
-            Element ogElement = doc.select("meta[property=og:url]").first();
-            String url = ogElement.attr("content");
-
-            return url;
+            return ogElement.attr("content");
         } catch (Exception e) {
             throw new ParsingException(e.getMessage(), e);
         }

src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchEngine.java

@@ -1,9 +1,5 @@
 package org.schabi.newpipe.extractor.services.soundcloud;
 
-import java.io.IOException;
-import java.net.URLEncoder;
-import java.util.EnumSet;
-
 import org.json.JSONArray;
 import org.json.JSONObject;
 import org.schabi.newpipe.extractor.Downloader;
@@ -12,6 +8,10 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.search.InfoItemSearchCollector;
 import org.schabi.newpipe.extractor.search.SearchEngine;
 
+import java.io.IOException;
+import java.net.URLEncoder;
+import java.util.EnumSet;
+
 public class SoundcloudSearchEngine extends SearchEngine {
     public static final String CHARSET_UTF_8 = "UTF-8";
 

src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java

@@ -13,61 +13,48 @@ import java.io.IOException;
 
 public class SoundcloudService extends StreamingService {
 
-    public SoundcloudService(int id) {
-        super(id);
+    public SoundcloudService(int id, String name) {
+        super(id, name);
     }
 
     @Override
-    public ServiceInfo getServiceInfo() {
-        ServiceInfo serviceInfo = new ServiceInfo();
-        serviceInfo.name = "Soundcloud";
-        return serviceInfo;
-    }
-
-    @Override
-    public StreamExtractor getStreamExtractorInstance(String url)
-            throws ExtractionException, IOException {
-        UrlIdHandler urlIdHandler = SoundcloudStreamUrlIdHandler.getInstance();
-        if (urlIdHandler.acceptUrl(url)) {
-            return new SoundcloudStreamExtractor(urlIdHandler, url, getServiceId());
-        } else {
-            throw new IllegalArgumentException("supplied String is not a valid Soundcloud URL");
-        }
-    }
-
-    @Override
-    public SearchEngine getSearchEngineInstance() {
+    public SearchEngine getSearchEngine() {
         return new SoundcloudSearchEngine(getServiceId());
     }
 
     @Override
-    public UrlIdHandler getStreamUrlIdHandlerInstance() {
+    public UrlIdHandler getStreamUrlIdHandler() {
         return SoundcloudStreamUrlIdHandler.getInstance();
     }
 
     @Override
-    public UrlIdHandler getChannelUrlIdHandlerInstance() {
+    public UrlIdHandler getChannelUrlIdHandler() {
         return SoundcloudChannelUrlIdHandler.getInstance();
     }
 
-
     @Override
-    public UrlIdHandler getPlaylistUrlIdHandlerInstance() {
+    public UrlIdHandler getPlaylistUrlIdHandler() {
         return SoundcloudPlaylistUrlIdHandler.getInstance();
     }
 
+
     @Override
-    public ChannelExtractor getChannelExtractorInstance(String url) throws ExtractionException, IOException {
-        return new SoundcloudChannelExtractor(getChannelUrlIdHandlerInstance(), url, getServiceId());
+    public StreamExtractor getStreamExtractor(String url) throws IOException, ExtractionException {
+        return new SoundcloudStreamExtractor(this, url);
     }
 
     @Override
-    public PlaylistExtractor getPlaylistExtractorInstance(String url) throws ExtractionException, IOException {
-        return new SoundcloudPlaylistExtractor(getPlaylistUrlIdHandlerInstance(), url, getServiceId());
+    public ChannelExtractor getChannelExtractor(String url, String nextStreamsUrl) throws IOException, ExtractionException {
+        return new SoundcloudChannelExtractor(this, url, nextStreamsUrl);
     }
 
     @Override
-    public SuggestionExtractor getSuggestionExtractorInstance() {
+    public PlaylistExtractor getPlaylistExtractor(String url, String nextStreamsUrl) throws IOException, ExtractionException {
+        return new SoundcloudPlaylistExtractor(this, url, nextStreamsUrl);
+    }
+
+    @Override
+    public SuggestionExtractor getSuggestionExtractor() {
         return new SoundcloudSuggestionExtractor(getServiceId());
     }
 }

src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java

@@ -1,19 +1,14 @@
 package org.schabi.newpipe.extractor.services.soundcloud;
 
-import java.io.IOException;
-import java.util.List;
-import java.util.Vector;
-
 import org.json.JSONArray;
 import org.json.JSONObject;
 import org.schabi.newpipe.extractor.Downloader;
 import org.schabi.newpipe.extractor.MediaFormat;
 import org.schabi.newpipe.extractor.NewPipe;
-import org.schabi.newpipe.extractor.UrlIdHandler;
+import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
-import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
 import org.schabi.newpipe.extractor.stream.AudioStream;
 import org.schabi.newpipe.extractor.stream.StreamExtractor;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
@@ -21,33 +16,39 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
 import org.schabi.newpipe.extractor.stream.StreamType;
 import org.schabi.newpipe.extractor.stream.VideoStream;
 import org.schabi.newpipe.extractor.utils.Parser;
-import org.schabi.newpipe.extractor.utils.Parser.RegexException;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
 
 public class SoundcloudStreamExtractor extends StreamExtractor {
-    private String pageUrl;
-    private String trackId;
     private JSONObject track;
 
-    public SoundcloudStreamExtractor(UrlIdHandler urlIdHandler, String pageUrl, int serviceId) throws ExtractionException, IOException {
-        super(urlIdHandler, pageUrl, serviceId);
+    public SoundcloudStreamExtractor(StreamingService service, String url) throws IOException, ExtractionException {
+        super(service, url);
+    }
 
-        Downloader dl = NewPipe.getDownloader();
-
-        trackId = urlIdHandler.getId(pageUrl);
-        String apiUrl = "https://api-v2.soundcloud.com/tracks/" + trackId
-                + "?client_id=" + SoundcloudParsingHelper.clientId();
-
-        String response = dl.download(apiUrl);
-        track = new JSONObject(response);
+    @Override
+    public void fetchPage() throws IOException, ExtractionException {
+        track = SoundcloudParsingHelper.resolveFor(getOriginalUrl());
 
         if (!track.getString("policy").equals("ALLOW") && !track.getString("policy").equals("MONETIZE")) {
             throw new ContentNotAvailableException("Content not available: policy " + track.getString("policy"));
         }
     }
 
+    @Override
+    public String getCleanUrl() {
+        try {
+            return track.getString("permalink_url");
+        } catch (Exception e) {
+            return getOriginalUrl();
+        }
+    }
+
     @Override
     public String getId() {
-        return trackId;
+        return track.getInt("id") + "";
     }
 
     @Override
@@ -96,11 +97,11 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
     }
 
     @Override
-    public List<AudioStream> getAudioStreams() throws ReCaptchaException, IOException, RegexException {
-        Vector<AudioStream> audioStreams = new Vector<>();
+    public List<AudioStream> getAudioStreams() throws IOException, ExtractionException {
+        List<AudioStream> audioStreams = new ArrayList<>();
         Downloader dl = NewPipe.getDownloader();
 
-        String apiUrl = "https://api.soundcloud.com/i1/tracks/" + trackId + "/streams"
+        String apiUrl = "https://api.soundcloud.com/i1/tracks/" + getId() + "/streams"
                 + "?client_id=" + SoundcloudParsingHelper.clientId();
 
         String response = dl.download(apiUrl);
@@ -113,20 +114,20 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
     }
 
     @Override
-    public List<VideoStream> getVideoStreams() {
+    public List<VideoStream> getVideoStreams() throws IOException, ExtractionException {
         return null;
     }
 
     @Override
-    public List<VideoStream> getVideoOnlyStreams() {
-       return null;
+    public List<VideoStream> getVideoOnlyStreams() throws IOException, ExtractionException {
+        return null;
     }
 
     @Override
     public int getTimeStamp() throws ParsingException {
         String timeStamp;
         try {
-            timeStamp = Parser.matchGroup1("(#t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)", pageUrl);
+            timeStamp = Parser.matchGroup1("(#t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)", getOriginalUrl());
         } catch (Parser.RegexException e) {
             // catch this instantly since an url does not necessarily have to have a time stamp
 
@@ -190,16 +191,16 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
     }
 
     @Override
-    public StreamInfoItemExtractor getNextVideo() {
+    public StreamInfoItemExtractor getNextVideo() throws IOException, ExtractionException {
         return null;
     }
 
     @Override
-    public StreamInfoItemCollector getRelatedVideos() throws ReCaptchaException, IOException, ParsingException {
-        StreamInfoItemCollector collector = getStreamPreviewInfoCollector();
+    public StreamInfoItemCollector getRelatedVideos() throws IOException, ExtractionException {
+        StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
         Downloader dl = NewPipe.getDownloader();
 
-        String apiUrl = "https://api-v2.soundcloud.com/tracks/" + trackId + "/related"
+        String apiUrl = "https://api-v2.soundcloud.com/tracks/" + getId() + "/related"
                 + "?client_id=" + SoundcloudParsingHelper.clientId();
 
         String response = dl.download(apiUrl);

src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamInfoItemExtractor.java

@@ -35,7 +35,7 @@ public class SoundcloudStreamInfoItemExtractor implements StreamInfoItemExtracto
 
     @Override
     public String getUploadDate() throws ParsingException {
-        return SoundcloudParsingHelper.toTimeAgoString(searchResult.getString("created_at"));
+        return SoundcloudParsingHelper.toDateString(searchResult.getString("created_at"));
     }
 
     @Override

src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamUrlIdHandler.java

@@ -1,10 +1,7 @@
 package org.schabi.newpipe.extractor.services.soundcloud;
 
-import org.json.JSONObject;
 import org.jsoup.Jsoup;
-import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
-import org.schabi.newpipe.extractor.Downloader;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.UrlIdHandler;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
@@ -13,6 +10,7 @@ import org.schabi.newpipe.extractor.utils.Parser;
 public class SoundcloudStreamUrlIdHandler implements UrlIdHandler {
 
     private static final SoundcloudStreamUrlIdHandler instance = new SoundcloudStreamUrlIdHandler();
+
     private SoundcloudStreamUrlIdHandler() {
     }
 
@@ -23,13 +21,7 @@ public class SoundcloudStreamUrlIdHandler implements UrlIdHandler {
     @Override
     public String getUrl(String videoId) throws ParsingException {
         try {
-            Downloader dl = NewPipe.getDownloader();
-
-            String response = dl.download("https://api-v2.soundcloud.com/tracks/" + videoId
-                    + "?client_id=" + SoundcloudParsingHelper.clientId());
-            JSONObject responseObject = new JSONObject(response);
-
-            return responseObject.getString("permalink_url");
+            return SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/tracks/" + videoId);
         } catch (Exception e) {
             throw new ParsingException(e.getMessage(), e);
         }
@@ -38,15 +30,7 @@ public class SoundcloudStreamUrlIdHandler implements UrlIdHandler {
     @Override
     public String getId(String url) throws ParsingException {
         try {
-            Downloader dl = NewPipe.getDownloader();
-
-            String response = dl.download(url);
-            Document doc = Jsoup.parse(response);
-
-            Element androidElement = doc.select("meta[property=al:android:url]").first();
-            String id = androidElement.attr("content").substring(20);
-
-            return id;
+            return SoundcloudParsingHelper.resolveIdWithEmbedPlayer(url);
         } catch (Exception e) {
             throw new ParsingException(e.getMessage(), e);
         }
@@ -55,15 +39,10 @@ public class SoundcloudStreamUrlIdHandler implements UrlIdHandler {
     @Override
     public String cleanUrl(String complexUrl) throws ParsingException {
         try {
-            Downloader dl = NewPipe.getDownloader();
+            Element ogElement = Jsoup.parse(NewPipe.getDownloader().download(complexUrl))
+                    .select("meta[property=og:url]").first();
 
-            String response = dl.download(complexUrl);
-            Document doc = Jsoup.parse(response);
-
-            Element ogElement = doc.select("meta[property=og:url]").first();
-            String url = ogElement.attr("content");
-
-            return url;
+            return ogElement.attr("content");
         } catch (Exception e) {
             throw new ParsingException(e.getMessage(), e);
         }

src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSuggestionExtractor.java

@@ -1,10 +1,5 @@
 package org.schabi.newpipe.extractor.services.soundcloud;
 
-import java.io.IOException;
-import java.net.URLEncoder;
-import java.util.ArrayList;
-import java.util.List;
-
 import org.json.JSONArray;
 import org.json.JSONObject;
 import org.schabi.newpipe.extractor.Downloader;
@@ -13,6 +8,11 @@ import org.schabi.newpipe.extractor.SuggestionExtractor;
 import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
 import org.schabi.newpipe.extractor.utils.Parser.RegexException;
 
+import java.io.IOException;
+import java.net.URLEncoder;
+import java.util.ArrayList;
+import java.util.List;
+
 public class SoundcloudSuggestionExtractor extends SuggestionExtractor {
 
     public static final String CHARSET_UTF_8 = "UTF-8";

src/main/java/org/schabi/newpipe/extractor/services/youtube/ItagItem.java

@@ -43,7 +43,7 @@ public class ItagItem {
             // Disable Opus codec as it's not well supported in older devices
 //          new ItagItem(249, AUDIO, WEBMA, 50),
 //          new ItagItem(250, AUDIO, WEBMA, 70),
-//          new ItagItem(251, AUDIO, WEBMA, 16),
+//          new ItagItem(251, AUDIO, WEBMA, 160),
             new ItagItem(171, AUDIO, WEBMA, 128),
             new ItagItem(172, AUDIO, WEBMA, 256),
             new ItagItem(139, AUDIO, M4A, 48),

src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java

@@ -8,7 +8,7 @@ import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
 import org.schabi.newpipe.extractor.Downloader;
 import org.schabi.newpipe.extractor.NewPipe;
-import org.schabi.newpipe.extractor.UrlIdHandler;
+import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.channel.ChannelExtractor;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
@@ -44,6 +44,7 @@ import java.io.IOException;
 @SuppressWarnings("WeakerAccess")
 public class YoutubeChannelExtractor extends ChannelExtractor {
     private static final String CHANNEL_FEED_BASE = "https://www.youtube.com/feeds/videos.xml?channel_id=";
+    private static final String CHANNEL_URL_PARAMETERS = "/videos?view=0&flow=list&sort=dd&live_view=10000";
 
     private Document doc;
     /**
@@ -51,29 +52,26 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
      */
     private Document nextStreamsAjax;
 
-    /*//////////////////////////////////////////////////////////////////////////
-    // Variables for cache purposes (not "select" the current document all over again)
-    //////////////////////////////////////////////////////////////////////////*/
-    private String channelId;
-    private String channelName;
-    private String avatarUrl;
-    private String bannerUrl;
-    private String feedUrl;
-    private long subscriberCount = -1;
+    public YoutubeChannelExtractor(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException {
+        super(service, url, nextStreamsUrl);
+    }
 
-    public YoutubeChannelExtractor(UrlIdHandler urlIdHandler, String url, int serviceId) throws ExtractionException, IOException {
-        super(urlIdHandler, urlIdHandler.cleanUrl(url), serviceId);
-        fetchDocument();
+    @Override
+    public void fetchPage() throws IOException, ExtractionException {
+        Downloader downloader = NewPipe.getDownloader();
+
+        String userUrl = getCleanUrl() + CHANNEL_URL_PARAMETERS;
+        String pageContent = downloader.download(userUrl);
+        doc = Jsoup.parse(pageContent, userUrl);
+
+        nextStreamsUrl = getNextStreamsUrlFrom(doc);
+        nextStreamsAjax = null;
     }
 
     @Override
     public String getChannelId() throws ParsingException {
         try {
-            if (channelId == null) {
-                channelId = getUrlIdHandler().getId(getUrl());
-            }
-
-            return channelId;
+            return getUrlIdHandler().getId(getCleanUrl());
         } catch (Exception e) {
             throw new ParsingException("Could not get channel id");
         }
@@ -82,11 +80,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
     @Override
     public String getChannelName() throws ParsingException {
         try {
-            if (channelName == null) {
-                channelName = doc.select("span[class=\"qualified-channel-title-text\"]").first().select("a").first().text();
-            }
-
-            return channelName;
+            return doc.select("span[class=\"qualified-channel-title-text\"]").first().select("a").first().text();
         } catch (Exception e) {
             throw new ParsingException("Could not get channel name");
         }
@@ -95,11 +89,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
     @Override
     public String getAvatarUrl() throws ParsingException {
         try {
-            if (avatarUrl == null) {
-                avatarUrl = doc.select("img[class=\"channel-header-profile-image\"]").first().attr("abs:src");
-            }
-
-            return avatarUrl;
+            return doc.select("img[class=\"channel-header-profile-image\"]").first().attr("abs:src");
         } catch (Exception e) {
             throw new ParsingException("Could not get avatar", e);
         }
@@ -108,59 +98,47 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
     @Override
     public String getBannerUrl() throws ParsingException {
         try {
-            if (bannerUrl == null) {
-                Element el = doc.select("div[id=\"gh-banner\"]").first().select("style").first();
-                String cssContent = el.html();
-                String url = "https:" + Parser.matchGroup1("url\\(([^)]+)\\)", cssContent);
+            Element el = doc.select("div[id=\"gh-banner\"]").first().select("style").first();
+            String cssContent = el.html();
+            String url = "https:" + Parser.matchGroup1("url\\(([^)]+)\\)", cssContent);
 
-                bannerUrl = url.contains("s.ytimg.com") || url.contains("default_banner") ? null : url;
-            }
-
-            return bannerUrl;
+            return url.contains("s.ytimg.com") || url.contains("default_banner") ? null : url;
         } catch (Exception e) {
             throw new ParsingException("Could not get Banner", e);
         }
     }
 
-    @Override
-    public StreamInfoItemCollector getStreams() throws ParsingException {
-        StreamInfoItemCollector collector = getStreamPreviewInfoCollector();
-        Element ul = doc.select("ul[id=\"browse-items-primary\"]").first();
-        collectStreamsFrom(collector, ul);
-        return collector;
-    }
 
     @Override
     public long getSubscriberCount() throws ParsingException {
-
-        if (subscriberCount == -1) {
-            Element el = doc.select("span[class*=\"yt-subscription-button-subscriber-count\"]").first();
-            if (el != null) {
-                subscriberCount = Long.parseLong(Utils.removeNonDigitCharacters(el.text()));
-            } else {
-                throw new ParsingException("Could not get subscriber count");
-            }
+        Element el = doc.select("span[class*=\"yt-subscription-button-subscriber-count\"]").first();
+        if (el != null) {
+            return Long.parseLong(Utils.removeNonDigitCharacters(el.text()));
+        } else {
+            throw new ParsingException("Could not get subscriber count");
         }
-
-        return subscriberCount;
     }
 
     @Override
     public String getFeedUrl() throws ParsingException {
         try {
-            if (feedUrl == null) {
-                String channelId = doc.getElementsByClass("yt-uix-subscription-button").first().attr("data-channel-external-id");
-                feedUrl = channelId == null ? "" : CHANNEL_FEED_BASE + channelId;
-            }
-
-            return feedUrl;
+            String channelId = doc.getElementsByClass("yt-uix-subscription-button").first().attr("data-channel-external-id");
+            return channelId == null ? "" : CHANNEL_FEED_BASE + channelId;
         } catch (Exception e) {
             throw new ParsingException("Could not get feed url", e);
         }
     }
 
     @Override
-    public StreamInfoItemCollector getNextStreams() throws ExtractionException, IOException {
+    public StreamInfoItemCollector getStreams() throws IOException, ExtractionException {
+        StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
+        Element ul = doc.select("ul[id=\"browse-items-primary\"]").first();
+        collectStreamsFrom(collector, ul);
+        return collector;
+    }
+
+    @Override
+    public NextItemsResult getNextStreams() throws IOException, ExtractionException {
         if (!hasMoreStreams()) {
             throw new ExtractionException("Channel doesn't have more streams");
         }
@@ -169,7 +147,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
         setupNextStreamsAjax(NewPipe.getDownloader());
         collectStreamsFrom(collector, nextStreamsAjax.select("body").first());
 
-        return collector;
+        return new NextItemsResult(collector.getItemList(), nextStreamsUrl);
     }
 
     private void setupNextStreamsAjax(Downloader downloader) throws IOException, ReCaptchaException, ParsingException {
@@ -182,8 +160,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
 
             String nextStreamsHtmlDataRaw = ajaxData.getString("load_more_widget_html");
             if (!nextStreamsHtmlDataRaw.isEmpty()) {
-                Document nextStreamsData = Jsoup.parse(nextStreamsHtmlDataRaw, nextStreamsUrl);
-                nextStreamsUrl = getNextStreamsUrl(nextStreamsData);
+                nextStreamsUrl = getNextStreamsUrlFrom(Jsoup.parse(nextStreamsHtmlDataRaw, nextStreamsUrl));
             } else {
                 nextStreamsUrl = "";
             }
@@ -192,7 +169,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
         }
     }
 
-    private String getNextStreamsUrl(Document d) throws ParsingException {
+    private String getNextStreamsUrlFrom(Document d) throws ParsingException {
         try {
             Element button = d.select("button[class*=\"yt-uix-load-more\"]").first();
             if (button != null) {
@@ -206,17 +183,6 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
         }
     }
 
-    private void fetchDocument() throws IOException, ReCaptchaException, ParsingException {
-        Downloader downloader = NewPipe.getDownloader();
-
-        String userUrl = getUrl() + "/videos?view=0&flow=list&sort=dd&live_view=10000";
-        String pageContent = downloader.download(userUrl);
-        doc = Jsoup.parse(pageContent, userUrl);
-
-        nextStreamsUrl = getNextStreamsUrl(doc);
-        nextStreamsAjax = null;
-    }
-
     private void collectStreamsFrom(StreamInfoItemCollector collector, Element element) throws ParsingException {
         collector.getItemList().clear();
 
@@ -230,7 +196,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
 
                     @Override
                     public boolean isAd() throws ParsingException {
-                        return !li.select("span[class*=\"icon-not-available\"]").isEmpty();
+                        return !li.select("span[class*=\"icon-not-available\"]").isEmpty() ||
+                                !li.select("span[class*=\"yt-badge-ad\"]").isEmpty();
                     }
 
                     @Override
@@ -259,7 +226,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
                     public int getDuration() throws ParsingException {
                         try {
                             return YoutubeParsingHelper.parseDurationString(
-                                    li.select("span[class=\"video-time\"]").first().text());
+                                    li.select("span[class*=\"video-time\"]").first().text());
                         } catch (Exception e) {
                             if (isLiveStream(li)) {
                                 // -1 for no duration

src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelInfoItemExtractor.java

@@ -68,7 +68,7 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor
     }
 
     @Override
-    public long getViewCount() throws ParsingException {
+    public long getStreamCount() throws ParsingException {
         Element metaEl = el.select("ul[class*=\"yt-lockup-meta-info\"]").first();
         if (metaEl == null) {
             return 0;

src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractor.java

@@ -7,6 +7,7 @@ import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
 import org.schabi.newpipe.extractor.Downloader;
 import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.UrlIdHandler;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
@@ -23,39 +24,31 @@ import java.io.IOException;
 @SuppressWarnings("WeakerAccess")
 public class YoutubePlaylistExtractor extends PlaylistExtractor {
 
-    private Document doc = null;
+    private Document doc;
     /**
      * It's lazily initialized (when getNextStreams is called)
      */
-    private Document nextStreamsAjax = null;
+    private Document nextStreamsAjax;
 
-    /*//////////////////////////////////////////////////////////////////////////
-    // Variables for cache purposes (not "select" the current document all over again)
-    //////////////////////////////////////////////////////////////////////////*/
-    private String playlistId;
-    private String playlistName;
-    private String avatarUrl;
-    private String bannerUrl;
+    public YoutubePlaylistExtractor(StreamingService service, String url, String nextStreamsUrl) throws IOException, ExtractionException {
+        super(service, url, nextStreamsUrl);
+    }
 
-    private long streamsCount;
+    @Override
+    public void fetchPage() throws IOException, ExtractionException {
+        Downloader downloader = NewPipe.getDownloader();
 
-    private String uploaderUrl;
-    private String uploaderName;
-    private String uploaderAvatarUrl;
+        String pageContent = downloader.download(getCleanUrl());
+        doc = Jsoup.parse(pageContent, getCleanUrl());
 
-    public YoutubePlaylistExtractor(UrlIdHandler urlIdHandler, String url, int serviceId) throws IOException, ExtractionException {
-        super(urlIdHandler, urlIdHandler.cleanUrl(url), serviceId);
-        fetchDocument();
+        nextStreamsUrl = getNextStreamsUrlFrom(doc);
+        nextStreamsAjax = null;
     }
 
     @Override
     public String getPlaylistId() throws ParsingException {
         try {
-            if (playlistId == null) {
-                playlistId = getUrlIdHandler().getId(getUrl());
-            }
-
-            return playlistId;
+            return getUrlIdHandler().getId(getCleanUrl());
         } catch (Exception e) {
             throw new ParsingException("Could not get playlist id");
         }
@@ -64,11 +57,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
     @Override
     public String getPlaylistName() throws ParsingException {
         try {
-            if (playlistName == null) {
-                playlistName = doc.select("div[id=pl-header] h1[class=pl-header-title]").first().text();
-            }
-
-            return playlistName;
+            return doc.select("div[id=pl-header] h1[class=pl-header-title]").first().text();
         } catch (Exception e) {
             throw new ParsingException("Could not get playlist name");
         }
@@ -77,11 +66,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
     @Override
     public String getAvatarUrl() throws ParsingException {
         try {
-            if (avatarUrl == null) {
-                avatarUrl = doc.select("div[id=pl-header] div[class=pl-header-thumb] img").first().attr("abs:src");
-            }
-
-            return avatarUrl;
+            return doc.select("div[id=pl-header] div[class=pl-header-thumb] img").first().attr("abs:src");
         } catch (Exception e) {
             throw new ParsingException("Could not get playlist avatar");
         }
@@ -90,18 +75,16 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
     @Override
     public String getBannerUrl() throws ParsingException {
         try {
-            if (bannerUrl == null) {
-                Element el = doc.select("div[id=\"gh-banner\"] style").first();
-                String cssContent = el.html();
-                String url = "https:" + Parser.matchGroup1("url\\((.*)\\)", cssContent);
-                if (url.contains("s.ytimg.com")) {
-                    bannerUrl = null;
-                } else {
-                    bannerUrl = url.substring(0, url.indexOf(");"));
-                }
+            Element el = doc.select("div[id=\"gh-banner\"] style").first();
+            String cssContent = el.html();
+            String url = "https:" + Parser.matchGroup1("url\\((.*)\\)", cssContent);
+            if (url.contains("s.ytimg.com")) {
+                return null;
+            } else {
+                return url.substring(0, url.indexOf(");"));
             }
 
-            return bannerUrl;
+
         } catch (Exception e) {
             throw new ParsingException("Could not get playlist Banner");
         }
@@ -110,11 +93,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
     @Override
     public String getUploaderUrl() throws ParsingException {
         try {
-            if (uploaderUrl == null) {
-                uploaderUrl = doc.select("ul[class=\"pl-header-details\"] li").first().select("a").first().attr("abs:href");
-            }
-
-            return uploaderUrl;
+            return doc.select("ul[class=\"pl-header-details\"] li").first().select("a").first().attr("abs:href");
         } catch (Exception e) {
             throw new ParsingException("Could not get playlist uploader name");
         }
@@ -123,11 +102,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
     @Override
     public String getUploaderName() throws ParsingException {
         try {
-            if (uploaderName == null) {
-                uploaderName = doc.select("span[class=\"qualified-channel-title-text\"]").first().select("a").first().text();
-            }
-
-            return uploaderName;
+            return doc.select("span[class=\"qualified-channel-title-text\"]").first().select("a").first().text();
         } catch (Exception e) {
             throw new ParsingException("Could not get playlist uploader name");
         }
@@ -136,54 +111,46 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
     @Override
     public String getUploaderAvatarUrl() throws ParsingException {
         try {
-            if (uploaderAvatarUrl == null) {
-                uploaderAvatarUrl = doc.select("div[id=gh-banner] img[class=channel-header-profile-image]").first().attr("abs:src");
-            }
-
-            return uploaderAvatarUrl;
+            return doc.select("div[id=gh-banner] img[class=channel-header-profile-image]").first().attr("abs:src");
         } catch (Exception e) {
             throw new ParsingException("Could not get playlist uploader avatar");
         }
     }
 
     @Override
-    public long getStreamsCount() throws ParsingException {
-        if (streamsCount <= 0) {
-            String input;
+    public long getStreamCount() throws ParsingException {
+        String input;
 
-            try {
-                input = doc.select("ul[class=\"pl-header-details\"] li").get(1).text();
-            } catch (IndexOutOfBoundsException e) {
-                throw new ParsingException("Could not get video count from playlist", e);
-            }
-
-            try {
-                streamsCount = Long.parseLong(Utils.removeNonDigitCharacters(input));
-            } catch (NumberFormatException e) {
-                // When there's no videos in a playlist, there's no number in the "innerHtml",
-                // all characters that is not a number is removed, so we try to parse a empty string
-                if (!input.isEmpty()) {
-                    streamsCount = 0;
-                } else {
-                    throw new ParsingException("Could not handle input: " + input, e);
-                }
-            }
+        try {
+            input = doc.select("ul[class=\"pl-header-details\"] li").get(1).text();
+        } catch (IndexOutOfBoundsException e) {
+            throw new ParsingException("Could not get video count from playlist", e);
         }
 
-        return streamsCount;
+        try {
+            return Long.parseLong(Utils.removeNonDigitCharacters(input));
+        } catch (NumberFormatException e) {
+            // When there's no videos in a playlist, there's no number in the "innerHtml",
+            // all characters that is not a number is removed, so we try to parse a empty string
+            if (!input.isEmpty()) {
+                return 0;
+            } else {
+                throw new ParsingException("Could not handle input: " + input, e);
+            }
+        }
     }
 
     @Override
-    public StreamInfoItemCollector getStreams() throws ParsingException {
-        StreamInfoItemCollector collector = getStreamPreviewInfoCollector();
+    public StreamInfoItemCollector getStreams() throws IOException, ExtractionException {
+        StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
         Element tbody = doc.select("tbody[id=\"pl-load-more-destination\"]").first();
         collectStreamsFrom(collector, tbody);
         return collector;
     }
 
     @Override
-    public StreamInfoItemCollector getNextStreams() throws ExtractionException, IOException {
-        if (!hasMoreStreams()){
+    public NextItemsResult getNextStreams() throws IOException, ExtractionException {
+        if (!hasMoreStreams()) {
             throw new ExtractionException("Playlist doesn't have more streams");
         }
 
@@ -191,7 +158,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
         setupNextStreamsAjax(NewPipe.getDownloader());
         collectStreamsFrom(collector, nextStreamsAjax.select("tbody[id=\"pl-load-more-destination\"]").first());
 
-        return collector;
+        return new NextItemsResult(collector.getItemList(), nextStreamsUrl);
     }
 
     private void setupNextStreamsAjax(Downloader downloader) throws IOException, ReCaptchaException, ParsingException {
@@ -204,8 +171,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
 
             String nextStreamsHtmlDataRaw = ajaxData.getString("load_more_widget_html");
             if (!nextStreamsHtmlDataRaw.isEmpty()) {
-                final Document nextStreamsData = Jsoup.parse(nextStreamsHtmlDataRaw, nextStreamsUrl);
-                nextStreamsUrl = getNextStreamsUrl(nextStreamsData);
+                nextStreamsUrl = getNextStreamsUrlFrom(Jsoup.parse(nextStreamsHtmlDataRaw, nextStreamsUrl));
             } else {
                 nextStreamsUrl = "";
             }
@@ -214,17 +180,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
         }
     }
 
-    private void fetchDocument() throws IOException, ReCaptchaException, ParsingException {
-        Downloader downloader = NewPipe.getDownloader();
-
-        String pageContent = downloader.download(getUrl());
-        doc = Jsoup.parse(pageContent, getUrl());
-
-        nextStreamsUrl = getNextStreamsUrl(doc);
-        nextStreamsAjax = null;
-    }
-
-    private String getNextStreamsUrl(Document d) throws ParsingException {
+    private String getNextStreamsUrlFrom(Document d) throws ParsingException {
         try {
             Element button = d.select("button[class*=\"yt-uix-load-more\"]").first();
             if (button != null) {
@@ -241,7 +197,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
     private void collectStreamsFrom(StreamInfoItemCollector collector, Element element) throws ParsingException {
         collector.getItemList().clear();
 
-        final YoutubeStreamUrlIdHandler youtubeStreamUrlIdHandler = YoutubeStreamUrlIdHandler.getInstance();
+        final UrlIdHandler streamUrlIdHandler = getService().getStreamUrlIdHandler();
         for (final Element li : element.children()) {
             collector.commit(new StreamInfoItemExtractor() {
                 @Override
@@ -252,7 +208,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
                 @Override
                 public String getWebPageUrl() throws ParsingException {
                     try {
-                        return youtubeStreamUrlIdHandler.getUrl(li.attr("data-video-id"));
+                        return streamUrlIdHandler.getUrl(li.attr("data-video-id"));
                     } catch (Exception e) {
                         throw new ParsingException("Could not get web page url for the video", e);
                     }
@@ -300,7 +256,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
                 @Override
                 public String getThumbnailUrl() throws ParsingException {
                     try {
-                        return "https://i.ytimg.com/vi/" + youtubeStreamUrlIdHandler.getId(getWebPageUrl()) + "/hqdefault.jpg";
+                        return "https://i.ytimg.com/vi/" + streamUrlIdHandler.getId(getWebPageUrl()) + "/hqdefault.jpg";
                     } catch (Exception e) {
                         throw new ParsingException("Could not get thumbnail url", e);
                     }

src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java

@@ -34,61 +34,48 @@ import java.io.IOException;
 
 public class YoutubeService extends StreamingService {
 
-    public YoutubeService(int id) {
-        super(id);
+    public YoutubeService(int id, String name) {
+        super(id, name);
     }
 
     @Override
-    public ServiceInfo getServiceInfo() {
-        ServiceInfo serviceInfo = new ServiceInfo();
-        serviceInfo.name = "Youtube";
-        return serviceInfo;
-    }
-
-    @Override
-    public StreamExtractor getStreamExtractorInstance(String url)
-            throws ExtractionException, IOException {
-        UrlIdHandler urlIdHandler = YoutubeStreamUrlIdHandler.getInstance();
-        if (urlIdHandler.acceptUrl(url)) {
-            return new YoutubeStreamExtractor(urlIdHandler, url, getServiceId());
-        } else {
-            throw new IllegalArgumentException("supplied String is not a valid Youtube URL");
-        }
-    }
-
-    @Override
-    public SearchEngine getSearchEngineInstance() {
+    public SearchEngine getSearchEngine() {
         return new YoutubeSearchEngine(getServiceId());
     }
 
     @Override
-    public UrlIdHandler getStreamUrlIdHandlerInstance() {
+    public UrlIdHandler getStreamUrlIdHandler() {
         return YoutubeStreamUrlIdHandler.getInstance();
     }
 
     @Override
-    public UrlIdHandler getChannelUrlIdHandlerInstance() {
+    public UrlIdHandler getChannelUrlIdHandler() {
         return YoutubeChannelUrlIdHandler.getInstance();
     }
 
-
     @Override
-    public UrlIdHandler getPlaylistUrlIdHandlerInstance() {
+    public UrlIdHandler getPlaylistUrlIdHandler() {
         return YoutubePlaylistUrlIdHandler.getInstance();
     }
 
+
     @Override
-    public ChannelExtractor getChannelExtractorInstance(String url) throws ExtractionException, IOException {
-        return new YoutubeChannelExtractor(getChannelUrlIdHandlerInstance(), url, getServiceId());
+    public StreamExtractor getStreamExtractor(String url) throws IOException, ExtractionException {
+        return new YoutubeStreamExtractor(this, url);
     }
 
     @Override
-    public PlaylistExtractor getPlaylistExtractorInstance(String url) throws ExtractionException, IOException {
-        return new YoutubePlaylistExtractor(getPlaylistUrlIdHandlerInstance(), url, getServiceId());
+    public ChannelExtractor getChannelExtractor(String url, String nextStreamsUrl) throws IOException, ExtractionException {
+        return new YoutubeChannelExtractor(this, url, nextStreamsUrl);
     }
 
     @Override
-    public SuggestionExtractor getSuggestionExtractorInstance() {
+    public PlaylistExtractor getPlaylistExtractor(String url, String nextStreamsUrl) throws IOException, ExtractionException {
+        return new YoutubePlaylistExtractor(this, url, nextStreamsUrl);
+    }
+
+    @Override
+    public SuggestionExtractor getSuggestionExtractor() {
         return new YoutubeSuggestionExtractor(getServiceId());
     }
 }

src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java

@@ -10,7 +10,7 @@ import org.mozilla.javascript.Function;
 import org.mozilla.javascript.ScriptableObject;
 import org.schabi.newpipe.extractor.Downloader;
 import org.schabi.newpipe.extractor.NewPipe;
-import org.schabi.newpipe.extractor.UrlIdHandler;
+import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
@@ -26,9 +26,9 @@ import org.schabi.newpipe.extractor.utils.Parser;
 import org.schabi.newpipe.extractor.utils.Utils;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
-import java.util.Vector;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -80,12 +80,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     /*//////////////////////////////////////////////////////////////////////////*/
 
     private Document doc;
-    private final String dirtyUrl;
 
-    public YoutubeStreamExtractor(UrlIdHandler urlIdHandler, String pageUrl, int serviceId) throws ExtractionException, IOException {
-        super(urlIdHandler, urlIdHandler.cleanUrl(pageUrl), serviceId);
-        dirtyUrl = pageUrl;
-        fetchDocument();
+    public YoutubeStreamExtractor(StreamingService service, String url) throws IOException, ExtractionException {
+        super(service, url);
     }
 
     /*//////////////////////////////////////////////////////////////////////////
@@ -95,7 +92,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     @Override
     public String getId() throws ParsingException {
         try {
-            return getUrlIdHandler().getId(getUrl());
+            return getUrlIdHandler().getId(getCleanUrl());
         } catch (Exception e) {
             throw new ParsingException("Could not get stream id");
         }
@@ -238,8 +235,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     }
 
     @Override
-    public List<AudioStream> getAudioStreams() throws ParsingException {
-        Vector<AudioStream> audioStreams = new Vector<>();
+    public List<AudioStream> getAudioStreams() throws IOException, ExtractionException {
+        List<AudioStream> audioStreams = new ArrayList<>();
         try {
             String encodedUrlMap;
             // playerArgs could be null if the video is age restricted
@@ -288,8 +285,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     }
 
     @Override
-    public List<VideoStream> getVideoStreams() throws ParsingException {
-        Vector<VideoStream> videoStreams = new Vector<>();
+    public List<VideoStream> getVideoStreams() throws IOException, ExtractionException {
+        List<VideoStream> videoStreams = new ArrayList<>();
 
         try {
             String encodedUrlMap;
@@ -342,8 +339,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     }
 
     @Override
-    public List<VideoStream> getVideoOnlyStreams() throws ParsingException {
-        Vector<VideoStream> videoOnlyStreams = new Vector<>();
+    public List<VideoStream> getVideoOnlyStreams() throws IOException, ExtractionException {
+        List<VideoStream> videoOnlyStreams = new ArrayList<>();
 
         try {
             String encodedUrlMap;
@@ -405,7 +402,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     public int getTimeStamp() throws ParsingException {
         String timeStamp;
         try {
-            timeStamp = Parser.matchGroup1("((#|&|\\?)t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)", dirtyUrl);
+            timeStamp = Parser.matchGroup1("((#|&|\\?)t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)", getOriginalUrl());
         } catch (Parser.RegexException e) {
             // catch this instantly since an url does not necessarily have to have a time stamp
 
@@ -516,7 +513,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     }
 
     @Override
-    public StreamInfoItemExtractor getNextVideo() throws ParsingException {
+    public StreamInfoItemExtractor getNextVideo() throws IOException, ExtractionException {
         try {
             return extractVideoPreviewInfo(doc.select("div[class=\"watch-sidebar-section\"]").first()
                     .select("li").first());
@@ -526,9 +523,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     }
 
     @Override
-    public StreamInfoItemCollector getRelatedVideos() throws ParsingException {
+    public StreamInfoItemCollector getRelatedVideos() throws IOException, ExtractionException {
         try {
-            StreamInfoItemCollector collector = getStreamPreviewInfoCollector();
+            StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
             Element ul = doc.select("ul[id=\"watch-related\"]").first();
             if (ul != null) {
                 for (Element li : ul.children()) {
@@ -617,11 +614,12 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     // cached values
     private static volatile String decryptionCode = "";
 
-    private void fetchDocument() throws IOException, ReCaptchaException, ParsingException {
+    @Override
+    public void fetchPage() throws IOException, ExtractionException {
         Downloader downloader = NewPipe.getDownloader();
 
-        String pageContent = downloader.download(getUrl());
-        doc = Jsoup.parse(pageContent, getUrl());
+        String pageContent = downloader.download(getCleanUrl());
+        doc = Jsoup.parse(pageContent, getCleanUrl());
 
         JSONObject ytPlayerConfig;
         String playerUrl;
@@ -632,7 +630,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
 
         // Check if the video is age restricted
         if (pageContent.contains("<meta property=\"og:restrictions:age")) {
-            playerUrl = getPlayerUrlFromRestrictedVideo(getUrl());
+            playerUrl = getPlayerUrlFromRestrictedVideo();
             isAgeRestricted = true;
         } else {
             ytPlayerConfig = getPlayerConfig(pageContent);
@@ -683,7 +681,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
             throw new ParsingException("Could not parse yt player config", e);
         }
         if (isLiveStream) {
-            throw new LiveStreamException("This is a Life stream. Can't use those right now.");
+            throw new LiveStreamException("This is a Live stream. Can't use those right now.");
         }
 
         return playerArgs;
@@ -709,12 +707,11 @@ public class YoutubeStreamExtractor extends StreamExtractor {
         }
     }
 
-    private String getPlayerUrlFromRestrictedVideo(String pageUrl) throws ParsingException, ReCaptchaException {
+    private String getPlayerUrlFromRestrictedVideo() throws ParsingException, ReCaptchaException {
         try {
             Downloader downloader = NewPipe.getDownloader();
             String playerUrl = "";
-            String videoId = getUrlIdHandler().getId(pageUrl);
-            String embedUrl = "https://www.youtube.com/embed/" + videoId;
+            String embedUrl = "https://www.youtube.com/embed/" + getId();
             String embedPageContent = downloader.download(embedUrl);
             //todo: find out if this can be reapaced by Parser.matchGroup1()
             Pattern assetsPattern = Pattern.compile("\"assets\":.+?\"js\":\\s*(\"[^\"]+\")");
@@ -811,7 +808,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
 
             @Override
             public boolean isAd() throws ParsingException {
-                return !li.select("span[class*=\"icon-not-available\"]").isEmpty();
+                return !li.select("span[class*=\"icon-not-available\"]").isEmpty() ||
+                        !li.select("span[class*=\"yt-badge-ad\"]").isEmpty();
             }
 
             @Override
@@ -829,8 +827,17 @@ public class YoutubeStreamExtractor extends StreamExtractor {
 
             @Override
             public int getDuration() throws ParsingException {
-                return YoutubeParsingHelper.parseDurationString(
-                        li.select("span.video-time").first().text());
+                try {
+                    return YoutubeParsingHelper.parseDurationString(
+                            li.select("span[class*=\"video-time\"]").first().text());
+                } catch (Exception e) {
+                    if (isLiveStream(li)) {
+                        // -1 for no duration
+                        return -1;
+                    } else {
+                        throw new ParsingException("Could not get Duration: " + getTitle(), e);
+                    }
+                }
             }
 
             @Override
@@ -845,12 +852,6 @@ public class YoutubeStreamExtractor extends StreamExtractor {
 
             @Override
             public long getViewCount() throws ParsingException {
-                //this line is unused
-                //String views = li.select("span.view-count").first().text();
-
-                //Log.i(TAG, "title:"+info.title);
-                //Log.i(TAG, "view count:"+views);
-
                 try {
                     return Long.parseLong(Utils.removeNonDigitCharacters(
                             li.select("span.view-count").first().text()));
@@ -875,6 +876,19 @@ public class YoutubeStreamExtractor extends StreamExtractor {
                 }
                 return thumbnailUrl;
             }
+
+            private boolean isLiveStream(Element item) {
+                Element bla = item.select("span[class*=\"yt-badge-live\"]").first();
+
+                if (bla == null) {
+                    // sometimes livestreams dont have badges but sill are live streams
+                    // if video time is not available we most likly have an offline livestream
+                    if (item.select("span[class*=\"video-time\"]").first() == null) {
+                        return true;
+                    }
+                }
+                return bla != null;
+            }
         };
     }
 }

src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamInfoItemExtractor.java

@@ -59,7 +59,7 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
     public int getDuration() throws ParsingException {
         try {
             return YoutubeParsingHelper.parseDurationString(
-                    item.select("span[class=\"video-time\"]").first().text());
+                    item.select("span[class*=\"video-time\"]").first().text());
         } catch (Exception e) {
             if (isLiveStream(item)) {
                 // -1 for no duration
@@ -104,16 +104,14 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
             if (div == null) {
                 return -1;
             } else {
-                input = div.select("li").get(1)
-                        .text();
+                input = div.select("li").get(1).text();
             }
         } catch (IndexOutOfBoundsException e) {
             if (isLiveStream(item)) {
                 // -1 for no view count
                 return -1;
             } else {
-                throw new ParsingException(
-                        "Could not parse yt-lockup-meta although available: " + getTitle(), e);
+                throw new ParsingException("Could not parse yt-lockup-meta although available: " + getTitle(), e);
             }
         }
 
@@ -161,7 +159,8 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
 
     @Override
     public boolean isAd() throws ParsingException {
-        return !item.select("span[class*=\"icon-not-available\"]").isEmpty();
+        return !item.select("span[class*=\"icon-not-available\"]").isEmpty() ||
+                !item.select("span[class*=\"yt-badge-ad\"]").isEmpty();
     }
 
     private boolean isLiveStream(Element item) {

src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSuggestionExtractor.java

@@ -1,25 +1,17 @@
 package org.schabi.newpipe.extractor.services.youtube;
 
+import org.json.JSONArray;
 import org.schabi.newpipe.extractor.Downloader;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.SuggestionExtractor;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
-import org.w3c.dom.Node;
-import org.w3c.dom.NodeList;
-import org.xml.sax.InputSource;
-import org.xml.sax.SAXException;
 
-import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.net.URLEncoder;
 import java.util.ArrayList;
 import java.util.List;
 
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
-import javax.xml.parsers.ParserConfigurationException;
-
 /*
  * Created by Christian Schabesberger on 28.09.16.
  *
@@ -49,51 +41,24 @@ public class YoutubeSuggestionExtractor extends SuggestionExtractor {
     }
 
     @Override
-    public List<String> suggestionList(
-            String query, String contentCountry)
-            throws ExtractionException, IOException {
+    public List<String> suggestionList(String query, String contentCountry) throws IOException, ExtractionException {
+        Downloader dl = NewPipe.getDownloader();
         List<String> suggestions = new ArrayList<>();
 
-        Downloader dl = NewPipe.getDownloader();
-
         String url = "https://suggestqueries.google.com/complete/search"
-                + "?client=" + ""
-                + "&output=" + "toolbar"
+                + "?client=" + "firefox" // 'toolbar' for xml
                 + "&ds=" + "yt"
                 + "&hl=" + URLEncoder.encode(contentCountry, CHARSET_UTF_8)
                 + "&q=" + URLEncoder.encode(query, CHARSET_UTF_8);
 
-
         String response = dl.download(url);
-
-        //TODO: Parse xml data using Jsoup not done
-        DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
-        DocumentBuilder dBuilder;
-        org.w3c.dom.Document doc = null;
-
         try {
-            dBuilder = dbFactory.newDocumentBuilder();
-            doc = dBuilder.parse(new InputSource(
-                    new ByteArrayInputStream(response.getBytes(CHARSET_UTF_8))));
-            doc.getDocumentElement().normalize();
-        } catch (ParserConfigurationException | SAXException | IOException e) {
-            throw new ParsingException("Could not parse document.");
-        }
-
-        try {
-            NodeList nList = doc.getElementsByTagName("CompleteSuggestion");
-            for (int temp = 0; temp < nList.getLength(); temp++) {
-
-                NodeList nList1 = doc.getElementsByTagName("suggestion");
-                Node nNode1 = nList1.item(temp);
-                if (nNode1.getNodeType() == Node.ELEMENT_NODE) {
-                    org.w3c.dom.Element eElement = (org.w3c.dom.Element) nNode1;
-                    suggestions.add(eElement.getAttribute("data"));
-                }
-            }
-            return suggestions;
+            JSONArray suggestionsArray = new JSONArray(response).getJSONArray(1);
+            for (Object suggestion : suggestionsArray) suggestions.add(suggestion.toString());
         } catch (Exception e) {
-            throw new ParsingException("Could not get suggestions form document.", e);
+            throw new ParsingException("Could not parse suggestions response.", e);
         }
+
+        return suggestions;
     }
 }

src/main/java/org/schabi/newpipe/extractor/stream/Stream.java

@@ -13,7 +13,7 @@ public abstract class Stream implements Serializable {
     }
 
     /**
-     * Reveals whether two streams are the same, but have different urls
+     * Reveals whether two streams have the same stats (format and bitrate, for example)
      */
     public boolean equalStats(Stream cmp) {
         return cmp != null && format == cmp.format;

src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java

@@ -21,9 +21,10 @@ package org.schabi.newpipe.extractor.stream;
  */
 
 import org.schabi.newpipe.extractor.Extractor;
+import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.UrlIdHandler;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
-import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
 
 import java.io.IOException;
 import java.util.List;
@@ -33,8 +34,14 @@ import java.util.List;
  */
 public abstract class StreamExtractor extends Extractor {
 
-    public StreamExtractor(UrlIdHandler urlIdHandler, String url, int serviceId) {
-        super(urlIdHandler, serviceId, url);
+    public StreamExtractor(StreamingService service, String url) throws IOException, ExtractionException {
+        super(service, url);
+        fetchPage();
+    }
+
+    @Override
+    protected UrlIdHandler getUrlIdHandler() throws ParsingException {
+        return getService().getStreamUrlIdHandler();
     }
 
     public abstract String getId() throws ParsingException;
@@ -48,22 +55,22 @@ public abstract class StreamExtractor extends Extractor {
     public abstract String getUploadDate() throws ParsingException;
     public abstract String getThumbnailUrl() throws ParsingException;
     public abstract String getUploaderThumbnailUrl() throws ParsingException;
-    public abstract List<AudioStream> getAudioStreams() throws ParsingException, ReCaptchaException, IOException;
-    public abstract List<VideoStream> getVideoStreams() throws ParsingException;
-    public abstract List<VideoStream> getVideoOnlyStreams() throws ParsingException;
+    public abstract List<AudioStream> getAudioStreams() throws IOException, ExtractionException;
+    public abstract List<VideoStream> getVideoStreams() throws IOException, ExtractionException;
+    public abstract List<VideoStream> getVideoOnlyStreams() throws IOException, ExtractionException;
     public abstract String getDashMpdUrl() throws ParsingException;
     public abstract int getAgeLimit() throws ParsingException;
     public abstract String getAverageRating() throws ParsingException;
     public abstract int getLikeCount() throws ParsingException;
     public abstract int getDislikeCount() throws ParsingException;
-    public abstract StreamInfoItemExtractor getNextVideo() throws ParsingException;
-    public abstract StreamInfoItemCollector getRelatedVideos() throws ParsingException, ReCaptchaException, IOException;
+    public abstract StreamInfoItemExtractor getNextVideo() throws IOException, ExtractionException;
+    public abstract StreamInfoItemCollector getRelatedVideos() throws IOException, ExtractionException;
     public abstract StreamType getStreamType() throws ParsingException;
 
     /**
      * Analyses the webpage's document and extracts any error message there might be.
      *
-     * @return  Error message; null if there is no error message.
+     * @return Error message; null if there is no error message.
      */
     public abstract String getErrorMessage();
 }

src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java

@@ -2,14 +2,18 @@ package org.schabi.newpipe.extractor.stream;
 
 import org.schabi.newpipe.extractor.Info;
 import org.schabi.newpipe.extractor.InfoItem;
+import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.ServiceList;
+import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.utils.DashMpdParser;
 import org.schabi.newpipe.extractor.utils.Utils;
 
 import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.ArrayList;
 import java.util.List;
-import java.util.Vector;
 
 /*
  * Created by Christian Schabesberger on 26.08.15.
@@ -46,11 +50,23 @@ public class StreamInfo extends Info {
     public StreamInfo() {
     }
 
+    public static StreamInfo getInfo(String url) throws IOException, ExtractionException {
+        return getInfo(NewPipe.getServiceByUrl(url), url);
+    }
+
+    public static StreamInfo getInfo(ServiceList serviceItem, String url) throws IOException, ExtractionException {
+        return getInfo(serviceItem.getService(), url);
+    }
+
+    public static StreamInfo getInfo(StreamingService service, String url) throws IOException, ExtractionException {
+        return getInfo(service.getStreamExtractor(url));
+    }
+
     /**
      * Fills out the video info fields which are common to all services.
      * Probably needs to be overridden by subclasses
      */
-    public static StreamInfo getVideoInfo(StreamExtractor extractor) throws ExtractionException {
+    public static StreamInfo getInfo(StreamExtractor extractor) throws ExtractionException {
         StreamInfo streamInfo = new StreamInfo();
 
         try {
@@ -80,7 +96,7 @@ public class StreamInfo extends Info {
         // if one of these is not available an exception is meant to be thrown directly into the frontend.
 
         streamInfo.service_id = extractor.getServiceId();
-        streamInfo.url = extractor.getUrl();
+        streamInfo.url = extractor.getCleanUrl();
         streamInfo.stream_type = extractor.getStreamType();
         streamInfo.id = extractor.getId();
         streamInfo.name = extractor.getTitle();
@@ -128,15 +144,12 @@ public class StreamInfo extends Info {
         }
 
         // Lists can be null if a exception was thrown during extraction
-        if (streamInfo.video_streams == null) streamInfo.video_streams = new Vector<>();
-        if (streamInfo.video_only_streams == null) streamInfo.video_only_streams = new Vector<>();
-        if (streamInfo.audio_streams == null) streamInfo.audio_streams = new Vector<>();
+        if (streamInfo.video_streams == null) streamInfo.video_streams = new ArrayList<>();
+        if (streamInfo.video_only_streams == null) streamInfo.video_only_streams = new ArrayList<>();
+        if (streamInfo.audio_streams == null) streamInfo.audio_streams = new ArrayList<>();
 
         if (streamInfo.dashMpdUrl != null && !streamInfo.dashMpdUrl.isEmpty()) {
             try {
-                // Will try to find in the dash manifest for any stream that the ItagItem has (by the id),
-                // it has video, video only and audio streams and will only add to the list if it don't
-                // find a similar stream in the respective lists (calling Stream#equalStats).
                 DashMpdParser.getStreams(streamInfo);
             } catch (Exception e) {
                 // Sometimes we receive 403 (forbidden) error when trying to download the manifest,
@@ -246,6 +259,8 @@ public class StreamInfo extends Info {
             streamInfo.addException(e);
         }
 
+        if (streamInfo.related_streams == null) streamInfo.related_streams = new ArrayList<>();
+
         return streamInfo;
     }
 
@@ -278,7 +293,7 @@ public class StreamInfo extends Info {
     public int dislike_count = -1;
     public String average_rating;
     public StreamInfoItem next_video;
-    public List<InfoItem> related_streams = new Vector<>();
+    public List<InfoItem> related_streams;
     //in seconds. some metadata is not passed using a StreamInfo object!
     public int start_position = 0;
 }

src/main/java/org/schabi/newpipe/extractor/utils/DashMpdParser.java

@@ -53,7 +53,13 @@ public class DashMpdParser {
     }
 
     /**
-     * Download manifest and return nodelist with elements of tag "AdaptationSet"
+     * Will try to download (using {@link StreamInfo#dashMpdUrl}) and parse the dash manifest,
+     * then it will search for any stream that the ItagItem has (by the id).
+     * <p>
+     * It has video, video only and audio streams and will only add to the list if it don't
+     * find a similar stream in the respective lists (calling {@link Stream#equalStats}).
+     *
+     * @param streamInfo where the parsed streams will be added
      */
     public static void getStreams(StreamInfo streamInfo) throws DashMpdParsingException, ReCaptchaException {
         String dashDoc;

src/main/java/org/schabi/newpipe/extractor/utils/Utils.java

@@ -9,7 +9,7 @@ public class Utils {
      * Remove all non-digit characters from a string.<p>
      * Examples:<br/>
      * <ul><li>1 234 567 views -> 1234567</li>
-     * <li>$ 31,133.124 -> 31133124</li></ul>
+     * <li>$31,133.124 -> 31133124</li></ul>
      *
      * @param toRemove string to remove non-digit chars
      * @return a string that contains only digits
@@ -21,15 +21,23 @@ public class Utils {
     /**
      * Check if throwable have the cause
      */
-    public static boolean hasCauseThrowable(Throwable throwable, Class<?> causeToCheck) {
+    public static boolean hasCauseThrowable(Throwable throwable, Class<?>... causesToCheck) {
         // Check if getCause is not the same as cause (the getCause is already the root),
         // as it will cause a infinite loop if it is
-        Throwable cause,  getCause = throwable;
+        Throwable cause, getCause = throwable;
+
+        for (Class<?> causesEl : causesToCheck) {
+            if (throwable.getClass().isAssignableFrom(causesEl)) {
+                return true;
+            }
+        }
 
         while ((cause = throwable.getCause()) != null && getCause != cause) {
             getCause = cause;
-            if (cause.getClass().isAssignableFrom(causeToCheck)) {
-                return true;
+            for (Class<?> causesEl : causesToCheck) {
+                if (cause.getClass().isAssignableFrom(causesEl)) {
+                    return true;
+                }
             }
         }
         return false;

src/test/java/org/schabi/newpipe/Downloader.java

@@ -14,7 +14,7 @@ import java.util.Map;
 import javax.net.ssl.HttpsURLConnection;
 
 
-/**
+/*
  * Created by Christian Schabesberger on 28.01.16.
  *
  * Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
@@ -35,16 +35,17 @@ import javax.net.ssl.HttpsURLConnection;
  */
 
 public class Downloader implements org.schabi.newpipe.extractor.Downloader {
-    
+
     private static final String USER_AGENT = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0";
     private static String mCookies = "";
 
     private static Downloader instance = null;
 
-    private Downloader() {}
+    private Downloader() {
+    }
 
     public static Downloader getInstance() {
-        if(instance == null) {
+        if (instance == null) {
             synchronized (Downloader.class) {
                 if (instance == null) {
                     instance = new Downloader();
@@ -62,11 +63,14 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
         return Downloader.mCookies;
     }
 
-    /**Download the text file at the supplied URL as in download(String),
+    /**
+     * Download the text file at the supplied URL as in download(String),
      * but set the HTTP header field "Accept-Language" to the supplied string.
-     * @param siteUrl the URL of the text file to return the contents of
+     *
+     * @param siteUrl  the URL of the text file to return the contents of
      * @param language the language (usually a 2-character code) to set as the preferred language
-     * @return the contents of the specified text file*/
+     * @return the contents of the specified text file
+     */
     public String download(String siteUrl, String language) throws IOException, ReCaptchaException {
         Map<String, String> requestProperties = new HashMap<>();
         requestProperties.put("Accept-Language", language);
@@ -74,29 +78,35 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
     }
 
 
-    /**Download the text file at the supplied URL as in download(String),
+    /**
+     * Download the text file at the supplied URL as in download(String),
      * but set the HTTP header field "Accept-Language" to the supplied string.
-     * @param siteUrl the URL of the text file to return the contents of
+     *
+     * @param siteUrl          the URL of the text file to return the contents of
      * @param customProperties set request header properties
      * @return the contents of the specified text file
-     * @throws IOException*/
+     * @throws IOException
+     */
     public String download(String siteUrl, Map<String, String> customProperties) throws IOException, ReCaptchaException {
         URL url = new URL(siteUrl);
         HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
         Iterator it = customProperties.entrySet().iterator();
-        while(it.hasNext()) {
-            Map.Entry pair = (Map.Entry)it.next();
-            con.setRequestProperty((String)pair.getKey(), (String)pair.getValue());
+        while (it.hasNext()) {
+            Map.Entry pair = (Map.Entry) it.next();
+            con.setRequestProperty((String) pair.getKey(), (String) pair.getValue());
         }
         return dl(con);
     }
 
-    /**Common functionality between download(String url) and download(String url, String language)*/
+    /**
+     * Common functionality between download(String url) and download(String url, String language)
+     */
     private static String dl(HttpsURLConnection con) throws IOException, ReCaptchaException {
         StringBuilder response = new StringBuilder();
         BufferedReader in = null;
 
         try {
+            con.setReadTimeout(30 * 1000);// 30s
             con.setRequestMethod("GET");
             con.setRequestProperty("User-Agent", USER_AGENT);
 
@@ -108,13 +118,13 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
                     new InputStreamReader(con.getInputStream()));
             String inputLine;
 
-            while((inputLine = in.readLine()) != null) {
+            while ((inputLine = in.readLine()) != null) {
                 response.append(inputLine);
             }
-        } catch(UnknownHostException uhe) {//thrown when there's no internet connection
+        } catch (UnknownHostException uhe) {//thrown when there's no internet connection
             throw new IOException("unknown host or no network", uhe);
             //Toast.makeText(getActivity(), uhe.getMessage(), Toast.LENGTH_LONG).show();
-        } catch(Exception e) {
+        } catch (Exception e) {
             /*
              * HTTP 429 == Too Many Request
              * Receive from Youtube.com = ReCaptcha challenge request
@@ -123,9 +133,10 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
             if (con.getResponseCode() == 429) {
                 throw new ReCaptchaException("reCaptcha Challenge requested");
             }
-            throw new IOException(e);
+
+            throw new IOException(con.getResponseCode() + " " + con.getResponseMessage(), e);
         } finally {
-            if(in != null) {
+            if (in != null) {
                 in.close();
             }
         }
@@ -133,10 +144,13 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
         return response.toString();
     }
 
-    /**Download (via HTTP) the text file located at the supplied URL, and return its contents.
+    /**
+     * Download (via HTTP) the text file located at the supplied URL, and return its contents.
      * Primarily intended for downloading web pages.
+     *
      * @param siteUrl the URL of the text file to download
-     * @return the contents of the specified text file*/
+     * @return the contents of the specified text file
+     */
     public String download(String siteUrl) throws IOException, ReCaptchaException {
         URL url = new URL(siteUrl);
         HttpsURLConnection con = (HttpsURLConnection) url.openConnection();

src/test/java/org/schabi/newpipe/extractor/NewPipeTest.java

@@ -0,0 +1,74 @@
+package org.schabi.newpipe.extractor;
+
+import org.junit.Test;
+
+import java.util.HashSet;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertTrue;
+import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
+import static org.schabi.newpipe.extractor.ServiceList.Youtube;
+import static org.schabi.newpipe.extractor.NewPipe.getServiceByUrl;
+
+public class NewPipeTest {
+    @Test
+    public void getAllServicesTest() throws Exception {
+        assertEquals(NewPipe.getServices().length, ServiceList.values().length);
+    }
+
+    @Test
+    public void testAllServicesHaveDifferentId() throws Exception {
+        HashSet<Integer> servicesId = new HashSet<>();
+        for (StreamingService streamingService : NewPipe.getServices()) {
+            String errorMsg = "There are services with the same id = " + streamingService.getServiceId() + " (current service > " + streamingService.getServiceInfo().name + ")";
+
+            assertTrue(errorMsg, servicesId.add(streamingService.getServiceId()));
+        }
+    }
+
+    @Test
+    public void getServiceWithId() throws Exception {
+        assertEquals(NewPipe.getService(Youtube.getId()), Youtube.getService());
+        assertEquals(NewPipe.getService(SoundCloud.getId()), SoundCloud.getService());
+
+        assertNotEquals(NewPipe.getService(SoundCloud.getId()), Youtube.getService());
+    }
+
+    @Test
+    public void getServiceWithName() throws Exception {
+        assertEquals(NewPipe.getService(Youtube.getServiceInfo().name), Youtube.getService());
+        assertEquals(NewPipe.getService(SoundCloud.getServiceInfo().name), SoundCloud.getService());
+
+        assertNotEquals(NewPipe.getService(Youtube.getServiceInfo().name), SoundCloud.getService());
+    }
+
+    @Test
+    public void getServiceWithUrl() throws Exception {
+        assertEquals(getServiceByUrl("https://www.youtube.com/watch?v=_r6CgaFNAGg"), Youtube.getService());
+        assertEquals(getServiceByUrl("https://www.youtube.com/channel/UCi2bIyFtz-JdI-ou8kaqsqg"), Youtube.getService());
+        assertEquals(getServiceByUrl("https://www.youtube.com/playlist?list=PLRqwX-V7Uu6ZiZxtDDRCi6uhfTH4FilpH"), Youtube.getService());
+        assertEquals(getServiceByUrl("https://soundcloud.com/shupemoosic/pegboard-nerds-try-this"), SoundCloud.getService());
+        assertEquals(getServiceByUrl("https://soundcloud.com/deluxe314/sets/pegboard-nerds"), SoundCloud.getService());
+        assertEquals(getServiceByUrl("https://soundcloud.com/pegboardnerds"), SoundCloud.getService());
+
+        assertNotEquals(getServiceByUrl("https://soundcloud.com/pegboardnerds"), Youtube.getService());
+        assertNotEquals(getServiceByUrl("https://www.youtube.com/playlist?list=PLRqwX-V7Uu6ZiZxtDDRCi6uhfTH4FilpH"), SoundCloud.getService());
+    }
+
+    @Test
+    public void getIdWithServiceName() throws Exception {
+        assertEquals(NewPipe.getIdOfService(Youtube.getServiceInfo().name), Youtube.getId());
+        assertEquals(NewPipe.getIdOfService(SoundCloud.getServiceInfo().name), SoundCloud.getId());
+
+        assertNotEquals(NewPipe.getIdOfService(SoundCloud.getServiceInfo().name), Youtube.getId());
+    }
+
+    @Test
+    public void getServiceNameWithId() throws Exception {
+        assertEquals(NewPipe.getNameOfService(Youtube.getId()), Youtube.getServiceInfo().name);
+        assertEquals(NewPipe.getNameOfService(SoundCloud.getId()), SoundCloud.getServiceInfo().name);
+
+        assertNotEquals(NewPipe.getNameOfService(Youtube.getId()), SoundCloud.getServiceInfo().name);
+    }
+}

src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java

@@ -1,16 +1,17 @@
 package org.schabi.newpipe.extractor.services.youtube;
 
-import static junit.framework.Assert.assertEquals;
-import static junit.framework.Assert.assertNotNull;
-import static junit.framework.Assert.assertTrue;
-
 import org.junit.Before;
 import org.junit.Test;
 import org.schabi.newpipe.Downloader;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.channel.ChannelExtractor;
 
-/**
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.schabi.newpipe.extractor.ServiceList.Youtube;
+
+/*
  * Created by Christian Schabesberger on 12.09.16.
  *
  * Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
@@ -41,8 +42,8 @@ public class YoutubeChannelExtractorTest  {
     @Before
     public void setUp() throws Exception {
         NewPipe.init(Downloader.getInstance());
-        extractor = NewPipe.getService("Youtube")
-                .getChannelExtractorInstance("https://www.youtube.com/channel/UCYJ61XIK64sp6ZFFS8sctxw");
+        extractor = Youtube.getService()
+                .getChannelExtractor("https://www.youtube.com/channel/UCYJ61XIK64sp6ZFFS8sctxw");
     }
 
     @Test
@@ -61,7 +62,7 @@ public class YoutubeChannelExtractorTest  {
     }
 
     @Test
-    public void testGetBannerurl() throws Exception {
+    public void testGetBannerUrl() throws Exception {
         assertTrue(extractor.getBannerUrl(), extractor.getBannerUrl().contains("yt3"));
     }
 
@@ -81,9 +82,10 @@ public class YoutubeChannelExtractorTest  {
     }
 
     @Test
-    public void testHasNextPage() throws Exception {
-        // this particular example (link) has a next page !!!
-        assertTrue("no next page link found", extractor.hasMoreStreams());
+    public void testHasMoreStreams() throws Exception {
+        // Setup the streams
+        extractor.getStreams();
+        assertTrue("don't have more streams", extractor.hasMoreStreams());
     }
 
     @Test
@@ -92,16 +94,11 @@ public class YoutubeChannelExtractorTest  {
     }
 
     @Test
-    public void testGetNextPage() throws Exception {
-        extractor = NewPipe.getService("Youtube")
-                .getChannelExtractorInstance("https://www.youtube.com/channel/UCYJ61XIK64sp6ZFFS8sctxw");
-        assertTrue("next page didn't have content", !extractor.getStreams().getItemList().isEmpty());
+    public void testGetNextStreams() throws Exception {
+        // Setup the streams
+        extractor.getStreams();
+        assertTrue("extractor didn't have next streams", !extractor.getNextStreams().nextItemsList.isEmpty());
+        assertTrue("extractor didn't have more streams after getNextStreams", extractor.hasMoreStreams());
     }
 
-    @Test
-    public void testGetNextNextPageUrl() throws Exception {
-        extractor = NewPipe.getService("Youtube")
-                .getChannelExtractorInstance("https://www.youtube.com/channel/UCYJ61XIK64sp6ZFFS8sctxw");
-        assertTrue("next page didn't have content", extractor.hasMoreStreams());
-    }
 }

src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java

@@ -0,0 +1,98 @@
+package org.schabi.newpipe.extractor.services.youtube;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.schabi.newpipe.Downloader;
+import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.schabi.newpipe.extractor.ServiceList.Youtube;
+
+/**
+ * Test for {@link PlaylistExtractor}
+ */
+
+public class YoutubePlaylistExtractorTest {
+    private PlaylistExtractor extractor;
+
+    @Before
+    public void setUp() throws Exception {
+        NewPipe.init(Downloader.getInstance());
+        extractor = Youtube.getService()
+                .getPlaylistExtractor("https://www.youtube.com/playlist?list=PL7XlqX4npddfrdpMCxBnNZXg2GFll7t5y");
+    }
+
+    @Test
+    public void testGetDownloader()  throws Exception {
+        assertNotNull(NewPipe.getDownloader());
+    }
+
+    @Test
+    public void testGetId() throws Exception {
+        assertEquals(extractor.getPlaylistId(), "PL7XlqX4npddfrdpMCxBnNZXg2GFll7t5y");
+    }
+
+    @Test
+    public void testGetName() throws Exception {
+        assertEquals(extractor.getPlaylistName(), "important videos");
+    }
+
+    @Test
+    public void testGetAvatarUrl() throws Exception {
+        assertTrue(extractor.getAvatarUrl(), extractor.getAvatarUrl().contains("yt"));
+    }
+
+    @Test
+    public void testGetBannerUrl() throws Exception {
+        assertTrue(extractor.getBannerUrl(), extractor.getBannerUrl().contains("yt"));
+    }
+
+    @Test
+    public void testGetUploaderUrl() throws Exception {
+        assertTrue(extractor.getUploaderUrl(), extractor.getUploaderUrl().contains("youtube.com"));
+    }
+
+    @Test
+    public void testGetUploaderName() throws Exception {
+        assertTrue(extractor.getUploaderName(), !extractor.getUploaderName().isEmpty());
+    }
+
+    @Test
+    public void testGetUploaderAvatarUrl() throws Exception {
+        assertTrue(extractor.getUploaderAvatarUrl(), extractor.getUploaderAvatarUrl().contains("yt"));
+    }
+
+    @Test
+    public void testGetStreamsCount() throws Exception {
+        assertTrue("error in the streams count", extractor.getStreamCount() > 100);
+    }
+
+    @Test
+    public void testGetStreams() throws Exception {
+        assertTrue("no streams are received", !extractor.getStreams().getItemList().isEmpty());
+    }
+
+    @Test
+    public void testGetStreamsErrors() throws Exception {
+        assertTrue("errors during stream list extraction", extractor.getStreams().getErrors().isEmpty());
+    }
+
+    @Test
+    public void testHasMoreStreams() throws Exception {
+        // Setup the streams
+        extractor.getStreams();
+        assertTrue("extractor didn't have more streams", extractor.hasMoreStreams());
+    }
+
+    @Test
+    public void testGetNextStreams() throws Exception {
+        // Setup the streams
+        extractor.getStreams();
+        assertTrue("extractor didn't have next streams", !extractor.getNextStreams().nextItemsList.isEmpty());
+        assertTrue("extractor didn't have more streams after getNextStreams", extractor.hasMoreStreams());
+    }
+
+}

src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngineAllTest.java

@@ -9,11 +9,12 @@ import org.schabi.newpipe.extractor.search.SearchResult;
 
 import java.util.EnumSet;
 
-import static junit.framework.Assert.assertFalse;
-import static junit.framework.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.schabi.newpipe.extractor.ServiceList.Youtube;
 
 
-/**
+/*
  * Created by Christian Schabesberger on 29.12.15.
  *
  * Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
@@ -42,7 +43,7 @@ public class YoutubeSearchEngineAllTest {
     @Before
     public void setUp() throws Exception {
         NewPipe.init(Downloader.getInstance());
-        SearchEngine engine = NewPipe.getService("Youtube").getSearchEngineInstance();
+        SearchEngine engine = Youtube.getService().getSearchEngine();
 
         // Youtube will suggest "asdf" instead of "asdgff"
         // keep in mind that the suggestions can change by country (the parameter "de")

src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngineChannelTest.java

@@ -10,12 +10,13 @@ import org.schabi.newpipe.extractor.search.SearchResult;
 
 import java.util.EnumSet;
 
-import static junit.framework.Assert.assertEquals;
-import static junit.framework.Assert.assertFalse;
-import static junit.framework.Assert.assertTrue;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.schabi.newpipe.extractor.ServiceList.Youtube;
 
 
-/**
+/*
  * Created by Christian Schabesberger on 29.12.15.
  *
  * Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
@@ -44,7 +45,7 @@ public class YoutubeSearchEngineChannelTest {
     @Before
     public void setUp() throws Exception {
         NewPipe.init(Downloader.getInstance());
-        SearchEngine engine = NewPipe.getService("Youtube").getSearchEngineInstance();
+        SearchEngine engine = Youtube.getService().getSearchEngine();
 
         // Youtube will suggest "gronkh" instead of "grrunkh"
         // keep in mind that the suggestions can change by country (the parameter "de")
@@ -59,7 +60,9 @@ public class YoutubeSearchEngineChannelTest {
 
     @Test
     public void testChannelItemType() {
-        assertEquals(result.resultList.get(0).info_type, InfoItem.InfoType.CHANNEL);
+        for (InfoItem infoItem : result.resultList) {
+            assertEquals(InfoItem.InfoType.CHANNEL, infoItem.info_type);
+        }
     }
 
     @Test

src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngineStreamTest.java

@@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor.services.youtube;
 
 import org.junit.Before;
 import org.junit.Test;
+
 import org.schabi.newpipe.Downloader;
 import org.schabi.newpipe.extractor.InfoItem;
 import org.schabi.newpipe.extractor.NewPipe;
@@ -10,12 +11,13 @@ import org.schabi.newpipe.extractor.search.SearchResult;
 
 import java.util.EnumSet;
 
-import static junit.framework.Assert.assertEquals;
-import static junit.framework.Assert.assertFalse;
-import static junit.framework.Assert.assertTrue;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.schabi.newpipe.extractor.ServiceList.Youtube;
 
 
-/**
+/*
  * Created by Christian Schabesberger on 29.12.15.
  *
  * Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
@@ -44,7 +46,7 @@ public class YoutubeSearchEngineStreamTest {
     @Before
     public void setUp() throws Exception {
         NewPipe.init(Downloader.getInstance());
-        SearchEngine engine = NewPipe.getService("Youtube").getSearchEngineInstance();
+        SearchEngine engine = Youtube.getService().getSearchEngine();
 
         // Youtube will suggest "results" instead of "rsults",
         // keep in mind that the suggestions can change by country (the parameter "de")
@@ -58,8 +60,10 @@ public class YoutubeSearchEngineStreamTest {
     }
 
     @Test
-    public void testChannelItemType() {
-        assertEquals(result.resultList.get(0).info_type, InfoItem.InfoType.STREAM);
+    public void testStreamItemType() {
+        for (InfoItem infoItem : result.resultList) {
+            assertEquals(InfoItem.InfoType.STREAM, infoItem.info_type);
+        }
     }
 
     @Test

src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java

@@ -1,21 +1,23 @@
 package org.schabi.newpipe.extractor.services.youtube;
 
-import static junit.framework.Assert.assertTrue;
-
-import java.io.IOException;
-
 import org.junit.Before;
 import org.junit.Test;
 import org.schabi.newpipe.Downloader;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
-import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
 import org.schabi.newpipe.extractor.stream.StreamExtractor;
+import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
 import org.schabi.newpipe.extractor.stream.StreamType;
 import org.schabi.newpipe.extractor.stream.VideoStream;
 
-/**
+import java.io.IOException;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.schabi.newpipe.extractor.ServiceList.Youtube;
+
+/*
  * Created by Christian Schabesberger on 30.12.15.
  *
  * Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
@@ -45,8 +47,7 @@ public class YoutubeStreamExtractorDefaultTest {
     @Before
     public void setUp() throws Exception {
         NewPipe.init(Downloader.getInstance());
-        extractor = NewPipe.getService("Youtube")
-                .getStreamExtractorInstance("https://www.youtube.com/watch?v=YQHsXMglC9A");
+        extractor = Youtube.getService().getStreamExtractor("https://www.youtube.com/watch?v=YQHsXMglC9A");
     }
 
     @Test
@@ -56,10 +57,8 @@ public class YoutubeStreamExtractorDefaultTest {
     }
 
     @Test
-    public void testGetValidTimeStamp() throws ExtractionException, IOException {
-        StreamExtractor extractor =
-                NewPipe.getService("Youtube")
-                        .getStreamExtractorInstance("https://youtu.be/FmG385_uUys?t=174");
+    public void testGetValidTimeStamp() throws IOException, ExtractionException {
+        StreamExtractor extractor = Youtube.getService().getStreamExtractor("https://youtu.be/FmG385_uUys?t=174");
         assertTrue(Integer.toString(extractor.getTimeStamp()),
                 extractor.getTimeStamp() == 174);
     }
@@ -113,12 +112,12 @@ public class YoutubeStreamExtractorDefaultTest {
     }
 
     @Test
-    public void testGetAudioStreams() throws ParsingException, ReCaptchaException, IOException {
+    public void testGetAudioStreams() throws IOException, ExtractionException {
         assertTrue(!extractor.getAudioStreams().isEmpty());
     }
 
     @Test
-    public void testGetVideoStreams() throws ParsingException {
+    public void testGetVideoStreams() throws IOException, ExtractionException {
         for(VideoStream s : extractor.getVideoStreams()) {
             assertTrue(s.url,
                     s.url.contains(HTTPS));
@@ -138,4 +137,11 @@ public class YoutubeStreamExtractorDefaultTest {
         assertTrue(extractor.getDashMpdUrl(),
                 extractor.getDashMpdUrl() != null || !extractor.getDashMpdUrl().isEmpty());
     }
+
+    @Test
+    public void testGetRelatedVideos() throws ExtractionException, IOException {
+        StreamInfoItemCollector relatedVideos = extractor.getRelatedVideos();
+        assertFalse(relatedVideos.getItemList().isEmpty());
+        assertTrue(relatedVideos.getErrors().isEmpty());
+    }
 }

src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorGemaTest.java

@@ -0,0 +1,53 @@
+package org.schabi.newpipe.extractor.services.youtube;
+
+import org.junit.Ignore;
+import org.junit.Test;
+import org.schabi.newpipe.Downloader;
+import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+
+import java.io.IOException;
+
+import static org.junit.Assert.fail;
+import static org.schabi.newpipe.extractor.ServiceList.Youtube;
+
+/*
+ * Created by Christian Schabesberger on 30.12.15.
+ *
+ * Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
+ * YoutubeVideoExtractorGema.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/**
+ * This exception is only thrown in Germany.
+ *
+ * WARNING: Deactivate this Test Case before uploading it to Github, otherwise CI will fail.
+ */
+@Ignore
+public class YoutubeStreamExtractorGemaTest {
+
+    @Test
+    public void testGemaError() throws IOException, ExtractionException {
+        try {
+            NewPipe.init(Downloader.getInstance());
+            Youtube.getService().getStreamExtractor("https://www.youtube.com/watch?v=3O1_3zBUKM8");
+
+            fail("GemaException should be thrown");
+        } catch (YoutubeStreamExtractor.GemaException ignored) {
+            // Exception was thrown, Gema error detection is working.
+        }
+    }
+}

src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorRestrictedTest.java

@@ -0,0 +1,108 @@
+package org.schabi.newpipe.extractor.services.youtube;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.schabi.newpipe.Downloader;
+import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.stream.StreamExtractor;
+import org.schabi.newpipe.extractor.stream.VideoStream;
+
+import java.io.IOException;
+
+import static org.junit.Assert.assertTrue;
+import static org.schabi.newpipe.extractor.ServiceList.Youtube;
+
+/**
+ * Test for {@link YoutubeStreamUrlIdHandler}
+ */
+public class YoutubeStreamExtractorRestrictedTest {
+    public static final String HTTPS = "https://";
+    private StreamExtractor extractor;
+
+    @Before
+    public void setUp() throws Exception {
+        NewPipe.init(Downloader.getInstance());
+        extractor = Youtube.getService()
+                .getStreamExtractor("https://www.youtube.com/watch?v=i6JTvzrpBy0");
+    }
+
+    @Test
+    public void testGetInvalidTimeStamp() throws ParsingException {
+        assertTrue(Integer.toString(extractor.getTimeStamp()),
+                extractor.getTimeStamp() <= 0);
+    }
+
+    @Test
+    public void testGetValidTimeStamp() throws IOException, ExtractionException {
+        StreamExtractor extractor= Youtube.getService()
+                .getStreamExtractor("https://youtu.be/FmG385_uUys?t=174");
+        assertTrue(Integer.toString(extractor.getTimeStamp()),
+                extractor.getTimeStamp() == 174);
+    }
+
+    @Test
+    public void testGetAgeLimit() throws ParsingException {
+        assertTrue(extractor.getAgeLimit() == 18);
+    }
+
+    @Test
+    public void testGetTitle() throws ParsingException {
+        assertTrue(!extractor.getTitle().isEmpty());
+    }
+
+    @Test
+    public void testGetDescription() throws ParsingException {
+        assertTrue(extractor.getDescription() != null);
+    }
+
+    @Test
+    public void testGetUploader() throws ParsingException {
+        assertTrue(!extractor.getUploader().isEmpty());
+    }
+
+    @Test
+    public void testGetLength() throws ParsingException {
+        assertTrue(extractor.getLength() > 0);
+    }
+
+    @Test
+    public void testGetViews() throws ParsingException {
+        assertTrue(extractor.getLength() > 0);
+    }
+
+    @Test
+    public void testGetUploadDate() throws ParsingException {
+        assertTrue(extractor.getUploadDate().length() > 0);
+    }
+
+    @Test
+    public void testGetThumbnailUrl() throws ParsingException {
+        assertTrue(extractor.getThumbnailUrl(),
+                extractor.getThumbnailUrl().contains(HTTPS));
+    }
+
+    @Test
+    public void testGetUploaderThumbnailUrl() throws ParsingException {
+        assertTrue(extractor.getUploaderThumbnailUrl(),
+                extractor.getUploaderThumbnailUrl().contains(HTTPS));
+    }
+
+    @Test
+    public void testGetAudioStreams() throws IOException, ExtractionException {
+        // audiostream not always necessary
+        assertTrue(!extractor.getAudioStreams().isEmpty());
+    }
+
+    @Test
+    public void testGetVideoStreams() throws IOException, ExtractionException {
+        for(VideoStream s : extractor.getVideoStreams()) {
+            assertTrue(s.url,
+                    s.url.contains(HTTPS));
+            assertTrue(s.resolution.length() > 0);
+            assertTrue(Integer.toString(s.format),
+                    0 <= s.format && s.format <= 4);
+        }
+    }
+}

src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamUrlIdHandlerTest.java

@@ -0,0 +1,117 @@
+package org.schabi.newpipe.extractor.services.youtube;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.schabi.newpipe.Downloader;
+import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.exceptions.FoundAdException;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Test for {@link YoutubeStreamUrlIdHandler}
+ */
+public class YoutubeStreamUrlIdHandlerTest {
+    private static String AD_URL = "https://googleads.g.doubleclick.net/aclk?sa=l&ai=C-2IPgeVTWPf4GcOStgfOnIOADf78n61GvKmmobYDrgIQASDj-5MDKAJg9ZXOgeAEoAGgy_T-A8gBAakC2gkpmquIsT6oAwGqBJMBT9BgD5kVgbN0dX602bFFaDw9vsxq-We-S8VkrXVBi6W_e7brZ36GCz1WO3EPEeklYuJjXLUowwCOKsd-8xr1UlS_tusuFJv9iX35xoBHKTRvs8-0aDbfEIm6in37QDfFuZjqgEMB8-tg0Jn_Pf1RU5OzbuU40B4Gy25NUTnOxhDKthOhKBUSZEksCEerUV8GMu10iAXCxquwApIFBggDEAEYAaAGGsgGlIjthrUDgAfItIsBqAemvhvYBwHSCAUIgGEQAbgT6AE&num=1&sig=AOD64_1DybDd4qAm5O7o9UAbTNRdqXXHFQ&ctype=21&video_id=dMO_IXYPZew&client=ca-pub-6219811747049371&adurl=http://www.youtube.com/watch%3Fv%3DdMO_IXYPZew";
+    private YoutubeStreamUrlIdHandler urlIdHandler;
+
+    @Before
+    public void setUp() throws Exception {
+        urlIdHandler = YoutubeStreamUrlIdHandler.getInstance();
+        NewPipe.init(Downloader.getInstance());
+    }
+
+    @Test(expected = NullPointerException.class)
+    public void getIdWithNullAsUrl() throws ParsingException {
+        urlIdHandler.getId(null);
+    }
+
+    @Test(expected = FoundAdException.class)
+    public void getIdForAd() throws ParsingException {
+        urlIdHandler.getId(AD_URL);
+    }
+
+    @Test
+    public void getIdForInvalidUrls() throws ParsingException {
+        List<String> invalidUrls = new ArrayList<>(50);
+        invalidUrls.add("https://www.youtube.com/watch?v=jZViOEv90d");
+        invalidUrls.add("https://www.youtube.com/watchjZViOEv90d");
+        invalidUrls.add("https://www.youtube.com/");
+        for(String invalidUrl: invalidUrls) {
+            Throwable exception = null;
+            try {
+                urlIdHandler.getId(invalidUrl);
+            } catch (ParsingException e) {
+                exception = e;
+            }
+            if(exception == null) {
+                fail("Expected ParsingException for url: " + invalidUrl);
+            }
+        }
+    }
+    @Test
+    public void getId() throws Exception {
+        assertEquals("jZViOEv90dI", urlIdHandler.getId("https://www.youtube.com/watch?v=jZViOEv90dI"));
+        assertEquals("W-fFHeTX70Q", urlIdHandler.getId("https://www.youtube.com/watch?v=W-fFHeTX70Q"));
+        assertEquals("jZViOEv90dI", urlIdHandler.getId("https://www.youtube.com/watch?v=jZViOEv90dI?t=100"));
+        assertEquals("jZViOEv90dI", urlIdHandler.getId("https://WWW.YouTube.com/watch?v=jZViOEv90dI?t=100"));
+        assertEquals("jZViOEv90dI", urlIdHandler.getId("HTTPS://www.youtube.com/watch?v=jZViOEv90dI?t=100"));
+        assertEquals("jZViOEv90dI", urlIdHandler.getId("https://youtu.be/jZViOEv90dI?t=9s"));
+        assertEquals("jZViOEv90dI", urlIdHandler.getId("HTTPS://Youtu.be/jZViOEv90dI?t=9s"));
+        assertEquals("uEJuoEs1UxY", urlIdHandler.getId("http://www.youtube.com/watch_popup?v=uEJuoEs1UxY"));
+        assertEquals("uEJuoEs1UxY", urlIdHandler.getId("http://www.Youtube.com/watch_popup?v=uEJuoEs1UxY"));
+        assertEquals("jZViOEv90dI", urlIdHandler.getId("https://www.youtube.com/embed/jZViOEv90dI"));
+        assertEquals("jZViOEv90dI", urlIdHandler.getId("https://www.youtube-nocookie.com/embed/jZViOEv90dI"));
+        assertEquals("jZViOEv90dI", urlIdHandler.getId("http://www.youtube.com/watch?v=jZViOEv90dI"));
+        assertEquals("jZViOEv90dI", urlIdHandler.getId("http://youtube.com/watch?v=jZViOEv90dI"));
+        assertEquals("jZViOEv90dI", urlIdHandler.getId("http://youtu.be/jZViOEv90dI?t=9s"));
+        assertEquals("7_WWz2DSnT8", urlIdHandler.getId("https://youtu.be/7_WWz2DSnT8"));
+        assertEquals("oy6NvWeVruY", urlIdHandler.getId("https://m.youtube.com/watch?v=oy6NvWeVruY"));
+        assertEquals("jZViOEv90dI", urlIdHandler.getId("http://www.youtube.com/embed/jZViOEv90dI"));
+        assertEquals("jZViOEv90dI", urlIdHandler.getId("http://www.Youtube.com/embed/jZViOEv90dI"));
+        assertEquals("jZViOEv90dI", urlIdHandler.getId("http://www.youtube-nocookie.com/embed/jZViOEv90dI"));
+        assertEquals("EhxJLojIE_o", urlIdHandler.getId("http://www.youtube.com/attribution_link?a=JdfC0C9V6ZI&u=%2Fwatch%3Fv%3DEhxJLojIE_o%26feature%3Dshare"));
+        assertEquals("jZViOEv90dI", urlIdHandler.getId("vnd.youtube://www.youtube.com/watch?v=jZViOEv90dI"));
+        assertEquals("jZViOEv90dI", urlIdHandler.getId("vnd.youtube:jZViOEv90dI"));
+
+        // Shared links
+        String sharedId = "7JIArTByb3E";
+        String realId = "Q7JsK50NGaA";
+        assertEquals(realId, urlIdHandler.getId("vnd.youtube://www.YouTube.com/shared?ci=" + sharedId + "&feature=twitter-deep-link"));
+        assertEquals(realId, urlIdHandler.getId("vnd.youtube://www.youtube.com/shared?ci=" + sharedId ));
+        assertEquals(realId, urlIdHandler.getId("https://www.youtube.com/shared?ci=" + sharedId));
+    }
+
+
+    @Test
+    public void testAcceptUrl() {
+        assertTrue(urlIdHandler.acceptUrl("https://www.youtube.com/watch?v=jZViOEv90dI"));
+        assertTrue(urlIdHandler.acceptUrl("https://www.youtube.com/watch?v=jZViOEv90dI?t=100"));
+        assertTrue(urlIdHandler.acceptUrl("https://WWW.YouTube.com/watch?v=jZViOEv90dI?t=100"));
+        assertTrue(urlIdHandler.acceptUrl("HTTPS://www.youtube.com/watch?v=jZViOEv90dI?t=100"));
+        assertTrue(urlIdHandler.acceptUrl("https://youtu.be/jZViOEv90dI?t=9s"));
+        //assertTrue(urlIdHandler.acceptUrl("https://www.youtube.com/watch/jZViOEv90dI"));
+        assertTrue(urlIdHandler.acceptUrl("https://www.youtube.com/embed/jZViOEv90dI"));
+        assertTrue(urlIdHandler.acceptUrl("https://www.youtube-nocookie.com/embed/jZViOEv90dI"));
+        assertTrue(urlIdHandler.acceptUrl("http://www.youtube.com/watch?v=jZViOEv90dI"));
+        assertTrue(urlIdHandler.acceptUrl("http://youtu.be/jZViOEv90dI?t=9s"));
+        assertTrue(urlIdHandler.acceptUrl("http://www.youtube.com/embed/jZViOEv90dI"));
+        assertTrue(urlIdHandler.acceptUrl("http://www.youtube-nocookie.com/embed/jZViOEv90dI"));
+        assertTrue(urlIdHandler.acceptUrl("http://www.youtube.com/attribution_link?a=JdfC0C9V6ZI&u=%2Fwatch%3Fv%3DEhxJLojIE_o%26feature%3Dshare"));
+        assertTrue(urlIdHandler.acceptUrl("vnd.youtube://www.youtube.com/watch?v=jZViOEv90dI"));
+        assertTrue(urlIdHandler.acceptUrl("vnd.youtube:jZViOEv90dI"));
+
+        assertTrue(urlIdHandler.acceptUrl("vnd.youtube:jZViOEv90dI"));
+
+        String sharedId = "8A940MXKFmQ";
+        assertTrue(urlIdHandler.acceptUrl("vnd.youtube://www.youtube.com/shared?ci=" + sharedId + "&feature=twitter-deep-link"));
+        assertTrue(urlIdHandler.acceptUrl("vnd.youtube://www.youtube.com/shared?ci=" + sharedId ));
+        assertTrue(urlIdHandler.acceptUrl("https://www.youtube.com/shared?ci=" + sharedId));
+    }
+}

src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSuggestionExtractorTest.java

@@ -0,0 +1,51 @@
+package org.schabi.newpipe.extractor.services.youtube;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.schabi.newpipe.Downloader;
+import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.SuggestionExtractor;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+
+import java.io.IOException;
+
+import static org.junit.Assert.assertFalse;
+import static org.schabi.newpipe.extractor.ServiceList.Youtube;
+
+/*
+ * Created by Christian Schabesberger on 18.11.16.
+ *
+ * Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
+ * YoutubeSuggestionExtractorTest.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/**
+ * Test for {@link SuggestionExtractor}
+ */
+public class YoutubeSuggestionExtractorTest {
+    private SuggestionExtractor suggestionExtractor;
+
+    @Before
+    public void setUp() throws Exception {
+        NewPipe.init(Downloader.getInstance());
+        suggestionExtractor = Youtube.getService().getSuggestionExtractor();
+    }
+
+    @Test
+    public void testIfSuggestions() throws IOException, ExtractionException {
+        assertFalse(suggestionExtractor.suggestionList("hello", "de").isEmpty());
+    }
+}