Fix some bugs discovered by the new tests

src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java

@@ -1,5 +1,6 @@
 package org.schabi.newpipe.extractor.services.soundcloud;
 
+import com.grack.nanojson.JsonArray;
 import com.grack.nanojson.JsonObject;
 import com.grack.nanojson.JsonParser;
 import com.grack.nanojson.JsonParserException;
@@ -14,6 +15,8 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
 import javax.annotation.Nonnull;
 import java.io.IOException;
 
+import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
+
 @SuppressWarnings("WeakerAccess")
 public class SoundcloudChannelExtractor extends ChannelExtractor {
     private String userId;
@@ -30,7 +33,7 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
     public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
 
         userId = getUrlIdHandler().getId(getOriginalUrl());
-        String apiUrl = "https://api.soundcloud.com/users/" + userId +
+        String apiUrl = "https://api-v2.soundcloud.com/users/" + userId +
                 "?client_id=" + SoundcloudParsingHelper.clientId();
 
         String response = downloader.download(apiUrl);
@@ -44,7 +47,7 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
     @Nonnull
     @Override
     public String getCleanUrl() {
-        return user.isString("permalink_url") ? user.getString("permalink_url") : getOriginalUrl();
+        return user.isString("permalink_url") ? replaceHttpWithHttps(user.getString("permalink_url")) : getOriginalUrl();
     }
 
     @Nonnull
@@ -66,11 +69,10 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
 
     @Override
     public String getBannerUrl() {
-        try {
+        return user.getObject("visuals", new JsonObject())
-            return user.getObject("visuals").getArray("visuals").getObject(0).getString("visual_url", "");
+                .getArray("visuals", new JsonArray())
-        } catch (NullPointerException e) {
+                .getObject(0, new JsonObject())
-            return null;
+                .getString("visual_url");
-        }
     }
 
     @Override

src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelInfoItemExtractor.java

@@ -3,6 +3,8 @@ package org.schabi.newpipe.extractor.services.soundcloud;
 import com.grack.nanojson.JsonObject;
 import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
 
+import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
+
 public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtractor {
     private final JsonObject itemObject;
 
@@ -17,7 +19,7 @@ public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtrac
 
     @Override
     public String getUrl() {
-        return itemObject.getString("permalink_url");
+        return replaceHttpWithHttps(itemObject.getString("permalink_url"));
     }
 
     @Override

src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelUrlIdHandler.java

@@ -8,6 +8,8 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.utils.Parser;
 import org.schabi.newpipe.extractor.utils.Utils;
 
+import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
+
 public class SoundcloudChannelUrlIdHandler implements UrlIdHandler {
     private static final SoundcloudChannelUrlIdHandler instance = new SoundcloudChannelUrlIdHandler();
     private final String URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/[0-9a-z_-]+" +
@@ -45,7 +47,7 @@ public class SoundcloudChannelUrlIdHandler implements UrlIdHandler {
             Element ogElement = Jsoup.parse(NewPipe.getDownloader().download(complexUrl))
                     .select("meta[property=og:url]").first();
 
-            return ogElement.attr("content");
+            return replaceHttpWithHttps(ogElement.attr("content"));
         } catch (Exception e) {
             throw new ParsingException(e.getMessage(), e);
         }

src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java

@@ -23,11 +23,10 @@ import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.util.Date;
 
+import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
+
 public class SoundcloudParsingHelper {
     private static String clientId;
-
-    private static final String HTTP = "http://";
-    private static final String HTTPS = "https://";
     
     private SoundcloudParsingHelper() {
     }
@@ -215,13 +214,6 @@ public class SoundcloudParsingHelper {
         return getStreamsFromApi(collector, apiUrl, false);
     }
 
-    private static String replaceHttpWithHttps(final String url) {
-        if(!url.isEmpty() && url.startsWith(HTTP)) {
-            return HTTPS + url.substring(HTTP.length());
-        }
-        return url;
-    }
-
     @Nonnull
     static String getUploaderUrl(JsonObject object) {
         String url = object.getObject("user").getString("permalink_url", "");

src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java

@@ -14,6 +14,8 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
 import javax.annotation.Nonnull;
 import java.io.IOException;
 
+import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
+
 @SuppressWarnings("WeakerAccess")
 public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
     private String playlistId;
@@ -45,7 +47,7 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
     @Nonnull
     @Override
     public String getCleanUrl() {
-        return playlist.isString("permalink_url") ? playlist.getString("permalink_url") : getOriginalUrl();
+        return playlist.isString("permalink_url") ? replaceHttpWithHttps(playlist.getString("permalink_url")) : getOriginalUrl();
     }
 
     @Nonnull
@@ -62,7 +64,26 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
 
     @Override
     public String getThumbnailUrl() {
-        return playlist.getString("artwork_url");
+        String artworkUrl = playlist.getString("artwork_url");
+
+        if (artworkUrl == null) {
+            // If the thumbnail is null, traverse the items list and get a valid one,
+            // if it also fails, return null
+            try {
+                final StreamInfoItemsCollector infoItems = getInfoItems();
+                if (infoItems.getItemList().isEmpty()) return null;
+
+                for (StreamInfoItem item : infoItems.getItemList()) {
+                    final String thumbnailUrl = item.getThumbnailUrl();
+                    if (thumbnailUrl == null || thumbnailUrl.isEmpty()) continue;
+
+                    return thumbnailUrl;
+                }
+            } catch (Exception ignored) {
+            }
+        }
+
+        return artworkUrl;
     }
 
     @Override

src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistInfoItemExtractor.java

@@ -4,38 +4,40 @@ import com.grack.nanojson.JsonObject;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor;
 
+import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
+
 public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtractor {
     private static final String USER_KEY = "user";
     private static final String AVATAR_URL_KEY = "avatar_url";
     private static final String ARTWORK_URL_KEY = "artwork_url";
 
-    private final JsonObject searchResult;
+    private final JsonObject itemObject;
 
-    public SoundcloudPlaylistInfoItemExtractor(JsonObject searchResult) {
+    public SoundcloudPlaylistInfoItemExtractor(JsonObject itemObject) {
-        this.searchResult = searchResult;
+        this.itemObject = itemObject;
     }
 
     @Override
-    public String getName() throws ParsingException {
+    public String getName() {
-        return searchResult.getString("title");
+        return itemObject.getString("title");
     }
 
     @Override
-    public String getUrl() throws ParsingException {
+    public String getUrl() {
-        return searchResult.getString("permalink_url");
+        return replaceHttpWithHttps(itemObject.getString("permalink_url"));
     }
 
     @Override
     public String getThumbnailUrl() throws ParsingException {
         // Over-engineering at its finest
-        if (searchResult.isString(ARTWORK_URL_KEY)) {
+        if (itemObject.isString(ARTWORK_URL_KEY)) {
-            final String artworkUrl = searchResult.getString(ARTWORK_URL_KEY, "");
+            final String artworkUrl = itemObject.getString(ARTWORK_URL_KEY, "");
             if (!artworkUrl.isEmpty()) return artworkUrl;
         }
 
         try {
             // Look for artwork url inside the track list
-            for (Object track : searchResult.getArray("tracks")) {
+            for (Object track : itemObject.getArray("tracks")) {
                 final JsonObject trackObject = (JsonObject) track;
 
                 // First look for track artwork url
@@ -55,7 +57,7 @@ public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtr
 
         try {
             // Last resort, use user avatar url. If still not found, then throw exception.
-            return searchResult.getObject(USER_KEY).getString(AVATAR_URL_KEY, "");
+            return itemObject.getObject(USER_KEY).getString(AVATAR_URL_KEY, "");
         } catch (Exception e) {
             throw new ParsingException("Failed to extract playlist thumbnail url", e);
         }
@@ -64,14 +66,14 @@ public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtr
     @Override
     public String getUploaderName() throws ParsingException {
         try {
-            return searchResult.getObject(USER_KEY).getString("username");
+            return itemObject.getObject(USER_KEY).getString("username");
         } catch (Exception e) {
             throw new ParsingException("Failed to extract playlist uploader", e);
         }
     }
 
     @Override
-    public long getStreamCount() throws ParsingException {
+    public long getStreamCount() {
-        return searchResult.getNumber("track_count", 0).longValue();
+        return itemObject.getNumber("track_count", 0).longValue();
     }
 }

src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistUrlIdHandler.java

@@ -8,6 +8,8 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.utils.Parser;
 import org.schabi.newpipe.extractor.utils.Utils;
 
+import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
+
 public class SoundcloudPlaylistUrlIdHandler implements UrlIdHandler {
     private static final SoundcloudPlaylistUrlIdHandler instance = new SoundcloudPlaylistUrlIdHandler();
     private final String URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/[0-9a-z_-]+" +
@@ -45,7 +47,7 @@ public class SoundcloudPlaylistUrlIdHandler implements UrlIdHandler {
             Element ogElement = Jsoup.parse(NewPipe.getDownloader().download(complexUrl))
                     .select("meta[property=og:url]").first();
 
-            return ogElement.attr("content");
+            return replaceHttpWithHttps(ogElement.attr("content"));
         } catch (Exception e) {
             throw new ParsingException(e.getMessage(), e);
         }

src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java

@@ -13,7 +13,11 @@ import javax.annotation.Nonnull;
 import java.io.IOException;
 import java.io.UnsupportedEncodingException;
 import java.net.URLEncoder;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
 
 public class SoundcloudStreamExtractor extends StreamExtractor {
     private JsonObject track;
@@ -35,7 +39,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
     @Nonnull
     @Override
     public String getCleanUrl() {
-        return track.isString("permalink_url") ? track.getString("permalink_url") : getOriginalUrl();
+        return track.isString("permalink_url") ? replaceHttpWithHttps(track.getString("permalink_url")) : getOriginalUrl();
     }
 
     @Nonnull

src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamInfoItemExtractor.java

@@ -5,52 +5,54 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
 import org.schabi.newpipe.extractor.stream.StreamType;
 
+import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
+
 public class SoundcloudStreamInfoItemExtractor implements StreamInfoItemExtractor {
 
-    protected final JsonObject searchResult;
+    protected final JsonObject itemObject;
 
-    public SoundcloudStreamInfoItemExtractor(JsonObject searchResult) {
+    public SoundcloudStreamInfoItemExtractor(JsonObject itemObject) {
-        this.searchResult = searchResult;
+        this.itemObject = itemObject;
     }
 
     @Override
     public String getUrl() {
-        return searchResult.getString("permalink_url");
+        return replaceHttpWithHttps(itemObject.getString("permalink_url"));
     }
 
     @Override
     public String getName() {
-        return searchResult.getString("title");
+        return itemObject.getString("title");
     }
 
     @Override
     public long getDuration() {
-        return searchResult.getNumber("duration", 0).longValue() / 1000L;
+        return itemObject.getNumber("duration", 0).longValue() / 1000L;
     }
 
     @Override
     public String getUploaderName() {
-        return searchResult.getObject("user").getString("username");
+        return itemObject.getObject("user").getString("username");
     }
 
     @Override
     public String getUploaderUrl() {
-        return searchResult.getObject("user").getString("permalink_url");
+        return replaceHttpWithHttps(itemObject.getObject("user").getString("permalink_url"));
     }
 
     @Override
     public String getUploadDate() throws ParsingException {
-        return SoundcloudParsingHelper.toDateString(searchResult.getString("created_at"));
+        return SoundcloudParsingHelper.toDateString(itemObject.getString("created_at"));
     }
 
     @Override
     public long getViewCount() {
-        return searchResult.getNumber("playback_count", 0).longValue();
+        return itemObject.getNumber("playback_count", 0).longValue();
     }
 
     @Override
     public String getThumbnailUrl() {
-        return searchResult.getString("artwork_url");
+        return itemObject.getString("artwork_url");
     }
 
     @Override

src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamUrlIdHandler.java

@@ -8,6 +8,8 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.utils.Parser;
 import org.schabi.newpipe.extractor.utils.Utils;
 
+import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
+
 public class SoundcloudStreamUrlIdHandler implements UrlIdHandler {
     private static final SoundcloudStreamUrlIdHandler instance = new SoundcloudStreamUrlIdHandler();
     private final String URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/[0-9a-z_-]+" +
@@ -48,7 +50,7 @@ public class SoundcloudStreamUrlIdHandler implements UrlIdHandler {
             Element ogElement = Jsoup.parse(NewPipe.getDownloader().download(complexUrl))
                     .select("meta[property=og:url]").first();
 
-            return ogElement.attr("content");
+            return replaceHttpWithHttps(ogElement.attr("content"));
         } catch (Exception e) {
             throw new ParsingException(e.getMessage(), e);
         }

src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java

@@ -175,7 +175,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
             throw new ParsingException("Could not parse json data for next streams", pe);
         }
 
-        final Document ajaxHtml = Jsoup.parse(ajaxJson.getString("content_html"));
+        final Document ajaxHtml = Jsoup.parse(ajaxJson.getString("content_html"), pageUrl);
         collectStreamsFrom(collector, ajaxHtml.select("body").first());
 
         return new InfoItemPage<>(collector, getNextPageUrlFromAjaxPage(ajaxJson, pageUrl));

src/main/java/org/schabi/newpipe/extractor/utils/Utils.java

@@ -6,6 +6,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import java.util.List;
 
 public class Utils {
+
     private Utils() {
         //no instance
     }
@@ -46,5 +47,17 @@ public class Utils {
             System.err.println("----------------");
         }
     }
+
+    private static final String HTTP = "http://";
+    private static final String HTTPS = "https://";
+
+    public static String replaceHttpWithHttps(final String url) {
+        if (url == null) return null;
+
+        if(!url.isEmpty() && url.startsWith(HTTP)) {
+            return HTTPS + url.substring(HTTP.length());
+        }
+        return url;
+    }
 }