Merge pull request #810 from TiA4f8R/delivery-methods-v2

Support delivery methods other than progressive HTTP

extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampRadioStreamExtractor.java

@@ -30,9 +30,12 @@ import java.util.List;
 import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.BASE_API_URL;
 import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.BASE_URL;
 import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImageUrl;
+import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
 
 public class BandcampRadioStreamExtractor extends BandcampStreamExtractor {
 
+    private static final String OPUS_LO = "opus-lo";
+    private static final String MP3_128 = "mp3-128";
     private JsonObject showInfo;
 
     public BandcampRadioStreamExtractor(final StreamingService service,
@@ -116,23 +119,27 @@ public class BandcampRadioStreamExtractor extends BandcampStreamExtractor {
 
     @Override
     public List<AudioStream> getAudioStreams() {
-        final ArrayList<AudioStream> list = new ArrayList<>();
+        final List<AudioStream> audioStreams = new ArrayList<>();
         final JsonObject streams = showInfo.getObject("audio_stream");
 
-        if (streams.has("opus-lo")) {
+        if (streams.has(MP3_128)) {
-            list.add(new AudioStream(
+            audioStreams.add(new AudioStream.Builder()
-                    streams.getString("opus-lo"),
+                    .setId(MP3_128)
-                    MediaFormat.OPUS, 100
+                    .setContent(streams.getString(MP3_128), true)
-            ));
+                    .setMediaFormat(MediaFormat.MP3)
-        }
+                    .setAverageBitrate(128)
-        if (streams.has("mp3-128")) {
+                    .build());
-            list.add(new AudioStream(
-                    streams.getString("mp3-128"),
-                    MediaFormat.MP3, 128
-            ));
         }
 
-        return list;
+        if (streams.has(OPUS_LO)) {
+            audioStreams.add(new AudioStream.Builder()
+                    .setId(OPUS_LO)
+                    .setContent(streams.getString(OPUS_LO), true)
+                    .setMediaFormat(MediaFormat.OPUS)
+                    .setAverageBitrate(100).build());
+        }
+
+        return audioStreams;
     }
 
     @Nonnull
@@ -156,14 +163,14 @@ public class BandcampRadioStreamExtractor extends BandcampStreamExtractor {
     @Override
     public String getLicence() {
         // Contrary to other Bandcamp streams, radio streams don't have a license
-        return "";
+        return EMPTY_STRING;
     }
 
     @Nonnull
     @Override
     public String getCategory() {
         // Contrary to other Bandcamp streams, radio streams don't have categories
-        return "";
+        return EMPTY_STRING;
     }
 
     @Nonnull

extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampStreamExtractor.java

@@ -3,6 +3,8 @@
 package org.schabi.newpipe.extractor.services.bandcamp.extractors;
 
 import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.getImageUrl;
+import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
+import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
 
 import com.grack.nanojson.JsonObject;
 import com.grack.nanojson.JsonParserException;
@@ -10,7 +12,6 @@ import com.grack.nanojson.JsonParserException;
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
-import org.jsoup.select.Elements;
 import org.schabi.newpipe.extractor.MediaFormat;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.downloader.Downloader;
@@ -27,16 +28,15 @@ import org.schabi.newpipe.extractor.stream.VideoStream;
 import org.schabi.newpipe.extractor.utils.JsonUtils;
 import org.schabi.newpipe.extractor.utils.Utils;
 
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
-
+import java.util.stream.Collectors;
-import javax.annotation.Nonnull;
-import javax.annotation.Nullable;
 
 public class BandcampStreamExtractor extends StreamExtractor {
-
     private JsonObject albumJson;
     private JsonObject current;
     private Document document;
@@ -88,7 +88,7 @@ public class BandcampStreamExtractor extends StreamExtractor {
     public String getUploaderUrl() throws ParsingException {
         final String[] parts = getUrl().split("/");
         // https: (/) (/) * .bandcamp.com (/) and leave out the rest
-        return "https://" + parts[2] + "/";
+        return HTTPS + parts[2] + "/";
     }
 
     @Nonnull
@@ -119,10 +119,10 @@ public class BandcampStreamExtractor extends StreamExtractor {
     @Override
     public String getThumbnailUrl() throws ParsingException {
         if (albumJson.isNull("art_id")) {
-            return Utils.EMPTY_STRING;
+            return EMPTY_STRING;
-        } else {
-            return getImageUrl(albumJson.getLong("art_id"), true);
         }
+
+        return getImageUrl(albumJson.getLong("art_id"), true);
     }
 
     @Nonnull
@@ -139,24 +139,26 @@ public class BandcampStreamExtractor extends StreamExtractor {
     public Description getDescription() {
         final String s = Utils.nonEmptyAndNullJoin(
                 "\n\n",
-                new String[]{
+                new String[] {
                         current.getString("about"),
                         current.getString("lyrics"),
                         current.getString("credits")
-                }
+                });
-        );
         return new Description(s, Description.PLAIN_TEXT);
     }
 
     @Override
     public List<AudioStream> getAudioStreams() {
         final List<AudioStream> audioStreams = new ArrayList<>();
-
+        audioStreams.add(new AudioStream.Builder()
-        audioStreams.add(new AudioStream(
+                .setId("mp3-128")
-                albumJson.getArray("trackinfo").getObject(0)
+                .setContent(albumJson.getArray("trackinfo")
-                        .getObject("file").getString("mp3-128"),
+                        .getObject(0)
-                MediaFormat.MP3, 128
+                        .getObject("file")
-        ));
+                        .getString("mp3-128"), true)
+                .setMediaFormat(MediaFormat.MP3)
+                .setAverageBitrate(128)
+                .build());
         return audioStreams;
     }
 
@@ -184,11 +186,11 @@ public class BandcampStreamExtractor extends StreamExtractor {
     @Override
     public PlaylistInfoItemsCollector getRelatedItems() {
         final PlaylistInfoItemsCollector collector = new PlaylistInfoItemsCollector(getServiceId());
-        final Elements recommendedAlbums = document.getElementsByClass("recommended-album");
+        document.getElementsByClass("recommended-album")
+                .stream()
+                .map(BandcampRelatedPlaylistInfoItemExtractor::new)
+                .forEach(collector::commit);
 
-        for (final Element album : recommendedAlbums) {
-            collector.commit(new BandcampRelatedPlaylistInfoItemExtractor(album));
-        }
         return collector;
     }
 
@@ -200,15 +202,17 @@ public class BandcampStreamExtractor extends StreamExtractor {
                 .flatMap(element -> element.getElementsByClass("tag").stream())
                 .map(Element::text)
                 .findFirst()
-                .orElse("");
+                .orElse(EMPTY_STRING);
     }
 
     @Nonnull
     @Override
     public String getLicence() {
-        /* Tests resulted in this mapping of ints to licence:
+        /*
+        Tests resulted in this mapping of ints to licence:
         https://cloud.disroot.org/s/ZTWBxbQ9fKRmRWJ/preview (screenshot from a Bandcamp artist's
-        account) */
+        account)
+        */
 
         switch (current.getInt("license_type")) {
             case 1:
@@ -233,14 +237,9 @@ public class BandcampStreamExtractor extends StreamExtractor {
     @Nonnull
     @Override
     public List<String> getTags() {
-        final Elements tagElements = document.getElementsByAttributeValue("itemprop", "keywords");
+        return document.getElementsByAttributeValue("itemprop", "keywords")
-
+                .stream()
-        final List<String> tags = new ArrayList<>();
+                .map(Element::text)
-
+                .collect(Collectors.toList());
-        for (final Element e : tagElements) {
-            tags.add(e.text());
-        }
-
-        return tags;
     }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCLiveStreamExtractor.java

@@ -10,18 +10,30 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
 import org.schabi.newpipe.extractor.stream.AudioStream;
+import org.schabi.newpipe.extractor.stream.DeliveryMethod;
 import org.schabi.newpipe.extractor.stream.Description;
+import org.schabi.newpipe.extractor.stream.Stream;
 import org.schabi.newpipe.extractor.stream.StreamExtractor;
 import org.schabi.newpipe.extractor.stream.StreamType;
 import org.schabi.newpipe.extractor.stream.VideoStream;
 
 import java.io.IOException;
-import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
+import java.util.function.Function;
+import java.util.stream.Collectors;
 
 import javax.annotation.Nonnull;
 
+import static org.schabi.newpipe.extractor.stream.AudioStream.UNKNOWN_BITRATE;
+import static org.schabi.newpipe.extractor.stream.Stream.ID_UNKNOWN;
+import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
+
 public class MediaCCCLiveStreamExtractor extends StreamExtractor {
+    private static final String STREAMS = "streams";
+    private static final String URLS = "urls";
+    private static final String URL = "url";
+
     private JsonObject conference = null;
     private String group = "";
     private JsonObject room = null;
@@ -34,19 +46,22 @@ public class MediaCCCLiveStreamExtractor extends StreamExtractor {
     @Override
     public void onFetchPage(@Nonnull final Downloader downloader)
             throws IOException, ExtractionException {
-        final JsonArray doc =
+        final JsonArray doc = MediaCCCParsingHelper.getLiveStreams(downloader,
-                MediaCCCParsingHelper.getLiveStreams(downloader, getExtractorLocalization());
+                getExtractorLocalization());
-        // find correct room
+        // Find the correct room
         for (int c = 0; c < doc.size(); c++) {
-            conference = doc.getObject(c);
+            final JsonObject conferenceObject = doc.getObject(c);
-            final JsonArray groups = conference.getArray("groups");
+            final JsonArray groups = conferenceObject.getArray("groups");
             for (int g = 0; g < groups.size(); g++) {
-                group = groups.getObject(g).getString("group");
+                final String groupObject = groups.getObject(g).getString("group");
                 final JsonArray rooms = groups.getObject(g).getArray("rooms");
                 for (int r = 0; r < rooms.size(); r++) {
-                    room = rooms.getObject(r);
+                    final JsonObject roomObject = rooms.getObject(r);
-                    if (getId().equals(
+                    if (getId().equals(conferenceObject.getString("slug") + "/"
-                            conference.getString("slug") + "/" + room.getString("slug"))) {
+                            + roomObject.getString("slug"))) {
+                        conference = conferenceObject;
+                        group = groupObject;
+                        room = roomObject;
                         return;
                     }
                 }
@@ -91,69 +106,155 @@ public class MediaCCCLiveStreamExtractor extends StreamExtractor {
         return conference.getString("conference");
     }
 
+    /**
+     * Get the URL of the first DASH stream found.
+     *
+     * <p>
+     * There can be several DASH streams, so the URL of the first one found is returned by this
+     * method.
+     * </p>
+     *
+     * <p>
+     * You can find the other DASH video streams by using {@link #getVideoStreams()}
+     * </p>
+     */
+    @Nonnull
+    @Override
+    public String getDashMpdUrl() throws ParsingException {
+        return getManifestOfDeliveryMethodWanted("dash");
+    }
+
+    /**
+     * Get the URL of the first HLS stream found.
+     *
+     * <p>
+     * There can be several HLS streams, so the URL of the first one found is returned by this
+     * method.
+     * </p>
+     *
+     * <p>
+     * You can find the other HLS video streams by using {@link #getVideoStreams()}
+     * </p>
+     */
     @Nonnull
     @Override
     public String getHlsUrl() {
-        // TODO: There are multiple HLS streams.
+        return getManifestOfDeliveryMethodWanted("hls");
-        //       Make getHlsUrl() and getDashMpdUrl() return lists of VideoStreams,
+    }
-        //       so the user can choose a resolution.
+
-        for (int s = 0; s < room.getArray("streams").size(); s++) {
+    @Nonnull
-            final JsonObject stream = room.getArray("streams").getObject(s);
+    private String getManifestOfDeliveryMethodWanted(@Nonnull final String deliveryMethod) {
-            if (stream.getString("type").equals("video")) {
+        return room.getArray(STREAMS).stream()
-                if (stream.has("hls")) {
+                .filter(JsonObject.class::isInstance)
-                    return stream.getObject("urls").getObject("hls").getString("url");
+                .map(JsonObject.class::cast)
-                }
+                .map(streamObject -> streamObject.getObject(URLS))
-            }
+                .filter(urls -> urls.has(deliveryMethod))
-        }
+                .map(urls -> urls.getObject(deliveryMethod).getString(URL, EMPTY_STRING))
-        return "";
+                .findFirst()
+                .orElse(EMPTY_STRING);
     }
 
     @Override
     public List<AudioStream> getAudioStreams() throws IOException, ExtractionException {
-        final List<AudioStream> audioStreams = new ArrayList<>();
+        return getStreams("audio",
-        for (int s = 0; s < room.getArray("streams").size(); s++) {
+                dto -> {
-            final JsonObject stream = room.getArray("streams").getObject(s);
+                    final AudioStream.Builder builder = new AudioStream.Builder()
-            if (stream.getString("type").equals("audio")) {
+                            .setId(dto.urlValue.getString("tech", ID_UNKNOWN))
-                for (final String type : stream.getObject("urls").keySet()) {
+                            .setContent(dto.urlValue.getString(URL), true)
-                    final JsonObject url = stream.getObject("urls").getObject(type);
+                            .setAverageBitrate(UNKNOWN_BITRATE);
-                    audioStreams.add(new AudioStream(url.getString("url"),
+
-                            MediaFormat.getFromSuffix(type), -1));
+                    if ("hls".equals(dto.urlKey)) {
-                }
+                        // We don't know with the type string what media format will
-            }
+                        // have HLS streams.
-        }
+                        // However, the tech string may contain some information
-        return audioStreams;
+                        // about the media format used.
+                        return builder.setDeliveryMethod(DeliveryMethod.HLS)
+                                .build();
+                    }
+
+                    return builder.setMediaFormat(MediaFormat.getFromSuffix(dto.urlKey))
+                            .build();
+                });
     }
 
     @Override
     public List<VideoStream> getVideoStreams() throws IOException, ExtractionException {
-        final List<VideoStream> videoStreams = new ArrayList<>();
+        return getStreams("video",
-        for (int s = 0; s < room.getArray("streams").size(); s++) {
+                dto -> {
-            final JsonObject stream = room.getArray("streams").getObject(s);
+                    final JsonArray videoSize = dto.streamJsonObj.getArray("videoSize");
-            if (stream.getString("type").equals("video")) {
+
-                final String resolution = stream.getArray("videoSize").getInt(0) + "x"
+                    final VideoStream.Builder builder = new VideoStream.Builder()
-                        + stream.getArray("videoSize").getInt(1);
+                            .setId(dto.urlValue.getString("tech", ID_UNKNOWN))
-                for (final String type : stream.getObject("urls").keySet()) {
+                            .setContent(dto.urlValue.getString(URL), true)
-                    if (!type.equals("hls")) {
+                            .setIsVideoOnly(false)
-                        final JsonObject url = stream.getObject("urls").getObject(type);
+                            .setResolution(videoSize.getInt(0) + "x" + videoSize.getInt(1));
-                        videoStreams.add(new VideoStream(
+
-                                url.getString("url"),
+                    if ("hls".equals(dto.urlKey)) {
-                                MediaFormat.getFromSuffix(type),
+                        // We don't know with the type string what media format will
-                                resolution));
+                        // have HLS streams.
+                        // However, the tech string may contain some information
+                        // about the media format used.
+                        return builder.setDeliveryMethod(DeliveryMethod.HLS)
+                                .build();
                     }
-                }
+
-            }
+                    return builder.setMediaFormat(MediaFormat.getFromSuffix(dto.urlKey))
+                            .build();
+                });
+    }
+
+
+    /**
+     * This is just an internal class used in {@link #getStreams(String, Function)} to tie together
+     * the stream json object, its URL key and its URL value. An object of this class would be
+     * temporary and the three values it holds would be <b>convert</b>ed to a proper {@link Stream}
+     * object based on the wanted stream type.
+     */
+    private static final class MediaCCCLiveStreamMapperDTO {
+        final JsonObject streamJsonObj;
+        final String urlKey;
+        final JsonObject urlValue;
+
+        MediaCCCLiveStreamMapperDTO(final JsonObject streamJsonObj,
+                                    final String urlKey,
+                                    final JsonObject urlValue) {
+            this.streamJsonObj = streamJsonObj;
+            this.urlKey = urlKey;
+            this.urlValue = urlValue;
         }
-        return videoStreams;
+    }
+
+    private <T extends Stream> List<T> getStreams(
+            @Nonnull final String streamType,
+            @Nonnull final Function<MediaCCCLiveStreamMapperDTO, T> converter) {
+        return room.getArray(STREAMS).stream()
+                // Ensure that we use only process JsonObjects
+                .filter(JsonObject.class::isInstance)
+                .map(JsonObject.class::cast)
+                // Only process streams of requested type
+                .filter(streamJsonObj -> streamType.equals(streamJsonObj.getString("type")))
+                // Flatmap Urls and ensure that we use only process JsonObjects
+                .flatMap(streamJsonObj -> streamJsonObj.getObject(URLS).entrySet().stream()
+                        .filter(e -> e.getValue() instanceof JsonObject)
+                        .map(e -> new MediaCCCLiveStreamMapperDTO(
+                                streamJsonObj,
+                                e.getKey(),
+                                (JsonObject) e.getValue())))
+                // The DASH manifest will be extracted with getDashMpdUrl
+                .filter(dto -> !"dash".equals(dto.urlKey))
+                // Convert
+                .map(converter)
+                .collect(Collectors.toList());
     }
 
     @Override
     public List<VideoStream> getVideoOnlyStreams() {
-        return null;
+        return Collections.emptyList();
     }
 
     @Override
     public StreamType getStreamType() throws ParsingException {
-        return StreamType.LIVE_STREAM; // TODO: video and audio only streams are both available
+        return StreamType.LIVE_STREAM;
     }
 
     @Nonnull

extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java

@@ -1,5 +1,8 @@
 package org.schabi.newpipe.extractor.services.media_ccc.extractors;
 
+import static org.schabi.newpipe.extractor.stream.AudioStream.UNKNOWN_BITRATE;
+import static org.schabi.newpipe.extractor.stream.Stream.ID_UNKNOWN;
+
 import com.grack.nanojson.JsonArray;
 import com.grack.nanojson.JsonObject;
 import com.grack.nanojson.JsonParser;
@@ -99,7 +102,7 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
             final JsonObject recording = recordings.getObject(i);
             final String mimeType = recording.getString("mime_type");
             if (mimeType.startsWith("audio")) {
-                //first we need to resolve the actual video data from CDN
+                // First we need to resolve the actual video data from the CDN
                 final MediaFormat mediaFormat;
                 if (mimeType.endsWith("opus")) {
                     mediaFormat = MediaFormat.OPUS;
@@ -108,11 +111,18 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
                 } else if (mimeType.endsWith("ogg")) {
                     mediaFormat = MediaFormat.OGG;
                 } else {
-                    throw new ExtractionException("Unknown media format: " + mimeType);
+                    mediaFormat = null;
                 }
 
-                audioStreams.add(new AudioStream(recording.getString("recording_url"),
+                // Not checking containsSimilarStream here, since MediaCCC does not provide enough
-                        mediaFormat, -1));
+                // information to decide whether two streams are similar. Hence that method would
+                // always return false, e.g. even for different language variations.
+                audioStreams.add(new AudioStream.Builder()
+                        .setId(recording.getString("filename", ID_UNKNOWN))
+                        .setContent(recording.getString("recording_url"), true)
+                        .setMediaFormat(mediaFormat)
+                        .setAverageBitrate(UNKNOWN_BITRATE)
+                        .build());
             }
         }
         return audioStreams;
@@ -126,21 +136,29 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
             final JsonObject recording = recordings.getObject(i);
             final String mimeType = recording.getString("mime_type");
             if (mimeType.startsWith("video")) {
-                //first we need to resolve the actual video data from CDN
+                // First we need to resolve the actual video data from the CDN
-
                 final MediaFormat mediaFormat;
                 if (mimeType.endsWith("webm")) {
                     mediaFormat = MediaFormat.WEBM;
                 } else if (mimeType.endsWith("mp4")) {
                     mediaFormat = MediaFormat.MPEG_4;
                 } else {
-                    throw new ExtractionException("Unknown media format: " + mimeType);
+                    mediaFormat = null;
                 }
 
-                videoStreams.add(new VideoStream(recording.getString("recording_url"),
+                // Not checking containsSimilarStream here, since MediaCCC does not provide enough
-                        mediaFormat, recording.getInt("height") + "p"));
+                // information to decide whether two streams are similar. Hence that method would
+                // always return false, e.g. even for different language variations.
+                videoStreams.add(new VideoStream.Builder()
+                        .setId(recording.getString("filename", ID_UNKNOWN))
+                        .setContent(recording.getString("recording_url"), true)
+                        .setIsVideoOnly(false)
+                        .setMediaFormat(mediaFormat)
+                        .setResolution(recording.getInt("height") + "p")
+                        .build());
             }
         }
+
         return videoStreams;
     }
 
@@ -163,7 +181,8 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
             conferenceData = JsonParser.object()
                     .from(downloader.get(data.getString("conference_url")).responseBody());
         } catch (final JsonParserException jpe) {
-            throw new ExtractionException("Could not parse json returned by url: " + videoUrl, jpe);
+            throw new ExtractionException("Could not parse json returned by URL: " + videoUrl,
+                    jpe);
         }
     }
 

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java

@@ -22,6 +22,7 @@ import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
 import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeSearchQueryHandlerFactory;
 import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeStreamLinkHandlerFactory;
 import org.schabi.newpipe.extractor.stream.AudioStream;
+import org.schabi.newpipe.extractor.stream.DeliveryMethod;
 import org.schabi.newpipe.extractor.stream.Description;
 import org.schabi.newpipe.extractor.stream.Stream;
 import org.schabi.newpipe.extractor.stream.StreamExtractor;
@@ -39,14 +40,30 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Locale;
+import java.util.stream.Collectors;
 
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
+import static org.schabi.newpipe.extractor.stream.AudioStream.UNKNOWN_BITRATE;
+import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
+
 public class PeertubeStreamExtractor extends StreamExtractor {
+    private static final String ACCOUNT_HOST = "account.host";
+    private static final String ACCOUNT_NAME = "account.name";
+    private static final String FILES = "files";
+    private static final String FILE_DOWNLOAD_URL = "fileDownloadUrl";
+    private static final String FILE_URL = "fileUrl";
+    private static final String PLAYLIST_URL = "playlistUrl";
+    private static final String RESOLUTION_ID = "resolution.id";
+    private static final String STREAMING_PLAYLISTS = "streamingPlaylists";
+
     private final String baseUrl;
     private JsonObject json;
+
     private final List<SubtitlesStream> subtitles = new ArrayList<>();
+    private final List<AudioStream> audioStreams = new ArrayList<>();
+    private final List<VideoStream> videoStreams = new ArrayList<>();
 
     public PeertubeStreamExtractor(final StreamingService service, final LinkHandler linkHandler)
             throws ParsingException {
@@ -85,9 +102,8 @@ public class PeertubeStreamExtractor extends StreamExtractor {
         } catch (final ParsingException e) {
             return Description.EMPTY_DESCRIPTION;
         }
-
         if (text.length() == 250 && text.substring(247).equals("...")) {
-            //if description is shortened, get full description
+            // If description is shortened, get full description
             final Downloader dl = NewPipe.getDownloader();
             try {
                 final Response response = dl.get(baseUrl
@@ -95,8 +111,8 @@ public class PeertubeStreamExtractor extends StreamExtractor {
                         + getId() + "/description");
                 final JsonObject jsonObject = JsonParser.object().from(response.responseBody());
                 text = JsonUtils.getString(jsonObject, "description");
-            } catch (ReCaptchaException | IOException | JsonParserException e) {
+            } catch (final IOException | ReCaptchaException | JsonParserException ignored) {
-                e.printStackTrace();
+                // Something went wrong when getting the full description, use the shortened one
             }
         }
         return new Description(text, Description.MARKDOWN);
@@ -119,8 +135,8 @@ public class PeertubeStreamExtractor extends StreamExtractor {
 
     @Override
     public long getTimeStamp() throws ParsingException {
-        final long timestamp =
+        final long timestamp = getTimestampSeconds(
-                getTimestampSeconds("((#|&|\\?)start=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)");
+                "((#|&|\\?)start=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)");
 
         if (timestamp == -2) {
             // regex for timestamp was not found
@@ -148,10 +164,10 @@ public class PeertubeStreamExtractor extends StreamExtractor {
     @Nonnull
     @Override
     public String getUploaderUrl() throws ParsingException {
-        final String name = JsonUtils.getString(json, "account.name");
+        final String name = JsonUtils.getString(json, ACCOUNT_NAME);
-        final String host = JsonUtils.getString(json, "account.host");
+        final String host = JsonUtils.getString(json, ACCOUNT_HOST);
-        return getService().getChannelLHFactory()
+        return getService().getChannelLHFactory().fromId("accounts/" + name + "@" + host, baseUrl)
-                .fromId("accounts/" + name + "@" + host, baseUrl).getUrl();
+                .getUrl();
     }
 
     @Nonnull
@@ -199,77 +215,51 @@ public class PeertubeStreamExtractor extends StreamExtractor {
     @Nonnull
     @Override
     public String getHlsUrl() {
-        return json.getArray("streamingPlaylists").getObject(0).getString("playlistUrl");
+        assertPageFetched();
+
+        if (getStreamType() == StreamType.VIDEO_STREAM
+                && !isNullOrEmpty(json.getObject(FILES))) {
+            return json.getObject(FILES).getString(PLAYLIST_URL, EMPTY_STRING);
+        }
+
+        return json.getArray(STREAMING_PLAYLISTS).getObject(0).getString(PLAYLIST_URL,
+                EMPTY_STRING);
     }
 
     @Override
-    public List<AudioStream> getAudioStreams() {
+    public List<AudioStream> getAudioStreams() throws ParsingException {
-        return Collections.emptyList();
+        assertPageFetched();
+
+        /*
+        Some videos have audio streams; others don't.
+        So an audio stream may be available if a video stream is available.
+        Audio streams are also not returned as separated streams for livestreams.
+        That's why the extraction of audio streams is only run when there are video streams
+        extracted and when the content is not a livestream.
+         */
+        if (audioStreams.isEmpty() && videoStreams.isEmpty()
+                && getStreamType() == StreamType.VIDEO_STREAM) {
+            getStreams();
+        }
+
+        return audioStreams;
     }
 
     @Override
     public List<VideoStream> getVideoStreams() throws ExtractionException {
         assertPageFetched();
-        final List<VideoStream> videoStreams = new ArrayList<>();
 
-        // mp4
+        if (videoStreams.isEmpty()) {
-        try {
+            if (getStreamType() == StreamType.VIDEO_STREAM) {
-            videoStreams.addAll(getVideoStreamsFromArray(json.getArray("files")));
+                getStreams();
-        } catch (final Exception ignored) { }
+            } else {
-
+                extractLiveVideoStreams();
-        // HLS
-        try {
-            final JsonArray streamingPlaylists = json.getArray("streamingPlaylists");
-            for (final Object p : streamingPlaylists) {
-                if (!(p instanceof JsonObject)) {
-                    continue;
-                }
-                final JsonObject playlist = (JsonObject) p;
-                videoStreams.addAll(getVideoStreamsFromArray(playlist.getArray("files")));
             }
-        } catch (final Exception e) {
-            throw new ParsingException("Could not get video streams", e);
-        }
-
-        if (getStreamType() == StreamType.LIVE_STREAM) {
-            videoStreams.add(new VideoStream(getHlsUrl(), MediaFormat.MPEG_4, "720p"));
         }
 
         return videoStreams;
     }
 
-    private List<VideoStream> getVideoStreamsFromArray(final JsonArray streams)
-            throws ParsingException {
-        try {
-            final List<VideoStream> videoStreams = new ArrayList<>();
-            for (final Object s : streams) {
-                if (!(s instanceof JsonObject)) {
-                    continue;
-                }
-                final JsonObject stream = (JsonObject) s;
-                final String url;
-                if (stream.has("fileDownloadUrl")) {
-                    url = JsonUtils.getString(stream, "fileDownloadUrl");
-                } else {
-                    url = JsonUtils.getString(stream, "fileUrl");
-                }
-                final String torrentUrl = JsonUtils.getString(stream, "torrentUrl");
-                final String resolution = JsonUtils.getString(stream, "resolution.label");
-                final String extension = url.substring(url.lastIndexOf(".") + 1);
-                final MediaFormat format = MediaFormat.getFromSuffix(extension);
-                final VideoStream videoStream
-                        = new VideoStream(url, torrentUrl, format, resolution);
-                if (!Stream.containSimilarStream(videoStream, videoStreams)) {
-                    videoStreams.add(videoStream);
-                }
-            }
-            return videoStreams;
-        } catch (final Exception e) {
-            throw new ParsingException("Could not get video streams from array");
-        }
-
-    }
-
     @Override
     public List<VideoStream> getVideoOnlyStreams() {
         return Collections.emptyList();
@@ -284,13 +274,9 @@ public class PeertubeStreamExtractor extends StreamExtractor {
     @Nonnull
     @Override
     public List<SubtitlesStream> getSubtitles(final MediaFormat format) {
-        final List<SubtitlesStream> filteredSubs = new ArrayList<>();
+        return subtitles.stream()
-        for (final SubtitlesStream sub : subtitles) {
+                .filter(sub -> sub.getFormat() == format)
-            if (sub.getFormat() == format) {
+                .collect(Collectors.toList());
-                filteredSubs.add(sub);
-            }
-        }
-        return filteredSubs;
     }
 
     @Override
@@ -304,8 +290,8 @@ public class PeertubeStreamExtractor extends StreamExtractor {
         final List<String> tags = getTags();
         final String apiUrl;
         if (tags.isEmpty()) {
-            apiUrl = baseUrl + "/api/v1/accounts/" + JsonUtils.getString(json, "account.name")
+            apiUrl = baseUrl + "/api/v1/accounts/" + JsonUtils.getString(json, ACCOUNT_NAME)
-                    + "@" + JsonUtils.getString(json, "account.host")
+                    + "@" + JsonUtils.getString(json, ACCOUNT_HOST)
                     + "/videos?start=0&count=8";
         } else {
             apiUrl = getRelatedItemsUrl(tags);
@@ -314,7 +300,8 @@ public class PeertubeStreamExtractor extends StreamExtractor {
         if (Utils.isBlank(apiUrl)) {
             return null;
         } else {
-            final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
+            final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(
+                    getServiceId());
             getStreamsFromApi(collector, apiUrl);
             return collector;
         }
@@ -332,11 +319,13 @@ public class PeertubeStreamExtractor extends StreamExtractor {
         try {
             return JsonUtils.getString(json, "support");
         } catch (final ParsingException e) {
-            return "";
+            return EMPTY_STRING;
         }
     }
 
-    private String getRelatedItemsUrl(final List<String> tags) throws UnsupportedEncodingException {
+    @Nonnull
+    private String getRelatedItemsUrl(@Nonnull final List<String> tags)
+            throws UnsupportedEncodingException {
         final String url = baseUrl + PeertubeSearchQueryHandlerFactory.SEARCH_ENDPOINT;
         final StringBuilder params = new StringBuilder();
         params.append("start=0&count=8&sort=-createdAt");
@@ -348,7 +337,7 @@ public class PeertubeStreamExtractor extends StreamExtractor {
     }
 
     private void getStreamsFromApi(final StreamInfoItemsCollector collector, final String apiUrl)
-            throws ReCaptchaException, IOException, ParsingException {
+            throws IOException, ReCaptchaException, ParsingException {
         final Response response = getDownloader().get(apiUrl);
         JsonObject relatedVideosJson = null;
         if (response != null && !Utils.isBlank(response.responseBody())) {
@@ -365,21 +354,20 @@ public class PeertubeStreamExtractor extends StreamExtractor {
     }
 
     private void collectStreamsFrom(final StreamInfoItemsCollector collector,
-                                    final JsonObject jsonObject)
+                                    final JsonObject jsonObject) throws ParsingException {
-            throws ParsingException {
         final JsonArray contents;
         try {
             contents = (JsonArray) JsonUtils.getValue(jsonObject, "data");
         } catch (final Exception e) {
-            throw new ParsingException("unable to extract related videos", e);
+            throw new ParsingException("Could not extract related videos", e);
         }
 
         for (final Object c : contents) {
             if (c instanceof JsonObject) {
                 final JsonObject item = (JsonObject) c;
-                final PeertubeStreamInfoItemExtractor extractor
+                final PeertubeStreamInfoItemExtractor extractor =
-                        = new PeertubeStreamInfoItemExtractor(item, baseUrl);
+                        new PeertubeStreamInfoItemExtractor(item, baseUrl);
-                //do not add the same stream in related streams
+                // Do not add the same stream in related streams
                 if (!extractor.getUrl().equals(getUrl())) {
                     collector.commit(extractor);
                 }
@@ -395,7 +383,7 @@ public class PeertubeStreamExtractor extends StreamExtractor {
         if (response != null) {
             setInitialData(response.responseBody());
         } else {
-            throw new ExtractionException("Unable to extract PeerTube channel data");
+            throw new ExtractionException("Could not extract PeerTube channel data");
         }
 
         loadSubtitles();
@@ -405,10 +393,10 @@ public class PeertubeStreamExtractor extends StreamExtractor {
         try {
             json = JsonParser.object().from(responseBody);
         } catch (final JsonParserException e) {
-            throw new ExtractionException("Unable to extract PeerTube stream data", e);
+            throw new ExtractionException("Could not extract PeerTube stream data", e);
         }
         if (json == null) {
-            throw new ExtractionException("Unable to extract PeerTube stream data");
+            throw new ExtractionException("Could not extract PeerTube stream data");
         }
         PeertubeParsingHelper.validate(json);
     }
@@ -429,16 +417,233 @@ public class PeertubeStreamExtractor extends StreamExtractor {
                         final String ext = url.substring(url.lastIndexOf(".") + 1);
                         final MediaFormat fmt = MediaFormat.getFromSuffix(ext);
                         if (fmt != null && !isNullOrEmpty(languageCode)) {
-                            subtitles.add(new SubtitlesStream(fmt, languageCode, url, false));
+                            subtitles.add(new SubtitlesStream.Builder()
+                                    .setContent(url, true)
+                                    .setMediaFormat(fmt)
+                                    .setLanguageCode(languageCode)
+                                    .setAutoGenerated(false)
+                                    .build());
                         }
                     }
                 }
-            } catch (final Exception e) {
+            } catch (final Exception ignored) {
-                // ignore all exceptions
+                // Ignore all exceptions
             }
         }
     }
 
+    private void extractLiveVideoStreams() throws ParsingException {
+        try {
+            final JsonArray streamingPlaylists = json.getArray(STREAMING_PLAYLISTS);
+            streamingPlaylists.stream()
+                    .filter(JsonObject.class::isInstance)
+                    .map(JsonObject.class::cast)
+                    .map(stream -> new VideoStream.Builder()
+                            .setId(String.valueOf(stream.getInt("id", -1)))
+                            .setContent(stream.getString(PLAYLIST_URL, EMPTY_STRING), true)
+                            .setIsVideoOnly(false)
+                            .setResolution(EMPTY_STRING)
+                            .setMediaFormat(MediaFormat.MPEG_4)
+                            .setDeliveryMethod(DeliveryMethod.HLS)
+                            .build())
+                    // Don't use the containsSimilarStream method because it will always return
+                    // false so if there are multiples HLS URLs returned, only the first will be
+                    // extracted in this case.
+                    .forEachOrdered(videoStreams::add);
+        } catch (final Exception e) {
+            throw new ParsingException("Could not get video streams", e);
+        }
+    }
+
+    private void getStreams() throws ParsingException {
+        // Progressive streams
+        getStreamsFromArray(json.getArray(FILES), EMPTY_STRING);
+
+        // HLS streams
+        try {
+            for (final JsonObject playlist : json.getArray(STREAMING_PLAYLISTS).stream()
+                    .filter(JsonObject.class::isInstance)
+                    .map(JsonObject.class::cast)
+                    .collect(Collectors.toList())) {
+                getStreamsFromArray(playlist.getArray(FILES), playlist.getString(PLAYLIST_URL));
+            }
+        } catch (final Exception e) {
+            throw new ParsingException("Could not get streams", e);
+        }
+    }
+
+    private void getStreamsFromArray(@Nonnull final JsonArray streams,
+                                     final String playlistUrl) throws ParsingException {
+        try {
+            /*
+            Starting with version 3.4.0 of PeerTube, the HLS playlist of stream resolutions
+            contains the UUID of the streams, so we can't use the same method to get the URL of
+            the HLS playlist without fetching the master playlist.
+            These UUIDs are the same as the ones returned into the fileUrl and fileDownloadUrl
+            strings.
+            */
+            final boolean isInstanceUsingRandomUuidsForHlsStreams = !isNullOrEmpty(playlistUrl)
+                    && playlistUrl.endsWith("-master.m3u8");
+
+            for (final JsonObject stream : streams.stream()
+                    .filter(JsonObject.class::isInstance)
+                    .map(JsonObject.class::cast)
+                    .collect(Collectors.toList())) {
+
+                // Extract stream version of streams first
+                final String url = JsonUtils.getString(stream,
+                        stream.has(FILE_URL) ? FILE_URL : FILE_DOWNLOAD_URL);
+                if (isNullOrEmpty(url)) {
+                    // Not a valid stream URL
+                    return;
+                }
+
+                final String resolution = JsonUtils.getString(stream, "resolution.label");
+                final String idSuffix = stream.has(FILE_URL) ? FILE_URL : FILE_DOWNLOAD_URL;
+
+                if (resolution.toLowerCase().contains("audio")) {
+                    // An audio stream
+                    addNewAudioStream(stream, isInstanceUsingRandomUuidsForHlsStreams, resolution,
+                            idSuffix, url, playlistUrl);
+                } else {
+                    // A video stream
+                    addNewVideoStream(stream, isInstanceUsingRandomUuidsForHlsStreams, resolution,
+                            idSuffix, url, playlistUrl);
+                }
+            }
+        } catch (final Exception e) {
+            throw new ParsingException("Could not get streams from array", e);
+        }
+    }
+
+    @Nonnull
+    private String getHlsPlaylistUrlFromFragmentedFileUrl(
+            @Nonnull final JsonObject streamJsonObject,
+            @Nonnull final String idSuffix,
+            @Nonnull final String format,
+            @Nonnull final String url) throws ParsingException {
+        final String streamUrl = FILE_DOWNLOAD_URL.equals(idSuffix)
+                ? JsonUtils.getString(streamJsonObject, FILE_URL)
+                : url;
+        return streamUrl.replace("-fragmented." + format, ".m3u8");
+    }
+
+    @Nonnull
+    private String getHlsPlaylistUrlFromMasterPlaylist(@Nonnull final JsonObject streamJsonObject,
+                                                       @Nonnull final String playlistUrl)
+            throws ParsingException {
+        return playlistUrl.replace("master", JsonUtils.getNumber(streamJsonObject,
+                RESOLUTION_ID).toString());
+    }
+
+    private void addNewAudioStream(@Nonnull final JsonObject streamJsonObject,
+                                   final boolean isInstanceUsingRandomUuidsForHlsStreams,
+                                   @Nonnull final String resolution,
+                                   @Nonnull final String idSuffix,
+                                   @Nonnull final String url,
+                                   @Nullable final String playlistUrl) throws ParsingException {
+        final String extension = url.substring(url.lastIndexOf(".") + 1);
+        final MediaFormat format = MediaFormat.getFromSuffix(extension);
+        final String id = resolution + "-" + extension;
+
+        // Add progressive HTTP streams first
+        audioStreams.add(new AudioStream.Builder()
+                .setId(id + "-" + idSuffix + "-" + DeliveryMethod.PROGRESSIVE_HTTP)
+                .setContent(url, true)
+                .setMediaFormat(format)
+                .setAverageBitrate(UNKNOWN_BITRATE)
+                .build());
+
+        // Then add HLS streams
+        if (!isNullOrEmpty(playlistUrl)) {
+            final String hlsStreamUrl;
+            if (isInstanceUsingRandomUuidsForHlsStreams) {
+                hlsStreamUrl = getHlsPlaylistUrlFromFragmentedFileUrl(streamJsonObject, idSuffix,
+                        extension, url);
+
+            } else {
+                hlsStreamUrl = getHlsPlaylistUrlFromMasterPlaylist(streamJsonObject, playlistUrl);
+            }
+            final AudioStream audioStream = new AudioStream.Builder()
+                    .setId(id + "-" + DeliveryMethod.HLS)
+                    .setContent(hlsStreamUrl, true)
+                    .setDeliveryMethod(DeliveryMethod.HLS)
+                    .setMediaFormat(format)
+                    .setAverageBitrate(UNKNOWN_BITRATE)
+                    .setManifestUrl(playlistUrl)
+                    .build();
+            if (!Stream.containSimilarStream(audioStream, audioStreams)) {
+                audioStreams.add(audioStream);
+            }
+        }
+
+        // Finally, add torrent URLs
+        final String torrentUrl = JsonUtils.getString(streamJsonObject, "torrentUrl");
+        if (!isNullOrEmpty(torrentUrl)) {
+            audioStreams.add(new AudioStream.Builder()
+                    .setId(id + "-" + idSuffix + "-" + DeliveryMethod.TORRENT)
+                    .setContent(torrentUrl, true)
+                    .setDeliveryMethod(DeliveryMethod.TORRENT)
+                    .setMediaFormat(format)
+                    .setAverageBitrate(UNKNOWN_BITRATE)
+                    .build());
+        }
+    }
+
+    private void addNewVideoStream(@Nonnull final JsonObject streamJsonObject,
+                                   final boolean isInstanceUsingRandomUuidsForHlsStreams,
+                                   @Nonnull final String resolution,
+                                   @Nonnull final String idSuffix,
+                                   @Nonnull final String url,
+                                   @Nullable final String playlistUrl) throws ParsingException {
+        final String extension = url.substring(url.lastIndexOf(".") + 1);
+        final MediaFormat format = MediaFormat.getFromSuffix(extension);
+        final String id = resolution + "-" + extension;
+
+        // Add progressive HTTP streams first
+        videoStreams.add(new VideoStream.Builder()
+                .setId(id + "-" + idSuffix + "-" + DeliveryMethod.PROGRESSIVE_HTTP)
+                .setContent(url, true)
+                .setIsVideoOnly(false)
+                .setResolution(resolution)
+                .setMediaFormat(format)
+                .build());
+
+        // Then add HLS streams
+        if (!isNullOrEmpty(playlistUrl)) {
+            final String hlsStreamUrl = isInstanceUsingRandomUuidsForHlsStreams
+                    ? getHlsPlaylistUrlFromFragmentedFileUrl(streamJsonObject, idSuffix, extension,
+                            url)
+                    : getHlsPlaylistUrlFromMasterPlaylist(streamJsonObject, playlistUrl);
+
+            final VideoStream videoStream = new VideoStream.Builder()
+                    .setId(id + "-" + DeliveryMethod.HLS)
+                    .setContent(hlsStreamUrl, true)
+                    .setIsVideoOnly(false)
+                    .setDeliveryMethod(DeliveryMethod.HLS)
+                    .setResolution(resolution)
+                    .setMediaFormat(format)
+                    .setManifestUrl(playlistUrl)
+                    .build();
+            if (!Stream.containSimilarStream(videoStream, videoStreams)) {
+                videoStreams.add(videoStream);
+            }
+        }
+
+        // Add finally torrent URLs
+        final String torrentUrl = JsonUtils.getString(streamJsonObject, "torrentUrl");
+        if (!isNullOrEmpty(torrentUrl)) {
+            videoStreams.add(new VideoStream.Builder()
+                    .setId(id + "-" + idSuffix + "-" + DeliveryMethod.TORRENT)
+                    .setContent(torrentUrl, true)
+                    .setIsVideoOnly(false)
+                    .setDeliveryMethod(DeliveryMethod.TORRENT)
+                    .setResolution(resolution)
+                    .setMediaFormat(format)
+                    .build());
+        }
+    }
+
     @Nonnull
     @Override
     public String getName() throws ParsingException {
@@ -448,7 +653,7 @@ public class PeertubeStreamExtractor extends StreamExtractor {
     @Nonnull
     @Override
     public String getHost() throws ParsingException {
-        return JsonUtils.getString(json, "account.host");
+        return JsonUtils.getString(json, ACCOUNT_HOST);
     }
 
     @Nonnull

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java

@@ -1,6 +1,9 @@
 package org.schabi.newpipe.extractor.services.soundcloud.extractors;
 
 import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL;
+import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.clientId;
+import static org.schabi.newpipe.extractor.stream.AudioStream.UNKNOWN_BITRATE;
+import static org.schabi.newpipe.extractor.stream.Stream.ID_UNKNOWN;
 import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
 import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
 import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
@@ -25,7 +28,9 @@ import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
 import org.schabi.newpipe.extractor.localization.DateWrapper;
 import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper;
 import org.schabi.newpipe.extractor.stream.AudioStream;
+import org.schabi.newpipe.extractor.stream.DeliveryMethod;
 import org.schabi.newpipe.extractor.stream.Description;
+import org.schabi.newpipe.extractor.stream.Stream;
 import org.schabi.newpipe.extractor.stream.StreamExtractor;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
 import org.schabi.newpipe.extractor.stream.StreamType;
@@ -58,13 +63,16 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
         final String policy = track.getString("policy", EMPTY_STRING);
         if (!policy.equals("ALLOW") && !policy.equals("MONETIZE")) {
             isAvailable = false;
+
             if (policy.equals("SNIP")) {
                 throw new SoundCloudGoPlusContentException();
             }
+
             if (policy.equals("BLOCK")) {
                 throw new GeographicRestrictionException(
                         "This track is not available in user's country");
             }
+
             throw new ContentNotAvailableException("Content not available: policy " + policy);
         }
     }
@@ -72,7 +80,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
     @Nonnull
     @Override
     public String getId() {
-        return track.getInt("id") + EMPTY_STRING;
+        return String.valueOf(track.getInt("id"));
     }
 
     @Nonnull
@@ -168,118 +176,205 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
 
         try {
             final JsonArray transcodings = track.getObject("media").getArray("transcodings");
-            if (transcodings != null) {
+            if (!isNullOrEmpty(transcodings)) {
                 // Get information about what stream formats are available
                 extractAudioStreams(transcodings, checkMp3ProgressivePresence(transcodings),
                         audioStreams);
             }
+
+            extractDownloadableFileIfAvailable(audioStreams);
         } catch (final NullPointerException e) {
-            throw new ExtractionException("Could not get SoundCloud's tracks audio URL", e);
+            throw new ExtractionException("Could not get audio streams", e);
         }
 
         return audioStreams;
     }
 
-    private static boolean checkMp3ProgressivePresence(final JsonArray transcodings) {
+    private static boolean checkMp3ProgressivePresence(@Nonnull final JsonArray transcodings) {
-        boolean presence = false;
+        return transcodings.stream()
-        for (final Object transcoding : transcodings) {
+                .filter(JsonObject.class::isInstance)
-            final JsonObject transcodingJsonObject = (JsonObject) transcoding;
+                .map(JsonObject.class::cast)
-            if (transcodingJsonObject.getString("preset").contains("mp3")
+                .anyMatch(transcodingJsonObject -> transcodingJsonObject.getString("preset")
-                    && transcodingJsonObject.getObject("format").getString("protocol")
+                        .contains("mp3") && transcodingJsonObject.getObject("format")
-                            .equals("progressive")) {
+                        .getString("protocol").equals("progressive"));
-                presence = true;
-                break;
-            }
-        }
-        return presence;
     }
 
     @Nonnull
-    private static String getTranscodingUrl(final String endpointUrl,
+    private String getTranscodingUrl(final String endpointUrl)
-                                            final String protocol)
             throws IOException, ExtractionException {
-        final Downloader downloader = NewPipe.getDownloader();
+        final String apiStreamUrl = endpointUrl + "?client_id=" + clientId();
-        final String apiStreamUrl = endpointUrl + "?client_id="
+        final String response = NewPipe.getDownloader().get(apiStreamUrl).responseBody();
-                + SoundcloudParsingHelper.clientId();
-        final String response = downloader.get(apiStreamUrl).responseBody();
         final JsonObject urlObject;
         try {
             urlObject = JsonParser.object().from(response);
         } catch (final JsonParserException e) {
-            throw new ParsingException("Could not parse streamable url", e);
+            throw new ParsingException("Could not parse streamable URL", e);
         }
-        final String urlString = urlObject.getString("url");
 
-        if (protocol.equals("progressive")) {
+        return urlObject.getString("url");
-            return urlString;
-        } else if (protocol.equals("hls")) {
-            try {
-                return getSingleUrlFromHlsManifest(urlString);
-            } catch (final ParsingException ignored) {
-            }
-        }
-        // else, unknown protocol
-        return "";
     }
 
-    private static void extractAudioStreams(final JsonArray transcodings,
+    @Nullable
-                                            final boolean mp3ProgressiveInStreams,
+    private String getDownloadUrl(@Nonnull final String trackId)
-                                            final List<AudioStream> audioStreams) {
+            throws IOException, ExtractionException {
-        for (final Object transcoding : transcodings) {
+        final String response = NewPipe.getDownloader().get(SOUNDCLOUD_API_V2_URL + "tracks/"
-            final JsonObject transcodingJsonObject = (JsonObject) transcoding;
+                + trackId + "/download" + "?client_id=" + clientId()).responseBody();
-            final String url = transcodingJsonObject.getString("url");
-            if (isNullOrEmpty(url)) {
-                continue;
-            }
-            final String mediaUrl;
-            final String preset = transcodingJsonObject.getString("preset");
-            final String protocol = transcodingJsonObject.getObject("format")
-                    .getString("protocol");
-            MediaFormat mediaFormat = null;
-            int bitrate = 0;
-            if (preset.contains("mp3")) {
-                // Don't add the MP3 HLS stream if there is a progressive stream present
-                // because the two have the same bitrate
-                if (mp3ProgressiveInStreams && protocol.equals("hls")) {
-                    continue;
-                }
-                mediaFormat = MediaFormat.MP3;
-                bitrate = 128;
-            } else if (preset.contains("opus")) {
-                mediaFormat = MediaFormat.OPUS;
-                bitrate = 64;
-            }
 
-            if (mediaFormat != null) {
+        final JsonObject downloadJsonObject;
-                try {
+        try {
-                    mediaUrl = getTranscodingUrl(url, protocol);
+            downloadJsonObject = JsonParser.object().from(response);
-                    if (!mediaUrl.isEmpty()) {
+        } catch (final JsonParserException e) {
-                        audioStreams.add(new AudioStream(mediaUrl, mediaFormat, bitrate));
+            throw new ParsingException("Could not parse download URL", e);
+        }
+        final String redirectUri = downloadJsonObject.getString("redirectUri");
+        if (!isNullOrEmpty(redirectUri)) {
+            return redirectUri;
+        }
+        return null;
+    }
+
+    private void extractAudioStreams(@Nonnull final JsonArray transcodings,
+                                     final boolean mp3ProgressiveInStreams,
+                                     final List<AudioStream> audioStreams) {
+        transcodings.stream()
+                .filter(JsonObject.class::isInstance)
+                .map(JsonObject.class::cast)
+                .forEachOrdered(transcoding -> {
+                    final String url = transcoding.getString("url");
+                    if (isNullOrEmpty(url)) {
+                        return;
                     }
-                } catch (final Exception ignored) {
+
-                    // something went wrong when parsing this transcoding, don't add it to
+                    final String preset = transcoding.getString("preset", ID_UNKNOWN);
-                    // audioStreams
+                    final String protocol = transcoding.getObject("format").getString("protocol");
+                    final AudioStream.Builder builder = new AudioStream.Builder()
+                            .setId(preset);
+
+                    try {
+                        // streamUrl can be either the MP3 progressive stream URL or the
+                        // manifest URL of the HLS MP3 stream (if there is no MP3 progressive
+                        // stream, see above)
+                        final String streamUrl = getTranscodingUrl(url);
+
+                        if (preset.contains("mp3")) {
+                            // Don't add the MP3 HLS stream if there is a progressive stream
+                            // present because the two have the same bitrate
+                            final boolean isHls = protocol.equals("hls");
+                            if (mp3ProgressiveInStreams && isHls) {
+                                return;
+                            }
+
+                            builder.setMediaFormat(MediaFormat.MP3);
+                            builder.setAverageBitrate(128);
+
+                            if (isHls) {
+                                builder.setDeliveryMethod(DeliveryMethod.HLS);
+                                builder.setContent(streamUrl, true);
+
+                                final AudioStream hlsStream = builder.build();
+                                if (!Stream.containSimilarStream(hlsStream, audioStreams)) {
+                                    audioStreams.add(hlsStream);
+                                }
+
+                                final String progressiveHlsUrl =
+                                        getSingleUrlFromHlsManifest(streamUrl);
+                                builder.setDeliveryMethod(DeliveryMethod.PROGRESSIVE_HTTP);
+                                builder.setContent(progressiveHlsUrl, true);
+
+                                final AudioStream progressiveHlsStream = builder.build();
+                                if (!Stream.containSimilarStream(
+                                        progressiveHlsStream, audioStreams)) {
+                                    audioStreams.add(progressiveHlsStream);
+                                }
+
+                                // The MP3 HLS stream has been added in both versions (HLS and
+                                // progressive with the manifest parsing trick), so we need to
+                                // continue (otherwise the code would try to add again the stream,
+                                // which would be not added because the containsSimilarStream
+                                // method would return false and an audio stream object would be
+                                // created for nothing)
+                                return;
+                            } else {
+                                builder.setContent(streamUrl, true);
+                            }
+                        } else if (preset.contains("opus")) {
+                            // The HLS manifest trick doesn't work for opus streams
+                            builder.setContent(streamUrl, true);
+                            builder.setMediaFormat(MediaFormat.OPUS);
+                            builder.setAverageBitrate(64);
+                            builder.setDeliveryMethod(DeliveryMethod.HLS);
+                        } else {
+                            // Unknown format, skip to the next audio stream
+                            return;
+                        }
+
+                        final AudioStream audioStream = builder.build();
+                        if (!Stream.containSimilarStream(audioStream, audioStreams)) {
+                            audioStreams.add(audioStream);
+                        }
+                    } catch (final ExtractionException | IOException ignored) {
+                        // Something went wrong when trying to get and add this audio stream,
+                        // skip to the next one
+                    }
+                });
+    }
+
+    /**
+     * Add the downloadable format if it is available.
+     *
+     * <p>
+     * A track can have the {@code downloadable} boolean set to {@code true}, but it doesn't mean
+     * we can download it.
+     * </p>
+     *
+     * <p>
+     * If the value of the {@code has_download_left} boolean is {@code true}, the track can be
+     * downloaded, and not otherwise.
+     * </p>
+     *
+     * @param audioStreams the audio streams to which the downloadable file is added
+     */
+    public void extractDownloadableFileIfAvailable(final List<AudioStream> audioStreams) {
+        if (track.getBoolean("downloadable") && track.getBoolean("has_downloads_left")) {
+            try {
+                final String downloadUrl = getDownloadUrl(getId());
+                if (!isNullOrEmpty(downloadUrl)) {
+                    audioStreams.add(new AudioStream.Builder()
+                            .setId("original-format")
+                            .setContent(downloadUrl, true)
+                            .setAverageBitrate(UNKNOWN_BITRATE)
+                            .build());
                 }
+            } catch (final Exception ignored) {
+                // If something went wrong when trying to get the download URL, ignore the
+                // exception throw because this "stream" is not necessary to play the track
             }
         }
     }
 
     /**
-     * Parses a SoundCloud HLS manifest to get a single URL of HLS streams.
+     * Parses a SoundCloud HLS MP3 manifest to get a single URL of HLS streams.
+     *
      * <p>
-     * This method downloads the provided manifest URL, find all web occurrences in the manifest,
+     * This method downloads the provided manifest URL, finds all web occurrences in the manifest,
-     * get the last segment URL, changes its segment range to {@code 0/track-length} and return
+     * gets the last segment URL, changes its segment range to {@code 0/track-length}, and return
-     * this string.
+     * this as a string.
+     * </p>
+     *
+     * <p>
+     * This was working before for Opus streams, but has been broken by SoundCloud.
+     * </p>
+     *
      * @param  hlsManifestUrl the URL of the manifest to be parsed
      * @return a single URL that contains a range equal to the length of the track
      */
-    private static String getSingleUrlFromHlsManifest(final String hlsManifestUrl)
+    @Nonnull
+    private static String getSingleUrlFromHlsManifest(@Nonnull final String hlsManifestUrl)
             throws ParsingException {
-        final Downloader dl = NewPipe.getDownloader();
         final String hlsManifestResponse;
 
         try {
-            hlsManifestResponse = dl.get(hlsManifestUrl).responseBody();
+            hlsManifestResponse = NewPipe.getDownloader().get(hlsManifestUrl).responseBody();
         } catch (final IOException | ReCaptchaException e) {
             throw new ParsingException("Could not get SoundCloud HLS manifest");
         }
@@ -288,12 +383,13 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
         for (int l = lines.length - 1; l >= 0; l--) {
             final String line = lines[l];
             // Get the last URL from manifest, because it contains the range of the stream
-            if (line.trim().length() != 0 && !line.startsWith("#") && line.startsWith("https")) {
+            if (line.trim().length() != 0 && !line.startsWith("#") && line.startsWith(HTTPS)) {
                 final String[] hlsLastRangeUrlArray = line.split("/");
                 return HTTPS + hlsLastRangeUrlArray[2] + "/media/0/" + hlsLastRangeUrlArray[5]
                         + "/" + hlsLastRangeUrlArray[6];
             }
         }
+
         throw new ParsingException("Could not get any URL from HLS manifest");
     }
 
@@ -326,7 +422,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
         final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
 
         final String apiUrl = SOUNDCLOUD_API_V2_URL + "tracks/" + urlEncode(getId())
-                + "/related?client_id=" + urlEncode(SoundcloudParsingHelper.clientId());
+                + "/related?client_id=" + urlEncode(clientId());
 
         SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl);
         return collector;
@@ -355,19 +451,19 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
         // Tags are separated by spaces, but they can be multiple words escaped by quotes "
         final String[] tagList = track.getString("tag_list").split(" ");
         final List<String> tags = new ArrayList<>();
-        String escapedTag = "";
+        final StringBuilder escapedTag = new StringBuilder();
         boolean isEscaped = false;
         for (final String tag : tagList) {
             if (tag.startsWith("\"")) {
-                escapedTag += tag.replace("\"", "");
+                escapedTag.append(tag.replace("\"", ""));
                 isEscaped = true;
             } else if (isEscaped) {
                 if (tag.endsWith("\"")) {
-                    escapedTag += " " + tag.replace("\"", "");
+                    escapedTag.append(" ").append(tag.replace("\"", ""));
                     isEscaped = false;
-                    tags.add(escapedTag);
+                    tags.add(escapedTag.toString());
                 } else {
-                    escapedTag += " " + tag;
+                    escapedTag.append(" ").append(tag);
                 }
             } else if (!tag.isEmpty()) {
                 tags.add(tag);

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/DeliveryType.java

@@ -0,0 +1,55 @@
+package org.schabi.newpipe.extractor.services.youtube;
+
+/**
+ * Streaming format types used by YouTube in their streams.
+ *
+ * <p>
+ * It is different from {@link org.schabi.newpipe.extractor.stream.DeliveryMethod delivery methods}!
+ * </p>
+ */
+public enum DeliveryType {
+
+    /**
+     * YouTube's progressive delivery method, which works with HTTP range headers.
+     * (Note that official clients use the corresponding parameter instead.)
+     *
+     * <p>
+     * Initialization and index ranges are available to get metadata (the corresponding values
+     * are returned in the player response).
+     * </p>
+     */
+    PROGRESSIVE,
+
+    /**
+     * YouTube's OTF delivery method which uses a sequence parameter to get segments of
+     * streams.
+     *
+     * <p>
+     * The first sequence (which can be fetched with the {@code &sq=0} parameter) contains all the
+     * metadata needed to build the stream source (sidx boxes, segment length, segment count,
+     * duration, ...).
+     * </p>
+     *
+     * <p>
+     * Only used for videos; mostly those with a small amount of views, or ended livestreams
+     * which have just been re-encoded as normal videos.
+     * </p>
+     */
+    OTF,
+
+    /**
+     * YouTube's delivery method for livestreams which uses a sequence parameter to get
+     * segments of streams.
+     *
+     * <p>
+     * Each sequence (which can be fetched with the {@code &sq=0} parameter) contains its own
+     * metadata (sidx boxes, segment length, ...), which make no need of an initialization
+     * segment.
+     * </p>
+     *
+     * <p>
+     * Only used for livestreams (ended or running).
+     * </p>
+     */
+    LIVE
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ItagItem.java

@@ -14,16 +14,20 @@ import org.schabi.newpipe.extractor.MediaFormat;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 
 import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+import java.io.Serializable;
+
+public class ItagItem implements Serializable {
 
-public class ItagItem {
     /**
-     * List can be found here
+     * List can be found here:
-     * https://github.com/ytdl-org/youtube-dl/blob/9fc5eaf/youtube_dl/extractor/youtube.py#L1071
+     * https://github.com/ytdl-org/youtube-dl/blob/e988fa4/youtube_dl/extractor/youtube.py#L1195
      */
     private static final ItagItem[] ITAG_LIST = {
             /////////////////////////////////////////////////////
-            // VIDEO     ID  Type   Format  Resolution  FPS  ///
+            // VIDEO     ID  Type   Format  Resolution  FPS  ////
-            ///////////////////////////////////////////////////
+            /////////////////////////////////////////////////////
             new ItagItem(17, VIDEO, v3GPP, "144p"),
             new ItagItem(36, VIDEO, v3GPP, "240p"),
 
@@ -41,8 +45,8 @@ public class ItagItem {
             new ItagItem(45, VIDEO, WEBM, "720p"),
             new ItagItem(46, VIDEO, WEBM, "1080p"),
 
-            ////////////////////////////////////////////////////////////////////
+            //////////////////////////////////////////////////////////////////
-            // AUDIO     ID      ItagType          Format        Bitrate    ///
+            // AUDIO     ID      ItagType          Format        Bitrate    //
             //////////////////////////////////////////////////////////////////
             new ItagItem(171, AUDIO, WEBMA, 128),
             new ItagItem(172, AUDIO, WEBMA, 256),
@@ -54,8 +58,8 @@ public class ItagItem {
             new ItagItem(251, AUDIO, WEBMA_OPUS, 160),
 
             /// VIDEO ONLY ////////////////////////////////////////////
-            //           ID      Type     Format  Resolution  FPS  ///
+            //           ID      Type     Format  Resolution  FPS  ////
-            /////////////////////////////////////////////////////////
+            ///////////////////////////////////////////////////////////
             new ItagItem(160, VIDEO_ONLY, MPEG_4, "144p"),
             new ItagItem(133, VIDEO_ONLY, MPEG_4, "240p"),
             new ItagItem(134, VIDEO_ONLY, MPEG_4, "360p"),
@@ -102,14 +106,26 @@ public class ItagItem {
     public static ItagItem getItag(final int itagId) throws ParsingException {
         for (final ItagItem item : ITAG_LIST) {
             if (itagId == item.id) {
-                return item;
+                return new ItagItem(item);
             }
         }
-        throw new ParsingException("itag=" + itagId + " not supported");
+        throw new ParsingException("itag " + itagId + " is not supported");
     }
 
     /*//////////////////////////////////////////////////////////////////////////
-    // Contructors and misc
+    // Static constants
+    //////////////////////////////////////////////////////////////////////////*/
+
+    public static final int AVERAGE_BITRATE_UNKNOWN = -1;
+    public static final int SAMPLE_RATE_UNKNOWN = -1;
+    public static final int FPS_NOT_APPLICABLE_OR_UNKNOWN = -1;
+    public static final int TARGET_DURATION_SEC_UNKNOWN = -1;
+    public static final int AUDIO_CHANNELS_NOT_APPLICABLE_OR_UNKNOWN = -1;
+    public static final long CONTENT_LENGTH_UNKNOWN = -1;
+    public static final long APPROX_DURATION_MS_UNKNOWN = -1;
+
+    /*//////////////////////////////////////////////////////////////////////////
+    // Constructors and misc
     //////////////////////////////////////////////////////////////////////////*/
 
     public enum ItagType {
@@ -134,8 +150,6 @@ public class ItagItem {
 
     /**
      * Constructor for videos.
-     *
-     * @param resolution string that will be used in the frontend
      */
     public ItagItem(final int id,
                     final ItagType type,
@@ -159,22 +173,58 @@ public class ItagItem {
         this.avgBitrate = avgBitrate;
     }
 
-    private final MediaFormat mediaFormat;
+    /**
-
+     * Copy constructor of the {@link ItagItem} class.
+     *
+     * @param itagItem the {@link ItagItem} to copy its properties into a new {@link ItagItem}
+     */
+    public ItagItem(@Nonnull final ItagItem itagItem) {
+        this.mediaFormat = itagItem.mediaFormat;
+        this.id = itagItem.id;
+        this.itagType = itagItem.itagType;
+        this.avgBitrate = itagItem.avgBitrate;
+        this.sampleRate = itagItem.sampleRate;
+        this.audioChannels = itagItem.audioChannels;
+        this.resolutionString = itagItem.resolutionString;
+        this.fps = itagItem.fps;
+        this.bitrate = itagItem.bitrate;
+        this.width = itagItem.width;
+        this.height = itagItem.height;
+        this.initStart = itagItem.initStart;
+        this.initEnd = itagItem.initEnd;
+        this.indexStart = itagItem.indexStart;
+        this.indexEnd = itagItem.indexEnd;
+        this.quality = itagItem.quality;
+        this.codec = itagItem.codec;
+        this.targetDurationSec = itagItem.targetDurationSec;
+        this.approxDurationMs = itagItem.approxDurationMs;
+        this.contentLength = itagItem.contentLength;
+    }
 
     public MediaFormat getMediaFormat() {
         return mediaFormat;
     }
 
+    private final MediaFormat mediaFormat;
+
     public final int id;
     public final ItagType itagType;
 
     // Audio fields
-    public int avgBitrate = -1;
+    /** @deprecated Use {@link #getAverageBitrate()} instead. */
+    @Deprecated
+    public int avgBitrate = AVERAGE_BITRATE_UNKNOWN;
+    private int sampleRate = SAMPLE_RATE_UNKNOWN;
+    private int audioChannels = AUDIO_CHANNELS_NOT_APPLICABLE_OR_UNKNOWN;
 
     // Video fields
+    /** @deprecated Use {@link #getResolutionString()} instead. */
+    @Deprecated
     public String resolutionString;
-    public int fps = -1;
+
+    /** @deprecated Use {@link #getFps()} and {@link #setFps(int)} instead. */
+    @Deprecated
+    public int fps = FPS_NOT_APPLICABLE_OR_UNKNOWN;
 
     // Fields for Dash
     private int bitrate;
@@ -186,6 +236,9 @@ public class ItagItem {
     private int indexEnd;
     private String quality;
     private String codec;
+    private int targetDurationSec = TARGET_DURATION_SEC_UNKNOWN;
+    private long approxDurationMs = APPROX_DURATION_MS_UNKNOWN;
+    private long contentLength = CONTENT_LENGTH_UNKNOWN;
 
     public int getBitrate() {
         return bitrate;
@@ -211,6 +264,43 @@ public class ItagItem {
         this.height = height;
     }
 
+    /**
+     * Get the frame rate.
+     *
+     * <p>
+     * It is set to the {@code fps} value returned in the corresponding itag in the YouTube player
+     * response.
+     * </p>
+     *
+     * <p>
+     * It defaults to the standard value associated with this itag.
+     * </p>
+     *
+     * <p>
+     * Note that this value is only known for video itags, so {@link
+     * #FPS_NOT_APPLICABLE_OR_UNKNOWN} is returned for non video itags.
+     * </p>
+     *
+     * @return the frame rate or {@link #FPS_NOT_APPLICABLE_OR_UNKNOWN}
+     */
+    public int getFps() {
+        return fps;
+    }
+
+    /**
+     * Set the frame rate.
+     *
+     * <p>
+     * It is only known for video itags, so {@link #FPS_NOT_APPLICABLE_OR_UNKNOWN} is set/used for
+     * non video itags or if the sample rate value is less than or equal to 0.
+     * </p>
+     *
+     * @param fps the frame rate
+     */
+    public void setFps(final int fps) {
+        this.fps = fps > 0 ? fps : FPS_NOT_APPLICABLE_OR_UNKNOWN;
+    }
+
     public int getInitStart() {
         return initStart;
     }
@@ -251,6 +341,21 @@ public class ItagItem {
         this.quality = quality;
     }
 
+    /**
+     * Get the resolution string associated with this {@code ItagItem}.
+     *
+     * <p>
+     * It is only known for video itags.
+     * </p>
+     *
+     * @return the resolution string associated with this {@code ItagItem} or
+     * {@code null}.
+     */
+    @Nullable
+    public String getResolutionString() {
+        return resolutionString;
+    }
+
     public String getCodec() {
         return codec;
     }
@@ -258,4 +363,180 @@ public class ItagItem {
     public void setCodec(final String codec) {
         this.codec = codec;
     }
+
+    /**
+     * Get the average bitrate.
+     *
+     * <p>
+     * It is only known for audio itags, so {@link #AVERAGE_BITRATE_UNKNOWN} is always returned for
+     * other itag types.
+     * </p>
+     *
+     * <p>
+     * Bitrate of video itags and precise bitrate of audio itags can be known using
+     * {@link #getBitrate()}.
+     * </p>
+     *
+     * @return the average bitrate or {@link #AVERAGE_BITRATE_UNKNOWN}
+     * @see #getBitrate()
+     */
+    public int getAverageBitrate() {
+        return avgBitrate;
+    }
+
+    /**
+     * Get the sample rate.
+     *
+     * <p>
+     * It is only known for audio itags, so {@link #SAMPLE_RATE_UNKNOWN} is returned for non audio
+     * itags, or if the sample rate is unknown.
+     * </p>
+     *
+     * @return the sample rate or {@link #SAMPLE_RATE_UNKNOWN}
+     */
+    public int getSampleRate() {
+        return sampleRate;
+    }
+
+    /**
+     * Set the sample rate.
+     *
+     * <p>
+     * It is only known for audio itags, so {@link #SAMPLE_RATE_UNKNOWN} is set/used for non audio
+     * itags, or if the sample rate value is less than or equal to 0.
+     * </p>
+     *
+     * @param sampleRate the sample rate of an audio itag
+     */
+    public void setSampleRate(final int sampleRate) {
+        this.sampleRate = sampleRate > 0 ? sampleRate : SAMPLE_RATE_UNKNOWN;
+    }
+
+    /**
+     * Get the number of audio channels.
+     *
+     * <p>
+     * It is only known for audio itags, so {@link #AUDIO_CHANNELS_NOT_APPLICABLE_OR_UNKNOWN} is
+     * returned for non audio itags, or if it is unknown.
+     * </p>
+     *
+     * @return the number of audio channels or {@link #AUDIO_CHANNELS_NOT_APPLICABLE_OR_UNKNOWN}
+     */
+    public int getAudioChannels() {
+        return audioChannels;
+    }
+
+    /**
+     * Set the number of audio channels.
+     *
+     * <p>
+     * It is only known for audio itags, so {@link #AUDIO_CHANNELS_NOT_APPLICABLE_OR_UNKNOWN} is
+     * set/used for non audio itags, or if the {@code audioChannels} value is less than or equal to
+     * 0.
+     * </p>
+     *
+     * @param audioChannels the number of audio channels of an audio itag
+     */
+    public void setAudioChannels(final int audioChannels) {
+        this.audioChannels = audioChannels > 0
+                ? audioChannels
+                : AUDIO_CHANNELS_NOT_APPLICABLE_OR_UNKNOWN;
+    }
+
+    /**
+     * Get the {@code targetDurationSec} value.
+     *
+     * <p>
+     * This value is the average time in seconds of the duration of sequences of livestreams and
+     * ended livestreams. It is only returned by YouTube for these stream types, and makes no sense
+     * for videos, so {@link #TARGET_DURATION_SEC_UNKNOWN} is returned for those.
+     * </p>
+     *
+     * @return the {@code targetDurationSec} value or {@link #TARGET_DURATION_SEC_UNKNOWN}
+     */
+    public int getTargetDurationSec() {
+        return targetDurationSec;
+    }
+
+    /**
+     * Set the {@code targetDurationSec} value.
+     *
+     * <p>
+     * This value is the average time in seconds of the duration of sequences of livestreams and
+     * ended livestreams.
+     * </p>
+     *
+     * <p>
+     * It is only returned for these stream types by YouTube and makes no sense for videos, so
+     * {@link #TARGET_DURATION_SEC_UNKNOWN} will be set/used for video streams or if this value is
+     * less than or equal to 0.
+     * </p>
+     *
+     * @param targetDurationSec the target duration of a segment of streams which are using the
+     *                          live delivery method type
+     */
+    public void setTargetDurationSec(final int targetDurationSec) {
+        this.targetDurationSec = targetDurationSec > 0
+                ? targetDurationSec
+                : TARGET_DURATION_SEC_UNKNOWN;
+    }
+
+    /**
+     * Get the {@code approxDurationMs} value.
+     *
+     * <p>
+     * It is only known for DASH progressive streams, so {@link #APPROX_DURATION_MS_UNKNOWN} is
+     * returned for other stream types or if this value is less than or equal to 0.
+     * </p>
+     *
+     * @return the {@code approxDurationMs} value or {@link #APPROX_DURATION_MS_UNKNOWN}
+     */
+    public long getApproxDurationMs() {
+        return approxDurationMs;
+    }
+
+    /**
+     * Set the {@code approxDurationMs} value.
+     *
+     * <p>
+     * It is only known for DASH progressive streams, so {@link #APPROX_DURATION_MS_UNKNOWN} is
+     * set/used for other stream types or if this value is less than or equal to 0.
+     * </p>
+     *
+     * @param approxDurationMs the approximate duration of a DASH progressive stream, in
+     *                         milliseconds
+     */
+    public void setApproxDurationMs(final long approxDurationMs) {
+        this.approxDurationMs = approxDurationMs > 0
+                ? approxDurationMs
+                : APPROX_DURATION_MS_UNKNOWN;
+    }
+
+    /**
+     * Get the {@code contentLength} value.
+     *
+     * <p>
+     * It is only known for DASH progressive streams, so {@link #CONTENT_LENGTH_UNKNOWN} is
+     * returned for other stream types or if this value is less than or equal to 0.
+     * </p>
+     *
+     * @return the {@code contentLength} value or {@link #CONTENT_LENGTH_UNKNOWN}
+     */
+    public long getContentLength() {
+        return contentLength;
+    }
+
+    /**
+     * Set the content length of stream.
+     *
+     * <p>
+     * It is only known for DASH progressive streams, so {@link #CONTENT_LENGTH_UNKNOWN} is
+     * set/used for other stream types or if this value is less than or equal to 0.
+     * </p>
+     *
+     * @param contentLength the content length of a DASH progressive stream
+     */
+    public void setContentLength(final long contentLength) {
+        this.contentLength = contentLength > 0 ? contentLength : CONTENT_LENGTH_UNKNOWN;
+    }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java

@@ -71,6 +71,7 @@ import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Random;
+import java.util.regex.Pattern;
 
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
@@ -246,6 +247,11 @@ public final class YoutubeParsingHelper {
     private static final String FEED_BASE_CHANNEL_ID =
             "https://www.youtube.com/feeds/videos.xml?channel_id=";
     private static final String FEED_BASE_USER = "https://www.youtube.com/feeds/videos.xml?user=";
+    private static final Pattern C_WEB_PATTERN = Pattern.compile("&c=WEB");
+    private static final Pattern C_TVHTML5_SIMPLY_EMBEDDED_PLAYER_PATTERN =
+            Pattern.compile("&c=TVHTML5_SIMPLY_EMBEDDED_PLAYER");
+    private static final Pattern C_ANDROID_PATTERN = Pattern.compile("&c=ANDROID");
+    private static final Pattern C_IOS_PATTERN = Pattern.compile("&c=IOS");
 
     private static boolean isGoogleURL(final String url) {
         final String cachedUrl = extractCachedUrlIfNeeded(url);
@@ -1190,7 +1196,7 @@ public final class YoutubeParsingHelper {
             @Nonnull final Localization localization,
             @Nonnull final ContentCountry contentCountry,
             @Nonnull final String videoId) {
-                // @formatter:off
+        // @formatter:off
         return JsonObject.builder()
                 .object("context")
                     .object("client")
@@ -1258,8 +1264,7 @@ public final class YoutubeParsingHelper {
         // Spoofing an Android 12 device with the hardcoded version of the Android app
         return "com.google.android.youtube/" + MOBILE_YOUTUBE_CLIENT_VERSION
                 + " (Linux; U; Android 12; "
-                + (localization != null ? localization.getCountryCode()
+                + (localization != null ? localization : Localization.DEFAULT).getCountryCode()
-                        : Localization.DEFAULT.getCountryCode())
                 + ") gzip";
     }
 
@@ -1278,10 +1283,8 @@ public final class YoutubeParsingHelper {
     public static String getIosUserAgent(@Nullable final Localization localization) {
         // Spoofing an iPhone running iOS 15.4 with the hardcoded mobile client version
         return "com.google.ios.youtube/" + MOBILE_YOUTUBE_CLIENT_VERSION
-                + "(" + IOS_DEVICE_MODEL
+                + "(" + IOS_DEVICE_MODEL + "; U; CPU iOS 15_4 like Mac OS X; "
-                + "; U; CPU iOS 15_4 like Mac OS X; "
+                + (localization != null ? localization : Localization.DEFAULT).getCountryCode()
-                + (localization != null ? localization.getCountryCode()
-                        : Localization.DEFAULT.getCountryCode())
                 + ")";
     }
 
@@ -1588,4 +1591,46 @@ public final class YoutubeParsingHelper {
         return RandomStringFromAlphabetGenerator.generate(
                 CONTENT_PLAYBACK_NONCE_ALPHABET, 12, numberGenerator);
     }
+
+    /**
+     * Check if the streaming URL is from the YouTube {@code WEB} client.
+     *
+     * @param url the streaming URL to be checked.
+     * @return true if it's a {@code WEB} streaming URL, false otherwise
+     */
+    public static boolean isWebStreamingUrl(@Nonnull final String url) {
+        return Parser.isMatch(C_WEB_PATTERN, url);
+    }
+
+    /**
+     * Check if the streaming URL is a URL from the YouTube {@code TVHTML5_SIMPLY_EMBEDDED_PLAYER}
+     * client.
+     *
+     * @param url the streaming URL on which check if it's a {@code TVHTML5_SIMPLY_EMBEDDED_PLAYER}
+     *            streaming URL.
+     * @return true if it's a {@code TVHTML5_SIMPLY_EMBEDDED_PLAYER} streaming URL, false otherwise
+     */
+    public static boolean isTvHtml5SimplyEmbeddedPlayerStreamingUrl(@Nonnull final String url) {
+        return Parser.isMatch(C_TVHTML5_SIMPLY_EMBEDDED_PLAYER_PATTERN, url);
+    }
+
+    /**
+     * Check if the streaming URL is a URL from the YouTube {@code ANDROID} client.
+     *
+     * @param url the streaming URL to be checked.
+     * @return true if it's a {@code ANDROID} streaming URL, false otherwise
+     */
+    public static boolean isAndroidStreamingUrl(@Nonnull final String url) {
+        return Parser.isMatch(C_ANDROID_PATTERN, url);
+    }
+
+    /**
+     * Check if the streaming URL is a URL from the YouTube {@code IOS} client.
+     *
+     * @param url the streaming URL on which check if it's a {@code IOS} streaming URL.
+     * @return true if it's a {@code IOS} streaming URL, false otherwise
+     */
+    public static boolean isIosStreamingUrl(@Nonnull final String url) {
+        return Parser.isMatch(C_IOS_PATTERN, url);
+    }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/CreationException.java

@@ -0,0 +1,63 @@
+package org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators;
+
+import javax.annotation.Nonnull;
+
+/**
+ * Exception that is thrown when a YouTube DASH manifest creator encounters a problem
+ * while creating a manifest.
+ */
+public final class CreationException extends RuntimeException {
+
+    /**
+     * Create a new {@link CreationException} with a detail message.
+     *
+     * @param message the detail message to add in the exception
+     */
+    public CreationException(final String message) {
+        super(message);
+    }
+
+    /**
+     * Create a new {@link CreationException} with a detail message and a cause.
+     * @param message the detail message to add in the exception
+     * @param cause   the exception cause of this {@link CreationException}
+     */
+    public CreationException(final String message, final Exception cause) {
+        super(message, cause);
+    }
+
+    // Methods to create exceptions easily without having to use big exception messages and to
+    // reduce duplication
+
+    /**
+     * Create a new {@link CreationException} with a cause and the following detail message format:
+     * <br>
+     * {@code "Could not add " + element + " element", cause}, where {@code element} is an element
+     * of a DASH manifest.
+     *
+     * @param element the element which was not added to the DASH document
+     * @param cause   the exception which prevented addition of the element to the DASH document
+     * @return a new {@link CreationException}
+     */
+    @Nonnull
+    public static CreationException couldNotAddElement(final String element,
+                                                       final Exception cause) {
+        return new CreationException("Could not add " + element + " element", cause);
+    }
+
+    /**
+     * Create a new {@link CreationException} with a cause and the following detail message format:
+     * <br>
+     * {@code "Could not add " + element + " element: " + reason}, where {@code element} is an
+     * element of a DASH manifest and {@code reason} the reason why this element cannot be added to
+     * the DASH document.
+     *
+     * @param element the element which was not added to the DASH document
+     * @param reason  the reason message of why the element has been not added to the DASH document
+     * @return a new {@link CreationException}
+     */
+    @Nonnull
+    public static CreationException couldNotAddElement(final String element, final String reason) {
+        return new CreationException("Could not add " + element + " element: " + reason);
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeDashManifestCreatorsUtils.java

@@ -0,0 +1,757 @@
+package org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators;
+
+import org.schabi.newpipe.extractor.MediaFormat;
+import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.downloader.Downloader;
+import org.schabi.newpipe.extractor.downloader.Response;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.services.youtube.DeliveryType;
+import org.schabi.newpipe.extractor.services.youtube.ItagItem;
+import org.schabi.newpipe.extractor.utils.ManifestCreatorCache;
+import org.w3c.dom.Attr;
+import org.w3c.dom.DOMException;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+
+import javax.annotation.Nonnull;
+import javax.xml.XMLConstants;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.transform.OutputKeys;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerException;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+
+import java.io.IOException;
+import java.io.StringWriter;
+import java.nio.charset.StandardCharsets;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addClientInfoHeaders;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getAndroidUserAgent;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getIosUserAgent;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isAndroidStreamingUrl;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isIosStreamingUrl;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isTvHtml5SimplyEmbeddedPlayerStreamingUrl;
+import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isWebStreamingUrl;
+import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
+
+/**
+ * Utilities and constants for YouTube DASH manifest creators.
+ *
+ * <p>
+ * This class includes common methods of manifest creators and useful constants.
+ * </p>
+ *
+ * <p>
+ * Generation of DASH documents and their conversion as a string is done using external classes
+ * from {@link org.w3c.dom} and {@link javax.xml} packages.
+ * </p>
+ */
+public final class YoutubeDashManifestCreatorsUtils {
+
+    private YoutubeDashManifestCreatorsUtils() {
+    }
+
+    /**
+     * The redirect count limit that this class uses, which is the same limit as OkHttp.
+     */
+    public static final int MAXIMUM_REDIRECT_COUNT = 20;
+
+    /**
+     * URL parameter of the first sequence for live, post-live-DVR and OTF streams.
+     */
+    public static final String SQ_0 = "&sq=0";
+
+    /**
+     * URL parameter of the first stream request made by official clients.
+     */
+    public static final String RN_0 = "&rn=0";
+
+    /**
+     * URL parameter specific to web clients. When this param is added, if a redirection occurs,
+     * the server will not redirect clients to the redirect URL. Instead, it will provide this URL
+     * as the response body.
+     */
+    public static final String ALR_YES = "&alr=yes";
+
+    // XML elements of DASH MPD manifests
+    // see https://www.brendanlong.com/the-structure-of-an-mpeg-dash-mpd.html
+    public static final String MPD = "MPD";
+    public static final String PERIOD = "Period";
+    public static final String ADAPTATION_SET = "AdaptationSet";
+    public static final String ROLE = "Role";
+    public static final String REPRESENTATION = "Representation";
+    public static final String AUDIO_CHANNEL_CONFIGURATION = "AudioChannelConfiguration";
+    public static final String SEGMENT_TEMPLATE = "SegmentTemplate";
+    public static final String SEGMENT_TIMELINE = "SegmentTimeline";
+    public static final String BASE_URL = "BaseURL";
+    public static final String SEGMENT_BASE = "SegmentBase";
+    public static final String INITIALIZATION = "Initialization";
+
+    /**
+     * Create an attribute with {@link Document#createAttribute(String)}, assign to it the provided
+     * name and value, then add it to the provided element using {@link
+     * Element#setAttributeNode(Attr)}.
+     *
+     * @param element element to which to add the created node
+     * @param doc     document to use to create the attribute
+     * @param name    name of the attribute
+     * @param value   value of the attribute, will be set using {@link Attr#setValue(String)}
+     */
+    public static void setAttribute(final Element element,
+                                    final Document doc,
+                                    final String name,
+                                    final String value) {
+        final Attr attr = doc.createAttribute(name);
+        attr.setValue(value);
+        element.setAttributeNode(attr);
+    }
+
+    /**
+     * Generate a {@link Document} with common manifest creator elements added to it.
+     *
+     * <p>
+     * Those are:
+     * <ul>
+     *     <li>{@code MPD} (using {@link #generateDocumentAndMpdElement(long)});</li>
+     *     <li>{@code Period} (using {@link #generatePeriodElement(Document)});</li>
+     *     <li>{@code AdaptationSet} (using {@link #generateAdaptationSetElement(Document,
+     *     ItagItem)});</li>
+     *     <li>{@code Role} (using {@link #generateRoleElement(Document)});</li>
+     *     <li>{@code Representation} (using {@link #generateRepresentationElement(Document,
+     *     ItagItem)});</li>
+     *     <li>and, for audio streams, {@code AudioChannelConfiguration} (using
+     *     {@link #generateAudioChannelConfigurationElement(Document, ItagItem)}).</li>
+     * </ul>
+     * </p>
+     *
+     * @param itagItem the {@link ItagItem} associated to the stream, which must not be null
+     * @param streamDuration the duration of the stream, in milliseconds
+     * @return a {@link Document} with the common elements added in it
+     */
+    @Nonnull
+    public static Document generateDocumentAndDoCommonElementsGeneration(
+            @Nonnull final ItagItem itagItem,
+            final long streamDuration) throws CreationException {
+        final Document doc = generateDocumentAndMpdElement(streamDuration);
+
+        generatePeriodElement(doc);
+        generateAdaptationSetElement(doc, itagItem);
+        generateRoleElement(doc);
+        generateRepresentationElement(doc, itagItem);
+        if (itagItem.itagType == ItagItem.ItagType.AUDIO) {
+            generateAudioChannelConfigurationElement(doc, itagItem);
+        }
+
+        return doc;
+    }
+
+    /**
+     * Create a {@link Document} instance and generate the {@code <MPD>} element of the manifest.
+     *
+     * <p>
+     * The generated {@code <MPD>} element looks like the manifest returned into the player
+     * response of videos:
+     * </p>
+     *
+     * <p>
+     * {@code <MPD xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+     * xmlns="urn:mpeg:DASH:schema:MPD:2011"
+     * xsi:schemaLocation="urn:mpeg:DASH:schema:MPD:2011 DASH-MPD.xsd" minBufferTime="PT1.500S"
+     * profiles="urn:mpeg:dash:profile:isoff-main:2011" type="static"
+     * mediaPresentationDuration="PT$duration$S">}
+     * (where {@code $duration$} represents the duration in seconds (a number with 3 digits after
+     * the decimal point)).
+     * </p>
+     *
+     * @param duration the duration of the stream, in milliseconds
+     * @return a {@link Document} instance which contains a {@code <MPD>} element
+     */
+    @Nonnull
+    public static Document generateDocumentAndMpdElement(final long duration)
+            throws CreationException {
+        try {
+            final Document doc = newDocument();
+
+            final Element mpdElement = doc.createElement(MPD);
+            doc.appendChild(mpdElement);
+
+            setAttribute(mpdElement, doc, "xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance");
+            setAttribute(mpdElement, doc, "xmlns", "urn:mpeg:DASH:schema:MPD:2011");
+            setAttribute(mpdElement, doc, "xsi:schemaLocation",
+                    "urn:mpeg:DASH:schema:MPD:2011 DASH-MPD.xsd");
+            setAttribute(mpdElement, doc, "minBufferTime", "PT1.500S");
+            setAttribute(mpdElement, doc, "profiles", "urn:mpeg:dash:profile:full:2011");
+            setAttribute(mpdElement, doc, "type", "static");
+            setAttribute(mpdElement, doc, "mediaPresentationDuration",
+                    String.format(Locale.ENGLISH, "PT%.3fS", duration / 1000.0));
+
+            return doc;
+        } catch (final Exception e) {
+            throw new CreationException(
+                    "Could not generate the DASH manifest or append the MPD doc to it", e);
+        }
+    }
+
+    /**
+     * Generate the {@code <Period>} element, appended as a child of the {@code <MPD>} element.
+     *
+     * <p>
+     * The {@code <MPD>} element needs to be generated before this element with
+     * {@link #generateDocumentAndMpdElement(long)}.
+     * </p>
+     *
+     * @param doc the {@link Document} on which the the {@code <Period>} element will be appended
+     */
+    public static void generatePeriodElement(@Nonnull final Document doc)
+            throws CreationException {
+        try {
+            final Element mpdElement = (Element) doc.getElementsByTagName(MPD).item(0);
+            final Element periodElement = doc.createElement(PERIOD);
+            mpdElement.appendChild(periodElement);
+        } catch (final DOMException e) {
+            throw CreationException.couldNotAddElement(PERIOD, e);
+        }
+    }
+
+    /**
+     * Generate the {@code <AdaptationSet>} element, appended as a child of the {@code <Period>}
+     * element.
+     *
+     * <p>
+     * The {@code <Period>} element needs to be generated before this element with
+     * {@link #generatePeriodElement(Document)}.
+     * </p>
+     *
+     * @param doc the {@link Document} on which the {@code <Period>} element will be appended
+     * @param itagItem the {@link ItagItem} corresponding to the stream, which must not be null
+     */
+    public static void generateAdaptationSetElement(@Nonnull final Document doc,
+                                                    @Nonnull final ItagItem itagItem)
+            throws CreationException {
+        try {
+            final Element periodElement = (Element) doc.getElementsByTagName(PERIOD)
+                    .item(0);
+            final Element adaptationSetElement = doc.createElement(ADAPTATION_SET);
+
+            setAttribute(adaptationSetElement, doc, "id", "0");
+
+            final MediaFormat mediaFormat = itagItem.getMediaFormat();
+            if (mediaFormat == null || isNullOrEmpty(mediaFormat.getMimeType())) {
+                throw CreationException.couldNotAddElement(ADAPTATION_SET,
+                        "the MediaFormat or its mime type is null or empty");
+            }
+
+            setAttribute(adaptationSetElement, doc, "mimeType", mediaFormat.getMimeType());
+            setAttribute(adaptationSetElement, doc, "subsegmentAlignment", "true");
+
+            periodElement.appendChild(adaptationSetElement);
+        } catch (final DOMException e) {
+            throw CreationException.couldNotAddElement(ADAPTATION_SET, e);
+        }
+    }
+
+    /**
+     * Generate the {@code <Role>} element, appended as a child of the {@code <AdaptationSet>}
+     * element.
+     *
+     * <p>
+     * This element, with its attributes and values, is:
+     * </p>
+     *
+     * <p>
+     * {@code <Role schemeIdUri="urn:mpeg:DASH:role:2011" value="main"/>}
+     * </p>
+     *
+     * <p>
+     * The {@code <AdaptationSet>} element needs to be generated before this element with
+     * {@link #generateAdaptationSetElement(Document, ItagItem)}).
+     * </p>
+     *
+     * @param doc the {@link Document} on which the the {@code <Role>} element will be appended
+     */
+    public static void generateRoleElement(@Nonnull final Document doc)
+            throws CreationException {
+        try {
+            final Element adaptationSetElement = (Element) doc.getElementsByTagName(
+                    ADAPTATION_SET).item(0);
+            final Element roleElement = doc.createElement(ROLE);
+
+            setAttribute(roleElement, doc, "schemeIdUri", "urn:mpeg:DASH:role:2011");
+            setAttribute(roleElement, doc, "value", "main");
+
+            adaptationSetElement.appendChild(roleElement);
+        } catch (final DOMException e) {
+            throw CreationException.couldNotAddElement(ROLE, e);
+        }
+    }
+
+    /**
+     * Generate the {@code <Representation>} element, appended as a child of the
+     * {@code <AdaptationSet>} element.
+     *
+     * <p>
+     * The {@code <AdaptationSet>} element needs to be generated before this element with
+     * {@link #generateAdaptationSetElement(Document, ItagItem)}).
+     * </p>
+     *
+     * @param doc the {@link Document} on which the the {@code <SegmentTimeline>} element will be
+     *            appended
+     * @param itagItem the {@link ItagItem} to use, which must not be null
+     */
+    public static void generateRepresentationElement(@Nonnull final Document doc,
+                                                     @Nonnull final ItagItem itagItem)
+            throws CreationException {
+        try {
+            final Element adaptationSetElement = (Element) doc.getElementsByTagName(
+                    ADAPTATION_SET).item(0);
+            final Element representationElement = doc.createElement(REPRESENTATION);
+
+            final int id = itagItem.id;
+            if (id <= 0) {
+                throw CreationException.couldNotAddElement(REPRESENTATION,
+                        "the id of the ItagItem is <= 0");
+            }
+            setAttribute(representationElement, doc, "id", String.valueOf(id));
+
+            final String codec = itagItem.getCodec();
+            if (isNullOrEmpty(codec)) {
+                throw CreationException.couldNotAddElement(ADAPTATION_SET,
+                        "the codec value of the ItagItem is null or empty");
+            }
+            setAttribute(representationElement, doc, "codecs", codec);
+            setAttribute(representationElement, doc, "startWithSAP", "1");
+            setAttribute(representationElement, doc, "maxPlayoutRate", "1");
+
+            final int bitrate = itagItem.getBitrate();
+            if (bitrate <= 0) {
+                throw CreationException.couldNotAddElement(REPRESENTATION,
+                        "the bitrate of the ItagItem is <= 0");
+            }
+            setAttribute(representationElement, doc, "bandwidth", String.valueOf(bitrate));
+
+            if (itagItem.itagType == ItagItem.ItagType.VIDEO
+                    || itagItem.itagType == ItagItem.ItagType.VIDEO_ONLY) {
+                final int height = itagItem.getHeight();
+                final int width = itagItem.getWidth();
+                if (height <= 0 && width <= 0) {
+                    throw CreationException.couldNotAddElement(REPRESENTATION,
+                            "both width and height of the ItagItem are <= 0");
+                }
+
+                if (width > 0) {
+                    setAttribute(representationElement, doc, "width", String.valueOf(width));
+                }
+                setAttribute(representationElement, doc, "height",
+                        String.valueOf(itagItem.getHeight()));
+
+                final int fps = itagItem.getFps();
+                if (fps > 0) {
+                    setAttribute(representationElement, doc, "frameRate", String.valueOf(fps));
+                }
+            }
+
+            if (itagItem.itagType == ItagItem.ItagType.AUDIO && itagItem.getSampleRate() > 0) {
+                final Attr audioSamplingRateAttribute = doc.createAttribute(
+                        "audioSamplingRate");
+                audioSamplingRateAttribute.setValue(String.valueOf(itagItem.getSampleRate()));
+            }
+
+            adaptationSetElement.appendChild(representationElement);
+        } catch (final DOMException e) {
+            throw CreationException.couldNotAddElement(REPRESENTATION, e);
+        }
+    }
+
+    /**
+     * Generate the {@code <AudioChannelConfiguration>} element, appended as a child of the
+     * {@code <Representation>} element.
+     *
+     * <p>
+     * This method is only used when generating DASH manifests of audio streams.
+     * </p>
+     *
+     * <p>
+     * It will produce the following element:
+     * <br>
+     * {@code <AudioChannelConfiguration
+     * schemeIdUri="urn:mpeg:dash:23003:3:audio_channel_configuration:2011"
+     * value="audioChannelsValue"}
+     * <br>
+     * (where {@code audioChannelsValue} is get from the {@link ItagItem} passed as the second
+     * parameter of this method)
+     * </p>
+     *
+     * <p>
+     * The {@code <Representation>} element needs to be generated before this element with
+     * {@link #generateRepresentationElement(Document, ItagItem)}).
+     * </p>
+     *
+     * @param doc the {@link Document} on which the {@code <AudioChannelConfiguration>} element will
+     *            be appended
+     * @param itagItem the {@link ItagItem} to use, which must not be null
+     */
+    public static void generateAudioChannelConfigurationElement(
+            @Nonnull final Document doc,
+            @Nonnull final ItagItem itagItem) throws CreationException {
+        try {
+            final Element representationElement = (Element) doc.getElementsByTagName(
+                    REPRESENTATION).item(0);
+            final Element audioChannelConfigurationElement = doc.createElement(
+                    AUDIO_CHANNEL_CONFIGURATION);
+
+            setAttribute(audioChannelConfigurationElement, doc, "schemeIdUri",
+                    "urn:mpeg:dash:23003:3:audio_channel_configuration:2011");
+
+            if (itagItem.getAudioChannels() <= 0) {
+                throw new CreationException("the number of audioChannels in the ItagItem is <= 0: "
+                        + itagItem.getAudioChannels());
+            }
+            setAttribute(audioChannelConfigurationElement, doc, "value",
+                    String.valueOf(itagItem.getAudioChannels()));
+
+            representationElement.appendChild(audioChannelConfigurationElement);
+        } catch (final DOMException e) {
+            throw CreationException.couldNotAddElement(AUDIO_CHANNEL_CONFIGURATION, e);
+        }
+    }
+
+    /**
+     * Convert a DASH manifest {@link Document doc} to a string and cache it.
+     *
+     * @param originalBaseStreamingUrl the original base URL of the stream
+     * @param doc                      the doc to be converted
+     * @param manifestCreatorCache     the {@link ManifestCreatorCache} on which store the string
+     *                                 generated
+     * @return the DASH manifest {@link Document doc} converted to a string
+     */
+    public static String buildAndCacheResult(
+            @Nonnull final String originalBaseStreamingUrl,
+            @Nonnull final Document doc,
+            @Nonnull final ManifestCreatorCache<String, String> manifestCreatorCache)
+            throws CreationException {
+
+        try {
+            final String documentXml = documentToXml(doc);
+            manifestCreatorCache.put(originalBaseStreamingUrl, documentXml);
+            return documentXml;
+        } catch (final Exception e) {
+            throw new CreationException(
+                    "Could not convert the DASH manifest generated to a string", e);
+        }
+    }
+
+    /**
+     * Generate the {@code <SegmentTemplate>} element, appended as a child of the
+     * {@code <Representation>} element.
+     *
+     * <p>
+     * This method is only used when generating DASH manifests from OTF and post-live-DVR streams.
+     * </p>
+     *
+     * <p>
+     * It will produce a {@code <SegmentTemplate>} element with the following attributes:
+     * <ul>
+     *     <li>{@code startNumber}, which takes the value {@code 0} for post-live-DVR streams and
+     *     {@code 1} for OTF streams;</li>
+     *     <li>{@code timescale}, which is always {@code 1000};</li>
+     *     <li>{@code media}, which is the base URL of the stream on which is appended
+     *     {@code &sq=$Number$};</li>
+     *     <li>{@code initialization} (only for OTF streams), which is the base URL of the stream
+     *     on which is appended {@link #SQ_0}.</li>
+     * </ul>
+     * </p>
+     *
+     * <p>
+     * The {@code <Representation>} element needs to be generated before this element with
+     * {@link #generateRepresentationElement(Document, ItagItem)}).
+     * </p>
+     *
+     * @param doc          the {@link Document} on which the {@code <SegmentTemplate>} element will
+     *                     be appended
+     * @param baseUrl      the base URL of the OTF/post-live-DVR stream
+     * @param deliveryType the stream {@link DeliveryType delivery type}, which must be either
+     * {@link DeliveryType#OTF OTF} or {@link DeliveryType#LIVE LIVE}
+     */
+    public static void generateSegmentTemplateElement(@Nonnull final Document doc,
+                                                      @Nonnull final String baseUrl,
+                                                      final DeliveryType deliveryType)
+            throws CreationException {
+        if (deliveryType != DeliveryType.OTF && deliveryType != DeliveryType.LIVE) {
+            throw CreationException.couldNotAddElement(SEGMENT_TEMPLATE, "invalid delivery type: "
+                    + deliveryType);
+        }
+
+        try {
+            final Element representationElement = (Element) doc.getElementsByTagName(
+                    REPRESENTATION).item(0);
+            final Element segmentTemplateElement = doc.createElement(SEGMENT_TEMPLATE);
+
+            // The first sequence of post DVR streams is the beginning of the video stream and not
+            // an initialization segment
+            setAttribute(segmentTemplateElement, doc, "startNumber",
+                    deliveryType == DeliveryType.LIVE ? "0" : "1");
+            setAttribute(segmentTemplateElement, doc, "timescale", "1000");
+
+            // Post-live-DVR/ended livestreams streams don't require an initialization sequence
+            if (deliveryType != DeliveryType.LIVE) {
+                setAttribute(segmentTemplateElement, doc, "initialization", baseUrl + SQ_0);
+            }
+
+            setAttribute(segmentTemplateElement, doc, "media", baseUrl + "&sq=$Number$");
+
+            representationElement.appendChild(segmentTemplateElement);
+        } catch (final DOMException e) {
+            throw CreationException.couldNotAddElement(SEGMENT_TEMPLATE, e);
+        }
+    }
+
+    /**
+     * Generate the {@code <SegmentTimeline>} element, appended as a child of the
+     * {@code <SegmentTemplate>} element.
+     *
+     * <p>
+     * The {@code <SegmentTemplate>} element needs to be generated before this element with
+     * {@link #generateSegmentTemplateElement(Document, String, DeliveryType)}.
+     * </p>
+     *
+     * @param doc the {@link Document} on which the the {@code <SegmentTimeline>} element will be
+     *            appended
+     */
+    public static void generateSegmentTimelineElement(@Nonnull final Document doc)
+            throws CreationException {
+        try {
+            final Element segmentTemplateElement = (Element) doc.getElementsByTagName(
+                    SEGMENT_TEMPLATE).item(0);
+            final Element segmentTimelineElement = doc.createElement(SEGMENT_TIMELINE);
+
+            segmentTemplateElement.appendChild(segmentTimelineElement);
+        } catch (final DOMException e) {
+            throw CreationException.couldNotAddElement(SEGMENT_TIMELINE, e);
+        }
+    }
+
+    /**
+     * Get the "initialization" {@link Response response} of a stream.
+     *
+     * <p>This method fetches, for OTF streams and for post-live-DVR streams:
+     *     <ul>
+     *         <li>the base URL of the stream, to which are appended {@link #SQ_0} and
+     *         {@link #RN_0} parameters, with a {@code GET} request for streaming URLs from HTML5
+     *         clients and a {@code POST} request for the ones from the {@code ANDROID} and the
+     *         {@code IOS} clients;</li>
+     *         <li>for streaming URLs from HTML5 clients, the {@link #ALR_YES} param is also added.
+     *         </li>
+     *     </ul>
+     * </p>
+     *
+     * @param baseStreamingUrl the base URL of the stream, which must not be null
+     * @param itagItem         the {@link ItagItem} of stream, which must not be null
+     * @param deliveryType     the {@link DeliveryType} of the stream
+     * @return the "initialization" response, without redirections on the network on which the
+     * request(s) is/are made
+     */
+    @SuppressWarnings("checkstyle:FinalParameters")
+    @Nonnull
+    public static Response getInitializationResponse(@Nonnull String baseStreamingUrl,
+                                                     @Nonnull final ItagItem itagItem,
+                                                     final DeliveryType deliveryType)
+            throws CreationException {
+        final boolean isHtml5StreamingUrl = isWebStreamingUrl(baseStreamingUrl)
+                || isTvHtml5SimplyEmbeddedPlayerStreamingUrl(baseStreamingUrl);
+        final boolean isAndroidStreamingUrl = isAndroidStreamingUrl(baseStreamingUrl);
+        final boolean isIosStreamingUrl = isIosStreamingUrl(baseStreamingUrl);
+        if (isHtml5StreamingUrl) {
+            baseStreamingUrl += ALR_YES;
+        }
+        baseStreamingUrl = appendRnSqParamsIfNeeded(baseStreamingUrl, deliveryType);
+
+        final Downloader downloader = NewPipe.getDownloader();
+        if (isHtml5StreamingUrl) {
+            final String mimeTypeExpected = itagItem.getMediaFormat().getMimeType();
+            if (!isNullOrEmpty(mimeTypeExpected)) {
+                return getStreamingWebUrlWithoutRedirects(downloader, baseStreamingUrl,
+                        mimeTypeExpected);
+            }
+        } else if (isAndroidStreamingUrl || isIosStreamingUrl) {
+            try {
+                final Map<String, List<String>> headers = new HashMap<>();
+                headers.put("User-Agent", Collections.singletonList(
+                        isAndroidStreamingUrl ? getAndroidUserAgent(null)
+                                : getIosUserAgent(null)));
+                final byte[] emptyBody = "".getBytes(StandardCharsets.UTF_8);
+                return downloader.post(baseStreamingUrl, headers, emptyBody);
+            } catch (final IOException | ExtractionException e) {
+                throw new CreationException("Could not get the "
+                        + (isIosStreamingUrl ? "ANDROID" : "IOS") + " streaming URL response", e);
+            }
+        }
+
+        try {
+            return downloader.get(baseStreamingUrl);
+        } catch (final IOException | ExtractionException e) {
+            throw new CreationException("Could not get the streaming URL response", e);
+        }
+    }
+
+    /**
+     * Generate a new {@link DocumentBuilder} secured from XXE attacks, on platforms which
+     * support setting {@link XMLConstants#ACCESS_EXTERNAL_DTD} and
+     * {@link XMLConstants#ACCESS_EXTERNAL_SCHEMA} in {@link DocumentBuilderFactory} instances.
+     *
+     * @return an instance of {@link Document} secured against XXE attacks on supported platforms,
+     *         that should then be convertible to an XML string without security problems
+     */
+    private static Document newDocument() throws ParserConfigurationException {
+        final DocumentBuilderFactory documentBuilderFactory = DocumentBuilderFactory.newInstance();
+        try {
+            documentBuilderFactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, "");
+            documentBuilderFactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_SCHEMA, "");
+        } catch (final Exception ignored) {
+            // Ignore exceptions as setting these attributes to secure XML generation is not
+            // supported by all platforms (like the Android implementation)
+        }
+
+        return documentBuilderFactory.newDocumentBuilder().newDocument();
+    }
+
+    /**
+     * Generate a new {@link TransformerFactory} secured from XXE attacks, on platforms which
+     * support setting {@link XMLConstants#ACCESS_EXTERNAL_DTD} and
+     * {@link XMLConstants#ACCESS_EXTERNAL_SCHEMA} in {@link TransformerFactory} instances.
+     *
+     * @param doc the doc to convert, which must have been created using {@link #newDocument()} to
+     *            properly prevent XXE attacks
+     * @return the doc converted to an XML string, making sure there can't be XXE attacks
+     */
+    // Sonar warning is suppressed because it is still shown even if we apply its solution
+    @SuppressWarnings("squid:S2755")
+    private static String documentToXml(@Nonnull final Document doc)
+            throws TransformerException {
+
+        final TransformerFactory transformerFactory = TransformerFactory.newInstance();
+        try {
+            transformerFactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, "");
+            transformerFactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_SCHEMA, "");
+        } catch (final Exception ignored) {
+            // Ignore exceptions as setting these attributes to secure XML generation is not
+            // supported by all platforms (like the Android implementation)
+        }
+
+        final Transformer transformer = transformerFactory.newTransformer();
+        transformer.setOutputProperty(OutputKeys.VERSION, "1.0");
+        transformer.setOutputProperty(OutputKeys.ENCODING, "UTF-8");
+        transformer.setOutputProperty(OutputKeys.STANDALONE, "no");
+
+        final StringWriter result = new StringWriter();
+        transformer.transform(new DOMSource(doc), new StreamResult(result));
+
+        return result.toString();
+    }
+
+    /**
+     * Append {@link #SQ_0} for post-live-DVR and OTF streams and {@link #RN_0} to all streams.
+     *
+     * @param baseStreamingUrl the base streaming URL to which the parameter(s) are being appended
+     * @param deliveryType     the {@link DeliveryType} of the stream
+     * @return the base streaming URL to which the param(s) are appended, depending on the
+     * {@link DeliveryType} of the stream
+     */
+    @Nonnull
+    private static String appendRnSqParamsIfNeeded(@Nonnull final String baseStreamingUrl,
+                                                   @Nonnull final DeliveryType deliveryType) {
+        return baseStreamingUrl + (deliveryType == DeliveryType.PROGRESSIVE ? "" : SQ_0) + RN_0;
+    }
+
+    /**
+     * Get a URL on which no redirection between playback hosts should be present on the network
+     * and/or IP used to fetch the streaming URL, for HTML5 clients.
+     *
+     * <p>This method will follow redirects which works in the following way:
+     *     <ol>
+     *         <li>the {@link #ALR_YES} param is appended to all streaming URLs</li>
+     *         <li>if no redirection occurs, the video server will return the streaming data;</li>
+     *         <li>if a redirection occurs, the server will respond with HTTP status code 200 and a
+     *         {@code text/plain} mime type. The redirection URL is the response body;</li>
+     *         <li>the redirection URL is requested and the steps above from step 2 are repeated,
+     *         until too many redirects are reached of course (the maximum number of redirects is
+     *         {@link #MAXIMUM_REDIRECT_COUNT the same as OkHttp}).</li>
+     *     </ol>
+     * </p>
+     *
+     * <p>
+     * For non-HTML5 clients, redirections are managed in the standard way in
+     * {@link #getInitializationResponse(String, ItagItem, DeliveryType)}.
+     * </p>
+     *
+     * @param downloader               the {@link Downloader} instance to be used
+     * @param streamingUrl             the streaming URL which we are trying to get a streaming URL
+     *                                 without any redirection on the network and/or IP used
+     * @param responseMimeTypeExpected the response mime type expected from Google video servers
+     * @return the {@link Response} of the stream, which should have no redirections
+     */
+    @SuppressWarnings("checkstyle:FinalParameters")
+    @Nonnull
+    private static Response getStreamingWebUrlWithoutRedirects(
+            @Nonnull final Downloader downloader,
+            @Nonnull String streamingUrl,
+            @Nonnull final String responseMimeTypeExpected)
+            throws CreationException {
+        try {
+            final Map<String, List<String>> headers = new HashMap<>();
+            addClientInfoHeaders(headers);
+
+            String responseMimeType = "";
+
+            int redirectsCount = 0;
+            while (!responseMimeType.equals(responseMimeTypeExpected)
+                    && redirectsCount < MAXIMUM_REDIRECT_COUNT) {
+                final Response response = downloader.get(streamingUrl, headers);
+
+                final int responseCode = response.responseCode();
+                if (responseCode != 200) {
+                    throw new CreationException(
+                            "Could not get the initialization URL: HTTP response code "
+                                    + responseCode);
+                }
+
+                // A valid HTTP 1.0+ response should include a Content-Type header, so we can
+                // require that the response from video servers has this header.
+                responseMimeType = Objects.requireNonNull(response.getHeader("Content-Type"),
+                        "Could not get the Content-Type header from the response headers");
+
+                // The response body is the redirection URL
+                if (responseMimeType.equals("text/plain")) {
+                    streamingUrl = response.responseBody();
+                    redirectsCount++;
+                } else {
+                    return response;
+                }
+            }
+
+            if (redirectsCount >= MAXIMUM_REDIRECT_COUNT) {
+                throw new CreationException(
+                        "Too many redirects when trying to get the the streaming URL response of a "
+                                + "HTML5 client");
+            }
+
+            // This should never be reached, but is required because we don't want to return null
+            // here
+            throw new CreationException(
+                    "Could not get the streaming URL response of a HTML5 client: unreachable code "
+                            + "reached!");
+        } catch (final IOException | ExtractionException e) {
+            throw new CreationException(
+                    "Could not get the streaming URL response of a HTML5 client", e);
+        }
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeOtfDashManifestCreator.java

@@ -0,0 +1,265 @@
+package org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators;
+
+import org.schabi.newpipe.extractor.downloader.Response;
+import org.schabi.newpipe.extractor.services.youtube.DeliveryType;
+import org.schabi.newpipe.extractor.services.youtube.ItagItem;
+import org.schabi.newpipe.extractor.utils.ManifestCreatorCache;
+import org.schabi.newpipe.extractor.utils.Utils;
+import org.w3c.dom.DOMException;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+
+import javax.annotation.Nonnull;
+import java.util.Arrays;
+import java.util.Objects;
+
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.ALR_YES;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.RN_0;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.SEGMENT_TIMELINE;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.SQ_0;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.buildAndCacheResult;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.generateDocumentAndDoCommonElementsGeneration;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.generateSegmentTemplateElement;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.generateSegmentTimelineElement;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.getInitializationResponse;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.setAttribute;
+import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
+import static org.schabi.newpipe.extractor.utils.Utils.isBlank;
+
+/**
+ * Class which generates DASH manifests of YouTube {@link DeliveryType#OTF OTF streams}.
+ */
+public final class YoutubeOtfDashManifestCreator {
+
+    /**
+     * Cache of DASH manifests generated for OTF streams.
+     */
+    private static final ManifestCreatorCache<String, String> OTF_STREAMS_CACHE
+            = new ManifestCreatorCache<>();
+
+    private YoutubeOtfDashManifestCreator() {
+    }
+
+    /**
+     * Create DASH manifests from a YouTube OTF stream.
+     *
+     * <p>
+     * OTF streams are YouTube-DASH specific streams which work with sequences and without the need
+     * to get a manifest (even if one is provided, it is not used by official clients).
+     * </p>
+     *
+     * <p>
+     * They can be found only on videos; mostly those with a small amount of views, or ended
+     * livestreams which have just been re-encoded as normal videos.
+     * </p>
+     *
+     * <p>This method needs:
+     *     <ul>
+     *         <li>the base URL of the stream (which, if you try to access to it, returns HTTP
+     *         status code 404 after redirects, and if the URL is valid);</li>
+     *         <li>an {@link ItagItem}, which needs to contain the following information:
+     *              <ul>
+     *                  <li>its type (see {@link ItagItem.ItagType}), to identify if the content is
+     *                  an audio or a video stream;</li>
+     *                  <li>its bitrate;</li>
+     *                  <li>its mime type;</li>
+     *                  <li>its codec(s);</li>
+     *                  <li>for an audio stream: its audio channels;</li>
+     *                  <li>for a video stream: its width and height.</li>
+     *              </ul>
+     *         </li>
+     *         <li>the duration of the video, which will be used if the duration could not be
+     *         parsed from the first sequence of the stream.</li>
+     *     </ul>
+     * </p>
+     *
+     * <p>In order to generate the DASH manifest, this method will:
+     *      <ul>
+     *          <li>request the first sequence of the stream (the base URL on which the first
+     *          sequence parameter is appended (see {@link YoutubeDashManifestCreatorsUtils#SQ_0}))
+     *          with a {@code POST} or {@code GET} request (depending of the client on which the
+     *          streaming URL comes from is a mobile one ({@code POST}) or not ({@code GET}));</li>
+     *          <li>follow its redirection(s), if any;</li>
+     *          <li>save the last URL, remove the first sequence parameter;</li>
+     *          <li>use the information provided in the {@link ItagItem} to generate all
+     *          elements of the DASH manifest.</li>
+     *      </ul>
+     * </p>
+     *
+     * <p>
+     * If the duration cannot be extracted, the {@code durationSecondsFallback} value will be used
+     * as the stream duration.
+     * </p>
+     *
+     * @param otfBaseStreamingUrl     the base URL of the OTF stream, which must not be null
+     * @param itagItem                the {@link ItagItem} corresponding to the stream, which
+     *                                must not be null
+     * @param durationSecondsFallback the duration of the video, which will be used if the duration
+     *                                could not be extracted from the first sequence
+     * @return the manifest generated into a string
+     */
+    @Nonnull
+    public static String fromOtfStreamingUrl(
+            @Nonnull final String otfBaseStreamingUrl,
+            @Nonnull final ItagItem itagItem,
+            final long durationSecondsFallback) throws CreationException {
+        if (OTF_STREAMS_CACHE.containsKey(otfBaseStreamingUrl)) {
+            return Objects.requireNonNull(OTF_STREAMS_CACHE.get(otfBaseStreamingUrl)).getSecond();
+        }
+
+        String realOtfBaseStreamingUrl = otfBaseStreamingUrl;
+        // Try to avoid redirects when streaming the content by saving the last URL we get
+        // from video servers.
+        final Response response = getInitializationResponse(realOtfBaseStreamingUrl,
+                itagItem, DeliveryType.OTF);
+        realOtfBaseStreamingUrl = response.latestUrl().replace(SQ_0, EMPTY_STRING)
+                .replace(RN_0, EMPTY_STRING).replace(ALR_YES, EMPTY_STRING);
+
+        final int responseCode = response.responseCode();
+        if (responseCode != 200) {
+            throw new CreationException("Could not get the initialization URL: response code "
+                    + responseCode);
+        }
+
+        final String[] segmentDuration;
+
+        try {
+            final String[] segmentsAndDurationsResponseSplit = response.responseBody()
+                    // Get the lines with the durations and the following
+                    .split("Segment-Durations-Ms: ")[1]
+                    // Remove the other lines
+                    .split("\n")[0]
+                    // Get all durations and repetitions which are separated by a comma
+                    .split(",");
+            final int lastIndex = segmentsAndDurationsResponseSplit.length - 1;
+            if (isBlank(segmentsAndDurationsResponseSplit[lastIndex])) {
+                segmentDuration = Arrays.copyOf(segmentsAndDurationsResponseSplit, lastIndex);
+            } else {
+                segmentDuration = segmentsAndDurationsResponseSplit;
+            }
+        } catch (final Exception e) {
+            throw new CreationException("Could not get segment durations", e);
+        }
+
+        long streamDuration;
+        try {
+            streamDuration = getStreamDuration(segmentDuration);
+        } catch (final CreationException e) {
+            streamDuration = durationSecondsFallback * 1000;
+        }
+
+        final Document doc = generateDocumentAndDoCommonElementsGeneration(itagItem,
+                streamDuration);
+
+        generateSegmentTemplateElement(doc, realOtfBaseStreamingUrl, DeliveryType.OTF);
+        generateSegmentTimelineElement(doc);
+        generateSegmentElementsForOtfStreams(segmentDuration, doc);
+
+        return buildAndCacheResult(otfBaseStreamingUrl, doc, OTF_STREAMS_CACHE);
+    }
+
+    /**
+     * @return the cache of DASH manifests generated for OTF streams
+     */
+    @Nonnull
+    public static ManifestCreatorCache<String, String> getCache() {
+        return OTF_STREAMS_CACHE;
+    }
+
+    /**
+     * Generate segment elements for OTF streams.
+     *
+     * <p>
+     * By parsing by the first media sequence, we know how many durations and repetitions there are
+     * so we just have to loop into segment durations to generate the following elements for each
+     * duration repeated X times:
+     * </p>
+     *
+     * <p>
+     * {@code <S d="segmentDuration" r="durationRepetition" />}
+     * </p>
+     *
+     * <p>
+     * If there is no repetition of the duration between two segments, the {@code r} attribute is
+     * not added to the {@code S} element, as it is not needed.
+     * </p>
+     *
+     * <p>
+     * These elements will be appended as children of the {@code <SegmentTimeline>} element, which
+     * needs to be generated before these elements with
+     * {@link YoutubeDashManifestCreatorsUtils#generateSegmentTimelineElement(Document)}.
+     * </p>
+     *
+     * @param segmentDurations the sequences "length" or "length(r=repeat_count" extracted with the
+     *                         regular expressions
+     * @param doc              the {@link Document} on which the {@code <S>} elements will be
+     *                         appended
+     */
+    private static void generateSegmentElementsForOtfStreams(
+            @Nonnull final String[] segmentDurations,
+            @Nonnull final Document doc) throws CreationException {
+        try {
+            final Element segmentTimelineElement = (Element) doc.getElementsByTagName(
+                    SEGMENT_TIMELINE).item(0);
+
+            for (final String segmentDuration : segmentDurations) {
+                final Element sElement = doc.createElement("S");
+
+                final String[] segmentLengthRepeat = segmentDuration.split("\\(r=");
+                // make sure segmentLengthRepeat[0], which is the length, is convertible to int
+                Integer.parseInt(segmentLengthRepeat[0]);
+
+                // There are repetitions of a segment duration in other segments
+                if (segmentLengthRepeat.length > 1) {
+                    final int segmentRepeatCount = Integer.parseInt(
+                            Utils.removeNonDigitCharacters(segmentLengthRepeat[1]));
+                    setAttribute(sElement, doc, "r", String.valueOf(segmentRepeatCount));
+                }
+                setAttribute(sElement, doc, "d", segmentLengthRepeat[0]);
+
+                segmentTimelineElement.appendChild(sElement);
+            }
+
+        } catch (final DOMException | IllegalStateException | IndexOutOfBoundsException
+                       | NumberFormatException e) {
+            throw CreationException.couldNotAddElement("segment (S)", e);
+        }
+    }
+
+    /**
+     * Get the duration of an OTF stream.
+     *
+     * <p>
+     * The duration of OTF streams is not returned into the player response and needs to be
+     * calculated by adding the duration of each segment.
+     * </p>
+     *
+     * @param segmentDuration the segment duration object extracted from the initialization
+     *                        sequence of the stream
+     * @return the duration of the OTF stream, in milliseconds
+     */
+    private static long getStreamDuration(@Nonnull final String[] segmentDuration)
+            throws CreationException {
+        try {
+            long streamLengthMs = 0;
+
+            for (final String segDuration : segmentDuration) {
+                final String[] segmentLengthRepeat = segDuration.split("\\(r=");
+                long segmentRepeatCount = 0;
+
+                // There are repetitions of a segment duration in other segments
+                if (segmentLengthRepeat.length > 1) {
+                    segmentRepeatCount = Long.parseLong(Utils.removeNonDigitCharacters(
+                            segmentLengthRepeat[1]));
+                }
+
+                final long segmentLength = Integer.parseInt(segmentLengthRepeat[0]);
+                streamLengthMs += segmentLength + segmentRepeatCount * segmentLength;
+            }
+
+            return streamLengthMs;
+        } catch (final NumberFormatException e) {
+            throw new CreationException("Could not get stream length from sequences list", e);
+        }
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubePostLiveStreamDvrDashManifestCreator.java

@@ -0,0 +1,217 @@
+package org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators;
+
+import org.schabi.newpipe.extractor.downloader.Response;
+import org.schabi.newpipe.extractor.services.youtube.DeliveryType;
+import org.schabi.newpipe.extractor.services.youtube.ItagItem;
+import org.schabi.newpipe.extractor.utils.ManifestCreatorCache;
+import org.w3c.dom.DOMException;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+
+import javax.annotation.Nonnull;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.ALR_YES;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.RN_0;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.SEGMENT_TIMELINE;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.SQ_0;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.buildAndCacheResult;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.generateDocumentAndDoCommonElementsGeneration;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.generateSegmentTemplateElement;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.generateSegmentTimelineElement;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.getInitializationResponse;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.setAttribute;
+import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
+import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
+
+/**
+ * Class which generates DASH manifests of YouTube post-live DVR streams (which use the
+ * {@link DeliveryType#LIVE LIVE delivery type}).
+ */
+public final class YoutubePostLiveStreamDvrDashManifestCreator {
+
+    /**
+     * Cache of DASH manifests generated for post-live-DVR streams.
+     */
+    private static final ManifestCreatorCache<String, String> POST_LIVE_DVR_STREAMS_CACHE
+            = new ManifestCreatorCache<>();
+
+    private YoutubePostLiveStreamDvrDashManifestCreator() {
+    }
+
+    /**
+     * Create DASH manifests from a YouTube post-live-DVR stream/ended livestream.
+     *
+     * <p>
+     * Post-live-DVR streams/ended livestreams are one of the YouTube DASH specific streams which
+     * works with sequences and without the need to get a manifest (even if one is provided but not
+     * used by main clients (and is not complete for big ended livestreams because it doesn't
+     * return the full stream)).
+     * </p>
+     *
+     * <p>
+     * They can be found only on livestreams which have ended very recently (a few hours, most of
+     * the time)
+     * </p>
+     *
+     * <p>This method needs:
+     *     <ul>
+     *         <li>the base URL of the stream (which, if you try to access to it, returns HTTP
+     *         status code 404 after redirects, and if the URL is valid);</li>
+     *         <li>an {@link ItagItem}, which needs to contain the following information:
+     *              <ul>
+     *                  <li>its type (see {@link ItagItem.ItagType}), to identify if the content is
+     *                  an audio or a video stream;</li>
+     *                  <li>its bitrate;</li>
+     *                  <li>its mime type;</li>
+     *                  <li>its codec(s);</li>
+     *                  <li>for an audio stream: its audio channels;</li>
+     *                  <li>for a video stream: its width and height.</li>
+     *              </ul>
+     *         </li>
+     *         <li>the duration of the video, which will be used if the duration could not be
+     *         parsed from the first sequence of the stream.</li>
+     *     </ul>
+     * </p>
+     *
+     * <p>In order to generate the DASH manifest, this method will:
+     *      <ul>
+     *          <li>request the first sequence of the stream (the base URL on which the first
+     *          sequence parameter is appended (see {@link YoutubeDashManifestCreatorsUtils#SQ_0}))
+     *          with a {@code POST} or {@code GET} request (depending of the client on which the
+     *          streaming URL comes from is a mobile one ({@code POST}) or not ({@code GET}));</li>
+     *          <li>follow its redirection(s), if any;</li>
+     *          <li>save the last URL, remove the first sequence parameters;</li>
+     *          <li>use the information provided in the {@link ItagItem} to generate all elements
+     *          of the DASH manifest.</li>
+     *      </ul>
+     * </p>
+     *
+     * <p>
+     * If the duration cannot be extracted, the {@code durationSecondsFallback} value will be used
+     * as the stream duration.
+     * </p>
+     *
+     * @param postLiveStreamDvrStreamingUrl the base URL of the post-live-DVR stream/ended
+     *                                      livestream, which must not be null
+     * @param itagItem                      the {@link ItagItem} corresponding to the stream, which
+     *                                      must not be null
+     * @param targetDurationSec             the target duration of each sequence, in seconds (this
+     *                                      value is returned with the {@code targetDurationSec}
+     *                                      field for each stream in YouTube's player response)
+     * @param durationSecondsFallback       the duration of the ended livestream, which will be
+     *                                      used if the duration could not be extracted from the
+     *                                      first sequence
+     * @return the manifest generated into a string
+     */
+    @Nonnull
+    public static String fromPostLiveStreamDvrStreamingUrl(
+            @Nonnull final String postLiveStreamDvrStreamingUrl,
+            @Nonnull final ItagItem itagItem,
+            final int targetDurationSec,
+            final long durationSecondsFallback) throws CreationException {
+        if (POST_LIVE_DVR_STREAMS_CACHE.containsKey(postLiveStreamDvrStreamingUrl)) {
+            return Objects.requireNonNull(
+                    POST_LIVE_DVR_STREAMS_CACHE.get(postLiveStreamDvrStreamingUrl)).getSecond();
+        }
+
+        String realPostLiveStreamDvrStreamingUrl = postLiveStreamDvrStreamingUrl;
+        final String streamDurationString;
+        final String segmentCount;
+
+        if (targetDurationSec <= 0) {
+            throw new CreationException("targetDurationSec value is <= 0: " + targetDurationSec);
+        }
+
+        try {
+            // Try to avoid redirects when streaming the content by saving the latest URL we get
+            // from video servers.
+            final Response response = getInitializationResponse(realPostLiveStreamDvrStreamingUrl,
+                    itagItem, DeliveryType.LIVE);
+            realPostLiveStreamDvrStreamingUrl = response.latestUrl().replace(SQ_0, EMPTY_STRING)
+                    .replace(RN_0, EMPTY_STRING).replace(ALR_YES, EMPTY_STRING);
+
+            final int responseCode = response.responseCode();
+            if (responseCode != 200) {
+                throw new CreationException(
+                        "Could not get the initialization sequence: response code " + responseCode);
+            }
+
+            final Map<String, List<String>> responseHeaders = response.responseHeaders();
+            streamDurationString = responseHeaders.get("X-Head-Time-Millis").get(0);
+            segmentCount = responseHeaders.get("X-Head-Seqnum").get(0);
+        } catch (final IndexOutOfBoundsException e) {
+            throw new CreationException(
+                    "Could not get the value of the X-Head-Time-Millis or the X-Head-Seqnum header",
+                    e);
+        }
+
+        if (isNullOrEmpty(segmentCount)) {
+            throw new CreationException("Could not get the number of segments");
+        }
+
+        long streamDuration;
+        try {
+            streamDuration = Long.parseLong(streamDurationString);
+        } catch (final NumberFormatException e) {
+            streamDuration = durationSecondsFallback;
+        }
+
+        final Document doc = generateDocumentAndDoCommonElementsGeneration(itagItem,
+                streamDuration);
+
+        generateSegmentTemplateElement(doc, realPostLiveStreamDvrStreamingUrl,
+                DeliveryType.LIVE);
+        generateSegmentTimelineElement(doc);
+        generateSegmentElementForPostLiveDvrStreams(doc, targetDurationSec, segmentCount);
+
+        return buildAndCacheResult(postLiveStreamDvrStreamingUrl, doc,
+                POST_LIVE_DVR_STREAMS_CACHE);
+    }
+
+    /**
+     * @return the cache of DASH manifests generated for post-live-DVR streams
+     */
+    @Nonnull
+    public static ManifestCreatorCache<String, String> getCache() {
+        return POST_LIVE_DVR_STREAMS_CACHE;
+    }
+
+    /**
+     * Generate the segment ({@code <S>}) element.
+     *
+     * <p>
+     * We don't know the exact duration of segments for post-live-DVR streams but an
+     * average instead (which is the {@code targetDurationSec} value), so we can use the following
+     * structure to generate the segment timeline for DASH manifests of ended livestreams:
+     * <br>
+     * {@code <S d="targetDurationSecValue" r="segmentCount" />}
+     * </p>
+     *
+     * @param doc                   the {@link Document} on which the {@code <S>} element will
+     *                              be appended
+     * @param targetDurationSeconds the {@code targetDurationSec} value from YouTube player
+     *                              response's stream
+     * @param segmentCount          the number of segments, extracted by {@link
+     *                              #fromPostLiveStreamDvrStreamingUrl(String, ItagItem, int, long)}
+     */
+    private static void generateSegmentElementForPostLiveDvrStreams(
+            @Nonnull final Document doc,
+            final int targetDurationSeconds,
+            @Nonnull final String segmentCount) throws CreationException {
+        try {
+            final Element segmentTimelineElement = (Element) doc.getElementsByTagName(
+                    SEGMENT_TIMELINE).item(0);
+            final Element sElement = doc.createElement("S");
+
+            setAttribute(sElement, doc, "d", String.valueOf(targetDurationSeconds * 1000));
+            setAttribute(sElement, doc, "r", segmentCount);
+
+            segmentTimelineElement.appendChild(sElement);
+        } catch (final DOMException e) {
+            throw CreationException.couldNotAddElement("segment (S)", e);
+        }
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeProgressiveDashManifestCreator.java

@@ -0,0 +1,235 @@
+package org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators;
+
+import org.schabi.newpipe.extractor.services.youtube.DeliveryType;
+import org.schabi.newpipe.extractor.services.youtube.ItagItem;
+import org.schabi.newpipe.extractor.utils.ManifestCreatorCache;
+import org.w3c.dom.DOMException;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+
+import javax.annotation.Nonnull;
+import java.util.Objects;
+
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.BASE_URL;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.INITIALIZATION;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.MPD;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.REPRESENTATION;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.SEGMENT_BASE;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.buildAndCacheResult;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.generateDocumentAndDoCommonElementsGeneration;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.setAttribute;
+
+/**
+ * Class which generates DASH manifests of {@link DeliveryType#PROGRESSIVE YouTube progressive}
+ * streams.
+ */
+public final class YoutubeProgressiveDashManifestCreator {
+
+    /**
+     * Cache of DASH manifests generated for progressive streams.
+     */
+    private static final ManifestCreatorCache<String, String> PROGRESSIVE_STREAMS_CACHE
+            = new ManifestCreatorCache<>();
+
+    private YoutubeProgressiveDashManifestCreator() {
+    }
+
+    /**
+     * Create DASH manifests from a YouTube progressive stream.
+     *
+     * <p>
+     * Progressive streams are YouTube DASH streams which work with range requests and without the
+     * need to get a manifest.
+     * </p>
+     *
+     * <p>
+     * They can be found on all videos, and for all streams for most of videos which come from a
+     * YouTube partner, and on videos with a large number of views.
+     * </p>
+     *
+     * <p>This method needs:
+     *     <ul>
+     *         <li>the base URL of the stream (which, if you try to access to it, returns the whole
+     *         stream, after redirects, and if the URL is valid);</li>
+     *         <li>an {@link ItagItem}, which needs to contain the following information:
+     *              <ul>
+     *                  <li>its type (see {@link ItagItem.ItagType}), to identify if the content is
+     *                  an audio or a video stream;</li>
+     *                  <li>its bitrate;</li>
+     *                  <li>its mime type;</li>
+     *                  <li>its codec(s);</li>
+     *                  <li>for an audio stream: its audio channels;</li>
+     *                  <li>for a video stream: its width and height.</li>
+     *              </ul>
+     *         </li>
+     *         <li>the duration of the video (parameter {@code durationSecondsFallback}), which
+     *         will be used as the stream duration if the duration could not be parsed from the
+     *         {@link ItagItem}.</li>
+     *     </ul>
+     * </p>
+     *
+     * @param progressiveStreamingBaseUrl the base URL of the progressive stream, which must not be
+     *                                    null
+     * @param itagItem                    the {@link ItagItem} corresponding to the stream, which
+     *                                    must not be null
+     * @param durationSecondsFallback     the duration of the progressive stream which will be used
+     *                                    if the duration could not be extracted from the
+     *                                    {@link ItagItem}
+     * @return the manifest generated into a string
+     */
+    @Nonnull
+    public static String fromProgressiveStreamingUrl(
+            @Nonnull final String progressiveStreamingBaseUrl,
+            @Nonnull final ItagItem itagItem,
+            final long durationSecondsFallback) throws CreationException {
+        if (PROGRESSIVE_STREAMS_CACHE.containsKey(progressiveStreamingBaseUrl)) {
+            return Objects.requireNonNull(
+                    PROGRESSIVE_STREAMS_CACHE.get(progressiveStreamingBaseUrl)).getSecond();
+        }
+
+        final long itagItemDuration = itagItem.getApproxDurationMs();
+        final long streamDuration;
+        if (itagItemDuration != -1) {
+            streamDuration = itagItemDuration;
+        } else {
+            if (durationSecondsFallback > 0) {
+                streamDuration = durationSecondsFallback * 1000;
+            } else {
+                throw CreationException.couldNotAddElement(MPD, "the duration of the stream "
+                        + "could not be determined and durationSecondsFallback is <= 0");
+            }
+        }
+
+        final Document doc = generateDocumentAndDoCommonElementsGeneration(itagItem,
+                streamDuration);
+
+        generateBaseUrlElement(doc, progressiveStreamingBaseUrl);
+        generateSegmentBaseElement(doc, itagItem);
+        generateInitializationElement(doc, itagItem);
+
+        return buildAndCacheResult(progressiveStreamingBaseUrl, doc,
+                PROGRESSIVE_STREAMS_CACHE);
+    }
+
+    /**
+     * @return the cache of DASH manifests generated for progressive streams
+     */
+    @Nonnull
+    public static ManifestCreatorCache<String, String> getCache() {
+        return PROGRESSIVE_STREAMS_CACHE;
+    }
+
+    /**
+     * Generate the {@code <BaseURL>} element, appended as a child of the
+     * {@code <Representation>} element.
+     *
+     * <p>
+     * The {@code <Representation>} element needs to be generated before this element with
+     * {@link YoutubeDashManifestCreatorsUtils#generateRepresentationElement(Document, ItagItem)}).
+     * </p>
+     *
+     * @param doc the {@link Document} on which the {@code <BaseURL>} element will be appended
+     * @param baseUrl  the base URL of the stream, which must not be null and will be set as the
+     *                 content of the {@code <BaseURL>} element
+     */
+    private static void generateBaseUrlElement(@Nonnull final Document doc,
+                                               @Nonnull final String baseUrl)
+            throws CreationException {
+        try {
+            final Element representationElement = (Element) doc.getElementsByTagName(
+                    REPRESENTATION).item(0);
+            final Element baseURLElement = doc.createElement(BASE_URL);
+            baseURLElement.setTextContent(baseUrl);
+            representationElement.appendChild(baseURLElement);
+        } catch (final DOMException e) {
+            throw CreationException.couldNotAddElement(BASE_URL, e);
+        }
+    }
+
+    /**
+     * Generate the {@code <SegmentBase>} element, appended as a child of the
+     * {@code <Representation>} element.
+     *
+     * <p>
+     * It generates the following element:
+     * <br>
+     * {@code <SegmentBase indexRange="indexStart-indexEnd" />}
+     * <br>
+     * (where {@code indexStart} and {@code indexEnd} are gotten from the {@link ItagItem} passed
+     * as the second parameter)
+     * </p>
+     *
+     * <p>
+     * The {@code <Representation>} element needs to be generated before this element with
+     * {@link YoutubeDashManifestCreatorsUtils#generateRepresentationElement(Document, ItagItem)}),
+     * and the {@code BaseURL} element with {@link #generateBaseUrlElement(Document, String)}
+     * should be generated too.
+     * </p>
+     *
+     * @param doc the {@link Document} on which the {@code <SegmentBase>} element will be appended
+     * @param itagItem the {@link ItagItem} to use, which must not be null
+     */
+    private static void generateSegmentBaseElement(@Nonnull final Document doc,
+                                                   @Nonnull final ItagItem itagItem)
+            throws CreationException {
+        try {
+            final Element representationElement = (Element) doc.getElementsByTagName(
+                    REPRESENTATION).item(0);
+            final Element segmentBaseElement = doc.createElement(SEGMENT_BASE);
+
+            final String range = itagItem.getIndexStart() + "-" + itagItem.getIndexEnd();
+            if (itagItem.getIndexStart() < 0 || itagItem.getIndexEnd() < 0) {
+                throw CreationException.couldNotAddElement(SEGMENT_BASE,
+                        "ItagItem's indexStart or " + "indexEnd are < 0: " + range);
+            }
+            setAttribute(segmentBaseElement, doc, "indexRange", range);
+
+            representationElement.appendChild(segmentBaseElement);
+        } catch (final DOMException e) {
+            throw CreationException.couldNotAddElement(SEGMENT_BASE, e);
+        }
+    }
+
+    /**
+     * Generate the {@code <Initialization>} element, appended as a child of the
+     * {@code <SegmentBase>} element.
+     *
+     * <p>
+     * It generates the following element:
+     * <br>
+     * {@code <Initialization range="initStart-initEnd"/>}
+     * <br>
+     * (where {@code indexStart} and {@code indexEnd} are gotten from the {@link ItagItem} passed
+     * as the second parameter)
+     * </p>
+     *
+     * <p>
+     * The {@code <SegmentBase>} element needs to be generated before this element with
+     * {@link #generateSegmentBaseElement(Document, ItagItem)}).
+     * </p>
+     *
+     * @param doc the {@link Document} on which the {@code <Initialization>} element will be
+     *            appended
+     * @param itagItem the {@link ItagItem} to use, which must not be null
+     */
+    private static void generateInitializationElement(@Nonnull final Document doc,
+                                                      @Nonnull final ItagItem itagItem)
+            throws CreationException {
+        try {
+            final Element segmentBaseElement = (Element) doc.getElementsByTagName(
+                    SEGMENT_BASE).item(0);
+            final Element initializationElement = doc.createElement(INITIALIZATION);
+
+            final String range = itagItem.getInitStart() + "-" + itagItem.getInitEnd();
+            if (itagItem.getInitStart() < 0 || itagItem.getInitEnd() < 0) {
+                throw CreationException.couldNotAddElement(INITIALIZATION,
+                        "ItagItem's initStart and/or " + "initEnd are/is < 0: " + range);
+            }
+            setAttribute(initializationElement, doc, "range", range);
+
+            segmentBaseElement.appendChild(initializationElement);
+        } catch (final DOMException e) {
+            throw CreationException.couldNotAddElement(INITIALIZATION, e);
+        }
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/ItagInfo.java

@@ -0,0 +1,80 @@
+package org.schabi.newpipe.extractor.services.youtube.extractors;
+
+import org.schabi.newpipe.extractor.services.youtube.ItagItem;
+
+import javax.annotation.Nonnull;
+import java.io.Serializable;
+
+/**
+ * Class to build easier {@link org.schabi.newpipe.extractor.stream.Stream}s for
+ * {@link YoutubeStreamExtractor}.
+ *
+ * <p>
+ * It stores, per stream:
+ * <ul>
+ *     <li>its content (the URL/the base URL of streams);</li>
+ *     <li>whether its content is the URL the content itself or the base URL;</li>
+ *     <li>its associated {@link ItagItem}.</li>
+ * </ul>
+ * </p>
+ */
+final class ItagInfo implements Serializable {
+    @Nonnull
+    private final String content;
+    @Nonnull
+    private final ItagItem itagItem;
+    private boolean isUrl;
+
+    /**
+     * Creates a new {@code ItagInfo} instance.
+     *
+     * @param content  the content of the stream, which must be not null
+     * @param itagItem the {@link ItagItem} associated with the stream, which must be not null
+     */
+    ItagInfo(@Nonnull final String content,
+             @Nonnull final ItagItem itagItem) {
+        this.content = content;
+        this.itagItem = itagItem;
+    }
+
+    /**
+     * Sets whether the stream is a URL.
+     *
+     * @param isUrl whether the content is a URL
+     */
+    void setIsUrl(final boolean isUrl) {
+        this.isUrl = isUrl;
+    }
+
+    /**
+     * Gets the content stored in this {@code ItagInfo} instance, which is either the URL to the
+     * content itself or the base URL.
+     *
+     * @return the content stored in this {@code ItagInfo} instance
+     */
+    @Nonnull
+    String getContent() {
+        return content;
+    }
+
+    /**
+     * Gets the {@link ItagItem} associated with this {@code ItagInfo} instance.
+     *
+     * @return the {@link ItagItem} associated with this {@code ItagInfo} instance, which is not
+     * null
+     */
+    @Nonnull
+    ItagItem getItagItem() {
+        return itagItem;
+    }
+
+    /**
+     * Gets whether the content stored is the URL to the content itself or the base URL of it.
+     *
+     * @return whether the content stored is the URL to the content itself or the base URL of it
+     * @see #getContent() for more details
+     */
+    boolean getIsUrl() {
+        return isUrl;
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java

@@ -1,10 +1,32 @@
+/*
+ * Created by Christian Schabesberger on 06.08.15.
+ *
+ * Copyright (C) Christian Schabesberger 2019 <chris.schabesberger@mailbox.org>
+ * YoutubeStreamExtractor.java is part of NewPipe Extractor.
+ *
+ * NewPipe Extractor is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe Extractor is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe Extractor. If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package org.schabi.newpipe.extractor.services.youtube.extractors;
 
+import static org.schabi.newpipe.extractor.services.youtube.ItagItem.APPROX_DURATION_MS_UNKNOWN;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.CONTENT_CHECK_OK;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.CPN;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.RACY_CHECK_OK;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.VIDEO_ID;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.createDesktopPlayerBody;
+import static org.schabi.newpipe.extractor.services.youtube.ItagItem.CONTENT_LENGTH_UNKNOWN;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.generateContentPlaybackNonce;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.generateTParameter;
@@ -50,6 +72,7 @@ import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
 import org.schabi.newpipe.extractor.services.youtube.YoutubeThrottlingDecrypter;
 import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
 import org.schabi.newpipe.extractor.stream.AudioStream;
+import org.schabi.newpipe.extractor.stream.DeliveryMethod;
 import org.schabi.newpipe.extractor.stream.Description;
 import org.schabi.newpipe.extractor.stream.Frameset;
 import org.schabi.newpipe.extractor.stream.Stream;
@@ -64,7 +87,6 @@ import org.schabi.newpipe.extractor.utils.Parser;
 import org.schabi.newpipe.extractor.utils.Utils;
 
 import java.io.IOException;
-import java.io.UnsupportedEncodingException;
 import java.nio.charset.StandardCharsets;
 import java.time.LocalDate;
 import java.time.OffsetDateTime;
@@ -72,7 +94,6 @@ import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
-import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
@@ -82,26 +103,6 @@ import java.util.stream.Collectors;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
-/*
- * Created by Christian Schabesberger on 06.08.15.
- *
- * Copyright (C) Christian Schabesberger 2019 <chris.schabesberger@mailbox.org>
- * YoutubeStreamExtractor.java is part of NewPipe.
- *
- * NewPipe is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * NewPipe is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
- */
-
 public class YoutubeStreamExtractor extends StreamExtractor {
     /*//////////////////////////////////////////////////////////////////////////
     // Exceptions
@@ -113,7 +114,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
         }
     }
 
-    /*//////////////////////////////////////////////////////////////////////////*/
+    /*////////////////////////////////////////////////////////////////////////*/
 
     @Nullable
     private static String cachedDeobfuscationCode = null;
@@ -140,8 +141,6 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     private JsonObject playerMicroFormatRenderer;
     private int ageLimit = -1;
     private StreamType streamType;
-    @Nullable
-    private List<SubtitlesStream> subtitles = null;
 
     // We need to store the contentPlaybackNonces because we need to append them to videoplayback
     // URLs (with the cpn parameter).
@@ -580,73 +579,25 @@ public class YoutubeStreamExtractor extends StreamExtractor {
                 .orElse(EMPTY_STRING);
     }
 
-    @FunctionalInterface
-    interface StreamTypeStreamBuilderHelper<T extends Stream> {
-        T buildStream(String url, ItagItem itagItem);
-    }
-
-    /**
-     * Abstract method for
-     * {@link #getAudioStreams()}, {@link #getVideoOnlyStreams()} and {@link #getVideoStreams()}.
-     *
-     * @param itags           A map of Urls + ItagItems
-     * @param streamBuilder   Builds the stream from the provided data
-     * @param exMsgStreamType Stream type inside the exception message e.g. "video streams"
-     * @param <T>             Type of the stream
-     * @return
-     * @throws ExtractionException
-     */
-    private <T extends Stream> List<T> getStreamsByType(
-            final Map<String, ItagItem> itags,
-            final StreamTypeStreamBuilderHelper<T> streamBuilder,
-            final String exMsgStreamType
-    ) throws ExtractionException {
-        final List<T> streams = new ArrayList<>();
-
-        try {
-            for (final Map.Entry<String, ItagItem> entry : itags.entrySet()) {
-                final String url = tryDecryptUrl(entry.getKey(), getId());
-
-                final T stream = streamBuilder.buildStream(url, entry.getValue());
-                if (!Stream.containSimilarStream(stream, streams)) {
-                    streams.add(stream);
-                }
-            }
-        } catch (final Exception e) {
-            throw new ParsingException("Could not get " + exMsgStreamType, e);
-        }
-
-        return streams;
-    }
-
     @Override
     public List<AudioStream> getAudioStreams() throws ExtractionException {
         assertPageFetched();
-        return getStreamsByType(
+        return getItags(ADAPTIVE_FORMATS, ItagItem.ItagType.AUDIO,
-                getItags(ADAPTIVE_FORMATS, ItagItem.ItagType.AUDIO),
+                getAudioStreamBuilderHelper(), "audio");
-                AudioStream::new,
-                "audio streams"
-        );
     }
 
     @Override
     public List<VideoStream> getVideoStreams() throws ExtractionException {
         assertPageFetched();
-        return getStreamsByType(
+        return getItags(FORMATS, ItagItem.ItagType.VIDEO,
-                getItags(FORMATS, ItagItem.ItagType.VIDEO),
+                getVideoStreamBuilderHelper(false), "video");
-                (url, itag) -> new VideoStream(url, false, itag),
-                "video streams"
-        );
     }
 
     @Override
     public List<VideoStream> getVideoOnlyStreams() throws ExtractionException {
         assertPageFetched();
-        return getStreamsByType(
+        return getItags(ADAPTIVE_FORMATS, ItagItem.ItagType.VIDEO_ONLY,
-                getItags(ADAPTIVE_FORMATS, ItagItem.ItagType.VIDEO_ONLY),
+                getVideoStreamBuilderHelper(true), "video-only");
-                (url, itag) -> new VideoStream(url, true, itag),
-                "video only streams"
-        );
     }
 
     /**
@@ -672,18 +623,15 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     @Nonnull
     public List<SubtitlesStream> getSubtitles(final MediaFormat format) throws ParsingException {
         assertPageFetched();
-        if (subtitles != null) {
-            // Already calculated
-            return subtitles;
-        }
 
+        // We cannot store the subtitles list because the media format may change
+        final List<SubtitlesStream> subtitlesToReturn = new ArrayList<>();
         final JsonObject renderer = playerResponse.getObject("captions")
                 .getObject("playerCaptionsTracklistRenderer");
         final JsonArray captionsArray = renderer.getArray("captionTracks");
         // TODO: use this to apply auto translation to different language from a source language
         // final JsonArray autoCaptionsArray = renderer.getArray("translationLanguages");
 
-        subtitles = new ArrayList<>();
         for (int i = 0; i < captionsArray.size(); i++) {
             final String languageCode = captionsArray.getObject(i).getString("languageCode");
             final String baseUrl = captionsArray.getObject(i).getString("baseUrl");
@@ -692,15 +640,21 @@ public class YoutubeStreamExtractor extends StreamExtractor {
             if (languageCode != null && baseUrl != null && vssId != null) {
                 final boolean isAutoGenerated = vssId.startsWith("a.");
                 final String cleanUrl = baseUrl
-                        .replaceAll("&fmt=[^&]*", "") // Remove preexisting format if exists
+                        // Remove preexisting format if exists
-                        .replaceAll("&tlang=[^&]*", ""); // Remove translation language
+                        .replaceAll("&fmt=[^&]*", "")
+                        // Remove translation language
+                        .replaceAll("&tlang=[^&]*", "");
 
-                subtitles.add(new SubtitlesStream(format, languageCode,
+                subtitlesToReturn.add(new SubtitlesStream.Builder()
-                        cleanUrl + "&fmt=" + format.getSuffix(), isAutoGenerated));
+                        .setContent(cleanUrl + "&fmt=" + format.getSuffix(), true)
+                        .setMediaFormat(format)
+                        .setLanguageCode(languageCode)
+                        .setAutoGenerated(isAutoGenerated)
+                        .build());
             }
         }
 
-        return subtitles;
+        return subtitlesToReturn;
     }
 
     @Override
@@ -711,9 +665,10 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     }
 
     private void setStreamType() {
-        if (playerResponse.getObject("playabilityStatus").has("liveStreamability")
+        if (playerResponse.getObject("playabilityStatus").has("liveStreamability")) {
-                || playerResponse.getObject("videoDetails").getBoolean("isPostLiveDvr", false)) {
             streamType = StreamType.LIVE_STREAM;
+        } else if (playerResponse.getObject("videoDetails").getBoolean("isPostLiveDvr", false)) {
+            streamType = StreamType.POST_LIVE_STREAM;
         } else {
             streamType = StreamType.VIDEO_STREAM;
         }
@@ -788,6 +743,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     private static final String STREAMING_DATA = "streamingData";
     private static final String PLAYER = "player";
     private static final String NEXT = "next";
+    private static final String SIGNATURE_CIPHER = "signatureCipher";
+    private static final String CIPHER = "cipher";
 
     private static final String[] REGEXES = {
             "(?:\\b|[^a-zA-Z0-9$])([a-zA-Z0-9$]{2,})\\s*=\\s*function\\(\\s*a\\s*\\)"
@@ -827,7 +784,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
 
         final JsonObject playabilityStatus = playerResponse.getObject("playabilityStatus");
 
-        final boolean ageRestricted = playabilityStatus.getString("reason", EMPTY_STRING)
+        final boolean isAgeRestricted = playabilityStatus.getString("reason", EMPTY_STRING)
                 .contains("age");
 
         setStreamType();
@@ -837,12 +794,12 @@ public class YoutubeStreamExtractor extends StreamExtractor {
                 fetchTvHtml5EmbedJsonPlayer(contentCountry, localization, videoId);
             } catch (final Exception ignored) {
             }
-
-            // Refresh the stream type because the stream type may be not properly known for
-            // age-restricted videos
-            setStreamType();
         }
 
+        // Refresh the stream type because the stream type may be not properly known for
+        // age-restricted videos
+        setStreamType();
+
         if (html5StreamingData == null && playerResponse.has(STREAMING_DATA)) {
             html5StreamingData = playerResponse.getObject(STREAMING_DATA);
         }
@@ -866,7 +823,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
                 .getBytes(StandardCharsets.UTF_8);
         nextResponse = getJsonPostResponse(NEXT, body, localization);
 
-        if ((!ageRestricted && streamType == StreamType.VIDEO_STREAM)
+        if ((!isAgeRestricted && streamType == StreamType.VIDEO_STREAM)
                 || isAndroidClientFetchForced) {
             try {
                 fetchAndroidMobileJsonPlayer(contentCountry, localization, videoId);
@@ -874,7 +831,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
             }
         }
 
-        if ((!ageRestricted && streamType == StreamType.LIVE_STREAM)
+        if ((!isAgeRestricted && streamType == StreamType.LIVE_STREAM)
                 || isIosClientFetchForced) {
             try {
                 fetchIosMobileJsonPlayer(contentCountry, localization, videoId);
@@ -1184,103 +1141,254 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     }
 
     @Nonnull
-    private Map<String, ItagItem> getItags(@Nonnull final String streamingDataKey,
+    private <T extends Stream> List<T> getItags(
-                                           @Nonnull final ItagItem.ItagType itagTypeWanted) {
+            final String streamingDataKey,
-        final Map<String, ItagItem> urlAndItags = new LinkedHashMap<>();
+            final ItagItem.ItagType itagTypeWanted,
-        if (html5StreamingData == null && androidStreamingData == null
+            final java.util.function.Function<ItagInfo, T> streamBuilderHelper,
-                && iosStreamingData == null) {
+            final String streamTypeExceptionMessage) throws ParsingException {
-            return urlAndItags;
+        try {
+            final String videoId = getId();
+            final List<T> streamList = new ArrayList<>();
+
+            java.util.stream.Stream.of(
+                    // Use the androidStreamingData object first because there is no n param and no
+                    // signatureCiphers in streaming URLs of the Android client
+                    new Pair<>(androidStreamingData, androidCpn),
+                    new Pair<>(html5StreamingData, html5Cpn),
+                    // Use the iosStreamingData object in the last position because most of the
+                    // available streams can be extracted with the Android and web clients and also
+                    // because the iOS client is only enabled by default on livestreams
+                    new Pair<>(iosStreamingData, iosCpn)
+            )
+                    .flatMap(pair -> getStreamsFromStreamingDataKey(videoId, pair.getFirst(),
+                            streamingDataKey, itagTypeWanted, pair.getSecond()))
+                    .map(streamBuilderHelper)
+                    .forEachOrdered(stream -> {
+                        if (!Stream.containSimilarStream(stream, streamList)) {
+                            streamList.add(stream);
+                        }
+                    });
+
+            return streamList;
+        } catch (final Exception e) {
+            throw new ParsingException(
+                    "Could not get " + streamTypeExceptionMessage + " streams", e);
         }
+    }
 
-        final List<Pair<JsonObject, String>> streamingDataAndCpnLoopList = new ArrayList<>();
+    /**
-        // Use the androidStreamingData object first because there is no n param and no
+     * Get the stream builder helper which will be used to build {@link AudioStream}s in
-        // signatureCiphers in streaming URLs of the Android client
+     * {@link #getItags(String, ItagItem.ItagType, java.util.function.Function, String)}
-        streamingDataAndCpnLoopList.add(new Pair<>(androidStreamingData, androidCpn));
+     *
-        streamingDataAndCpnLoopList.add(new Pair<>(html5StreamingData, html5Cpn));
+     * <p>
-        // Use the iosStreamingData object in the last position because most of the available
+     * The {@code StreamBuilderHelper} will set the following attributes in the
-        // streams can be extracted with the Android and web clients and also because the iOS
+     * {@link AudioStream}s built:
-        // client is only enabled by default on livestreams
+     * <ul>
-        streamingDataAndCpnLoopList.add(new Pair<>(iosStreamingData, iosCpn));
+     *     <li>the {@link ItagItem}'s id of the stream as its id;</li>
+     *     <li>{@link ItagInfo#getContent()} and {@link ItagInfo#getIsUrl()} as its content and
+     *     and as the value of {@code isUrl};</li>
+     *     <li>the media format returned by the {@link ItagItem} as its media format;</li>
+     *     <li>its average bitrate with the value returned by {@link
+     *     ItagItem#getAverageBitrate()};</li>
+     *     <li>the {@link ItagItem};</li>
+     *     <li>the {@link DeliveryMethod#DASH DASH delivery method}, for OTF streams, live streams
+     *     and ended streams.</li>
+     * </ul>
+     * </p>
+     *
+     * <p>
+     * Note that the {@link ItagItem} comes from an {@link ItagInfo} instance.
+     * </p>
+     *
+     * @return a stream builder helper to build {@link AudioStream}s
+     */
+    @Nonnull
+    private java.util.function.Function<ItagInfo, AudioStream> getAudioStreamBuilderHelper() {
+        return (itagInfo) -> {
+            final ItagItem itagItem = itagInfo.getItagItem();
+            final AudioStream.Builder builder = new AudioStream.Builder()
+                    .setId(String.valueOf(itagItem.id))
+                    .setContent(itagInfo.getContent(), itagInfo.getIsUrl())
+                    .setMediaFormat(itagItem.getMediaFormat())
+                    .setAverageBitrate(itagItem.getAverageBitrate())
+                    .setItagItem(itagItem);
 
-        for (final Pair<JsonObject, String> pair : streamingDataAndCpnLoopList) {
+            if (streamType == StreamType.LIVE_STREAM
-            urlAndItags.putAll(getStreamsFromStreamingDataKey(pair.getFirst(), streamingDataKey,
+                    || streamType == StreamType.POST_LIVE_STREAM
-                    itagTypeWanted, pair.getSecond()));
+                    || !itagInfo.getIsUrl()) {
-        }
+                // For YouTube videos on OTF streams and for all streams of post-live streams
+                // and live streams, only the DASH delivery method can be used.
+                builder.setDeliveryMethod(DeliveryMethod.DASH);
+            }
 
-        return urlAndItags;
+            return builder.build();
+        };
+    }
+
+    /**
+     * Get the stream builder helper which will be used to build {@link VideoStream}s in
+     * {@link #getItags(String, ItagItem.ItagType, java.util.function.Function, String)}
+     *
+     * <p>
+     * The {@code StreamBuilderHelper} will set the following attributes in the
+     * {@link VideoStream}s built:
+     * <ul>
+     *     <li>the {@link ItagItem}'s id of the stream as its id;</li>
+     *     <li>{@link ItagInfo#getContent()} and {@link ItagInfo#getIsUrl()} as its content and
+     *     and as the value of {@code isUrl};</li>
+     *     <li>the media format returned by the {@link ItagItem} as its media format;</li>
+     *     <li>whether it is video-only with the {@code areStreamsVideoOnly} parameter</li>
+     *     <li>the {@link ItagItem};</li>
+     *     <li>the resolution, by trying to use, in this order:
+     *         <ol>
+     *             <li>the height returned by the {@link ItagItem} + {@code p} + the frame rate if
+     *             it is more than 30;</li>
+     *             <li>the default resolution string from the {@link ItagItem};</li>
+     *             <li>an {@link Utils#EMPTY_STRING empty string}.</li>
+     *         </ol>
+     *     </li>
+     *     <li>the {@link DeliveryMethod#DASH DASH delivery method}, for OTF streams, live streams
+     *     and ended streams.</li>
+     * </ul>
+     *
+     * <p>
+     * Note that the {@link ItagItem} comes from an {@link ItagInfo} instance.
+     * </p>
+     *
+     * @param areStreamsVideoOnly whether the stream builder helper will set the video
+     *                            streams as video-only streams
+     * @return a stream builder helper to build {@link VideoStream}s
+     */
+    @Nonnull
+    private java.util.function.Function<ItagInfo, VideoStream> getVideoStreamBuilderHelper(
+            final boolean areStreamsVideoOnly) {
+        return (itagInfo) -> {
+            final ItagItem itagItem = itagInfo.getItagItem();
+            final VideoStream.Builder builder = new VideoStream.Builder()
+                    .setId(String.valueOf(itagItem.id))
+                    .setContent(itagInfo.getContent(), itagInfo.getIsUrl())
+                    .setMediaFormat(itagItem.getMediaFormat())
+                    .setIsVideoOnly(areStreamsVideoOnly)
+                    .setItagItem(itagItem);
+
+            final String resolutionString = itagItem.getResolutionString();
+            builder.setResolution(resolutionString != null ? resolutionString
+                    : EMPTY_STRING);
+
+            if (streamType != StreamType.VIDEO_STREAM || !itagInfo.getIsUrl()) {
+                // For YouTube videos on OTF streams and for all streams of post-live streams
+                // and live streams, only the DASH delivery method can be used.
+                builder.setDeliveryMethod(DeliveryMethod.DASH);
+            }
+
+            return builder.build();
+        };
     }
 
     @Nonnull
-    private Map<String, ItagItem> getStreamsFromStreamingDataKey(
+    private java.util.stream.Stream<ItagInfo> getStreamsFromStreamingDataKey(
+            final String videoId,
             final JsonObject streamingData,
-            @Nonnull final String streamingDataKey,
+            final String streamingDataKey,
             @Nonnull final ItagItem.ItagType itagTypeWanted,
             @Nonnull final String contentPlaybackNonce) {
         if (streamingData == null || !streamingData.has(streamingDataKey)) {
-            return Collections.emptyMap();
+            return java.util.stream.Stream.empty();
         }
 
-        final Map<String, ItagItem> urlAndItagsFromStreamingDataObject = new LinkedHashMap<>();
+        return streamingData.getArray(streamingDataKey).stream()
-        final JsonArray formats = streamingData.getArray(streamingDataKey);
+                .filter(JsonObject.class::isInstance)
-        for (int i = 0; i < formats.size(); i++) {
+                .map(JsonObject.class::cast)
-            final JsonObject formatData = formats.getObject(i);
+                .map(formatData -> {
-            final int itag = formatData.getInt("itag");
+                    try {
-
+                        final ItagItem itagItem = ItagItem.getItag(formatData.getInt("itag"));
-            if (!ItagItem.isSupported(itag)) {
+                        if (itagItem.itagType == itagTypeWanted) {
-                continue;
+                            return buildAndAddItagInfoToList(videoId, formatData, itagItem,
-            }
+                                    itagItem.itagType, contentPlaybackNonce);
-
+                        }
-            try {
+                    } catch (final IOException | ExtractionException ignored) {
-                final ItagItem itagItem = ItagItem.getItag(itag);
+                        // if the itag is not supported and getItag fails, we end up here
-                if (itagItem.itagType != itagTypeWanted) {
+                    }
-                    continue;
+                    return null;
-                }
+                })
-
+                .filter(Objects::nonNull);
-                // Ignore streams that are delivered using YouTube's OTF format,
-                // as those only work with DASH and not with progressive HTTP.
-                if ("FORMAT_STREAM_TYPE_OTF".equalsIgnoreCase(formatData.getString("type"))) {
-                    continue;
-                }
-
-                final String streamUrl;
-                if (formatData.has("url")) {
-                    streamUrl = formatData.getString("url");
-                } else {
-                    // This url has an obfuscated signature
-                    final String cipherString = formatData.has("cipher")
-                            ? formatData.getString("cipher")
-                            : formatData.getString("signatureCipher");
-                    final Map<String, String> cipher = Parser.compatParseMap(
-                            cipherString);
-                    streamUrl = cipher.get("url") + "&" + cipher.get("sp") + "="
-                            + deobfuscateSignature(cipher.get("s"));
-                }
-
-                final JsonObject initRange = formatData.getObject("initRange");
-                final JsonObject indexRange = formatData.getObject("indexRange");
-                final String mimeType = formatData.getString("mimeType", EMPTY_STRING);
-                final String codec = mimeType.contains("codecs")
-                        ? mimeType.split("\"")[1]
-                        : EMPTY_STRING;
-
-                itagItem.setBitrate(formatData.getInt("bitrate"));
-                itagItem.setWidth(formatData.getInt("width"));
-                itagItem.setHeight(formatData.getInt("height"));
-                itagItem.setInitStart(Integer.parseInt(initRange.getString("start", "-1")));
-                itagItem.setInitEnd(Integer.parseInt(initRange.getString("end", "-1")));
-                itagItem.setIndexStart(Integer.parseInt(indexRange.getString("start", "-1")));
-                itagItem.setIndexEnd(Integer.parseInt(indexRange.getString("end", "-1")));
-                itagItem.fps = formatData.getInt("fps");
-                itagItem.setQuality(formatData.getString("quality"));
-                itagItem.setCodec(codec);
-
-                urlAndItagsFromStreamingDataObject.put(streamUrl, itagItem);
-            } catch (final UnsupportedEncodingException | ParsingException ignored) {
-            }
-        }
-        return urlAndItagsFromStreamingDataObject;
     }
 
+    private ItagInfo buildAndAddItagInfoToList(
+            @Nonnull final String videoId,
+            @Nonnull final JsonObject formatData,
+            @Nonnull final ItagItem itagItem,
+            @Nonnull final ItagItem.ItagType itagType,
+            @Nonnull final String contentPlaybackNonce) throws IOException, ExtractionException {
+        String streamUrl;
+        if (formatData.has("url")) {
+            streamUrl = formatData.getString("url");
+        } else {
+            // This url has an obfuscated signature
+            final String cipherString = formatData.has(CIPHER)
+                    ? formatData.getString(CIPHER)
+                    : formatData.getString(SIGNATURE_CIPHER);
+            final Map<String, String> cipher = Parser.compatParseMap(
+                    cipherString);
+            streamUrl = cipher.get("url") + "&" + cipher.get("sp") + "="
+                    + deobfuscateSignature(cipher.get("s"));
+        }
+
+        // Add the content playback nonce to the stream URL
+        streamUrl += "&" + CPN + "=" + contentPlaybackNonce;
+
+        // Decrypt the n parameter if it is present
+        streamUrl = tryDecryptUrl(streamUrl, videoId);
+
+        final JsonObject initRange = formatData.getObject("initRange");
+        final JsonObject indexRange = formatData.getObject("indexRange");
+        final String mimeType = formatData.getString("mimeType", EMPTY_STRING);
+        final String codec = mimeType.contains("codecs")
+                ? mimeType.split("\"")[1] : EMPTY_STRING;
+
+        itagItem.setBitrate(formatData.getInt("bitrate"));
+        itagItem.setWidth(formatData.getInt("width"));
+        itagItem.setHeight(formatData.getInt("height"));
+        itagItem.setInitStart(Integer.parseInt(initRange.getString("start", "-1")));
+        itagItem.setInitEnd(Integer.parseInt(initRange.getString("end", "-1")));
+        itagItem.setIndexStart(Integer.parseInt(indexRange.getString("start", "-1")));
+        itagItem.setIndexEnd(Integer.parseInt(indexRange.getString("end", "-1")));
+        itagItem.setQuality(formatData.getString("quality"));
+        itagItem.setCodec(codec);
+
+        if (streamType == StreamType.LIVE_STREAM || streamType == StreamType.POST_LIVE_STREAM) {
+            itagItem.setTargetDurationSec(formatData.getInt("targetDurationSec"));
+        } else if (itagType == ItagItem.ItagType.VIDEO
+                || itagType == ItagItem.ItagType.VIDEO_ONLY) {
+            itagItem.setFps(formatData.getInt("fps"));
+        } else if (itagType == ItagItem.ItagType.AUDIO) {
+            // YouTube return the audio sample rate as a string
+            itagItem.setSampleRate(Integer.parseInt(formatData.getString("audioSampleRate")));
+            itagItem.setAudioChannels(formatData.getInt("audioChannels"));
+        }
+
+        // YouTube return the content length and the approximate duration as strings
+        itagItem.setContentLength(Long.parseLong(formatData.getString("contentLength",
+                String.valueOf(CONTENT_LENGTH_UNKNOWN))));
+        itagItem.setApproxDurationMs(Long.parseLong(formatData.getString("approxDurationMs",
+                String.valueOf(APPROX_DURATION_MS_UNKNOWN))));
+
+        final ItagInfo itagInfo = new ItagInfo(streamUrl, itagItem);
+
+        if (streamType == StreamType.VIDEO_STREAM) {
+            itagInfo.setIsUrl(!formatData.getString("type", EMPTY_STRING)
+                    .equalsIgnoreCase("FORMAT_STREAM_TYPE_OTF"));
+        } else {
+            // We are currently not able to generate DASH manifests for running
+            // livestreams, so because of the requirements of StreamInfo
+            // objects, return these streams as DASH URL streams (even if they
+            // are not playable).
+            // Ended livestreams are returned as non URL streams
+            itagInfo.setIsUrl(streamType != StreamType.POST_LIVE_STREAM);
+        }
+
+        return itagInfo;
+    }
 
     @Nonnull
     @Override

extractor/src/main/java/org/schabi/newpipe/extractor/stream/AudioStream.java

@@ -4,30 +4,35 @@ package org.schabi.newpipe.extractor.stream;
  * Created by Christian Schabesberger on 04.03.16.
  *
  * Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
- * AudioStream.java is part of NewPipe.
+ * AudioStream.java is part of NewPipe Extractor.
  *
- * NewPipe is free software: you can redistribute it and/or modify
+ * NewPipe Extractor is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
  * the Free Software Foundation, either version 3 of the License, or
  * (at your option) any later version.
  *
- * NewPipe is distributed in the hope that it will be useful,
+ * NewPipe Extractor is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
- * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ * along with NewPipe Extractor. If not, see <https://www.gnu.org/licenses/>.
  */
 
 import org.schabi.newpipe.extractor.MediaFormat;
 import org.schabi.newpipe.extractor.services.youtube.ItagItem;
 
-public class AudioStream extends Stream {
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+public final class AudioStream extends Stream {
+    public static final int UNKNOWN_BITRATE = -1;
+
     private final int averageBitrate;
 
-    // Fields for Dash
+    // Fields for DASH
-    private int itag;
+    private int itag = ITAG_NOT_AVAILABLE_OR_NOT_APPLICABLE;
     private int bitrate;
     private int initStart;
     private int initEnd;
@@ -35,37 +40,241 @@ public class AudioStream extends Stream {
     private int indexEnd;
     private String quality;
     private String codec;
+    @Nullable
+    private ItagItem itagItem;
 
     /**
-     * Create a new audio stream
+     * Class to build {@link AudioStream} objects.
-     * @param url the url
-     * @param format the format
-     * @param averageBitrate the average bitrate
      */
-    public AudioStream(final String url,
+    @SuppressWarnings("checkstyle:hiddenField")
-                       final MediaFormat format,
+    public static final class Builder {
-                       final int averageBitrate) {
+        private String id;
-        super(url, format);
+        private String content;
+        private boolean isUrl;
+        private DeliveryMethod deliveryMethod = DeliveryMethod.PROGRESSIVE_HTTP;
+        @Nullable
+        private MediaFormat mediaFormat;
+        @Nullable
+        private String manifestUrl;
+        private int averageBitrate = UNKNOWN_BITRATE;
+        @Nullable
+        private ItagItem itagItem;
+
+        /**
+         * Create a new {@link Builder} instance with its default values.
+         */
+        public Builder() {
+        }
+
+        /**
+         * Set the identifier of the {@link AudioStream}.
+         *
+         * <p>
+         * It <b>must not be null</b> and should be non empty.
+         * </p>
+         *
+         * <p>
+         * If you are not able to get an identifier, use the static constant {@link
+         * Stream#ID_UNKNOWN ID_UNKNOWN} of the {@link Stream} class.
+         * </p>
+         *
+         * @param id the identifier of the {@link AudioStream}, which must not be null
+         * @return this {@link Builder} instance
+         */
+        public Builder setId(@Nonnull final String id) {
+            this.id = id;
+            return this;
+        }
+
+        /**
+         * Set the content of the {@link AudioStream}.
+         *
+         * <p>
+         * It must not be null, and should be non empty.
+         * </p>
+         *
+         * @param content the content of the {@link AudioStream}
+         * @param isUrl   whether the content is a URL
+         * @return this {@link Builder} instance
+         */
+        public Builder setContent(@Nonnull final String content,
+                                  final boolean isUrl) {
+            this.content = content;
+            this.isUrl = isUrl;
+            return this;
+        }
+
+        /**
+         * Set the {@link MediaFormat} used by the {@link AudioStream}.
+         *
+         * <p>
+         * It should be one of the audio {@link MediaFormat}s ({@link MediaFormat#M4A M4A},
+         * {@link MediaFormat#WEBMA WEBMA}, {@link MediaFormat#MP3 MP3}, {@link MediaFormat#OPUS
+         * OPUS}, {@link MediaFormat#OGG OGG}, or {@link MediaFormat#WEBMA_OPUS WEBMA_OPUS}) but
+         * can be {@code null} if the media format could not be determined.
+         * </p>
+         *
+         * <p>
+         * The default value is {@code null}.
+         * </p>
+         *
+         * @param mediaFormat the {@link MediaFormat} of the {@link AudioStream}, which can be null
+         * @return this {@link Builder} instance
+         */
+        public Builder setMediaFormat(@Nullable final MediaFormat mediaFormat) {
+            this.mediaFormat = mediaFormat;
+            return this;
+        }
+
+        /**
+         * Set the {@link DeliveryMethod} of the {@link AudioStream}.
+         *
+         * <p>
+         * It must not be null.
+         * </p>
+         *
+         * <p>
+         * The default delivery method is {@link DeliveryMethod#PROGRESSIVE_HTTP}.
+         * </p>
+         *
+         * @param deliveryMethod the {@link DeliveryMethod} of the {@link AudioStream}, which must
+         *                       not be null
+         * @return this {@link Builder} instance
+         */
+        public Builder setDeliveryMethod(@Nonnull final DeliveryMethod deliveryMethod) {
+            this.deliveryMethod = deliveryMethod;
+            return this;
+        }
+
+        /**
+         * Sets the URL of the manifest this stream comes from (if applicable, otherwise null).
+         *
+         * @param manifestUrl the URL of the manifest this stream comes from or {@code null}
+         * @return this {@link Builder} instance
+         */
+        public Builder setManifestUrl(@Nullable final String manifestUrl) {
+            this.manifestUrl = manifestUrl;
+            return this;
+        }
+
+        /**
+         * Set the average bitrate of the {@link AudioStream}.
+         *
+         * <p>
+         * The default value is {@link #UNKNOWN_BITRATE}.
+         * </p>
+         *
+         * @param averageBitrate the average bitrate of the {@link AudioStream}, which should
+         *                       positive
+         * @return this {@link Builder} instance
+         */
+        public Builder setAverageBitrate(final int averageBitrate) {
+            this.averageBitrate = averageBitrate;
+            return this;
+        }
+
+        /**
+         * Set the {@link ItagItem} corresponding to the {@link AudioStream}.
+         *
+         * <p>
+         * {@link ItagItem}s are YouTube specific objects, so they are only known for this service
+         * and can be null.
+         * </p>
+         *
+         * <p>
+         * The default value is {@code null}.
+         * </p>
+         *
+         * @param itagItem the {@link ItagItem} of the {@link AudioStream}, which can be null
+         * @return this {@link Builder} instance
+         */
+        public Builder setItagItem(@Nullable final ItagItem itagItem) {
+            this.itagItem = itagItem;
+            return this;
+        }
+
+        /**
+         * Build an {@link AudioStream} using the builder's current values.
+         *
+         * <p>
+         * The identifier and the content (and so the {@code isUrl} boolean) properties must have
+         * been set.
+         * </p>
+         *
+         * @return a new {@link AudioStream} using the builder's current values
+         * @throws IllegalStateException if {@code id}, {@code content} (and so {@code isUrl}) or
+         * {@code deliveryMethod} have been not set, or have been set as {@code null}
+         */
+        @Nonnull
+        public AudioStream build() {
+            if (id == null) {
+                throw new IllegalStateException(
+                        "The identifier of the audio stream has been not set or is null. If you "
+                                + "are not able to get an identifier, use the static constant "
+                                + "ID_UNKNOWN of the Stream class.");
+            }
+
+            if (content == null) {
+                throw new IllegalStateException("The content of the audio stream has been not set "
+                        + "or is null. Please specify a non-null one with setContent.");
+            }
+
+            if (deliveryMethod == null) {
+                throw new IllegalStateException(
+                        "The delivery method of the audio stream has been set as null, which is "
+                                + "not allowed. Pass a valid one instead with setDeliveryMethod.");
+            }
+
+            return new AudioStream(id, content, isUrl, mediaFormat, deliveryMethod, averageBitrate,
+                    manifestUrl, itagItem);
+        }
+    }
+
+
+    /**
+     * Create a new audio stream.
+     *
+     * @param id             the identifier which uniquely identifies the stream, e.g. for YouTube
+     *                       this would be the itag
+     * @param content        the content or the URL of the stream, depending on whether isUrl is
+     *                       true
+     * @param isUrl          whether content is the URL or the actual content of e.g. a DASH
+     *                       manifest
+     * @param format         the {@link MediaFormat} used by the stream, which can be null
+     * @param deliveryMethod the {@link DeliveryMethod} of the stream
+     * @param averageBitrate the average bitrate of the stream (which can be unknown, see
+     *                       {@link #UNKNOWN_BITRATE})
+     * @param itagItem       the {@link ItagItem} corresponding to the stream, which cannot be null
+     * @param manifestUrl    the URL of the manifest this stream comes from (if applicable,
+     *                       otherwise null)
+     */
+    @SuppressWarnings("checkstyle:ParameterNumber")
+    private AudioStream(@Nonnull final String id,
+                        @Nonnull final String content,
+                        final boolean isUrl,
+                        @Nullable final MediaFormat format,
+                        @Nonnull final DeliveryMethod deliveryMethod,
+                        final int averageBitrate,
+                        @Nullable final String manifestUrl,
+                        @Nullable final ItagItem itagItem) {
+        super(id, content, isUrl, format, deliveryMethod, manifestUrl);
+        if (itagItem != null) {
+            this.itagItem = itagItem;
+            this.itag = itagItem.id;
+            this.quality = itagItem.getQuality();
+            this.bitrate = itagItem.getBitrate();
+            this.initStart = itagItem.getInitStart();
+            this.initEnd = itagItem.getInitEnd();
+            this.indexStart = itagItem.getIndexStart();
+            this.indexEnd = itagItem.getIndexEnd();
+            this.codec = itagItem.getCodec();
+        }
         this.averageBitrate = averageBitrate;
     }
 
     /**
-     * Create a new audio stream
+     * {@inheritDoc}
-     * @param url the url
-     * @param itag the ItagItem of the Stream
      */
-    public AudioStream(final String url, final ItagItem itag) {
-        this(url, itag.getMediaFormat(), itag.avgBitrate);
-        this.itag = itag.id;
-        this.quality = itag.getQuality();
-        this.bitrate = itag.getBitrate();
-        this.initStart = itag.getInitStart();
-        this.initEnd = itag.getInitEnd();
-        this.indexStart = itag.getIndexStart();
-        this.indexEnd = itag.getIndexEnd();
-        this.codec = itag.getCodec();
-    }
-
     @Override
     public boolean equalStats(final Stream cmp) {
         return super.equalStats(cmp) && cmp instanceof AudioStream
@@ -73,42 +282,102 @@ public class AudioStream extends Stream {
     }
 
     /**
-     * Get the average bitrate
+     * Get the average bitrate of the stream.
-     * @return the average bitrate or -1
+     *
+     * @return the average bitrate or {@link #UNKNOWN_BITRATE} if it is unknown
      */
     public int getAverageBitrate() {
         return averageBitrate;
     }
 
+    /**
+     * Get the itag identifier of the stream.
+     *
+     * <p>
+     * Always equals to {@link #ITAG_NOT_AVAILABLE_OR_NOT_APPLICABLE} for other streams than the
+     * ones of the YouTube service.
+     * </p>
+     *
+     * @return the number of the {@link ItagItem} passed in the constructor of the audio stream.
+     */
     public int getItag() {
         return itag;
     }
 
+    /**
+     * Get the bitrate of the stream.
+     *
+     * @return the bitrate set from the {@link ItagItem} passed in the constructor of the stream.
+     */
     public int getBitrate() {
         return bitrate;
     }
 
+    /**
+     * Get the initialization start of the stream.
+     *
+     * @return the initialization start value set from the {@link ItagItem} passed in the
+     * constructor of the stream.
+     */
     public int getInitStart() {
         return initStart;
     }
 
+    /**
+     * Get the initialization end of the stream.
+     *
+     * @return the initialization end value set from the {@link ItagItem} passed in the constructor
+     * of the stream.
+     */
     public int getInitEnd() {
         return initEnd;
     }
 
+    /**
+     * Get the index start of the stream.
+     *
+     * @return the index start value set from the {@link ItagItem} passed in the constructor of the
+     * stream.
+     */
     public int getIndexStart() {
         return indexStart;
     }
 
+    /**
+     * Get the index end of the stream.
+     *
+     * @return the index end value set from the {@link ItagItem} passed in the constructor of the
+     * stream.
+     */
     public int getIndexEnd() {
         return indexEnd;
     }
 
+    /**
+     * Get the quality of the stream.
+     *
+     * @return the quality label set from the {@link ItagItem} passed in the constructor of the
+     * stream.
+     */
     public String getQuality() {
         return quality;
     }
 
+    /**
+     * Get the codec of the stream.
+     *
+     * @return the codec set from the {@link ItagItem} passed in the constructor of the stream.
+     */
     public String getCodec() {
         return codec;
     }
+
+    /**
+     * {@inheritDoc}
+     */
+    @Override
+    @Nullable
+    public ItagItem getItagItem() {
+        return itagItem;
+    }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/stream/DeliveryMethod.java

@@ -0,0 +1,53 @@
+package org.schabi.newpipe.extractor.stream;
+
+/**
+ * An enum to represent the different delivery methods of {@link Stream streams} which are returned
+ * by the extractor.
+ */
+public enum DeliveryMethod {
+
+    /**
+     * Used for {@link Stream}s served using the progressive HTTP streaming method.
+     */
+    PROGRESSIVE_HTTP,
+
+    /**
+     * Used for {@link Stream}s served using the DASH (Dynamic Adaptive Streaming over HTTP)
+     * adaptive streaming method.
+     *
+     * @see <a href="https://en.wikipedia.org/wiki/Dynamic_Adaptive_Streaming_over_HTTP">the
+     * Dynamic Adaptive Streaming over HTTP Wikipedia page</a> and <a href="https://dashif.org/">
+     * DASH Industry Forum's website</a> for more information about the DASH delivery method
+     */
+    DASH,
+
+    /**
+     * Used for {@link Stream}s served using the HLS (HTTP Live Streaming) adaptive streaming
+     * method.
+     *
+     * @see <a href="https://en.wikipedia.org/wiki/HTTP_Live_Streaming">the HTTP Live Streaming
+     * page</a> and <a href="https://developer.apple.com/streaming">Apple's developers website page
+     * about HTTP Live Streaming</a> for more information about the HLS delivery method
+     */
+    HLS,
+
+    /**
+     * Used for {@link Stream}s served using the SmoothStreaming adaptive streaming method.
+     *
+     * @see <a href="https://en.wikipedia.org/wiki/Adaptive_bitrate_streaming
+     * #Microsoft_Smooth_Streaming_(MSS)">Wikipedia's page about adaptive bitrate streaming,
+     * section <i>Microsoft Smooth Streaming (MSS)</i></a> for more information about the
+     * SmoothStreaming delivery method
+     */
+    SS,
+
+    /**
+     * Used for {@link Stream}s served via a torrent file.
+     *
+     * @see <a href="https://en.wikipedia.org/wiki/BitTorrent">Wikipedia's BitTorrent's page</a>,
+     * <a href="https://en.wikipedia.org/wiki/Torrent_file">Wikipedia's page about torrent files
+     * </a> and <a href="https://www.bittorrent.org">Bitorrent's website</a> for more information
+     * about the BitTorrent protocol
+     */
+    TORRENT
+}

extractor/src/main/java/org/schabi/newpipe/extractor/stream/Stream.java

@@ -1,68 +1,72 @@
 package org.schabi.newpipe.extractor.stream;
 
-import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
-
 import org.schabi.newpipe.extractor.MediaFormat;
+import org.schabi.newpipe.extractor.services.youtube.ItagItem;
 
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
 import java.io.Serializable;
 import java.util.List;
 
+import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
+
 /**
- * Creates a stream object from url, format and optional torrent url
+ * Abstract class which represents streams in the extractor.
  */
 public abstract class Stream implements Serializable {
-    private final MediaFormat mediaFormat;
+    public static final int FORMAT_ID_UNKNOWN = -1;
-    private final String url;
+    public static final String ID_UNKNOWN = " ";
-    private final String torrentUrl;
 
     /**
-     * @deprecated Use {@link #getFormat()}  or {@link #getFormatId()}
+     * An integer to represent that the itag ID returned is not available (only for YouTube; this
-     */
+     * should never happen) or not applicable (for other services than YouTube).
-    @Deprecated
-    public final int format;
-
-    /**
-     * Instantiates a new stream object.
      *
-     * @param url    the url
+     * <p>
-     * @param format the format
+     * An itag should not have a negative value, so {@code -1} is used for this constant.
+     * </p>
      */
-    public Stream(final String url, final MediaFormat format) {
+    public static final int ITAG_NOT_AVAILABLE_OR_NOT_APPLICABLE = -1;
-        this(url, null, format);
+
-    }
+    private final String id;
+    @Nullable private final MediaFormat mediaFormat;
+    private final String content;
+    private final boolean isUrl;
+    private final DeliveryMethod deliveryMethod;
+    @Nullable private final String manifestUrl;
 
     /**
-     * Instantiates a new stream object.
+     * Instantiates a new {@code Stream} object.
      *
-     * @param url        the url
+     * @param id             the identifier which uniquely identifies the file, e.g. for YouTube
-     * @param torrentUrl the url to torrent file, example
+     *                       this would be the itag
-     *                   https://webtorrent.io/torrents/big-buck-bunny.torrent
+     * @param content        the content or URL, depending on whether isUrl is true
-     * @param format     the format
+     * @param isUrl          whether content is the URL or the actual content of e.g. a DASH
+     *                       manifest
+     * @param format         the {@link MediaFormat}, which can be null
+     * @param deliveryMethod the delivery method of the stream
+     * @param manifestUrl    the URL of the manifest this stream comes from (if applicable,
+     *                       otherwise null)
      */
-    public Stream(final String url, final String torrentUrl, final MediaFormat format) {
+    public Stream(final String id,
-        this.url = url;
+                  final String content,
-        this.torrentUrl = torrentUrl;
+                  final boolean isUrl,
-        //noinspection deprecation
+                  @Nullable final MediaFormat format,
-        this.format = format.id;
+                  final DeliveryMethod deliveryMethod,
+                  @Nullable final String manifestUrl) {
+        this.id = id;
+        this.content = content;
+        this.isUrl = isUrl;
         this.mediaFormat = format;
+        this.deliveryMethod = deliveryMethod;
+        this.manifestUrl = manifestUrl;
     }
 
     /**
-     * Reveals whether two streams have the same stats (format and bitrate, for example)
+     * Checks if the list already contains a stream with the same statistics.
-     */
+     *
-    public boolean equalStats(final Stream cmp) {
+     * @param stream the stream to be compared against the streams in the stream list
-        return cmp != null && getFormatId() == cmp.getFormatId();
+     * @param streamList the list of {@link Stream}s which will be compared
-    }
+     * @return whether the list already contains one stream with equals stats
-
-    /**
-     * Reveals whether two Streams are equal
-     */
-    public boolean equals(final Stream cmp) {
-        return equalStats(cmp) && url.equals(cmp.url);
-    }
-
-    /**
-     * Check if the list already contains one stream with equals stats
      */
     public static boolean containSimilarStream(final Stream stream,
                                                final List<? extends Stream> streamList) {
@@ -78,38 +82,126 @@ public abstract class Stream implements Serializable {
     }
 
     /**
-     * Gets the url.
+     * Reveals whether two streams have the same statistics ({@link MediaFormat media format} and
+     * {@link DeliveryMethod delivery method}).
      *
-     * @return the url
+     * <p>
+     * If the {@link MediaFormat media format} of the stream is unknown, the streams are compared
+     * by using only the {@link DeliveryMethod delivery method} and their ID.
+     * </p>
+     *
+     * <p>
+     * Note: This method always returns false if the stream passed is null.
+     * </p>
+     *
+     * @param other the stream object to be compared to this stream object
+     * @return whether the stream have the same stats or not, based on the criteria above
      */
+    public boolean equalStats(@Nullable final Stream other) {
+        if (other == null || mediaFormat == null || other.mediaFormat == null) {
+            return false;
+        }
+        return mediaFormat.id == other.mediaFormat.id && deliveryMethod == other.deliveryMethod
+                && isUrl == other.isUrl;
+    }
+
+    /**
+     * Gets the identifier of this stream, e.g. the itag for YouTube.
+     *
+     * <p>
+     * It should normally be unique, but {@link #ID_UNKNOWN} may be returned as the identifier if
+     * the one used by the stream extractor cannot be extracted, which could happen if the
+     * extractor uses a value from a streaming service.
+     * </p>
+     *
+     * @return the identifier (which may be {@link #ID_UNKNOWN})
+     */
+    public String getId() {
+        return id;
+    }
+
+    /**
+     * Gets the URL of this stream if the content is a URL, or {@code null} otherwise.
+     *
+     * @return the URL if the content is a URL, {@code null} otherwise
+     * @deprecated Use {@link #getContent()} instead.
+     */
+    @Deprecated
+    @Nullable
     public String getUrl() {
-        return url;
+        return isUrl ? content : null;
     }
 
     /**
-     * Gets the torrent url.
+     * Gets the content or URL.
      *
-     * @return the torrent url, example https://webtorrent.io/torrents/big-buck-bunny.torrent
+     * @return the content or URL
      */
-    public String getTorrentUrl() {
+    public String getContent() {
-        return torrentUrl;
+        return content;
     }
 
     /**
-     * Gets the format.
+     * Returns whether the content is a URL or not.
+     *
+     * @return {@code true} if the content of this stream is a URL, {@code false} if it's the
+     * actual content
+     */
+    public boolean isUrl() {
+        return isUrl;
+    }
+
+    /**
+     * Gets the {@link MediaFormat}, which can be null.
      *
      * @return the format
      */
+    @Nullable
     public MediaFormat getFormat() {
         return mediaFormat;
     }
 
     /**
-     * Gets the format id.
+     * Gets the format ID, which can be unknown.
      *
-     * @return the format id
+     * @return the format ID or {@link #FORMAT_ID_UNKNOWN}
      */
     public int getFormatId() {
-        return mediaFormat.id;
+        if (mediaFormat != null) {
+            return mediaFormat.id;
+        }
+        return FORMAT_ID_UNKNOWN;
     }
+
+    /**
+     * Gets the {@link DeliveryMethod}.
+     *
+     * @return the delivery method
+     */
+    @Nonnull
+    public DeliveryMethod getDeliveryMethod() {
+        return deliveryMethod;
+    }
+
+    /**
+     * Gets the URL of the manifest this stream comes from (if applicable, otherwise null).
+     *
+     * @return the URL of the manifest this stream comes from or {@code null}
+     */
+    @Nullable
+    public String getManifestUrl() {
+        return manifestUrl;
+    }
+
+    /**
+     * Gets the {@link ItagItem} of a stream.
+     *
+     * <p>
+     * If the stream is not from YouTube, this value will always be null.
+     * </p>
+     *
+     * @return the {@link ItagItem} of the stream or {@code null}
+     */
+    @Nullable
+    public abstract ItagItem getItagItem();
 }

extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java

@@ -9,7 +9,6 @@ import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
 import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.localization.DateWrapper;
-import org.schabi.newpipe.extractor.utils.DashMpdParser;
 import org.schabi.newpipe.extractor.utils.ExtractorHelper;
 
 import java.io.IOException;
@@ -26,24 +25,24 @@ import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
  * Created by Christian Schabesberger on 26.08.15.
  *
  * Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
- * StreamInfo.java is part of NewPipe.
+ * StreamInfo.java is part of NewPipe Extractor.
  *
- * NewPipe is free software: you can redistribute it and/or modify
+ * NewPipe Extractor is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
  * the Free Software Foundation, either version 3 of the License, or
  * (at your option) any later version.
  *
- * NewPipe is distributed in the hope that it will be useful,
+ * NewPipe Extractor is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
- * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ * along with NewPipe Extractor. If not, see <https://www.gnu.org/licenses/>.
  */
 
 /**
- * Info object for opened videos, ie the video ready to play.
+ * Info object for opened contents, i.e. the content ready to play.
  */
 public class StreamInfo extends Info {
 
@@ -69,27 +68,26 @@ public class StreamInfo extends Info {
         return getInfo(NewPipe.getServiceByUrl(url), url);
     }
 
-    public static StreamInfo getInfo(final StreamingService service,
+    public static StreamInfo getInfo(@Nonnull final StreamingService service,
                                      final String url) throws IOException, ExtractionException {
         return getInfo(service.getStreamExtractor(url));
     }
 
-    public static StreamInfo getInfo(final StreamExtractor extractor)
+    public static StreamInfo getInfo(@Nonnull final StreamExtractor extractor)
             throws ExtractionException, IOException {
         extractor.fetchPage();
+        final StreamInfo streamInfo;
         try {
-            final StreamInfo streamInfo = extractImportantData(extractor);
+            streamInfo = extractImportantData(extractor);
             extractStreams(streamInfo, extractor);
             extractOptionalData(streamInfo, extractor);
             return streamInfo;
 
         } catch (final ExtractionException e) {
-            // Currently YouTube does not distinguish between age restricted videos and
+            // Currently, YouTube does not distinguish between age restricted videos and videos
-            // videos blocked
+            // blocked by country. This means that during the initialisation of the extractor, the
-            // by country. This means that during the initialisation of the extractor, the
+            // extractor will assume that a video is age restricted while in reality it is blocked
-            // extractor
+            // by country.
-            // will assume that a video is age restricted while in reality it it blocked by
-            // country.
             //
             // We will now detect whether the video is blocked by country or not.
 
@@ -102,22 +100,27 @@ public class StreamInfo extends Info {
         }
     }
 
-    private static StreamInfo extractImportantData(final StreamExtractor extractor)
+    @Nonnull
+    private static StreamInfo extractImportantData(@Nonnull final StreamExtractor extractor)
             throws ExtractionException {
-        /* ---- important data, without the video can't be displayed goes here: ---- */
+        // Important data, without it the content can't be displayed.
-        // if one of these is not available an exception is meant to be thrown directly
+        // If one of these is not available, the frontend will receive an exception directly.
-        // into the frontend.
 
+        final int serviceId = extractor.getServiceId();
         final String url = extractor.getUrl();
+        final String originalUrl = extractor.getOriginalUrl();
         final StreamType streamType = extractor.getStreamType();
         final String id = extractor.getId();
         final String name = extractor.getName();
         final int ageLimit = extractor.getAgeLimit();
 
-        // suppress always-non-null warning as here we double-check it really is not null
+        // Suppress always-non-null warning as here we double-check it really is not null
         //noinspection ConstantConditions
-        if (streamType == StreamType.NONE || isNullOrEmpty(url) || isNullOrEmpty(id)
+        if (streamType == StreamType.NONE
-                || name == null /* but it can be empty of course */ || ageLimit == -1) {
+                || isNullOrEmpty(url)
+                || isNullOrEmpty(id)
+                || name == null /* but it can be empty of course */
+                || ageLimit == -1) {
             throw new ExtractionException("Some important stream information was not given.");
         }
 
@@ -125,16 +128,18 @@ public class StreamInfo extends Info {
                 streamType, id, name, ageLimit);
     }
 
-    private static void extractStreams(final StreamInfo streamInfo, final StreamExtractor extractor)
+
+    private static void extractStreams(final StreamInfo streamInfo,
+                                       final StreamExtractor extractor)
             throws ExtractionException {
-        /* ---- stream extraction goes here ---- */
+        /* ---- Stream extraction goes here ---- */
-        // At least one type of stream has to be available,
+        // At least one type of stream has to be available, otherwise an exception will be thrown
-        // otherwise an exception will be thrown directly into the frontend.
+        // directly into the frontend.
 
         try {
             streamInfo.setDashMpdUrl(extractor.getDashMpdUrl());
         } catch (final Exception e) {
-            streamInfo.addError(new ExtractionException("Couldn't get Dash manifest", e));
+            streamInfo.addError(new ExtractionException("Couldn't get DASH manifest", e));
         }
 
         try {
@@ -151,12 +156,14 @@ public class StreamInfo extends Info {
         } catch (final Exception e) {
             streamInfo.addError(new ExtractionException("Couldn't get audio streams", e));
         }
+
         /* Extract video stream url */
         try {
             streamInfo.setVideoStreams(extractor.getVideoStreams());
         } catch (final Exception e) {
             streamInfo.addError(new ExtractionException("Couldn't get video streams", e));
         }
+
         /* Extract video only stream url */
         try {
             streamInfo.setVideoOnlyStreams(extractor.getVideoOnlyStreams());
@@ -164,7 +171,7 @@ public class StreamInfo extends Info {
             streamInfo.addError(new ExtractionException("Couldn't get video only streams", e));
         }
 
-        // Lists can be null if a exception was thrown during extraction
+        // Lists can be null if an exception was thrown during extraction
         if (streamInfo.getVideoStreams() == null) {
             streamInfo.setVideoStreams(Collections.emptyList());
         }
@@ -175,37 +182,9 @@ public class StreamInfo extends Info {
             streamInfo.setAudioStreams(Collections.emptyList());
         }
 
-        Exception dashMpdError = null;
+        // Either audio or video has to be available, otherwise we didn't get a stream (since
-        if (!isNullOrEmpty(streamInfo.getDashMpdUrl())) {
+        // videoOnly are optional, they don't count).
-            try {
-                final DashMpdParser.ParserResult result = DashMpdParser.getStreams(streamInfo);
-                streamInfo.getVideoOnlyStreams().addAll(result.getVideoOnlyStreams());
-                streamInfo.getAudioStreams().addAll(result.getAudioStreams());
-                streamInfo.getVideoStreams().addAll(result.getVideoStreams());
-                streamInfo.segmentedVideoOnlyStreams = result.getSegmentedVideoOnlyStreams();
-                streamInfo.segmentedAudioStreams = result.getSegmentedAudioStreams();
-                streamInfo.segmentedVideoStreams = result.getSegmentedVideoStreams();
-            } catch (final Exception e) {
-                // Sometimes we receive 403 (forbidden) error when trying to download the
-                // manifest (similar to what happens with youtube-dl),
-                // just skip the exception (but store it somewhere), as we later check if we
-                // have streams anyway.
-                dashMpdError = e;
-            }
-        }
-
-        // Either audio or video has to be available, otherwise we didn't get a stream
-        // (since videoOnly are optional, they don't count).
         if ((streamInfo.videoStreams.isEmpty()) && (streamInfo.audioStreams.isEmpty())) {
-
-            if (dashMpdError != null) {
-                // If we don't have any video or audio and the dashMpd 'errored', add it to the
-                // error list
-                // (it's optional and it don't get added automatically, but it's good to have
-                // some additional error context)
-                streamInfo.addError(dashMpdError);
-            }
-
             throw new StreamExtractException(
                     "Could not get any stream. See error variable to get further details.");
         }
@@ -214,11 +193,9 @@ public class StreamInfo extends Info {
     @SuppressWarnings("MethodLength")
     private static void extractOptionalData(final StreamInfo streamInfo,
                                             final StreamExtractor extractor) {
-        /* ---- optional data goes here: ---- */
+        /* ---- Optional data goes here: ---- */
-        // If one of these fails, the frontend needs to handle that they are not
+        // If one of these fails, the frontend needs to handle that they are not available.
-        // available.
+        // Exceptions are therefore not thrown into the frontend, but stored into the error list,
-        // Exceptions are therefore not thrown into the frontend, but stored into the
-        // error List,
         // so the frontend can afterwards check where errors happened.
 
         try {
@@ -314,7 +291,7 @@ public class StreamInfo extends Info {
             streamInfo.addError(e);
         }
 
-        //additional info
+        // Additional info
         try {
             streamInfo.setHost(extractor.getHost());
         } catch (final Exception e) {
@@ -360,15 +337,14 @@ public class StreamInfo extends Info {
         } catch (final Exception e) {
             streamInfo.addError(e);
         }
-
         try {
             streamInfo.setPreviewFrames(extractor.getFrames());
         } catch (final Exception e) {
             streamInfo.addError(e);
         }
 
-        streamInfo
+        streamInfo.setRelatedItems(ExtractorHelper.getRelatedItemsOrLogError(streamInfo,
-                .setRelatedItems(ExtractorHelper.getRelatedItemsOrLogError(streamInfo, extractor));
+                extractor));
     }
 
     private StreamType streamType;
@@ -398,11 +374,6 @@ public class StreamInfo extends Info {
     private List<VideoStream> videoOnlyStreams = new ArrayList<>();
 
     private String dashMpdUrl = "";
-    private List<VideoStream> segmentedVideoStreams = new ArrayList<>();
-    private List<AudioStream> segmentedAudioStreams = new ArrayList<>();
-    private List<VideoStream> segmentedVideoOnlyStreams = new ArrayList<>();
-
-
     private String hlsUrl = "";
     private List<InfoItem> relatedItems = new ArrayList<>();
 
@@ -625,30 +596,6 @@ public class StreamInfo extends Info {
         this.dashMpdUrl = dashMpdUrl;
     }
 
-    public List<VideoStream> getSegmentedVideoStreams() {
-        return segmentedVideoStreams;
-    }
-
-    public void setSegmentedVideoStreams(final List<VideoStream> segmentedVideoStreams) {
-        this.segmentedVideoStreams = segmentedVideoStreams;
-    }
-
-    public List<AudioStream> getSegmentedAudioStreams() {
-        return segmentedAudioStreams;
-    }
-
-    public void setSegmentedAudioStreams(final List<AudioStream> segmentedAudioStreams) {
-        this.segmentedAudioStreams = segmentedAudioStreams;
-    }
-
-    public List<VideoStream> getSegmentedVideoOnlyStreams() {
-        return segmentedVideoOnlyStreams;
-    }
-
-    public void setSegmentedVideoOnlyStreams(final List<VideoStream> segmentedVideoOnlyStreams) {
-        this.segmentedVideoOnlyStreams = segmentedVideoOnlyStreams;
-    }
-
     public String getHlsUrl() {
         return hlsUrl;
     }

extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamType.java

@@ -1,10 +1,74 @@
 package org.schabi.newpipe.extractor.stream;
 
+/**
+ * An enum representing the stream type of a {@link StreamInfo} extracted by a {@link
+ * StreamExtractor}.
+ */
 public enum StreamType {
-    NONE,   // placeholder to check if stream type was checked or not
+
+    /**
+     * Placeholder to check if the stream type was checked or not. It doesn't make sense to use this
+     * enum constant outside of the extractor as it will never be returned by an {@link
+     * org.schabi.newpipe.extractor.Extractor} and is only used internally.
+     */
+    NONE,
+
+    /**
+     * A normal video stream, usually with audio. Note that the {@link StreamInfo} <strong>can also
+     * provide audio-only {@link AudioStream}s</strong> in addition to video or video-only {@link
+     * VideoStream}s.
+     */
     VIDEO_STREAM,
+
+    /**
+     * An audio-only stream. There should be no {@link VideoStream}s available! In order to prevent
+     * unexpected behaviors, when {@link StreamExtractor}s return this stream type, they should
+     * ensure that no video stream is returned in {@link StreamExtractor#getVideoStreams()} and
+     * {@link StreamExtractor#getVideoOnlyStreams()}.
+     */
     AUDIO_STREAM,
+
+    /**
+     * A video live stream, usually with audio. Note that the {@link StreamInfo} <strong>can also
+     * provide audio-only {@link AudioStream}s</strong> in addition to video or video-only {@link
+     * VideoStream}s.
+     */
     LIVE_STREAM,
+
+    /**
+     * An audio-only live stream. There should be no {@link VideoStream}s available! In order to
+     * prevent unexpected behaviors, when {@link StreamExtractor}s return this stream type, they
+     * should ensure that no video stream is returned in {@link StreamExtractor#getVideoStreams()}
+     * and {@link StreamExtractor#getVideoOnlyStreams()}.
+     */
     AUDIO_LIVE_STREAM,
-    FILE
+
+    /**
+     * A video live stream that has just ended but has not yet been encoded into a normal video
+     * stream. Note that the {@link StreamInfo} <strong>can also provide audio-only {@link
+     * AudioStream}s</strong> in addition to video or video-only {@link VideoStream}s.
+     *
+     * <p>
+     * Note that most of the content of an ended live video (or audio) may be extracted as {@link
+     * #VIDEO_STREAM regular video contents} (or {@link #AUDIO_STREAM regular audio contents})
+     * later, because the service may encode them again later as normal video/audio streams. That's
+     * the case on YouTube, for example.
+     * </p>
+     */
+    POST_LIVE_STREAM,
+
+    /**
+     * An audio live stream that has just ended but has not yet been encoded into a normal audio
+     * stream. There should be no {@link VideoStream}s available! In order to prevent unexpected
+     * behaviors, when {@link StreamExtractor}s return this stream type, they should ensure that no
+     * video stream is returned in {@link StreamExtractor#getVideoStreams()} and
+     * {@link StreamExtractor#getVideoOnlyStreams()}.
+     *
+     * <p>
+     * Note that most of ended live audio streams extracted with this value are processed as
+     * {@link #AUDIO_STREAM regular audio streams} later, because the service may encode them
+     * again later.
+     * </p>
+     */
+    POST_LIVE_AUDIO_STREAM
 }

extractor/src/main/java/org/schabi/newpipe/extractor/stream/SubtitlesStream.java

@@ -1,53 +1,286 @@
 package org.schabi.newpipe.extractor.stream;
 
 import org.schabi.newpipe.extractor.MediaFormat;
+import org.schabi.newpipe.extractor.services.youtube.ItagItem;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
 
-import java.io.Serializable;
 import java.util.Locale;
 
-public class SubtitlesStream extends Stream implements Serializable {
+import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
+
+public final class SubtitlesStream extends Stream {
     private final MediaFormat format;
     private final Locale locale;
     private final boolean autoGenerated;
     private final String code;
 
-    public SubtitlesStream(final MediaFormat format,
+    /**
-                           final String languageCode,
+     * Class to build {@link SubtitlesStream} objects.
-                           final String url,
+     */
-                           final boolean autoGenerated) {
+    @SuppressWarnings("checkstyle:HiddenField")
-        super(url, format);
+    public static final class Builder {
+        private String id;
+        private String content;
+        private boolean isUrl;
+        private DeliveryMethod deliveryMethod = DeliveryMethod.PROGRESSIVE_HTTP;
+        @Nullable
+        private MediaFormat mediaFormat;
+        @Nullable
+        private String manifestUrl;
+        private String languageCode;
+        // Use of the Boolean class instead of the primitive type needed for setter call check
+        private Boolean autoGenerated;
+
+        /**
+         * Create a new {@link Builder} instance with default values.
+         */
+        public Builder() {
+        }
+
+        /**
+         * Set the identifier of the {@link SubtitlesStream}.
+         *
+         * @param id the identifier of the {@link SubtitlesStream}, which should not be null
+         *           (otherwise the fallback to create the identifier will be used when building
+         *           the builder)
+         * @return this {@link Builder} instance
+         */
+        public Builder setId(@Nonnull final String id) {
+            this.id = id;
+            return this;
+        }
+
+        /**
+         * Set the content of the {@link SubtitlesStream}.
+         *
+         * <p>
+         * It must not be null, and should be non empty.
+         * </p>
+         *
+         * @param content the content of the {@link SubtitlesStream}, which must not be null
+         * @param isUrl   whether the content is a URL
+         * @return this {@link Builder} instance
+         */
+        public Builder setContent(@Nonnull final String content,
+                                  final boolean isUrl) {
+            this.content = content;
+            this.isUrl = isUrl;
+            return this;
+        }
+
+        /**
+         * Set the {@link MediaFormat} used by the {@link SubtitlesStream}.
+         *
+         * <p>
+         * It should be one of the subtitles {@link MediaFormat}s ({@link MediaFormat#SRT SRT},
+         * {@link MediaFormat#TRANSCRIPT1 TRANSCRIPT1}, {@link MediaFormat#TRANSCRIPT2
+         * TRANSCRIPT2}, {@link MediaFormat#TRANSCRIPT3 TRANSCRIPT3}, {@link MediaFormat#TTML
+         * TTML}, or {@link MediaFormat#VTT VTT}) but can be {@code null} if the media format could
+         * not be determined.
+         * </p>
+         *
+         * <p>
+         * The default value is {@code null}.
+         * </p>
+         *
+         * @param mediaFormat the {@link MediaFormat} of the {@link SubtitlesStream}, which can be
+         *                    null
+         * @return this {@link Builder} instance
+         */
+        public Builder setMediaFormat(@Nullable final MediaFormat mediaFormat) {
+            this.mediaFormat = mediaFormat;
+            return this;
+        }
+
+        /**
+         * Set the {@link DeliveryMethod} of the {@link SubtitlesStream}.
+         *
+         * <p>
+         * It must not be null.
+         * </p>
+         *
+         * <p>
+         * The default delivery method is {@link DeliveryMethod#PROGRESSIVE_HTTP}.
+         * </p>
+         *
+         * @param deliveryMethod the {@link DeliveryMethod} of the {@link SubtitlesStream}, which
+         *                       must not be null
+         * @return this {@link Builder} instance
+         */
+        public Builder setDeliveryMethod(@Nonnull final DeliveryMethod deliveryMethod) {
+            this.deliveryMethod = deliveryMethod;
+            return this;
+        }
+
+        /**
+         * Sets the URL of the manifest this stream comes from (if applicable, otherwise null).
+         *
+         * @param manifestUrl the URL of the manifest this stream comes from or {@code null}
+         * @return this {@link Builder} instance
+         */
+        public Builder setManifestUrl(@Nullable final String manifestUrl) {
+            this.manifestUrl = manifestUrl;
+            return this;
+        }
+
+        /**
+         * Set the language code of the {@link SubtitlesStream}.
+         *
+         * <p>
+         * It <b>must not be null</b> and should not be an empty string.
+         * </p>
+         *
+         * @param languageCode the language code of the {@link SubtitlesStream}
+         * @return this {@link Builder} instance
+         */
+        public Builder setLanguageCode(@Nonnull final String languageCode) {
+            this.languageCode = languageCode;
+            return this;
+        }
+
+        /**
+         * Set whether the subtitles have been auto-generated by the streaming service.
+         *
+         * @param autoGenerated whether the subtitles have been generated by the streaming
+         *                      service
+         * @return this {@link Builder} instance
+         */
+        public Builder setAutoGenerated(final boolean autoGenerated) {
+            this.autoGenerated = autoGenerated;
+            return this;
+        }
+
+        /**
+         * Build a {@link SubtitlesStream} using the builder's current values.
+         *
+         * <p>
+         * The content (and so the {@code isUrl} boolean), the language code and the {@code
+         * isAutoGenerated} properties must have been set.
+         * </p>
+         *
+         * <p>
+         * If no identifier has been set, an identifier will be generated using the language code
+         * and the media format suffix, if the media format is known.
+         * </p>
+         *
+         * @return a new {@link SubtitlesStream} using the builder's current values
+         * @throws IllegalStateException if {@code id}, {@code content} (and so {@code isUrl}),
+         * {@code deliveryMethod}, {@code languageCode} or the {@code isAutogenerated} have been
+         * not set, or have been set as {@code null}
+         */
+        @Nonnull
+        public SubtitlesStream build() {
+            if (content == null) {
+                throw new IllegalStateException("No valid content was specified. Please specify a "
+                        + "valid one with setContent.");
+            }
+
+            if (deliveryMethod == null) {
+                throw new IllegalStateException(
+                        "The delivery method of the subtitles stream has been set as null, which "
+                                + "is not allowed. Pass a valid one instead with"
+                                + "setDeliveryMethod.");
+            }
+
+            if (languageCode == null) {
+                throw new IllegalStateException("The language code of the subtitles stream has "
+                        + "been not set or is null. Make sure you specified an non null language "
+                        + "code with setLanguageCode.");
+            }
+
+            if (autoGenerated == null) {
+                throw new IllegalStateException("The subtitles stream has been not set as an "
+                        + "autogenerated subtitles stream or not. Please specify this information "
+                        + "with setIsAutoGenerated.");
+            }
+
+            if (id == null) {
+                id = languageCode + (mediaFormat != null ? "." + mediaFormat.suffix
+                        : EMPTY_STRING);
+            }
+
+            return new SubtitlesStream(id, content, isUrl, mediaFormat, deliveryMethod,
+                    languageCode, autoGenerated, manifestUrl);
+        }
+    }
+
+    /**
+     * Create a new subtitles stream.
+     *
+     * @param id             the identifier which uniquely identifies the stream, e.g. for YouTube
+     *                       this would be the itag
+     * @param content        the content or the URL of the stream, depending on whether isUrl is
+     *                       true
+     * @param isUrl          whether content is the URL or the actual content of e.g. a DASH
+     *                       manifest
+     * @param mediaFormat    the {@link MediaFormat} used by the stream
+     * @param deliveryMethod the {@link DeliveryMethod} of the stream
+     * @param languageCode   the language code of the stream
+     * @param autoGenerated  whether the subtitles are auto-generated by the streaming service
+     * @param manifestUrl    the URL of the manifest this stream comes from (if applicable,
+     *                       otherwise null)
+     */
+    @SuppressWarnings("checkstyle:ParameterNumber")
+    private SubtitlesStream(@Nonnull final String id,
+                            @Nonnull final String content,
+                            final boolean isUrl,
+                            @Nullable final MediaFormat mediaFormat,
+                            @Nonnull final DeliveryMethod deliveryMethod,
+                            @Nonnull final String languageCode,
+                            final boolean autoGenerated,
+                            @Nullable final String manifestUrl) {
+        super(id, content, isUrl, mediaFormat, deliveryMethod, manifestUrl);
 
         /*
-        * Locale.forLanguageTag only for API >= 21
+         * Locale.forLanguageTag only for Android API >= 21
-        * Locale.Builder only for API >= 21
+         * Locale.Builder only for Android API >= 21
-        * Country codes doesn't work well without
+         * Country codes doesn't work well without
-        */
+         */
         final String[] splits = languageCode.split("-");
         switch (splits.length) {
-            default:
-                this.locale = new Locale(splits[0]);
-                break;
-            case 3:
-                // complex variants doesn't work!
-                this.locale = new Locale(splits[0], splits[1], splits[2]);
-                break;
             case 2:
                 this.locale = new Locale(splits[0], splits[1]);
                 break;
+            case 3:
+                // Complex variants don't work!
+                this.locale = new Locale(splits[0], splits[1], splits[2]);
+                break;
+            default:
+                this.locale = new Locale(splits[0]);
+                break;
         }
+
         this.code = languageCode;
-        this.format = format;
+        this.format = mediaFormat;
         this.autoGenerated = autoGenerated;
     }
 
+    /**
+     * Get the extension of the subtitles.
+     *
+     * @return the extension of the subtitles
+     */
     public String getExtension() {
         return format.suffix;
     }
 
+    /**
+     * Return whether if the subtitles are auto-generated.
+     * <p>
+     * Some streaming services can generate subtitles for their contents, like YouTube.
+     * </p>
+     *
+     * @return {@code true} if the subtitles are auto-generated, {@code false} otherwise
+     */
     public boolean isAutoGenerated() {
         return autoGenerated;
     }
 
+    /**
+     * {@inheritDoc}
+     */
     @Override
     public boolean equalStats(final Stream cmp) {
         return super.equalStats(cmp)
@@ -56,16 +289,42 @@ public class SubtitlesStream extends Stream implements Serializable {
                 && autoGenerated == ((SubtitlesStream) cmp).autoGenerated;
     }
 
+    /**
+     * Get the display language name of the subtitles.
+     *
+     * @return the display language name of the subtitles
+     */
     public String getDisplayLanguageName() {
         return locale.getDisplayName(locale);
     }
 
+    /**
+     * Get the language tag of the subtitles.
+     *
+     * @return the language tag of the subtitles
+     */
     public String getLanguageTag() {
         return code;
     }
 
+    /**
+     * Get the {@link Locale locale} of the subtitles.
+     *
+     * @return the {@link Locale locale} of the subtitles
+     */
     public Locale getLocale() {
         return locale;
     }
 
+    /**
+     * No subtitles which are currently extracted use an {@link ItagItem}, so {@code null} is
+     * returned by this method.
+     *
+     * @return {@code null}
+     */
+    @Nullable
+    @Override
+    public ItagItem getItagItem() {
+        return null;
+    }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/stream/VideoStream.java

@@ -4,31 +4,41 @@ package org.schabi.newpipe.extractor.stream;
  * Created by Christian Schabesberger on 04.03.16.
  *
  * Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
- * VideoStream.java is part of NewPipe.
+ * VideoStream.java is part of NewPipe Extractor.
  *
- * NewPipe is free software: you can redistribute it and/or modify
+ * NewPipe Extractor is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
  * the Free Software Foundation, either version 3 of the License, or
  * (at your option) any later version.
  *
- * NewPipe is distributed in the hope that it will be useful,
+ * NewPipe Extractor is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
- * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ * along with NewPipe Extractor. If not, see <https://www.gnu.org/licenses/>.
  */
 
 import org.schabi.newpipe.extractor.MediaFormat;
 import org.schabi.newpipe.extractor.services.youtube.ItagItem;
 
-public class VideoStream extends Stream {
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+public final class VideoStream extends Stream {
+    public static final String RESOLUTION_UNKNOWN = "";
+
+    /** @deprecated Use {@link #getResolution()} instead. */
+    @Deprecated
     public final String resolution;
+
+    /** @deprecated Use {@link #isVideoOnly()} instead. */
+    @Deprecated
     public final boolean isVideoOnly;
 
-    // Fields for Dash
+    // Fields for DASH
-    private int itag;
+    private int itag = ITAG_NOT_AVAILABLE_OR_NOT_APPLICABLE;
     private int bitrate;
     private int initStart;
     private int initEnd;
@@ -39,118 +49,437 @@ public class VideoStream extends Stream {
     private int fps;
     private String quality;
     private String codec;
+    @Nullable private ItagItem itagItem;
 
-    public VideoStream(final String url, final MediaFormat format, final String resolution) {
+    /**
-        this(url, format, resolution, false);
+     * Class to build {@link VideoStream} objects.
+     */
+    @SuppressWarnings("checkstyle:hiddenField")
+    public static final class Builder {
+        private String id;
+        private String content;
+        private boolean isUrl;
+        private DeliveryMethod deliveryMethod = DeliveryMethod.PROGRESSIVE_HTTP;
+        @Nullable
+        private MediaFormat mediaFormat;
+        @Nullable
+        private String manifestUrl;
+        // Use of the Boolean class instead of the primitive type needed for setter call check
+        private Boolean isVideoOnly;
+        private String resolution;
+        @Nullable
+        private ItagItem itagItem;
+
+        /**
+         * Create a new {@link Builder} instance with its default values.
+         */
+        public Builder() {
+        }
+
+        /**
+         * Set the identifier of the {@link VideoStream}.
+         *
+         * <p>
+         * It must not be null, and should be non empty.
+         * </p>
+         *
+         * <p>
+         * If you are not able to get an identifier, use the static constant {@link
+         * Stream#ID_UNKNOWN ID_UNKNOWN} of the {@link Stream} class.
+         * </p>
+         *
+         * @param id the identifier of the {@link VideoStream}, which must not be null
+         * @return this {@link Builder} instance
+         */
+        public Builder setId(@Nonnull final String id) {
+            this.id = id;
+            return this;
+        }
+
+        /**
+         * Set the content of the {@link VideoStream}.
+         *
+         * <p>
+         * It must not be null, and should be non empty.
+         * </p>
+         *
+         * @param content the content of the {@link VideoStream}
+         * @param isUrl   whether the content is a URL
+         * @return this {@link Builder} instance
+         */
+        public Builder setContent(@Nonnull final String content,
+                                  final boolean isUrl) {
+            this.content = content;
+            this.isUrl = isUrl;
+            return this;
+        }
+
+        /**
+         * Set the {@link MediaFormat} used by the {@link VideoStream}.
+         *
+         * <p>
+         * It should be one of the video {@link MediaFormat}s ({@link MediaFormat#MPEG_4 MPEG_4},
+         * {@link MediaFormat#v3GPP v3GPP}, or {@link MediaFormat#WEBM WEBM}) but can be {@code
+         * null} if the media format could not be determined.
+         * </p>
+         *
+         * <p>
+         * The default value is {@code null}.
+         * </p>
+         *
+         * @param mediaFormat the {@link MediaFormat} of the {@link VideoStream}, which can be null
+         * @return this {@link Builder} instance
+         */
+        public Builder setMediaFormat(@Nullable final MediaFormat mediaFormat) {
+            this.mediaFormat = mediaFormat;
+            return this;
+        }
+
+        /**
+         * Set the {@link DeliveryMethod} of the {@link VideoStream}.
+         *
+         * <p>
+         * It must not be null.
+         * </p>
+         *
+         * <p>
+         * The default delivery method is {@link DeliveryMethod#PROGRESSIVE_HTTP}.
+         * </p>
+         *
+         * @param deliveryMethod the {@link DeliveryMethod} of the {@link VideoStream}, which must
+         *                       not be null
+         * @return this {@link Builder} instance
+         */
+        public Builder setDeliveryMethod(@Nonnull final DeliveryMethod deliveryMethod) {
+            this.deliveryMethod = deliveryMethod;
+            return this;
+        }
+
+        /**
+         * Sets the URL of the manifest this stream comes from (if applicable, otherwise null).
+         *
+         * @param manifestUrl the URL of the manifest this stream comes from or {@code null}
+         * @return this {@link Builder} instance
+         */
+        public Builder setManifestUrl(@Nullable final String manifestUrl) {
+            this.manifestUrl = manifestUrl;
+            return this;
+        }
+
+        /**
+         * Set whether the {@link VideoStream} is video-only.
+         *
+         * <p>
+         * This property must be set before building the {@link VideoStream}.
+         * </p>
+         *
+         * @param isVideoOnly whether the {@link VideoStream} is video-only
+         * @return this {@link Builder} instance
+         */
+        public Builder setIsVideoOnly(final boolean isVideoOnly) {
+            this.isVideoOnly = isVideoOnly;
+            return this;
+        }
+
+        /**
+         * Set the resolution of the {@link VideoStream}.
+         *
+         * <p>
+         * This resolution can be used by clients to know the quality of the video stream.
+         * </p>
+         *
+         * <p>
+         * If you are not able to know the resolution, you should use {@link #RESOLUTION_UNKNOWN}
+         * as the resolution of the video stream.
+         * </p>
+         *
+         * <p>
+         * It must be set before building the builder and not null.
+         * </p>
+         *
+         * @param resolution the resolution of the {@link VideoStream}
+         * @return this {@link Builder} instance
+         */
+        public Builder setResolution(@Nonnull final String resolution) {
+            this.resolution = resolution;
+            return this;
+        }
+
+        /**
+         * Set the {@link ItagItem} corresponding to the {@link VideoStream}.
+         *
+         * <p>
+         * {@link ItagItem}s are YouTube specific objects, so they are only known for this service
+         * and can be null.
+         * </p>
+         *
+         * <p>
+         * The default value is {@code null}.
+         * </p>
+         *
+         * @param itagItem the {@link ItagItem} of the {@link VideoStream}, which can be null
+         * @return this {@link Builder} instance
+         */
+        public Builder setItagItem(@Nullable final ItagItem itagItem) {
+            this.itagItem = itagItem;
+            return this;
+        }
+
+        /**
+         * Build a {@link VideoStream} using the builder's current values.
+         *
+         * <p>
+         * The identifier, the content (and so the {@code isUrl} boolean), the {@code isVideoOnly}
+         * and the {@code resolution} properties must have been set.
+         * </p>
+         *
+         * @return a new {@link VideoStream} using the builder's current values
+         * @throws IllegalStateException if {@code id}, {@code content} (and so {@code isUrl}),
+         * {@code deliveryMethod}, {@code isVideoOnly} or {@code resolution} have been not set, or
+         * have been set as {@code null}
+         */
+        @Nonnull
+        public VideoStream build() {
+            if (id == null) {
+                throw new IllegalStateException(
+                        "The identifier of the video stream has been not set or is null. If you "
+                                + "are not able to get an identifier, use the static constant "
+                                + "ID_UNKNOWN of the Stream class.");
+            }
+
+            if (content == null) {
+                throw new IllegalStateException("The content of the video stream has been not set "
+                        + "or is null. Please specify a non-null one with setContent.");
+            }
+
+            if (deliveryMethod == null) {
+                throw new IllegalStateException(
+                        "The delivery method of the video stream has been set as null, which is "
+                                + "not allowed. Pass a valid one instead with setDeliveryMethod.");
+            }
+
+            if (isVideoOnly == null) {
+                throw new IllegalStateException("The video stream has been not set as a "
+                        + "video-only stream or as a video stream with embedded audio. Please "
+                        + "specify this information with setIsVideoOnly.");
+            }
+
+            if (resolution == null) {
+                throw new IllegalStateException(
+                        "The resolution of the video stream has been not set. Please specify it "
+                                + "with setResolution (use an empty string if you are not able to "
+                                + "get it).");
+            }
+
+            return new VideoStream(id, content, isUrl, mediaFormat, deliveryMethod, resolution,
+                    isVideoOnly, manifestUrl, itagItem);
+        }
     }
 
-    public VideoStream(final String url,
+    /**
-                       final MediaFormat format,
+     * Create a new video stream.
-                       final String resolution,
+     *
-                       final boolean isVideoOnly) {
+     * @param id             the identifier which uniquely identifies the stream, e.g. for YouTube
-        this(url, null, format, resolution, isVideoOnly);
+     *                       this would be the itag
-    }
+     * @param content        the content or the URL of the stream, depending on whether isUrl is
-
+     *                       true
-    public VideoStream(final String url, final boolean isVideoOnly, final ItagItem itag) {
+     * @param isUrl          whether content is the URL or the actual content of e.g. a DASH
-        this(url, itag.getMediaFormat(), itag.resolutionString, isVideoOnly);
+     *                       manifest
-        this.itag = itag.id;
+     * @param format         the {@link MediaFormat} used by the stream, which can be null
-        this.bitrate = itag.getBitrate();
+     * @param deliveryMethod the {@link DeliveryMethod} of the stream
-        this.initStart = itag.getInitStart();
+     * @param resolution     the resolution of the stream
-        this.initEnd = itag.getInitEnd();
+     * @param isVideoOnly    whether the stream is video-only
-        this.indexStart = itag.getIndexStart();
+     * @param itagItem       the {@link ItagItem} corresponding to the stream, which cannot be null
-        this.indexEnd = itag.getIndexEnd();
+     * @param manifestUrl    the URL of the manifest this stream comes from (if applicable,
-        this.codec = itag.getCodec();
+     *                       otherwise null)
-        this.height = itag.getHeight();
+     */
-        this.width = itag.getWidth();
+    @SuppressWarnings("checkstyle:ParameterNumber")
-        this.quality = itag.getQuality();
+    private VideoStream(@Nonnull final String id,
-        this.fps = itag.fps;
+                        @Nonnull final String content,
-    }
+                        final boolean isUrl,
-
+                        @Nullable final MediaFormat format,
-    public VideoStream(final String url,
+                        @Nonnull final DeliveryMethod deliveryMethod,
-                       final String torrentUrl,
+                        @Nonnull final String resolution,
-                       final MediaFormat format,
+                        final boolean isVideoOnly,
-                       final String resolution) {
+                        @Nullable final String manifestUrl,
-        this(url, torrentUrl, format, resolution, false);
+                        @Nullable final ItagItem itagItem) {
-    }
+        super(id, content, isUrl, format, deliveryMethod, manifestUrl);
-
+        if (itagItem != null) {
-    public VideoStream(final String url,
+            this.itagItem = itagItem;
-                       final String torrentUrl,
+            this.itag = itagItem.id;
-                       final MediaFormat format,
+            this.bitrate = itagItem.getBitrate();
-                       final String resolution,
+            this.initStart = itagItem.getInitStart();
-                       final boolean isVideoOnly) {
+            this.initEnd = itagItem.getInitEnd();
-        super(url, torrentUrl, format);
+            this.indexStart = itagItem.getIndexStart();
+            this.indexEnd = itagItem.getIndexEnd();
+            this.codec = itagItem.getCodec();
+            this.height = itagItem.getHeight();
+            this.width = itagItem.getWidth();
+            this.quality = itagItem.getQuality();
+            this.fps = itagItem.getFps();
+        }
         this.resolution = resolution;
         this.isVideoOnly = isVideoOnly;
     }
 
+    /**
+     * {@inheritDoc}
+     */
     @Override
     public boolean equalStats(final Stream cmp) {
-        return super.equalStats(cmp) && cmp instanceof VideoStream
+        return super.equalStats(cmp)
+                && cmp instanceof VideoStream
                 && resolution.equals(((VideoStream) cmp).resolution)
                 && isVideoOnly == ((VideoStream) cmp).isVideoOnly;
     }
 
     /**
-     * Get the video resolution
+     * Get the video resolution.
      *
-     * @return the video resolution
+     * <p>
+     * It can be unknown for some streams, like for HLS master playlists. In this case,
+     * {@link #RESOLUTION_UNKNOWN} is returned by this method.
+     * </p>
+     *
+     * @return the video resolution or {@link #RESOLUTION_UNKNOWN}
      */
+    @Nonnull
     public String getResolution() {
         return resolution;
     }
 
     /**
-     * Check if the video is video only.
+     * Return whether the stream is video-only.
-     * <p>
-     * Video only streams have no audio
      *
-     * @return {@code true} if this stream is vid
+     * <p>
+     * Video-only streams have no audio.
+     * </p>
+     *
+     * @return {@code true} if this stream is video-only, {@code false} otherwise
      */
     public boolean isVideoOnly() {
         return isVideoOnly;
     }
 
+    /**
+     * Get the itag identifier of the stream.
+     *
+     * <p>
+     * Always equals to {@link #ITAG_NOT_AVAILABLE_OR_NOT_APPLICABLE} for other streams than the
+     * ones of the YouTube service.
+     * </p>
+     *
+     * @return the number of the {@link ItagItem} passed in the constructor of the video stream.
+     */
     public int getItag() {
         return itag;
     }
 
+    /**
+     * Get the bitrate of the stream.
+     *
+     * @return the bitrate set from the {@link ItagItem} passed in the constructor of the stream.
+     */
     public int getBitrate() {
         return bitrate;
     }
 
+    /**
+     * Get the initialization start of the stream.
+     *
+     * @return the initialization start value set from the {@link ItagItem} passed in the
+     * constructor of the
+     * stream.
+     */
     public int getInitStart() {
         return initStart;
     }
 
+    /**
+     * Get the initialization end of the stream.
+     *
+     * @return the initialization end value set from the {@link ItagItem} passed in the constructor
+     * of the stream.
+     */
     public int getInitEnd() {
         return initEnd;
     }
 
+    /**
+     * Get the index start of the stream.
+     *
+     * @return the index start value set from the {@link ItagItem} passed in the constructor of the
+     * stream.
+     */
     public int getIndexStart() {
         return indexStart;
     }
 
+    /**
+     * Get the index end of the stream.
+     *
+     * @return the index end value set from the {@link ItagItem} passed in the constructor of the
+     * stream.
+     */
     public int getIndexEnd() {
         return indexEnd;
     }
 
+    /**
+     * Get the width of the video stream.
+     *
+     * @return the width set from the {@link ItagItem} passed in the constructor of the
+     * stream.
+     */
     public int getWidth() {
         return width;
     }
 
+    /**
+     * Get the height of the video stream.
+     *
+     * @return the height set from the {@link ItagItem} passed in the constructor of the
+     * stream.
+     */
     public int getHeight() {
         return height;
     }
 
+    /**
+     * Get the frames per second of the video stream.
+     *
+     * @return the frames per second set from the {@link ItagItem} passed in the constructor of the
+     * stream.
+     */
     public int getFps() {
         return fps;
     }
 
+    /**
+     * Get the quality of the stream.
+     *
+     * @return the quality label set from the {@link ItagItem} passed in the constructor of the
+     * stream.
+     */
     public String getQuality() {
         return quality;
     }
 
+    /**
+     * Get the codec of the stream.
+     *
+     * @return the codec set from the {@link ItagItem} passed in the constructor of the stream.
+     */
     public String getCodec() {
         return codec;
     }
+
+    /**
+     * {@inheritDoc}
+     */
+    @Override
+    @Nullable
+    public ItagItem getItagItem() {
+        return itagItem;
+    }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/utils/DashMpdParser.java

@@ -1,225 +0,0 @@
-package org.schabi.newpipe.extractor.utils;
-
-import org.schabi.newpipe.extractor.MediaFormat;
-import org.schabi.newpipe.extractor.NewPipe;
-import org.schabi.newpipe.extractor.downloader.Downloader;
-import org.schabi.newpipe.extractor.exceptions.ParsingException;
-import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
-import org.schabi.newpipe.extractor.services.youtube.ItagItem;
-import org.schabi.newpipe.extractor.stream.AudioStream;
-import org.schabi.newpipe.extractor.stream.Stream;
-import org.schabi.newpipe.extractor.stream.StreamInfo;
-import org.schabi.newpipe.extractor.stream.VideoStream;
-import org.w3c.dom.Document;
-import org.w3c.dom.Element;
-import org.w3c.dom.Node;
-import org.w3c.dom.NodeList;
-
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.List;
-
-/*
- * Created by Christian Schabesberger on 02.02.16.
- *
- * Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
- * DashMpdParser.java is part of NewPipe.
- *
- * NewPipe is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * NewPipe is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-public final class DashMpdParser {
-
-    private DashMpdParser() {
-    }
-
-    public static class DashMpdParsingException extends ParsingException {
-        DashMpdParsingException(final String message, final Exception e) {
-            super(message, e);
-        }
-    }
-
-    public static class ParserResult {
-        private final List<VideoStream> videoStreams;
-        private final List<AudioStream> audioStreams;
-        private final List<VideoStream> videoOnlyStreams;
-
-        private final List<VideoStream> segmentedVideoStreams;
-        private final List<AudioStream> segmentedAudioStreams;
-        private final List<VideoStream> segmentedVideoOnlyStreams;
-
-
-        public ParserResult(final List<VideoStream> videoStreams,
-                            final List<AudioStream> audioStreams,
-                            final List<VideoStream> videoOnlyStreams,
-                            final List<VideoStream> segmentedVideoStreams,
-                            final List<AudioStream> segmentedAudioStreams,
-                            final List<VideoStream> segmentedVideoOnlyStreams) {
-            this.videoStreams = videoStreams;
-            this.audioStreams = audioStreams;
-            this.videoOnlyStreams = videoOnlyStreams;
-            this.segmentedVideoStreams = segmentedVideoStreams;
-            this.segmentedAudioStreams = segmentedAudioStreams;
-            this.segmentedVideoOnlyStreams = segmentedVideoOnlyStreams;
-        }
-
-        public List<VideoStream> getVideoStreams() {
-            return videoStreams;
-        }
-
-        public List<AudioStream> getAudioStreams() {
-            return audioStreams;
-        }
-
-        public List<VideoStream> getVideoOnlyStreams() {
-            return videoOnlyStreams;
-        }
-
-        public List<VideoStream> getSegmentedVideoStreams() {
-            return segmentedVideoStreams;
-        }
-
-        public List<AudioStream> getSegmentedAudioStreams() {
-            return segmentedAudioStreams;
-        }
-
-        public List<VideoStream> getSegmentedVideoOnlyStreams() {
-            return segmentedVideoOnlyStreams;
-        }
-    }
-
-    /**
-     * Will try to download (using {@link StreamInfo#getDashMpdUrl()}) and parse the dash manifest,
-     * then it will search for any stream that the ItagItem has (by the id).
-     * <p>
-     * It has video, video only and audio streams and will only add to the list if it don't
-     * find a similar stream in the respective lists (calling {@link Stream#equalStats}).
-     * <p>
-     * Info about dash MPD can be found
-     * <a href="https://www.brendanlong.com/the-structure-of-an-mpeg-dash-mpd.html">here</a>.
-     *
-     * @param streamInfo where the parsed streams will be added
-     */
-    public static ParserResult getStreams(final StreamInfo streamInfo)
-            throws DashMpdParsingException, ReCaptchaException {
-        final String dashDoc;
-        final Downloader downloader = NewPipe.getDownloader();
-        try {
-            dashDoc = downloader.get(streamInfo.getDashMpdUrl()).responseBody();
-        } catch (final IOException ioe) {
-            throw new DashMpdParsingException(
-                    "Could not get dash mpd: " + streamInfo.getDashMpdUrl(), ioe);
-        }
-
-        try {
-            final DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
-            final DocumentBuilder builder = factory.newDocumentBuilder();
-            final InputStream stream = new ByteArrayInputStream(dashDoc.getBytes());
-
-            final Document doc = builder.parse(stream);
-            final NodeList representationList = doc.getElementsByTagName("Representation");
-
-            final List<VideoStream> videoStreams = new ArrayList<>();
-            final List<AudioStream> audioStreams = new ArrayList<>();
-            final List<VideoStream> videoOnlyStreams = new ArrayList<>();
-
-            final List<VideoStream> segmentedVideoStreams = new ArrayList<>();
-            final List<AudioStream> segmentedAudioStreams = new ArrayList<>();
-            final List<VideoStream> segmentedVideoOnlyStreams = new ArrayList<>();
-
-            for (int i = 0; i < representationList.getLength(); i++) {
-                final Element representation = (Element) representationList.item(i);
-                try {
-                    final String mimeType
-                            = ((Element) representation.getParentNode()).getAttribute("mimeType");
-                    final String id = representation.getAttribute("id");
-                    final String url = representation
-                            .getElementsByTagName("BaseURL").item(0).getTextContent();
-                    final ItagItem itag = ItagItem.getItag(Integer.parseInt(id));
-                    final Node segmentationList
-                            = representation.getElementsByTagName("SegmentList").item(0);
-
-                    // If SegmentList is not null this means that BaseUrl is not representing the
-                    // url to the stream. Instead we need to add the "media=" value from the
-                    // <SegementURL/> tags inside the <SegmentList/> tag in order to get a full
-                    // working url. However each of these is just pointing to a part of the video,
-                    // so we can not return a URL with a working stream here. Instead of putting
-                    // those streams into the list of regular stream urls we put them in a for
-                    // example "segmentedVideoStreams" list.
-
-                    final MediaFormat mediaFormat = MediaFormat.getFromMimeType(mimeType);
-
-                    if (itag.itagType.equals(ItagItem.ItagType.AUDIO)) {
-                        if (segmentationList == null) {
-                            final AudioStream audioStream
-                                    = new AudioStream(url, mediaFormat, itag.avgBitrate);
-                            if (!Stream.containSimilarStream(audioStream,
-                                    streamInfo.getAudioStreams())) {
-                                audioStreams.add(audioStream);
-                            }
-                        } else {
-                            segmentedAudioStreams.add(
-                                    new AudioStream(id, mediaFormat, itag.avgBitrate));
-                        }
-                    } else {
-                        final boolean isVideoOnly
-                                = itag.itagType.equals(ItagItem.ItagType.VIDEO_ONLY);
-
-                        if (segmentationList == null) {
-                            final VideoStream videoStream = new VideoStream(url,
-                                    mediaFormat,
-                                    itag.resolutionString,
-                                    isVideoOnly);
-
-                            if (isVideoOnly) {
-                                if (!Stream.containSimilarStream(videoStream,
-                                        streamInfo.getVideoOnlyStreams())) {
-                                    videoOnlyStreams.add(videoStream);
-                                }
-                            } else if (!Stream.containSimilarStream(videoStream,
-                                    streamInfo.getVideoStreams())) {
-                                videoStreams.add(videoStream);
-                            }
-                        } else {
-                            final VideoStream videoStream = new VideoStream(id,
-                                    mediaFormat,
-                                    itag.resolutionString,
-                                    isVideoOnly);
-
-                            if (isVideoOnly) {
-                                segmentedVideoOnlyStreams.add(videoStream);
-                            } else {
-                                segmentedVideoStreams.add(videoStream);
-                            }
-                        }
-                    }
-                } catch (final Exception ignored) {
-                }
-            }
-            return new ParserResult(
-                    videoStreams,
-                    audioStreams,
-                    videoOnlyStreams,
-                    segmentedVideoStreams,
-                    segmentedAudioStreams,
-                    segmentedVideoOnlyStreams);
-        } catch (final Exception e) {
-            throw new DashMpdParsingException("Could not parse Dash mpd", e);
-        }
-    }
-}

extractor/src/main/java/org/schabi/newpipe/extractor/utils/ManifestCreatorCache.java

@@ -0,0 +1,255 @@
+package org.schabi.newpipe.extractor.utils;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * A {@link Serializable serializable} cache class used by the extractor to cache manifests
+ * generated with extractor's manifests generators.
+ *
+ * <p>
+ * It relies internally on a {@link ConcurrentHashMap} to allow concurrent access to the cache.
+ * </p>
+ *
+ * @param <K> the type of cache keys, which must be {@link Serializable serializable}
+ * @param <V> the type of the second element of {@link Pair pairs} used as values of the cache,
+ *            which must be {@link Serializable serializable}
+ */
+public final class ManifestCreatorCache<K extends Serializable, V extends Serializable>
+        implements Serializable {
+
+    /**
+     * The default maximum size of a manifest cache.
+     */
+    public static final int DEFAULT_MAXIMUM_SIZE = Integer.MAX_VALUE;
+
+    /**
+     * The default clear factor of a manifest cache.
+     */
+    public static final double DEFAULT_CLEAR_FACTOR = 0.75;
+
+    /**
+     * The {@link ConcurrentHashMap} used internally as the cache of manifests.
+     */
+    private final ConcurrentHashMap<K, Pair<Integer, V>> concurrentHashMap;
+
+    /**
+     * The maximum size of the cache.
+     *
+     * <p>
+     * The default value is {@link #DEFAULT_MAXIMUM_SIZE}.
+     * </p>
+     */
+    private int maximumSize = DEFAULT_MAXIMUM_SIZE;
+
+    /**
+     * The clear factor of the cache, which is a double between {@code 0} and {@code 1} excluded.
+     *
+     * <p>
+     * The default value is {@link #DEFAULT_CLEAR_FACTOR}.
+     * </p>
+     */
+    private double clearFactor = DEFAULT_CLEAR_FACTOR;
+
+    /**
+     * Creates a new {@link ManifestCreatorCache}.
+     */
+    public ManifestCreatorCache() {
+        concurrentHashMap = new ConcurrentHashMap<>();
+    }
+
+    /**
+     * Tests if the specified key is in the cache.
+     *
+     * @param key the key to test its presence in the cache
+     * @return {@code true} if the key is in the cache, {@code false} otherwise.
+     */
+    public boolean containsKey(final K key) {
+        return concurrentHashMap.containsKey(key);
+    }
+
+    /**
+     * Returns the value to which the specified key is mapped, or {@code null} if the cache
+     * contains no mapping for the key.
+     *
+     * @param key the key to which getting its value
+     * @return the value to which the specified key is mapped, or {@code null}
+     */
+    @Nullable
+    public Pair<Integer, V> get(final K key) {
+        return concurrentHashMap.get(key);
+    }
+
+    /**
+     * Adds a new element to the cache.
+     *
+     * <p>
+     * If the cache limit is reached, oldest elements will be cleared first using the load factor
+     * and the maximum size.
+     * </p>
+     *
+     * @param key   the key to put
+     * @param value the value to associate to the key
+     *
+     * @return the previous value associated with the key, or {@code null} if there was no mapping
+     * for the key (note that a null return can also indicate that the cache previously associated
+     * {@code null} with the key).
+     */
+    @Nullable
+    public V put(final K key, final V value) {
+        if (!concurrentHashMap.containsKey(key) && concurrentHashMap.size() == maximumSize) {
+            final int newCacheSize = (int) Math.round(maximumSize * clearFactor);
+            keepNewestEntries(newCacheSize != 0 ? newCacheSize : 1);
+        }
+
+        final Pair<Integer, V> returnValue = concurrentHashMap.put(key,
+                new Pair<>(concurrentHashMap.size(), value));
+        return returnValue == null ? null : returnValue.getSecond();
+    }
+
+    /**
+     * Clears the cached manifests.
+     *
+     * <p>
+     * The cache will be empty after this method is called.
+     * </p>
+     */
+    public void clear() {
+        concurrentHashMap.clear();
+    }
+
+    /**
+     * Resets the cache.
+     *
+     * <p>
+     * The cache will be empty and the clear factor and the maximum size will be reset to their
+     * default values.
+     * </p>
+     *
+     * @see #clear()
+     * @see #resetClearFactor()
+     * @see #resetMaximumSize()
+     */
+    public void reset() {
+        clear();
+        resetClearFactor();
+        resetMaximumSize();
+    }
+
+    /**
+     * @return the number of cached manifests in the cache
+     */
+    public int size() {
+        return concurrentHashMap.size();
+    }
+
+    /**
+     * @return the maximum size of the cache
+     */
+    public long getMaximumSize() {
+        return maximumSize;
+    }
+
+    /**
+     * Sets the maximum size of the cache.
+     *
+     * If the current cache size is more than the new maximum size, the percentage of one less the
+     * clear factor of the maximum new size of manifests in the cache will be removed.
+     *
+     * @param maximumSize the new maximum size of the cache
+     * @throws IllegalArgumentException if {@code maximumSize} is less than or equal to 0
+     */
+    public void setMaximumSize(final int maximumSize) {
+        if (maximumSize <= 0) {
+            throw new IllegalArgumentException("Invalid maximum size");
+        }
+
+        if (maximumSize < this.maximumSize && !concurrentHashMap.isEmpty()) {
+            final int newCacheSize = (int) Math.round(maximumSize * clearFactor);
+            keepNewestEntries(newCacheSize != 0 ? newCacheSize : 1);
+        }
+
+        this.maximumSize = maximumSize;
+    }
+
+    /**
+     * Resets the maximum size of the cache to its {@link #DEFAULT_MAXIMUM_SIZE default value}.
+     */
+    public void resetMaximumSize() {
+        this.maximumSize = DEFAULT_MAXIMUM_SIZE;
+    }
+
+    /**
+     * @return the current clear factor of the cache, used when the cache limit size is reached
+     */
+    public double getClearFactor() {
+        return clearFactor;
+    }
+
+    /**
+     * Sets the clear factor of the cache, used when the cache limit size is reached.
+     *
+     * <p>
+     * The clear factor must be a double between {@code 0} excluded and {@code 1} excluded.
+     * </p>
+     *
+     * <p>
+     * Note that it will be only used the next time the cache size limit is reached.
+     * </p>
+     *
+     * @param clearFactor the new clear factor of the cache
+     * @throws IllegalArgumentException if the clear factor passed a parameter is invalid
+     */
+    public void setClearFactor(final double clearFactor) {
+        if (clearFactor <= 0 || clearFactor >= 1) {
+            throw new IllegalArgumentException("Invalid clear factor");
+        }
+
+        this.clearFactor = clearFactor;
+    }
+
+    /**
+     * Resets the clear factor to its {@link #DEFAULT_CLEAR_FACTOR default value}.
+     */
+    public void resetClearFactor() {
+        this.clearFactor = DEFAULT_CLEAR_FACTOR;
+    }
+
+    @Nonnull
+    @Override
+    public String toString() {
+        return "ManifestCreatorCache[clearFactor=" + clearFactor + ", maximumSize=" + maximumSize
+                + ", concurrentHashMap=" + concurrentHashMap + "]";
+    }
+
+    /**
+     * Keeps only the newest entries in a cache.
+     *
+     * <p>
+     * This method will first collect the entries to remove by looping through the concurrent hash
+     * map
+     * </p>
+     *
+     * @param newLimit the new limit of the cache
+     */
+    private void keepNewestEntries(final int newLimit) {
+        final int difference = concurrentHashMap.size() - newLimit;
+        final ArrayList<Map.Entry<K, Pair<Integer, V>>> entriesToRemove = new ArrayList<>();
+
+        concurrentHashMap.entrySet().forEach(entry -> {
+            final Pair<Integer, V> value = entry.getValue();
+            if (value.getFirst() < difference) {
+                entriesToRemove.add(entry);
+            } else {
+                value.setFirst(value.getFirst() - difference);
+            }
+        });
+
+        entriesToRemove.forEach(entry -> concurrentHashMap.remove(entry.getKey(),
+                entry.getValue()));
+    }
+}

extractor/src/test/java/org/schabi/newpipe/extractor/ExtractorAsserts.java

@@ -15,6 +15,8 @@ import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
+import org.schabi.newpipe.extractor.utils.Utils;
+
 public class ExtractorAsserts {
     public static void assertEmptyErrors(String message, List<Throwable> errors) {
         if (!errors.isEmpty()) {
@@ -64,6 +66,14 @@ public class ExtractorAsserts {
         }
     }
 
+    public static void assertNotBlank(String stringToCheck) {
+        assertNotBlank(stringToCheck, null);
+    }
+
+    public static void assertNotBlank(String stringToCheck, @Nullable String message) {
+        assertFalse(Utils.isBlank(stringToCheck), message);
+    }
+
     public static void assertGreater(final long expected, final long actual) {
         assertGreater(expected, actual, actual + " is not > " + expected);
     }

extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultStreamExtractorTest.java

@@ -271,13 +271,20 @@ public abstract class DefaultStreamExtractorTest extends DefaultExtractorTest<St
             assertFalse(videoStreams.isEmpty());
 
             for (final VideoStream stream : videoStreams) {
-                assertIsSecureUrl(stream.getUrl());
+                if (stream.isUrl()) {
-                assertFalse(stream.getResolution().isEmpty());
+                    assertIsSecureUrl(stream.getContent());
-
+                }
-                final int formatId = stream.getFormatId();
+                final StreamType streamType = extractor().getStreamType();
-                // see MediaFormat: video stream formats range from 0 to 0x100
+                // On some video streams, the resolution can be empty and the format be unknown,
-                assertTrue(0 <= formatId && formatId < 0x100,
+                // especially on livestreams (like streams with HLS master playlists)
-                        "format id does not fit a video stream: " + formatId);
+                if (streamType != StreamType.LIVE_STREAM
+                        && streamType != StreamType.AUDIO_LIVE_STREAM) {
+                    assertFalse(stream.getResolution().isEmpty());
+                    final int formatId = stream.getFormatId();
+                    // see MediaFormat: video stream formats range from 0 to 0x100
+                    assertTrue(0 <= formatId && formatId < 0x100,
+                            "Format id does not fit a video stream: " + formatId);
+                }
             }
         } else {
             assertTrue(videoStreams.isEmpty());
@@ -294,12 +301,17 @@ public abstract class DefaultStreamExtractorTest extends DefaultExtractorTest<St
             assertFalse(audioStreams.isEmpty());
 
             for (final AudioStream stream : audioStreams) {
-                assertIsSecureUrl(stream.getUrl());
+                if (stream.isUrl()) {
+                    assertIsSecureUrl(stream.getContent());
+                }
 
-                final int formatId = stream.getFormatId();
+                // The media format can be unknown on some audio streams
-                // see MediaFormat: video stream formats range from 0x100 to 0x1000
+                if (stream.getFormat() != null) {
-                assertTrue(0x100 <= formatId && formatId < 0x1000,
+                    final int formatId = stream.getFormat().id;
-                        "format id does not fit an audio stream: " + formatId);
+                    // see MediaFormat: audio stream formats range from 0x100 to 0x1000
+                    assertTrue(0x100 <= formatId && formatId < 0x1000,
+                            "Format id does not fit an audio stream: " + formatId);
+                }
             }
         } else {
             assertTrue(audioStreams.isEmpty());
@@ -316,12 +328,14 @@ public abstract class DefaultStreamExtractorTest extends DefaultExtractorTest<St
             assertFalse(subtitles.isEmpty());
 
             for (final SubtitlesStream stream : subtitles) {
-                assertIsSecureUrl(stream.getUrl());
+                if (stream.isUrl()) {
+                    assertIsSecureUrl(stream.getContent());
+                }
 
                 final int formatId = stream.getFormatId();
                 // see MediaFormat: video stream formats range from 0x1000 to 0x10000
                 assertTrue(0x1000 <= formatId && formatId < 0x10000,
-                        "format id does not fit a subtitles stream: " + formatId);
+                        "Format id does not fit a subtitles stream: " + formatId);
             }
         } else {
             assertTrue(subtitles.isEmpty());
@@ -344,7 +358,8 @@ public abstract class DefaultStreamExtractorTest extends DefaultExtractorTest<St
             assertTrue(dashMpdUrl.isEmpty());
         } else {
             assertIsSecureUrl(dashMpdUrl);
-            ExtractorAsserts.assertContains(expectedDashMpdUrlContains(), extractor().getDashMpdUrl());
+            ExtractorAsserts.assertContains(expectedDashMpdUrlContains(),
+                    extractor().getDashMpdUrl());
         }
     }
 

extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java

@@ -12,6 +12,7 @@ import org.schabi.newpipe.extractor.exceptions.GeographicRestrictionException;
 import org.schabi.newpipe.extractor.exceptions.SoundCloudGoPlusContentException;
 import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest;
 import org.schabi.newpipe.extractor.stream.AudioStream;
+import org.schabi.newpipe.extractor.stream.DeliveryMethod;
 import org.schabi.newpipe.extractor.stream.StreamExtractor;
 import org.schabi.newpipe.extractor.stream.StreamType;
 
@@ -21,7 +22,7 @@ import java.util.List;
 
 import javax.annotation.Nullable;
 
-import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.*;
 import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
 
 public class SoundcloudStreamExtractorTest {
@@ -187,18 +188,27 @@ public class SoundcloudStreamExtractorTest {
             super.testAudioStreams();
             final List<AudioStream> audioStreams = extractor.getAudioStreams();
             assertEquals(2, audioStreams.size());
-            for (final AudioStream audioStream : audioStreams) {
+            audioStreams.forEach(audioStream -> {
-                final String mediaUrl = audioStream.getUrl();
+                final DeliveryMethod deliveryMethod = audioStream.getDeliveryMethod();
+                final String mediaUrl = audioStream.getContent();
                 if (audioStream.getFormat() == MediaFormat.OPUS) {
-                    // assert that it's an OPUS 64 kbps media URL with a single range which comes from an HLS SoundCloud CDN
+                    // Assert that it's an OPUS 64 kbps media URL with a single range which comes
+                    // from an HLS SoundCloud CDN
                     ExtractorAsserts.assertContains("-hls-opus-media.sndcdn.com", mediaUrl);
                     ExtractorAsserts.assertContains(".64.opus", mediaUrl);
+                    assertSame(DeliveryMethod.HLS, deliveryMethod,
+                            "Wrong delivery method for stream " + audioStream.getId() + ": "
+                                    + deliveryMethod);
+                } else if (audioStream.getFormat() == MediaFormat.MP3) {
+                    // Assert that it's a MP3 128 kbps media URL which comes from a progressive
+                    // SoundCloud CDN
+                    ExtractorAsserts.assertContains("-media.sndcdn.com/bKOA7Pwbut93.128.mp3",
+                            mediaUrl);
+                    assertSame(DeliveryMethod.PROGRESSIVE_HTTP, deliveryMethod,
+                            "Wrong delivery method for stream " + audioStream.getId() + ": "
+                                    + deliveryMethod);
                 }
-                if (audioStream.getFormat() == MediaFormat.MP3) {
+            });
-                    // assert that it's a MP3 128 kbps media URL which comes from a progressive SoundCloud CDN
-                    ExtractorAsserts.assertContains("-media.sndcdn.com/bKOA7Pwbut93.128.mp3", mediaUrl);
-                }
-            }
         }
     }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeDashManifestCreatorsTest.java

@@ -0,0 +1,363 @@
+package org.schabi.newpipe.extractor.services.youtube;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.schabi.newpipe.downloader.DownloaderTestImpl;
+import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.CreationException;
+import org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeOtfDashManifestCreator;
+import org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeProgressiveDashManifestCreator;
+import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
+import org.schabi.newpipe.extractor.stream.DeliveryMethod;
+import org.schabi.newpipe.extractor.stream.Stream;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+import org.xml.sax.InputSource;
+
+import javax.annotation.Nonnull;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import java.io.StringReader;
+import java.util.List;
+import java.util.Random;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import static org.junit.jupiter.api.Assertions.assertAll;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.schabi.newpipe.extractor.ExtractorAsserts.assertGreater;
+import static org.schabi.newpipe.extractor.ExtractorAsserts.assertGreaterOrEqual;
+import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsValidUrl;
+import static org.schabi.newpipe.extractor.ExtractorAsserts.assertNotBlank;
+import static org.schabi.newpipe.extractor.ServiceList.YouTube;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.ADAPTATION_SET;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.AUDIO_CHANNEL_CONFIGURATION;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.BASE_URL;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.INITIALIZATION;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.MPD;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.PERIOD;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.REPRESENTATION;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.ROLE;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.SEGMENT_BASE;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.SEGMENT_TEMPLATE;
+import static org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeDashManifestCreatorsUtils.SEGMENT_TIMELINE;
+import static org.schabi.newpipe.extractor.utils.Utils.isBlank;
+
+/**
+ * Test for YouTube DASH manifest creators.
+ *
+ * <p>
+ * Tests the generation of OTF and progressive manifests.
+ * </p>
+ *
+ * <p>
+ * We cannot test the generation of DASH manifests for ended livestreams because these videos will
+ * be re-encoded as normal videos later, so we can't use a specific video.
+ * </p>
+ *
+ * <p>
+ * The generation of DASH manifests for OTF streams, which can be tested, uses a video licenced
+ * under the Creative Commons Attribution licence (reuse allowed): {@code A New Era of Open?
+ * COVID-19 and the Pursuit for Equitable Solutions} (<a href=
+ * "https://www.youtube.com/watch?v=DJ8GQUNUXGM">https://www.youtube.com/watch?v=DJ8GQUNUXGM</a>)
+ * </p>
+ *
+ * <p>
+ * We couldn't use mocks for these tests because the streaming URLs needs to fetched and the IP
+ * address used to get these URLs is required (used as a param in the URLs; without it, video
+ * servers return 403/Forbidden HTTP response code).
+ * </p>
+ *
+ * <p>
+ * So the real downloader will be used everytime on this test class.
+ * </p>
+ */
+class YoutubeDashManifestCreatorsTest {
+    // Setting a higher number may let Google video servers return 403s
+    private static final int MAX_STREAMS_TO_TEST_PER_METHOD = 3;
+    private static final String url = "https://www.youtube.com/watch?v=DJ8GQUNUXGM";
+    private static YoutubeStreamExtractor extractor;
+    private static long videoLength;
+
+    @BeforeAll
+    public static void setUp() throws Exception {
+        YoutubeParsingHelper.resetClientVersionAndKey();
+        YoutubeParsingHelper.setNumberGenerator(new Random(1));
+        NewPipe.init(DownloaderTestImpl.getInstance());
+
+        extractor = (YoutubeStreamExtractor) YouTube.getStreamExtractor(url);
+        extractor.fetchPage();
+        videoLength = extractor.getLength();
+    }
+
+    @Test
+    void testOtfStreams() throws Exception {
+        assertDashStreams(extractor.getVideoOnlyStreams());
+        assertDashStreams(extractor.getAudioStreams());
+
+        // no video stream with audio uses the DASH delivery method (YouTube OTF stream type)
+        assertEquals(0, assertFilterStreams(extractor.getVideoStreams(),
+                DeliveryMethod.DASH).size());
+    }
+
+    @Test
+    void testProgressiveStreams() throws Exception {
+        assertProgressiveStreams(extractor.getVideoOnlyStreams());
+        assertProgressiveStreams(extractor.getAudioStreams());
+
+        // we are not able to generate DASH manifests of video formats with audio
+        assertThrows(CreationException.class,
+                () -> assertProgressiveStreams(extractor.getVideoStreams()));
+    }
+
+    private void assertDashStreams(final List<? extends Stream> streams) throws Exception {
+
+        for (final Stream stream : assertFilterStreams(streams, DeliveryMethod.DASH)) {
+            //noinspection ConstantConditions
+            final String manifest = YoutubeOtfDashManifestCreator.fromOtfStreamingUrl(
+                    stream.getContent(), stream.getItagItem(), videoLength);
+            assertNotBlank(manifest);
+
+            assertManifestGenerated(
+                    manifest,
+                    stream.getItagItem(),
+                    document -> assertAll(
+                            () -> assertSegmentTemplateElement(document),
+                            () -> assertSegmentTimelineAndSElements(document)
+                    )
+            );
+        }
+    }
+
+    private void assertProgressiveStreams(final List<? extends Stream> streams) throws Exception {
+
+        for (final Stream stream : assertFilterStreams(streams, DeliveryMethod.PROGRESSIVE_HTTP)) {
+            //noinspection ConstantConditions
+            final String manifest =
+                    YoutubeProgressiveDashManifestCreator.fromProgressiveStreamingUrl(
+                            stream.getContent(), stream.getItagItem(), videoLength);
+            assertNotBlank(manifest);
+
+            assertManifestGenerated(
+                    manifest,
+                    stream.getItagItem(),
+                    document -> assertAll(
+                            () -> assertBaseUrlElement(document),
+                            () -> assertSegmentBaseElement(document, stream.getItagItem()),
+                            () -> assertInitializationElement(document, stream.getItagItem())
+                    )
+            );
+        }
+    }
+
+    @Nonnull
+    private List<? extends Stream> assertFilterStreams(
+            @Nonnull final List<? extends Stream> streams,
+            final DeliveryMethod deliveryMethod) {
+
+        final List<? extends Stream> filteredStreams = streams.stream()
+                .filter(stream -> stream.getDeliveryMethod() == deliveryMethod)
+                .limit(MAX_STREAMS_TO_TEST_PER_METHOD)
+                .collect(Collectors.toList());
+
+        assertAll(filteredStreams.stream()
+                .flatMap(stream -> java.util.stream.Stream.of(
+                        () -> assertNotBlank(stream.getContent()),
+                        () -> assertNotNull(stream.getItagItem())
+                ))
+        );
+
+        return filteredStreams;
+    }
+
+    private void assertManifestGenerated(final String dashManifest,
+                                         final ItagItem itagItem,
+                                         final Consumer<Document> additionalAsserts)
+            throws Exception {
+
+        final DocumentBuilderFactory documentBuilderFactory = DocumentBuilderFactory
+                .newInstance();
+        final DocumentBuilder documentBuilder = documentBuilderFactory.newDocumentBuilder();
+        final Document document = documentBuilder.parse(new InputSource(
+                new StringReader(dashManifest)));
+
+        assertAll(
+                () -> assertMpdElement(document),
+                () -> assertPeriodElement(document),
+                () -> assertAdaptationSetElement(document, itagItem),
+                () -> assertRoleElement(document),
+                () -> assertRepresentationElement(document, itagItem),
+                () -> {
+                    if (itagItem.itagType.equals(ItagItem.ItagType.AUDIO)) {
+                        assertAudioChannelConfigurationElement(document, itagItem);
+                    }
+                },
+                () -> additionalAsserts.accept(document)
+        );
+    }
+
+    private void assertMpdElement(@Nonnull final Document document) {
+        final Element element = (Element) document.getElementsByTagName(MPD).item(0);
+        assertNotNull(element);
+        assertNull(element.getParentNode().getNodeValue());
+
+        final String mediaPresentationDuration = element.getAttribute("mediaPresentationDuration");
+        assertNotNull(mediaPresentationDuration);
+        assertTrue(mediaPresentationDuration.startsWith("PT"));
+    }
+
+    private void assertPeriodElement(@Nonnull final Document document) {
+        assertGetElement(document, PERIOD, MPD);
+    }
+
+    private void assertAdaptationSetElement(@Nonnull final Document document,
+                                            @Nonnull final ItagItem itagItem) {
+        final Element element = assertGetElement(document, ADAPTATION_SET, PERIOD);
+        assertAttrEquals(itagItem.getMediaFormat().getMimeType(), element, "mimeType");
+    }
+
+    private void assertRoleElement(@Nonnull final Document document) {
+        assertGetElement(document, ROLE, ADAPTATION_SET);
+    }
+
+    private void assertRepresentationElement(@Nonnull final Document document,
+                                             @Nonnull final ItagItem itagItem) {
+        final Element element = assertGetElement(document, REPRESENTATION, ADAPTATION_SET);
+
+        assertAttrEquals(itagItem.getBitrate(), element, "bandwidth");
+        assertAttrEquals(itagItem.getCodec(), element, "codecs");
+
+        if (itagItem.itagType == ItagItem.ItagType.VIDEO_ONLY
+                || itagItem.itagType == ItagItem.ItagType.VIDEO) {
+            assertAttrEquals(itagItem.getFps(), element, "frameRate");
+            assertAttrEquals(itagItem.getHeight(), element, "height");
+            assertAttrEquals(itagItem.getWidth(), element, "width");
+        }
+
+        assertAttrEquals(itagItem.id, element, "id");
+    }
+
+    private void assertAudioChannelConfigurationElement(@Nonnull final Document document,
+                                                        @Nonnull final ItagItem itagItem) {
+        final Element element = assertGetElement(document, AUDIO_CHANNEL_CONFIGURATION,
+                REPRESENTATION);
+        assertAttrEquals(itagItem.getAudioChannels(), element, "value");
+    }
+
+    private void assertSegmentTemplateElement(@Nonnull final Document document) {
+        final Element element = assertGetElement(document, SEGMENT_TEMPLATE, REPRESENTATION);
+
+        final String initializationValue = element.getAttribute("initialization");
+        assertIsValidUrl(initializationValue);
+        assertTrue(initializationValue.endsWith("&sq=0"));
+
+        final String mediaValue = element.getAttribute("media");
+        assertIsValidUrl(mediaValue);
+        assertTrue(mediaValue.endsWith("&sq=$Number$"));
+
+        assertEquals("1", element.getAttribute("startNumber"));
+    }
+
+    private void assertSegmentTimelineAndSElements(@Nonnull final Document document) {
+        final Element element = assertGetElement(document, SEGMENT_TIMELINE, SEGMENT_TEMPLATE);
+        final NodeList childNodes = element.getChildNodes();
+        assertGreater(0, childNodes.getLength());
+
+        assertAll(IntStream.range(0, childNodes.getLength())
+                .mapToObj(childNodes::item)
+                .map(Element.class::cast)
+                .flatMap(sElement -> java.util.stream.Stream.of(
+                        () -> assertEquals("S", sElement.getTagName()),
+                        () -> assertGreater(0, Integer.parseInt(sElement.getAttribute("d"))),
+                        () -> {
+                            final String rValue = sElement.getAttribute("r");
+                            // A segment duration can or can't be repeated, so test the next segment
+                            // if there is no r attribute
+                            if (!isBlank(rValue)) {
+                                assertGreater(0, Integer.parseInt(rValue));
+                            }
+                        }
+                    )
+                )
+        );
+    }
+
+    private void assertBaseUrlElement(@Nonnull final Document document) {
+        final Element element = assertGetElement(document, BASE_URL, REPRESENTATION);
+        assertIsValidUrl(element.getTextContent());
+    }
+
+    private void assertSegmentBaseElement(@Nonnull final Document document,
+                                          @Nonnull final ItagItem itagItem) {
+        final Element element = assertGetElement(document, SEGMENT_BASE, REPRESENTATION);
+        assertRangeEquals(itagItem.getIndexStart(), itagItem.getIndexEnd(), element, "indexRange");
+    }
+
+    private void assertInitializationElement(@Nonnull final Document document,
+                                             @Nonnull final ItagItem itagItem) {
+        final Element element = assertGetElement(document, INITIALIZATION, SEGMENT_BASE);
+        assertRangeEquals(itagItem.getInitStart(), itagItem.getInitEnd(), element, "range");
+    }
+
+
+    private void assertAttrEquals(final int expected,
+                                  @Nonnull final Element element,
+                                  final String attribute) {
+
+        final int actual = Integer.parseInt(element.getAttribute(attribute));
+        assertAll(
+                () -> assertGreater(0, actual),
+                () -> assertEquals(expected, actual)
+        );
+    }
+
+    private void assertAttrEquals(final String expected,
+                                  @Nonnull final Element element,
+                                  final String attribute) {
+        final String actual = element.getAttribute(attribute);
+        assertAll(
+                () -> assertNotBlank(actual),
+                () -> assertEquals(expected, actual)
+        );
+    }
+
+    private void assertRangeEquals(final int expectedStart,
+                                   final int expectedEnd,
+                                   @Nonnull final Element element,
+                                   final String attribute) {
+        final String range = element.getAttribute(attribute);
+        assertNotBlank(range);
+        final String[] rangeParts = range.split("-");
+        assertEquals(2, rangeParts.length);
+
+        final int actualStart = Integer.parseInt(rangeParts[0]);
+        final int actualEnd = Integer.parseInt(rangeParts[1]);
+
+        assertAll(
+                () -> assertGreaterOrEqual(0, actualStart),
+                () -> assertEquals(expectedStart, actualStart),
+                () -> assertGreater(0, actualEnd),
+                () -> assertEquals(expectedEnd, actualEnd)
+        );
+    }
+
+    @Nonnull
+    private Element assertGetElement(@Nonnull final Document document,
+                                     final String tagName,
+                                     final String expectedParentTagName) {
+
+        final Element element = (Element) document.getElementsByTagName(tagName).item(0);
+        assertNotNull(element);
+        assertTrue(element.getParentNode().isEqualNode(
+                document.getElementsByTagName(expectedParentTagName).item(0)),
+                "Element with tag name \"" + tagName + "\" does not have a parent node"
+                + " with tag name \"" + expectedParentTagName + "\"");
+        return element;
+    }
+}

extractor/src/test/java/org/schabi/newpipe/extractor/utils/ManifestCreatorCacheTest.java

@@ -0,0 +1,74 @@
+package org.schabi.newpipe.extractor.utils;
+
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+class ManifestCreatorCacheTest {
+    @Test
+    void basicMaximumSizeAndResetTest() {
+        final ManifestCreatorCache<String, String> cache = new ManifestCreatorCache<>();
+
+        // 30 elements set -> cache resized to 23 -> 5 new elements set to the cache -> 28
+        cache.setMaximumSize(30);
+        setCacheContent(cache);
+        assertEquals(28, cache.size(),
+                "Wrong cache size with default clear factor and 30 as the maximum size");
+        cache.reset();
+
+        assertEquals(0, cache.size(),
+                "The cache has been not cleared after a reset call (wrong cache size)");
+        assertEquals(ManifestCreatorCache.DEFAULT_MAXIMUM_SIZE, cache.getMaximumSize(),
+                "Wrong maximum size after cache reset");
+        assertEquals(ManifestCreatorCache.DEFAULT_CLEAR_FACTOR, cache.getClearFactor(),
+                "Wrong clear factor after cache reset");
+    }
+
+    @Test
+    void maximumSizeAndClearFactorSettersAndResettersTest() {
+        final ManifestCreatorCache<String, String> cache = new ManifestCreatorCache<>();
+        cache.setMaximumSize(20);
+        cache.setClearFactor(0.5);
+
+        setCacheContent(cache);
+        // 30 elements set -> cache resized to 10 -> 5 new elements set to the cache -> 15
+        assertEquals(15, cache.size(),
+                "Wrong cache size with 0.5 as the clear factor and 20 as the maximum size");
+
+        // Clear factor and maximum size getters tests
+        assertEquals(0.5, cache.getClearFactor(),
+                "Wrong clear factor gotten from clear factor getter");
+        assertEquals(20, cache.getMaximumSize(),
+                "Wrong maximum cache size gotten from maximum size getter");
+
+        // Resetters tests
+        cache.resetMaximumSize();
+        assertEquals(ManifestCreatorCache.DEFAULT_MAXIMUM_SIZE, cache.getMaximumSize(),
+                "Wrong maximum cache size gotten from maximum size getter after maximum size "
+                        + "resetter call");
+
+        cache.resetClearFactor();
+        assertEquals(ManifestCreatorCache.DEFAULT_CLEAR_FACTOR, cache.getClearFactor(),
+                "Wrong clear factor gotten from clear factor getter after clear factor resetter "
+                        + "call");
+    }
+
+    /**
+     * Adds sample strings to the provided manifest creator cache, in order to test clear factor and
+     * maximum size.
+     * @param cache the cache to fill with some data
+     */
+    private static void setCacheContent(final ManifestCreatorCache<String, String> cache) {
+        int i = 0;
+        while (i < 26) {
+            cache.put(String.valueOf((char) ('a' + i)), "V");
+            ++i;
+        }
+
+        i = 0;
+        while (i < 9) {
+            cache.put("a" + (char) ('a' + i), "V");
+            ++i;
+        }
+    }
+}

extractor/src/test/java/org/schabi/newpipe/extractor/utils/UtilsTest.java

@@ -3,13 +3,14 @@ package org.schabi.newpipe.extractor.utils;
 import org.junit.jupiter.api.Test;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 
+import javax.annotation.Nonnull;
 import java.util.Arrays;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
-public class UtilsTest {
+class UtilsTest {
     @Test
-    public void testMixedNumberWordToLong() throws ParsingException {
+    void testMixedNumberWordToLong() throws ParsingException {
         assertEquals(10, Utils.mixedNumberWordToLong("10"));
         assertEquals(10.5e3, Utils.mixedNumberWordToLong("10.5K"), 0.0);
         assertEquals(10.5e6, Utils.mixedNumberWordToLong("10.5M"), 0.0);
@@ -18,13 +19,13 @@ public class UtilsTest {
     }
 
     @Test
-    public void testJoin() {
+    void testJoin() {
         assertEquals("some,random,stuff", Utils.join(",", Arrays.asList("some", "random", "stuff")));
         assertEquals("some,random,not-null,stuff", Utils.nonEmptyAndNullJoin(",", new String[]{"some", "null", "random", "", "not-null", null, "stuff"}));
     }
 
     @Test
-    public void testGetBaseUrl() throws ParsingException {
+    void testGetBaseUrl() throws ParsingException {
         assertEquals("https://www.youtube.com", Utils.getBaseUrl("https://www.youtube.com/watch?v=Hu80uDzh8RY"));
         assertEquals("vnd.youtube", Utils.getBaseUrl("vnd.youtube://www.youtube.com/watch?v=jZViOEv90dI"));
         assertEquals("vnd.youtube", Utils.getBaseUrl("vnd.youtube:jZViOEv90dI"));
@@ -33,7 +34,7 @@ public class UtilsTest {
     }
 
     @Test
-    public void testFollowGoogleRedirect() {
+    void testFollowGoogleRedirect() {
         assertEquals("https://www.youtube.com/watch?v=Hu80uDzh8RY",
                 Utils.followGoogleRedirectIfNeeded("https://www.google.it/url?sa=t&rct=j&q=&esrc=s&cd=&cad=rja&uact=8&url=https%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DHu80uDzh8RY&source=video"));
         assertEquals("https://www.youtube.com/watch?v=0b6cFWG45kA",