[media.ccc.de] Play live streams

extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCService.java

@@ -20,8 +20,6 @@ import org.schabi.newpipe.extractor.stream.StreamExtractor;
 import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
 import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
 
-import java.io.IOException;
-
 import static java.util.Arrays.asList;
 import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO;
 import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.VIDEO;
@@ -58,6 +56,9 @@ public class MediaCCCService extends StreamingService {
 
     @Override
     public StreamExtractor getStreamExtractor(final LinkHandler linkHandler) {
+        if (MediaCCCParsingHelper.isLiveStreamId(linkHandler.getId())) {
+            return new MediaCCCLiveStreamExtractor(this, linkHandler);
+        }
         return new MediaCCCStreamExtractor(this, linkHandler);
     }
 
@@ -108,9 +109,9 @@ public class MediaCCCService extends StreamingService {
                                                      final String url, final String kioskId)
                         throws ExtractionException {
                     return new MediaCCCLiveStreamKiosk(MediaCCCService.this,
-                            new MediaCCCLiveStreamListLinkHandlerFactory().fromUrl(url), kioskId);
+                            new MediaCCCLiveListLinkHandlerFactory().fromUrl(url), kioskId);
                 }
-            }, new MediaCCCLiveStreamListLinkHandlerFactory(), "live");
+            }, new MediaCCCLiveListLinkHandlerFactory(), "live");
 
             list.setDefaultKiosk("recent");
         } catch (Exception e) {

extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCLiveStreamExtractor.java

@@ -1,75 +1,63 @@
 package org.schabi.newpipe.extractor.services.media_ccc.extractors;
 
+import com.grack.nanojson.JsonArray;
 import com.grack.nanojson.JsonObject;
+import org.schabi.newpipe.extractor.MediaFormat;
+import org.schabi.newpipe.extractor.MetaInfo;
+import org.schabi.newpipe.extractor.StreamingService;
+import org.schabi.newpipe.extractor.downloader.Downloader;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
 import org.schabi.newpipe.extractor.localization.DateWrapper;
-import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
+import org.schabi.newpipe.extractor.stream.*;
-import org.schabi.newpipe.extractor.stream.StreamType;
 
+import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Locale;
 
-public class MediaCCCLiveStreamExtractor implements StreamInfoItemExtractor {
+public class MediaCCCLiveStreamExtractor extends StreamExtractor {
+    private JsonArray doc = null;
+    private JsonObject conference = null;
+    private String group = "";
+    private JsonObject room = null;
 
-    private final JsonObject conferenceInfo;
+    public MediaCCCLiveStreamExtractor(StreamingService service, LinkHandler linkHandler) {
-    private final String group;
+        super(service, linkHandler);
-    private final JsonObject roomInfo;
-
-    public MediaCCCLiveStreamExtractor(JsonObject conferenceInfo, String group, JsonObject roomInfo) {
-        this.conferenceInfo = conferenceInfo;
-        this.group = group;
-        this.roomInfo = roomInfo;
     }
 
     @Override
-    public String getName() throws ParsingException {
+    public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
-        return roomInfo.getString("schedulename");
+        doc = MediaCCCParsingHelper.getLiveStreams(downloader, getExtractorLocalization());
-    }
+        // find correct room
-
+        for (int c = 0; c < doc.size(); c++) {
-    @Override
+            final JsonObject conference = doc.getObject(c);
-    public String getUrl() throws ParsingException {
+            final JsonArray groups = conference.getArray("groups");
-        return roomInfo.getString("link");
+            for (int g = 0; g < groups.size(); g++) {
-    }
+                final String group = groups.getObject(g).getString("group");
-
+                final JsonArray rooms = groups.getObject(g).getArray("rooms");
-    @Override
+                for (int r = 0; r < rooms.size(); r++) {
-    public String getThumbnailUrl() throws ParsingException {
+                    final JsonObject room = rooms.getObject(r);
-        return roomInfo.getString("thumb");
+                    if (getId().equals(conference.getString("slug") + "/" + room.getString("slug"))) {
-    }
+                        this.conference = conference;
-
+                        this.group = group;
-    @Override
+                        this.room = room;
-    public StreamType getStreamType() throws ParsingException {
+                        return;
-        boolean isVideo = false;
+                    }
-        for (Object stream : roomInfo.getArray("streams")) {
+                }
-            if ("video".equals(((JsonObject) stream).getString("type"))) {
-                isVideo = true;
-                break;
             }
         }
-        return isVideo ? StreamType.LIVE_STREAM :  StreamType.AUDIO_LIVE_STREAM;
+        throw new ExtractionException("Could not find room matching id: '" + getId() + "'");
     }
 
+    @Nonnull
     @Override
-    public boolean isAd() throws ParsingException {
+    public String getName() throws ParsingException {
-        return false;
+        return room.getString("display");
-    }
-
-    @Override
-    public long getDuration() throws ParsingException {
-        return 0;
-    }
-
-    @Override
-    public long getViewCount() throws ParsingException {
-        return -1;
-    }
-
-    @Override
-    public String getUploaderName() throws ParsingException {
-        return conferenceInfo.getString("conference");
-    }
-
-    @Override
-    public String getUploaderUrl() throws ParsingException {
-        return "https://media.ccc.de/c/" + conferenceInfo.getString("slug");
     }
 
     @Nullable
@@ -83,4 +71,230 @@ public class MediaCCCLiveStreamExtractor implements StreamInfoItemExtractor {
     public DateWrapper getUploadDate() throws ParsingException {
         return null;
     }
+
+    @Nonnull
+    @Override
+    public String getThumbnailUrl() throws ParsingException {
+        return room.getString("thumb");
+    }
+
+    @Nonnull
+    @Override
+    public Description getDescription() throws ParsingException {
+        return new Description(conference.getString("description") + " - " + group, Description.PLAIN_TEXT);
+    }
+
+    @Override
+    public int getAgeLimit() {
+        return 0;
+    }
+
+    @Override
+    public long getLength() {
+        return 0;
+    }
+
+    @Override
+    public long getTimeStamp() throws ParsingException {
+        return 0;
+    }
+
+    @Override
+    public long getViewCount() {
+        return -1;
+    }
+
+    @Override
+    public long getLikeCount() {
+        return -1;
+    }
+
+    @Override
+    public long getDislikeCount() {
+        return -1;
+    }
+
+    @Nonnull
+    @Override
+    public String getUploaderUrl() throws ParsingException {
+        return "https://streaming.media.ccc.de/" + conference.getString("slug");
+    }
+
+    @Nonnull
+    @Override
+    public String getUploaderName() throws ParsingException {
+        return conference.getString("conference");
+    }
+
+    @Nonnull
+    @Override
+    public String getUploaderAvatarUrl() {
+        return "";
+    }
+
+    @Nonnull
+    @Override
+    public String getSubChannelUrl() {
+        return "";
+    }
+
+    @Nonnull
+    @Override
+    public String getSubChannelName() {
+        return "";
+    }
+
+    @Nonnull
+    @Override
+    public String getSubChannelAvatarUrl() {
+        return "";
+    }
+
+    @Nonnull
+    @Override
+    public String getDashMpdUrl() throws ParsingException {
+        return "";
+    }
+
+    @Nonnull
+    @Override
+    public String getHlsUrl() {
+        // TODO: There are multiple HLS streams.
+        //       Make getHlsUrl() and getDashMpdUrl() return lists of VideoStreams, so the user can choose a resolution.
+        for (int s = 0; s < room.getArray("streams").size(); s++) {
+            final JsonObject stream = room.getArray("streams").getObject(s);
+            if (stream.getString("type").equals("video")) {
+                final String resolution = stream.getArray("videoSize").getInt(0) + "x"
+                        + stream.getArray("videoSize").getInt(1);
+                if (stream.has("hls")) {
+                    return stream.getObject("urls").getObject("hls").getString("url");
+                }
+            }
+        }
+        return "";
+    }
+
+    @Override
+    public List<AudioStream> getAudioStreams() throws IOException, ExtractionException {
+        final List<AudioStream> audioStreams = new ArrayList<>();
+        for (int s = 0; s < room.getArray("streams").size(); s++) {
+            final JsonObject stream = room.getArray("streams").getObject(s);
+            if (stream.getString("type").equals("audio")) {
+                for (final String type :stream.getObject("urls").keySet()) {
+                    final JsonObject url = stream.getObject("urls").getObject(type);
+                    audioStreams.add(new AudioStream(url.getString("url"), MediaFormat.getFromSuffix(type), -1));
+                }
+            }
+        }
+        return audioStreams;
+    }
+
+    @Override
+    public List<VideoStream> getVideoStreams() throws IOException, ExtractionException {
+        final List<VideoStream> videoStreams = new ArrayList<>();
+        for (int s = 0; s < room.getArray("streams").size(); s++) {
+            final JsonObject stream = room.getArray("streams").getObject(s);
+            if (stream.getString("type").equals("video")) {
+                final String resolution = stream.getArray("videoSize").getInt(0) + "x"
+                        + stream.getArray("videoSize").getInt(1);
+                for (final String type :stream.getObject("urls").keySet()) {
+                    if (!type.equals("hls")) {
+                        final JsonObject url = stream.getObject("urls").getObject(type);
+                        videoStreams.add(new VideoStream(
+                                url.getString("url"),
+                                MediaFormat.getFromSuffix(type),
+                                resolution));
+                    }
+                }
+            }
+        }
+        return videoStreams;
+    }
+
+    @Override
+    public List<VideoStream> getVideoOnlyStreams() throws IOException, ExtractionException {
+        return null;
+    }
+
+    @Nonnull
+    @Override
+    public List<SubtitlesStream> getSubtitlesDefault(){
+        return Collections.emptyList();
+    }
+
+    @Nonnull
+    @Override
+    public List<SubtitlesStream> getSubtitles(MediaFormat format) {
+        return Collections.emptyList();
+    }
+
+    @Override
+    public StreamType getStreamType() throws ParsingException {
+        return StreamType.LIVE_STREAM; // TODO: video and audio only streams are both available
+    }
+
+    @Nullable
+    @Override
+    public StreamInfoItemsCollector getRelatedStreams() {
+        return null;
+    }
+
+    @Override
+    public String getErrorMessage() {
+        return null;
+    }
+
+    @Nonnull
+    @Override
+    public String getHost() throws ParsingException {
+        return null;
+    }
+
+    @Nonnull
+    @Override
+    public String getPrivacy() {
+        return "Public";
+    }
+
+    @Nonnull
+    @Override
+    public String getCategory() {
+        return group;
+    }
+
+    @Nonnull
+    @Override
+    public String getLicence() {
+        return "";
+    }
+
+    @Nullable
+    @Override
+    public Locale getLanguageInfo() {
+        return null;
+    }
+
+    @Nonnull
+    @Override
+    public List<String> getTags() {
+        return Collections.emptyList();
+    }
+
+    @Nonnull
+    @Override
+    public String getSupportInfo() {
+        return "";
+    }
+
+    @Nonnull
+    @Override
+    public List<StreamSegment> getStreamSegments() {
+        return Collections.emptyList();
+    }
+
+    @Nonnull
+    @Override
+    public List<MetaInfo> getMetaInfo() {
+        return Collections.emptyList();
+    }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCLiveStreamKioskExtractor.java

@@ -0,0 +1,87 @@
+package org.schabi.newpipe.extractor.services.media_ccc.extractors;
+
+import com.grack.nanojson.JsonObject;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.localization.DateWrapper;
+import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
+import org.schabi.newpipe.extractor.stream.StreamType;
+
+import javax.annotation.Nullable;
+
+public class MediaCCCLiveStreamKioskExtractor implements StreamInfoItemExtractor {
+
+    private final JsonObject conferenceInfo;
+    private final String group;
+    private final JsonObject roomInfo;
+
+    public MediaCCCLiveStreamKioskExtractor(final JsonObject conferenceInfo, final String group,
+                                            final JsonObject roomInfo) {
+        this.conferenceInfo = conferenceInfo;
+        this.group = group;
+        this.roomInfo = roomInfo;
+    }
+
+    @Override
+    public String getName() throws ParsingException {
+        return roomInfo.getString("schedulename");
+    }
+
+    @Override
+    public String getUrl() throws ParsingException {
+        return roomInfo.getString("link");
+    }
+
+    @Override
+    public String getThumbnailUrl() throws ParsingException {
+        return roomInfo.getString("thumb");
+    }
+
+    @Override
+    public StreamType getStreamType() throws ParsingException {
+        boolean isVideo = false;
+        for (Object stream : roomInfo.getArray("streams")) {
+            if ("video".equals(((JsonObject) stream).getString("type"))) {
+                isVideo = true;
+                break;
+            }
+        }
+        return isVideo ? StreamType.LIVE_STREAM :  StreamType.AUDIO_LIVE_STREAM;
+    }
+
+    @Override
+    public boolean isAd() throws ParsingException {
+        return false;
+    }
+
+    @Override
+    public long getDuration() throws ParsingException {
+        return 0;
+    }
+
+    @Override
+    public long getViewCount() throws ParsingException {
+        return -1;
+    }
+
+    @Override
+    public String getUploaderName() throws ParsingException {
+        return conferenceInfo.getString("conference");
+    }
+
+    @Override
+    public String getUploaderUrl() throws ParsingException {
+        return "https://media.ccc.de/c/" + conferenceInfo.getString("slug");
+    }
+
+    @Nullable
+    @Override
+    public String getTextualUploadDate() throws ParsingException {
+        return null;
+    }
+
+    @Nullable
+    @Override
+    public DateWrapper getUploadDate() throws ParsingException {
+        return null;
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCParsingHelper.java

@@ -1,11 +1,23 @@
 package org.schabi.newpipe.extractor.services.media_ccc.extractors;
 
+import com.grack.nanojson.JsonArray;
+import com.grack.nanojson.JsonObject;
+import com.grack.nanojson.JsonParser;
+import com.grack.nanojson.JsonParserException;
+import org.schabi.newpipe.extractor.downloader.Downloader;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
+import org.schabi.newpipe.extractor.localization.Localization;
 
+import java.io.IOException;
 import java.time.OffsetDateTime;
 import java.time.format.DateTimeParseException;
+import java.util.regex.Pattern;
 
 public final class MediaCCCParsingHelper {
+    private static JsonArray liveStreams = null;
+
     private MediaCCCParsingHelper() { }
 
     public static OffsetDateTime parseDateFrom(final String textualUploadDate) throws ParsingException {
@@ -15,4 +27,24 @@ public final class MediaCCCParsingHelper {
             throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e);
         }
     }
+
+    public static boolean isLiveStreamId(final String url) {
+        final String pattern = "\\w+/\\w+";
+        return Pattern.matches(pattern, url); // {conference_slug}/{room_slug}
+    }
+
+    public static JsonArray getLiveStreams(final Downloader downloader, final Localization localization) throws ExtractionException {
+        if (liveStreams == null) {
+            try {
+                final String site = downloader.get("https://streaming.media.ccc.de/streams/v2.json",
+                        localization).responseBody();
+                liveStreams = JsonParser.array().from(site);
+            } catch (IOException | ReCaptchaException e) {
+                throw new ExtractionException("Could not get live stream JSON.", e);
+            } catch (JsonParserException e) {
+                throw new ExtractionException("Could not parse JSON.", e);
+            }
+        }
+        return liveStreams;
+    }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCLiveListLinkHandlerFactory.java

@@ -6,8 +6,8 @@ import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
 import java.util.List;
 import java.util.regex.Pattern;
 
-public class MediaCCCLiveStreamListLinkHandlerFactory extends ListLinkHandlerFactory {
+public class MediaCCCLiveListLinkHandlerFactory extends ListLinkHandlerFactory {
-    private static final String streamPattern = "^(https?://)?streaming.media.ccc.de$";
+    private static final String streamPattern = "^(?:https?://)?media\\.ccc\\.de/live$";
 
     @Override
     public String getId(String url) throws ParsingException {
@@ -22,6 +22,6 @@ public class MediaCCCLiveStreamListLinkHandlerFactory extends ListLinkHandlerFac
     @Override
     public String getUrl(String id, List<String> contentFilter, String sortFilter) throws ParsingException {
         // FIXME: wrong URL; should be https://streaming.media.ccc.de/{conference_slug}/{room_slug}
-        return "https://streaming.media.ccc.de/" + id;
+        return "https://media.ccc.de/live";
     }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCLiveStreamLinkHandlerFactory.java

@@ -0,0 +1,30 @@
+package org.schabi.newpipe.extractor.services.media_ccc.linkHandler;
+
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
+import org.schabi.newpipe.extractor.utils.Parser;
+
+public class MediaCCCLiveStreamLinkHandlerFactory extends LinkHandlerFactory {
+    public static final String VIDEO_API_ENDPOINT = "https://api.media.ccc.de/public/events/";
+    private static final String VIDEO_PATH = "https://streaming.media.ccc.de/v/";
+    private static final String ID_PATTERN = "(?:(?:(?:api\\.)?media\\.ccc\\.de/public/events/)|(?:media\\.ccc\\.de/v/))([^/?&#]*)";
+
+    @Override
+    public String getId(final String url) throws ParsingException {
+        return Parser.matchGroup1(ID_PATTERN, url);
+    }
+
+    @Override
+    public String getUrl(final String id) throws ParsingException {
+        return VIDEO_PATH + id;
+    }
+
+    @Override
+    public boolean onAcceptUrl(final String url) {
+        try {
+            return getId(url) != null;
+        } catch (ParsingException e) {
+            return false;
+        }
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCStreamLinkHandlerFactory.java

@@ -2,20 +2,36 @@ package org.schabi.newpipe.extractor.services.media_ccc.linkHandler;
 
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
+import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCParsingHelper;
 import org.schabi.newpipe.extractor.utils.Parser;
 
 public class MediaCCCStreamLinkHandlerFactory extends LinkHandlerFactory {
     public static final String VIDEO_API_ENDPOINT = "https://api.media.ccc.de/public/events/";
     private static final String VIDEO_PATH = "https://media.ccc.de/v/";
-    private static final String ID_PATTERN = "(?:(?:(?:api\\.)?media\\.ccc\\.de/public/events/)|(?:media\\.ccc\\.de/v/))([^/?&#]*)";
+    private static final String RECORDING_ID_PATTERN = "(?:(?:(?:api\\.)?media\\.ccc\\.de/public/events/)|(?:media\\.ccc\\.de/v/))([^/?&#]*)";
+    private static final String LIVE_STREAM_API_ENDPOINT = "https://streaming.media.ccc.de/streams/v2.json";
+    private static final String LIVE_STREAM_PATH = "https://streaming.media.ccc.de/";
+    private static final String LIVE_STREAM_ID_PATTERN = "streaming\\.media\\.ccc\\.de\\/(\\w+\\/\\w+)";
 
     @Override
     public String getId(final String url) throws ParsingException {
-        return Parser.matchGroup1(ID_PATTERN, url);
+        String streamId = null;
+        try {
+            streamId = Parser.matchGroup1(LIVE_STREAM_ID_PATTERN, url);
+        } catch (Parser.RegexException ignored) {
+
+        }
+        if (streamId == null) {
+            return Parser.matchGroup1(RECORDING_ID_PATTERN, url);
+        }
+        return streamId;
     }
 
     @Override
     public String getUrl(final String id) throws ParsingException {
+        if (MediaCCCParsingHelper.isLiveStreamId(id)) {
+            return LIVE_STREAM_PATH + id;
+        }
         return VIDEO_PATH + id;
     }