New option to use dedicated feed sources for services that support it

YouTube, for example, has a dedicated feed which was built to be used
like this. It comes with some caveats though, like lacking enough
information about the items and returning a limited amount of them.

Nonetheless, a nice option for users that like speedy updates but don't
mind this issue.

app/src/main/java/org/schabi/newpipe/local/feed/service/FeedLoadService.kt

@@ -40,8 +40,9 @@ import org.reactivestreams.Subscriber
 import org.reactivestreams.Subscription
 import org.schabi.newpipe.MainActivity.DEBUG
 import org.schabi.newpipe.R
-import org.schabi.newpipe.extractor.channel.ChannelInfo
+import org.schabi.newpipe.extractor.ListInfo
 import org.schabi.newpipe.extractor.exceptions.ReCaptchaException
+import org.schabi.newpipe.extractor.stream.StreamInfoItem
 import org.schabi.newpipe.local.feed.FeedDatabaseManager
 import org.schabi.newpipe.local.feed.service.FeedEventManager.Event.*
 import org.schabi.newpipe.local.feed.service.FeedEventManager.postEvent
@@ -109,11 +110,14 @@ class FeedLoadService : Service() {
         val defaultSharedPreferences = PreferenceManager.getDefaultSharedPreferences(this)
 
         val groupId = intent.getLongExtra(EXTRA_GROUP_ID, -1)
+        val useFeedExtractor = defaultSharedPreferences
+                .getBoolean(getString(R.string.feed_use_dedicated_fetch_method_key), false)
+
         val thresholdOutdatedMinutesString = defaultSharedPreferences
                 .getString(getString(R.string.feed_update_threshold_key), getString(R.string.feed_update_threshold_default_value))
         val thresholdOutdatedMinutes = thresholdOutdatedMinutesString!!.toInt()
 
-        startLoading(groupId, thresholdOutdatedMinutes)
+        startLoading(groupId, useFeedExtractor, thresholdOutdatedMinutes)
 
         return START_NOT_STICKY
     }
@@ -142,7 +146,7 @@ class FeedLoadService : Service() {
 
     private class RequestException(val subscriptionId: Long, message: String, cause: Throwable) : Exception(message, cause) {
         companion object {
-            fun wrapList(subscriptionId: Long, info: ChannelInfo): List<Throwable> {
+            fun wrapList(subscriptionId: Long, info: ListInfo<StreamInfoItem>): List<Throwable> {
                 val toReturn = ArrayList<Throwable>(info.errors.size)
                 for (error in info.errors) {
                     toReturn.add(RequestException(subscriptionId, info.serviceId.toString() + ":" + info.url, error))
@@ -152,7 +156,7 @@ class FeedLoadService : Service() {
         }
     }
 
-    private fun startLoading(groupId: Long = -1, thresholdOutdatedMinutes: Int) {
+    private fun startLoading(groupId: Long = -1, useFeedExtractor: Boolean, thresholdOutdatedMinutes: Int) {
         feedResultsHolder = ResultsHolder()
 
         val outdatedThreshold = Calendar.getInstance().apply {
@@ -187,14 +191,21 @@ class FeedLoadService : Service() {
                 .runOn(Schedulers.io())
                 .map { subscriptionEntity ->
                     try {
-                        val channelInfo = ExtractorHelper
+                        val listInfo = if (useFeedExtractor) {
+                            ExtractorHelper
+                                    .getFeedInfoFallbackToChannelInfo(subscriptionEntity.serviceId, subscriptionEntity.url)
+                                    .blockingGet()
+                        } else {
+                            ExtractorHelper
                                     .getChannelInfo(subscriptionEntity.serviceId, subscriptionEntity.url, true)
                                     .blockingGet()
-                        return@map Notification.createOnNext(Pair(subscriptionEntity.uid, channelInfo))
+                        } as ListInfo<StreamInfoItem>
+
+                        return@map Notification.createOnNext(Pair(subscriptionEntity.uid, listInfo))
                     } catch (e: Throwable) {
                         val request = "${subscriptionEntity.serviceId}:${subscriptionEntity.url}"
                         val wrapper = RequestException(subscriptionEntity.uid, request, e)
-                        return@map Notification.createOnError<Pair<Long, ChannelInfo>>(wrapper)
+                        return@map Notification.createOnError<Pair<Long, ListInfo<StreamInfoItem>>>(wrapper)
                     }
                 }
                 .sequential()
@@ -219,14 +230,14 @@ class FeedLoadService : Service() {
     }
 
     private val resultSubscriber
-        get() = object : Subscriber<List<Notification<Pair<Long, ChannelInfo>>>> {
+        get() = object : Subscriber<List<Notification<Pair<Long, ListInfo<StreamInfoItem>>>>> {
 
             override fun onSubscribe(s: Subscription) {
                 loadingSubscription = s
                 s.request(java.lang.Long.MAX_VALUE)
             }
 
-            override fun onNext(notification: List<Notification<Pair<Long, ChannelInfo>>>) {
+            override fun onNext(notification: List<Notification<Pair<Long, ListInfo<StreamInfoItem>>>>) {
                 if (DEBUG) Log.v(TAG, "onNext() → $notification")
             }
 
@@ -271,7 +282,7 @@ class FeedLoadService : Service() {
             }
         }
 
-    private val databaseConsumer: Consumer<List<Notification<Pair<Long, ChannelInfo>>>>
+    private val databaseConsumer: Consumer<List<Notification<Pair<Long, ListInfo<StreamInfoItem>>>>>
         get() = Consumer {
             feedDatabaseManager.database().runInTransaction {
                 for (notification in it) {
@@ -300,7 +311,8 @@ class FeedLoadService : Service() {
             }
         }
 
-    private val errorHandlingConsumer: Consumer<Notification<Pair<Long, ChannelInfo>>>
+
+    private val errorHandlingConsumer: Consumer<Notification<Pair<Long, ListInfo<StreamInfoItem>>>>
         get() = Consumer {
             if (it.isOnError) {
                 var error = it.error!!
@@ -317,7 +329,7 @@ class FeedLoadService : Service() {
             }
         }
 
-    private val notificationsConsumer: Consumer<Notification<Pair<Long, ChannelInfo>>>
+    private val notificationsConsumer: Consumer<Notification<Pair<Long, ListInfo<StreamInfoItem>>>>
         get() = Consumer { onItemCompleted(it.value?.second?.name) }
 
     private fun onItemCompleted(updateDescription: String?) {

app/src/main/java/org/schabi/newpipe/local/subscription/SubscriptionManager.kt

@@ -7,7 +7,10 @@ import io.reactivex.schedulers.Schedulers
 import org.schabi.newpipe.NewPipeDatabase
 import org.schabi.newpipe.database.subscription.SubscriptionDAO
 import org.schabi.newpipe.database.subscription.SubscriptionEntity
+import org.schabi.newpipe.extractor.ListInfo
 import org.schabi.newpipe.extractor.channel.ChannelInfo
+import org.schabi.newpipe.extractor.feed.FeedInfo
+import org.schabi.newpipe.extractor.stream.StreamInfoItem
 import org.schabi.newpipe.local.feed.FeedDatabaseManager
 
 class SubscriptionManager(context: Context) {
@@ -40,9 +43,14 @@ class SubscriptionManager(context: Context) {
                 }
             }
 
-    fun updateFromInfo(subscriptionId: Long, info: ChannelInfo) {
+    fun updateFromInfo(subscriptionId: Long, info: ListInfo<StreamInfoItem>) {
         val subscriptionEntity = subscriptionTable.getSubscription(subscriptionId)
+
+        if (info is FeedInfo) {
+            subscriptionEntity.name = info.name
+        } else if (info is ChannelInfo) {
             subscriptionEntity.setData(info.name, info.avatarUrl, info.description, info.subscriberCount)
+        }
 
         subscriptionTable.update(subscriptionEntity)
     }

app/src/main/java/org/schabi/newpipe/util/ExtractorHelper.java

@@ -31,18 +31,23 @@ import org.schabi.newpipe.ReCaptchaActivity;
 import org.schabi.newpipe.extractor.Info;
 import org.schabi.newpipe.extractor.InfoItem;
 import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage;
+import org.schabi.newpipe.extractor.ListInfo;
 import org.schabi.newpipe.extractor.NewPipe;
-import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
+import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.channel.ChannelInfo;
 import org.schabi.newpipe.extractor.comments.CommentsInfo;
 import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
+import org.schabi.newpipe.extractor.feed.FeedExtractor;
+import org.schabi.newpipe.extractor.feed.FeedInfo;
 import org.schabi.newpipe.extractor.kiosk.KioskInfo;
 import org.schabi.newpipe.extractor.playlist.PlaylistInfo;
 import org.schabi.newpipe.extractor.search.SearchInfo;
 import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
 import org.schabi.newpipe.extractor.stream.StreamInfo;
+import org.schabi.newpipe.extractor.stream.StreamInfoItem;
+import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
 import org.schabi.newpipe.report.ErrorActivity;
 import org.schabi.newpipe.report.UserAction;
 
@@ -131,6 +136,22 @@ public final class ExtractorHelper {
                 ChannelInfo.getMoreItems(NewPipe.getService(serviceId), url, nextStreamsUrl));
     }
 
+    public static Single<ListInfo<StreamInfoItem>> getFeedInfoFallbackToChannelInfo(final int serviceId,
+                                                                                    final String url) {
+        final Maybe<ListInfo<StreamInfoItem>> maybeFeedInfo = Maybe.fromCallable(() -> {
+            final StreamingService service = NewPipe.getService(serviceId);
+            final FeedExtractor feedExtractor = service.getFeedExtractor(url);
+
+            if (feedExtractor == null) {
+                return null;
+            }
+
+            return FeedInfo.getInfo(feedExtractor);
+        });
+
+        return maybeFeedInfo.switchIfEmpty(getChannelInfo(serviceId, url, true));
+    }
+
     public static Single<CommentsInfo> getCommentsInfo(final int serviceId,
                                                        final String url,
                                                        boolean forceLoad) {

app/src/main/res/values/settings_keys.xml

@@ -202,6 +202,7 @@
         <item>720</item>
         <item>1440</item>
     </string-array>
+    <string name="feed_use_dedicated_fetch_method_key" translatable="false">feed_use_dedicated_fetch_method</string>
 
     <string name="import_data" translatable="false">import_data</string>
     <string name="export_data" translatable="false">export_data</string>

app/src/main/res/values/strings.xml

@@ -616,4 +616,6 @@
     <string name="feed_update_threshold_title">Feed update threshold</string>
     <string name="feed_update_threshold_summary">Time after last update before a subscription is considered outdated — %s</string>
     <string name="feed_update_threshold_option_always_update">Always update</string>
+    <string name="feed_use_dedicated_fetch_method_title">Fetch from dedicated feed when available</string>
+    <string name="feed_use_dedicated_fetch_method_summary">Available in some services, it is usually much faster but may return a limited amount of items and often incomplete information (e.g. no duration, item type, no live status).</string>
 </resources>

app/src/main/res/xml/content_settings.xml

@@ -102,5 +102,13 @@
             android:entryValues="@array/feed_update_threshold_values"
             android:title="@string/feed_update_threshold_title"
             android:summary="@string/feed_update_threshold_summary"/>
+
+        <SwitchPreference
+            app:iconSpaceReserved="false"
+            android:defaultValue="false"
+            android:key="@string/feed_use_dedicated_fetch_method_key"
+            android:title="@string/feed_use_dedicated_fetch_method_title"
+            android:summary="@string/feed_use_dedicated_fetch_method_summary"/>
+
     </PreferenceCategory>
 </PreferenceScreen>