be-gay-do-crimes
/
NewPipe
mirror of https://github.com/TeamNewPipe/NewPipe
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #11743 from TeamNewPipe/slower-feed 

Throttle feed loading to avoid YouTube rate limits
This commit is contained in:
Stypox 2024-11-24 16:35:13 +01:00 committed by GitHub
parent 8bf24e6b14 726c12e934
commit acf2e88cb3
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 27 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
app/src/main/java/org/schabi/newpipe/local/feed/service/FeedLoadManager.kt
View File

@ -17,6 +17,7 @@ import org.schabi.newpipe.database.subscription.NotificationMode
import org.schabi.newpipe.database.subscription.SubscriptionEntity
import org.schabi.newpipe.extractor.Info
import org.schabi.newpipe.extractor.NewPipe
import org.schabi.newpipe.extractor.ServiceList
import org.schabi.newpipe.extractor.feed.FeedInfo
import org.schabi.newpipe.extractor.stream.StreamInfoItem
import org.schabi.newpipe.ktx.getStringSafe
@ -90,6 +91,10 @@ class FeedLoadManager(private val context: Context) {
else -> feedDatabaseManager.outdatedSubscriptionsForGroup(groupId, outdatedThreshold)
}
// like `currentProgress`, but counts the number of YouTube extractions that have begun, so
// they can be properly throttled every once in a while (see doOnNext below)
val youtubeExtractionCount = AtomicInteger()
return outdatedSubscriptions
.take(1)
.doOnNext {
@ -105,6 +110,15 @@ class FeedLoadManager(private val context: Context) {
.observeOn(Schedulers.io())
.flatMap { Flowable.fromIterable(it) }
.takeWhile { !cancelSignal.get() }
.doOnNext { subscriptionEntity ->
// throttle YouTube extractions once every BATCH_SIZE to avoid being rate limited
if (subscriptionEntity.serviceId == ServiceList.YouTube.serviceId) {
val previousCount = youtubeExtractionCount.getAndIncrement()
if (previousCount != 0 && previousCount % BATCH_SIZE == 0) {
Thread.sleep(DELAY_BETWEEN_BATCHES_MILLIS.random())
}
}
}
.parallel(PARALLEL_EXTRACTIONS, PARALLEL_EXTRACTIONS * 2)
.runOn(Schedulers.io(), PARALLEL_EXTRACTIONS * 2)
.filter { !cancelSignal.get() }
@ -328,7 +342,19 @@ class FeedLoadManager(private val context: Context) {
/**
* How many extractions will be running in parallel.
*/
private const val PARALLEL_EXTRACTIONS = 6
private const val PARALLEL_EXTRACTIONS = 3
/**
* How many YouTube extractions to perform before waiting [DELAY_BETWEEN_BATCHES_MILLIS]
* to avoid being rate limited
*/
private const val BATCH_SIZE = 50
/**
* Wait a random delay in this range once every [BATCH_SIZE] YouTube extractions to avoid
* being rate limited
*/
private val DELAY_BETWEEN_BATCHES_MILLIS = (6000L..12000L)
/**
* Number of items to buffer to mass-insert in the database.