How to the library in your app
This commit is contained in:
Clone Conflict
parent
b2a76985cd
commit
ea20ca0b00
|
|
@ -0,0 +1,251 @@
|
|||
= Using Newpipe Extractor
|
||||
|
||||
|
||||
== Add dependencies to gradle:
|
||||
|
||||
- *project* build.gradle:
|
||||
|
||||
[,gradle]
|
||||
----
|
||||
repositories {
|
||||
...
|
||||
maven { url 'https://jitpack.io' }
|
||||
}
|
||||
----
|
||||
|
||||
.or in settings.build
|
||||
[%collapsible]
|
||||
======
|
||||
this is properly for android ;)
|
||||
[,gradle]
|
||||
----
|
||||
repositories {
|
||||
...
|
||||
maven { url 'https://jitpack.io' }
|
||||
}
|
||||
----
|
||||
======
|
||||
|
||||
|
||||
|
||||
- *module*/app build.gradle:
|
||||
|
||||
image:https://img.shields.io/github/v/release/TeamNewPipe/NewPipeExtractor.svg[]
|
||||
|
||||
[,gradle]
|
||||
----
|
||||
// NewPipe Libraries
|
||||
implementation 'com.github.TeamNewPipe:NewPipeExtractor:0.21.12'
|
||||
implementation 'com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751'
|
||||
|
||||
|
||||
// HTTP client
|
||||
//noinspection GradleDependency --> do not update okhttp to keep supporting Android 4.4 users
|
||||
implementation "com.squareup.okhttp3:okhttp:3.12.13"
|
||||
|
||||
----
|
||||
|
||||
NOTE: You could really use any http client other than okhttp.
|
||||
|
||||
|
||||
== Configure for android projects:
|
||||
|
||||
Make sure you are using Java 11.
|
||||
[,gradle]
|
||||
```
|
||||
android {
|
||||
...
|
||||
compileOptions {
|
||||
coreLibraryDesugaringEnabled true
|
||||
|
||||
sourceCompatibility JavaVersion.VERSION_11
|
||||
targetCompatibility JavaVersion.VERSION_11
|
||||
}
|
||||
|
||||
// if you are using kotlin:
|
||||
kotlinOptions {
|
||||
jvmTarget = JavaVersion.VERSION_11
|
||||
useIR = true
|
||||
}
|
||||
}
|
||||
|
||||
dependencies {
|
||||
...
|
||||
|
||||
// desugar
|
||||
coreLibraryDesugaring 'com.android.tools:desugar_jdk_libs:1.1.5'
|
||||
}
|
||||
```
|
||||
|
||||
=== Proguard rules if you've [minify] enable:
|
||||
|
||||
the `proguard-rules.pro` file:
|
||||
|
||||
```
|
||||
...
|
||||
|
||||
# For NewPipeExtractor
|
||||
-dontobfuscate
|
||||
-keep class org.schabi.newpipe.extractor.timeago.patterns.** { *; }
|
||||
-keep class org.ocpsoft.prettytime.i18n.** { *; }
|
||||
|
||||
-keep class org.mozilla.javascript.** { *; }
|
||||
|
||||
-keep class org.mozilla.classfile.ClassFileWriter
|
||||
-keep class com.google.android.exoplayer2.** { *; }
|
||||
|
||||
-dontwarn org.mozilla.javascript.tools.**
|
||||
-dontwarn android.arch.util.paging.CountedDataSource
|
||||
-dontwarn android.arch.persistence.room.paging.LimitOffsetDataSource
|
||||
|
||||
|
||||
|
||||
# Rules for OkHttp. Copy paste from https://github.com/square/okhttp
|
||||
-dontwarn okhttp3.**
|
||||
-dontwarn okio.**
|
||||
-dontwarn javax.annotation.**
|
||||
# A resource is loaded with a relative path so the package of this class must be preserved.
|
||||
-keepnames class okhttp3.internal.publicsuffix.PublicSuffixDatabase
|
||||
-keepclassmembers class * implements java.io.Serializable {
|
||||
static final long serialVersionUID;
|
||||
!static !transient <fields>;
|
||||
private void writeObject(java.io.ObjectOutputStream);
|
||||
private void readObject(java.io.ObjectInputStream);
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
=== Implementing [Downloader] (okhttp)
|
||||
|
||||
Copy paste from Newpipe ;)
|
||||
|
||||
```
|
||||
package com.mycompany.hello
|
||||
|
||||
import okhttp3.OkHttpClient
|
||||
import org.schabi.newpipe.extractor.downloader.Downloader
|
||||
import org.schabi.newpipe.extractor.downloader.Request
|
||||
import org.schabi.newpipe.extractor.downloader.Response
|
||||
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException
|
||||
|
||||
import okhttp3.RequestBody
|
||||
|
||||
class DownloaderImpl(
|
||||
private val okhttp : OkHttpClient
|
||||
) : Downloader() {
|
||||
|
||||
companion object {
|
||||
const val USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; rv:78.0) Gecko/20100101 Firefox/78.0"
|
||||
}
|
||||
|
||||
override fun execute(request: Request): Response {
|
||||
val httpMethod = request.httpMethod()
|
||||
val url = request.url()
|
||||
val headers = request.headers()
|
||||
val dataToSend = request.dataToSend()
|
||||
|
||||
var requestBody: RequestBody? = null
|
||||
if (dataToSend != null) {
|
||||
requestBody = RequestBody.create(null, dataToSend)
|
||||
}
|
||||
|
||||
val requestBuilder = okhttp3.Request.Builder()
|
||||
.method(httpMethod, requestBody).url(url)
|
||||
.addHeader("User-Agent", USER_AGENT)
|
||||
|
||||
for ((headerName, headerValueList) in headers) {
|
||||
if (headerValueList.size > 1) {
|
||||
requestBuilder.removeHeader(headerName)
|
||||
for (headerValue in headerValueList) {
|
||||
requestBuilder.addHeader(headerName, headerValue)
|
||||
}
|
||||
} else if (headerValueList.size == 1) {
|
||||
requestBuilder.header(headerName, headerValueList[0])
|
||||
}
|
||||
}
|
||||
|
||||
val response: okhttp3.Response = okhttp.newCall(requestBuilder.build()).execute()
|
||||
|
||||
if (response.code() == 429) {
|
||||
response.close()
|
||||
throw ReCaptchaException("reCaptcha Challenge requested", url)
|
||||
}
|
||||
|
||||
val body = response.body()
|
||||
var responseBodyToReturn: String? = null
|
||||
|
||||
if (body != null) {
|
||||
responseBodyToReturn = body.string()
|
||||
}
|
||||
|
||||
val latestUrl = response.request().url().toString()
|
||||
return Response(
|
||||
response.code(), response.message(), response.headers().toMultimap(),
|
||||
responseBodyToReturn, latestUrl
|
||||
)
|
||||
}
|
||||
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
|
||||
== Using the library!
|
||||
|
||||
[,kotlin]
|
||||
```
|
||||
// init Okhttp client
|
||||
val okhttp = OkHttpClient.Builder()
|
||||
.readTimeout(30, TimeUnit.SECONDS)
|
||||
.build()
|
||||
|
||||
// setting the downloader.
|
||||
NewPipe.init(DownloaderImpl(okhttp))
|
||||
|
||||
// getting youtube service
|
||||
val extractor = ServiceList.Youtube
|
||||
```
|
||||
|
||||
|
||||
=== Getting suggestions:
|
||||
|
||||
[,kotlin]
|
||||
```
|
||||
extractor.suggestionExtractor.suggestionList("Hello")
|
||||
```
|
||||
|
||||
|
||||
|
||||
=== Search for videos:
|
||||
|
||||
[,kotlin]
|
||||
```
|
||||
val search = extractor.getSearchExtractor("Hello")
|
||||
search.fetchPage()
|
||||
|
||||
// result list. Contains videos/channels etc..
|
||||
// InfoItem include type, name, thumbnail...
|
||||
print(search.initialPage.items)
|
||||
```
|
||||
|
||||
|
||||
=== Get video:
|
||||
|
||||
[,kotlin]
|
||||
```
|
||||
val page = extractor.getStreamExtractor("https://www.youtube.com/watch?v=mIuYmbO9uBM")
|
||||
page.fetchPage()
|
||||
|
||||
// list of video streams (urls)
|
||||
page.videoStreams
|
||||
```
|
||||
|
||||
|
||||
== Final note:
|
||||
This is very humble guide on how to use newpipe to scrap youtube *specificly*
|
||||
|
||||
You can explore the library by your own
|
||||
|
||||
image::https://imgur.com/2ciR9kq.png[width=400]
|
||||
|
||||
I write this for me first to not forget and also in the hope that it will help some one :)
|
||||
Loading…
Reference in New Issue