From e2cbf40957eb2a0ebd0085bf0d504cb30ef3931b Mon Sep 17 00:00:00 2001 From: Stypox Date: Fri, 10 Apr 2020 18:48:40 +0200 Subject: [PATCH 1/3] Fix recaptcha activity for another type of recaptcha page Try to get cookies from pages even when they start loading, because non-html pages like ones with `pbj=1` never stop loading. Try to extract the cookie directly from the redirection url, by looking at the field "google_abuse=". Add `GOOGLE_ABUSE_EXEMPTION=` to the youtube recaptcha cookies. --- .../org/schabi/newpipe/ReCaptchaActivity.java | 84 ++++++++++++++++--- 1 file changed, 74 insertions(+), 10 deletions(-) diff --git a/app/src/main/java/org/schabi/newpipe/ReCaptchaActivity.java b/app/src/main/java/org/schabi/newpipe/ReCaptchaActivity.java index a8a83e13e..0c9538037 100644 --- a/app/src/main/java/org/schabi/newpipe/ReCaptchaActivity.java +++ b/app/src/main/java/org/schabi/newpipe/ReCaptchaActivity.java @@ -7,11 +7,14 @@ import android.util.Log; import android.view.Menu; import android.view.MenuItem; import android.webkit.CookieManager; +import android.webkit.WebResourceRequest; import android.webkit.WebSettings; import android.webkit.WebView; import android.webkit.WebViewClient; import androidx.annotation.NonNull; +import androidx.annotation.Nullable; +import androidx.annotation.RequiresApi; import androidx.appcompat.app.ActionBar; import androidx.appcompat.app.AppCompatActivity; import androidx.appcompat.widget.Toolbar; @@ -19,6 +22,9 @@ import androidx.core.app.NavUtils; import org.schabi.newpipe.util.ThemeHelper; +import java.io.UnsupportedEncodingException; +import java.net.URLDecoder; + /* * Created by beneth on 06.12.16. * @@ -71,10 +77,33 @@ public class ReCaptchaActivity extends AppCompatActivity { webSettings.setJavaScriptEnabled(true); webView.setWebViewClient(new WebViewClient() { + @RequiresApi(api = Build.VERSION_CODES.LOLLIPOP) + @Override + public boolean shouldOverrideUrlLoading(final WebView view, + final WebResourceRequest request) { + String url = request.getUrl().toString(); + if (MainActivity.DEBUG) { + Log.d(TAG, "shouldOverrideUrlLoading: request.url=" + url); + } + + handleCookiesFromUrl(url); + return false; + } + + @Override + public boolean shouldOverrideUrlLoading(final WebView view, final String url) { + if (MainActivity.DEBUG) { + Log.d(TAG, "shouldOverrideUrlLoading: url=" + url); + } + + handleCookiesFromUrl(url); + return false; + } + @Override public void onPageFinished(final WebView view, final String url) { super.onPageFinished(view, url); - handleCookies(url); + handleCookiesFromUrl(url); } }); @@ -124,7 +153,11 @@ public class ReCaptchaActivity extends AppCompatActivity { } private void saveCookiesAndFinish() { - handleCookies(webView.getUrl()); // try to get cookies of unclosed page + handleCookiesFromUrl(webView.getUrl()); // try to get cookies of unclosed page + if (MainActivity.DEBUG) { + Log.d(TAG, "saveCookiesAndFinish: foundCookies=" + foundCookies); + } + if (!foundCookies.isEmpty()) { // give cookies to Downloader class DownloaderImpl.getInstance().setCookies(foundCookies); @@ -137,23 +170,54 @@ public class ReCaptchaActivity extends AppCompatActivity { } - private void handleCookies(final String url) { - String cookies = CookieManager.getInstance().getCookie(url); + private void handleCookiesFromUrl(final @Nullable String url) { if (MainActivity.DEBUG) { - Log.d(TAG, "handleCookies: " - + "url=" + url + "; cookies=" + (cookies == null ? "null" : cookies)); + Log.d(TAG, "handleCookiesFromUrl: url=" + (url == null ? "null" : url)); } - if (cookies == null) { + + if (url == null) { + return; + } + + String cookies = CookieManager.getInstance().getCookie(url); + handleCookies(cookies); + + // sometimes cookies are inside the url + int abuseStart = url.indexOf("google_abuse="); + if (abuseStart != -1) { + int abuseEnd = url.indexOf("+path"); + + try { + String abuseCookie = url.substring(abuseStart + 13, abuseEnd); + abuseCookie = URLDecoder.decode(abuseCookie, "UTF-8"); + handleCookies(abuseCookie); + } catch (UnsupportedEncodingException | StringIndexOutOfBoundsException e) { + if (MainActivity.DEBUG) { + e.printStackTrace(); + Log.d(TAG, "handleCookiesFromUrl: invalid google abuse starting at " + + abuseStart + " and ending at " + abuseEnd + " for url " + url); + } + } + } + } + + private void handleCookies(final @Nullable String cookies) { + if (MainActivity.DEBUG) { + Log.d(TAG, "handleCookies: cookies=" + (cookies == null ? "null" : cookies)); + } + + if (cookies == null || foundCookies.contains(cookies)) { return; } addYoutubeCookies(cookies); - // add other methods to extract cookies here + // add here methods to extract cookies for other services } - private void addYoutubeCookies(@NonNull final String cookies) { + private void addYoutubeCookies(final @NonNull String cookies) { if (cookies.contains("s_gl=") || cookies.contains("goojf=") - || cookies.contains("VISITOR_INFO1_LIVE=")) { + || cookies.contains("VISITOR_INFO1_LIVE=") + || cookies.contains("GOOGLE_ABUSE_EXEMPTION=")) { // youtube seems to also need the other cookies: addCookie(cookies); } From b8efef7c7a9f3fce8e57a5fb4f6cf1b0f5a3cfd3 Mon Sep 17 00:00:00 2001 From: Stypox Date: Sat, 11 Apr 2020 11:50:56 +0200 Subject: [PATCH 2/3] Remove duplicate check if cookies are already present --- app/src/main/java/org/schabi/newpipe/ReCaptchaActivity.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/src/main/java/org/schabi/newpipe/ReCaptchaActivity.java b/app/src/main/java/org/schabi/newpipe/ReCaptchaActivity.java index 0c9538037..9741372df 100644 --- a/app/src/main/java/org/schabi/newpipe/ReCaptchaActivity.java +++ b/app/src/main/java/org/schabi/newpipe/ReCaptchaActivity.java @@ -206,7 +206,7 @@ public class ReCaptchaActivity extends AppCompatActivity { Log.d(TAG, "handleCookies: cookies=" + (cookies == null ? "null" : cookies)); } - if (cookies == null || foundCookies.contains(cookies)) { + if (cookies == null) { return; } From ee5ce0c8096cf735588b7b9d13bdc74d39f0f69d Mon Sep 17 00:00:00 2001 From: Stypox Date: Sat, 11 Apr 2020 11:51:40 +0200 Subject: [PATCH 3/3] Save and restore cookies to/from preferences So that the user does not have to solve a recaptcha every time he opens the app --- .../debug/java/org/schabi/newpipe/DebugApp.java | 4 +++- app/src/main/java/org/schabi/newpipe/App.java | 13 ++++++++++++- .../java/org/schabi/newpipe/ReCaptchaActivity.java | 14 +++++++++++--- app/src/main/res/values/settings_keys.xml | 1 + 4 files changed, 27 insertions(+), 5 deletions(-) diff --git a/app/src/debug/java/org/schabi/newpipe/DebugApp.java b/app/src/debug/java/org/schabi/newpipe/DebugApp.java index 4d763aeb1..6bcf71035 100644 --- a/app/src/debug/java/org/schabi/newpipe/DebugApp.java +++ b/app/src/debug/java/org/schabi/newpipe/DebugApp.java @@ -40,8 +40,10 @@ public class DebugApp extends App { @Override protected Downloader getDownloader() { - return DownloaderImpl.init(new OkHttpClient.Builder() + DownloaderImpl downloader = DownloaderImpl.init(new OkHttpClient.Builder() .addNetworkInterceptor(new StethoInterceptor())); + setCookiesToDownloader(downloader); + return downloader; } private void initStetho() { diff --git a/app/src/main/java/org/schabi/newpipe/App.java b/app/src/main/java/org/schabi/newpipe/App.java index 7e9df0bed..4d05c69cc 100644 --- a/app/src/main/java/org/schabi/newpipe/App.java +++ b/app/src/main/java/org/schabi/newpipe/App.java @@ -5,10 +5,12 @@ import android.app.Application; import android.app.NotificationChannel; import android.app.NotificationManager; import android.content.Context; +import android.content.SharedPreferences; import android.os.Build; import android.util.Log; import androidx.annotation.Nullable; +import androidx.preference.PreferenceManager; import com.nostra13.universalimageloader.cache.memory.impl.LRULimitedMemoryCache; import com.nostra13.universalimageloader.core.ImageLoader; @@ -125,7 +127,16 @@ public class App extends Application { } protected Downloader getDownloader() { - return DownloaderImpl.init(null); + DownloaderImpl downloader = DownloaderImpl.init(null); + setCookiesToDownloader(downloader); + return downloader; + } + + protected void setCookiesToDownloader(final DownloaderImpl downloader) { + final SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences( + getApplicationContext()); + final String key = getApplicationContext().getString(R.string.recaptcha_cookies_key); + downloader.setCookies(prefs.getString(key, "")); } private void configureRxJavaErrorHandler() { diff --git a/app/src/main/java/org/schabi/newpipe/ReCaptchaActivity.java b/app/src/main/java/org/schabi/newpipe/ReCaptchaActivity.java index 9741372df..49fb6b179 100644 --- a/app/src/main/java/org/schabi/newpipe/ReCaptchaActivity.java +++ b/app/src/main/java/org/schabi/newpipe/ReCaptchaActivity.java @@ -1,6 +1,7 @@ package org.schabi.newpipe; import android.content.Intent; +import android.content.SharedPreferences; import android.os.Build; import android.os.Bundle; import android.util.Log; @@ -19,6 +20,7 @@ import androidx.appcompat.app.ActionBar; import androidx.appcompat.app.AppCompatActivity; import androidx.appcompat.widget.Toolbar; import androidx.core.app.NavUtils; +import androidx.preference.PreferenceManager; import org.schabi.newpipe.util.ThemeHelper; @@ -159,6 +161,12 @@ public class ReCaptchaActivity extends AppCompatActivity { } if (!foundCookies.isEmpty()) { + // save cookies to preferences + final SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences( + getApplicationContext()); + final String key = getApplicationContext().getString(R.string.recaptcha_cookies_key); + prefs.edit().putString(key, foundCookies).apply(); + // give cookies to Downloader class DownloaderImpl.getInstance().setCookies(foundCookies); setResult(RESULT_OK); @@ -170,7 +178,7 @@ public class ReCaptchaActivity extends AppCompatActivity { } - private void handleCookiesFromUrl(final @Nullable String url) { + private void handleCookiesFromUrl(@Nullable final String url) { if (MainActivity.DEBUG) { Log.d(TAG, "handleCookiesFromUrl: url=" + (url == null ? "null" : url)); } @@ -201,7 +209,7 @@ public class ReCaptchaActivity extends AppCompatActivity { } } - private void handleCookies(final @Nullable String cookies) { + private void handleCookies(@Nullable final String cookies) { if (MainActivity.DEBUG) { Log.d(TAG, "handleCookies: cookies=" + (cookies == null ? "null" : cookies)); } @@ -214,7 +222,7 @@ public class ReCaptchaActivity extends AppCompatActivity { // add here methods to extract cookies for other services } - private void addYoutubeCookies(final @NonNull String cookies) { + private void addYoutubeCookies(@NonNull final String cookies) { if (cookies.contains("s_gl=") || cookies.contains("goojf=") || cookies.contains("VISITOR_INFO1_LIVE=") || cookies.contains("GOOGLE_ABUSE_EXEMPTION=")) { diff --git a/app/src/main/res/values/settings_keys.xml b/app/src/main/res/values/settings_keys.xml index b5eeae6aa..ca8528fef 100644 --- a/app/src/main/res/values/settings_keys.xml +++ b/app/src/main/res/values/settings_keys.xml @@ -1129,4 +1129,5 @@ @string/grid + recaptcha_cookies_key