diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index 447d01ddc..5ce399d91 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -153,6 +153,9 @@ android:label="@string/title_activity_channel" android:theme="@style/AppTheme.NoActionBar" /> + \ No newline at end of file diff --git a/app/src/main/java/org/schabi/newpipe/App.java b/app/src/main/java/org/schabi/newpipe/App.java index 7e93969bc..f9b95c457 100644 --- a/app/src/main/java/org/schabi/newpipe/App.java +++ b/app/src/main/java/org/schabi/newpipe/App.java @@ -62,7 +62,7 @@ public class App extends Application { } //init NewPipe - NewPipe.init(new Downloader()); + NewPipe.init(Downloader.getInstance()); // Initialize image loader ImageLoaderConfiguration config = new ImageLoaderConfiguration.Builder(this).build(); diff --git a/app/src/main/java/org/schabi/newpipe/Downloader.java b/app/src/main/java/org/schabi/newpipe/Downloader.java index 04098a4d7..748d6bd4d 100644 --- a/app/src/main/java/org/schabi/newpipe/Downloader.java +++ b/app/src/main/java/org/schabi/newpipe/Downloader.java @@ -1,5 +1,7 @@ package org.schabi.newpipe; +import org.schabi.newpipe.extractor.exceptions.reCaptchaException; + import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; @@ -35,13 +37,37 @@ import javax.net.ssl.HttpsURLConnection; public class Downloader implements org.schabi.newpipe.extractor.Downloader { private static final String USER_AGENT = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0"; + private static String mCookies = ""; + + private static Downloader instance = null; + + private Downloader() {} + + public static Downloader getInstance() { + if(instance == null) { + synchronized (Downloader.class) { + if (instance == null) { + instance = new Downloader(); + } + } + } + return instance; + } + + public static synchronized void setCookies(String cookies) { + Downloader.mCookies = cookies; + } + + public static synchronized String getCookies() { + return Downloader.mCookies; + } /**Download the text file at the supplied URL as in download(String), * but set the HTTP header field "Accept-Language" to the supplied string. * @param siteUrl the URL of the text file to return the contents of * @param language the language (usually a 2-character code) to set as the preferred language * @return the contents of the specified text file*/ - public String download(String siteUrl, String language) throws IOException { + public String download(String siteUrl, String language) throws IOException, reCaptchaException { Map requestProperties = new HashMap<>(); requestProperties.put("Accept-Language", language); return download(siteUrl, requestProperties); @@ -54,7 +80,7 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader { * @param customProperties set request header properties * @return the contents of the specified text file * @throws IOException*/ - public String download(String siteUrl, Map customProperties) throws IOException { + public String download(String siteUrl, Map customProperties) throws IOException, reCaptchaException { URL url = new URL(siteUrl); HttpsURLConnection con = (HttpsURLConnection) url.openConnection(); Iterator it = customProperties.entrySet().iterator(); @@ -66,7 +92,7 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader { } /**Common functionality between download(String url) and download(String url, String language)*/ - private static String dl(HttpsURLConnection con) throws IOException { + private static String dl(HttpsURLConnection con) throws IOException, reCaptchaException { StringBuilder response = new StringBuilder(); BufferedReader in = null; @@ -74,6 +100,10 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader { con.setRequestMethod("GET"); con.setRequestProperty("User-Agent", USER_AGENT); + if (getCookies().length() > 0) { + con.setRequestProperty("Cookie", getCookies()); + } + in = new BufferedReader( new InputStreamReader(con.getInputStream())); String inputLine; @@ -85,6 +115,14 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader { throw new IOException("unknown host or no network", uhe); //Toast.makeText(getActivity(), uhe.getMessage(), Toast.LENGTH_LONG).show(); } catch(Exception e) { + /* + * HTTP 429 == Too Many Request + * Receive from Youtube.com = ReCaptcha challenge request + * See : https://github.com/rg3/youtube-dl/issues/5138 + */ + if (con.getResponseCode() == 429) { + throw new reCaptchaException("reCaptcha Challenge requested"); + } throw new IOException(e); } finally { if(in != null) { @@ -99,7 +137,7 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader { * Primarily intended for downloading web pages. * @param siteUrl the URL of the text file to download * @return the contents of the specified text file*/ - public String download(String siteUrl) throws IOException { + public String download(String siteUrl) throws IOException, reCaptchaException { URL url = new URL(siteUrl); HttpsURLConnection con = (HttpsURLConnection) url.openConnection(); //HttpsURLConnection con = NetCipher.getHttpsURLConnection(url); diff --git a/app/src/main/java/org/schabi/newpipe/MainActivity.java b/app/src/main/java/org/schabi/newpipe/MainActivity.java index 5d4729e90..bfeb74287 100644 --- a/app/src/main/java/org/schabi/newpipe/MainActivity.java +++ b/app/src/main/java/org/schabi/newpipe/MainActivity.java @@ -46,6 +46,7 @@ public class MainActivity extends AppCompatActivity { .findFragmentById(R.id.search_fragment); } + @Override public boolean onCreateOptionsMenu(Menu menu) { super.onCreateOptionsMenu(menu); MenuInflater inflater = getMenuInflater(); diff --git a/app/src/main/java/org/schabi/newpipe/ReCaptchaActivity.java b/app/src/main/java/org/schabi/newpipe/ReCaptchaActivity.java new file mode 100644 index 000000000..f503bfcff --- /dev/null +++ b/app/src/main/java/org/schabi/newpipe/ReCaptchaActivity.java @@ -0,0 +1,138 @@ +package org.schabi.newpipe; + +import android.app.Activity; +import android.content.Intent; +import android.os.Build; +import android.os.Bundle; +import android.support.v4.app.NavUtils; +import android.support.v7.app.ActionBar; +import android.support.v7.app.AppCompatActivity; +import android.view.MenuItem; +import android.webkit.CookieManager; +import android.webkit.ValueCallback; +import android.webkit.WebSettings; +import android.webkit.WebView; +import android.webkit.WebViewClient; + +/** + * Created by beneth on 06.12.16. + * + * Copyright (C) Christian Schabesberger 2015 + * ReCaptchaActivity.java is part of NewPipe. + * + * NewPipe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe. If not, see . + */ +public class ReCaptchaActivity extends AppCompatActivity { + public static final String TAG = ReCaptchaActivity.class.toString(); + public static final String YT_URL = "https://www.youtube.com"; + + @Override + protected void onCreate(Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + setContentView(R.layout.activity_recaptcha); + + ActionBar actionBar = getSupportActionBar(); + actionBar.setDisplayHomeAsUpEnabled(true); + actionBar.setTitle(R.string.reCaptcha_title); + actionBar.setDisplayShowTitleEnabled(true); + + WebView myWebView = (WebView) findViewById(R.id.reCaptchaWebView); + + // Enable Javascript + WebSettings webSettings = myWebView.getSettings(); + webSettings.setJavaScriptEnabled(true); + + ReCaptchaWebViewClient webClient = new ReCaptchaWebViewClient(this); + myWebView.setWebViewClient(webClient); + + // Cleaning cache, history and cookies from webView + myWebView.clearCache(true); + myWebView.clearHistory(); + android.webkit.CookieManager cookieManager = CookieManager.getInstance(); + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { + cookieManager.removeAllCookies(new ValueCallback() { + @Override + public void onReceiveValue(Boolean aBoolean) {} + }); + } else { + cookieManager.removeAllCookie(); + } + + myWebView.loadUrl(YT_URL); + } + + private class ReCaptchaWebViewClient extends WebViewClient { + private Activity context; + private String mCookies; + + ReCaptchaWebViewClient(Activity ctx) { + context = ctx; + } + + @Override + public void onPageFinished(WebView view, String url) { + String cookies = CookieManager.getInstance().getCookie(url); + + // find cookies : s_gl & goojf and Add cookies to Downloader + if (find_access_cookies(cookies)) { + // Give cookies to Downloader class + Downloader.setCookies(mCookies); + + // Closing activity and return to parent. + Intent intent = new Intent(context, org.schabi.newpipe.MainActivity.class); + intent.addFlags(Intent.FLAG_ACTIVITY_CLEAR_TOP); + NavUtils.navigateUpTo(context, intent); + } + } + + private boolean find_access_cookies(String cookies) { + boolean ret = false; + String c_s_gl = ""; + String c_goojf = ""; + + String[] parts = cookies.split("; "); + for (String part : parts) { + if (part.trim().startsWith("s_gl")) { + c_s_gl = part.trim(); + } + if (part.trim().startsWith("goojf")) { + c_goojf = part.trim(); + } + } + if (c_s_gl.length() > 0 && c_goojf.length() > 0) { + ret = true; + //mCookies = c_s_gl + "; " + c_goojf; + // Youtube seems to also need the other cookies: + mCookies = cookies; + } + + return ret; + } + } + + @Override + public boolean onOptionsItemSelected(MenuItem item) { + int id = item.getItemId(); + switch (id) { + case android.R.id.home: { + Intent intent = new Intent(this, org.schabi.newpipe.MainActivity.class); + intent.addFlags(Intent.FLAG_ACTIVITY_CLEAR_TOP); + NavUtils.navigateUpTo(this, intent); + return true; + } + default: + return false; + } + } +} diff --git a/app/src/main/java/org/schabi/newpipe/extractor/DashMpdParser.java b/app/src/main/java/org/schabi/newpipe/extractor/DashMpdParser.java index db26e3871..7dcbc16a8 100644 --- a/app/src/main/java/org/schabi/newpipe/extractor/DashMpdParser.java +++ b/app/src/main/java/org/schabi/newpipe/extractor/DashMpdParser.java @@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor; import android.util.Xml; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.exceptions.reCaptchaException; import org.schabi.newpipe.extractor.stream_info.AudioStream; import org.xmlpull.v1.XmlPullParser; @@ -43,13 +44,15 @@ public class DashMpdParser { } public static List getAudioStreams(String dashManifestUrl) - throws DashMpdParsingException { + throws DashMpdParsingException, reCaptchaException { String dashDoc; Downloader downloader = NewPipe.getDownloader(); try { dashDoc = downloader.download(dashManifestUrl); } catch(IOException ioe) { throw new DashMpdParsingException("Could not get dash mpd: " + dashManifestUrl, ioe); + } catch (reCaptchaException e) { + throw new reCaptchaException("reCaptcha Challenge needed"); } Vector audioStreams = new Vector<>(); try { diff --git a/app/src/main/java/org/schabi/newpipe/extractor/Downloader.java b/app/src/main/java/org/schabi/newpipe/extractor/Downloader.java index 14475dba7..f25375113 100644 --- a/app/src/main/java/org/schabi/newpipe/extractor/Downloader.java +++ b/app/src/main/java/org/schabi/newpipe/extractor/Downloader.java @@ -1,5 +1,7 @@ package org.schabi.newpipe.extractor; +import org.schabi.newpipe.extractor.exceptions.reCaptchaException; + import java.io.IOException; import java.util.Map; @@ -31,7 +33,7 @@ public interface Downloader { * @param language the language (usually a 2-character code) to set as the preferred language * @return the contents of the specified text file * @throws IOException*/ - String download(String siteUrl, String language) throws IOException; + String download(String siteUrl, String language) throws IOException, reCaptchaException; /**Download the text file at the supplied URL as in download(String), * but set the HTTP header field "Accept-Language" to the supplied string. @@ -39,12 +41,12 @@ public interface Downloader { * @param customProperties set request header properties * @return the contents of the specified text file * @throws IOException*/ - String download(String siteUrl, Map customProperties) throws IOException; + String download(String siteUrl, Map customProperties) throws IOException, reCaptchaException; /**Download (via HTTP) the text file located at the supplied URL, and return its contents. * Primarily intended for downloading web pages. * @param siteUrl the URL of the text file to download * @return the contents of the specified text file * @throws IOException*/ - String download(String siteUrl) throws IOException; + String download(String siteUrl) throws IOException, reCaptchaException; } diff --git a/app/src/main/java/org/schabi/newpipe/extractor/exceptions/reCaptchaException.java b/app/src/main/java/org/schabi/newpipe/extractor/exceptions/reCaptchaException.java new file mode 100644 index 000000000..a9107f15e --- /dev/null +++ b/app/src/main/java/org/schabi/newpipe/extractor/exceptions/reCaptchaException.java @@ -0,0 +1,27 @@ +package org.schabi.newpipe.extractor.exceptions; + +/** + * Created by beneth on 07.12.16. + * + * Copyright (C) Christian Schabesberger 2016 + * reCaptchaException.java is part of NewPipe. + * + * NewPipe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe. If not, see . + */ + +public class reCaptchaException extends ExtractionException { + public reCaptchaException(String message) { + super(message); + } +} diff --git a/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java b/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java index e1cbdd1a6..aa07bf96f 100644 --- a/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java +++ b/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java @@ -11,6 +11,7 @@ import org.mozilla.javascript.ScriptableObject; import org.schabi.newpipe.extractor.AbstractStreamInfo; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.exceptions.reCaptchaException; import org.schabi.newpipe.extractor.stream_info.AudioStream; import org.schabi.newpipe.extractor.Downloader; import org.schabi.newpipe.extractor.NewPipe; @@ -280,7 +281,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { } } - private String getPlayerUrlFromRestrictedVideo(String pageUrl) throws ParsingException { + private String getPlayerUrlFromRestrictedVideo(String pageUrl) throws ParsingException, reCaptchaException { try { Downloader downloader = NewPipe.getDownloader(); String playerUrl = ""; @@ -302,6 +303,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { } catch (IOException e) { throw new ParsingException( "Could load decryption code form restricted video for the Youtube service.", e); + } catch (reCaptchaException e) { + throw new reCaptchaException("reCaptcha Challenge requested"); } } diff --git a/app/src/main/java/org/schabi/newpipe/report/ErrorActivity.java b/app/src/main/java/org/schabi/newpipe/report/ErrorActivity.java index 49cfe0471..85151efb3 100644 --- a/app/src/main/java/org/schabi/newpipe/report/ErrorActivity.java +++ b/app/src/main/java/org/schabi/newpipe/report/ErrorActivity.java @@ -473,7 +473,7 @@ public class ErrorActivity extends AppCompatActivity { public void run() { String ipRange = "none"; try { - Downloader dl = new Downloader(); + Downloader dl = Downloader.getInstance(); String ip = dl.download("https://ifcfg.me/ip"); ipRange = Parser.matchGroup1("([0-9]*\\.[0-9]*\\.)[0-9]*\\.[0-9]*", ip) diff --git a/app/src/main/java/org/schabi/newpipe/search_fragment/SearchInfoItemFragment.java b/app/src/main/java/org/schabi/newpipe/search_fragment/SearchInfoItemFragment.java index 782f7a78b..e710bbbd9 100644 --- a/app/src/main/java/org/schabi/newpipe/search_fragment/SearchInfoItemFragment.java +++ b/app/src/main/java/org/schabi/newpipe/search_fragment/SearchInfoItemFragment.java @@ -18,6 +18,7 @@ import android.view.inputmethod.InputMethodManager; import android.widget.ProgressBar; import android.widget.Toast; +import org.schabi.newpipe.ReCaptchaActivity; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.search.SearchResult; import org.schabi.newpipe.info_list.InfoItemBuilder; @@ -149,7 +150,7 @@ public class SearchInfoItemFragment extends Fragment { } SearchWorker sw = SearchWorker.getInstance(); - sw.setSearchWorkerResultListner(new SearchWorker.SearchWorkerResultListner() { + sw.setSearchWorkerResultListener(new SearchWorker.SearchWorkerResultListener() { @Override public void onResult(SearchResult result) { infoListAdapter.addStreamItemList(result.resultList); @@ -174,6 +175,15 @@ public class SearchInfoItemFragment extends Fragment { isLoading = false; loadingIndicator.setVisibility(View.GONE); } + + @Override + public void onReCaptchaChallenge() { + Toast.makeText(getActivity(), "ReCaptcha Challenge requested", + Toast.LENGTH_LONG).show(); + // Starting ReCaptcha Challenge Activity + Intent i = new Intent(getActivity(), ReCaptchaActivity.class); + getActivity().startActivity(i); + } }); } diff --git a/app/src/main/java/org/schabi/newpipe/search_fragment/SearchWorker.java b/app/src/main/java/org/schabi/newpipe/search_fragment/SearchWorker.java index b6be1e259..7ce0d30c5 100644 --- a/app/src/main/java/org/schabi/newpipe/search_fragment/SearchWorker.java +++ b/app/src/main/java/org/schabi/newpipe/search_fragment/SearchWorker.java @@ -8,6 +8,7 @@ import android.util.Log; import android.view.View; import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.reCaptchaException; import org.schabi.newpipe.extractor.search.SearchEngine; import org.schabi.newpipe.extractor.search.SearchResult; import org.schabi.newpipe.report.ErrorActivity; @@ -40,10 +41,11 @@ import java.io.IOException; public class SearchWorker { private static final String TAG = SearchWorker.class.toString(); - public interface SearchWorkerResultListner { + public interface SearchWorkerResultListener { void onResult(SearchResult result); void onNothingFound(final int stringResource); void onError(String message); + void onReCaptchaChallenge(); } private class ResultRunnable implements Runnable { @@ -56,7 +58,7 @@ public class SearchWorker { @Override public void run() { if(this.requestId == SearchWorker.this.requestId) { - searchWorkerResultListner.onResult(result); + searchWorkerResultListener.onResult(result); } } } @@ -121,11 +123,18 @@ public class SearchWorker { } // hard errors: + } catch (reCaptchaException e) { + h.post(new Runnable() { + @Override + public void run() { + searchWorkerResultListener.onReCaptchaChallenge(); + } + }); } catch(IOException e) { h.post(new Runnable() { @Override public void run() { - searchWorkerResultListner.onNothingFound(R.string.network_error); + searchWorkerResultListener.onNothingFound(R.string.network_error); } }); e.printStackTrace(); @@ -133,7 +142,7 @@ public class SearchWorker { h.post(new Runnable() { @Override public void run() { - searchWorkerResultListner.onError(e.getMessage()); + searchWorkerResultListener.onError(e.getMessage()); } }); } catch(ExtractionException e) { @@ -155,7 +164,7 @@ public class SearchWorker { } private static SearchWorker searchWorker = null; - private SearchWorkerResultListner searchWorkerResultListner = null; + private SearchWorkerResultListener searchWorkerResultListener = null; private SearchRunnable runnable = null; private int requestId = 0; //prevents running requests that have already ben expired @@ -163,8 +172,8 @@ public class SearchWorker { return searchWorker == null ? (searchWorker = new SearchWorker()) : searchWorker; } - public void setSearchWorkerResultListner(SearchWorkerResultListner listener) { - searchWorkerResultListner = listener; + public void setSearchWorkerResultListener(SearchWorkerResultListener listener) { + searchWorkerResultListener = listener; } private SearchWorker() { diff --git a/app/src/main/res/layout/activity_recaptcha.xml b/app/src/main/res/layout/activity_recaptcha.xml new file mode 100644 index 000000000..0a3b03010 --- /dev/null +++ b/app/src/main/res/layout/activity_recaptcha.xml @@ -0,0 +1,10 @@ + + + + + \ No newline at end of file diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index dfd247dd7..208cac70d 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -269,6 +269,8 @@ "when a user tries to pick up one of cards.\n\n" Settings + reCaptcha + reCaptcha Challenge