Merge branch 'reCaptcha' of git://github.com/be-neth/NewPipe into be-neth-reCaptcha

app/src/main/AndroidManifest.xml

@@ -153,6 +153,9 @@
             android:label="@string/title_activity_channel"
             android:theme="@style/AppTheme.NoActionBar" />
 
+        <activity
+            android:name=".ReCaptchaActivity"
+            android:label="@string/reCaptchaActivity" />
     </application>
 
 </manifest>

app/src/main/java/org/schabi/newpipe/App.java

@@ -62,7 +62,7 @@ public class App extends Application {
         }
 
         //init NewPipe
-        NewPipe.init(new Downloader());
+        NewPipe.init(Downloader.getInstance());
 
         // Initialize image loader
         ImageLoaderConfiguration config = new ImageLoaderConfiguration.Builder(this).build();

app/src/main/java/org/schabi/newpipe/Downloader.java

@@ -1,5 +1,7 @@
 package org.schabi.newpipe;
 
+import org.schabi.newpipe.extractor.exceptions.reCaptchaException;
+
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStreamReader;
@@ -35,13 +37,37 @@ import javax.net.ssl.HttpsURLConnection;
 public class Downloader implements org.schabi.newpipe.extractor.Downloader {
     
     private static final String USER_AGENT = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0";
+    private static String mCookies = "";
+
+    private static Downloader instance = null;
+
+    private Downloader() {}
+
+    public static Downloader getInstance() {
+        if(instance == null) {
+            synchronized (Downloader.class) {
+                if (instance == null) {
+                    instance = new Downloader();
+                }
+            }
+        }
+        return instance;
+    }
+
+    public static synchronized void setCookies(String cookies) {
+        Downloader.mCookies = cookies;
+    }
+
+    public static synchronized String getCookies() {
+        return Downloader.mCookies;
+    }
 
     /**Download the text file at the supplied URL as in download(String),
      * but set the HTTP header field "Accept-Language" to the supplied string.
      * @param siteUrl the URL of the text file to return the contents of
      * @param language the language (usually a 2-character code) to set as the preferred language
      * @return the contents of the specified text file*/
-    public String download(String siteUrl, String language) throws IOException {
+    public String download(String siteUrl, String language) throws IOException, reCaptchaException {
         Map<String, String> requestProperties = new HashMap<>();
         requestProperties.put("Accept-Language", language);
         return download(siteUrl, requestProperties);
@@ -54,7 +80,7 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
      * @param customProperties set request header properties
      * @return the contents of the specified text file
      * @throws IOException*/
-    public String download(String siteUrl, Map<String, String> customProperties) throws IOException {
+    public String download(String siteUrl, Map<String, String> customProperties) throws IOException, reCaptchaException {
         URL url = new URL(siteUrl);
         HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
         Iterator it = customProperties.entrySet().iterator();
@@ -66,7 +92,7 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
     }
 
     /**Common functionality between download(String url) and download(String url, String language)*/
-    private static String dl(HttpsURLConnection con) throws IOException {
+    private static String dl(HttpsURLConnection con) throws IOException, reCaptchaException {
         StringBuilder response = new StringBuilder();
         BufferedReader in = null;
 
@@ -74,6 +100,10 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
             con.setRequestMethod("GET");
             con.setRequestProperty("User-Agent", USER_AGENT);
 
+            if (getCookies().length() > 0) {
+                con.setRequestProperty("Cookie", getCookies());
+            }
+
             in = new BufferedReader(
                     new InputStreamReader(con.getInputStream()));
             String inputLine;
@@ -85,6 +115,14 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
             throw new IOException("unknown host or no network", uhe);
             //Toast.makeText(getActivity(), uhe.getMessage(), Toast.LENGTH_LONG).show();
         } catch(Exception e) {
+            /*
+             * HTTP 429 == Too Many Request
+             * Receive from Youtube.com = ReCaptcha challenge request
+             * See : https://github.com/rg3/youtube-dl/issues/5138
+             */
+            if (con.getResponseCode() == 429) {
+                throw new reCaptchaException("reCaptcha Challenge requested");
+            }
             throw new IOException(e);
         } finally {
             if(in != null) {
@@ -99,7 +137,7 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
      * Primarily intended for downloading web pages.
      * @param siteUrl the URL of the text file to download
      * @return the contents of the specified text file*/
-    public String download(String siteUrl) throws IOException {
+    public String download(String siteUrl) throws IOException, reCaptchaException {
         URL url = new URL(siteUrl);
         HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
         //HttpsURLConnection con = NetCipher.getHttpsURLConnection(url);

app/src/main/java/org/schabi/newpipe/MainActivity.java

@@ -46,6 +46,7 @@ public class MainActivity extends AppCompatActivity {
                 .findFragmentById(R.id.search_fragment);
     }
 
+    @Override
     public boolean onCreateOptionsMenu(Menu menu) {
         super.onCreateOptionsMenu(menu);
         MenuInflater inflater = getMenuInflater();

app/src/main/java/org/schabi/newpipe/ReCaptchaActivity.java

@@ -0,0 +1,138 @@
+package org.schabi.newpipe;
+
+import android.app.Activity;
+import android.content.Intent;
+import android.os.Build;
+import android.os.Bundle;
+import android.support.v4.app.NavUtils;
+import android.support.v7.app.ActionBar;
+import android.support.v7.app.AppCompatActivity;
+import android.view.MenuItem;
+import android.webkit.CookieManager;
+import android.webkit.ValueCallback;
+import android.webkit.WebSettings;
+import android.webkit.WebView;
+import android.webkit.WebViewClient;
+
+/**
+ * Created by beneth <bmauduit@beneth.fr> on 06.12.16.
+ *
+ * Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
+ * ReCaptchaActivity.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+public class ReCaptchaActivity extends AppCompatActivity {
+    public static final String TAG = ReCaptchaActivity.class.toString();
+    public static final String YT_URL = "https://www.youtube.com";
+
+    @Override
+    protected void onCreate(Bundle savedInstanceState) {
+        super.onCreate(savedInstanceState);
+        setContentView(R.layout.activity_recaptcha);
+
+        ActionBar actionBar = getSupportActionBar();
+        actionBar.setDisplayHomeAsUpEnabled(true);
+        actionBar.setTitle(R.string.reCaptcha_title);
+        actionBar.setDisplayShowTitleEnabled(true);
+
+        WebView myWebView = (WebView) findViewById(R.id.reCaptchaWebView);
+
+        // Enable Javascript
+        WebSettings webSettings = myWebView.getSettings();
+        webSettings.setJavaScriptEnabled(true);
+
+        ReCaptchaWebViewClient webClient = new ReCaptchaWebViewClient(this);
+        myWebView.setWebViewClient(webClient);
+
+        // Cleaning cache, history and cookies from webView
+        myWebView.clearCache(true);
+        myWebView.clearHistory();
+        android.webkit.CookieManager cookieManager = CookieManager.getInstance();
+        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {
+            cookieManager.removeAllCookies(new ValueCallback<Boolean>() {
+                @Override
+                public void onReceiveValue(Boolean aBoolean) {}
+            });
+        } else {
+            cookieManager.removeAllCookie();
+        }
+
+        myWebView.loadUrl(YT_URL);
+    }
+
+    private class ReCaptchaWebViewClient extends WebViewClient {
+        private Activity context;
+        private String mCookies;
+
+        ReCaptchaWebViewClient(Activity ctx) {
+            context = ctx;
+        }
+
+        @Override
+        public void onPageFinished(WebView view, String url) {
+            String cookies = CookieManager.getInstance().getCookie(url);
+
+            // find cookies : s_gl & goojf and Add cookies to Downloader
+            if (find_access_cookies(cookies)) {
+                // Give cookies to Downloader class
+                Downloader.setCookies(mCookies);
+
+                // Closing activity and return to parent.
+                Intent intent = new Intent(context, org.schabi.newpipe.MainActivity.class);
+                intent.addFlags(Intent.FLAG_ACTIVITY_CLEAR_TOP);
+                NavUtils.navigateUpTo(context, intent);
+            }
+        }
+
+        private boolean find_access_cookies(String cookies) {
+            boolean ret = false;
+            String c_s_gl = "";
+            String c_goojf = "";
+
+            String[] parts = cookies.split("; ");
+            for (String part : parts) {
+                if (part.trim().startsWith("s_gl")) {
+                    c_s_gl = part.trim();
+                }
+                if (part.trim().startsWith("goojf")) {
+                    c_goojf = part.trim();
+                }
+            }
+            if (c_s_gl.length() > 0 && c_goojf.length() > 0) {
+                ret = true;
+                //mCookies = c_s_gl + "; " + c_goojf;
+                // Youtube seems to also need the other cookies:
+                mCookies = cookies;
+            }
+
+            return ret;
+        }
+    }
+
+    @Override
+    public boolean onOptionsItemSelected(MenuItem item) {
+        int id = item.getItemId();
+        switch (id) {
+            case android.R.id.home: {
+                Intent intent = new Intent(this, org.schabi.newpipe.MainActivity.class);
+                intent.addFlags(Intent.FLAG_ACTIVITY_CLEAR_TOP);
+                NavUtils.navigateUpTo(this, intent);
+                return true;
+            }
+            default:
+                return false;
+        }
+    }
+}

app/src/main/java/org/schabi/newpipe/extractor/DashMpdParser.java

@@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor;
 import android.util.Xml;
 
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.exceptions.reCaptchaException;
 import org.schabi.newpipe.extractor.stream_info.AudioStream;
 import org.xmlpull.v1.XmlPullParser;
 
@@ -43,13 +44,15 @@ public class DashMpdParser {
     }
 
     public static List<AudioStream> getAudioStreams(String dashManifestUrl)
-            throws DashMpdParsingException {
+            throws DashMpdParsingException, reCaptchaException {
         String dashDoc;
         Downloader downloader = NewPipe.getDownloader();
         try {
             dashDoc = downloader.download(dashManifestUrl);
         } catch(IOException ioe) {
             throw new DashMpdParsingException("Could not get dash mpd: " + dashManifestUrl, ioe);
+        } catch (reCaptchaException e) {
+            throw new reCaptchaException("reCaptcha Challenge needed");
         }
         Vector<AudioStream> audioStreams = new Vector<>();
         try {

app/src/main/java/org/schabi/newpipe/extractor/Downloader.java

@@ -1,5 +1,7 @@
 package org.schabi.newpipe.extractor;
 
+import org.schabi.newpipe.extractor.exceptions.reCaptchaException;
+
 import java.io.IOException;
 import java.util.Map;
 
@@ -31,7 +33,7 @@ public interface Downloader {
      * @param language the language (usually a 2-character code) to set as the preferred language
      * @return the contents of the specified text file
      * @throws IOException*/
-    String download(String siteUrl, String language) throws IOException;
+    String download(String siteUrl, String language) throws IOException, reCaptchaException;
 
     /**Download the text file at the supplied URL as in download(String),
      * but set the HTTP header field "Accept-Language" to the supplied string.
@@ -39,12 +41,12 @@ public interface Downloader {
      * @param customProperties set request header properties
      * @return the contents of the specified text file
      * @throws IOException*/
-    String download(String siteUrl, Map<String, String> customProperties) throws IOException;
+    String download(String siteUrl, Map<String, String> customProperties) throws IOException, reCaptchaException;
 
     /**Download (via HTTP) the text file located at the supplied URL, and return its contents.
      * Primarily intended for downloading web pages.
      * @param siteUrl the URL of the text file to download
      * @return the contents of the specified text file
      * @throws IOException*/
-    String download(String siteUrl) throws IOException;
+    String download(String siteUrl) throws IOException, reCaptchaException;
 }

app/src/main/java/org/schabi/newpipe/extractor/exceptions/reCaptchaException.java

@@ -0,0 +1,27 @@
+package org.schabi.newpipe.extractor.exceptions;
+
+/**
+ * Created by beneth <bmauduit@beneth.fr> on 07.12.16.
+ *
+ * Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
+ * reCaptchaException.java is part of NewPipe.
+ *
+ * NewPipe is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+public class reCaptchaException extends ExtractionException {
+    public reCaptchaException(String message) {
+        super(message);
+    }
+}

app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java

@@ -11,6 +11,7 @@ import org.mozilla.javascript.ScriptableObject;
 import org.schabi.newpipe.extractor.AbstractStreamInfo;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.exceptions.reCaptchaException;
 import org.schabi.newpipe.extractor.stream_info.AudioStream;
 import org.schabi.newpipe.extractor.Downloader;
 import org.schabi.newpipe.extractor.NewPipe;
@@ -280,7 +281,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
         }
     }
 
-    private String getPlayerUrlFromRestrictedVideo(String pageUrl) throws ParsingException {
+    private String getPlayerUrlFromRestrictedVideo(String pageUrl) throws ParsingException, reCaptchaException {
         try {
             Downloader downloader = NewPipe.getDownloader();
             String playerUrl = "";
@@ -302,6 +303,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
         } catch (IOException e) {
             throw new ParsingException(
                     "Could load decryption code form restricted video for the Youtube service.", e);
+        } catch (reCaptchaException e) {
+            throw new reCaptchaException("reCaptcha Challenge requested");
         }
     }
 

app/src/main/java/org/schabi/newpipe/report/ErrorActivity.java

@@ -473,7 +473,7 @@ public class ErrorActivity extends AppCompatActivity {
         public void run() {
             String ipRange = "none";
             try {
-                Downloader dl = new Downloader();
+                Downloader dl = Downloader.getInstance();
                 String ip = dl.download("https://ifcfg.me/ip");
 
                 ipRange = Parser.matchGroup1("([0-9]*\\.[0-9]*\\.)[0-9]*\\.[0-9]*", ip)

app/src/main/java/org/schabi/newpipe/search_fragment/SearchInfoItemFragment.java

@@ -18,6 +18,7 @@ import android.view.inputmethod.InputMethodManager;
 import android.widget.ProgressBar;
 import android.widget.Toast;
 
+import org.schabi.newpipe.ReCaptchaActivity;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.search.SearchResult;
 import org.schabi.newpipe.info_list.InfoItemBuilder;
@@ -149,7 +150,7 @@ public class SearchInfoItemFragment extends Fragment {
         }
 
         SearchWorker sw = SearchWorker.getInstance();
-        sw.setSearchWorkerResultListner(new SearchWorker.SearchWorkerResultListner() {
+        sw.setSearchWorkerResultListener(new SearchWorker.SearchWorkerResultListener() {
             @Override
             public void onResult(SearchResult result) {
                 infoListAdapter.addStreamItemList(result.resultList);
@@ -174,6 +175,15 @@ public class SearchInfoItemFragment extends Fragment {
                 isLoading = false;
                 loadingIndicator.setVisibility(View.GONE);
             }
+
+            @Override
+            public void onReCaptchaChallenge() {
+                Toast.makeText(getActivity(), "ReCaptcha Challenge requested",
+                        Toast.LENGTH_LONG).show();
+                // Starting ReCaptcha Challenge Activity
+                Intent i = new Intent(getActivity(), ReCaptchaActivity.class);
+                getActivity().startActivity(i);
+            }
         });
     }
 

app/src/main/java/org/schabi/newpipe/search_fragment/SearchWorker.java

@@ -8,6 +8,7 @@ import android.util.Log;
 import android.view.View;
 
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.exceptions.reCaptchaException;
 import org.schabi.newpipe.extractor.search.SearchEngine;
 import org.schabi.newpipe.extractor.search.SearchResult;
 import org.schabi.newpipe.report.ErrorActivity;
@@ -40,10 +41,11 @@ import java.io.IOException;
 public class SearchWorker {
     private static final String TAG = SearchWorker.class.toString();
 
-    public interface SearchWorkerResultListner {
+    public interface SearchWorkerResultListener {
         void onResult(SearchResult result);
         void onNothingFound(final int stringResource);
         void onError(String message);
+        void onReCaptchaChallenge();
     }
 
     private class ResultRunnable implements Runnable {
@@ -56,7 +58,7 @@ public class SearchWorker {
         @Override
         public void run() {
             if(this.requestId == SearchWorker.this.requestId) {
-                searchWorkerResultListner.onResult(result);
+                searchWorkerResultListener.onResult(result);
             }
         }
     }
@@ -121,11 +123,18 @@ public class SearchWorker {
 
                 }
                 // hard errors:
+            } catch (reCaptchaException e) {
+                h.post(new Runnable() {
+                    @Override
+                    public void run() {
+                        searchWorkerResultListener.onReCaptchaChallenge();
+                    }
+                });
             } catch(IOException e) {
                 h.post(new Runnable() {
                     @Override
                     public void run() {
-                        searchWorkerResultListner.onNothingFound(R.string.network_error);
+                        searchWorkerResultListener.onNothingFound(R.string.network_error);
                     }
                 });
                 e.printStackTrace();
@@ -133,7 +142,7 @@ public class SearchWorker {
                 h.post(new Runnable() {
                     @Override
                     public void run() {
-                        searchWorkerResultListner.onError(e.getMessage());
+                        searchWorkerResultListener.onError(e.getMessage());
                     }
                 });
             } catch(ExtractionException e) {
@@ -155,7 +164,7 @@ public class SearchWorker {
     }
 
     private static SearchWorker searchWorker = null;
-    private SearchWorkerResultListner searchWorkerResultListner = null;
+    private SearchWorkerResultListener searchWorkerResultListener = null;
     private SearchRunnable runnable = null;
     private int requestId = 0;     //prevents running requests that have already ben expired
 
@@ -163,8 +172,8 @@ public class SearchWorker {
         return searchWorker == null ? (searchWorker = new SearchWorker()) : searchWorker;
     }
 
-    public void setSearchWorkerResultListner(SearchWorkerResultListner listener) {
+    public void setSearchWorkerResultListener(SearchWorkerResultListener listener) {
-        searchWorkerResultListner = listener;
+        searchWorkerResultListener = listener;
     }
 
     private SearchWorker() {

app/src/main/res/layout/activity_recaptcha.xml

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="utf-8"?>
+<LinearLayout xmlns:android="http://schemas.android.com/apk/res/android"
+    android:orientation="vertical" android:layout_width="match_parent"
+    android:layout_height="match_parent">
+
+    <WebView
+        android:layout_width="match_parent"
+        android:layout_height="match_parent"
+        android:id="@+id/reCaptchaWebView" />
+</LinearLayout>

app/src/main/res/values/strings.xml

@@ -269,6 +269,8 @@
         "when a user tries to pick up one of cards.\n\n"
     </string>
     <string name="action_settings">Settings</string>
+    <string name="reCaptchaActivity">reCaptcha</string>
+    <string name="reCaptcha_title">reCaptcha Challenge</string>
 
     <!-- End of GigaGet's Strings -->