Merge remote-tracking branch 'origin/master'

This commit is contained in:
Weblate 2016-12-25 12:44:13 +01:00
commit 66b73d1592
28 changed files with 287 additions and 36 deletions

View File

@ -8,11 +8,12 @@ android:
- build-tools-23.0.3
# The SDK version used to compile NewPipe
- android-24
- android-25
# Additional components
- extra-android-support
- extra-android-m2repository
- extra-google-m2repository
# Emulators
- sys-img-armeabi-v7a-android-21
@ -33,3 +34,7 @@ before_script:
- adb shell input keyevent 82 &
script: ./gradlew --info build connectedCheck
licenses:
- '.+'

View File

@ -8,8 +8,8 @@ android {
applicationId "org.schabi.newpipe"
minSdkVersion 15
targetSdkVersion 25
versionCode 19
versionName "0.8.5"
versionCode 20
versionName "0.8.6"
}
buildTypes {
release {

View File

@ -32,7 +32,7 @@ public class YoutubeChannelExtractorTest extends AndroidTestCase {
@Override
public void setUp() throws Exception {
super.setUp();
NewPipe.init(new Downloader());
NewPipe.init(Downloader.getInstance());
extractor = NewPipe.getService("Youtube")
.getChannelExtractorInstance("https://www.youtube.com/channel/UCYJ61XIK64sp6ZFFS8sctxw", 0);
}

View File

@ -35,7 +35,7 @@ public class YoutubeSearchEngineTest extends AndroidTestCase {
@Override
public void setUp() throws Exception {
super.setUp();
NewPipe.init(new Downloader());
NewPipe.init(Downloader.getInstance());
SearchEngine engine = NewPipe.getService("Youtube").getSearchEngineInstance();
result = engine.search("this is something boring", 0, "de").getSearchResult();

View File

@ -38,7 +38,7 @@ public class YoutubeSearchResultTest extends AndroidTestCase {
@Override
public void setUp() throws Exception {
super.setUp();
NewPipe.init(new Downloader());
NewPipe.init(Downloader.getInstance());
SuggestionExtractor engine = new YoutubeSuggestionExtractor(0);
suggestionReply = engine.suggestionList("hello", "de");
}

View File

@ -38,7 +38,7 @@ public class YoutubeStreamExtractorDefaultTest extends AndroidTestCase {
public void setUp() throws Exception {
super.setUp();
NewPipe.init(new Downloader());
NewPipe.init(Downloader.getInstance());
extractor = NewPipe.getService("Youtube")
.getExtractorInstance("https://www.youtube.com/watch?v=YQHsXMglC9A");
}

View File

@ -39,7 +39,7 @@ public class YoutubeStreamExtractorGemaTest extends AndroidTestCase {
public void testGemaError() throws IOException, ExtractionException {
if(testActive) {
try {
NewPipe.init(new Downloader());
NewPipe.init(Downloader.getInstance());
NewPipe.getService("Youtube")
.getExtractorInstance("https://www.youtube.com/watch?v=3O1_3zBUKM8");
} catch(YoutubeStreamExtractor.GemaException ge) {

View File

@ -38,9 +38,9 @@ public class YoutubeStreamExtractorLiveStreamTest extends AndroidTestCase {
//todo: make the extractor not throw over a livestream
/*
NewPipe.init(new Downloader());
NewPipe.init(Downloader.getInstance());
extractor = NewPipe.getService("Youtube")
.getExtractorInstance("https://www.youtube.com/watch?v=J0s6NjqdjLE", new Downloader());
.getExtractorInstance("https://www.youtube.com/watch?v=J0s6NjqdjLE", Downloader.getInstance());
*/
}

View File

@ -17,7 +17,7 @@ public class YoutubeStreamExtractorRestrictedTest extends AndroidTestCase {
public void setUp() throws Exception {
super.setUp();
NewPipe.init(new Downloader());
NewPipe.init(Downloader.getInstance());
extractor = NewPipe.getService("Youtube")
.getExtractorInstance("https://www.youtube.com/watch?v=i6JTvzrpBy0");
}

View File

@ -153,6 +153,9 @@
android:label="@string/title_activity_channel"
android:theme="@style/AppTheme.NoActionBar" />
<activity
android:name=".ReCaptchaActivity"
android:label="@string/reCaptchaActivity" />
</application>
</manifest>

View File

@ -62,7 +62,7 @@ public class App extends Application {
}
//init NewPipe
NewPipe.init(new Downloader());
NewPipe.init(Downloader.getInstance());
// Initialize image loader
ImageLoaderConfiguration config = new ImageLoaderConfiguration.Builder(this).build();

View File

@ -1,5 +1,7 @@
package org.schabi.newpipe;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
@ -35,13 +37,37 @@ import javax.net.ssl.HttpsURLConnection;
public class Downloader implements org.schabi.newpipe.extractor.Downloader {
private static final String USER_AGENT = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0";
private static String mCookies = "";
private static Downloader instance = null;
private Downloader() {}
public static Downloader getInstance() {
if(instance == null) {
synchronized (Downloader.class) {
if (instance == null) {
instance = new Downloader();
}
}
}
return instance;
}
public static synchronized void setCookies(String cookies) {
Downloader.mCookies = cookies;
}
public static synchronized String getCookies() {
return Downloader.mCookies;
}
/**Download the text file at the supplied URL as in download(String),
* but set the HTTP header field "Accept-Language" to the supplied string.
* @param siteUrl the URL of the text file to return the contents of
* @param language the language (usually a 2-character code) to set as the preferred language
* @return the contents of the specified text file*/
public String download(String siteUrl, String language) throws IOException {
public String download(String siteUrl, String language) throws IOException, ReCaptchaException {
Map<String, String> requestProperties = new HashMap<>();
requestProperties.put("Accept-Language", language);
return download(siteUrl, requestProperties);
@ -54,7 +80,7 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
* @param customProperties set request header properties
* @return the contents of the specified text file
* @throws IOException*/
public String download(String siteUrl, Map<String, String> customProperties) throws IOException {
public String download(String siteUrl, Map<String, String> customProperties) throws IOException, ReCaptchaException {
URL url = new URL(siteUrl);
HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
Iterator it = customProperties.entrySet().iterator();
@ -66,7 +92,7 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
}
/**Common functionality between download(String url) and download(String url, String language)*/
private static String dl(HttpsURLConnection con) throws IOException {
private static String dl(HttpsURLConnection con) throws IOException, ReCaptchaException {
StringBuilder response = new StringBuilder();
BufferedReader in = null;
@ -74,6 +100,10 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
con.setRequestMethod("GET");
con.setRequestProperty("User-Agent", USER_AGENT);
if (getCookies().length() > 0) {
con.setRequestProperty("Cookie", getCookies());
}
in = new BufferedReader(
new InputStreamReader(con.getInputStream()));
String inputLine;
@ -85,6 +115,14 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
throw new IOException("unknown host or no network", uhe);
//Toast.makeText(getActivity(), uhe.getMessage(), Toast.LENGTH_LONG).show();
} catch(Exception e) {
/*
* HTTP 429 == Too Many Request
* Receive from Youtube.com = ReCaptcha challenge request
* See : https://github.com/rg3/youtube-dl/issues/5138
*/
if (con.getResponseCode() == 429) {
throw new ReCaptchaException("reCaptcha Challenge requested");
}
throw new IOException(e);
} finally {
if(in != null) {
@ -99,7 +137,7 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
* Primarily intended for downloading web pages.
* @param siteUrl the URL of the text file to download
* @return the contents of the specified text file*/
public String download(String siteUrl) throws IOException {
public String download(String siteUrl) throws IOException, ReCaptchaException {
URL url = new URL(siteUrl);
HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
//HttpsURLConnection con = NetCipher.getHttpsURLConnection(url);

View File

@ -46,6 +46,7 @@ public class MainActivity extends AppCompatActivity {
.findFragmentById(R.id.search_fragment);
}
@Override
public boolean onCreateOptionsMenu(Menu menu) {
super.onCreateOptionsMenu(menu);
MenuInflater inflater = getMenuInflater();

View File

@ -0,0 +1,138 @@
package org.schabi.newpipe;
import android.app.Activity;
import android.content.Intent;
import android.os.Build;
import android.os.Bundle;
import android.support.v4.app.NavUtils;
import android.support.v7.app.ActionBar;
import android.support.v7.app.AppCompatActivity;
import android.view.MenuItem;
import android.webkit.CookieManager;
import android.webkit.ValueCallback;
import android.webkit.WebSettings;
import android.webkit.WebView;
import android.webkit.WebViewClient;
/**
* Created by beneth <bmauduit@beneth.fr> on 06.12.16.
*
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
* ReCaptchaActivity.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class ReCaptchaActivity extends AppCompatActivity {
public static final String TAG = ReCaptchaActivity.class.toString();
public static final String YT_URL = "https://www.youtube.com";
@Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
setContentView(R.layout.activity_recaptcha);
ActionBar actionBar = getSupportActionBar();
actionBar.setDisplayHomeAsUpEnabled(true);
actionBar.setTitle(R.string.reCaptcha_title);
actionBar.setDisplayShowTitleEnabled(true);
WebView myWebView = (WebView) findViewById(R.id.reCaptchaWebView);
// Enable Javascript
WebSettings webSettings = myWebView.getSettings();
webSettings.setJavaScriptEnabled(true);
ReCaptchaWebViewClient webClient = new ReCaptchaWebViewClient(this);
myWebView.setWebViewClient(webClient);
// Cleaning cache, history and cookies from webView
myWebView.clearCache(true);
myWebView.clearHistory();
android.webkit.CookieManager cookieManager = CookieManager.getInstance();
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {
cookieManager.removeAllCookies(new ValueCallback<Boolean>() {
@Override
public void onReceiveValue(Boolean aBoolean) {}
});
} else {
cookieManager.removeAllCookie();
}
myWebView.loadUrl(YT_URL);
}
private class ReCaptchaWebViewClient extends WebViewClient {
private Activity context;
private String mCookies;
ReCaptchaWebViewClient(Activity ctx) {
context = ctx;
}
@Override
public void onPageFinished(WebView view, String url) {
String cookies = CookieManager.getInstance().getCookie(url);
// find cookies : s_gl & goojf and Add cookies to Downloader
if (find_access_cookies(cookies)) {
// Give cookies to Downloader class
Downloader.setCookies(mCookies);
// Closing activity and return to parent.
Intent intent = new Intent(context, org.schabi.newpipe.MainActivity.class);
intent.addFlags(Intent.FLAG_ACTIVITY_CLEAR_TOP);
NavUtils.navigateUpTo(context, intent);
}
}
private boolean find_access_cookies(String cookies) {
boolean ret = false;
String c_s_gl = "";
String c_goojf = "";
String[] parts = cookies.split("; ");
for (String part : parts) {
if (part.trim().startsWith("s_gl")) {
c_s_gl = part.trim();
}
if (part.trim().startsWith("goojf")) {
c_goojf = part.trim();
}
}
if (c_s_gl.length() > 0 && c_goojf.length() > 0) {
ret = true;
//mCookies = c_s_gl + "; " + c_goojf;
// Youtube seems to also need the other cookies:
mCookies = cookies;
}
return ret;
}
}
@Override
public boolean onOptionsItemSelected(MenuItem item) {
int id = item.getItemId();
switch (id) {
case android.R.id.home: {
Intent intent = new Intent(this, org.schabi.newpipe.MainActivity.class);
intent.addFlags(Intent.FLAG_ACTIVITY_CLEAR_TOP);
NavUtils.navigateUpTo(this, intent);
return true;
}
default:
return false;
}
}
}

View File

@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor;
import android.util.Xml;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.stream_info.AudioStream;
import org.xmlpull.v1.XmlPullParser;
@ -43,13 +44,15 @@ public class DashMpdParser {
}
public static List<AudioStream> getAudioStreams(String dashManifestUrl)
throws DashMpdParsingException {
throws DashMpdParsingException, ReCaptchaException {
String dashDoc;
Downloader downloader = NewPipe.getDownloader();
try {
dashDoc = downloader.download(dashManifestUrl);
} catch(IOException ioe) {
throw new DashMpdParsingException("Could not get dash mpd: " + dashManifestUrl, ioe);
} catch (ReCaptchaException e) {
throw new ReCaptchaException("reCaptcha Challenge needed");
}
Vector<AudioStream> audioStreams = new Vector<>();
try {

View File

@ -1,5 +1,7 @@
package org.schabi.newpipe.extractor;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import java.io.IOException;
import java.util.Map;
@ -31,7 +33,7 @@ public interface Downloader {
* @param language the language (usually a 2-character code) to set as the preferred language
* @return the contents of the specified text file
* @throws IOException*/
String download(String siteUrl, String language) throws IOException;
String download(String siteUrl, String language) throws IOException, ReCaptchaException;
/**Download the text file at the supplied URL as in download(String),
* but set the HTTP header field "Accept-Language" to the supplied string.
@ -39,12 +41,12 @@ public interface Downloader {
* @param customProperties set request header properties
* @return the contents of the specified text file
* @throws IOException*/
String download(String siteUrl, Map<String, String> customProperties) throws IOException;
String download(String siteUrl, Map<String, String> customProperties) throws IOException, ReCaptchaException;
/**Download (via HTTP) the text file located at the supplied URL, and return its contents.
* Primarily intended for downloading web pages.
* @param siteUrl the URL of the text file to download
* @return the contents of the specified text file
* @throws IOException*/
String download(String siteUrl) throws IOException;
String download(String siteUrl) throws IOException, ReCaptchaException;
}

View File

@ -0,0 +1,27 @@
package org.schabi.newpipe.extractor.exceptions;
/**
* Created by beneth <bmauduit@beneth.fr> on 07.12.16.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* ReCaptchaException.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class ReCaptchaException extends ExtractionException {
public ReCaptchaException(String message) {
super(message);
}
}

View File

@ -136,7 +136,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
if(!isAjaxPage) {
Element el = doc.select("div[id=\"gh-banner\"]").first().select("style").first();
String cssContent = el.html();
String url = "https:" + Parser.matchGroup1("url\\((.*)\\)", cssContent);
String url = "https:" + Parser.matchGroup1("url\\(([^)]+)\\)", cssContent);
if (url.contains("s.ytimg.com")) {
bannerUrl = null;
} else {

View File

@ -53,7 +53,7 @@ public class YoutubeSearchEngine extends SearchEngine {
String url = "https://www.youtube.com/results"
+ "?search_query=" + URLEncoder.encode(query, CHARSET_UTF_8)
+ "&page=" + Integer.toString(page)
+ "&page=" + Integer.toString(page + 1)
+ "&filters=" + "video";
String site;

View File

@ -11,6 +11,7 @@ import org.mozilla.javascript.ScriptableObject;
import org.schabi.newpipe.extractor.AbstractStreamInfo;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.stream_info.AudioStream;
import org.schabi.newpipe.extractor.Downloader;
import org.schabi.newpipe.extractor.NewPipe;
@ -280,7 +281,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
}
private String getPlayerUrlFromRestrictedVideo(String pageUrl) throws ParsingException {
private String getPlayerUrlFromRestrictedVideo(String pageUrl) throws ParsingException, ReCaptchaException {
try {
Downloader downloader = NewPipe.getDownloader();
String playerUrl = "";
@ -302,6 +303,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
} catch (IOException e) {
throw new ParsingException(
"Could load decryption code form restricted video for the Youtube service.", e);
} catch (ReCaptchaException e) {
throw new ReCaptchaException("reCaptcha Challenge requested");
}
}

View File

@ -473,7 +473,7 @@ public class ErrorActivity extends AppCompatActivity {
public void run() {
String ipRange = "none";
try {
Downloader dl = new Downloader();
Downloader dl = Downloader.getInstance();
String ip = dl.download("https://ifcfg.me/ip");
ipRange = Parser.matchGroup1("([0-9]*\\.[0-9]*\\.)[0-9]*\\.[0-9]*", ip)

View File

@ -18,6 +18,7 @@ import android.view.inputmethod.InputMethodManager;
import android.widget.ProgressBar;
import android.widget.Toast;
import org.schabi.newpipe.ReCaptchaActivity;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.search.SearchResult;
import org.schabi.newpipe.info_list.InfoItemBuilder;
@ -149,7 +150,7 @@ public class SearchInfoItemFragment extends Fragment {
}
SearchWorker sw = SearchWorker.getInstance();
sw.setSearchWorkerResultListner(new SearchWorker.SearchWorkerResultListner() {
sw.setSearchWorkerResultListener(new SearchWorker.SearchWorkerResultListener() {
@Override
public void onResult(SearchResult result) {
infoListAdapter.addStreamItemList(result.resultList);
@ -174,6 +175,15 @@ public class SearchInfoItemFragment extends Fragment {
isLoading = false;
loadingIndicator.setVisibility(View.GONE);
}
@Override
public void onReCaptchaChallenge() {
Toast.makeText(getActivity(), "ReCaptcha Challenge requested",
Toast.LENGTH_LONG).show();
// Starting ReCaptcha Challenge Activity
Intent i = new Intent(getActivity(), ReCaptchaActivity.class);
getActivity().startActivity(i);
}
});
}

View File

@ -8,6 +8,7 @@ import android.util.Log;
import android.view.View;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.search.SearchEngine;
import org.schabi.newpipe.extractor.search.SearchResult;
import org.schabi.newpipe.report.ErrorActivity;
@ -40,10 +41,11 @@ import java.io.IOException;
public class SearchWorker {
private static final String TAG = SearchWorker.class.toString();
public interface SearchWorkerResultListner {
public interface SearchWorkerResultListener {
void onResult(SearchResult result);
void onNothingFound(final int stringResource);
void onError(String message);
void onReCaptchaChallenge();
}
private class ResultRunnable implements Runnable {
@ -56,7 +58,7 @@ public class SearchWorker {
@Override
public void run() {
if(this.requestId == SearchWorker.this.requestId) {
searchWorkerResultListner.onResult(result);
searchWorkerResultListener.onResult(result);
}
}
}
@ -121,11 +123,18 @@ public class SearchWorker {
}
// hard errors:
} catch (ReCaptchaException e) {
h.post(new Runnable() {
@Override
public void run() {
searchWorkerResultListener.onReCaptchaChallenge();
}
});
} catch(IOException e) {
h.post(new Runnable() {
@Override
public void run() {
searchWorkerResultListner.onNothingFound(R.string.network_error);
searchWorkerResultListener.onNothingFound(R.string.network_error);
}
});
e.printStackTrace();
@ -133,7 +142,7 @@ public class SearchWorker {
h.post(new Runnable() {
@Override
public void run() {
searchWorkerResultListner.onError(e.getMessage());
searchWorkerResultListener.onError(e.getMessage());
}
});
} catch(ExtractionException e) {
@ -155,7 +164,7 @@ public class SearchWorker {
}
private static SearchWorker searchWorker = null;
private SearchWorkerResultListner searchWorkerResultListner = null;
private SearchWorkerResultListener searchWorkerResultListener = null;
private SearchRunnable runnable = null;
private int requestId = 0; //prevents running requests that have already ben expired
@ -163,8 +172,8 @@ public class SearchWorker {
return searchWorker == null ? (searchWorker = new SearchWorker()) : searchWorker;
}
public void setSearchWorkerResultListner(SearchWorkerResultListner listener) {
searchWorkerResultListner = listener;
public void setSearchWorkerResultListener(SearchWorkerResultListener listener) {
searchWorkerResultListener = listener;
}
private SearchWorker() {

View File

@ -0,0 +1,10 @@
<?xml version="1.0" encoding="utf-8"?>
<LinearLayout xmlns:android="http://schemas.android.com/apk/res/android"
android:orientation="vertical" android:layout_width="match_parent"
android:layout_height="match_parent">
<WebView
android:layout_width="match_parent"
android:layout_height="match_parent"
android:id="@+id/reCaptchaWebView" />
</LinearLayout>

View File

@ -48,7 +48,7 @@
android:paddingLeft="@dimen/video_item_search_duration_horizontal_padding"
android:textAppearance="?android:attr/textAppearanceSmall"
android:textSize="@dimen/video_item_search_duration_text_size"
android:background="@color/duration_dackground_color"
android:background="@color/duration_background_color"
android:textColor="@color/duration_text_color"/>
</RelativeLayout>

View File

@ -13,7 +13,7 @@
<color name="dark_youtube_accent_color">#FFFFFF</color>
<!-- Miscellaneous -->
<color name="duration_dackground_color">#AA000000</color>
<color name="duration_background_color">#AA000000</color>
<color name="duration_text_color">#EEFFFFFF</color>
<color name="video_overlay_color">#66000000</color>
<color name="background_notification_color">#323232</color>
@ -34,4 +34,4 @@
<color name="gray">#616161</color>
</resources>
</resources>

View File

@ -269,6 +269,8 @@
"when a user tries to pick up one of cards.\n\n"
</string>
<string name="action_settings">Settings</string>
<string name="reCaptchaActivity">reCaptcha</string>
<string name="reCaptcha_title">reCaptcha Challenge</string>
<!-- End of GigaGet's Strings -->

View File

@ -5,7 +5,7 @@ buildscript {
jcenter()
}
dependencies {
classpath 'com.android.tools.build:gradle:2.2.2'
classpath 'com.android.tools.build:gradle:2.2.3'
// NOTE: Do not place your application dependencies here; they belong
// in the individual module build.gradle files