be-gay-do-crimes
/
NewPipe
mirror of https://github.com/TeamNewPipe/NewPipe
1
0
Fork 0
Code Issues Releases Wiki Activity

use java error system in the crawler

This commit is contained in:
Christian Schabesberger 2016-01-31 19:57:30 +01:00
parent 46c2db310a
commit 7f12b58722
17 changed files with 877 additions and 673 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
app/src/androidTest/java/org/schabi/newpipe/services/youtube/YoutubeSearchEngineTest.java
View File

@ -3,7 +3,9 @@ package org.schabi.newpipe.services.youtube;
import android.test.AndroidTestCase; import android.test.AndroidTestCase;
import org.schabi.newpipe.crawler.VideoPreviewInfo; import org.schabi.newpipe.crawler.VideoPreviewInfo;
import org.schabi.newpipe.services.SearchEngine; import org.schabi.newpipe.crawler.SearchEngine;
import org.schabi.newpipe.crawler.services.youtube.YoutubeSearchEngine;
import org.schabi.newpipe.Downloader;
import java.util.ArrayList; import java.util.ArrayList;
@ -35,8 +37,9 @@ public class YoutubeSearchEngineTest extends AndroidTestCase {
public void setUp() throws Exception{ public void setUp() throws Exception{
super.setUp(); super.setUp();
SearchEngine engine = new YoutubeSearchEngine(); SearchEngine engine = new YoutubeSearchEngine();
result = engine.search("https://www.youtube.com/results?search_query=bla", 0, "de"); result = engine.search("https://www.youtube.com/results?search_query=bla",
suggestionReply = engine.suggestionList("hello"); 0, "de", new Downloader());
suggestionReply = engine.suggestionList("hello", new Downloader());
} }
public void testIfNoErrorOccur() { public void testIfNoErrorOccur() {

53
app/src/androidTest/java/org/schabi/newpipe/services/youtube/YoutubeVideoExtractorDefaultTest.java
View File

@ -1,9 +1,14 @@
package org.schabi.newpipe.services.youtube; package org.schabi.newpipe.services.youtube;
import android.test.AndroidTestCase; import android.test.AndroidTestCase;
import android.util.Log;
import org.schabi.newpipe.services.VideoInfo; import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.crawler.CrawlingException;
import org.schabi.newpipe.crawler.ParsingException;
import org.schabi.newpipe.crawler.services.youtube.YoutubeVideoExtractor;
import org.schabi.newpipe.crawler.VideoInfo;
import java.io.IOException;
/** /**
* Created by the-scrabi on 30.12.15. * Created by the-scrabi on 30.12.15.
@ -28,58 +33,58 @@ import org.schabi.newpipe.services.VideoInfo;
public class YoutubeVideoExtractorDefaultTest extends AndroidTestCase { public class YoutubeVideoExtractorDefaultTest extends AndroidTestCase {
private YoutubeVideoExtractor extractor; private YoutubeVideoExtractor extractor;
public void setUp() { public void setUp() throws IOException, CrawlingException {
extractor = new YoutubeVideoExtractor("https://www.youtube.com/watch?v=FmG385_uUys"); extractor = new YoutubeVideoExtractor("https://www.youtube.com/watch?v=FmG385_uUys",
new Downloader());
} }
public void testGetErrorCode() { public void testGetInvalidTimeStamp() throws ParsingException {
assertEquals(extractor.getErrorCode(), VideoInfo.NO_ERROR);
}
public void testGetErrorMessage() {
assertEquals(extractor.getErrorMessage(), "");
}
public void testGetTimeStamp() {
assertTrue(Integer.toString(extractor.getTimeStamp()), assertTrue(Integer.toString(extractor.getTimeStamp()),
extractor.getTimeStamp() >= 0); extractor.getTimeStamp() <= 0);
} }
public void testGetTitle() { public void testGetValidTimeStamp() throws CrawlingException, IOException {
YoutubeVideoExtractor extractor =
new YoutubeVideoExtractor("https://youtu.be/FmG385_uUys?t=174", new Downloader());
assertTrue(Integer.toString(extractor.getTimeStamp()),
extractor.getTimeStamp() == 174);
}
public void testGetTitle() throws ParsingException {
assertTrue(!extractor.getTitle().isEmpty()); assertTrue(!extractor.getTitle().isEmpty());
} }
public void testGetDescription() { public void testGetDescription() throws ParsingException {
assertTrue(extractor.getDescription() != null); assertTrue(extractor.getDescription() != null);
} }
public void testGetUploader() { public void testGetUploader() throws ParsingException {
assertTrue(!extractor.getUploader().isEmpty()); assertTrue(!extractor.getUploader().isEmpty());
} }
public void testGetLength() { public void testGetLength() throws ParsingException {
assertTrue(extractor.getLength() > 0); assertTrue(extractor.getLength() > 0);
} }
public void testGetViews() { public void testGetViews() throws ParsingException {
assertTrue(extractor.getLength() > 0); assertTrue(extractor.getLength() > 0);
} }
public void testGetUploadDate() { public void testGetUploadDate() throws ParsingException {
assertTrue(extractor.getUploadDate().length() > 0); assertTrue(extractor.getUploadDate().length() > 0);
} }
public void testGetThumbnailUrl() { public void testGetThumbnailUrl() throws ParsingException {
assertTrue(extractor.getThumbnailUrl(), assertTrue(extractor.getThumbnailUrl(),
extractor.getThumbnailUrl().contains("https://")); extractor.getThumbnailUrl().contains("https://"));
} }
public void testGetUploaderThumbnailUrl() { public void testGetUploaderThumbnailUrl() throws ParsingException {
assertTrue(extractor.getUploaderThumbnailUrl(), assertTrue(extractor.getUploaderThumbnailUrl(),
extractor.getUploaderThumbnailUrl().contains("https://")); extractor.getUploaderThumbnailUrl().contains("https://"));
} }
public void testGetAudioStreams() { public void testGetAudioStreams() throws ParsingException {
for(VideoInfo.AudioStream s : extractor.getAudioStreams()) { for(VideoInfo.AudioStream s : extractor.getAudioStreams()) {
assertTrue(s.url, assertTrue(s.url,
s.url.contains("https://")); s.url.contains("https://"));
@ -88,7 +93,7 @@ public class YoutubeVideoExtractorDefaultTest extends AndroidTestCase {
} }
} }
public void testGetVideoStreams() { public void testGetVideoStreams() throws ParsingException {
for(VideoInfo.VideoStream s : extractor.getVideoStreams()) { for(VideoInfo.VideoStream s : extractor.getVideoStreams()) {
assertTrue(s.url, assertTrue(s.url,
s.url.contains("https://")); s.url.contains("https://"));

40
app/src/androidTest/java/org/schabi/newpipe/services/youtube/YoutubeVideoExtractorGemaTest.java
View File

@ -2,7 +2,13 @@ package org.schabi.newpipe.services.youtube;
import android.test.AndroidTestCase; import android.test.AndroidTestCase;
import org.schabi.newpipe.services.VideoInfo; import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.crawler.CrawlingException;
import org.schabi.newpipe.crawler.services.youtube.YoutubeVideoExtractor;
import org.schabi.newpipe.crawler.VideoInfo;
import org.schabi.newpipe.Downloader;
import java.io.IOException;
/** /**
* Created by the-scrabi on 30.12.15. * Created by the-scrabi on 30.12.15.
@ -29,31 +35,17 @@ import org.schabi.newpipe.services.VideoInfo;
public class YoutubeVideoExtractorGemaTest extends AndroidTestCase { public class YoutubeVideoExtractorGemaTest extends AndroidTestCase {
// Deaktivate this Test Case bevore uploading it githup, otherwise CI will fail. // Deaktivate this Test Case bevore uploading it githup, otherwise CI will fail.
private static final boolean testActive = false; private static final boolean testActive = true;
public void testGemaError() throws IOException, CrawlingException {
private YoutubeVideoExtractor extractor;
public void setUp() {
if(testActive) { if(testActive) {
extractor = new YoutubeVideoExtractor("https://www.youtube.com/watch?v=3O1_3zBUKM8"); try {
} new YoutubeVideoExtractor("https://www.youtube.com/watch?v=3O1_3zBUKM8",
} new Downloader());
assertTrue("Gema exception not thrown", false);
public void testGetErrorCode() { } catch(YoutubeVideoExtractor.GemaException ge) {
if(testActive) { assertTrue(true);
assertEquals(extractor.getErrorCode(), VideoInfo.ERROR_BLOCKED_BY_GEMA); }
} else {
assertTrue(true);
}
}
public void testGetErrorMessage() {
if(testActive) {
assertTrue(extractor.getErrorMessage(),
extractor.getErrorMessage().contains("GEMA"));
} else {
assertTrue(true);
} }
} }
} }

54
app/src/main/java/org/schabi/newpipe/Downloader.java
View File

@ -39,42 +39,39 @@ public class Downloader implements org.schabi.newpipe.crawler.Downloader {
* @param siteUrl the URL of the text file to return the contents of * @param siteUrl the URL of the text file to return the contents of
* @param language the language (usually a 2-character code) to set as the preferred language * @param language the language (usually a 2-character code) to set as the preferred language
* @return the contents of the specified text file*/ * @return the contents of the specified text file*/
public String download(String siteUrl, String language) { public String download(String siteUrl, String language) throws IOException {
String ret = ""; URL url = new URL(siteUrl);
try { //HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
URL url = new URL(siteUrl); HttpsURLConnection con = NetCipher.getHttpsURLConnection(url);
//HttpsURLConnection con = (HttpsURLConnection) url.openConnection(); con.setRequestProperty("Accept-Language", language);
HttpsURLConnection con = NetCipher.getHttpsURLConnection(url); return dl(con);
con.setRequestProperty("Accept-Language", language);
ret = dl(con);
}
catch(Exception e) {
e.printStackTrace();
}
return ret;
} }
/**Common functionality between download(String url) and download(String url, String language)*/ /**Common functionality between download(String url) and download(String url, String language)*/
private static String dl(HttpsURLConnection con) throws IOException { private static String dl(HttpsURLConnection con) throws IOException {
StringBuilder response = new StringBuilder(); StringBuilder response = new StringBuilder();
BufferedReader in = null;
try { try {
con.setRequestMethod("GET"); con.setRequestMethod("GET");
con.setRequestProperty("User-Agent", USER_AGENT); con.setRequestProperty("User-Agent", USER_AGENT);
BufferedReader in = new BufferedReader( in = new BufferedReader(
new InputStreamReader(con.getInputStream())); new InputStreamReader(con.getInputStream()));
String inputLine; String inputLine;
while((inputLine = in.readLine()) != null) { while((inputLine = in.readLine()) != null) {
response.append(inputLine); response.append(inputLine);
} }
in.close(); } catch(UnknownHostException uhe) {//thrown when there's no internet connection
throw new IOException("unknown host or no network", uhe);
}
catch(UnknownHostException uhe) {//thrown when there's no internet connection
uhe.printStackTrace();
//Toast.makeText(getActivity(), uhe.getMessage(), Toast.LENGTH_LONG).show(); //Toast.makeText(getActivity(), uhe.getMessage(), Toast.LENGTH_LONG).show();
} catch(Exception e) {
throw new IOException(e);
} finally {
if(in != null) {
in.close();
}
} }
return response.toString(); return response.toString();
@ -84,19 +81,10 @@ public class Downloader implements org.schabi.newpipe.crawler.Downloader {
* Primarily intended for downloading web pages. * Primarily intended for downloading web pages.
* @param siteUrl the URL of the text file to download * @param siteUrl the URL of the text file to download
* @return the contents of the specified text file*/ * @return the contents of the specified text file*/
public String download(String siteUrl) { public String download(String siteUrl) throws IOException {
String ret = ""; URL url = new URL(siteUrl);
HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
try { //HttpsURLConnection con = NetCipher.getHttpsURLConnection(url);
URL url = new URL(siteUrl); return dl(con);
HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
//HttpsURLConnection con = NetCipher.getHttpsURLConnection(url);
ret = dl(con);
}
catch(Exception e) {
e.printStackTrace();
}
return ret;
} }
} }

258
app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java
View File

@ -31,15 +31,20 @@ import android.widget.TextView;
import android.view.MenuItem; import android.view.MenuItem;
import android.widget.Toast; import android.widget.Toast;
import java.io.IOException;
import java.net.URL; import java.net.URL;
import java.nio.charset.MalformedInputException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Vector; import java.util.Vector;
import org.schabi.newpipe.crawler.CrawlingException;
import org.schabi.newpipe.crawler.ParsingException;
import org.schabi.newpipe.crawler.VideoPreviewInfo; import org.schabi.newpipe.crawler.VideoPreviewInfo;
import org.schabi.newpipe.crawler.VideoExtractor; import org.schabi.newpipe.crawler.VideoExtractor;
import org.schabi.newpipe.crawler.ServiceList; import org.schabi.newpipe.crawler.ServiceList;
import org.schabi.newpipe.crawler.StreamingService; import org.schabi.newpipe.crawler.StreamingService;
import org.schabi.newpipe.crawler.VideoInfo; import org.schabi.newpipe.crawler.VideoInfo;
import org.schabi.newpipe.crawler.services.youtube.YoutubeVideoExtractor;
/** /**
@ -68,7 +73,6 @@ public class VideoItemDetailFragment extends Fragment {
* The fragment argument representing the item ID that this fragment * The fragment argument representing the item ID that this fragment
* represents. * represents.
*/ */
//public static final String ARG_ITEM_ID = "item_id";
public static final String VIDEO_URL = "video_url"; public static final String VIDEO_URL = "video_url";
public static final String STREAMING_SERVICE = "streaming_service"; public static final String STREAMING_SERVICE = "streaming_service";
public static final String AUTO_PLAY = "auto_play"; public static final String AUTO_PLAY = "auto_play";
@ -87,7 +91,6 @@ public class VideoItemDetailFragment extends Fragment {
private FloatingActionButton playVideoButton; private FloatingActionButton playVideoButton;
private final Point initialThumbnailPos = new Point(0, 0); private final Point initialThumbnailPos = new Point(0, 0);
public interface OnInvokeCreateOptionsMenuListener { public interface OnInvokeCreateOptionsMenuListener {
void createOptionsMenu(); void createOptionsMenu();
} }
@ -108,45 +111,64 @@ public class VideoItemDetailFragment extends Fragment {
@Override @Override
public void run() { public void run() {
try { try {
this.videoExtractor = service.getExtractorInstance(videoUrl, new Downloader()); videoExtractor = service.getExtractorInstance(videoUrl, new Downloader());
VideoInfo videoInfo = videoExtractor.getVideoInfo(); VideoInfo videoInfo = videoExtractor.getVideoInfo();
h.post(new VideoResultReturnedRunnable(videoInfo)); h.post(new VideoResultReturnedRunnable(videoInfo));
if (videoInfo.errorCode == VideoInfo.NO_ERROR) { h.post(new SetThumbnailRunnable(
BitmapFactory.decodeStream(
new URL(videoInfo.thumbnail_url)
.openConnection()
.getInputStream()),
SetThumbnailRunnable.VIDEO_THUMBNAIL));
h.post(new SetThumbnailRunnable(
BitmapFactory.decodeStream(
new URL(videoInfo.uploader_thumbnail_url)
.openConnection()
.getInputStream()),
SetThumbnailRunnable.CHANNEL_THUMBNAIL));
if (showNextVideoItem) {
h.post(new SetThumbnailRunnable( h.post(new SetThumbnailRunnable(
BitmapFactory.decodeStream( BitmapFactory.decodeStream(
new URL(videoInfo.thumbnail_url) new URL(videoInfo.nextVideo.thumbnail_url)
.openConnection() .openConnection()
.getInputStream()), .getInputStream()),
SetThumbnailRunnable.VIDEO_THUMBNAIL)); SetThumbnailRunnable.NEXT_VIDEO_THUMBNAIL));
h.post(new SetThumbnailRunnable(
BitmapFactory.decodeStream(
new URL(videoInfo.uploader_thumbnail_url)
.openConnection()
.getInputStream()),
SetThumbnailRunnable.CHANNEL_THUMBNAIL));
if(showNextVideoItem) {
h.post(new SetThumbnailRunnable(
BitmapFactory.decodeStream(
new URL(videoInfo.nextVideo.thumbnail_url)
.openConnection()
.getInputStream()),
SetThumbnailRunnable.NEXT_VIDEO_THUMBNAIL));
}
} }
} catch (Exception e) { } catch (MalformedInputException e) {
postNewErrorToast(h, R.string.could_not_load_thumbnails);
e.printStackTrace();
} catch (IOException e) {
postNewErrorToast(h, R.string.network_error);
e.printStackTrace();
}
// custom service related exceptions
catch (YoutubeVideoExtractor.DecryptException de) {
postNewErrorToast(h, R.string.youtube_signature_decryption_error);
de.printStackTrace();
} catch (YoutubeVideoExtractor.GemaException ge) {
h.post(new Runnable() { h.post(new Runnable() {
@Override @Override
public void run() { public void run() {
progressBar.setVisibility(View.GONE); onErrorBlockedByGema();
// This is poor style, but unless we have better error handling in the }
// crawler, this may not be better. });
Toast.makeText(VideoItemDetailFragment.this.getActivity(), }
R.string.network_error, Toast.LENGTH_LONG).show(); // ----------------------------------------
catch(VideoExtractor.ContentNotAvailableException e) {
h.post(new Runnable() {
@Override
public void run() {
onNotSpecifiedContentError();
} }
}); });
e.printStackTrace(); e.printStackTrace();
} catch (ParsingException e) {
postNewErrorToast(h, e.getMessage());
e.printStackTrace();
} catch(Exception e) {
postNewErrorToast(h, R.string.general_error);
e.printStackTrace();
} }
} }
} }
@ -213,7 +235,7 @@ public class VideoItemDetailFragment extends Fragment {
private void updateInfo(VideoInfo info) { private void updateInfo(VideoInfo info) {
currentVideoInfo = info; currentVideoInfo = info;
Resources res = activity.getResources();
try { try {
VideoInfoItemViewCreator videoItemViewCreator = VideoInfoItemViewCreator videoItemViewCreator =
new VideoInfoItemViewCreator(LayoutInflater.from(getActivity())); new VideoInfoItemViewCreator(LayoutInflater.from(getActivity()));
@ -226,107 +248,80 @@ public class VideoItemDetailFragment extends Fragment {
TextView thumbsDownView = (TextView) activity.findViewById(R.id.detailThumbsDownCountView); TextView thumbsDownView = (TextView) activity.findViewById(R.id.detailThumbsDownCountView);
TextView uploadDateView = (TextView) activity.findViewById(R.id.detailUploadDateView); TextView uploadDateView = (TextView) activity.findViewById(R.id.detailUploadDateView);
TextView descriptionView = (TextView) activity.findViewById(R.id.detailDescriptionView); TextView descriptionView = (TextView) activity.findViewById(R.id.detailDescriptionView);
ImageView thumbnailView = (ImageView) activity.findViewById(R.id.detailThumbnailView);
FrameLayout nextVideoFrame = (FrameLayout) activity.findViewById(R.id.detailNextVideoFrame); FrameLayout nextVideoFrame = (FrameLayout) activity.findViewById(R.id.detailNextVideoFrame);
RelativeLayout nextVideoRootFrame = RelativeLayout nextVideoRootFrame =
(RelativeLayout) activity.findViewById(R.id.detailNextVideoRootLayout); (RelativeLayout) activity.findViewById(R.id.detailNextVideoRootLayout);
Button backgroundButton = (Button)
activity.findViewById(R.id.detailVideoThumbnailWindowBackgroundButton);
progressBar.setVisibility(View.GONE); progressBar.setVisibility(View.GONE);
switch (info.errorCode) {
case VideoInfo.NO_ERROR: { View nextVideoView = videoItemViewCreator
View nextVideoView = videoItemViewCreator .getViewFromVideoInfoItem(null, nextVideoFrame, info.nextVideo, getContext());
.getViewFromVideoInfoItem(null, nextVideoFrame, info.nextVideo, getContext()); nextVideoFrame.addView(nextVideoView);
nextVideoFrame.addView(nextVideoView);
Button nextVideoButton = (Button) activity.findViewById(R.id.detailNextVideoButton); Button nextVideoButton = (Button) activity.findViewById(R.id.detailNextVideoButton);
Button similarVideosButton = (Button) activity.findViewById(R.id.detailShowSimilarButton); Button similarVideosButton = (Button) activity.findViewById(R.id.detailShowSimilarButton);
textContentLayout.setVisibility(View.VISIBLE); textContentLayout.setVisibility(View.VISIBLE);
playVideoButton.setVisibility(View.VISIBLE); playVideoButton.setVisibility(View.VISIBLE);
if (!showNextVideoItem) { if (!showNextVideoItem) {
nextVideoRootFrame.setVisibility(View.GONE); nextVideoRootFrame.setVisibility(View.GONE);
similarVideosButton.setVisibility(View.GONE); similarVideosButton.setVisibility(View.GONE);
} }
videoTitleView.setText(info.title); videoTitleView.setText(info.title);
uploaderView.setText(info.uploader); uploaderView.setText(info.uploader);
actionBarHandler.setChannelName(info.uploader); actionBarHandler.setChannelName(info.uploader);
String localizedViewCount = Localization.localizeViewCount(info.view_count, getContext()); String localizedViewCount = Localization.localizeViewCount(info.view_count, getContext());
viewCountView.setText(localizedViewCount); viewCountView.setText(localizedViewCount);
String localizedLikeCount = Localization.localizeNumber(info.like_count, getContext()); String localizedLikeCount = Localization.localizeNumber(info.like_count, getContext());
thumbsUpView.setText(localizedLikeCount); thumbsUpView.setText(localizedLikeCount);
String localizedDislikeCount = Localization.localizeNumber(info.dislike_count, getContext()); String localizedDislikeCount = Localization.localizeNumber(info.dislike_count, getContext());
thumbsDownView.setText(localizedDislikeCount); thumbsDownView.setText(localizedDislikeCount);
String localizedDate = Localization.localizeDate(info.upload_date, getContext()); String localizedDate = Localization.localizeDate(info.upload_date, getContext());
uploadDateView.setText(localizedDate); uploadDateView.setText(localizedDate);
descriptionView.setText(Html.fromHtml(info.description)); descriptionView.setText(Html.fromHtml(info.description));
descriptionView.setMovementMethod(LinkMovementMethod.getInstance()); descriptionView.setMovementMethod(LinkMovementMethod.getInstance());
actionBarHandler.setServiceId(streamingServiceId); actionBarHandler.setServiceId(streamingServiceId);
actionBarHandler.setVideoInfo(info.webpage_url, info.title); actionBarHandler.setVideoInfo(info.webpage_url, info.title);
actionBarHandler.setStartPosition(info.startPosition); actionBarHandler.setStartPosition(info.startPosition);
// parse streams // parse streams
Vector<VideoInfo.VideoStream> streamsToUse = new Vector<>(); Vector<VideoInfo.VideoStream> streamsToUse = new Vector<>();
for (VideoInfo.VideoStream i : info.videoStreams) { for (VideoInfo.VideoStream i : info.videoStreams) {
if (useStream(i, streamsToUse)) { if (useStream(i, streamsToUse)) {
streamsToUse.add(i); streamsToUse.add(i);
} }
} }
VideoInfo.VideoStream[] streamList = new VideoInfo.VideoStream[streamsToUse.size()]; VideoInfo.VideoStream[] streamList = new VideoInfo.VideoStream[streamsToUse.size()];
for (int i = 0; i < streamList.length; i++) { for (int i = 0; i < streamList.length; i++) {
streamList[i] = streamsToUse.get(i); streamList[i] = streamsToUse.get(i);
} }
actionBarHandler.setStreams(streamList, info.audioStreams); actionBarHandler.setStreams(streamList, info.audioStreams);
nextVideoButton.setOnClickListener(new View.OnClickListener() { nextVideoButton.setOnClickListener(new View.OnClickListener() {
@Override @Override
public void onClick(View v) { public void onClick(View v) {
Intent detailIntent = Intent detailIntent =
new Intent(getActivity(), VideoItemDetailActivity.class); new Intent(getActivity(), VideoItemDetailActivity.class);
/*detailIntent.putExtra( /*detailIntent.putExtra(
VideoItemDetailFragment.ARG_ITEM_ID, currentVideoInfo.nextVideo.id); */ VideoItemDetailFragment.ARG_ITEM_ID, currentVideoInfo.nextVideo.id); */
detailIntent.putExtra( detailIntent.putExtra(
VideoItemDetailFragment.VIDEO_URL, currentVideoInfo.nextVideo.webpage_url); VideoItemDetailFragment.VIDEO_URL, currentVideoInfo.nextVideo.webpage_url);
detailIntent.putExtra(VideoItemDetailFragment.STREAMING_SERVICE, streamingServiceId); detailIntent.putExtra(VideoItemDetailFragment.STREAMING_SERVICE, streamingServiceId);
startActivity(detailIntent); startActivity(detailIntent);
}
});
} }
break; });
case VideoInfo.ERROR_BLOCKED_BY_GEMA:
thumbnailView.setImageBitmap(BitmapFactory.decodeResource(
getResources(), R.drawable.gruese_die_gema));
backgroundButton.setOnClickListener(new View.OnClickListener() {
@Override
public void onClick(View v) {
Intent intent = new Intent();
intent.setAction(Intent.ACTION_VIEW);
intent.setData(Uri.parse(activity.getString(R.string.c3s_url)));
activity.startActivity(intent);
}
});
break;
case VideoInfo.ERROR_NO_SPECIFIED_ERROR:
thumbnailView.setImageBitmap(BitmapFactory.decodeResource(
getResources(), R.drawable.not_available_monkey));
Toast.makeText(activity, info.errorMessage, Toast.LENGTH_LONG)
.show();
break;
default:
Log.e(TAG, "Video Available Status not known.");
}
if(autoPlayEnabled) { if(autoPlayEnabled) {
actionBarHandler.playVideo(); actionBarHandler.playVideo();
@ -337,6 +332,37 @@ public class VideoItemDetailFragment extends Fragment {
} }
} }
private void onErrorBlockedByGema() {
Button backgroundButton = (Button)
activity.findViewById(R.id.detailVideoThumbnailWindowBackgroundButton);
ImageView thumbnailView = (ImageView) activity.findViewById(R.id.detailThumbnailView);
progressBar.setVisibility(View.GONE);
thumbnailView.setImageBitmap(BitmapFactory.decodeResource(
getResources(), R.drawable.gruese_die_gema));
backgroundButton.setOnClickListener(new View.OnClickListener() {
@Override
public void onClick(View v) {
Intent intent = new Intent();
intent.setAction(Intent.ACTION_VIEW);
intent.setData(Uri.parse(activity.getString(R.string.c3s_url)));
activity.startActivity(intent);
}
});
Toast.makeText(VideoItemDetailFragment.this.getActivity(),
R.string.blocked_by_gema, Toast.LENGTH_LONG).show();
}
private void onNotSpecifiedContentError() {
ImageView thumbnailView = (ImageView) activity.findViewById(R.id.detailThumbnailView);
progressBar.setVisibility(View.GONE);
thumbnailView.setImageBitmap(BitmapFactory.decodeResource(
getResources(), R.drawable.not_available_monkey));
Toast.makeText(activity, R.string.content_not_available, Toast.LENGTH_LONG)
.show();
}
private boolean useStream(VideoInfo.VideoStream stream, Vector<VideoInfo.VideoStream> streams) { private boolean useStream(VideoInfo.VideoStream stream, Vector<VideoInfo.VideoStream> streams) {
for(VideoInfo.VideoStream i : streams) { for(VideoInfo.VideoStream i : streams) {
if(i.resolution.equals(stream.resolution)) { if(i.resolution.equals(stream.resolution)) {
@ -465,4 +491,24 @@ public class VideoItemDetailFragment extends Fragment {
public void setOnInvokeCreateOptionsMenuListener(OnInvokeCreateOptionsMenuListener listener) { public void setOnInvokeCreateOptionsMenuListener(OnInvokeCreateOptionsMenuListener listener) {
this.onInvokeCreateOptionsMenuListener = listener; this.onInvokeCreateOptionsMenuListener = listener;
} }
private void postNewErrorToast(Handler h, final int stringResource) {
h.post(new Runnable() {
@Override
public void run() {
Toast.makeText(VideoItemDetailFragment.this.getActivity(),
stringResource, Toast.LENGTH_LONG).show();
}
});
}
private void postNewErrorToast(Handler h, final String message) {
h.post(new Runnable() {
@Override
public void run() {
Toast.makeText(VideoItemDetailFragment.this.getActivity(),
message, Toast.LENGTH_LONG).show();
}
});
}
} }

30
app/src/main/java/org/schabi/newpipe/VideoItemListFragment.java
View File

@ -15,10 +15,12 @@ import android.widget.AbsListView;
import android.widget.ListView; import android.widget.ListView;
import android.widget.Toast; import android.widget.Toast;
import java.io.IOException;
import java.net.URL; import java.net.URL;
import java.util.List; import java.util.List;
import java.util.Vector; import java.util.Vector;
import org.schabi.newpipe.crawler.CrawlingException;
import org.schabi.newpipe.crawler.VideoPreviewInfo; import org.schabi.newpipe.crawler.VideoPreviewInfo;
import org.schabi.newpipe.crawler.SearchEngine; import org.schabi.newpipe.crawler.SearchEngine;
import org.schabi.newpipe.crawler.StreamingService; import org.schabi.newpipe.crawler.StreamingService;
@ -116,17 +118,15 @@ public class VideoItemListFragment extends ListFragment {
if(runs) { if(runs) {
h.post(new ResultRunnable(result, requestId)); h.post(new ResultRunnable(result, requestId));
} }
} catch(Exception e) { } catch(IOException e) {
postNewErrorToast(h, R.string.network_error);
e.printStackTrace();
} catch(CrawlingException ce) {
postNewErrorToast(h, R.string.parsing_error);
ce.printStackTrace();
} catch(Exception e) {
postNewErrorToast(h, R.string.general_error);
e.printStackTrace(); e.printStackTrace();
h.post(new Runnable() {
@Override
public void run() {
setListShown(true);
Toast.makeText(getActivity(), getString(R.string.network_error),
Toast.LENGTH_SHORT).show();
}
});
} }
} }
} }
@ -386,4 +386,14 @@ public class VideoItemListFragment extends ListFragment {
mActivatedPosition = position; mActivatedPosition = position;
} }
private void postNewErrorToast(Handler h, final int stringResource) {
h.post(new Runnable() {
@Override
public void run() {
setListShown(true);
Toast.makeText(getActivity(), getString(R.string.network_error),
Toast.LENGTH_SHORT).show();
}
});
}
} }

37
app/src/main/java/org/schabi/newpipe/crawler/CrawlingException.java Normal file
View File

@ -0,0 +1,37 @@
package org.schabi.newpipe.crawler;
/**
* Created by Christian Schabesberger on 30.01.16.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* CrawlingException.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class CrawlingException extends Exception {
public CrawlingException() {}
public CrawlingException(String message) {
super(message);
}
public CrawlingException(Throwable cause) {
super(cause);
}
public CrawlingException(String message, Throwable cause) {
super(message, cause);
}
}

12
app/src/main/java/org/schabi/newpipe/crawler/Downloader.java
View File

@ -1,5 +1,7 @@
package org.schabi.newpipe.crawler; package org.schabi.newpipe.crawler;
import java.io.IOException;
/** /**
* Created by Christian Schabesberger on 28.01.16. * Created by Christian Schabesberger on 28.01.16.
* *
@ -26,12 +28,14 @@ public interface Downloader {
* but set the HTTP header field "Accept-Language" to the supplied string. * but set the HTTP header field "Accept-Language" to the supplied string.
* @param siteUrl the URL of the text file to return the contents of * @param siteUrl the URL of the text file to return the contents of
* @param language the language (usually a 2-character code) to set as the preferred language * @param language the language (usually a 2-character code) to set as the preferred language
* @return the contents of the specified text file*/ * @return the contents of the specified text file
String download(String siteUrl, String language); * @throws IOException*/
String download(String siteUrl, String language) throws IOException;
/**Download (via HTTP) the text file located at the supplied URL, and return its contents. /**Download (via HTTP) the text file located at the supplied URL, and return its contents.
* Primarily intended for downloading web pages. * Primarily intended for downloading web pages.
* @param siteUrl the URL of the text file to download * @param siteUrl the URL of the text file to download
* @return the contents of the specified text file*/ * @return the contents of the specified text file
String download(String siteUrl); * @throws IOException*/
String download(String siteUrl) throws IOException;
} }

35
app/src/main/java/org/schabi/newpipe/crawler/ParsingException.java Normal file
View File

@ -0,0 +1,35 @@
package org.schabi.newpipe.crawler;
/**
* Created by Christian Schabesberger on 31.01.16.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* ParsingException.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class ParsingException extends CrawlingException {
public ParsingException() {}
public ParsingException(String message) {
super(message);
}
public ParsingException(Throwable cause) {
super(cause);
}
public ParsingException(String message, Throwable cause) {
super(message, cause);
}
}

7
app/src/main/java/org/schabi/newpipe/crawler/SearchEngine.java
View File

@ -1,5 +1,6 @@
package org.schabi.newpipe.crawler; package org.schabi.newpipe.crawler;
import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Vector; import java.util.Vector;
@ -31,8 +32,10 @@ public interface SearchEngine {
public final Vector<VideoPreviewInfo> resultList = new Vector<>(); public final Vector<VideoPreviewInfo> resultList = new Vector<>();
} }
ArrayList<String> suggestionList(String query, Downloader dl); ArrayList<String> suggestionList(String query, Downloader dl)
throws CrawlingException, IOException;
//Result search(String query, int page); //Result search(String query, int page);
Result search(String query, int page, String contentCountry, Downloader dl); Result search(String query, int page, String contentCountry, Downloader dl)
throws CrawlingException, IOException;
} }

4
app/src/main/java/org/schabi/newpipe/crawler/StreamingService.java
View File

@ -1,5 +1,7 @@
package org.schabi.newpipe.crawler; package org.schabi.newpipe.crawler;
import java.io.IOException;
/** /**
* Created by Christian Schabesberger on 23.08.15. * Created by Christian Schabesberger on 23.08.15.
* *
@ -25,7 +27,7 @@ public interface StreamingService {
public String name = ""; public String name = "";
} }
ServiceInfo getServiceInfo(); ServiceInfo getServiceInfo();
VideoExtractor getExtractorInstance(String url, Downloader downloader); VideoExtractor getExtractorInstance(String url, Downloader downloader) throws IOException, CrawlingException;
SearchEngine getSearchEngineInstance(); SearchEngine getSearchEngineInstance();
/**When a VIEW_ACTION is caught this function will test if the url delivered within the calling /**When a VIEW_ACTION is caught this function will test if the url delivered within the calling

178
app/src/main/java/org/schabi/newpipe/crawler/VideoExtractor.java
View File

@ -22,8 +22,43 @@ package org.schabi.newpipe.crawler;
/**Scrapes information from a video streaming service (eg, YouTube).*/ /**Scrapes information from a video streaming service (eg, YouTube).*/
@SuppressWarnings("ALL") @SuppressWarnings("ALL")
public abstract class VideoExtractor { public abstract class VideoExtractor {
public class ExctractorInitException extends CrawlingException {
public ExctractorInitException() {}
public ExctractorInitException(String message) {
super(message);
}
public ExctractorInitException(Throwable cause) {
super(cause);
}
public ExctractorInitException(String message, Throwable cause) {
super(message, cause);
}
}
public class RegexException extends ParsingException {
public RegexException() {}
public RegexException(String message) {
super(message);
}
}
public class ContentNotAvailableException extends ParsingException {
public ContentNotAvailableException() {}
public ContentNotAvailableException(String message) {
super(message);
}
public ContentNotAvailableException(Throwable cause) {
super(cause);
}
public ContentNotAvailableException(String message, Throwable cause) {
super(message, cause);
}
}
protected final String pageUrl; protected final String pageUrl;
protected VideoInfo videoInfo; protected VideoInfo videoInfo;
@ -34,7 +69,7 @@ public abstract class VideoExtractor {
/**Fills out the video info fields which are common to all services. /**Fills out the video info fields which are common to all services.
* Probably needs to be overridden by subclasses*/ * Probably needs to be overridden by subclasses*/
public VideoInfo getVideoInfo() public VideoInfo getVideoInfo() throws CrawlingException
{ {
if(videoInfo == null) { if(videoInfo == null) {
videoInfo = new VideoInfo(); videoInfo = new VideoInfo();
@ -44,90 +79,83 @@ public abstract class VideoExtractor {
videoInfo.webpage_url = pageUrl; videoInfo.webpage_url = pageUrl;
} }
if(getErrorCode() == VideoInfo.NO_ERROR) {
if (videoInfo.title.isEmpty()) { if (videoInfo.title.isEmpty()) {
videoInfo.title = getTitle(); videoInfo.title = getTitle();
}
if (videoInfo.duration < 1) {
videoInfo.duration = getLength();
}
if (videoInfo.uploader.isEmpty()) {
videoInfo.uploader = getUploader();
}
if (videoInfo.description.isEmpty()) {
videoInfo.description = getDescription();
}
if (videoInfo.view_count == -1) {
videoInfo.view_count = getViews();
}
if (videoInfo.upload_date.isEmpty()) {
videoInfo.upload_date = getUploadDate();
}
if (videoInfo.thumbnail_url.isEmpty()) {
videoInfo.thumbnail_url = getThumbnailUrl();
}
if (videoInfo.id.isEmpty()) {
videoInfo.id = getVideoId(pageUrl);
}
/** Load and extract audio*/
if (videoInfo.audioStreams == null) {
videoInfo.audioStreams = getAudioStreams();
}
/** Extract video stream url*/
if (videoInfo.videoStreams == null) {
videoInfo.videoStreams = getVideoStreams();
}
if (videoInfo.uploader_thumbnail_url.isEmpty()) {
videoInfo.uploader_thumbnail_url = getUploaderThumbnailUrl();
}
if (videoInfo.startPosition < 0) {
videoInfo.startPosition = getTimeStamp();
}
if(videoInfo.dashMpdUrl.isEmpty()) {
videoInfo.dashMpdUrl = getDashMpdUrl();
}
} else {
videoInfo.errorCode = getErrorCode();
videoInfo.errorMessage = getErrorMessage();
} }
if (videoInfo.duration < 1) {
videoInfo.duration = getLength();
}
if (videoInfo.uploader.isEmpty()) {
videoInfo.uploader = getUploader();
}
if (videoInfo.description.isEmpty()) {
videoInfo.description = getDescription();
}
if (videoInfo.view_count == -1) {
videoInfo.view_count = getViews();
}
if (videoInfo.upload_date.isEmpty()) {
videoInfo.upload_date = getUploadDate();
}
if (videoInfo.thumbnail_url.isEmpty()) {
videoInfo.thumbnail_url = getThumbnailUrl();
}
if (videoInfo.id.isEmpty()) {
videoInfo.id = getVideoId(pageUrl);
}
/** Load and extract audio*/
if (videoInfo.audioStreams == null) {
videoInfo.audioStreams = getAudioStreams();
}
/** Extract video stream url*/
if (videoInfo.videoStreams == null) {
videoInfo.videoStreams = getVideoStreams();
}
if (videoInfo.uploader_thumbnail_url.isEmpty()) {
videoInfo.uploader_thumbnail_url = getUploaderThumbnailUrl();
}
if (videoInfo.startPosition < 0) {
videoInfo.startPosition = getTimeStamp();
}
if(videoInfo.dashMpdUrl.isEmpty()) {
videoInfo.dashMpdUrl = getDashMpdUrl();
}
//Bitmap thumbnail = null; //Bitmap thumbnail = null;
//Bitmap uploader_thumbnail = null; //Bitmap uploader_thumbnail = null;
//int videoAvailableStatus = VIDEO_AVAILABLE; //int videoAvailableStatus = VIDEO_AVAILABLE;
return videoInfo; return videoInfo;
} }
//todo: add licence field
public abstract int getErrorCode();
public abstract String getErrorMessage();
//todo: remove these functions, or make them static, otherwise its useles, to have them here //todo: remove these functions, or make them static, otherwise its useles, to have them here
public abstract String getVideoUrl(String videoId); public abstract String getVideoUrl(String videoId);
public abstract String getVideoId(String siteUrl); public abstract String getVideoId(String siteUrl) throws ParsingException;
/////////////////////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////////////////////////
public abstract int getTimeStamp(); public abstract int getTimeStamp() throws ParsingException;
public abstract String getTitle(); public abstract String getTitle() throws ParsingException;
public abstract String getDescription(); public abstract String getDescription() throws ParsingException;
public abstract String getUploader(); public abstract String getUploader() throws ParsingException;
public abstract int getLength(); public abstract int getLength() throws ParsingException;
public abstract long getViews(); public abstract long getViews() throws ParsingException;
public abstract String getUploadDate(); public abstract String getUploadDate() throws ParsingException;
public abstract String getThumbnailUrl(); public abstract String getThumbnailUrl() throws ParsingException;
public abstract String getUploaderThumbnailUrl(); public abstract String getUploaderThumbnailUrl() throws ParsingException;
public abstract VideoInfo.AudioStream[] getAudioStreams(); public abstract VideoInfo.AudioStream[] getAudioStreams() throws ParsingException;
public abstract VideoInfo.VideoStream[] getVideoStreams(); public abstract VideoInfo.VideoStream[] getVideoStreams() throws ParsingException;
public abstract String getDashMpdUrl(); public abstract String getDashMpdUrl() throws ParsingException;
public abstract int getAgeLimit() throws ParsingException;
} }

11
app/src/main/java/org/schabi/newpipe/crawler/VideoInfo.java
View File

@ -26,14 +26,6 @@ import java.util.List;
@SuppressWarnings("ALL") @SuppressWarnings("ALL")
public class VideoInfo extends AbstractVideoInfo { public class VideoInfo extends AbstractVideoInfo {
// If a video could not be parsed, this predefined error codes
// will be returned AND can be parsed by the frontend of the app.
// Error codes:
public final static int NO_ERROR = 0x0;
public final static int ERROR_NO_SPECIFIED_ERROR = 0x1;
// GEMA a german music colecting society.
public final static int ERROR_BLOCKED_BY_GEMA = 0x2;
public String uploader_thumbnail_url = ""; public String uploader_thumbnail_url = "";
public String description = ""; public String description = "";
public VideoStream[] videoStreams = null; public VideoStream[] videoStreams = null;
@ -43,8 +35,6 @@ public class VideoInfo extends AbstractVideoInfo {
// crawling such a file is not service dependent. Therefore getting audio only streams by yust // crawling such a file is not service dependent. Therefore getting audio only streams by yust
// providing the dash mpd fille will be possible in the future. // providing the dash mpd fille will be possible in the future.
public String dashMpdUrl = ""; public String dashMpdUrl = "";
public int errorCode = NO_ERROR;
public String errorMessage = "";
public int duration = -1; public int duration = -1;
/*YouTube-specific fields /*YouTube-specific fields
@ -60,7 +50,6 @@ public class VideoInfo extends AbstractVideoInfo {
public VideoInfo() {} public VideoInfo() {}
/**Creates a new VideoInfo object from an existing AbstractVideoInfo. /**Creates a new VideoInfo object from an existing AbstractVideoInfo.
* All the shared properties are copied to the new VideoInfo.*/ * All the shared properties are copied to the new VideoInfo.*/
@SuppressWarnings("WeakerAccess") @SuppressWarnings("WeakerAccess")

162
app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeSearchEngine.java
View File

@ -6,8 +6,11 @@ import android.util.Log;
import org.jsoup.Jsoup; import org.jsoup.Jsoup;
import org.jsoup.nodes.Document; import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element; import org.jsoup.nodes.Element;
import org.schabi.newpipe.crawler.CrawlingException;
import org.schabi.newpipe.crawler.Downloader; import org.schabi.newpipe.crawler.Downloader;
import org.schabi.newpipe.crawler.ParsingException;
import org.schabi.newpipe.crawler.SearchEngine; import org.schabi.newpipe.crawler.SearchEngine;
import org.schabi.newpipe.crawler.VideoExtractor;
import org.schabi.newpipe.crawler.VideoPreviewInfo; import org.schabi.newpipe.crawler.VideoPreviewInfo;
import org.w3c.dom.Node; import org.w3c.dom.Node;
import org.w3c.dom.NodeList; import org.w3c.dom.NodeList;
@ -49,8 +52,8 @@ public class YoutubeSearchEngine implements SearchEngine {
private static final String TAG = YoutubeSearchEngine.class.toString(); private static final String TAG = YoutubeSearchEngine.class.toString();
@Override @Override
public Result search(String query, int page, String languageCode, Downloader downloader) { public Result search(String query, int page, String languageCode, Downloader downloader) throws IOException, ParsingException {
//String contentCountry = PreferenceManager.getDefaultSharedPreferences(this).getString(getString(R.string., ""); Result result = new Result();
Uri.Builder builder = new Uri.Builder(); Uri.Builder builder = new Uri.Builder();
builder.scheme("https") builder.scheme("https")
.authority("www.youtube.com") .authority("www.youtube.com")
@ -63,22 +66,19 @@ public class YoutubeSearchEngine implements SearchEngine {
String url = builder.build().toString(); String url = builder.build().toString();
//if we've been passed a valid language code, append it to the URL //if we've been passed a valid language code, append it to the URL
if(!languageCode.isEmpty()) { if(!languageCode.isEmpty()) {
//assert Pattern.matches("[a-z]{2}(-([A-Z]{2}|[0-9]{1,3}))?", languageCode); //assert Pattern.matches("[a-z]{2}(-([A-Z]{2}|[0-9]{1,3}))?", languageCode);
site = downloader.download(url, languageCode); site = downloader.download(url, languageCode);
} }
else { else {
site = downloader.download(url); site = downloader.download(url);
} }
try {
Document doc = Jsoup.parse(site, url); Document doc = Jsoup.parse(site, url);
Result result = new Result(); Element list = doc.select("ol[class=\"item-section\"]").first();
Element list = doc.select("ol[class=\"item-section\"]").first();
for (Element item : list.children()) {
int i = 0;
for(Element item : list.children()) {
i++;
/* First we need to determine which kind of item we are working with. /* First we need to determine which kind of item we are working with.
Youtube depicts five different kinds of items on its search result page. These are Youtube depicts five different kinds of items on its search result page. These are
regular videos, playlists, channels, two types of video suggestions, and a "no video regular videos, playlists, channels, two types of video suggestions, and a "no video
@ -90,57 +90,61 @@ public class YoutubeSearchEngine implements SearchEngine {
playlists now. playlists now.
*/ */
Element el; Element el;
// both types of spell correction item // both types of spell correction item
if(!((el = item.select("div[class*=\"spell-correction\"]").first()) == null)) { if (!((el = item.select("div[class*=\"spell-correction\"]").first()) == null)) {
result.suggestion = el.select("a").first().text(); result.suggestion = el.select("a").first().text();
// search message item // search message item
} else if(!((el = item.select("div[class*=\"search-message\"]").first()) == null)) { } else if (!((el = item.select("div[class*=\"search-message\"]").first()) == null)) {
result.errorMessage = el.text(); result.errorMessage = el.text();
// video item type // video item type
} else if(!((el = item.select("div[class*=\"yt-lockup-video\"").first()) == null)) { } else if (!((el = item.select("div[class*=\"yt-lockup-video\"").first()) == null)) {
VideoPreviewInfo resultItem = new VideoPreviewInfo(); VideoPreviewInfo resultItem = new VideoPreviewInfo();
Element dl = el.select("h3").first().select("a").first(); Element dl = el.select("h3").first().select("a").first();
resultItem.webpage_url = dl.attr("abs:href"); resultItem.webpage_url = dl.attr("abs:href");
try { try {
Pattern p = Pattern.compile("v=([0-9a-zA-Z-]*)"); Pattern p = Pattern.compile("v=([0-9a-zA-Z-]*)");
Matcher m = p.matcher(resultItem.webpage_url); Matcher m = p.matcher(resultItem.webpage_url);
resultItem.id=m.group(1); resultItem.id = m.group(1);
} catch (Exception e) { } catch (Exception e) {
//e.printStackTrace(); //e.printStackTrace();
}
resultItem.title = dl.text();
resultItem.duration = item.select("span[class=\"video-time\"]").first().text();
resultItem.uploader = item.select("div[class=\"yt-lockup-byline\"]").first()
.select("a").first()
.text();
resultItem.upload_date = item.select("div[class=\"yt-lockup-meta\"]").first()
.select("li").first()
.text();
Element te = item.select("div[class=\"yt-thumb video-thumb\"]").first()
.select("img").first();
resultItem.thumbnail_url = te.attr("abs:src");
// Sometimes youtube sends links to gif files which somehow seem to not exist
// anymore. Items with such gif also offer a secondary image source. So we are going
// to use that if we've caught such an item.
if (resultItem.thumbnail_url.contains(".gif")) {
resultItem.thumbnail_url = te.attr("abs:data-thumb");
}
result.resultList.add(resultItem);
} else {
//noinspection ConstantConditions
Log.e(TAG, "unexpected element found:\"" + el + "\"");
} }
resultItem.title = dl.text();
resultItem.duration = item.select("span[class=\"video-time\"]").first().text();
resultItem.uploader = item.select("div[class=\"yt-lockup-byline\"]").first()
.select("a").first()
.text();
resultItem.upload_date = item.select("div[class=\"yt-lockup-meta\"]").first()
.select("li").first()
.text();
Element te = item.select("div[class=\"yt-thumb video-thumb\"]").first()
.select("img").first();
resultItem.thumbnail_url = te.attr("abs:src");
// Sometimes youtube sends links to gif files which somehow seem to not exist
// anymore. Items with such gif also offer a secondary image source. So we are going
// to use that if we've caught such an item.
if(resultItem.thumbnail_url.contains(".gif")) {
resultItem.thumbnail_url = te.attr("abs:data-thumb");
}
result.resultList.add(resultItem);
} else {
//noinspection ConstantConditions
Log.e(TAG, "unexpected element found:\""+el+"\"");
} }
} catch(Exception e) {
throw new ParsingException(e);
} }
return result; return result;
} }
@Override @Override
public ArrayList<String> suggestionList(String query, Downloader dl) { public ArrayList<String> suggestionList(String query, Downloader dl)
throws IOException, ParsingException {
ArrayList<String> suggestions = new ArrayList<>(); ArrayList<String> suggestions = new ArrayList<>();
@ -155,36 +159,42 @@ public class YoutubeSearchEngine implements SearchEngine {
.appendQueryParameter("q", query); .appendQueryParameter("q", query);
String url = builder.build().toString(); String url = builder.build().toString();
String response = dl.download(url); String response = dl.download(url);
//TODO: Parse xml data using Jsoup not done
DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
DocumentBuilder dBuilder;
org.w3c.dom.Document doc = null;
try { try {
dBuilder = dbFactory.newDocumentBuilder();
doc = dBuilder.parse(new InputSource(new ByteArrayInputStream(response.getBytes("utf-8"))));
doc.getDocumentElement().normalize();
}catch (ParserConfigurationException | SAXException | IOException e) {
e.printStackTrace();
}
if(doc!=null){ //TODO: Parse xml data using Jsoup not done
NodeList nList = doc.getElementsByTagName("CompleteSuggestion"); DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
for (int temp = 0; temp < nList.getLength(); temp++) { DocumentBuilder dBuilder;
org.w3c.dom.Document doc = null;
NodeList nList1 = doc.getElementsByTagName("suggestion"); try {
Node nNode1 = nList1.item(temp); dBuilder = dbFactory.newDocumentBuilder();
if (nNode1.getNodeType() == Node.ELEMENT_NODE) { doc = dBuilder.parse(new InputSource(new ByteArrayInputStream(response.getBytes("utf-8"))));
org.w3c.dom.Element eElement = (org.w3c.dom.Element) nNode1; doc.getDocumentElement().normalize();
suggestions.add(eElement.getAttribute("data")); } catch (ParserConfigurationException | SAXException | IOException e) {
} e.printStackTrace();
} }
}else {
Log.e(TAG, "GREAT FUCKING ERROR"); if (doc != null) {
NodeList nList = doc.getElementsByTagName("CompleteSuggestion");
for (int temp = 0; temp < nList.getLength(); temp++) {
NodeList nList1 = doc.getElementsByTagName("suggestion");
Node nNode1 = nList1.item(temp);
if (nNode1.getNodeType() == Node.ELEMENT_NODE) {
org.w3c.dom.Element eElement = (org.w3c.dom.Element) nNode1;
suggestions.add(eElement.getAttribute("data"));
}
}
} else {
Log.e(TAG, "GREAT FUCKING ERROR");
}
return suggestions;
} catch(Exception e) {
throw new ParsingException(e);
} }
return suggestions;
} }
} }

7
app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeService.java
View File

@ -1,10 +1,13 @@
package org.schabi.newpipe.crawler.services.youtube; package org.schabi.newpipe.crawler.services.youtube;
import org.schabi.newpipe.crawler.CrawlingException;
import org.schabi.newpipe.crawler.Downloader; import org.schabi.newpipe.crawler.Downloader;
import org.schabi.newpipe.crawler.StreamingService; import org.schabi.newpipe.crawler.StreamingService;
import org.schabi.newpipe.crawler.VideoExtractor; import org.schabi.newpipe.crawler.VideoExtractor;
import org.schabi.newpipe.crawler.SearchEngine; import org.schabi.newpipe.crawler.SearchEngine;
import java.io.IOException;
/** /**
* Created by Christian Schabesberger on 23.08.15. * Created by Christian Schabesberger on 23.08.15.
@ -34,9 +37,9 @@ public class YoutubeService implements StreamingService {
return serviceInfo; return serviceInfo;
} }
@Override @Override
public VideoExtractor getExtractorInstance(String url, Downloader downloader) { public VideoExtractor getExtractorInstance(String url, Downloader downloader) throws CrawlingException, IOException {
if(acceptUrl(url)) { if(acceptUrl(url)) {
return new YoutubeVideoExtractor(url, downloader); return new YoutubeVideoExtractor(url, downloader) ;
} }
else { else {
throw new IllegalArgumentException("supplied String is not a valid Youtube URL"); throw new IllegalArgumentException("supplied String is not a valid Youtube URL");

647
app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeVideoExtractor.java
View File

@ -12,13 +12,16 @@ import org.jsoup.parser.Parser;
import org.mozilla.javascript.Context; import org.mozilla.javascript.Context;
import org.mozilla.javascript.Function; import org.mozilla.javascript.Function;
import org.mozilla.javascript.ScriptableObject; import org.mozilla.javascript.ScriptableObject;
import org.schabi.newpipe.crawler.CrawlingException;
import org.schabi.newpipe.crawler.Downloader; import org.schabi.newpipe.crawler.Downloader;
import org.schabi.newpipe.crawler.ParsingException;
import org.schabi.newpipe.crawler.VideoExtractor; import org.schabi.newpipe.crawler.VideoExtractor;
import org.schabi.newpipe.crawler.MediaFormat; import org.schabi.newpipe.crawler.MediaFormat;
import org.schabi.newpipe.crawler.VideoInfo; import org.schabi.newpipe.crawler.VideoInfo;
import org.schabi.newpipe.crawler.VideoPreviewInfo; import org.schabi.newpipe.crawler.VideoPreviewInfo;
import org.xmlpull.v1.XmlPullParser; import org.xmlpull.v1.XmlPullParser;
import java.io.IOException;
import java.io.StringReader; import java.io.StringReader;
import java.net.URLDecoder; import java.net.URLDecoder;
import java.util.HashMap; import java.util.HashMap;
@ -49,11 +52,28 @@ import java.util.regex.Pattern;
public class YoutubeVideoExtractor extends VideoExtractor { public class YoutubeVideoExtractor extends VideoExtractor {
public class DecryptException extends ParsingException {
DecryptException(Throwable cause) {
super(cause);
}
DecryptException(String message, Throwable cause) {
super(message, cause);
}
}
// special content not available exceptions
public class GemaException extends ContentNotAvailableException {
GemaException(String message) {
super(message);
}
}
// ----------------
private static final String TAG = YoutubeVideoExtractor.class.toString(); private static final String TAG = YoutubeVideoExtractor.class.toString();
private final Document doc; private final Document doc;
private JSONObject jsonObj;
private JSONObject playerArgs; private JSONObject playerArgs;
private int errorCode = VideoInfo.NO_ERROR;
private String errorMessage = ""; private String errorMessage = "";
// static values // static values
@ -64,29 +84,32 @@ public class YoutubeVideoExtractor extends VideoExtractor {
private Downloader downloader; private Downloader downloader;
public YoutubeVideoExtractor(String pageUrl, Downloader dl) { public YoutubeVideoExtractor(String pageUrl, Downloader dl) throws CrawlingException, IOException {
//most common videoInfo fields are now set in our superclass, for all services //most common videoInfo fields are now set in our superclass, for all services
super(pageUrl, dl); super(pageUrl, dl);
downloader = dl; downloader = dl;
String pageContent = downloader.download(cleanUrl(pageUrl)); String pageContent = downloader.download(cleanUrl(pageUrl));
doc = Jsoup.parse(pageContent, pageUrl); doc = Jsoup.parse(pageContent, pageUrl);
String ytPlayerConfigRaw;
JSONObject ytPlayerConfig;
//attempt to load the youtube js player JSON arguments //attempt to load the youtube js player JSON arguments
try { try {
String jsonString = matchGroup1("ytplayer.config\\s*=\\s*(\\{.*?\\});", pageContent); ytPlayerConfigRaw = matchGroup1("ytplayer.config\\s*=\\s*(\\{.*?\\});", pageContent);
//todo: implement this by try and catch. TESTING THE STRING AGAINST EMPTY IS CONSIDERED POOR STYLE !!! ytPlayerConfig = new JSONObject(ytPlayerConfigRaw);
if(jsonString.isEmpty()) { playerArgs = ytPlayerConfig.getJSONObject("args");
errorCode = findErrorReason(doc); } catch (RegexException e) {
return; String errorReason = findErrorReason(doc);
switch(errorReason) {
case "GEMA":
throw new GemaException(errorReason);
case "":
throw new ParsingException("player config empty", e);
default:
throw new ContentNotAvailableException("Content not available", e);
} }
} catch (JSONException e) {
jsonObj = new JSONObject(jsonString); throw new ParsingException("Could not parse yt player config");
playerArgs = jsonObj.getJSONObject("args");
} catch (Exception e) {//if this fails, the video is most likely not available.
// Determining why is done later.
videoInfo.errorCode = VideoInfo.ERROR_NO_SPECIFIED_ERROR;
Log.e(TAG, "Could not load JSON data for Youtube video \""+pageUrl+"\". This most likely means the video is unavailable");
} }
//---------------------------------- //----------------------------------
@ -94,102 +117,92 @@ public class YoutubeVideoExtractor extends VideoExtractor {
//---------------------------------- //----------------------------------
if (decryptionCode.isEmpty()) { if (decryptionCode.isEmpty()) {
try { try {
// The Youtube service needs to be initialized by downloading the // The Youtube service needs to be initialized by downloading the
// js-Youtube-player. This is done in order to get the algorithm // js-Youtube-player. This is done in order to get the algorithm
// for decrypting cryptic signatures inside certain stream urls. // for decrypting cryptic signatures inside certain stream urls.
JSONObject ytAssets = jsonObj.getJSONObject("assets"); JSONObject ytAssets = ytPlayerConfig.getJSONObject("assets");
String playerUrl = ytAssets.getString("js"); String playerUrl = ytAssets.getString("js");
if (playerUrl.startsWith("//")) { if (playerUrl.startsWith("//")) {
playerUrl = "https:" + playerUrl; playerUrl = "https:" + playerUrl;
} }
decryptionCode = loadDecryptionCode(playerUrl); decryptionCode = loadDecryptionCode(playerUrl);
} catch (Exception e){ } catch (JSONException e) {
Log.e(TAG, "Could not load decryption code for the Youtube service."); throw new ParsingException(
e.printStackTrace(); "Could not load decryption code for the Youtube service.", e);
} }
} }
} }
@Override @Override
public String getTitle() { public String getTitle() throws ParsingException {
try {//json player args method try {//json player args method
return playerArgs.getString("title"); return playerArgs.getString("title");
} catch(JSONException je) {//html <meta> method } catch(JSONException je) {//html <meta> method
je.printStackTrace(); je.printStackTrace();
Log.w(TAG, "failed to load title from JSON args; trying to extract it from HTML"); Log.w(TAG, "failed to load title from JSON args; trying to extract it from HTML");
} try { // fall through to fall-back try { // fall through to fall-back
return doc.select("meta[name=title]").attr("content"); return doc.select("meta[name=title]").attr("content");
} catch (Exception e) { } catch (Exception e) {
Log.e(TAG, "failed permanently to load title."); throw new ParsingException("failed permanently to load title.", e);
e.printStackTrace(); }
return "";
} }
} }
@Override @Override
public String getDescription() { public String getDescription() throws ParsingException {
try { try {
return doc.select("p[id=\"eow-description\"]").first().html(); return doc.select("p[id=\"eow-description\"]").first().html();
} catch (Exception e) {//todo: add fallback method } catch (Exception e) {//todo: add fallback method <-- there is no ... as long as i know
Log.e(TAG, "failed to load description."); throw new ParsingException("failed to load description.", e);
e.printStackTrace();
return "";
} }
} }
@Override @Override
public String getUploader() { public String getUploader() throws ParsingException {
try {//json player args method try {//json player args method
return playerArgs.getString("author"); return playerArgs.getString("author");
} catch(JSONException je) { } catch(JSONException je) {
je.printStackTrace(); je.printStackTrace();
Log.w(TAG, "failed to load uploader name from JSON args; trying to extract it from HTML"); Log.w(TAG,
"failed to load uploader name from JSON args; trying to extract it from HTML");
} try {//fall through to fallback HTML method } try {//fall through to fallback HTML method
return doc.select("div.yt-user-info").first().text(); return doc.select("div.yt-user-info").first().text();
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); throw new ParsingException("failed permanently to load uploader name.", e);
Log.e(TAG, "failed permanently to load uploader name.");
return "";
} }
} }
@Override @Override
public int getLength() { public int getLength() throws ParsingException {
try { try {
return playerArgs.getInt("length_seconds"); return playerArgs.getInt("length_seconds");
} catch (JSONException je) {//todo: find fallback method } catch (JSONException e) {//todo: find fallback method
Log.e(TAG, "failed to load video duration from JSON args"); throw new ParsingException("failed to load video duration from JSON args", e);
je.printStackTrace();
return -1;
} }
} }
@Override @Override
public long getViews() { public long getViews() throws ParsingException {
try { try {
String viewCountString = doc.select("meta[itemprop=interactionCount]").attr("content"); String viewCountString = doc.select("meta[itemprop=interactionCount]").attr("content");
return Long.parseLong(viewCountString); return Long.parseLong(viewCountString);
} catch (Exception e) {//todo: find fallback method } catch (Exception e) {//todo: find fallback method
Log.e(TAG, "failed to number of views"); throw new ParsingException("failed to number of views", e);
e.printStackTrace();
return -1;
} }
} }
@Override @Override
public String getUploadDate() { public String getUploadDate() throws ParsingException {
try { try {
return doc.select("meta[itemprop=datePublished]").attr("content"); return doc.select("meta[itemprop=datePublished]").attr("content");
} catch (Exception e) {//todo: add fallback method } catch (Exception e) {//todo: add fallback method
Log.e(TAG, "failed to get upload date."); throw new ParsingException("failed to get upload date.", e);
e.printStackTrace();
return "";
} }
} }
@Override @Override
public String getThumbnailUrl() { public String getThumbnailUrl() throws ParsingException {
//first attempt getting a small image version //first attempt getting a small image version
//in the html extracting part we try to get a thumbnail with a higher resolution //in the html extracting part we try to get a thumbnail with a higher resolution
// Try to get high resolution thumbnail if it fails use low res from the player instead // Try to get high resolution thumbnail if it fails use low res from the player instead
@ -197,148 +210,98 @@ public class YoutubeVideoExtractor extends VideoExtractor {
return doc.select("link[itemprop=\"thumbnailUrl\"]").first().attr("abs:href"); return doc.select("link[itemprop=\"thumbnailUrl\"]").first().attr("abs:href");
} catch(Exception e) { } catch(Exception e) {
Log.w(TAG, "Could not find high res Thumbnail. Using low res instead"); Log.w(TAG, "Could not find high res Thumbnail. Using low res instead");
//fall through to fallback } try { //fall through to fallback
} try {
return playerArgs.getString("thumbnail_url"); return playerArgs.getString("thumbnail_url");
} catch (JSONException je) { } catch (JSONException je) {
je.printStackTrace(); throw new ParsingException(
Log.w(TAG, "failed to extract thumbnail URL from JSON args; trying to extract it from HTML"); "failed to extract thumbnail URL from JSON args; trying to extract it from HTML", je);
return "";
} }
} }
@Override @Override
public String getUploaderThumbnailUrl() { public String getUploaderThumbnailUrl() throws ParsingException {
try { try {
return doc.select("a[class*=\"yt-user-photo\"]").first() return doc.select("a[class*=\"yt-user-photo\"]").first()
.select("img").first() .select("img").first()
.attr("abs:data-thumb"); .attr("abs:data-thumb");
} catch (Exception e) {//todo: add fallback method } catch (Exception e) {//todo: add fallback method
Log.e(TAG, "failed to get uploader thumbnail URL."); throw new ParsingException("failed to get uploader thumbnail URL.", e);
e.printStackTrace();
return "";
} }
} }
@Override @Override
public String getDashMpdUrl() { public String getDashMpdUrl() throws ParsingException {
try { try {
return playerArgs.getString("dashmpd"); return playerArgs.getString("dashmpd");
} catch(NullPointerException e) { } catch(NullPointerException e) {
Log.e(TAG, "Could not find \"dashmpd\" upon the player args (maybe no dash manifest available)."); throw new ParsingException(
"Could not find \"dashmpd\" upon the player args (maybe no dash manifest available).", e);
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); throw new ParsingException(e);
} }
return "";
} }
@Override @Override
public VideoInfo.AudioStream[] getAudioStreams() { public VideoInfo.AudioStream[] getAudioStreams() throws ParsingException {
try { try {
String dashManifest = playerArgs.getString("dashmpd"); String dashManifest = playerArgs.getString("dashmpd");
Log.d(TAG, dashManifest);
return parseDashManifest(dashManifest, decryptionCode); return parseDashManifest(dashManifest, decryptionCode);
} catch (NullPointerException e) { } catch (NullPointerException e) {
Log.e(TAG, "Could not find \"dashmpd\" upon the player args (maybe no dash manifest available)."); throw new ParsingException(
"Could not find \"dashmpd\" upon the player args (maybe no dash manifest available).", e);
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); throw new ParsingException(e);
} }
return new VideoInfo.AudioStream[0];
} }
@Override @Override
public VideoInfo.VideoStream[] getVideoStreams() { public VideoInfo.VideoStream[] getVideoStreams() throws ParsingException {
Vector<VideoInfo.VideoStream> videoStreams = new Vector<>();
try{ try{
//------------------------------------
// extract video stream url
//------------------------------------
Vector<VideoInfo.VideoStream> videoStreams = new Vector<>();
// The following line belongs to dash audio stuff.
// We can't use dash audio, unless we have exoplayer as our main player.
//String adaptive_fmts = playerArgs.getString("adaptive_fmts");
String encoded_url_map = playerArgs.getString("url_encoded_fmt_stream_map"); String encoded_url_map = playerArgs.getString("url_encoded_fmt_stream_map");
for(String url_data_str : encoded_url_map.split(",")) { for(String url_data_str : encoded_url_map.split(",")) {
Map<String, String> tags = new HashMap<>(); try {
for(String raw_tag : Parser.unescapeEntities(url_data_str, true).split("&")) { Map<String, String> tags = new HashMap<>();
String[] split_tag = raw_tag.split("="); for (String raw_tag : Parser.unescapeEntities(url_data_str, true).split("&")) {
tags.put(split_tag[0], split_tag[1]); String[] split_tag = raw_tag.split("=");
} tags.put(split_tag[0], split_tag[1]);
}
int itag = Integer.parseInt(tags.get("itag")); int itag = Integer.parseInt(tags.get("itag"));
String streamUrl = URLDecoder.decode(tags.get("url"), "UTF-8"); String streamUrl = URLDecoder.decode(tags.get("url"), "UTF-8");
// if video has a signature: decrypt it and add it to the url // if video has a signature: decrypt it and add it to the url
if(tags.get("s") != null) { if (tags.get("s") != null) {
streamUrl = streamUrl + "&signature=" + decryptSignature(tags.get("s"), decryptionCode); streamUrl = streamUrl + "&signature="
} + decryptSignature(tags.get("s"), decryptionCode);
}
if(resolveFormat(itag) != -1) { if (resolveFormat(itag) != -1) {
videoStreams.add(new VideoInfo.VideoStream( videoStreams.add(new VideoInfo.VideoStream(
streamUrl, streamUrl,
resolveFormat(itag), resolveFormat(itag),
resolveResolutionString(itag))); resolveResolutionString(itag)));
}
} catch (Exception e) {
Log.w(TAG, "Could not get Video stream.");
e.printStackTrace();
} }
} }
return videoStreams.toArray(new VideoInfo.VideoStream[videoStreams.size()]);
} catch (Exception e) { } catch (Exception e) {
Log.e(TAG, "Failed to get video stream"); throw new ParsingException("Failed to get video streams", e);
e.printStackTrace();
return new VideoInfo.VideoStream[0];
} }
}
/**These lists only contain itag formats that are supported by the common Android Video player. if(videoStreams.isEmpty()) {
However if you are looking for a list showing all itag formats, look at throw new ParsingException("Failed to get any video stream");
https://github.com/rg3/youtube-dl/issues/1687 */
@SuppressWarnings("WeakerAccess")
public static int resolveFormat(int itag) {
switch(itag) {
// !!! lists only supported formats !!!
// video
case 17: return MediaFormat.v3GPP.id;
case 18: return MediaFormat.MPEG_4.id;
case 22: return MediaFormat.MPEG_4.id;
case 36: return MediaFormat.v3GPP.id;
case 37: return MediaFormat.MPEG_4.id;
case 38: return MediaFormat.MPEG_4.id;
case 43: return MediaFormat.WEBM.id;
case 44: return MediaFormat.WEBM.id;
case 45: return MediaFormat.WEBM.id;
case 46: return MediaFormat.WEBM.id;
default:
//Log.i(TAG, "Itag " + Integer.toString(itag) + " not known or not supported.");
return -1;
} }
}
@SuppressWarnings("WeakerAccess") return videoStreams.toArray(new VideoInfo.VideoStream[videoStreams.size()]);
public static String resolveResolutionString(int itag) {
switch(itag) {
case 17: return "144p";
case 18: return "360p";
case 22: return "720p";
case 36: return "240p";
case 37: return "1080p";
case 38: return "1080p";
case 43: return "360p";
case 44: return "480p";
case 45: return "720p";
case 46: return "1080p";
default:
//Log.i(TAG, "Itag " + Integer.toString(itag) + " not known or not supported.");
return null;
}
} }
@SuppressWarnings("WeakerAccess") @SuppressWarnings("WeakerAccess")
@Override @Override
public String getVideoId(String url) { public String getVideoId(String url) throws ParsingException {
String id; String id;
String pat; String pat;
@ -349,16 +312,16 @@ public class YoutubeVideoExtractor extends VideoExtractor {
pat = "youtu\\.be/([a-zA-Z0-9_-]{11})"; pat = "youtu\\.be/([a-zA-Z0-9_-]{11})";
} }
else { else {
Log.e(TAG, "Error could not parse url: " + url); throw new ParsingException("Error no suitable url: " + url);
return "";
} }
id = matchGroup1(pat, url); id = matchGroup1(pat, url);
if(!id.isEmpty()){ if(!id.isEmpty()){
//Log.i(TAG, "string \""+url+"\" matches!"); //Log.i(TAG, "string \""+url+"\" matches!");
return id; return id;
} else {
throw new ParsingException("Error could not parse url: " + url);
} }
//Log.i(TAG, "string \""+url+"\" does not match.");
return "";
} }
@SuppressWarnings("WeakerAccess") @SuppressWarnings("WeakerAccess")
@ -370,116 +333,136 @@ public class YoutubeVideoExtractor extends VideoExtractor {
/**Attempts to parse (and return) the offset to start playing the video from. /**Attempts to parse (and return) the offset to start playing the video from.
* @return the offset (in seconds), or 0 if no timestamp is found.*/ * @return the offset (in seconds), or 0 if no timestamp is found.*/
@Override @Override
public int getTimeStamp(){ public int getTimeStamp() throws ParsingException {
String timeStamp = matchGroup1("((#|&)t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)", pageUrl); //todo: add unit test for timestamp
String timeStamp;
try {
timeStamp = matchGroup1("((#|&|\\?)t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)", pageUrl);
} catch (RegexException e) {
// catch this instantly since an url does not necessarily have to have a time stamp
// -2 because well the testing system will then know its the regex that failed :/
// not good i know
return -2;
}
//TODO: test this //TODO: test this
if(!timeStamp.isEmpty()) { if(!timeStamp.isEmpty()) {
String secondsString = matchGroup1("(\\d{1,3})s", timeStamp); try {
String minutesString = matchGroup1("(\\d{1,3})m", timeStamp); String secondsString = "";
String hoursString = matchGroup1("(\\d{1,3})h", timeStamp); String minutesString = "";
String hoursString = "";
try {
secondsString = matchGroup1("(\\d{1,3})s", timeStamp);
minutesString = matchGroup1("(\\d{1,3})m", timeStamp);
hoursString = matchGroup1("(\\d{1,3})h", timeStamp);
} catch (Exception e) {
//it could be that time is given in another method
if (secondsString.isEmpty() //if nothing was got,
&& minutesString.isEmpty()//treat as unlabelled seconds
&& hoursString.isEmpty()) {
secondsString = matchGroup1("t=(\\d{1,3})", timeStamp);
}
}
if(secondsString.isEmpty()//if nothing was got, int seconds = (secondsString.isEmpty() ? 0 : Integer.parseInt(secondsString));
&& minutesString.isEmpty()//treat as unlabelled seconds int minutes = (minutesString.isEmpty() ? 0 : Integer.parseInt(minutesString));
&& hoursString.isEmpty()) int hours = (hoursString.isEmpty() ? 0 : Integer.parseInt(hoursString));
secondsString = matchGroup1("t=(\\d{1,3})", timeStamp);
int seconds = (secondsString.isEmpty() ? 0 : Integer.parseInt(secondsString)); int ret = seconds + (60 * minutes) + (3600 * hours);//don't trust BODMAS!
int minutes = (minutesString.isEmpty() ? 0 : Integer.parseInt(minutesString)); //Log.d(TAG, "derived timestamp value:"+ret);
int hours = (hoursString.isEmpty() ? 0 : Integer.parseInt(hoursString)); return ret;
//the ordering varies internationally
} catch (ParsingException e) {
throw new ParsingException("Could not get timestamp.", e);
}
} else {
return -1;
}
}
int ret = seconds + (60*minutes) + (3600*hours);//don't trust BODMAS! @Override
//Log.d(TAG, "derived timestamp value:"+ret); public int getAgeLimit() throws ParsingException {
return ret; // Not yet implemented.
//the ordering varies internationally // Also you need to be logged in to see age restricted videos on youtube,
}//else, return default 0 // therefore NP is not able to receive such videos.
return 0; return 0;
} }
@Override @Override
public VideoInfo getVideoInfo() { public VideoInfo getVideoInfo() throws CrawlingException {
//todo: @medovax i like your work, but what the fuck:
videoInfo = super.getVideoInfo(); videoInfo = super.getVideoInfo();
if(errorCode == VideoInfo.NO_ERROR) { //todo: replace this with a call to getVideoId, if possible
//todo: replace this with a call to getVideoId, if possible //videoInfo.id = matchGroup1("v=([0-9a-zA-Z_-]{11})", pageUrl);
videoInfo.id = matchGroup1("v=([0-9a-zA-Z_-]{11})", pageUrl); videoInfo.id = getVideoId(pageUrl);
if (videoInfo.audioStreams == null if (videoInfo.audioStreams == null
|| videoInfo.audioStreams.length == 0) { || videoInfo.audioStreams.length == 0) {
Log.e(TAG, "uninitialised audio streams!"); Log.e(TAG, "uninitialised audio streams!");
}
if (videoInfo.videoStreams == null
|| videoInfo.videoStreams.length == 0) {
Log.e(TAG, "uninitialised video streams!");
}
videoInfo.age_limit = 0;
//average rating
try {
videoInfo.average_rating = playerArgs.getString("avg_rating");
} catch (JSONException e) {
e.printStackTrace();
}
//---------------------------------------
// extracting information from html page
//---------------------------------------
String likesString = "";
String dislikesString = "";
try {
// likes
likesString = doc.select("button.like-button-renderer-like-button").first()
.select("span.yt-uix-button-content").first().text();
videoInfo.like_count = Integer.parseInt(likesString.replaceAll("[^\\d]", ""));
// dislikes
dislikesString = doc.select("button.like-button-renderer-dislike-button").first()
.select("span.yt-uix-button-content").first().text();
videoInfo.dislike_count = Integer.parseInt(dislikesString.replaceAll("[^\\d]", ""));
} catch (NumberFormatException nfe) {
Log.e(TAG, "failed to parse likesString \"" + likesString + "\" and dislikesString \"" +
dislikesString + "\" as integers");
} catch (Exception e) {
// if it fails we know that the video does not offer dislikes.
e.printStackTrace();
videoInfo.like_count = 0;
videoInfo.dislike_count = 0;
}
// next video
videoInfo.nextVideo = extractVideoPreviewInfo(doc.select("div[class=\"watch-sidebar-section\"]").first()
.select("li").first());
// related videos
Vector<VideoPreviewInfo> relatedVideos = new Vector<>();
for (Element li : doc.select("ul[id=\"watch-related\"]").first().children()) {
// first check if we have a playlist. If so leave them out
if (li.select("a[class*=\"content-link\"]").first() != null) {
relatedVideos.add(extractVideoPreviewInfo(li));
}
}
//todo: replace conversion
videoInfo.relatedVideos = relatedVideos;
//videoInfo.relatedVideos = relatedVideos.toArray(new VideoPreviewInfo[relatedVideos.size()]);
} }
if (videoInfo.videoStreams == null
|| videoInfo.videoStreams.length == 0) {
Log.e(TAG, "uninitialised video streams!");
}
videoInfo.age_limit = 0;
//average rating
try {
videoInfo.average_rating = playerArgs.getString("avg_rating");
} catch (JSONException e) {
e.printStackTrace();
}
//---------------------------------------
// extracting information from html page
//---------------------------------------
String likesString = "";
String dislikesString = "";
try {
// likes
likesString = doc.select("button.like-button-renderer-like-button").first()
.select("span.yt-uix-button-content").first().text();
videoInfo.like_count = Integer.parseInt(likesString.replaceAll("[^\\d]", ""));
// dislikes
dislikesString = doc.select("button.like-button-renderer-dislike-button").first()
.select("span.yt-uix-button-content").first().text();
videoInfo.dislike_count = Integer.parseInt(dislikesString.replaceAll("[^\\d]", ""));
} catch (NumberFormatException nfe) {
Log.e(TAG, "failed to parse likesString \"" + likesString + "\" and dislikesString \"" +
dislikesString + "\" as integers");
} catch (Exception e) {
// if it fails we know that the video does not offer dislikes.
e.printStackTrace();
videoInfo.like_count = 0;
videoInfo.dislike_count = 0;
}
// next video
videoInfo.nextVideo = extractVideoPreviewInfo(doc.select("div[class=\"watch-sidebar-section\"]").first()
.select("li").first());
// related videos
Vector<VideoPreviewInfo> relatedVideos = new Vector<>();
for (Element li : doc.select("ul[id=\"watch-related\"]").first().children()) {
// first check if we have a playlist. If so leave them out
if (li.select("a[class*=\"content-link\"]").first() != null) {
relatedVideos.add(extractVideoPreviewInfo(li));
}
}
//todo: replace conversion
videoInfo.relatedVideos = relatedVideos;
//videoInfo.relatedVideos = relatedVideos.toArray(new VideoPreviewInfo[relatedVideos.size()]);
return videoInfo; return videoInfo;
} }
@Override private VideoInfo.AudioStream[] parseDashManifest(String dashManifest, String decryptoinCode) throws RegexException, DecryptException {
public int getErrorCode() {
return errorCode;
}
@Override
public String getErrorMessage() {
return errorMessage;
}
private VideoInfo.AudioStream[] parseDashManifest(String dashManifest, String decryptoinCode) {
if(!dashManifest.contains("/signature/")) { if(!dashManifest.contains("/signature/")) {
String encryptedSig = matchGroup1("/s/([a-fA-F0-9\\.]+)", dashManifest); String encryptedSig = matchGroup1("/s/([a-fA-F0-9\\.]+)", dashManifest);
String decryptedSig; String decryptedSig;
@ -487,7 +470,12 @@ public class YoutubeVideoExtractor extends VideoExtractor {
decryptedSig = decryptSignature(encryptedSig, decryptoinCode); decryptedSig = decryptSignature(encryptedSig, decryptoinCode);
dashManifest = dashManifest.replace("/s/" + encryptedSig, "/signature/" + decryptedSig); dashManifest = dashManifest.replace("/s/" + encryptedSig, "/signature/" + decryptedSig);
} }
String dashDoc = downloader.download(dashManifest); String dashDoc;
try {
dashDoc = downloader.download(dashManifest);
} catch(IOException ioe) {
throw new DecryptException("Could not get dash mpd", ioe);
}
Vector<VideoInfo.AudioStream> audioStreams = new Vector<>(); Vector<VideoInfo.AudioStream> audioStreams = new Vector<>();
try { try {
XmlPullParser parser = Xml.newPullParser(); XmlPullParser parser = Xml.newPullParser();
@ -545,63 +533,67 @@ public class YoutubeVideoExtractor extends VideoExtractor {
/**Provides information about links to other videos on the video page, such as related videos. /**Provides information about links to other videos on the video page, such as related videos.
* This is encapsulated in a VideoPreviewInfo object, * This is encapsulated in a VideoPreviewInfo object,
* which is a subset of the fields in a full VideoInfo.*/ * which is a subset of the fields in a full VideoInfo.*/
private VideoPreviewInfo extractVideoPreviewInfo(Element li) { private VideoPreviewInfo extractVideoPreviewInfo(Element li) throws ParsingException {
VideoPreviewInfo info = new VideoPreviewInfo(); VideoPreviewInfo info = new VideoPreviewInfo();
info.webpage_url = li.select("a.content-link").first()
.attr("abs:href");
try { try {
info.webpage_url = li.select("a.content-link").first()
.attr("abs:href");
info.id = matchGroup1("v=([0-9a-zA-Z-]*)", info.webpage_url); info.id = matchGroup1("v=([0-9a-zA-Z-]*)", info.webpage_url);
//todo: check NullPointerException causing
info.title = li.select("span.title").first().text();
//this page causes the NullPointerException, after finding it by searching for "tjvg":
//https://www.youtube.com/watch?v=Uqg0aEhLFAg
//this line is unused
//String views = li.select("span.view-count").first().text();
//Log.i(TAG, "title:"+info.title);
//Log.i(TAG, "view count:"+views);
try {
info.view_count = Long.parseLong(li.select("span.view-count")
.first().text().replaceAll("[^\\d]", ""));
} catch (NullPointerException e) {//related videos sometimes have no view count
info.view_count = 0;
}
info.uploader = li.select("span.g-hovercard").first().text();
info.duration = li.select("span.video-time").first().text();
Element img = li.select("img").first();
info.thumbnail_url = img.attr("abs:src");
// Sometimes youtube sends links to gif files which somehow seem to not exist
// anymore. Items with such gif also offer a secondary image source. So we are going
// to use that if we caught such an item.
if (info.thumbnail_url.contains(".gif")) {
info.thumbnail_url = img.attr("data-thumb");
}
if (info.thumbnail_url.startsWith("//")) {
info.thumbnail_url = "https:" + info.thumbnail_url;
}
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); throw new ParsingException(e);
}
//todo: check NullPointerException causing
info.title = li.select("span.title").first().text();
//this page causes the NullPointerException, after finding it by searching for "tjvg":
//https://www.youtube.com/watch?v=Uqg0aEhLFAg
//this line is unused
//String views = li.select("span.view-count").first().text();
//Log.i(TAG, "title:"+info.title);
//Log.i(TAG, "view count:"+views);
try {
info.view_count = Long.parseLong(li.select("span.view-count")
.first().text().replaceAll("[^\\d]", ""));
} catch (NullPointerException e) {//related videos sometimes have no view count
info.view_count = 0;
}
info.uploader = li.select("span.g-hovercard").first().text();
info.duration = li.select("span.video-time").first().text();
Element img = li.select("img").first();
info.thumbnail_url = img.attr("abs:src");
// Sometimes youtube sends links to gif files which somehow seem to not exist
// anymore. Items with such gif also offer a secondary image source. So we are going
// to use that if we caught such an item.
if(info.thumbnail_url.contains(".gif")) {
info.thumbnail_url = img.attr("data-thumb");
}
if(info.thumbnail_url.startsWith("//")) {
info.thumbnail_url = "https:" + info.thumbnail_url;
} }
return info; return info;
} }
private String loadDecryptionCode(String playerUrl) { private String loadDecryptionCode(String playerUrl) throws DecryptException {
String playerCode = downloader.download(playerUrl); String decryptionFuncName;
String decryptionFuncName = ""; String decryptionFunc;
String decryptionFunc = "";
String helperObjectName; String helperObjectName;
String helperObject = ""; String helperObject;
String callerFunc = "function " + DECRYPTION_FUNC_NAME + "(a){return %%(a);}"; String callerFunc = "function " + DECRYPTION_FUNC_NAME + "(a){return %%(a);}";
String decryptionCode; String decryptionCode;
try { try {
String playerCode = downloader.download(playerUrl);
decryptionFuncName = matchGroup1("\\.sig\\|\\|([a-zA-Z0-9$]+)\\(", playerCode); decryptionFuncName = matchGroup1("\\.sig\\|\\|([a-zA-Z0-9$]+)\\(", playerCode);
String functionPattern = "(" + decryptionFuncName.replace("$", "\\$") +"=function\\([a-zA-Z0-9_]*\\)\\{.+?\\})"; String functionPattern = "(" + decryptionFuncName.replace("$", "\\$") + "=function\\([a-zA-Z0-9_]*\\)\\{.+?\\})";
decryptionFunc = "var " + matchGroup1(functionPattern, playerCode) + ";"; decryptionFunc = "var " + matchGroup1(functionPattern, playerCode) + ";";
helperObjectName = matchGroup1(";([A-Za-z0-9_\\$]{2})\\...\\(", decryptionFunc); helperObjectName = matchGroup1(";([A-Za-z0-9_\\$]{2})\\...\\(", decryptionFunc);
@ -609,17 +601,20 @@ public class YoutubeVideoExtractor extends VideoExtractor {
String helperPattern = "(var " + helperObjectName.replace("$", "\\$") + "=\\{.+?\\}\\};)"; String helperPattern = "(var " + helperObjectName.replace("$", "\\$") + "=\\{.+?\\}\\};)";
helperObject = matchGroup1(helperPattern, playerCode); helperObject = matchGroup1(helperPattern, playerCode);
} catch (Exception e) {
e.printStackTrace();
}
callerFunc = callerFunc.replace("%%", decryptionFuncName); callerFunc = callerFunc.replace("%%", decryptionFuncName);
decryptionCode = helperObject + decryptionFunc + callerFunc; decryptionCode = helperObject + decryptionFunc + callerFunc;
} catch(IOException ioe) {
throw new DecryptException("Could not load decrypt function", ioe);
} catch(Exception e) {
throw new DecryptException("Could not parse decrypt function ", e);
}
return decryptionCode; return decryptionCode;
} }
private String decryptSignature(String encryptedSig, String decryptionCode) { private String decryptSignature(String encryptedSig, String decryptionCode)
throws DecryptException{
Context context = Context.enter(); Context context = Context.enter();
context.setOptimizationLevel(-1); context.setOptimizationLevel(-1);
Object result = null; Object result = null;
@ -629,17 +624,18 @@ public class YoutubeVideoExtractor extends VideoExtractor {
Function decryptionFunc = (Function) scope.get("decrypt", scope); Function decryptionFunc = (Function) scope.get("decrypt", scope);
result = decryptionFunc.call(context, scope, scope, new Object[]{encryptedSig}); result = decryptionFunc.call(context, scope, scope, new Object[]{encryptedSig});
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); throw new DecryptException(e);
} finally {
Context.exit();
} }
Context.exit();
return (result == null ? "" : result.toString()); return (result == null ? "" : result.toString());
} }
private String cleanUrl(String complexUrl) { private String cleanUrl(String complexUrl) throws ParsingException {
return getVideoUrl(getVideoId(complexUrl)); return getVideoUrl(getVideoId(complexUrl));
} }
private String matchGroup1(String pattern, String input) { private String matchGroup1(String pattern, String input) throws RegexException {
Pattern pat = Pattern.compile(pattern); Pattern pat = Pattern.compile(pattern);
Matcher mat = pat.matcher(input); Matcher mat = pat.matcher(input);
boolean foundMatch = mat.find(); boolean foundMatch = mat.find();
@ -647,17 +643,64 @@ public class YoutubeVideoExtractor extends VideoExtractor {
return mat.group(1); return mat.group(1);
} }
else { else {
Log.e(TAG, "failed to find pattern \""+pattern+"\" inside of \""+input+"\""); //Log.e(TAG, "failed to find pattern \""+pattern+"\" inside of \""+input+"\"");
new Exception("failed to find pattern \""+pattern+"\"").printStackTrace(); throw new RegexException("failed to find pattern \""+pattern+" inside of "+input+"\"");
return "";
} }
} }
private int findErrorReason(Document doc) { private String findErrorReason(Document doc) {
errorMessage = doc.select("h1[id=\"unavailable-message\"]").first().text(); errorMessage = doc.select("h1[id=\"unavailable-message\"]").first().text();
if(errorMessage.contains("GEMA")) { if(errorMessage.contains("GEMA")) {
return VideoInfo.ERROR_BLOCKED_BY_GEMA; // Gema sometimes blocks youtube music content in germany:
// https://www.gema.de/en/
// Detailed description:
// https://en.wikipedia.org/wiki/GEMA_%28German_organization%29
return "GEMA";
}
return "";
}
/**These lists only contain itag formats that are supported by the common Android Video player.
However if you are looking for a list showing all itag formats, look at
https://github.com/rg3/youtube-dl/issues/1687 */
@SuppressWarnings("WeakerAccess")
public static int resolveFormat(int itag) {
switch(itag) {
// !!! lists only supported formats !!!
// video
case 17: return MediaFormat.v3GPP.id;
case 18: return MediaFormat.MPEG_4.id;
case 22: return MediaFormat.MPEG_4.id;
case 36: return MediaFormat.v3GPP.id;
case 37: return MediaFormat.MPEG_4.id;
case 38: return MediaFormat.MPEG_4.id;
case 43: return MediaFormat.WEBM.id;
case 44: return MediaFormat.WEBM.id;
case 45: return MediaFormat.WEBM.id;
case 46: return MediaFormat.WEBM.id;
default:
//Log.i(TAG, "Itag " + Integer.toString(itag) + " not known or not supported.");
return -1;
}
}
@SuppressWarnings("WeakerAccess")
public static String resolveResolutionString(int itag) {
switch(itag) {
case 17: return "144p";
case 18: return "360p";
case 22: return "720p";
case 36: return "240p";
case 37: return "1080p";
case 38: return "1080p";
case 43: return "360p";
case 44: return "480p";
case 45: return "720p";
case 46: return "1080p";
default:
//Log.i(TAG, "Itag " + Integer.toString(itag) + " not known or not supported.");
return null;
} }
return VideoInfo.ERROR_NO_SPECIFIED_ERROR;
} }
} }

6
app/src/main/res/values/strings.xml
View File

@ -65,7 +65,13 @@
<string name="background_player_playing_toast">Playing in background</string> <string name="background_player_playing_toast">Playing in background</string>
<string name="c3s_url" translatable="false">https://www.c3s.cc/</string> <string name="c3s_url" translatable="false">https://www.c3s.cc/</string>
<string name="play_btn_text">Play</string> <string name="play_btn_text">Play</string>
<string name="general_error">Error</string>
<string name="network_error">Network error</string> <string name="network_error">Network error</string>
<string name="could_not_load_thumbnails">Could not load Thumbnails</string>
<string name="youtube_signature_decryption_error">Could not decrypt video url signature.</string>
<string name="parsing_error">Could not parse website.</string>
<string name="content_not_available">Content not available.</string>
<string name="blocked_by_gema">Blocked by GEMA.</string>
<!-- Content descriptions (for better accessibility) --> <!-- Content descriptions (for better accessibility) -->
<string name="list_thumbnail_view_description">Video preview thumbnail</string> <string name="list_thumbnail_view_description">Video preview thumbnail</string>