Merge remote-tracking branch 'origin/dev' into bandcamp

This commit is contained in:
TobiGr 2021-01-15 21:49:58 +01:00
commit 78c2113094
150 changed files with 4468 additions and 348 deletions

29
.github/workflows/ci.yml vendored Normal file
View File

@ -0,0 +1,29 @@
name: CI
on:
push:
branches:
- dev
- master
pull_request:
jobs:
build-and-test:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: set up JDK 1.8
uses: actions/setup-java@v1.4.3
with:
java-version: 1.8
- name: Cache Gradle dependencies
uses: actions/cache@v2
with:
path: ~/.gradle/caches
key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }}
restore-keys: ${{ runner.os }}-gradle
- name: Build and run Tests
run: ./gradlew check --stacktrace -Ddownloader=MOCK

33
.github/workflows/docs.yml vendored Normal file
View File

@ -0,0 +1,33 @@
name: Build and deploy JavaDocs
on:
push:
branches:
- master
jobs:
build-and-deploy-docs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: set up JDK 1.8
uses: actions/setup-java@v1.4.3
with:
java-version: 1.8
- name: Cache Gradle dependencies
uses: actions/cache@v2
with:
path: ~/.gradle/caches
key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }}
restore-keys: ${{ runner.os }}-gradle
- name: Build JavaDocs
run: ./gradlew aggregatedJavadocs
- name: Deploy JavaDocs
uses: peaceiris/actions-gh-pages@v3
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
publish_dir: ./build/docs

View File

@ -1,14 +0,0 @@
language: java
jdk:
- openjdk8
script:
- ./gradlew check aggregatedJavadocs
deploy:
provider: pages
skip_cleanup: true
github_token: $GITHUB_TOKEN # Set in travis-ci.org dashboard
local_dir: build/docs
on:
branch: master

View File

@ -1,6 +1,6 @@
# NewPipe Extractor
[![Build Status](https://travis-ci.org/TeamNewPipe/NewPipeExtractor.svg?branch=master)](https://travis-ci.org/TeamNewPipe/NewPipeExtractor) [![JIT Pack Badge](https://jitpack.io/v/TeamNewPipe/NewPipeExtractor.svg)](https://jitpack.io/#TeamNewPipe/NewPipeExtractor) [JDoc](https://teamnewpipe.github.io/NewPipeExtractor/javadoc/) • [Documentation](https://teamnewpipe.github.io/documentation/)
[![Build Status](https://github.com/TeamNewPipe/NewPipeExtractor/workflows/CI/badge.svg?branch=dev&event=push)](https://github.com/TeamNewPipe/NewPipeExtractor/actions) [![JIT Pack Badge](https://jitpack.io/v/TeamNewPipe/NewPipeExtractor.svg)](https://jitpack.io/#TeamNewPipe/NewPipeExtractor) [JDoc](https://teamnewpipe.github.io/NewPipeExtractor/javadoc/) • [Documentation](https://teamnewpipe.github.io/documentation/)
NewPipe Extractor is a library for extracting things from streaming sites. It is a core component of [NewPipe](https://github.com/TeamNewPipe/NewPipe), but could be used independently.
@ -11,7 +11,7 @@ NewPipe Extractor is available at JitPack's Maven repo.
If you're using Gradle, you could add NewPipe Extractor as a dependency with the following steps:
1. Add `maven { url 'https://jitpack.io' }` to the `repositories` in your `build.gradle`.
2. Add `implementation 'com.github.TeamNewPipe:NewPipeExtractor:v0.20.6'`the `dependencies` in your `build.gradle`. Replace `v0.20.6` with the latest release.
2. Add `implementation 'com.github.TeamNewPipe:NewPipeExtractor:v0.20.8'`the `dependencies` in your `build.gradle`. Replace `v0.20.8` with the latest release.
**Note:** To use NewPipe Extractor in projects with a `minSdkVersion` below 26, [API desugaring](https://developer.android.com/studio/write/java8-support#library-desugaring) is required.

View File

@ -5,7 +5,7 @@ allprojects {
sourceCompatibility = 1.8
targetCompatibility = 1.8
version 'v0.20.5'
version 'v0.20.8'
group 'com.github.TeamNewPipe'
repositories {
@ -43,7 +43,7 @@ task aggregatedJavadocs(type: Javadoc, group: 'Documentation') {
destinationDir = file("$buildDir/docs/javadoc")
title = "$project.name $version"
// options.memberLevel = JavadocMemberLevel.PRIVATE
options.links 'https://docs.oracle.com/javase/7/docs/api/'
options.links 'https://docs.oracle.com/javase/8/docs/api/'
options.encoding 'UTF-8'
subprojects.each { project ->

View File

@ -1,3 +1,10 @@
test {
// Pass on downloader type to tests for different CI jobs. See DownloaderFactory.java and ci.yml
if (System.properties.containsKey('downloader')) {
systemProperty('downloader', System.getProperty('downloader'))
}
}
dependencies {
implementation project(':timeago-parser')
@ -7,6 +14,7 @@ dependencies {
implementation 'com.github.spotbugs:spotbugs-annotations:4.0.2'
implementation 'org.nibor.autolink:autolink:0.10.0'
testImplementation 'junit:junit:4.13'
testImplementation 'junit:junit:4.13.1'
testImplementation "com.squareup.okhttp3:okhttp:3.12.11"
testImplementation 'com.google.code.gson:gson:2.8.6'
}

View File

@ -3,8 +3,10 @@ package org.schabi.newpipe.extractor;
import org.schabi.newpipe.extractor.exceptions.FoundAdException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import javax.annotation.Nullable;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
/*
@ -32,17 +34,31 @@ public abstract class InfoItemsCollector<I extends InfoItem, E extends InfoItemE
private final List<I> itemList = new ArrayList<>();
private final List<Throwable> errors = new ArrayList<>();
private final int serviceId;
@Nullable
private final Comparator<I> comparator;
/**
* Create a new collector with no comparator / sorting function
* @param serviceId the service id
*/
public InfoItemsCollector(final int serviceId) {
this(serviceId, null);
}
/**
* Create a new collector
* @param serviceId the service id
*/
public InfoItemsCollector(int serviceId) {
public InfoItemsCollector(final int serviceId, @Nullable final Comparator<I> comparator) {
this.serviceId = serviceId;
this.comparator = comparator;
}
@Override
public List<I> getItems() {
if (comparator != null) {
itemList.sort(comparator);
}
return Collections.unmodifiableList(itemList);
}

View File

@ -0,0 +1,76 @@
package org.schabi.newpipe.extractor;
import org.schabi.newpipe.extractor.stream.Description;
import java.io.Serializable;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import javax.annotation.Nonnull;
public class MetaInfo implements Serializable {
private String title = "";
private Description content;
private List<URL> urls = new ArrayList<>();
private List<String> urlTexts = new ArrayList<>();
public MetaInfo(@Nonnull final String title, @Nonnull final Description content,
@Nonnull final List<URL> urls, @Nonnull final List<String> urlTexts) {
this.title = title;
this.content = content;
this.urls = urls;
this.urlTexts = urlTexts;
}
public MetaInfo() {
}
/**
* @return Title of the info. Can be empty.
*/
@Nonnull
public String getTitle() {
return title;
}
public void setTitle(@Nonnull final String title) {
this.title = title;
}
@Nonnull
public Description getContent() {
return content;
}
public void setContent(@Nonnull final Description content) {
this.content = content;
}
@Nonnull
public List<URL> getUrls() {
return urls;
}
public void setUrls(@Nonnull final List<URL> urls) {
this.urls = urls;
}
public void addUrl(@Nonnull final URL url) {
urls.add(url);
}
@Nonnull
public List<String> getUrlTexts() {
return urlTexts;
}
public void setUrlTexts(@Nonnull final List<String> urlTexts) {
this.urlTexts = urlTexts;
}
public void addUrlText(@Nonnull final String urlText) {
urlTexts.add(urlText);
}
}

View File

@ -16,6 +16,7 @@ public class CommentsInfoItem extends InfoItem {
@Nullable
private DateWrapper uploadDate;
private int likeCount;
private boolean heartedByUploader;
public CommentsInfoItem(int serviceId, String url, String name) {
super(InfoType.COMMENT, serviceId, url, name);
@ -85,4 +86,12 @@ public class CommentsInfoItem extends InfoItem {
public void setLikeCount(int likeCount) {
this.likeCount = likeCount;
}
public void setHeartedByUploader(boolean isHeartedByUploader) {
this.heartedByUploader = isHeartedByUploader;
}
public boolean getHeartedByUploader() {
return this.heartedByUploader;
}
}

View File

@ -40,4 +40,9 @@ public interface CommentsInfoItemExtractor extends InfoItemExtractor {
String getUploaderName() throws ParsingException;
String getUploaderAvatarUrl() throws ParsingException;
/**
* Whether the comment has been hearted by the uploader
*/
boolean getHeartedByUploader() throws ParsingException;
}

View File

@ -70,6 +70,12 @@ public class CommentsInfoItemsCollector extends InfoItemsCollector<CommentsInfoI
addError(e);
}
try {
resultItem.setHeartedByUploader(extractor.getHeartedByUploader());
} catch (Exception e) {
addError(e);
}
return resultItem;
}

View File

@ -241,4 +241,27 @@ public class Request {
return headers;
}
/*//////////////////////////////////////////////////////////////////////////
// Generated
//////////////////////////////////////////////////////////////////////////*/
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
Request request = (Request) o;
return httpMethod.equals(request.httpMethod) &&
url.equals(request.url) &&
headers.equals(request.headers) &&
Arrays.equals(dataToSend, request.dataToSend) &&
Objects.equals(localization, request.localization);
}
@Override
public int hashCode() {
int result = Objects.hash(httpMethod, url, headers, localization);
result = 31 * result + Arrays.hashCode(dataToSend);
return result;
}
}

View File

@ -45,11 +45,11 @@ public abstract class KioskExtractor<T extends InfoItem> extends ListExtractor<T
}
/**
* Id should be the name of the kiosk, tho Id is used for identifing it in the frontend,
* Id should be the name of the kiosk, tho Id is used for identifying it in the frontend,
* so id should be kept in english.
* In order to get the name of the kiosk in the desired language we have to
* crawl if from the website.
* @return the tranlsated version of id
* @return the translated version of id
* @throws ParsingException
*/
@Nonnull

View File

@ -74,7 +74,7 @@ public abstract class ListLinkHandlerFactory extends LinkHandlerFactory {
* however it should not be overridden by the actual implementation.
*
* @param id
* @return the url coresponding to id without any filters applied
* @return the url corresponding to id without any filters applied
*/
public String getUrl(String id) throws ParsingException {
return getUrl(id, new ArrayList<String>(0), "");

View File

@ -42,7 +42,6 @@ public abstract class SearchQueryHandlerFactory extends ListLinkHandlerFactory {
* It's not mandatory for NewPipe to handle the Url
*
* @param url
* @return
*/
@Override
public boolean onAcceptUrl(String url) {

View File

@ -2,12 +2,14 @@ package org.schabi.newpipe.extractor.search;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.MetaInfo;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
import javax.annotation.Nonnull;
import java.util.List;
public abstract class SearchExtractor extends ListExtractor<InfoItem> {
@ -57,4 +59,15 @@ public abstract class SearchExtractor extends ListExtractor<InfoItem> {
* @return whether the results comes from a corrected query or not.
*/
public abstract boolean isCorrectedSearch() throws ParsingException;
/**
* Meta information about the search query.
* <p>
* Example: on YouTube, if you search for "Covid-19",
* there is a box with information from the WHO about Covid-19 and a link to the WHO's website.
* @return additional meta information about the search query
* @throws ParsingException
*/
@Nonnull
public abstract List<MetaInfo> getMetaInfo() throws ParsingException;
}

View File

@ -1,20 +1,20 @@
package org.schabi.newpipe.extractor.search;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.ListInfo;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.*;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
import org.schabi.newpipe.extractor.utils.ExtractorHelper;
import java.io.IOException;
import java.util.List;
import javax.annotation.Nonnull;
public class SearchInfo extends ListInfo<InfoItem> {
private String searchString;
private String searchSuggestion;
private boolean isCorrectedSearch;
private List<MetaInfo> metaInfo;
public SearchInfo(int serviceId,
SearchQueryHandler qIHandler,
@ -51,6 +51,11 @@ public class SearchInfo extends ListInfo<InfoItem> {
} catch (Exception e) {
info.addError(e);
}
try {
info.setMetaInfo(extractor.getMetaInfo());
} catch (Exception e) {
info.addError(e);
}
ListExtractor.InfoItemsPage<InfoItem> page = ExtractorHelper.getItemsPageOrLogError(info, extractor);
info.setRelatedItems(page.getItems());
@ -87,4 +92,13 @@ public class SearchInfo extends ListInfo<InfoItem> {
public void setSearchSuggestion(String searchSuggestion) {
this.searchSuggestion = searchSuggestion;
}
@Nonnull
public List<MetaInfo> getMetaInfo() {
return metaInfo;
}
public void setMetaInfo(@Nonnull List<MetaInfo> metaInfo) {
this.metaInfo = metaInfo;
}
}

View File

@ -14,14 +14,8 @@ import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCConferenceExtractor;
import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCConferenceKiosk;
import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCSearchExtractor;
import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCStreamExtractor;
import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCConferenceLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCConferencesListLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCSearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCStreamLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.media_ccc.extractors.*;
import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.*;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
@ -62,6 +56,9 @@ public class MediaCCCService extends StreamingService {
@Override
public StreamExtractor getStreamExtractor(final LinkHandler linkHandler) {
if (MediaCCCParsingHelper.isLiveStreamId(linkHandler.getId())) {
return new MediaCCCLiveStreamExtractor(this, linkHandler);
}
return new MediaCCCStreamExtractor(this, linkHandler);
}
@ -95,7 +92,28 @@ public class MediaCCCService extends StreamingService {
new MediaCCCConferencesListLinkHandlerFactory().fromUrl(url), kioskId);
}
}, new MediaCCCConferencesListLinkHandlerFactory(), "conferences");
list.setDefaultKiosk("conferences");
list.addKioskEntry(new KioskList.KioskExtractorFactory() {
@Override
public KioskExtractor createNewKiosk(final StreamingService streamingService,
final String url, final String kioskId)
throws ExtractionException {
return new MediaCCCRecentKiosk(MediaCCCService.this,
new MediaCCCRecentListLinkHandlerFactory().fromUrl(url), kioskId);
}
}, new MediaCCCRecentListLinkHandlerFactory(), "recent");
list.addKioskEntry(new KioskList.KioskExtractorFactory() {
@Override
public KioskExtractor createNewKiosk(final StreamingService streamingService,
final String url, final String kioskId)
throws ExtractionException {
return new MediaCCCLiveStreamKiosk(MediaCCCService.this,
new MediaCCCLiveListLinkHandlerFactory().fromUrl(url), kioskId);
}
}, new MediaCCCLiveListLinkHandlerFactory(), "live");
list.setDefaultKiosk("recent");
} catch (Exception e) {
throw new ExtractionException(e);
}

View File

@ -0,0 +1,300 @@
package org.schabi.newpipe.extractor.services.media_ccc.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.MetaInfo;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.stream.*;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
public class MediaCCCLiveStreamExtractor extends StreamExtractor {
private JsonArray doc = null;
private JsonObject conference = null;
private String group = "";
private JsonObject room = null;
public MediaCCCLiveStreamExtractor(StreamingService service, LinkHandler linkHandler) {
super(service, linkHandler);
}
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
doc = MediaCCCParsingHelper.getLiveStreams(downloader, getExtractorLocalization());
// find correct room
for (int c = 0; c < doc.size(); c++) {
final JsonObject conference = doc.getObject(c);
final JsonArray groups = conference.getArray("groups");
for (int g = 0; g < groups.size(); g++) {
final String group = groups.getObject(g).getString("group");
final JsonArray rooms = groups.getObject(g).getArray("rooms");
for (int r = 0; r < rooms.size(); r++) {
final JsonObject room = rooms.getObject(r);
if (getId().equals(conference.getString("slug") + "/" + room.getString("slug"))) {
this.conference = conference;
this.group = group;
this.room = room;
return;
}
}
}
}
throw new ExtractionException("Could not find room matching id: '" + getId() + "'");
}
@Nonnull
@Override
public String getName() throws ParsingException {
return room.getString("display");
}
@Nullable
@Override
public String getTextualUploadDate() throws ParsingException {
return null;
}
@Nullable
@Override
public DateWrapper getUploadDate() throws ParsingException {
return null;
}
@Nonnull
@Override
public String getThumbnailUrl() throws ParsingException {
return room.getString("thumb");
}
@Nonnull
@Override
public Description getDescription() throws ParsingException {
return new Description(conference.getString("description") + " - " + group, Description.PLAIN_TEXT);
}
@Override
public int getAgeLimit() {
return 0;
}
@Override
public long getLength() {
return 0;
}
@Override
public long getTimeStamp() throws ParsingException {
return 0;
}
@Override
public long getViewCount() {
return -1;
}
@Override
public long getLikeCount() {
return -1;
}
@Override
public long getDislikeCount() {
return -1;
}
@Nonnull
@Override
public String getUploaderUrl() throws ParsingException {
return "https://streaming.media.ccc.de/" + conference.getString("slug");
}
@Nonnull
@Override
public String getUploaderName() throws ParsingException {
return conference.getString("conference");
}
@Nonnull
@Override
public String getUploaderAvatarUrl() {
return "";
}
@Nonnull
@Override
public String getSubChannelUrl() {
return "";
}
@Nonnull
@Override
public String getSubChannelName() {
return "";
}
@Nonnull
@Override
public String getSubChannelAvatarUrl() {
return "";
}
@Nonnull
@Override
public String getDashMpdUrl() throws ParsingException {
return "";
}
@Nonnull
@Override
public String getHlsUrl() {
// TODO: There are multiple HLS streams.
// Make getHlsUrl() and getDashMpdUrl() return lists of VideoStreams, so the user can choose a resolution.
for (int s = 0; s < room.getArray("streams").size(); s++) {
final JsonObject stream = room.getArray("streams").getObject(s);
if (stream.getString("type").equals("video")) {
final String resolution = stream.getArray("videoSize").getInt(0) + "x"
+ stream.getArray("videoSize").getInt(1);
if (stream.has("hls")) {
return stream.getObject("urls").getObject("hls").getString("url");
}
}
}
return "";
}
@Override
public List<AudioStream> getAudioStreams() throws IOException, ExtractionException {
final List<AudioStream> audioStreams = new ArrayList<>();
for (int s = 0; s < room.getArray("streams").size(); s++) {
final JsonObject stream = room.getArray("streams").getObject(s);
if (stream.getString("type").equals("audio")) {
for (final String type :stream.getObject("urls").keySet()) {
final JsonObject url = stream.getObject("urls").getObject(type);
audioStreams.add(new AudioStream(url.getString("url"), MediaFormat.getFromSuffix(type), -1));
}
}
}
return audioStreams;
}
@Override
public List<VideoStream> getVideoStreams() throws IOException, ExtractionException {
final List<VideoStream> videoStreams = new ArrayList<>();
for (int s = 0; s < room.getArray("streams").size(); s++) {
final JsonObject stream = room.getArray("streams").getObject(s);
if (stream.getString("type").equals("video")) {
final String resolution = stream.getArray("videoSize").getInt(0) + "x"
+ stream.getArray("videoSize").getInt(1);
for (final String type :stream.getObject("urls").keySet()) {
if (!type.equals("hls")) {
final JsonObject url = stream.getObject("urls").getObject(type);
videoStreams.add(new VideoStream(
url.getString("url"),
MediaFormat.getFromSuffix(type),
resolution));
}
}
}
}
return videoStreams;
}
@Override
public List<VideoStream> getVideoOnlyStreams() throws IOException, ExtractionException {
return null;
}
@Nonnull
@Override
public List<SubtitlesStream> getSubtitlesDefault(){
return Collections.emptyList();
}
@Nonnull
@Override
public List<SubtitlesStream> getSubtitles(MediaFormat format) {
return Collections.emptyList();
}
@Override
public StreamType getStreamType() throws ParsingException {
return StreamType.LIVE_STREAM; // TODO: video and audio only streams are both available
}
@Nullable
@Override
public StreamInfoItemsCollector getRelatedStreams() {
return null;
}
@Override
public String getErrorMessage() {
return null;
}
@Nonnull
@Override
public String getHost() throws ParsingException {
return null;
}
@Nonnull
@Override
public String getPrivacy() {
return "Public";
}
@Nonnull
@Override
public String getCategory() {
return group;
}
@Nonnull
@Override
public String getLicence() {
return "";
}
@Nullable
@Override
public Locale getLanguageInfo() {
return null;
}
@Nonnull
@Override
public List<String> getTags() {
return Collections.emptyList();
}
@Nonnull
@Override
public String getSupportInfo() {
return "";
}
@Nonnull
@Override
public List<StreamSegment> getStreamSegments() {
return Collections.emptyList();
}
@Nonnull
@Override
public List<MetaInfo> getMetaInfo() {
return Collections.emptyList();
}
}

View File

@ -0,0 +1,62 @@
package org.schabi.newpipe.extractor.services.media_ccc.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import javax.annotation.Nonnull;
import java.io.IOException;
public class MediaCCCLiveStreamKiosk extends KioskExtractor<StreamInfoItem> {
public JsonArray doc;
public MediaCCCLiveStreamKiosk(StreamingService streamingService, ListLinkHandler linkHandler, String kioskId) {
super(streamingService, linkHandler, kioskId);
}
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
doc = MediaCCCParsingHelper.getLiveStreams(downloader, getExtractorLocalization());
}
@Nonnull
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
for (int c = 0; c < doc.size(); c++) {
final JsonObject conference = doc.getObject(c);
final JsonArray groups = conference.getArray("groups");
for (int g = 0; g < groups.size(); g++) {
final String group = groups.getObject(g).getString("group");
final JsonArray rooms = groups.getObject(g).getArray("rooms");
for (int r = 0; r < rooms.size(); r++) {
final JsonObject room = rooms.getObject(r);
collector.commit(new MediaCCCLiveStreamKioskExtractor(conference, group, room));
}
}
}
return new InfoItemsPage<>(collector, null);
}
@Override
public InfoItemsPage<StreamInfoItem> getPage(Page page) throws IOException, ExtractionException {
return InfoItemsPage.emptyPage();
}
@Nonnull
@Override
public String getName() throws ParsingException {
return "live";
}
}

View File

@ -0,0 +1,87 @@
package org.schabi.newpipe.extractor.services.media_ccc.extractors;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamType;
import javax.annotation.Nullable;
public class MediaCCCLiveStreamKioskExtractor implements StreamInfoItemExtractor {
private final JsonObject conferenceInfo;
private final String group;
private final JsonObject roomInfo;
public MediaCCCLiveStreamKioskExtractor(final JsonObject conferenceInfo, final String group,
final JsonObject roomInfo) {
this.conferenceInfo = conferenceInfo;
this.group = group;
this.roomInfo = roomInfo;
}
@Override
public String getName() throws ParsingException {
return roomInfo.getObject("talks").getObject("current").getString("title");
}
@Override
public String getUrl() throws ParsingException {
return roomInfo.getString("link");
}
@Override
public String getThumbnailUrl() throws ParsingException {
return roomInfo.getString("thumb");
}
@Override
public StreamType getStreamType() throws ParsingException {
boolean isVideo = false;
for (Object stream : roomInfo.getArray("streams")) {
if ("video".equals(((JsonObject) stream).getString("type"))) {
isVideo = true;
break;
}
}
return isVideo ? StreamType.LIVE_STREAM : StreamType.AUDIO_LIVE_STREAM;
}
@Override
public boolean isAd() throws ParsingException {
return false;
}
@Override
public long getDuration() throws ParsingException {
return 0;
}
@Override
public long getViewCount() throws ParsingException {
return -1;
}
@Override
public String getUploaderName() throws ParsingException {
return conferenceInfo.getString("conference") + " - " + group + " - " + roomInfo.getString("display");
}
@Override
public String getUploaderUrl() throws ParsingException {
return "https://media.ccc.de/c/" + conferenceInfo.getString("slug");
}
@Nullable
@Override
public String getTextualUploadDate() throws ParsingException {
return null;
}
@Nullable
@Override
public DateWrapper getUploadDate() throws ParsingException {
return null;
}
}

View File

@ -1,11 +1,24 @@
package org.schabi.newpipe.extractor.services.media_ccc.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.localization.Localization;
import java.io.IOException;
import java.time.OffsetDateTime;
import java.time.format.DateTimeParseException;
import java.util.Locale;
import java.util.regex.Pattern;
public final class MediaCCCParsingHelper {
private static final Pattern LIVE_STREAM_ID_PATTERN = Pattern.compile("\\w+/\\w+"); // {conference_slug}/{room_slug}
private static JsonArray liveStreams = null;
private MediaCCCParsingHelper() { }
public static OffsetDateTime parseDateFrom(final String textualUploadDate) throws ParsingException {
@ -15,4 +28,40 @@ public final class MediaCCCParsingHelper {
throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e);
}
}
/**
* Check whether an id is a live stream id
* @param id the {@code id} to check
* @return returns {@code true} if the {@code id} is formatted like {@code {conference_slug}/{room_slug}};
* {@code false} otherwise
*/
public static boolean isLiveStreamId(final String id) {
return LIVE_STREAM_ID_PATTERN.matcher(id).find();
}
/**
* Get currently available live streams from
* <a href="https://streaming.media.ccc.de/streams/v2.json">https://streaming.media.ccc.de/streams/v2.json</a>.
* Use this method to cache requests, because they can get quite big.
* TODO: implement better caching policy (max-age: 3 min)
* @param downloader The downloader to use for making the request
* @param localization The localization to be used. Will most likely be ignored.
* @return {@link JsonArray} containing current conferences and info about their rooms and streams.
* @throws ExtractionException if the data could not be fetched or the retrieved data could not be parsed to a {@link JsonArray}
*/
public static JsonArray getLiveStreams(final Downloader downloader, final Localization localization)
throws ExtractionException {
if (liveStreams == null) {
try {
final String site = downloader.get("https://streaming.media.ccc.de/streams/v2.json",
localization).responseBody();
liveStreams = JsonParser.array().from(site);
} catch (IOException | ReCaptchaException e) {
throw new ExtractionException("Could not get live stream JSON.", e);
} catch (JsonParserException e) {
throw new ExtractionException("Could not parse JSON.", e);
}
}
return liveStreams;
}
}

View File

@ -0,0 +1,68 @@
package org.schabi.newpipe.extractor.services.media_ccc.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.util.Comparator;
public class MediaCCCRecentKiosk extends KioskExtractor<StreamInfoItem> {
private JsonObject doc;
public MediaCCCRecentKiosk(StreamingService streamingService, ListLinkHandler linkHandler, String kioskId) {
super(streamingService, linkHandler, kioskId);
}
@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
final String site = downloader.get("https://api.media.ccc.de/public/events/recent",
getExtractorLocalization()).responseBody();
try {
doc = JsonParser.object().from(site);
} catch (JsonParserException jpe) {
throw new ExtractionException("Could not parse json.", jpe);
}
}
@Nonnull
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
final JsonArray events = doc.getArray("events");
// Streams in the recent kiosk are not ordered by the release date.
// Sort them to have the latest stream at the beginning of the list.
Comparator<StreamInfoItem> comparator = Comparator.comparing(
streamInfoItem -> streamInfoItem.getUploadDate().offsetDateTime());
comparator = comparator.reversed();
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId(), comparator);
for (int i = 0; i < events.size(); i++) {
collector.commit(new MediaCCCRecentKioskExtractor(events.getObject(i)));
}
return new InfoItemsPage<>(collector, null);
}
@Override
public InfoItemsPage<StreamInfoItem> getPage(Page page) throws IOException, ExtractionException {
return InfoItemsPage.emptyPage();
}
@Nonnull
@Override
public String getName() throws ParsingException {
return "recent";
}
}

View File

@ -0,0 +1,84 @@
package org.schabi.newpipe.extractor.services.media_ccc.extractors;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCConferenceLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamType;
import javax.annotation.Nullable;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
public class MediaCCCRecentKioskExtractor implements StreamInfoItemExtractor {
private final JsonObject event;
public MediaCCCRecentKioskExtractor(final JsonObject event) {
this.event = event;
}
@Override
public String getName() throws ParsingException {
return event.getString("title");
}
@Override
public String getUrl() throws ParsingException {
return event.getString("frontend_link");
}
@Override
public String getThumbnailUrl() throws ParsingException {
return event.getString("thumb_url");
}
@Override
public StreamType getStreamType() throws ParsingException {
return StreamType.VIDEO_STREAM;
}
@Override
public boolean isAd() {
return false;
}
@Override
public long getDuration() throws ParsingException {
// duration and length have the same value
// see https://github.com/voc/voctoweb/blob/master/app/views/public/shared/_event.json.jbuilder
return event.getInt("duration");
}
@Override
public long getViewCount() throws ParsingException {
return event.getInt("view_count");
}
@Override
public String getUploaderName() throws ParsingException {
return event.getString("conference_title");
}
@Override
public String getUploaderUrl() throws ParsingException {
return new MediaCCCConferenceLinkHandlerFactory()
.fromUrl(event.getString("conference_url")) // API URL
.getUrl(); // web URL
}
@Nullable
@Override
public String getTextualUploadDate() throws ParsingException {
return event.getString("date");
}
@Nullable
@Override
public DateWrapper getUploadDate() throws ParsingException {
final ZonedDateTime zonedDateTime = ZonedDateTime.parse(event.getString("date"),
DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSzzzz"));
return new DateWrapper(zonedDateTime.toOffsetDateTime(), false);
}
}

View File

@ -6,6 +6,7 @@ import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.MetaInfo;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelInfoItem;
@ -20,6 +21,7 @@ import org.schabi.newpipe.extractor.services.media_ccc.extractors.infoItems.Medi
import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCConferencesListLinkHandlerFactory;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import javax.annotation.Nonnull;
@ -55,6 +57,12 @@ public class MediaCCCSearchExtractor extends SearchExtractor {
return false;
}
@Nonnull
@Override
public List<MetaInfo> getMetaInfo() {
return Collections.emptyList();
}
@Nonnull
@Override
public InfoItemsPage<InfoItem> getInitialPage() {
@ -73,8 +81,13 @@ public class MediaCCCSearchExtractor extends SearchExtractor {
|| getLinkHandler().getContentFilters().isEmpty()) {
JsonArray events = doc.getArray("events");
for (int i = 0; i < events.size(); i++) {
searchItems.commit(new MediaCCCStreamInfoItemExtractor(
events.getObject(i)));
// Ensure only uploaded talks are shown in the search results.
// If the release date is null, the talk has not been held or uploaded yet
// and no streams are going to be available anyway.
if (events.getObject(i).getString("release_date") != null) {
searchItems.commit(new MediaCCCStreamInfoItemExtractor(
events.getObject(i)));
}
}
}
return new InfoItemsPage<>(searchItems, null);
@ -97,7 +110,7 @@ public class MediaCCCSearchExtractor extends SearchExtractor {
try {
doc = JsonParser.object().from(site);
} catch (JsonParserException jpe) {
throw new ExtractionException("Could not parse json.", jpe);
throw new ExtractionException("Could not parse JSON.", jpe);
}
}
if (getLinkHandler().getContentFilters().contains(CONFERENCES)

View File

@ -6,6 +6,7 @@ import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.MetaInfo;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
@ -16,6 +17,7 @@ import org.schabi.newpipe.extractor.stream.AudioStream;
import org.schabi.newpipe.extractor.stream.Description;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamSegment;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.stream.SubtitlesStream;
import org.schabi.newpipe.extractor.stream.VideoStream;
@ -294,4 +296,16 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
public String getSupportInfo() {
return "";
}
@Nonnull
@Override
public List<StreamSegment> getStreamSegments() {
return Collections.emptyList();
}
@Nonnull
@Override
public List<MetaInfo> getMetaInfo() {
return Collections.emptyList();
}
}

View File

@ -7,7 +7,7 @@ import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
public class MediaCCCConferenceInfoItemExtractor implements ChannelInfoItemExtractor {
private JsonObject conference;
private final JsonObject conference;
public MediaCCCConferenceInfoItemExtractor(final JsonObject conference) {
this.conference = conference;

View File

@ -38,8 +38,7 @@ public class MediaCCCStreamInfoItemExtractor implements StreamInfoItemExtractor
@Override
public String getUploaderName() {
return event.getString("conference_url")
.replaceFirst("https://(api\\.)?media\\.ccc\\.de/public/conferences/", "");
return event.getString("conference_title");
}
@Override
@ -56,7 +55,11 @@ public class MediaCCCStreamInfoItemExtractor implements StreamInfoItemExtractor
@Nullable
@Override
public DateWrapper getUploadDate() throws ParsingException {
return new DateWrapper(MediaCCCParsingHelper.parseDateFrom(getTextualUploadDate()));
final String date = getTextualUploadDate();
if (date == null) {
return null; // event is in the future...
}
return new DateWrapper(MediaCCCParsingHelper.parseDateFrom(date));
}
@Override

View File

@ -0,0 +1,27 @@
package org.schabi.newpipe.extractor.services.media_ccc.linkHandler;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import java.util.List;
import java.util.regex.Pattern;
public class MediaCCCLiveListLinkHandlerFactory extends ListLinkHandlerFactory {
private static final String streamPattern = "^(?:https?://)?media\\.ccc\\.de/live$";
@Override
public String getId(String url) throws ParsingException {
return "live";
}
@Override
public boolean onAcceptUrl(String url) throws ParsingException {
return Pattern.matches(streamPattern, url);
}
@Override
public String getUrl(String id, List<String> contentFilter, String sortFilter) throws ParsingException {
// FIXME: wrong URL; should be https://streaming.media.ccc.de/{conference_slug}/{room_slug}
return "https://media.ccc.de/live";
}
}

View File

@ -0,0 +1,30 @@
package org.schabi.newpipe.extractor.services.media_ccc.linkHandler;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
import org.schabi.newpipe.extractor.utils.Parser;
public class MediaCCCLiveStreamLinkHandlerFactory extends LinkHandlerFactory {
public static final String VIDEO_API_ENDPOINT = "https://api.media.ccc.de/public/events/";
private static final String VIDEO_PATH = "https://streaming.media.ccc.de/v/";
private static final String ID_PATTERN = "(?:(?:(?:api\\.)?media\\.ccc\\.de/public/events/)|(?:media\\.ccc\\.de/v/))([^/?&#]*)";
@Override
public String getId(final String url) throws ParsingException {
return Parser.matchGroup1(ID_PATTERN, url);
}
@Override
public String getUrl(final String id) throws ParsingException {
return VIDEO_PATH + id;
}
@Override
public boolean onAcceptUrl(final String url) {
try {
return getId(url) != null;
} catch (ParsingException e) {
return false;
}
}
}

View File

@ -0,0 +1,25 @@
package org.schabi.newpipe.extractor.services.media_ccc.linkHandler;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
import java.util.List;
import java.util.regex.Pattern;
public class MediaCCCRecentListLinkHandlerFactory extends ListLinkHandlerFactory {
private static final String pattern = "^(https?://)?media\\.ccc\\.de/recent/?$";
@Override
public String getId(String url) {
return "recent";
}
@Override
public boolean onAcceptUrl(String url) {
return Pattern.matches(pattern, url);
}
@Override
public String getUrl(String id, List<String> contentFilter, String sortFilter) {
return "https://media.ccc.de/recent";
}
}

View File

@ -33,7 +33,7 @@ public class MediaCCCSearchQueryHandlerFactory extends SearchQueryHandlerFactory
return "https://media.ccc.de/public/events/search?q="
+ URLEncoder.encode(query, "UTF-8");
} catch (UnsupportedEncodingException e) {
throw new ParsingException("Could not create search string with querry: " + query, e);
throw new ParsingException("Could not create search string with query: " + query, e);
}
}
}

View File

@ -2,20 +2,36 @@ package org.schabi.newpipe.extractor.services.media_ccc.linkHandler;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCParsingHelper;
import org.schabi.newpipe.extractor.utils.Parser;
public class MediaCCCStreamLinkHandlerFactory extends LinkHandlerFactory {
public static final String VIDEO_API_ENDPOINT = "https://api.media.ccc.de/public/events/";
private static final String VIDEO_PATH = "https://media.ccc.de/v/";
private static final String ID_PATTERN = "(?:(?:(?:api\\.)?media\\.ccc\\.de/public/events/)|(?:media\\.ccc\\.de/v/))([^/?&#]*)";
private static final String RECORDING_ID_PATTERN = "(?:(?:(?:api\\.)?media\\.ccc\\.de/public/events/)|(?:media\\.ccc\\.de/v/))([^/?&#]*)";
private static final String LIVE_STREAM_API_ENDPOINT = "https://streaming.media.ccc.de/streams/v2.json";
private static final String LIVE_STREAM_PATH = "https://streaming.media.ccc.de/";
private static final String LIVE_STREAM_ID_PATTERN = "streaming\\.media\\.ccc\\.de\\/(\\w+\\/\\w+)";
@Override
public String getId(final String url) throws ParsingException {
return Parser.matchGroup1(ID_PATTERN, url);
String streamId = null;
try {
streamId = Parser.matchGroup1(LIVE_STREAM_ID_PATTERN, url);
} catch (Parser.RegexException ignored) {
}
if (streamId == null) {
return Parser.matchGroup1(RECORDING_ID_PATTERN, url);
}
return streamId;
}
@Override
public String getUrl(final String id) throws ParsingException {
if (MediaCCCParsingHelper.isLiveStreamId(id)) {
return LIVE_STREAM_PATH + id;
}
return VIDEO_PATH + id;
}

View File

@ -6,6 +6,7 @@ import org.schabi.newpipe.extractor.InfoItemsCollector;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeSepiaStreamInfoItemExtractor;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeStreamInfoItemExtractor;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Parser;
@ -63,6 +64,19 @@ public class PeertubeParsingHelper {
}
public static void collectStreamsFrom(final InfoItemsCollector collector, final JsonObject json, final String baseUrl) throws ParsingException {
collectStreamsFrom(collector, json, baseUrl, false);
}
/**
* Collect stream from json with collector
*
* @param collector the collector used to collect information
* @param json the file to retrieve data from
* @param baseUrl the base Url of the instance
* @param sepia if we should use PeertubeSepiaStreamInfoItemExtractor
* @throws ParsingException
*/
public static void collectStreamsFrom(final InfoItemsCollector collector, final JsonObject json, final String baseUrl, boolean sepia) throws ParsingException {
final JsonArray contents;
try {
contents = (JsonArray) JsonUtils.getValue(json, "data");
@ -73,9 +87,15 @@ public class PeertubeParsingHelper {
for (final Object c : contents) {
if (c instanceof JsonObject) {
final JsonObject item = (JsonObject) c;
final PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item, baseUrl);
PeertubeStreamInfoItemExtractor extractor;
if (sepia) {
extractor = new PeertubeSepiaStreamInfoItemExtractor(item, baseUrl);
} else {
extractor = new PeertubeStreamInfoItemExtractor(item, baseUrl);
}
collector.commit(extractor);
}
}
}
}

View File

@ -15,6 +15,8 @@ import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
import java.util.List;
import static java.util.Arrays.asList;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.COMMENTS;
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.VIDEO;
@ -59,7 +61,12 @@ public class PeertubeService extends StreamingService {
@Override
public SearchExtractor getSearchExtractor(SearchQueryHandler queryHandler) {
return new PeertubeSearchExtractor(this, queryHandler);
final List<String> contentFilters = queryHandler.getContentFilters();
boolean external = false;
if (contentFilters.size() > 0 && contentFilters.get(0).startsWith("sepia_")) {
external = true;
}
return new PeertubeSearchExtractor(this, queryHandler, external);
}
@Override

View File

@ -88,6 +88,11 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac
return baseUrl + value;
}
@Override
public boolean getHeartedByUploader() throws ParsingException {
return false;
}
@Override
public String getUploaderName() throws ParsingException {
return JsonUtils.getString(item, "account.name") + "@" + JsonUtils.getString(item, "account.host");

View File

@ -4,6 +4,7 @@ import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.MetaInfo;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
@ -17,6 +18,8 @@ import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import javax.annotation.Nonnull;
@ -27,8 +30,17 @@ import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelp
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class PeertubeSearchExtractor extends SearchExtractor {
// if we should use PeertubeSepiaStreamInfoItemExtractor
private boolean sepia;
public PeertubeSearchExtractor(StreamingService service, SearchQueryHandler linkHandler) {
this(service, linkHandler, false);
}
public PeertubeSearchExtractor(StreamingService service, SearchQueryHandler linkHandler, boolean sepia) {
super(service, linkHandler);
this.sepia = sepia;
}
@Nonnull
@ -42,6 +54,12 @@ public class PeertubeSearchExtractor extends SearchExtractor {
return false;
}
@Nonnull
@Override
public List<MetaInfo> getMetaInfo() {
return Collections.emptyList();
}
@Override
public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionException {
final String pageUrl = getUrl() + "&" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE;
@ -70,7 +88,7 @@ public class PeertubeSearchExtractor extends SearchExtractor {
final long total = json.getLong("total");
final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId());
collectStreamsFrom(collector, json, getBaseUrl());
collectStreamsFrom(collector, json, getBaseUrl(), sepia);
return new InfoItemsPage<>(collector, PeertubeParsingHelper.getNextPage(page.getUrl(), total));
} else {

View File

@ -0,0 +1,22 @@
package org.schabi.newpipe.extractor.services.peertube.extractors;
import com.grack.nanojson.JsonObject;
/**
* A StreamInfoItem collected from SepiaSearch
*/
public class PeertubeSepiaStreamInfoItemExtractor extends PeertubeStreamInfoItemExtractor {
public PeertubeSepiaStreamInfoItemExtractor(JsonObject item, String baseUrl) {
super(item, baseUrl);
final String embedUrl = super.item.getString("embedUrl");
final String embedPath = super.item.getString("embedPath");
final String itemBaseUrl = embedUrl.replace(embedPath, "");
setBaseUrl(itemBaseUrl);
// Usually, all videos, pictures and other content are hosted on the instance,
// or can be accessed by the same URL path if the instance with baseUrl federates the one where the video is actually uploaded
// But it can't be accessed with Sepiasearch, so we use the item's instance as base URL
}
}

View File

@ -4,8 +4,8 @@ import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.MetaInfo;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
@ -23,12 +23,15 @@ import org.schabi.newpipe.extractor.stream.Description;
import org.schabi.newpipe.extractor.stream.Stream;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamSegment;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.stream.SubtitlesStream;
import org.schabi.newpipe.extractor.stream.VideoStream;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
@ -37,9 +40,6 @@ import java.util.Collections;
import java.util.List;
import java.util.Locale;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
public class PeertubeStreamExtractor extends StreamExtractor {
private final String baseUrl;
private JsonObject json;
@ -268,7 +268,9 @@ public class PeertubeStreamExtractor extends StreamExtractor {
final List<String> tags = getTags();
final String apiUrl;
if (tags.isEmpty()) {
apiUrl = getUploaderUrl() + "/videos?start=0&count=8";
apiUrl = baseUrl + "/api/v1/accounts/" + JsonUtils.getString(json, "account.name")
+ "@" + JsonUtils.getString(json, "account.host") +
"/videos?start=0&count=8";
} else {
apiUrl = getRelatedStreamsUrl(tags);
}
@ -302,6 +304,18 @@ public class PeertubeStreamExtractor extends StreamExtractor {
}
}
@Nonnull
@Override
public List<StreamSegment> getStreamSegments() {
return Collections.emptyList();
}
@Nonnull
@Override
public List<MetaInfo> getMetaInfo() {
return Collections.emptyList();
}
private String getRelatedStreamsUrl(final List<String> tags) throws UnsupportedEncodingException {
final String url = baseUrl + PeertubeSearchQueryHandlerFactory.SEARCH_ENDPOINT;
final StringBuilder params = new StringBuilder();

View File

@ -1,7 +1,6 @@
package org.schabi.newpipe.extractor.services.peertube.extractors;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.ServiceList;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.localization.DateWrapper;
@ -11,8 +10,9 @@ import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.utils.JsonUtils;
public class PeertubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
protected final JsonObject item;
private final String baseUrl;
private String baseUrl;
public PeertubeStreamInfoItemExtractor(final JsonObject item, final String baseUrl) {
this.item = item;
@ -84,4 +84,8 @@ public class PeertubeStreamInfoItemExtractor implements StreamInfoItemExtractor
public long getDuration() {
return item.getLong("duration");
}
protected void setBaseUrl(final String baseUrl) {
this.baseUrl = baseUrl;
}
}

View File

@ -12,6 +12,8 @@ public class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory
public static final String CHARSET_UTF_8 = "UTF-8";
public static final String VIDEOS = "videos";
public static final String SEPIA_VIDEOS = "sepia_videos"; // sepia is the global index
public static final String SEPIA_BASE_URL = "https://sepiasearch.org";
public static final String SEARCH_ENDPOINT = "/api/v1/search/videos";
public static PeertubeSearchQueryHandlerFactory getInstance() {
@ -20,7 +22,12 @@ public class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory
@Override
public String getUrl(String searchString, List<String> contentFilters, String sortFilter) throws ParsingException {
String baseUrl = ServiceList.PeerTube.getBaseUrl();
String baseUrl;
if (contentFilters.size() > 0 && contentFilters.get(0).startsWith("sepia_")) {
baseUrl = SEPIA_BASE_URL;
} else {
baseUrl = ServiceList.PeerTube.getBaseUrl();
}
return getUrl(searchString, contentFilters, sortFilter, baseUrl);
}
@ -38,6 +45,9 @@ public class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory
@Override
public String[] getAvailableContentFilter() {
return new String[]{VIDEOS};
return new String[]{
VIDEOS,
SEPIA_VIDEOS
};
}
}

View File

@ -39,6 +39,11 @@ public class SoundcloudCommentsInfoItemExtractor implements CommentsInfoItemExtr
return json.getObject("user").getString("avatar_url");
}
@Override
public boolean getHeartedByUploader() throws ParsingException {
return false;
}
@Override
public String getUploaderUrl() {
return json.getObject("user").getString("permalink_url");

View File

@ -8,6 +8,7 @@ import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.InfoItemExtractor;
import org.schabi.newpipe.extractor.InfoItemsCollector;
import org.schabi.newpipe.extractor.MetaInfo;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
@ -22,6 +23,8 @@ import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Collections;
import java.util.List;
import javax.annotation.Nonnull;
@ -47,6 +50,12 @@ public class SoundcloudSearchExtractor extends SearchExtractor {
return false;
}
@Nonnull
@Override
public List<MetaInfo> getMetaInfo() {
return Collections.emptyList();
}
@Nonnull
@Override
public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionException {

View File

@ -6,6 +6,7 @@ import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.MetaInfo;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
@ -20,6 +21,7 @@ import org.schabi.newpipe.extractor.stream.AudioStream;
import org.schabi.newpipe.extractor.stream.Description;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamSegment;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.stream.SubtitlesStream;
import org.schabi.newpipe.extractor.stream.VideoStream;
@ -320,4 +322,16 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
public String getSupportInfo() {
return "";
}
@Nonnull
@Override
public List<StreamSegment> getStreamSegments() {
return Collections.emptyList();
}
@Nonnull
@Override
public List<MetaInfo> getMetaInfo() {
return Collections.emptyList();
}
}

View File

@ -60,7 +60,7 @@ public class ItagItem {
new ItagItem(278, VIDEO_ONLY, WEBM, "144p"),
new ItagItem(242, VIDEO_ONLY, WEBM, "240p"),
// new ItagItem(243, VIDEO_ONLY, WEBM, "360p"),
new ItagItem(243, VIDEO_ONLY, WEBM, "360p"),
new ItagItem(244, VIDEO_ONLY, WEBM, "480p"),
new ItagItem(245, VIDEO_ONLY, WEBM, "480p"),
new ItagItem(246, VIDEO_ONLY, WEBM, "480p"),

View File

@ -1,5 +1,6 @@
package org.schabi.newpipe.extractor.services.youtube;
import com.grack.nanojson.*;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
@ -8,6 +9,7 @@ import com.grack.nanojson.JsonWriter;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.schabi.newpipe.extractor.MetaInfo;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
@ -15,9 +17,12 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.stream.Description;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
@ -28,13 +33,11 @@ import java.time.LocalDate;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.time.format.DateTimeParseException;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.*;
import static org.schabi.newpipe.extractor.NewPipe.getDownloader;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.*;
import static org.schabi.newpipe.extractor.utils.Utils.HTTP;
import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
@ -76,41 +79,35 @@ public class YoutubeParsingHelper {
private static final String FEED_BASE_CHANNEL_ID = "https://www.youtube.com/feeds/videos.xml?channel_id=";
private static final String FEED_BASE_USER = "https://www.youtube.com/feeds/videos.xml?user=";
private static final String[] RECAPTCHA_DETECTION_SELECTORS = {
"form[action*=\"/das_captcha\"]",
"input[name*=\"action_recaptcha_verify\"]"
};
public static Document parseAndCheckPage(final String url, final Response response) throws ReCaptchaException {
final Document document = Jsoup.parse(response.responseBody(), url);
for (String detectionSelector : RECAPTCHA_DETECTION_SELECTORS) {
if (!document.select(detectionSelector).isEmpty()) {
throw new ReCaptchaException("reCAPTCHA challenge requested (detected with selector: \"" + detectionSelector + "\")", url);
}
private static boolean isGoogleURL(String url) {
url = extractCachedUrlIfNeeded(url);
try {
final URL u = new URL(url);
final String host = u.getHost();
return host.startsWith("google.") || host.startsWith("m.google.");
} catch (MalformedURLException e) {
return false;
}
return document;
}
public static boolean isYoutubeURL(URL url) {
String host = url.getHost();
public static boolean isYoutubeURL(final URL url) {
final String host = url.getHost();
return host.equalsIgnoreCase("youtube.com") || host.equalsIgnoreCase("www.youtube.com")
|| host.equalsIgnoreCase("m.youtube.com") || host.equalsIgnoreCase("music.youtube.com");
}
public static boolean isYoutubeServiceURL(URL url) {
String host = url.getHost();
public static boolean isYoutubeServiceURL(final URL url) {
final String host = url.getHost();
return host.equalsIgnoreCase("www.youtube-nocookie.com") || host.equalsIgnoreCase("youtu.be");
}
public static boolean isHooktubeURL(URL url) {
String host = url.getHost();
public static boolean isHooktubeURL(final URL url) {
final String host = url.getHost();
return host.equalsIgnoreCase("hooktube.com");
}
public static boolean isInvidioURL(URL url) {
String host = url.getHost();
public static boolean isInvidioURL(final URL url) {
final String host = url.getHost();
return host.equalsIgnoreCase("invidio.us")
|| host.equalsIgnoreCase("dev.invidio.us")
|| host.equalsIgnoreCase("www.invidio.us")
@ -184,7 +181,7 @@ public class YoutubeParsingHelper {
}
}
public static OffsetDateTime parseDateFrom(String textualUploadDate) throws ParsingException {
public static OffsetDateTime parseDateFrom(final String textualUploadDate) throws ParsingException {
try {
return OffsetDateTime.parse(textualUploadDate);
} catch (DateTimeParseException e) {
@ -208,12 +205,12 @@ public class YoutubeParsingHelper {
/**
* Checks if the given playlist id is a YouTube Music Mix (auto-generated playlist)
* Ids from a YouTube Music Mix start with "RDAMVM"
* Ids from a YouTube Music Mix start with "RDAMVM" or "RDCLAK"
* @param playlistId
* @return Whether given id belongs to a YouTube Music Mix
*/
public static boolean isYoutubeMusicMixId(final String playlistId) {
return playlistId.startsWith("RDAMVM");
return playlistId.startsWith("RDAMVM") || playlistId.startsWith("RDCLAK");
}
/**
* Checks if the given playlist id is a YouTube Channel Mix (auto-generated playlist)
@ -229,25 +226,25 @@ public class YoutubeParsingHelper {
* @throws ParsingException If the playlistId is a Channel Mix or not a mix.
*/
public static String extractVideoIdFromMixId(final String playlistId) throws ParsingException {
if (playlistId.startsWith("RDMM")) { //My Mix
if (playlistId.startsWith("RDMM")) { // My Mix
return playlistId.substring(4);
} else if (playlistId.startsWith("RDAMVM")) { //Music mix
} else if (isYoutubeMusicMixId(playlistId)) { // starts with "RDAMVM" or "RDCLAK"
return playlistId.substring(6);
} else if (playlistId.startsWith("RMCM")) { //Channel mix
//Channel mix are build with RMCM{channelId}, so videoId can't be determined
} else if (isYoutubeChannelMixId(playlistId)) { // starts with "RMCM"
// Channel mix are build with RMCM{channelId}, so videoId can't be determined
throw new ParsingException("Video id could not be determined from mix id: " + playlistId);
} else if (playlistId.startsWith("RD")) { // Normal mix
} else if (isYoutubeMixId(playlistId)) { // normal mix, starts with "RD"
return playlistId.substring(2);
} else { //not a mix
} else { // not a mix
throw new ParsingException("Video id could not be determined from mix id: " + playlistId);
}
}
public static JsonObject getInitialData(String html) throws ParsingException {
public static JsonObject getInitialData(final String html) throws ParsingException {
try {
try {
final String initialData = Parser.matchGroup1("window\\[\"ytInitialData\"\\]\\s*=\\s*(\\{.*?\\});", html);
@ -264,10 +261,9 @@ public class YoutubeParsingHelper {
public static boolean isHardcodedClientVersionValid() throws IOException, ExtractionException {
final String url = "https://www.youtube.com/results?search_query=test&pbj=1";
Map<String, List<String>> headers = new HashMap<>();
final Map<String, List<String>> headers = new HashMap<>();
headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
headers.put("X-YouTube-Client-Version",
Collections.singletonList(HARDCODED_CLIENT_VERSION));
headers.put("X-YouTube-Client-Version", Collections.singletonList(HARDCODED_CLIENT_VERSION));
final String response = getDownloader().get(url, headers).responseBody();
return response.length() > 50; // ensure to have a valid response
@ -357,6 +353,16 @@ public class YoutubeParsingHelper {
return key;
}
/**
* Only use in tests.
*
* Quick-and-dirty solution to reset global state in between test classes.
*/
static void resetClientVersionAndKey() {
clientVersion = null;
key = null;
}
public static boolean areHardcodedYoutubeMusicKeysValid() throws IOException, ReCaptchaException {
final String url = "https://music.youtube.com/youtubei/v1/search?alt=json&key=" + HARDCODED_YOUTUBE_MUSIC_KEYS[0];
@ -390,14 +396,14 @@ public class YoutubeParsingHelper {
.end().done().getBytes("UTF-8");
// @formatter:on
Map<String, List<String>> headers = new HashMap<>();
final Map<String, List<String>> headers = new HashMap<>();
headers.put("X-YouTube-Client-Name", Collections.singletonList(HARDCODED_YOUTUBE_MUSIC_KEYS[1]));
headers.put("X-YouTube-Client-Version", Collections.singletonList(HARDCODED_YOUTUBE_MUSIC_KEYS[2]));
headers.put("Origin", Collections.singletonList("https://music.youtube.com"));
headers.put("Referer", Collections.singletonList("music.youtube.com"));
headers.put("Content-Type", Collections.singletonList("application/json"));
String response = getDownloader().post(url, headers, json).responseBody();
final String response = getDownloader().post(url, headers, json).responseBody();
return response.length() > 50; // ensure to have a valid response
}
@ -432,6 +438,7 @@ public class YoutubeParsingHelper {
return youtubeMusicKeys = new String[]{key, clientName, clientVersion};
}
@Nullable
public static String getUrlFromNavigationEndpoint(JsonObject navigationEndpoint) throws ParsingException {
if (navigationEndpoint.has("urlEndpoint")) {
String internUrl = navigationEndpoint.getObject("urlEndpoint").getString("url");
@ -493,6 +500,7 @@ public class YoutubeParsingHelper {
* @param html whether to return HTML, by parsing the navigationEndpoint
* @return text in the JSON object or {@code null}
*/
@Nullable
public static String getTextFromObject(JsonObject textObject, boolean html) throws ParsingException {
if (isNullOrEmpty(textObject)) return null;
@ -500,8 +508,8 @@ public class YoutubeParsingHelper {
if (textObject.getArray("runs").isEmpty()) return null;
StringBuilder textBuilder = new StringBuilder();
for (Object textPart : textObject.getArray("runs")) {
final StringBuilder textBuilder = new StringBuilder();
for (final Object textPart : textObject.getArray("runs")) {
String text = ((JsonObject) textPart).getString("text");
if (html && ((JsonObject) textPart).has("navigationEndpoint")) {
String url = getUrlFromNavigationEndpoint(((JsonObject) textPart).getObject("navigationEndpoint"));
@ -523,6 +531,7 @@ public class YoutubeParsingHelper {
return text;
}
@Nullable
public static String getTextFromObject(JsonObject textObject) throws ParsingException {
return getTextFromObject(textObject, false);
}
@ -650,4 +659,124 @@ public class YoutubeParsingHelper {
}
}
}
@Nonnull
public static List<MetaInfo> getMetaInfo(final JsonArray contents) throws ParsingException {
final List<MetaInfo> metaInfo = new ArrayList<>();
for (final Object content : contents) {
final JsonObject resultObject = (JsonObject) content;
if (resultObject.has("itemSectionRenderer")) {
for (final Object sectionContentObject :
resultObject.getObject("itemSectionRenderer").getArray("contents")) {
final JsonObject sectionContent = (JsonObject) sectionContentObject;
if (sectionContent.has("infoPanelContentRenderer")) {
metaInfo.add(getInfoPanelContent(sectionContent.getObject("infoPanelContentRenderer")));
}
if (sectionContent.has("clarificationRenderer")) {
metaInfo.add(getClarificationRendererContent(sectionContent.getObject("clarificationRenderer")
));
}
}
}
}
return metaInfo;
}
@Nonnull
private static MetaInfo getInfoPanelContent(final JsonObject infoPanelContentRenderer)
throws ParsingException {
final MetaInfo metaInfo = new MetaInfo();
final StringBuilder sb = new StringBuilder();
for (final Object paragraph : infoPanelContentRenderer.getArray("paragraphs")) {
if (sb.length() != 0) {
sb.append("<br>");
}
sb.append(YoutubeParsingHelper.getTextFromObject((JsonObject) paragraph));
}
metaInfo.setContent(new Description(sb.toString(), Description.HTML));
if (infoPanelContentRenderer.has("sourceEndpoint")) {
final String metaInfoLinkUrl = YoutubeParsingHelper.getUrlFromNavigationEndpoint(
infoPanelContentRenderer.getObject("sourceEndpoint"));
try {
metaInfo.addUrl(new URL(Objects.requireNonNull(extractCachedUrlIfNeeded(metaInfoLinkUrl))));
} catch (final NullPointerException | MalformedURLException e) {
throw new ParsingException("Could not get metadata info URL", e);
}
final String metaInfoLinkText = YoutubeParsingHelper.getTextFromObject(
infoPanelContentRenderer.getObject("inlineSource"));
if (isNullOrEmpty(metaInfoLinkText)) {
throw new ParsingException("Could not get metadata info link text.");
}
metaInfo.addUrlText(metaInfoLinkText);
}
return metaInfo;
}
@Nonnull
private static MetaInfo getClarificationRendererContent(final JsonObject clarificationRenderer)
throws ParsingException {
final MetaInfo metaInfo = new MetaInfo();
final String title = YoutubeParsingHelper.getTextFromObject(clarificationRenderer.getObject("contentTitle"));
final String text = YoutubeParsingHelper.getTextFromObject(clarificationRenderer.getObject("text"));
if (title == null || text == null) {
throw new ParsingException("Could not extract clarification renderer content");
}
metaInfo.setTitle(title);
metaInfo.setContent(new Description(text, Description.PLAIN_TEXT));
if (clarificationRenderer.has("actionButton")) {
final JsonObject actionButton = clarificationRenderer.getObject("actionButton")
.getObject("buttonRenderer");
try {
final String url = YoutubeParsingHelper.getUrlFromNavigationEndpoint(actionButton.getObject("command"));
metaInfo.addUrl(new URL(Objects.requireNonNull(extractCachedUrlIfNeeded(url))));
} catch (final NullPointerException | MalformedURLException e) {
throw new ParsingException("Could not get metadata info URL", e);
}
final String metaInfoLinkText = YoutubeParsingHelper.getTextFromObject(
actionButton.getObject("text"));
if (isNullOrEmpty(metaInfoLinkText)) {
throw new ParsingException("Could not get metadata info link text.");
}
metaInfo.addUrlText(metaInfoLinkText);
}
if (clarificationRenderer.has("secondaryEndpoint") && clarificationRenderer.has("secondarySource")) {
final String url = getUrlFromNavigationEndpoint(clarificationRenderer.getObject("secondaryEndpoint"));
// ignore Google URLs, because those point to a Google search about "Covid-19"
if (url != null && !isGoogleURL(url)) {
try {
metaInfo.addUrl(new URL(url));
final String description = getTextFromObject(clarificationRenderer.getObject("secondarySource"));
metaInfo.addUrlText(description == null ? url : description);
} catch (MalformedURLException e) {
throw new ParsingException("Could not get metadata info secondary URL", e);
}
}
}
return metaInfo;
}
/**
* Sometimes, YouTube provides URLs which use Google's cache. They look like
* {@code https://webcache.googleusercontent.com/search?q=cache:CACHED_URL}
* @param url the URL which might refer to the Google's webcache
* @return the URL which is referring to the original site
*/
public static String extractCachedUrlIfNeeded(final String url) {
if (url == null) {
return null;
}
if (url.contains("webcache.googleusercontent.com")) {
return url.split("cache:")[1];
}
return url;
}
}

View File

@ -111,7 +111,8 @@ public class YoutubeService extends StreamingService {
@Override
public PlaylistExtractor getPlaylistExtractor(final ListLinkHandler linkHandler) {
if (YoutubeParsingHelper.isYoutubeMixId(linkHandler.getId())) {
if (YoutubeParsingHelper.isYoutubeMixId(linkHandler.getId())
&& !YoutubeParsingHelper.isYoutubeMusicMixId(linkHandler.getId())) {
return new YoutubeMixPlaylistExtractor(this, linkHandler);
} else {
return new YoutubePlaylistExtractor(this, linkHandler);

View File

@ -115,6 +115,11 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
}
}
@Override
public boolean getHeartedByUploader() throws ParsingException {
return json.has("creatorHeart");
}
@Override
public String getUploaderName() throws ParsingException {
try {

View File

@ -7,6 +7,7 @@ import com.grack.nanojson.JsonParserException;
import com.grack.nanojson.JsonWriter;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.MetaInfo;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
@ -19,6 +20,8 @@ import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Utils;
@ -163,6 +166,12 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
return !showingResultsForRenderer.isEmpty();
}
@Nonnull
@Override
public List<MetaInfo> getMetaInfo() {
return Collections.emptyList();
}
@Nonnull
@Override
public InfoItemsPage<InfoItem> getInitialPage() throws ExtractionException, IOException {
@ -251,16 +260,29 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
final TimeAgoParser timeAgoParser = getTimeAgoParser();
for (Object item : videos) {
final JsonObject info = ((JsonObject) item).getObject("musicResponsiveListItemRenderer", null);
final JsonObject info = ((JsonObject) item)
.getObject("musicResponsiveListItemRenderer", null);
if (info != null) {
final String displayPolicy = info.getString("musicItemRendererDisplayPolicy", EMPTY_STRING);
if (displayPolicy.equals("MUSIC_ITEM_RENDERER_DISPLAY_POLICY_GREY_OUT")) {
continue; // no info about video URL available
}
final JsonObject flexColumnRenderer = info
.getArray("flexColumns")
.getObject(1)
.getObject("musicResponsiveListItemFlexColumnRenderer");
final JsonArray descriptionElements = flexColumnRenderer
.getObject("text")
.getArray("runs");
final String searchType = getLinkHandler().getContentFilters().get(0);
if (searchType.equals(MUSIC_SONGS) || searchType.equals(MUSIC_VIDEOS)) {
collector.commit(new YoutubeStreamInfoItemExtractor(info, timeAgoParser) {
@Override
public String getUrl() throws ParsingException {
final String url = getUrlFromNavigationEndpoint(info.getObject("doubleTapCommand"));
if (!isNullOrEmpty(url)) {
return url;
final String id = info.getObject("playlistItemData").getString("videoId");
if (!isNullOrEmpty(id)) {
return "https://music.youtube.com/watch?v=" + id;
}
throw new ParsingException("Could not get url");
}
@ -277,8 +299,9 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
@Override
public long getDuration() throws ParsingException {
final String duration = getTextFromObject(info.getArray("flexColumns").getObject(3)
.getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text"));
final String duration = descriptionElements
.getObject(descriptionElements.size() - 1)
.getString("text");
if (!isNullOrEmpty(duration)) {
return YoutubeParsingHelper.parseDurationString(duration);
}
@ -287,8 +310,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
@Override
public String getUploaderName() throws ParsingException {
final String name = getTextFromObject(info.getArray("flexColumns").getObject(1)
.getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text"));
final String name = descriptionElements.getObject(0).getString("text");
if (!isNullOrEmpty(name)) {
return name;
}
@ -339,8 +361,9 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
if (searchType.equals(MUSIC_SONGS)) {
return -1;
}
final String viewCount = getTextFromObject(info.getArray("flexColumns").getObject(2)
.getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text"));
final String viewCount = descriptionElements
.getObject(descriptionElements.size() - 3)
.getString("text");
if (!isNullOrEmpty(viewCount)) {
return Utils.mixedNumberWordToLong(viewCount);
}
@ -444,9 +467,27 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
@Override
public String getUrl() throws ParsingException {
final String url = getUrlFromNavigationEndpoint(info.getObject("doubleTapCommand"));
if (!isNullOrEmpty(url)) {
return url;
String playlistId = info.getObject("menu")
.getObject("menuRenderer")
.getArray("items")
.getObject(4)
.getObject("toggleMenuServiceItemRenderer")
.getObject("toggledServiceEndpoint")
.getObject("likeEndpoint")
.getObject("target")
.getString("playlistId");
if (isNullOrEmpty(playlistId)) {
playlistId = info.getObject("overlay")
.getObject("musicItemThumbnailOverlayRenderer")
.getObject("content")
.getObject("musicPlayButtonRenderer")
.getObject("playNavigationEndpoint")
.getObject("watchPlaylistEndpoint")
.getString("playlistId");
}
if (!isNullOrEmpty(playlistId)) {
return "https://music.youtube.com/playlist?list=" + playlistId;
}
throw new ParsingException("Could not get url");
}
@ -455,11 +496,9 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
public String getUploaderName() throws ParsingException {
final String name;
if (searchType.equals(MUSIC_ALBUMS)) {
name = getTextFromObject(info.getArray("flexColumns").getObject(2)
.getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text"));
name = descriptionElements.getObject(2).getString("text");
} else {
name = getTextFromObject(info.getArray("flexColumns").getObject(1)
.getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text"));
name = descriptionElements.getObject(0).getString("text");
}
if (!isNullOrEmpty(name)) {
return name;
@ -472,8 +511,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
if (searchType.equals(MUSIC_ALBUMS)) {
return ITEM_COUNT_UNKNOWN;
}
final String count = getTextFromObject(info.getArray("flexColumns").getObject(2)
.getObject("musicResponsiveListItemFlexColumnRenderer").getObject("text"));
final String count = descriptionElements.getObject(2).getString("text");
if (!isNullOrEmpty(count)) {
if (count.contains("100+")) {
return ITEM_COUNT_MORE_THAN_100;

View File

@ -7,6 +7,7 @@ import com.grack.nanojson.JsonParserException;
import com.grack.nanojson.JsonWriter;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.MetaInfo;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
@ -16,13 +17,11 @@ import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.*;
import javax.annotation.Nonnull;
@ -106,6 +105,13 @@ public class YoutubeSearchExtractor extends SearchExtractor {
return !showingResultsForRenderer.isEmpty();
}
@Override
public List<MetaInfo> getMetaInfo() throws ParsingException {
return YoutubeParsingHelper.getMetaInfo(
initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer")
.getObject("primaryContents").getObject("sectionListRenderer").getArray("contents"));
}
@Nonnull
@Override
public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionException {

View File

@ -13,6 +13,7 @@ import org.mozilla.javascript.Context;
import org.mozilla.javascript.Function;
import org.mozilla.javascript.ScriptableObject;
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.MetaInfo;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
@ -35,6 +36,7 @@ import org.schabi.newpipe.extractor.stream.Stream;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamSegment;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.stream.SubtitlesStream;
import org.schabi.newpipe.extractor.stream.VideoStream;
@ -44,6 +46,9 @@ import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.net.URL;
import java.time.LocalDate;
import java.time.OffsetDateTime;
import java.time.format.DateTimeFormatter;
@ -837,8 +842,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
if (playerJsUrl.startsWith("//")) {
playerJsUrl = HTTPS + playerJsUrl;
} else if (playerJsUrl.startsWith("/")) {
// sometimes https://youtube.com part has to be added manually
playerJsUrl = HTTPS + "//youtube.com" + playerJsUrl;
// sometimes https://www.youtube.com part has to be added manually
playerJsUrl = HTTPS + "//www.youtube.com" + playerJsUrl;
}
cachedDeobfuscationCode = loadDeobfuscationCode(playerJsUrl);
@ -987,7 +992,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
for (int i = 1; i < spec.length; ++i) {
final String[] parts = spec[i].split("#");
if (parts.length != 8) {
if (parts.length != 8 || Integer.parseInt(parts[5]) == 0) {
continue;
}
final int frameWidth = Integer.parseInt(parts[0]);
@ -1011,6 +1016,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
frameWidth,
frameHeight,
totalCount,
Integer.parseInt(parts[5]),
framesPerPageX,
framesPerPageY
));
@ -1062,4 +1068,67 @@ public class YoutubeStreamExtractor extends StreamExtractor {
public String getSupportInfo() {
return "";
}
@Nonnull
@Override
public List<StreamSegment> getStreamSegments() throws ParsingException {
final ArrayList<StreamSegment> segments = new ArrayList<>();
if (initialData.has("engagementPanels")) {
final JsonArray panels = initialData.getArray("engagementPanels");
JsonArray segmentsArray = null;
// Search for correct panel containing the data
for (int i = 0; i < panels.size(); i++) {
if (panels.getObject(i).getObject("engagementPanelSectionListRenderer")
.getString("panelIdentifier").equals("engagement-panel-macro-markers")) {
segmentsArray = panels.getObject(i).getObject("engagementPanelSectionListRenderer")
.getObject("content").getObject("macroMarkersListRenderer").getArray("contents");
break;
}
}
if (segmentsArray != null) {
final long duration = getLength();
for (final Object object : segmentsArray) {
final JsonObject segmentJson = ((JsonObject) object).getObject("macroMarkersListItemRenderer");
final int startTimeSeconds = segmentJson.getObject("onTap").getObject("watchEndpoint")
.getInt("startTimeSeconds", -1);
if (startTimeSeconds == -1) {
throw new ParsingException("Could not get stream segment start time.");
}
if (startTimeSeconds > duration) {
break;
}
final String title = getTextFromObject(segmentJson.getObject("title"));
if (isNullOrEmpty(title)) {
throw new ParsingException("Could not get stream segment title.");
}
final StreamSegment segment = new StreamSegment(title, startTimeSeconds);
segment.setUrl(getUrl() + "?t=" + startTimeSeconds);
if (segmentJson.has("thumbnail")) {
final JsonArray previewsArray = segmentJson.getObject("thumbnail").getArray("thumbnails");
if (!previewsArray.isEmpty()) {
// Assume that the thumbnail with the highest resolution is at the last position
final String url = previewsArray.getObject(previewsArray.size() - 1).getString("url");
segment.setPreviewUrl(fixThumbnailUrl(url));
}
}
segments.add(segment);
}
}
}
return segments;
}
@Nonnull
@Override
public List<MetaInfo> getMetaInfo() throws ParsingException {
return YoutubeParsingHelper.getMetaInfo(
initialData.getObject("contents").getObject("twoColumnWatchNextResults")
.getObject("results").getObject("results").getArray("contents"));
}
}

View File

@ -33,7 +33,7 @@ public class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory {
if (!Utils.isHTTP(urlObj) || !(YoutubeParsingHelper.isYoutubeURL(urlObj)
|| YoutubeParsingHelper.isInvidioURL(urlObj))) {
throw new ParsingException("the url given is not a Youtube-URL");
throw new ParsingException("the url given is not a YouTube-URL");
}
final String path = urlObj.getPath();
@ -44,7 +44,7 @@ public class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory {
final String listID = Utils.getQueryValue(urlObj, "list");
if (listID == null) {
throw new ParsingException("the url given does not include a playlist");
throw new ParsingException("the URL given does not include a playlist");
}
if (!listID.matches("[a-zA-Z0-9_-]{10,}")) {
@ -52,11 +52,6 @@ public class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory {
"the list-ID given in the URL does not match the list pattern");
}
if (YoutubeParsingHelper.isYoutubeMusicMixId(listID)) {
throw new ContentNotSupportedException(
"YouTube Music Mix playlists are not yet supported");
}
if (YoutubeParsingHelper.isYoutubeChannelMixId(listID)
&& Utils.getQueryValue(urlObj, "v") == null) {
//Video id can't be determined from the channel mix id. See YoutubeParsingHelper#extractVideoIdFromMixId
@ -65,7 +60,7 @@ public class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory {
return listID;
} catch (final Exception exception) {
throw new ParsingException("Error could not parse url :" + exception.getMessage(),
throw new ParsingException("Error could not parse URL: " + exception.getMessage(),
exception);
}
}
@ -81,9 +76,8 @@ public class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory {
}
/**
* * If it is a mix (auto-generated playlist) URL, return a {@link LinkHandler} where the URL is
* like
* <code>https://youtube.com/watch?v=videoId&list=playlistId</code>.
* If it is a mix (auto-generated playlist) URL, return a {@link LinkHandler} where the URL is like
* {@code https://youtube.com/watch?v=videoId&list=playlistId}
* <p>Otherwise use super</p>
*/
@Override
@ -103,7 +97,7 @@ public class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory {
getSortFilter(url));
}
} catch (MalformedURLException exception) {
throw new ParsingException("Error could not parse url :" + exception.getMessage(),
throw new ParsingException("Error could not parse URL: " + exception.getMessage(),
exception);
}
return super.fromUrl(url);

View File

@ -8,12 +8,14 @@ public final class Frameset {
private int frameWidth;
private int frameHeight;
private int totalCount;
private int durationPerFrame;
private int framesPerPageX;
private int framesPerPageY;
public Frameset(List<String> urls, int frameWidth, int frameHeight, int totalCount, int framesPerPageX, int framesPerPageY) {
public Frameset(List<String> urls, int frameWidth, int frameHeight, int totalCount, int durationPerFrame, int framesPerPageX, int framesPerPageY) {
this.urls = urls;
this.totalCount = totalCount;
this.durationPerFrame = durationPerFrame;
this.frameWidth = frameWidth;
this.frameHeight = frameHeight;
this.framesPerPageX = framesPerPageX;
@ -61,4 +63,48 @@ public final class Frameset {
public int getFrameHeight() {
return frameHeight;
}
/**
* @return duration per frame in milliseconds
*/
public int getDurationPerFrame() {
return durationPerFrame;
}
/**
* Returns the information for the frame at stream position.
*
* @param position Position in milliseconds
* @return An <code>int</code>-array containing the bounds and URL where the indexes are specified as
* followed:
*
* <ul>
* <li><code>0</code>: Index of the URL</li>
* <li><code>1</code>: Left bound</li>
* <li><code>2</code>: Top bound</li>
* <li><code>3</code>: Right bound</li>
* <li><code>4</code>: Bottom bound</li>
* </ul>
*/
public int[] getFrameBoundsAt(long position) {
if (position < 0 || position > ((totalCount + 1) * durationPerFrame)) {
// Return the first frame as fallback
return new int[] { 0, 0, 0, frameWidth, frameHeight };
}
final int framesPerStoryboard = framesPerPageX * framesPerPageY;
final int absoluteFrameNumber = Math.min((int) (position / durationPerFrame), totalCount);
final int relativeFrameNumber = absoluteFrameNumber % framesPerStoryboard;
final int rowIndex = Math.floorDiv(relativeFrameNumber, framesPerPageX);
final int columnIndex = relativeFrameNumber % framesPerPageY;
return new int[] {
/* storyboardIndex */ Math.floorDiv(absoluteFrameNumber, framesPerStoryboard),
/* left */ columnIndex * frameWidth,
/* top */ rowIndex * frameHeight,
/* right */ columnIndex * frameWidth + frameWidth,
/* bottom */ rowIndex * frameHeight + frameHeight };
}
}

View File

@ -22,6 +22,7 @@ package org.schabi.newpipe.extractor.stream;
import org.schabi.newpipe.extractor.Extractor;
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.MetaInfo;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
@ -476,4 +477,28 @@ public abstract class StreamExtractor extends Extractor {
*/
@Nonnull
public abstract String getSupportInfo() throws ParsingException;
/**
* The list of stream segments by timestamps for the stream.
* If the segment list is not available you can simply return an empty list.
*
* @return The list of segments of the stream or an empty list.
* @throws ParsingException
*/
@Nonnull
public abstract List<StreamSegment> getStreamSegments() throws ParsingException;
/**
* Meta information about the stream.
* <p>
* This can be information about the stream creator (e.g. if the creator is a public broadcaster)
* or further information on the topic (e.g. hints that the video might contain conspiracy theories
* or contains information about a current health situation like the Covid-19 pandemic).
* </p>
* The meta information often contains links to external sources like Wikipedia or the WHO.
* @return The meta info of the stream or an empty List if not provided.
* @throws ParsingException
*/
@Nonnull
public abstract List<MetaInfo> getMetaInfo() throws ParsingException;
}

View File

@ -1,9 +1,6 @@
package org.schabi.newpipe.extractor.stream;
import org.schabi.newpipe.extractor.Info;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.*;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
@ -13,9 +10,12 @@ import org.schabi.newpipe.extractor.utils.ExtractorHelper;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
/*
@ -324,6 +324,16 @@ public class StreamInfo extends Info {
} catch (Exception e) {
streamInfo.addError(e);
}
try {
streamInfo.setStreamSegments(extractor.getStreamSegments());
} catch (Exception e) {
streamInfo.addError(e);
}
try {
streamInfo.setMetaInfo(extractor.getMetaInfo());
} catch (Exception e) {
streamInfo.addError(e);
}
streamInfo.setRelatedStreams(ExtractorHelper.getRelatedVideosOrLogError(streamInfo, extractor));
@ -373,6 +383,8 @@ public class StreamInfo extends Info {
private String support = "";
private Locale language = null;
private List<String> tags = new ArrayList<>();
private List<StreamSegment> streamSegments = new ArrayList<>();
private List<MetaInfo> metaInfo = new ArrayList<>();
/**
* Get the stream type
@ -670,4 +682,21 @@ public class StreamInfo extends Info {
public String getSupportInfo() {
return this.support;
}
public List<StreamSegment> getStreamSegments() {
return streamSegments;
}
public void setStreamSegments(List<StreamSegment> streamSegments) {
this.streamSegments = streamSegments;
}
public void setMetaInfo(final List<MetaInfo> metaInfo) {
this.metaInfo = metaInfo;
}
@Nonnull
public List<MetaInfo> getMetaInfo() {
return this.metaInfo;
}
}

View File

@ -5,6 +5,7 @@ import org.schabi.newpipe.extractor.InfoItemsCollector;
import org.schabi.newpipe.extractor.exceptions.FoundAdException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import java.util.Comparator;
import java.util.List;
import java.util.Vector;
@ -34,6 +35,10 @@ public class StreamInfoItemsCollector extends InfoItemsCollector<StreamInfoItem,
super(serviceId);
}
public StreamInfoItemsCollector(int serviceId, Comparator<StreamInfoItem> comparator) {
super(serviceId, comparator);
}
@Override
public StreamInfoItem extract(StreamInfoItemExtractor extractor) throws ParsingException {
if (extractor.isAd()) {

View File

@ -0,0 +1,69 @@
package org.schabi.newpipe.extractor.stream;
import javax.annotation.Nullable;
import java.io.Serializable;
public class StreamSegment implements Serializable {
/**
* Title of this segment
*/
private String title;
/**
* Timestamp of the starting point in seconds
*/
private int startTimeSeconds;
/**
* Direct url to this segment. This can be null if the service doesn't provide such function.
*/
@Nullable
public String url;
/**
* Preview url for this segment. This can be null if the service doesn't provide such function
* or there is no resource found.
*/
@Nullable
private String previewUrl = null;
public StreamSegment(String title, int startTimeSeconds) {
this.title = title;
this.startTimeSeconds = startTimeSeconds;
}
public String getTitle() {
return title;
}
public void setTitle(final String title) {
this.title = title;
}
public int getStartTimeSeconds() {
return startTimeSeconds;
}
public void setStartTimeSeconds(final int startTimeSeconds) {
this.startTimeSeconds = startTimeSeconds;
}
@Nullable
public String getUrl() {
return url;
}
public void setUrl(@Nullable final String url) {
this.url = url;
}
@Nullable
public String getPreviewUrl() {
return previewUrl;
}
public void setPreviewUrl(@Nullable final String previewUrl) {
this.previewUrl = previewUrl;
}
}

View File

@ -1,67 +1,67 @@
package org.schabi.newpipe.extractor.stream;
import org.schabi.newpipe.extractor.MediaFormat;
import java.io.Serializable;
import java.util.Locale;
public class SubtitlesStream extends Stream implements Serializable {
private final MediaFormat format;
private final Locale locale;
private final boolean autoGenerated;
private final String code;
public SubtitlesStream(MediaFormat format, String languageCode, String url, boolean autoGenerated) {
super(url, format);
/*
* Locale.forLanguageTag only for API >= 21
* Locale.Builder only for API >= 21
* Country codes doesn't work well without
*/
final String[] splits = languageCode.split("-");
switch (splits.length) {
default:
this.locale = new Locale(splits[0]);
break;
case 3:
this.locale = new Locale(splits[0], splits[1], splits[2]);// complex variants doesn't work!
break;
case 2:
this.locale = new Locale(splits[0], splits[1]);
break;
}
this.code = languageCode;
this.format = format;
this.autoGenerated = autoGenerated;
}
public String getExtension() {
return format.suffix;
}
public boolean isAutoGenerated() {
return autoGenerated;
}
@Override
public boolean equalStats(Stream cmp) {
return super.equalStats(cmp) &&
cmp instanceof SubtitlesStream &&
code.equals(((SubtitlesStream) cmp).code) &&
autoGenerated == ((SubtitlesStream) cmp).autoGenerated;
}
public String getDisplayLanguageName() {
return locale.getDisplayName(locale);
}
public String getLanguageTag() {
return code;
}
public Locale getLocale() {
return locale;
}
}
package org.schabi.newpipe.extractor.stream;
import org.schabi.newpipe.extractor.MediaFormat;
import java.io.Serializable;
import java.util.Locale;
public class SubtitlesStream extends Stream implements Serializable {
private final MediaFormat format;
private final Locale locale;
private final boolean autoGenerated;
private final String code;
public SubtitlesStream(MediaFormat format, String languageCode, String url, boolean autoGenerated) {
super(url, format);
/*
* Locale.forLanguageTag only for API >= 21
* Locale.Builder only for API >= 21
* Country codes doesn't work well without
*/
final String[] splits = languageCode.split("-");
switch (splits.length) {
default:
this.locale = new Locale(splits[0]);
break;
case 3:
this.locale = new Locale(splits[0], splits[1], splits[2]);// complex variants doesn't work!
break;
case 2:
this.locale = new Locale(splits[0], splits[1]);
break;
}
this.code = languageCode;
this.format = format;
this.autoGenerated = autoGenerated;
}
public String getExtension() {
return format.suffix;
}
public boolean isAutoGenerated() {
return autoGenerated;
}
@Override
public boolean equalStats(Stream cmp) {
return super.equalStats(cmp) &&
cmp instanceof SubtitlesStream &&
code.equals(((SubtitlesStream) cmp).code) &&
autoGenerated == ((SubtitlesStream) cmp).autoGenerated;
}
public String getDisplayLanguageName() {
return locale.getDisplayName(locale);
}
public String getLanguageTag() {
return code;
}
public Locale getLocale() {
return locale;
}
}

View File

@ -11,7 +11,7 @@ public class DonationLinkHelper {
}
public enum AffiliateService {
NO_AFILIATE,
NO_AFFILIATE,
AMAZON,
}
@ -33,7 +33,7 @@ public class DonationLinkHelper {
URL url = new URL(fixLink(link));
switch (url.getHost()) {
case "amzn.to": return AffiliateService.AMAZON;
default: return AffiliateService.NO_AFILIATE;
default: return AffiliateService.NO_AFFILIATE;
}
}

View File

@ -0,0 +1,50 @@
package org.schabi.newpipe.downloader;
import org.schabi.newpipe.extractor.downloader.Downloader;
import java.io.IOException;
public class DownloaderFactory {
public final static String RESOURCE_PATH = "src/test/resources/org/schabi/newpipe/extractor/";
private final static DownloaderType DEFAULT_DOWNLOADER = DownloaderType.REAL;
/**
* <p>
* Returns a implementation of a {@link Downloader}.
* </p>
* <p>
* If the system property "downloader" is set and is one of {@link DownloaderType},
* then a downloader of that type is returned.
* It can be passed in with gradle by adding the argument -Ddownloader=abcd,
* where abcd is one of {@link DownloaderType}
* </p>
* <p>
* Otherwise it falls back to {@link DownloaderFactory#DEFAULT_DOWNLOADER}.
* Change this during development on the local machine to use a different downloader.
* </p>
*
* @param path The path to the folder where mocks are saved/retrieved.
* Preferably starting with {@link DownloaderFactory#RESOURCE_PATH}
*/
public Downloader getDownloader(String path) throws IOException {
DownloaderType type;
try {
type = DownloaderType.valueOf(System.getProperty("downloader"));
} catch (Exception e) {
type = DEFAULT_DOWNLOADER;
}
switch (type) {
case REAL:
return DownloaderTestImpl.getInstance();
case MOCK:
return new MockDownloader(path);
case RECORDING:
return new RecordingDownloader(path);
default:
throw new UnsupportedOperationException("Unknown downloader type: " + type.toString());
}
}
}

View File

@ -1,4 +1,4 @@
package org.schabi.newpipe;
package org.schabi.newpipe.downloader;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Request;

View File

@ -0,0 +1,5 @@
package org.schabi.newpipe.downloader;
public enum DownloaderType {
REAL, MOCK, RECORDING
}

View File

@ -0,0 +1,53 @@
package org.schabi.newpipe.downloader;
import com.google.gson.GsonBuilder;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Request;
import org.schabi.newpipe.extractor.downloader.Response;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import javax.annotation.Nonnull;
/**
* <p>
* Mocks requests by using json files created by {@link RecordingDownloader}
* </p>
*/
class MockDownloader extends Downloader {
private final String path;
private final Map<Request, Response> mocks;
public MockDownloader(@Nonnull String path) throws IOException {
this.path = path;
this.mocks = new HashMap<>();
File folder = new File(path);
for (File file : folder.listFiles()) {
if (file.getName().startsWith(RecordingDownloader.FILE_NAME_PREFIX)) {
final FileReader reader = new FileReader(file);
final TestRequestResponse response = new GsonBuilder()
.create()
.fromJson(reader, TestRequestResponse.class);
reader.close();
mocks.put(response.getRequest(), response.getResponse());
}
}
}
@Override
public Response execute(@Nonnull Request request) {
Response result = mocks.get(request);
if (result == null) {
throw new NullPointerException("No mock response for request with url '" + request.url()
+ "' exists in path '" + path + "'.\nPlease make sure to run the tests with " +
"the RecordingDownloader first after changes.");
}
return result;
}
}

View File

@ -0,0 +1,76 @@
package org.schabi.newpipe.downloader;
import com.google.gson.GsonBuilder;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Request;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import javax.annotation.Nonnull;
/**
* <p>
* Relays requests to {@link DownloaderTestImpl} and saves the request/response pair into a json file.
* </p>
* <p>
* Those files are used by {@link MockDownloader}.
* </p>
* <p>
* The files <b>must</b> be created on the local dev environment
* and recreated when the requests made by a test class change.
* </p>
*/
class RecordingDownloader extends Downloader {
public final static String FILE_NAME_PREFIX = "generated_mock_";
private int index = 0;
private final String path;
/**
* Creates the folder described by {@code stringPath} if it does not exists.
* Deletes existing files starting with {@link RecordingDownloader#FILE_NAME_PREFIX}.
* @param stringPath Path to the folder where the json files will be saved to.
*/
public RecordingDownloader(String stringPath) throws IOException {
this.path = stringPath;
Path path = Paths.get(stringPath);
File folder = path.toFile();
if (folder.exists()) {
for (File file : folder.listFiles()) {
if (file.getName().startsWith(RecordingDownloader.FILE_NAME_PREFIX)) {
file.delete();
}
}
} else {
Files.createDirectories(path);
}
}
@Override
public Response execute(@Nonnull Request request) throws IOException, ReCaptchaException {
Downloader downloader = DownloaderTestImpl.getInstance();
Response response = downloader.execute(request);
File outputFile = new File(path + File.separator + FILE_NAME_PREFIX + index + ".json");
index++;
outputFile.createNewFile();
FileWriter writer = new FileWriter(outputFile);
new GsonBuilder()
.setPrettyPrinting()
.create()
.toJson(new TestRequestResponse(request, response), writer);
writer.flush();
writer.close();
return response;
}
}

View File

@ -0,0 +1,22 @@
package org.schabi.newpipe.downloader;
import org.schabi.newpipe.extractor.downloader.Request;
import org.schabi.newpipe.extractor.downloader.Response;
final class TestRequestResponse {
private final Request request;
private final Response response;
public TestRequestResponse(Request request, Response response) {
this.request = request;
this.response = response;
}
public Request getRequest() {
return request;
}
public Response getResponse() {
return response;
}
}

View File

@ -1,12 +1,20 @@
package org.schabi.newpipe.extractor.services;
import org.junit.Test;
import org.schabi.newpipe.extractor.MetaInfo;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import javax.annotation.Nullable;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Collections;
import java.util.List;
import java.util.stream.Collectors;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmpty;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
@ -20,6 +28,10 @@ public abstract class DefaultSearchExtractorTest extends DefaultListExtractorTes
return false;
}
public List<MetaInfo> expectedMetaInfo() throws MalformedURLException {
return Collections.emptyList();
}
@Test
@Override
public void testSearchString() throws Exception {
@ -41,4 +53,34 @@ public abstract class DefaultSearchExtractorTest extends DefaultListExtractorTes
public void testSearchCorrected() throws Exception {
assertEquals(isCorrectedSearch(), extractor().isCorrectedSearch());
}
/**
* @see DefaultStreamExtractorTest#testMetaInfo()
*/
@Test
public void testMetaInfo() throws Exception {
final List<MetaInfo> metaInfoList = extractor().getMetaInfo();
final List<MetaInfo> expectedMetaInfoList = expectedMetaInfo();
for (final MetaInfo expectedMetaInfo : expectedMetaInfoList) {
final List<String> texts = metaInfoList.stream()
.map(metaInfo -> metaInfo.getContent().getContent())
.collect(Collectors.toList());
final List<String> titles = metaInfoList.stream().map(MetaInfo::getTitle).collect(Collectors.toList());
final List<URL> urls = metaInfoList.stream().flatMap(info -> info.getUrls().stream())
.collect(Collectors.toList());
final List<String> urlTexts = metaInfoList.stream().flatMap(info -> info.getUrlTexts().stream())
.collect(Collectors.toList());
assertTrue(texts.contains(expectedMetaInfo.getContent().getContent()));
assertTrue(titles.contains(expectedMetaInfo.getTitle()));
for (final String expectedUrlText : expectedMetaInfo.getUrlTexts()) {
assertTrue(urlTexts.contains(expectedUrlText));
}
for (final URL expectedUrl : expectedMetaInfo.getUrls()) {
assertTrue(urls.contains(expectedUrl));
}
}
}
}

View File

@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor.services;
import org.junit.Test;
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.MetaInfo;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.stream.AudioStream;
import org.schabi.newpipe.extractor.stream.Description;
@ -15,9 +16,12 @@ import org.schabi.newpipe.extractor.stream.VideoStream;
import javax.annotation.Nullable;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
import java.util.stream.Collectors;
import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.MatcherAssert.assertThat;
@ -66,6 +70,8 @@ public abstract class DefaultStreamExtractorTest extends DefaultExtractorTest<St
public Locale expectedLanguageInfo() { return null; } // default: no language info available
public List<String> expectedTags() { return Collections.emptyList(); } // default: no tags
public String expectedSupportInfo() { return ""; } // default: no support info available
public int expectedStreamSegmentsCount() { return -1; } // return 0 or greater to test (default is -1 to ignore)
public List<MetaInfo> expectedMetaInfo() throws MalformedURLException { return Collections.emptyList(); } // default: no metadata info available
@Test
@Override
@ -332,6 +338,8 @@ public abstract class DefaultStreamExtractorTest extends DefaultExtractorTest<St
assertIsValidUrl(url);
assertIsSecureUrl(url);
}
assertTrue(f.getDurationPerFrame() > 0);
assertEquals(f.getFrameBoundsAt(0)[3], f.getFrameWidth());
}
} else {
assertTrue(frames.isEmpty());
@ -379,4 +387,42 @@ public abstract class DefaultStreamExtractorTest extends DefaultExtractorTest<St
public void testSupportInfo() throws Exception {
assertEquals(expectedSupportInfo(), extractor().getSupportInfo());
}
@Test
public void testStreamSegmentsCount() throws Exception {
if (expectedStreamSegmentsCount() >= 0) {
assertEquals(expectedStreamSegmentsCount(), extractor().getStreamSegments().size());
}
}
/**
* @see DefaultSearchExtractorTest#testMetaInfo()
*/
@Test
public void testMetaInfo() throws Exception {
final List<MetaInfo> metaInfoList = extractor().getMetaInfo();
final List<MetaInfo> expectedMetaInfoList = expectedMetaInfo();
for (final MetaInfo expectedMetaInfo : expectedMetaInfoList) {
final List<String> texts = metaInfoList.stream()
.map((metaInfo) -> metaInfo.getContent().getContent())
.collect(Collectors.toList());
final List<String> titles = metaInfoList.stream().map(MetaInfo::getTitle).collect(Collectors.toList());
final List<URL> urls = metaInfoList.stream().flatMap(info -> info.getUrls().stream())
.collect(Collectors.toList());
final List<String> urlTexts = metaInfoList.stream().flatMap(info -> info.getUrlTexts().stream())
.collect(Collectors.toList());
assertTrue(texts.contains(expectedMetaInfo.getContent().getContent()));
assertTrue(titles.contains(expectedMetaInfo.getTitle()));
for (final String expectedUrlText : expectedMetaInfo.getUrlTexts()) {
assertTrue(urlTexts.contains(expectedUrlText));
}
for (final URL expectedUrl : expectedMetaInfo.getUrls()) {
assertTrue(urls.contains(expectedUrl));
}
}
}
}

View File

@ -2,7 +2,7 @@ package org.schabi.newpipe.extractor.services.media_ccc;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCConferenceExtractor;

View File

@ -2,7 +2,7 @@ package org.schabi.newpipe.extractor.services.media_ccc;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCConferenceLinkHandlerFactory;

View File

@ -2,7 +2,7 @@ package org.schabi.newpipe.extractor.services.media_ccc;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
@ -24,7 +24,7 @@ public class MediaCCCConferenceListExtractorTest {
@BeforeClass
public static void setUpClass() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = MediaCCC.getKioskList().getDefaultKioskExtractor();
extractor = MediaCCC.getKioskList().getExtractorById("conferences", null);
extractor.fetchPage();
}

View File

@ -0,0 +1,37 @@
package org.schabi.newpipe.extractor.services.media_ccc;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
import java.util.List;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.MediaCCC;
public class MediaCCCLiveStreamListExtractorTest {
private static KioskExtractor extractor;
@BeforeClass
public static void setUpClass() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = MediaCCC.getKioskList().getExtractorById("live", null);
extractor.fetchPage();
}
@Test
public void getConferencesListTest() throws Exception {
final List<InfoItem> a = extractor.getInitialPage().getItems();
for (int i = 0; i < a.size(); i++) {
final InfoItem b = a.get(i);
assertNotNull(a.get(i).getName());
assertTrue(a.get(i).getName().length() >= 1);
}
}
}

View File

@ -2,7 +2,7 @@ package org.schabi.newpipe.extractor.services.media_ccc;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCStreamExtractor;
import org.schabi.newpipe.extractor.stream.AudioStream;

View File

@ -0,0 +1,43 @@
package org.schabi.newpipe.extractor.services.media_ccc;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import java.time.OffsetDateTime;
import java.util.List;
import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.ServiceList.MediaCCC;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class MediaCCCRecentListExtractorTest {
private static KioskExtractor extractor;
@BeforeClass
public static void setUpClass() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = MediaCCC.getKioskList().getExtractorById("recent", null);
extractor.fetchPage();
}
@Test
@Ignore("TODO fix")
public void testStreamList() throws Exception {
final List<StreamInfoItem> items = extractor.getInitialPage().getItems();
assertEquals(100, items.size());
for (final StreamInfoItem item: items) {
assertFalse(isNullOrEmpty(item.getName()));
assertTrue(item.getDuration() > 0);
assertTrue(isNullOrEmpty(item.getUploaderName())); // we do not get the uploader name
assertTrue(item.getUploadDate().offsetDateTime().isBefore(OffsetDateTime.now()));
assertTrue(item.getUploadDate().offsetDateTime().isAfter(OffsetDateTime.now().minusYears(1)));
}
}
}

View File

@ -2,7 +2,7 @@ package org.schabi.newpipe.extractor.services.media_ccc;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest;
@ -57,6 +57,7 @@ public class MediaCCCStreamExtractorTest {
@Override public boolean expectedHasSubtitles() { return false; }
@Override public boolean expectedHasFrames() { return false; }
@Override public List<String> expectedTags() { return Arrays.asList("gpn18", "105"); }
@Override public int expectedStreamSegmentsCount() { return 0; }
@Override
@Test

View File

@ -2,7 +2,7 @@ package org.schabi.newpipe.extractor.services.media_ccc;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCStreamLinkHandlerFactory;

View File

@ -1,7 +1,7 @@
package org.schabi.newpipe.extractor.services.media_ccc.search;
import org.junit.BeforeClass;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;

View File

@ -3,7 +3,7 @@ package org.schabi.newpipe.extractor.services.peertube;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
@ -19,16 +19,17 @@ import static org.schabi.newpipe.extractor.services.DefaultTests.*;
* Test for {@link PeertubeAccountExtractor}
*/
public class PeertubeAccountExtractorTest {
public static class KDE implements BaseChannelExtractorTest {
public static class Framasoft implements BaseChannelExtractorTest {
private static PeertubeAccountExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
// setting instance might break test when running in parallel
PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host"));
PeerTube.setInstance(new PeertubeInstance("https://framatube.org", "Framatube"));
extractor = (PeertubeAccountExtractor) PeerTube
.getChannelExtractor("https://peertube.mastodon.host/accounts/kde");
.getChannelExtractor("https://framatube.org/accounts/framasoft");
extractor.fetchPage();
}
@ -43,22 +44,22 @@ public class PeertubeAccountExtractorTest {
@Test
public void testName() throws ParsingException {
assertEquals("The KDE Community", extractor.getName());
assertEquals("Framasoft", extractor.getName());
}
@Test
public void testId() throws ParsingException {
assertEquals("accounts/kde", extractor.getId());
assertEquals("accounts/framasoft", extractor.getId());
}
@Test
public void testUrl() throws ParsingException {
assertEquals("https://peertube.mastodon.host/accounts/kde", extractor.getUrl());
assertEquals("https://framatube.org/accounts/framasoft", extractor.getUrl());
}
@Test
public void testOriginalUrl() throws ParsingException {
assertEquals("https://peertube.mastodon.host/accounts/kde", extractor.getOriginalUrl());
assertEquals("https://framatube.org/accounts/framasoft", extractor.getOriginalUrl());
}
/*//////////////////////////////////////////////////////////////////////////
@ -96,25 +97,26 @@ public class PeertubeAccountExtractorTest {
@Test
public void testFeedUrl() throws ParsingException {
assertEquals("https://peertube.mastodon.host/feeds/videos.xml?accountId=32465", extractor.getFeedUrl());
assertEquals("https://framatube.org/feeds/videos.xml?accountId=3", extractor.getFeedUrl());
}
@Test
@Ignore("TODO fix")
public void testSubscriberCount() throws ParsingException {
assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 5);
assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 500);
}
}
public static class Booteille implements BaseChannelExtractorTest {
public static class FreeSoftwareFoundation implements BaseChannelExtractorTest {
private static PeertubeAccountExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
// setting instance might break test when running in parallel
PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host"));
PeerTube.setInstance(new PeertubeInstance("https://framatube.org", "Framatube"));
extractor = (PeertubeAccountExtractor) PeerTube
.getChannelExtractor("https://peertube.mastodon.host/api/v1/accounts/booteille");
.getChannelExtractor("https://framatube.org/api/v1/accounts/fsf");
extractor.fetchPage();
}
@ -139,22 +141,22 @@ public class PeertubeAccountExtractorTest {
@Test
public void testName() throws ParsingException {
assertEquals("booteille", extractor.getName());
assertEquals("Free Software Foundation", extractor.getName());
}
@Test
public void testId() throws ParsingException {
assertEquals("accounts/booteille", extractor.getId());
assertEquals("accounts/fsf", extractor.getId());
}
@Test
public void testUrl() throws ParsingException {
assertEquals("https://peertube.mastodon.host/accounts/booteille", extractor.getUrl());
assertEquals("https://framatube.org/accounts/fsf", extractor.getUrl());
}
@Test
public void testOriginalUrl() throws ParsingException {
assertEquals("https://peertube.mastodon.host/api/v1/accounts/booteille", extractor.getOriginalUrl());
assertEquals("https://framatube.org/api/v1/accounts/fsf", extractor.getOriginalUrl());
}
/*//////////////////////////////////////////////////////////////////////////
@ -185,20 +187,19 @@ public class PeertubeAccountExtractorTest {
assertIsSecureUrl(extractor.getAvatarUrl());
}
@Ignore
@Test
public void testBannerUrl() throws ParsingException {
assertIsSecureUrl(extractor.getBannerUrl());
assertNull(extractor.getBannerUrl());
}
@Test
public void testFeedUrl() throws ParsingException {
assertEquals("https://peertube.mastodon.host/feeds/videos.xml?accountId=1753", extractor.getFeedUrl());
assertEquals("https://framatube.org/feeds/videos.xml?accountId=8178", extractor.getFeedUrl());
}
@Test
public void testSubscriberCount() throws ParsingException {
assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 1);
assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 100);
}
}
}

View File

@ -3,7 +3,7 @@ package org.schabi.newpipe.extractor.services.peertube;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
@ -19,16 +19,17 @@ import static org.schabi.newpipe.extractor.services.DefaultTests.*;
* Test for {@link PeertubeChannelExtractor}
*/
public class PeertubeChannelExtractorTest {
public static class DanDAugeTutoriels implements BaseChannelExtractorTest {
public static class LaQuadratureDuNet implements BaseChannelExtractorTest {
private static PeertubeChannelExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
// setting instance might break test when running in parallel
PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host"));
PeerTube.setInstance(new PeertubeInstance("https://framatube.org", "Framatube"));
extractor = (PeertubeChannelExtractor) PeerTube
.getChannelExtractor("https://peertube.mastodon.host/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa");
.getChannelExtractor("https://framatube.org/video-channels/lqdn_channel@video.lqdn.fr/videos");
extractor.fetchPage();
}
@ -43,22 +44,22 @@ public class PeertubeChannelExtractorTest {
@Test
public void testName() throws ParsingException {
assertEquals("Dan d'Auge tutoriels", extractor.getName());
assertEquals("La Quadrature du Net", extractor.getName());
}
@Test
public void testId() throws ParsingException {
assertEquals("video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa", extractor.getId());
assertEquals("video-channels/lqdn_channel@video.lqdn.fr", extractor.getId());
}
@Test
public void testUrl() throws ParsingException {
assertEquals("https://peertube.mastodon.host/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa", extractor.getUrl());
assertEquals("https://framatube.org/video-channels/lqdn_channel@video.lqdn.fr", extractor.getUrl());
}
@Test
public void testOriginalUrl() throws ParsingException {
assertEquals("https://peertube.mastodon.host/video-channels/7682d9f2-07be-4622-862e-93ec812e2ffa", extractor.getOriginalUrl());
assertEquals("https://framatube.org/video-channels/lqdn_channel@video.lqdn.fr/videos", extractor.getOriginalUrl());
}
/*//////////////////////////////////////////////////////////////////////////
@ -86,12 +87,12 @@ public class PeertubeChannelExtractorTest {
@Test
public void testParentChannelName() throws ParsingException {
assertEquals("libux", extractor.getParentChannelName());
assertEquals("lqdn", extractor.getParentChannelName());
}
@Test
public void testParentChannelUrl() throws ParsingException {
assertEquals("https://peertube.mastodon.host/accounts/libux", extractor.getParentChannelUrl());
assertEquals("https://video.lqdn.fr/accounts/lqdn", extractor.getParentChannelUrl());
}
@Test
@ -111,25 +112,26 @@ public class PeertubeChannelExtractorTest {
@Test
public void testFeedUrl() throws ParsingException {
assertEquals("https://peertube.mastodon.host/feeds/videos.xml?videoChannelId=1361", extractor.getFeedUrl());
assertEquals("https://framatube.org/feeds/videos.xml?videoChannelId=1126", extractor.getFeedUrl());
}
@Test
public void testSubscriberCount() throws ParsingException {
assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 4);
assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 230);
}
}
public static class Divers implements BaseChannelExtractorTest {
public static class ChatSceptique implements BaseChannelExtractorTest {
private static PeertubeChannelExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
// setting instance might break test when running in parallel
PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host"));
PeerTube.setInstance(new PeertubeInstance("https://framatube.org", "Framatube"));
extractor = (PeertubeChannelExtractor) PeerTube
.getChannelExtractor("https://peertube.mastodon.host/api/v1/video-channels/35080089-79b6-45fc-96ac-37e4d46a4457");
.getChannelExtractor("https://framatube.org/api/v1/video-channels/chatsceptique@skeptikon.fr");
extractor.fetchPage();
}
@ -154,22 +156,22 @@ public class PeertubeChannelExtractorTest {
@Test
public void testName() throws ParsingException {
assertEquals("Divers", extractor.getName());
assertEquals("Chat Sceptique", extractor.getName());
}
@Test
public void testId() throws ParsingException {
assertEquals("video-channels/35080089-79b6-45fc-96ac-37e4d46a4457", extractor.getId());
assertEquals("video-channels/chatsceptique@skeptikon.fr", extractor.getId());
}
@Test
public void testUrl() throws ParsingException {
assertEquals("https://peertube.mastodon.host/video-channels/35080089-79b6-45fc-96ac-37e4d46a4457", extractor.getUrl());
assertEquals("https://framatube.org/video-channels/chatsceptique@skeptikon.fr", extractor.getUrl());
}
@Test
public void testOriginalUrl() throws ParsingException {
assertEquals("https://peertube.mastodon.host/api/v1/video-channels/35080089-79b6-45fc-96ac-37e4d46a4457", extractor.getOriginalUrl());
assertEquals("https://framatube.org/api/v1/video-channels/chatsceptique@skeptikon.fr", extractor.getOriginalUrl());
}
/*//////////////////////////////////////////////////////////////////////////
@ -197,12 +199,12 @@ public class PeertubeChannelExtractorTest {
@Test
public void testParentChannelName() throws ParsingException {
assertEquals("booteille", extractor.getParentChannelName());
assertEquals("nathan", extractor.getParentChannelName());
}
@Test
public void testParentChannelUrl() throws ParsingException {
assertEquals("https://peertube.mastodon.host/accounts/booteille", extractor.getParentChannelUrl());
assertEquals("https://skeptikon.fr/accounts/nathan", extractor.getParentChannelUrl());
}
@Test
@ -215,20 +217,19 @@ public class PeertubeChannelExtractorTest {
assertIsSecureUrl(extractor.getAvatarUrl());
}
@Ignore
@Test
public void testBannerUrl() throws ParsingException {
assertIsSecureUrl(extractor.getBannerUrl());
assertNull(extractor.getBannerUrl());
}
@Test
public void testFeedUrl() throws ParsingException {
assertEquals("https://peertube.mastodon.host/feeds/videos.xml?videoChannelId=1227", extractor.getFeedUrl());
assertEquals("https://framatube.org/feeds/videos.xml?videoChannelId=137", extractor.getFeedUrl());
}
@Test
public void testSubscriberCount() throws ParsingException {
assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 2);
assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 700);
}
}
}

View File

@ -2,7 +2,7 @@ package org.schabi.newpipe.extractor.services.peertube;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory;

View File

@ -2,7 +2,7 @@ package org.schabi.newpipe.extractor.services.peertube;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.Page;

View File

@ -2,7 +2,7 @@ package org.schabi.newpipe.extractor.services.peertube;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeCommentsLinkHandlerFactory;

View File

@ -1,8 +1,9 @@
package org.schabi.newpipe.extractor.services.peertube;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubePlaylistExtractor;
@ -44,11 +45,13 @@ public class PeertubePlaylistExtractorTest {
}
@Test
@Ignore("TODO fix")
public void testGetUploaderName() throws ParsingException {
assertEquals("Méta de Choc", extractor.getUploaderName());
}
@Test
@Ignore("TODO fix")
public void testGetStreamCount() throws ParsingException {
assertEquals(35, extractor.getStreamCount());
}
@ -59,6 +62,7 @@ public class PeertubePlaylistExtractorTest {
}
@Test
@Ignore("TODO fix")
public void testGetSubChannelName() throws ParsingException {
assertEquals("SHOCKING !", extractor.getSubChannelName());
}

View File

@ -2,7 +2,7 @@ package org.schabi.newpipe.extractor.services.peertube;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubePlaylistLinkHandlerFactory;

View File

@ -1,8 +1,9 @@
package org.schabi.newpipe.extractor.services.peertube;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
@ -92,8 +93,17 @@ public class PeertubeStreamExtractorTest {
@Override public String expectedLicence() { return "Attribution - Share Alike"; }
@Override public Locale expectedLanguageInfo() { return Locale.forLanguageTag("en"); }
@Override public List<String> expectedTags() { return Arrays.asList("framasoft", "peertube"); }
@Override public int expectedStreamSegmentsCount() { return 0; }
@Override
@Test
@Ignore("TODO fix")
public void testSubChannelName() throws Exception {
super.testSubChannelName();
}
}
@Ignore("TODO fix")
public static class AgeRestricted extends DefaultStreamExtractorTest {
private static final String ID = "dbd8e5e1-c527-49b6-b70c-89101dbb9c08";
private static final String INSTANCE = "https://nocensoring.net";

View File

@ -2,7 +2,7 @@ package org.schabi.newpipe.extractor.services.peertube;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeStreamLinkHandlerFactory;

View File

@ -2,7 +2,7 @@ package org.schabi.newpipe.extractor.services.peertube;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.BaseListExtractorTest;
@ -13,6 +13,7 @@ import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
import static org.schabi.newpipe.extractor.services.DefaultTests.*;
public class PeertubeTrendingExtractorTest {
public static class Trending implements BaseListExtractorTest {
private static PeertubeTrendingExtractor extractor;
@ -20,7 +21,7 @@ public class PeertubeTrendingExtractorTest {
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
// setting instance might break test when running in parallel
PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host"));
PeerTube.setInstance(new PeertubeInstance("https://framatube.org", "Framatube"));
extractor = (PeertubeTrendingExtractor) PeerTube.getKioskList()
.getExtractorById("Trending", null);
extractor.fetchPage();
@ -47,12 +48,12 @@ public class PeertubeTrendingExtractorTest {
@Test
public void testUrl() throws ParsingException {
assertEquals("https://peertube.mastodon.host/api/v1/videos?sort=-trending", extractor.getUrl());
assertEquals("https://framatube.org/api/v1/videos?sort=-trending", extractor.getUrl());
}
@Test
public void testOriginalUrl() throws ParsingException {
assertEquals("https://peertube.mastodon.host/api/v1/videos?sort=-trending", extractor.getOriginalUrl());
assertEquals("https://framatube.org/api/v1/videos?sort=-trending", extractor.getOriginalUrl());
}
/*//////////////////////////////////////////////////////////////////////////

View File

@ -2,7 +2,7 @@ package org.schabi.newpipe.extractor.services.peertube;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;

View File

@ -2,7 +2,7 @@ package org.schabi.newpipe.extractor.services.peertube.search;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage;
import org.schabi.newpipe.extractor.NewPipe;
@ -10,6 +10,7 @@ import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.DefaultSearchExtractorTest;
import org.schabi.newpipe.extractor.services.peertube.PeertubeInstance;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeSearchQueryHandlerFactory;
import javax.annotation.Nullable;
@ -21,6 +22,29 @@ import static org.schabi.newpipe.extractor.services.peertube.linkHandler.Peertub
public class PeertubeSearchExtractorTest {
public static class All extends DefaultSearchExtractorTest {
private static SearchExtractor extractor;
private static final String QUERY = "fsf";
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
// setting instance might break test when running in parallel
PeerTube.setInstance(new PeertubeInstance("https://framatube.org", "Framatube"));
extractor = PeerTube.getSearchExtractor(QUERY);
extractor.fetchPage();
}
@Override public SearchExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return PeerTube; }
@Override public String expectedName() { return QUERY; }
@Override public String expectedId() { return QUERY; }
@Override public String expectedUrlContains() { return "/search/videos?search=" + QUERY; }
@Override public String expectedOriginalUrlContains() { return "/search/videos?search=" + QUERY; }
@Override public String expectedSearchString() { return QUERY; }
@Nullable @Override public String expectedSearchSuggestion() { return null; }
}
public static class SepiaSearch extends DefaultSearchExtractorTest {
private static SearchExtractor extractor;
private static final String QUERY = "kde";
@ -28,8 +52,8 @@ public class PeertubeSearchExtractorTest {
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
// setting instance might break test when running in parallel
PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host"));
extractor = PeerTube.getSearchExtractor(QUERY);
PeerTube.setInstance(new PeertubeInstance("https://framatube.org", "Framatube"));
extractor = PeerTube.getSearchExtractor(QUERY, singletonList(PeertubeSearchQueryHandlerFactory.SEPIA_VIDEOS), "");
extractor.fetchPage();
}

View File

@ -1,9 +1,12 @@
package org.schabi.newpipe.extractor.services.peertube.search;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.schabi.newpipe.extractor.services.peertube.PeertubeInstance;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeSearchQueryHandlerFactory;
import static java.util.Collections.singletonList;
import static org.junit.Assert.assertEquals;
import static org.schabi.newpipe.extractor.ServiceList.PeerTube;
@ -16,11 +19,14 @@ public class PeertubeSearchQHTest {
}
@Test
@Ignore("TODO fix")
public void testRegularValues() throws Exception {
assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=asdf", PeerTube.getSearchQHFactory().fromQuery("asdf").getUrl());
assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=hans", PeerTube.getSearchQHFactory().fromQuery("hans").getUrl());
assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=Poifj%26jaijf", PeerTube.getSearchQHFactory().fromQuery("Poifj&jaijf").getUrl());
assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=G%C3%BCl%C3%BCm", PeerTube.getSearchQHFactory().fromQuery("Gülüm").getUrl());
assertEquals("https://peertube.mastodon.host/api/v1/search/videos?search=%3Fj%24%29H%C2%A7B", PeerTube.getSearchQHFactory().fromQuery("?j$)H§B").getUrl());
assertEquals("https://sepiasearch.org/api/v1/search/videos?search=%3Fj%24%29H%C2%A7B", PeerTube.getSearchQHFactory().fromQuery("?j$)H§B", singletonList(PeertubeSearchQueryHandlerFactory.SEPIA_VIDEOS), "").getUrl());
assertEquals("https://anotherpeertubeindex.com/api/v1/search/videos?search=%3Fj%24%29H%C2%A7B", PeerTube.getSearchQHFactory().fromQuery("?j$)H§B", singletonList(PeertubeSearchQueryHandlerFactory.SEPIA_VIDEOS), "", "https://anotherpeertubeindex.com").getUrl());
}
}

View File

@ -2,7 +2,7 @@ package org.schabi.newpipe.extractor.services.soundcloud;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;

View File

@ -2,7 +2,7 @@ package org.schabi.newpipe.extractor.services.soundcloud;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.BaseListExtractorTest;

View File

@ -2,7 +2,7 @@ package org.schabi.newpipe.extractor.services.soundcloud;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.soundcloud.linkHandler.SoundcloudChartsLinkHandlerFactory;

View File

@ -3,7 +3,7 @@ package org.schabi.newpipe.extractor.services.soundcloud;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import static org.junit.Assert.assertTrue;

View File

@ -1,8 +1,9 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
@ -41,6 +42,7 @@ public class SoundcloudPlaylistExtractorTest {
}
@Test
@Ignore("TODO fix")
public void testName() {
assertEquals("THE PERFECT LUV TAPE®", extractor.getName());
}
@ -361,6 +363,7 @@ public class SoundcloudPlaylistExtractorTest {
}
@Test
@Ignore("TODO fix")
public void testMoreRelatedItems() throws Exception {
try {
defaultTestMoreItems(extractor);

View File

@ -1,11 +1,9 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException;
import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamType;
@ -55,6 +53,7 @@ public class SoundcloudStreamExtractorTest {
@Override public boolean expectedHasVideoStreams() { return false; }
@Override public boolean expectedHasSubtitles() { return false; }
@Override public boolean expectedHasFrames() { return false; }
@Override public int expectedStreamSegmentsCount() { return 0; }
}
}

View File

@ -1,8 +1,9 @@
package org.schabi.newpipe.extractor.services.soundcloud;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.soundcloud.linkHandler.SoundcloudStreamLinkHandlerFactory;
@ -25,6 +26,7 @@ public class SoundcloudStreamLinkHandlerFactoryTest {
}
@Test(expected = IllegalArgumentException.class)
@Ignore("TODO fix")
public void getIdWithNullAsUrl() throws ParsingException {
linkHandler.fromUrl(null).getId();
}

View File

@ -2,7 +2,7 @@ package org.schabi.newpipe.extractor.services.soundcloud;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.ServiceList;
import org.schabi.newpipe.extractor.exceptions.ParsingException;

View File

@ -2,7 +2,7 @@ package org.schabi.newpipe.extractor.services.soundcloud;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;

View File

@ -1,8 +1,9 @@
package org.schabi.newpipe.extractor.services.soundcloud.search;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage;
import org.schabi.newpipe.extractor.NewPipe;
@ -113,6 +114,7 @@ public class SoundcloudSearchExtractorTest {
public static class PagingTest {
@Test
@Ignore("TODO fix")
public void duplicatedItemsCheck() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
final SearchExtractor extractor = SoundCloud.getSearchExtractor("cirque du soleil", singletonList(TRACKS), "");

View File

@ -1,8 +1,9 @@
package org.schabi.newpipe.extractor.services.soundcloud.search;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import static java.util.Arrays.asList;
@ -23,6 +24,7 @@ public class SoundcloudSearchQHTest {
}
@Test
@Ignore("TODO fix")
public void testRegularValues() throws Exception {
assertEquals("https://api-v2.soundcloud.com/search?q=asdf&limit=10&offset=0",
removeClientId(SoundCloud.getSearchQHFactory().fromQuery("asdf").getUrl()));

Some files were not shown because too many files have changed in this diff Show More