diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index b6649d5ed..ca60293ea 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -25,6 +25,7 @@ import static org.schabi.newpipe.extractor.utils.Utils.HTTP; import static org.schabi.newpipe.extractor.utils.Utils.HTTPS; import static org.schabi.newpipe.extractor.utils.Utils.getStringResultFromRegexArray; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; +import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonBuilder; @@ -32,8 +33,8 @@ import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; import com.grack.nanojson.JsonWriter; -import org.jsoup.nodes.Entities; +import org.jsoup.nodes.Entities; import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.Image.ResolutionLevel; import org.schabi.newpipe.extractor.MetaInfo; @@ -71,6 +72,7 @@ import java.util.Objects; import java.util.Optional; import java.util.Random; import java.util.Set; +import java.util.function.Consumer; import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -1080,6 +1082,16 @@ public final class YoutubeParsingHelper { .replaceAll(" {2}", "  "); } + @Nonnull + public static String getTextFromObjectOrThrow(final JsonObject textObject, final String error) + throws ParsingException { + final String result = getTextFromObject(textObject); + if (result == null) { + throw new ParsingException("Could not extract text: " + error); + } + return result; + } + @Nullable public static String getTextFromObject(final JsonObject textObject) { return getTextFromObject(textObject, false); @@ -1664,11 +1676,16 @@ public final class YoutubeParsingHelper { .getObject("infoPanelContentRenderer"))); } if (sectionContent.has("clarificationRenderer")) { - metaInfo.add(getClarificationRendererContent(sectionContent + metaInfo.add(getClarificationRenderer(sectionContent .getObject("clarificationRenderer") )); } - + if (sectionContent.has("emergencyOneboxRenderer")) { + getEmergencyOneboxRenderer( + sectionContent.getObject("emergencyOneboxRenderer"), + metaInfo::add + ); + } } } } @@ -1709,7 +1726,7 @@ public final class YoutubeParsingHelper { } @Nonnull - private static MetaInfo getClarificationRendererContent( + private static MetaInfo getClarificationRenderer( @Nonnull final JsonObject clarificationRenderer) throws ParsingException { final MetaInfo metaInfo = new MetaInfo(); @@ -1762,6 +1779,55 @@ public final class YoutubeParsingHelper { return metaInfo; } + @Nonnull + private static void getEmergencyOneboxRenderer( + @Nonnull final JsonObject emergencyOneboxRenderer, + final Consumer addMetaInfo + ) throws ParsingException { + final List supportRenderers = emergencyOneboxRenderer.entrySet().stream() + .filter((a) -> a.getValue() instanceof JsonObject + && ((JsonObject) a.getValue()).has("singleActionEmergencySupportRenderer")) + .map((a) -> ((JsonObject) a.getValue()) + .getObject("singleActionEmergencySupportRenderer")) + .collect(Collectors.toList()); + + if (supportRenderers.isEmpty()) { + throw new ParsingException("Could not extract any meta info from emergency renderer"); + } + + for (final JsonObject r : supportRenderers) { + final MetaInfo metaInfo = new MetaInfo(); + + // usually an encouragement like "We are with you" + final String title = getTextFromObjectOrThrow(r.getObject("title"), "title"); + // usually a phone number + final String action = getTextFromObjectOrThrow(r.getObject("actionText"), "action"); + // usually details about the phone number + final String details = getTextFromObjectOrThrow(r.getObject("detailsText"), "details"); + // usually the name of an association + final String urlText = getTextFromObjectOrThrow(r.getObject("navigationText"), + "urlText"); + + metaInfo.setTitle(title); + metaInfo.setContent(new Description(details + "\n" + action, Description.PLAIN_TEXT)); + metaInfo.addUrlText(urlText); + + // usually the webpage of the association + final String url = getUrlFromNavigationEndpoint(r.getObject("navigationEndpoint")); + if (url == null) { + throw new ParsingException("Could not extract emergency renderer url"); + } + + try { + metaInfo.addUrl(new URL(replaceHttpWithHttps(url))); + } catch (final MalformedURLException e) { + throw new ParsingException("Could not parse emergency renderer url", e); + } + + addMetaInfo.accept(metaInfo); + } + } + /** * Sometimes, YouTube provides URLs which use Google's cache. They look like * {@code https://webcache.googleusercontent.com/search?q=cache:CACHED_URL} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java index d6705d615..7612ec06c 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java @@ -446,5 +446,15 @@ public class YoutubeSearchExtractorTest { @Override public String expectedOriginalUrlContains() throws Exception { return "youtube.com/results?search_query=" + Utils.encodeUrlUtf8(QUERY); } @Override public String expectedSearchString() { return QUERY; } @Nullable @Override public String expectedSearchSuggestion() { return null; } + + @Test + @Override + public void testMetaInfo() throws Exception { + final List metaInfoList = extractor().getMetaInfo(); + + // the meta info will have different text and language depending on where in the world + // the connection is established from, so we can't check the actual content + assertEquals(1, metaInfoList.size()); + } } }